blob: d63c8cc37b32949d12677da671b345719fc9ada6 [file] [log] [blame]
Reid Spencer5f016e22007-07-11 17:01:13 +00001//===--- Lexer.cpp - C Language Family Lexer ------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner0bc735f2007-12-29 19:59:25 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Reid Spencer5f016e22007-07-11 17:01:13 +00007//
8//===----------------------------------------------------------------------===//
9//
Chris Lattnerd2177732007-07-20 16:59:19 +000010// This file implements the Lexer and Token interfaces.
Reid Spencer5f016e22007-07-11 17:01:13 +000011//
12//===----------------------------------------------------------------------===//
13//
14// TODO: GCC Diagnostics emitted by the lexer:
15// PEDWARN: (form feed|vertical tab) in preprocessing directive
16//
17// Universal characters, unicode, char mapping:
18// WARNING: `%.*s' is not in NFKC
19// WARNING: `%.*s' is not in NFC
20//
21// Other:
22// TODO: Options to support:
23// -fexec-charset,-fwide-exec-charset
24//
25//===----------------------------------------------------------------------===//
26
27#include "clang/Lex/Lexer.h"
28#include "clang/Lex/Preprocessor.h"
29#include "clang/Basic/Diagnostic.h"
Chris Lattner9dc1f532007-07-20 16:37:10 +000030#include "clang/Basic/SourceManager.h"
Chris Lattner409a0362007-07-22 18:38:25 +000031#include "llvm/Support/Compiler.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000032#include "llvm/Support/MemoryBuffer.h"
33#include <cctype>
34using namespace clang;
35
36static void InitCharacterInfo();
37
Chris Lattnerdbf388b2007-10-07 08:47:24 +000038//===----------------------------------------------------------------------===//
39// Token Class Implementation
40//===----------------------------------------------------------------------===//
41
42/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
43bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
Douglas Gregorbec1c9d2008-12-01 21:46:47 +000044 if (IdentifierInfo *II = getIdentifierInfo())
45 return II->getObjCKeywordID() == objcKey;
46 return false;
Chris Lattnerdbf388b2007-10-07 08:47:24 +000047}
48
49/// getObjCKeywordID - Return the ObjC keyword kind.
50tok::ObjCKeywordKind Token::getObjCKeywordID() const {
51 IdentifierInfo *specId = getIdentifierInfo();
52 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
53}
54
Chris Lattner53702cd2007-12-13 01:59:49 +000055
Chris Lattnerdbf388b2007-10-07 08:47:24 +000056//===----------------------------------------------------------------------===//
57// Lexer Class Implementation
58//===----------------------------------------------------------------------===//
59
60
Chris Lattner168ae2d2007-10-17 20:41:00 +000061/// Lexer constructor - Create a new lexer object for the specified buffer
62/// with the specified preprocessor managing the lexing process. This lexer
63/// assumes that the associated file buffer and Preprocessor objects will
64/// outlive it, so it doesn't take ownership of either of them.
Chris Lattner2b2453a2009-01-17 06:22:33 +000065Lexer::Lexer(SourceLocation fileloc, Preprocessor &PP,
Chris Lattner25bdb512007-07-20 16:52:03 +000066 const char *BufStart, const char *BufEnd)
Chris Lattner2b2453a2009-01-17 06:22:33 +000067// FIXME: This is really horrible and only needed for _Pragma lexers, split this
68// out of the main lexer path!
69 : PreprocessorLexer(&PP,
70 PP.getSourceManager().getCanonicalFileID(
71 PP.getSourceManager().getSpellingLoc(fileloc))),
72 FileLoc(fileloc),
73 Features(PP.getLangOptions()) {
Chris Lattner25bdb512007-07-20 16:52:03 +000074
Chris Lattner2b2453a2009-01-17 06:22:33 +000075 SourceManager &SourceMgr = PP.getSourceManager();
76 const llvm::MemoryBuffer *InputFile = SourceMgr.getBuffer(getFileID());
Chris Lattner25bdb512007-07-20 16:52:03 +000077
Reid Spencer5f016e22007-07-11 17:01:13 +000078 Is_PragmaLexer = false;
Reid Spencer5f016e22007-07-11 17:01:13 +000079 InitCharacterInfo();
Chris Lattner448cec42007-07-22 18:44:36 +000080
81 // BufferStart must always be InputFile->getBufferStart().
82 BufferStart = InputFile->getBufferStart();
83
84 // BufferPtr and BufferEnd can start out somewhere inside the current buffer.
85 // If unspecified, they starts at the start/end of the buffer.
86 BufferPtr = BufStart ? BufStart : BufferStart;
Chris Lattner25bdb512007-07-20 16:52:03 +000087 BufferEnd = BufEnd ? BufEnd : InputFile->getBufferEnd();
88
Reid Spencer5f016e22007-07-11 17:01:13 +000089 assert(BufferEnd[0] == 0 &&
90 "We assume that the input buffer has a null character at the end"
91 " to simplify lexing!");
Chris Lattner25bdb512007-07-20 16:52:03 +000092
Reid Spencer5f016e22007-07-11 17:01:13 +000093 // Start of the file is a start of line.
94 IsAtStartOfLine = true;
95
96 // We are not after parsing a #.
97 ParsingPreprocessorDirective = false;
98
99 // We are not after parsing #include.
100 ParsingFilename = false;
101
102 // We are not in raw mode. Raw mode disables diagnostics and interpretation
103 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used
104 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
105 // or otherwise skipping over tokens.
106 LexingRawMode = false;
107
Chris Lattnerd88dc482008-10-12 04:05:48 +0000108 // Default to keeping comments if the preprocessor wants them.
109 ExtendedTokenMode = 0;
Chris Lattner2b2453a2009-01-17 06:22:33 +0000110 SetCommentRetentionState(PP.getCommentRetentionState());
Reid Spencer5f016e22007-07-11 17:01:13 +0000111}
112
Chris Lattner168ae2d2007-10-17 20:41:00 +0000113/// Lexer constructor - Create a new raw lexer object. This object is only
Chris Lattner590f0cc2008-10-12 01:15:46 +0000114/// suitable for calls to 'LexRawToken'. This lexer assumes that the text
115/// range will outlive it, so it doesn't take ownership of it.
Chris Lattner168ae2d2007-10-17 20:41:00 +0000116Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
Chris Lattner590f0cc2008-10-12 01:15:46 +0000117 const char *BufStart, const char *BufEnd,
118 const llvm::MemoryBuffer *FromFile)
Chris Lattnerc6fe32a2009-01-17 03:48:08 +0000119 : FileLoc(fileloc), Features(features) {
Ted Kremenek41938c82008-11-19 21:57:25 +0000120
Chris Lattner168ae2d2007-10-17 20:41:00 +0000121 Is_PragmaLexer = false;
122 InitCharacterInfo();
123
Chris Lattner8527b712008-10-12 01:23:27 +0000124 // If a MemoryBuffer was specified, use its start as BufferStart. This affects
125 // the source location objects produced by this lexer.
Chris Lattner590f0cc2008-10-12 01:15:46 +0000126 BufferStart = FromFile ? FromFile->getBufferStart() : BufStart;
Chris Lattner168ae2d2007-10-17 20:41:00 +0000127 BufferPtr = BufStart;
128 BufferEnd = BufEnd;
129
130 assert(BufferEnd[0] == 0 &&
131 "We assume that the input buffer has a null character at the end"
132 " to simplify lexing!");
133
134 // Start of the file is a start of line.
135 IsAtStartOfLine = true;
136
137 // We are not after parsing a #.
138 ParsingPreprocessorDirective = false;
139
140 // We are not after parsing #include.
141 ParsingFilename = false;
142
143 // We *are* in raw mode.
144 LexingRawMode = true;
145
Chris Lattnera2c7ad92008-10-12 01:34:51 +0000146 // Default to not keeping comments in raw mode.
Chris Lattnerd88dc482008-10-12 04:05:48 +0000147 ExtendedTokenMode = 0;
Chris Lattner168ae2d2007-10-17 20:41:00 +0000148}
149
150
Reid Spencer5f016e22007-07-11 17:01:13 +0000151/// Stringify - Convert the specified string into a C string, with surrounding
152/// ""'s, and with escaped \ and " characters.
153std::string Lexer::Stringify(const std::string &Str, bool Charify) {
154 std::string Result = Str;
155 char Quote = Charify ? '\'' : '"';
156 for (unsigned i = 0, e = Result.size(); i != e; ++i) {
157 if (Result[i] == '\\' || Result[i] == Quote) {
158 Result.insert(Result.begin()+i, '\\');
159 ++i; ++e;
160 }
161 }
162 return Result;
163}
164
Chris Lattnerd8e30832007-07-24 06:57:14 +0000165/// Stringify - Convert the specified string into a C string by escaping '\'
166/// and " characters. This does not add surrounding ""'s to the string.
167void Lexer::Stringify(llvm::SmallVectorImpl<char> &Str) {
168 for (unsigned i = 0, e = Str.size(); i != e; ++i) {
169 if (Str[i] == '\\' || Str[i] == '"') {
170 Str.insert(Str.begin()+i, '\\');
171 ++i; ++e;
172 }
173 }
174}
175
Reid Spencer5f016e22007-07-11 17:01:13 +0000176
Chris Lattner9a611942007-10-17 21:18:47 +0000177/// MeasureTokenLength - Relex the token at the specified location and return
178/// its length in bytes in the input file. If the token needs cleaning (e.g.
179/// includes a trigraph or an escaped newline) then this count includes bytes
180/// that are part of that.
181unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
182 const SourceManager &SM) {
183 // If this comes from a macro expansion, we really do want the macro name, not
184 // the token this macro expanded to.
Chris Lattnerf7cf85b2009-01-16 07:36:28 +0000185 Loc = SM.getInstantiationLoc(Loc);
Chris Lattner9a611942007-10-17 21:18:47 +0000186
187 const char *StrData = SM.getCharacterData(Loc);
188
189 // TODO: this could be special cased for common tokens like identifiers, ')',
190 // etc to make this faster, if it mattered. Just look at StrData[0] to handle
191 // all obviously single-char tokens. This could use
192 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
193 // something.
Chris Lattner2b2453a2009-01-17 06:22:33 +0000194 const char *BufEnd = SM.getBufferData(Loc).second;
Chris Lattner9a611942007-10-17 21:18:47 +0000195
196 // Create a langops struct and enable trigraphs. This is sufficient for
197 // measuring tokens.
198 LangOptions LangOpts;
199 LangOpts.Trigraphs = true;
200
201 // Create a lexer starting at the beginning of this token.
202 Lexer TheLexer(Loc, LangOpts, StrData, BufEnd);
203 Token TheTok;
Chris Lattner590f0cc2008-10-12 01:15:46 +0000204 TheLexer.LexFromRawLexer(TheTok);
Chris Lattner9a611942007-10-17 21:18:47 +0000205 return TheTok.getLength();
206}
207
Reid Spencer5f016e22007-07-11 17:01:13 +0000208//===----------------------------------------------------------------------===//
209// Character information.
210//===----------------------------------------------------------------------===//
211
212static unsigned char CharInfo[256];
213
214enum {
215 CHAR_HORZ_WS = 0x01, // ' ', '\t', '\f', '\v'. Note, no '\0'
216 CHAR_VERT_WS = 0x02, // '\r', '\n'
217 CHAR_LETTER = 0x04, // a-z,A-Z
218 CHAR_NUMBER = 0x08, // 0-9
219 CHAR_UNDER = 0x10, // _
220 CHAR_PERIOD = 0x20 // .
221};
222
223static void InitCharacterInfo() {
224 static bool isInited = false;
225 if (isInited) return;
226 isInited = true;
227
228 // Intiialize the CharInfo table.
229 // TODO: statically initialize this.
230 CharInfo[(int)' '] = CharInfo[(int)'\t'] =
231 CharInfo[(int)'\f'] = CharInfo[(int)'\v'] = CHAR_HORZ_WS;
232 CharInfo[(int)'\n'] = CharInfo[(int)'\r'] = CHAR_VERT_WS;
233
234 CharInfo[(int)'_'] = CHAR_UNDER;
235 CharInfo[(int)'.'] = CHAR_PERIOD;
236 for (unsigned i = 'a'; i <= 'z'; ++i)
237 CharInfo[i] = CharInfo[i+'A'-'a'] = CHAR_LETTER;
238 for (unsigned i = '0'; i <= '9'; ++i)
239 CharInfo[i] = CHAR_NUMBER;
240}
241
242/// isIdentifierBody - Return true if this is the body character of an
243/// identifier, which is [a-zA-Z0-9_].
244static inline bool isIdentifierBody(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +0000245 return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER)) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000246}
247
248/// isHorizontalWhitespace - Return true if this character is horizontal
249/// whitespace: ' ', '\t', '\f', '\v'. Note that this returns false for '\0'.
250static inline bool isHorizontalWhitespace(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +0000251 return (CharInfo[c] & CHAR_HORZ_WS) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000252}
253
254/// isWhitespace - Return true if this character is horizontal or vertical
255/// whitespace: ' ', '\t', '\f', '\v', '\n', '\r'. Note that this returns false
256/// for '\0'.
257static inline bool isWhitespace(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +0000258 return (CharInfo[c] & (CHAR_HORZ_WS|CHAR_VERT_WS)) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000259}
260
261/// isNumberBody - Return true if this is the body character of an
262/// preprocessing number, which is [a-zA-Z0-9_.].
263static inline bool isNumberBody(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +0000264 return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD)) ?
265 true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000266}
267
268
269//===----------------------------------------------------------------------===//
270// Diagnostics forwarding code.
271//===----------------------------------------------------------------------===//
272
Chris Lattner409a0362007-07-22 18:38:25 +0000273/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
274/// lexer buffer was all instantiated at a single point, perform the mapping.
275/// This is currently only used for _Pragma implementation, so it is the slow
276/// path of the hot getSourceLocation method. Do not allow it to be inlined.
277static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
278 SourceLocation FileLoc,
279 unsigned CharNo) DISABLE_INLINE;
280static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
281 SourceLocation FileLoc,
282 unsigned CharNo) {
283 // Otherwise, we're lexing "mapped tokens". This is used for things like
284 // _Pragma handling. Combine the instantiation location of FileLoc with the
Chris Lattnerdf7c17a2009-01-16 07:00:02 +0000285 // spelling location.
Chris Lattner409a0362007-07-22 18:38:25 +0000286 SourceManager &SourceMgr = PP.getSourceManager();
287
Chris Lattnerf7cf85b2009-01-16 07:36:28 +0000288 // Create a new SLoc which is expanded from Instantiation(FileLoc) but whose
Chris Lattnerdf7c17a2009-01-16 07:00:02 +0000289 // characters come from spelling(FileLoc)+Offset.
Chris Lattnerf7cf85b2009-01-16 07:36:28 +0000290 SourceLocation InstLoc = SourceMgr.getInstantiationLoc(FileLoc);
Chris Lattnerdf7c17a2009-01-16 07:00:02 +0000291 SourceLocation SpellingLoc = SourceMgr.getSpellingLoc(FileLoc);
292 SpellingLoc = SourceLocation::getFileLoc(SpellingLoc.getFileID(), CharNo);
Chris Lattnerf7cf85b2009-01-16 07:36:28 +0000293 return SourceMgr.getInstantiationLoc(SpellingLoc, InstLoc);
Chris Lattner409a0362007-07-22 18:38:25 +0000294}
295
Reid Spencer5f016e22007-07-11 17:01:13 +0000296/// getSourceLocation - Return a source location identifier for the specified
297/// offset in the current file.
298SourceLocation Lexer::getSourceLocation(const char *Loc) const {
Chris Lattner448cec42007-07-22 18:44:36 +0000299 assert(Loc >= BufferStart && Loc <= BufferEnd &&
Reid Spencer5f016e22007-07-11 17:01:13 +0000300 "Location out of range for this buffer!");
Chris Lattner9dc1f532007-07-20 16:37:10 +0000301
302 // In the normal case, we're just lexing from a simple file buffer, return
303 // the file id from FileLoc with the offset specified.
Chris Lattner448cec42007-07-22 18:44:36 +0000304 unsigned CharNo = Loc-BufferStart;
Chris Lattner9dc1f532007-07-20 16:37:10 +0000305 if (FileLoc.isFileID())
306 return SourceLocation::getFileLoc(FileLoc.getFileID(), CharNo);
307
Chris Lattner2b2453a2009-01-17 06:22:33 +0000308 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
309 // tokens are lexed from where the _Pragma was defined.
Chris Lattner168ae2d2007-10-17 20:41:00 +0000310 assert(PP && "This doesn't work on raw lexers");
311 return GetMappedTokenLoc(*PP, FileLoc, CharNo);
Reid Spencer5f016e22007-07-11 17:01:13 +0000312}
313
Reid Spencer5f016e22007-07-11 17:01:13 +0000314/// Diag - Forwarding function for diagnostics. This translate a source
315/// position in the current buffer into a SourceLocation object for rendering.
Chris Lattner3cbfe2c2008-11-22 00:59:29 +0000316DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
Chris Lattner3692b092008-11-18 07:59:24 +0000317 return PP->Diag(getSourceLocation(Loc), DiagID);
Reid Spencer5f016e22007-07-11 17:01:13 +0000318}
Reid Spencer5f016e22007-07-11 17:01:13 +0000319
320//===----------------------------------------------------------------------===//
321// Trigraph and Escaped Newline Handling Code.
322//===----------------------------------------------------------------------===//
323
324/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
325/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
326static char GetTrigraphCharForLetter(char Letter) {
327 switch (Letter) {
328 default: return 0;
329 case '=': return '#';
330 case ')': return ']';
331 case '(': return '[';
332 case '!': return '|';
333 case '\'': return '^';
334 case '>': return '}';
335 case '/': return '\\';
336 case '<': return '{';
337 case '-': return '~';
338 }
339}
340
341/// DecodeTrigraphChar - If the specified character is a legal trigraph when
342/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled,
343/// return the result character. Finally, emit a warning about trigraph use
344/// whether trigraphs are enabled or not.
345static char DecodeTrigraphChar(const char *CP, Lexer *L) {
346 char Res = GetTrigraphCharForLetter(*CP);
Chris Lattner3692b092008-11-18 07:59:24 +0000347 if (!Res || !L) return Res;
348
349 if (!L->getFeatures().Trigraphs) {
Chris Lattner74d15df2008-11-22 02:02:22 +0000350 if (!L->isLexingRawMode())
351 L->Diag(CP-2, diag::trigraph_ignored);
Chris Lattner3692b092008-11-18 07:59:24 +0000352 return 0;
Reid Spencer5f016e22007-07-11 17:01:13 +0000353 }
Chris Lattner3692b092008-11-18 07:59:24 +0000354
Chris Lattner74d15df2008-11-22 02:02:22 +0000355 if (!L->isLexingRawMode())
356 L->Diag(CP-2, diag::trigraph_converted) << std::string()+Res;
Reid Spencer5f016e22007-07-11 17:01:13 +0000357 return Res;
358}
359
360/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
361/// get its size, and return it. This is tricky in several cases:
362/// 1. If currently at the start of a trigraph, we warn about the trigraph,
363/// then either return the trigraph (skipping 3 chars) or the '?',
364/// depending on whether trigraphs are enabled or not.
365/// 2. If this is an escaped newline (potentially with whitespace between
366/// the backslash and newline), implicitly skip the newline and return
367/// the char after it.
368/// 3. If this is a UCN, return it. FIXME: C++ UCN's?
369///
370/// This handles the slow/uncommon case of the getCharAndSize method. Here we
371/// know that we can accumulate into Size, and that we have already incremented
372/// Ptr by Size bytes.
373///
374/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
375/// be updated to match.
376///
377char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
Chris Lattnerd2177732007-07-20 16:59:19 +0000378 Token *Tok) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000379 // If we have a slash, look for an escaped newline.
380 if (Ptr[0] == '\\') {
381 ++Size;
382 ++Ptr;
383Slash:
384 // Common case, backslash-char where the char is not whitespace.
385 if (!isWhitespace(Ptr[0])) return '\\';
386
387 // See if we have optional whitespace characters followed by a newline.
388 {
389 unsigned SizeTmp = 0;
390 do {
391 ++SizeTmp;
392 if (Ptr[SizeTmp-1] == '\n' || Ptr[SizeTmp-1] == '\r') {
393 // Remember that this token needs to be cleaned.
Chris Lattnerd2177732007-07-20 16:59:19 +0000394 if (Tok) Tok->setFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +0000395
396 // Warn if there was whitespace between the backslash and newline.
Chris Lattner74d15df2008-11-22 02:02:22 +0000397 if (SizeTmp != 1 && Tok && !isLexingRawMode())
Reid Spencer5f016e22007-07-11 17:01:13 +0000398 Diag(Ptr, diag::backslash_newline_space);
399
400 // If this is a \r\n or \n\r, skip the newlines.
401 if ((Ptr[SizeTmp] == '\r' || Ptr[SizeTmp] == '\n') &&
402 Ptr[SizeTmp-1] != Ptr[SizeTmp])
403 ++SizeTmp;
404
405 // Found backslash<whitespace><newline>. Parse the char after it.
406 Size += SizeTmp;
407 Ptr += SizeTmp;
408 // Use slow version to accumulate a correct size field.
409 return getCharAndSizeSlow(Ptr, Size, Tok);
410 }
411 } while (isWhitespace(Ptr[SizeTmp]));
412 }
413
414 // Otherwise, this is not an escaped newline, just return the slash.
415 return '\\';
416 }
417
418 // If this is a trigraph, process it.
419 if (Ptr[0] == '?' && Ptr[1] == '?') {
420 // If this is actually a legal trigraph (not something like "??x"), emit
421 // a trigraph warning. If so, and if trigraphs are enabled, return it.
422 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) {
423 // Remember that this token needs to be cleaned.
Chris Lattnerd2177732007-07-20 16:59:19 +0000424 if (Tok) Tok->setFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +0000425
426 Ptr += 3;
427 Size += 3;
428 if (C == '\\') goto Slash;
429 return C;
430 }
431 }
432
433 // If this is neither, return a single character.
434 ++Size;
435 return *Ptr;
436}
437
438
439/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
440/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size,
441/// and that we have already incremented Ptr by Size bytes.
442///
443/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
444/// be updated to match.
445char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
446 const LangOptions &Features) {
447 // If we have a slash, look for an escaped newline.
448 if (Ptr[0] == '\\') {
449 ++Size;
450 ++Ptr;
451Slash:
452 // Common case, backslash-char where the char is not whitespace.
453 if (!isWhitespace(Ptr[0])) return '\\';
454
455 // See if we have optional whitespace characters followed by a newline.
456 {
457 unsigned SizeTmp = 0;
458 do {
459 ++SizeTmp;
460 if (Ptr[SizeTmp-1] == '\n' || Ptr[SizeTmp-1] == '\r') {
461
462 // If this is a \r\n or \n\r, skip the newlines.
463 if ((Ptr[SizeTmp] == '\r' || Ptr[SizeTmp] == '\n') &&
464 Ptr[SizeTmp-1] != Ptr[SizeTmp])
465 ++SizeTmp;
466
467 // Found backslash<whitespace><newline>. Parse the char after it.
468 Size += SizeTmp;
469 Ptr += SizeTmp;
470
471 // Use slow version to accumulate a correct size field.
472 return getCharAndSizeSlowNoWarn(Ptr, Size, Features);
473 }
474 } while (isWhitespace(Ptr[SizeTmp]));
475 }
476
477 // Otherwise, this is not an escaped newline, just return the slash.
478 return '\\';
479 }
480
481 // If this is a trigraph, process it.
482 if (Features.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
483 // If this is actually a legal trigraph (not something like "??x"), return
484 // it.
485 if (char C = GetTrigraphCharForLetter(Ptr[2])) {
486 Ptr += 3;
487 Size += 3;
488 if (C == '\\') goto Slash;
489 return C;
490 }
491 }
492
493 // If this is neither, return a single character.
494 ++Size;
495 return *Ptr;
496}
497
498//===----------------------------------------------------------------------===//
499// Helper methods for lexing.
500//===----------------------------------------------------------------------===//
501
Chris Lattnerd2177732007-07-20 16:59:19 +0000502void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000503 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
504 unsigned Size;
505 unsigned char C = *CurPtr++;
506 while (isIdentifierBody(C)) {
507 C = *CurPtr++;
508 }
509 --CurPtr; // Back up over the skipped character.
510
511 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline
512 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN.
513 // FIXME: UCNs.
514 if (C != '\\' && C != '?' && (C != '$' || !Features.DollarIdents)) {
515FinishIdentifier:
516 const char *IdStart = BufferPtr;
Chris Lattner9e6293d2008-10-12 04:51:35 +0000517 FormTokenWithChars(Result, CurPtr, tok::identifier);
Reid Spencer5f016e22007-07-11 17:01:13 +0000518
519 // If we are in raw mode, return this identifier raw. There is no need to
520 // look up identifier information or attempt to macro expand it.
521 if (LexingRawMode) return;
522
523 // Fill in Result.IdentifierInfo, looking up the identifier in the
524 // identifier table.
Chris Lattner168ae2d2007-10-17 20:41:00 +0000525 PP->LookUpIdentifierInfo(Result, IdStart);
Reid Spencer5f016e22007-07-11 17:01:13 +0000526
527 // Finally, now that we know we have an identifier, pass this off to the
528 // preprocessor, which may macro expand it or something.
Chris Lattner168ae2d2007-10-17 20:41:00 +0000529 return PP->HandleIdentifier(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +0000530 }
531
532 // Otherwise, $,\,? in identifier found. Enter slower path.
533
534 C = getCharAndSize(CurPtr, Size);
535 while (1) {
536 if (C == '$') {
537 // If we hit a $ and they are not supported in identifiers, we are done.
538 if (!Features.DollarIdents) goto FinishIdentifier;
539
540 // Otherwise, emit a diagnostic and continue.
Chris Lattner74d15df2008-11-22 02:02:22 +0000541 if (!isLexingRawMode())
542 Diag(CurPtr, diag::ext_dollar_in_identifier);
Reid Spencer5f016e22007-07-11 17:01:13 +0000543 CurPtr = ConsumeChar(CurPtr, Size, Result);
544 C = getCharAndSize(CurPtr, Size);
545 continue;
546 } else if (!isIdentifierBody(C)) { // FIXME: UCNs.
547 // Found end of identifier.
548 goto FinishIdentifier;
549 }
550
551 // Otherwise, this character is good, consume it.
552 CurPtr = ConsumeChar(CurPtr, Size, Result);
553
554 C = getCharAndSize(CurPtr, Size);
555 while (isIdentifierBody(C)) { // FIXME: UCNs.
556 CurPtr = ConsumeChar(CurPtr, Size, Result);
557 C = getCharAndSize(CurPtr, Size);
558 }
559 }
560}
561
562
Nate Begeman5253c7f2008-04-14 02:26:39 +0000563/// LexNumericConstant - Lex the remainder of a integer or floating point
Reid Spencer5f016e22007-07-11 17:01:13 +0000564/// constant. From[-1] is the first character lexed. Return the end of the
565/// constant.
Chris Lattnerd2177732007-07-20 16:59:19 +0000566void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000567 unsigned Size;
568 char C = getCharAndSize(CurPtr, Size);
569 char PrevCh = 0;
570 while (isNumberBody(C)) { // FIXME: UCNs?
571 CurPtr = ConsumeChar(CurPtr, Size, Result);
572 PrevCh = C;
573 C = getCharAndSize(CurPtr, Size);
574 }
575
576 // If we fell out, check for a sign, due to 1e+12. If we have one, continue.
577 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e'))
578 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
579
580 // If we have a hex FP constant, continue.
Chris Lattner49842122008-11-22 07:39:03 +0000581 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p') &&
582 (Features.HexFloats || !Features.NoExtensions))
Reid Spencer5f016e22007-07-11 17:01:13 +0000583 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
584
Reid Spencer5f016e22007-07-11 17:01:13 +0000585 // Update the location of token as well as BufferPtr.
Chris Lattner9e6293d2008-10-12 04:51:35 +0000586 FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
Reid Spencer5f016e22007-07-11 17:01:13 +0000587}
588
589/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
590/// either " or L".
Chris Lattnerd88dc482008-10-12 04:05:48 +0000591void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000592 const char *NulCharacter = 0; // Does this string contain the \0 character?
593
594 char C = getAndAdvanceChar(CurPtr, Result);
595 while (C != '"') {
596 // Skip escaped characters.
597 if (C == '\\') {
598 // Skip the escaped character.
599 C = getAndAdvanceChar(CurPtr, Result);
600 } else if (C == '\n' || C == '\r' || // Newline.
601 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
Chris Lattner74d15df2008-11-22 02:02:22 +0000602 if (!isLexingRawMode())
603 Diag(BufferPtr, diag::err_unterminated_string);
Chris Lattner9e6293d2008-10-12 04:51:35 +0000604 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
Reid Spencer5f016e22007-07-11 17:01:13 +0000605 return;
606 } else if (C == 0) {
607 NulCharacter = CurPtr-1;
608 }
609 C = getAndAdvanceChar(CurPtr, Result);
610 }
611
612 // If a nul character existed in the string, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +0000613 if (NulCharacter && !isLexingRawMode())
614 Diag(NulCharacter, diag::null_in_string);
Reid Spencer5f016e22007-07-11 17:01:13 +0000615
Reid Spencer5f016e22007-07-11 17:01:13 +0000616 // Update the location of the token as well as the BufferPtr instance var.
Chris Lattner9e6293d2008-10-12 04:51:35 +0000617 FormTokenWithChars(Result, CurPtr,
618 Wide ? tok::wide_string_literal : tok::string_literal);
Reid Spencer5f016e22007-07-11 17:01:13 +0000619}
620
621/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
622/// after having lexed the '<' character. This is used for #include filenames.
Chris Lattnerd2177732007-07-20 16:59:19 +0000623void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000624 const char *NulCharacter = 0; // Does this string contain the \0 character?
625
626 char C = getAndAdvanceChar(CurPtr, Result);
627 while (C != '>') {
628 // Skip escaped characters.
629 if (C == '\\') {
630 // Skip the escaped character.
631 C = getAndAdvanceChar(CurPtr, Result);
632 } else if (C == '\n' || C == '\r' || // Newline.
633 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
Chris Lattner74d15df2008-11-22 02:02:22 +0000634 if (!isLexingRawMode())
635 Diag(BufferPtr, diag::err_unterminated_string);
Chris Lattner9e6293d2008-10-12 04:51:35 +0000636 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
Reid Spencer5f016e22007-07-11 17:01:13 +0000637 return;
638 } else if (C == 0) {
639 NulCharacter = CurPtr-1;
640 }
641 C = getAndAdvanceChar(CurPtr, Result);
642 }
643
644 // If a nul character existed in the string, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +0000645 if (NulCharacter && !isLexingRawMode())
646 Diag(NulCharacter, diag::null_in_string);
Reid Spencer5f016e22007-07-11 17:01:13 +0000647
Reid Spencer5f016e22007-07-11 17:01:13 +0000648 // Update the location of token as well as BufferPtr.
Chris Lattner9e6293d2008-10-12 04:51:35 +0000649 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
Reid Spencer5f016e22007-07-11 17:01:13 +0000650}
651
652
653/// LexCharConstant - Lex the remainder of a character constant, after having
654/// lexed either ' or L'.
Chris Lattnerd2177732007-07-20 16:59:19 +0000655void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000656 const char *NulCharacter = 0; // Does this character contain the \0 character?
657
658 // Handle the common case of 'x' and '\y' efficiently.
659 char C = getAndAdvanceChar(CurPtr, Result);
660 if (C == '\'') {
Chris Lattner74d15df2008-11-22 02:02:22 +0000661 if (!isLexingRawMode())
662 Diag(BufferPtr, diag::err_empty_character);
Chris Lattner9e6293d2008-10-12 04:51:35 +0000663 FormTokenWithChars(Result, CurPtr, tok::unknown);
Reid Spencer5f016e22007-07-11 17:01:13 +0000664 return;
665 } else if (C == '\\') {
666 // Skip the escaped character.
667 // FIXME: UCN's.
668 C = getAndAdvanceChar(CurPtr, Result);
669 }
670
671 if (C && C != '\n' && C != '\r' && CurPtr[0] == '\'') {
672 ++CurPtr;
673 } else {
674 // Fall back on generic code for embedded nulls, newlines, wide chars.
675 do {
676 // Skip escaped characters.
677 if (C == '\\') {
678 // Skip the escaped character.
679 C = getAndAdvanceChar(CurPtr, Result);
680 } else if (C == '\n' || C == '\r' || // Newline.
681 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
Chris Lattner74d15df2008-11-22 02:02:22 +0000682 if (!isLexingRawMode())
683 Diag(BufferPtr, diag::err_unterminated_char);
Chris Lattner9e6293d2008-10-12 04:51:35 +0000684 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
Reid Spencer5f016e22007-07-11 17:01:13 +0000685 return;
686 } else if (C == 0) {
687 NulCharacter = CurPtr-1;
688 }
689 C = getAndAdvanceChar(CurPtr, Result);
690 } while (C != '\'');
691 }
692
Chris Lattner74d15df2008-11-22 02:02:22 +0000693 if (NulCharacter && !isLexingRawMode())
694 Diag(NulCharacter, diag::null_in_char);
Reid Spencer5f016e22007-07-11 17:01:13 +0000695
Reid Spencer5f016e22007-07-11 17:01:13 +0000696 // Update the location of token as well as BufferPtr.
Chris Lattner9e6293d2008-10-12 04:51:35 +0000697 FormTokenWithChars(Result, CurPtr, tok::char_constant);
Reid Spencer5f016e22007-07-11 17:01:13 +0000698}
699
700/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
701/// Update BufferPtr to point to the next non-whitespace character and return.
Chris Lattnerd88dc482008-10-12 04:05:48 +0000702///
703/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
704///
705bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000706 // Whitespace - Skip it, then return the token after the whitespace.
707 unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently.
708 while (1) {
709 // Skip horizontal whitespace very aggressively.
710 while (isHorizontalWhitespace(Char))
711 Char = *++CurPtr;
712
Daniel Dunbarddd3e8b2008-11-25 00:20:22 +0000713 // Otherwise if we have something other than whitespace, we're done.
Reid Spencer5f016e22007-07-11 17:01:13 +0000714 if (Char != '\n' && Char != '\r')
715 break;
716
717 if (ParsingPreprocessorDirective) {
718 // End of preprocessor directive line, let LexTokenInternal handle this.
719 BufferPtr = CurPtr;
Chris Lattnerd88dc482008-10-12 04:05:48 +0000720 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000721 }
722
723 // ok, but handle newline.
724 // The returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +0000725 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +0000726 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +0000727 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +0000728 Char = *++CurPtr;
729 }
730
731 // If this isn't immediately after a newline, there is leading space.
732 char PrevChar = CurPtr[-1];
733 if (PrevChar != '\n' && PrevChar != '\r')
Chris Lattnerd2177732007-07-20 16:59:19 +0000734 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +0000735
Chris Lattnerd88dc482008-10-12 04:05:48 +0000736 // If the client wants us to return whitespace, return it now.
737 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +0000738 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattnerd88dc482008-10-12 04:05:48 +0000739 return true;
740 }
741
Reid Spencer5f016e22007-07-11 17:01:13 +0000742 BufferPtr = CurPtr;
Chris Lattnerd88dc482008-10-12 04:05:48 +0000743 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000744}
745
746// SkipBCPLComment - We have just read the // characters from input. Skip until
747// we find the newline character thats terminate the comment. Then update
Chris Lattner2d381892008-10-12 04:15:42 +0000748/// BufferPtr and return. If we're in KeepCommentMode, this will form the token
749/// and return true.
Chris Lattnerd2177732007-07-20 16:59:19 +0000750bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000751 // If BCPL comments aren't explicitly enabled for this language, emit an
752 // extension warning.
Chris Lattner74d15df2008-11-22 02:02:22 +0000753 if (!Features.BCPLComment && !isLexingRawMode()) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000754 Diag(BufferPtr, diag::ext_bcpl_comment);
755
756 // Mark them enabled so we only emit one warning for this translation
757 // unit.
758 Features.BCPLComment = true;
759 }
760
761 // Scan over the body of the comment. The common case, when scanning, is that
762 // the comment contains normal ascii characters with nothing interesting in
763 // them. As such, optimize for this case with the inner loop.
764 char C;
765 do {
766 C = *CurPtr;
767 // FIXME: Speedup BCPL comment lexing. Just scan for a \n or \r character.
768 // If we find a \n character, scan backwards, checking to see if it's an
769 // escaped newline, like we do for block comments.
770
771 // Skip over characters in the fast loop.
772 while (C != 0 && // Potentially EOF.
773 C != '\\' && // Potentially escaped newline.
774 C != '?' && // Potentially trigraph.
775 C != '\n' && C != '\r') // Newline or DOS-style newline.
776 C = *++CurPtr;
777
778 // If this is a newline, we're done.
779 if (C == '\n' || C == '\r')
780 break; // Found the newline? Break out!
781
782 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to
Chris Lattnerbc3e9842008-12-12 07:34:39 +0000783 // properly decode the character. Read it in raw mode to avoid emitting
784 // diagnostics about things like trigraphs. If we see an escaped newline,
785 // we'll handle it below.
Reid Spencer5f016e22007-07-11 17:01:13 +0000786 const char *OldPtr = CurPtr;
Chris Lattnerbc3e9842008-12-12 07:34:39 +0000787 bool OldRawMode = isLexingRawMode();
788 LexingRawMode = true;
Reid Spencer5f016e22007-07-11 17:01:13 +0000789 C = getAndAdvanceChar(CurPtr, Result);
Chris Lattnerbc3e9842008-12-12 07:34:39 +0000790 LexingRawMode = OldRawMode;
Reid Spencer5f016e22007-07-11 17:01:13 +0000791
792 // If we read multiple characters, and one of those characters was a \r or
793 // \n, then we had an escaped newline within the comment. Emit diagnostic
794 // unless the next line is also a // comment.
795 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') {
796 for (; OldPtr != CurPtr; ++OldPtr)
797 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
798 // Okay, we found a // comment that ends in a newline, if the next
799 // line is also a // comment, but has spaces, don't emit a diagnostic.
800 if (isspace(C)) {
801 const char *ForwardPtr = CurPtr;
802 while (isspace(*ForwardPtr)) // Skip whitespace.
803 ++ForwardPtr;
804 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
805 break;
806 }
807
Chris Lattner74d15df2008-11-22 02:02:22 +0000808 if (!isLexingRawMode())
809 Diag(OldPtr-1, diag::ext_multi_line_bcpl_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +0000810 break;
811 }
812 }
813
814 if (CurPtr == BufferEnd+1) { --CurPtr; break; }
815 } while (C != '\n' && C != '\r');
816
817 // Found but did not consume the newline.
818
819 // If we are returning comments as tokens, return this comment as a token.
Chris Lattnerfa95a012008-10-12 03:22:02 +0000820 if (inKeepCommentMode())
Reid Spencer5f016e22007-07-11 17:01:13 +0000821 return SaveBCPLComment(Result, CurPtr);
822
823 // If we are inside a preprocessor directive and we see the end of line,
824 // return immediately, so that the lexer can return this as an EOM token.
825 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
826 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +0000827 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000828 }
829
830 // Otherwise, eat the \n character. We don't care if this is a \n\r or
Chris Lattner7a4f0042008-10-12 00:23:07 +0000831 // \r\n sequence. This is an efficiency hack (because we know the \n can't
Chris Lattnerd88dc482008-10-12 04:05:48 +0000832 // contribute to another token), it isn't needed for correctness. Note that
833 // this is ok even in KeepWhitespaceMode, because we would have returned the
834 /// comment above in that mode.
Reid Spencer5f016e22007-07-11 17:01:13 +0000835 ++CurPtr;
836
837 // The next returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +0000838 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +0000839 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +0000840 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +0000841 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +0000842 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000843}
844
845/// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in
846/// an appropriate way and return it.
Chris Lattnerd2177732007-07-20 16:59:19 +0000847bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
Chris Lattner9e6293d2008-10-12 04:51:35 +0000848 // If we're not in a preprocessor directive, just return the // comment
849 // directly.
850 FormTokenWithChars(Result, CurPtr, tok::comment);
Reid Spencer5f016e22007-07-11 17:01:13 +0000851
Chris Lattner9e6293d2008-10-12 04:51:35 +0000852 if (!ParsingPreprocessorDirective)
853 return true;
854
855 // If this BCPL-style comment is in a macro definition, transmogrify it into
856 // a C-style block comment.
857 std::string Spelling = PP->getSpelling(Result);
858 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not bcpl comment?");
859 Spelling[1] = '*'; // Change prefix to "/*".
860 Spelling += "*/"; // add suffix.
861
862 Result.setKind(tok::comment);
863 Result.setLocation(PP->CreateString(&Spelling[0], Spelling.size(),
864 Result.getLocation()));
865 Result.setLength(Spelling.size());
Chris Lattner2d381892008-10-12 04:15:42 +0000866 return true;
Reid Spencer5f016e22007-07-11 17:01:13 +0000867}
868
869/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
870/// character (either \n or \r) is part of an escaped newline sequence. Issue a
Chris Lattner47a2b402008-12-12 07:14:34 +0000871/// diagnostic if so. We know that the newline is inside of a block comment.
Reid Spencer5f016e22007-07-11 17:01:13 +0000872static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
873 Lexer *L) {
874 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
875
876 // Back up off the newline.
877 --CurPtr;
878
879 // If this is a two-character newline sequence, skip the other character.
880 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
881 // \n\n or \r\r -> not escaped newline.
882 if (CurPtr[0] == CurPtr[1])
883 return false;
884 // \n\r or \r\n -> skip the newline.
885 --CurPtr;
886 }
887
888 // If we have horizontal whitespace, skip over it. We allow whitespace
889 // between the slash and newline.
890 bool HasSpace = false;
891 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) {
892 --CurPtr;
893 HasSpace = true;
894 }
895
896 // If we have a slash, we know this is an escaped newline.
897 if (*CurPtr == '\\') {
898 if (CurPtr[-1] != '*') return false;
899 } else {
900 // It isn't a slash, is it the ?? / trigraph?
901 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' ||
902 CurPtr[-3] != '*')
903 return false;
904
905 // This is the trigraph ending the comment. Emit a stern warning!
906 CurPtr -= 2;
907
908 // If no trigraphs are enabled, warn that we ignored this trigraph and
909 // ignore this * character.
910 if (!L->getFeatures().Trigraphs) {
Chris Lattner74d15df2008-11-22 02:02:22 +0000911 if (!L->isLexingRawMode())
912 L->Diag(CurPtr, diag::trigraph_ignored_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +0000913 return false;
914 }
Chris Lattner74d15df2008-11-22 02:02:22 +0000915 if (!L->isLexingRawMode())
916 L->Diag(CurPtr, diag::trigraph_ends_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +0000917 }
918
919 // Warn about having an escaped newline between the */ characters.
Chris Lattner74d15df2008-11-22 02:02:22 +0000920 if (!L->isLexingRawMode())
921 L->Diag(CurPtr, diag::escaped_newline_block_comment_end);
Reid Spencer5f016e22007-07-11 17:01:13 +0000922
923 // If there was space between the backslash and newline, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +0000924 if (HasSpace && !L->isLexingRawMode())
925 L->Diag(CurPtr, diag::backslash_newline_space);
Reid Spencer5f016e22007-07-11 17:01:13 +0000926
927 return true;
928}
929
930#ifdef __SSE2__
931#include <emmintrin.h>
932#elif __ALTIVEC__
933#include <altivec.h>
934#undef bool
935#endif
936
937/// SkipBlockComment - We have just read the /* characters from input. Read
938/// until we find the */ characters that terminate the comment. Note that we
939/// don't bother decoding trigraphs or escaped newlines in block comments,
940/// because they cannot cause the comment to end. The only thing that can
941/// happen is the comment could end with an escaped newline between the */ end
942/// of comment.
Chris Lattner2d381892008-10-12 04:15:42 +0000943///
944/// If KeepCommentMode is enabled, this forms a token from the comment and
945/// returns true.
Chris Lattnerd2177732007-07-20 16:59:19 +0000946bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000947 // Scan one character past where we should, looking for a '/' character. Once
948 // we find it, check to see if it was preceeded by a *. This common
949 // optimization helps people who like to put a lot of * characters in their
950 // comments.
Chris Lattner8146b682007-07-21 23:43:37 +0000951
952 // The first character we get with newlines and trigraphs skipped to handle
953 // the degenerate /*/ case below correctly if the * has an escaped newline
954 // after it.
955 unsigned CharSize;
956 unsigned char C = getCharAndSize(CurPtr, CharSize);
957 CurPtr += CharSize;
Reid Spencer5f016e22007-07-11 17:01:13 +0000958 if (C == 0 && CurPtr == BufferEnd+1) {
Chris Lattner74d15df2008-11-22 02:02:22 +0000959 if (!isLexingRawMode())
Chris Lattner0af57422008-10-12 01:31:51 +0000960 Diag(BufferPtr, diag::err_unterminated_block_comment);
Chris Lattner31f0eca2008-10-12 04:19:49 +0000961 --CurPtr;
962
963 // KeepWhitespaceMode should return this broken comment as a token. Since
964 // it isn't a well formed comment, just return it as an 'unknown' token.
965 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +0000966 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattner31f0eca2008-10-12 04:19:49 +0000967 return true;
968 }
969
970 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +0000971 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +0000972 }
973
Chris Lattner8146b682007-07-21 23:43:37 +0000974 // Check to see if the first character after the '/*' is another /. If so,
975 // then this slash does not end the block comment, it is part of it.
976 if (C == '/')
977 C = *CurPtr++;
978
Reid Spencer5f016e22007-07-11 17:01:13 +0000979 while (1) {
980 // Skip over all non-interesting characters until we find end of buffer or a
981 // (probably ending) '/' character.
982 if (CurPtr + 24 < BufferEnd) {
983 // While not aligned to a 16-byte boundary.
984 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0)
985 C = *CurPtr++;
986
987 if (C == '/') goto FoundSlash;
988
989#ifdef __SSE2__
990 __m128i Slashes = _mm_set_epi8('/', '/', '/', '/', '/', '/', '/', '/',
991 '/', '/', '/', '/', '/', '/', '/', '/');
992 while (CurPtr+16 <= BufferEnd &&
993 _mm_movemask_epi8(_mm_cmpeq_epi8(*(__m128i*)CurPtr, Slashes)) == 0)
994 CurPtr += 16;
995#elif __ALTIVEC__
996 __vector unsigned char Slashes = {
997 '/', '/', '/', '/', '/', '/', '/', '/',
998 '/', '/', '/', '/', '/', '/', '/', '/'
999 };
1000 while (CurPtr+16 <= BufferEnd &&
1001 !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes))
1002 CurPtr += 16;
1003#else
1004 // Scan for '/' quickly. Many block comments are very large.
1005 while (CurPtr[0] != '/' &&
1006 CurPtr[1] != '/' &&
1007 CurPtr[2] != '/' &&
1008 CurPtr[3] != '/' &&
1009 CurPtr+4 < BufferEnd) {
1010 CurPtr += 4;
1011 }
1012#endif
1013
1014 // It has to be one of the bytes scanned, increment to it and read one.
1015 C = *CurPtr++;
1016 }
1017
1018 // Loop to scan the remainder.
1019 while (C != '/' && C != '\0')
1020 C = *CurPtr++;
1021
1022 FoundSlash:
1023 if (C == '/') {
1024 if (CurPtr[-2] == '*') // We found the final */. We're done!
1025 break;
1026
1027 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
1028 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) {
1029 // We found the final */, though it had an escaped newline between the
1030 // * and /. We're done!
1031 break;
1032 }
1033 }
1034 if (CurPtr[0] == '*' && CurPtr[1] != '/') {
1035 // If this is a /* inside of the comment, emit a warning. Don't do this
1036 // if this is a /*/, which will end the comment. This misses cases with
1037 // embedded escaped newlines, but oh well.
Chris Lattner74d15df2008-11-22 02:02:22 +00001038 if (!isLexingRawMode())
1039 Diag(CurPtr-1, diag::warn_nested_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00001040 }
1041 } else if (C == 0 && CurPtr == BufferEnd+1) {
Chris Lattner74d15df2008-11-22 02:02:22 +00001042 if (!isLexingRawMode())
1043 Diag(BufferPtr, diag::err_unterminated_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00001044 // Note: the user probably forgot a */. We could continue immediately
1045 // after the /*, but this would involve lexing a lot of what really is the
1046 // comment, which surely would confuse the parser.
Chris Lattner31f0eca2008-10-12 04:19:49 +00001047 --CurPtr;
1048
1049 // KeepWhitespaceMode should return this broken comment as a token. Since
1050 // it isn't a well formed comment, just return it as an 'unknown' token.
1051 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001052 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattner31f0eca2008-10-12 04:19:49 +00001053 return true;
1054 }
1055
1056 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00001057 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001058 }
1059 C = *CurPtr++;
1060 }
1061
1062 // If we are returning comments as tokens, return this comment as a token.
Chris Lattnerfa95a012008-10-12 03:22:02 +00001063 if (inKeepCommentMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001064 FormTokenWithChars(Result, CurPtr, tok::comment);
Chris Lattner2d381892008-10-12 04:15:42 +00001065 return true;
Reid Spencer5f016e22007-07-11 17:01:13 +00001066 }
1067
1068 // It is common for the tokens immediately after a /**/ comment to be
1069 // whitespace. Instead of going through the big switch, handle it
Chris Lattnerd88dc482008-10-12 04:05:48 +00001070 // efficiently now. This is safe even in KeepWhitespaceMode because we would
1071 // have already returned above with the comment as a token.
Reid Spencer5f016e22007-07-11 17:01:13 +00001072 if (isHorizontalWhitespace(*CurPtr)) {
Chris Lattnerd2177732007-07-20 16:59:19 +00001073 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001074 SkipWhitespace(Result, CurPtr+1);
Chris Lattner2d381892008-10-12 04:15:42 +00001075 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001076 }
1077
1078 // Otherwise, just return so that the next character will be lexed as a token.
1079 BufferPtr = CurPtr;
Chris Lattnerd2177732007-07-20 16:59:19 +00001080 Result.setFlag(Token::LeadingSpace);
Chris Lattner2d381892008-10-12 04:15:42 +00001081 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001082}
1083
1084//===----------------------------------------------------------------------===//
1085// Primary Lexing Entry Points
1086//===----------------------------------------------------------------------===//
1087
Reid Spencer5f016e22007-07-11 17:01:13 +00001088/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
1089/// uninterpreted string. This switches the lexer out of directive mode.
1090std::string Lexer::ReadToEndOfLine() {
1091 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
1092 "Must be in a preprocessing directive!");
1093 std::string Result;
Chris Lattnerd2177732007-07-20 16:59:19 +00001094 Token Tmp;
Reid Spencer5f016e22007-07-11 17:01:13 +00001095
1096 // CurPtr - Cache BufferPtr in an automatic variable.
1097 const char *CurPtr = BufferPtr;
1098 while (1) {
1099 char Char = getAndAdvanceChar(CurPtr, Tmp);
1100 switch (Char) {
1101 default:
1102 Result += Char;
1103 break;
1104 case 0: // Null.
1105 // Found end of file?
1106 if (CurPtr-1 != BufferEnd) {
1107 // Nope, normal character, continue.
1108 Result += Char;
1109 break;
1110 }
1111 // FALL THROUGH.
1112 case '\r':
1113 case '\n':
1114 // Okay, we found the end of the line. First, back up past the \0, \r, \n.
1115 assert(CurPtr[-1] == Char && "Trigraphs for newline?");
1116 BufferPtr = CurPtr-1;
1117
1118 // Next, lex the character, which should handle the EOM transition.
1119 Lex(Tmp);
Chris Lattner22f6bbc2007-10-09 18:02:16 +00001120 assert(Tmp.is(tok::eom) && "Unexpected token!");
Reid Spencer5f016e22007-07-11 17:01:13 +00001121
1122 // Finally, we're done, return the string we found.
1123 return Result;
1124 }
1125 }
1126}
1127
1128/// LexEndOfFile - CurPtr points to the end of this file. Handle this
1129/// condition, reporting diagnostics and handling other edge cases as required.
1130/// This returns true if Result contains a token, false if PP.Lex should be
1131/// called again.
Chris Lattnerd2177732007-07-20 16:59:19 +00001132bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001133 // If we hit the end of the file while parsing a preprocessor directive,
1134 // end the preprocessor directive first. The next token returned will
1135 // then be the end of file.
1136 if (ParsingPreprocessorDirective) {
1137 // Done parsing the "line".
1138 ParsingPreprocessorDirective = false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001139 // Update the location of token as well as BufferPtr.
Chris Lattner9e6293d2008-10-12 04:51:35 +00001140 FormTokenWithChars(Result, CurPtr, tok::eom);
Reid Spencer5f016e22007-07-11 17:01:13 +00001141
1142 // Restore comment saving mode, in case it was disabled for directive.
Chris Lattnerf744d132008-10-12 03:27:19 +00001143 SetCommentRetentionState(PP->getCommentRetentionState());
Reid Spencer5f016e22007-07-11 17:01:13 +00001144 return true; // Have a token.
1145 }
1146
1147 // If we are in raw mode, return this event as an EOF token. Let the caller
1148 // that put us in raw mode handle the event.
Chris Lattner74d15df2008-11-22 02:02:22 +00001149 if (isLexingRawMode()) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001150 Result.startToken();
1151 BufferPtr = BufferEnd;
Chris Lattner9e6293d2008-10-12 04:51:35 +00001152 FormTokenWithChars(Result, BufferEnd, tok::eof);
Reid Spencer5f016e22007-07-11 17:01:13 +00001153 return true;
1154 }
1155
1156 // Otherwise, issue diagnostics for unterminated #if and missing newline.
1157
1158 // If we are in a #if directive, emit an error.
1159 while (!ConditionalStack.empty()) {
Chris Lattner30c64762008-11-22 06:22:39 +00001160 PP->Diag(ConditionalStack.back().IfLoc,
1161 diag::err_pp_unterminated_conditional);
Reid Spencer5f016e22007-07-11 17:01:13 +00001162 ConditionalStack.pop_back();
1163 }
1164
Chris Lattnerb25e5d72008-04-12 05:54:25 +00001165 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue
1166 // a pedwarn.
1167 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r'))
Reid Spencer5f016e22007-07-11 17:01:13 +00001168 Diag(BufferEnd, diag::ext_no_newline_eof);
1169
1170 BufferPtr = CurPtr;
1171
1172 // Finally, let the preprocessor handle this.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001173 return PP->HandleEndOfFile(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00001174}
1175
1176/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
1177/// the specified lexer will return a tok::l_paren token, 0 if it is something
1178/// else and 2 if there are no more tokens in the buffer controlled by the
1179/// lexer.
1180unsigned Lexer::isNextPPTokenLParen() {
1181 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
1182
1183 // Switch to 'skipping' mode. This will ensure that we can lex a token
1184 // without emitting diagnostics, disables macro expansion, and will cause EOF
1185 // to return an EOF token instead of popping the include stack.
1186 LexingRawMode = true;
1187
1188 // Save state that can be changed while lexing so that we can restore it.
1189 const char *TmpBufferPtr = BufferPtr;
1190
Chris Lattnerd2177732007-07-20 16:59:19 +00001191 Token Tok;
Reid Spencer5f016e22007-07-11 17:01:13 +00001192 Tok.startToken();
1193 LexTokenInternal(Tok);
1194
1195 // Restore state that may have changed.
1196 BufferPtr = TmpBufferPtr;
1197
1198 // Restore the lexer back to non-skipping mode.
1199 LexingRawMode = false;
1200
Chris Lattner22f6bbc2007-10-09 18:02:16 +00001201 if (Tok.is(tok::eof))
Reid Spencer5f016e22007-07-11 17:01:13 +00001202 return 2;
Chris Lattner22f6bbc2007-10-09 18:02:16 +00001203 return Tok.is(tok::l_paren);
Reid Spencer5f016e22007-07-11 17:01:13 +00001204}
1205
1206
1207/// LexTokenInternal - This implements a simple C family lexer. It is an
1208/// extremely performance critical piece of code. This assumes that the buffer
1209/// has a null character at the end of the file. Return true if an error
1210/// occurred and compilation should terminate, false if normal. This returns a
1211/// preprocessing token, not a normal token, as such, it is an internal
1212/// interface. It assumes that the Flags of result have been cleared before
1213/// calling this.
Chris Lattnerd2177732007-07-20 16:59:19 +00001214void Lexer::LexTokenInternal(Token &Result) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001215LexNextToken:
1216 // New token, can't need cleaning yet.
Chris Lattnerd2177732007-07-20 16:59:19 +00001217 Result.clearFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +00001218 Result.setIdentifierInfo(0);
1219
1220 // CurPtr - Cache BufferPtr in an automatic variable.
1221 const char *CurPtr = BufferPtr;
1222
1223 // Small amounts of horizontal whitespace is very common between tokens.
1224 if ((*CurPtr == ' ') || (*CurPtr == '\t')) {
1225 ++CurPtr;
1226 while ((*CurPtr == ' ') || (*CurPtr == '\t'))
1227 ++CurPtr;
Chris Lattnerd88dc482008-10-12 04:05:48 +00001228
1229 // If we are keeping whitespace and other tokens, just return what we just
1230 // skipped. The next lexer invocation will return the token after the
1231 // whitespace.
1232 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001233 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattnerd88dc482008-10-12 04:05:48 +00001234 return;
1235 }
1236
Reid Spencer5f016e22007-07-11 17:01:13 +00001237 BufferPtr = CurPtr;
Chris Lattnerd2177732007-07-20 16:59:19 +00001238 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001239 }
1240
1241 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below.
1242
1243 // Read a character, advancing over it.
1244 char Char = getAndAdvanceChar(CurPtr, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001245 tok::TokenKind Kind;
1246
Reid Spencer5f016e22007-07-11 17:01:13 +00001247 switch (Char) {
1248 case 0: // Null.
1249 // Found end of file?
1250 if (CurPtr-1 == BufferEnd) {
1251 // Read the PP instance variable into an automatic variable, because
1252 // LexEndOfFile will often delete 'this'.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001253 Preprocessor *PPCache = PP;
Reid Spencer5f016e22007-07-11 17:01:13 +00001254 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file.
1255 return; // Got a token to return.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001256 assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
1257 return PPCache->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00001258 }
1259
Chris Lattner74d15df2008-11-22 02:02:22 +00001260 if (!isLexingRawMode())
1261 Diag(CurPtr-1, diag::null_in_file);
Chris Lattnerd2177732007-07-20 16:59:19 +00001262 Result.setFlag(Token::LeadingSpace);
Chris Lattnerd88dc482008-10-12 04:05:48 +00001263 if (SkipWhitespace(Result, CurPtr))
1264 return; // KeepWhitespaceMode
1265
Reid Spencer5f016e22007-07-11 17:01:13 +00001266 goto LexNextToken; // GCC isn't tail call eliminating.
1267 case '\n':
1268 case '\r':
1269 // If we are inside a preprocessor directive and we see the end of line,
1270 // we know we are done with the directive, so return an EOM token.
1271 if (ParsingPreprocessorDirective) {
1272 // Done parsing the "line".
1273 ParsingPreprocessorDirective = false;
1274
1275 // Restore comment saving mode, in case it was disabled for directive.
Chris Lattnerf744d132008-10-12 03:27:19 +00001276 SetCommentRetentionState(PP->getCommentRetentionState());
Reid Spencer5f016e22007-07-11 17:01:13 +00001277
1278 // Since we consumed a newline, we are back at the start of a line.
1279 IsAtStartOfLine = true;
1280
Chris Lattner9e6293d2008-10-12 04:51:35 +00001281 Kind = tok::eom;
Reid Spencer5f016e22007-07-11 17:01:13 +00001282 break;
1283 }
1284 // The returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +00001285 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00001286 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +00001287 Result.clearFlag(Token::LeadingSpace);
Chris Lattnerd88dc482008-10-12 04:05:48 +00001288
1289 if (SkipWhitespace(Result, CurPtr))
1290 return; // KeepWhitespaceMode
Reid Spencer5f016e22007-07-11 17:01:13 +00001291 goto LexNextToken; // GCC isn't tail call eliminating.
1292 case ' ':
1293 case '\t':
1294 case '\f':
1295 case '\v':
Chris Lattner8133cfc2007-07-22 06:29:05 +00001296 SkipHorizontalWhitespace:
Chris Lattnerd2177732007-07-20 16:59:19 +00001297 Result.setFlag(Token::LeadingSpace);
Chris Lattnerd88dc482008-10-12 04:05:48 +00001298 if (SkipWhitespace(Result, CurPtr))
1299 return; // KeepWhitespaceMode
Chris Lattner8133cfc2007-07-22 06:29:05 +00001300
1301 SkipIgnoredUnits:
1302 CurPtr = BufferPtr;
1303
1304 // If the next token is obviously a // or /* */ comment, skip it efficiently
1305 // too (without going through the big switch stmt).
Chris Lattner8402c732009-01-16 22:39:25 +00001306 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
1307 Features.BCPLComment) {
Chris Lattner8133cfc2007-07-22 06:29:05 +00001308 SkipBCPLComment(Result, CurPtr+2);
1309 goto SkipIgnoredUnits;
Chris Lattnerfa95a012008-10-12 03:22:02 +00001310 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
Chris Lattner8133cfc2007-07-22 06:29:05 +00001311 SkipBlockComment(Result, CurPtr+2);
1312 goto SkipIgnoredUnits;
1313 } else if (isHorizontalWhitespace(*CurPtr)) {
1314 goto SkipHorizontalWhitespace;
1315 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001316 goto LexNextToken; // GCC isn't tail call eliminating.
1317
Chris Lattner3a570772008-01-03 17:58:54 +00001318 // C99 6.4.4.1: Integer Constants.
1319 // C99 6.4.4.2: Floating Constants.
1320 case '0': case '1': case '2': case '3': case '4':
1321 case '5': case '6': case '7': case '8': case '9':
1322 // Notify MIOpt that we read a non-whitespace/non-comment token.
1323 MIOpt.ReadToken();
1324 return LexNumericConstant(Result, CurPtr);
1325
1326 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
Reid Spencer5f016e22007-07-11 17:01:13 +00001327 // Notify MIOpt that we read a non-whitespace/non-comment token.
1328 MIOpt.ReadToken();
1329 Char = getCharAndSize(CurPtr, SizeTmp);
1330
1331 // Wide string literal.
1332 if (Char == '"')
1333 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
1334 true);
1335
1336 // Wide character constant.
1337 if (Char == '\'')
1338 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
1339 // FALL THROUGH, treating L like the start of an identifier.
1340
1341 // C99 6.4.2: Identifiers.
1342 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
1343 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N':
1344 case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U':
1345 case 'V': case 'W': case 'X': case 'Y': case 'Z':
1346 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
1347 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
1348 case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u':
1349 case 'v': case 'w': case 'x': case 'y': case 'z':
1350 case '_':
1351 // Notify MIOpt that we read a non-whitespace/non-comment token.
1352 MIOpt.ReadToken();
1353 return LexIdentifier(Result, CurPtr);
Chris Lattner3a570772008-01-03 17:58:54 +00001354
1355 case '$': // $ in identifiers.
1356 if (Features.DollarIdents) {
Chris Lattner74d15df2008-11-22 02:02:22 +00001357 if (!isLexingRawMode())
1358 Diag(CurPtr-1, diag::ext_dollar_in_identifier);
Chris Lattner3a570772008-01-03 17:58:54 +00001359 // Notify MIOpt that we read a non-whitespace/non-comment token.
1360 MIOpt.ReadToken();
1361 return LexIdentifier(Result, CurPtr);
1362 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001363
Chris Lattner9e6293d2008-10-12 04:51:35 +00001364 Kind = tok::unknown;
Chris Lattner3a570772008-01-03 17:58:54 +00001365 break;
Reid Spencer5f016e22007-07-11 17:01:13 +00001366
1367 // C99 6.4.4: Character Constants.
1368 case '\'':
1369 // Notify MIOpt that we read a non-whitespace/non-comment token.
1370 MIOpt.ReadToken();
1371 return LexCharConstant(Result, CurPtr);
1372
1373 // C99 6.4.5: String Literals.
1374 case '"':
1375 // Notify MIOpt that we read a non-whitespace/non-comment token.
1376 MIOpt.ReadToken();
1377 return LexStringLiteral(Result, CurPtr, false);
1378
1379 // C99 6.4.6: Punctuators.
1380 case '?':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001381 Kind = tok::question;
Reid Spencer5f016e22007-07-11 17:01:13 +00001382 break;
1383 case '[':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001384 Kind = tok::l_square;
Reid Spencer5f016e22007-07-11 17:01:13 +00001385 break;
1386 case ']':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001387 Kind = tok::r_square;
Reid Spencer5f016e22007-07-11 17:01:13 +00001388 break;
1389 case '(':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001390 Kind = tok::l_paren;
Reid Spencer5f016e22007-07-11 17:01:13 +00001391 break;
1392 case ')':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001393 Kind = tok::r_paren;
Reid Spencer5f016e22007-07-11 17:01:13 +00001394 break;
1395 case '{':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001396 Kind = tok::l_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00001397 break;
1398 case '}':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001399 Kind = tok::r_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00001400 break;
1401 case '.':
1402 Char = getCharAndSize(CurPtr, SizeTmp);
1403 if (Char >= '0' && Char <= '9') {
1404 // Notify MIOpt that we read a non-whitespace/non-comment token.
1405 MIOpt.ReadToken();
1406
1407 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
1408 } else if (Features.CPlusPlus && Char == '*') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001409 Kind = tok::periodstar;
Reid Spencer5f016e22007-07-11 17:01:13 +00001410 CurPtr += SizeTmp;
1411 } else if (Char == '.' &&
1412 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001413 Kind = tok::ellipsis;
Reid Spencer5f016e22007-07-11 17:01:13 +00001414 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
1415 SizeTmp2, Result);
1416 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001417 Kind = tok::period;
Reid Spencer5f016e22007-07-11 17:01:13 +00001418 }
1419 break;
1420 case '&':
1421 Char = getCharAndSize(CurPtr, SizeTmp);
1422 if (Char == '&') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001423 Kind = tok::ampamp;
Reid Spencer5f016e22007-07-11 17:01:13 +00001424 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1425 } else if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001426 Kind = tok::ampequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001427 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1428 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001429 Kind = tok::amp;
Reid Spencer5f016e22007-07-11 17:01:13 +00001430 }
1431 break;
1432 case '*':
1433 if (getCharAndSize(CurPtr, SizeTmp) == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001434 Kind = tok::starequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001435 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1436 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001437 Kind = tok::star;
Reid Spencer5f016e22007-07-11 17:01:13 +00001438 }
1439 break;
1440 case '+':
1441 Char = getCharAndSize(CurPtr, SizeTmp);
1442 if (Char == '+') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001443 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001444 Kind = tok::plusplus;
Reid Spencer5f016e22007-07-11 17:01:13 +00001445 } else if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001446 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001447 Kind = tok::plusequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001448 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001449 Kind = tok::plus;
Reid Spencer5f016e22007-07-11 17:01:13 +00001450 }
1451 break;
1452 case '-':
1453 Char = getCharAndSize(CurPtr, SizeTmp);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001454 if (Char == '-') { // --
Reid Spencer5f016e22007-07-11 17:01:13 +00001455 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001456 Kind = tok::minusminus;
Reid Spencer5f016e22007-07-11 17:01:13 +00001457 } else if (Char == '>' && Features.CPlusPlus &&
Chris Lattner9e6293d2008-10-12 04:51:35 +00001458 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->*
Reid Spencer5f016e22007-07-11 17:01:13 +00001459 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
1460 SizeTmp2, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001461 Kind = tok::arrowstar;
1462 } else if (Char == '>') { // ->
Reid Spencer5f016e22007-07-11 17:01:13 +00001463 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001464 Kind = tok::arrow;
1465 } else if (Char == '=') { // -=
Reid Spencer5f016e22007-07-11 17:01:13 +00001466 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001467 Kind = tok::minusequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001468 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001469 Kind = tok::minus;
Reid Spencer5f016e22007-07-11 17:01:13 +00001470 }
1471 break;
1472 case '~':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001473 Kind = tok::tilde;
Reid Spencer5f016e22007-07-11 17:01:13 +00001474 break;
1475 case '!':
1476 if (getCharAndSize(CurPtr, SizeTmp) == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001477 Kind = tok::exclaimequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001478 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1479 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001480 Kind = tok::exclaim;
Reid Spencer5f016e22007-07-11 17:01:13 +00001481 }
1482 break;
1483 case '/':
1484 // 6.4.9: Comments
1485 Char = getCharAndSize(CurPtr, SizeTmp);
1486 if (Char == '/') { // BCPL comment.
Chris Lattner8402c732009-01-16 22:39:25 +00001487 // Even if BCPL comments are disabled (e.g. in C89 mode), we generally
1488 // want to lex this as a comment. There is one problem with this though,
1489 // that in one particular corner case, this can change the behavior of the
1490 // resultant program. For example, In "foo //**/ bar", C89 would lex
1491 // this as "foo / bar" and langauges with BCPL comments would lex it as
1492 // "foo". Check to see if the character after the second slash is a '*'.
1493 // If so, we will lex that as a "/" instead of the start of a comment.
1494 if (Features.BCPLComment ||
1495 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') {
1496 if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
1497 return; // KeepCommentMode
Chris Lattner2d381892008-10-12 04:15:42 +00001498
Chris Lattner8402c732009-01-16 22:39:25 +00001499 // It is common for the tokens immediately after a // comment to be
1500 // whitespace (indentation for the next line). Instead of going through
1501 // the big switch, handle it efficiently now.
1502 goto SkipIgnoredUnits;
1503 }
1504 }
1505
1506 if (Char == '*') { // /**/ comment.
Reid Spencer5f016e22007-07-11 17:01:13 +00001507 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
Chris Lattner2d381892008-10-12 04:15:42 +00001508 return; // KeepCommentMode
1509 goto LexNextToken; // GCC isn't tail call eliminating.
Chris Lattner8402c732009-01-16 22:39:25 +00001510 }
1511
1512 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001513 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001514 Kind = tok::slashequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001515 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001516 Kind = tok::slash;
Reid Spencer5f016e22007-07-11 17:01:13 +00001517 }
1518 break;
1519 case '%':
1520 Char = getCharAndSize(CurPtr, SizeTmp);
1521 if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001522 Kind = tok::percentequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001523 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1524 } else if (Features.Digraphs && Char == '>') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001525 Kind = tok::r_brace; // '%>' -> '}'
Reid Spencer5f016e22007-07-11 17:01:13 +00001526 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1527 } else if (Features.Digraphs && Char == ':') {
1528 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1529 Char = getCharAndSize(CurPtr, SizeTmp);
1530 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001531 Kind = tok::hashhash; // '%:%:' -> '##'
Reid Spencer5f016e22007-07-11 17:01:13 +00001532 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
1533 SizeTmp2, Result);
1534 } else if (Char == '@' && Features.Microsoft) { // %:@ -> #@ -> Charize
Reid Spencer5f016e22007-07-11 17:01:13 +00001535 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner74d15df2008-11-22 02:02:22 +00001536 if (!isLexingRawMode())
1537 Diag(BufferPtr, diag::charize_microsoft_ext);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001538 Kind = tok::hashat;
Reid Spencer5f016e22007-07-11 17:01:13 +00001539 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001540 Kind = tok::hash; // '%:' -> '#'
Reid Spencer5f016e22007-07-11 17:01:13 +00001541
1542 // We parsed a # character. If this occurs at the start of the line,
1543 // it's actually the start of a preprocessing directive. Callback to
1544 // the preprocessor to handle it.
1545 // FIXME: -fpreprocessed mode??
1546 if (Result.isAtStartOfLine() && !LexingRawMode) {
1547 BufferPtr = CurPtr;
Chris Lattner168ae2d2007-10-17 20:41:00 +00001548 PP->HandleDirective(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00001549
1550 // As an optimization, if the preprocessor didn't switch lexers, tail
1551 // recurse.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001552 if (PP->isCurrentLexer(this)) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001553 // Start a new token. If this is a #include or something, the PP may
1554 // want us starting at the beginning of the line again. If so, set
1555 // the StartOfLine flag.
1556 if (IsAtStartOfLine) {
Chris Lattnerd2177732007-07-20 16:59:19 +00001557 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00001558 IsAtStartOfLine = false;
1559 }
1560 goto LexNextToken; // GCC isn't tail call eliminating.
1561 }
1562
Chris Lattner168ae2d2007-10-17 20:41:00 +00001563 return PP->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00001564 }
1565 }
1566 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001567 Kind = tok::percent;
Reid Spencer5f016e22007-07-11 17:01:13 +00001568 }
1569 break;
1570 case '<':
1571 Char = getCharAndSize(CurPtr, SizeTmp);
1572 if (ParsingFilename) {
1573 return LexAngledStringLiteral(Result, CurPtr+SizeTmp);
1574 } else if (Char == '<' &&
1575 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001576 Kind = tok::lesslessequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001577 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
1578 SizeTmp2, Result);
1579 } else if (Char == '<') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001580 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001581 Kind = tok::lessless;
Reid Spencer5f016e22007-07-11 17:01:13 +00001582 } else if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001583 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001584 Kind = tok::lessequal;
1585 } else if (Features.Digraphs && Char == ':') { // '<:' -> '['
Reid Spencer5f016e22007-07-11 17:01:13 +00001586 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001587 Kind = tok::l_square;
1588 } else if (Features.Digraphs && Char == '%') { // '<%' -> '{'
Reid Spencer5f016e22007-07-11 17:01:13 +00001589 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001590 Kind = tok::l_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00001591 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001592 Kind = tok::less;
Reid Spencer5f016e22007-07-11 17:01:13 +00001593 }
1594 break;
1595 case '>':
1596 Char = getCharAndSize(CurPtr, SizeTmp);
1597 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001598 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001599 Kind = tok::greaterequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001600 } else if (Char == '>' &&
1601 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001602 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
1603 SizeTmp2, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001604 Kind = tok::greatergreaterequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001605 } else if (Char == '>') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001606 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001607 Kind = tok::greatergreater;
Reid Spencer5f016e22007-07-11 17:01:13 +00001608 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001609 Kind = tok::greater;
Reid Spencer5f016e22007-07-11 17:01:13 +00001610 }
1611 break;
1612 case '^':
1613 Char = getCharAndSize(CurPtr, SizeTmp);
1614 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00001615 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001616 Kind = tok::caretequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001617 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001618 Kind = tok::caret;
Reid Spencer5f016e22007-07-11 17:01:13 +00001619 }
1620 break;
1621 case '|':
1622 Char = getCharAndSize(CurPtr, SizeTmp);
1623 if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001624 Kind = tok::pipeequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001625 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1626 } else if (Char == '|') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001627 Kind = tok::pipepipe;
Reid Spencer5f016e22007-07-11 17:01:13 +00001628 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1629 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001630 Kind = tok::pipe;
Reid Spencer5f016e22007-07-11 17:01:13 +00001631 }
1632 break;
1633 case ':':
1634 Char = getCharAndSize(CurPtr, SizeTmp);
1635 if (Features.Digraphs && Char == '>') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001636 Kind = tok::r_square; // ':>' -> ']'
Reid Spencer5f016e22007-07-11 17:01:13 +00001637 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1638 } else if (Features.CPlusPlus && Char == ':') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001639 Kind = tok::coloncolon;
Reid Spencer5f016e22007-07-11 17:01:13 +00001640 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1641 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001642 Kind = tok::colon;
Reid Spencer5f016e22007-07-11 17:01:13 +00001643 }
1644 break;
1645 case ';':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001646 Kind = tok::semi;
Reid Spencer5f016e22007-07-11 17:01:13 +00001647 break;
1648 case '=':
1649 Char = getCharAndSize(CurPtr, SizeTmp);
1650 if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001651 Kind = tok::equalequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001652 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1653 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001654 Kind = tok::equal;
Reid Spencer5f016e22007-07-11 17:01:13 +00001655 }
1656 break;
1657 case ',':
Chris Lattner9e6293d2008-10-12 04:51:35 +00001658 Kind = tok::comma;
Reid Spencer5f016e22007-07-11 17:01:13 +00001659 break;
1660 case '#':
1661 Char = getCharAndSize(CurPtr, SizeTmp);
1662 if (Char == '#') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001663 Kind = tok::hashhash;
Reid Spencer5f016e22007-07-11 17:01:13 +00001664 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1665 } else if (Char == '@' && Features.Microsoft) { // #@ -> Charize
Chris Lattner9e6293d2008-10-12 04:51:35 +00001666 Kind = tok::hashat;
Chris Lattner74d15df2008-11-22 02:02:22 +00001667 if (!isLexingRawMode())
1668 Diag(BufferPtr, diag::charize_microsoft_ext);
Reid Spencer5f016e22007-07-11 17:01:13 +00001669 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
1670 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001671 Kind = tok::hash;
Reid Spencer5f016e22007-07-11 17:01:13 +00001672 // We parsed a # character. If this occurs at the start of the line,
1673 // it's actually the start of a preprocessing directive. Callback to
1674 // the preprocessor to handle it.
1675 // FIXME: -fpreprocessed mode??
1676 if (Result.isAtStartOfLine() && !LexingRawMode) {
1677 BufferPtr = CurPtr;
Chris Lattner168ae2d2007-10-17 20:41:00 +00001678 PP->HandleDirective(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00001679
1680 // As an optimization, if the preprocessor didn't switch lexers, tail
1681 // recurse.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001682 if (PP->isCurrentLexer(this)) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001683 // Start a new token. If this is a #include or something, the PP may
1684 // want us starting at the beginning of the line again. If so, set
1685 // the StartOfLine flag.
1686 if (IsAtStartOfLine) {
Chris Lattnerd2177732007-07-20 16:59:19 +00001687 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00001688 IsAtStartOfLine = false;
1689 }
1690 goto LexNextToken; // GCC isn't tail call eliminating.
1691 }
Chris Lattner168ae2d2007-10-17 20:41:00 +00001692 return PP->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00001693 }
1694 }
1695 break;
1696
Chris Lattner3a570772008-01-03 17:58:54 +00001697 case '@':
1698 // Objective C support.
1699 if (CurPtr[-1] == '@' && Features.ObjC1)
Chris Lattner9e6293d2008-10-12 04:51:35 +00001700 Kind = tok::at;
Chris Lattner3a570772008-01-03 17:58:54 +00001701 else
Chris Lattner9e6293d2008-10-12 04:51:35 +00001702 Kind = tok::unknown;
Chris Lattner3a570772008-01-03 17:58:54 +00001703 break;
1704
Reid Spencer5f016e22007-07-11 17:01:13 +00001705 case '\\':
1706 // FIXME: UCN's.
1707 // FALL THROUGH.
1708 default:
Chris Lattner9e6293d2008-10-12 04:51:35 +00001709 Kind = tok::unknown;
Reid Spencer5f016e22007-07-11 17:01:13 +00001710 break;
1711 }
1712
1713 // Notify MIOpt that we read a non-whitespace/non-comment token.
1714 MIOpt.ReadToken();
1715
1716 // Update the location of token as well as BufferPtr.
Chris Lattner9e6293d2008-10-12 04:51:35 +00001717 FormTokenWithChars(Result, CurPtr, Kind);
Reid Spencer5f016e22007-07-11 17:01:13 +00001718}