blob: 8ae6e98a4f3eed2b568631ab084c69e378298c1c [file] [log] [blame]
Reid Spencer5f016e22007-07-11 17:01:13 +00001//===--- Lexer.cpp - C Language Family Lexer ------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner0bc735f2007-12-29 19:59:25 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Reid Spencer5f016e22007-07-11 17:01:13 +00007//
8//===----------------------------------------------------------------------===//
9//
Chris Lattnerd2177732007-07-20 16:59:19 +000010// This file implements the Lexer and Token interfaces.
Reid Spencer5f016e22007-07-11 17:01:13 +000011//
12//===----------------------------------------------------------------------===//
13//
14// TODO: GCC Diagnostics emitted by the lexer:
15// PEDWARN: (form feed|vertical tab) in preprocessing directive
16//
17// Universal characters, unicode, char mapping:
18// WARNING: `%.*s' is not in NFKC
19// WARNING: `%.*s' is not in NFC
20//
21// Other:
22// TODO: Options to support:
23// -fexec-charset,-fwide-exec-charset
24//
25//===----------------------------------------------------------------------===//
26
27#include "clang/Lex/Lexer.h"
28#include "clang/Lex/Preprocessor.h"
Chris Lattner500d3292009-01-29 05:15:15 +000029#include "clang/Lex/LexDiagnostic.h"
Douglas Gregor55817af2010-08-25 17:04:25 +000030#include "clang/Lex/CodeCompletionHandler.h"
Chris Lattner9dc1f532007-07-20 16:37:10 +000031#include "clang/Basic/SourceManager.h"
Douglas Gregorf033f1d2010-07-20 20:18:03 +000032#include "llvm/ADT/StringSwitch.h"
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +000033#include "llvm/ADT/STLExtras.h"
Chris Lattner409a0362007-07-22 18:38:25 +000034#include "llvm/Support/Compiler.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000035#include "llvm/Support/MemoryBuffer.h"
Craig Topper2fa4e862011-08-11 04:06:15 +000036#include <cstring>
Reid Spencer5f016e22007-07-11 17:01:13 +000037using namespace clang;
38
Chris Lattnera2bf1052009-12-17 05:29:40 +000039static void InitCharacterInfo();
Reid Spencer5f016e22007-07-11 17:01:13 +000040
Chris Lattnerdbf388b2007-10-07 08:47:24 +000041//===----------------------------------------------------------------------===//
42// Token Class Implementation
43//===----------------------------------------------------------------------===//
44
Mike Stump1eb44332009-09-09 15:08:12 +000045/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
Chris Lattnerdbf388b2007-10-07 08:47:24 +000046bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
Douglas Gregorbec1c9d2008-12-01 21:46:47 +000047 if (IdentifierInfo *II = getIdentifierInfo())
48 return II->getObjCKeywordID() == objcKey;
49 return false;
Chris Lattnerdbf388b2007-10-07 08:47:24 +000050}
51
52/// getObjCKeywordID - Return the ObjC keyword kind.
53tok::ObjCKeywordKind Token::getObjCKeywordID() const {
54 IdentifierInfo *specId = getIdentifierInfo();
55 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
56}
57
Chris Lattner53702cd2007-12-13 01:59:49 +000058
Chris Lattnerdbf388b2007-10-07 08:47:24 +000059//===----------------------------------------------------------------------===//
60// Lexer Class Implementation
61//===----------------------------------------------------------------------===//
62
David Blaikie99ba9e32011-12-20 02:48:34 +000063void Lexer::anchor() { }
64
Mike Stump1eb44332009-09-09 15:08:12 +000065void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
Chris Lattner22d91ca2009-01-17 06:55:17 +000066 const char *BufEnd) {
Chris Lattnera2bf1052009-12-17 05:29:40 +000067 InitCharacterInfo();
Mike Stump1eb44332009-09-09 15:08:12 +000068
Chris Lattner22d91ca2009-01-17 06:55:17 +000069 BufferStart = BufStart;
70 BufferPtr = BufPtr;
71 BufferEnd = BufEnd;
Mike Stump1eb44332009-09-09 15:08:12 +000072
Chris Lattner22d91ca2009-01-17 06:55:17 +000073 assert(BufEnd[0] == 0 &&
74 "We assume that the input buffer has a null character at the end"
75 " to simplify lexing!");
Mike Stump1eb44332009-09-09 15:08:12 +000076
Eric Christopher156119d2011-04-09 00:01:04 +000077 // Check whether we have a BOM in the beginning of the buffer. If yes - act
78 // accordingly. Right now we support only UTF-8 with and without BOM, so, just
79 // skip the UTF-8 BOM if it's present.
80 if (BufferStart == BufferPtr) {
81 // Determine the size of the BOM.
Chris Lattner5f9e2722011-07-23 10:55:15 +000082 StringRef Buf(BufferStart, BufferEnd - BufferStart);
Eli Friedman969f9d42011-05-10 17:11:21 +000083 size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
Eric Christopher156119d2011-04-09 00:01:04 +000084 .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM
85 .Default(0);
86
87 // Skip the BOM.
88 BufferPtr += BOMLength;
89 }
90
Chris Lattner22d91ca2009-01-17 06:55:17 +000091 Is_PragmaLexer = false;
Richard Smithd5e1d602011-10-12 00:37:51 +000092 CurrentConflictMarkerState = CMK_None;
Eric Christopher156119d2011-04-09 00:01:04 +000093
Chris Lattner22d91ca2009-01-17 06:55:17 +000094 // Start of the file is a start of line.
95 IsAtStartOfLine = true;
Mike Stump1eb44332009-09-09 15:08:12 +000096
Chris Lattner22d91ca2009-01-17 06:55:17 +000097 // We are not after parsing a #.
98 ParsingPreprocessorDirective = false;
Mike Stump1eb44332009-09-09 15:08:12 +000099
Chris Lattner22d91ca2009-01-17 06:55:17 +0000100 // We are not after parsing #include.
101 ParsingFilename = false;
Mike Stump1eb44332009-09-09 15:08:12 +0000102
Chris Lattner22d91ca2009-01-17 06:55:17 +0000103 // We are not in raw mode. Raw mode disables diagnostics and interpretation
104 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used
105 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
106 // or otherwise skipping over tokens.
107 LexingRawMode = false;
Mike Stump1eb44332009-09-09 15:08:12 +0000108
Chris Lattner22d91ca2009-01-17 06:55:17 +0000109 // Default to not keeping comments.
110 ExtendedTokenMode = 0;
111}
112
Chris Lattner0770dab2009-01-17 07:56:59 +0000113/// Lexer constructor - Create a new lexer object for the specified buffer
114/// with the specified preprocessor managing the lexing process. This lexer
115/// assumes that the associated file buffer and Preprocessor objects will
116/// outlive it, so it doesn't take ownership of either of them.
Chris Lattner6e290142009-11-30 04:18:44 +0000117Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP)
Chris Lattner88d3ac12009-01-17 08:03:42 +0000118 : PreprocessorLexer(&PP, FID),
119 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
120 Features(PP.getLangOptions()) {
Mike Stump1eb44332009-09-09 15:08:12 +0000121
Chris Lattner0770dab2009-01-17 07:56:59 +0000122 InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(),
123 InputFile->getBufferEnd());
Mike Stump1eb44332009-09-09 15:08:12 +0000124
Chris Lattner0770dab2009-01-17 07:56:59 +0000125 // Default to keeping comments if the preprocessor wants them.
126 SetCommentRetentionState(PP.getCommentRetentionState());
127}
Chris Lattnerdbf388b2007-10-07 08:47:24 +0000128
Chris Lattner168ae2d2007-10-17 20:41:00 +0000129/// Lexer constructor - Create a new raw lexer object. This object is only
Chris Lattner590f0cc2008-10-12 01:15:46 +0000130/// suitable for calls to 'LexRawToken'. This lexer assumes that the text
131/// range will outlive it, so it doesn't take ownership of it.
Chris Lattner168ae2d2007-10-17 20:41:00 +0000132Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
Chris Lattnerde96c0f2009-01-17 07:42:27 +0000133 const char *BufStart, const char *BufPtr, const char *BufEnd)
Chris Lattnerc6fe32a2009-01-17 03:48:08 +0000134 : FileLoc(fileloc), Features(features) {
Chris Lattner22d91ca2009-01-17 06:55:17 +0000135
Chris Lattner22d91ca2009-01-17 06:55:17 +0000136 InitLexer(BufStart, BufPtr, BufEnd);
Mike Stump1eb44332009-09-09 15:08:12 +0000137
Chris Lattner168ae2d2007-10-17 20:41:00 +0000138 // We *are* in raw mode.
139 LexingRawMode = true;
Chris Lattner168ae2d2007-10-17 20:41:00 +0000140}
141
Chris Lattner025c3a62009-01-17 07:35:14 +0000142/// Lexer constructor - Create a new raw lexer object. This object is only
143/// suitable for calls to 'LexRawToken'. This lexer assumes that the text
144/// range will outlive it, so it doesn't take ownership of it.
Chris Lattner6e290142009-11-30 04:18:44 +0000145Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
146 const SourceManager &SM, const LangOptions &features)
Chris Lattner025c3a62009-01-17 07:35:14 +0000147 : FileLoc(SM.getLocForStartOfFile(FID)), Features(features) {
Chris Lattner025c3a62009-01-17 07:35:14 +0000148
Mike Stump1eb44332009-09-09 15:08:12 +0000149 InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(),
Chris Lattner025c3a62009-01-17 07:35:14 +0000150 FromFile->getBufferEnd());
Mike Stump1eb44332009-09-09 15:08:12 +0000151
Chris Lattner025c3a62009-01-17 07:35:14 +0000152 // We *are* in raw mode.
153 LexingRawMode = true;
154}
155
Chris Lattner42e00d12009-01-17 08:27:52 +0000156/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
157/// _Pragma expansion. This has a variety of magic semantics that this method
158/// sets up. It returns a new'd Lexer that must be delete'd when done.
159///
160/// On entrance to this routine, TokStartLoc is a macro location which has a
161/// spelling loc that indicates the bytes to be lexed for the token and an
Chandler Carruth433db062011-07-14 08:20:40 +0000162/// expansion location that indicates where all lexed tokens should be
Chris Lattner42e00d12009-01-17 08:27:52 +0000163/// "expanded from".
164///
165/// FIXME: It would really be nice to make _Pragma just be a wrapper around a
166/// normal lexer that remaps tokens as they fly by. This would require making
167/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer
168/// interface that could handle this stuff. This would pull GetMappedTokenLoc
169/// out of the critical path of the lexer!
170///
Mike Stump1eb44332009-09-09 15:08:12 +0000171Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
Chandler Carruth433db062011-07-14 08:20:40 +0000172 SourceLocation ExpansionLocStart,
173 SourceLocation ExpansionLocEnd,
Chris Lattnerbcc2a672009-01-19 06:46:35 +0000174 unsigned TokLen, Preprocessor &PP) {
Chris Lattner42e00d12009-01-17 08:27:52 +0000175 SourceManager &SM = PP.getSourceManager();
Chris Lattner42e00d12009-01-17 08:27:52 +0000176
177 // Create the lexer as if we were going to lex the file normally.
Chris Lattnera11d6172009-01-19 07:46:45 +0000178 FileID SpellingFID = SM.getFileID(SpellingLoc);
Chris Lattner6e290142009-11-30 04:18:44 +0000179 const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID);
180 Lexer *L = new Lexer(SpellingFID, InputFile, PP);
Mike Stump1eb44332009-09-09 15:08:12 +0000181
Chris Lattner42e00d12009-01-17 08:27:52 +0000182 // Now that the lexer is created, change the start/end locations so that we
183 // just lex the subsection of the file that we want. This is lexing from a
184 // scratch buffer.
185 const char *StrData = SM.getCharacterData(SpellingLoc);
Mike Stump1eb44332009-09-09 15:08:12 +0000186
Chris Lattner42e00d12009-01-17 08:27:52 +0000187 L->BufferPtr = StrData;
188 L->BufferEnd = StrData+TokLen;
Chris Lattner1fa49532009-03-08 08:08:45 +0000189 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
Chris Lattner42e00d12009-01-17 08:27:52 +0000190
191 // Set the SourceLocation with the remapping information. This ensures that
192 // GetMappedTokenLoc will remap the tokens as they are lexed.
Chandler Carruthbf340e42011-07-26 03:03:05 +0000193 L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID),
194 ExpansionLocStart,
195 ExpansionLocEnd, TokLen);
Mike Stump1eb44332009-09-09 15:08:12 +0000196
Chris Lattner42e00d12009-01-17 08:27:52 +0000197 // Ensure that the lexer thinks it is inside a directive, so that end \n will
Peter Collingbourne84021552011-02-28 02:37:51 +0000198 // return an EOD token.
Chris Lattner42e00d12009-01-17 08:27:52 +0000199 L->ParsingPreprocessorDirective = true;
Mike Stump1eb44332009-09-09 15:08:12 +0000200
Chris Lattner42e00d12009-01-17 08:27:52 +0000201 // This lexer really is for _Pragma.
202 L->Is_PragmaLexer = true;
203 return L;
204}
205
Chris Lattner168ae2d2007-10-17 20:41:00 +0000206
Reid Spencer5f016e22007-07-11 17:01:13 +0000207/// Stringify - Convert the specified string into a C string, with surrounding
208/// ""'s, and with escaped \ and " characters.
209std::string Lexer::Stringify(const std::string &Str, bool Charify) {
210 std::string Result = Str;
211 char Quote = Charify ? '\'' : '"';
212 for (unsigned i = 0, e = Result.size(); i != e; ++i) {
213 if (Result[i] == '\\' || Result[i] == Quote) {
214 Result.insert(Result.begin()+i, '\\');
215 ++i; ++e;
216 }
217 }
218 return Result;
219}
220
Chris Lattnerd8e30832007-07-24 06:57:14 +0000221/// Stringify - Convert the specified string into a C string by escaping '\'
222/// and " characters. This does not add surrounding ""'s to the string.
Chris Lattner5f9e2722011-07-23 10:55:15 +0000223void Lexer::Stringify(SmallVectorImpl<char> &Str) {
Chris Lattnerd8e30832007-07-24 06:57:14 +0000224 for (unsigned i = 0, e = Str.size(); i != e; ++i) {
225 if (Str[i] == '\\' || Str[i] == '"') {
226 Str.insert(Str.begin()+i, '\\');
227 ++i; ++e;
228 }
229 }
230}
231
Chris Lattnerb0607272010-11-17 07:26:20 +0000232//===----------------------------------------------------------------------===//
233// Token Spelling
234//===----------------------------------------------------------------------===//
235
236/// getSpelling() - Return the 'spelling' of this token. The spelling of a
237/// token are the characters used to represent the token in the source file
238/// after trigraph expansion and escaped-newline folding. In particular, this
239/// wants to get the true, uncanonicalized, spelling of things like digraphs
240/// UCNs, etc.
Chris Lattner5f9e2722011-07-23 10:55:15 +0000241StringRef Lexer::getSpelling(SourceLocation loc,
242 SmallVectorImpl<char> &buffer,
John McCall834e3f62011-03-08 07:59:04 +0000243 const SourceManager &SM,
244 const LangOptions &options,
245 bool *invalid) {
246 // Break down the source location.
247 std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
248
249 // Try to the load the file buffer.
250 bool invalidTemp = false;
Chris Lattner5f9e2722011-07-23 10:55:15 +0000251 StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
John McCall834e3f62011-03-08 07:59:04 +0000252 if (invalidTemp) {
253 if (invalid) *invalid = true;
Chris Lattner5f9e2722011-07-23 10:55:15 +0000254 return StringRef();
John McCall834e3f62011-03-08 07:59:04 +0000255 }
256
257 const char *tokenBegin = file.data() + locInfo.second;
258
259 // Lex from the start of the given location.
260 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options,
261 file.begin(), tokenBegin, file.end());
262 Token token;
263 lexer.LexFromRawLexer(token);
264
265 unsigned length = token.getLength();
266
267 // Common case: no need for cleaning.
268 if (!token.needsCleaning())
Chris Lattner5f9e2722011-07-23 10:55:15 +0000269 return StringRef(tokenBegin, length);
John McCall834e3f62011-03-08 07:59:04 +0000270
271 // Hard case, we need to relex the characters into the string.
272 buffer.clear();
273 buffer.reserve(length);
274
275 for (const char *ti = tokenBegin, *te = ti + length; ti != te; ) {
276 unsigned charSize;
277 buffer.push_back(Lexer::getCharAndSizeNoWarn(ti, charSize, options));
278 ti += charSize;
279 }
280
Chris Lattner5f9e2722011-07-23 10:55:15 +0000281 return StringRef(buffer.data(), buffer.size());
John McCall834e3f62011-03-08 07:59:04 +0000282}
283
284/// getSpelling() - Return the 'spelling' of this token. The spelling of a
285/// token are the characters used to represent the token in the source file
286/// after trigraph expansion and escaped-newline folding. In particular, this
287/// wants to get the true, uncanonicalized, spelling of things like digraphs
288/// UCNs, etc.
Chris Lattnerb0607272010-11-17 07:26:20 +0000289std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
290 const LangOptions &Features, bool *Invalid) {
291 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
292
293 // If this token contains nothing interesting, return it directly.
294 bool CharDataInvalid = false;
295 const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
296 &CharDataInvalid);
297 if (Invalid)
298 *Invalid = CharDataInvalid;
299 if (CharDataInvalid)
300 return std::string();
301
302 if (!Tok.needsCleaning())
303 return std::string(TokStart, TokStart+Tok.getLength());
304
305 std::string Result;
306 Result.reserve(Tok.getLength());
307
308 // Otherwise, hard case, relex the characters into the string.
309 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
310 Ptr != End; ) {
311 unsigned CharSize;
312 Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
313 Ptr += CharSize;
314 }
315 assert(Result.size() != unsigned(Tok.getLength()) &&
316 "NeedsCleaning flag set on something that didn't need cleaning!");
317 return Result;
318}
319
320/// getSpelling - This method is used to get the spelling of a token into a
321/// preallocated buffer, instead of as an std::string. The caller is required
322/// to allocate enough space for the token, which is guaranteed to be at least
323/// Tok.getLength() bytes long. The actual length of the token is returned.
324///
325/// Note that this method may do two possible things: it may either fill in
326/// the buffer specified with characters, or it may *change the input pointer*
327/// to point to a constant buffer with the data already in it (avoiding a
328/// copy). The caller is not allowed to modify the returned buffer pointer
329/// if an internal buffer is returned.
330unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer,
331 const SourceManager &SourceMgr,
332 const LangOptions &Features, bool *Invalid) {
333 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000334
335 const char *TokStart = 0;
336 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
337 if (Tok.is(tok::raw_identifier))
338 TokStart = Tok.getRawIdentifierData();
339 else if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
340 // Just return the string from the identifier table, which is very quick.
Chris Lattnerb0607272010-11-17 07:26:20 +0000341 Buffer = II->getNameStart();
342 return II->getLength();
343 }
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000344
345 // NOTE: this can be checked even after testing for an IdentifierInfo.
Chris Lattnerb0607272010-11-17 07:26:20 +0000346 if (Tok.isLiteral())
347 TokStart = Tok.getLiteralData();
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000348
Chris Lattnerb0607272010-11-17 07:26:20 +0000349 if (TokStart == 0) {
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000350 // Compute the start of the token in the input lexer buffer.
Chris Lattnerb0607272010-11-17 07:26:20 +0000351 bool CharDataInvalid = false;
352 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
353 if (Invalid)
354 *Invalid = CharDataInvalid;
355 if (CharDataInvalid) {
356 Buffer = "";
357 return 0;
358 }
359 }
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000360
Chris Lattnerb0607272010-11-17 07:26:20 +0000361 // If this token contains nothing interesting, return it directly.
362 if (!Tok.needsCleaning()) {
363 Buffer = TokStart;
364 return Tok.getLength();
365 }
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000366
Chris Lattnerb0607272010-11-17 07:26:20 +0000367 // Otherwise, hard case, relex the characters into the string.
368 char *OutBuf = const_cast<char*>(Buffer);
369 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
370 Ptr != End; ) {
371 unsigned CharSize;
372 *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
373 Ptr += CharSize;
374 }
375 assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
376 "NeedsCleaning flag set on something that didn't need cleaning!");
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000377
Chris Lattnerb0607272010-11-17 07:26:20 +0000378 return OutBuf-Buffer;
379}
380
381
382
Douglas Gregor33e9abd2010-01-22 19:49:59 +0000383static bool isWhitespace(unsigned char c);
Reid Spencer5f016e22007-07-11 17:01:13 +0000384
Chris Lattner9a611942007-10-17 21:18:47 +0000385/// MeasureTokenLength - Relex the token at the specified location and return
386/// its length in bytes in the input file. If the token needs cleaning (e.g.
387/// includes a trigraph or an escaped newline) then this count includes bytes
388/// that are part of that.
389unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
Chris Lattner2c78b872009-04-14 23:22:57 +0000390 const SourceManager &SM,
391 const LangOptions &LangOpts) {
Chris Lattner9a611942007-10-17 21:18:47 +0000392 // TODO: this could be special cased for common tokens like identifiers, ')',
393 // etc to make this faster, if it mattered. Just look at StrData[0] to handle
Mike Stump1eb44332009-09-09 15:08:12 +0000394 // all obviously single-char tokens. This could use
Chris Lattner9a611942007-10-17 21:18:47 +0000395 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
396 // something.
Chris Lattnerde7aeef2009-01-26 00:43:02 +0000397
398 // If this comes from a macro expansion, we really do want the macro name, not
399 // the token this macro expanded to.
Chandler Carruth40278532011-07-25 16:49:02 +0000400 Loc = SM.getExpansionLoc(Loc);
Chris Lattner363fdc22009-01-26 22:24:27 +0000401 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
Douglas Gregorf715ca12010-03-16 00:06:06 +0000402 bool Invalid = false;
Chris Lattner5f9e2722011-07-23 10:55:15 +0000403 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
Douglas Gregorf715ca12010-03-16 00:06:06 +0000404 if (Invalid)
Douglas Gregoraea67db2010-03-15 22:54:52 +0000405 return 0;
Benjamin Kramerf6ac97b2010-03-16 14:14:31 +0000406
407 const char *StrData = Buffer.data()+LocInfo.second;
Chris Lattner83503942009-01-17 08:30:10 +0000408
Douglas Gregor33e9abd2010-01-22 19:49:59 +0000409 if (isWhitespace(StrData[0]))
410 return 0;
411
Chris Lattner9a611942007-10-17 21:18:47 +0000412 // Create a lexer starting at the beginning of this token.
Sebastian Redlc3526d82010-09-30 01:03:03 +0000413 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
414 Buffer.begin(), StrData, Buffer.end());
Chris Lattner39de7402009-10-14 15:04:18 +0000415 TheLexer.SetCommentRetentionState(true);
Chris Lattner9a611942007-10-17 21:18:47 +0000416 Token TheTok;
Chris Lattner590f0cc2008-10-12 01:15:46 +0000417 TheLexer.LexFromRawLexer(TheTok);
Chris Lattner9a611942007-10-17 21:18:47 +0000418 return TheTok.getLength();
419}
420
Argyrios Kyrtzidis0e870622011-08-17 00:31:23 +0000421static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
422 const SourceManager &SM,
423 const LangOptions &LangOpts) {
424 assert(Loc.isFileID());
Douglas Gregora8e5c5b2010-07-22 20:22:31 +0000425 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
Douglas Gregor3de84242011-01-31 22:42:36 +0000426 if (LocInfo.first.isInvalid())
427 return Loc;
428
Douglas Gregora8e5c5b2010-07-22 20:22:31 +0000429 bool Invalid = false;
Chris Lattner5f9e2722011-07-23 10:55:15 +0000430 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
Douglas Gregora8e5c5b2010-07-22 20:22:31 +0000431 if (Invalid)
432 return Loc;
433
434 // Back up from the current location until we hit the beginning of a line
435 // (or the buffer). We'll relex from that point.
436 const char *BufStart = Buffer.data();
Douglas Gregor3de84242011-01-31 22:42:36 +0000437 if (LocInfo.second >= Buffer.size())
438 return Loc;
439
Douglas Gregora8e5c5b2010-07-22 20:22:31 +0000440 const char *StrData = BufStart+LocInfo.second;
441 if (StrData[0] == '\n' || StrData[0] == '\r')
442 return Loc;
443
444 const char *LexStart = StrData;
445 while (LexStart != BufStart) {
446 if (LexStart[0] == '\n' || LexStart[0] == '\r') {
447 ++LexStart;
448 break;
449 }
450
451 --LexStart;
452 }
453
454 // Create a lexer starting at the beginning of this token.
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +0000455 SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second);
Douglas Gregora8e5c5b2010-07-22 20:22:31 +0000456 Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end());
457 TheLexer.SetCommentRetentionState(true);
458
459 // Lex tokens until we find the token that contains the source location.
460 Token TheTok;
461 do {
462 TheLexer.LexFromRawLexer(TheTok);
463
464 if (TheLexer.getBufferLocation() > StrData) {
465 // Lexing this token has taken the lexer past the source location we're
466 // looking for. If the current token encompasses our source location,
467 // return the beginning of that token.
468 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
469 return TheTok.getLocation();
470
471 // We ended up skipping over the source location entirely, which means
472 // that it points into whitespace. We're done here.
473 break;
474 }
475 } while (TheTok.getKind() != tok::eof);
476
477 // We've passed our source location; just return the original source location.
478 return Loc;
479}
480
Argyrios Kyrtzidis0e870622011-08-17 00:31:23 +0000481SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
482 const SourceManager &SM,
483 const LangOptions &LangOpts) {
484 if (Loc.isFileID())
485 return getBeginningOfFileToken(Loc, SM, LangOpts);
486
487 if (!SM.isMacroArgExpansion(Loc))
488 return Loc;
489
490 SourceLocation FileLoc = SM.getSpellingLoc(Loc);
491 SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts);
492 std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc);
Chandler Carruthae9f85b2012-01-15 09:03:45 +0000493 std::pair<FileID, unsigned> BeginFileLocInfo
494 = SM.getDecomposedLoc(BeginFileLoc);
Argyrios Kyrtzidis0e870622011-08-17 00:31:23 +0000495 assert(FileLocInfo.first == BeginFileLocInfo.first &&
496 FileLocInfo.second >= BeginFileLocInfo.second);
Chandler Carruthae9f85b2012-01-15 09:03:45 +0000497 return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second);
Argyrios Kyrtzidis0e870622011-08-17 00:31:23 +0000498}
499
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000500namespace {
501 enum PreambleDirectiveKind {
502 PDK_Skipped,
503 PDK_StartIf,
504 PDK_EndIf,
505 PDK_Unknown
506 };
507}
508
Douglas Gregorf4f6c9d2010-07-26 21:36:20 +0000509std::pair<unsigned, bool>
Argyrios Kyrtzidis03c107a2011-08-25 20:39:19 +0000510Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer,
511 const LangOptions &Features, unsigned MaxLines) {
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000512 // Create a lexer starting at the beginning of the file. Note that we use a
513 // "fake" file source location at offset 1 so that the lexer will track our
514 // position within the file.
515 const unsigned StartOffset = 1;
516 SourceLocation StartLoc = SourceLocation::getFromRawEncoding(StartOffset);
Argyrios Kyrtzidis03c107a2011-08-25 20:39:19 +0000517 Lexer TheLexer(StartLoc, Features, Buffer->getBufferStart(),
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000518 Buffer->getBufferStart(), Buffer->getBufferEnd());
519
520 bool InPreprocessorDirective = false;
521 Token TheTok;
522 Token IfStartTok;
523 unsigned IfCount = 0;
Argyrios Kyrtzidisc8c97a02011-09-04 03:32:04 +0000524
525 unsigned MaxLineOffset = 0;
526 if (MaxLines) {
527 const char *CurPtr = Buffer->getBufferStart();
528 unsigned CurLine = 0;
529 while (CurPtr != Buffer->getBufferEnd()) {
530 char ch = *CurPtr++;
531 if (ch == '\n') {
532 ++CurLine;
533 if (CurLine == MaxLines)
534 break;
535 }
536 }
537 if (CurPtr != Buffer->getBufferEnd())
538 MaxLineOffset = CurPtr - Buffer->getBufferStart();
539 }
Douglas Gregordf95a132010-08-09 20:45:32 +0000540
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000541 do {
542 TheLexer.LexFromRawLexer(TheTok);
543
544 if (InPreprocessorDirective) {
545 // If we've hit the end of the file, we're done.
546 if (TheTok.getKind() == tok::eof) {
547 InPreprocessorDirective = false;
548 break;
549 }
550
551 // If we haven't hit the end of the preprocessor directive, skip this
552 // token.
553 if (!TheTok.isAtStartOfLine())
554 continue;
555
556 // We've passed the end of the preprocessor directive, and will look
557 // at this token again below.
558 InPreprocessorDirective = false;
559 }
560
Douglas Gregordf95a132010-08-09 20:45:32 +0000561 // Keep track of the # of lines in the preamble.
562 if (TheTok.isAtStartOfLine()) {
Argyrios Kyrtzidisc8c97a02011-09-04 03:32:04 +0000563 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;
Douglas Gregordf95a132010-08-09 20:45:32 +0000564
565 // If we were asked to limit the number of lines in the preamble,
566 // and we're about to exceed that limit, we're done.
Argyrios Kyrtzidisc8c97a02011-09-04 03:32:04 +0000567 if (MaxLineOffset && TokOffset >= MaxLineOffset)
Douglas Gregordf95a132010-08-09 20:45:32 +0000568 break;
569 }
570
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000571 // Comments are okay; skip over them.
572 if (TheTok.getKind() == tok::comment)
573 continue;
574
575 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
576 // This is the start of a preprocessor directive.
577 Token HashTok = TheTok;
578 InPreprocessorDirective = true;
579
Joerg Sonnenberger19207f12011-07-20 00:14:37 +0000580 // Figure out which directive this is. Since we're lexing raw tokens,
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000581 // we don't have an identifier table available. Instead, just look at
582 // the raw identifier to recognize and categorize preprocessor directives.
583 TheLexer.LexFromRawLexer(TheTok);
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000584 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
Chris Lattner5f9e2722011-07-23 10:55:15 +0000585 StringRef Keyword(TheTok.getRawIdentifierData(),
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000586 TheTok.getLength());
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000587 PreambleDirectiveKind PDK
588 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
589 .Case("include", PDK_Skipped)
590 .Case("__include_macros", PDK_Skipped)
591 .Case("define", PDK_Skipped)
592 .Case("undef", PDK_Skipped)
593 .Case("line", PDK_Skipped)
594 .Case("error", PDK_Skipped)
595 .Case("pragma", PDK_Skipped)
596 .Case("import", PDK_Skipped)
597 .Case("include_next", PDK_Skipped)
598 .Case("warning", PDK_Skipped)
599 .Case("ident", PDK_Skipped)
600 .Case("sccs", PDK_Skipped)
601 .Case("assert", PDK_Skipped)
602 .Case("unassert", PDK_Skipped)
603 .Case("if", PDK_StartIf)
604 .Case("ifdef", PDK_StartIf)
605 .Case("ifndef", PDK_StartIf)
606 .Case("elif", PDK_Skipped)
607 .Case("else", PDK_Skipped)
608 .Case("endif", PDK_EndIf)
609 .Default(PDK_Unknown);
610
611 switch (PDK) {
612 case PDK_Skipped:
613 continue;
614
615 case PDK_StartIf:
616 if (IfCount == 0)
617 IfStartTok = HashTok;
618
619 ++IfCount;
620 continue;
621
622 case PDK_EndIf:
623 // Mismatched #endif. The preamble ends here.
624 if (IfCount == 0)
625 break;
626
627 --IfCount;
628 continue;
629
630 case PDK_Unknown:
631 // We don't know what this directive is; stop at the '#'.
632 break;
633 }
634 }
635
636 // We only end up here if we didn't recognize the preprocessor
637 // directive or it was one that can't occur in the preamble at this
638 // point. Roll back the current token to the location of the '#'.
639 InPreprocessorDirective = false;
640 TheTok = HashTok;
641 }
642
Douglas Gregordf95a132010-08-09 20:45:32 +0000643 // We hit a token that we don't recognize as being in the
644 // "preprocessing only" part of the file, so we're no longer in
645 // the preamble.
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000646 break;
647 } while (true);
648
649 SourceLocation End = IfCount? IfStartTok.getLocation() : TheTok.getLocation();
Douglas Gregorf4f6c9d2010-07-26 21:36:20 +0000650 return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(),
651 IfCount? IfStartTok.isAtStartOfLine()
652 : TheTok.isAtStartOfLine());
Douglas Gregorf033f1d2010-07-20 20:18:03 +0000653}
654
Chris Lattner7ef5c272010-11-17 07:05:50 +0000655
656/// AdvanceToTokenCharacter - Given a location that specifies the start of a
657/// token, return a new location that specifies a character within the token.
658SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
659 unsigned CharNo,
660 const SourceManager &SM,
661 const LangOptions &Features) {
Chandler Carruth433db062011-07-14 08:20:40 +0000662 // Figure out how many physical characters away the specified expansion
Chris Lattner7ef5c272010-11-17 07:05:50 +0000663 // character is. This needs to take into consideration newlines and
664 // trigraphs.
665 bool Invalid = false;
666 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid);
667
668 // If they request the first char of the token, we're trivially done.
669 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
670 return TokStart;
671
672 unsigned PhysOffset = 0;
673
674 // The usual case is that tokens don't contain anything interesting. Skip
675 // over the uninteresting characters. If a token only consists of simple
676 // chars, this method is extremely fast.
677 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
678 if (CharNo == 0)
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +0000679 return TokStart.getLocWithOffset(PhysOffset);
Chris Lattner7ef5c272010-11-17 07:05:50 +0000680 ++TokPtr, --CharNo, ++PhysOffset;
681 }
682
683 // If we have a character that may be a trigraph or escaped newline, use a
684 // lexer to parse it correctly.
685 for (; CharNo; --CharNo) {
686 unsigned Size;
687 Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features);
688 TokPtr += Size;
689 PhysOffset += Size;
690 }
691
692 // Final detail: if we end up on an escaped newline, we want to return the
693 // location of the actual byte of the token. For example foo\<newline>bar
694 // advanced by 3 should return the location of b, not of \\. One compounding
695 // detail of this is that the escape may be made by a trigraph.
696 if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
697 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
698
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +0000699 return TokStart.getLocWithOffset(PhysOffset);
Chris Lattner7ef5c272010-11-17 07:05:50 +0000700}
701
702/// \brief Computes the source location just past the end of the
703/// token at this source location.
704///
705/// This routine can be used to produce a source location that
706/// points just past the end of the token referenced by \p Loc, and
707/// is generally used when a diagnostic needs to point just after a
708/// token where it expected something different that it received. If
709/// the returned source location would not be meaningful (e.g., if
710/// it points into a macro), this routine returns an invalid
711/// source location.
712///
713/// \param Offset an offset from the end of the token, where the source
714/// location should refer to. The default offset (0) produces a source
715/// location pointing just past the end of the token; an offset of 1 produces
716/// a source location pointing to the last character in the token, etc.
717SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
718 const SourceManager &SM,
719 const LangOptions &Features) {
Argyrios Kyrtzidis7ddf6b22011-06-24 17:58:59 +0000720 if (Loc.isInvalid())
Chris Lattner7ef5c272010-11-17 07:05:50 +0000721 return SourceLocation();
Argyrios Kyrtzidis7ddf6b22011-06-24 17:58:59 +0000722
723 if (Loc.isMacroID()) {
Argyrios Kyrtzidis69bda4c2012-01-19 15:59:08 +0000724 if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, Features, &Loc))
Chandler Carruth433db062011-07-14 08:20:40 +0000725 return SourceLocation(); // Points inside the macro expansion.
Argyrios Kyrtzidis7ddf6b22011-06-24 17:58:59 +0000726 }
727
Chris Lattner7ef5c272010-11-17 07:05:50 +0000728 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, Features);
729 if (Len > Offset)
730 Len = Len - Offset;
731 else
732 return Loc;
733
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +0000734 return Loc.getLocWithOffset(Len);
Chris Lattner7ef5c272010-11-17 07:05:50 +0000735}
736
Argyrios Kyrtzidis7a759602011-07-07 21:54:45 +0000737/// \brief Returns true if the given MacroID location points at the first
Chandler Carruth433db062011-07-14 08:20:40 +0000738/// token of the macro expansion.
739bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
Douglas Gregorf62d43d2011-07-19 16:10:42 +0000740 const SourceManager &SM,
Argyrios Kyrtzidis69bda4c2012-01-19 15:59:08 +0000741 const LangOptions &LangOpts,
742 SourceLocation *MacroBegin) {
Argyrios Kyrtzidis7a759602011-07-07 21:54:45 +0000743 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
744
745 std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc);
746 // FIXME: If the token comes from the macro token paste operator ('##')
747 // this function will always return false;
748 if (infoLoc.second > 0)
749 return false; // Does not point at the start of token.
750
Chandler Carruth433db062011-07-14 08:20:40 +0000751 SourceLocation expansionLoc =
Chandler Carruth17287622011-07-26 04:56:51 +0000752 SM.getSLocEntry(infoLoc.first).getExpansion().getExpansionLocStart();
Argyrios Kyrtzidis69bda4c2012-01-19 15:59:08 +0000753 if (expansionLoc.isFileID()) {
754 // No other macro expansions, this is the first.
755 if (MacroBegin)
756 *MacroBegin = expansionLoc;
757 return true;
758 }
Argyrios Kyrtzidis7a759602011-07-07 21:54:45 +0000759
Argyrios Kyrtzidis69bda4c2012-01-19 15:59:08 +0000760 return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin);
Argyrios Kyrtzidis7a759602011-07-07 21:54:45 +0000761}
762
763/// \brief Returns true if the given MacroID location points at the last
Chandler Carruth433db062011-07-14 08:20:40 +0000764/// token of the macro expansion.
765bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
Argyrios Kyrtzidis69bda4c2012-01-19 15:59:08 +0000766 const SourceManager &SM,
767 const LangOptions &LangOpts,
768 SourceLocation *MacroEnd) {
Argyrios Kyrtzidis7a759602011-07-07 21:54:45 +0000769 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
770
771 SourceLocation spellLoc = SM.getSpellingLoc(loc);
772 unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts);
773 if (tokLen == 0)
774 return false;
775
776 FileID FID = SM.getFileID(loc);
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +0000777 SourceLocation afterLoc = loc.getLocWithOffset(tokLen+1);
Argyrios Kyrtzidisf8c50652011-08-23 21:02:30 +0000778 if (SM.isInFileID(afterLoc, FID))
779 return false; // Still in the same FileID, does not point to the last token.
Argyrios Kyrtzidis7a759602011-07-07 21:54:45 +0000780
781 // FIXME: If the token comes from the macro token paste operator ('##')
782 // or the stringify operator ('#') this function will always return false;
Argyrios Kyrtzidisf8c50652011-08-23 21:02:30 +0000783
Chandler Carruth433db062011-07-14 08:20:40 +0000784 SourceLocation expansionLoc =
Chandler Carruth17287622011-07-26 04:56:51 +0000785 SM.getSLocEntry(FID).getExpansion().getExpansionLocEnd();
Argyrios Kyrtzidis69bda4c2012-01-19 15:59:08 +0000786 if (expansionLoc.isFileID()) {
787 // No other macro expansions.
788 if (MacroEnd)
789 *MacroEnd = expansionLoc;
790 return true;
791 }
Argyrios Kyrtzidis7a759602011-07-07 21:54:45 +0000792
Argyrios Kyrtzidis69bda4c2012-01-19 15:59:08 +0000793 return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd);
Argyrios Kyrtzidis7a759602011-07-07 21:54:45 +0000794}
795
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +0000796static CharSourceRange makeRangeFromFileLocs(SourceLocation Begin,
797 SourceLocation End,
798 const SourceManager &SM,
799 const LangOptions &LangOpts) {
800 assert(Begin.isFileID() && End.isFileID());
801 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts);
802 if (End.isInvalid())
803 return CharSourceRange();
804
805 // Break down the source locations.
806 FileID FID;
807 unsigned BeginOffs;
808 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin);
809 if (FID.isInvalid())
810 return CharSourceRange();
811
812 unsigned EndOffs;
813 if (!SM.isInFileID(End, FID, &EndOffs) ||
814 BeginOffs > EndOffs)
815 return CharSourceRange();
816
817 return CharSourceRange::getCharRange(Begin, End);
818}
819
Argyrios Kyrtzidis11b652d2012-01-19 15:59:14 +0000820/// \brief Accepts a token source range and returns a character range with
821/// file locations.
822/// Returns a null range if a part of the range resides inside a macro
823/// expansion or the range does not reside on the same FileID.
824CharSourceRange Lexer::makeFileCharRange(SourceRange TokenRange,
825 const SourceManager &SM,
826 const LangOptions &LangOpts) {
827 SourceLocation Begin = TokenRange.getBegin();
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +0000828 SourceLocation End = TokenRange.getEnd();
829 if (Begin.isInvalid() || End.isInvalid())
Argyrios Kyrtzidis11b652d2012-01-19 15:59:14 +0000830 return CharSourceRange();
831
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +0000832 if (Begin.isFileID() && End.isFileID())
833 return makeRangeFromFileLocs(Begin, End, SM, LangOpts);
834
835 if (Begin.isMacroID() && End.isFileID()) {
Argyrios Kyrtzidis11b652d2012-01-19 15:59:14 +0000836 if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin))
837 return CharSourceRange();
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +0000838 return makeRangeFromFileLocs(Begin, End, SM, LangOpts);
839 }
Argyrios Kyrtzidis11b652d2012-01-19 15:59:14 +0000840
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +0000841 if (Begin.isFileID() && End.isMacroID()) {
842 if (!isAtEndOfMacroExpansion(End, SM, LangOpts, &End))
843 return CharSourceRange();
844 return makeRangeFromFileLocs(Begin, End, SM, LangOpts);
845 }
Argyrios Kyrtzidis11b652d2012-01-19 15:59:14 +0000846
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +0000847 assert(Begin.isMacroID() && End.isMacroID());
848 SourceLocation MacroBegin, MacroEnd;
849 if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) &&
850 isAtEndOfMacroExpansion(End, SM, LangOpts, &MacroEnd))
851 return makeRangeFromFileLocs(MacroBegin, MacroEnd, SM, LangOpts);
852
853 FileID FID;
854 unsigned BeginOffs;
855 llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin);
856 if (FID.isInvalid())
Argyrios Kyrtzidise64d9032012-01-19 15:59:19 +0000857 return CharSourceRange();
858
Argyrios Kyrtzidis11b652d2012-01-19 15:59:14 +0000859 unsigned EndOffs;
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +0000860 if (!SM.isInFileID(End, FID, &EndOffs) ||
861 BeginOffs > EndOffs)
Argyrios Kyrtzidis11b652d2012-01-19 15:59:14 +0000862 return CharSourceRange();
863
Argyrios Kyrtzidisd9806c92012-01-20 16:52:43 +0000864 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
865 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
866 if (Expansion.isMacroArgExpansion() &&
867 Expansion.getSpellingLoc().isFileID()) {
868 SourceLocation SpellLoc = Expansion.getSpellingLoc();
869 return makeRangeFromFileLocs(SpellLoc.getLocWithOffset(BeginOffs),
870 SpellLoc.getLocWithOffset(EndOffs),
871 SM, LangOpts);
872 }
873
874 return CharSourceRange();
Argyrios Kyrtzidis11b652d2012-01-19 15:59:14 +0000875}
876
Argyrios Kyrtzidise64d9032012-01-19 15:59:19 +0000877StringRef Lexer::getSourceText(CharSourceRange Range,
878 const SourceManager &SM,
879 const LangOptions &LangOpts,
880 bool *Invalid) {
881 if (Range.isTokenRange())
882 Range = makeFileCharRange(Range.getAsRange(), SM, LangOpts);
883
884 if (Range.isInvalid() ||
885 Range.getBegin().isMacroID() || Range.getEnd().isMacroID()) {
886 if (Invalid) *Invalid = true;
887 return StringRef();
888 }
889
890 // Break down the source location.
891 std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin());
892 if (beginInfo.first.isInvalid()) {
893 if (Invalid) *Invalid = true;
894 return StringRef();
895 }
896
897 unsigned EndOffs;
898 if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) ||
899 beginInfo.second > EndOffs) {
900 if (Invalid) *Invalid = true;
901 return StringRef();
902 }
903
904 // Try to the load the file buffer.
905 bool invalidTemp = false;
906 StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp);
907 if (invalidTemp) {
908 if (Invalid) *Invalid = true;
909 return StringRef();
910 }
911
912 if (Invalid) *Invalid = false;
913 return file.substr(beginInfo.second, EndOffs - beginInfo.second);
914}
915
Anna Zaksc2a8d6c2012-01-18 20:17:16 +0000916StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
917 const SourceManager &SM,
918 const LangOptions &LangOpts) {
919 assert(Loc.isMacroID() && "Only reasonble to call this on macros");
Argyrios Kyrtzidis7f6cf972012-01-23 16:58:33 +0000920
921 // Find the location of the immediate macro expansion.
922 while (1) {
923 FileID FID = SM.getFileID(Loc);
924 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
925 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
926 Loc = Expansion.getExpansionLocStart();
927 if (!Expansion.isMacroArgExpansion())
928 break;
929
930 // For macro arguments we need to check that the argument did not come
931 // from an inner macro, e.g: "MAC1( MAC2(foo) )"
932
933 // Loc points to the argument id of the macro definition, move to the
934 // macro expansion.
Anna Zaksc2a8d6c2012-01-18 20:17:16 +0000935 Loc = SM.getImmediateExpansionRange(Loc).first;
Argyrios Kyrtzidis7f6cf972012-01-23 16:58:33 +0000936 SourceLocation SpellLoc = Expansion.getSpellingLoc();
937 if (SpellLoc.isFileID())
938 break; // No inner macro.
939
940 // If spelling location resides in the same FileID as macro expansion
941 // location, it means there is no inner macro.
942 FileID MacroFID = SM.getFileID(Loc);
943 if (SM.isInFileID(SpellLoc, MacroFID))
944 break;
945
946 // Argument came from inner macro.
947 Loc = SpellLoc;
948 }
Anna Zaksc2a8d6c2012-01-18 20:17:16 +0000949
950 // Find the spelling location of the start of the non-argument expansion
951 // range. This is where the macro name was spelled in order to begin
952 // expanding this macro.
Argyrios Kyrtzidis7f6cf972012-01-23 16:58:33 +0000953 Loc = SM.getSpellingLoc(Loc);
Anna Zaksc2a8d6c2012-01-18 20:17:16 +0000954
955 // Dig out the buffer where the macro name was spelled and the extents of the
956 // name so that we can render it into the expansion note.
957 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc);
958 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
959 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first);
960 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength);
961}
962
Reid Spencer5f016e22007-07-11 17:01:13 +0000963//===----------------------------------------------------------------------===//
964// Character information.
965//===----------------------------------------------------------------------===//
966
Reid Spencer5f016e22007-07-11 17:01:13 +0000967enum {
968 CHAR_HORZ_WS = 0x01, // ' ', '\t', '\f', '\v'. Note, no '\0'
969 CHAR_VERT_WS = 0x02, // '\r', '\n'
970 CHAR_LETTER = 0x04, // a-z,A-Z
971 CHAR_NUMBER = 0x08, // 0-9
972 CHAR_UNDER = 0x10, // _
Craig Topper2fa4e862011-08-11 04:06:15 +0000973 CHAR_PERIOD = 0x20, // .
974 CHAR_RAWDEL = 0x40 // {}[]#<>%:;?*+-/^&|~!=,"'
Reid Spencer5f016e22007-07-11 17:01:13 +0000975};
976
Chris Lattner03b98662009-07-07 17:09:54 +0000977// Statically initialize CharInfo table based on ASCII character set
978// Reference: FreeBSD 7.2 /usr/share/misc/ascii
Chris Lattnera2bf1052009-12-17 05:29:40 +0000979static const unsigned char CharInfo[256] =
Chris Lattner03b98662009-07-07 17:09:54 +0000980{
981// 0 NUL 1 SOH 2 STX 3 ETX
982// 4 EOT 5 ENQ 6 ACK 7 BEL
983 0 , 0 , 0 , 0 ,
984 0 , 0 , 0 , 0 ,
985// 8 BS 9 HT 10 NL 11 VT
986//12 NP 13 CR 14 SO 15 SI
987 0 , CHAR_HORZ_WS, CHAR_VERT_WS, CHAR_HORZ_WS,
988 CHAR_HORZ_WS, CHAR_VERT_WS, 0 , 0 ,
989//16 DLE 17 DC1 18 DC2 19 DC3
990//20 DC4 21 NAK 22 SYN 23 ETB
991 0 , 0 , 0 , 0 ,
992 0 , 0 , 0 , 0 ,
993//24 CAN 25 EM 26 SUB 27 ESC
994//28 FS 29 GS 30 RS 31 US
995 0 , 0 , 0 , 0 ,
996 0 , 0 , 0 , 0 ,
997//32 SP 33 ! 34 " 35 #
998//36 $ 37 % 38 & 39 '
Craig Topper2fa4e862011-08-11 04:06:15 +0000999 CHAR_HORZ_WS, CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
1000 0 , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
Chris Lattner03b98662009-07-07 17:09:54 +00001001//40 ( 41 ) 42 * 43 +
1002//44 , 45 - 46 . 47 /
Craig Topper2fa4e862011-08-11 04:06:15 +00001003 0 , 0 , CHAR_RAWDEL , CHAR_RAWDEL ,
1004 CHAR_RAWDEL , CHAR_RAWDEL , CHAR_PERIOD , CHAR_RAWDEL ,
Chris Lattner03b98662009-07-07 17:09:54 +00001005//48 0 49 1 50 2 51 3
1006//52 4 53 5 54 6 55 7
1007 CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER ,
1008 CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER ,
1009//56 8 57 9 58 : 59 ;
1010//60 < 61 = 62 > 63 ?
Craig Topper2fa4e862011-08-11 04:06:15 +00001011 CHAR_NUMBER , CHAR_NUMBER , CHAR_RAWDEL , CHAR_RAWDEL ,
1012 CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
Chris Lattner03b98662009-07-07 17:09:54 +00001013//64 @ 65 A 66 B 67 C
1014//68 D 69 E 70 F 71 G
1015 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1016 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1017//72 H 73 I 74 J 75 K
1018//76 L 77 M 78 N 79 O
1019 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1020 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1021//80 P 81 Q 82 R 83 S
1022//84 T 85 U 86 V 87 W
1023 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1024 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1025//88 X 89 Y 90 Z 91 [
1026//92 \ 93 ] 94 ^ 95 _
Craig Topper2fa4e862011-08-11 04:06:15 +00001027 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL ,
1028 0 , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_UNDER ,
Chris Lattner03b98662009-07-07 17:09:54 +00001029//96 ` 97 a 98 b 99 c
1030//100 d 101 e 102 f 103 g
1031 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1032 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1033//104 h 105 i 106 j 107 k
1034//108 l 109 m 110 n 111 o
1035 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1036 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1037//112 p 113 q 114 r 115 s
1038//116 t 117 u 118 v 119 w
1039 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1040 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
1041//120 x 121 y 122 z 123 {
Craig Topper2fa4e862011-08-11 04:06:15 +00001042//124 | 125 } 126 ~ 127 DEL
1043 CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL ,
1044 CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 0
Chris Lattner03b98662009-07-07 17:09:54 +00001045};
1046
Chris Lattnera2bf1052009-12-17 05:29:40 +00001047static void InitCharacterInfo() {
Reid Spencer5f016e22007-07-11 17:01:13 +00001048 static bool isInited = false;
1049 if (isInited) return;
Chris Lattner03b98662009-07-07 17:09:54 +00001050 // check the statically-initialized CharInfo table
1051 assert(CHAR_HORZ_WS == CharInfo[(int)' ']);
1052 assert(CHAR_HORZ_WS == CharInfo[(int)'\t']);
1053 assert(CHAR_HORZ_WS == CharInfo[(int)'\f']);
1054 assert(CHAR_HORZ_WS == CharInfo[(int)'\v']);
1055 assert(CHAR_VERT_WS == CharInfo[(int)'\n']);
1056 assert(CHAR_VERT_WS == CharInfo[(int)'\r']);
1057 assert(CHAR_UNDER == CharInfo[(int)'_']);
1058 assert(CHAR_PERIOD == CharInfo[(int)'.']);
1059 for (unsigned i = 'a'; i <= 'z'; ++i) {
1060 assert(CHAR_LETTER == CharInfo[i]);
1061 assert(CHAR_LETTER == CharInfo[i+'A'-'a']);
1062 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001063 for (unsigned i = '0'; i <= '9'; ++i)
Chris Lattner03b98662009-07-07 17:09:54 +00001064 assert(CHAR_NUMBER == CharInfo[i]);
Steve Naroff7b682652009-12-08 16:38:12 +00001065
Chris Lattner03b98662009-07-07 17:09:54 +00001066 isInited = true;
Reid Spencer5f016e22007-07-11 17:01:13 +00001067}
1068
Chris Lattner03b98662009-07-07 17:09:54 +00001069
Reid Spencer5f016e22007-07-11 17:01:13 +00001070/// isIdentifierBody - Return true if this is the body character of an
1071/// identifier, which is [a-zA-Z0-9_].
1072static inline bool isIdentifierBody(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +00001073 return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER)) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001074}
1075
1076/// isHorizontalWhitespace - Return true if this character is horizontal
1077/// whitespace: ' ', '\t', '\f', '\v'. Note that this returns false for '\0'.
1078static inline bool isHorizontalWhitespace(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +00001079 return (CharInfo[c] & CHAR_HORZ_WS) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001080}
1081
Anna Zaksaca25bc2011-07-27 21:43:43 +00001082/// isVerticalWhitespace - Return true if this character is vertical
1083/// whitespace: '\n', '\r'. Note that this returns false for '\0'.
1084static inline bool isVerticalWhitespace(unsigned char c) {
1085 return (CharInfo[c] & CHAR_VERT_WS) ? true : false;
1086}
1087
Reid Spencer5f016e22007-07-11 17:01:13 +00001088/// isWhitespace - Return true if this character is horizontal or vertical
1089/// whitespace: ' ', '\t', '\f', '\v', '\n', '\r'. Note that this returns false
1090/// for '\0'.
1091static inline bool isWhitespace(unsigned char c) {
Hartmut Kaiser95c062b2007-10-18 12:47:01 +00001092 return (CharInfo[c] & (CHAR_HORZ_WS|CHAR_VERT_WS)) ? true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001093}
1094
1095/// isNumberBody - Return true if this is the body character of an
1096/// preprocessing number, which is [a-zA-Z0-9_.].
1097static inline bool isNumberBody(unsigned char c) {
Mike Stump1eb44332009-09-09 15:08:12 +00001098 return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD)) ?
Hartmut Kaiser95c062b2007-10-18 12:47:01 +00001099 true : false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001100}
1101
Craig Topper2fa4e862011-08-11 04:06:15 +00001102/// isRawStringDelimBody - Return true if this is the body character of a
1103/// raw string delimiter.
1104static inline bool isRawStringDelimBody(unsigned char c) {
1105 return (CharInfo[c] &
1106 (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD|CHAR_RAWDEL)) ?
1107 true : false;
1108}
1109
Reid Spencer5f016e22007-07-11 17:01:13 +00001110
1111//===----------------------------------------------------------------------===//
1112// Diagnostics forwarding code.
1113//===----------------------------------------------------------------------===//
1114
Chris Lattner409a0362007-07-22 18:38:25 +00001115/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
Chandler Carruth433db062011-07-14 08:20:40 +00001116/// lexer buffer was all expanded at a single point, perform the mapping.
Chris Lattner409a0362007-07-22 18:38:25 +00001117/// This is currently only used for _Pragma implementation, so it is the slow
1118/// path of the hot getSourceLocation method. Do not allow it to be inlined.
Chandler Carruth14bd9652010-10-23 08:44:57 +00001119static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
1120 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
Chris Lattner409a0362007-07-22 18:38:25 +00001121static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
1122 SourceLocation FileLoc,
Chris Lattnerde7aeef2009-01-26 00:43:02 +00001123 unsigned CharNo, unsigned TokLen) {
Chandler Carruth433db062011-07-14 08:20:40 +00001124 assert(FileLoc.isMacroID() && "Must be a macro expansion");
Mike Stump1eb44332009-09-09 15:08:12 +00001125
Chris Lattner409a0362007-07-22 18:38:25 +00001126 // Otherwise, we're lexing "mapped tokens". This is used for things like
Chandler Carruth433db062011-07-14 08:20:40 +00001127 // _Pragma handling. Combine the expansion location of FileLoc with the
Chris Lattnerdf7c17a2009-01-16 07:00:02 +00001128 // spelling location.
Chris Lattnere7fb4842009-02-15 20:52:18 +00001129 SourceManager &SM = PP.getSourceManager();
Mike Stump1eb44332009-09-09 15:08:12 +00001130
Chandler Carruth433db062011-07-14 08:20:40 +00001131 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
Chris Lattnerdf7c17a2009-01-16 07:00:02 +00001132 // characters come from spelling(FileLoc)+Offset.
Chris Lattnere7fb4842009-02-15 20:52:18 +00001133 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc);
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +00001134 SpellingLoc = SpellingLoc.getLocWithOffset(CharNo);
Mike Stump1eb44332009-09-09 15:08:12 +00001135
Chris Lattnere7fb4842009-02-15 20:52:18 +00001136 // Figure out the expansion loc range, which is the range covered by the
1137 // original _Pragma(...) sequence.
1138 std::pair<SourceLocation,SourceLocation> II =
Chandler Carruth999f7392011-07-25 20:52:21 +00001139 SM.getImmediateExpansionRange(FileLoc);
Mike Stump1eb44332009-09-09 15:08:12 +00001140
Chandler Carruthbf340e42011-07-26 03:03:05 +00001141 return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen);
Chris Lattner409a0362007-07-22 18:38:25 +00001142}
1143
Reid Spencer5f016e22007-07-11 17:01:13 +00001144/// getSourceLocation - Return a source location identifier for the specified
1145/// offset in the current file.
Chris Lattnerde7aeef2009-01-26 00:43:02 +00001146SourceLocation Lexer::getSourceLocation(const char *Loc,
1147 unsigned TokLen) const {
Chris Lattner448cec42007-07-22 18:44:36 +00001148 assert(Loc >= BufferStart && Loc <= BufferEnd &&
Reid Spencer5f016e22007-07-11 17:01:13 +00001149 "Location out of range for this buffer!");
Chris Lattner9dc1f532007-07-20 16:37:10 +00001150
1151 // In the normal case, we're just lexing from a simple file buffer, return
1152 // the file id from FileLoc with the offset specified.
Chris Lattner448cec42007-07-22 18:44:36 +00001153 unsigned CharNo = Loc-BufferStart;
Chris Lattner9dc1f532007-07-20 16:37:10 +00001154 if (FileLoc.isFileID())
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +00001155 return FileLoc.getLocWithOffset(CharNo);
Mike Stump1eb44332009-09-09 15:08:12 +00001156
Chris Lattner2b2453a2009-01-17 06:22:33 +00001157 // Otherwise, this is the _Pragma lexer case, which pretends that all of the
1158 // tokens are lexed from where the _Pragma was defined.
Chris Lattner168ae2d2007-10-17 20:41:00 +00001159 assert(PP && "This doesn't work on raw lexers");
Chris Lattnerde7aeef2009-01-26 00:43:02 +00001160 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen);
Reid Spencer5f016e22007-07-11 17:01:13 +00001161}
1162
Reid Spencer5f016e22007-07-11 17:01:13 +00001163/// Diag - Forwarding function for diagnostics. This translate a source
1164/// position in the current buffer into a SourceLocation object for rendering.
Chris Lattner3cbfe2c2008-11-22 00:59:29 +00001165DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
Chris Lattner3692b092008-11-18 07:59:24 +00001166 return PP->Diag(getSourceLocation(Loc), DiagID);
Reid Spencer5f016e22007-07-11 17:01:13 +00001167}
Reid Spencer5f016e22007-07-11 17:01:13 +00001168
1169//===----------------------------------------------------------------------===//
1170// Trigraph and Escaped Newline Handling Code.
1171//===----------------------------------------------------------------------===//
1172
1173/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
1174/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
1175static char GetTrigraphCharForLetter(char Letter) {
1176 switch (Letter) {
1177 default: return 0;
1178 case '=': return '#';
1179 case ')': return ']';
1180 case '(': return '[';
1181 case '!': return '|';
1182 case '\'': return '^';
1183 case '>': return '}';
1184 case '/': return '\\';
1185 case '<': return '{';
1186 case '-': return '~';
1187 }
1188}
1189
1190/// DecodeTrigraphChar - If the specified character is a legal trigraph when
1191/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled,
1192/// return the result character. Finally, emit a warning about trigraph use
1193/// whether trigraphs are enabled or not.
1194static char DecodeTrigraphChar(const char *CP, Lexer *L) {
1195 char Res = GetTrigraphCharForLetter(*CP);
Chris Lattner3692b092008-11-18 07:59:24 +00001196 if (!Res || !L) return Res;
Mike Stump1eb44332009-09-09 15:08:12 +00001197
Chris Lattner3692b092008-11-18 07:59:24 +00001198 if (!L->getFeatures().Trigraphs) {
Chris Lattner74d15df2008-11-22 02:02:22 +00001199 if (!L->isLexingRawMode())
1200 L->Diag(CP-2, diag::trigraph_ignored);
Chris Lattner3692b092008-11-18 07:59:24 +00001201 return 0;
Reid Spencer5f016e22007-07-11 17:01:13 +00001202 }
Mike Stump1eb44332009-09-09 15:08:12 +00001203
Chris Lattner74d15df2008-11-22 02:02:22 +00001204 if (!L->isLexingRawMode())
Chris Lattner5f9e2722011-07-23 10:55:15 +00001205 L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1);
Reid Spencer5f016e22007-07-11 17:01:13 +00001206 return Res;
1207}
1208
Chris Lattner24f0e482009-04-18 22:05:41 +00001209/// getEscapedNewLineSize - Return the size of the specified escaped newline,
1210/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
Mike Stump1eb44332009-09-09 15:08:12 +00001211/// trigraph equivalent on entry to this function.
Chris Lattner24f0e482009-04-18 22:05:41 +00001212unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
1213 unsigned Size = 0;
1214 while (isWhitespace(Ptr[Size])) {
1215 ++Size;
Mike Stump1eb44332009-09-09 15:08:12 +00001216
Chris Lattner24f0e482009-04-18 22:05:41 +00001217 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r')
1218 continue;
1219
1220 // If this is a \r\n or \n\r, skip the other half.
1221 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') &&
1222 Ptr[Size-1] != Ptr[Size])
1223 ++Size;
Mike Stump1eb44332009-09-09 15:08:12 +00001224
Chris Lattner24f0e482009-04-18 22:05:41 +00001225 return Size;
Mike Stump1eb44332009-09-09 15:08:12 +00001226 }
1227
Chris Lattner24f0e482009-04-18 22:05:41 +00001228 // Not an escaped newline, must be a \t or something else.
1229 return 0;
1230}
1231
Chris Lattner03374952009-04-18 22:27:02 +00001232/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
1233/// them), skip over them and return the first non-escaped-newline found,
1234/// otherwise return P.
1235const char *Lexer::SkipEscapedNewLines(const char *P) {
1236 while (1) {
1237 const char *AfterEscape;
1238 if (*P == '\\') {
1239 AfterEscape = P+1;
1240 } else if (*P == '?') {
1241 // If not a trigraph for escape, bail out.
1242 if (P[1] != '?' || P[2] != '/')
1243 return P;
1244 AfterEscape = P+3;
1245 } else {
1246 return P;
1247 }
Mike Stump1eb44332009-09-09 15:08:12 +00001248
Chris Lattner03374952009-04-18 22:27:02 +00001249 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape);
1250 if (NewLineSize == 0) return P;
1251 P = AfterEscape+NewLineSize;
1252 }
1253}
1254
Anna Zaksaca25bc2011-07-27 21:43:43 +00001255/// \brief Checks that the given token is the first token that occurs after the
1256/// given location (this excludes comments and whitespace). Returns the location
1257/// immediately after the specified token. If the token is not found or the
1258/// location is inside a macro, the returned source location will be invalid.
1259SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc,
1260 tok::TokenKind TKind,
1261 const SourceManager &SM,
1262 const LangOptions &LangOpts,
1263 bool SkipTrailingWhitespaceAndNewLine) {
1264 if (Loc.isMacroID()) {
Argyrios Kyrtzidis69bda4c2012-01-19 15:59:08 +00001265 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
Anna Zaksaca25bc2011-07-27 21:43:43 +00001266 return SourceLocation();
Anna Zaksaca25bc2011-07-27 21:43:43 +00001267 }
1268 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);
1269
1270 // Break down the source location.
1271 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
1272
1273 // Try to load the file buffer.
1274 bool InvalidTemp = false;
1275 llvm::StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp);
1276 if (InvalidTemp)
1277 return SourceLocation();
1278
1279 const char *TokenBegin = File.data() + LocInfo.second;
1280
1281 // Lex from the start of the given location.
1282 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(),
1283 TokenBegin, File.end());
1284 // Find the token.
1285 Token Tok;
1286 lexer.LexFromRawLexer(Tok);
1287 if (Tok.isNot(TKind))
1288 return SourceLocation();
1289 SourceLocation TokenLoc = Tok.getLocation();
1290
1291 // Calculate how much whitespace needs to be skipped if any.
1292 unsigned NumWhitespaceChars = 0;
1293 if (SkipTrailingWhitespaceAndNewLine) {
1294 const char *TokenEnd = SM.getCharacterData(TokenLoc) +
1295 Tok.getLength();
1296 unsigned char C = *TokenEnd;
1297 while (isHorizontalWhitespace(C)) {
1298 C = *(++TokenEnd);
1299 NumWhitespaceChars++;
1300 }
1301 if (isVerticalWhitespace(C))
1302 NumWhitespaceChars++;
1303 }
1304
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +00001305 return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars);
Anna Zaksaca25bc2011-07-27 21:43:43 +00001306}
Chris Lattner24f0e482009-04-18 22:05:41 +00001307
Reid Spencer5f016e22007-07-11 17:01:13 +00001308/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
1309/// get its size, and return it. This is tricky in several cases:
1310/// 1. If currently at the start of a trigraph, we warn about the trigraph,
1311/// then either return the trigraph (skipping 3 chars) or the '?',
1312/// depending on whether trigraphs are enabled or not.
1313/// 2. If this is an escaped newline (potentially with whitespace between
1314/// the backslash and newline), implicitly skip the newline and return
1315/// the char after it.
1316/// 3. If this is a UCN, return it. FIXME: C++ UCN's?
1317///
1318/// This handles the slow/uncommon case of the getCharAndSize method. Here we
1319/// know that we can accumulate into Size, and that we have already incremented
1320/// Ptr by Size bytes.
1321///
1322/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
1323/// be updated to match.
1324///
1325char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
Chris Lattnerd2177732007-07-20 16:59:19 +00001326 Token *Tok) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001327 // If we have a slash, look for an escaped newline.
1328 if (Ptr[0] == '\\') {
1329 ++Size;
1330 ++Ptr;
1331Slash:
1332 // Common case, backslash-char where the char is not whitespace.
1333 if (!isWhitespace(Ptr[0])) return '\\';
Mike Stump1eb44332009-09-09 15:08:12 +00001334
Chris Lattner5636a3b2009-06-23 05:15:06 +00001335 // See if we have optional whitespace characters between the slash and
1336 // newline.
Chris Lattner24f0e482009-04-18 22:05:41 +00001337 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1338 // Remember that this token needs to be cleaned.
1339 if (Tok) Tok->setFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +00001340
Chris Lattner24f0e482009-04-18 22:05:41 +00001341 // Warn if there was whitespace between the backslash and newline.
Chris Lattner5636a3b2009-06-23 05:15:06 +00001342 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
Chris Lattner24f0e482009-04-18 22:05:41 +00001343 Diag(Ptr, diag::backslash_newline_space);
Mike Stump1eb44332009-09-09 15:08:12 +00001344
Chris Lattner24f0e482009-04-18 22:05:41 +00001345 // Found backslash<whitespace><newline>. Parse the char after it.
1346 Size += EscapedNewLineSize;
1347 Ptr += EscapedNewLineSize;
Argyrios Kyrtzidisf132dca2011-12-21 20:19:55 +00001348
Argyrios Kyrtzidis04a94bc2011-12-22 04:38:07 +00001349 // If the char that we finally got was a \n, then we must have had
1350 // something like \<newline><newline>. We don't want to consume the
1351 // second newline.
1352 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0')
1353 return ' ';
Argyrios Kyrtzidisf132dca2011-12-21 20:19:55 +00001354
Chris Lattner24f0e482009-04-18 22:05:41 +00001355 // Use slow version to accumulate a correct size field.
1356 return getCharAndSizeSlow(Ptr, Size, Tok);
1357 }
Mike Stump1eb44332009-09-09 15:08:12 +00001358
Reid Spencer5f016e22007-07-11 17:01:13 +00001359 // Otherwise, this is not an escaped newline, just return the slash.
1360 return '\\';
1361 }
Mike Stump1eb44332009-09-09 15:08:12 +00001362
Reid Spencer5f016e22007-07-11 17:01:13 +00001363 // If this is a trigraph, process it.
1364 if (Ptr[0] == '?' && Ptr[1] == '?') {
1365 // If this is actually a legal trigraph (not something like "??x"), emit
1366 // a trigraph warning. If so, and if trigraphs are enabled, return it.
1367 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) {
1368 // Remember that this token needs to be cleaned.
Chris Lattnerd2177732007-07-20 16:59:19 +00001369 if (Tok) Tok->setFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +00001370
1371 Ptr += 3;
1372 Size += 3;
1373 if (C == '\\') goto Slash;
1374 return C;
1375 }
1376 }
Mike Stump1eb44332009-09-09 15:08:12 +00001377
Reid Spencer5f016e22007-07-11 17:01:13 +00001378 // If this is neither, return a single character.
1379 ++Size;
1380 return *Ptr;
1381}
1382
1383
1384/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
1385/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size,
1386/// and that we have already incremented Ptr by Size bytes.
1387///
1388/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
1389/// be updated to match.
1390char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
1391 const LangOptions &Features) {
1392 // If we have a slash, look for an escaped newline.
1393 if (Ptr[0] == '\\') {
1394 ++Size;
1395 ++Ptr;
1396Slash:
1397 // Common case, backslash-char where the char is not whitespace.
1398 if (!isWhitespace(Ptr[0])) return '\\';
Mike Stump1eb44332009-09-09 15:08:12 +00001399
Reid Spencer5f016e22007-07-11 17:01:13 +00001400 // See if we have optional whitespace characters followed by a newline.
Chris Lattner24f0e482009-04-18 22:05:41 +00001401 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
1402 // Found backslash<whitespace><newline>. Parse the char after it.
1403 Size += EscapedNewLineSize;
1404 Ptr += EscapedNewLineSize;
Mike Stump1eb44332009-09-09 15:08:12 +00001405
Argyrios Kyrtzidis04a94bc2011-12-22 04:38:07 +00001406 // If the char that we finally got was a \n, then we must have had
1407 // something like \<newline><newline>. We don't want to consume the
1408 // second newline.
1409 if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0')
1410 return ' ';
Argyrios Kyrtzidisf132dca2011-12-21 20:19:55 +00001411
Chris Lattner24f0e482009-04-18 22:05:41 +00001412 // Use slow version to accumulate a correct size field.
1413 return getCharAndSizeSlowNoWarn(Ptr, Size, Features);
1414 }
Mike Stump1eb44332009-09-09 15:08:12 +00001415
Reid Spencer5f016e22007-07-11 17:01:13 +00001416 // Otherwise, this is not an escaped newline, just return the slash.
1417 return '\\';
1418 }
Mike Stump1eb44332009-09-09 15:08:12 +00001419
Reid Spencer5f016e22007-07-11 17:01:13 +00001420 // If this is a trigraph, process it.
1421 if (Features.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
1422 // If this is actually a legal trigraph (not something like "??x"), return
1423 // it.
1424 if (char C = GetTrigraphCharForLetter(Ptr[2])) {
1425 Ptr += 3;
1426 Size += 3;
1427 if (C == '\\') goto Slash;
1428 return C;
1429 }
1430 }
Mike Stump1eb44332009-09-09 15:08:12 +00001431
Reid Spencer5f016e22007-07-11 17:01:13 +00001432 // If this is neither, return a single character.
1433 ++Size;
1434 return *Ptr;
1435}
1436
1437//===----------------------------------------------------------------------===//
1438// Helper methods for lexing.
1439//===----------------------------------------------------------------------===//
1440
Douglas Gregorf4f6c9d2010-07-26 21:36:20 +00001441/// \brief Routine that indiscriminately skips bytes in the source file.
1442void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) {
1443 BufferPtr += Bytes;
1444 if (BufferPtr > BufferEnd)
1445 BufferPtr = BufferEnd;
1446 IsAtStartOfLine = StartOfLine;
1447}
1448
Chris Lattnerd2177732007-07-20 16:59:19 +00001449void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001450 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
1451 unsigned Size;
1452 unsigned char C = *CurPtr++;
Chris Lattnercd991db2010-01-11 02:38:50 +00001453 while (isIdentifierBody(C))
Reid Spencer5f016e22007-07-11 17:01:13 +00001454 C = *CurPtr++;
Chris Lattnercd991db2010-01-11 02:38:50 +00001455
Reid Spencer5f016e22007-07-11 17:01:13 +00001456 --CurPtr; // Back up over the skipped character.
1457
1458 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline
1459 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN.
1460 // FIXME: UCNs.
Chris Lattnercd991db2010-01-11 02:38:50 +00001461 //
1462 // TODO: Could merge these checks into a CharInfo flag to make the comparison
1463 // cheaper
Reid Spencer5f016e22007-07-11 17:01:13 +00001464 if (C != '\\' && C != '?' && (C != '$' || !Features.DollarIdents)) {
1465FinishIdentifier:
1466 const char *IdStart = BufferPtr;
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +00001467 FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
1468 Result.setRawIdentifierData(IdStart);
Mike Stump1eb44332009-09-09 15:08:12 +00001469
Reid Spencer5f016e22007-07-11 17:01:13 +00001470 // If we are in raw mode, return this identifier raw. There is no need to
1471 // look up identifier information or attempt to macro expand it.
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +00001472 if (LexingRawMode)
1473 return;
Mike Stump1eb44332009-09-09 15:08:12 +00001474
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +00001475 // Fill in Result.IdentifierInfo and update the token kind,
1476 // looking up the identifier in the identifier table.
1477 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result);
Mike Stump1eb44332009-09-09 15:08:12 +00001478
Reid Spencer5f016e22007-07-11 17:01:13 +00001479 // Finally, now that we know we have an identifier, pass this off to the
1480 // preprocessor, which may macro expand it or something.
Chris Lattnerd1186fa2009-01-21 07:45:14 +00001481 if (II->isHandleIdentifierCase())
Chris Lattner6a170eb2009-01-21 07:43:11 +00001482 PP->HandleIdentifier(Result);
Douglas Gregor6aa52ec2011-08-26 23:56:07 +00001483
Chris Lattner6a170eb2009-01-21 07:43:11 +00001484 return;
Reid Spencer5f016e22007-07-11 17:01:13 +00001485 }
Mike Stump1eb44332009-09-09 15:08:12 +00001486
Reid Spencer5f016e22007-07-11 17:01:13 +00001487 // Otherwise, $,\,? in identifier found. Enter slower path.
Mike Stump1eb44332009-09-09 15:08:12 +00001488
Reid Spencer5f016e22007-07-11 17:01:13 +00001489 C = getCharAndSize(CurPtr, Size);
1490 while (1) {
1491 if (C == '$') {
1492 // If we hit a $ and they are not supported in identifiers, we are done.
1493 if (!Features.DollarIdents) goto FinishIdentifier;
Mike Stump1eb44332009-09-09 15:08:12 +00001494
Reid Spencer5f016e22007-07-11 17:01:13 +00001495 // Otherwise, emit a diagnostic and continue.
Chris Lattner74d15df2008-11-22 02:02:22 +00001496 if (!isLexingRawMode())
1497 Diag(CurPtr, diag::ext_dollar_in_identifier);
Reid Spencer5f016e22007-07-11 17:01:13 +00001498 CurPtr = ConsumeChar(CurPtr, Size, Result);
1499 C = getCharAndSize(CurPtr, Size);
1500 continue;
1501 } else if (!isIdentifierBody(C)) { // FIXME: UCNs.
1502 // Found end of identifier.
1503 goto FinishIdentifier;
1504 }
1505
1506 // Otherwise, this character is good, consume it.
1507 CurPtr = ConsumeChar(CurPtr, Size, Result);
1508
1509 C = getCharAndSize(CurPtr, Size);
1510 while (isIdentifierBody(C)) { // FIXME: UCNs.
1511 CurPtr = ConsumeChar(CurPtr, Size, Result);
1512 C = getCharAndSize(CurPtr, Size);
1513 }
1514 }
1515}
1516
Douglas Gregora75ec432010-08-30 14:50:47 +00001517/// isHexaLiteral - Return true if Start points to a hex constant.
Chris Lattner4a551002010-08-30 17:11:14 +00001518/// in microsoft mode (where this is supposed to be several different tokens).
Chris Lattner6ab55eb2010-08-31 16:42:00 +00001519static bool isHexaLiteral(const char *Start, const LangOptions &Features) {
1520 unsigned Size;
1521 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, Features);
1522 if (C1 != '0')
1523 return false;
1524 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, Features);
1525 return (C2 == 'x' || C2 == 'X');
Douglas Gregora75ec432010-08-30 14:50:47 +00001526}
Reid Spencer5f016e22007-07-11 17:01:13 +00001527
Nate Begeman5253c7f2008-04-14 02:26:39 +00001528/// LexNumericConstant - Lex the remainder of a integer or floating point
Reid Spencer5f016e22007-07-11 17:01:13 +00001529/// constant. From[-1] is the first character lexed. Return the end of the
1530/// constant.
Chris Lattnerd2177732007-07-20 16:59:19 +00001531void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001532 unsigned Size;
1533 char C = getCharAndSize(CurPtr, Size);
1534 char PrevCh = 0;
1535 while (isNumberBody(C)) { // FIXME: UCNs?
1536 CurPtr = ConsumeChar(CurPtr, Size, Result);
1537 PrevCh = C;
1538 C = getCharAndSize(CurPtr, Size);
1539 }
Mike Stump1eb44332009-09-09 15:08:12 +00001540
Reid Spencer5f016e22007-07-11 17:01:13 +00001541 // If we fell out, check for a sign, due to 1e+12. If we have one, continue.
Chris Lattnerb2f4a202010-08-30 17:09:08 +00001542 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
1543 // If we are in Microsoft mode, don't continue if the constant is hex.
1544 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
Francois Pichet62ec1f22011-09-17 17:15:52 +00001545 if (!Features.MicrosoftExt || !isHexaLiteral(BufferPtr, Features))
Chris Lattnerb2f4a202010-08-30 17:09:08 +00001546 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
1547 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001548
1549 // If we have a hex FP constant, continue.
Douglas Gregor46717302011-10-12 18:51:02 +00001550 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p'))
Reid Spencer5f016e22007-07-11 17:01:13 +00001551 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
Mike Stump1eb44332009-09-09 15:08:12 +00001552
Reid Spencer5f016e22007-07-11 17:01:13 +00001553 // Update the location of token as well as BufferPtr.
Chris Lattner47246be2009-01-26 19:29:26 +00001554 const char *TokStart = BufferPtr;
Chris Lattner9e6293d2008-10-12 04:51:35 +00001555 FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
Chris Lattner47246be2009-01-26 19:29:26 +00001556 Result.setLiteralData(TokStart);
Reid Spencer5f016e22007-07-11 17:01:13 +00001557}
1558
1559/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
Douglas Gregor5cee1192011-07-27 05:40:30 +00001560/// either " or L" or u8" or u" or U".
1561void Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
1562 tok::TokenKind Kind) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001563 const char *NulCharacter = 0; // Does this string contain the \0 character?
Mike Stump1eb44332009-09-09 15:08:12 +00001564
Richard Smith661a9962011-10-15 01:18:56 +00001565 if (!isLexingRawMode() &&
1566 (Kind == tok::utf8_string_literal ||
1567 Kind == tok::utf16_string_literal ||
1568 Kind == tok::utf32_string_literal))
1569 Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal);
1570
Reid Spencer5f016e22007-07-11 17:01:13 +00001571 char C = getAndAdvanceChar(CurPtr, Result);
1572 while (C != '"') {
Chris Lattner571339c2010-05-30 23:27:38 +00001573 // Skip escaped characters. Escaped newlines will already be processed by
1574 // getAndAdvanceChar.
1575 if (C == '\\')
Reid Spencer5f016e22007-07-11 17:01:13 +00001576 C = getAndAdvanceChar(CurPtr, Result);
Douglas Gregor33611e02010-05-30 22:59:50 +00001577
Chris Lattner571339c2010-05-30 23:27:38 +00001578 if (C == '\n' || C == '\r' || // Newline.
Douglas Gregor33611e02010-05-30 22:59:50 +00001579 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00001580 if (!isLexingRawMode() && !Features.AsmPreprocessor)
Argyrios Kyrtzidisff1ed982011-02-15 23:45:31 +00001581 Diag(BufferPtr, diag::warn_unterminated_string);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001582 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
Reid Spencer5f016e22007-07-11 17:01:13 +00001583 return;
Reid Spencer5f016e22007-07-11 17:01:13 +00001584 }
Chris Lattner571339c2010-05-30 23:27:38 +00001585
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00001586 if (C == 0) {
1587 if (isCodeCompletionPoint(CurPtr-1)) {
1588 PP->CodeCompleteNaturalLanguage();
1589 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1590 return cutOffLexing();
1591 }
1592
Chris Lattner571339c2010-05-30 23:27:38 +00001593 NulCharacter = CurPtr-1;
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00001594 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001595 C = getAndAdvanceChar(CurPtr, Result);
1596 }
Mike Stump1eb44332009-09-09 15:08:12 +00001597
Reid Spencer5f016e22007-07-11 17:01:13 +00001598 // If a nul character existed in the string, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +00001599 if (NulCharacter && !isLexingRawMode())
1600 Diag(NulCharacter, diag::null_in_string);
Reid Spencer5f016e22007-07-11 17:01:13 +00001601
Reid Spencer5f016e22007-07-11 17:01:13 +00001602 // Update the location of the token as well as the BufferPtr instance var.
Chris Lattner47246be2009-01-26 19:29:26 +00001603 const char *TokStart = BufferPtr;
Douglas Gregor5cee1192011-07-27 05:40:30 +00001604 FormTokenWithChars(Result, CurPtr, Kind);
Chris Lattner47246be2009-01-26 19:29:26 +00001605 Result.setLiteralData(TokStart);
Reid Spencer5f016e22007-07-11 17:01:13 +00001606}
1607
Craig Topper2fa4e862011-08-11 04:06:15 +00001608/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
1609/// having lexed R", LR", u8R", uR", or UR".
1610void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
1611 tok::TokenKind Kind) {
1612 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
1613 // Between the initial and final double quote characters of the raw string,
1614 // any transformations performed in phases 1 and 2 (trigraphs,
1615 // universal-character-names, and line splicing) are reverted.
1616
Richard Smith661a9962011-10-15 01:18:56 +00001617 if (!isLexingRawMode())
1618 Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal);
1619
Craig Topper2fa4e862011-08-11 04:06:15 +00001620 unsigned PrefixLen = 0;
1621
1622 while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen]))
1623 ++PrefixLen;
1624
1625 // If the last character was not a '(', then we didn't lex a valid delimiter.
1626 if (CurPtr[PrefixLen] != '(') {
1627 if (!isLexingRawMode()) {
1628 const char *PrefixEnd = &CurPtr[PrefixLen];
1629 if (PrefixLen == 16) {
1630 Diag(PrefixEnd, diag::err_raw_delim_too_long);
1631 } else {
1632 Diag(PrefixEnd, diag::err_invalid_char_raw_delim)
1633 << StringRef(PrefixEnd, 1);
1634 }
1635 }
1636
1637 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
1638 // it's possible the '"' was intended to be part of the raw string, but
1639 // there's not much we can do about that.
1640 while (1) {
1641 char C = *CurPtr++;
1642
1643 if (C == '"')
1644 break;
1645 if (C == 0 && CurPtr-1 == BufferEnd) {
1646 --CurPtr;
1647 break;
1648 }
1649 }
1650
1651 FormTokenWithChars(Result, CurPtr, tok::unknown);
1652 return;
1653 }
1654
1655 // Save prefix and move CurPtr past it
1656 const char *Prefix = CurPtr;
1657 CurPtr += PrefixLen + 1; // skip over prefix and '('
1658
1659 while (1) {
1660 char C = *CurPtr++;
1661
1662 if (C == ')') {
1663 // Check for prefix match and closing quote.
1664 if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
1665 CurPtr += PrefixLen + 1; // skip over prefix and '"'
1666 break;
1667 }
1668 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
1669 if (!isLexingRawMode())
1670 Diag(BufferPtr, diag::err_unterminated_raw_string)
1671 << StringRef(Prefix, PrefixLen);
1672 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1673 return;
1674 }
1675 }
1676
1677 // Update the location of token as well as BufferPtr.
1678 const char *TokStart = BufferPtr;
1679 FormTokenWithChars(Result, CurPtr, Kind);
1680 Result.setLiteralData(TokStart);
1681}
1682
Reid Spencer5f016e22007-07-11 17:01:13 +00001683/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
1684/// after having lexed the '<' character. This is used for #include filenames.
Chris Lattnerd2177732007-07-20 16:59:19 +00001685void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001686 const char *NulCharacter = 0; // Does this string contain the \0 character?
Chris Lattner9cb51ce2009-04-17 23:56:52 +00001687 const char *AfterLessPos = CurPtr;
Reid Spencer5f016e22007-07-11 17:01:13 +00001688 char C = getAndAdvanceChar(CurPtr, Result);
1689 while (C != '>') {
1690 // Skip escaped characters.
1691 if (C == '\\') {
1692 // Skip the escaped character.
1693 C = getAndAdvanceChar(CurPtr, Result);
1694 } else if (C == '\n' || C == '\r' || // Newline.
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00001695 (C == 0 && (CurPtr-1 == BufferEnd || // End of file.
1696 isCodeCompletionPoint(CurPtr-1)))) {
Chris Lattner9cb51ce2009-04-17 23:56:52 +00001697 // If the filename is unterminated, then it must just be a lone <
1698 // character. Return this as such.
1699 FormTokenWithChars(Result, AfterLessPos, tok::less);
Reid Spencer5f016e22007-07-11 17:01:13 +00001700 return;
1701 } else if (C == 0) {
1702 NulCharacter = CurPtr-1;
1703 }
1704 C = getAndAdvanceChar(CurPtr, Result);
1705 }
Mike Stump1eb44332009-09-09 15:08:12 +00001706
Reid Spencer5f016e22007-07-11 17:01:13 +00001707 // If a nul character existed in the string, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +00001708 if (NulCharacter && !isLexingRawMode())
1709 Diag(NulCharacter, diag::null_in_string);
Mike Stump1eb44332009-09-09 15:08:12 +00001710
Reid Spencer5f016e22007-07-11 17:01:13 +00001711 // Update the location of token as well as BufferPtr.
Chris Lattner47246be2009-01-26 19:29:26 +00001712 const char *TokStart = BufferPtr;
Chris Lattner9e6293d2008-10-12 04:51:35 +00001713 FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
Chris Lattner47246be2009-01-26 19:29:26 +00001714 Result.setLiteralData(TokStart);
Reid Spencer5f016e22007-07-11 17:01:13 +00001715}
1716
1717
1718/// LexCharConstant - Lex the remainder of a character constant, after having
Douglas Gregor5cee1192011-07-27 05:40:30 +00001719/// lexed either ' or L' or u' or U'.
1720void Lexer::LexCharConstant(Token &Result, const char *CurPtr,
1721 tok::TokenKind Kind) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001722 const char *NulCharacter = 0; // Does this character contain the \0 character?
1723
Richard Smith661a9962011-10-15 01:18:56 +00001724 if (!isLexingRawMode() &&
1725 (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant))
1726 Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal);
1727
Reid Spencer5f016e22007-07-11 17:01:13 +00001728 char C = getAndAdvanceChar(CurPtr, Result);
1729 if (C == '\'') {
Chris Lattner33ab3f62009-03-18 21:10:12 +00001730 if (!isLexingRawMode() && !Features.AsmPreprocessor)
Chris Lattner74d15df2008-11-22 02:02:22 +00001731 Diag(BufferPtr, diag::err_empty_character);
Chris Lattner9e6293d2008-10-12 04:51:35 +00001732 FormTokenWithChars(Result, CurPtr, tok::unknown);
Reid Spencer5f016e22007-07-11 17:01:13 +00001733 return;
Chris Lattnerd80f7862010-07-07 23:24:27 +00001734 }
1735
1736 while (C != '\'') {
1737 // Skip escaped characters.
1738 if (C == '\\') {
1739 // Skip the escaped character.
1740 // FIXME: UCN's
1741 C = getAndAdvanceChar(CurPtr, Result);
1742 } else if (C == '\n' || C == '\r' || // Newline.
1743 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file.
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00001744 if (!isLexingRawMode() && !Features.AsmPreprocessor)
Argyrios Kyrtzidisff1ed982011-02-15 23:45:31 +00001745 Diag(BufferPtr, diag::warn_unterminated_char);
Chris Lattnerd80f7862010-07-07 23:24:27 +00001746 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1747 return;
1748 } else if (C == 0) {
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00001749 if (isCodeCompletionPoint(CurPtr-1)) {
1750 PP->CodeCompleteNaturalLanguage();
1751 FormTokenWithChars(Result, CurPtr-1, tok::unknown);
1752 return cutOffLexing();
1753 }
1754
Chris Lattnerd80f7862010-07-07 23:24:27 +00001755 NulCharacter = CurPtr-1;
1756 }
Reid Spencer5f016e22007-07-11 17:01:13 +00001757 C = getAndAdvanceChar(CurPtr, Result);
1758 }
Mike Stump1eb44332009-09-09 15:08:12 +00001759
Chris Lattnerd80f7862010-07-07 23:24:27 +00001760 // If a nul character existed in the character, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +00001761 if (NulCharacter && !isLexingRawMode())
1762 Diag(NulCharacter, diag::null_in_char);
Reid Spencer5f016e22007-07-11 17:01:13 +00001763
Reid Spencer5f016e22007-07-11 17:01:13 +00001764 // Update the location of token as well as BufferPtr.
Chris Lattner47246be2009-01-26 19:29:26 +00001765 const char *TokStart = BufferPtr;
Douglas Gregor5cee1192011-07-27 05:40:30 +00001766 FormTokenWithChars(Result, CurPtr, Kind);
Chris Lattner47246be2009-01-26 19:29:26 +00001767 Result.setLiteralData(TokStart);
Reid Spencer5f016e22007-07-11 17:01:13 +00001768}
1769
1770/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
1771/// Update BufferPtr to point to the next non-whitespace character and return.
Chris Lattnerd88dc482008-10-12 04:05:48 +00001772///
1773/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
1774///
1775bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001776 // Whitespace - Skip it, then return the token after the whitespace.
1777 unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently.
1778 while (1) {
1779 // Skip horizontal whitespace very aggressively.
1780 while (isHorizontalWhitespace(Char))
1781 Char = *++CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001782
Daniel Dunbarddd3e8b2008-11-25 00:20:22 +00001783 // Otherwise if we have something other than whitespace, we're done.
Reid Spencer5f016e22007-07-11 17:01:13 +00001784 if (Char != '\n' && Char != '\r')
1785 break;
Mike Stump1eb44332009-09-09 15:08:12 +00001786
Reid Spencer5f016e22007-07-11 17:01:13 +00001787 if (ParsingPreprocessorDirective) {
1788 // End of preprocessor directive line, let LexTokenInternal handle this.
1789 BufferPtr = CurPtr;
Chris Lattnerd88dc482008-10-12 04:05:48 +00001790 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001791 }
Mike Stump1eb44332009-09-09 15:08:12 +00001792
Reid Spencer5f016e22007-07-11 17:01:13 +00001793 // ok, but handle newline.
1794 // The returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +00001795 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00001796 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +00001797 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001798 Char = *++CurPtr;
1799 }
1800
1801 // If this isn't immediately after a newline, there is leading space.
1802 char PrevChar = CurPtr[-1];
1803 if (PrevChar != '\n' && PrevChar != '\r')
Chris Lattnerd2177732007-07-20 16:59:19 +00001804 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001805
Chris Lattnerd88dc482008-10-12 04:05:48 +00001806 // If the client wants us to return whitespace, return it now.
1807 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001808 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattnerd88dc482008-10-12 04:05:48 +00001809 return true;
1810 }
Mike Stump1eb44332009-09-09 15:08:12 +00001811
Reid Spencer5f016e22007-07-11 17:01:13 +00001812 BufferPtr = CurPtr;
Chris Lattnerd88dc482008-10-12 04:05:48 +00001813 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001814}
1815
1816// SkipBCPLComment - We have just read the // characters from input. Skip until
1817// we find the newline character thats terminate the comment. Then update
Chris Lattner046c2272010-01-18 22:35:47 +00001818/// BufferPtr and return.
1819///
1820/// If we're in KeepCommentMode or any CommentHandler has inserted
1821/// some tokens, this will store the first token and return true.
Chris Lattnerd2177732007-07-20 16:59:19 +00001822bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001823 // If BCPL comments aren't explicitly enabled for this language, emit an
1824 // extension warning.
Chris Lattner74d15df2008-11-22 02:02:22 +00001825 if (!Features.BCPLComment && !isLexingRawMode()) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001826 Diag(BufferPtr, diag::ext_bcpl_comment);
Mike Stump1eb44332009-09-09 15:08:12 +00001827
Reid Spencer5f016e22007-07-11 17:01:13 +00001828 // Mark them enabled so we only emit one warning for this translation
1829 // unit.
1830 Features.BCPLComment = true;
1831 }
Mike Stump1eb44332009-09-09 15:08:12 +00001832
Reid Spencer5f016e22007-07-11 17:01:13 +00001833 // Scan over the body of the comment. The common case, when scanning, is that
1834 // the comment contains normal ascii characters with nothing interesting in
1835 // them. As such, optimize for this case with the inner loop.
1836 char C;
1837 do {
1838 C = *CurPtr;
Reid Spencer5f016e22007-07-11 17:01:13 +00001839 // Skip over characters in the fast loop.
1840 while (C != 0 && // Potentially EOF.
Reid Spencer5f016e22007-07-11 17:01:13 +00001841 C != '\n' && C != '\r') // Newline or DOS-style newline.
1842 C = *++CurPtr;
1843
Benjamin Kramer1daa58e2011-09-05 07:19:39 +00001844 const char *NextLine = CurPtr;
1845 if (C != 0) {
1846 // We found a newline, see if it's escaped.
1847 const char *EscapePtr = CurPtr-1;
1848 while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace.
1849 --EscapePtr;
1850
1851 if (*EscapePtr == '\\') // Escaped newline.
1852 CurPtr = EscapePtr;
1853 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' &&
1854 EscapePtr[-2] == '?') // Trigraph-escaped newline.
1855 CurPtr = EscapePtr-2;
1856 else
1857 break; // This is a newline, we're done.
1858
1859 C = *CurPtr;
1860 }
Mike Stump1eb44332009-09-09 15:08:12 +00001861
Reid Spencer5f016e22007-07-11 17:01:13 +00001862 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to
Chris Lattnerbc3e9842008-12-12 07:34:39 +00001863 // properly decode the character. Read it in raw mode to avoid emitting
1864 // diagnostics about things like trigraphs. If we see an escaped newline,
1865 // we'll handle it below.
Reid Spencer5f016e22007-07-11 17:01:13 +00001866 const char *OldPtr = CurPtr;
Chris Lattnerbc3e9842008-12-12 07:34:39 +00001867 bool OldRawMode = isLexingRawMode();
1868 LexingRawMode = true;
Reid Spencer5f016e22007-07-11 17:01:13 +00001869 C = getAndAdvanceChar(CurPtr, Result);
Chris Lattnerbc3e9842008-12-12 07:34:39 +00001870 LexingRawMode = OldRawMode;
Chris Lattneread616c2009-04-05 00:26:41 +00001871
Benjamin Kramer1daa58e2011-09-05 07:19:39 +00001872 // If we only read only one character, then no special handling is needed.
1873 // We're done and can skip forward to the newline.
1874 if (C != 0 && CurPtr == OldPtr+1) {
1875 CurPtr = NextLine;
1876 break;
1877 }
1878
Reid Spencer5f016e22007-07-11 17:01:13 +00001879 // If we read multiple characters, and one of those characters was a \r or
1880 // \n, then we had an escaped newline within the comment. Emit diagnostic
1881 // unless the next line is also a // comment.
1882 if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') {
1883 for (; OldPtr != CurPtr; ++OldPtr)
1884 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
1885 // Okay, we found a // comment that ends in a newline, if the next
1886 // line is also a // comment, but has spaces, don't emit a diagnostic.
Benjamin Kramer5d6ae282011-09-05 07:19:35 +00001887 if (isWhitespace(C)) {
Reid Spencer5f016e22007-07-11 17:01:13 +00001888 const char *ForwardPtr = CurPtr;
Benjamin Kramer5d6ae282011-09-05 07:19:35 +00001889 while (isWhitespace(*ForwardPtr)) // Skip whitespace.
Reid Spencer5f016e22007-07-11 17:01:13 +00001890 ++ForwardPtr;
1891 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
1892 break;
1893 }
Mike Stump1eb44332009-09-09 15:08:12 +00001894
Chris Lattner74d15df2008-11-22 02:02:22 +00001895 if (!isLexingRawMode())
1896 Diag(OldPtr-1, diag::ext_multi_line_bcpl_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00001897 break;
1898 }
1899 }
Mike Stump1eb44332009-09-09 15:08:12 +00001900
Douglas Gregor55817af2010-08-25 17:04:25 +00001901 if (CurPtr == BufferEnd+1) {
Douglas Gregor55817af2010-08-25 17:04:25 +00001902 --CurPtr;
1903 break;
1904 }
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00001905
1906 if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
1907 PP->CodeCompleteNaturalLanguage();
1908 cutOffLexing();
1909 return false;
1910 }
1911
Reid Spencer5f016e22007-07-11 17:01:13 +00001912 } while (C != '\n' && C != '\r');
1913
Chris Lattner3d0ad582010-02-03 21:06:21 +00001914 // Found but did not consume the newline. Notify comment handlers about the
1915 // comment unless we're in a #if 0 block.
1916 if (PP && !isLexingRawMode() &&
1917 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
1918 getSourceLocation(CurPtr)))) {
Chris Lattner046c2272010-01-18 22:35:47 +00001919 BufferPtr = CurPtr;
1920 return true; // A token has to be returned.
1921 }
Mike Stump1eb44332009-09-09 15:08:12 +00001922
Reid Spencer5f016e22007-07-11 17:01:13 +00001923 // If we are returning comments as tokens, return this comment as a token.
Chris Lattnerfa95a012008-10-12 03:22:02 +00001924 if (inKeepCommentMode())
Reid Spencer5f016e22007-07-11 17:01:13 +00001925 return SaveBCPLComment(Result, CurPtr);
1926
1927 // If we are inside a preprocessor directive and we see the end of line,
Peter Collingbourne84021552011-02-28 02:37:51 +00001928 // return immediately, so that the lexer can return this as an EOD token.
Reid Spencer5f016e22007-07-11 17:01:13 +00001929 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
1930 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00001931 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001932 }
Mike Stump1eb44332009-09-09 15:08:12 +00001933
Reid Spencer5f016e22007-07-11 17:01:13 +00001934 // Otherwise, eat the \n character. We don't care if this is a \n\r or
Chris Lattner7a4f0042008-10-12 00:23:07 +00001935 // \r\n sequence. This is an efficiency hack (because we know the \n can't
Chris Lattnerd88dc482008-10-12 04:05:48 +00001936 // contribute to another token), it isn't needed for correctness. Note that
1937 // this is ok even in KeepWhitespaceMode, because we would have returned the
1938 /// comment above in that mode.
Reid Spencer5f016e22007-07-11 17:01:13 +00001939 ++CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001940
Reid Spencer5f016e22007-07-11 17:01:13 +00001941 // The next returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +00001942 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00001943 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +00001944 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00001945 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00001946 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00001947}
1948
1949/// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in
1950/// an appropriate way and return it.
Chris Lattnerd2177732007-07-20 16:59:19 +00001951bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00001952 // If we're not in a preprocessor directive, just return the // comment
1953 // directly.
1954 FormTokenWithChars(Result, CurPtr, tok::comment);
Mike Stump1eb44332009-09-09 15:08:12 +00001955
Chris Lattner9e6293d2008-10-12 04:51:35 +00001956 if (!ParsingPreprocessorDirective)
1957 return true;
Mike Stump1eb44332009-09-09 15:08:12 +00001958
Chris Lattner9e6293d2008-10-12 04:51:35 +00001959 // If this BCPL-style comment is in a macro definition, transmogrify it into
1960 // a C-style block comment.
Douglas Gregor453091c2010-03-16 22:30:13 +00001961 bool Invalid = false;
1962 std::string Spelling = PP->getSpelling(Result, &Invalid);
1963 if (Invalid)
1964 return true;
1965
Chris Lattner9e6293d2008-10-12 04:51:35 +00001966 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not bcpl comment?");
1967 Spelling[1] = '*'; // Change prefix to "/*".
1968 Spelling += "*/"; // add suffix.
Mike Stump1eb44332009-09-09 15:08:12 +00001969
Chris Lattner9e6293d2008-10-12 04:51:35 +00001970 Result.setKind(tok::comment);
Chris Lattner47246be2009-01-26 19:29:26 +00001971 PP->CreateString(&Spelling[0], Spelling.size(), Result,
Abramo Bagnaraa08529c2011-10-03 18:39:03 +00001972 Result.getLocation(), Result.getLocation());
Chris Lattner2d381892008-10-12 04:15:42 +00001973 return true;
Reid Spencer5f016e22007-07-11 17:01:13 +00001974}
1975
1976/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
1977/// character (either \n or \r) is part of an escaped newline sequence. Issue a
Chris Lattner47a2b402008-12-12 07:14:34 +00001978/// diagnostic if so. We know that the newline is inside of a block comment.
Mike Stump1eb44332009-09-09 15:08:12 +00001979static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
Reid Spencer5f016e22007-07-11 17:01:13 +00001980 Lexer *L) {
1981 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
Mike Stump1eb44332009-09-09 15:08:12 +00001982
Reid Spencer5f016e22007-07-11 17:01:13 +00001983 // Back up off the newline.
1984 --CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00001985
Reid Spencer5f016e22007-07-11 17:01:13 +00001986 // If this is a two-character newline sequence, skip the other character.
1987 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
1988 // \n\n or \r\r -> not escaped newline.
1989 if (CurPtr[0] == CurPtr[1])
1990 return false;
1991 // \n\r or \r\n -> skip the newline.
1992 --CurPtr;
1993 }
Mike Stump1eb44332009-09-09 15:08:12 +00001994
Reid Spencer5f016e22007-07-11 17:01:13 +00001995 // If we have horizontal whitespace, skip over it. We allow whitespace
1996 // between the slash and newline.
1997 bool HasSpace = false;
1998 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) {
1999 --CurPtr;
2000 HasSpace = true;
2001 }
Mike Stump1eb44332009-09-09 15:08:12 +00002002
Reid Spencer5f016e22007-07-11 17:01:13 +00002003 // If we have a slash, we know this is an escaped newline.
2004 if (*CurPtr == '\\') {
2005 if (CurPtr[-1] != '*') return false;
2006 } else {
2007 // It isn't a slash, is it the ?? / trigraph?
2008 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' ||
2009 CurPtr[-3] != '*')
2010 return false;
Mike Stump1eb44332009-09-09 15:08:12 +00002011
Reid Spencer5f016e22007-07-11 17:01:13 +00002012 // This is the trigraph ending the comment. Emit a stern warning!
2013 CurPtr -= 2;
2014
2015 // If no trigraphs are enabled, warn that we ignored this trigraph and
2016 // ignore this * character.
2017 if (!L->getFeatures().Trigraphs) {
Chris Lattner74d15df2008-11-22 02:02:22 +00002018 if (!L->isLexingRawMode())
2019 L->Diag(CurPtr, diag::trigraph_ignored_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00002020 return false;
2021 }
Chris Lattner74d15df2008-11-22 02:02:22 +00002022 if (!L->isLexingRawMode())
2023 L->Diag(CurPtr, diag::trigraph_ends_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00002024 }
Mike Stump1eb44332009-09-09 15:08:12 +00002025
Reid Spencer5f016e22007-07-11 17:01:13 +00002026 // Warn about having an escaped newline between the */ characters.
Chris Lattner74d15df2008-11-22 02:02:22 +00002027 if (!L->isLexingRawMode())
2028 L->Diag(CurPtr, diag::escaped_newline_block_comment_end);
Mike Stump1eb44332009-09-09 15:08:12 +00002029
Reid Spencer5f016e22007-07-11 17:01:13 +00002030 // If there was space between the backslash and newline, warn about it.
Chris Lattner74d15df2008-11-22 02:02:22 +00002031 if (HasSpace && !L->isLexingRawMode())
2032 L->Diag(CurPtr, diag::backslash_newline_space);
Mike Stump1eb44332009-09-09 15:08:12 +00002033
Reid Spencer5f016e22007-07-11 17:01:13 +00002034 return true;
2035}
2036
2037#ifdef __SSE2__
2038#include <emmintrin.h>
2039#elif __ALTIVEC__
2040#include <altivec.h>
2041#undef bool
2042#endif
2043
2044/// SkipBlockComment - We have just read the /* characters from input. Read
2045/// until we find the */ characters that terminate the comment. Note that we
2046/// don't bother decoding trigraphs or escaped newlines in block comments,
2047/// because they cannot cause the comment to end. The only thing that can
2048/// happen is the comment could end with an escaped newline between the */ end
2049/// of comment.
Chris Lattner2d381892008-10-12 04:15:42 +00002050///
Chris Lattner046c2272010-01-18 22:35:47 +00002051/// If we're in KeepCommentMode or any CommentHandler has inserted
2052/// some tokens, this will store the first token and return true.
Chris Lattnerd2177732007-07-20 16:59:19 +00002053bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00002054 // Scan one character past where we should, looking for a '/' character. Once
Chris Lattnerfc8f0e12011-04-15 05:22:18 +00002055 // we find it, check to see if it was preceded by a *. This common
Reid Spencer5f016e22007-07-11 17:01:13 +00002056 // optimization helps people who like to put a lot of * characters in their
2057 // comments.
Chris Lattner8146b682007-07-21 23:43:37 +00002058
2059 // The first character we get with newlines and trigraphs skipped to handle
2060 // the degenerate /*/ case below correctly if the * has an escaped newline
2061 // after it.
2062 unsigned CharSize;
2063 unsigned char C = getCharAndSize(CurPtr, CharSize);
2064 CurPtr += CharSize;
Reid Spencer5f016e22007-07-11 17:01:13 +00002065 if (C == 0 && CurPtr == BufferEnd+1) {
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002066 if (!isLexingRawMode())
Chris Lattner0af57422008-10-12 01:31:51 +00002067 Diag(BufferPtr, diag::err_unterminated_block_comment);
Chris Lattner31f0eca2008-10-12 04:19:49 +00002068 --CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00002069
Chris Lattner31f0eca2008-10-12 04:19:49 +00002070 // KeepWhitespaceMode should return this broken comment as a token. Since
2071 // it isn't a well formed comment, just return it as an 'unknown' token.
2072 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002073 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattner31f0eca2008-10-12 04:19:49 +00002074 return true;
2075 }
Mike Stump1eb44332009-09-09 15:08:12 +00002076
Chris Lattner31f0eca2008-10-12 04:19:49 +00002077 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00002078 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00002079 }
Mike Stump1eb44332009-09-09 15:08:12 +00002080
Chris Lattner8146b682007-07-21 23:43:37 +00002081 // Check to see if the first character after the '/*' is another /. If so,
2082 // then this slash does not end the block comment, it is part of it.
2083 if (C == '/')
2084 C = *CurPtr++;
Mike Stump1eb44332009-09-09 15:08:12 +00002085
Reid Spencer5f016e22007-07-11 17:01:13 +00002086 while (1) {
2087 // Skip over all non-interesting characters until we find end of buffer or a
2088 // (probably ending) '/' character.
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002089 if (CurPtr + 24 < BufferEnd &&
2090 // If there is a code-completion point avoid the fast scan because it
2091 // doesn't check for '\0'.
2092 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
Reid Spencer5f016e22007-07-11 17:01:13 +00002093 // While not aligned to a 16-byte boundary.
2094 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0)
2095 C = *CurPtr++;
Mike Stump1eb44332009-09-09 15:08:12 +00002096
Reid Spencer5f016e22007-07-11 17:01:13 +00002097 if (C == '/') goto FoundSlash;
2098
2099#ifdef __SSE2__
Benjamin Kramer3f6f4e62011-11-22 18:56:46 +00002100 __m128i Slashes = _mm_set1_epi8('/');
2101 while (CurPtr+16 <= BufferEnd) {
2102 int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(__m128i*)CurPtr, Slashes));
2103 if (cmp != 0) {
Benjamin Kramer6300f5b2011-11-22 20:39:31 +00002104 // Adjust the pointer to point directly after the first slash. It's
2105 // not necessary to set C here, it will be overwritten at the end of
2106 // the outer loop.
2107 CurPtr += llvm::CountTrailingZeros_32(cmp) + 1;
Benjamin Kramer3f6f4e62011-11-22 18:56:46 +00002108 goto FoundSlash;
2109 }
Reid Spencer5f016e22007-07-11 17:01:13 +00002110 CurPtr += 16;
Benjamin Kramer3f6f4e62011-11-22 18:56:46 +00002111 }
Reid Spencer5f016e22007-07-11 17:01:13 +00002112#elif __ALTIVEC__
2113 __vector unsigned char Slashes = {
Mike Stump1eb44332009-09-09 15:08:12 +00002114 '/', '/', '/', '/', '/', '/', '/', '/',
Reid Spencer5f016e22007-07-11 17:01:13 +00002115 '/', '/', '/', '/', '/', '/', '/', '/'
2116 };
2117 while (CurPtr+16 <= BufferEnd &&
2118 !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes))
2119 CurPtr += 16;
Mike Stump1eb44332009-09-09 15:08:12 +00002120#else
Reid Spencer5f016e22007-07-11 17:01:13 +00002121 // Scan for '/' quickly. Many block comments are very large.
2122 while (CurPtr[0] != '/' &&
2123 CurPtr[1] != '/' &&
2124 CurPtr[2] != '/' &&
2125 CurPtr[3] != '/' &&
2126 CurPtr+4 < BufferEnd) {
2127 CurPtr += 4;
2128 }
2129#endif
Mike Stump1eb44332009-09-09 15:08:12 +00002130
Reid Spencer5f016e22007-07-11 17:01:13 +00002131 // It has to be one of the bytes scanned, increment to it and read one.
2132 C = *CurPtr++;
2133 }
Mike Stump1eb44332009-09-09 15:08:12 +00002134
Reid Spencer5f016e22007-07-11 17:01:13 +00002135 // Loop to scan the remainder.
2136 while (C != '/' && C != '\0')
2137 C = *CurPtr++;
Mike Stump1eb44332009-09-09 15:08:12 +00002138
Reid Spencer5f016e22007-07-11 17:01:13 +00002139 if (C == '/') {
Benjamin Kramer3f6f4e62011-11-22 18:56:46 +00002140 FoundSlash:
Reid Spencer5f016e22007-07-11 17:01:13 +00002141 if (CurPtr[-2] == '*') // We found the final */. We're done!
2142 break;
Mike Stump1eb44332009-09-09 15:08:12 +00002143
Reid Spencer5f016e22007-07-11 17:01:13 +00002144 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
2145 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) {
2146 // We found the final */, though it had an escaped newline between the
2147 // * and /. We're done!
2148 break;
2149 }
2150 }
2151 if (CurPtr[0] == '*' && CurPtr[1] != '/') {
2152 // If this is a /* inside of the comment, emit a warning. Don't do this
2153 // if this is a /*/, which will end the comment. This misses cases with
2154 // embedded escaped newlines, but oh well.
Chris Lattner74d15df2008-11-22 02:02:22 +00002155 if (!isLexingRawMode())
2156 Diag(CurPtr-1, diag::warn_nested_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00002157 }
2158 } else if (C == 0 && CurPtr == BufferEnd+1) {
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002159 if (!isLexingRawMode())
Chris Lattner74d15df2008-11-22 02:02:22 +00002160 Diag(BufferPtr, diag::err_unterminated_block_comment);
Reid Spencer5f016e22007-07-11 17:01:13 +00002161 // Note: the user probably forgot a */. We could continue immediately
2162 // after the /*, but this would involve lexing a lot of what really is the
2163 // comment, which surely would confuse the parser.
Chris Lattner31f0eca2008-10-12 04:19:49 +00002164 --CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00002165
Chris Lattner31f0eca2008-10-12 04:19:49 +00002166 // KeepWhitespaceMode should return this broken comment as a token. Since
2167 // it isn't a well formed comment, just return it as an 'unknown' token.
2168 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002169 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattner31f0eca2008-10-12 04:19:49 +00002170 return true;
2171 }
Mike Stump1eb44332009-09-09 15:08:12 +00002172
Chris Lattner31f0eca2008-10-12 04:19:49 +00002173 BufferPtr = CurPtr;
Chris Lattner2d381892008-10-12 04:15:42 +00002174 return false;
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002175 } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
2176 PP->CodeCompleteNaturalLanguage();
2177 cutOffLexing();
2178 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00002179 }
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002180
Reid Spencer5f016e22007-07-11 17:01:13 +00002181 C = *CurPtr++;
2182 }
Mike Stump1eb44332009-09-09 15:08:12 +00002183
Chris Lattner3d0ad582010-02-03 21:06:21 +00002184 // Notify comment handlers about the comment unless we're in a #if 0 block.
2185 if (PP && !isLexingRawMode() &&
2186 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
2187 getSourceLocation(CurPtr)))) {
Chris Lattner046c2272010-01-18 22:35:47 +00002188 BufferPtr = CurPtr;
2189 return true; // A token has to be returned.
2190 }
Douglas Gregor2e222532009-07-02 17:08:52 +00002191
Reid Spencer5f016e22007-07-11 17:01:13 +00002192 // If we are returning comments as tokens, return this comment as a token.
Chris Lattnerfa95a012008-10-12 03:22:02 +00002193 if (inKeepCommentMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002194 FormTokenWithChars(Result, CurPtr, tok::comment);
Chris Lattner2d381892008-10-12 04:15:42 +00002195 return true;
Reid Spencer5f016e22007-07-11 17:01:13 +00002196 }
2197
2198 // It is common for the tokens immediately after a /**/ comment to be
2199 // whitespace. Instead of going through the big switch, handle it
Chris Lattnerd88dc482008-10-12 04:05:48 +00002200 // efficiently now. This is safe even in KeepWhitespaceMode because we would
2201 // have already returned above with the comment as a token.
Reid Spencer5f016e22007-07-11 17:01:13 +00002202 if (isHorizontalWhitespace(*CurPtr)) {
Chris Lattnerd2177732007-07-20 16:59:19 +00002203 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00002204 SkipWhitespace(Result, CurPtr+1);
Chris Lattner2d381892008-10-12 04:15:42 +00002205 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00002206 }
2207
2208 // Otherwise, just return so that the next character will be lexed as a token.
2209 BufferPtr = CurPtr;
Chris Lattnerd2177732007-07-20 16:59:19 +00002210 Result.setFlag(Token::LeadingSpace);
Chris Lattner2d381892008-10-12 04:15:42 +00002211 return false;
Reid Spencer5f016e22007-07-11 17:01:13 +00002212}
2213
2214//===----------------------------------------------------------------------===//
2215// Primary Lexing Entry Points
2216//===----------------------------------------------------------------------===//
2217
Reid Spencer5f016e22007-07-11 17:01:13 +00002218/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
2219/// uninterpreted string. This switches the lexer out of directive mode.
2220std::string Lexer::ReadToEndOfLine() {
2221 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
2222 "Must be in a preprocessing directive!");
2223 std::string Result;
Chris Lattnerd2177732007-07-20 16:59:19 +00002224 Token Tmp;
Reid Spencer5f016e22007-07-11 17:01:13 +00002225
2226 // CurPtr - Cache BufferPtr in an automatic variable.
2227 const char *CurPtr = BufferPtr;
2228 while (1) {
2229 char Char = getAndAdvanceChar(CurPtr, Tmp);
2230 switch (Char) {
2231 default:
2232 Result += Char;
2233 break;
2234 case 0: // Null.
2235 // Found end of file?
2236 if (CurPtr-1 != BufferEnd) {
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002237 if (isCodeCompletionPoint(CurPtr-1)) {
2238 PP->CodeCompleteNaturalLanguage();
2239 cutOffLexing();
2240 return Result;
2241 }
2242
Reid Spencer5f016e22007-07-11 17:01:13 +00002243 // Nope, normal character, continue.
2244 Result += Char;
2245 break;
2246 }
2247 // FALL THROUGH.
2248 case '\r':
2249 case '\n':
2250 // Okay, we found the end of the line. First, back up past the \0, \r, \n.
2251 assert(CurPtr[-1] == Char && "Trigraphs for newline?");
2252 BufferPtr = CurPtr-1;
Mike Stump1eb44332009-09-09 15:08:12 +00002253
Peter Collingbourne84021552011-02-28 02:37:51 +00002254 // Next, lex the character, which should handle the EOD transition.
Reid Spencer5f016e22007-07-11 17:01:13 +00002255 Lex(Tmp);
Douglas Gregor55817af2010-08-25 17:04:25 +00002256 if (Tmp.is(tok::code_completion)) {
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002257 if (PP)
2258 PP->CodeCompleteNaturalLanguage();
Douglas Gregor55817af2010-08-25 17:04:25 +00002259 Lex(Tmp);
2260 }
Peter Collingbourne84021552011-02-28 02:37:51 +00002261 assert(Tmp.is(tok::eod) && "Unexpected token!");
Mike Stump1eb44332009-09-09 15:08:12 +00002262
Reid Spencer5f016e22007-07-11 17:01:13 +00002263 // Finally, we're done, return the string we found.
2264 return Result;
2265 }
2266 }
2267}
2268
2269/// LexEndOfFile - CurPtr points to the end of this file. Handle this
2270/// condition, reporting diagnostics and handling other edge cases as required.
2271/// This returns true if Result contains a token, false if PP.Lex should be
2272/// called again.
Chris Lattnerd2177732007-07-20 16:59:19 +00002273bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
Reid Spencer5f016e22007-07-11 17:01:13 +00002274 // If we hit the end of the file while parsing a preprocessor directive,
2275 // end the preprocessor directive first. The next token returned will
2276 // then be the end of file.
2277 if (ParsingPreprocessorDirective) {
2278 // Done parsing the "line".
2279 ParsingPreprocessorDirective = false;
Reid Spencer5f016e22007-07-11 17:01:13 +00002280 // Update the location of token as well as BufferPtr.
Peter Collingbourne84021552011-02-28 02:37:51 +00002281 FormTokenWithChars(Result, CurPtr, tok::eod);
Mike Stump1eb44332009-09-09 15:08:12 +00002282
Reid Spencer5f016e22007-07-11 17:01:13 +00002283 // Restore comment saving mode, in case it was disabled for directive.
Chris Lattnerf744d132008-10-12 03:27:19 +00002284 SetCommentRetentionState(PP->getCommentRetentionState());
Reid Spencer5f016e22007-07-11 17:01:13 +00002285 return true; // Have a token.
Mike Stump1eb44332009-09-09 15:08:12 +00002286 }
Douglas Gregor86d9a522009-09-21 16:56:56 +00002287
Reid Spencer5f016e22007-07-11 17:01:13 +00002288 // If we are in raw mode, return this event as an EOF token. Let the caller
2289 // that put us in raw mode handle the event.
Chris Lattner74d15df2008-11-22 02:02:22 +00002290 if (isLexingRawMode()) {
Reid Spencer5f016e22007-07-11 17:01:13 +00002291 Result.startToken();
2292 BufferPtr = BufferEnd;
Chris Lattner9e6293d2008-10-12 04:51:35 +00002293 FormTokenWithChars(Result, BufferEnd, tok::eof);
Reid Spencer5f016e22007-07-11 17:01:13 +00002294 return true;
2295 }
Douglas Gregor86d9a522009-09-21 16:56:56 +00002296
Douglas Gregorf44e8542010-08-24 19:08:16 +00002297 // Issue diagnostics for unterminated #if and missing newline.
2298
Reid Spencer5f016e22007-07-11 17:01:13 +00002299 // If we are in a #if directive, emit an error.
2300 while (!ConditionalStack.empty()) {
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002301 if (PP->getCodeCompletionFileLoc() != FileLoc)
Douglas Gregor2d474ba2010-08-12 17:04:55 +00002302 PP->Diag(ConditionalStack.back().IfLoc,
2303 diag::err_pp_unterminated_conditional);
Reid Spencer5f016e22007-07-11 17:01:13 +00002304 ConditionalStack.pop_back();
2305 }
Mike Stump1eb44332009-09-09 15:08:12 +00002306
Chris Lattnerb25e5d72008-04-12 05:54:25 +00002307 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue
2308 // a pedwarn.
2309 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r'))
Mike Stump20d0ee52009-04-02 02:29:42 +00002310 Diag(BufferEnd, diag::ext_no_newline_eof)
Douglas Gregor849b2432010-03-31 17:46:05 +00002311 << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n");
Mike Stump1eb44332009-09-09 15:08:12 +00002312
Reid Spencer5f016e22007-07-11 17:01:13 +00002313 BufferPtr = CurPtr;
2314
2315 // Finally, let the preprocessor handle this.
Chris Lattner168ae2d2007-10-17 20:41:00 +00002316 return PP->HandleEndOfFile(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00002317}
2318
2319/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
2320/// the specified lexer will return a tok::l_paren token, 0 if it is something
2321/// else and 2 if there are no more tokens in the buffer controlled by the
2322/// lexer.
2323unsigned Lexer::isNextPPTokenLParen() {
2324 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
Mike Stump1eb44332009-09-09 15:08:12 +00002325
Reid Spencer5f016e22007-07-11 17:01:13 +00002326 // Switch to 'skipping' mode. This will ensure that we can lex a token
2327 // without emitting diagnostics, disables macro expansion, and will cause EOF
2328 // to return an EOF token instead of popping the include stack.
2329 LexingRawMode = true;
Mike Stump1eb44332009-09-09 15:08:12 +00002330
Reid Spencer5f016e22007-07-11 17:01:13 +00002331 // Save state that can be changed while lexing so that we can restore it.
2332 const char *TmpBufferPtr = BufferPtr;
Chris Lattnera864cf72009-04-24 07:15:46 +00002333 bool inPPDirectiveMode = ParsingPreprocessorDirective;
Mike Stump1eb44332009-09-09 15:08:12 +00002334
Chris Lattnerd2177732007-07-20 16:59:19 +00002335 Token Tok;
Reid Spencer5f016e22007-07-11 17:01:13 +00002336 Tok.startToken();
2337 LexTokenInternal(Tok);
Mike Stump1eb44332009-09-09 15:08:12 +00002338
Reid Spencer5f016e22007-07-11 17:01:13 +00002339 // Restore state that may have changed.
2340 BufferPtr = TmpBufferPtr;
Chris Lattnera864cf72009-04-24 07:15:46 +00002341 ParsingPreprocessorDirective = inPPDirectiveMode;
Mike Stump1eb44332009-09-09 15:08:12 +00002342
Reid Spencer5f016e22007-07-11 17:01:13 +00002343 // Restore the lexer back to non-skipping mode.
2344 LexingRawMode = false;
Mike Stump1eb44332009-09-09 15:08:12 +00002345
Chris Lattner22f6bbc2007-10-09 18:02:16 +00002346 if (Tok.is(tok::eof))
Reid Spencer5f016e22007-07-11 17:01:13 +00002347 return 2;
Chris Lattner22f6bbc2007-10-09 18:02:16 +00002348 return Tok.is(tok::l_paren);
Reid Spencer5f016e22007-07-11 17:01:13 +00002349}
2350
Chris Lattner34f349d2009-12-14 06:16:57 +00002351/// FindConflictEnd - Find the end of a version control conflict marker.
Richard Smithd5e1d602011-10-12 00:37:51 +00002352static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
2353 ConflictMarkerKind CMK) {
2354 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>";
2355 size_t TermLen = CMK == CMK_Perforce ? 5 : 7;
2356 StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen);
2357 size_t Pos = RestOfBuffer.find(Terminator);
Chris Lattner5f9e2722011-07-23 10:55:15 +00002358 while (Pos != StringRef::npos) {
Chris Lattner34f349d2009-12-14 06:16:57 +00002359 // Must occur at start of line.
2360 if (RestOfBuffer[Pos-1] != '\r' &&
2361 RestOfBuffer[Pos-1] != '\n') {
Richard Smithd5e1d602011-10-12 00:37:51 +00002362 RestOfBuffer = RestOfBuffer.substr(Pos+TermLen);
2363 Pos = RestOfBuffer.find(Terminator);
Chris Lattner34f349d2009-12-14 06:16:57 +00002364 continue;
2365 }
2366 return RestOfBuffer.data()+Pos;
2367 }
2368 return 0;
2369}
2370
2371/// IsStartOfConflictMarker - If the specified pointer is the start of a version
2372/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
2373/// and recover nicely. This returns true if it is a conflict marker and false
2374/// if not.
2375bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
2376 // Only a conflict marker if it starts at the beginning of a line.
2377 if (CurPtr != BufferStart &&
2378 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
2379 return false;
2380
Richard Smithd5e1d602011-10-12 00:37:51 +00002381 // Check to see if we have <<<<<<< or >>>>.
2382 if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") &&
2383 (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> "))
Chris Lattner34f349d2009-12-14 06:16:57 +00002384 return false;
2385
2386 // If we have a situation where we don't care about conflict markers, ignore
2387 // it.
Richard Smithd5e1d602011-10-12 00:37:51 +00002388 if (CurrentConflictMarkerState || isLexingRawMode())
Chris Lattner34f349d2009-12-14 06:16:57 +00002389 return false;
2390
Richard Smithd5e1d602011-10-12 00:37:51 +00002391 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce;
2392
2393 // Check to see if there is an ending marker somewhere in the buffer at the
2394 // start of a line to terminate this conflict marker.
2395 if (FindConflictEnd(CurPtr, BufferEnd, Kind)) {
Chris Lattner34f349d2009-12-14 06:16:57 +00002396 // We found a match. We are really in a conflict marker.
2397 // Diagnose this, and ignore to the end of line.
2398 Diag(CurPtr, diag::err_conflict_marker);
Richard Smithd5e1d602011-10-12 00:37:51 +00002399 CurrentConflictMarkerState = Kind;
Chris Lattner34f349d2009-12-14 06:16:57 +00002400
2401 // Skip ahead to the end of line. We know this exists because the
2402 // end-of-conflict marker starts with \r or \n.
2403 while (*CurPtr != '\r' && *CurPtr != '\n') {
2404 assert(CurPtr != BufferEnd && "Didn't find end of line");
2405 ++CurPtr;
2406 }
2407 BufferPtr = CurPtr;
2408 return true;
2409 }
2410
2411 // No end of conflict marker found.
2412 return false;
2413}
2414
2415
Richard Smithd5e1d602011-10-12 00:37:51 +00002416/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
2417/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
2418/// is the end of a conflict marker. Handle it by ignoring up until the end of
2419/// the line. This returns true if it is a conflict marker and false if not.
Chris Lattner34f349d2009-12-14 06:16:57 +00002420bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
2421 // Only a conflict marker if it starts at the beginning of a line.
2422 if (CurPtr != BufferStart &&
2423 CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
2424 return false;
2425
2426 // If we have a situation where we don't care about conflict markers, ignore
2427 // it.
Richard Smithd5e1d602011-10-12 00:37:51 +00002428 if (!CurrentConflictMarkerState || isLexingRawMode())
Chris Lattner34f349d2009-12-14 06:16:57 +00002429 return false;
2430
Richard Smithd5e1d602011-10-12 00:37:51 +00002431 // Check to see if we have the marker (4 characters in a row).
2432 for (unsigned i = 1; i != 4; ++i)
Chris Lattner34f349d2009-12-14 06:16:57 +00002433 if (CurPtr[i] != CurPtr[0])
2434 return false;
2435
2436 // If we do have it, search for the end of the conflict marker. This could
2437 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might
2438 // be the end of conflict marker.
Richard Smithd5e1d602011-10-12 00:37:51 +00002439 if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
2440 CurrentConflictMarkerState)) {
Chris Lattner34f349d2009-12-14 06:16:57 +00002441 CurPtr = End;
2442
2443 // Skip ahead to the end of line.
2444 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
2445 ++CurPtr;
2446
2447 BufferPtr = CurPtr;
2448
2449 // No longer in the conflict marker.
Richard Smithd5e1d602011-10-12 00:37:51 +00002450 CurrentConflictMarkerState = CMK_None;
Chris Lattner34f349d2009-12-14 06:16:57 +00002451 return true;
2452 }
2453
2454 return false;
2455}
2456
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002457bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
2458 if (PP && PP->isCodeCompletionEnabled()) {
Argyrios Kyrtzidisa64ccef2011-09-19 20:40:19 +00002459 SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart);
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002460 return Loc == PP->getCodeCompletionLoc();
2461 }
2462
2463 return false;
2464}
2465
Reid Spencer5f016e22007-07-11 17:01:13 +00002466
2467/// LexTokenInternal - This implements a simple C family lexer. It is an
2468/// extremely performance critical piece of code. This assumes that the buffer
Chris Lattnerefb173d2009-07-07 05:05:42 +00002469/// has a null character at the end of the file. This returns a preprocessing
2470/// token, not a normal token, as such, it is an internal interface. It assumes
2471/// that the Flags of result have been cleared before calling this.
Chris Lattnerd2177732007-07-20 16:59:19 +00002472void Lexer::LexTokenInternal(Token &Result) {
Reid Spencer5f016e22007-07-11 17:01:13 +00002473LexNextToken:
2474 // New token, can't need cleaning yet.
Chris Lattnerd2177732007-07-20 16:59:19 +00002475 Result.clearFlag(Token::NeedsCleaning);
Reid Spencer5f016e22007-07-11 17:01:13 +00002476 Result.setIdentifierInfo(0);
Mike Stump1eb44332009-09-09 15:08:12 +00002477
Reid Spencer5f016e22007-07-11 17:01:13 +00002478 // CurPtr - Cache BufferPtr in an automatic variable.
2479 const char *CurPtr = BufferPtr;
2480
2481 // Small amounts of horizontal whitespace is very common between tokens.
2482 if ((*CurPtr == ' ') || (*CurPtr == '\t')) {
2483 ++CurPtr;
2484 while ((*CurPtr == ' ') || (*CurPtr == '\t'))
2485 ++CurPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00002486
Chris Lattnerd88dc482008-10-12 04:05:48 +00002487 // If we are keeping whitespace and other tokens, just return what we just
2488 // skipped. The next lexer invocation will return the token after the
2489 // whitespace.
2490 if (isKeepWhitespaceMode()) {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002491 FormTokenWithChars(Result, CurPtr, tok::unknown);
Chris Lattnerd88dc482008-10-12 04:05:48 +00002492 return;
2493 }
Mike Stump1eb44332009-09-09 15:08:12 +00002494
Reid Spencer5f016e22007-07-11 17:01:13 +00002495 BufferPtr = CurPtr;
Chris Lattnerd2177732007-07-20 16:59:19 +00002496 Result.setFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00002497 }
Mike Stump1eb44332009-09-09 15:08:12 +00002498
Reid Spencer5f016e22007-07-11 17:01:13 +00002499 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below.
Mike Stump1eb44332009-09-09 15:08:12 +00002500
Reid Spencer5f016e22007-07-11 17:01:13 +00002501 // Read a character, advancing over it.
2502 char Char = getAndAdvanceChar(CurPtr, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002503 tok::TokenKind Kind;
Mike Stump1eb44332009-09-09 15:08:12 +00002504
Reid Spencer5f016e22007-07-11 17:01:13 +00002505 switch (Char) {
2506 case 0: // Null.
2507 // Found end of file?
2508 if (CurPtr-1 == BufferEnd) {
2509 // Read the PP instance variable into an automatic variable, because
2510 // LexEndOfFile will often delete 'this'.
Chris Lattner168ae2d2007-10-17 20:41:00 +00002511 Preprocessor *PPCache = PP;
Reid Spencer5f016e22007-07-11 17:01:13 +00002512 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file.
2513 return; // Got a token to return.
Chris Lattner168ae2d2007-10-17 20:41:00 +00002514 assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
2515 return PPCache->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00002516 }
Mike Stump1eb44332009-09-09 15:08:12 +00002517
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +00002518 // Check if we are performing code completion.
2519 if (isCodeCompletionPoint(CurPtr-1)) {
2520 // Return the code-completion token.
2521 Result.startToken();
2522 FormTokenWithChars(Result, CurPtr, tok::code_completion);
2523 return;
2524 }
2525
Chris Lattner74d15df2008-11-22 02:02:22 +00002526 if (!isLexingRawMode())
2527 Diag(CurPtr-1, diag::null_in_file);
Chris Lattnerd2177732007-07-20 16:59:19 +00002528 Result.setFlag(Token::LeadingSpace);
Chris Lattnerd88dc482008-10-12 04:05:48 +00002529 if (SkipWhitespace(Result, CurPtr))
2530 return; // KeepWhitespaceMode
Mike Stump1eb44332009-09-09 15:08:12 +00002531
Reid Spencer5f016e22007-07-11 17:01:13 +00002532 goto LexNextToken; // GCC isn't tail call eliminating.
Chris Lattnera2bf1052009-12-17 05:29:40 +00002533
2534 case 26: // DOS & CP/M EOF: "^Z".
2535 // If we're in Microsoft extensions mode, treat this as end of file.
Francois Pichet62ec1f22011-09-17 17:15:52 +00002536 if (Features.MicrosoftExt) {
Chris Lattnera2bf1052009-12-17 05:29:40 +00002537 // Read the PP instance variable into an automatic variable, because
2538 // LexEndOfFile will often delete 'this'.
2539 Preprocessor *PPCache = PP;
2540 if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file.
2541 return; // Got a token to return.
2542 assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
2543 return PPCache->Lex(Result);
2544 }
2545 // If Microsoft extensions are disabled, this is just random garbage.
2546 Kind = tok::unknown;
2547 break;
2548
Reid Spencer5f016e22007-07-11 17:01:13 +00002549 case '\n':
2550 case '\r':
2551 // If we are inside a preprocessor directive and we see the end of line,
Peter Collingbourne84021552011-02-28 02:37:51 +00002552 // we know we are done with the directive, so return an EOD token.
Reid Spencer5f016e22007-07-11 17:01:13 +00002553 if (ParsingPreprocessorDirective) {
2554 // Done parsing the "line".
2555 ParsingPreprocessorDirective = false;
Mike Stump1eb44332009-09-09 15:08:12 +00002556
Reid Spencer5f016e22007-07-11 17:01:13 +00002557 // Restore comment saving mode, in case it was disabled for directive.
Chris Lattnerf744d132008-10-12 03:27:19 +00002558 SetCommentRetentionState(PP->getCommentRetentionState());
Mike Stump1eb44332009-09-09 15:08:12 +00002559
Reid Spencer5f016e22007-07-11 17:01:13 +00002560 // Since we consumed a newline, we are back at the start of a line.
2561 IsAtStartOfLine = true;
Mike Stump1eb44332009-09-09 15:08:12 +00002562
Peter Collingbourne84021552011-02-28 02:37:51 +00002563 Kind = tok::eod;
Reid Spencer5f016e22007-07-11 17:01:13 +00002564 break;
2565 }
2566 // The returned token is at the start of the line.
Chris Lattnerd2177732007-07-20 16:59:19 +00002567 Result.setFlag(Token::StartOfLine);
Reid Spencer5f016e22007-07-11 17:01:13 +00002568 // No leading whitespace seen so far.
Chris Lattnerd2177732007-07-20 16:59:19 +00002569 Result.clearFlag(Token::LeadingSpace);
Mike Stump1eb44332009-09-09 15:08:12 +00002570
Chris Lattnerd88dc482008-10-12 04:05:48 +00002571 if (SkipWhitespace(Result, CurPtr))
2572 return; // KeepWhitespaceMode
Reid Spencer5f016e22007-07-11 17:01:13 +00002573 goto LexNextToken; // GCC isn't tail call eliminating.
2574 case ' ':
2575 case '\t':
2576 case '\f':
2577 case '\v':
Chris Lattner8133cfc2007-07-22 06:29:05 +00002578 SkipHorizontalWhitespace:
Chris Lattnerd2177732007-07-20 16:59:19 +00002579 Result.setFlag(Token::LeadingSpace);
Chris Lattnerd88dc482008-10-12 04:05:48 +00002580 if (SkipWhitespace(Result, CurPtr))
2581 return; // KeepWhitespaceMode
Chris Lattner8133cfc2007-07-22 06:29:05 +00002582
2583 SkipIgnoredUnits:
2584 CurPtr = BufferPtr;
Mike Stump1eb44332009-09-09 15:08:12 +00002585
Chris Lattner8133cfc2007-07-22 06:29:05 +00002586 // If the next token is obviously a // or /* */ comment, skip it efficiently
2587 // too (without going through the big switch stmt).
Chris Lattner8402c732009-01-16 22:39:25 +00002588 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
Daniel Dunbar2ed42282011-03-18 21:23:38 +00002589 Features.BCPLComment && !Features.TraditionalCPP) {
Chris Lattner046c2272010-01-18 22:35:47 +00002590 if (SkipBCPLComment(Result, CurPtr+2))
2591 return; // There is a token to return.
Chris Lattner8133cfc2007-07-22 06:29:05 +00002592 goto SkipIgnoredUnits;
Chris Lattnerfa95a012008-10-12 03:22:02 +00002593 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
Chris Lattner046c2272010-01-18 22:35:47 +00002594 if (SkipBlockComment(Result, CurPtr+2))
2595 return; // There is a token to return.
Chris Lattner8133cfc2007-07-22 06:29:05 +00002596 goto SkipIgnoredUnits;
2597 } else if (isHorizontalWhitespace(*CurPtr)) {
2598 goto SkipHorizontalWhitespace;
2599 }
Reid Spencer5f016e22007-07-11 17:01:13 +00002600 goto LexNextToken; // GCC isn't tail call eliminating.
Chris Lattnera2bf1052009-12-17 05:29:40 +00002601
Chris Lattner3a570772008-01-03 17:58:54 +00002602 // C99 6.4.4.1: Integer Constants.
2603 // C99 6.4.4.2: Floating Constants.
2604 case '0': case '1': case '2': case '3': case '4':
2605 case '5': case '6': case '7': case '8': case '9':
2606 // Notify MIOpt that we read a non-whitespace/non-comment token.
2607 MIOpt.ReadToken();
2608 return LexNumericConstant(Result, CurPtr);
Mike Stump1eb44332009-09-09 15:08:12 +00002609
Douglas Gregor5cee1192011-07-27 05:40:30 +00002610 case 'u': // Identifier (uber) or C++0x UTF-8 or UTF-16 string literal
2611 // Notify MIOpt that we read a non-whitespace/non-comment token.
2612 MIOpt.ReadToken();
2613
2614 if (Features.CPlusPlus0x) {
2615 Char = getCharAndSize(CurPtr, SizeTmp);
2616
2617 // UTF-16 string literal
2618 if (Char == '"')
2619 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
2620 tok::utf16_string_literal);
2621
2622 // UTF-16 character constant
2623 if (Char == '\'')
2624 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
2625 tok::utf16_char_constant);
2626
Craig Topper2fa4e862011-08-11 04:06:15 +00002627 // UTF-16 raw string literal
2628 if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
2629 return LexRawStringLiteral(Result,
2630 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2631 SizeTmp2, Result),
2632 tok::utf16_string_literal);
2633
2634 if (Char == '8') {
2635 char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2);
2636
2637 // UTF-8 string literal
2638 if (Char2 == '"')
2639 return LexStringLiteral(Result,
2640 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2641 SizeTmp2, Result),
2642 tok::utf8_string_literal);
2643
2644 if (Char2 == 'R') {
2645 unsigned SizeTmp3;
2646 char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
2647 // UTF-8 raw string literal
2648 if (Char3 == '"') {
2649 return LexRawStringLiteral(Result,
2650 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2651 SizeTmp2, Result),
2652 SizeTmp3, Result),
2653 tok::utf8_string_literal);
2654 }
2655 }
2656 }
Douglas Gregor5cee1192011-07-27 05:40:30 +00002657 }
2658
2659 // treat u like the start of an identifier.
2660 return LexIdentifier(Result, CurPtr);
2661
2662 case 'U': // Identifier (Uber) or C++0x UTF-32 string literal
2663 // Notify MIOpt that we read a non-whitespace/non-comment token.
2664 MIOpt.ReadToken();
2665
2666 if (Features.CPlusPlus0x) {
2667 Char = getCharAndSize(CurPtr, SizeTmp);
2668
2669 // UTF-32 string literal
2670 if (Char == '"')
2671 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
2672 tok::utf32_string_literal);
2673
2674 // UTF-32 character constant
2675 if (Char == '\'')
2676 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
2677 tok::utf32_char_constant);
Craig Topper2fa4e862011-08-11 04:06:15 +00002678
2679 // UTF-32 raw string literal
2680 if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
2681 return LexRawStringLiteral(Result,
2682 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2683 SizeTmp2, Result),
2684 tok::utf32_string_literal);
Douglas Gregor5cee1192011-07-27 05:40:30 +00002685 }
2686
2687 // treat U like the start of an identifier.
2688 return LexIdentifier(Result, CurPtr);
2689
Craig Topper2fa4e862011-08-11 04:06:15 +00002690 case 'R': // Identifier or C++0x raw string literal
2691 // Notify MIOpt that we read a non-whitespace/non-comment token.
2692 MIOpt.ReadToken();
2693
2694 if (Features.CPlusPlus0x) {
2695 Char = getCharAndSize(CurPtr, SizeTmp);
2696
2697 if (Char == '"')
2698 return LexRawStringLiteral(Result,
2699 ConsumeChar(CurPtr, SizeTmp, Result),
2700 tok::string_literal);
2701 }
2702
2703 // treat R like the start of an identifier.
2704 return LexIdentifier(Result, CurPtr);
2705
Chris Lattner3a570772008-01-03 17:58:54 +00002706 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz").
Reid Spencer5f016e22007-07-11 17:01:13 +00002707 // Notify MIOpt that we read a non-whitespace/non-comment token.
2708 MIOpt.ReadToken();
2709 Char = getCharAndSize(CurPtr, SizeTmp);
2710
2711 // Wide string literal.
2712 if (Char == '"')
2713 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
Douglas Gregor5cee1192011-07-27 05:40:30 +00002714 tok::wide_string_literal);
Reid Spencer5f016e22007-07-11 17:01:13 +00002715
Craig Topper2fa4e862011-08-11 04:06:15 +00002716 // Wide raw string literal.
2717 if (Features.CPlusPlus0x && Char == 'R' &&
2718 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
2719 return LexRawStringLiteral(Result,
2720 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2721 SizeTmp2, Result),
2722 tok::wide_string_literal);
2723
Reid Spencer5f016e22007-07-11 17:01:13 +00002724 // Wide character constant.
2725 if (Char == '\'')
Douglas Gregor5cee1192011-07-27 05:40:30 +00002726 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
2727 tok::wide_char_constant);
Reid Spencer5f016e22007-07-11 17:01:13 +00002728 // FALL THROUGH, treating L like the start of an identifier.
Mike Stump1eb44332009-09-09 15:08:12 +00002729
Reid Spencer5f016e22007-07-11 17:01:13 +00002730 // C99 6.4.2: Identifiers.
2731 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
2732 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N':
Craig Topper2fa4e862011-08-11 04:06:15 +00002733 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/
Reid Spencer5f016e22007-07-11 17:01:13 +00002734 case 'V': case 'W': case 'X': case 'Y': case 'Z':
2735 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
2736 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
Douglas Gregor5cee1192011-07-27 05:40:30 +00002737 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/
Reid Spencer5f016e22007-07-11 17:01:13 +00002738 case 'v': case 'w': case 'x': case 'y': case 'z':
2739 case '_':
2740 // Notify MIOpt that we read a non-whitespace/non-comment token.
2741 MIOpt.ReadToken();
2742 return LexIdentifier(Result, CurPtr);
Chris Lattner3a570772008-01-03 17:58:54 +00002743
2744 case '$': // $ in identifiers.
2745 if (Features.DollarIdents) {
Chris Lattner74d15df2008-11-22 02:02:22 +00002746 if (!isLexingRawMode())
2747 Diag(CurPtr-1, diag::ext_dollar_in_identifier);
Chris Lattner3a570772008-01-03 17:58:54 +00002748 // Notify MIOpt that we read a non-whitespace/non-comment token.
2749 MIOpt.ReadToken();
2750 return LexIdentifier(Result, CurPtr);
2751 }
Mike Stump1eb44332009-09-09 15:08:12 +00002752
Chris Lattner9e6293d2008-10-12 04:51:35 +00002753 Kind = tok::unknown;
Chris Lattner3a570772008-01-03 17:58:54 +00002754 break;
Mike Stump1eb44332009-09-09 15:08:12 +00002755
Reid Spencer5f016e22007-07-11 17:01:13 +00002756 // C99 6.4.4: Character Constants.
2757 case '\'':
2758 // Notify MIOpt that we read a non-whitespace/non-comment token.
2759 MIOpt.ReadToken();
Douglas Gregor5cee1192011-07-27 05:40:30 +00002760 return LexCharConstant(Result, CurPtr, tok::char_constant);
Reid Spencer5f016e22007-07-11 17:01:13 +00002761
2762 // C99 6.4.5: String Literals.
2763 case '"':
2764 // Notify MIOpt that we read a non-whitespace/non-comment token.
2765 MIOpt.ReadToken();
Douglas Gregor5cee1192011-07-27 05:40:30 +00002766 return LexStringLiteral(Result, CurPtr, tok::string_literal);
Reid Spencer5f016e22007-07-11 17:01:13 +00002767
2768 // C99 6.4.6: Punctuators.
2769 case '?':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002770 Kind = tok::question;
Reid Spencer5f016e22007-07-11 17:01:13 +00002771 break;
2772 case '[':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002773 Kind = tok::l_square;
Reid Spencer5f016e22007-07-11 17:01:13 +00002774 break;
2775 case ']':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002776 Kind = tok::r_square;
Reid Spencer5f016e22007-07-11 17:01:13 +00002777 break;
2778 case '(':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002779 Kind = tok::l_paren;
Reid Spencer5f016e22007-07-11 17:01:13 +00002780 break;
2781 case ')':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002782 Kind = tok::r_paren;
Reid Spencer5f016e22007-07-11 17:01:13 +00002783 break;
2784 case '{':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002785 Kind = tok::l_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00002786 break;
2787 case '}':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002788 Kind = tok::r_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00002789 break;
2790 case '.':
2791 Char = getCharAndSize(CurPtr, SizeTmp);
2792 if (Char >= '0' && Char <= '9') {
2793 // Notify MIOpt that we read a non-whitespace/non-comment token.
2794 MIOpt.ReadToken();
2795
2796 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
2797 } else if (Features.CPlusPlus && Char == '*') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002798 Kind = tok::periodstar;
Reid Spencer5f016e22007-07-11 17:01:13 +00002799 CurPtr += SizeTmp;
2800 } else if (Char == '.' &&
2801 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002802 Kind = tok::ellipsis;
Reid Spencer5f016e22007-07-11 17:01:13 +00002803 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2804 SizeTmp2, Result);
2805 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002806 Kind = tok::period;
Reid Spencer5f016e22007-07-11 17:01:13 +00002807 }
2808 break;
2809 case '&':
2810 Char = getCharAndSize(CurPtr, SizeTmp);
2811 if (Char == '&') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002812 Kind = tok::ampamp;
Reid Spencer5f016e22007-07-11 17:01:13 +00002813 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2814 } else if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002815 Kind = tok::ampequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002816 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2817 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002818 Kind = tok::amp;
Reid Spencer5f016e22007-07-11 17:01:13 +00002819 }
2820 break;
Mike Stump1eb44332009-09-09 15:08:12 +00002821 case '*':
Reid Spencer5f016e22007-07-11 17:01:13 +00002822 if (getCharAndSize(CurPtr, SizeTmp) == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002823 Kind = tok::starequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002824 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2825 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002826 Kind = tok::star;
Reid Spencer5f016e22007-07-11 17:01:13 +00002827 }
2828 break;
2829 case '+':
2830 Char = getCharAndSize(CurPtr, SizeTmp);
2831 if (Char == '+') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002832 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002833 Kind = tok::plusplus;
Reid Spencer5f016e22007-07-11 17:01:13 +00002834 } else if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002835 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002836 Kind = tok::plusequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002837 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002838 Kind = tok::plus;
Reid Spencer5f016e22007-07-11 17:01:13 +00002839 }
2840 break;
2841 case '-':
2842 Char = getCharAndSize(CurPtr, SizeTmp);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002843 if (Char == '-') { // --
Reid Spencer5f016e22007-07-11 17:01:13 +00002844 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002845 Kind = tok::minusminus;
Mike Stump1eb44332009-09-09 15:08:12 +00002846 } else if (Char == '>' && Features.CPlusPlus &&
Chris Lattner9e6293d2008-10-12 04:51:35 +00002847 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->*
Reid Spencer5f016e22007-07-11 17:01:13 +00002848 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2849 SizeTmp2, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002850 Kind = tok::arrowstar;
2851 } else if (Char == '>') { // ->
Reid Spencer5f016e22007-07-11 17:01:13 +00002852 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002853 Kind = tok::arrow;
2854 } else if (Char == '=') { // -=
Reid Spencer5f016e22007-07-11 17:01:13 +00002855 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002856 Kind = tok::minusequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002857 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002858 Kind = tok::minus;
Reid Spencer5f016e22007-07-11 17:01:13 +00002859 }
2860 break;
2861 case '~':
Chris Lattner9e6293d2008-10-12 04:51:35 +00002862 Kind = tok::tilde;
Reid Spencer5f016e22007-07-11 17:01:13 +00002863 break;
2864 case '!':
2865 if (getCharAndSize(CurPtr, SizeTmp) == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002866 Kind = tok::exclaimequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002867 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2868 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002869 Kind = tok::exclaim;
Reid Spencer5f016e22007-07-11 17:01:13 +00002870 }
2871 break;
2872 case '/':
2873 // 6.4.9: Comments
2874 Char = getCharAndSize(CurPtr, SizeTmp);
2875 if (Char == '/') { // BCPL comment.
Chris Lattner8402c732009-01-16 22:39:25 +00002876 // Even if BCPL comments are disabled (e.g. in C89 mode), we generally
2877 // want to lex this as a comment. There is one problem with this though,
2878 // that in one particular corner case, this can change the behavior of the
2879 // resultant program. For example, In "foo //**/ bar", C89 would lex
2880 // this as "foo / bar" and langauges with BCPL comments would lex it as
2881 // "foo". Check to see if the character after the second slash is a '*'.
2882 // If so, we will lex that as a "/" instead of the start of a comment.
Daniel Dunbar2ed42282011-03-18 21:23:38 +00002883 // However, we never do this in -traditional-cpp mode.
2884 if ((Features.BCPLComment ||
2885 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') &&
2886 !Features.TraditionalCPP) {
Chris Lattner8402c732009-01-16 22:39:25 +00002887 if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
Chris Lattner046c2272010-01-18 22:35:47 +00002888 return; // There is a token to return.
Mike Stump1eb44332009-09-09 15:08:12 +00002889
Chris Lattner8402c732009-01-16 22:39:25 +00002890 // It is common for the tokens immediately after a // comment to be
2891 // whitespace (indentation for the next line). Instead of going through
2892 // the big switch, handle it efficiently now.
2893 goto SkipIgnoredUnits;
2894 }
2895 }
Mike Stump1eb44332009-09-09 15:08:12 +00002896
Chris Lattner8402c732009-01-16 22:39:25 +00002897 if (Char == '*') { // /**/ comment.
Reid Spencer5f016e22007-07-11 17:01:13 +00002898 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
Chris Lattner046c2272010-01-18 22:35:47 +00002899 return; // There is a token to return.
Chris Lattner2d381892008-10-12 04:15:42 +00002900 goto LexNextToken; // GCC isn't tail call eliminating.
Chris Lattner8402c732009-01-16 22:39:25 +00002901 }
Mike Stump1eb44332009-09-09 15:08:12 +00002902
Chris Lattner8402c732009-01-16 22:39:25 +00002903 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002904 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002905 Kind = tok::slashequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002906 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002907 Kind = tok::slash;
Reid Spencer5f016e22007-07-11 17:01:13 +00002908 }
2909 break;
2910 case '%':
2911 Char = getCharAndSize(CurPtr, SizeTmp);
2912 if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002913 Kind = tok::percentequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00002914 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2915 } else if (Features.Digraphs && Char == '>') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002916 Kind = tok::r_brace; // '%>' -> '}'
Reid Spencer5f016e22007-07-11 17:01:13 +00002917 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2918 } else if (Features.Digraphs && Char == ':') {
2919 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2920 Char = getCharAndSize(CurPtr, SizeTmp);
2921 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002922 Kind = tok::hashhash; // '%:%:' -> '##'
Reid Spencer5f016e22007-07-11 17:01:13 +00002923 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2924 SizeTmp2, Result);
Francois Pichet62ec1f22011-09-17 17:15:52 +00002925 } else if (Char == '@' && Features.MicrosoftExt) {// %:@ -> #@ -> Charize
Reid Spencer5f016e22007-07-11 17:01:13 +00002926 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner74d15df2008-11-22 02:02:22 +00002927 if (!isLexingRawMode())
Ted Kremenek66d5ce12011-10-17 21:47:53 +00002928 Diag(BufferPtr, diag::ext_charize_microsoft);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002929 Kind = tok::hashat;
Chris Lattnere91e9322009-03-18 20:58:27 +00002930 } else { // '%:' -> '#'
Reid Spencer5f016e22007-07-11 17:01:13 +00002931 // We parsed a # character. If this occurs at the start of the line,
2932 // it's actually the start of a preprocessing directive. Callback to
2933 // the preprocessor to handle it.
2934 // FIXME: -fpreprocessed mode??
Chris Lattner766703b2009-05-13 06:10:29 +00002935 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) {
Chris Lattnere91e9322009-03-18 20:58:27 +00002936 FormTokenWithChars(Result, CurPtr, tok::hash);
Chris Lattner168ae2d2007-10-17 20:41:00 +00002937 PP->HandleDirective(Result);
Mike Stump1eb44332009-09-09 15:08:12 +00002938
Reid Spencer5f016e22007-07-11 17:01:13 +00002939 // As an optimization, if the preprocessor didn't switch lexers, tail
2940 // recurse.
Chris Lattner168ae2d2007-10-17 20:41:00 +00002941 if (PP->isCurrentLexer(this)) {
Reid Spencer5f016e22007-07-11 17:01:13 +00002942 // Start a new token. If this is a #include or something, the PP may
2943 // want us starting at the beginning of the line again. If so, set
Chris Lattner515f43f2010-04-12 23:04:41 +00002944 // the StartOfLine flag and clear LeadingSpace.
Reid Spencer5f016e22007-07-11 17:01:13 +00002945 if (IsAtStartOfLine) {
Chris Lattnerd2177732007-07-20 16:59:19 +00002946 Result.setFlag(Token::StartOfLine);
Chris Lattner515f43f2010-04-12 23:04:41 +00002947 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00002948 IsAtStartOfLine = false;
2949 }
2950 goto LexNextToken; // GCC isn't tail call eliminating.
2951 }
Mike Stump1eb44332009-09-09 15:08:12 +00002952
Chris Lattner168ae2d2007-10-17 20:41:00 +00002953 return PP->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00002954 }
Mike Stump1eb44332009-09-09 15:08:12 +00002955
Chris Lattnere91e9322009-03-18 20:58:27 +00002956 Kind = tok::hash;
Reid Spencer5f016e22007-07-11 17:01:13 +00002957 }
2958 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00002959 Kind = tok::percent;
Reid Spencer5f016e22007-07-11 17:01:13 +00002960 }
2961 break;
2962 case '<':
2963 Char = getCharAndSize(CurPtr, SizeTmp);
2964 if (ParsingFilename) {
Chris Lattner9cb51ce2009-04-17 23:56:52 +00002965 return LexAngledStringLiteral(Result, CurPtr);
Reid Spencer5f016e22007-07-11 17:01:13 +00002966 } else if (Char == '<') {
Chris Lattner34f349d2009-12-14 06:16:57 +00002967 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
2968 if (After == '=') {
2969 Kind = tok::lesslessequal;
2970 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2971 SizeTmp2, Result);
2972 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) {
2973 // If this is actually a '<<<<<<<' version control conflict marker,
2974 // recognize it as such and recover nicely.
2975 goto LexNextToken;
Richard Smithd5e1d602011-10-12 00:37:51 +00002976 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) {
2977 // If this is '<<<<' and we're in a Perforce-style conflict marker,
2978 // ignore it.
2979 goto LexNextToken;
Peter Collingbourne1b791d62011-02-09 21:08:21 +00002980 } else if (Features.CUDA && After == '<') {
2981 Kind = tok::lesslessless;
2982 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
2983 SizeTmp2, Result);
Chris Lattner34f349d2009-12-14 06:16:57 +00002984 } else {
2985 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
2986 Kind = tok::lessless;
2987 }
Reid Spencer5f016e22007-07-11 17:01:13 +00002988 } else if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00002989 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00002990 Kind = tok::lessequal;
2991 } else if (Features.Digraphs && Char == ':') { // '<:' -> '['
Richard Smith87a1e192011-04-14 18:36:27 +00002992 if (Features.CPlusPlus0x &&
2993 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') {
2994 // C++0x [lex.pptoken]p3:
2995 // Otherwise, if the next three characters are <:: and the subsequent
2996 // character is neither : nor >, the < is treated as a preprocessor
2997 // token by itself and not as the first character of the alternative
2998 // token <:.
2999 unsigned SizeTmp3;
3000 char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
3001 if (After != ':' && After != '>') {
3002 Kind = tok::less;
Richard Smith661a9962011-10-15 01:18:56 +00003003 if (!isLexingRawMode())
3004 Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon);
Richard Smith87a1e192011-04-14 18:36:27 +00003005 break;
3006 }
3007 }
3008
Reid Spencer5f016e22007-07-11 17:01:13 +00003009 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00003010 Kind = tok::l_square;
3011 } else if (Features.Digraphs && Char == '%') { // '<%' -> '{'
Reid Spencer5f016e22007-07-11 17:01:13 +00003012 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00003013 Kind = tok::l_brace;
Reid Spencer5f016e22007-07-11 17:01:13 +00003014 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003015 Kind = tok::less;
Reid Spencer5f016e22007-07-11 17:01:13 +00003016 }
3017 break;
3018 case '>':
3019 Char = getCharAndSize(CurPtr, SizeTmp);
3020 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00003021 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00003022 Kind = tok::greaterequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00003023 } else if (Char == '>') {
Chris Lattner34f349d2009-12-14 06:16:57 +00003024 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
3025 if (After == '=') {
3026 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3027 SizeTmp2, Result);
3028 Kind = tok::greatergreaterequal;
Richard Smithd5e1d602011-10-12 00:37:51 +00003029 } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) {
3030 // If this is actually a '>>>>' conflict marker, recognize it as such
3031 // and recover nicely.
3032 goto LexNextToken;
Chris Lattner34f349d2009-12-14 06:16:57 +00003033 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) {
3034 // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
3035 goto LexNextToken;
Peter Collingbourne1b791d62011-02-09 21:08:21 +00003036 } else if (Features.CUDA && After == '>') {
3037 Kind = tok::greatergreatergreater;
3038 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
3039 SizeTmp2, Result);
Chris Lattner34f349d2009-12-14 06:16:57 +00003040 } else {
3041 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3042 Kind = tok::greatergreater;
3043 }
3044
Reid Spencer5f016e22007-07-11 17:01:13 +00003045 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003046 Kind = tok::greater;
Reid Spencer5f016e22007-07-11 17:01:13 +00003047 }
3048 break;
3049 case '^':
3050 Char = getCharAndSize(CurPtr, SizeTmp);
3051 if (Char == '=') {
Reid Spencer5f016e22007-07-11 17:01:13 +00003052 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Chris Lattner9e6293d2008-10-12 04:51:35 +00003053 Kind = tok::caretequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00003054 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003055 Kind = tok::caret;
Reid Spencer5f016e22007-07-11 17:01:13 +00003056 }
3057 break;
3058 case '|':
3059 Char = getCharAndSize(CurPtr, SizeTmp);
3060 if (Char == '=') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003061 Kind = tok::pipeequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00003062 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3063 } else if (Char == '|') {
Chris Lattner34f349d2009-12-14 06:16:57 +00003064 // If this is '|||||||' and we're in a conflict marker, ignore it.
3065 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1))
3066 goto LexNextToken;
Chris Lattner9e6293d2008-10-12 04:51:35 +00003067 Kind = tok::pipepipe;
Reid Spencer5f016e22007-07-11 17:01:13 +00003068 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3069 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003070 Kind = tok::pipe;
Reid Spencer5f016e22007-07-11 17:01:13 +00003071 }
3072 break;
3073 case ':':
3074 Char = getCharAndSize(CurPtr, SizeTmp);
3075 if (Features.Digraphs && Char == '>') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003076 Kind = tok::r_square; // ':>' -> ']'
Reid Spencer5f016e22007-07-11 17:01:13 +00003077 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3078 } else if (Features.CPlusPlus && Char == ':') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003079 Kind = tok::coloncolon;
Reid Spencer5f016e22007-07-11 17:01:13 +00003080 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Mike Stump1eb44332009-09-09 15:08:12 +00003081 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003082 Kind = tok::colon;
Reid Spencer5f016e22007-07-11 17:01:13 +00003083 }
3084 break;
3085 case ';':
Chris Lattner9e6293d2008-10-12 04:51:35 +00003086 Kind = tok::semi;
Reid Spencer5f016e22007-07-11 17:01:13 +00003087 break;
3088 case '=':
3089 Char = getCharAndSize(CurPtr, SizeTmp);
3090 if (Char == '=') {
Richard Smithd5e1d602011-10-12 00:37:51 +00003091 // If this is '====' and we're in a conflict marker, ignore it.
Chris Lattner34f349d2009-12-14 06:16:57 +00003092 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1))
3093 goto LexNextToken;
3094
Chris Lattner9e6293d2008-10-12 04:51:35 +00003095 Kind = tok::equalequal;
Reid Spencer5f016e22007-07-11 17:01:13 +00003096 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Mike Stump1eb44332009-09-09 15:08:12 +00003097 } else {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003098 Kind = tok::equal;
Reid Spencer5f016e22007-07-11 17:01:13 +00003099 }
3100 break;
3101 case ',':
Chris Lattner9e6293d2008-10-12 04:51:35 +00003102 Kind = tok::comma;
Reid Spencer5f016e22007-07-11 17:01:13 +00003103 break;
3104 case '#':
3105 Char = getCharAndSize(CurPtr, SizeTmp);
3106 if (Char == '#') {
Chris Lattner9e6293d2008-10-12 04:51:35 +00003107 Kind = tok::hashhash;
Reid Spencer5f016e22007-07-11 17:01:13 +00003108 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
Francois Pichet62ec1f22011-09-17 17:15:52 +00003109 } else if (Char == '@' && Features.MicrosoftExt) { // #@ -> Charize
Chris Lattner9e6293d2008-10-12 04:51:35 +00003110 Kind = tok::hashat;
Chris Lattner74d15df2008-11-22 02:02:22 +00003111 if (!isLexingRawMode())
Ted Kremenek66d5ce12011-10-17 21:47:53 +00003112 Diag(BufferPtr, diag::ext_charize_microsoft);
Reid Spencer5f016e22007-07-11 17:01:13 +00003113 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
3114 } else {
Reid Spencer5f016e22007-07-11 17:01:13 +00003115 // We parsed a # character. If this occurs at the start of the line,
3116 // it's actually the start of a preprocessing directive. Callback to
3117 // the preprocessor to handle it.
3118 // FIXME: -fpreprocessed mode??
Chris Lattner766703b2009-05-13 06:10:29 +00003119 if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) {
Chris Lattnere91e9322009-03-18 20:58:27 +00003120 FormTokenWithChars(Result, CurPtr, tok::hash);
Chris Lattner168ae2d2007-10-17 20:41:00 +00003121 PP->HandleDirective(Result);
Mike Stump1eb44332009-09-09 15:08:12 +00003122
Reid Spencer5f016e22007-07-11 17:01:13 +00003123 // As an optimization, if the preprocessor didn't switch lexers, tail
3124 // recurse.
Chris Lattner168ae2d2007-10-17 20:41:00 +00003125 if (PP->isCurrentLexer(this)) {
Reid Spencer5f016e22007-07-11 17:01:13 +00003126 // Start a new token. If this is a #include or something, the PP may
3127 // want us starting at the beginning of the line again. If so, set
Chris Lattner515f43f2010-04-12 23:04:41 +00003128 // the StartOfLine flag and clear LeadingSpace.
Reid Spencer5f016e22007-07-11 17:01:13 +00003129 if (IsAtStartOfLine) {
Chris Lattnerd2177732007-07-20 16:59:19 +00003130 Result.setFlag(Token::StartOfLine);
Chris Lattner515f43f2010-04-12 23:04:41 +00003131 Result.clearFlag(Token::LeadingSpace);
Reid Spencer5f016e22007-07-11 17:01:13 +00003132 IsAtStartOfLine = false;
3133 }
3134 goto LexNextToken; // GCC isn't tail call eliminating.
3135 }
Chris Lattner168ae2d2007-10-17 20:41:00 +00003136 return PP->Lex(Result);
Reid Spencer5f016e22007-07-11 17:01:13 +00003137 }
Mike Stump1eb44332009-09-09 15:08:12 +00003138
Chris Lattnere91e9322009-03-18 20:58:27 +00003139 Kind = tok::hash;
Reid Spencer5f016e22007-07-11 17:01:13 +00003140 }
3141 break;
3142
Chris Lattner3a570772008-01-03 17:58:54 +00003143 case '@':
3144 // Objective C support.
3145 if (CurPtr[-1] == '@' && Features.ObjC1)
Chris Lattner9e6293d2008-10-12 04:51:35 +00003146 Kind = tok::at;
Chris Lattner3a570772008-01-03 17:58:54 +00003147 else
Chris Lattner9e6293d2008-10-12 04:51:35 +00003148 Kind = tok::unknown;
Chris Lattner3a570772008-01-03 17:58:54 +00003149 break;
Mike Stump1eb44332009-09-09 15:08:12 +00003150
Reid Spencer5f016e22007-07-11 17:01:13 +00003151 case '\\':
3152 // FIXME: UCN's.
3153 // FALL THROUGH.
3154 default:
Chris Lattner9e6293d2008-10-12 04:51:35 +00003155 Kind = tok::unknown;
Reid Spencer5f016e22007-07-11 17:01:13 +00003156 break;
3157 }
Mike Stump1eb44332009-09-09 15:08:12 +00003158
Reid Spencer5f016e22007-07-11 17:01:13 +00003159 // Notify MIOpt that we read a non-whitespace/non-comment token.
3160 MIOpt.ReadToken();
3161
3162 // Update the location of token as well as BufferPtr.
Chris Lattner9e6293d2008-10-12 04:51:35 +00003163 FormTokenWithChars(Result, CurPtr, Kind);
Reid Spencer5f016e22007-07-11 17:01:13 +00003164}