Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1 | //===--- Lexer.cpp - C Language Family Lexer ------------------------------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
Chris Lattner | 5b12ab8 | 2007-12-29 19:59:25 +0000 | [diff] [blame] | 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 10 | // This file implements the Lexer and Token interfaces. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 13 | |
| 14 | #include "clang/Lex/Lexer.h" |
Chandler Carruth | 5553d0d | 2014-01-07 11:51:46 +0000 | [diff] [blame] | 15 | #include "UnicodeCharSets.h" |
Jordan Rose | a2100d7 | 2013-02-08 22:30:22 +0000 | [diff] [blame] | 16 | #include "clang/Basic/CharInfo.h" |
Chris Lattner | dc5c055 | 2007-07-20 16:37:10 +0000 | [diff] [blame] | 17 | #include "clang/Basic/SourceManager.h" |
Chandler Carruth | 3a02247 | 2012-12-04 09:13:33 +0000 | [diff] [blame] | 18 | #include "clang/Lex/CodeCompletionHandler.h" |
| 19 | #include "clang/Lex/LexDiagnostic.h" |
Richard Smith | 2a98862 | 2013-09-24 04:06:10 +0000 | [diff] [blame] | 20 | #include "clang/Lex/LiteralSupport.h" |
Chandler Carruth | 3a02247 | 2012-12-04 09:13:33 +0000 | [diff] [blame] | 21 | #include "clang/Lex/Preprocessor.h" |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 22 | #include "llvm/ADT/STLExtras.h" |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 23 | #include "llvm/ADT/StringExtras.h" |
Chandler Carruth | 3a02247 | 2012-12-04 09:13:33 +0000 | [diff] [blame] | 24 | #include "llvm/ADT/StringSwitch.h" |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 25 | #include "llvm/Support/Compiler.h" |
Dmitri Gribenko | 9feeef4 | 2013-01-30 12:06:08 +0000 | [diff] [blame] | 26 | #include "llvm/Support/ConvertUTF.h" |
Chris Lattner | 739e739 | 2007-04-29 07:12:06 +0000 | [diff] [blame] | 27 | #include "llvm/Support/MemoryBuffer.h" |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 28 | #include <cstring> |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 29 | using namespace clang; |
| 30 | |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 31 | //===----------------------------------------------------------------------===// |
| 32 | // Token Class Implementation |
| 33 | //===----------------------------------------------------------------------===// |
| 34 | |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 35 | /// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 36 | bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { |
Douglas Gregor | 90abb6d | 2008-12-01 21:46:47 +0000 | [diff] [blame] | 37 | if (IdentifierInfo *II = getIdentifierInfo()) |
| 38 | return II->getObjCKeywordID() == objcKey; |
| 39 | return false; |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 40 | } |
| 41 | |
| 42 | /// getObjCKeywordID - Return the ObjC keyword kind. |
| 43 | tok::ObjCKeywordKind Token::getObjCKeywordID() const { |
| 44 | IdentifierInfo *specId = getIdentifierInfo(); |
| 45 | return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; |
| 46 | } |
| 47 | |
Chris Lattner | 67671ed | 2007-12-13 01:59:49 +0000 | [diff] [blame] | 48 | |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 49 | //===----------------------------------------------------------------------===// |
| 50 | // Lexer Class Implementation |
| 51 | //===----------------------------------------------------------------------===// |
| 52 | |
David Blaikie | 68e081d | 2011-12-20 02:48:34 +0000 | [diff] [blame] | 53 | void Lexer::anchor() { } |
| 54 | |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 55 | void Lexer::InitLexer(const char *BufStart, const char *BufPtr, |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 56 | const char *BufEnd) { |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 57 | BufferStart = BufStart; |
| 58 | BufferPtr = BufPtr; |
| 59 | BufferEnd = BufEnd; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 60 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 61 | assert(BufEnd[0] == 0 && |
| 62 | "We assume that the input buffer has a null character at the end" |
| 63 | " to simplify lexing!"); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 64 | |
Eric Christopher | 7f36a79 | 2011-04-09 00:01:04 +0000 | [diff] [blame] | 65 | // Check whether we have a BOM in the beginning of the buffer. If yes - act |
| 66 | // accordingly. Right now we support only UTF-8 with and without BOM, so, just |
| 67 | // skip the UTF-8 BOM if it's present. |
| 68 | if (BufferStart == BufferPtr) { |
| 69 | // Determine the size of the BOM. |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 70 | StringRef Buf(BufferStart, BufferEnd - BufferStart); |
Eli Friedman | 86a5101 | 2011-05-10 17:11:21 +0000 | [diff] [blame] | 71 | size_t BOMLength = llvm::StringSwitch<size_t>(Buf) |
Eric Christopher | 7f36a79 | 2011-04-09 00:01:04 +0000 | [diff] [blame] | 72 | .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM |
| 73 | .Default(0); |
| 74 | |
| 75 | // Skip the BOM. |
| 76 | BufferPtr += BOMLength; |
| 77 | } |
| 78 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 79 | Is_PragmaLexer = false; |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 80 | CurrentConflictMarkerState = CMK_None; |
Eric Christopher | 7f36a79 | 2011-04-09 00:01:04 +0000 | [diff] [blame] | 81 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 82 | // Start of the file is a start of line. |
| 83 | IsAtStartOfLine = true; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 84 | IsAtPhysicalStartOfLine = true; |
| 85 | |
| 86 | HasLeadingSpace = false; |
| 87 | HasLeadingEmptyMacro = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 88 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 89 | // We are not after parsing a #. |
| 90 | ParsingPreprocessorDirective = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 91 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 92 | // We are not after parsing #include. |
| 93 | ParsingFilename = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 94 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 95 | // We are not in raw mode. Raw mode disables diagnostics and interpretation |
| 96 | // of tokens (e.g. identifiers, thus disabling macro expansion). It is used |
| 97 | // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block |
| 98 | // or otherwise skipping over tokens. |
| 99 | LexingRawMode = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 100 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 101 | // Default to not keeping comments. |
| 102 | ExtendedTokenMode = 0; |
| 103 | } |
| 104 | |
Chris Lattner | 5965a28 | 2009-01-17 07:56:59 +0000 | [diff] [blame] | 105 | /// Lexer constructor - Create a new lexer object for the specified buffer |
| 106 | /// with the specified preprocessor managing the lexing process. This lexer |
| 107 | /// assumes that the associated file buffer and Preprocessor objects will |
| 108 | /// outlive it, so it doesn't take ownership of either of them. |
Chris Lattner | 710bb87 | 2009-11-30 04:18:44 +0000 | [diff] [blame] | 109 | Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) |
Chris Lattner | c809089 | 2009-01-17 08:03:42 +0000 | [diff] [blame] | 110 | : PreprocessorLexer(&PP, FID), |
| 111 | FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 112 | LangOpts(PP.getLangOpts()) { |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 113 | |
Chris Lattner | 5965a28 | 2009-01-17 07:56:59 +0000 | [diff] [blame] | 114 | InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), |
| 115 | InputFile->getBufferEnd()); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 116 | |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 117 | resetExtendedTokenMode(); |
| 118 | } |
| 119 | |
| 120 | void Lexer::resetExtendedTokenMode() { |
| 121 | assert(PP && "Cannot reset token mode without a preprocessor"); |
| 122 | if (LangOpts.TraditionalCPP) |
| 123 | SetKeepWhitespaceMode(true); |
| 124 | else |
| 125 | SetCommentRetentionState(PP->getCommentRetentionState()); |
Chris Lattner | 5965a28 | 2009-01-17 07:56:59 +0000 | [diff] [blame] | 126 | } |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 127 | |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 128 | /// Lexer constructor - Create a new raw lexer object. This object is only |
Dmitri Gribenko | 702b732 | 2012-06-08 23:19:37 +0000 | [diff] [blame] | 129 | /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text |
Chris Lattner | 50c9050 | 2008-10-12 01:15:46 +0000 | [diff] [blame] | 130 | /// range will outlive it, so it doesn't take ownership of it. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 131 | Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, |
Chris Lattner | fcf6452 | 2009-01-17 07:42:27 +0000 | [diff] [blame] | 132 | const char *BufStart, const char *BufPtr, const char *BufEnd) |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 133 | : FileLoc(fileloc), LangOpts(langOpts) { |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 134 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 135 | InitLexer(BufStart, BufPtr, BufEnd); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 136 | |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 137 | // We *are* in raw mode. |
| 138 | LexingRawMode = true; |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 139 | } |
| 140 | |
Chris Lattner | 08354fe | 2009-01-17 07:35:14 +0000 | [diff] [blame] | 141 | /// Lexer constructor - Create a new raw lexer object. This object is only |
Dmitri Gribenko | 702b732 | 2012-06-08 23:19:37 +0000 | [diff] [blame] | 142 | /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text |
Chris Lattner | 08354fe | 2009-01-17 07:35:14 +0000 | [diff] [blame] | 143 | /// range will outlive it, so it doesn't take ownership of it. |
Chris Lattner | 710bb87 | 2009-11-30 04:18:44 +0000 | [diff] [blame] | 144 | Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 145 | const SourceManager &SM, const LangOptions &langOpts) |
| 146 | : FileLoc(SM.getLocForStartOfFile(FID)), LangOpts(langOpts) { |
Chris Lattner | 08354fe | 2009-01-17 07:35:14 +0000 | [diff] [blame] | 147 | |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 148 | InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), |
Chris Lattner | 08354fe | 2009-01-17 07:35:14 +0000 | [diff] [blame] | 149 | FromFile->getBufferEnd()); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 150 | |
Chris Lattner | 08354fe | 2009-01-17 07:35:14 +0000 | [diff] [blame] | 151 | // We *are* in raw mode. |
| 152 | LexingRawMode = true; |
| 153 | } |
| 154 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 155 | /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for |
| 156 | /// _Pragma expansion. This has a variety of magic semantics that this method |
| 157 | /// sets up. It returns a new'd Lexer that must be delete'd when done. |
| 158 | /// |
| 159 | /// On entrance to this routine, TokStartLoc is a macro location which has a |
| 160 | /// spelling loc that indicates the bytes to be lexed for the token and an |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 161 | /// expansion location that indicates where all lexed tokens should be |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 162 | /// "expanded from". |
| 163 | /// |
Alp Toker | 7755aff | 2014-05-18 18:37:59 +0000 | [diff] [blame] | 164 | /// TODO: It would really be nice to make _Pragma just be a wrapper around a |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 165 | /// normal lexer that remaps tokens as they fly by. This would require making |
| 166 | /// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer |
| 167 | /// interface that could handle this stuff. This would pull GetMappedTokenLoc |
| 168 | /// out of the critical path of the lexer! |
| 169 | /// |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 170 | Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 171 | SourceLocation ExpansionLocStart, |
| 172 | SourceLocation ExpansionLocEnd, |
Chris Lattner | 29a2a19 | 2009-01-19 06:46:35 +0000 | [diff] [blame] | 173 | unsigned TokLen, Preprocessor &PP) { |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 174 | SourceManager &SM = PP.getSourceManager(); |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 175 | |
| 176 | // Create the lexer as if we were going to lex the file normally. |
Chris Lattner | cbc35ecb | 2009-01-19 07:46:45 +0000 | [diff] [blame] | 177 | FileID SpellingFID = SM.getFileID(SpellingLoc); |
Chris Lattner | 710bb87 | 2009-11-30 04:18:44 +0000 | [diff] [blame] | 178 | const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); |
| 179 | Lexer *L = new Lexer(SpellingFID, InputFile, PP); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 180 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 181 | // Now that the lexer is created, change the start/end locations so that we |
| 182 | // just lex the subsection of the file that we want. This is lexing from a |
| 183 | // scratch buffer. |
| 184 | const char *StrData = SM.getCharacterData(SpellingLoc); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 185 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 186 | L->BufferPtr = StrData; |
| 187 | L->BufferEnd = StrData+TokLen; |
Chris Lattner | fa217bd | 2009-03-08 08:08:45 +0000 | [diff] [blame] | 188 | assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 189 | |
| 190 | // Set the SourceLocation with the remapping information. This ensures that |
| 191 | // GetMappedTokenLoc will remap the tokens as they are lexed. |
Chandler Carruth | 115b077 | 2011-07-26 03:03:05 +0000 | [diff] [blame] | 192 | L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), |
| 193 | ExpansionLocStart, |
| 194 | ExpansionLocEnd, TokLen); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 195 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 196 | // Ensure that the lexer thinks it is inside a directive, so that end \n will |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 197 | // return an EOD token. |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 198 | L->ParsingPreprocessorDirective = true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 199 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 200 | // This lexer really is for _Pragma. |
| 201 | L->Is_PragmaLexer = true; |
| 202 | return L; |
| 203 | } |
| 204 | |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 205 | |
Chris Lattner | e3e81ea | 2006-07-03 01:13:26 +0000 | [diff] [blame] | 206 | /// Stringify - Convert the specified string into a C string, with surrounding |
| 207 | /// ""'s, and with escaped \ and " characters. |
Chris Lattner | ecc39e9 | 2006-07-15 05:23:31 +0000 | [diff] [blame] | 208 | std::string Lexer::Stringify(const std::string &Str, bool Charify) { |
Chris Lattner | e3e81ea | 2006-07-03 01:13:26 +0000 | [diff] [blame] | 209 | std::string Result = Str; |
Chris Lattner | ecc39e9 | 2006-07-15 05:23:31 +0000 | [diff] [blame] | 210 | char Quote = Charify ? '\'' : '"'; |
Chris Lattner | e3e81ea | 2006-07-03 01:13:26 +0000 | [diff] [blame] | 211 | for (unsigned i = 0, e = Result.size(); i != e; ++i) { |
Chris Lattner | ecc39e9 | 2006-07-15 05:23:31 +0000 | [diff] [blame] | 212 | if (Result[i] == '\\' || Result[i] == Quote) { |
Chris Lattner | e3e81ea | 2006-07-03 01:13:26 +0000 | [diff] [blame] | 213 | Result.insert(Result.begin()+i, '\\'); |
| 214 | ++i; ++e; |
| 215 | } |
| 216 | } |
Chris Lattner | ecc39e9 | 2006-07-15 05:23:31 +0000 | [diff] [blame] | 217 | return Result; |
Chris Lattner | e3e81ea | 2006-07-03 01:13:26 +0000 | [diff] [blame] | 218 | } |
| 219 | |
Chris Lattner | 4c4a245 | 2007-07-24 06:57:14 +0000 | [diff] [blame] | 220 | /// Stringify - Convert the specified string into a C string by escaping '\' |
| 221 | /// and " characters. This does not add surrounding ""'s to the string. |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 222 | void Lexer::Stringify(SmallVectorImpl<char> &Str) { |
Chris Lattner | 4c4a245 | 2007-07-24 06:57:14 +0000 | [diff] [blame] | 223 | for (unsigned i = 0, e = Str.size(); i != e; ++i) { |
| 224 | if (Str[i] == '\\' || Str[i] == '"') { |
| 225 | Str.insert(Str.begin()+i, '\\'); |
| 226 | ++i; ++e; |
| 227 | } |
| 228 | } |
| 229 | } |
| 230 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 231 | //===----------------------------------------------------------------------===// |
| 232 | // Token Spelling |
| 233 | //===----------------------------------------------------------------------===// |
| 234 | |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 235 | /// \brief Slow case of getSpelling. Extract the characters comprising the |
| 236 | /// spelling of this token from the provided input buffer. |
| 237 | static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, |
| 238 | const LangOptions &LangOpts, char *Spelling) { |
| 239 | assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); |
| 240 | |
| 241 | size_t Length = 0; |
| 242 | const char *BufEnd = BufPtr + Tok.getLength(); |
| 243 | |
| 244 | if (Tok.is(tok::string_literal)) { |
| 245 | // Munch the encoding-prefix and opening double-quote. |
| 246 | while (BufPtr < BufEnd) { |
| 247 | unsigned Size; |
| 248 | Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); |
| 249 | BufPtr += Size; |
| 250 | |
| 251 | if (Spelling[Length - 1] == '"') |
| 252 | break; |
| 253 | } |
| 254 | |
| 255 | // Raw string literals need special handling; trigraph expansion and line |
| 256 | // splicing do not occur within their d-char-sequence nor within their |
| 257 | // r-char-sequence. |
| 258 | if (Length >= 2 && |
| 259 | Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { |
| 260 | // Search backwards from the end of the token to find the matching closing |
| 261 | // quote. |
| 262 | const char *RawEnd = BufEnd; |
| 263 | do --RawEnd; while (*RawEnd != '"'); |
| 264 | size_t RawLength = RawEnd - BufPtr + 1; |
| 265 | |
| 266 | // Everything between the quotes is included verbatim in the spelling. |
| 267 | memcpy(Spelling + Length, BufPtr, RawLength); |
| 268 | Length += RawLength; |
| 269 | BufPtr += RawLength; |
| 270 | |
| 271 | // The rest of the token is lexed normally. |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | while (BufPtr < BufEnd) { |
| 276 | unsigned Size; |
| 277 | Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); |
| 278 | BufPtr += Size; |
| 279 | } |
| 280 | |
| 281 | assert(Length < Tok.getLength() && |
| 282 | "NeedsCleaning flag set on token that didn't need cleaning!"); |
| 283 | return Length; |
| 284 | } |
| 285 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 286 | /// getSpelling() - Return the 'spelling' of this token. The spelling of a |
| 287 | /// token are the characters used to represent the token in the source file |
| 288 | /// after trigraph expansion and escaped-newline folding. In particular, this |
| 289 | /// wants to get the true, uncanonicalized, spelling of things like digraphs |
| 290 | /// UCNs, etc. |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 291 | StringRef Lexer::getSpelling(SourceLocation loc, |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 292 | SmallVectorImpl<char> &buffer, |
| 293 | const SourceManager &SM, |
| 294 | const LangOptions &options, |
| 295 | bool *invalid) { |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 296 | // Break down the source location. |
| 297 | std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); |
| 298 | |
| 299 | // Try to the load the file buffer. |
| 300 | bool invalidTemp = false; |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 301 | StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 302 | if (invalidTemp) { |
| 303 | if (invalid) *invalid = true; |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 304 | return StringRef(); |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 305 | } |
| 306 | |
| 307 | const char *tokenBegin = file.data() + locInfo.second; |
| 308 | |
| 309 | // Lex from the start of the given location. |
| 310 | Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, |
| 311 | file.begin(), tokenBegin, file.end()); |
| 312 | Token token; |
| 313 | lexer.LexFromRawLexer(token); |
| 314 | |
| 315 | unsigned length = token.getLength(); |
| 316 | |
| 317 | // Common case: no need for cleaning. |
| 318 | if (!token.needsCleaning()) |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 319 | return StringRef(tokenBegin, length); |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 320 | |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 321 | // Hard case, we need to relex the characters into the string. |
| 322 | buffer.resize(length); |
| 323 | buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 324 | return StringRef(buffer.data(), buffer.size()); |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 325 | } |
| 326 | |
| 327 | /// getSpelling() - Return the 'spelling' of this token. The spelling of a |
| 328 | /// token are the characters used to represent the token in the source file |
| 329 | /// after trigraph expansion and escaped-newline folding. In particular, this |
| 330 | /// wants to get the true, uncanonicalized, spelling of things like digraphs |
| 331 | /// UCNs, etc. |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 332 | std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 333 | const LangOptions &LangOpts, bool *Invalid) { |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 334 | assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 335 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 336 | bool CharDataInvalid = false; |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 337 | const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 338 | &CharDataInvalid); |
| 339 | if (Invalid) |
| 340 | *Invalid = CharDataInvalid; |
| 341 | if (CharDataInvalid) |
| 342 | return std::string(); |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 343 | |
| 344 | // If this token contains nothing interesting, return it directly. |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 345 | if (!Tok.needsCleaning()) |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 346 | return std::string(TokStart, TokStart + Tok.getLength()); |
| 347 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 348 | std::string Result; |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 349 | Result.resize(Tok.getLength()); |
| 350 | Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 351 | return Result; |
| 352 | } |
| 353 | |
| 354 | /// getSpelling - This method is used to get the spelling of a token into a |
| 355 | /// preallocated buffer, instead of as an std::string. The caller is required |
| 356 | /// to allocate enough space for the token, which is guaranteed to be at least |
| 357 | /// Tok.getLength() bytes long. The actual length of the token is returned. |
| 358 | /// |
| 359 | /// Note that this method may do two possible things: it may either fill in |
| 360 | /// the buffer specified with characters, or it may *change the input pointer* |
| 361 | /// to point to a constant buffer with the data already in it (avoiding a |
| 362 | /// copy). The caller is not allowed to modify the returned buffer pointer |
| 363 | /// if an internal buffer is returned. |
| 364 | unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, |
| 365 | const SourceManager &SourceMgr, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 366 | const LangOptions &LangOpts, bool *Invalid) { |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 367 | assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 368 | |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 369 | const char *TokStart = nullptr; |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 370 | // NOTE: this has to be checked *before* testing for an IdentifierInfo. |
| 371 | if (Tok.is(tok::raw_identifier)) |
Alp Toker | 2d57cea | 2014-05-17 04:53:25 +0000 | [diff] [blame] | 372 | TokStart = Tok.getRawIdentifier().data(); |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 373 | else if (!Tok.hasUCN()) { |
| 374 | if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { |
| 375 | // Just return the string from the identifier table, which is very quick. |
| 376 | Buffer = II->getNameStart(); |
| 377 | return II->getLength(); |
| 378 | } |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 379 | } |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 380 | |
| 381 | // NOTE: this can be checked even after testing for an IdentifierInfo. |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 382 | if (Tok.isLiteral()) |
| 383 | TokStart = Tok.getLiteralData(); |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 384 | |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 385 | if (!TokStart) { |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 386 | // Compute the start of the token in the input lexer buffer. |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 387 | bool CharDataInvalid = false; |
| 388 | TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); |
| 389 | if (Invalid) |
| 390 | *Invalid = CharDataInvalid; |
| 391 | if (CharDataInvalid) { |
| 392 | Buffer = ""; |
| 393 | return 0; |
| 394 | } |
| 395 | } |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 396 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 397 | // If this token contains nothing interesting, return it directly. |
| 398 | if (!Tok.needsCleaning()) { |
| 399 | Buffer = TokStart; |
| 400 | return Tok.getLength(); |
| 401 | } |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 402 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 403 | // Otherwise, hard case, relex the characters into the string. |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 404 | return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 405 | } |
| 406 | |
| 407 | |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 408 | /// MeasureTokenLength - Relex the token at the specified location and return |
| 409 | /// its length in bytes in the input file. If the token needs cleaning (e.g. |
| 410 | /// includes a trigraph or an escaped newline) then this count includes bytes |
| 411 | /// that are part of that. |
| 412 | unsigned Lexer::MeasureTokenLength(SourceLocation Loc, |
Chris Lattner | 184e65d | 2009-04-14 23:22:57 +0000 | [diff] [blame] | 413 | const SourceManager &SM, |
| 414 | const LangOptions &LangOpts) { |
Argyrios Kyrtzidis | 86f1a93 | 2013-01-07 19:16:18 +0000 | [diff] [blame] | 415 | Token TheTok; |
| 416 | if (getRawToken(Loc, TheTok, SM, LangOpts)) |
| 417 | return 0; |
| 418 | return TheTok.getLength(); |
| 419 | } |
| 420 | |
| 421 | /// \brief Relex the token at the specified location. |
| 422 | /// \returns true if there was a failure, false on success. |
| 423 | bool Lexer::getRawToken(SourceLocation Loc, Token &Result, |
| 424 | const SourceManager &SM, |
Fariborz Jahanian | d38ad47 | 2013-08-20 00:07:23 +0000 | [diff] [blame] | 425 | const LangOptions &LangOpts, |
| 426 | bool IgnoreWhiteSpace) { |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 427 | // TODO: this could be special cased for common tokens like identifiers, ')', |
| 428 | // etc to make this faster, if it mattered. Just look at StrData[0] to handle |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 429 | // all obviously single-char tokens. This could use |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 430 | // Lexer::isObviouslySimpleCharacter for example to handle identifiers or |
| 431 | // something. |
Chris Lattner | 4fa2362 | 2009-01-26 00:43:02 +0000 | [diff] [blame] | 432 | |
| 433 | // If this comes from a macro expansion, we really do want the macro name, not |
| 434 | // the token this macro expanded to. |
Chandler Carruth | 35f5320 | 2011-07-25 16:49:02 +0000 | [diff] [blame] | 435 | Loc = SM.getExpansionLoc(Loc); |
Chris Lattner | d381721 | 2009-01-26 22:24:27 +0000 | [diff] [blame] | 436 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
Douglas Gregor | e0fbb83 | 2010-03-16 00:06:06 +0000 | [diff] [blame] | 437 | bool Invalid = false; |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 438 | StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
Douglas Gregor | e0fbb83 | 2010-03-16 00:06:06 +0000 | [diff] [blame] | 439 | if (Invalid) |
Argyrios Kyrtzidis | 86f1a93 | 2013-01-07 19:16:18 +0000 | [diff] [blame] | 440 | return true; |
Benjamin Kramer | eb92dc0 | 2010-03-16 14:14:31 +0000 | [diff] [blame] | 441 | |
| 442 | const char *StrData = Buffer.data()+LocInfo.second; |
Chris Lattner | 5509d53 | 2009-01-17 08:30:10 +0000 | [diff] [blame] | 443 | |
Fariborz Jahanian | d38ad47 | 2013-08-20 00:07:23 +0000 | [diff] [blame] | 444 | if (!IgnoreWhiteSpace && isWhitespace(StrData[0])) |
Argyrios Kyrtzidis | 86f1a93 | 2013-01-07 19:16:18 +0000 | [diff] [blame] | 445 | return true; |
Douglas Gregor | 562c1f9 | 2010-01-22 19:49:59 +0000 | [diff] [blame] | 446 | |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 447 | // Create a lexer starting at the beginning of this token. |
Sebastian Redl | 5175230 | 2010-09-30 01:03:03 +0000 | [diff] [blame] | 448 | Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, |
| 449 | Buffer.begin(), StrData, Buffer.end()); |
Chris Lattner | a3d4f16 | 2009-10-14 15:04:18 +0000 | [diff] [blame] | 450 | TheLexer.SetCommentRetentionState(true); |
Argyrios Kyrtzidis | 86f1a93 | 2013-01-07 19:16:18 +0000 | [diff] [blame] | 451 | TheLexer.LexFromRawLexer(Result); |
| 452 | return false; |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 453 | } |
| 454 | |
Argyrios Kyrtzidis | 161868d | 2011-08-17 00:31:23 +0000 | [diff] [blame] | 455 | static SourceLocation getBeginningOfFileToken(SourceLocation Loc, |
| 456 | const SourceManager &SM, |
| 457 | const LangOptions &LangOpts) { |
| 458 | assert(Loc.isFileID()); |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 459 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
Douglas Gregor | 86af984 | 2011-01-31 22:42:36 +0000 | [diff] [blame] | 460 | if (LocInfo.first.isInvalid()) |
| 461 | return Loc; |
| 462 | |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 463 | bool Invalid = false; |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 464 | StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 465 | if (Invalid) |
| 466 | return Loc; |
| 467 | |
| 468 | // Back up from the current location until we hit the beginning of a line |
| 469 | // (or the buffer). We'll relex from that point. |
| 470 | const char *BufStart = Buffer.data(); |
Douglas Gregor | 86af984 | 2011-01-31 22:42:36 +0000 | [diff] [blame] | 471 | if (LocInfo.second >= Buffer.size()) |
| 472 | return Loc; |
| 473 | |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 474 | const char *StrData = BufStart+LocInfo.second; |
| 475 | if (StrData[0] == '\n' || StrData[0] == '\r') |
| 476 | return Loc; |
| 477 | |
| 478 | const char *LexStart = StrData; |
| 479 | while (LexStart != BufStart) { |
| 480 | if (LexStart[0] == '\n' || LexStart[0] == '\r') { |
| 481 | ++LexStart; |
| 482 | break; |
| 483 | } |
| 484 | |
| 485 | --LexStart; |
| 486 | } |
| 487 | |
| 488 | // Create a lexer starting at the beginning of this token. |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 489 | SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 490 | Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); |
| 491 | TheLexer.SetCommentRetentionState(true); |
| 492 | |
| 493 | // Lex tokens until we find the token that contains the source location. |
| 494 | Token TheTok; |
| 495 | do { |
| 496 | TheLexer.LexFromRawLexer(TheTok); |
| 497 | |
| 498 | if (TheLexer.getBufferLocation() > StrData) { |
| 499 | // Lexing this token has taken the lexer past the source location we're |
| 500 | // looking for. If the current token encompasses our source location, |
| 501 | // return the beginning of that token. |
| 502 | if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) |
| 503 | return TheTok.getLocation(); |
| 504 | |
| 505 | // We ended up skipping over the source location entirely, which means |
| 506 | // that it points into whitespace. We're done here. |
| 507 | break; |
| 508 | } |
| 509 | } while (TheTok.getKind() != tok::eof); |
| 510 | |
| 511 | // We've passed our source location; just return the original source location. |
| 512 | return Loc; |
| 513 | } |
| 514 | |
Argyrios Kyrtzidis | 161868d | 2011-08-17 00:31:23 +0000 | [diff] [blame] | 515 | SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, |
| 516 | const SourceManager &SM, |
| 517 | const LangOptions &LangOpts) { |
| 518 | if (Loc.isFileID()) |
| 519 | return getBeginningOfFileToken(Loc, SM, LangOpts); |
| 520 | |
| 521 | if (!SM.isMacroArgExpansion(Loc)) |
| 522 | return Loc; |
| 523 | |
| 524 | SourceLocation FileLoc = SM.getSpellingLoc(Loc); |
| 525 | SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); |
| 526 | std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); |
Chandler Carruth | 5b15a9b | 2012-01-15 09:03:45 +0000 | [diff] [blame] | 527 | std::pair<FileID, unsigned> BeginFileLocInfo |
| 528 | = SM.getDecomposedLoc(BeginFileLoc); |
Argyrios Kyrtzidis | 161868d | 2011-08-17 00:31:23 +0000 | [diff] [blame] | 529 | assert(FileLocInfo.first == BeginFileLocInfo.first && |
| 530 | FileLocInfo.second >= BeginFileLocInfo.second); |
Chandler Carruth | 5b15a9b | 2012-01-15 09:03:45 +0000 | [diff] [blame] | 531 | return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); |
Argyrios Kyrtzidis | 161868d | 2011-08-17 00:31:23 +0000 | [diff] [blame] | 532 | } |
| 533 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 534 | namespace { |
| 535 | enum PreambleDirectiveKind { |
| 536 | PDK_Skipped, |
| 537 | PDK_StartIf, |
| 538 | PDK_EndIf, |
| 539 | PDK_Unknown |
| 540 | }; |
| 541 | } |
| 542 | |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 543 | std::pair<unsigned, bool> Lexer::ComputePreamble(StringRef Buffer, |
David Blaikie | 3d95d85 | 2014-08-11 22:08:06 +0000 | [diff] [blame] | 544 | const LangOptions &LangOpts, |
| 545 | unsigned MaxLines) { |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 546 | // Create a lexer starting at the beginning of the file. Note that we use a |
| 547 | // "fake" file source location at offset 1 so that the lexer will track our |
| 548 | // position within the file. |
| 549 | const unsigned StartOffset = 1; |
Argyrios Kyrtzidis | d53d0da | 2012-10-25 01:51:45 +0000 | [diff] [blame] | 550 | SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 551 | Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(), |
| 552 | Buffer.end()); |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 553 | TheLexer.SetCommentRetentionState(true); |
Argyrios Kyrtzidis | d53d0da | 2012-10-25 01:51:45 +0000 | [diff] [blame] | 554 | |
| 555 | // StartLoc will differ from FileLoc if there is a BOM that was skipped. |
| 556 | SourceLocation StartLoc = TheLexer.getSourceLocation(); |
| 557 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 558 | bool InPreprocessorDirective = false; |
| 559 | Token TheTok; |
| 560 | Token IfStartTok; |
| 561 | unsigned IfCount = 0; |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 562 | SourceLocation ActiveCommentLoc; |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 563 | |
| 564 | unsigned MaxLineOffset = 0; |
| 565 | if (MaxLines) { |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 566 | const char *CurPtr = Buffer.begin(); |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 567 | unsigned CurLine = 0; |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 568 | while (CurPtr != Buffer.end()) { |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 569 | char ch = *CurPtr++; |
| 570 | if (ch == '\n') { |
| 571 | ++CurLine; |
| 572 | if (CurLine == MaxLines) |
| 573 | break; |
| 574 | } |
| 575 | } |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 576 | if (CurPtr != Buffer.end()) |
| 577 | MaxLineOffset = CurPtr - Buffer.begin(); |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 578 | } |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 579 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 580 | do { |
| 581 | TheLexer.LexFromRawLexer(TheTok); |
| 582 | |
| 583 | if (InPreprocessorDirective) { |
| 584 | // If we've hit the end of the file, we're done. |
| 585 | if (TheTok.getKind() == tok::eof) { |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 586 | break; |
| 587 | } |
| 588 | |
| 589 | // If we haven't hit the end of the preprocessor directive, skip this |
| 590 | // token. |
| 591 | if (!TheTok.isAtStartOfLine()) |
| 592 | continue; |
| 593 | |
| 594 | // We've passed the end of the preprocessor directive, and will look |
| 595 | // at this token again below. |
| 596 | InPreprocessorDirective = false; |
| 597 | } |
| 598 | |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 599 | // Keep track of the # of lines in the preamble. |
| 600 | if (TheTok.isAtStartOfLine()) { |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 601 | unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 602 | |
| 603 | // If we were asked to limit the number of lines in the preamble, |
| 604 | // and we're about to exceed that limit, we're done. |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 605 | if (MaxLineOffset && TokOffset >= MaxLineOffset) |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 606 | break; |
| 607 | } |
| 608 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 609 | // Comments are okay; skip over them. |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 610 | if (TheTok.getKind() == tok::comment) { |
| 611 | if (ActiveCommentLoc.isInvalid()) |
| 612 | ActiveCommentLoc = TheTok.getLocation(); |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 613 | continue; |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 614 | } |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 615 | |
| 616 | if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { |
| 617 | // This is the start of a preprocessor directive. |
| 618 | Token HashTok = TheTok; |
| 619 | InPreprocessorDirective = true; |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 620 | ActiveCommentLoc = SourceLocation(); |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 621 | |
Joerg Sonnenberger | da5d2b7 | 2011-07-20 00:14:37 +0000 | [diff] [blame] | 622 | // Figure out which directive this is. Since we're lexing raw tokens, |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 623 | // we don't have an identifier table available. Instead, just look at |
| 624 | // the raw identifier to recognize and categorize preprocessor directives. |
| 625 | TheLexer.LexFromRawLexer(TheTok); |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 626 | if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { |
Alp Toker | 2d57cea | 2014-05-17 04:53:25 +0000 | [diff] [blame] | 627 | StringRef Keyword = TheTok.getRawIdentifier(); |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 628 | PreambleDirectiveKind PDK |
| 629 | = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) |
| 630 | .Case("include", PDK_Skipped) |
| 631 | .Case("__include_macros", PDK_Skipped) |
| 632 | .Case("define", PDK_Skipped) |
| 633 | .Case("undef", PDK_Skipped) |
| 634 | .Case("line", PDK_Skipped) |
| 635 | .Case("error", PDK_Skipped) |
| 636 | .Case("pragma", PDK_Skipped) |
| 637 | .Case("import", PDK_Skipped) |
| 638 | .Case("include_next", PDK_Skipped) |
| 639 | .Case("warning", PDK_Skipped) |
| 640 | .Case("ident", PDK_Skipped) |
| 641 | .Case("sccs", PDK_Skipped) |
| 642 | .Case("assert", PDK_Skipped) |
| 643 | .Case("unassert", PDK_Skipped) |
| 644 | .Case("if", PDK_StartIf) |
| 645 | .Case("ifdef", PDK_StartIf) |
| 646 | .Case("ifndef", PDK_StartIf) |
| 647 | .Case("elif", PDK_Skipped) |
| 648 | .Case("else", PDK_Skipped) |
| 649 | .Case("endif", PDK_EndIf) |
| 650 | .Default(PDK_Unknown); |
| 651 | |
| 652 | switch (PDK) { |
| 653 | case PDK_Skipped: |
| 654 | continue; |
| 655 | |
| 656 | case PDK_StartIf: |
| 657 | if (IfCount == 0) |
| 658 | IfStartTok = HashTok; |
| 659 | |
| 660 | ++IfCount; |
| 661 | continue; |
| 662 | |
| 663 | case PDK_EndIf: |
| 664 | // Mismatched #endif. The preamble ends here. |
| 665 | if (IfCount == 0) |
| 666 | break; |
| 667 | |
| 668 | --IfCount; |
| 669 | continue; |
| 670 | |
| 671 | case PDK_Unknown: |
| 672 | // We don't know what this directive is; stop at the '#'. |
| 673 | break; |
| 674 | } |
| 675 | } |
| 676 | |
| 677 | // We only end up here if we didn't recognize the preprocessor |
| 678 | // directive or it was one that can't occur in the preamble at this |
| 679 | // point. Roll back the current token to the location of the '#'. |
| 680 | InPreprocessorDirective = false; |
| 681 | TheTok = HashTok; |
| 682 | } |
| 683 | |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 684 | // We hit a token that we don't recognize as being in the |
| 685 | // "preprocessing only" part of the file, so we're no longer in |
| 686 | // the preamble. |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 687 | break; |
| 688 | } while (true); |
| 689 | |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 690 | SourceLocation End; |
| 691 | if (IfCount) |
| 692 | End = IfStartTok.getLocation(); |
| 693 | else if (ActiveCommentLoc.isValid()) |
| 694 | End = ActiveCommentLoc; // don't truncate a decl comment. |
| 695 | else |
| 696 | End = TheTok.getLocation(); |
| 697 | |
Douglas Gregor | 3f4bea0 | 2010-07-26 21:36:20 +0000 | [diff] [blame] | 698 | return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), |
| 699 | IfCount? IfStartTok.isAtStartOfLine() |
| 700 | : TheTok.isAtStartOfLine()); |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 701 | } |
| 702 | |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 703 | |
| 704 | /// AdvanceToTokenCharacter - Given a location that specifies the start of a |
| 705 | /// token, return a new location that specifies a character within the token. |
| 706 | SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, |
| 707 | unsigned CharNo, |
| 708 | const SourceManager &SM, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 709 | const LangOptions &LangOpts) { |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 710 | // Figure out how many physical characters away the specified expansion |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 711 | // character is. This needs to take into consideration newlines and |
| 712 | // trigraphs. |
| 713 | bool Invalid = false; |
| 714 | const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); |
| 715 | |
| 716 | // If they request the first char of the token, we're trivially done. |
| 717 | if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) |
| 718 | return TokStart; |
| 719 | |
| 720 | unsigned PhysOffset = 0; |
| 721 | |
| 722 | // The usual case is that tokens don't contain anything interesting. Skip |
| 723 | // over the uninteresting characters. If a token only consists of simple |
| 724 | // chars, this method is extremely fast. |
| 725 | while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { |
| 726 | if (CharNo == 0) |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 727 | return TokStart.getLocWithOffset(PhysOffset); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 728 | ++TokPtr, --CharNo, ++PhysOffset; |
| 729 | } |
| 730 | |
| 731 | // If we have a character that may be a trigraph or escaped newline, use a |
| 732 | // lexer to parse it correctly. |
| 733 | for (; CharNo; --CharNo) { |
| 734 | unsigned Size; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 735 | Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 736 | TokPtr += Size; |
| 737 | PhysOffset += Size; |
| 738 | } |
| 739 | |
| 740 | // Final detail: if we end up on an escaped newline, we want to return the |
| 741 | // location of the actual byte of the token. For example foo\<newline>bar |
| 742 | // advanced by 3 should return the location of b, not of \\. One compounding |
| 743 | // detail of this is that the escape may be made by a trigraph. |
| 744 | if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) |
| 745 | PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; |
| 746 | |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 747 | return TokStart.getLocWithOffset(PhysOffset); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 748 | } |
| 749 | |
| 750 | /// \brief Computes the source location just past the end of the |
| 751 | /// token at this source location. |
| 752 | /// |
| 753 | /// This routine can be used to produce a source location that |
| 754 | /// points just past the end of the token referenced by \p Loc, and |
| 755 | /// is generally used when a diagnostic needs to point just after a |
| 756 | /// token where it expected something different that it received. If |
| 757 | /// the returned source location would not be meaningful (e.g., if |
| 758 | /// it points into a macro), this routine returns an invalid |
| 759 | /// source location. |
| 760 | /// |
| 761 | /// \param Offset an offset from the end of the token, where the source |
| 762 | /// location should refer to. The default offset (0) produces a source |
| 763 | /// location pointing just past the end of the token; an offset of 1 produces |
| 764 | /// a source location pointing to the last character in the token, etc. |
| 765 | SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, |
| 766 | const SourceManager &SM, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 767 | const LangOptions &LangOpts) { |
Argyrios Kyrtzidis | 2cfce18 | 2011-06-24 17:58:59 +0000 | [diff] [blame] | 768 | if (Loc.isInvalid()) |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 769 | return SourceLocation(); |
Argyrios Kyrtzidis | 2cfce18 | 2011-06-24 17:58:59 +0000 | [diff] [blame] | 770 | |
| 771 | if (Loc.isMacroID()) { |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 772 | if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 773 | return SourceLocation(); // Points inside the macro expansion. |
Argyrios Kyrtzidis | 2cfce18 | 2011-06-24 17:58:59 +0000 | [diff] [blame] | 774 | } |
| 775 | |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 776 | unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 777 | if (Len > Offset) |
| 778 | Len = Len - Offset; |
| 779 | else |
| 780 | return Loc; |
| 781 | |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 782 | return Loc.getLocWithOffset(Len); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 783 | } |
| 784 | |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 785 | /// \brief Returns true if the given MacroID location points at the first |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 786 | /// token of the macro expansion. |
| 787 | bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, |
Douglas Gregor | 925296b | 2011-07-19 16:10:42 +0000 | [diff] [blame] | 788 | const SourceManager &SM, |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 789 | const LangOptions &LangOpts, |
| 790 | SourceLocation *MacroBegin) { |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 791 | assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); |
| 792 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 793 | SourceLocation expansionLoc; |
| 794 | if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc)) |
| 795 | return false; |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 796 | |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 797 | if (expansionLoc.isFileID()) { |
| 798 | // No other macro expansions, this is the first. |
| 799 | if (MacroBegin) |
| 800 | *MacroBegin = expansionLoc; |
| 801 | return true; |
| 802 | } |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 803 | |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 804 | return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 805 | } |
| 806 | |
| 807 | /// \brief Returns true if the given MacroID location points at the last |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 808 | /// token of the macro expansion. |
| 809 | bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 810 | const SourceManager &SM, |
| 811 | const LangOptions &LangOpts, |
| 812 | SourceLocation *MacroEnd) { |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 813 | assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); |
| 814 | |
| 815 | SourceLocation spellLoc = SM.getSpellingLoc(loc); |
| 816 | unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); |
| 817 | if (tokLen == 0) |
| 818 | return false; |
| 819 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 820 | SourceLocation afterLoc = loc.getLocWithOffset(tokLen); |
| 821 | SourceLocation expansionLoc; |
| 822 | if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc)) |
| 823 | return false; |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 824 | |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 825 | if (expansionLoc.isFileID()) { |
| 826 | // No other macro expansions. |
| 827 | if (MacroEnd) |
| 828 | *MacroEnd = expansionLoc; |
| 829 | return true; |
| 830 | } |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 831 | |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 832 | return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 833 | } |
| 834 | |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 835 | static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 836 | const SourceManager &SM, |
| 837 | const LangOptions &LangOpts) { |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 838 | SourceLocation Begin = Range.getBegin(); |
| 839 | SourceLocation End = Range.getEnd(); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 840 | assert(Begin.isFileID() && End.isFileID()); |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 841 | if (Range.isTokenRange()) { |
| 842 | End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); |
| 843 | if (End.isInvalid()) |
| 844 | return CharSourceRange(); |
| 845 | } |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 846 | |
| 847 | // Break down the source locations. |
| 848 | FileID FID; |
| 849 | unsigned BeginOffs; |
Benjamin Kramer | 867ea1d | 2014-03-02 13:01:17 +0000 | [diff] [blame] | 850 | std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 851 | if (FID.isInvalid()) |
| 852 | return CharSourceRange(); |
| 853 | |
| 854 | unsigned EndOffs; |
| 855 | if (!SM.isInFileID(End, FID, &EndOffs) || |
| 856 | BeginOffs > EndOffs) |
| 857 | return CharSourceRange(); |
| 858 | |
| 859 | return CharSourceRange::getCharRange(Begin, End); |
| 860 | } |
| 861 | |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 862 | CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 863 | const SourceManager &SM, |
| 864 | const LangOptions &LangOpts) { |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 865 | SourceLocation Begin = Range.getBegin(); |
| 866 | SourceLocation End = Range.getEnd(); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 867 | if (Begin.isInvalid() || End.isInvalid()) |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 868 | return CharSourceRange(); |
| 869 | |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 870 | if (Begin.isFileID() && End.isFileID()) |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 871 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 872 | |
| 873 | if (Begin.isMacroID() && End.isFileID()) { |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 874 | if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) |
| 875 | return CharSourceRange(); |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 876 | Range.setBegin(Begin); |
| 877 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 878 | } |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 879 | |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 880 | if (Begin.isFileID() && End.isMacroID()) { |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 881 | if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, |
| 882 | &End)) || |
| 883 | (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, |
| 884 | &End))) |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 885 | return CharSourceRange(); |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 886 | Range.setEnd(End); |
| 887 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 888 | } |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 889 | |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 890 | assert(Begin.isMacroID() && End.isMacroID()); |
| 891 | SourceLocation MacroBegin, MacroEnd; |
| 892 | if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 893 | ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, |
| 894 | &MacroEnd)) || |
| 895 | (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, |
| 896 | &MacroEnd)))) { |
| 897 | Range.setBegin(MacroBegin); |
| 898 | Range.setEnd(MacroEnd); |
| 899 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
| 900 | } |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 901 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 902 | bool Invalid = false; |
| 903 | const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin), |
| 904 | &Invalid); |
| 905 | if (Invalid) |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 906 | return CharSourceRange(); |
| 907 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 908 | if (BeginEntry.getExpansion().isMacroArgExpansion()) { |
| 909 | const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End), |
| 910 | &Invalid); |
| 911 | if (Invalid) |
| 912 | return CharSourceRange(); |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 913 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 914 | if (EndEntry.getExpansion().isMacroArgExpansion() && |
| 915 | BeginEntry.getExpansion().getExpansionLocStart() == |
| 916 | EndEntry.getExpansion().getExpansionLocStart()) { |
| 917 | Range.setBegin(SM.getImmediateSpellingLoc(Begin)); |
| 918 | Range.setEnd(SM.getImmediateSpellingLoc(End)); |
| 919 | return makeFileCharRange(Range, SM, LangOpts); |
| 920 | } |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 921 | } |
| 922 | |
| 923 | return CharSourceRange(); |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 924 | } |
| 925 | |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 926 | StringRef Lexer::getSourceText(CharSourceRange Range, |
| 927 | const SourceManager &SM, |
| 928 | const LangOptions &LangOpts, |
| 929 | bool *Invalid) { |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 930 | Range = makeFileCharRange(Range, SM, LangOpts); |
| 931 | if (Range.isInvalid()) { |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 932 | if (Invalid) *Invalid = true; |
| 933 | return StringRef(); |
| 934 | } |
| 935 | |
| 936 | // Break down the source location. |
| 937 | std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); |
| 938 | if (beginInfo.first.isInvalid()) { |
| 939 | if (Invalid) *Invalid = true; |
| 940 | return StringRef(); |
| 941 | } |
| 942 | |
| 943 | unsigned EndOffs; |
| 944 | if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || |
| 945 | beginInfo.second > EndOffs) { |
| 946 | if (Invalid) *Invalid = true; |
| 947 | return StringRef(); |
| 948 | } |
| 949 | |
| 950 | // Try to the load the file buffer. |
| 951 | bool invalidTemp = false; |
| 952 | StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); |
| 953 | if (invalidTemp) { |
| 954 | if (Invalid) *Invalid = true; |
| 955 | return StringRef(); |
| 956 | } |
| 957 | |
| 958 | if (Invalid) *Invalid = false; |
| 959 | return file.substr(beginInfo.second, EndOffs - beginInfo.second); |
| 960 | } |
| 961 | |
Anna Zaks | 1bea4bf | 2012-01-18 20:17:16 +0000 | [diff] [blame] | 962 | StringRef Lexer::getImmediateMacroName(SourceLocation Loc, |
| 963 | const SourceManager &SM, |
| 964 | const LangOptions &LangOpts) { |
| 965 | assert(Loc.isMacroID() && "Only reasonble to call this on macros"); |
Argyrios Kyrtzidis | abff5f1 | 2012-01-23 16:58:33 +0000 | [diff] [blame] | 966 | |
| 967 | // Find the location of the immediate macro expansion. |
| 968 | while (1) { |
| 969 | FileID FID = SM.getFileID(Loc); |
| 970 | const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); |
| 971 | const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); |
| 972 | Loc = Expansion.getExpansionLocStart(); |
| 973 | if (!Expansion.isMacroArgExpansion()) |
| 974 | break; |
| 975 | |
| 976 | // For macro arguments we need to check that the argument did not come |
| 977 | // from an inner macro, e.g: "MAC1( MAC2(foo) )" |
| 978 | |
| 979 | // Loc points to the argument id of the macro definition, move to the |
| 980 | // macro expansion. |
Anna Zaks | 1bea4bf | 2012-01-18 20:17:16 +0000 | [diff] [blame] | 981 | Loc = SM.getImmediateExpansionRange(Loc).first; |
Argyrios Kyrtzidis | abff5f1 | 2012-01-23 16:58:33 +0000 | [diff] [blame] | 982 | SourceLocation SpellLoc = Expansion.getSpellingLoc(); |
| 983 | if (SpellLoc.isFileID()) |
| 984 | break; // No inner macro. |
| 985 | |
| 986 | // If spelling location resides in the same FileID as macro expansion |
| 987 | // location, it means there is no inner macro. |
| 988 | FileID MacroFID = SM.getFileID(Loc); |
| 989 | if (SM.isInFileID(SpellLoc, MacroFID)) |
| 990 | break; |
| 991 | |
| 992 | // Argument came from inner macro. |
| 993 | Loc = SpellLoc; |
| 994 | } |
Anna Zaks | 1bea4bf | 2012-01-18 20:17:16 +0000 | [diff] [blame] | 995 | |
| 996 | // Find the spelling location of the start of the non-argument expansion |
| 997 | // range. This is where the macro name was spelled in order to begin |
| 998 | // expanding this macro. |
Argyrios Kyrtzidis | abff5f1 | 2012-01-23 16:58:33 +0000 | [diff] [blame] | 999 | Loc = SM.getSpellingLoc(Loc); |
Anna Zaks | 1bea4bf | 2012-01-18 20:17:16 +0000 | [diff] [blame] | 1000 | |
| 1001 | // Dig out the buffer where the macro name was spelled and the extents of the |
| 1002 | // name so that we can render it into the expansion note. |
| 1003 | std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); |
| 1004 | unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); |
| 1005 | StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); |
| 1006 | return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); |
| 1007 | } |
| 1008 | |
Jordan Rose | 288c421 | 2012-06-07 01:10:31 +0000 | [diff] [blame] | 1009 | bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { |
Jordan Rose | a2100d7 | 2013-02-08 22:30:22 +0000 | [diff] [blame] | 1010 | return isIdentifierBody(c, LangOpts.DollarIdents); |
Jordan Rose | 288c421 | 2012-06-07 01:10:31 +0000 | [diff] [blame] | 1011 | } |
| 1012 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1013 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1014 | //===----------------------------------------------------------------------===// |
| 1015 | // Diagnostics forwarding code. |
| 1016 | //===----------------------------------------------------------------------===// |
| 1017 | |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1018 | /// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 1019 | /// lexer buffer was all expanded at a single point, perform the mapping. |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1020 | /// This is currently only used for _Pragma implementation, so it is the slow |
| 1021 | /// path of the hot getSourceLocation method. Do not allow it to be inlined. |
Chandler Carruth | c3ce584 | 2010-10-23 08:44:57 +0000 | [diff] [blame] | 1022 | static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( |
| 1023 | Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1024 | static SourceLocation GetMappedTokenLoc(Preprocessor &PP, |
| 1025 | SourceLocation FileLoc, |
Chris Lattner | 4fa2362 | 2009-01-26 00:43:02 +0000 | [diff] [blame] | 1026 | unsigned CharNo, unsigned TokLen) { |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 1027 | assert(FileLoc.isMacroID() && "Must be a macro expansion"); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1028 | |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1029 | // Otherwise, we're lexing "mapped tokens". This is used for things like |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 1030 | // _Pragma handling. Combine the expansion location of FileLoc with the |
Chris Lattner | 53e384f | 2009-01-16 07:00:02 +0000 | [diff] [blame] | 1031 | // spelling location. |
Chris Lattner | 9dc9c20 | 2009-02-15 20:52:18 +0000 | [diff] [blame] | 1032 | SourceManager &SM = PP.getSourceManager(); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1033 | |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 1034 | // Create a new SLoc which is expanded from Expansion(FileLoc) but whose |
Chris Lattner | 53e384f | 2009-01-16 07:00:02 +0000 | [diff] [blame] | 1035 | // characters come from spelling(FileLoc)+Offset. |
Chris Lattner | 9dc9c20 | 2009-02-15 20:52:18 +0000 | [diff] [blame] | 1036 | SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 1037 | SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1038 | |
Chris Lattner | 9dc9c20 | 2009-02-15 20:52:18 +0000 | [diff] [blame] | 1039 | // Figure out the expansion loc range, which is the range covered by the |
| 1040 | // original _Pragma(...) sequence. |
| 1041 | std::pair<SourceLocation,SourceLocation> II = |
Chandler Carruth | ca75758 | 2011-07-25 20:52:21 +0000 | [diff] [blame] | 1042 | SM.getImmediateExpansionRange(FileLoc); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1043 | |
Chandler Carruth | 115b077 | 2011-07-26 03:03:05 +0000 | [diff] [blame] | 1044 | return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1045 | } |
| 1046 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1047 | /// getSourceLocation - Return a source location identifier for the specified |
| 1048 | /// offset in the current file. |
Chris Lattner | 4fa2362 | 2009-01-26 00:43:02 +0000 | [diff] [blame] | 1049 | SourceLocation Lexer::getSourceLocation(const char *Loc, |
| 1050 | unsigned TokLen) const { |
Chris Lattner | 5d1c027 | 2007-07-22 18:44:36 +0000 | [diff] [blame] | 1051 | assert(Loc >= BufferStart && Loc <= BufferEnd && |
Chris Lattner | 4cca5ba | 2006-07-02 20:05:54 +0000 | [diff] [blame] | 1052 | "Location out of range for this buffer!"); |
Chris Lattner | dc5c055 | 2007-07-20 16:37:10 +0000 | [diff] [blame] | 1053 | |
| 1054 | // In the normal case, we're just lexing from a simple file buffer, return |
| 1055 | // the file id from FileLoc with the offset specified. |
Chris Lattner | 5d1c027 | 2007-07-22 18:44:36 +0000 | [diff] [blame] | 1056 | unsigned CharNo = Loc-BufferStart; |
Chris Lattner | dc5c055 | 2007-07-20 16:37:10 +0000 | [diff] [blame] | 1057 | if (FileLoc.isFileID()) |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 1058 | return FileLoc.getLocWithOffset(CharNo); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1059 | |
Chris Lattner | d32480d | 2009-01-17 06:22:33 +0000 | [diff] [blame] | 1060 | // Otherwise, this is the _Pragma lexer case, which pretends that all of the |
| 1061 | // tokens are lexed from where the _Pragma was defined. |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 1062 | assert(PP && "This doesn't work on raw lexers"); |
Chris Lattner | 4fa2362 | 2009-01-26 00:43:02 +0000 | [diff] [blame] | 1063 | return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1064 | } |
| 1065 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1066 | /// Diag - Forwarding function for diagnostics. This translate a source |
| 1067 | /// position in the current buffer into a SourceLocation object for rendering. |
Chris Lattner | 427c9c1 | 2008-11-22 00:59:29 +0000 | [diff] [blame] | 1068 | DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { |
Chris Lattner | 907dfe9 | 2008-11-18 07:59:24 +0000 | [diff] [blame] | 1069 | return PP->Diag(getSourceLocation(Loc), DiagID); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1070 | } |
| 1071 | |
| 1072 | //===----------------------------------------------------------------------===// |
| 1073 | // Trigraph and Escaped Newline Handling Code. |
| 1074 | //===----------------------------------------------------------------------===// |
| 1075 | |
| 1076 | /// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, |
| 1077 | /// return the decoded trigraph letter it corresponds to, or '\0' if nothing. |
| 1078 | static char GetTrigraphCharForLetter(char Letter) { |
| 1079 | switch (Letter) { |
| 1080 | default: return 0; |
| 1081 | case '=': return '#'; |
| 1082 | case ')': return ']'; |
| 1083 | case '(': return '['; |
| 1084 | case '!': return '|'; |
| 1085 | case '\'': return '^'; |
| 1086 | case '>': return '}'; |
| 1087 | case '/': return '\\'; |
| 1088 | case '<': return '{'; |
| 1089 | case '-': return '~'; |
| 1090 | } |
| 1091 | } |
| 1092 | |
| 1093 | /// DecodeTrigraphChar - If the specified character is a legal trigraph when |
| 1094 | /// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, |
| 1095 | /// return the result character. Finally, emit a warning about trigraph use |
| 1096 | /// whether trigraphs are enabled or not. |
| 1097 | static char DecodeTrigraphChar(const char *CP, Lexer *L) { |
| 1098 | char Res = GetTrigraphCharForLetter(*CP); |
Chris Lattner | 907dfe9 | 2008-11-18 07:59:24 +0000 | [diff] [blame] | 1099 | if (!Res || !L) return Res; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1100 | |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1101 | if (!L->getLangOpts().Trigraphs) { |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1102 | if (!L->isLexingRawMode()) |
| 1103 | L->Diag(CP-2, diag::trigraph_ignored); |
Chris Lattner | 907dfe9 | 2008-11-18 07:59:24 +0000 | [diff] [blame] | 1104 | return 0; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1105 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1106 | |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1107 | if (!L->isLexingRawMode()) |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 1108 | L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1109 | return Res; |
| 1110 | } |
| 1111 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1112 | /// getEscapedNewLineSize - Return the size of the specified escaped newline, |
| 1113 | /// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1114 | /// trigraph equivalent on entry to this function. |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1115 | unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { |
| 1116 | unsigned Size = 0; |
| 1117 | while (isWhitespace(Ptr[Size])) { |
| 1118 | ++Size; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1119 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1120 | if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') |
| 1121 | continue; |
| 1122 | |
| 1123 | // If this is a \r\n or \n\r, skip the other half. |
| 1124 | if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && |
| 1125 | Ptr[Size-1] != Ptr[Size]) |
| 1126 | ++Size; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1127 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1128 | return Size; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1129 | } |
| 1130 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1131 | // Not an escaped newline, must be a \t or something else. |
| 1132 | return 0; |
| 1133 | } |
| 1134 | |
Chris Lattner | 38b2cde | 2009-04-18 22:27:02 +0000 | [diff] [blame] | 1135 | /// SkipEscapedNewLines - If P points to an escaped newline (or a series of |
| 1136 | /// them), skip over them and return the first non-escaped-newline found, |
| 1137 | /// otherwise return P. |
| 1138 | const char *Lexer::SkipEscapedNewLines(const char *P) { |
| 1139 | while (1) { |
| 1140 | const char *AfterEscape; |
| 1141 | if (*P == '\\') { |
| 1142 | AfterEscape = P+1; |
| 1143 | } else if (*P == '?') { |
| 1144 | // If not a trigraph for escape, bail out. |
| 1145 | if (P[1] != '?' || P[2] != '/') |
| 1146 | return P; |
| 1147 | AfterEscape = P+3; |
| 1148 | } else { |
| 1149 | return P; |
| 1150 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1151 | |
Chris Lattner | 38b2cde | 2009-04-18 22:27:02 +0000 | [diff] [blame] | 1152 | unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); |
| 1153 | if (NewLineSize == 0) return P; |
| 1154 | P = AfterEscape+NewLineSize; |
| 1155 | } |
| 1156 | } |
| 1157 | |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1158 | /// \brief Checks that the given token is the first token that occurs after the |
| 1159 | /// given location (this excludes comments and whitespace). Returns the location |
| 1160 | /// immediately after the specified token. If the token is not found or the |
| 1161 | /// location is inside a macro, the returned source location will be invalid. |
| 1162 | SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, |
| 1163 | tok::TokenKind TKind, |
| 1164 | const SourceManager &SM, |
| 1165 | const LangOptions &LangOpts, |
| 1166 | bool SkipTrailingWhitespaceAndNewLine) { |
| 1167 | if (Loc.isMacroID()) { |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 1168 | if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1169 | return SourceLocation(); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1170 | } |
| 1171 | Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); |
| 1172 | |
| 1173 | // Break down the source location. |
| 1174 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
| 1175 | |
| 1176 | // Try to load the file buffer. |
| 1177 | bool InvalidTemp = false; |
Dmitri Gribenko | f857950 | 2013-01-12 19:30:44 +0000 | [diff] [blame] | 1178 | StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1179 | if (InvalidTemp) |
| 1180 | return SourceLocation(); |
| 1181 | |
| 1182 | const char *TokenBegin = File.data() + LocInfo.second; |
| 1183 | |
| 1184 | // Lex from the start of the given location. |
| 1185 | Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), |
| 1186 | TokenBegin, File.end()); |
| 1187 | // Find the token. |
| 1188 | Token Tok; |
| 1189 | lexer.LexFromRawLexer(Tok); |
| 1190 | if (Tok.isNot(TKind)) |
| 1191 | return SourceLocation(); |
| 1192 | SourceLocation TokenLoc = Tok.getLocation(); |
| 1193 | |
| 1194 | // Calculate how much whitespace needs to be skipped if any. |
| 1195 | unsigned NumWhitespaceChars = 0; |
| 1196 | if (SkipTrailingWhitespaceAndNewLine) { |
| 1197 | const char *TokenEnd = SM.getCharacterData(TokenLoc) + |
| 1198 | Tok.getLength(); |
| 1199 | unsigned char C = *TokenEnd; |
| 1200 | while (isHorizontalWhitespace(C)) { |
| 1201 | C = *(++TokenEnd); |
| 1202 | NumWhitespaceChars++; |
| 1203 | } |
Eli Friedman | b699e61 | 2012-11-14 01:28:38 +0000 | [diff] [blame] | 1204 | |
| 1205 | // Skip \r, \n, \r\n, or \n\r |
| 1206 | if (C == '\n' || C == '\r') { |
| 1207 | char PrevC = C; |
| 1208 | C = *(++TokenEnd); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1209 | NumWhitespaceChars++; |
Eli Friedman | b699e61 | 2012-11-14 01:28:38 +0000 | [diff] [blame] | 1210 | if ((C == '\n' || C == '\r') && C != PrevC) |
| 1211 | NumWhitespaceChars++; |
| 1212 | } |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1213 | } |
| 1214 | |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 1215 | return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1216 | } |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1217 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1218 | /// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, |
| 1219 | /// get its size, and return it. This is tricky in several cases: |
| 1220 | /// 1. If currently at the start of a trigraph, we warn about the trigraph, |
| 1221 | /// then either return the trigraph (skipping 3 chars) or the '?', |
| 1222 | /// depending on whether trigraphs are enabled or not. |
| 1223 | /// 2. If this is an escaped newline (potentially with whitespace between |
| 1224 | /// the backslash and newline), implicitly skip the newline and return |
| 1225 | /// the char after it. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1226 | /// |
| 1227 | /// This handles the slow/uncommon case of the getCharAndSize method. Here we |
| 1228 | /// know that we can accumulate into Size, and that we have already incremented |
| 1229 | /// Ptr by Size bytes. |
| 1230 | /// |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1231 | /// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should |
| 1232 | /// be updated to match. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1233 | /// |
| 1234 | char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 1235 | Token *Tok) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1236 | // If we have a slash, look for an escaped newline. |
| 1237 | if (Ptr[0] == '\\') { |
| 1238 | ++Size; |
| 1239 | ++Ptr; |
| 1240 | Slash: |
| 1241 | // Common case, backslash-char where the char is not whitespace. |
| 1242 | if (!isWhitespace(Ptr[0])) return '\\'; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1243 | |
Chris Lattner | c183595 | 2009-06-23 05:15:06 +0000 | [diff] [blame] | 1244 | // See if we have optional whitespace characters between the slash and |
| 1245 | // newline. |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1246 | if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { |
| 1247 | // Remember that this token needs to be cleaned. |
| 1248 | if (Tok) Tok->setFlag(Token::NeedsCleaning); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1249 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1250 | // Warn if there was whitespace between the backslash and newline. |
Chris Lattner | c183595 | 2009-06-23 05:15:06 +0000 | [diff] [blame] | 1251 | if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1252 | Diag(Ptr, diag::backslash_newline_space); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1253 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1254 | // Found backslash<whitespace><newline>. Parse the char after it. |
| 1255 | Size += EscapedNewLineSize; |
| 1256 | Ptr += EscapedNewLineSize; |
Argyrios Kyrtzidis | e5cdd08 | 2011-12-21 20:19:55 +0000 | [diff] [blame] | 1257 | |
Argyrios Kyrtzidis | 8a26c4d | 2011-12-22 04:38:07 +0000 | [diff] [blame] | 1258 | // If the char that we finally got was a \n, then we must have had |
| 1259 | // something like \<newline><newline>. We don't want to consume the |
| 1260 | // second newline. |
| 1261 | if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') |
| 1262 | return ' '; |
Argyrios Kyrtzidis | e5cdd08 | 2011-12-21 20:19:55 +0000 | [diff] [blame] | 1263 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1264 | // Use slow version to accumulate a correct size field. |
| 1265 | return getCharAndSizeSlow(Ptr, Size, Tok); |
| 1266 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1267 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1268 | // Otherwise, this is not an escaped newline, just return the slash. |
| 1269 | return '\\'; |
| 1270 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1271 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1272 | // If this is a trigraph, process it. |
| 1273 | if (Ptr[0] == '?' && Ptr[1] == '?') { |
| 1274 | // If this is actually a legal trigraph (not something like "??x"), emit |
| 1275 | // a trigraph warning. If so, and if trigraphs are enabled, return it. |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1276 | if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : nullptr)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1277 | // Remember that this token needs to be cleaned. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 1278 | if (Tok) Tok->setFlag(Token::NeedsCleaning); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1279 | |
| 1280 | Ptr += 3; |
| 1281 | Size += 3; |
| 1282 | if (C == '\\') goto Slash; |
| 1283 | return C; |
| 1284 | } |
| 1285 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1286 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1287 | // If this is neither, return a single character. |
| 1288 | ++Size; |
| 1289 | return *Ptr; |
| 1290 | } |
| 1291 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1292 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1293 | /// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the |
| 1294 | /// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, |
| 1295 | /// and that we have already incremented Ptr by Size bytes. |
| 1296 | /// |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1297 | /// NOTE: When this method is updated, getCharAndSizeSlow (above) should |
| 1298 | /// be updated to match. |
| 1299 | char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1300 | const LangOptions &LangOpts) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1301 | // If we have a slash, look for an escaped newline. |
| 1302 | if (Ptr[0] == '\\') { |
| 1303 | ++Size; |
| 1304 | ++Ptr; |
| 1305 | Slash: |
| 1306 | // Common case, backslash-char where the char is not whitespace. |
| 1307 | if (!isWhitespace(Ptr[0])) return '\\'; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1308 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1309 | // See if we have optional whitespace characters followed by a newline. |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1310 | if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { |
| 1311 | // Found backslash<whitespace><newline>. Parse the char after it. |
| 1312 | Size += EscapedNewLineSize; |
| 1313 | Ptr += EscapedNewLineSize; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1314 | |
Argyrios Kyrtzidis | 8a26c4d | 2011-12-22 04:38:07 +0000 | [diff] [blame] | 1315 | // If the char that we finally got was a \n, then we must have had |
| 1316 | // something like \<newline><newline>. We don't want to consume the |
| 1317 | // second newline. |
| 1318 | if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') |
| 1319 | return ' '; |
Argyrios Kyrtzidis | e5cdd08 | 2011-12-21 20:19:55 +0000 | [diff] [blame] | 1320 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1321 | // Use slow version to accumulate a correct size field. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1322 | return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1323 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1324 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1325 | // Otherwise, this is not an escaped newline, just return the slash. |
| 1326 | return '\\'; |
| 1327 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1328 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1329 | // If this is a trigraph, process it. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1330 | if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1331 | // If this is actually a legal trigraph (not something like "??x"), return |
| 1332 | // it. |
| 1333 | if (char C = GetTrigraphCharForLetter(Ptr[2])) { |
| 1334 | Ptr += 3; |
| 1335 | Size += 3; |
| 1336 | if (C == '\\') goto Slash; |
| 1337 | return C; |
| 1338 | } |
| 1339 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1340 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1341 | // If this is neither, return a single character. |
| 1342 | ++Size; |
| 1343 | return *Ptr; |
| 1344 | } |
| 1345 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1346 | //===----------------------------------------------------------------------===// |
| 1347 | // Helper methods for lexing. |
| 1348 | //===----------------------------------------------------------------------===// |
| 1349 | |
Douglas Gregor | 3f4bea0 | 2010-07-26 21:36:20 +0000 | [diff] [blame] | 1350 | /// \brief Routine that indiscriminately skips bytes in the source file. |
| 1351 | void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { |
| 1352 | BufferPtr += Bytes; |
| 1353 | if (BufferPtr > BufferEnd) |
| 1354 | BufferPtr = BufferEnd; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1355 | // FIXME: What exactly does the StartOfLine bit mean? There are two |
| 1356 | // possible meanings for the "start" of the line: the first token on the |
| 1357 | // unexpanded line, or the first token on the expanded line. |
Douglas Gregor | 3f4bea0 | 2010-07-26 21:36:20 +0000 | [diff] [blame] | 1358 | IsAtStartOfLine = StartOfLine; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1359 | IsAtPhysicalStartOfLine = StartOfLine; |
Douglas Gregor | 3f4bea0 | 2010-07-26 21:36:20 +0000 | [diff] [blame] | 1360 | } |
| 1361 | |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1362 | static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1363 | if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
| 1364 | static const llvm::sys::UnicodeCharSet C11AllowedIDChars( |
| 1365 | C11AllowedIDCharRanges); |
| 1366 | return C11AllowedIDChars.contains(C); |
| 1367 | } else if (LangOpts.CPlusPlus) { |
| 1368 | static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( |
| 1369 | CXX03AllowedIDCharRanges); |
| 1370 | return CXX03AllowedIDChars.contains(C); |
| 1371 | } else { |
| 1372 | static const llvm::sys::UnicodeCharSet C99AllowedIDChars( |
| 1373 | C99AllowedIDCharRanges); |
| 1374 | return C99AllowedIDChars.contains(C); |
| 1375 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1376 | } |
| 1377 | |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1378 | static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { |
| 1379 | assert(isAllowedIDChar(C, LangOpts)); |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1380 | if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
| 1381 | static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars( |
| 1382 | C11DisallowedInitialIDCharRanges); |
| 1383 | return !C11DisallowedInitialIDChars.contains(C); |
| 1384 | } else if (LangOpts.CPlusPlus) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1385 | return true; |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1386 | } else { |
| 1387 | static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( |
| 1388 | C99DisallowedInitialIDCharRanges); |
| 1389 | return !C99DisallowedInitialIDChars.contains(C); |
| 1390 | } |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1391 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1392 | |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1393 | static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, |
| 1394 | const char *End) { |
| 1395 | return CharSourceRange::getCharRange(L.getSourceLocation(Begin), |
| 1396 | L.getSourceLocation(End)); |
| 1397 | } |
| 1398 | |
| 1399 | static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, |
| 1400 | CharSourceRange Range, bool IsFirst) { |
| 1401 | // Check C99 compatibility. |
Alp Toker | d4a3f0e | 2014-06-15 23:30:39 +0000 | [diff] [blame] | 1402 | if (!Diags.isIgnored(diag::warn_c99_compat_unicode_id, Range.getBegin())) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1403 | enum { |
| 1404 | CannotAppearInIdentifier = 0, |
| 1405 | CannotStartIdentifier |
| 1406 | }; |
| 1407 | |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1408 | static const llvm::sys::UnicodeCharSet C99AllowedIDChars( |
| 1409 | C99AllowedIDCharRanges); |
| 1410 | static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( |
| 1411 | C99DisallowedInitialIDCharRanges); |
| 1412 | if (!C99AllowedIDChars.contains(C)) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1413 | Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) |
| 1414 | << Range |
| 1415 | << CannotAppearInIdentifier; |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1416 | } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1417 | Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) |
| 1418 | << Range |
| 1419 | << CannotStartIdentifier; |
| 1420 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1421 | } |
| 1422 | |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1423 | // Check C++98 compatibility. |
Alp Toker | d4a3f0e | 2014-06-15 23:30:39 +0000 | [diff] [blame] | 1424 | if (!Diags.isIgnored(diag::warn_cxx98_compat_unicode_id, Range.getBegin())) { |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1425 | static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( |
| 1426 | CXX03AllowedIDCharRanges); |
| 1427 | if (!CXX03AllowedIDChars.contains(C)) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1428 | Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) |
| 1429 | << Range; |
| 1430 | } |
| 1431 | } |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1432 | } |
| 1433 | |
| 1434 | bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size, |
| 1435 | Token &Result) { |
| 1436 | const char *UCNPtr = CurPtr + Size; |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1437 | uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/nullptr); |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1438 | if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) |
| 1439 | return false; |
| 1440 | |
| 1441 | if (!isLexingRawMode()) |
| 1442 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, |
| 1443 | makeCharRange(*this, CurPtr, UCNPtr), |
| 1444 | /*IsFirst=*/false); |
| 1445 | |
| 1446 | Result.setFlag(Token::HasUCN); |
| 1447 | if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || |
| 1448 | (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) |
| 1449 | CurPtr = UCNPtr; |
| 1450 | else |
| 1451 | while (CurPtr != UCNPtr) |
| 1452 | (void)getAndAdvanceChar(CurPtr, Result); |
| 1453 | return true; |
| 1454 | } |
| 1455 | |
| 1456 | bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) { |
| 1457 | const char *UnicodePtr = CurPtr; |
| 1458 | UTF32 CodePoint; |
| 1459 | ConversionResult Result = |
| 1460 | llvm::convertUTF8Sequence((const UTF8 **)&UnicodePtr, |
| 1461 | (const UTF8 *)BufferEnd, |
| 1462 | &CodePoint, |
| 1463 | strictConversion); |
| 1464 | if (Result != conversionOK || |
| 1465 | !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) |
| 1466 | return false; |
| 1467 | |
| 1468 | if (!isLexingRawMode()) |
| 1469 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, |
| 1470 | makeCharRange(*this, CurPtr, UnicodePtr), |
| 1471 | /*IsFirst=*/false); |
| 1472 | |
| 1473 | CurPtr = UnicodePtr; |
| 1474 | return true; |
| 1475 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1476 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1477 | bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1478 | // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] |
| 1479 | unsigned Size; |
| 1480 | unsigned char C = *CurPtr++; |
Chris Lattner | 21d9b9a | 2010-01-11 02:38:50 +0000 | [diff] [blame] | 1481 | while (isIdentifierBody(C)) |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1482 | C = *CurPtr++; |
Chris Lattner | 21d9b9a | 2010-01-11 02:38:50 +0000 | [diff] [blame] | 1483 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1484 | --CurPtr; // Back up over the skipped character. |
| 1485 | |
| 1486 | // Fast path, no $,\,? in identifier found. '\' might be an escaped newline |
| 1487 | // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. |
Chris Lattner | 21d9b9a | 2010-01-11 02:38:50 +0000 | [diff] [blame] | 1488 | // |
Jordan Rose | a2100d7 | 2013-02-08 22:30:22 +0000 | [diff] [blame] | 1489 | // TODO: Could merge these checks into an InfoTable flag to make the |
| 1490 | // comparison cheaper |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1491 | if (isASCII(C) && C != '\\' && C != '?' && |
| 1492 | (C != '$' || !LangOpts.DollarIdents)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1493 | FinishIdentifier: |
Chris Lattner | cefc768 | 2006-07-08 08:28:12 +0000 | [diff] [blame] | 1494 | const char *IdStart = BufferPtr; |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 1495 | FormTokenWithChars(Result, CurPtr, tok::raw_identifier); |
| 1496 | Result.setRawIdentifierData(IdStart); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1497 | |
Chris Lattner | 0f1f505 | 2006-07-20 04:16:23 +0000 | [diff] [blame] | 1498 | // If we are in raw mode, return this identifier raw. There is no need to |
| 1499 | // look up identifier information or attempt to macro expand it. |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 1500 | if (LexingRawMode) |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1501 | return true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1502 | |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 1503 | // Fill in Result.IdentifierInfo and update the token kind, |
| 1504 | // looking up the identifier in the identifier table. |
| 1505 | IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1506 | |
Chris Lattner | c5a0006 | 2006-06-18 16:41:01 +0000 | [diff] [blame] | 1507 | // Finally, now that we know we have an identifier, pass this off to the |
| 1508 | // preprocessor, which may macro expand it or something. |
Chris Lattner | 8256b97 | 2009-01-21 07:45:14 +0000 | [diff] [blame] | 1509 | if (II->isHandleIdentifierCase()) |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1510 | return PP->HandleIdentifier(Result); |
Douglas Gregor | 0814253 | 2011-08-26 23:56:07 +0000 | [diff] [blame] | 1511 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1512 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1513 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1514 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1515 | // Otherwise, $,\,? in identifier found. Enter slower path. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1516 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1517 | C = getCharAndSize(CurPtr, Size); |
| 1518 | while (1) { |
| 1519 | if (C == '$') { |
| 1520 | // If we hit a $ and they are not supported in identifiers, we are done. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1521 | if (!LangOpts.DollarIdents) goto FinishIdentifier; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1522 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1523 | // Otherwise, emit a diagnostic and continue. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1524 | if (!isLexingRawMode()) |
| 1525 | Diag(CurPtr, diag::ext_dollar_in_identifier); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1526 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
| 1527 | C = getCharAndSize(CurPtr, Size); |
| 1528 | continue; |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1529 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1530 | } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1531 | C = getCharAndSize(CurPtr, Size); |
| 1532 | continue; |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1533 | } else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1534 | C = getCharAndSize(CurPtr, Size); |
| 1535 | continue; |
| 1536 | } else if (!isIdentifierBody(C)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1537 | goto FinishIdentifier; |
| 1538 | } |
| 1539 | |
| 1540 | // Otherwise, this character is good, consume it. |
| 1541 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
| 1542 | |
| 1543 | C = getCharAndSize(CurPtr, Size); |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1544 | while (isIdentifierBody(C)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1545 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
| 1546 | C = getCharAndSize(CurPtr, Size); |
| 1547 | } |
| 1548 | } |
| 1549 | } |
| 1550 | |
Douglas Gregor | 759ef23 | 2010-08-30 14:50:47 +0000 | [diff] [blame] | 1551 | /// isHexaLiteral - Return true if Start points to a hex constant. |
Chris Lattner | 5f183aa | 2010-08-30 17:11:14 +0000 | [diff] [blame] | 1552 | /// in microsoft mode (where this is supposed to be several different tokens). |
Eli Friedman | 324adad | 2012-08-31 02:29:37 +0000 | [diff] [blame] | 1553 | bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { |
Chris Lattner | 0f0492e | 2010-08-31 16:42:00 +0000 | [diff] [blame] | 1554 | unsigned Size; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1555 | char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); |
Chris Lattner | 0f0492e | 2010-08-31 16:42:00 +0000 | [diff] [blame] | 1556 | if (C1 != '0') |
| 1557 | return false; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1558 | char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); |
Chris Lattner | 0f0492e | 2010-08-31 16:42:00 +0000 | [diff] [blame] | 1559 | return (C2 == 'x' || C2 == 'X'); |
Douglas Gregor | 759ef23 | 2010-08-30 14:50:47 +0000 | [diff] [blame] | 1560 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1561 | |
Nate Begeman | 5eee933 | 2008-04-14 02:26:39 +0000 | [diff] [blame] | 1562 | /// LexNumericConstant - Lex the remainder of a integer or floating point |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1563 | /// constant. From[-1] is the first character lexed. Return the end of the |
| 1564 | /// constant. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1565 | bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1566 | unsigned Size; |
| 1567 | char C = getCharAndSize(CurPtr, Size); |
| 1568 | char PrevCh = 0; |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1569 | while (isPreprocessingNumberBody(C)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1570 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
| 1571 | PrevCh = C; |
| 1572 | C = getCharAndSize(CurPtr, Size); |
| 1573 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1574 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1575 | // If we fell out, check for a sign, due to 1e+12. If we have one, continue. |
Chris Lattner | 7a9e9e7 | 2010-08-30 17:09:08 +0000 | [diff] [blame] | 1576 | if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { |
| 1577 | // If we are in Microsoft mode, don't continue if the constant is hex. |
| 1578 | // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1579 | if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) |
Chris Lattner | 7a9e9e7 | 2010-08-30 17:09:08 +0000 | [diff] [blame] | 1580 | return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); |
| 1581 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1582 | |
| 1583 | // If we have a hex FP constant, continue. |
Richard Smith | e6799dd | 2012-06-15 05:07:49 +0000 | [diff] [blame] | 1584 | if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { |
| 1585 | // Outside C99, we accept hexadecimal floating point numbers as a |
| 1586 | // not-quite-conforming extension. Only do so if this looks like it's |
| 1587 | // actually meant to be a hexfloat, and not if it has a ud-suffix. |
| 1588 | bool IsHexFloat = true; |
| 1589 | if (!LangOpts.C99) { |
| 1590 | if (!isHexaLiteral(BufferPtr, LangOpts)) |
| 1591 | IsHexFloat = false; |
| 1592 | else if (std::find(BufferPtr, CurPtr, '_') != CurPtr) |
| 1593 | IsHexFloat = false; |
| 1594 | } |
| 1595 | if (IsHexFloat) |
| 1596 | return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); |
| 1597 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1598 | |
Richard Smith | fde9485 | 2013-09-26 03:33:06 +0000 | [diff] [blame] | 1599 | // If we have a digit separator, continue. |
Aaron Ballman | dd69ef3 | 2014-08-19 15:55:55 +0000 | [diff] [blame] | 1600 | if (C == '\'' && getLangOpts().CPlusPlus14) { |
Richard Smith | fde9485 | 2013-09-26 03:33:06 +0000 | [diff] [blame] | 1601 | unsigned NextSize; |
| 1602 | char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts()); |
Richard Smith | 7f2707a | 2013-09-26 18:13:20 +0000 | [diff] [blame] | 1603 | if (isIdentifierBody(Next)) { |
Richard Smith | fde9485 | 2013-09-26 03:33:06 +0000 | [diff] [blame] | 1604 | if (!isLexingRawMode()) |
| 1605 | Diag(CurPtr, diag::warn_cxx11_compat_digit_separator); |
| 1606 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
Richard Smith | 35ddad0 | 2014-02-28 20:06:02 +0000 | [diff] [blame] | 1607 | CurPtr = ConsumeChar(CurPtr, NextSize, Result); |
Richard Smith | fde9485 | 2013-09-26 03:33:06 +0000 | [diff] [blame] | 1608 | return LexNumericConstant(Result, CurPtr); |
| 1609 | } |
| 1610 | } |
| 1611 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1612 | // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue. |
| 1613 | if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) |
| 1614 | return LexNumericConstant(Result, CurPtr); |
| 1615 | if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) |
| 1616 | return LexNumericConstant(Result, CurPtr); |
| 1617 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1618 | // Update the location of token as well as BufferPtr. |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1619 | const char *TokStart = BufferPtr; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1620 | FormTokenWithChars(Result, CurPtr, tok::numeric_constant); |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1621 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1622 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1623 | } |
| 1624 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1625 | /// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes |
Richard Smith | 3e4a60a | 2012-03-07 03:13:00 +0000 | [diff] [blame] | 1626 | /// in C++11, or warn on a ud-suffix in C++98. |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1627 | const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr, |
| 1628 | bool IsStringLiteral) { |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1629 | assert(getLangOpts().CPlusPlus); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1630 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1631 | // Maximally munch an identifier. |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1632 | unsigned Size; |
| 1633 | char C = getCharAndSize(CurPtr, Size); |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1634 | bool Consumed = false; |
Richard Smith | 0df56f4 | 2012-03-08 02:39:21 +0000 | [diff] [blame] | 1635 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1636 | if (!isIdentifierHead(C)) { |
| 1637 | if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) |
| 1638 | Consumed = true; |
| 1639 | else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) |
| 1640 | Consumed = true; |
| 1641 | else |
| 1642 | return CurPtr; |
| 1643 | } |
| 1644 | |
| 1645 | if (!getLangOpts().CPlusPlus11) { |
| 1646 | if (!isLexingRawMode()) |
| 1647 | Diag(CurPtr, |
| 1648 | C == '_' ? diag::warn_cxx11_compat_user_defined_literal |
| 1649 | : diag::warn_cxx11_compat_reserved_user_defined_literal) |
| 1650 | << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); |
| 1651 | return CurPtr; |
| 1652 | } |
| 1653 | |
| 1654 | // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix |
| 1655 | // that does not start with an underscore is ill-formed. As a conforming |
| 1656 | // extension, we treat all such suffixes as if they had whitespace before |
| 1657 | // them. We assume a suffix beginning with a UCN or UTF-8 character is more |
| 1658 | // likely to be a ud-suffix than a macro, however, and accept that. |
| 1659 | if (!Consumed) { |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1660 | bool IsUDSuffix = false; |
| 1661 | if (C == '_') |
| 1662 | IsUDSuffix = true; |
Aaron Ballman | dd69ef3 | 2014-08-19 15:55:55 +0000 | [diff] [blame] | 1663 | else if (IsStringLiteral && getLangOpts().CPlusPlus14) { |
Richard Smith | 2a98862 | 2013-09-24 04:06:10 +0000 | [diff] [blame] | 1664 | // In C++1y, we need to look ahead a few characters to see if this is a |
| 1665 | // valid suffix for a string literal or a numeric literal (this could be |
| 1666 | // the 'operator""if' defining a numeric literal operator). |
Richard Smith | 5acb759 | 2013-09-24 22:13:21 +0000 | [diff] [blame] | 1667 | const unsigned MaxStandardSuffixLength = 3; |
Richard Smith | 2a98862 | 2013-09-24 04:06:10 +0000 | [diff] [blame] | 1668 | char Buffer[MaxStandardSuffixLength] = { C }; |
| 1669 | unsigned Consumed = Size; |
| 1670 | unsigned Chars = 1; |
| 1671 | while (true) { |
| 1672 | unsigned NextSize; |
| 1673 | char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize, |
| 1674 | getLangOpts()); |
| 1675 | if (!isIdentifierBody(Next)) { |
| 1676 | // End of suffix. Check whether this is on the whitelist. |
| 1677 | IsUDSuffix = (Chars == 1 && Buffer[0] == 's') || |
| 1678 | NumericLiteralParser::isValidUDSuffix( |
| 1679 | getLangOpts(), StringRef(Buffer, Chars)); |
| 1680 | break; |
| 1681 | } |
| 1682 | |
| 1683 | if (Chars == MaxStandardSuffixLength) |
| 1684 | // Too long: can't be a standard suffix. |
| 1685 | break; |
| 1686 | |
| 1687 | Buffer[Chars++] = Next; |
| 1688 | Consumed += NextSize; |
| 1689 | } |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1690 | } |
| 1691 | |
| 1692 | if (!IsUDSuffix) { |
Richard Smith | 0df56f4 | 2012-03-08 02:39:21 +0000 | [diff] [blame] | 1693 | if (!isLexingRawMode()) |
Alp Toker | bfa3934 | 2014-01-14 12:51:41 +0000 | [diff] [blame] | 1694 | Diag(CurPtr, getLangOpts().MSVCCompat |
| 1695 | ? diag::ext_ms_reserved_user_defined_literal |
| 1696 | : diag::ext_reserved_user_defined_literal) |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1697 | << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); |
Richard Smith | 3e4a60a | 2012-03-07 03:13:00 +0000 | [diff] [blame] | 1698 | return CurPtr; |
| 1699 | } |
| 1700 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1701 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1702 | } |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1703 | |
| 1704 | Result.setFlag(Token::HasUDSuffix); |
| 1705 | while (true) { |
| 1706 | C = getCharAndSize(CurPtr, Size); |
| 1707 | if (isIdentifierBody(C)) { CurPtr = ConsumeChar(CurPtr, Size, Result); } |
| 1708 | else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {} |
| 1709 | else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) {} |
| 1710 | else break; |
| 1711 | } |
| 1712 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1713 | return CurPtr; |
| 1714 | } |
| 1715 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1716 | /// LexStringLiteral - Lex the remainder of a string literal, after having lexed |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1717 | /// either " or L" or u8" or u" or U". |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1718 | bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr, |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1719 | tok::TokenKind Kind) { |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1720 | // Does this string contain the \0 character? |
| 1721 | const char *NulCharacter = nullptr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1722 | |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 1723 | if (!isLexingRawMode() && |
| 1724 | (Kind == tok::utf8_string_literal || |
| 1725 | Kind == tok::utf16_string_literal || |
Richard Smith | 06d274f | 2013-03-11 18:01:42 +0000 | [diff] [blame] | 1726 | Kind == tok::utf32_string_literal)) |
| 1727 | Diag(BufferPtr, getLangOpts().CPlusPlus |
| 1728 | ? diag::warn_cxx98_compat_unicode_literal |
| 1729 | : diag::warn_c99_compat_unicode_literal); |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 1730 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1731 | char C = getAndAdvanceChar(CurPtr, Result); |
| 1732 | while (C != '"') { |
Chris Lattner | 52d96ac | 2010-05-30 23:27:38 +0000 | [diff] [blame] | 1733 | // Skip escaped characters. Escaped newlines will already be processed by |
| 1734 | // getAndAdvanceChar. |
| 1735 | if (C == '\\') |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1736 | C = getAndAdvanceChar(CurPtr, Result); |
Douglas Gregor | fe4a410 | 2010-05-30 22:59:50 +0000 | [diff] [blame] | 1737 | |
Chris Lattner | 52d96ac | 2010-05-30 23:27:38 +0000 | [diff] [blame] | 1738 | if (C == '\n' || C == '\r' || // Newline. |
Douglas Gregor | fe4a410 | 2010-05-30 22:59:50 +0000 | [diff] [blame] | 1739 | (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1740 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
Richard Smith | 608c0b6 | 2012-06-28 07:51:56 +0000 | [diff] [blame] | 1741 | Diag(BufferPtr, diag::ext_unterminated_string); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1742 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1743 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1744 | } |
Chris Lattner | 52d96ac | 2010-05-30 23:27:38 +0000 | [diff] [blame] | 1745 | |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1746 | if (C == 0) { |
| 1747 | if (isCodeCompletionPoint(CurPtr-1)) { |
| 1748 | PP->CodeCompleteNaturalLanguage(); |
| 1749 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1750 | cutOffLexing(); |
| 1751 | return true; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1752 | } |
| 1753 | |
Chris Lattner | 52d96ac | 2010-05-30 23:27:38 +0000 | [diff] [blame] | 1754 | NulCharacter = CurPtr-1; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1755 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1756 | C = getAndAdvanceChar(CurPtr, Result); |
| 1757 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1758 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1759 | // If we are in C++11, lex the optional ud-suffix. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1760 | if (getLangOpts().CPlusPlus) |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1761 | CurPtr = LexUDSuffix(Result, CurPtr, true); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1762 | |
Chris Lattner | 5a78a02 | 2006-07-20 06:02:19 +0000 | [diff] [blame] | 1763 | // If a nul character existed in the string, warn about it. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1764 | if (NulCharacter && !isLexingRawMode()) |
| 1765 | Diag(NulCharacter, diag::null_in_string); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1766 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1767 | // Update the location of the token as well as the BufferPtr instance var. |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1768 | const char *TokStart = BufferPtr; |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1769 | FormTokenWithChars(Result, CurPtr, Kind); |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1770 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1771 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1772 | } |
| 1773 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1774 | /// LexRawStringLiteral - Lex the remainder of a raw string literal, after |
| 1775 | /// having lexed R", LR", u8R", uR", or UR". |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1776 | bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1777 | tok::TokenKind Kind) { |
| 1778 | // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: |
| 1779 | // Between the initial and final double quote characters of the raw string, |
| 1780 | // any transformations performed in phases 1 and 2 (trigraphs, |
| 1781 | // universal-character-names, and line splicing) are reverted. |
| 1782 | |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 1783 | if (!isLexingRawMode()) |
| 1784 | Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); |
| 1785 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1786 | unsigned PrefixLen = 0; |
| 1787 | |
| 1788 | while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) |
| 1789 | ++PrefixLen; |
| 1790 | |
| 1791 | // If the last character was not a '(', then we didn't lex a valid delimiter. |
| 1792 | if (CurPtr[PrefixLen] != '(') { |
| 1793 | if (!isLexingRawMode()) { |
| 1794 | const char *PrefixEnd = &CurPtr[PrefixLen]; |
| 1795 | if (PrefixLen == 16) { |
| 1796 | Diag(PrefixEnd, diag::err_raw_delim_too_long); |
| 1797 | } else { |
| 1798 | Diag(PrefixEnd, diag::err_invalid_char_raw_delim) |
| 1799 | << StringRef(PrefixEnd, 1); |
| 1800 | } |
| 1801 | } |
| 1802 | |
| 1803 | // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, |
| 1804 | // it's possible the '"' was intended to be part of the raw string, but |
| 1805 | // there's not much we can do about that. |
| 1806 | while (1) { |
| 1807 | char C = *CurPtr++; |
| 1808 | |
| 1809 | if (C == '"') |
| 1810 | break; |
| 1811 | if (C == 0 && CurPtr-1 == BufferEnd) { |
| 1812 | --CurPtr; |
| 1813 | break; |
| 1814 | } |
| 1815 | } |
| 1816 | |
| 1817 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1818 | return true; |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1819 | } |
| 1820 | |
| 1821 | // Save prefix and move CurPtr past it |
| 1822 | const char *Prefix = CurPtr; |
| 1823 | CurPtr += PrefixLen + 1; // skip over prefix and '(' |
| 1824 | |
| 1825 | while (1) { |
| 1826 | char C = *CurPtr++; |
| 1827 | |
| 1828 | if (C == ')') { |
| 1829 | // Check for prefix match and closing quote. |
| 1830 | if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { |
| 1831 | CurPtr += PrefixLen + 1; // skip over prefix and '"' |
| 1832 | break; |
| 1833 | } |
| 1834 | } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. |
| 1835 | if (!isLexingRawMode()) |
| 1836 | Diag(BufferPtr, diag::err_unterminated_raw_string) |
| 1837 | << StringRef(Prefix, PrefixLen); |
| 1838 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1839 | return true; |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1840 | } |
| 1841 | } |
| 1842 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1843 | // If we are in C++11, lex the optional ud-suffix. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1844 | if (getLangOpts().CPlusPlus) |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1845 | CurPtr = LexUDSuffix(Result, CurPtr, true); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1846 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1847 | // Update the location of token as well as BufferPtr. |
| 1848 | const char *TokStart = BufferPtr; |
| 1849 | FormTokenWithChars(Result, CurPtr, Kind); |
| 1850 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1851 | return true; |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1852 | } |
| 1853 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1854 | /// LexAngledStringLiteral - Lex the remainder of an angled string literal, |
| 1855 | /// after having lexed the '<' character. This is used for #include filenames. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1856 | bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1857 | // Does this string contain the \0 character? |
| 1858 | const char *NulCharacter = nullptr; |
Chris Lattner | b40289b | 2009-04-17 23:56:52 +0000 | [diff] [blame] | 1859 | const char *AfterLessPos = CurPtr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1860 | char C = getAndAdvanceChar(CurPtr, Result); |
| 1861 | while (C != '>') { |
| 1862 | // Skip escaped characters. |
| 1863 | if (C == '\\') { |
| 1864 | // Skip the escaped character. |
Dmitri Gribenko | 4aa05c5 | 2012-07-30 17:59:40 +0000 | [diff] [blame] | 1865 | getAndAdvanceChar(CurPtr, Result); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1866 | } else if (C == '\n' || C == '\r' || // Newline. |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1867 | (C == 0 && (CurPtr-1 == BufferEnd || // End of file. |
| 1868 | isCodeCompletionPoint(CurPtr-1)))) { |
Chris Lattner | b40289b | 2009-04-17 23:56:52 +0000 | [diff] [blame] | 1869 | // If the filename is unterminated, then it must just be a lone < |
| 1870 | // character. Return this as such. |
| 1871 | FormTokenWithChars(Result, AfterLessPos, tok::less); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1872 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1873 | } else if (C == 0) { |
| 1874 | NulCharacter = CurPtr-1; |
| 1875 | } |
| 1876 | C = getAndAdvanceChar(CurPtr, Result); |
| 1877 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1878 | |
Chris Lattner | 5a78a02 | 2006-07-20 06:02:19 +0000 | [diff] [blame] | 1879 | // If a nul character existed in the string, warn about it. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1880 | if (NulCharacter && !isLexingRawMode()) |
| 1881 | Diag(NulCharacter, diag::null_in_string); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1882 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1883 | // Update the location of token as well as BufferPtr. |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1884 | const char *TokStart = BufferPtr; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1885 | FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1886 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1887 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1888 | } |
| 1889 | |
| 1890 | |
| 1891 | /// LexCharConstant - Lex the remainder of a character constant, after having |
Richard Smith | 3e3a705 | 2014-11-08 06:08:42 +0000 | [diff] [blame] | 1892 | /// lexed either ' or L' or u8' or u' or U'. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1893 | bool Lexer::LexCharConstant(Token &Result, const char *CurPtr, |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1894 | tok::TokenKind Kind) { |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1895 | // Does this character contain the \0 character? |
| 1896 | const char *NulCharacter = nullptr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1897 | |
Richard Smith | 3e3a705 | 2014-11-08 06:08:42 +0000 | [diff] [blame] | 1898 | if (!isLexingRawMode()) { |
| 1899 | if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant) |
| 1900 | Diag(BufferPtr, getLangOpts().CPlusPlus |
| 1901 | ? diag::warn_cxx98_compat_unicode_literal |
| 1902 | : diag::warn_c99_compat_unicode_literal); |
| 1903 | else if (Kind == tok::utf8_char_constant) |
| 1904 | Diag(BufferPtr, diag::warn_cxx14_compat_u8_character_literal); |
| 1905 | } |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 1906 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1907 | char C = getAndAdvanceChar(CurPtr, Result); |
| 1908 | if (C == '\'') { |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1909 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
Richard Smith | 608c0b6 | 2012-06-28 07:51:56 +0000 | [diff] [blame] | 1910 | Diag(BufferPtr, diag::ext_empty_character); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1911 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1912 | return true; |
Chris Lattner | 86851b8 | 2010-07-07 23:24:27 +0000 | [diff] [blame] | 1913 | } |
| 1914 | |
| 1915 | while (C != '\'') { |
| 1916 | // Skip escaped characters. |
Nico Weber | 4e27038 | 2012-11-17 20:25:54 +0000 | [diff] [blame] | 1917 | if (C == '\\') |
| 1918 | C = getAndAdvanceChar(CurPtr, Result); |
| 1919 | |
| 1920 | if (C == '\n' || C == '\r' || // Newline. |
| 1921 | (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1922 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
Richard Smith | 608c0b6 | 2012-06-28 07:51:56 +0000 | [diff] [blame] | 1923 | Diag(BufferPtr, diag::ext_unterminated_char); |
Chris Lattner | 86851b8 | 2010-07-07 23:24:27 +0000 | [diff] [blame] | 1924 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1925 | return true; |
Nico Weber | 4e27038 | 2012-11-17 20:25:54 +0000 | [diff] [blame] | 1926 | } |
| 1927 | |
| 1928 | if (C == 0) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1929 | if (isCodeCompletionPoint(CurPtr-1)) { |
| 1930 | PP->CodeCompleteNaturalLanguage(); |
| 1931 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1932 | cutOffLexing(); |
| 1933 | return true; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1934 | } |
| 1935 | |
Chris Lattner | 86851b8 | 2010-07-07 23:24:27 +0000 | [diff] [blame] | 1936 | NulCharacter = CurPtr-1; |
| 1937 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1938 | C = getAndAdvanceChar(CurPtr, Result); |
| 1939 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1940 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1941 | // If we are in C++11, lex the optional ud-suffix. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1942 | if (getLangOpts().CPlusPlus) |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1943 | CurPtr = LexUDSuffix(Result, CurPtr, false); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1944 | |
Chris Lattner | 86851b8 | 2010-07-07 23:24:27 +0000 | [diff] [blame] | 1945 | // If a nul character existed in the character, warn about it. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1946 | if (NulCharacter && !isLexingRawMode()) |
| 1947 | Diag(NulCharacter, diag::null_in_char); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1948 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1949 | // Update the location of token as well as BufferPtr. |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1950 | const char *TokStart = BufferPtr; |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1951 | FormTokenWithChars(Result, CurPtr, Kind); |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1952 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1953 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1954 | } |
| 1955 | |
| 1956 | /// SkipWhitespace - Efficiently skip over a series of whitespace characters. |
| 1957 | /// Update BufferPtr to point to the next non-whitespace character and return. |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 1958 | /// |
| 1959 | /// This method forms a token and returns true if KeepWhitespaceMode is enabled. |
| 1960 | /// |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1961 | bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr, |
| 1962 | bool &TokAtPhysicalStartOfLine) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1963 | // Whitespace - Skip it, then return the token after the whitespace. |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 1964 | bool SawNewline = isVerticalWhitespace(CurPtr[-1]); |
| 1965 | |
Richard Smith | 0f7f6f1a | 2013-05-10 02:36:35 +0000 | [diff] [blame] | 1966 | unsigned char Char = *CurPtr; |
| 1967 | |
| 1968 | // Skip consecutive spaces efficiently. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1969 | while (1) { |
| 1970 | // Skip horizontal whitespace very aggressively. |
| 1971 | while (isHorizontalWhitespace(Char)) |
| 1972 | Char = *++CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1973 | |
Daniel Dunbar | 5c4cc09 | 2008-11-25 00:20:22 +0000 | [diff] [blame] | 1974 | // Otherwise if we have something other than whitespace, we're done. |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 1975 | if (!isVerticalWhitespace(Char)) |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1976 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1977 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1978 | if (ParsingPreprocessorDirective) { |
| 1979 | // End of preprocessor directive line, let LexTokenInternal handle this. |
| 1980 | BufferPtr = CurPtr; |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 1981 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1982 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1983 | |
Richard Smith | 0f7f6f1a | 2013-05-10 02:36:35 +0000 | [diff] [blame] | 1984 | // OK, but handle newline. |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 1985 | SawNewline = true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1986 | Char = *++CurPtr; |
| 1987 | } |
| 1988 | |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 1989 | // If the client wants us to return whitespace, return it now. |
| 1990 | if (isKeepWhitespaceMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1991 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1992 | if (SawNewline) { |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 1993 | IsAtStartOfLine = true; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1994 | IsAtPhysicalStartOfLine = true; |
| 1995 | } |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 1996 | // FIXME: The next token will not have LeadingSpace set. |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 1997 | return true; |
| 1998 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1999 | |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2000 | // If this isn't immediately after a newline, there is leading space. |
| 2001 | char PrevChar = CurPtr[-1]; |
| 2002 | bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); |
| 2003 | |
| 2004 | Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2005 | if (SawNewline) { |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2006 | Result.setFlag(Token::StartOfLine); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2007 | TokAtPhysicalStartOfLine = true; |
| 2008 | } |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2009 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2010 | BufferPtr = CurPtr; |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2011 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2012 | } |
| 2013 | |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2014 | /// We have just read the // characters from input. Skip until we find the |
| 2015 | /// newline character thats terminate the comment. Then update BufferPtr and |
| 2016 | /// return. |
Chris Lattner | 87d0208 | 2010-01-18 22:35:47 +0000 | [diff] [blame] | 2017 | /// |
| 2018 | /// If we're in KeepCommentMode or any CommentHandler has inserted |
| 2019 | /// some tokens, this will store the first token and return true. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2020 | bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, |
| 2021 | bool &TokAtPhysicalStartOfLine) { |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2022 | // If Line comments aren't explicitly enabled for this language, emit an |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2023 | // extension warning. |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2024 | if (!LangOpts.LineComment && !isLexingRawMode()) { |
| 2025 | Diag(BufferPtr, diag::ext_line_comment); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2026 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2027 | // Mark them enabled so we only emit one warning for this translation |
| 2028 | // unit. |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2029 | LangOpts.LineComment = true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2030 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2031 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2032 | // Scan over the body of the comment. The common case, when scanning, is that |
| 2033 | // the comment contains normal ascii characters with nothing interesting in |
| 2034 | // them. As such, optimize for this case with the inner loop. |
| 2035 | char C; |
| 2036 | do { |
| 2037 | C = *CurPtr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2038 | // Skip over characters in the fast loop. |
| 2039 | while (C != 0 && // Potentially EOF. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2040 | C != '\n' && C != '\r') // Newline or DOS-style newline. |
| 2041 | C = *++CurPtr; |
| 2042 | |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2043 | const char *NextLine = CurPtr; |
| 2044 | if (C != 0) { |
| 2045 | // We found a newline, see if it's escaped. |
| 2046 | const char *EscapePtr = CurPtr-1; |
Alp Toker | 6de6da6 | 2013-12-14 23:32:31 +0000 | [diff] [blame] | 2047 | bool HasSpace = false; |
| 2048 | while (isHorizontalWhitespace(*EscapePtr)) { // Skip whitespace. |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2049 | --EscapePtr; |
Alp Toker | 6de6da6 | 2013-12-14 23:32:31 +0000 | [diff] [blame] | 2050 | HasSpace = true; |
| 2051 | } |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2052 | |
| 2053 | if (*EscapePtr == '\\') // Escaped newline. |
| 2054 | CurPtr = EscapePtr; |
| 2055 | else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && |
| 2056 | EscapePtr[-2] == '?') // Trigraph-escaped newline. |
| 2057 | CurPtr = EscapePtr-2; |
| 2058 | else |
| 2059 | break; // This is a newline, we're done. |
Alp Toker | 6de6da6 | 2013-12-14 23:32:31 +0000 | [diff] [blame] | 2060 | |
| 2061 | // If there was space between the backslash and newline, warn about it. |
| 2062 | if (HasSpace && !isLexingRawMode()) |
| 2063 | Diag(EscapePtr, diag::backslash_newline_space); |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2064 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2065 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2066 | // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to |
Chris Lattner | e141a9e | 2008-12-12 07:34:39 +0000 | [diff] [blame] | 2067 | // properly decode the character. Read it in raw mode to avoid emitting |
| 2068 | // diagnostics about things like trigraphs. If we see an escaped newline, |
| 2069 | // we'll handle it below. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2070 | const char *OldPtr = CurPtr; |
Chris Lattner | e141a9e | 2008-12-12 07:34:39 +0000 | [diff] [blame] | 2071 | bool OldRawMode = isLexingRawMode(); |
| 2072 | LexingRawMode = true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2073 | C = getAndAdvanceChar(CurPtr, Result); |
Chris Lattner | e141a9e | 2008-12-12 07:34:39 +0000 | [diff] [blame] | 2074 | LexingRawMode = OldRawMode; |
Chris Lattner | ecdaf40 | 2009-04-05 00:26:41 +0000 | [diff] [blame] | 2075 | |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2076 | // If we only read only one character, then no special handling is needed. |
| 2077 | // We're done and can skip forward to the newline. |
| 2078 | if (C != 0 && CurPtr == OldPtr+1) { |
| 2079 | CurPtr = NextLine; |
| 2080 | break; |
| 2081 | } |
| 2082 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2083 | // If we read multiple characters, and one of those characters was a \r or |
Chris Lattner | ff591e2 | 2007-06-09 06:07:22 +0000 | [diff] [blame] | 2084 | // \n, then we had an escaped newline within the comment. Emit diagnostic |
| 2085 | // unless the next line is also a // comment. |
| 2086 | if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2087 | for (; OldPtr != CurPtr; ++OldPtr) |
| 2088 | if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { |
Chris Lattner | ff591e2 | 2007-06-09 06:07:22 +0000 | [diff] [blame] | 2089 | // Okay, we found a // comment that ends in a newline, if the next |
| 2090 | // line is also a // comment, but has spaces, don't emit a diagnostic. |
Benjamin Kramer | dbfb18a | 2011-09-05 07:19:35 +0000 | [diff] [blame] | 2091 | if (isWhitespace(C)) { |
Chris Lattner | ff591e2 | 2007-06-09 06:07:22 +0000 | [diff] [blame] | 2092 | const char *ForwardPtr = CurPtr; |
Benjamin Kramer | dbfb18a | 2011-09-05 07:19:35 +0000 | [diff] [blame] | 2093 | while (isWhitespace(*ForwardPtr)) // Skip whitespace. |
Chris Lattner | ff591e2 | 2007-06-09 06:07:22 +0000 | [diff] [blame] | 2094 | ++ForwardPtr; |
| 2095 | if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') |
| 2096 | break; |
| 2097 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2098 | |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2099 | if (!isLexingRawMode()) |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2100 | Diag(OldPtr-1, diag::ext_multi_line_line_comment); |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2101 | break; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2102 | } |
| 2103 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2104 | |
Douglas Gregor | 1158370 | 2010-08-25 17:04:25 +0000 | [diff] [blame] | 2105 | if (CurPtr == BufferEnd+1) { |
Douglas Gregor | 1158370 | 2010-08-25 17:04:25 +0000 | [diff] [blame] | 2106 | --CurPtr; |
| 2107 | break; |
| 2108 | } |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2109 | |
| 2110 | if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { |
| 2111 | PP->CodeCompleteNaturalLanguage(); |
| 2112 | cutOffLexing(); |
| 2113 | return false; |
| 2114 | } |
| 2115 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2116 | } while (C != '\n' && C != '\r'); |
| 2117 | |
Chris Lattner | 93ddf80 | 2010-02-03 21:06:21 +0000 | [diff] [blame] | 2118 | // Found but did not consume the newline. Notify comment handlers about the |
| 2119 | // comment unless we're in a #if 0 block. |
| 2120 | if (PP && !isLexingRawMode() && |
| 2121 | PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), |
| 2122 | getSourceLocation(CurPtr)))) { |
Chris Lattner | 87d0208 | 2010-01-18 22:35:47 +0000 | [diff] [blame] | 2123 | BufferPtr = CurPtr; |
| 2124 | return true; // A token has to be returned. |
| 2125 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2126 | |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2127 | // If we are returning comments as tokens, return this comment as a token. |
Chris Lattner | 8637abd | 2008-10-12 03:22:02 +0000 | [diff] [blame] | 2128 | if (inKeepCommentMode()) |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2129 | return SaveLineComment(Result, CurPtr); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2130 | |
| 2131 | // If we are inside a preprocessor directive and we see the end of line, |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2132 | // return immediately, so that the lexer can return this as an EOD token. |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2133 | if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2134 | BufferPtr = CurPtr; |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2135 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2136 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2137 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2138 | // Otherwise, eat the \n character. We don't care if this is a \n\r or |
Chris Lattner | 87e97ea | 2008-10-12 00:23:07 +0000 | [diff] [blame] | 2139 | // \r\n sequence. This is an efficiency hack (because we know the \n can't |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2140 | // contribute to another token), it isn't needed for correctness. Note that |
| 2141 | // this is ok even in KeepWhitespaceMode, because we would have returned the |
| 2142 | /// comment above in that mode. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2143 | ++CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2144 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2145 | // The next returned token is at the start of the line. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2146 | Result.setFlag(Token::StartOfLine); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2147 | TokAtPhysicalStartOfLine = true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2148 | // No leading whitespace seen so far. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2149 | Result.clearFlag(Token::LeadingSpace); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2150 | BufferPtr = CurPtr; |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2151 | return false; |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2152 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2153 | |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2154 | /// If in save-comment mode, package up this Line comment in an appropriate |
| 2155 | /// way and return it. |
| 2156 | bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2157 | // If we're not in a preprocessor directive, just return the // comment |
| 2158 | // directly. |
| 2159 | FormTokenWithChars(Result, CurPtr, tok::comment); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2160 | |
David Blaikie | d532124 | 2012-06-06 18:52:13 +0000 | [diff] [blame] | 2161 | if (!ParsingPreprocessorDirective || LexingRawMode) |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2162 | return true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2163 | |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2164 | // If this Line-style comment is in a macro definition, transmogrify it into |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2165 | // a C-style block comment. |
Douglas Gregor | dc970f0 | 2010-03-16 22:30:13 +0000 | [diff] [blame] | 2166 | bool Invalid = false; |
| 2167 | std::string Spelling = PP->getSpelling(Result, &Invalid); |
| 2168 | if (Invalid) |
| 2169 | return true; |
| 2170 | |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2171 | assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2172 | Spelling[1] = '*'; // Change prefix to "/*". |
| 2173 | Spelling += "*/"; // add suffix. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2174 | |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2175 | Result.setKind(tok::comment); |
Dmitri Gribenko | b8e9e75 | 2012-09-24 21:07:17 +0000 | [diff] [blame] | 2176 | PP->CreateString(Spelling, Result, |
Abramo Bagnara | e398e60 | 2011-10-03 18:39:03 +0000 | [diff] [blame] | 2177 | Result.getLocation(), Result.getLocation()); |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2178 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2179 | } |
| 2180 | |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2181 | /// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline |
David Blaikie | 987bcf9 | 2012-06-06 18:43:20 +0000 | [diff] [blame] | 2182 | /// character (either \\n or \\r) is part of an escaped newline sequence. Issue |
| 2183 | /// a diagnostic if so. We know that the newline is inside of a block comment. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2184 | static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, |
Chris Lattner | 1f58305 | 2006-06-18 06:53:56 +0000 | [diff] [blame] | 2185 | Lexer *L) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2186 | assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2187 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2188 | // Back up off the newline. |
| 2189 | --CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2190 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2191 | // If this is a two-character newline sequence, skip the other character. |
| 2192 | if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { |
| 2193 | // \n\n or \r\r -> not escaped newline. |
| 2194 | if (CurPtr[0] == CurPtr[1]) |
| 2195 | return false; |
| 2196 | // \n\r or \r\n -> skip the newline. |
| 2197 | --CurPtr; |
| 2198 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2199 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2200 | // If we have horizontal whitespace, skip over it. We allow whitespace |
| 2201 | // between the slash and newline. |
| 2202 | bool HasSpace = false; |
| 2203 | while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { |
| 2204 | --CurPtr; |
| 2205 | HasSpace = true; |
| 2206 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2207 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2208 | // If we have a slash, we know this is an escaped newline. |
| 2209 | if (*CurPtr == '\\') { |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2210 | if (CurPtr[-1] != '*') return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2211 | } else { |
| 2212 | // It isn't a slash, is it the ?? / trigraph? |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2213 | if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || |
| 2214 | CurPtr[-3] != '*') |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2215 | return false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2216 | |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2217 | // This is the trigraph ending the comment. Emit a stern warning! |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2218 | CurPtr -= 2; |
| 2219 | |
| 2220 | // If no trigraphs are enabled, warn that we ignored this trigraph and |
| 2221 | // ignore this * character. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 2222 | if (!L->getLangOpts().Trigraphs) { |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2223 | if (!L->isLexingRawMode()) |
| 2224 | L->Diag(CurPtr, diag::trigraph_ignored_block_comment); |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2225 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2226 | } |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2227 | if (!L->isLexingRawMode()) |
| 2228 | L->Diag(CurPtr, diag::trigraph_ends_block_comment); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2229 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2230 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2231 | // Warn about having an escaped newline between the */ characters. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2232 | if (!L->isLexingRawMode()) |
| 2233 | L->Diag(CurPtr, diag::escaped_newline_block_comment_end); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2234 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2235 | // If there was space between the backslash and newline, warn about it. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2236 | if (HasSpace && !L->isLexingRawMode()) |
| 2237 | L->Diag(CurPtr, diag::backslash_newline_space); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2238 | |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2239 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2240 | } |
| 2241 | |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2242 | #ifdef __SSE2__ |
| 2243 | #include <emmintrin.h> |
Chris Lattner | 9f6604f | 2006-10-30 20:01:22 +0000 | [diff] [blame] | 2244 | #elif __ALTIVEC__ |
| 2245 | #include <altivec.h> |
| 2246 | #undef bool |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2247 | #endif |
| 2248 | |
James Dennett | f442d24 | 2012-06-17 03:40:43 +0000 | [diff] [blame] | 2249 | /// We have just read from input the / and * characters that started a comment. |
| 2250 | /// Read until we find the * and / characters that terminate the comment. |
| 2251 | /// Note that we don't bother decoding trigraphs or escaped newlines in block |
| 2252 | /// comments, because they cannot cause the comment to end. The only thing |
| 2253 | /// that can happen is the comment could end with an escaped newline between |
| 2254 | /// the terminating * and /. |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2255 | /// |
Chris Lattner | 87d0208 | 2010-01-18 22:35:47 +0000 | [diff] [blame] | 2256 | /// If we're in KeepCommentMode or any CommentHandler has inserted |
| 2257 | /// some tokens, this will store the first token and return true. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2258 | bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr, |
| 2259 | bool &TokAtPhysicalStartOfLine) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2260 | // Scan one character past where we should, looking for a '/' character. Once |
Chris Lattner | 57540c5 | 2011-04-15 05:22:18 +0000 | [diff] [blame] | 2261 | // we find it, check to see if it was preceded by a *. This common |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2262 | // optimization helps people who like to put a lot of * characters in their |
| 2263 | // comments. |
Chris Lattner | c850ad6 | 2007-07-21 23:43:37 +0000 | [diff] [blame] | 2264 | |
| 2265 | // The first character we get with newlines and trigraphs skipped to handle |
| 2266 | // the degenerate /*/ case below correctly if the * has an escaped newline |
| 2267 | // after it. |
| 2268 | unsigned CharSize; |
| 2269 | unsigned char C = getCharAndSize(CurPtr, CharSize); |
| 2270 | CurPtr += CharSize; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2271 | if (C == 0 && CurPtr == BufferEnd+1) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2272 | if (!isLexingRawMode()) |
Chris Lattner | 7c2e980 | 2008-10-12 01:31:51 +0000 | [diff] [blame] | 2273 | Diag(BufferPtr, diag::err_unterminated_block_comment); |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2274 | --CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2275 | |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2276 | // KeepWhitespaceMode should return this broken comment as a token. Since |
| 2277 | // it isn't a well formed comment, just return it as an 'unknown' token. |
| 2278 | if (isKeepWhitespaceMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2279 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2280 | return true; |
| 2281 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2282 | |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2283 | BufferPtr = CurPtr; |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2284 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2285 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2286 | |
Chris Lattner | c850ad6 | 2007-07-21 23:43:37 +0000 | [diff] [blame] | 2287 | // Check to see if the first character after the '/*' is another /. If so, |
| 2288 | // then this slash does not end the block comment, it is part of it. |
| 2289 | if (C == '/') |
| 2290 | C = *CurPtr++; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2291 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2292 | while (1) { |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2293 | // Skip over all non-interesting characters until we find end of buffer or a |
| 2294 | // (probably ending) '/' character. |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2295 | if (CurPtr + 24 < BufferEnd && |
| 2296 | // If there is a code-completion point avoid the fast scan because it |
| 2297 | // doesn't check for '\0'. |
| 2298 | !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2299 | // While not aligned to a 16-byte boundary. |
| 2300 | while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) |
| 2301 | C = *CurPtr++; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2302 | |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2303 | if (C == '/') goto FoundSlash; |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2304 | |
| 2305 | #ifdef __SSE2__ |
Roman Divacky | 6150990 | 2014-04-03 18:04:52 +0000 | [diff] [blame] | 2306 | __m128i Slashes = _mm_set1_epi8('/'); |
| 2307 | while (CurPtr+16 <= BufferEnd) { |
| 2308 | int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, |
| 2309 | Slashes)); |
Benjamin Kramer | 3885737 | 2011-11-22 18:56:46 +0000 | [diff] [blame] | 2310 | if (cmp != 0) { |
Benjamin Kramer | 900f1de | 2011-11-22 20:39:31 +0000 | [diff] [blame] | 2311 | // Adjust the pointer to point directly after the first slash. It's |
| 2312 | // not necessary to set C here, it will be overwritten at the end of |
| 2313 | // the outer loop. |
Michael J. Spencer | 8c39840 | 2013-05-24 21:42:04 +0000 | [diff] [blame] | 2314 | CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1; |
Benjamin Kramer | 3885737 | 2011-11-22 18:56:46 +0000 | [diff] [blame] | 2315 | goto FoundSlash; |
| 2316 | } |
Roman Divacky | 6150990 | 2014-04-03 18:04:52 +0000 | [diff] [blame] | 2317 | CurPtr += 16; |
Benjamin Kramer | 3885737 | 2011-11-22 18:56:46 +0000 | [diff] [blame] | 2318 | } |
Chris Lattner | 9f6604f | 2006-10-30 20:01:22 +0000 | [diff] [blame] | 2319 | #elif __ALTIVEC__ |
| 2320 | __vector unsigned char Slashes = { |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2321 | '/', '/', '/', '/', '/', '/', '/', '/', |
Chris Lattner | 9f6604f | 2006-10-30 20:01:22 +0000 | [diff] [blame] | 2322 | '/', '/', '/', '/', '/', '/', '/', '/' |
| 2323 | }; |
| 2324 | while (CurPtr+16 <= BufferEnd && |
Jay Foad | 6af95d3 | 2014-10-29 14:42:12 +0000 | [diff] [blame] | 2325 | !vec_any_eq(*(const vector unsigned char*)CurPtr, Slashes)) |
Chris Lattner | 9f6604f | 2006-10-30 20:01:22 +0000 | [diff] [blame] | 2326 | CurPtr += 16; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2327 | #else |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2328 | // Scan for '/' quickly. Many block comments are very large. |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2329 | while (CurPtr[0] != '/' && |
| 2330 | CurPtr[1] != '/' && |
| 2331 | CurPtr[2] != '/' && |
| 2332 | CurPtr[3] != '/' && |
| 2333 | CurPtr+4 < BufferEnd) { |
| 2334 | CurPtr += 4; |
| 2335 | } |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2336 | #endif |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2337 | |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2338 | // It has to be one of the bytes scanned, increment to it and read one. |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2339 | C = *CurPtr++; |
| 2340 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2341 | |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2342 | // Loop to scan the remainder. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2343 | while (C != '/' && C != '\0') |
| 2344 | C = *CurPtr++; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2345 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2346 | if (C == '/') { |
Benjamin Kramer | 3885737 | 2011-11-22 18:56:46 +0000 | [diff] [blame] | 2347 | FoundSlash: |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2348 | if (CurPtr[-2] == '*') // We found the final */. We're done! |
| 2349 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2350 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2351 | if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { |
Chris Lattner | 1f58305 | 2006-06-18 06:53:56 +0000 | [diff] [blame] | 2352 | if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2353 | // We found the final */, though it had an escaped newline between the |
| 2354 | // * and /. We're done! |
| 2355 | break; |
| 2356 | } |
| 2357 | } |
| 2358 | if (CurPtr[0] == '*' && CurPtr[1] != '/') { |
| 2359 | // If this is a /* inside of the comment, emit a warning. Don't do this |
| 2360 | // if this is a /*/, which will end the comment. This misses cases with |
| 2361 | // embedded escaped newlines, but oh well. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2362 | if (!isLexingRawMode()) |
| 2363 | Diag(CurPtr-1, diag::warn_nested_block_comment); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2364 | } |
| 2365 | } else if (C == 0 && CurPtr == BufferEnd+1) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2366 | if (!isLexingRawMode()) |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2367 | Diag(BufferPtr, diag::err_unterminated_block_comment); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2368 | // Note: the user probably forgot a */. We could continue immediately |
| 2369 | // after the /*, but this would involve lexing a lot of what really is the |
| 2370 | // comment, which surely would confuse the parser. |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2371 | --CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2372 | |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2373 | // KeepWhitespaceMode should return this broken comment as a token. Since |
| 2374 | // it isn't a well formed comment, just return it as an 'unknown' token. |
| 2375 | if (isKeepWhitespaceMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2376 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2377 | return true; |
| 2378 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2379 | |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2380 | BufferPtr = CurPtr; |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2381 | return false; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2382 | } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { |
| 2383 | PP->CodeCompleteNaturalLanguage(); |
| 2384 | cutOffLexing(); |
| 2385 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2386 | } |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2387 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2388 | C = *CurPtr++; |
| 2389 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2390 | |
Chris Lattner | 93ddf80 | 2010-02-03 21:06:21 +0000 | [diff] [blame] | 2391 | // Notify comment handlers about the comment unless we're in a #if 0 block. |
| 2392 | if (PP && !isLexingRawMode() && |
| 2393 | PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), |
| 2394 | getSourceLocation(CurPtr)))) { |
Chris Lattner | 87d0208 | 2010-01-18 22:35:47 +0000 | [diff] [blame] | 2395 | BufferPtr = CurPtr; |
| 2396 | return true; // A token has to be returned. |
| 2397 | } |
Douglas Gregor | c6d5edd | 2009-07-02 17:08:52 +0000 | [diff] [blame] | 2398 | |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2399 | // If we are returning comments as tokens, return this comment as a token. |
Chris Lattner | 8637abd | 2008-10-12 03:22:02 +0000 | [diff] [blame] | 2400 | if (inKeepCommentMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2401 | FormTokenWithChars(Result, CurPtr, tok::comment); |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2402 | return true; |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2403 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2404 | |
| 2405 | // It is common for the tokens immediately after a /**/ comment to be |
| 2406 | // whitespace. Instead of going through the big switch, handle it |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2407 | // efficiently now. This is safe even in KeepWhitespaceMode because we would |
| 2408 | // have already returned above with the comment as a token. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2409 | if (isHorizontalWhitespace(*CurPtr)) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2410 | SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine); |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2411 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2412 | } |
| 2413 | |
| 2414 | // Otherwise, just return so that the next character will be lexed as a token. |
| 2415 | BufferPtr = CurPtr; |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2416 | Result.setFlag(Token::LeadingSpace); |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2417 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2418 | } |
| 2419 | |
| 2420 | //===----------------------------------------------------------------------===// |
| 2421 | // Primary Lexing Entry Points |
| 2422 | //===----------------------------------------------------------------------===// |
| 2423 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2424 | /// ReadToEndOfLine - Read the rest of the current preprocessor line as an |
| 2425 | /// uninterpreted string. This switches the lexer out of directive mode. |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2426 | void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2427 | assert(ParsingPreprocessorDirective && ParsingFilename == false && |
| 2428 | "Must be in a preprocessing directive!"); |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2429 | Token Tmp; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2430 | |
| 2431 | // CurPtr - Cache BufferPtr in an automatic variable. |
| 2432 | const char *CurPtr = BufferPtr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2433 | while (1) { |
| 2434 | char Char = getAndAdvanceChar(CurPtr, Tmp); |
| 2435 | switch (Char) { |
| 2436 | default: |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2437 | if (Result) |
| 2438 | Result->push_back(Char); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2439 | break; |
| 2440 | case 0: // Null. |
| 2441 | // Found end of file? |
| 2442 | if (CurPtr-1 != BufferEnd) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2443 | if (isCodeCompletionPoint(CurPtr-1)) { |
| 2444 | PP->CodeCompleteNaturalLanguage(); |
| 2445 | cutOffLexing(); |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2446 | return; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2447 | } |
| 2448 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2449 | // Nope, normal character, continue. |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2450 | if (Result) |
| 2451 | Result->push_back(Char); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2452 | break; |
| 2453 | } |
| 2454 | // FALL THROUGH. |
| 2455 | case '\r': |
| 2456 | case '\n': |
| 2457 | // Okay, we found the end of the line. First, back up past the \0, \r, \n. |
| 2458 | assert(CurPtr[-1] == Char && "Trigraphs for newline?"); |
| 2459 | BufferPtr = CurPtr-1; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2460 | |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2461 | // Next, lex the character, which should handle the EOD transition. |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2462 | Lex(Tmp); |
Douglas Gregor | 1158370 | 2010-08-25 17:04:25 +0000 | [diff] [blame] | 2463 | if (Tmp.is(tok::code_completion)) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2464 | if (PP) |
| 2465 | PP->CodeCompleteNaturalLanguage(); |
Douglas Gregor | 1158370 | 2010-08-25 17:04:25 +0000 | [diff] [blame] | 2466 | Lex(Tmp); |
| 2467 | } |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2468 | assert(Tmp.is(tok::eod) && "Unexpected token!"); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2469 | |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2470 | // Finally, we're done; |
| 2471 | return; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2472 | } |
| 2473 | } |
| 2474 | } |
| 2475 | |
| 2476 | /// LexEndOfFile - CurPtr points to the end of this file. Handle this |
| 2477 | /// condition, reporting diagnostics and handling other edge cases as required. |
Chris Lattner | 2183a6e | 2006-07-18 06:36:12 +0000 | [diff] [blame] | 2478 | /// This returns true if Result contains a token, false if PP.Lex should be |
| 2479 | /// called again. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2480 | bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2481 | // If we hit the end of the file while parsing a preprocessor directive, |
| 2482 | // end the preprocessor directive first. The next token returned will |
| 2483 | // then be the end of file. |
| 2484 | if (ParsingPreprocessorDirective) { |
| 2485 | // Done parsing the "line". |
| 2486 | ParsingPreprocessorDirective = false; |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 2487 | // Update the location of token as well as BufferPtr. |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2488 | FormTokenWithChars(Result, CurPtr, tok::eod); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2489 | |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2490 | // Restore comment saving mode, in case it was disabled for directive. |
Alp Toker | 08c2500 | 2013-12-13 17:04:55 +0000 | [diff] [blame] | 2491 | if (PP) |
| 2492 | resetExtendedTokenMode(); |
Chris Lattner | 2183a6e | 2006-07-18 06:36:12 +0000 | [diff] [blame] | 2493 | return true; // Have a token. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2494 | } |
Douglas Gregor | 3545ff4 | 2009-09-21 16:56:56 +0000 | [diff] [blame] | 2495 | |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2496 | // If we are in raw mode, return this event as an EOF token. Let the caller |
| 2497 | // that put us in raw mode handle the event. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2498 | if (isLexingRawMode()) { |
Chris Lattner | 8c20487 | 2006-10-14 05:19:21 +0000 | [diff] [blame] | 2499 | Result.startToken(); |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2500 | BufferPtr = BufferEnd; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2501 | FormTokenWithChars(Result, BufferEnd, tok::eof); |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2502 | return true; |
Chris Lattner | d8aee0e | 2006-07-11 05:04:55 +0000 | [diff] [blame] | 2503 | } |
Douglas Gregor | 3545ff4 | 2009-09-21 16:56:56 +0000 | [diff] [blame] | 2504 | |
Douglas Gregor | 3a7ad25 | 2010-08-24 19:08:16 +0000 | [diff] [blame] | 2505 | // Issue diagnostics for unterminated #if and missing newline. |
| 2506 | |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2507 | // If we are in a #if directive, emit an error. |
| 2508 | while (!ConditionalStack.empty()) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2509 | if (PP->getCodeCompletionFileLoc() != FileLoc) |
Douglas Gregor | 02690ba | 2010-08-12 17:04:55 +0000 | [diff] [blame] | 2510 | PP->Diag(ConditionalStack.back().IfLoc, |
| 2511 | diag::err_pp_unterminated_conditional); |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2512 | ConditionalStack.pop_back(); |
| 2513 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2514 | |
Chris Lattner | 8f96d04 | 2008-04-12 05:54:25 +0000 | [diff] [blame] | 2515 | // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue |
| 2516 | // a pedwarn. |
Jordan Rose | 4c55d45 | 2013-08-23 15:42:01 +0000 | [diff] [blame] | 2517 | if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) { |
| 2518 | DiagnosticsEngine &Diags = PP->getDiagnostics(); |
| 2519 | SourceLocation EndLoc = getSourceLocation(BufferEnd); |
| 2520 | unsigned DiagID; |
| 2521 | |
| 2522 | if (LangOpts.CPlusPlus11) { |
| 2523 | // C++11 [lex.phases] 2.2 p2 |
| 2524 | // Prefer the C++98 pedantic compatibility warning over the generic, |
| 2525 | // non-extension, user-requested "missing newline at EOF" warning. |
Alp Toker | d4a3f0e | 2014-06-15 23:30:39 +0000 | [diff] [blame] | 2526 | if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc)) { |
Jordan Rose | 4c55d45 | 2013-08-23 15:42:01 +0000 | [diff] [blame] | 2527 | DiagID = diag::warn_cxx98_compat_no_newline_eof; |
| 2528 | } else { |
| 2529 | DiagID = diag::warn_no_newline_eof; |
| 2530 | } |
| 2531 | } else { |
| 2532 | DiagID = diag::ext_no_newline_eof; |
| 2533 | } |
| 2534 | |
| 2535 | Diag(BufferEnd, DiagID) |
| 2536 | << FixItHint::CreateInsertion(EndLoc, "\n"); |
| 2537 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2538 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2539 | BufferPtr = CurPtr; |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2540 | |
| 2541 | // Finally, let the preprocessor handle this. |
Jordan Rose | 127f6ee | 2012-06-15 23:33:51 +0000 | [diff] [blame] | 2542 | return PP->HandleEndOfFile(Result, isPragmaLexer()); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2543 | } |
| 2544 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2545 | /// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from |
| 2546 | /// the specified lexer will return a tok::l_paren token, 0 if it is something |
| 2547 | /// else and 2 if there are no more tokens in the buffer controlled by the |
| 2548 | /// lexer. |
| 2549 | unsigned Lexer::isNextPPTokenLParen() { |
| 2550 | assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2551 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2552 | // Switch to 'skipping' mode. This will ensure that we can lex a token |
| 2553 | // without emitting diagnostics, disables macro expansion, and will cause EOF |
| 2554 | // to return an EOF token instead of popping the include stack. |
| 2555 | LexingRawMode = true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2556 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2557 | // Save state that can be changed while lexing so that we can restore it. |
| 2558 | const char *TmpBufferPtr = BufferPtr; |
Chris Lattner | 40493eb | 2009-04-24 07:15:46 +0000 | [diff] [blame] | 2559 | bool inPPDirectiveMode = ParsingPreprocessorDirective; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2560 | bool atStartOfLine = IsAtStartOfLine; |
| 2561 | bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; |
| 2562 | bool leadingSpace = HasLeadingSpace; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2563 | |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2564 | Token Tok; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2565 | Lex(Tok); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2566 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2567 | // Restore state that may have changed. |
| 2568 | BufferPtr = TmpBufferPtr; |
Chris Lattner | 40493eb | 2009-04-24 07:15:46 +0000 | [diff] [blame] | 2569 | ParsingPreprocessorDirective = inPPDirectiveMode; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2570 | HasLeadingSpace = leadingSpace; |
| 2571 | IsAtStartOfLine = atStartOfLine; |
| 2572 | IsAtPhysicalStartOfLine = atPhysicalStartOfLine; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2573 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2574 | // Restore the lexer back to non-skipping mode. |
| 2575 | LexingRawMode = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2576 | |
Chris Lattner | 98c1f7c | 2007-10-09 18:02:16 +0000 | [diff] [blame] | 2577 | if (Tok.is(tok::eof)) |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2578 | return 2; |
Chris Lattner | 98c1f7c | 2007-10-09 18:02:16 +0000 | [diff] [blame] | 2579 | return Tok.is(tok::l_paren); |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2580 | } |
| 2581 | |
James Dennett | f442d24 | 2012-06-17 03:40:43 +0000 | [diff] [blame] | 2582 | /// \brief Find the end of a version control conflict marker. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2583 | static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, |
| 2584 | ConflictMarkerKind CMK) { |
| 2585 | const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; |
| 2586 | size_t TermLen = CMK == CMK_Perforce ? 5 : 7; |
| 2587 | StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen); |
| 2588 | size_t Pos = RestOfBuffer.find(Terminator); |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 2589 | while (Pos != StringRef::npos) { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2590 | // Must occur at start of line. |
| 2591 | if (RestOfBuffer[Pos-1] != '\r' && |
| 2592 | RestOfBuffer[Pos-1] != '\n') { |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2593 | RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); |
| 2594 | Pos = RestOfBuffer.find(Terminator); |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2595 | continue; |
| 2596 | } |
| 2597 | return RestOfBuffer.data()+Pos; |
| 2598 | } |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 2599 | return nullptr; |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2600 | } |
| 2601 | |
| 2602 | /// IsStartOfConflictMarker - If the specified pointer is the start of a version |
| 2603 | /// control conflict marker like '<<<<<<<', recognize it as such, emit an error |
| 2604 | /// and recover nicely. This returns true if it is a conflict marker and false |
| 2605 | /// if not. |
| 2606 | bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { |
| 2607 | // Only a conflict marker if it starts at the beginning of a line. |
| 2608 | if (CurPtr != BufferStart && |
| 2609 | CurPtr[-1] != '\n' && CurPtr[-1] != '\r') |
| 2610 | return false; |
| 2611 | |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2612 | // Check to see if we have <<<<<<< or >>>>. |
| 2613 | if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") && |
| 2614 | (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> ")) |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2615 | return false; |
| 2616 | |
| 2617 | // If we have a situation where we don't care about conflict markers, ignore |
| 2618 | // it. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2619 | if (CurrentConflictMarkerState || isLexingRawMode()) |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2620 | return false; |
| 2621 | |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2622 | ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; |
| 2623 | |
| 2624 | // Check to see if there is an ending marker somewhere in the buffer at the |
| 2625 | // start of a line to terminate this conflict marker. |
| 2626 | if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2627 | // We found a match. We are really in a conflict marker. |
| 2628 | // Diagnose this, and ignore to the end of line. |
| 2629 | Diag(CurPtr, diag::err_conflict_marker); |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2630 | CurrentConflictMarkerState = Kind; |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2631 | |
| 2632 | // Skip ahead to the end of line. We know this exists because the |
| 2633 | // end-of-conflict marker starts with \r or \n. |
| 2634 | while (*CurPtr != '\r' && *CurPtr != '\n') { |
| 2635 | assert(CurPtr != BufferEnd && "Didn't find end of line"); |
| 2636 | ++CurPtr; |
| 2637 | } |
| 2638 | BufferPtr = CurPtr; |
| 2639 | return true; |
| 2640 | } |
| 2641 | |
| 2642 | // No end of conflict marker found. |
| 2643 | return false; |
| 2644 | } |
| 2645 | |
| 2646 | |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2647 | /// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if |
| 2648 | /// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it |
| 2649 | /// is the end of a conflict marker. Handle it by ignoring up until the end of |
| 2650 | /// the line. This returns true if it is a conflict marker and false if not. |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2651 | bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { |
| 2652 | // Only a conflict marker if it starts at the beginning of a line. |
| 2653 | if (CurPtr != BufferStart && |
| 2654 | CurPtr[-1] != '\n' && CurPtr[-1] != '\r') |
| 2655 | return false; |
| 2656 | |
| 2657 | // If we have a situation where we don't care about conflict markers, ignore |
| 2658 | // it. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2659 | if (!CurrentConflictMarkerState || isLexingRawMode()) |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2660 | return false; |
| 2661 | |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2662 | // Check to see if we have the marker (4 characters in a row). |
| 2663 | for (unsigned i = 1; i != 4; ++i) |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2664 | if (CurPtr[i] != CurPtr[0]) |
| 2665 | return false; |
| 2666 | |
| 2667 | // If we do have it, search for the end of the conflict marker. This could |
| 2668 | // fail if it got skipped with a '#if 0' or something. Note that CurPtr might |
| 2669 | // be the end of conflict marker. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2670 | if (const char *End = FindConflictEnd(CurPtr, BufferEnd, |
| 2671 | CurrentConflictMarkerState)) { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2672 | CurPtr = End; |
| 2673 | |
| 2674 | // Skip ahead to the end of line. |
| 2675 | while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') |
| 2676 | ++CurPtr; |
| 2677 | |
| 2678 | BufferPtr = CurPtr; |
| 2679 | |
| 2680 | // No longer in the conflict marker. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2681 | CurrentConflictMarkerState = CMK_None; |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2682 | return true; |
| 2683 | } |
| 2684 | |
| 2685 | return false; |
| 2686 | } |
| 2687 | |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2688 | bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { |
| 2689 | if (PP && PP->isCodeCompletionEnabled()) { |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 2690 | SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2691 | return Loc == PP->getCodeCompletionLoc(); |
| 2692 | } |
| 2693 | |
| 2694 | return false; |
| 2695 | } |
| 2696 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2697 | uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, |
| 2698 | Token *Result) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2699 | unsigned CharSize; |
| 2700 | char Kind = getCharAndSize(StartPtr, CharSize); |
| 2701 | |
| 2702 | unsigned NumHexDigits; |
| 2703 | if (Kind == 'u') |
| 2704 | NumHexDigits = 4; |
| 2705 | else if (Kind == 'U') |
| 2706 | NumHexDigits = 8; |
| 2707 | else |
| 2708 | return 0; |
| 2709 | |
Jordan Rose | c0cba27 | 2013-01-27 20:12:04 +0000 | [diff] [blame] | 2710 | if (!LangOpts.CPlusPlus && !LangOpts.C99) { |
Jordan Rose | cccbdbf | 2013-01-28 17:49:02 +0000 | [diff] [blame] | 2711 | if (Result && !isLexingRawMode()) |
| 2712 | Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); |
Jordan Rose | c0cba27 | 2013-01-27 20:12:04 +0000 | [diff] [blame] | 2713 | return 0; |
| 2714 | } |
| 2715 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2716 | const char *CurPtr = StartPtr + CharSize; |
| 2717 | const char *KindLoc = &CurPtr[-1]; |
| 2718 | |
| 2719 | uint32_t CodePoint = 0; |
| 2720 | for (unsigned i = 0; i < NumHexDigits; ++i) { |
| 2721 | char C = getCharAndSize(CurPtr, CharSize); |
| 2722 | |
| 2723 | unsigned Value = llvm::hexDigitValue(C); |
| 2724 | if (Value == -1U) { |
| 2725 | if (Result && !isLexingRawMode()) { |
| 2726 | if (i == 0) { |
| 2727 | Diag(BufferPtr, diag::warn_ucn_escape_no_digits) |
| 2728 | << StringRef(KindLoc, 1); |
| 2729 | } else { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2730 | Diag(BufferPtr, diag::warn_ucn_escape_incomplete); |
Jordan Rose | 62db506 | 2013-01-24 20:50:52 +0000 | [diff] [blame] | 2731 | |
| 2732 | // If the user wrote \U1234, suggest a fixit to \u. |
| 2733 | if (i == 4 && NumHexDigits == 8) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2734 | CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); |
Jordan Rose | 62db506 | 2013-01-24 20:50:52 +0000 | [diff] [blame] | 2735 | Diag(KindLoc, diag::note_ucn_four_not_eight) |
| 2736 | << FixItHint::CreateReplacement(URange, "u"); |
| 2737 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2738 | } |
| 2739 | } |
Jordan Rose | c0cba27 | 2013-01-27 20:12:04 +0000 | [diff] [blame] | 2740 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2741 | return 0; |
| 2742 | } |
| 2743 | |
| 2744 | CodePoint <<= 4; |
| 2745 | CodePoint += Value; |
| 2746 | |
| 2747 | CurPtr += CharSize; |
| 2748 | } |
| 2749 | |
| 2750 | if (Result) { |
| 2751 | Result->setFlag(Token::HasUCN); |
NAKAMURA Takumi | e8f83db | 2013-01-25 14:57:21 +0000 | [diff] [blame] | 2752 | if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2753 | StartPtr = CurPtr; |
| 2754 | else |
| 2755 | while (StartPtr != CurPtr) |
| 2756 | (void)getAndAdvanceChar(StartPtr, *Result); |
| 2757 | } else { |
| 2758 | StartPtr = CurPtr; |
| 2759 | } |
| 2760 | |
Justin Bogner | 5353513 | 2013-10-21 05:02:28 +0000 | [diff] [blame] | 2761 | // Don't apply C family restrictions to UCNs in assembly mode |
| 2762 | if (LangOpts.AsmPreprocessor) |
| 2763 | return CodePoint; |
| 2764 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2765 | // C99 6.4.3p2: A universal character name shall not specify a character whose |
| 2766 | // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or |
| 2767 | // 0060 (`), nor one in the range D800 through DFFF inclusive.) |
| 2768 | // C++11 [lex.charset]p2: If the hexadecimal value for a |
| 2769 | // universal-character-name corresponds to a surrogate code point (in the |
| 2770 | // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, |
| 2771 | // if the hexadecimal value for a universal-character-name outside the |
| 2772 | // c-char-sequence, s-char-sequence, or r-char-sequence of a character or |
| 2773 | // string literal corresponds to a control character (in either of the |
| 2774 | // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the |
| 2775 | // basic source character set, the program is ill-formed. |
| 2776 | if (CodePoint < 0xA0) { |
| 2777 | if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) |
| 2778 | return CodePoint; |
| 2779 | |
| 2780 | // We don't use isLexingRawMode() here because we need to warn about bad |
| 2781 | // UCNs even when skipping preprocessing tokens in a #if block. |
| 2782 | if (Result && PP) { |
| 2783 | if (CodePoint < 0x20 || CodePoint >= 0x7F) |
| 2784 | Diag(BufferPtr, diag::err_ucn_control_character); |
| 2785 | else { |
| 2786 | char C = static_cast<char>(CodePoint); |
| 2787 | Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); |
| 2788 | } |
| 2789 | } |
| 2790 | |
| 2791 | return 0; |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2792 | |
| 2793 | } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2794 | // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2795 | // We don't use isLexingRawMode() here because we need to diagnose bad |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2796 | // UCNs even when skipping preprocessing tokens in a #if block. |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2797 | if (Result && PP) { |
| 2798 | if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) |
| 2799 | Diag(BufferPtr, diag::warn_ucn_escape_surrogate); |
| 2800 | else |
| 2801 | Diag(BufferPtr, diag::err_ucn_escape_invalid); |
| 2802 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2803 | return 0; |
| 2804 | } |
| 2805 | |
| 2806 | return CodePoint; |
| 2807 | } |
| 2808 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2809 | bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C, |
| 2810 | const char *CurPtr) { |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 2811 | static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars( |
| 2812 | UnicodeWhitespaceCharRanges); |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 2813 | if (!isLexingRawMode() && !PP->isPreprocessedOutput() && |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 2814 | UnicodeWhitespaceChars.contains(C)) { |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 2815 | Diag(BufferPtr, diag::ext_unicode_whitespace) |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2816 | << makeCharRange(*this, BufferPtr, CurPtr); |
Jordan Rose | 4246ae0 | 2013-01-24 20:50:50 +0000 | [diff] [blame] | 2817 | |
| 2818 | Result.setFlag(Token::LeadingSpace); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2819 | return true; |
Jordan Rose | 4246ae0 | 2013-01-24 20:50:50 +0000 | [diff] [blame] | 2820 | } |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2821 | return false; |
| 2822 | } |
Jordan Rose | 4246ae0 | 2013-01-24 20:50:50 +0000 | [diff] [blame] | 2823 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2824 | bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2825 | if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { |
| 2826 | if (!isLexingRawMode() && !ParsingPreprocessorDirective && |
| 2827 | !PP->isPreprocessedOutput()) { |
| 2828 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, |
| 2829 | makeCharRange(*this, BufferPtr, CurPtr), |
| 2830 | /*IsFirst=*/true); |
| 2831 | } |
| 2832 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2833 | MIOpt.ReadToken(); |
| 2834 | return LexIdentifier(Result, CurPtr); |
| 2835 | } |
| 2836 | |
Jordan Rose | cc53834 | 2013-01-31 19:48:48 +0000 | [diff] [blame] | 2837 | if (!isLexingRawMode() && !ParsingPreprocessorDirective && |
| 2838 | !PP->isPreprocessedOutput() && |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2839 | !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2840 | // Non-ASCII characters tend to creep into source code unintentionally. |
| 2841 | // Instead of letting the parser complain about the unknown token, |
| 2842 | // just drop the character. |
| 2843 | // Note that we can /only/ do this when the non-ASCII character is actually |
| 2844 | // spelled as Unicode, not written as a UCN. The standard requires that |
| 2845 | // we not throw away any possible preprocessor tokens, but there's a |
| 2846 | // loophole in the mapping of Unicode characters to basic character set |
| 2847 | // characters that allows us to map these particular characters to, say, |
| 2848 | // whitespace. |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 2849 | Diag(BufferPtr, diag::err_non_ascii) |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2850 | << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2851 | |
| 2852 | BufferPtr = CurPtr; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2853 | return false; |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2854 | } |
| 2855 | |
| 2856 | // Otherwise, we have an explicit UCN or a character that's unlikely to show |
| 2857 | // up by accident. |
| 2858 | MIOpt.ReadToken(); |
| 2859 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2860 | return true; |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2861 | } |
| 2862 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2863 | void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { |
| 2864 | IsAtStartOfLine = Result.isAtStartOfLine(); |
| 2865 | HasLeadingSpace = Result.hasLeadingSpace(); |
| 2866 | HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro(); |
| 2867 | // Note that this doesn't affect IsAtPhysicalStartOfLine. |
| 2868 | } |
| 2869 | |
| 2870 | bool Lexer::Lex(Token &Result) { |
| 2871 | // Start a new token. |
| 2872 | Result.startToken(); |
| 2873 | |
| 2874 | // Set up misc whitespace flags for LexTokenInternal. |
| 2875 | if (IsAtStartOfLine) { |
| 2876 | Result.setFlag(Token::StartOfLine); |
| 2877 | IsAtStartOfLine = false; |
| 2878 | } |
| 2879 | |
| 2880 | if (HasLeadingSpace) { |
| 2881 | Result.setFlag(Token::LeadingSpace); |
| 2882 | HasLeadingSpace = false; |
| 2883 | } |
| 2884 | |
| 2885 | if (HasLeadingEmptyMacro) { |
| 2886 | Result.setFlag(Token::LeadingEmptyMacro); |
| 2887 | HasLeadingEmptyMacro = false; |
| 2888 | } |
| 2889 | |
| 2890 | bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; |
| 2891 | IsAtPhysicalStartOfLine = false; |
Eli Friedman | 29749d2 | 2013-09-19 01:51:23 +0000 | [diff] [blame] | 2892 | bool isRawLex = isLexingRawMode(); |
| 2893 | (void) isRawLex; |
| 2894 | bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine); |
| 2895 | // (After the LexTokenInternal call, the lexer might be destroyed.) |
| 2896 | assert((returnedToken || !isRawLex) && "Raw lex must succeed"); |
| 2897 | return returnedToken; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2898 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2899 | |
| 2900 | /// LexTokenInternal - This implements a simple C family lexer. It is an |
| 2901 | /// extremely performance critical piece of code. This assumes that the buffer |
Chris Lattner | 5c34938 | 2009-07-07 05:05:42 +0000 | [diff] [blame] | 2902 | /// has a null character at the end of the file. This returns a preprocessing |
| 2903 | /// token, not a normal token, as such, it is an internal interface. It assumes |
| 2904 | /// that the Flags of result have been cleared before calling this. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2905 | bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2906 | LexNextToken: |
| 2907 | // New token, can't need cleaning yet. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2908 | Result.clearFlag(Token::NeedsCleaning); |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 2909 | Result.setIdentifierInfo(nullptr); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2910 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2911 | // CurPtr - Cache BufferPtr in an automatic variable. |
| 2912 | const char *CurPtr = BufferPtr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2913 | |
Chris Lattner | eb54b59 | 2006-07-10 06:34:27 +0000 | [diff] [blame] | 2914 | // Small amounts of horizontal whitespace is very common between tokens. |
| 2915 | if ((*CurPtr == ' ') || (*CurPtr == '\t')) { |
| 2916 | ++CurPtr; |
| 2917 | while ((*CurPtr == ' ') || (*CurPtr == '\t')) |
| 2918 | ++CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2919 | |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2920 | // If we are keeping whitespace and other tokens, just return what we just |
| 2921 | // skipped. The next lexer invocation will return the token after the |
| 2922 | // whitespace. |
| 2923 | if (isKeepWhitespaceMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2924 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2925 | // FIXME: The next token will not have LeadingSpace set. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2926 | return true; |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2927 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2928 | |
Chris Lattner | eb54b59 | 2006-07-10 06:34:27 +0000 | [diff] [blame] | 2929 | BufferPtr = CurPtr; |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2930 | Result.setFlag(Token::LeadingSpace); |
Chris Lattner | eb54b59 | 2006-07-10 06:34:27 +0000 | [diff] [blame] | 2931 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2932 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2933 | unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2934 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2935 | // Read a character, advancing over it. |
| 2936 | char Char = getAndAdvanceChar(CurPtr, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2937 | tok::TokenKind Kind; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2938 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2939 | switch (Char) { |
| 2940 | case 0: // Null. |
| 2941 | // Found end of file? |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2942 | if (CurPtr-1 == BufferEnd) |
| 2943 | return LexEndOfFile(Result, CurPtr-1); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2944 | |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2945 | // Check if we are performing code completion. |
| 2946 | if (isCodeCompletionPoint(CurPtr-1)) { |
| 2947 | // Return the code-completion token. |
| 2948 | Result.startToken(); |
| 2949 | FormTokenWithChars(Result, CurPtr, tok::code_completion); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2950 | return true; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2951 | } |
| 2952 | |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2953 | if (!isLexingRawMode()) |
| 2954 | Diag(CurPtr-1, diag::null_in_file); |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2955 | Result.setFlag(Token::LeadingSpace); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2956 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 2957 | return true; // KeepWhitespaceMode |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2958 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2959 | // We know the lexer hasn't changed, so just try again with this lexer. |
| 2960 | // (We manually eliminate the tail call to avoid recursion.) |
| 2961 | goto LexNextToken; |
Chris Lattner | 3dfff97 | 2009-12-17 05:29:40 +0000 | [diff] [blame] | 2962 | |
| 2963 | case 26: // DOS & CP/M EOF: "^Z". |
| 2964 | // If we're in Microsoft extensions mode, treat this as end of file. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2965 | if (LangOpts.MicrosoftExt) |
| 2966 | return LexEndOfFile(Result, CurPtr-1); |
| 2967 | |
Chris Lattner | 3dfff97 | 2009-12-17 05:29:40 +0000 | [diff] [blame] | 2968 | // If Microsoft extensions are disabled, this is just random garbage. |
| 2969 | Kind = tok::unknown; |
| 2970 | break; |
| 2971 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2972 | case '\n': |
| 2973 | case '\r': |
| 2974 | // If we are inside a preprocessor directive and we see the end of line, |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2975 | // we know we are done with the directive, so return an EOD token. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2976 | if (ParsingPreprocessorDirective) { |
| 2977 | // Done parsing the "line". |
| 2978 | ParsingPreprocessorDirective = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2979 | |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2980 | // Restore comment saving mode, in case it was disabled for directive. |
David Blaikie | 2af2b30 | 2012-06-15 00:47:13 +0000 | [diff] [blame] | 2981 | if (PP) |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2982 | resetExtendedTokenMode(); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2983 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2984 | // Since we consumed a newline, we are back at the start of a line. |
| 2985 | IsAtStartOfLine = true; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2986 | IsAtPhysicalStartOfLine = true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2987 | |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2988 | Kind = tok::eod; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2989 | break; |
| 2990 | } |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2991 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2992 | // No leading whitespace seen so far. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2993 | Result.clearFlag(Token::LeadingSpace); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2994 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2995 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 2996 | return true; // KeepWhitespaceMode |
| 2997 | |
| 2998 | // We only saw whitespace, so just try again with this lexer. |
| 2999 | // (We manually eliminate the tail call to avoid recursion.) |
| 3000 | goto LexNextToken; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3001 | case ' ': |
| 3002 | case '\t': |
| 3003 | case '\f': |
| 3004 | case '\v': |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3005 | SkipHorizontalWhitespace: |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 3006 | Result.setFlag(Token::LeadingSpace); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3007 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 3008 | return true; // KeepWhitespaceMode |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3009 | |
| 3010 | SkipIgnoredUnits: |
| 3011 | CurPtr = BufferPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3012 | |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3013 | // If the next token is obviously a // or /* */ comment, skip it efficiently |
| 3014 | // too (without going through the big switch stmt). |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3015 | if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && |
Eli Friedman | cefc7ea | 2013-08-28 20:53:32 +0000 | [diff] [blame] | 3016 | LangOpts.LineComment && |
| 3017 | (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3018 | if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) |
| 3019 | return true; // There is a token to return. |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3020 | goto SkipIgnoredUnits; |
Chris Lattner | 8637abd | 2008-10-12 03:22:02 +0000 | [diff] [blame] | 3021 | } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3022 | if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) |
| 3023 | return true; // There is a token to return. |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3024 | goto SkipIgnoredUnits; |
| 3025 | } else if (isHorizontalWhitespace(*CurPtr)) { |
| 3026 | goto SkipHorizontalWhitespace; |
| 3027 | } |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3028 | // We only saw whitespace, so just try again with this lexer. |
| 3029 | // (We manually eliminate the tail call to avoid recursion.) |
| 3030 | goto LexNextToken; |
Chris Lattner | 3dfff97 | 2009-12-17 05:29:40 +0000 | [diff] [blame] | 3031 | |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3032 | // C99 6.4.4.1: Integer Constants. |
| 3033 | // C99 6.4.4.2: Floating Constants. |
| 3034 | case '0': case '1': case '2': case '3': case '4': |
| 3035 | case '5': case '6': case '7': case '8': case '9': |
| 3036 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3037 | MIOpt.ReadToken(); |
| 3038 | return LexNumericConstant(Result, CurPtr); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3039 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3040 | case 'u': // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3041 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3042 | MIOpt.ReadToken(); |
| 3043 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3044 | if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3045 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3046 | |
| 3047 | // UTF-16 string literal |
| 3048 | if (Char == '"') |
| 3049 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3050 | tok::utf16_string_literal); |
| 3051 | |
| 3052 | // UTF-16 character constant |
| 3053 | if (Char == '\'') |
| 3054 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3055 | tok::utf16_char_constant); |
| 3056 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3057 | // UTF-16 raw string literal |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3058 | if (Char == 'R' && LangOpts.CPlusPlus11 && |
| 3059 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3060 | return LexRawStringLiteral(Result, |
| 3061 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3062 | SizeTmp2, Result), |
| 3063 | tok::utf16_string_literal); |
| 3064 | |
| 3065 | if (Char == '8') { |
| 3066 | char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); |
| 3067 | |
| 3068 | // UTF-8 string literal |
| 3069 | if (Char2 == '"') |
| 3070 | return LexStringLiteral(Result, |
| 3071 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3072 | SizeTmp2, Result), |
| 3073 | tok::utf8_string_literal); |
Richard Smith | 3e3a705 | 2014-11-08 06:08:42 +0000 | [diff] [blame] | 3074 | if (Char2 == '\'' && LangOpts.CPlusPlus1z) |
| 3075 | return LexCharConstant( |
| 3076 | Result, ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3077 | SizeTmp2, Result), |
| 3078 | tok::utf8_char_constant); |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3079 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3080 | if (Char2 == 'R' && LangOpts.CPlusPlus11) { |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3081 | unsigned SizeTmp3; |
| 3082 | char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); |
| 3083 | // UTF-8 raw string literal |
| 3084 | if (Char3 == '"') { |
| 3085 | return LexRawStringLiteral(Result, |
| 3086 | ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3087 | SizeTmp2, Result), |
| 3088 | SizeTmp3, Result), |
| 3089 | tok::utf8_string_literal); |
| 3090 | } |
| 3091 | } |
| 3092 | } |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3093 | } |
| 3094 | |
| 3095 | // treat u like the start of an identifier. |
| 3096 | return LexIdentifier(Result, CurPtr); |
| 3097 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3098 | case 'U': // Identifier (Uber) or C11/C++11 UTF-32 string literal |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3099 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3100 | MIOpt.ReadToken(); |
| 3101 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3102 | if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3103 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3104 | |
| 3105 | // UTF-32 string literal |
| 3106 | if (Char == '"') |
| 3107 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3108 | tok::utf32_string_literal); |
| 3109 | |
| 3110 | // UTF-32 character constant |
| 3111 | if (Char == '\'') |
| 3112 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3113 | tok::utf32_char_constant); |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3114 | |
| 3115 | // UTF-32 raw string literal |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3116 | if (Char == 'R' && LangOpts.CPlusPlus11 && |
| 3117 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3118 | return LexRawStringLiteral(Result, |
| 3119 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3120 | SizeTmp2, Result), |
| 3121 | tok::utf32_string_literal); |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3122 | } |
| 3123 | |
| 3124 | // treat U like the start of an identifier. |
| 3125 | return LexIdentifier(Result, CurPtr); |
| 3126 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3127 | case 'R': // Identifier or C++0x raw string literal |
| 3128 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3129 | MIOpt.ReadToken(); |
| 3130 | |
Richard Smith | 2bf7fdb | 2013-01-02 11:42:31 +0000 | [diff] [blame] | 3131 | if (LangOpts.CPlusPlus11) { |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3132 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3133 | |
| 3134 | if (Char == '"') |
| 3135 | return LexRawStringLiteral(Result, |
| 3136 | ConsumeChar(CurPtr, SizeTmp, Result), |
| 3137 | tok::string_literal); |
| 3138 | } |
| 3139 | |
| 3140 | // treat R like the start of an identifier. |
| 3141 | return LexIdentifier(Result, CurPtr); |
| 3142 | |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3143 | case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3144 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3145 | MIOpt.ReadToken(); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3146 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3147 | |
| 3148 | // Wide string literal. |
| 3149 | if (Char == '"') |
Chris Lattner | d3e9895 | 2006-10-06 05:22:26 +0000 | [diff] [blame] | 3150 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3151 | tok::wide_string_literal); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3152 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3153 | // Wide raw string literal. |
Richard Smith | 2bf7fdb | 2013-01-02 11:42:31 +0000 | [diff] [blame] | 3154 | if (LangOpts.CPlusPlus11 && Char == 'R' && |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3155 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
| 3156 | return LexRawStringLiteral(Result, |
| 3157 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3158 | SizeTmp2, Result), |
| 3159 | tok::wide_string_literal); |
| 3160 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3161 | // Wide character constant. |
| 3162 | if (Char == '\'') |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3163 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3164 | tok::wide_char_constant); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3165 | // FALL THROUGH, treating L like the start of an identifier. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3166 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3167 | // C99 6.4.2: Identifiers. |
| 3168 | case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': |
| 3169 | case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3170 | case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3171 | case 'V': case 'W': case 'X': case 'Y': case 'Z': |
| 3172 | case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': |
| 3173 | case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3174 | case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3175 | case 'v': case 'w': case 'x': case 'y': case 'z': |
| 3176 | case '_': |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3177 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3178 | MIOpt.ReadToken(); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3179 | return LexIdentifier(Result, CurPtr); |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3180 | |
| 3181 | case '$': // $ in identifiers. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3182 | if (LangOpts.DollarIdents) { |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 3183 | if (!isLexingRawMode()) |
| 3184 | Diag(CurPtr-1, diag::ext_dollar_in_identifier); |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3185 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3186 | MIOpt.ReadToken(); |
| 3187 | return LexIdentifier(Result, CurPtr); |
| 3188 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3189 | |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3190 | Kind = tok::unknown; |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3191 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3192 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3193 | // C99 6.4.4: Character Constants. |
| 3194 | case '\'': |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3195 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3196 | MIOpt.ReadToken(); |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3197 | return LexCharConstant(Result, CurPtr, tok::char_constant); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3198 | |
| 3199 | // C99 6.4.5: String Literals. |
| 3200 | case '"': |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3201 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3202 | MIOpt.ReadToken(); |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3203 | return LexStringLiteral(Result, CurPtr, tok::string_literal); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3204 | |
| 3205 | // C99 6.4.6: Punctuators. |
| 3206 | case '?': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3207 | Kind = tok::question; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3208 | break; |
| 3209 | case '[': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3210 | Kind = tok::l_square; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3211 | break; |
| 3212 | case ']': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3213 | Kind = tok::r_square; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3214 | break; |
| 3215 | case '(': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3216 | Kind = tok::l_paren; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3217 | break; |
| 3218 | case ')': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3219 | Kind = tok::r_paren; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3220 | break; |
| 3221 | case '{': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3222 | Kind = tok::l_brace; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3223 | break; |
| 3224 | case '}': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3225 | Kind = tok::r_brace; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3226 | break; |
| 3227 | case '.': |
| 3228 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3229 | if (Char >= '0' && Char <= '9') { |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3230 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3231 | MIOpt.ReadToken(); |
| 3232 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3233 | return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3234 | } else if (LangOpts.CPlusPlus && Char == '*') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3235 | Kind = tok::periodstar; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3236 | CurPtr += SizeTmp; |
| 3237 | } else if (Char == '.' && |
| 3238 | getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3239 | Kind = tok::ellipsis; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3240 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3241 | SizeTmp2, Result); |
| 3242 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3243 | Kind = tok::period; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3244 | } |
| 3245 | break; |
| 3246 | case '&': |
| 3247 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3248 | if (Char == '&') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3249 | Kind = tok::ampamp; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3250 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3251 | } else if (Char == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3252 | Kind = tok::ampequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3253 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3254 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3255 | Kind = tok::amp; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3256 | } |
| 3257 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3258 | case '*': |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3259 | if (getCharAndSize(CurPtr, SizeTmp) == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3260 | Kind = tok::starequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3261 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3262 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3263 | Kind = tok::star; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3264 | } |
| 3265 | break; |
| 3266 | case '+': |
| 3267 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3268 | if (Char == '+') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3269 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3270 | Kind = tok::plusplus; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3271 | } else if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3272 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3273 | Kind = tok::plusequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3274 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3275 | Kind = tok::plus; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3276 | } |
| 3277 | break; |
| 3278 | case '-': |
| 3279 | Char = getCharAndSize(CurPtr, SizeTmp); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3280 | if (Char == '-') { // -- |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3281 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3282 | Kind = tok::minusminus; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3283 | } else if (Char == '>' && LangOpts.CPlusPlus && |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3284 | getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3285 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3286 | SizeTmp2, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3287 | Kind = tok::arrowstar; |
| 3288 | } else if (Char == '>') { // -> |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3289 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3290 | Kind = tok::arrow; |
| 3291 | } else if (Char == '=') { // -= |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3292 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3293 | Kind = tok::minusequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3294 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3295 | Kind = tok::minus; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3296 | } |
| 3297 | break; |
| 3298 | case '~': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3299 | Kind = tok::tilde; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3300 | break; |
| 3301 | case '!': |
| 3302 | if (getCharAndSize(CurPtr, SizeTmp) == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3303 | Kind = tok::exclaimequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3304 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3305 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3306 | Kind = tok::exclaim; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3307 | } |
| 3308 | break; |
| 3309 | case '/': |
| 3310 | // 6.4.9: Comments |
| 3311 | Char = getCharAndSize(CurPtr, SizeTmp); |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 3312 | if (Char == '/') { // Line comment. |
| 3313 | // Even if Line comments are disabled (e.g. in C89 mode), we generally |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3314 | // want to lex this as a comment. There is one problem with this though, |
| 3315 | // that in one particular corner case, this can change the behavior of the |
| 3316 | // resultant program. For example, In "foo //**/ bar", C89 would lex |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 3317 | // this as "foo / bar" and langauges with Line comments would lex it as |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3318 | // "foo". Check to see if the character after the second slash is a '*'. |
| 3319 | // If so, we will lex that as a "/" instead of the start of a comment. |
Jordan Rose | 864b810 | 2013-03-05 22:51:04 +0000 | [diff] [blame] | 3320 | // However, we never do this if we are just preprocessing. |
Eli Friedman | cefc7ea | 2013-08-28 20:53:32 +0000 | [diff] [blame] | 3321 | bool TreatAsComment = LangOpts.LineComment && |
| 3322 | (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP); |
Jordan Rose | 864b810 | 2013-03-05 22:51:04 +0000 | [diff] [blame] | 3323 | if (!TreatAsComment) |
| 3324 | if (!(PP && PP->isPreprocessedOutput())) |
| 3325 | TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; |
| 3326 | |
| 3327 | if (TreatAsComment) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3328 | if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3329 | TokAtPhysicalStartOfLine)) |
| 3330 | return true; // There is a token to return. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3331 | |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3332 | // It is common for the tokens immediately after a // comment to be |
| 3333 | // whitespace (indentation for the next line). Instead of going through |
| 3334 | // the big switch, handle it efficiently now. |
| 3335 | goto SkipIgnoredUnits; |
| 3336 | } |
| 3337 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3338 | |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3339 | if (Char == '*') { // /**/ comment. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3340 | if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3341 | TokAtPhysicalStartOfLine)) |
| 3342 | return true; // There is a token to return. |
| 3343 | |
| 3344 | // We only saw whitespace, so just try again with this lexer. |
| 3345 | // (We manually eliminate the tail call to avoid recursion.) |
| 3346 | goto LexNextToken; |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3347 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3348 | |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3349 | if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3350 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3351 | Kind = tok::slashequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3352 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3353 | Kind = tok::slash; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3354 | } |
| 3355 | break; |
| 3356 | case '%': |
| 3357 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3358 | if (Char == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3359 | Kind = tok::percentequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3360 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3361 | } else if (LangOpts.Digraphs && Char == '>') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3362 | Kind = tok::r_brace; // '%>' -> '}' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3363 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3364 | } else if (LangOpts.Digraphs && Char == ':') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3365 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | 2b271db | 2006-07-15 05:41:09 +0000 | [diff] [blame] | 3366 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3367 | if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3368 | Kind = tok::hashhash; // '%:%:' -> '##' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3369 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3370 | SizeTmp2, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3371 | } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize |
Chris Lattner | 2b271db | 2006-07-15 05:41:09 +0000 | [diff] [blame] | 3372 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 3373 | if (!isLexingRawMode()) |
Ted Kremenek | a08713c | 2011-10-17 21:47:53 +0000 | [diff] [blame] | 3374 | Diag(BufferPtr, diag::ext_charize_microsoft); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3375 | Kind = tok::hashat; |
Chris Lattner | 2534324a | 2009-03-18 20:58:27 +0000 | [diff] [blame] | 3376 | } else { // '%:' -> '#' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3377 | // We parsed a # character. If this occurs at the start of the line, |
| 3378 | // it's actually the start of a preprocessing directive. Callback to |
| 3379 | // the preprocessor to handle it. |
Alp Toker | 7755aff | 2014-05-18 18:37:59 +0000 | [diff] [blame] | 3380 | // TODO: -fpreprocessed mode?? |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3381 | if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) |
Argyrios Kyrtzidis | 36675b7 | 2012-11-13 01:02:40 +0000 | [diff] [blame] | 3382 | goto HandleDirective; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3383 | |
Chris Lattner | 2534324a | 2009-03-18 20:58:27 +0000 | [diff] [blame] | 3384 | Kind = tok::hash; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3385 | } |
| 3386 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3387 | Kind = tok::percent; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3388 | } |
| 3389 | break; |
| 3390 | case '<': |
| 3391 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3392 | if (ParsingFilename) { |
Chris Lattner | b40289b | 2009-04-17 23:56:52 +0000 | [diff] [blame] | 3393 | return LexAngledStringLiteral(Result, CurPtr); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3394 | } else if (Char == '<') { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3395 | char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); |
| 3396 | if (After == '=') { |
| 3397 | Kind = tok::lesslessequal; |
| 3398 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3399 | SizeTmp2, Result); |
| 3400 | } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { |
| 3401 | // If this is actually a '<<<<<<<' version control conflict marker, |
| 3402 | // recognize it as such and recover nicely. |
| 3403 | goto LexNextToken; |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 3404 | } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { |
| 3405 | // If this is '<<<<' and we're in a Perforce-style conflict marker, |
| 3406 | // ignore it. |
| 3407 | goto LexNextToken; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3408 | } else if (LangOpts.CUDA && After == '<') { |
Peter Collingbourne | c1270f5 | 2011-02-09 21:08:21 +0000 | [diff] [blame] | 3409 | Kind = tok::lesslessless; |
| 3410 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3411 | SizeTmp2, Result); |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3412 | } else { |
| 3413 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3414 | Kind = tok::lessless; |
| 3415 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3416 | } else if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3417 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3418 | Kind = tok::lessequal; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3419 | } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' |
Richard Smith | 2bf7fdb | 2013-01-02 11:42:31 +0000 | [diff] [blame] | 3420 | if (LangOpts.CPlusPlus11 && |
Richard Smith | f7b6202 | 2011-04-14 18:36:27 +0000 | [diff] [blame] | 3421 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { |
| 3422 | // C++0x [lex.pptoken]p3: |
| 3423 | // Otherwise, if the next three characters are <:: and the subsequent |
| 3424 | // character is neither : nor >, the < is treated as a preprocessor |
| 3425 | // token by itself and not as the first character of the alternative |
| 3426 | // token <:. |
| 3427 | unsigned SizeTmp3; |
| 3428 | char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); |
| 3429 | if (After != ':' && After != '>') { |
| 3430 | Kind = tok::less; |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 3431 | if (!isLexingRawMode()) |
| 3432 | Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); |
Richard Smith | f7b6202 | 2011-04-14 18:36:27 +0000 | [diff] [blame] | 3433 | break; |
| 3434 | } |
| 3435 | } |
| 3436 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3437 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3438 | Kind = tok::l_square; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3439 | } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3440 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3441 | Kind = tok::l_brace; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3442 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3443 | Kind = tok::less; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3444 | } |
| 3445 | break; |
| 3446 | case '>': |
| 3447 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3448 | if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3449 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3450 | Kind = tok::greaterequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3451 | } else if (Char == '>') { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3452 | char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); |
| 3453 | if (After == '=') { |
| 3454 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3455 | SizeTmp2, Result); |
| 3456 | Kind = tok::greatergreaterequal; |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 3457 | } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { |
| 3458 | // If this is actually a '>>>>' conflict marker, recognize it as such |
| 3459 | // and recover nicely. |
| 3460 | goto LexNextToken; |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3461 | } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { |
| 3462 | // If this is '>>>>>>>' and we're in a conflict marker, ignore it. |
| 3463 | goto LexNextToken; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3464 | } else if (LangOpts.CUDA && After == '>') { |
Peter Collingbourne | c1270f5 | 2011-02-09 21:08:21 +0000 | [diff] [blame] | 3465 | Kind = tok::greatergreatergreater; |
| 3466 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3467 | SizeTmp2, Result); |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3468 | } else { |
| 3469 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3470 | Kind = tok::greatergreater; |
| 3471 | } |
| 3472 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3473 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3474 | Kind = tok::greater; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3475 | } |
| 3476 | break; |
| 3477 | case '^': |
| 3478 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3479 | if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3480 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3481 | Kind = tok::caretequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3482 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3483 | Kind = tok::caret; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3484 | } |
| 3485 | break; |
| 3486 | case '|': |
| 3487 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3488 | if (Char == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3489 | Kind = tok::pipeequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3490 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3491 | } else if (Char == '|') { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3492 | // If this is '|||||||' and we're in a conflict marker, ignore it. |
| 3493 | if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) |
| 3494 | goto LexNextToken; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3495 | Kind = tok::pipepipe; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3496 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3497 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3498 | Kind = tok::pipe; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3499 | } |
| 3500 | break; |
| 3501 | case ':': |
| 3502 | Char = getCharAndSize(CurPtr, SizeTmp); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3503 | if (LangOpts.Digraphs && Char == '>') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3504 | Kind = tok::r_square; // ':>' -> ']' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3505 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3506 | } else if (LangOpts.CPlusPlus && Char == ':') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3507 | Kind = tok::coloncolon; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3508 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3509 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3510 | Kind = tok::colon; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3511 | } |
| 3512 | break; |
| 3513 | case ';': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3514 | Kind = tok::semi; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3515 | break; |
| 3516 | case '=': |
| 3517 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3518 | if (Char == '=') { |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 3519 | // If this is '====' and we're in a conflict marker, ignore it. |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3520 | if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) |
| 3521 | goto LexNextToken; |
| 3522 | |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3523 | Kind = tok::equalequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3524 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3525 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3526 | Kind = tok::equal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3527 | } |
| 3528 | break; |
| 3529 | case ',': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3530 | Kind = tok::comma; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3531 | break; |
| 3532 | case '#': |
| 3533 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3534 | if (Char == '#') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3535 | Kind = tok::hashhash; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3536 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3537 | } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3538 | Kind = tok::hashat; |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 3539 | if (!isLexingRawMode()) |
Ted Kremenek | a08713c | 2011-10-17 21:47:53 +0000 | [diff] [blame] | 3540 | Diag(BufferPtr, diag::ext_charize_microsoft); |
Chris Lattner | 2b271db | 2006-07-15 05:41:09 +0000 | [diff] [blame] | 3541 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3542 | } else { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3543 | // We parsed a # character. If this occurs at the start of the line, |
| 3544 | // it's actually the start of a preprocessing directive. Callback to |
| 3545 | // the preprocessor to handle it. |
Alp Toker | 7755aff | 2014-05-18 18:37:59 +0000 | [diff] [blame] | 3546 | // TODO: -fpreprocessed mode?? |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3547 | if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) |
Argyrios Kyrtzidis | 36675b7 | 2012-11-13 01:02:40 +0000 | [diff] [blame] | 3548 | goto HandleDirective; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3549 | |
Chris Lattner | 2534324a | 2009-03-18 20:58:27 +0000 | [diff] [blame] | 3550 | Kind = tok::hash; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3551 | } |
| 3552 | break; |
| 3553 | |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3554 | case '@': |
| 3555 | // Objective C support. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3556 | if (CurPtr[-1] == '@' && LangOpts.ObjC1) |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3557 | Kind = tok::at; |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3558 | else |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3559 | Kind = tok::unknown; |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3560 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3561 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3562 | // UCNs (C99 6.4.3, C++11 [lex.charset]p2) |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3563 | case '\\': |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3564 | if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) { |
| 3565 | if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { |
| 3566 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 3567 | return true; // KeepWhitespaceMode |
| 3568 | |
| 3569 | // We only saw whitespace, so just try again with this lexer. |
| 3570 | // (We manually eliminate the tail call to avoid recursion.) |
| 3571 | goto LexNextToken; |
| 3572 | } |
| 3573 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3574 | return LexUnicode(Result, CodePoint, CurPtr); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3575 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3576 | |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3577 | Kind = tok::unknown; |
Chris Lattner | 041bef8 | 2006-07-11 05:52:53 +0000 | [diff] [blame] | 3578 | break; |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3579 | |
| 3580 | default: { |
| 3581 | if (isASCII(Char)) { |
| 3582 | Kind = tok::unknown; |
| 3583 | break; |
| 3584 | } |
| 3585 | |
| 3586 | UTF32 CodePoint; |
| 3587 | |
| 3588 | // We can't just reset CurPtr to BufferPtr because BufferPtr may point to |
| 3589 | // an escaped newline. |
| 3590 | --CurPtr; |
Dmitri Gribenko | 9feeef4 | 2013-01-30 12:06:08 +0000 | [diff] [blame] | 3591 | ConversionResult Status = |
| 3592 | llvm::convertUTF8Sequence((const UTF8 **)&CurPtr, |
| 3593 | (const UTF8 *)BufferEnd, |
| 3594 | &CodePoint, |
| 3595 | strictConversion); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3596 | if (Status == conversionOK) { |
| 3597 | if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { |
| 3598 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 3599 | return true; // KeepWhitespaceMode |
| 3600 | |
| 3601 | // We only saw whitespace, so just try again with this lexer. |
| 3602 | // (We manually eliminate the tail call to avoid recursion.) |
| 3603 | goto LexNextToken; |
| 3604 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3605 | return LexUnicode(Result, CodePoint, CurPtr); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3606 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3607 | |
Jordan Rose | cc53834 | 2013-01-31 19:48:48 +0000 | [diff] [blame] | 3608 | if (isLexingRawMode() || ParsingPreprocessorDirective || |
| 3609 | PP->isPreprocessedOutput()) { |
Jordan Rose | f649795 | 2013-01-30 19:21:12 +0000 | [diff] [blame] | 3610 | ++CurPtr; |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 3611 | Kind = tok::unknown; |
| 3612 | break; |
| 3613 | } |
| 3614 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3615 | // Non-ASCII characters tend to creep into source code unintentionally. |
| 3616 | // Instead of letting the parser complain about the unknown token, |
Jordan Rose | 8b4af2a | 2013-01-25 00:20:28 +0000 | [diff] [blame] | 3617 | // just diagnose the invalid UTF-8, then drop the character. |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 3618 | Diag(CurPtr, diag::err_invalid_utf8); |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3619 | |
| 3620 | BufferPtr = CurPtr+1; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3621 | // We're pretending the character didn't exist, so just try again with |
| 3622 | // this lexer. |
| 3623 | // (We manually eliminate the tail call to avoid recursion.) |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3624 | goto LexNextToken; |
| 3625 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3626 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3627 | |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3628 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3629 | MIOpt.ReadToken(); |
| 3630 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 3631 | // Update the location of token as well as BufferPtr. |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3632 | FormTokenWithChars(Result, CurPtr, Kind); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3633 | return true; |
Argyrios Kyrtzidis | 36675b7 | 2012-11-13 01:02:40 +0000 | [diff] [blame] | 3634 | |
| 3635 | HandleDirective: |
| 3636 | // We parsed a # character and it's the start of a preprocessing directive. |
| 3637 | |
| 3638 | FormTokenWithChars(Result, CurPtr, tok::hash); |
| 3639 | PP->HandleDirective(Result); |
| 3640 | |
Argyrios Kyrtzidis | dc9fdaf | 2013-05-24 05:44:08 +0000 | [diff] [blame] | 3641 | if (PP->hadModuleLoaderFatalFailure()) { |
| 3642 | // With a fatal failure in the module loader, we abort parsing. |
| 3643 | assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof"); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3644 | return true; |
Argyrios Kyrtzidis | dc9fdaf | 2013-05-24 05:44:08 +0000 | [diff] [blame] | 3645 | } |
| 3646 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3647 | // We parsed the directive; lex a token with the new state. |
| 3648 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3649 | } |