Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 1 | //===- Lexer.cpp - C Language Family Lexer --------------------------------===// |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
Chris Lattner | 5b12ab8 | 2007-12-29 19:59:25 +0000 | [diff] [blame] | 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 10 | // This file implements the Lexer and Token interfaces. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 13 | |
| 14 | #include "clang/Lex/Lexer.h" |
Chandler Carruth | 5553d0d | 2014-01-07 11:51:46 +0000 | [diff] [blame] | 15 | #include "UnicodeCharSets.h" |
Jordan Rose | a2100d7 | 2013-02-08 22:30:22 +0000 | [diff] [blame] | 16 | #include "clang/Basic/CharInfo.h" |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 17 | #include "clang/Basic/IdentifierTable.h" |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 18 | #include "clang/Basic/LangOptions.h" |
| 19 | #include "clang/Basic/SourceLocation.h" |
Chris Lattner | dc5c055 | 2007-07-20 16:37:10 +0000 | [diff] [blame] | 20 | #include "clang/Basic/SourceManager.h" |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 21 | #include "clang/Basic/TokenKinds.h" |
Chandler Carruth | 3a02247 | 2012-12-04 09:13:33 +0000 | [diff] [blame] | 22 | #include "clang/Lex/LexDiagnostic.h" |
Richard Smith | 2a98862 | 2013-09-24 04:06:10 +0000 | [diff] [blame] | 23 | #include "clang/Lex/LiteralSupport.h" |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 24 | #include "clang/Lex/MultipleIncludeOpt.h" |
Chandler Carruth | 3a02247 | 2012-12-04 09:13:33 +0000 | [diff] [blame] | 25 | #include "clang/Lex/Preprocessor.h" |
Alex Lorenz | fb7654a | 2017-06-16 20:13:39 +0000 | [diff] [blame] | 26 | #include "clang/Lex/PreprocessorOptions.h" |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 27 | #include "clang/Lex/Token.h" |
| 28 | #include "clang/Basic/Diagnostic.h" |
| 29 | #include "clang/Basic/LLVM.h" |
| 30 | #include "clang/Basic/TokenKinds.h" |
| 31 | #include "llvm/ADT/None.h" |
| 32 | #include "llvm/ADT/Optional.h" |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 33 | #include "llvm/ADT/StringExtras.h" |
Chandler Carruth | 3a02247 | 2012-12-04 09:13:33 +0000 | [diff] [blame] | 34 | #include "llvm/ADT/StringSwitch.h" |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 35 | #include "llvm/ADT/StringRef.h" |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 36 | #include "llvm/Support/Compiler.h" |
Dmitri Gribenko | 9feeef4 | 2013-01-30 12:06:08 +0000 | [diff] [blame] | 37 | #include "llvm/Support/ConvertUTF.h" |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 38 | #include "llvm/Support/MathExtras.h" |
Chris Lattner | 739e739 | 2007-04-29 07:12:06 +0000 | [diff] [blame] | 39 | #include "llvm/Support/MemoryBuffer.h" |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 40 | #include "llvm/Support/UnicodeCharRanges.h" |
| 41 | #include <algorithm> |
| 42 | #include <cassert> |
| 43 | #include <cstddef> |
| 44 | #include <cstdint> |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 45 | #include <cstring> |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 46 | #include <string> |
| 47 | #include <tuple> |
| 48 | #include <utility> |
| 49 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 50 | using namespace clang; |
| 51 | |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 52 | //===----------------------------------------------------------------------===// |
| 53 | // Token Class Implementation |
| 54 | //===----------------------------------------------------------------------===// |
| 55 | |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 56 | /// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 57 | bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { |
Alex Lorenz | d475466 | 2017-05-17 11:08:36 +0000 | [diff] [blame] | 58 | if (isAnnotation()) |
| 59 | return false; |
Douglas Gregor | 90abb6d | 2008-12-01 21:46:47 +0000 | [diff] [blame] | 60 | if (IdentifierInfo *II = getIdentifierInfo()) |
| 61 | return II->getObjCKeywordID() == objcKey; |
| 62 | return false; |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 63 | } |
| 64 | |
| 65 | /// getObjCKeywordID - Return the ObjC keyword kind. |
| 66 | tok::ObjCKeywordKind Token::getObjCKeywordID() const { |
Alex Lorenz | d475466 | 2017-05-17 11:08:36 +0000 | [diff] [blame] | 67 | if (isAnnotation()) |
| 68 | return tok::objc_not_keyword; |
Chris Lattner | 4894f48 | 2007-10-07 08:47:24 +0000 | [diff] [blame] | 69 | IdentifierInfo *specId = getIdentifierInfo(); |
| 70 | return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; |
| 71 | } |
| 72 | |
| 73 | //===----------------------------------------------------------------------===// |
| 74 | // Lexer Class Implementation |
| 75 | //===----------------------------------------------------------------------===// |
| 76 | |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 77 | void Lexer::anchor() {} |
David Blaikie | 68e081d | 2011-12-20 02:48:34 +0000 | [diff] [blame] | 78 | |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 79 | void Lexer::InitLexer(const char *BufStart, const char *BufPtr, |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 80 | const char *BufEnd) { |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 81 | BufferStart = BufStart; |
| 82 | BufferPtr = BufPtr; |
| 83 | BufferEnd = BufEnd; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 84 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 85 | assert(BufEnd[0] == 0 && |
| 86 | "We assume that the input buffer has a null character at the end" |
| 87 | " to simplify lexing!"); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 88 | |
Eric Christopher | 7f36a79 | 2011-04-09 00:01:04 +0000 | [diff] [blame] | 89 | // Check whether we have a BOM in the beginning of the buffer. If yes - act |
| 90 | // accordingly. Right now we support only UTF-8 with and without BOM, so, just |
| 91 | // skip the UTF-8 BOM if it's present. |
| 92 | if (BufferStart == BufferPtr) { |
| 93 | // Determine the size of the BOM. |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 94 | StringRef Buf(BufferStart, BufferEnd - BufferStart); |
Eli Friedman | 86a5101 | 2011-05-10 17:11:21 +0000 | [diff] [blame] | 95 | size_t BOMLength = llvm::StringSwitch<size_t>(Buf) |
Eric Christopher | 7f36a79 | 2011-04-09 00:01:04 +0000 | [diff] [blame] | 96 | .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM |
| 97 | .Default(0); |
| 98 | |
| 99 | // Skip the BOM. |
| 100 | BufferPtr += BOMLength; |
| 101 | } |
| 102 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 103 | Is_PragmaLexer = false; |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 104 | CurrentConflictMarkerState = CMK_None; |
Eric Christopher | 7f36a79 | 2011-04-09 00:01:04 +0000 | [diff] [blame] | 105 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 106 | // Start of the file is a start of line. |
| 107 | IsAtStartOfLine = true; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 108 | IsAtPhysicalStartOfLine = true; |
| 109 | |
| 110 | HasLeadingSpace = false; |
| 111 | HasLeadingEmptyMacro = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 112 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 113 | // We are not after parsing a #. |
| 114 | ParsingPreprocessorDirective = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 115 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 116 | // We are not after parsing #include. |
| 117 | ParsingFilename = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 118 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 119 | // We are not in raw mode. Raw mode disables diagnostics and interpretation |
| 120 | // of tokens (e.g. identifiers, thus disabling macro expansion). It is used |
| 121 | // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block |
| 122 | // or otherwise skipping over tokens. |
| 123 | LexingRawMode = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 124 | |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 125 | // Default to not keeping comments. |
| 126 | ExtendedTokenMode = 0; |
| 127 | } |
| 128 | |
Chris Lattner | 5965a28 | 2009-01-17 07:56:59 +0000 | [diff] [blame] | 129 | /// Lexer constructor - Create a new lexer object for the specified buffer |
| 130 | /// with the specified preprocessor managing the lexing process. This lexer |
| 131 | /// assumes that the associated file buffer and Preprocessor objects will |
| 132 | /// outlive it, so it doesn't take ownership of either of them. |
Chris Lattner | 710bb87 | 2009-11-30 04:18:44 +0000 | [diff] [blame] | 133 | Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 134 | : PreprocessorLexer(&PP, FID), |
| 135 | FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), |
| 136 | LangOpts(PP.getLangOpts()) { |
Chris Lattner | 5965a28 | 2009-01-17 07:56:59 +0000 | [diff] [blame] | 137 | InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), |
| 138 | InputFile->getBufferEnd()); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 139 | |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 140 | resetExtendedTokenMode(); |
| 141 | } |
| 142 | |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 143 | /// Lexer constructor - Create a new raw lexer object. This object is only |
Dmitri Gribenko | 702b732 | 2012-06-08 23:19:37 +0000 | [diff] [blame] | 144 | /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text |
Chris Lattner | 50c9050 | 2008-10-12 01:15:46 +0000 | [diff] [blame] | 145 | /// range will outlive it, so it doesn't take ownership of it. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 146 | Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, |
Chris Lattner | fcf6452 | 2009-01-17 07:42:27 +0000 | [diff] [blame] | 147 | const char *BufStart, const char *BufPtr, const char *BufEnd) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 148 | : FileLoc(fileloc), LangOpts(langOpts) { |
Chris Lattner | f76b920 | 2009-01-17 06:55:17 +0000 | [diff] [blame] | 149 | InitLexer(BufStart, BufPtr, BufEnd); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 150 | |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 151 | // We *are* in raw mode. |
| 152 | LexingRawMode = true; |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 153 | } |
| 154 | |
Chris Lattner | 08354fe | 2009-01-17 07:35:14 +0000 | [diff] [blame] | 155 | /// Lexer constructor - Create a new raw lexer object. This object is only |
Dmitri Gribenko | 702b732 | 2012-06-08 23:19:37 +0000 | [diff] [blame] | 156 | /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text |
Chris Lattner | 08354fe | 2009-01-17 07:35:14 +0000 | [diff] [blame] | 157 | /// range will outlive it, so it doesn't take ownership of it. |
Chris Lattner | 710bb87 | 2009-11-30 04:18:44 +0000 | [diff] [blame] | 158 | Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 159 | const SourceManager &SM, const LangOptions &langOpts) |
Benjamin Kramer | f04f98d | 2015-03-06 14:15:57 +0000 | [diff] [blame] | 160 | : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile->getBufferStart(), |
| 161 | FromFile->getBufferStart(), FromFile->getBufferEnd()) {} |
Chris Lattner | 08354fe | 2009-01-17 07:35:14 +0000 | [diff] [blame] | 162 | |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 163 | void Lexer::resetExtendedTokenMode() { |
| 164 | assert(PP && "Cannot reset token mode without a preprocessor"); |
| 165 | if (LangOpts.TraditionalCPP) |
| 166 | SetKeepWhitespaceMode(true); |
| 167 | else |
| 168 | SetCommentRetentionState(PP->getCommentRetentionState()); |
| 169 | } |
| 170 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 171 | /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for |
| 172 | /// _Pragma expansion. This has a variety of magic semantics that this method |
| 173 | /// sets up. It returns a new'd Lexer that must be delete'd when done. |
| 174 | /// |
| 175 | /// On entrance to this routine, TokStartLoc is a macro location which has a |
| 176 | /// spelling loc that indicates the bytes to be lexed for the token and an |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 177 | /// expansion location that indicates where all lexed tokens should be |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 178 | /// "expanded from". |
| 179 | /// |
Alp Toker | 7755aff | 2014-05-18 18:37:59 +0000 | [diff] [blame] | 180 | /// TODO: It would really be nice to make _Pragma just be a wrapper around a |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 181 | /// normal lexer that remaps tokens as they fly by. This would require making |
| 182 | /// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer |
| 183 | /// interface that could handle this stuff. This would pull GetMappedTokenLoc |
| 184 | /// out of the critical path of the lexer! |
| 185 | /// |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 186 | Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 187 | SourceLocation ExpansionLocStart, |
| 188 | SourceLocation ExpansionLocEnd, |
Chris Lattner | 29a2a19 | 2009-01-19 06:46:35 +0000 | [diff] [blame] | 189 | unsigned TokLen, Preprocessor &PP) { |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 190 | SourceManager &SM = PP.getSourceManager(); |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 191 | |
| 192 | // Create the lexer as if we were going to lex the file normally. |
Chris Lattner | cbc35ecb | 2009-01-19 07:46:45 +0000 | [diff] [blame] | 193 | FileID SpellingFID = SM.getFileID(SpellingLoc); |
Chris Lattner | 710bb87 | 2009-11-30 04:18:44 +0000 | [diff] [blame] | 194 | const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); |
| 195 | Lexer *L = new Lexer(SpellingFID, InputFile, PP); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 196 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 197 | // Now that the lexer is created, change the start/end locations so that we |
| 198 | // just lex the subsection of the file that we want. This is lexing from a |
| 199 | // scratch buffer. |
| 200 | const char *StrData = SM.getCharacterData(SpellingLoc); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 201 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 202 | L->BufferPtr = StrData; |
| 203 | L->BufferEnd = StrData+TokLen; |
Chris Lattner | fa217bd | 2009-03-08 08:08:45 +0000 | [diff] [blame] | 204 | assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 205 | |
| 206 | // Set the SourceLocation with the remapping information. This ensures that |
| 207 | // GetMappedTokenLoc will remap the tokens as they are lexed. |
Chandler Carruth | 115b077 | 2011-07-26 03:03:05 +0000 | [diff] [blame] | 208 | L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), |
| 209 | ExpansionLocStart, |
| 210 | ExpansionLocEnd, TokLen); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 211 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 212 | // Ensure that the lexer thinks it is inside a directive, so that end \n will |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 213 | // return an EOD token. |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 214 | L->ParsingPreprocessorDirective = true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 215 | |
Chris Lattner | 757169b | 2009-01-17 08:27:52 +0000 | [diff] [blame] | 216 | // This lexer really is for _Pragma. |
| 217 | L->Is_PragmaLexer = true; |
| 218 | return L; |
| 219 | } |
| 220 | |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 221 | template <typename T> static void StringifyImpl(T &Str, char Quote) { |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 222 | typename T::size_type i = 0, e = Str.size(); |
| 223 | while (i < e) { |
| 224 | if (Str[i] == '\\' || Str[i] == Quote) { |
| 225 | Str.insert(Str.begin() + i, '\\'); |
| 226 | i += 2; |
| 227 | ++e; |
| 228 | } else if (Str[i] == '\n' || Str[i] == '\r') { |
| 229 | // Replace '\r\n' and '\n\r' to '\\' followed by 'n'. |
| 230 | if ((i < e - 1) && (Str[i + 1] == '\n' || Str[i + 1] == '\r') && |
| 231 | Str[i] != Str[i + 1]) { |
| 232 | Str[i] = '\\'; |
| 233 | Str[i + 1] = 'n'; |
| 234 | } else { |
| 235 | // Replace '\n' and '\r' to '\\' followed by 'n'. |
| 236 | Str[i] = '\\'; |
| 237 | Str.insert(Str.begin() + i + 1, 'n'); |
| 238 | ++e; |
| 239 | } |
| 240 | i += 2; |
| 241 | } else |
| 242 | ++i; |
| 243 | } |
| 244 | } |
| 245 | |
Rafael Espindola | c0f18a9 | 2015-06-01 20:00:16 +0000 | [diff] [blame] | 246 | std::string Lexer::Stringify(StringRef Str, bool Charify) { |
Chris Lattner | e3e81ea | 2006-07-03 01:13:26 +0000 | [diff] [blame] | 247 | std::string Result = Str; |
Chris Lattner | ecc39e9 | 2006-07-15 05:23:31 +0000 | [diff] [blame] | 248 | char Quote = Charify ? '\'' : '"'; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 249 | StringifyImpl(Result, Quote); |
Chris Lattner | ecc39e9 | 2006-07-15 05:23:31 +0000 | [diff] [blame] | 250 | return Result; |
Chris Lattner | e3e81ea | 2006-07-03 01:13:26 +0000 | [diff] [blame] | 251 | } |
| 252 | |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 253 | void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, '"'); } |
Chris Lattner | 4c4a245 | 2007-07-24 06:57:14 +0000 | [diff] [blame] | 254 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 255 | //===----------------------------------------------------------------------===// |
| 256 | // Token Spelling |
| 257 | //===----------------------------------------------------------------------===// |
| 258 | |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 259 | /// \brief Slow case of getSpelling. Extract the characters comprising the |
| 260 | /// spelling of this token from the provided input buffer. |
| 261 | static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, |
| 262 | const LangOptions &LangOpts, char *Spelling) { |
| 263 | assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); |
| 264 | |
| 265 | size_t Length = 0; |
| 266 | const char *BufEnd = BufPtr + Tok.getLength(); |
| 267 | |
Craig Topper | a6324c9 | 2015-10-22 15:35:21 +0000 | [diff] [blame] | 268 | if (tok::isStringLiteral(Tok.getKind())) { |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 269 | // Munch the encoding-prefix and opening double-quote. |
| 270 | while (BufPtr < BufEnd) { |
| 271 | unsigned Size; |
| 272 | Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); |
| 273 | BufPtr += Size; |
| 274 | |
| 275 | if (Spelling[Length - 1] == '"') |
| 276 | break; |
| 277 | } |
| 278 | |
| 279 | // Raw string literals need special handling; trigraph expansion and line |
| 280 | // splicing do not occur within their d-char-sequence nor within their |
| 281 | // r-char-sequence. |
| 282 | if (Length >= 2 && |
| 283 | Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { |
| 284 | // Search backwards from the end of the token to find the matching closing |
| 285 | // quote. |
| 286 | const char *RawEnd = BufEnd; |
| 287 | do --RawEnd; while (*RawEnd != '"'); |
| 288 | size_t RawLength = RawEnd - BufPtr + 1; |
| 289 | |
| 290 | // Everything between the quotes is included verbatim in the spelling. |
| 291 | memcpy(Spelling + Length, BufPtr, RawLength); |
| 292 | Length += RawLength; |
| 293 | BufPtr += RawLength; |
| 294 | |
| 295 | // The rest of the token is lexed normally. |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | while (BufPtr < BufEnd) { |
| 300 | unsigned Size; |
| 301 | Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); |
| 302 | BufPtr += Size; |
| 303 | } |
| 304 | |
| 305 | assert(Length < Tok.getLength() && |
| 306 | "NeedsCleaning flag set on token that didn't need cleaning!"); |
| 307 | return Length; |
| 308 | } |
| 309 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 310 | /// getSpelling() - Return the 'spelling' of this token. The spelling of a |
| 311 | /// token are the characters used to represent the token in the source file |
| 312 | /// after trigraph expansion and escaped-newline folding. In particular, this |
| 313 | /// wants to get the true, uncanonicalized, spelling of things like digraphs |
| 314 | /// UCNs, etc. |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 315 | StringRef Lexer::getSpelling(SourceLocation loc, |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 316 | SmallVectorImpl<char> &buffer, |
| 317 | const SourceManager &SM, |
| 318 | const LangOptions &options, |
| 319 | bool *invalid) { |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 320 | // Break down the source location. |
| 321 | std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); |
| 322 | |
| 323 | // Try to the load the file buffer. |
| 324 | bool invalidTemp = false; |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 325 | StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 326 | if (invalidTemp) { |
| 327 | if (invalid) *invalid = true; |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 328 | return {}; |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 329 | } |
| 330 | |
| 331 | const char *tokenBegin = file.data() + locInfo.second; |
| 332 | |
| 333 | // Lex from the start of the given location. |
| 334 | Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, |
| 335 | file.begin(), tokenBegin, file.end()); |
| 336 | Token token; |
| 337 | lexer.LexFromRawLexer(token); |
| 338 | |
| 339 | unsigned length = token.getLength(); |
| 340 | |
| 341 | // Common case: no need for cleaning. |
| 342 | if (!token.needsCleaning()) |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 343 | return StringRef(tokenBegin, length); |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 344 | |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 345 | // Hard case, we need to relex the characters into the string. |
| 346 | buffer.resize(length); |
| 347 | buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 348 | return StringRef(buffer.data(), buffer.size()); |
John McCall | 462c055 | 2011-03-08 07:59:04 +0000 | [diff] [blame] | 349 | } |
| 350 | |
| 351 | /// getSpelling() - Return the 'spelling' of this token. The spelling of a |
| 352 | /// token are the characters used to represent the token in the source file |
| 353 | /// after trigraph expansion and escaped-newline folding. In particular, this |
| 354 | /// wants to get the true, uncanonicalized, spelling of things like digraphs |
| 355 | /// UCNs, etc. |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 356 | std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 357 | const LangOptions &LangOpts, bool *Invalid) { |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 358 | assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 359 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 360 | bool CharDataInvalid = false; |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 361 | const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 362 | &CharDataInvalid); |
| 363 | if (Invalid) |
| 364 | *Invalid = CharDataInvalid; |
| 365 | if (CharDataInvalid) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 366 | return {}; |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 367 | |
| 368 | // If this token contains nothing interesting, return it directly. |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 369 | if (!Tok.needsCleaning()) |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 370 | return std::string(TokStart, TokStart + Tok.getLength()); |
| 371 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 372 | std::string Result; |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 373 | Result.resize(Tok.getLength()); |
| 374 | Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 375 | return Result; |
| 376 | } |
| 377 | |
| 378 | /// getSpelling - This method is used to get the spelling of a token into a |
| 379 | /// preallocated buffer, instead of as an std::string. The caller is required |
| 380 | /// to allocate enough space for the token, which is guaranteed to be at least |
| 381 | /// Tok.getLength() bytes long. The actual length of the token is returned. |
| 382 | /// |
| 383 | /// Note that this method may do two possible things: it may either fill in |
| 384 | /// the buffer specified with characters, or it may *change the input pointer* |
| 385 | /// to point to a constant buffer with the data already in it (avoiding a |
| 386 | /// copy). The caller is not allowed to modify the returned buffer pointer |
| 387 | /// if an internal buffer is returned. |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 388 | unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 389 | const SourceManager &SourceMgr, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 390 | const LangOptions &LangOpts, bool *Invalid) { |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 391 | assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 392 | |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 393 | const char *TokStart = nullptr; |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 394 | // NOTE: this has to be checked *before* testing for an IdentifierInfo. |
| 395 | if (Tok.is(tok::raw_identifier)) |
Alp Toker | 2d57cea | 2014-05-17 04:53:25 +0000 | [diff] [blame] | 396 | TokStart = Tok.getRawIdentifier().data(); |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 397 | else if (!Tok.hasUCN()) { |
| 398 | if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { |
| 399 | // Just return the string from the identifier table, which is very quick. |
| 400 | Buffer = II->getNameStart(); |
| 401 | return II->getLength(); |
| 402 | } |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 403 | } |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 404 | |
| 405 | // NOTE: this can be checked even after testing for an IdentifierInfo. |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 406 | if (Tok.isLiteral()) |
| 407 | TokStart = Tok.getLiteralData(); |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 408 | |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 409 | if (!TokStart) { |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 410 | // Compute the start of the token in the input lexer buffer. |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 411 | bool CharDataInvalid = false; |
| 412 | TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); |
| 413 | if (Invalid) |
| 414 | *Invalid = CharDataInvalid; |
| 415 | if (CharDataInvalid) { |
| 416 | Buffer = ""; |
| 417 | return 0; |
| 418 | } |
| 419 | } |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 420 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 421 | // If this token contains nothing interesting, return it directly. |
| 422 | if (!Tok.needsCleaning()) { |
| 423 | Buffer = TokStart; |
| 424 | return Tok.getLength(); |
| 425 | } |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 426 | |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 427 | // Otherwise, hard case, relex the characters into the string. |
Richard Smith | 9a67f47 | 2012-11-28 07:29:00 +0000 | [diff] [blame] | 428 | return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); |
Chris Lattner | 3972011 | 2010-11-17 07:26:20 +0000 | [diff] [blame] | 429 | } |
| 430 | |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 431 | /// MeasureTokenLength - Relex the token at the specified location and return |
| 432 | /// its length in bytes in the input file. If the token needs cleaning (e.g. |
| 433 | /// includes a trigraph or an escaped newline) then this count includes bytes |
| 434 | /// that are part of that. |
| 435 | unsigned Lexer::MeasureTokenLength(SourceLocation Loc, |
Chris Lattner | 184e65d | 2009-04-14 23:22:57 +0000 | [diff] [blame] | 436 | const SourceManager &SM, |
| 437 | const LangOptions &LangOpts) { |
Argyrios Kyrtzidis | 86f1a93 | 2013-01-07 19:16:18 +0000 | [diff] [blame] | 438 | Token TheTok; |
| 439 | if (getRawToken(Loc, TheTok, SM, LangOpts)) |
| 440 | return 0; |
| 441 | return TheTok.getLength(); |
| 442 | } |
| 443 | |
| 444 | /// \brief Relex the token at the specified location. |
| 445 | /// \returns true if there was a failure, false on success. |
| 446 | bool Lexer::getRawToken(SourceLocation Loc, Token &Result, |
| 447 | const SourceManager &SM, |
Fariborz Jahanian | d38ad47 | 2013-08-20 00:07:23 +0000 | [diff] [blame] | 448 | const LangOptions &LangOpts, |
| 449 | bool IgnoreWhiteSpace) { |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 450 | // TODO: this could be special cased for common tokens like identifiers, ')', |
| 451 | // etc to make this faster, if it mattered. Just look at StrData[0] to handle |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 452 | // all obviously single-char tokens. This could use |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 453 | // Lexer::isObviouslySimpleCharacter for example to handle identifiers or |
| 454 | // something. |
Chris Lattner | 4fa2362 | 2009-01-26 00:43:02 +0000 | [diff] [blame] | 455 | |
| 456 | // If this comes from a macro expansion, we really do want the macro name, not |
| 457 | // the token this macro expanded to. |
Chandler Carruth | 35f5320 | 2011-07-25 16:49:02 +0000 | [diff] [blame] | 458 | Loc = SM.getExpansionLoc(Loc); |
Chris Lattner | d381721 | 2009-01-26 22:24:27 +0000 | [diff] [blame] | 459 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
Douglas Gregor | e0fbb83 | 2010-03-16 00:06:06 +0000 | [diff] [blame] | 460 | bool Invalid = false; |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 461 | StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
Douglas Gregor | e0fbb83 | 2010-03-16 00:06:06 +0000 | [diff] [blame] | 462 | if (Invalid) |
Argyrios Kyrtzidis | 86f1a93 | 2013-01-07 19:16:18 +0000 | [diff] [blame] | 463 | return true; |
Benjamin Kramer | eb92dc0 | 2010-03-16 14:14:31 +0000 | [diff] [blame] | 464 | |
| 465 | const char *StrData = Buffer.data()+LocInfo.second; |
Chris Lattner | 5509d53 | 2009-01-17 08:30:10 +0000 | [diff] [blame] | 466 | |
Fariborz Jahanian | d38ad47 | 2013-08-20 00:07:23 +0000 | [diff] [blame] | 467 | if (!IgnoreWhiteSpace && isWhitespace(StrData[0])) |
Argyrios Kyrtzidis | 86f1a93 | 2013-01-07 19:16:18 +0000 | [diff] [blame] | 468 | return true; |
Douglas Gregor | 562c1f9 | 2010-01-22 19:49:59 +0000 | [diff] [blame] | 469 | |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 470 | // Create a lexer starting at the beginning of this token. |
Sebastian Redl | 5175230 | 2010-09-30 01:03:03 +0000 | [diff] [blame] | 471 | Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, |
| 472 | Buffer.begin(), StrData, Buffer.end()); |
Chris Lattner | a3d4f16 | 2009-10-14 15:04:18 +0000 | [diff] [blame] | 473 | TheLexer.SetCommentRetentionState(true); |
Argyrios Kyrtzidis | 86f1a93 | 2013-01-07 19:16:18 +0000 | [diff] [blame] | 474 | TheLexer.LexFromRawLexer(Result); |
| 475 | return false; |
Chris Lattner | 8e129c2 | 2007-10-17 21:18:47 +0000 | [diff] [blame] | 476 | } |
| 477 | |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 478 | /// Returns the pointer that points to the beginning of line that contains |
| 479 | /// the given offset, or null if the offset if invalid. |
| 480 | static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) { |
| 481 | const char *BufStart = Buffer.data(); |
| 482 | if (Offset >= Buffer.size()) |
| 483 | return nullptr; |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 484 | |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 485 | const char *LexStart = BufStart + Offset; |
| 486 | for (; LexStart != BufStart; --LexStart) { |
| 487 | if (isVerticalWhitespace(LexStart[0]) && |
| 488 | !Lexer::isNewLineEscaped(BufStart, LexStart)) { |
| 489 | // LexStart should point at first character of logical line. |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 490 | ++LexStart; |
| 491 | break; |
| 492 | } |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 493 | } |
| 494 | return LexStart; |
| 495 | } |
| 496 | |
Argyrios Kyrtzidis | 161868d | 2011-08-17 00:31:23 +0000 | [diff] [blame] | 497 | static SourceLocation getBeginningOfFileToken(SourceLocation Loc, |
| 498 | const SourceManager &SM, |
| 499 | const LangOptions &LangOpts) { |
| 500 | assert(Loc.isFileID()); |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 501 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
Douglas Gregor | 86af984 | 2011-01-31 22:42:36 +0000 | [diff] [blame] | 502 | if (LocInfo.first.isInvalid()) |
| 503 | return Loc; |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 504 | |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 505 | bool Invalid = false; |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 506 | StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 507 | if (Invalid) |
| 508 | return Loc; |
| 509 | |
| 510 | // Back up from the current location until we hit the beginning of a line |
| 511 | // (or the buffer). We'll relex from that point. |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 512 | const char *StrData = Buffer.data() + LocInfo.second; |
| 513 | const char *LexStart = findBeginningOfLine(Buffer, LocInfo.second); |
| 514 | if (!LexStart || LexStart == StrData) |
Douglas Gregor | 86af984 | 2011-01-31 22:42:36 +0000 | [diff] [blame] | 515 | return Loc; |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 516 | |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 517 | // Create a lexer starting at the beginning of this token. |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 518 | SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 519 | Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart, |
| 520 | Buffer.end()); |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 521 | TheLexer.SetCommentRetentionState(true); |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 522 | |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 523 | // Lex tokens until we find the token that contains the source location. |
| 524 | Token TheTok; |
| 525 | do { |
| 526 | TheLexer.LexFromRawLexer(TheTok); |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 527 | |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 528 | if (TheLexer.getBufferLocation() > StrData) { |
| 529 | // Lexing this token has taken the lexer past the source location we're |
| 530 | // looking for. If the current token encompasses our source location, |
| 531 | // return the beginning of that token. |
| 532 | if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) |
| 533 | return TheTok.getLocation(); |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 534 | |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 535 | // We ended up skipping over the source location entirely, which means |
| 536 | // that it points into whitespace. We're done here. |
| 537 | break; |
| 538 | } |
| 539 | } while (TheTok.getKind() != tok::eof); |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 540 | |
Douglas Gregor | cd8bdd0 | 2010-07-22 20:22:31 +0000 | [diff] [blame] | 541 | // We've passed our source location; just return the original source location. |
| 542 | return Loc; |
| 543 | } |
| 544 | |
Argyrios Kyrtzidis | 161868d | 2011-08-17 00:31:23 +0000 | [diff] [blame] | 545 | SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, |
| 546 | const SourceManager &SM, |
| 547 | const LangOptions &LangOpts) { |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 548 | if (Loc.isFileID()) |
| 549 | return getBeginningOfFileToken(Loc, SM, LangOpts); |
Argyrios Kyrtzidis | 161868d | 2011-08-17 00:31:23 +0000 | [diff] [blame] | 550 | |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 551 | if (!SM.isMacroArgExpansion(Loc)) |
| 552 | return Loc; |
| 553 | |
| 554 | SourceLocation FileLoc = SM.getSpellingLoc(Loc); |
| 555 | SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); |
| 556 | std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); |
| 557 | std::pair<FileID, unsigned> BeginFileLocInfo = |
| 558 | SM.getDecomposedLoc(BeginFileLoc); |
| 559 | assert(FileLocInfo.first == BeginFileLocInfo.first && |
| 560 | FileLocInfo.second >= BeginFileLocInfo.second); |
| 561 | return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); |
Argyrios Kyrtzidis | 161868d | 2011-08-17 00:31:23 +0000 | [diff] [blame] | 562 | } |
| 563 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 564 | namespace { |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 565 | |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 566 | enum PreambleDirectiveKind { |
| 567 | PDK_Skipped, |
| 568 | PDK_Unknown |
| 569 | }; |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 570 | |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 571 | } // namespace |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 572 | |
Cameron Desrochers | 84fd064 | 2017-09-20 19:03:37 +0000 | [diff] [blame] | 573 | PreambleBounds Lexer::ComputePreamble(StringRef Buffer, |
| 574 | const LangOptions &LangOpts, |
| 575 | unsigned MaxLines) { |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 576 | // Create a lexer starting at the beginning of the file. Note that we use a |
| 577 | // "fake" file source location at offset 1 so that the lexer will track our |
| 578 | // position within the file. |
| 579 | const unsigned StartOffset = 1; |
Argyrios Kyrtzidis | d53d0da | 2012-10-25 01:51:45 +0000 | [diff] [blame] | 580 | SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 581 | Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(), |
| 582 | Buffer.end()); |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 583 | TheLexer.SetCommentRetentionState(true); |
Argyrios Kyrtzidis | d53d0da | 2012-10-25 01:51:45 +0000 | [diff] [blame] | 584 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 585 | bool InPreprocessorDirective = false; |
| 586 | Token TheTok; |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 587 | SourceLocation ActiveCommentLoc; |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 588 | |
| 589 | unsigned MaxLineOffset = 0; |
| 590 | if (MaxLines) { |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 591 | const char *CurPtr = Buffer.begin(); |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 592 | unsigned CurLine = 0; |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 593 | while (CurPtr != Buffer.end()) { |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 594 | char ch = *CurPtr++; |
| 595 | if (ch == '\n') { |
| 596 | ++CurLine; |
| 597 | if (CurLine == MaxLines) |
| 598 | break; |
| 599 | } |
| 600 | } |
Rafael Espindola | cd0b380 | 2014-08-12 15:46:24 +0000 | [diff] [blame] | 601 | if (CurPtr != Buffer.end()) |
| 602 | MaxLineOffset = CurPtr - Buffer.begin(); |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 603 | } |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 604 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 605 | do { |
| 606 | TheLexer.LexFromRawLexer(TheTok); |
| 607 | |
| 608 | if (InPreprocessorDirective) { |
| 609 | // If we've hit the end of the file, we're done. |
| 610 | if (TheTok.getKind() == tok::eof) { |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 611 | break; |
| 612 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 613 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 614 | // If we haven't hit the end of the preprocessor directive, skip this |
| 615 | // token. |
| 616 | if (!TheTok.isAtStartOfLine()) |
| 617 | continue; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 618 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 619 | // We've passed the end of the preprocessor directive, and will look |
| 620 | // at this token again below. |
| 621 | InPreprocessorDirective = false; |
| 622 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 623 | |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 624 | // Keep track of the # of lines in the preamble. |
| 625 | if (TheTok.isAtStartOfLine()) { |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 626 | unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 627 | |
| 628 | // If we were asked to limit the number of lines in the preamble, |
| 629 | // and we're about to exceed that limit, we're done. |
Argyrios Kyrtzidis | a3deaee | 2011-09-04 03:32:04 +0000 | [diff] [blame] | 630 | if (MaxLineOffset && TokOffset >= MaxLineOffset) |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 631 | break; |
| 632 | } |
| 633 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 634 | // Comments are okay; skip over them. |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 635 | if (TheTok.getKind() == tok::comment) { |
| 636 | if (ActiveCommentLoc.isInvalid()) |
| 637 | ActiveCommentLoc = TheTok.getLocation(); |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 638 | continue; |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 639 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 640 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 641 | if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 642 | // This is the start of a preprocessor directive. |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 643 | Token HashTok = TheTok; |
| 644 | InPreprocessorDirective = true; |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 645 | ActiveCommentLoc = SourceLocation(); |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 646 | |
Joerg Sonnenberger | da5d2b7 | 2011-07-20 00:14:37 +0000 | [diff] [blame] | 647 | // Figure out which directive this is. Since we're lexing raw tokens, |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 648 | // we don't have an identifier table available. Instead, just look at |
| 649 | // the raw identifier to recognize and categorize preprocessor directives. |
| 650 | TheLexer.LexFromRawLexer(TheTok); |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 651 | if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { |
Alp Toker | 2d57cea | 2014-05-17 04:53:25 +0000 | [diff] [blame] | 652 | StringRef Keyword = TheTok.getRawIdentifier(); |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 653 | PreambleDirectiveKind PDK |
| 654 | = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) |
| 655 | .Case("include", PDK_Skipped) |
| 656 | .Case("__include_macros", PDK_Skipped) |
| 657 | .Case("define", PDK_Skipped) |
| 658 | .Case("undef", PDK_Skipped) |
| 659 | .Case("line", PDK_Skipped) |
| 660 | .Case("error", PDK_Skipped) |
| 661 | .Case("pragma", PDK_Skipped) |
| 662 | .Case("import", PDK_Skipped) |
| 663 | .Case("include_next", PDK_Skipped) |
| 664 | .Case("warning", PDK_Skipped) |
| 665 | .Case("ident", PDK_Skipped) |
| 666 | .Case("sccs", PDK_Skipped) |
| 667 | .Case("assert", PDK_Skipped) |
| 668 | .Case("unassert", PDK_Skipped) |
Erik Verbruggen | b34c79f | 2017-05-30 11:54:55 +0000 | [diff] [blame] | 669 | .Case("if", PDK_Skipped) |
| 670 | .Case("ifdef", PDK_Skipped) |
| 671 | .Case("ifndef", PDK_Skipped) |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 672 | .Case("elif", PDK_Skipped) |
| 673 | .Case("else", PDK_Skipped) |
Erik Verbruggen | b34c79f | 2017-05-30 11:54:55 +0000 | [diff] [blame] | 674 | .Case("endif", PDK_Skipped) |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 675 | .Default(PDK_Unknown); |
| 676 | |
| 677 | switch (PDK) { |
| 678 | case PDK_Skipped: |
| 679 | continue; |
| 680 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 681 | case PDK_Unknown: |
| 682 | // We don't know what this directive is; stop at the '#'. |
| 683 | break; |
| 684 | } |
| 685 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 686 | |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 687 | // We only end up here if we didn't recognize the preprocessor |
| 688 | // directive or it was one that can't occur in the preamble at this |
| 689 | // point. Roll back the current token to the location of the '#'. |
| 690 | InPreprocessorDirective = false; |
| 691 | TheTok = HashTok; |
| 692 | } |
| 693 | |
Douglas Gregor | 028d3e4 | 2010-08-09 20:45:32 +0000 | [diff] [blame] | 694 | // We hit a token that we don't recognize as being in the |
| 695 | // "preprocessing only" part of the file, so we're no longer in |
| 696 | // the preamble. |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 697 | break; |
| 698 | } while (true); |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 699 | |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 700 | SourceLocation End; |
Erik Verbruggen | b34c79f | 2017-05-30 11:54:55 +0000 | [diff] [blame] | 701 | if (ActiveCommentLoc.isValid()) |
Argyrios Kyrtzidis | 0903f8d | 2013-04-19 23:24:25 +0000 | [diff] [blame] | 702 | End = ActiveCommentLoc; // don't truncate a decl comment. |
| 703 | else |
| 704 | End = TheTok.getLocation(); |
| 705 | |
Cameron Desrochers | 84fd064 | 2017-09-20 19:03:37 +0000 | [diff] [blame] | 706 | return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(), |
Erik Verbruggen | b34c79f | 2017-05-30 11:54:55 +0000 | [diff] [blame] | 707 | TheTok.isAtStartOfLine()); |
Douglas Gregor | af82e35 | 2010-07-20 20:18:03 +0000 | [diff] [blame] | 708 | } |
| 709 | |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 710 | /// AdvanceToTokenCharacter - Given a location that specifies the start of a |
| 711 | /// token, return a new location that specifies a character within the token. |
| 712 | SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, |
| 713 | unsigned CharNo, |
| 714 | const SourceManager &SM, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 715 | const LangOptions &LangOpts) { |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 716 | // Figure out how many physical characters away the specified expansion |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 717 | // character is. This needs to take into consideration newlines and |
| 718 | // trigraphs. |
| 719 | bool Invalid = false; |
| 720 | const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 721 | |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 722 | // If they request the first char of the token, we're trivially done. |
| 723 | if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) |
| 724 | return TokStart; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 725 | |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 726 | unsigned PhysOffset = 0; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 727 | |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 728 | // The usual case is that tokens don't contain anything interesting. Skip |
| 729 | // over the uninteresting characters. If a token only consists of simple |
| 730 | // chars, this method is extremely fast. |
| 731 | while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { |
| 732 | if (CharNo == 0) |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 733 | return TokStart.getLocWithOffset(PhysOffset); |
Richard Trieu | cc3949d | 2016-02-18 22:34:54 +0000 | [diff] [blame] | 734 | ++TokPtr; |
| 735 | --CharNo; |
| 736 | ++PhysOffset; |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 737 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 738 | |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 739 | // If we have a character that may be a trigraph or escaped newline, use a |
| 740 | // lexer to parse it correctly. |
| 741 | for (; CharNo; --CharNo) { |
| 742 | unsigned Size; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 743 | Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 744 | TokPtr += Size; |
| 745 | PhysOffset += Size; |
| 746 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 747 | |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 748 | // Final detail: if we end up on an escaped newline, we want to return the |
| 749 | // location of the actual byte of the token. For example foo\<newline>bar |
| 750 | // advanced by 3 should return the location of b, not of \\. One compounding |
| 751 | // detail of this is that the escape may be made by a trigraph. |
| 752 | if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) |
| 753 | PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 754 | |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 755 | return TokStart.getLocWithOffset(PhysOffset); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 756 | } |
| 757 | |
| 758 | /// \brief Computes the source location just past the end of the |
| 759 | /// token at this source location. |
| 760 | /// |
| 761 | /// This routine can be used to produce a source location that |
| 762 | /// points just past the end of the token referenced by \p Loc, and |
| 763 | /// is generally used when a diagnostic needs to point just after a |
| 764 | /// token where it expected something different that it received. If |
| 765 | /// the returned source location would not be meaningful (e.g., if |
| 766 | /// it points into a macro), this routine returns an invalid |
| 767 | /// source location. |
| 768 | /// |
| 769 | /// \param Offset an offset from the end of the token, where the source |
| 770 | /// location should refer to. The default offset (0) produces a source |
| 771 | /// location pointing just past the end of the token; an offset of 1 produces |
| 772 | /// a source location pointing to the last character in the token, etc. |
| 773 | SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, |
| 774 | const SourceManager &SM, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 775 | const LangOptions &LangOpts) { |
Argyrios Kyrtzidis | 2cfce18 | 2011-06-24 17:58:59 +0000 | [diff] [blame] | 776 | if (Loc.isInvalid()) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 777 | return {}; |
Argyrios Kyrtzidis | 2cfce18 | 2011-06-24 17:58:59 +0000 | [diff] [blame] | 778 | |
| 779 | if (Loc.isMacroID()) { |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 780 | if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 781 | return {}; // Points inside the macro expansion. |
Argyrios Kyrtzidis | 2cfce18 | 2011-06-24 17:58:59 +0000 | [diff] [blame] | 782 | } |
| 783 | |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 784 | unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 785 | if (Len > Offset) |
| 786 | Len = Len - Offset; |
| 787 | else |
| 788 | return Loc; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 789 | |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 790 | return Loc.getLocWithOffset(Len); |
Chris Lattner | 2a6ee91 | 2010-11-17 07:05:50 +0000 | [diff] [blame] | 791 | } |
| 792 | |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 793 | /// \brief Returns true if the given MacroID location points at the first |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 794 | /// token of the macro expansion. |
| 795 | bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, |
Douglas Gregor | 925296b | 2011-07-19 16:10:42 +0000 | [diff] [blame] | 796 | const SourceManager &SM, |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 797 | const LangOptions &LangOpts, |
| 798 | SourceLocation *MacroBegin) { |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 799 | assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); |
| 800 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 801 | SourceLocation expansionLoc; |
| 802 | if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc)) |
| 803 | return false; |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 804 | |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 805 | if (expansionLoc.isFileID()) { |
| 806 | // No other macro expansions, this is the first. |
| 807 | if (MacroBegin) |
| 808 | *MacroBegin = expansionLoc; |
| 809 | return true; |
| 810 | } |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 811 | |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 812 | return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 813 | } |
| 814 | |
| 815 | /// \brief Returns true if the given MacroID location points at the last |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 816 | /// token of the macro expansion. |
| 817 | bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 818 | const SourceManager &SM, |
| 819 | const LangOptions &LangOpts, |
| 820 | SourceLocation *MacroEnd) { |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 821 | assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); |
| 822 | |
| 823 | SourceLocation spellLoc = SM.getSpellingLoc(loc); |
| 824 | unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); |
| 825 | if (tokLen == 0) |
| 826 | return false; |
| 827 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 828 | SourceLocation afterLoc = loc.getLocWithOffset(tokLen); |
| 829 | SourceLocation expansionLoc; |
| 830 | if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc)) |
| 831 | return false; |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 832 | |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 833 | if (expansionLoc.isFileID()) { |
| 834 | // No other macro expansions. |
| 835 | if (MacroEnd) |
| 836 | *MacroEnd = expansionLoc; |
| 837 | return true; |
| 838 | } |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 839 | |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 840 | return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); |
Argyrios Kyrtzidis | 61c58f7 | 2011-07-07 21:54:45 +0000 | [diff] [blame] | 841 | } |
| 842 | |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 843 | static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 844 | const SourceManager &SM, |
| 845 | const LangOptions &LangOpts) { |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 846 | SourceLocation Begin = Range.getBegin(); |
| 847 | SourceLocation End = Range.getEnd(); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 848 | assert(Begin.isFileID() && End.isFileID()); |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 849 | if (Range.isTokenRange()) { |
| 850 | End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); |
| 851 | if (End.isInvalid()) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 852 | return {}; |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 853 | } |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 854 | |
| 855 | // Break down the source locations. |
| 856 | FileID FID; |
| 857 | unsigned BeginOffs; |
Benjamin Kramer | 867ea1d | 2014-03-02 13:01:17 +0000 | [diff] [blame] | 858 | std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 859 | if (FID.isInvalid()) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 860 | return {}; |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 861 | |
| 862 | unsigned EndOffs; |
| 863 | if (!SM.isInFileID(End, FID, &EndOffs) || |
| 864 | BeginOffs > EndOffs) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 865 | return {}; |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 866 | |
| 867 | return CharSourceRange::getCharRange(Begin, End); |
| 868 | } |
| 869 | |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 870 | CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 871 | const SourceManager &SM, |
| 872 | const LangOptions &LangOpts) { |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 873 | SourceLocation Begin = Range.getBegin(); |
| 874 | SourceLocation End = Range.getEnd(); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 875 | if (Begin.isInvalid() || End.isInvalid()) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 876 | return {}; |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 877 | |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 878 | if (Begin.isFileID() && End.isFileID()) |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 879 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 880 | |
| 881 | if (Begin.isMacroID() && End.isFileID()) { |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 882 | if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 883 | return {}; |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 884 | Range.setBegin(Begin); |
| 885 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 886 | } |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 887 | |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 888 | if (Begin.isFileID() && End.isMacroID()) { |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 889 | if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, |
| 890 | &End)) || |
| 891 | (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, |
| 892 | &End))) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 893 | return {}; |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 894 | Range.setEnd(End); |
| 895 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 896 | } |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 897 | |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 898 | assert(Begin.isMacroID() && End.isMacroID()); |
| 899 | SourceLocation MacroBegin, MacroEnd; |
| 900 | if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 901 | ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, |
| 902 | &MacroEnd)) || |
| 903 | (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, |
| 904 | &MacroEnd)))) { |
| 905 | Range.setBegin(MacroBegin); |
| 906 | Range.setEnd(MacroEnd); |
| 907 | return makeRangeFromFileLocs(Range, SM, LangOpts); |
| 908 | } |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 909 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 910 | bool Invalid = false; |
| 911 | const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin), |
| 912 | &Invalid); |
| 913 | if (Invalid) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 914 | return {}; |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 915 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 916 | if (BeginEntry.getExpansion().isMacroArgExpansion()) { |
| 917 | const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End), |
| 918 | &Invalid); |
| 919 | if (Invalid) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 920 | return {}; |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 921 | |
Argyrios Kyrtzidis | 065d720 | 2013-05-16 21:37:39 +0000 | [diff] [blame] | 922 | if (EndEntry.getExpansion().isMacroArgExpansion() && |
| 923 | BeginEntry.getExpansion().getExpansionLocStart() == |
| 924 | EndEntry.getExpansion().getExpansionLocStart()) { |
| 925 | Range.setBegin(SM.getImmediateSpellingLoc(Begin)); |
| 926 | Range.setEnd(SM.getImmediateSpellingLoc(End)); |
| 927 | return makeFileCharRange(Range, SM, LangOpts); |
| 928 | } |
Argyrios Kyrtzidis | 85e7671 | 2012-01-20 16:52:43 +0000 | [diff] [blame] | 929 | } |
| 930 | |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 931 | return {}; |
Argyrios Kyrtzidis | a99e02d | 2012-01-19 15:59:14 +0000 | [diff] [blame] | 932 | } |
| 933 | |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 934 | StringRef Lexer::getSourceText(CharSourceRange Range, |
| 935 | const SourceManager &SM, |
| 936 | const LangOptions &LangOpts, |
| 937 | bool *Invalid) { |
Argyrios Kyrtzidis | 0d9e24b | 2012-02-03 05:58:29 +0000 | [diff] [blame] | 938 | Range = makeFileCharRange(Range, SM, LangOpts); |
| 939 | if (Range.isInvalid()) { |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 940 | if (Invalid) *Invalid = true; |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 941 | return {}; |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 942 | } |
| 943 | |
| 944 | // Break down the source location. |
| 945 | std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); |
| 946 | if (beginInfo.first.isInvalid()) { |
| 947 | if (Invalid) *Invalid = true; |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 948 | return {}; |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 949 | } |
| 950 | |
| 951 | unsigned EndOffs; |
| 952 | if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || |
| 953 | beginInfo.second > EndOffs) { |
| 954 | if (Invalid) *Invalid = true; |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 955 | return {}; |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 956 | } |
| 957 | |
| 958 | // Try to the load the file buffer. |
| 959 | bool invalidTemp = false; |
| 960 | StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); |
| 961 | if (invalidTemp) { |
| 962 | if (Invalid) *Invalid = true; |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 963 | return {}; |
Argyrios Kyrtzidis | 7838a2b | 2012-01-19 15:59:19 +0000 | [diff] [blame] | 964 | } |
| 965 | |
| 966 | if (Invalid) *Invalid = false; |
| 967 | return file.substr(beginInfo.second, EndOffs - beginInfo.second); |
| 968 | } |
| 969 | |
Anna Zaks | 1bea4bf | 2012-01-18 20:17:16 +0000 | [diff] [blame] | 970 | StringRef Lexer::getImmediateMacroName(SourceLocation Loc, |
| 971 | const SourceManager &SM, |
| 972 | const LangOptions &LangOpts) { |
| 973 | assert(Loc.isMacroID() && "Only reasonble to call this on macros"); |
Argyrios Kyrtzidis | abff5f1 | 2012-01-23 16:58:33 +0000 | [diff] [blame] | 974 | |
| 975 | // Find the location of the immediate macro expansion. |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 976 | while (true) { |
Argyrios Kyrtzidis | abff5f1 | 2012-01-23 16:58:33 +0000 | [diff] [blame] | 977 | FileID FID = SM.getFileID(Loc); |
| 978 | const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); |
| 979 | const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); |
| 980 | Loc = Expansion.getExpansionLocStart(); |
| 981 | if (!Expansion.isMacroArgExpansion()) |
| 982 | break; |
| 983 | |
| 984 | // For macro arguments we need to check that the argument did not come |
| 985 | // from an inner macro, e.g: "MAC1( MAC2(foo) )" |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 986 | |
Argyrios Kyrtzidis | abff5f1 | 2012-01-23 16:58:33 +0000 | [diff] [blame] | 987 | // Loc points to the argument id of the macro definition, move to the |
| 988 | // macro expansion. |
Anna Zaks | 1bea4bf | 2012-01-18 20:17:16 +0000 | [diff] [blame] | 989 | Loc = SM.getImmediateExpansionRange(Loc).first; |
Argyrios Kyrtzidis | abff5f1 | 2012-01-23 16:58:33 +0000 | [diff] [blame] | 990 | SourceLocation SpellLoc = Expansion.getSpellingLoc(); |
| 991 | if (SpellLoc.isFileID()) |
| 992 | break; // No inner macro. |
| 993 | |
| 994 | // If spelling location resides in the same FileID as macro expansion |
| 995 | // location, it means there is no inner macro. |
| 996 | FileID MacroFID = SM.getFileID(Loc); |
| 997 | if (SM.isInFileID(SpellLoc, MacroFID)) |
| 998 | break; |
| 999 | |
| 1000 | // Argument came from inner macro. |
| 1001 | Loc = SpellLoc; |
| 1002 | } |
Anna Zaks | 1bea4bf | 2012-01-18 20:17:16 +0000 | [diff] [blame] | 1003 | |
| 1004 | // Find the spelling location of the start of the non-argument expansion |
| 1005 | // range. This is where the macro name was spelled in order to begin |
| 1006 | // expanding this macro. |
Argyrios Kyrtzidis | abff5f1 | 2012-01-23 16:58:33 +0000 | [diff] [blame] | 1007 | Loc = SM.getSpellingLoc(Loc); |
Anna Zaks | 1bea4bf | 2012-01-18 20:17:16 +0000 | [diff] [blame] | 1008 | |
| 1009 | // Dig out the buffer where the macro name was spelled and the extents of the |
| 1010 | // name so that we can render it into the expansion note. |
| 1011 | std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); |
| 1012 | unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); |
| 1013 | StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); |
| 1014 | return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); |
| 1015 | } |
| 1016 | |
Richard Trieu | 3a5c958 | 2016-01-26 02:51:55 +0000 | [diff] [blame] | 1017 | StringRef Lexer::getImmediateMacroNameForDiagnostics( |
| 1018 | SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) { |
| 1019 | assert(Loc.isMacroID() && "Only reasonble to call this on macros"); |
| 1020 | // Walk past macro argument expanions. |
| 1021 | while (SM.isMacroArgExpansion(Loc)) |
| 1022 | Loc = SM.getImmediateExpansionRange(Loc).first; |
| 1023 | |
| 1024 | // If the macro's spelling has no FileID, then it's actually a token paste |
| 1025 | // or stringization (or similar) and not a macro at all. |
| 1026 | if (!SM.getFileEntryForID(SM.getFileID(SM.getSpellingLoc(Loc)))) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 1027 | return {}; |
Richard Trieu | 3a5c958 | 2016-01-26 02:51:55 +0000 | [diff] [blame] | 1028 | |
| 1029 | // Find the spelling location of the start of the non-argument expansion |
| 1030 | // range. This is where the macro name was spelled in order to begin |
| 1031 | // expanding this macro. |
| 1032 | Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).first); |
| 1033 | |
| 1034 | // Dig out the buffer where the macro name was spelled and the extents of the |
| 1035 | // name so that we can render it into the expansion note. |
| 1036 | std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); |
| 1037 | unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); |
| 1038 | StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); |
| 1039 | return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); |
| 1040 | } |
| 1041 | |
Jordan Rose | 288c421 | 2012-06-07 01:10:31 +0000 | [diff] [blame] | 1042 | bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { |
Jordan Rose | a2100d7 | 2013-02-08 22:30:22 +0000 | [diff] [blame] | 1043 | return isIdentifierBody(c, LangOpts.DollarIdents); |
Jordan Rose | 288c421 | 2012-06-07 01:10:31 +0000 | [diff] [blame] | 1044 | } |
| 1045 | |
Alexander Kornienko | cf007a7 | 2017-08-10 10:06:16 +0000 | [diff] [blame] | 1046 | bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) { |
| 1047 | assert(isVerticalWhitespace(Str[0])); |
| 1048 | if (Str - 1 < BufferStart) |
| 1049 | return false; |
| 1050 | |
| 1051 | if ((Str[0] == '\n' && Str[-1] == '\r') || |
| 1052 | (Str[0] == '\r' && Str[-1] == '\n')) { |
| 1053 | if (Str - 2 < BufferStart) |
| 1054 | return false; |
| 1055 | --Str; |
| 1056 | } |
| 1057 | --Str; |
| 1058 | |
| 1059 | // Rewind to first non-space character: |
| 1060 | while (Str > BufferStart && isHorizontalWhitespace(*Str)) |
| 1061 | --Str; |
| 1062 | |
| 1063 | return *Str == '\\'; |
| 1064 | } |
| 1065 | |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 1066 | StringRef Lexer::getIndentationForLine(SourceLocation Loc, |
| 1067 | const SourceManager &SM) { |
| 1068 | if (Loc.isInvalid() || Loc.isMacroID()) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 1069 | return {}; |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 1070 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
| 1071 | if (LocInfo.first.isInvalid()) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 1072 | return {}; |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 1073 | bool Invalid = false; |
| 1074 | StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
| 1075 | if (Invalid) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 1076 | return {}; |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 1077 | const char *Line = findBeginningOfLine(Buffer, LocInfo.second); |
| 1078 | if (!Line) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 1079 | return {}; |
Alex Lorenz | 9c5c2bf | 2017-05-05 16:42:44 +0000 | [diff] [blame] | 1080 | StringRef Rest = Buffer.substr(Line - Buffer.data()); |
| 1081 | size_t NumWhitespaceChars = Rest.find_first_not_of(" \t"); |
| 1082 | return NumWhitespaceChars == StringRef::npos |
| 1083 | ? "" |
| 1084 | : Rest.take_front(NumWhitespaceChars); |
| 1085 | } |
| 1086 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1087 | //===----------------------------------------------------------------------===// |
| 1088 | // Diagnostics forwarding code. |
| 1089 | //===----------------------------------------------------------------------===// |
| 1090 | |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1091 | /// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 1092 | /// lexer buffer was all expanded at a single point, perform the mapping. |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1093 | /// This is currently only used for _Pragma implementation, so it is the slow |
| 1094 | /// path of the hot getSourceLocation method. Do not allow it to be inlined. |
Chandler Carruth | c3ce584 | 2010-10-23 08:44:57 +0000 | [diff] [blame] | 1095 | static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( |
| 1096 | Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1097 | static SourceLocation GetMappedTokenLoc(Preprocessor &PP, |
| 1098 | SourceLocation FileLoc, |
Chris Lattner | 4fa2362 | 2009-01-26 00:43:02 +0000 | [diff] [blame] | 1099 | unsigned CharNo, unsigned TokLen) { |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 1100 | assert(FileLoc.isMacroID() && "Must be a macro expansion"); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1101 | |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1102 | // Otherwise, we're lexing "mapped tokens". This is used for things like |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 1103 | // _Pragma handling. Combine the expansion location of FileLoc with the |
Chris Lattner | 53e384f | 2009-01-16 07:00:02 +0000 | [diff] [blame] | 1104 | // spelling location. |
Chris Lattner | 9dc9c20 | 2009-02-15 20:52:18 +0000 | [diff] [blame] | 1105 | SourceManager &SM = PP.getSourceManager(); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1106 | |
Chandler Carruth | e2c09eb | 2011-07-14 08:20:40 +0000 | [diff] [blame] | 1107 | // Create a new SLoc which is expanded from Expansion(FileLoc) but whose |
Chris Lattner | 53e384f | 2009-01-16 07:00:02 +0000 | [diff] [blame] | 1108 | // characters come from spelling(FileLoc)+Offset. |
Chris Lattner | 9dc9c20 | 2009-02-15 20:52:18 +0000 | [diff] [blame] | 1109 | SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 1110 | SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1111 | |
Chris Lattner | 9dc9c20 | 2009-02-15 20:52:18 +0000 | [diff] [blame] | 1112 | // Figure out the expansion loc range, which is the range covered by the |
| 1113 | // original _Pragma(...) sequence. |
| 1114 | std::pair<SourceLocation,SourceLocation> II = |
Chandler Carruth | ca75758 | 2011-07-25 20:52:21 +0000 | [diff] [blame] | 1115 | SM.getImmediateExpansionRange(FileLoc); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1116 | |
Chandler Carruth | 115b077 | 2011-07-26 03:03:05 +0000 | [diff] [blame] | 1117 | return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); |
Chris Lattner | 619c174 | 2007-07-22 18:38:25 +0000 | [diff] [blame] | 1118 | } |
| 1119 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1120 | /// getSourceLocation - Return a source location identifier for the specified |
| 1121 | /// offset in the current file. |
Chris Lattner | 4fa2362 | 2009-01-26 00:43:02 +0000 | [diff] [blame] | 1122 | SourceLocation Lexer::getSourceLocation(const char *Loc, |
| 1123 | unsigned TokLen) const { |
Chris Lattner | 5d1c027 | 2007-07-22 18:44:36 +0000 | [diff] [blame] | 1124 | assert(Loc >= BufferStart && Loc <= BufferEnd && |
Chris Lattner | 4cca5ba | 2006-07-02 20:05:54 +0000 | [diff] [blame] | 1125 | "Location out of range for this buffer!"); |
Chris Lattner | dc5c055 | 2007-07-20 16:37:10 +0000 | [diff] [blame] | 1126 | |
| 1127 | // In the normal case, we're just lexing from a simple file buffer, return |
| 1128 | // the file id from FileLoc with the offset specified. |
Chris Lattner | 5d1c027 | 2007-07-22 18:44:36 +0000 | [diff] [blame] | 1129 | unsigned CharNo = Loc-BufferStart; |
Chris Lattner | dc5c055 | 2007-07-20 16:37:10 +0000 | [diff] [blame] | 1130 | if (FileLoc.isFileID()) |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 1131 | return FileLoc.getLocWithOffset(CharNo); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1132 | |
Chris Lattner | d32480d | 2009-01-17 06:22:33 +0000 | [diff] [blame] | 1133 | // Otherwise, this is the _Pragma lexer case, which pretends that all of the |
| 1134 | // tokens are lexed from where the _Pragma was defined. |
Chris Lattner | 02b436a | 2007-10-17 20:41:00 +0000 | [diff] [blame] | 1135 | assert(PP && "This doesn't work on raw lexers"); |
Chris Lattner | 4fa2362 | 2009-01-26 00:43:02 +0000 | [diff] [blame] | 1136 | return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1137 | } |
| 1138 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1139 | /// Diag - Forwarding function for diagnostics. This translate a source |
| 1140 | /// position in the current buffer into a SourceLocation object for rendering. |
Chris Lattner | 427c9c1 | 2008-11-22 00:59:29 +0000 | [diff] [blame] | 1141 | DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { |
Chris Lattner | 907dfe9 | 2008-11-18 07:59:24 +0000 | [diff] [blame] | 1142 | return PP->Diag(getSourceLocation(Loc), DiagID); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1143 | } |
| 1144 | |
| 1145 | //===----------------------------------------------------------------------===// |
| 1146 | // Trigraph and Escaped Newline Handling Code. |
| 1147 | //===----------------------------------------------------------------------===// |
| 1148 | |
| 1149 | /// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, |
| 1150 | /// return the decoded trigraph letter it corresponds to, or '\0' if nothing. |
| 1151 | static char GetTrigraphCharForLetter(char Letter) { |
| 1152 | switch (Letter) { |
| 1153 | default: return 0; |
| 1154 | case '=': return '#'; |
| 1155 | case ')': return ']'; |
| 1156 | case '(': return '['; |
| 1157 | case '!': return '|'; |
| 1158 | case '\'': return '^'; |
| 1159 | case '>': return '}'; |
| 1160 | case '/': return '\\'; |
| 1161 | case '<': return '{'; |
| 1162 | case '-': return '~'; |
| 1163 | } |
| 1164 | } |
| 1165 | |
| 1166 | /// DecodeTrigraphChar - If the specified character is a legal trigraph when |
| 1167 | /// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, |
| 1168 | /// return the result character. Finally, emit a warning about trigraph use |
| 1169 | /// whether trigraphs are enabled or not. |
| 1170 | static char DecodeTrigraphChar(const char *CP, Lexer *L) { |
| 1171 | char Res = GetTrigraphCharForLetter(*CP); |
Chris Lattner | 907dfe9 | 2008-11-18 07:59:24 +0000 | [diff] [blame] | 1172 | if (!Res || !L) return Res; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1173 | |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1174 | if (!L->getLangOpts().Trigraphs) { |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1175 | if (!L->isLexingRawMode()) |
| 1176 | L->Diag(CP-2, diag::trigraph_ignored); |
Chris Lattner | 907dfe9 | 2008-11-18 07:59:24 +0000 | [diff] [blame] | 1177 | return 0; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1178 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1179 | |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1180 | if (!L->isLexingRawMode()) |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 1181 | L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1182 | return Res; |
| 1183 | } |
| 1184 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1185 | /// getEscapedNewLineSize - Return the size of the specified escaped newline, |
| 1186 | /// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1187 | /// trigraph equivalent on entry to this function. |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1188 | unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { |
| 1189 | unsigned Size = 0; |
| 1190 | while (isWhitespace(Ptr[Size])) { |
| 1191 | ++Size; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1192 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1193 | if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') |
| 1194 | continue; |
| 1195 | |
| 1196 | // If this is a \r\n or \n\r, skip the other half. |
| 1197 | if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && |
| 1198 | Ptr[Size-1] != Ptr[Size]) |
| 1199 | ++Size; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1200 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1201 | return Size; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1202 | } |
| 1203 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1204 | // Not an escaped newline, must be a \t or something else. |
| 1205 | return 0; |
| 1206 | } |
| 1207 | |
Chris Lattner | 38b2cde | 2009-04-18 22:27:02 +0000 | [diff] [blame] | 1208 | /// SkipEscapedNewLines - If P points to an escaped newline (or a series of |
| 1209 | /// them), skip over them and return the first non-escaped-newline found, |
| 1210 | /// otherwise return P. |
| 1211 | const char *Lexer::SkipEscapedNewLines(const char *P) { |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 1212 | while (true) { |
Chris Lattner | 38b2cde | 2009-04-18 22:27:02 +0000 | [diff] [blame] | 1213 | const char *AfterEscape; |
| 1214 | if (*P == '\\') { |
| 1215 | AfterEscape = P+1; |
| 1216 | } else if (*P == '?') { |
| 1217 | // If not a trigraph for escape, bail out. |
| 1218 | if (P[1] != '?' || P[2] != '/') |
| 1219 | return P; |
Richard Smith | 4c132e5 | 2017-04-18 21:45:04 +0000 | [diff] [blame] | 1220 | // FIXME: Take LangOpts into account; the language might not |
| 1221 | // support trigraphs. |
Chris Lattner | 38b2cde | 2009-04-18 22:27:02 +0000 | [diff] [blame] | 1222 | AfterEscape = P+3; |
| 1223 | } else { |
| 1224 | return P; |
| 1225 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1226 | |
Chris Lattner | 38b2cde | 2009-04-18 22:27:02 +0000 | [diff] [blame] | 1227 | unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); |
| 1228 | if (NewLineSize == 0) return P; |
| 1229 | P = AfterEscape+NewLineSize; |
| 1230 | } |
| 1231 | } |
| 1232 | |
Alex Lorenz | ebbbb81 | 2017-11-03 18:11:22 +0000 | [diff] [blame] | 1233 | Optional<Token> Lexer::findNextToken(SourceLocation Loc, |
| 1234 | const SourceManager &SM, |
| 1235 | const LangOptions &LangOpts) { |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1236 | if (Loc.isMacroID()) { |
Argyrios Kyrtzidis | 1b07c34 | 2012-01-19 15:59:08 +0000 | [diff] [blame] | 1237 | if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) |
Alex Lorenz | ebbbb81 | 2017-11-03 18:11:22 +0000 | [diff] [blame] | 1238 | return None; |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1239 | } |
| 1240 | Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); |
| 1241 | |
| 1242 | // Break down the source location. |
| 1243 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); |
| 1244 | |
| 1245 | // Try to load the file buffer. |
| 1246 | bool InvalidTemp = false; |
Dmitri Gribenko | f857950 | 2013-01-12 19:30:44 +0000 | [diff] [blame] | 1247 | StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1248 | if (InvalidTemp) |
Alex Lorenz | ebbbb81 | 2017-11-03 18:11:22 +0000 | [diff] [blame] | 1249 | return None; |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1250 | |
| 1251 | const char *TokenBegin = File.data() + LocInfo.second; |
| 1252 | |
| 1253 | // Lex from the start of the given location. |
| 1254 | Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), |
| 1255 | TokenBegin, File.end()); |
| 1256 | // Find the token. |
| 1257 | Token Tok; |
| 1258 | lexer.LexFromRawLexer(Tok); |
Alex Lorenz | ebbbb81 | 2017-11-03 18:11:22 +0000 | [diff] [blame] | 1259 | return Tok; |
| 1260 | } |
| 1261 | |
| 1262 | /// \brief Checks that the given token is the first token that occurs after the |
| 1263 | /// given location (this excludes comments and whitespace). Returns the location |
| 1264 | /// immediately after the specified token. If the token is not found or the |
| 1265 | /// location is inside a macro, the returned source location will be invalid. |
| 1266 | SourceLocation Lexer::findLocationAfterToken( |
| 1267 | SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM, |
| 1268 | const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) { |
| 1269 | Optional<Token> Tok = findNextToken(Loc, SM, LangOpts); |
| 1270 | if (!Tok || Tok->isNot(TKind)) |
Eugene Zelenko | cb96ac6 | 2017-12-08 22:39:26 +0000 | [diff] [blame] | 1271 | return {}; |
Alex Lorenz | ebbbb81 | 2017-11-03 18:11:22 +0000 | [diff] [blame] | 1272 | SourceLocation TokenLoc = Tok->getLocation(); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1273 | |
| 1274 | // Calculate how much whitespace needs to be skipped if any. |
| 1275 | unsigned NumWhitespaceChars = 0; |
| 1276 | if (SkipTrailingWhitespaceAndNewLine) { |
Alex Lorenz | ebbbb81 | 2017-11-03 18:11:22 +0000 | [diff] [blame] | 1277 | const char *TokenEnd = SM.getCharacterData(TokenLoc) + Tok->getLength(); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1278 | unsigned char C = *TokenEnd; |
| 1279 | while (isHorizontalWhitespace(C)) { |
| 1280 | C = *(++TokenEnd); |
| 1281 | NumWhitespaceChars++; |
| 1282 | } |
Eli Friedman | b699e61 | 2012-11-14 01:28:38 +0000 | [diff] [blame] | 1283 | |
| 1284 | // Skip \r, \n, \r\n, or \n\r |
| 1285 | if (C == '\n' || C == '\r') { |
| 1286 | char PrevC = C; |
| 1287 | C = *(++TokenEnd); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1288 | NumWhitespaceChars++; |
Eli Friedman | b699e61 | 2012-11-14 01:28:38 +0000 | [diff] [blame] | 1289 | if ((C == '\n' || C == '\r') && C != PrevC) |
| 1290 | NumWhitespaceChars++; |
| 1291 | } |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1292 | } |
| 1293 | |
Alex Lorenz | ebbbb81 | 2017-11-03 18:11:22 +0000 | [diff] [blame] | 1294 | return TokenLoc.getLocWithOffset(Tok->getLength() + NumWhitespaceChars); |
Anna Zaks | 59a3c80 | 2011-07-27 21:43:43 +0000 | [diff] [blame] | 1295 | } |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1296 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1297 | /// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, |
| 1298 | /// get its size, and return it. This is tricky in several cases: |
| 1299 | /// 1. If currently at the start of a trigraph, we warn about the trigraph, |
| 1300 | /// then either return the trigraph (skipping 3 chars) or the '?', |
| 1301 | /// depending on whether trigraphs are enabled or not. |
| 1302 | /// 2. If this is an escaped newline (potentially with whitespace between |
| 1303 | /// the backslash and newline), implicitly skip the newline and return |
| 1304 | /// the char after it. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1305 | /// |
| 1306 | /// This handles the slow/uncommon case of the getCharAndSize method. Here we |
| 1307 | /// know that we can accumulate into Size, and that we have already incremented |
| 1308 | /// Ptr by Size bytes. |
| 1309 | /// |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1310 | /// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should |
| 1311 | /// be updated to match. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1312 | char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 1313 | Token *Tok) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1314 | // If we have a slash, look for an escaped newline. |
| 1315 | if (Ptr[0] == '\\') { |
| 1316 | ++Size; |
| 1317 | ++Ptr; |
| 1318 | Slash: |
| 1319 | // Common case, backslash-char where the char is not whitespace. |
| 1320 | if (!isWhitespace(Ptr[0])) return '\\'; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1321 | |
Chris Lattner | c183595 | 2009-06-23 05:15:06 +0000 | [diff] [blame] | 1322 | // See if we have optional whitespace characters between the slash and |
| 1323 | // newline. |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1324 | if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { |
| 1325 | // Remember that this token needs to be cleaned. |
| 1326 | if (Tok) Tok->setFlag(Token::NeedsCleaning); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1327 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1328 | // Warn if there was whitespace between the backslash and newline. |
Chris Lattner | c183595 | 2009-06-23 05:15:06 +0000 | [diff] [blame] | 1329 | if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1330 | Diag(Ptr, diag::backslash_newline_space); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1331 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1332 | // Found backslash<whitespace><newline>. Parse the char after it. |
| 1333 | Size += EscapedNewLineSize; |
| 1334 | Ptr += EscapedNewLineSize; |
Argyrios Kyrtzidis | e5cdd08 | 2011-12-21 20:19:55 +0000 | [diff] [blame] | 1335 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1336 | // Use slow version to accumulate a correct size field. |
| 1337 | return getCharAndSizeSlow(Ptr, Size, Tok); |
| 1338 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1339 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1340 | // Otherwise, this is not an escaped newline, just return the slash. |
| 1341 | return '\\'; |
| 1342 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1343 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1344 | // If this is a trigraph, process it. |
| 1345 | if (Ptr[0] == '?' && Ptr[1] == '?') { |
| 1346 | // If this is actually a legal trigraph (not something like "??x"), emit |
| 1347 | // a trigraph warning. If so, and if trigraphs are enabled, return it. |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1348 | if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : nullptr)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1349 | // Remember that this token needs to be cleaned. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 1350 | if (Tok) Tok->setFlag(Token::NeedsCleaning); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1351 | |
| 1352 | Ptr += 3; |
| 1353 | Size += 3; |
| 1354 | if (C == '\\') goto Slash; |
| 1355 | return C; |
| 1356 | } |
| 1357 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1358 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1359 | // If this is neither, return a single character. |
| 1360 | ++Size; |
| 1361 | return *Ptr; |
| 1362 | } |
| 1363 | |
| 1364 | /// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the |
| 1365 | /// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, |
| 1366 | /// and that we have already incremented Ptr by Size bytes. |
| 1367 | /// |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1368 | /// NOTE: When this method is updated, getCharAndSizeSlow (above) should |
| 1369 | /// be updated to match. |
| 1370 | char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1371 | const LangOptions &LangOpts) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1372 | // If we have a slash, look for an escaped newline. |
| 1373 | if (Ptr[0] == '\\') { |
| 1374 | ++Size; |
| 1375 | ++Ptr; |
| 1376 | Slash: |
| 1377 | // Common case, backslash-char where the char is not whitespace. |
| 1378 | if (!isWhitespace(Ptr[0])) return '\\'; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1379 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1380 | // See if we have optional whitespace characters followed by a newline. |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1381 | if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { |
| 1382 | // Found backslash<whitespace><newline>. Parse the char after it. |
| 1383 | Size += EscapedNewLineSize; |
| 1384 | Ptr += EscapedNewLineSize; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1385 | |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1386 | // Use slow version to accumulate a correct size field. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1387 | return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); |
Chris Lattner | fbce7aa | 2009-04-18 22:05:41 +0000 | [diff] [blame] | 1388 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1389 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1390 | // Otherwise, this is not an escaped newline, just return the slash. |
| 1391 | return '\\'; |
| 1392 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1393 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1394 | // If this is a trigraph, process it. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1395 | if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1396 | // If this is actually a legal trigraph (not something like "??x"), return |
| 1397 | // it. |
| 1398 | if (char C = GetTrigraphCharForLetter(Ptr[2])) { |
| 1399 | Ptr += 3; |
| 1400 | Size += 3; |
| 1401 | if (C == '\\') goto Slash; |
| 1402 | return C; |
| 1403 | } |
| 1404 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1405 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1406 | // If this is neither, return a single character. |
| 1407 | ++Size; |
| 1408 | return *Ptr; |
| 1409 | } |
| 1410 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1411 | //===----------------------------------------------------------------------===// |
| 1412 | // Helper methods for lexing. |
| 1413 | //===----------------------------------------------------------------------===// |
| 1414 | |
Cameron Desrochers | 84fd064 | 2017-09-20 19:03:37 +0000 | [diff] [blame] | 1415 | /// \brief Routine that indiscriminately sets the offset into the source file. |
| 1416 | void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) { |
| 1417 | BufferPtr = BufferStart + Offset; |
Douglas Gregor | 3f4bea0 | 2010-07-26 21:36:20 +0000 | [diff] [blame] | 1418 | if (BufferPtr > BufferEnd) |
| 1419 | BufferPtr = BufferEnd; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1420 | // FIXME: What exactly does the StartOfLine bit mean? There are two |
| 1421 | // possible meanings for the "start" of the line: the first token on the |
| 1422 | // unexpanded line, or the first token on the expanded line. |
Douglas Gregor | 3f4bea0 | 2010-07-26 21:36:20 +0000 | [diff] [blame] | 1423 | IsAtStartOfLine = StartOfLine; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1424 | IsAtPhysicalStartOfLine = StartOfLine; |
Douglas Gregor | 3f4bea0 | 2010-07-26 21:36:20 +0000 | [diff] [blame] | 1425 | } |
| 1426 | |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1427 | static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { |
Vinicius Tinti | 92e68c2 | 2015-11-20 23:42:39 +0000 | [diff] [blame] | 1428 | if (LangOpts.AsmPreprocessor) { |
| 1429 | return false; |
| 1430 | } else if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1431 | static const llvm::sys::UnicodeCharSet C11AllowedIDChars( |
| 1432 | C11AllowedIDCharRanges); |
| 1433 | return C11AllowedIDChars.contains(C); |
| 1434 | } else if (LangOpts.CPlusPlus) { |
| 1435 | static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( |
| 1436 | CXX03AllowedIDCharRanges); |
| 1437 | return CXX03AllowedIDChars.contains(C); |
| 1438 | } else { |
| 1439 | static const llvm::sys::UnicodeCharSet C99AllowedIDChars( |
| 1440 | C99AllowedIDCharRanges); |
| 1441 | return C99AllowedIDChars.contains(C); |
| 1442 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1443 | } |
| 1444 | |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1445 | static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { |
| 1446 | assert(isAllowedIDChar(C, LangOpts)); |
Vinicius Tinti | 92e68c2 | 2015-11-20 23:42:39 +0000 | [diff] [blame] | 1447 | if (LangOpts.AsmPreprocessor) { |
| 1448 | return false; |
| 1449 | } else if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1450 | static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars( |
| 1451 | C11DisallowedInitialIDCharRanges); |
| 1452 | return !C11DisallowedInitialIDChars.contains(C); |
| 1453 | } else if (LangOpts.CPlusPlus) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1454 | return true; |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1455 | } else { |
| 1456 | static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( |
| 1457 | C99DisallowedInitialIDCharRanges); |
| 1458 | return !C99DisallowedInitialIDChars.contains(C); |
| 1459 | } |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1460 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1461 | |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1462 | static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, |
| 1463 | const char *End) { |
| 1464 | return CharSourceRange::getCharRange(L.getSourceLocation(Begin), |
| 1465 | L.getSourceLocation(End)); |
| 1466 | } |
| 1467 | |
| 1468 | static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, |
| 1469 | CharSourceRange Range, bool IsFirst) { |
| 1470 | // Check C99 compatibility. |
Alp Toker | d4a3f0e | 2014-06-15 23:30:39 +0000 | [diff] [blame] | 1471 | if (!Diags.isIgnored(diag::warn_c99_compat_unicode_id, Range.getBegin())) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1472 | enum { |
| 1473 | CannotAppearInIdentifier = 0, |
| 1474 | CannotStartIdentifier |
| 1475 | }; |
| 1476 | |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1477 | static const llvm::sys::UnicodeCharSet C99AllowedIDChars( |
| 1478 | C99AllowedIDCharRanges); |
| 1479 | static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( |
| 1480 | C99DisallowedInitialIDCharRanges); |
| 1481 | if (!C99AllowedIDChars.contains(C)) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1482 | Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) |
| 1483 | << Range |
| 1484 | << CannotAppearInIdentifier; |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1485 | } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1486 | Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) |
| 1487 | << Range |
| 1488 | << CannotStartIdentifier; |
| 1489 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1490 | } |
| 1491 | |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1492 | // Check C++98 compatibility. |
Alp Toker | d4a3f0e | 2014-06-15 23:30:39 +0000 | [diff] [blame] | 1493 | if (!Diags.isIgnored(diag::warn_cxx98_compat_unicode_id, Range.getBegin())) { |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 1494 | static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( |
| 1495 | CXX03AllowedIDCharRanges); |
| 1496 | if (!CXX03AllowedIDChars.contains(C)) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 1497 | Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) |
| 1498 | << Range; |
| 1499 | } |
| 1500 | } |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1501 | } |
| 1502 | |
| 1503 | bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size, |
| 1504 | Token &Result) { |
| 1505 | const char *UCNPtr = CurPtr + Size; |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1506 | uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/nullptr); |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1507 | if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) |
| 1508 | return false; |
| 1509 | |
| 1510 | if (!isLexingRawMode()) |
| 1511 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, |
| 1512 | makeCharRange(*this, CurPtr, UCNPtr), |
| 1513 | /*IsFirst=*/false); |
| 1514 | |
| 1515 | Result.setFlag(Token::HasUCN); |
| 1516 | if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || |
| 1517 | (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) |
| 1518 | CurPtr = UCNPtr; |
| 1519 | else |
| 1520 | while (CurPtr != UCNPtr) |
| 1521 | (void)getAndAdvanceChar(CurPtr, Result); |
| 1522 | return true; |
| 1523 | } |
| 1524 | |
| 1525 | bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) { |
| 1526 | const char *UnicodePtr = CurPtr; |
Justin Lebar | 9091055 | 2016-09-30 00:38:45 +0000 | [diff] [blame] | 1527 | llvm::UTF32 CodePoint; |
| 1528 | llvm::ConversionResult Result = |
| 1529 | llvm::convertUTF8Sequence((const llvm::UTF8 **)&UnicodePtr, |
| 1530 | (const llvm::UTF8 *)BufferEnd, |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1531 | &CodePoint, |
Justin Lebar | 9091055 | 2016-09-30 00:38:45 +0000 | [diff] [blame] | 1532 | llvm::strictConversion); |
| 1533 | if (Result != llvm::conversionOK || |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1534 | !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) |
| 1535 | return false; |
| 1536 | |
| 1537 | if (!isLexingRawMode()) |
| 1538 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, |
| 1539 | makeCharRange(*this, CurPtr, UnicodePtr), |
| 1540 | /*IsFirst=*/false); |
| 1541 | |
| 1542 | CurPtr = UnicodePtr; |
| 1543 | return true; |
| 1544 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1545 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1546 | bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1547 | // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] |
| 1548 | unsigned Size; |
| 1549 | unsigned char C = *CurPtr++; |
Chris Lattner | 21d9b9a | 2010-01-11 02:38:50 +0000 | [diff] [blame] | 1550 | while (isIdentifierBody(C)) |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1551 | C = *CurPtr++; |
Chris Lattner | 21d9b9a | 2010-01-11 02:38:50 +0000 | [diff] [blame] | 1552 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1553 | --CurPtr; // Back up over the skipped character. |
| 1554 | |
| 1555 | // Fast path, no $,\,? in identifier found. '\' might be an escaped newline |
| 1556 | // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. |
Chris Lattner | 21d9b9a | 2010-01-11 02:38:50 +0000 | [diff] [blame] | 1557 | // |
Jordan Rose | a2100d7 | 2013-02-08 22:30:22 +0000 | [diff] [blame] | 1558 | // TODO: Could merge these checks into an InfoTable flag to make the |
| 1559 | // comparison cheaper |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1560 | if (isASCII(C) && C != '\\' && C != '?' && |
| 1561 | (C != '$' || !LangOpts.DollarIdents)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1562 | FinishIdentifier: |
Chris Lattner | cefc768 | 2006-07-08 08:28:12 +0000 | [diff] [blame] | 1563 | const char *IdStart = BufferPtr; |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 1564 | FormTokenWithChars(Result, CurPtr, tok::raw_identifier); |
| 1565 | Result.setRawIdentifierData(IdStart); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1566 | |
Chris Lattner | 0f1f505 | 2006-07-20 04:16:23 +0000 | [diff] [blame] | 1567 | // If we are in raw mode, return this identifier raw. There is no need to |
| 1568 | // look up identifier information or attempt to macro expand it. |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 1569 | if (LexingRawMode) |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1570 | return true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1571 | |
Abramo Bagnara | ea4f7c7 | 2010-12-22 08:23:18 +0000 | [diff] [blame] | 1572 | // Fill in Result.IdentifierInfo and update the token kind, |
| 1573 | // looking up the identifier in the identifier table. |
| 1574 | IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1575 | |
Chris Lattner | c5a0006 | 2006-06-18 16:41:01 +0000 | [diff] [blame] | 1576 | // Finally, now that we know we have an identifier, pass this off to the |
| 1577 | // preprocessor, which may macro expand it or something. |
Chris Lattner | 8256b97 | 2009-01-21 07:45:14 +0000 | [diff] [blame] | 1578 | if (II->isHandleIdentifierCase()) |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1579 | return PP->HandleIdentifier(Result); |
Vassil Vassilev | 644ea61 | 2016-07-27 14:56:59 +0000 | [diff] [blame] | 1580 | |
| 1581 | if (II->getTokenID() == tok::identifier && isCodeCompletionPoint(CurPtr) |
| 1582 | && II->getPPKeywordID() == tok::pp_not_keyword |
| 1583 | && II->getObjCKeywordID() == tok::objc_not_keyword) { |
| 1584 | // Return the code-completion token. |
| 1585 | Result.setKind(tok::code_completion); |
| 1586 | cutOffLexing(); |
| 1587 | return true; |
| 1588 | } |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1589 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1590 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1591 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1592 | // Otherwise, $,\,? in identifier found. Enter slower path. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1593 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1594 | C = getCharAndSize(CurPtr, Size); |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 1595 | while (true) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1596 | if (C == '$') { |
| 1597 | // If we hit a $ and they are not supported in identifiers, we are done. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1598 | if (!LangOpts.DollarIdents) goto FinishIdentifier; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1599 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1600 | // Otherwise, emit a diagnostic and continue. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1601 | if (!isLexingRawMode()) |
| 1602 | Diag(CurPtr, diag::ext_dollar_in_identifier); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1603 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
| 1604 | C = getCharAndSize(CurPtr, Size); |
| 1605 | continue; |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1606 | } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1607 | C = getCharAndSize(CurPtr, Size); |
| 1608 | continue; |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1609 | } else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1610 | C = getCharAndSize(CurPtr, Size); |
| 1611 | continue; |
| 1612 | } else if (!isIdentifierBody(C)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1613 | goto FinishIdentifier; |
| 1614 | } |
| 1615 | |
| 1616 | // Otherwise, this character is good, consume it. |
| 1617 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
| 1618 | |
| 1619 | C = getCharAndSize(CurPtr, Size); |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 1620 | while (isIdentifierBody(C)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1621 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
| 1622 | C = getCharAndSize(CurPtr, Size); |
| 1623 | } |
| 1624 | } |
| 1625 | } |
| 1626 | |
Douglas Gregor | 759ef23 | 2010-08-30 14:50:47 +0000 | [diff] [blame] | 1627 | /// isHexaLiteral - Return true if Start points to a hex constant. |
Chris Lattner | 5f183aa | 2010-08-30 17:11:14 +0000 | [diff] [blame] | 1628 | /// in microsoft mode (where this is supposed to be several different tokens). |
Eli Friedman | 324adad | 2012-08-31 02:29:37 +0000 | [diff] [blame] | 1629 | bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { |
Chris Lattner | 0f0492e | 2010-08-31 16:42:00 +0000 | [diff] [blame] | 1630 | unsigned Size; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1631 | char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); |
Chris Lattner | 0f0492e | 2010-08-31 16:42:00 +0000 | [diff] [blame] | 1632 | if (C1 != '0') |
| 1633 | return false; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1634 | char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); |
Chris Lattner | 0f0492e | 2010-08-31 16:42:00 +0000 | [diff] [blame] | 1635 | return (C2 == 'x' || C2 == 'X'); |
Douglas Gregor | 759ef23 | 2010-08-30 14:50:47 +0000 | [diff] [blame] | 1636 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1637 | |
Nate Begeman | 5eee933 | 2008-04-14 02:26:39 +0000 | [diff] [blame] | 1638 | /// LexNumericConstant - Lex the remainder of a integer or floating point |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1639 | /// constant. From[-1] is the first character lexed. Return the end of the |
| 1640 | /// constant. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1641 | bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1642 | unsigned Size; |
| 1643 | char C = getCharAndSize(CurPtr, Size); |
| 1644 | char PrevCh = 0; |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1645 | while (isPreprocessingNumberBody(C)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1646 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
| 1647 | PrevCh = C; |
| 1648 | C = getCharAndSize(CurPtr, Size); |
| 1649 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1650 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1651 | // If we fell out, check for a sign, due to 1e+12. If we have one, continue. |
Chris Lattner | 7a9e9e7 | 2010-08-30 17:09:08 +0000 | [diff] [blame] | 1652 | if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { |
| 1653 | // If we are in Microsoft mode, don't continue if the constant is hex. |
| 1654 | // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1655 | if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) |
Chris Lattner | 7a9e9e7 | 2010-08-30 17:09:08 +0000 | [diff] [blame] | 1656 | return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); |
| 1657 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1658 | |
| 1659 | // If we have a hex FP constant, continue. |
Richard Smith | e6799dd | 2012-06-15 05:07:49 +0000 | [diff] [blame] | 1660 | if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { |
Richard Smith | 560a357 | 2016-03-04 22:32:06 +0000 | [diff] [blame] | 1661 | // Outside C99 and C++17, we accept hexadecimal floating point numbers as a |
Richard Smith | e6799dd | 2012-06-15 05:07:49 +0000 | [diff] [blame] | 1662 | // not-quite-conforming extension. Only do so if this looks like it's |
| 1663 | // actually meant to be a hexfloat, and not if it has a ud-suffix. |
| 1664 | bool IsHexFloat = true; |
| 1665 | if (!LangOpts.C99) { |
| 1666 | if (!isHexaLiteral(BufferPtr, LangOpts)) |
| 1667 | IsHexFloat = false; |
Aaron Ballman | c351fba | 2017-12-04 20:27:34 +0000 | [diff] [blame] | 1668 | else if (!getLangOpts().CPlusPlus17 && |
Richard Smith | 560a357 | 2016-03-04 22:32:06 +0000 | [diff] [blame] | 1669 | std::find(BufferPtr, CurPtr, '_') != CurPtr) |
Richard Smith | e6799dd | 2012-06-15 05:07:49 +0000 | [diff] [blame] | 1670 | IsHexFloat = false; |
| 1671 | } |
| 1672 | if (IsHexFloat) |
| 1673 | return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); |
| 1674 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1675 | |
Richard Smith | fde9485 | 2013-09-26 03:33:06 +0000 | [diff] [blame] | 1676 | // If we have a digit separator, continue. |
Aaron Ballman | dd69ef3 | 2014-08-19 15:55:55 +0000 | [diff] [blame] | 1677 | if (C == '\'' && getLangOpts().CPlusPlus14) { |
Richard Smith | fde9485 | 2013-09-26 03:33:06 +0000 | [diff] [blame] | 1678 | unsigned NextSize; |
| 1679 | char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts()); |
Richard Smith | 7f2707a | 2013-09-26 18:13:20 +0000 | [diff] [blame] | 1680 | if (isIdentifierBody(Next)) { |
Richard Smith | fde9485 | 2013-09-26 03:33:06 +0000 | [diff] [blame] | 1681 | if (!isLexingRawMode()) |
| 1682 | Diag(CurPtr, diag::warn_cxx11_compat_digit_separator); |
| 1683 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
Richard Smith | 35ddad0 | 2014-02-28 20:06:02 +0000 | [diff] [blame] | 1684 | CurPtr = ConsumeChar(CurPtr, NextSize, Result); |
Richard Smith | fde9485 | 2013-09-26 03:33:06 +0000 | [diff] [blame] | 1685 | return LexNumericConstant(Result, CurPtr); |
| 1686 | } |
| 1687 | } |
| 1688 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1689 | // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue. |
| 1690 | if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) |
| 1691 | return LexNumericConstant(Result, CurPtr); |
| 1692 | if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) |
| 1693 | return LexNumericConstant(Result, CurPtr); |
| 1694 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1695 | // Update the location of token as well as BufferPtr. |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1696 | const char *TokStart = BufferPtr; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1697 | FormTokenWithChars(Result, CurPtr, tok::numeric_constant); |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1698 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1699 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1700 | } |
| 1701 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1702 | /// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes |
Richard Smith | 3e4a60a | 2012-03-07 03:13:00 +0000 | [diff] [blame] | 1703 | /// in C++11, or warn on a ud-suffix in C++98. |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1704 | const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr, |
| 1705 | bool IsStringLiteral) { |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1706 | assert(getLangOpts().CPlusPlus); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1707 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1708 | // Maximally munch an identifier. |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1709 | unsigned Size; |
| 1710 | char C = getCharAndSize(CurPtr, Size); |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1711 | bool Consumed = false; |
Richard Smith | 0df56f4 | 2012-03-08 02:39:21 +0000 | [diff] [blame] | 1712 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1713 | if (!isIdentifierHead(C)) { |
| 1714 | if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) |
| 1715 | Consumed = true; |
| 1716 | else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) |
| 1717 | Consumed = true; |
| 1718 | else |
| 1719 | return CurPtr; |
| 1720 | } |
| 1721 | |
| 1722 | if (!getLangOpts().CPlusPlus11) { |
| 1723 | if (!isLexingRawMode()) |
| 1724 | Diag(CurPtr, |
| 1725 | C == '_' ? diag::warn_cxx11_compat_user_defined_literal |
| 1726 | : diag::warn_cxx11_compat_reserved_user_defined_literal) |
| 1727 | << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); |
| 1728 | return CurPtr; |
| 1729 | } |
| 1730 | |
| 1731 | // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix |
| 1732 | // that does not start with an underscore is ill-formed. As a conforming |
| 1733 | // extension, we treat all such suffixes as if they had whitespace before |
| 1734 | // them. We assume a suffix beginning with a UCN or UTF-8 character is more |
| 1735 | // likely to be a ud-suffix than a macro, however, and accept that. |
| 1736 | if (!Consumed) { |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1737 | bool IsUDSuffix = false; |
| 1738 | if (C == '_') |
| 1739 | IsUDSuffix = true; |
Aaron Ballman | dd69ef3 | 2014-08-19 15:55:55 +0000 | [diff] [blame] | 1740 | else if (IsStringLiteral && getLangOpts().CPlusPlus14) { |
Richard Smith | 2a98862 | 2013-09-24 04:06:10 +0000 | [diff] [blame] | 1741 | // In C++1y, we need to look ahead a few characters to see if this is a |
| 1742 | // valid suffix for a string literal or a numeric literal (this could be |
| 1743 | // the 'operator""if' defining a numeric literal operator). |
Richard Smith | 5acb759 | 2013-09-24 22:13:21 +0000 | [diff] [blame] | 1744 | const unsigned MaxStandardSuffixLength = 3; |
Richard Smith | 2a98862 | 2013-09-24 04:06:10 +0000 | [diff] [blame] | 1745 | char Buffer[MaxStandardSuffixLength] = { C }; |
| 1746 | unsigned Consumed = Size; |
| 1747 | unsigned Chars = 1; |
| 1748 | while (true) { |
| 1749 | unsigned NextSize; |
| 1750 | char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize, |
| 1751 | getLangOpts()); |
| 1752 | if (!isIdentifierBody(Next)) { |
| 1753 | // End of suffix. Check whether this is on the whitelist. |
Eric Fiselier | cb2f326 | 2016-12-30 04:51:10 +0000 | [diff] [blame] | 1754 | const StringRef CompleteSuffix(Buffer, Chars); |
| 1755 | IsUDSuffix = StringLiteralParser::isValidUDSuffix(getLangOpts(), |
| 1756 | CompleteSuffix); |
Richard Smith | 2a98862 | 2013-09-24 04:06:10 +0000 | [diff] [blame] | 1757 | break; |
| 1758 | } |
| 1759 | |
| 1760 | if (Chars == MaxStandardSuffixLength) |
| 1761 | // Too long: can't be a standard suffix. |
| 1762 | break; |
| 1763 | |
| 1764 | Buffer[Chars++] = Next; |
| 1765 | Consumed += NextSize; |
| 1766 | } |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1767 | } |
| 1768 | |
| 1769 | if (!IsUDSuffix) { |
Richard Smith | 0df56f4 | 2012-03-08 02:39:21 +0000 | [diff] [blame] | 1770 | if (!isLexingRawMode()) |
Alp Toker | bfa3934 | 2014-01-14 12:51:41 +0000 | [diff] [blame] | 1771 | Diag(CurPtr, getLangOpts().MSVCCompat |
| 1772 | ? diag::ext_ms_reserved_user_defined_literal |
| 1773 | : diag::ext_reserved_user_defined_literal) |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1774 | << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); |
Richard Smith | 3e4a60a | 2012-03-07 03:13:00 +0000 | [diff] [blame] | 1775 | return CurPtr; |
| 1776 | } |
| 1777 | |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1778 | CurPtr = ConsumeChar(CurPtr, Size, Result); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1779 | } |
Richard Smith | 8b7258b | 2014-02-17 21:52:30 +0000 | [diff] [blame] | 1780 | |
| 1781 | Result.setFlag(Token::HasUDSuffix); |
| 1782 | while (true) { |
| 1783 | C = getCharAndSize(CurPtr, Size); |
| 1784 | if (isIdentifierBody(C)) { CurPtr = ConsumeChar(CurPtr, Size, Result); } |
| 1785 | else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {} |
| 1786 | else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) {} |
| 1787 | else break; |
| 1788 | } |
| 1789 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1790 | return CurPtr; |
| 1791 | } |
| 1792 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1793 | /// LexStringLiteral - Lex the remainder of a string literal, after having lexed |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1794 | /// either " or L" or u8" or u" or U". |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1795 | bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr, |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1796 | tok::TokenKind Kind) { |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1797 | // Does this string contain the \0 character? |
| 1798 | const char *NulCharacter = nullptr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1799 | |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 1800 | if (!isLexingRawMode() && |
| 1801 | (Kind == tok::utf8_string_literal || |
| 1802 | Kind == tok::utf16_string_literal || |
Richard Smith | 06d274f | 2013-03-11 18:01:42 +0000 | [diff] [blame] | 1803 | Kind == tok::utf32_string_literal)) |
| 1804 | Diag(BufferPtr, getLangOpts().CPlusPlus |
| 1805 | ? diag::warn_cxx98_compat_unicode_literal |
| 1806 | : diag::warn_c99_compat_unicode_literal); |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 1807 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1808 | char C = getAndAdvanceChar(CurPtr, Result); |
| 1809 | while (C != '"') { |
Chris Lattner | 52d96ac | 2010-05-30 23:27:38 +0000 | [diff] [blame] | 1810 | // Skip escaped characters. Escaped newlines will already be processed by |
| 1811 | // getAndAdvanceChar. |
| 1812 | if (C == '\\') |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1813 | C = getAndAdvanceChar(CurPtr, Result); |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 1814 | |
Chris Lattner | 52d96ac | 2010-05-30 23:27:38 +0000 | [diff] [blame] | 1815 | if (C == '\n' || C == '\r' || // Newline. |
Douglas Gregor | fe4a410 | 2010-05-30 22:59:50 +0000 | [diff] [blame] | 1816 | (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1817 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
Craig Topper | 7f5ff21 | 2015-11-14 02:09:55 +0000 | [diff] [blame] | 1818 | Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1819 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1820 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1821 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 1822 | |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1823 | if (C == 0) { |
| 1824 | if (isCodeCompletionPoint(CurPtr-1)) { |
| 1825 | PP->CodeCompleteNaturalLanguage(); |
| 1826 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1827 | cutOffLexing(); |
| 1828 | return true; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1829 | } |
| 1830 | |
Chris Lattner | 52d96ac | 2010-05-30 23:27:38 +0000 | [diff] [blame] | 1831 | NulCharacter = CurPtr-1; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1832 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1833 | C = getAndAdvanceChar(CurPtr, Result); |
| 1834 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1835 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1836 | // If we are in C++11, lex the optional ud-suffix. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1837 | if (getLangOpts().CPlusPlus) |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1838 | CurPtr = LexUDSuffix(Result, CurPtr, true); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1839 | |
Chris Lattner | 5a78a02 | 2006-07-20 06:02:19 +0000 | [diff] [blame] | 1840 | // If a nul character existed in the string, warn about it. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1841 | if (NulCharacter && !isLexingRawMode()) |
Craig Topper | 7f5ff21 | 2015-11-14 02:09:55 +0000 | [diff] [blame] | 1842 | Diag(NulCharacter, diag::null_in_char_or_string) << 1; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1843 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1844 | // Update the location of the token as well as the BufferPtr instance var. |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1845 | const char *TokStart = BufferPtr; |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1846 | FormTokenWithChars(Result, CurPtr, Kind); |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1847 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1848 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1849 | } |
| 1850 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1851 | /// LexRawStringLiteral - Lex the remainder of a raw string literal, after |
| 1852 | /// having lexed R", LR", u8R", uR", or UR". |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1853 | bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1854 | tok::TokenKind Kind) { |
| 1855 | // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: |
| 1856 | // Between the initial and final double quote characters of the raw string, |
| 1857 | // any transformations performed in phases 1 and 2 (trigraphs, |
| 1858 | // universal-character-names, and line splicing) are reverted. |
| 1859 | |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 1860 | if (!isLexingRawMode()) |
| 1861 | Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); |
| 1862 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1863 | unsigned PrefixLen = 0; |
| 1864 | |
| 1865 | while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) |
| 1866 | ++PrefixLen; |
| 1867 | |
| 1868 | // If the last character was not a '(', then we didn't lex a valid delimiter. |
| 1869 | if (CurPtr[PrefixLen] != '(') { |
| 1870 | if (!isLexingRawMode()) { |
| 1871 | const char *PrefixEnd = &CurPtr[PrefixLen]; |
| 1872 | if (PrefixLen == 16) { |
| 1873 | Diag(PrefixEnd, diag::err_raw_delim_too_long); |
| 1874 | } else { |
| 1875 | Diag(PrefixEnd, diag::err_invalid_char_raw_delim) |
| 1876 | << StringRef(PrefixEnd, 1); |
| 1877 | } |
| 1878 | } |
| 1879 | |
| 1880 | // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, |
| 1881 | // it's possible the '"' was intended to be part of the raw string, but |
| 1882 | // there's not much we can do about that. |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 1883 | while (true) { |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1884 | char C = *CurPtr++; |
| 1885 | |
| 1886 | if (C == '"') |
| 1887 | break; |
| 1888 | if (C == 0 && CurPtr-1 == BufferEnd) { |
| 1889 | --CurPtr; |
| 1890 | break; |
| 1891 | } |
| 1892 | } |
| 1893 | |
| 1894 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1895 | return true; |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1896 | } |
| 1897 | |
| 1898 | // Save prefix and move CurPtr past it |
| 1899 | const char *Prefix = CurPtr; |
| 1900 | CurPtr += PrefixLen + 1; // skip over prefix and '(' |
| 1901 | |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 1902 | while (true) { |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1903 | char C = *CurPtr++; |
| 1904 | |
| 1905 | if (C == ')') { |
| 1906 | // Check for prefix match and closing quote. |
| 1907 | if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { |
| 1908 | CurPtr += PrefixLen + 1; // skip over prefix and '"' |
| 1909 | break; |
| 1910 | } |
| 1911 | } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. |
| 1912 | if (!isLexingRawMode()) |
| 1913 | Diag(BufferPtr, diag::err_unterminated_raw_string) |
| 1914 | << StringRef(Prefix, PrefixLen); |
| 1915 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1916 | return true; |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1917 | } |
| 1918 | } |
| 1919 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1920 | // If we are in C++11, lex the optional ud-suffix. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1921 | if (getLangOpts().CPlusPlus) |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 1922 | CurPtr = LexUDSuffix(Result, CurPtr, true); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 1923 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1924 | // Update the location of token as well as BufferPtr. |
| 1925 | const char *TokStart = BufferPtr; |
| 1926 | FormTokenWithChars(Result, CurPtr, Kind); |
| 1927 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1928 | return true; |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 1929 | } |
| 1930 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1931 | /// LexAngledStringLiteral - Lex the remainder of an angled string literal, |
| 1932 | /// after having lexed the '<' character. This is used for #include filenames. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1933 | bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1934 | // Does this string contain the \0 character? |
| 1935 | const char *NulCharacter = nullptr; |
Chris Lattner | b40289b | 2009-04-17 23:56:52 +0000 | [diff] [blame] | 1936 | const char *AfterLessPos = CurPtr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1937 | char C = getAndAdvanceChar(CurPtr, Result); |
| 1938 | while (C != '>') { |
| 1939 | // Skip escaped characters. |
Kostya Serebryany | 6c2479b | 2015-05-04 22:30:29 +0000 | [diff] [blame] | 1940 | if (C == '\\' && CurPtr < BufferEnd) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1941 | // Skip the escaped character. |
Dmitri Gribenko | 4aa05c5 | 2012-07-30 17:59:40 +0000 | [diff] [blame] | 1942 | getAndAdvanceChar(CurPtr, Result); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1943 | } else if (C == '\n' || C == '\r' || // Newline. |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 1944 | (C == 0 && (CurPtr-1 == BufferEnd || // End of file. |
| 1945 | isCodeCompletionPoint(CurPtr-1)))) { |
Chris Lattner | b40289b | 2009-04-17 23:56:52 +0000 | [diff] [blame] | 1946 | // If the filename is unterminated, then it must just be a lone < |
| 1947 | // character. Return this as such. |
| 1948 | FormTokenWithChars(Result, AfterLessPos, tok::less); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1949 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1950 | } else if (C == 0) { |
| 1951 | NulCharacter = CurPtr-1; |
| 1952 | } |
| 1953 | C = getAndAdvanceChar(CurPtr, Result); |
| 1954 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1955 | |
Chris Lattner | 5a78a02 | 2006-07-20 06:02:19 +0000 | [diff] [blame] | 1956 | // If a nul character existed in the string, warn about it. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 1957 | if (NulCharacter && !isLexingRawMode()) |
Craig Topper | 7f5ff21 | 2015-11-14 02:09:55 +0000 | [diff] [blame] | 1958 | Diag(NulCharacter, diag::null_in_char_or_string) << 1; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 1959 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 1960 | // Update the location of token as well as BufferPtr. |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1961 | const char *TokStart = BufferPtr; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1962 | FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 1963 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1964 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1965 | } |
| 1966 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1967 | /// LexCharConstant - Lex the remainder of a character constant, after having |
Richard Smith | 3e3a705 | 2014-11-08 06:08:42 +0000 | [diff] [blame] | 1968 | /// lexed either ' or L' or u8' or u' or U'. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1969 | bool Lexer::LexCharConstant(Token &Result, const char *CurPtr, |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 1970 | tok::TokenKind Kind) { |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 1971 | // Does this character contain the \0 character? |
| 1972 | const char *NulCharacter = nullptr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1973 | |
Richard Smith | 3e3a705 | 2014-11-08 06:08:42 +0000 | [diff] [blame] | 1974 | if (!isLexingRawMode()) { |
| 1975 | if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant) |
| 1976 | Diag(BufferPtr, getLangOpts().CPlusPlus |
| 1977 | ? diag::warn_cxx98_compat_unicode_literal |
| 1978 | : diag::warn_c99_compat_unicode_literal); |
| 1979 | else if (Kind == tok::utf8_char_constant) |
| 1980 | Diag(BufferPtr, diag::warn_cxx14_compat_u8_character_literal); |
| 1981 | } |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 1982 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 1983 | char C = getAndAdvanceChar(CurPtr, Result); |
| 1984 | if (C == '\'') { |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1985 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
Richard Smith | 608c0b6 | 2012-06-28 07:51:56 +0000 | [diff] [blame] | 1986 | Diag(BufferPtr, diag::ext_empty_character); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 1987 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 1988 | return true; |
Chris Lattner | 86851b8 | 2010-07-07 23:24:27 +0000 | [diff] [blame] | 1989 | } |
| 1990 | |
| 1991 | while (C != '\'') { |
| 1992 | // Skip escaped characters. |
Nico Weber | 4e27038 | 2012-11-17 20:25:54 +0000 | [diff] [blame] | 1993 | if (C == '\\') |
| 1994 | C = getAndAdvanceChar(CurPtr, Result); |
| 1995 | |
| 1996 | if (C == '\n' || C == '\r' || // Newline. |
| 1997 | (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 1998 | if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) |
Craig Topper | 7f5ff21 | 2015-11-14 02:09:55 +0000 | [diff] [blame] | 1999 | Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 0; |
Chris Lattner | 86851b8 | 2010-07-07 23:24:27 +0000 | [diff] [blame] | 2000 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2001 | return true; |
Nico Weber | 4e27038 | 2012-11-17 20:25:54 +0000 | [diff] [blame] | 2002 | } |
| 2003 | |
| 2004 | if (C == 0) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2005 | if (isCodeCompletionPoint(CurPtr-1)) { |
| 2006 | PP->CodeCompleteNaturalLanguage(); |
| 2007 | FormTokenWithChars(Result, CurPtr-1, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2008 | cutOffLexing(); |
| 2009 | return true; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2010 | } |
| 2011 | |
Chris Lattner | 86851b8 | 2010-07-07 23:24:27 +0000 | [diff] [blame] | 2012 | NulCharacter = CurPtr-1; |
| 2013 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2014 | C = getAndAdvanceChar(CurPtr, Result); |
| 2015 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2016 | |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 2017 | // If we are in C++11, lex the optional ud-suffix. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 2018 | if (getLangOpts().CPlusPlus) |
Richard Smith | f4198b7 | 2013-07-23 08:14:48 +0000 | [diff] [blame] | 2019 | CurPtr = LexUDSuffix(Result, CurPtr, false); |
Richard Smith | e18f0fa | 2012-03-05 04:02:15 +0000 | [diff] [blame] | 2020 | |
Chris Lattner | 86851b8 | 2010-07-07 23:24:27 +0000 | [diff] [blame] | 2021 | // If a nul character existed in the character, warn about it. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2022 | if (NulCharacter && !isLexingRawMode()) |
Craig Topper | 7f5ff21 | 2015-11-14 02:09:55 +0000 | [diff] [blame] | 2023 | Diag(NulCharacter, diag::null_in_char_or_string) << 0; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2024 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 2025 | // Update the location of token as well as BufferPtr. |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 2026 | const char *TokStart = BufferPtr; |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 2027 | FormTokenWithChars(Result, CurPtr, Kind); |
Chris Lattner | 5a7971e | 2009-01-26 19:29:26 +0000 | [diff] [blame] | 2028 | Result.setLiteralData(TokStart); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2029 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2030 | } |
| 2031 | |
| 2032 | /// SkipWhitespace - Efficiently skip over a series of whitespace characters. |
| 2033 | /// Update BufferPtr to point to the next non-whitespace character and return. |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2034 | /// |
| 2035 | /// This method forms a token and returns true if KeepWhitespaceMode is enabled. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2036 | bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr, |
| 2037 | bool &TokAtPhysicalStartOfLine) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2038 | // Whitespace - Skip it, then return the token after the whitespace. |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2039 | bool SawNewline = isVerticalWhitespace(CurPtr[-1]); |
| 2040 | |
Richard Smith | 0f7f6f1a | 2013-05-10 02:36:35 +0000 | [diff] [blame] | 2041 | unsigned char Char = *CurPtr; |
| 2042 | |
| 2043 | // Skip consecutive spaces efficiently. |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 2044 | while (true) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2045 | // Skip horizontal whitespace very aggressively. |
| 2046 | while (isHorizontalWhitespace(Char)) |
| 2047 | Char = *++CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2048 | |
Daniel Dunbar | 5c4cc09 | 2008-11-25 00:20:22 +0000 | [diff] [blame] | 2049 | // Otherwise if we have something other than whitespace, we're done. |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2050 | if (!isVerticalWhitespace(Char)) |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2051 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2052 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2053 | if (ParsingPreprocessorDirective) { |
| 2054 | // End of preprocessor directive line, let LexTokenInternal handle this. |
| 2055 | BufferPtr = CurPtr; |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2056 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2057 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2058 | |
Richard Smith | 0f7f6f1a | 2013-05-10 02:36:35 +0000 | [diff] [blame] | 2059 | // OK, but handle newline. |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2060 | SawNewline = true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2061 | Char = *++CurPtr; |
| 2062 | } |
| 2063 | |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2064 | // If the client wants us to return whitespace, return it now. |
| 2065 | if (isKeepWhitespaceMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2066 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2067 | if (SawNewline) { |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2068 | IsAtStartOfLine = true; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2069 | IsAtPhysicalStartOfLine = true; |
| 2070 | } |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2071 | // FIXME: The next token will not have LeadingSpace set. |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2072 | return true; |
| 2073 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2074 | |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2075 | // If this isn't immediately after a newline, there is leading space. |
| 2076 | char PrevChar = CurPtr[-1]; |
| 2077 | bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); |
| 2078 | |
| 2079 | Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2080 | if (SawNewline) { |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2081 | Result.setFlag(Token::StartOfLine); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2082 | TokAtPhysicalStartOfLine = true; |
| 2083 | } |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 2084 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2085 | BufferPtr = CurPtr; |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2086 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2087 | } |
| 2088 | |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2089 | /// We have just read the // characters from input. Skip until we find the |
| 2090 | /// newline character thats terminate the comment. Then update BufferPtr and |
| 2091 | /// return. |
Chris Lattner | 87d0208 | 2010-01-18 22:35:47 +0000 | [diff] [blame] | 2092 | /// |
| 2093 | /// If we're in KeepCommentMode or any CommentHandler has inserted |
| 2094 | /// some tokens, this will store the first token and return true. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2095 | bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, |
| 2096 | bool &TokAtPhysicalStartOfLine) { |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2097 | // If Line comments aren't explicitly enabled for this language, emit an |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2098 | // extension warning. |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2099 | if (!LangOpts.LineComment && !isLexingRawMode()) { |
| 2100 | Diag(BufferPtr, diag::ext_line_comment); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2101 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2102 | // Mark them enabled so we only emit one warning for this translation |
| 2103 | // unit. |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2104 | LangOpts.LineComment = true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2105 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2106 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2107 | // Scan over the body of the comment. The common case, when scanning, is that |
| 2108 | // the comment contains normal ascii characters with nothing interesting in |
| 2109 | // them. As such, optimize for this case with the inner loop. |
Richard Smith | 1d2ae94 | 2017-04-17 23:44:51 +0000 | [diff] [blame] | 2110 | // |
| 2111 | // This loop terminates with CurPtr pointing at the newline (or end of buffer) |
| 2112 | // character that ends the line comment. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2113 | char C; |
Richard Smith | 1d2ae94 | 2017-04-17 23:44:51 +0000 | [diff] [blame] | 2114 | while (true) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2115 | C = *CurPtr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2116 | // Skip over characters in the fast loop. |
| 2117 | while (C != 0 && // Potentially EOF. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2118 | C != '\n' && C != '\r') // Newline or DOS-style newline. |
| 2119 | C = *++CurPtr; |
| 2120 | |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2121 | const char *NextLine = CurPtr; |
| 2122 | if (C != 0) { |
| 2123 | // We found a newline, see if it's escaped. |
| 2124 | const char *EscapePtr = CurPtr-1; |
Alp Toker | 6de6da6 | 2013-12-14 23:32:31 +0000 | [diff] [blame] | 2125 | bool HasSpace = false; |
| 2126 | while (isHorizontalWhitespace(*EscapePtr)) { // Skip whitespace. |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2127 | --EscapePtr; |
Alp Toker | 6de6da6 | 2013-12-14 23:32:31 +0000 | [diff] [blame] | 2128 | HasSpace = true; |
| 2129 | } |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2130 | |
Richard Smith | 4c132e5 | 2017-04-18 21:45:04 +0000 | [diff] [blame] | 2131 | if (*EscapePtr == '\\') |
| 2132 | // Escaped newline. |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2133 | CurPtr = EscapePtr; |
| 2134 | else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && |
Richard Smith | 4c132e5 | 2017-04-18 21:45:04 +0000 | [diff] [blame] | 2135 | EscapePtr[-2] == '?' && LangOpts.Trigraphs) |
| 2136 | // Trigraph-escaped newline. |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2137 | CurPtr = EscapePtr-2; |
| 2138 | else |
| 2139 | break; // This is a newline, we're done. |
Alp Toker | 6de6da6 | 2013-12-14 23:32:31 +0000 | [diff] [blame] | 2140 | |
| 2141 | // If there was space between the backslash and newline, warn about it. |
| 2142 | if (HasSpace && !isLexingRawMode()) |
| 2143 | Diag(EscapePtr, diag::backslash_newline_space); |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2144 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2145 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2146 | // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to |
Chris Lattner | e141a9e | 2008-12-12 07:34:39 +0000 | [diff] [blame] | 2147 | // properly decode the character. Read it in raw mode to avoid emitting |
| 2148 | // diagnostics about things like trigraphs. If we see an escaped newline, |
| 2149 | // we'll handle it below. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2150 | const char *OldPtr = CurPtr; |
Chris Lattner | e141a9e | 2008-12-12 07:34:39 +0000 | [diff] [blame] | 2151 | bool OldRawMode = isLexingRawMode(); |
| 2152 | LexingRawMode = true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2153 | C = getAndAdvanceChar(CurPtr, Result); |
Chris Lattner | e141a9e | 2008-12-12 07:34:39 +0000 | [diff] [blame] | 2154 | LexingRawMode = OldRawMode; |
Chris Lattner | ecdaf40 | 2009-04-05 00:26:41 +0000 | [diff] [blame] | 2155 | |
Benjamin Kramer | 17ff23c7 | 2011-09-05 07:19:39 +0000 | [diff] [blame] | 2156 | // If we only read only one character, then no special handling is needed. |
| 2157 | // We're done and can skip forward to the newline. |
| 2158 | if (C != 0 && CurPtr == OldPtr+1) { |
| 2159 | CurPtr = NextLine; |
| 2160 | break; |
| 2161 | } |
| 2162 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2163 | // If we read multiple characters, and one of those characters was a \r or |
Chris Lattner | ff591e2 | 2007-06-09 06:07:22 +0000 | [diff] [blame] | 2164 | // \n, then we had an escaped newline within the comment. Emit diagnostic |
| 2165 | // unless the next line is also a // comment. |
Alex Lorenz | d5bf436 | 2017-10-14 01:18:30 +0000 | [diff] [blame] | 2166 | if (CurPtr != OldPtr + 1 && C != '/' && |
| 2167 | (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2168 | for (; OldPtr != CurPtr; ++OldPtr) |
| 2169 | if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { |
Chris Lattner | ff591e2 | 2007-06-09 06:07:22 +0000 | [diff] [blame] | 2170 | // Okay, we found a // comment that ends in a newline, if the next |
| 2171 | // line is also a // comment, but has spaces, don't emit a diagnostic. |
Benjamin Kramer | dbfb18a | 2011-09-05 07:19:35 +0000 | [diff] [blame] | 2172 | if (isWhitespace(C)) { |
Chris Lattner | ff591e2 | 2007-06-09 06:07:22 +0000 | [diff] [blame] | 2173 | const char *ForwardPtr = CurPtr; |
Benjamin Kramer | dbfb18a | 2011-09-05 07:19:35 +0000 | [diff] [blame] | 2174 | while (isWhitespace(*ForwardPtr)) // Skip whitespace. |
Chris Lattner | ff591e2 | 2007-06-09 06:07:22 +0000 | [diff] [blame] | 2175 | ++ForwardPtr; |
| 2176 | if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') |
| 2177 | break; |
| 2178 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2179 | |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2180 | if (!isLexingRawMode()) |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2181 | Diag(OldPtr-1, diag::ext_multi_line_line_comment); |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2182 | break; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2183 | } |
| 2184 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2185 | |
Richard Smith | 1d2ae94 | 2017-04-17 23:44:51 +0000 | [diff] [blame] | 2186 | if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) { |
| 2187 | --CurPtr; |
| 2188 | break; |
Douglas Gregor | 1158370 | 2010-08-25 17:04:25 +0000 | [diff] [blame] | 2189 | } |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2190 | |
| 2191 | if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { |
| 2192 | PP->CodeCompleteNaturalLanguage(); |
| 2193 | cutOffLexing(); |
| 2194 | return false; |
| 2195 | } |
Richard Smith | 1d2ae94 | 2017-04-17 23:44:51 +0000 | [diff] [blame] | 2196 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2197 | |
Chris Lattner | 93ddf80 | 2010-02-03 21:06:21 +0000 | [diff] [blame] | 2198 | // Found but did not consume the newline. Notify comment handlers about the |
| 2199 | // comment unless we're in a #if 0 block. |
| 2200 | if (PP && !isLexingRawMode() && |
| 2201 | PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), |
| 2202 | getSourceLocation(CurPtr)))) { |
Chris Lattner | 87d0208 | 2010-01-18 22:35:47 +0000 | [diff] [blame] | 2203 | BufferPtr = CurPtr; |
| 2204 | return true; // A token has to be returned. |
| 2205 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2206 | |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2207 | // If we are returning comments as tokens, return this comment as a token. |
Chris Lattner | 8637abd | 2008-10-12 03:22:02 +0000 | [diff] [blame] | 2208 | if (inKeepCommentMode()) |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2209 | return SaveLineComment(Result, CurPtr); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2210 | |
| 2211 | // If we are inside a preprocessor directive and we see the end of line, |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2212 | // return immediately, so that the lexer can return this as an EOD token. |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2213 | if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2214 | BufferPtr = CurPtr; |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2215 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2216 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2217 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2218 | // Otherwise, eat the \n character. We don't care if this is a \n\r or |
Chris Lattner | 87e97ea | 2008-10-12 00:23:07 +0000 | [diff] [blame] | 2219 | // \r\n sequence. This is an efficiency hack (because we know the \n can't |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2220 | // contribute to another token), it isn't needed for correctness. Note that |
| 2221 | // this is ok even in KeepWhitespaceMode, because we would have returned the |
| 2222 | /// comment above in that mode. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2223 | ++CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2224 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2225 | // The next returned token is at the start of the line. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2226 | Result.setFlag(Token::StartOfLine); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2227 | TokAtPhysicalStartOfLine = true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2228 | // No leading whitespace seen so far. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2229 | Result.clearFlag(Token::LeadingSpace); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2230 | BufferPtr = CurPtr; |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2231 | return false; |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2232 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2233 | |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2234 | /// If in save-comment mode, package up this Line comment in an appropriate |
| 2235 | /// way and return it. |
| 2236 | bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2237 | // If we're not in a preprocessor directive, just return the // comment |
| 2238 | // directly. |
| 2239 | FormTokenWithChars(Result, CurPtr, tok::comment); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2240 | |
David Blaikie | d532124 | 2012-06-06 18:52:13 +0000 | [diff] [blame] | 2241 | if (!ParsingPreprocessorDirective || LexingRawMode) |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2242 | return true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2243 | |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2244 | // If this Line-style comment is in a macro definition, transmogrify it into |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2245 | // a C-style block comment. |
Douglas Gregor | dc970f0 | 2010-03-16 22:30:13 +0000 | [diff] [blame] | 2246 | bool Invalid = false; |
| 2247 | std::string Spelling = PP->getSpelling(Result, &Invalid); |
| 2248 | if (Invalid) |
| 2249 | return true; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2250 | |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 2251 | assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2252 | Spelling[1] = '*'; // Change prefix to "/*". |
| 2253 | Spelling += "*/"; // add suffix. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2254 | |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2255 | Result.setKind(tok::comment); |
Dmitri Gribenko | b8e9e75 | 2012-09-24 21:07:17 +0000 | [diff] [blame] | 2256 | PP->CreateString(Spelling, Result, |
Abramo Bagnara | e398e60 | 2011-10-03 18:39:03 +0000 | [diff] [blame] | 2257 | Result.getLocation(), Result.getLocation()); |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2258 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2259 | } |
| 2260 | |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2261 | /// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline |
David Blaikie | 987bcf9 | 2012-06-06 18:43:20 +0000 | [diff] [blame] | 2262 | /// character (either \\n or \\r) is part of an escaped newline sequence. Issue |
| 2263 | /// a diagnostic if so. We know that the newline is inside of a block comment. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2264 | static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, |
Chris Lattner | 1f58305 | 2006-06-18 06:53:56 +0000 | [diff] [blame] | 2265 | Lexer *L) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2266 | assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2267 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2268 | // Back up off the newline. |
| 2269 | --CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2270 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2271 | // If this is a two-character newline sequence, skip the other character. |
| 2272 | if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { |
| 2273 | // \n\n or \r\r -> not escaped newline. |
| 2274 | if (CurPtr[0] == CurPtr[1]) |
| 2275 | return false; |
| 2276 | // \n\r or \r\n -> skip the newline. |
| 2277 | --CurPtr; |
| 2278 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2279 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2280 | // If we have horizontal whitespace, skip over it. We allow whitespace |
| 2281 | // between the slash and newline. |
| 2282 | bool HasSpace = false; |
| 2283 | while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { |
| 2284 | --CurPtr; |
| 2285 | HasSpace = true; |
| 2286 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2287 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2288 | // If we have a slash, we know this is an escaped newline. |
| 2289 | if (*CurPtr == '\\') { |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2290 | if (CurPtr[-1] != '*') return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2291 | } else { |
| 2292 | // It isn't a slash, is it the ?? / trigraph? |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2293 | if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || |
| 2294 | CurPtr[-3] != '*') |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2295 | return false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2296 | |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2297 | // This is the trigraph ending the comment. Emit a stern warning! |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2298 | CurPtr -= 2; |
| 2299 | |
| 2300 | // If no trigraphs are enabled, warn that we ignored this trigraph and |
| 2301 | // ignore this * character. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 2302 | if (!L->getLangOpts().Trigraphs) { |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2303 | if (!L->isLexingRawMode()) |
| 2304 | L->Diag(CurPtr, diag::trigraph_ignored_block_comment); |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2305 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2306 | } |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2307 | if (!L->isLexingRawMode()) |
| 2308 | L->Diag(CurPtr, diag::trigraph_ends_block_comment); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2309 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2310 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2311 | // Warn about having an escaped newline between the */ characters. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2312 | if (!L->isLexingRawMode()) |
| 2313 | L->Diag(CurPtr, diag::escaped_newline_block_comment_end); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2314 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2315 | // If there was space between the backslash and newline, warn about it. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2316 | if (HasSpace && !L->isLexingRawMode()) |
| 2317 | L->Diag(CurPtr, diag::backslash_newline_space); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2318 | |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2319 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2320 | } |
| 2321 | |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2322 | #ifdef __SSE2__ |
| 2323 | #include <emmintrin.h> |
Chris Lattner | 9f6604f | 2006-10-30 20:01:22 +0000 | [diff] [blame] | 2324 | #elif __ALTIVEC__ |
| 2325 | #include <altivec.h> |
| 2326 | #undef bool |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2327 | #endif |
| 2328 | |
James Dennett | f442d24 | 2012-06-17 03:40:43 +0000 | [diff] [blame] | 2329 | /// We have just read from input the / and * characters that started a comment. |
| 2330 | /// Read until we find the * and / characters that terminate the comment. |
| 2331 | /// Note that we don't bother decoding trigraphs or escaped newlines in block |
| 2332 | /// comments, because they cannot cause the comment to end. The only thing |
| 2333 | /// that can happen is the comment could end with an escaped newline between |
| 2334 | /// the terminating * and /. |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2335 | /// |
Chris Lattner | 87d0208 | 2010-01-18 22:35:47 +0000 | [diff] [blame] | 2336 | /// If we're in KeepCommentMode or any CommentHandler has inserted |
| 2337 | /// some tokens, this will store the first token and return true. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2338 | bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr, |
| 2339 | bool &TokAtPhysicalStartOfLine) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2340 | // Scan one character past where we should, looking for a '/' character. Once |
Chris Lattner | 57540c5 | 2011-04-15 05:22:18 +0000 | [diff] [blame] | 2341 | // we find it, check to see if it was preceded by a *. This common |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2342 | // optimization helps people who like to put a lot of * characters in their |
| 2343 | // comments. |
Chris Lattner | c850ad6 | 2007-07-21 23:43:37 +0000 | [diff] [blame] | 2344 | |
| 2345 | // The first character we get with newlines and trigraphs skipped to handle |
| 2346 | // the degenerate /*/ case below correctly if the * has an escaped newline |
| 2347 | // after it. |
| 2348 | unsigned CharSize; |
| 2349 | unsigned char C = getCharAndSize(CurPtr, CharSize); |
| 2350 | CurPtr += CharSize; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2351 | if (C == 0 && CurPtr == BufferEnd+1) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2352 | if (!isLexingRawMode()) |
Chris Lattner | 7c2e980 | 2008-10-12 01:31:51 +0000 | [diff] [blame] | 2353 | Diag(BufferPtr, diag::err_unterminated_block_comment); |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2354 | --CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2355 | |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2356 | // KeepWhitespaceMode should return this broken comment as a token. Since |
| 2357 | // it isn't a well formed comment, just return it as an 'unknown' token. |
| 2358 | if (isKeepWhitespaceMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2359 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2360 | return true; |
| 2361 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2362 | |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2363 | BufferPtr = CurPtr; |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2364 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2365 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2366 | |
Chris Lattner | c850ad6 | 2007-07-21 23:43:37 +0000 | [diff] [blame] | 2367 | // Check to see if the first character after the '/*' is another /. If so, |
| 2368 | // then this slash does not end the block comment, it is part of it. |
| 2369 | if (C == '/') |
| 2370 | C = *CurPtr++; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2371 | |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 2372 | while (true) { |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2373 | // Skip over all non-interesting characters until we find end of buffer or a |
| 2374 | // (probably ending) '/' character. |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2375 | if (CurPtr + 24 < BufferEnd && |
| 2376 | // If there is a code-completion point avoid the fast scan because it |
| 2377 | // doesn't check for '\0'. |
| 2378 | !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2379 | // While not aligned to a 16-byte boundary. |
| 2380 | while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) |
| 2381 | C = *CurPtr++; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2382 | |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2383 | if (C == '/') goto FoundSlash; |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2384 | |
| 2385 | #ifdef __SSE2__ |
Roman Divacky | 6150990 | 2014-04-03 18:04:52 +0000 | [diff] [blame] | 2386 | __m128i Slashes = _mm_set1_epi8('/'); |
| 2387 | while (CurPtr+16 <= BufferEnd) { |
| 2388 | int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, |
| 2389 | Slashes)); |
Benjamin Kramer | 3885737 | 2011-11-22 18:56:46 +0000 | [diff] [blame] | 2390 | if (cmp != 0) { |
Benjamin Kramer | 900f1de | 2011-11-22 20:39:31 +0000 | [diff] [blame] | 2391 | // Adjust the pointer to point directly after the first slash. It's |
| 2392 | // not necessary to set C here, it will be overwritten at the end of |
| 2393 | // the outer loop. |
Michael J. Spencer | 8c39840 | 2013-05-24 21:42:04 +0000 | [diff] [blame] | 2394 | CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1; |
Benjamin Kramer | 3885737 | 2011-11-22 18:56:46 +0000 | [diff] [blame] | 2395 | goto FoundSlash; |
| 2396 | } |
Roman Divacky | 6150990 | 2014-04-03 18:04:52 +0000 | [diff] [blame] | 2397 | CurPtr += 16; |
Benjamin Kramer | 3885737 | 2011-11-22 18:56:46 +0000 | [diff] [blame] | 2398 | } |
Chris Lattner | 9f6604f | 2006-10-30 20:01:22 +0000 | [diff] [blame] | 2399 | #elif __ALTIVEC__ |
| 2400 | __vector unsigned char Slashes = { |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2401 | '/', '/', '/', '/', '/', '/', '/', '/', |
Chris Lattner | 9f6604f | 2006-10-30 20:01:22 +0000 | [diff] [blame] | 2402 | '/', '/', '/', '/', '/', '/', '/', '/' |
| 2403 | }; |
| 2404 | while (CurPtr+16 <= BufferEnd && |
Jay Foad | 6af95d3 | 2014-10-29 14:42:12 +0000 | [diff] [blame] | 2405 | !vec_any_eq(*(const vector unsigned char*)CurPtr, Slashes)) |
Chris Lattner | 9f6604f | 2006-10-30 20:01:22 +0000 | [diff] [blame] | 2406 | CurPtr += 16; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2407 | #else |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2408 | // Scan for '/' quickly. Many block comments are very large. |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2409 | while (CurPtr[0] != '/' && |
| 2410 | CurPtr[1] != '/' && |
| 2411 | CurPtr[2] != '/' && |
| 2412 | CurPtr[3] != '/' && |
| 2413 | CurPtr+4 < BufferEnd) { |
| 2414 | CurPtr += 4; |
| 2415 | } |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2416 | #endif |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2417 | |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2418 | // It has to be one of the bytes scanned, increment to it and read one. |
Chris Lattner | 6cc3e36 | 2006-10-27 04:12:35 +0000 | [diff] [blame] | 2419 | C = *CurPtr++; |
| 2420 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2421 | |
Chris Lattner | aded4a9 | 2006-10-27 04:42:31 +0000 | [diff] [blame] | 2422 | // Loop to scan the remainder. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2423 | while (C != '/' && C != '\0') |
| 2424 | C = *CurPtr++; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2425 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2426 | if (C == '/') { |
Benjamin Kramer | 3885737 | 2011-11-22 18:56:46 +0000 | [diff] [blame] | 2427 | FoundSlash: |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2428 | if (CurPtr[-2] == '*') // We found the final */. We're done! |
| 2429 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2430 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2431 | if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { |
Chris Lattner | 1f58305 | 2006-06-18 06:53:56 +0000 | [diff] [blame] | 2432 | if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2433 | // We found the final */, though it had an escaped newline between the |
| 2434 | // * and /. We're done! |
| 2435 | break; |
| 2436 | } |
| 2437 | } |
| 2438 | if (CurPtr[0] == '*' && CurPtr[1] != '/') { |
| 2439 | // If this is a /* inside of the comment, emit a warning. Don't do this |
| 2440 | // if this is a /*/, which will end the comment. This misses cases with |
| 2441 | // embedded escaped newlines, but oh well. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2442 | if (!isLexingRawMode()) |
| 2443 | Diag(CurPtr-1, diag::warn_nested_block_comment); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2444 | } |
| 2445 | } else if (C == 0 && CurPtr == BufferEnd+1) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2446 | if (!isLexingRawMode()) |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2447 | Diag(BufferPtr, diag::err_unterminated_block_comment); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2448 | // Note: the user probably forgot a */. We could continue immediately |
| 2449 | // after the /*, but this would involve lexing a lot of what really is the |
| 2450 | // comment, which surely would confuse the parser. |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2451 | --CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2452 | |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2453 | // KeepWhitespaceMode should return this broken comment as a token. Since |
| 2454 | // it isn't a well formed comment, just return it as an 'unknown' token. |
| 2455 | if (isKeepWhitespaceMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2456 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2457 | return true; |
| 2458 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2459 | |
Chris Lattner | 99e7d23 | 2008-10-12 04:19:49 +0000 | [diff] [blame] | 2460 | BufferPtr = CurPtr; |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2461 | return false; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2462 | } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { |
| 2463 | PP->CodeCompleteNaturalLanguage(); |
| 2464 | cutOffLexing(); |
| 2465 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2466 | } |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2467 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2468 | C = *CurPtr++; |
| 2469 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2470 | |
Chris Lattner | 93ddf80 | 2010-02-03 21:06:21 +0000 | [diff] [blame] | 2471 | // Notify comment handlers about the comment unless we're in a #if 0 block. |
| 2472 | if (PP && !isLexingRawMode() && |
| 2473 | PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), |
| 2474 | getSourceLocation(CurPtr)))) { |
Chris Lattner | 87d0208 | 2010-01-18 22:35:47 +0000 | [diff] [blame] | 2475 | BufferPtr = CurPtr; |
| 2476 | return true; // A token has to be returned. |
| 2477 | } |
Douglas Gregor | c6d5edd | 2009-07-02 17:08:52 +0000 | [diff] [blame] | 2478 | |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2479 | // If we are returning comments as tokens, return this comment as a token. |
Chris Lattner | 8637abd | 2008-10-12 03:22:02 +0000 | [diff] [blame] | 2480 | if (inKeepCommentMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2481 | FormTokenWithChars(Result, CurPtr, tok::comment); |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2482 | return true; |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2483 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2484 | |
| 2485 | // It is common for the tokens immediately after a /**/ comment to be |
| 2486 | // whitespace. Instead of going through the big switch, handle it |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 2487 | // efficiently now. This is safe even in KeepWhitespaceMode because we would |
| 2488 | // have already returned above with the comment as a token. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2489 | if (isHorizontalWhitespace(*CurPtr)) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2490 | SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine); |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2491 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2492 | } |
| 2493 | |
| 2494 | // Otherwise, just return so that the next character will be lexed as a token. |
| 2495 | BufferPtr = CurPtr; |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2496 | Result.setFlag(Token::LeadingSpace); |
Chris Lattner | e01e758 | 2008-10-12 04:15:42 +0000 | [diff] [blame] | 2497 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2498 | } |
| 2499 | |
| 2500 | //===----------------------------------------------------------------------===// |
| 2501 | // Primary Lexing Entry Points |
| 2502 | //===----------------------------------------------------------------------===// |
| 2503 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2504 | /// ReadToEndOfLine - Read the rest of the current preprocessor line as an |
| 2505 | /// uninterpreted string. This switches the lexer out of directive mode. |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2506 | void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2507 | assert(ParsingPreprocessorDirective && ParsingFilename == false && |
| 2508 | "Must be in a preprocessing directive!"); |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2509 | Token Tmp; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2510 | |
| 2511 | // CurPtr - Cache BufferPtr in an automatic variable. |
| 2512 | const char *CurPtr = BufferPtr; |
Eugene Zelenko | e95e7d5 | 2016-09-07 21:53:17 +0000 | [diff] [blame] | 2513 | while (true) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2514 | char Char = getAndAdvanceChar(CurPtr, Tmp); |
| 2515 | switch (Char) { |
| 2516 | default: |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2517 | if (Result) |
| 2518 | Result->push_back(Char); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2519 | break; |
| 2520 | case 0: // Null. |
| 2521 | // Found end of file? |
| 2522 | if (CurPtr-1 != BufferEnd) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2523 | if (isCodeCompletionPoint(CurPtr-1)) { |
| 2524 | PP->CodeCompleteNaturalLanguage(); |
| 2525 | cutOffLexing(); |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2526 | return; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2527 | } |
| 2528 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2529 | // Nope, normal character, continue. |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2530 | if (Result) |
| 2531 | Result->push_back(Char); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2532 | break; |
| 2533 | } |
| 2534 | // FALL THROUGH. |
Galina Kistanova | 39edaaa | 2017-06-03 06:25:47 +0000 | [diff] [blame] | 2535 | LLVM_FALLTHROUGH; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2536 | case '\r': |
| 2537 | case '\n': |
| 2538 | // Okay, we found the end of the line. First, back up past the \0, \r, \n. |
| 2539 | assert(CurPtr[-1] == Char && "Trigraphs for newline?"); |
| 2540 | BufferPtr = CurPtr-1; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2541 | |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2542 | // Next, lex the character, which should handle the EOD transition. |
Chris Lattner | cb28334 | 2006-06-18 06:48:37 +0000 | [diff] [blame] | 2543 | Lex(Tmp); |
Douglas Gregor | 1158370 | 2010-08-25 17:04:25 +0000 | [diff] [blame] | 2544 | if (Tmp.is(tok::code_completion)) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2545 | if (PP) |
| 2546 | PP->CodeCompleteNaturalLanguage(); |
Douglas Gregor | 1158370 | 2010-08-25 17:04:25 +0000 | [diff] [blame] | 2547 | Lex(Tmp); |
| 2548 | } |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2549 | assert(Tmp.is(tok::eod) && "Unexpected token!"); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2550 | |
Benjamin Kramer | e5fbc6c | 2012-05-18 19:32:16 +0000 | [diff] [blame] | 2551 | // Finally, we're done; |
| 2552 | return; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2553 | } |
| 2554 | } |
| 2555 | } |
| 2556 | |
| 2557 | /// LexEndOfFile - CurPtr points to the end of this file. Handle this |
| 2558 | /// condition, reporting diagnostics and handling other edge cases as required. |
Chris Lattner | 2183a6e | 2006-07-18 06:36:12 +0000 | [diff] [blame] | 2559 | /// This returns true if Result contains a token, false if PP.Lex should be |
| 2560 | /// called again. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2561 | bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2562 | // If we hit the end of the file while parsing a preprocessor directive, |
| 2563 | // end the preprocessor directive first. The next token returned will |
| 2564 | // then be the end of file. |
| 2565 | if (ParsingPreprocessorDirective) { |
| 2566 | // Done parsing the "line". |
| 2567 | ParsingPreprocessorDirective = false; |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 2568 | // Update the location of token as well as BufferPtr. |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 2569 | FormTokenWithChars(Result, CurPtr, tok::eod); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2570 | |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 2571 | // Restore comment saving mode, in case it was disabled for directive. |
Alp Toker | 08c2500 | 2013-12-13 17:04:55 +0000 | [diff] [blame] | 2572 | if (PP) |
| 2573 | resetExtendedTokenMode(); |
Chris Lattner | 2183a6e | 2006-07-18 06:36:12 +0000 | [diff] [blame] | 2574 | return true; // Have a token. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2575 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2576 | |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2577 | // If we are in raw mode, return this event as an EOF token. Let the caller |
| 2578 | // that put us in raw mode handle the event. |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 2579 | if (isLexingRawMode()) { |
Chris Lattner | 8c20487 | 2006-10-14 05:19:21 +0000 | [diff] [blame] | 2580 | Result.startToken(); |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2581 | BufferPtr = BufferEnd; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 2582 | FormTokenWithChars(Result, BufferEnd, tok::eof); |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2583 | return true; |
Chris Lattner | d8aee0e | 2006-07-11 05:04:55 +0000 | [diff] [blame] | 2584 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2585 | |
Erik Verbruggen | 795eee9 | 2017-07-05 09:44:07 +0000 | [diff] [blame] | 2586 | if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) { |
Erik Verbruggen | b34c79f | 2017-05-30 11:54:55 +0000 | [diff] [blame] | 2587 | PP->setRecordedPreambleConditionalStack(ConditionalStack); |
| 2588 | ConditionalStack.clear(); |
| 2589 | } |
| 2590 | |
Douglas Gregor | 3a7ad25 | 2010-08-24 19:08:16 +0000 | [diff] [blame] | 2591 | // Issue diagnostics for unterminated #if and missing newline. |
| 2592 | |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2593 | // If we are in a #if directive, emit an error. |
| 2594 | while (!ConditionalStack.empty()) { |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2595 | if (PP->getCodeCompletionFileLoc() != FileLoc) |
Douglas Gregor | 02690ba | 2010-08-12 17:04:55 +0000 | [diff] [blame] | 2596 | PP->Diag(ConditionalStack.back().IfLoc, |
| 2597 | diag::err_pp_unterminated_conditional); |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2598 | ConditionalStack.pop_back(); |
| 2599 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2600 | |
Chris Lattner | 8f96d04 | 2008-04-12 05:54:25 +0000 | [diff] [blame] | 2601 | // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue |
| 2602 | // a pedwarn. |
Jordan Rose | 4c55d45 | 2013-08-23 15:42:01 +0000 | [diff] [blame] | 2603 | if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) { |
| 2604 | DiagnosticsEngine &Diags = PP->getDiagnostics(); |
| 2605 | SourceLocation EndLoc = getSourceLocation(BufferEnd); |
| 2606 | unsigned DiagID; |
| 2607 | |
| 2608 | if (LangOpts.CPlusPlus11) { |
| 2609 | // C++11 [lex.phases] 2.2 p2 |
| 2610 | // Prefer the C++98 pedantic compatibility warning over the generic, |
| 2611 | // non-extension, user-requested "missing newline at EOF" warning. |
Alp Toker | d4a3f0e | 2014-06-15 23:30:39 +0000 | [diff] [blame] | 2612 | if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc)) { |
Jordan Rose | 4c55d45 | 2013-08-23 15:42:01 +0000 | [diff] [blame] | 2613 | DiagID = diag::warn_cxx98_compat_no_newline_eof; |
| 2614 | } else { |
| 2615 | DiagID = diag::warn_no_newline_eof; |
| 2616 | } |
| 2617 | } else { |
| 2618 | DiagID = diag::ext_no_newline_eof; |
| 2619 | } |
| 2620 | |
| 2621 | Diag(BufferEnd, DiagID) |
| 2622 | << FixItHint::CreateInsertion(EndLoc, "\n"); |
| 2623 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2624 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2625 | BufferPtr = CurPtr; |
Chris Lattner | 30a2fa1 | 2006-07-19 06:31:49 +0000 | [diff] [blame] | 2626 | |
| 2627 | // Finally, let the preprocessor handle this. |
Jordan Rose | 127f6ee | 2012-06-15 23:33:51 +0000 | [diff] [blame] | 2628 | return PP->HandleEndOfFile(Result, isPragmaLexer()); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 2629 | } |
| 2630 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2631 | /// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from |
| 2632 | /// the specified lexer will return a tok::l_paren token, 0 if it is something |
| 2633 | /// else and 2 if there are no more tokens in the buffer controlled by the |
| 2634 | /// lexer. |
| 2635 | unsigned Lexer::isNextPPTokenLParen() { |
| 2636 | assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2637 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2638 | // Switch to 'skipping' mode. This will ensure that we can lex a token |
| 2639 | // without emitting diagnostics, disables macro expansion, and will cause EOF |
| 2640 | // to return an EOF token instead of popping the include stack. |
| 2641 | LexingRawMode = true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2642 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2643 | // Save state that can be changed while lexing so that we can restore it. |
| 2644 | const char *TmpBufferPtr = BufferPtr; |
Chris Lattner | 40493eb | 2009-04-24 07:15:46 +0000 | [diff] [blame] | 2645 | bool inPPDirectiveMode = ParsingPreprocessorDirective; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2646 | bool atStartOfLine = IsAtStartOfLine; |
| 2647 | bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; |
| 2648 | bool leadingSpace = HasLeadingSpace; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2649 | |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 2650 | Token Tok; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2651 | Lex(Tok); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2652 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2653 | // Restore state that may have changed. |
| 2654 | BufferPtr = TmpBufferPtr; |
Chris Lattner | 40493eb | 2009-04-24 07:15:46 +0000 | [diff] [blame] | 2655 | ParsingPreprocessorDirective = inPPDirectiveMode; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2656 | HasLeadingSpace = leadingSpace; |
| 2657 | IsAtStartOfLine = atStartOfLine; |
| 2658 | IsAtPhysicalStartOfLine = atPhysicalStartOfLine; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2659 | |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2660 | // Restore the lexer back to non-skipping mode. |
| 2661 | LexingRawMode = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 2662 | |
Chris Lattner | 98c1f7c | 2007-10-09 18:02:16 +0000 | [diff] [blame] | 2663 | if (Tok.is(tok::eof)) |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2664 | return 2; |
Chris Lattner | 98c1f7c | 2007-10-09 18:02:16 +0000 | [diff] [blame] | 2665 | return Tok.is(tok::l_paren); |
Chris Lattner | 678c880 | 2006-07-11 05:46:12 +0000 | [diff] [blame] | 2666 | } |
| 2667 | |
James Dennett | f442d24 | 2012-06-17 03:40:43 +0000 | [diff] [blame] | 2668 | /// \brief Find the end of a version control conflict marker. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2669 | static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, |
| 2670 | ConflictMarkerKind CMK) { |
| 2671 | const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; |
| 2672 | size_t TermLen = CMK == CMK_Perforce ? 5 : 7; |
Benjamin Kramer | e550bbd | 2016-04-01 09:58:45 +0000 | [diff] [blame] | 2673 | auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(TermLen); |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2674 | size_t Pos = RestOfBuffer.find(Terminator); |
Chris Lattner | 0e62c1c | 2011-07-23 10:55:15 +0000 | [diff] [blame] | 2675 | while (Pos != StringRef::npos) { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2676 | // Must occur at start of line. |
David Majnemer | 5a54977 | 2014-12-14 04:53:11 +0000 | [diff] [blame] | 2677 | if (Pos == 0 || |
| 2678 | (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) { |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2679 | RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); |
| 2680 | Pos = RestOfBuffer.find(Terminator); |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2681 | continue; |
| 2682 | } |
| 2683 | return RestOfBuffer.data()+Pos; |
| 2684 | } |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 2685 | return nullptr; |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2686 | } |
| 2687 | |
| 2688 | /// IsStartOfConflictMarker - If the specified pointer is the start of a version |
| 2689 | /// control conflict marker like '<<<<<<<', recognize it as such, emit an error |
| 2690 | /// and recover nicely. This returns true if it is a conflict marker and false |
| 2691 | /// if not. |
| 2692 | bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { |
| 2693 | // Only a conflict marker if it starts at the beginning of a line. |
| 2694 | if (CurPtr != BufferStart && |
| 2695 | CurPtr[-1] != '\n' && CurPtr[-1] != '\r') |
| 2696 | return false; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2697 | |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2698 | // Check to see if we have <<<<<<< or >>>>. |
Benjamin Kramer | 22f24f6 | 2016-04-01 10:04:07 +0000 | [diff] [blame] | 2699 | if (!StringRef(CurPtr, BufferEnd - CurPtr).startswith("<<<<<<<") && |
| 2700 | !StringRef(CurPtr, BufferEnd - CurPtr).startswith(">>>> ")) |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2701 | return false; |
| 2702 | |
| 2703 | // If we have a situation where we don't care about conflict markers, ignore |
| 2704 | // it. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2705 | if (CurrentConflictMarkerState || isLexingRawMode()) |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2706 | return false; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2707 | |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2708 | ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; |
| 2709 | |
| 2710 | // Check to see if there is an ending marker somewhere in the buffer at the |
| 2711 | // start of a line to terminate this conflict marker. |
| 2712 | if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2713 | // We found a match. We are really in a conflict marker. |
| 2714 | // Diagnose this, and ignore to the end of line. |
| 2715 | Diag(CurPtr, diag::err_conflict_marker); |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2716 | CurrentConflictMarkerState = Kind; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2717 | |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2718 | // Skip ahead to the end of line. We know this exists because the |
| 2719 | // end-of-conflict marker starts with \r or \n. |
| 2720 | while (*CurPtr != '\r' && *CurPtr != '\n') { |
| 2721 | assert(CurPtr != BufferEnd && "Didn't find end of line"); |
| 2722 | ++CurPtr; |
| 2723 | } |
| 2724 | BufferPtr = CurPtr; |
| 2725 | return true; |
| 2726 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2727 | |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2728 | // No end of conflict marker found. |
| 2729 | return false; |
| 2730 | } |
| 2731 | |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2732 | /// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if |
| 2733 | /// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it |
| 2734 | /// is the end of a conflict marker. Handle it by ignoring up until the end of |
| 2735 | /// the line. This returns true if it is a conflict marker and false if not. |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2736 | bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { |
| 2737 | // Only a conflict marker if it starts at the beginning of a line. |
| 2738 | if (CurPtr != BufferStart && |
| 2739 | CurPtr[-1] != '\n' && CurPtr[-1] != '\r') |
| 2740 | return false; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2741 | |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2742 | // If we have a situation where we don't care about conflict markers, ignore |
| 2743 | // it. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2744 | if (!CurrentConflictMarkerState || isLexingRawMode()) |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2745 | return false; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2746 | |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2747 | // Check to see if we have the marker (4 characters in a row). |
| 2748 | for (unsigned i = 1; i != 4; ++i) |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2749 | if (CurPtr[i] != CurPtr[0]) |
| 2750 | return false; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2751 | |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2752 | // If we do have it, search for the end of the conflict marker. This could |
| 2753 | // fail if it got skipped with a '#if 0' or something. Note that CurPtr might |
| 2754 | // be the end of conflict marker. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2755 | if (const char *End = FindConflictEnd(CurPtr, BufferEnd, |
| 2756 | CurrentConflictMarkerState)) { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2757 | CurPtr = End; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2758 | |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2759 | // Skip ahead to the end of line. |
| 2760 | while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') |
| 2761 | ++CurPtr; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2762 | |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2763 | BufferPtr = CurPtr; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2764 | |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2765 | // No longer in the conflict marker. |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 2766 | CurrentConflictMarkerState = CMK_None; |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2767 | return true; |
| 2768 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 2769 | |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 2770 | return false; |
| 2771 | } |
| 2772 | |
Alex Lorenz | 1be800c5 | 2017-04-19 08:58:56 +0000 | [diff] [blame] | 2773 | static const char *findPlaceholderEnd(const char *CurPtr, |
| 2774 | const char *BufferEnd) { |
| 2775 | if (CurPtr == BufferEnd) |
| 2776 | return nullptr; |
| 2777 | BufferEnd -= 1; // Scan until the second last character. |
| 2778 | for (; CurPtr != BufferEnd; ++CurPtr) { |
| 2779 | if (CurPtr[0] == '#' && CurPtr[1] == '>') |
| 2780 | return CurPtr + 2; |
| 2781 | } |
| 2782 | return nullptr; |
| 2783 | } |
| 2784 | |
| 2785 | bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) { |
| 2786 | assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!"); |
Alex Lorenz | fb7654a | 2017-06-16 20:13:39 +0000 | [diff] [blame] | 2787 | if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode) |
Alex Lorenz | 1be800c5 | 2017-04-19 08:58:56 +0000 | [diff] [blame] | 2788 | return false; |
| 2789 | const char *End = findPlaceholderEnd(CurPtr + 1, BufferEnd); |
| 2790 | if (!End) |
| 2791 | return false; |
| 2792 | const char *Start = CurPtr - 1; |
| 2793 | if (!LangOpts.AllowEditorPlaceholders) |
| 2794 | Diag(Start, diag::err_placeholder_in_source); |
| 2795 | Result.startToken(); |
| 2796 | FormTokenWithChars(Result, End, tok::raw_identifier); |
| 2797 | Result.setRawIdentifierData(Start); |
| 2798 | PP->LookUpIdentifierInfo(Result); |
| 2799 | Result.setFlag(Token::IsEditorPlaceholder); |
| 2800 | BufferPtr = End; |
| 2801 | return true; |
| 2802 | } |
| 2803 | |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2804 | bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { |
| 2805 | if (PP && PP->isCodeCompletionEnabled()) { |
Argyrios Kyrtzidis | e6e67de | 2011-09-19 20:40:19 +0000 | [diff] [blame] | 2806 | SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 2807 | return Loc == PP->getCodeCompletionLoc(); |
| 2808 | } |
| 2809 | |
| 2810 | return false; |
| 2811 | } |
| 2812 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2813 | uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, |
| 2814 | Token *Result) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2815 | unsigned CharSize; |
| 2816 | char Kind = getCharAndSize(StartPtr, CharSize); |
| 2817 | |
| 2818 | unsigned NumHexDigits; |
| 2819 | if (Kind == 'u') |
| 2820 | NumHexDigits = 4; |
| 2821 | else if (Kind == 'U') |
| 2822 | NumHexDigits = 8; |
| 2823 | else |
| 2824 | return 0; |
| 2825 | |
Jordan Rose | c0cba27 | 2013-01-27 20:12:04 +0000 | [diff] [blame] | 2826 | if (!LangOpts.CPlusPlus && !LangOpts.C99) { |
Jordan Rose | cccbdbf | 2013-01-28 17:49:02 +0000 | [diff] [blame] | 2827 | if (Result && !isLexingRawMode()) |
| 2828 | Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); |
Jordan Rose | c0cba27 | 2013-01-27 20:12:04 +0000 | [diff] [blame] | 2829 | return 0; |
| 2830 | } |
| 2831 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2832 | const char *CurPtr = StartPtr + CharSize; |
| 2833 | const char *KindLoc = &CurPtr[-1]; |
| 2834 | |
| 2835 | uint32_t CodePoint = 0; |
| 2836 | for (unsigned i = 0; i < NumHexDigits; ++i) { |
| 2837 | char C = getCharAndSize(CurPtr, CharSize); |
| 2838 | |
| 2839 | unsigned Value = llvm::hexDigitValue(C); |
| 2840 | if (Value == -1U) { |
| 2841 | if (Result && !isLexingRawMode()) { |
| 2842 | if (i == 0) { |
| 2843 | Diag(BufferPtr, diag::warn_ucn_escape_no_digits) |
| 2844 | << StringRef(KindLoc, 1); |
| 2845 | } else { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2846 | Diag(BufferPtr, diag::warn_ucn_escape_incomplete); |
Jordan Rose | 62db506 | 2013-01-24 20:50:52 +0000 | [diff] [blame] | 2847 | |
| 2848 | // If the user wrote \U1234, suggest a fixit to \u. |
| 2849 | if (i == 4 && NumHexDigits == 8) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2850 | CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); |
Jordan Rose | 62db506 | 2013-01-24 20:50:52 +0000 | [diff] [blame] | 2851 | Diag(KindLoc, diag::note_ucn_four_not_eight) |
| 2852 | << FixItHint::CreateReplacement(URange, "u"); |
| 2853 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2854 | } |
| 2855 | } |
Jordan Rose | c0cba27 | 2013-01-27 20:12:04 +0000 | [diff] [blame] | 2856 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2857 | return 0; |
| 2858 | } |
| 2859 | |
| 2860 | CodePoint <<= 4; |
| 2861 | CodePoint += Value; |
| 2862 | |
| 2863 | CurPtr += CharSize; |
| 2864 | } |
| 2865 | |
| 2866 | if (Result) { |
| 2867 | Result->setFlag(Token::HasUCN); |
NAKAMURA Takumi | e8f83db | 2013-01-25 14:57:21 +0000 | [diff] [blame] | 2868 | if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2869 | StartPtr = CurPtr; |
| 2870 | else |
| 2871 | while (StartPtr != CurPtr) |
| 2872 | (void)getAndAdvanceChar(StartPtr, *Result); |
| 2873 | } else { |
| 2874 | StartPtr = CurPtr; |
| 2875 | } |
| 2876 | |
Justin Bogner | 5353513 | 2013-10-21 05:02:28 +0000 | [diff] [blame] | 2877 | // Don't apply C family restrictions to UCNs in assembly mode |
| 2878 | if (LangOpts.AsmPreprocessor) |
| 2879 | return CodePoint; |
| 2880 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2881 | // C99 6.4.3p2: A universal character name shall not specify a character whose |
| 2882 | // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or |
| 2883 | // 0060 (`), nor one in the range D800 through DFFF inclusive.) |
| 2884 | // C++11 [lex.charset]p2: If the hexadecimal value for a |
| 2885 | // universal-character-name corresponds to a surrogate code point (in the |
| 2886 | // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, |
| 2887 | // if the hexadecimal value for a universal-character-name outside the |
| 2888 | // c-char-sequence, s-char-sequence, or r-char-sequence of a character or |
| 2889 | // string literal corresponds to a control character (in either of the |
| 2890 | // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the |
| 2891 | // basic source character set, the program is ill-formed. |
| 2892 | if (CodePoint < 0xA0) { |
| 2893 | if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) |
| 2894 | return CodePoint; |
| 2895 | |
| 2896 | // We don't use isLexingRawMode() here because we need to warn about bad |
| 2897 | // UCNs even when skipping preprocessing tokens in a #if block. |
| 2898 | if (Result && PP) { |
| 2899 | if (CodePoint < 0x20 || CodePoint >= 0x7F) |
| 2900 | Diag(BufferPtr, diag::err_ucn_control_character); |
| 2901 | else { |
| 2902 | char C = static_cast<char>(CodePoint); |
| 2903 | Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); |
| 2904 | } |
| 2905 | } |
| 2906 | |
| 2907 | return 0; |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2908 | } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2909 | // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2910 | // We don't use isLexingRawMode() here because we need to diagnose bad |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2911 | // UCNs even when skipping preprocessing tokens in a #if block. |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2912 | if (Result && PP) { |
| 2913 | if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) |
| 2914 | Diag(BufferPtr, diag::warn_ucn_escape_surrogate); |
| 2915 | else |
| 2916 | Diag(BufferPtr, diag::err_ucn_escape_invalid); |
| 2917 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2918 | return 0; |
| 2919 | } |
| 2920 | |
| 2921 | return CodePoint; |
| 2922 | } |
| 2923 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2924 | bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C, |
| 2925 | const char *CurPtr) { |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 2926 | static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars( |
| 2927 | UnicodeWhitespaceCharRanges); |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 2928 | if (!isLexingRawMode() && !PP->isPreprocessedOutput() && |
Alexander Kornienko | 37d6b18 | 2013-08-29 12:12:31 +0000 | [diff] [blame] | 2929 | UnicodeWhitespaceChars.contains(C)) { |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 2930 | Diag(BufferPtr, diag::ext_unicode_whitespace) |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2931 | << makeCharRange(*this, BufferPtr, CurPtr); |
Jordan Rose | 4246ae0 | 2013-01-24 20:50:50 +0000 | [diff] [blame] | 2932 | |
| 2933 | Result.setFlag(Token::LeadingSpace); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2934 | return true; |
Jordan Rose | 4246ae0 | 2013-01-24 20:50:50 +0000 | [diff] [blame] | 2935 | } |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2936 | return false; |
| 2937 | } |
Jordan Rose | 4246ae0 | 2013-01-24 20:50:50 +0000 | [diff] [blame] | 2938 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2939 | bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2940 | if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { |
| 2941 | if (!isLexingRawMode() && !ParsingPreprocessorDirective && |
| 2942 | !PP->isPreprocessedOutput()) { |
| 2943 | maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, |
| 2944 | makeCharRange(*this, BufferPtr, CurPtr), |
| 2945 | /*IsFirst=*/true); |
| 2946 | } |
| 2947 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2948 | MIOpt.ReadToken(); |
| 2949 | return LexIdentifier(Result, CurPtr); |
| 2950 | } |
| 2951 | |
Jordan Rose | cc53834 | 2013-01-31 19:48:48 +0000 | [diff] [blame] | 2952 | if (!isLexingRawMode() && !ParsingPreprocessorDirective && |
| 2953 | !PP->isPreprocessedOutput() && |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2954 | !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2955 | // Non-ASCII characters tend to creep into source code unintentionally. |
| 2956 | // Instead of letting the parser complain about the unknown token, |
| 2957 | // just drop the character. |
| 2958 | // Note that we can /only/ do this when the non-ASCII character is actually |
| 2959 | // spelled as Unicode, not written as a UCN. The standard requires that |
| 2960 | // we not throw away any possible preprocessor tokens, but there's a |
| 2961 | // loophole in the mapping of Unicode characters to basic character set |
| 2962 | // characters that allows us to map these particular characters to, say, |
| 2963 | // whitespace. |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 2964 | Diag(BufferPtr, diag::err_non_ascii) |
Jordan Rose | 58c61e0 | 2013-02-09 01:10:25 +0000 | [diff] [blame] | 2965 | << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2966 | |
| 2967 | BufferPtr = CurPtr; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2968 | return false; |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2969 | } |
| 2970 | |
| 2971 | // Otherwise, we have an explicit UCN or a character that's unlikely to show |
| 2972 | // up by accident. |
| 2973 | MIOpt.ReadToken(); |
| 2974 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2975 | return true; |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 2976 | } |
| 2977 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 2978 | void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { |
| 2979 | IsAtStartOfLine = Result.isAtStartOfLine(); |
| 2980 | HasLeadingSpace = Result.hasLeadingSpace(); |
| 2981 | HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro(); |
| 2982 | // Note that this doesn't affect IsAtPhysicalStartOfLine. |
| 2983 | } |
| 2984 | |
| 2985 | bool Lexer::Lex(Token &Result) { |
| 2986 | // Start a new token. |
| 2987 | Result.startToken(); |
| 2988 | |
| 2989 | // Set up misc whitespace flags for LexTokenInternal. |
| 2990 | if (IsAtStartOfLine) { |
| 2991 | Result.setFlag(Token::StartOfLine); |
| 2992 | IsAtStartOfLine = false; |
| 2993 | } |
| 2994 | |
| 2995 | if (HasLeadingSpace) { |
| 2996 | Result.setFlag(Token::LeadingSpace); |
| 2997 | HasLeadingSpace = false; |
| 2998 | } |
| 2999 | |
| 3000 | if (HasLeadingEmptyMacro) { |
| 3001 | Result.setFlag(Token::LeadingEmptyMacro); |
| 3002 | HasLeadingEmptyMacro = false; |
| 3003 | } |
| 3004 | |
| 3005 | bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; |
| 3006 | IsAtPhysicalStartOfLine = false; |
Eli Friedman | 29749d2 | 2013-09-19 01:51:23 +0000 | [diff] [blame] | 3007 | bool isRawLex = isLexingRawMode(); |
| 3008 | (void) isRawLex; |
| 3009 | bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine); |
| 3010 | // (After the LexTokenInternal call, the lexer might be destroyed.) |
| 3011 | assert((returnedToken || !isRawLex) && "Raw lex must succeed"); |
| 3012 | return returnedToken; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3013 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3014 | |
| 3015 | /// LexTokenInternal - This implements a simple C family lexer. It is an |
| 3016 | /// extremely performance critical piece of code. This assumes that the buffer |
Chris Lattner | 5c34938 | 2009-07-07 05:05:42 +0000 | [diff] [blame] | 3017 | /// has a null character at the end of the file. This returns a preprocessing |
| 3018 | /// token, not a normal token, as such, it is an internal interface. It assumes |
| 3019 | /// that the Flags of result have been cleared before calling this. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3020 | bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3021 | LexNextToken: |
| 3022 | // New token, can't need cleaning yet. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 3023 | Result.clearFlag(Token::NeedsCleaning); |
Craig Topper | d2d442c | 2014-05-17 23:10:59 +0000 | [diff] [blame] | 3024 | Result.setIdentifierInfo(nullptr); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3025 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3026 | // CurPtr - Cache BufferPtr in an automatic variable. |
| 3027 | const char *CurPtr = BufferPtr; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3028 | |
Chris Lattner | eb54b59 | 2006-07-10 06:34:27 +0000 | [diff] [blame] | 3029 | // Small amounts of horizontal whitespace is very common between tokens. |
| 3030 | if ((*CurPtr == ' ') || (*CurPtr == '\t')) { |
| 3031 | ++CurPtr; |
| 3032 | while ((*CurPtr == ' ') || (*CurPtr == '\t')) |
| 3033 | ++CurPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3034 | |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 3035 | // If we are keeping whitespace and other tokens, just return what we just |
| 3036 | // skipped. The next lexer invocation will return the token after the |
| 3037 | // whitespace. |
| 3038 | if (isKeepWhitespaceMode()) { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3039 | FormTokenWithChars(Result, CurPtr, tok::unknown); |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 3040 | // FIXME: The next token will not have LeadingSpace set. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3041 | return true; |
Chris Lattner | 4d96344 | 2008-10-12 04:05:48 +0000 | [diff] [blame] | 3042 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3043 | |
Chris Lattner | eb54b59 | 2006-07-10 06:34:27 +0000 | [diff] [blame] | 3044 | BufferPtr = CurPtr; |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 3045 | Result.setFlag(Token::LeadingSpace); |
Chris Lattner | eb54b59 | 2006-07-10 06:34:27 +0000 | [diff] [blame] | 3046 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3047 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3048 | unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3049 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3050 | // Read a character, advancing over it. |
| 3051 | char Char = getAndAdvanceChar(CurPtr, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3052 | tok::TokenKind Kind; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3053 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3054 | switch (Char) { |
| 3055 | case 0: // Null. |
| 3056 | // Found end of file? |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3057 | if (CurPtr-1 == BufferEnd) |
| 3058 | return LexEndOfFile(Result, CurPtr-1); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3059 | |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 3060 | // Check if we are performing code completion. |
| 3061 | if (isCodeCompletionPoint(CurPtr-1)) { |
| 3062 | // Return the code-completion token. |
| 3063 | Result.startToken(); |
| 3064 | FormTokenWithChars(Result, CurPtr, tok::code_completion); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3065 | return true; |
Argyrios Kyrtzidis | 5cec2ae | 2011-09-04 03:32:15 +0000 | [diff] [blame] | 3066 | } |
| 3067 | |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 3068 | if (!isLexingRawMode()) |
| 3069 | Diag(CurPtr-1, diag::null_in_file); |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 3070 | Result.setFlag(Token::LeadingSpace); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3071 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 3072 | return true; // KeepWhitespaceMode |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3073 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3074 | // We know the lexer hasn't changed, so just try again with this lexer. |
| 3075 | // (We manually eliminate the tail call to avoid recursion.) |
| 3076 | goto LexNextToken; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 3077 | |
Chris Lattner | 3dfff97 | 2009-12-17 05:29:40 +0000 | [diff] [blame] | 3078 | case 26: // DOS & CP/M EOF: "^Z". |
| 3079 | // If we're in Microsoft extensions mode, treat this as end of file. |
Nico Weber | de2310b | 2015-12-29 23:17:27 +0000 | [diff] [blame] | 3080 | if (LangOpts.MicrosoftExt) { |
| 3081 | if (!isLexingRawMode()) |
| 3082 | Diag(CurPtr-1, diag::ext_ctrl_z_eof_microsoft); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3083 | return LexEndOfFile(Result, CurPtr-1); |
Nico Weber | de2310b | 2015-12-29 23:17:27 +0000 | [diff] [blame] | 3084 | } |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3085 | |
Chris Lattner | 3dfff97 | 2009-12-17 05:29:40 +0000 | [diff] [blame] | 3086 | // If Microsoft extensions are disabled, this is just random garbage. |
| 3087 | Kind = tok::unknown; |
| 3088 | break; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 3089 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3090 | case '\r': |
Erich Keane | e916d54 | 2017-09-05 17:32:36 +0000 | [diff] [blame] | 3091 | if (CurPtr[0] == '\n') |
Erich Keane | 5a2b322 | 2017-08-24 18:36:07 +0000 | [diff] [blame] | 3092 | Char = getAndAdvanceChar(CurPtr, Result); |
Erich Keane | e916d54 | 2017-09-05 17:32:36 +0000 | [diff] [blame] | 3093 | LLVM_FALLTHROUGH; |
| 3094 | case '\n': |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3095 | // If we are inside a preprocessor directive and we see the end of line, |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 3096 | // we know we are done with the directive, so return an EOD token. |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3097 | if (ParsingPreprocessorDirective) { |
| 3098 | // Done parsing the "line". |
| 3099 | ParsingPreprocessorDirective = false; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3100 | |
Chris Lattner | 457fc15 | 2006-07-29 06:30:25 +0000 | [diff] [blame] | 3101 | // Restore comment saving mode, in case it was disabled for directive. |
David Blaikie | 2af2b30 | 2012-06-15 00:47:13 +0000 | [diff] [blame] | 3102 | if (PP) |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 3103 | resetExtendedTokenMode(); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3104 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3105 | // Since we consumed a newline, we are back at the start of a line. |
| 3106 | IsAtStartOfLine = true; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3107 | IsAtPhysicalStartOfLine = true; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3108 | |
Peter Collingbourne | 2f1e36b | 2011-02-28 02:37:51 +0000 | [diff] [blame] | 3109 | Kind = tok::eod; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3110 | break; |
| 3111 | } |
Jordan Rose | cb8a1ac | 2013-02-21 18:53:19 +0000 | [diff] [blame] | 3112 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3113 | // No leading whitespace seen so far. |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 3114 | Result.clearFlag(Token::LeadingSpace); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3115 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3116 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 3117 | return true; // KeepWhitespaceMode |
| 3118 | |
| 3119 | // We only saw whitespace, so just try again with this lexer. |
| 3120 | // (We manually eliminate the tail call to avoid recursion.) |
| 3121 | goto LexNextToken; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3122 | case ' ': |
| 3123 | case '\t': |
| 3124 | case '\f': |
| 3125 | case '\v': |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3126 | SkipHorizontalWhitespace: |
Chris Lattner | 146762e | 2007-07-20 16:59:19 +0000 | [diff] [blame] | 3127 | Result.setFlag(Token::LeadingSpace); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3128 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 3129 | return true; // KeepWhitespaceMode |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3130 | |
| 3131 | SkipIgnoredUnits: |
| 3132 | CurPtr = BufferPtr; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3133 | |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3134 | // If the next token is obviously a // or /* */ comment, skip it efficiently |
| 3135 | // too (without going through the big switch stmt). |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3136 | if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && |
Eli Friedman | cefc7ea | 2013-08-28 20:53:32 +0000 | [diff] [blame] | 3137 | LangOpts.LineComment && |
| 3138 | (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3139 | if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) |
| 3140 | return true; // There is a token to return. |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3141 | goto SkipIgnoredUnits; |
Chris Lattner | 8637abd | 2008-10-12 03:22:02 +0000 | [diff] [blame] | 3142 | } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3143 | if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) |
| 3144 | return true; // There is a token to return. |
Chris Lattner | b9b8597 | 2007-07-22 06:29:05 +0000 | [diff] [blame] | 3145 | goto SkipIgnoredUnits; |
| 3146 | } else if (isHorizontalWhitespace(*CurPtr)) { |
| 3147 | goto SkipHorizontalWhitespace; |
| 3148 | } |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3149 | // We only saw whitespace, so just try again with this lexer. |
| 3150 | // (We manually eliminate the tail call to avoid recursion.) |
| 3151 | goto LexNextToken; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 3152 | |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3153 | // C99 6.4.4.1: Integer Constants. |
| 3154 | // C99 6.4.4.2: Floating Constants. |
| 3155 | case '0': case '1': case '2': case '3': case '4': |
| 3156 | case '5': case '6': case '7': case '8': case '9': |
| 3157 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3158 | MIOpt.ReadToken(); |
| 3159 | return LexNumericConstant(Result, CurPtr); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3160 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3161 | case 'u': // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3162 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3163 | MIOpt.ReadToken(); |
| 3164 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3165 | if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3166 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3167 | |
| 3168 | // UTF-16 string literal |
| 3169 | if (Char == '"') |
| 3170 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3171 | tok::utf16_string_literal); |
| 3172 | |
| 3173 | // UTF-16 character constant |
| 3174 | if (Char == '\'') |
| 3175 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3176 | tok::utf16_char_constant); |
| 3177 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3178 | // UTF-16 raw string literal |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3179 | if (Char == 'R' && LangOpts.CPlusPlus11 && |
| 3180 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3181 | return LexRawStringLiteral(Result, |
| 3182 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3183 | SizeTmp2, Result), |
| 3184 | tok::utf16_string_literal); |
| 3185 | |
| 3186 | if (Char == '8') { |
| 3187 | char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); |
| 3188 | |
| 3189 | // UTF-8 string literal |
| 3190 | if (Char2 == '"') |
| 3191 | return LexStringLiteral(Result, |
| 3192 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3193 | SizeTmp2, Result), |
| 3194 | tok::utf8_string_literal); |
Aaron Ballman | c351fba | 2017-12-04 20:27:34 +0000 | [diff] [blame] | 3195 | if (Char2 == '\'' && LangOpts.CPlusPlus17) |
Richard Smith | 3e3a705 | 2014-11-08 06:08:42 +0000 | [diff] [blame] | 3196 | return LexCharConstant( |
| 3197 | Result, ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3198 | SizeTmp2, Result), |
| 3199 | tok::utf8_char_constant); |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3200 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3201 | if (Char2 == 'R' && LangOpts.CPlusPlus11) { |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3202 | unsigned SizeTmp3; |
| 3203 | char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); |
| 3204 | // UTF-8 raw string literal |
| 3205 | if (Char3 == '"') { |
| 3206 | return LexRawStringLiteral(Result, |
| 3207 | ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3208 | SizeTmp2, Result), |
| 3209 | SizeTmp3, Result), |
| 3210 | tok::utf8_string_literal); |
| 3211 | } |
| 3212 | } |
| 3213 | } |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3214 | } |
| 3215 | |
| 3216 | // treat u like the start of an identifier. |
| 3217 | return LexIdentifier(Result, CurPtr); |
| 3218 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3219 | case 'U': // Identifier (Uber) or C11/C++11 UTF-32 string literal |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3220 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3221 | MIOpt.ReadToken(); |
| 3222 | |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3223 | if (LangOpts.CPlusPlus11 || LangOpts.C11) { |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3224 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3225 | |
| 3226 | // UTF-32 string literal |
| 3227 | if (Char == '"') |
| 3228 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3229 | tok::utf32_string_literal); |
| 3230 | |
| 3231 | // UTF-32 character constant |
| 3232 | if (Char == '\'') |
| 3233 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3234 | tok::utf32_char_constant); |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3235 | |
| 3236 | // UTF-32 raw string literal |
Richard Smith | 9b36209 | 2013-03-09 23:56:02 +0000 | [diff] [blame] | 3237 | if (Char == 'R' && LangOpts.CPlusPlus11 && |
| 3238 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3239 | return LexRawStringLiteral(Result, |
| 3240 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3241 | SizeTmp2, Result), |
| 3242 | tok::utf32_string_literal); |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3243 | } |
| 3244 | |
| 3245 | // treat U like the start of an identifier. |
| 3246 | return LexIdentifier(Result, CurPtr); |
| 3247 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3248 | case 'R': // Identifier or C++0x raw string literal |
| 3249 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3250 | MIOpt.ReadToken(); |
| 3251 | |
Richard Smith | 2bf7fdb | 2013-01-02 11:42:31 +0000 | [diff] [blame] | 3252 | if (LangOpts.CPlusPlus11) { |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3253 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3254 | |
| 3255 | if (Char == '"') |
| 3256 | return LexRawStringLiteral(Result, |
| 3257 | ConsumeChar(CurPtr, SizeTmp, Result), |
| 3258 | tok::string_literal); |
| 3259 | } |
| 3260 | |
| 3261 | // treat R like the start of an identifier. |
| 3262 | return LexIdentifier(Result, CurPtr); |
| 3263 | |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3264 | case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3265 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3266 | MIOpt.ReadToken(); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3267 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3268 | |
| 3269 | // Wide string literal. |
| 3270 | if (Char == '"') |
Chris Lattner | d3e9895 | 2006-10-06 05:22:26 +0000 | [diff] [blame] | 3271 | return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3272 | tok::wide_string_literal); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3273 | |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3274 | // Wide raw string literal. |
Richard Smith | 2bf7fdb | 2013-01-02 11:42:31 +0000 | [diff] [blame] | 3275 | if (LangOpts.CPlusPlus11 && Char == 'R' && |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3276 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') |
| 3277 | return LexRawStringLiteral(Result, |
| 3278 | ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3279 | SizeTmp2, Result), |
| 3280 | tok::wide_string_literal); |
| 3281 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3282 | // Wide character constant. |
| 3283 | if (Char == '\'') |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3284 | return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3285 | tok::wide_char_constant); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3286 | // FALL THROUGH, treating L like the start of an identifier. |
Galina Kistanova | 39edaaa | 2017-06-03 06:25:47 +0000 | [diff] [blame] | 3287 | LLVM_FALLTHROUGH; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3288 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3289 | // C99 6.4.2: Identifiers. |
| 3290 | case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': |
| 3291 | case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': |
Craig Topper | 54edcca | 2011-08-11 04:06:15 +0000 | [diff] [blame] | 3292 | case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3293 | case 'V': case 'W': case 'X': case 'Y': case 'Z': |
| 3294 | case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': |
| 3295 | case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3296 | case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3297 | case 'v': case 'w': case 'x': case 'y': case 'z': |
| 3298 | case '_': |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3299 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3300 | MIOpt.ReadToken(); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3301 | return LexIdentifier(Result, CurPtr); |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3302 | |
| 3303 | case '$': // $ in identifiers. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3304 | if (LangOpts.DollarIdents) { |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 3305 | if (!isLexingRawMode()) |
| 3306 | Diag(CurPtr-1, diag::ext_dollar_in_identifier); |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3307 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3308 | MIOpt.ReadToken(); |
| 3309 | return LexIdentifier(Result, CurPtr); |
| 3310 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3311 | |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3312 | Kind = tok::unknown; |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3313 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3314 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3315 | // C99 6.4.4: Character Constants. |
| 3316 | case '\'': |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3317 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3318 | MIOpt.ReadToken(); |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3319 | return LexCharConstant(Result, CurPtr, tok::char_constant); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3320 | |
| 3321 | // C99 6.4.5: String Literals. |
| 3322 | case '"': |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3323 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3324 | MIOpt.ReadToken(); |
Douglas Gregor | fb65e59 | 2011-07-27 05:40:30 +0000 | [diff] [blame] | 3325 | return LexStringLiteral(Result, CurPtr, tok::string_literal); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3326 | |
| 3327 | // C99 6.4.6: Punctuators. |
| 3328 | case '?': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3329 | Kind = tok::question; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3330 | break; |
| 3331 | case '[': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3332 | Kind = tok::l_square; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3333 | break; |
| 3334 | case ']': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3335 | Kind = tok::r_square; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3336 | break; |
| 3337 | case '(': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3338 | Kind = tok::l_paren; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3339 | break; |
| 3340 | case ')': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3341 | Kind = tok::r_paren; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3342 | break; |
| 3343 | case '{': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3344 | Kind = tok::l_brace; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3345 | break; |
| 3346 | case '}': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3347 | Kind = tok::r_brace; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3348 | break; |
| 3349 | case '.': |
| 3350 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3351 | if (Char >= '0' && Char <= '9') { |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3352 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3353 | MIOpt.ReadToken(); |
| 3354 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3355 | return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3356 | } else if (LangOpts.CPlusPlus && Char == '*') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3357 | Kind = tok::periodstar; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3358 | CurPtr += SizeTmp; |
| 3359 | } else if (Char == '.' && |
| 3360 | getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3361 | Kind = tok::ellipsis; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3362 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3363 | SizeTmp2, Result); |
| 3364 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3365 | Kind = tok::period; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3366 | } |
| 3367 | break; |
| 3368 | case '&': |
| 3369 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3370 | if (Char == '&') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3371 | Kind = tok::ampamp; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3372 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3373 | } else if (Char == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3374 | Kind = tok::ampequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3375 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3376 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3377 | Kind = tok::amp; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3378 | } |
| 3379 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3380 | case '*': |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3381 | if (getCharAndSize(CurPtr, SizeTmp) == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3382 | Kind = tok::starequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3383 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3384 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3385 | Kind = tok::star; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3386 | } |
| 3387 | break; |
| 3388 | case '+': |
| 3389 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3390 | if (Char == '+') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3391 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3392 | Kind = tok::plusplus; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3393 | } else if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3394 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3395 | Kind = tok::plusequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3396 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3397 | Kind = tok::plus; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3398 | } |
| 3399 | break; |
| 3400 | case '-': |
| 3401 | Char = getCharAndSize(CurPtr, SizeTmp); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3402 | if (Char == '-') { // -- |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3403 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3404 | Kind = tok::minusminus; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3405 | } else if (Char == '>' && LangOpts.CPlusPlus && |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3406 | getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3407 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3408 | SizeTmp2, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3409 | Kind = tok::arrowstar; |
| 3410 | } else if (Char == '>') { // -> |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3411 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3412 | Kind = tok::arrow; |
| 3413 | } else if (Char == '=') { // -= |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3414 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3415 | Kind = tok::minusequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3416 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3417 | Kind = tok::minus; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3418 | } |
| 3419 | break; |
| 3420 | case '~': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3421 | Kind = tok::tilde; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3422 | break; |
| 3423 | case '!': |
| 3424 | if (getCharAndSize(CurPtr, SizeTmp) == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3425 | Kind = tok::exclaimequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3426 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3427 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3428 | Kind = tok::exclaim; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3429 | } |
| 3430 | break; |
| 3431 | case '/': |
| 3432 | // 6.4.9: Comments |
| 3433 | Char = getCharAndSize(CurPtr, SizeTmp); |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 3434 | if (Char == '/') { // Line comment. |
| 3435 | // Even if Line comments are disabled (e.g. in C89 mode), we generally |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3436 | // want to lex this as a comment. There is one problem with this though, |
| 3437 | // that in one particular corner case, this can change the behavior of the |
| 3438 | // resultant program. For example, In "foo //**/ bar", C89 would lex |
Nico Weber | 158a31a | 2012-11-11 07:02:14 +0000 | [diff] [blame] | 3439 | // this as "foo / bar" and langauges with Line comments would lex it as |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3440 | // "foo". Check to see if the character after the second slash is a '*'. |
| 3441 | // If so, we will lex that as a "/" instead of the start of a comment. |
Jordan Rose | 864b810 | 2013-03-05 22:51:04 +0000 | [diff] [blame] | 3442 | // However, we never do this if we are just preprocessing. |
Eli Friedman | cefc7ea | 2013-08-28 20:53:32 +0000 | [diff] [blame] | 3443 | bool TreatAsComment = LangOpts.LineComment && |
| 3444 | (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP); |
Jordan Rose | 864b810 | 2013-03-05 22:51:04 +0000 | [diff] [blame] | 3445 | if (!TreatAsComment) |
| 3446 | if (!(PP && PP->isPreprocessedOutput())) |
| 3447 | TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; |
| 3448 | |
| 3449 | if (TreatAsComment) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3450 | if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3451 | TokAtPhysicalStartOfLine)) |
| 3452 | return true; // There is a token to return. |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3453 | |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3454 | // It is common for the tokens immediately after a // comment to be |
| 3455 | // whitespace (indentation for the next line). Instead of going through |
| 3456 | // the big switch, handle it efficiently now. |
| 3457 | goto SkipIgnoredUnits; |
| 3458 | } |
| 3459 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3460 | |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3461 | if (Char == '*') { // /**/ comment. |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3462 | if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), |
| 3463 | TokAtPhysicalStartOfLine)) |
| 3464 | return true; // There is a token to return. |
| 3465 | |
| 3466 | // We only saw whitespace, so just try again with this lexer. |
| 3467 | // (We manually eliminate the tail call to avoid recursion.) |
| 3468 | goto LexNextToken; |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3469 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3470 | |
Chris Lattner | 5882771 | 2009-01-16 22:39:25 +0000 | [diff] [blame] | 3471 | if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3472 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3473 | Kind = tok::slashequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3474 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3475 | Kind = tok::slash; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3476 | } |
| 3477 | break; |
| 3478 | case '%': |
| 3479 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3480 | if (Char == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3481 | Kind = tok::percentequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3482 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3483 | } else if (LangOpts.Digraphs && Char == '>') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3484 | Kind = tok::r_brace; // '%>' -> '}' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3485 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3486 | } else if (LangOpts.Digraphs && Char == ':') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3487 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | 2b271db | 2006-07-15 05:41:09 +0000 | [diff] [blame] | 3488 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3489 | if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3490 | Kind = tok::hashhash; // '%:%:' -> '##' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3491 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3492 | SizeTmp2, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3493 | } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize |
Chris Lattner | 2b271db | 2006-07-15 05:41:09 +0000 | [diff] [blame] | 3494 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 3495 | if (!isLexingRawMode()) |
Ted Kremenek | a08713c | 2011-10-17 21:47:53 +0000 | [diff] [blame] | 3496 | Diag(BufferPtr, diag::ext_charize_microsoft); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3497 | Kind = tok::hashat; |
Chris Lattner | 2534324a | 2009-03-18 20:58:27 +0000 | [diff] [blame] | 3498 | } else { // '%:' -> '#' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3499 | // We parsed a # character. If this occurs at the start of the line, |
| 3500 | // it's actually the start of a preprocessing directive. Callback to |
| 3501 | // the preprocessor to handle it. |
Alp Toker | 7755aff | 2014-05-18 18:37:59 +0000 | [diff] [blame] | 3502 | // TODO: -fpreprocessed mode?? |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3503 | if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) |
Argyrios Kyrtzidis | 36675b7 | 2012-11-13 01:02:40 +0000 | [diff] [blame] | 3504 | goto HandleDirective; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3505 | |
Chris Lattner | 2534324a | 2009-03-18 20:58:27 +0000 | [diff] [blame] | 3506 | Kind = tok::hash; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3507 | } |
| 3508 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3509 | Kind = tok::percent; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3510 | } |
| 3511 | break; |
| 3512 | case '<': |
| 3513 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3514 | if (ParsingFilename) { |
Chris Lattner | b40289b | 2009-04-17 23:56:52 +0000 | [diff] [blame] | 3515 | return LexAngledStringLiteral(Result, CurPtr); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3516 | } else if (Char == '<') { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3517 | char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); |
| 3518 | if (After == '=') { |
| 3519 | Kind = tok::lesslessequal; |
| 3520 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3521 | SizeTmp2, Result); |
| 3522 | } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { |
| 3523 | // If this is actually a '<<<<<<<' version control conflict marker, |
| 3524 | // recognize it as such and recover nicely. |
| 3525 | goto LexNextToken; |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 3526 | } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { |
| 3527 | // If this is '<<<<' and we're in a Perforce-style conflict marker, |
| 3528 | // ignore it. |
| 3529 | goto LexNextToken; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3530 | } else if (LangOpts.CUDA && After == '<') { |
Peter Collingbourne | c1270f5 | 2011-02-09 21:08:21 +0000 | [diff] [blame] | 3531 | Kind = tok::lesslessless; |
| 3532 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3533 | SizeTmp2, Result); |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3534 | } else { |
| 3535 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3536 | Kind = tok::lessless; |
| 3537 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3538 | } else if (Char == '=') { |
Richard Smith | edbf597 | 2017-12-01 01:07:10 +0000 | [diff] [blame] | 3539 | char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); |
| 3540 | if (After == '>') { |
| 3541 | if (getLangOpts().CPlusPlus2a) { |
| 3542 | if (!isLexingRawMode()) |
| 3543 | Diag(BufferPtr, diag::warn_cxx17_compat_spaceship); |
| 3544 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3545 | SizeTmp2, Result); |
| 3546 | Kind = tok::spaceship; |
| 3547 | break; |
| 3548 | } |
| 3549 | // Suggest adding a space between the '<=' and the '>' to avoid a |
| 3550 | // change in semantics if this turns up in C++ <=17 mode. |
| 3551 | if (getLangOpts().CPlusPlus && !isLexingRawMode()) { |
| 3552 | Diag(BufferPtr, diag::warn_cxx2a_compat_spaceship) |
| 3553 | << FixItHint::CreateInsertion( |
| 3554 | getSourceLocation(CurPtr + SizeTmp, SizeTmp2), " "); |
| 3555 | } |
| 3556 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3557 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3558 | Kind = tok::lessequal; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3559 | } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' |
Richard Smith | 2bf7fdb | 2013-01-02 11:42:31 +0000 | [diff] [blame] | 3560 | if (LangOpts.CPlusPlus11 && |
Richard Smith | f7b6202 | 2011-04-14 18:36:27 +0000 | [diff] [blame] | 3561 | getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { |
| 3562 | // C++0x [lex.pptoken]p3: |
| 3563 | // Otherwise, if the next three characters are <:: and the subsequent |
| 3564 | // character is neither : nor >, the < is treated as a preprocessor |
| 3565 | // token by itself and not as the first character of the alternative |
| 3566 | // token <:. |
| 3567 | unsigned SizeTmp3; |
| 3568 | char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); |
| 3569 | if (After != ':' && After != '>') { |
| 3570 | Kind = tok::less; |
Richard Smith | acd4d3d | 2011-10-15 01:18:56 +0000 | [diff] [blame] | 3571 | if (!isLexingRawMode()) |
| 3572 | Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); |
Richard Smith | f7b6202 | 2011-04-14 18:36:27 +0000 | [diff] [blame] | 3573 | break; |
| 3574 | } |
| 3575 | } |
| 3576 | |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3577 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3578 | Kind = tok::l_square; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3579 | } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3580 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3581 | Kind = tok::l_brace; |
Alex Lorenz | c1e32fc | 2017-10-11 00:41:20 +0000 | [diff] [blame] | 3582 | } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 && |
| 3583 | lexEditorPlaceholder(Result, CurPtr)) { |
Alex Lorenz | 1be800c5 | 2017-04-19 08:58:56 +0000 | [diff] [blame] | 3584 | return true; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3585 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3586 | Kind = tok::less; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3587 | } |
| 3588 | break; |
| 3589 | case '>': |
| 3590 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3591 | if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3592 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3593 | Kind = tok::greaterequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3594 | } else if (Char == '>') { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3595 | char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); |
| 3596 | if (After == '=') { |
| 3597 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3598 | SizeTmp2, Result); |
| 3599 | Kind = tok::greatergreaterequal; |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 3600 | } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { |
| 3601 | // If this is actually a '>>>>' conflict marker, recognize it as such |
| 3602 | // and recover nicely. |
| 3603 | goto LexNextToken; |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3604 | } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { |
| 3605 | // If this is '>>>>>>>' and we're in a conflict marker, ignore it. |
| 3606 | goto LexNextToken; |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3607 | } else if (LangOpts.CUDA && After == '>') { |
Peter Collingbourne | c1270f5 | 2011-02-09 21:08:21 +0000 | [diff] [blame] | 3608 | Kind = tok::greatergreatergreater; |
| 3609 | CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), |
| 3610 | SizeTmp2, Result); |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3611 | } else { |
| 3612 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3613 | Kind = tok::greatergreater; |
| 3614 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3615 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3616 | Kind = tok::greater; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3617 | } |
| 3618 | break; |
| 3619 | case '^': |
| 3620 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3621 | if (Char == '=') { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3622 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3623 | Kind = tok::caretequal; |
Anastasia Stulova | 735c6cd | 2016-02-03 15:17:14 +0000 | [diff] [blame] | 3624 | } else if (LangOpts.OpenCL && Char == '^') { |
| 3625 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3626 | Kind = tok::caretcaret; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3627 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3628 | Kind = tok::caret; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3629 | } |
| 3630 | break; |
| 3631 | case '|': |
| 3632 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3633 | if (Char == '=') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3634 | Kind = tok::pipeequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3635 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3636 | } else if (Char == '|') { |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3637 | // If this is '|||||||' and we're in a conflict marker, ignore it. |
| 3638 | if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) |
| 3639 | goto LexNextToken; |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3640 | Kind = tok::pipepipe; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3641 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
| 3642 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3643 | Kind = tok::pipe; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3644 | } |
| 3645 | break; |
| 3646 | case ':': |
| 3647 | Char = getCharAndSize(CurPtr, SizeTmp); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3648 | if (LangOpts.Digraphs && Char == '>') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3649 | Kind = tok::r_square; // ':>' -> ']' |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3650 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Aaron Ballman | 606093a | 2017-10-15 15:01:42 +0000 | [diff] [blame] | 3651 | } else if ((LangOpts.CPlusPlus || |
| 3652 | LangOpts.DoubleSquareBracketAttributes) && |
| 3653 | Char == ':') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3654 | Kind = tok::coloncolon; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3655 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3656 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3657 | Kind = tok::colon; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3658 | } |
| 3659 | break; |
| 3660 | case ';': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3661 | Kind = tok::semi; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3662 | break; |
| 3663 | case '=': |
| 3664 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3665 | if (Char == '=') { |
Richard Smith | a9e33d4 | 2011-10-12 00:37:51 +0000 | [diff] [blame] | 3666 | // If this is '====' and we're in a conflict marker, ignore it. |
Chris Lattner | 7c027ee | 2009-12-14 06:16:57 +0000 | [diff] [blame] | 3667 | if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) |
| 3668 | goto LexNextToken; |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 3669 | |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3670 | Kind = tok::equalequal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3671 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3672 | } else { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3673 | Kind = tok::equal; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3674 | } |
| 3675 | break; |
| 3676 | case ',': |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3677 | Kind = tok::comma; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3678 | break; |
| 3679 | case '#': |
| 3680 | Char = getCharAndSize(CurPtr, SizeTmp); |
| 3681 | if (Char == '#') { |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3682 | Kind = tok::hashhash; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3683 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3684 | } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3685 | Kind = tok::hashat; |
Chris Lattner | 6d27a16 | 2008-11-22 02:02:22 +0000 | [diff] [blame] | 3686 | if (!isLexingRawMode()) |
Ted Kremenek | a08713c | 2011-10-17 21:47:53 +0000 | [diff] [blame] | 3687 | Diag(BufferPtr, diag::ext_charize_microsoft); |
Chris Lattner | 2b271db | 2006-07-15 05:41:09 +0000 | [diff] [blame] | 3688 | CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3689 | } else { |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3690 | // We parsed a # character. If this occurs at the start of the line, |
| 3691 | // it's actually the start of a preprocessing directive. Callback to |
| 3692 | // the preprocessor to handle it. |
Alp Toker | 7755aff | 2014-05-18 18:37:59 +0000 | [diff] [blame] | 3693 | // TODO: -fpreprocessed mode?? |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3694 | if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) |
Argyrios Kyrtzidis | 36675b7 | 2012-11-13 01:02:40 +0000 | [diff] [blame] | 3695 | goto HandleDirective; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3696 | |
Chris Lattner | 2534324a | 2009-03-18 20:58:27 +0000 | [diff] [blame] | 3697 | Kind = tok::hash; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3698 | } |
| 3699 | break; |
| 3700 | |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3701 | case '@': |
| 3702 | // Objective C support. |
David Blaikie | bbafb8a | 2012-03-11 07:00:24 +0000 | [diff] [blame] | 3703 | if (CurPtr[-1] == '@' && LangOpts.ObjC1) |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3704 | Kind = tok::at; |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3705 | else |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3706 | Kind = tok::unknown; |
Chris Lattner | 2b15cf7 | 2008-01-03 17:58:54 +0000 | [diff] [blame] | 3707 | break; |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3708 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3709 | // UCNs (C99 6.4.3, C++11 [lex.charset]p2) |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3710 | case '\\': |
Sanne Wouda | db1bdf4 | 2017-04-07 10:13:00 +0000 | [diff] [blame] | 3711 | if (!LangOpts.AsmPreprocessor) { |
| 3712 | if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) { |
| 3713 | if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { |
| 3714 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 3715 | return true; // KeepWhitespaceMode |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3716 | |
Sanne Wouda | db1bdf4 | 2017-04-07 10:13:00 +0000 | [diff] [blame] | 3717 | // We only saw whitespace, so just try again with this lexer. |
| 3718 | // (We manually eliminate the tail call to avoid recursion.) |
| 3719 | goto LexNextToken; |
| 3720 | } |
| 3721 | |
| 3722 | return LexUnicode(Result, CodePoint, CurPtr); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3723 | } |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3724 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3725 | |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3726 | Kind = tok::unknown; |
Chris Lattner | 041bef8 | 2006-07-11 05:52:53 +0000 | [diff] [blame] | 3727 | break; |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3728 | |
| 3729 | default: { |
| 3730 | if (isASCII(Char)) { |
| 3731 | Kind = tok::unknown; |
| 3732 | break; |
| 3733 | } |
| 3734 | |
Justin Lebar | 9091055 | 2016-09-30 00:38:45 +0000 | [diff] [blame] | 3735 | llvm::UTF32 CodePoint; |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3736 | |
| 3737 | // We can't just reset CurPtr to BufferPtr because BufferPtr may point to |
| 3738 | // an escaped newline. |
| 3739 | --CurPtr; |
Justin Lebar | 9091055 | 2016-09-30 00:38:45 +0000 | [diff] [blame] | 3740 | llvm::ConversionResult Status = |
| 3741 | llvm::convertUTF8Sequence((const llvm::UTF8 **)&CurPtr, |
| 3742 | (const llvm::UTF8 *)BufferEnd, |
Dmitri Gribenko | 9feeef4 | 2013-01-30 12:06:08 +0000 | [diff] [blame] | 3743 | &CodePoint, |
Justin Lebar | 9091055 | 2016-09-30 00:38:45 +0000 | [diff] [blame] | 3744 | llvm::strictConversion); |
| 3745 | if (Status == llvm::conversionOK) { |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3746 | if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { |
| 3747 | if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) |
| 3748 | return true; // KeepWhitespaceMode |
| 3749 | |
| 3750 | // We only saw whitespace, so just try again with this lexer. |
| 3751 | // (We manually eliminate the tail call to avoid recursion.) |
| 3752 | goto LexNextToken; |
| 3753 | } |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3754 | return LexUnicode(Result, CodePoint, CurPtr); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3755 | } |
Taewook Oh | cebac48 | 2017-12-06 17:00:53 +0000 | [diff] [blame] | 3756 | |
Jordan Rose | cc53834 | 2013-01-31 19:48:48 +0000 | [diff] [blame] | 3757 | if (isLexingRawMode() || ParsingPreprocessorDirective || |
| 3758 | PP->isPreprocessedOutput()) { |
Jordan Rose | f649795 | 2013-01-30 19:21:12 +0000 | [diff] [blame] | 3759 | ++CurPtr; |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 3760 | Kind = tok::unknown; |
| 3761 | break; |
| 3762 | } |
| 3763 | |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3764 | // Non-ASCII characters tend to creep into source code unintentionally. |
| 3765 | // Instead of letting the parser complain about the unknown token, |
Jordan Rose | 8b4af2a | 2013-01-25 00:20:28 +0000 | [diff] [blame] | 3766 | // just diagnose the invalid UTF-8, then drop the character. |
Jordan Rose | 1744158 | 2013-01-30 01:52:57 +0000 | [diff] [blame] | 3767 | Diag(CurPtr, diag::err_invalid_utf8); |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3768 | |
| 3769 | BufferPtr = CurPtr+1; |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3770 | // We're pretending the character didn't exist, so just try again with |
| 3771 | // this lexer. |
| 3772 | // (We manually eliminate the tail call to avoid recursion.) |
Jordan Rose | 7f43ddd | 2013-01-24 20:50:46 +0000 | [diff] [blame] | 3773 | goto LexNextToken; |
| 3774 | } |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3775 | } |
Mike Stump | 11289f4 | 2009-09-09 15:08:12 +0000 | [diff] [blame] | 3776 | |
Chris Lattner | 371ac8a | 2006-07-04 07:11:10 +0000 | [diff] [blame] | 3777 | // Notify MIOpt that we read a non-whitespace/non-comment token. |
| 3778 | MIOpt.ReadToken(); |
| 3779 | |
Chris Lattner | d01e291 | 2006-06-18 16:22:51 +0000 | [diff] [blame] | 3780 | // Update the location of token as well as BufferPtr. |
Chris Lattner | b11c323 | 2008-10-12 04:51:35 +0000 | [diff] [blame] | 3781 | FormTokenWithChars(Result, CurPtr, Kind); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3782 | return true; |
Argyrios Kyrtzidis | 36675b7 | 2012-11-13 01:02:40 +0000 | [diff] [blame] | 3783 | |
| 3784 | HandleDirective: |
| 3785 | // We parsed a # character and it's the start of a preprocessing directive. |
| 3786 | |
| 3787 | FormTokenWithChars(Result, CurPtr, tok::hash); |
| 3788 | PP->HandleDirective(Result); |
| 3789 | |
Argyrios Kyrtzidis | dc9fdaf | 2013-05-24 05:44:08 +0000 | [diff] [blame] | 3790 | if (PP->hadModuleLoaderFatalFailure()) { |
| 3791 | // With a fatal failure in the module loader, we abort parsing. |
| 3792 | assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof"); |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3793 | return true; |
Argyrios Kyrtzidis | dc9fdaf | 2013-05-24 05:44:08 +0000 | [diff] [blame] | 3794 | } |
| 3795 | |
Eli Friedman | 0834a4b | 2013-09-19 00:41:32 +0000 | [diff] [blame] | 3796 | // We parsed the directive; lex a token with the new state. |
| 3797 | return false; |
Chris Lattner | 22eb972 | 2006-06-18 05:43:12 +0000 | [diff] [blame] | 3798 | } |