blob: 01ed3ce80a66f0bbdcc4495a7905c74e04f12601 [file] [log] [blame]
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +00001#include "clang/AST/CommentLexer.h"
Dmitri Gribenkoca7f80a2012-08-09 00:03:17 +00002#include "clang/AST/CommentCommandTraits.h"
Fariborz Jahanian6738e432013-05-04 00:47:28 +00003#include "clang/AST/CommentDiagnostic.h"
Dmitri Gribenkobcef3412013-02-09 15:16:58 +00004#include "clang/Basic/CharInfo.h"
Dmitri Gribenkob2e54822013-01-19 22:06:05 +00005#include "llvm/ADT/StringExtras.h"
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +00006#include "llvm/ADT/StringSwitch.h"
Dmitri Gribenko9feeef42013-01-30 12:06:08 +00007#include "llvm/Support/ConvertUTF.h"
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +00008#include "llvm/Support/ErrorHandling.h"
9
10namespace clang {
11namespace comments {
12
13void Token::dump(const Lexer &L, const SourceManager &SM) const {
14 llvm::errs() << "comments::Token Kind=" << Kind << " ";
15 Loc.dump(SM);
16 llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
17}
18
Dmitri Gribenko74f4d022013-02-10 11:54:22 +000019static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +000020 return isLetter(C);
Dmitri Gribenko4586df72012-07-27 20:37:06 +000021}
22
Dmitri Gribenko74f4d022013-02-10 11:54:22 +000023static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +000024 return isDigit(C);
Dmitri Gribenko4586df72012-07-27 20:37:06 +000025}
26
Dmitri Gribenko74f4d022013-02-10 11:54:22 +000027static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +000028 return isHexDigit(C);
Dmitri Gribenko4586df72012-07-27 20:37:06 +000029}
Dmitri Gribenko107618a2012-08-22 22:56:08 +000030
Dmitri Gribenko74f4d022013-02-10 11:54:22 +000031static inline StringRef convertCodePointToUTF8(
32 llvm::BumpPtrAllocator &Allocator,
33 unsigned CodePoint) {
Fariborz Jahanian7b3ae192013-01-29 23:42:26 +000034 char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
35 char *ResolvedPtr = Resolved;
Dmitri Gribenko9feeef42013-01-30 12:06:08 +000036 if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
Fariborz Jahanian7b3ae192013-01-29 23:42:26 +000037 return StringRef(Resolved, ResolvedPtr - Resolved);
38 else
39 return StringRef();
40}
Dmitri Gribenko28800da2013-01-30 14:29:28 +000041
Dmitri Gribenko74f4d022013-02-10 11:54:22 +000042namespace {
43
Dmitri Gribenko28800da2013-01-30 14:29:28 +000044#include "clang/AST/CommentHTMLTags.inc"
45#include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
46
47} // unnamed namespace
Fariborz Jahanian7b3ae192013-01-29 23:42:26 +000048
Dmitri Gribenko4586df72012-07-27 20:37:06 +000049StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
Dmitri Gribenko28800da2013-01-30 14:29:28 +000050 // Fast path, first check a few most widely used named character references.
Dmitri Gribenko4586df72012-07-27 20:37:06 +000051 return llvm::StringSwitch<StringRef>(Name)
52 .Case("amp", "&")
53 .Case("lt", "<")
54 .Case("gt", ">")
55 .Case("quot", "\"")
56 .Case("apos", "\'")
Dmitri Gribenko28800da2013-01-30 14:29:28 +000057 // Slow path.
58 .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
Fariborz Jahanian7b3ae192013-01-29 23:42:26 +000059}
Dmitri Gribenko4586df72012-07-27 20:37:06 +000060
61StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
62 unsigned CodePoint = 0;
63 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
64 assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
65 CodePoint *= 10;
66 CodePoint += Name[i] - '0';
67 }
Dmitri Gribenko28800da2013-01-30 14:29:28 +000068 return convertCodePointToUTF8(Allocator, CodePoint);
69}
Dmitri Gribenko4586df72012-07-27 20:37:06 +000070
Dmitri Gribenko28800da2013-01-30 14:29:28 +000071StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
72 unsigned CodePoint = 0;
73 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
74 CodePoint *= 16;
75 const char C = Name[i];
76 assert(isHTMLHexCharacterReferenceCharacter(C));
77 CodePoint += llvm::hexDigitValue(C);
78 }
79 return convertCodePointToUTF8(Allocator, CodePoint);
Dmitri Gribenko4586df72012-07-27 20:37:06 +000080}
81
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +000082void Lexer::skipLineStartingDecorations() {
83 // This function should be called only for C comments
84 assert(CommentState == LCS_InsideCComment);
85
86 if (BufferPtr == CommentEnd)
87 return;
88
89 switch (*BufferPtr) {
90 case ' ':
91 case '\t':
92 case '\f':
93 case '\v': {
94 const char *NewBufferPtr = BufferPtr;
95 NewBufferPtr++;
96 if (NewBufferPtr == CommentEnd)
97 return;
98
99 char C = *NewBufferPtr;
Dmitri Gribenkobcef3412013-02-09 15:16:58 +0000100 while (isHorizontalWhitespace(C)) {
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000101 NewBufferPtr++;
102 if (NewBufferPtr == CommentEnd)
103 return;
104 C = *NewBufferPtr;
105 }
106 if (C == '*')
107 BufferPtr = NewBufferPtr + 1;
108 break;
109 }
110 case '*':
111 BufferPtr++;
112 break;
113 }
114}
115
116namespace {
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000117/// Returns pointer to the first newline character in the string.
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000118const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
119 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +0000120 if (isVerticalWhitespace(*BufferPtr))
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000121 return BufferPtr;
122 }
123 return BufferEnd;
124}
125
126const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
127 if (BufferPtr == BufferEnd)
128 return BufferPtr;
129
130 if (*BufferPtr == '\n')
131 BufferPtr++;
132 else {
133 assert(*BufferPtr == '\r');
134 BufferPtr++;
135 if (BufferPtr != BufferEnd && *BufferPtr == '\n')
136 BufferPtr++;
137 }
138 return BufferPtr;
139}
140
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000141const char *skipNamedCharacterReference(const char *BufferPtr,
142 const char *BufferEnd) {
143 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
144 if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
145 return BufferPtr;
146 }
147 return BufferEnd;
148}
149
150const char *skipDecimalCharacterReference(const char *BufferPtr,
151 const char *BufferEnd) {
152 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
153 if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
154 return BufferPtr;
155 }
156 return BufferEnd;
157}
158
159const char *skipHexCharacterReference(const char *BufferPtr,
Dmitri Gribenko11f54e82013-08-23 17:48:41 +0000160 const char *BufferEnd) {
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000161 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
162 if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
163 return BufferPtr;
164 }
165 return BufferEnd;
166}
167
Dmitri Gribenko17709ae2012-07-09 21:32:40 +0000168bool isHTMLIdentifierStartingCharacter(char C) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +0000169 return isLetter(C);
Dmitri Gribenko17709ae2012-07-09 21:32:40 +0000170}
171
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000172bool isHTMLIdentifierCharacter(char C) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +0000173 return isAlphanumeric(C);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000174}
175
176const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
177 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
178 if (!isHTMLIdentifierCharacter(*BufferPtr))
179 return BufferPtr;
180 }
181 return BufferEnd;
182}
183
184/// Skip HTML string quoted in single or double quotes. Escaping quotes inside
185/// string allowed.
186///
187/// Returns pointer to closing quote.
188const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
189{
190 const char Quote = *BufferPtr;
191 assert(Quote == '\"' || Quote == '\'');
192
193 BufferPtr++;
194 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
195 const char C = *BufferPtr;
196 if (C == Quote && BufferPtr[-1] != '\\')
197 return BufferPtr;
198 }
199 return BufferEnd;
200}
201
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000202const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
203 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
204 if (!isWhitespace(*BufferPtr))
205 return BufferPtr;
206 }
207 return BufferEnd;
208}
209
Dmitri Gribenkoe4a39972012-07-18 23:01:58 +0000210bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
211 return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
212}
213
Dmitri Gribenkoad45ad62012-09-14 16:35:35 +0000214bool isCommandNameStartCharacter(char C) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +0000215 return isLetter(C);
Dmitri Gribenkoad45ad62012-09-14 16:35:35 +0000216}
217
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000218bool isCommandNameCharacter(char C) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +0000219 return isAlphanumeric(C);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000220}
221
222const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
223 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
224 if (!isCommandNameCharacter(*BufferPtr))
225 return BufferPtr;
226 }
227 return BufferEnd;
228}
229
230/// Return the one past end pointer for BCPL comments.
231/// Handles newlines escaped with backslash or trigraph for backslahs.
232const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
233 const char *CurPtr = BufferPtr;
234 while (CurPtr != BufferEnd) {
Dmitri Gribenkobcef3412013-02-09 15:16:58 +0000235 while (!isVerticalWhitespace(*CurPtr)) {
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000236 CurPtr++;
237 if (CurPtr == BufferEnd)
238 return BufferEnd;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000239 }
240 // We found a newline, check if it is escaped.
241 const char *EscapePtr = CurPtr - 1;
242 while(isHorizontalWhitespace(*EscapePtr))
243 EscapePtr--;
244
245 if (*EscapePtr == '\\' ||
246 (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
247 EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
248 // We found an escaped newline.
249 CurPtr = skipNewline(CurPtr, BufferEnd);
250 } else
251 return CurPtr; // Not an escaped newline.
252 }
253 return BufferEnd;
254}
255
256/// Return the one past end pointer for C comments.
257/// Very dumb, does not handle escaped newlines or trigraphs.
258const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
259 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
260 if (*BufferPtr == '*') {
261 assert(BufferPtr + 1 != BufferEnd);
262 if (*(BufferPtr + 1) == '/')
263 return BufferPtr;
264 }
265 }
266 llvm_unreachable("buffer end hit before '*/' was seen");
267}
Fariborz Jahanian6c7a1662013-05-08 19:21:00 +0000268
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000269} // unnamed namespace
270
271void Lexer::lexCommentText(Token &T) {
272 assert(CommentState == LCS_InsideBCPLComment ||
273 CommentState == LCS_InsideCComment);
274
275 switch (State) {
276 case LS_Normal:
277 break;
278 case LS_VerbatimBlockFirstLine:
279 lexVerbatimBlockFirstLine(T);
280 return;
281 case LS_VerbatimBlockBody:
282 lexVerbatimBlockBody(T);
283 return;
Dmitri Gribenko1669f702012-06-27 16:53:58 +0000284 case LS_VerbatimLineText:
285 lexVerbatimLineText(T);
286 return;
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000287 case LS_HTMLStartTag:
288 lexHTMLStartTag(T);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000289 return;
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000290 case LS_HTMLEndTag:
291 lexHTMLEndTag(T);
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000292 return;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000293 }
294
295 assert(State == LS_Normal);
296
297 const char *TokenPtr = BufferPtr;
298 assert(TokenPtr < CommentEnd);
299 while (TokenPtr != CommentEnd) {
300 switch(*TokenPtr) {
301 case '\\':
302 case '@': {
Dmitri Gribenkobcf7f4d2013-03-04 23:06:15 +0000303 // Commands that start with a backslash and commands that start with
304 // 'at' have equivalent semantics. But we keep information about the
305 // exact syntax in AST for comments.
306 tok::TokenKind CommandKind =
307 (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000308 TokenPtr++;
309 if (TokenPtr == CommentEnd) {
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000310 formTextToken(T, TokenPtr);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000311 return;
312 }
313 char C = *TokenPtr;
314 switch (C) {
315 default:
316 break;
317
318 case '\\': case '@': case '&': case '$':
319 case '#': case '<': case '>': case '%':
320 case '\"': case '.': case ':':
321 // This is one of \\ \@ \& \$ etc escape sequences.
322 TokenPtr++;
323 if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
324 // This is the \:: escape sequence.
325 TokenPtr++;
326 }
Dmitri Gribenko60ddd8a2012-06-27 16:30:35 +0000327 StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000328 formTokenWithChars(T, TokenPtr, tok::text);
Dmitri Gribenko60ddd8a2012-06-27 16:30:35 +0000329 T.setText(UnescapedText);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000330 return;
331 }
332
333 // Don't make zero-length commands.
Dmitri Gribenkoad45ad62012-09-14 16:35:35 +0000334 if (!isCommandNameStartCharacter(*TokenPtr)) {
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000335 formTextToken(T, TokenPtr);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000336 return;
337 }
338
339 TokenPtr = skipCommandName(TokenPtr, CommentEnd);
340 unsigned Length = TokenPtr - (BufferPtr + 1);
341
342 // Hardcoded support for lexing LaTeX formula commands
343 // \f$ \f[ \f] \f{ \f} as a single command.
344 if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
345 C = *TokenPtr;
346 if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
347 TokenPtr++;
348 Length++;
349 }
350 }
351
352 const StringRef CommandName(BufferPtr + 1, Length);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000353
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000354 const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
355 if (!Info) {
Fariborz Jahanianebb262f2013-05-08 20:29:57 +0000356 if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
Fariborz Jahanian6c7a1662013-05-08 19:21:00 +0000357 StringRef CorrectedName = Info->Name;
Benjamin Kramer2dece572013-12-01 15:09:32 +0000358 SourceLocation Loc = getSourceLocation(BufferPtr);
359 SourceRange CommandRange(Loc.getLocWithOffset(1),
360 getSourceLocation(TokenPtr));
361 Diag(Loc, diag::warn_correct_comment_command_name)
Fariborz Jahanian6c7a1662013-05-08 19:21:00 +0000362 << CommandName << CorrectedName
363 << FixItHint::CreateReplacement(CommandRange, CorrectedName);
364 } else {
Benjamin Kramer2dece572013-12-01 15:09:32 +0000365 formTokenWithChars(T, TokenPtr, tok::unknown_command);
366 T.setUnknownCommandName(CommandName);
Fariborz Jahanian6c7a1662013-05-08 19:21:00 +0000367 Diag(T.getLocation(), diag::warn_unknown_comment_command_name);
368 return;
369 }
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000370 }
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000371 if (Info->IsVerbatimBlockCommand) {
372 setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
373 return;
374 }
375 if (Info->IsVerbatimLineCommand) {
376 setupAndLexVerbatimLine(T, TokenPtr, Info);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000377 return;
378 }
Dmitri Gribenkobcf7f4d2013-03-04 23:06:15 +0000379 formTokenWithChars(T, TokenPtr, CommandKind);
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000380 T.setCommandID(Info->getID());
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000381 return;
382 }
383
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000384 case '&':
385 lexHTMLCharacterReference(T);
386 return;
387
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000388 case '<': {
389 TokenPtr++;
390 if (TokenPtr == CommentEnd) {
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000391 formTextToken(T, TokenPtr);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000392 return;
393 }
394 const char C = *TokenPtr;
Dmitri Gribenko17709ae2012-07-09 21:32:40 +0000395 if (isHTMLIdentifierStartingCharacter(C))
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000396 setupAndLexHTMLStartTag(T);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000397 else if (C == '/')
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000398 setupAndLexHTMLEndTag(T);
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000399 else
400 formTextToken(T, TokenPtr);
401
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000402 return;
403 }
404
405 case '\n':
406 case '\r':
407 TokenPtr = skipNewline(TokenPtr, CommentEnd);
408 formTokenWithChars(T, TokenPtr, tok::newline);
409
410 if (CommentState == LCS_InsideCComment)
411 skipLineStartingDecorations();
412 return;
413
414 default: {
Dmitri Gribenko10af67a2012-12-30 19:45:46 +0000415 size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr).
416 find_first_of("\n\r\\@&<");
417 if (End != StringRef::npos)
418 TokenPtr += End;
419 else
420 TokenPtr = CommentEnd;
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000421 formTextToken(T, TokenPtr);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000422 return;
423 }
424 }
425 }
426}
427
428void Lexer::setupAndLexVerbatimBlock(Token &T,
429 const char *TextBegin,
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000430 char Marker, const CommandInfo *Info) {
431 assert(Info->IsVerbatimBlockCommand);
432
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000433 VerbatimBlockEndCommandName.clear();
434 VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000435 VerbatimBlockEndCommandName.append(Info->EndCommandName);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000436
437 formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000438 T.setVerbatimBlockID(Info->getID());
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000439
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000440 // If there is a newline following the verbatim opening command, skip the
441 // newline so that we don't create an tok::verbatim_block_line with empty
442 // text content.
Dmitri Gribenkobcef3412013-02-09 15:16:58 +0000443 if (BufferPtr != CommentEnd &&
444 isVerticalWhitespace(*BufferPtr)) {
445 BufferPtr = skipNewline(BufferPtr, CommentEnd);
446 State = LS_VerbatimBlockBody;
447 return;
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000448 }
449
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000450 State = LS_VerbatimBlockFirstLine;
451}
452
453void Lexer::lexVerbatimBlockFirstLine(Token &T) {
Dmitri Gribenkoe4a39972012-07-18 23:01:58 +0000454again:
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000455 assert(BufferPtr < CommentEnd);
456
457 // FIXME: It would be better to scan the text once, finding either the block
458 // end command or newline.
459 //
460 // Extract current line.
461 const char *Newline = findNewline(BufferPtr, CommentEnd);
462 StringRef Line(BufferPtr, Newline - BufferPtr);
463
464 // Look for end command in current line.
465 size_t Pos = Line.find(VerbatimBlockEndCommandName);
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000466 const char *TextEnd;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000467 const char *NextLine;
468 if (Pos == StringRef::npos) {
469 // Current line is completely verbatim.
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000470 TextEnd = Newline;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000471 NextLine = skipNewline(Newline, CommentEnd);
472 } else if (Pos == 0) {
473 // Current line contains just an end command.
474 const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
Dmitri Gribenko60ddd8a2012-06-27 16:30:35 +0000475 StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000476 formTokenWithChars(T, End, tok::verbatim_block_end);
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000477 T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000478 State = LS_Normal;
479 return;
480 } else {
481 // There is some text, followed by end command. Extract text first.
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000482 TextEnd = BufferPtr + Pos;
483 NextLine = TextEnd;
Dmitri Gribenkoe4a39972012-07-18 23:01:58 +0000484 // If there is only whitespace before end command, skip whitespace.
485 if (isWhitespace(BufferPtr, TextEnd)) {
486 BufferPtr = TextEnd;
487 goto again;
488 }
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000489 }
490
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000491 StringRef Text(BufferPtr, TextEnd - BufferPtr);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000492 formTokenWithChars(T, NextLine, tok::verbatim_block_line);
Dmitri Gribenko60ddd8a2012-06-27 16:30:35 +0000493 T.setVerbatimBlockText(Text);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000494
495 State = LS_VerbatimBlockBody;
496}
497
498void Lexer::lexVerbatimBlockBody(Token &T) {
499 assert(State == LS_VerbatimBlockBody);
500
501 if (CommentState == LCS_InsideCComment)
502 skipLineStartingDecorations();
503
504 lexVerbatimBlockFirstLine(T);
505}
506
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000507void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
508 const CommandInfo *Info) {
509 assert(Info->IsVerbatimLineCommand);
Dmitri Gribenko1669f702012-06-27 16:53:58 +0000510 formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
Dmitri Gribenko7acbf002012-09-10 20:32:42 +0000511 T.setVerbatimLineID(Info->getID());
Dmitri Gribenko1669f702012-06-27 16:53:58 +0000512
513 State = LS_VerbatimLineText;
514}
515
516void Lexer::lexVerbatimLineText(Token &T) {
517 assert(State == LS_VerbatimLineText);
518
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000519 // Extract current line.
520 const char *Newline = findNewline(BufferPtr, CommentEnd);
Dmitri Gribenko1669f702012-06-27 16:53:58 +0000521 const StringRef Text(BufferPtr, Newline - BufferPtr);
522 formTokenWithChars(T, Newline, tok::verbatim_line_text);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000523 T.setVerbatimLineText(Text);
Dmitri Gribenko1669f702012-06-27 16:53:58 +0000524
525 State = LS_Normal;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000526}
527
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000528void Lexer::lexHTMLCharacterReference(Token &T) {
529 const char *TokenPtr = BufferPtr;
530 assert(*TokenPtr == '&');
531 TokenPtr++;
532 if (TokenPtr == CommentEnd) {
533 formTextToken(T, TokenPtr);
534 return;
535 }
536 const char *NamePtr;
537 bool isNamed = false;
538 bool isDecimal = false;
539 char C = *TokenPtr;
540 if (isHTMLNamedCharacterReferenceCharacter(C)) {
541 NamePtr = TokenPtr;
542 TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
543 isNamed = true;
544 } else if (C == '#') {
545 TokenPtr++;
546 if (TokenPtr == CommentEnd) {
547 formTextToken(T, TokenPtr);
548 return;
549 }
550 C = *TokenPtr;
551 if (isHTMLDecimalCharacterReferenceCharacter(C)) {
552 NamePtr = TokenPtr;
553 TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
554 isDecimal = true;
555 } else if (C == 'x' || C == 'X') {
556 TokenPtr++;
557 NamePtr = TokenPtr;
558 TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
559 } else {
560 formTextToken(T, TokenPtr);
561 return;
562 }
563 } else {
564 formTextToken(T, TokenPtr);
565 return;
566 }
567 if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
568 *TokenPtr != ';') {
569 formTextToken(T, TokenPtr);
570 return;
571 }
572 StringRef Name(NamePtr, TokenPtr - NamePtr);
573 TokenPtr++; // Skip semicolon.
574 StringRef Resolved;
Dmitri Gribenko28800da2013-01-30 14:29:28 +0000575 if (isNamed)
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000576 Resolved = resolveHTMLNamedCharacterReference(Name);
577 else if (isDecimal)
578 Resolved = resolveHTMLDecimalCharacterReference(Name);
579 else
580 Resolved = resolveHTMLHexCharacterReference(Name);
581
582 if (Resolved.empty()) {
583 formTextToken(T, TokenPtr);
584 return;
585 }
586 formTokenWithChars(T, TokenPtr, tok::text);
587 T.setText(Resolved);
588 return;
589}
590
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000591void Lexer::setupAndLexHTMLStartTag(Token &T) {
Dmitri Gribenko17709ae2012-07-09 21:32:40 +0000592 assert(BufferPtr[0] == '<' &&
593 isHTMLIdentifierStartingCharacter(BufferPtr[1]));
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000594 const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
Dmitri Gribenko60ddd8a2012-06-27 16:30:35 +0000595 StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
Dmitri Gribenko107618a2012-08-22 22:56:08 +0000596 if (!isHTMLTagName(Name)) {
597 formTextToken(T, TagNameEnd);
598 return;
599 }
600
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000601 formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
602 T.setHTMLTagStartName(Name);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000603
604 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
605
Dmitri Gribenko17709ae2012-07-09 21:32:40 +0000606 const char C = *BufferPtr;
607 if (BufferPtr != CommentEnd &&
Dmitri Gribenkof26054f2012-07-11 21:38:39 +0000608 (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000609 State = LS_HTMLStartTag;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000610}
611
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000612void Lexer::lexHTMLStartTag(Token &T) {
613 assert(State == LS_HTMLStartTag);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000614
615 const char *TokenPtr = BufferPtr;
616 char C = *TokenPtr;
617 if (isHTMLIdentifierCharacter(C)) {
618 TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
Dmitri Gribenko60ddd8a2012-06-27 16:30:35 +0000619 StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000620 formTokenWithChars(T, TokenPtr, tok::html_ident);
Dmitri Gribenko60ddd8a2012-06-27 16:30:35 +0000621 T.setHTMLIdent(Ident);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000622 } else {
623 switch (C) {
624 case '=':
625 TokenPtr++;
626 formTokenWithChars(T, TokenPtr, tok::html_equals);
627 break;
628 case '\"':
629 case '\'': {
630 const char *OpenQuote = TokenPtr;
631 TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
632 const char *ClosingQuote = TokenPtr;
633 if (TokenPtr != CommentEnd) // Skip closing quote.
634 TokenPtr++;
635 formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
636 T.setHTMLQuotedString(StringRef(OpenQuote + 1,
637 ClosingQuote - (OpenQuote + 1)));
638 break;
639 }
640 case '>':
641 TokenPtr++;
642 formTokenWithChars(T, TokenPtr, tok::html_greater);
Dmitri Gribenko17709ae2012-07-09 21:32:40 +0000643 State = LS_Normal;
644 return;
Dmitri Gribenkof26054f2012-07-11 21:38:39 +0000645 case '/':
646 TokenPtr++;
647 if (TokenPtr != CommentEnd && *TokenPtr == '>') {
648 TokenPtr++;
649 formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
Dmitri Gribenko4586df72012-07-27 20:37:06 +0000650 } else
651 formTextToken(T, TokenPtr);
652
Dmitri Gribenkof26054f2012-07-11 21:38:39 +0000653 State = LS_Normal;
654 return;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000655 }
656 }
657
658 // Now look ahead and return to normal state if we don't see any HTML tokens
659 // ahead.
660 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
661 if (BufferPtr == CommentEnd) {
662 State = LS_Normal;
663 return;
664 }
665
666 C = *BufferPtr;
Dmitri Gribenko17709ae2012-07-09 21:32:40 +0000667 if (!isHTMLIdentifierStartingCharacter(C) &&
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000668 C != '=' && C != '\"' && C != '\'' && C != '>') {
669 State = LS_Normal;
670 return;
671 }
672}
673
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000674void Lexer::setupAndLexHTMLEndTag(Token &T) {
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000675 assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
676
677 const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
678 const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
Dmitri Gribenko107618a2012-08-22 22:56:08 +0000679 StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
680 if (!isHTMLTagName(Name)) {
681 formTextToken(T, TagNameEnd);
682 return;
683 }
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000684
685 const char *End = skipWhitespace(TagNameEnd, CommentEnd);
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000686
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000687 formTokenWithChars(T, End, tok::html_end_tag);
Dmitri Gribenko107618a2012-08-22 22:56:08 +0000688 T.setHTMLTagEndName(Name);
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000689
690 if (BufferPtr != CommentEnd && *BufferPtr == '>')
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000691 State = LS_HTMLEndTag;
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000692}
693
Dmitri Gribenkoe00ffc72012-07-13 00:44:24 +0000694void Lexer::lexHTMLEndTag(Token &T) {
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000695 assert(BufferPtr != CommentEnd && *BufferPtr == '>');
696
697 formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
698 State = LS_Normal;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000699}
700
Fariborz Jahanian5b637072013-05-03 23:15:20 +0000701Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
702 const CommandTraits &Traits,
Dmitri Gribenko6bab9112012-08-31 10:35:30 +0000703 SourceLocation FileLoc,
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000704 const char *BufferStart, const char *BufferEnd):
Fariborz Jahanian5b637072013-05-03 23:15:20 +0000705 Allocator(Allocator), Diags(Diags), Traits(Traits),
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000706 BufferStart(BufferStart), BufferEnd(BufferEnd),
Dmitri Gribenko6bab9112012-08-31 10:35:30 +0000707 FileLoc(FileLoc), BufferPtr(BufferStart),
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000708 CommentState(LCS_BeforeComment), State(LS_Normal) {
709}
710
711void Lexer::lex(Token &T) {
712again:
713 switch (CommentState) {
714 case LCS_BeforeComment:
715 if (BufferPtr == BufferEnd) {
716 formTokenWithChars(T, BufferPtr, tok::eof);
717 return;
718 }
719
720 assert(*BufferPtr == '/');
721 BufferPtr++; // Skip first slash.
722 switch(*BufferPtr) {
723 case '/': { // BCPL comment.
724 BufferPtr++; // Skip second slash.
725
726 if (BufferPtr != BufferEnd) {
727 // Skip Doxygen magic marker, if it is present.
728 // It might be missing because of a typo //< or /*<, or because we
729 // merged this non-Doxygen comment into a bunch of Doxygen comments
730 // around it: /** ... */ /* ... */ /** ... */
731 const char C = *BufferPtr;
732 if (C == '/' || C == '!')
733 BufferPtr++;
734 }
735
736 // Skip less-than symbol that marks trailing comments.
737 // Skip it even if the comment is not a Doxygen one, because //< and /*<
738 // are frequent typos.
739 if (BufferPtr != BufferEnd && *BufferPtr == '<')
740 BufferPtr++;
741
742 CommentState = LCS_InsideBCPLComment;
Dmitri Gribenkoec925312012-07-06 00:28:32 +0000743 if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
744 State = LS_Normal;
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000745 CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
746 goto again;
747 }
748 case '*': { // C comment.
749 BufferPtr++; // Skip star.
750
751 // Skip Doxygen magic marker.
752 const char C = *BufferPtr;
753 if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
754 BufferPtr++;
755
756 // Skip less-than symbol that marks trailing comments.
757 if (BufferPtr != BufferEnd && *BufferPtr == '<')
758 BufferPtr++;
759
760 CommentState = LCS_InsideCComment;
761 State = LS_Normal;
762 CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
763 goto again;
764 }
765 default:
766 llvm_unreachable("second character of comment should be '/' or '*'");
767 }
768
769 case LCS_BetweenComments: {
770 // Consecutive comments are extracted only if there is only whitespace
771 // between them. So we can search for the start of the next comment.
772 const char *EndWhitespace = BufferPtr;
773 while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
774 EndWhitespace++;
775
776 // Turn any whitespace between comments (and there is only whitespace
Dmitri Gribenko17709ae2012-07-09 21:32:40 +0000777 // between them -- guaranteed by comment extraction) into a newline. We
778 // have two newlines between C comments in total (first one was synthesized
779 // after a comment).
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000780 formTokenWithChars(T, EndWhitespace, tok::newline);
781
782 CommentState = LCS_BeforeComment;
783 break;
784 }
785
786 case LCS_InsideBCPLComment:
787 case LCS_InsideCComment:
788 if (BufferPtr != CommentEnd) {
789 lexCommentText(T);
790 break;
791 } else {
792 // Skip C comment closing sequence.
793 if (CommentState == LCS_InsideCComment) {
794 assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
795 BufferPtr += 2;
796 assert(BufferPtr <= BufferEnd);
797
798 // Synthenize newline just after the C comment, regardless if there is
799 // actually a newline.
800 formTokenWithChars(T, BufferPtr, tok::newline);
801
802 CommentState = LCS_BetweenComments;
803 break;
804 } else {
805 // Don't synthesized a newline after BCPL comment.
806 CommentState = LCS_BetweenComments;
807 goto again;
808 }
809 }
810 }
811}
812
813StringRef Lexer::getSpelling(const Token &Tok,
814 const SourceManager &SourceMgr,
815 bool *Invalid) const {
816 SourceLocation Loc = Tok.getLocation();
817 std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
818
819 bool InvalidTemp = false;
820 StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
821 if (InvalidTemp) {
822 *Invalid = true;
823 return StringRef();
824 }
825
826 const char *Begin = File.data() + LocInfo.second;
827 return StringRef(Begin, Tok.getLength());
828}
829
Dmitri Gribenko5188c4b2012-06-26 20:39:18 +0000830} // end namespace comments
831} // end namespace clang
832