Preprocessor: preserve whitespace in -traditional-cpp mode.
Note that unlike GNU cpp we currently do not preserve whitespace in macros
(even in -traditional-cpp mode).
<rdar://problem/12897179>
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@175778 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 0590d9e..65ea5e3 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -122,8 +122,15 @@
InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(),
InputFile->getBufferEnd());
- // Default to keeping comments if the preprocessor wants them.
- SetCommentRetentionState(PP.getCommentRetentionState());
+ resetExtendedTokenMode();
+}
+
+void Lexer::resetExtendedTokenMode() {
+ assert(PP && "Cannot reset token mode without a preprocessor");
+ if (LangOpts.TraditionalCPP)
+ SetKeepWhitespaceMode(true);
+ else
+ SetCommentRetentionState(PP->getCommentRetentionState());
}
/// Lexer constructor - Create a new raw lexer object. This object is only
@@ -1844,6 +1851,8 @@
///
bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
// Whitespace - Skip it, then return the token after the whitespace.
+ bool SawNewline = isVerticalWhitespace(CurPtr[-1]);
+
unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently.
while (1) {
// Skip horizontal whitespace very aggressively.
@@ -1851,7 +1860,7 @@
Char = *++CurPtr;
// Otherwise if we have something other than whitespace, we're done.
- if (Char != '\n' && Char != '\r')
+ if (!isVerticalWhitespace(Char))
break;
if (ParsingPreprocessorDirective) {
@@ -1861,24 +1870,27 @@
}
// ok, but handle newline.
- // The returned token is at the start of the line.
- Result.setFlag(Token::StartOfLine);
- // No leading whitespace seen so far.
- Result.clearFlag(Token::LeadingSpace);
+ SawNewline = true;
Char = *++CurPtr;
}
- // If this isn't immediately after a newline, there is leading space.
- char PrevChar = CurPtr[-1];
- if (PrevChar != '\n' && PrevChar != '\r')
- Result.setFlag(Token::LeadingSpace);
-
// If the client wants us to return whitespace, return it now.
if (isKeepWhitespaceMode()) {
FormTokenWithChars(Result, CurPtr, tok::unknown);
+ if (SawNewline)
+ IsAtStartOfLine = true;
+ // FIXME: The next token will not have LeadingSpace set.
return true;
}
+ // If this isn't immediately after a newline, there is leading space.
+ char PrevChar = CurPtr[-1];
+ bool HasLeadingSpace = !isVerticalWhitespace(PrevChar);
+
+ Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+ if (SawNewline)
+ Result.setFlag(Token::StartOfLine);
+
BufferPtr = CurPtr;
return false;
}
@@ -2269,7 +2281,6 @@
// efficiently now. This is safe even in KeepWhitespaceMode because we would
// have already returned above with the comment as a token.
if (isHorizontalWhitespace(*CurPtr)) {
- Result.setFlag(Token::LeadingSpace);
SkipWhitespace(Result, CurPtr+1);
return false;
}
@@ -2351,7 +2362,7 @@
FormTokenWithChars(Result, CurPtr, tok::eod);
// Restore comment saving mode, in case it was disabled for directive.
- SetCommentRetentionState(PP->getCommentRetentionState());
+ resetExtendedTokenMode();
return true; // Have a token.
}
@@ -2718,6 +2729,7 @@
// whitespace.
if (isKeepWhitespaceMode()) {
FormTokenWithChars(Result, CurPtr, tok::unknown);
+ // FIXME: The next token will not have LeadingSpace set.
return;
}
@@ -2785,7 +2797,7 @@
// Restore comment saving mode, in case it was disabled for directive.
if (PP)
- SetCommentRetentionState(PP->getCommentRetentionState());
+ resetExtendedTokenMode();
// Since we consumed a newline, we are back at the start of a line.
IsAtStartOfLine = true;
@@ -2793,8 +2805,7 @@
Kind = tok::eod;
break;
}
- // The returned token is at the start of the line.
- Result.setFlag(Token::StartOfLine);
+
// No leading whitespace seen so far.
Result.clearFlag(Token::LeadingSpace);