Remove use of lookahead from _Pragma handling and from all other
internal lexing steps in the preprocessor.
It is not safe to use the preprocessor's token lookahead except when
operating on the final sequence of tokens that would be produced by
phase 4 of translation. Doing so corrupts the token lookahead cache used
by the parser. (See added testcase for an example.) Lookahead should
instead be viewed as a layer on top of the normal lexer.
Added assertions to catch any further incorrect uses of lookahead within
lexing actions.
llvm-svn: 358230
diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index 929d35a..90238de 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -23,6 +23,7 @@
// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
// be combined with the EnableBacktrackAtThisPos calls in reverse order.
void Preprocessor::EnableBacktrackAtThisPos() {
+ assert(LexLevel == 0 && "cannot use lookahead while lexing");
BacktrackPositions.push_back(CachedLexPos);
EnterCachingLexMode();
}
@@ -34,29 +35,6 @@
BacktrackPositions.pop_back();
}
-Preprocessor::CachedTokensRange Preprocessor::LastCachedTokenRange() {
- assert(isBacktrackEnabled());
- auto PrevCachedLexPos = BacktrackPositions.back();
- return CachedTokensRange{PrevCachedLexPos, CachedLexPos};
-}
-
-void Preprocessor::EraseCachedTokens(CachedTokensRange TokenRange) {
- assert(TokenRange.Begin <= TokenRange.End);
- if (CachedLexPos == TokenRange.Begin && TokenRange.Begin != TokenRange.End) {
- // We have backtracked to the start of the token range as we want to consume
- // them again. Erase the tokens only after consuming then.
- assert(!CachedTokenRangeToErase);
- CachedTokenRangeToErase = TokenRange;
- return;
- }
- // The cached tokens were committed, so they should be erased now.
- assert(TokenRange.End == CachedLexPos);
- CachedTokens.erase(CachedTokens.begin() + TokenRange.Begin,
- CachedTokens.begin() + TokenRange.End);
- CachedLexPos = TokenRange.Begin;
- ExitCachingLexMode();
-}
-
// Make Preprocessor re-lex the tokens that were lexed since
// EnableBacktrackAtThisPos() was previously called.
void Preprocessor::Backtrack() {
@@ -71,15 +49,12 @@
if (!InCachingLexMode())
return;
+ // The assert in EnterCachingLexMode should prevent this from happening.
+ assert(LexLevel == 1 &&
+ "should not use token caching within the preprocessor");
+
if (CachedLexPos < CachedTokens.size()) {
Result = CachedTokens[CachedLexPos++];
- // Erase the some of the cached tokens after they are consumed when
- // asked to do so.
- if (CachedTokenRangeToErase &&
- CachedTokenRangeToErase->End == CachedLexPos) {
- EraseCachedTokens(*CachedTokenRangeToErase);
- CachedTokenRangeToErase = None;
- }
return;
}
@@ -88,14 +63,14 @@
if (isBacktrackEnabled()) {
// Cache the lexed token.
- EnterCachingLexMode();
+ EnterCachingLexModeUnchecked();
CachedTokens.push_back(Result);
++CachedLexPos;
return;
}
if (CachedLexPos < CachedTokens.size()) {
- EnterCachingLexMode();
+ EnterCachingLexModeUnchecked();
} else {
// All cached tokens were consumed.
CachedTokens.clear();
@@ -104,11 +79,23 @@
}
void Preprocessor::EnterCachingLexMode() {
+ // The caching layer sits on top of all the other lexers, so it's incorrect
+ // to cache tokens while inside a nested lex action. The cached tokens would
+ // be retained after returning to the enclosing lex action and, at best,
+ // would appear at the wrong position in the token stream.
+ assert(LexLevel == 0 &&
+ "entered caching lex mode while lexing something else");
+
if (InCachingLexMode()) {
assert(CurLexerKind == CLK_CachingLexer && "Unexpected lexer kind");
return;
}
+ EnterCachingLexModeUnchecked();
+}
+
+void Preprocessor::EnterCachingLexModeUnchecked() {
+ assert(CurLexerKind != CLK_CachingLexer && "already in caching lex mode");
PushIncludeMacroStack();
CurLexerKind = CLK_CachingLexer;
}