[Lex] Allow to consume tokens while preprocessing
Summary:
By adding a hook to consume all tokens produced by the preprocessor.
The intention of this change is to make it possible to consume the
expanded tokens without re-runnig the preprocessor with minimal changes
to the preprocessor and minimal performance penalty when preprocessing
without recording the tokens.
The added hook is very low-level and reconstructing the expanded token
stream requires more work in the client code, the actual algorithm to
collect the tokens using this hook can be found in the follow-up change.
Reviewers: rsmith
Reviewed By: rsmith
Subscribers: eraman, nemanjai, kbarton, jsji, riccibruno, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D59885
llvm-svn: 361007
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index c6f457f..8abd696 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -69,9 +69,9 @@
DigraphToken.setLength(1);
// Push new tokens back to token stream.
- PP.EnterToken(ColonToken);
+ PP.EnterToken(ColonToken, /*IsReinject*/ true);
if (!AtDigraph)
- PP.EnterToken(DigraphToken);
+ PP.EnterToken(DigraphToken, /*IsReinject*/ true);
}
// Check for '<::' which should be '< ::' instead of '[:' when following
@@ -434,7 +434,7 @@
Token ColonColon;
PP.Lex(ColonColon);
ColonColon.setKind(tok::colon);
- PP.EnterToken(ColonColon);
+ PP.EnterToken(ColonColon, /*IsReinject*/ true);
break;
}
}
@@ -460,8 +460,8 @@
// mistyped '::' instead of ':'.
if (CorrectionFlagPtr && IsCorrectedToColon) {
ColonColon.setKind(tok::colon);
- PP.EnterToken(Tok);
- PP.EnterToken(ColonColon);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
+ PP.EnterToken(ColonColon, /*IsReinject*/ true);
Tok = Identifier;
break;
}
@@ -3300,7 +3300,8 @@
Toks.push_back(Tok);
// Re-enter the stored parenthesized tokens into the token stream, so we may
// parse them now.
- PP.EnterTokenStream(Toks, true /*DisableMacroExpansion*/);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion*/ true,
+ /*IsReinject*/ true);
// Drop the current token and bring the first cached one. It's the same token
// as when we entered this function.
ConsumeAnyToken();