[Lex] Allow to consume tokens while preprocessing
Summary:
By adding a hook to consume all tokens produced by the preprocessor.
The intention of this change is to make it possible to consume the
expanded tokens without re-runnig the preprocessor with minimal changes
to the preprocessor and minimal performance penalty when preprocessing
without recording the tokens.
The added hook is very low-level and reconstructing the expanded token
stream requires more work in the client code, the actual algorithm to
collect the tokens using this hook can be found in the follow-up change.
Reviewers: rsmith
Reviewed By: rsmith
Subscribers: eraman, nemanjai, kbarton, jsji, riccibruno, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D59885
llvm-svn: 361007
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
index 8aff163..2404a47 100644
--- a/clang/lib/Lex/Pragma.cpp
+++ b/clang/lib/Lex/Pragma.cpp
@@ -189,7 +189,8 @@
std::copy(Tokens.begin() + 1, Tokens.end(), Toks.get());
Toks[Tokens.size() - 1] = Tok;
Self.EnterTokenStream(std::move(Toks), Tokens.size(),
- /*DisableMacroExpansion*/ true);
+ /*DisableMacroExpansion*/ true,
+ /*IsReinject*/ true);
// ... and return the _Pragma token unchanged.
Tok = *Tokens.begin();
@@ -366,7 +367,8 @@
std::copy(PragmaToks.begin(), PragmaToks.end(), TokArray);
// Push the tokens onto the stack.
- EnterTokenStream(TokArray, PragmaToks.size(), true, true);
+ EnterTokenStream(TokArray, PragmaToks.size(), true, true,
+ /*IsReinject*/ false);
// With everything set up, lex this as a #pragma directive.
HandlePragmaDirective(PragmaLoc, PIK___pragma);
@@ -1028,7 +1030,7 @@
Crasher.startToken();
Crasher.setKind(tok::annot_pragma_parser_crash);
Crasher.setAnnotationRange(SourceRange(Tok.getLocation()));
- PP.EnterToken(Crasher);
+ PP.EnterToken(Crasher, /*IsReinject*/false);
} else if (II->isStr("dump")) {
Token Identifier;
PP.LexUnexpandedToken(Identifier);
@@ -1040,7 +1042,7 @@
SourceRange(Tok.getLocation(), Identifier.getLocation()));
DumpAnnot.setAnnotationValue(DumpII);
PP.DiscardUntilEndOfDirective();
- PP.EnterToken(DumpAnnot);
+ PP.EnterToken(DumpAnnot, /*IsReinject*/false);
} else {
PP.Diag(Identifier, diag::warn_pragma_debug_missing_argument)
<< II->getName();
@@ -1123,7 +1125,8 @@
Toks[0].setKind(tok::annot_pragma_captured);
Toks[0].setLocation(NameLoc);
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// Disable MSVC warning about runtime stack overflow.