[Lex] Allow to consume tokens while preprocessing
Summary:
By adding a hook to consume all tokens produced by the preprocessor.
The intention of this change is to make it possible to consume the
expanded tokens without re-runnig the preprocessor with minimal changes
to the preprocessor and minimal performance penalty when preprocessing
without recording the tokens.
The added hook is very low-level and reconstructing the expanded token
stream requires more work in the client code, the actual algorithm to
collect the tokens using this hook can be found in the follow-up change.
Reviewers: rsmith
Reviewed By: rsmith
Subscribers: eraman, nemanjai, kbarton, jsji, riccibruno, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D59885
llvm-svn: 361007
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index e884884..6f44b84 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -1040,7 +1040,7 @@
if (PP.isBacktrackEnabled())
PP.RevertCachedTokens(1);
else
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::annot_decltype);
setExprAnnotation(Tok,
@@ -1736,7 +1736,7 @@
// A semicolon was missing after this declaration. Diagnose and recover.
ExpectAndConsume(tok::semi, diag::err_expected_after,
DeclSpec::getSpecifierName(TagType, PPol));
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::semi);
}
} else
@@ -2013,7 +2013,7 @@
// Push this token back into the preprocessor and change our current token
// to ';' so that the rest of the code recovers as though there were an
// ';' after the definition.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::semi);
}
}
@@ -3280,7 +3280,7 @@
if (SuggestFixIt) {
LBraceDiag << FixItHint::CreateInsertion(BraceLoc, " {");
// Try recovering from missing { after base-clause.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::l_brace);
} else {
if (TagDecl)
@@ -3376,12 +3376,12 @@
diag::note_missing_end_of_definition_before) << D;
// Push '};' onto the token stream to recover.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
Tok.startToken();
Tok.setLocation(PP.getLocForEndOfToken(PrevTokLocation));
Tok.setKind(tok::semi);
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
Tok.setKind(tok::r_brace);
}