Allow the preprocessor to cache the lexed tokens, so that we can do efficient lookahead and backtracking.
1) New public methods added:
-EnableBacktrackAtThisPos
-DisableBacktrack
-Backtrack
-isBacktrackEnabled
2) LookAhead() implementation is replaced with a more efficient one.
3) LookNext() is removed.
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@54611 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/Lex/PPCaching.cpp b/lib/Lex/PPCaching.cpp
new file mode 100644
index 0000000..794e9c4
--- /dev/null
+++ b/lib/Lex/PPCaching.cpp
@@ -0,0 +1,63 @@
+//===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements pieces of the Preprocessor interface that manage the
+// caching of lexed tokens.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+using namespace clang;
+
+void Preprocessor::CachingLex(Token &Result) {
+ if (CachedLexPos < CachedTokens.size()) {
+ Result = CachedTokens[CachedLexPos++];
+ return;
+ }
+
+ ExitCachingLexMode();
+ Lex(Result);
+
+ if (!CacheTokens) {
+ // All cached tokens were consumed.
+ CachedTokens.clear();
+ CachedLexPos = 0;
+ return;
+ }
+
+ // We should cache the lexed token.
+
+ EnterCachingLexMode();
+ if (Result.isNot(tok::eof)) {
+ CachedTokens.push_back(Result);
+ ++CachedLexPos;
+ }
+}
+
+void Preprocessor::EnterCachingLexMode() {
+ if (InCachingLexMode())
+ return;
+
+ IncludeMacroStack.push_back(IncludeStackInfo(CurLexer, CurDirLookup,
+ CurTokenLexer));
+ CurLexer = 0;
+ CurTokenLexer = 0;
+}
+
+
+const Token &Preprocessor::PeekAhead(unsigned N) {
+ assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
+ ExitCachingLexMode();
+ for (unsigned C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
+ CachedTokens.push_back(Token());
+ Lex(CachedTokens.back());
+ }
+ EnterCachingLexMode();
+ return CachedTokens.back();
+}
diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index 1bedd5e..a14cbed 100644
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -60,94 +60,6 @@
return 0;
}
-/// LookAhead - This peeks ahead N tokens and returns that token without
-/// consuming any tokens. LookAhead(0) returns 'Tok', LookAhead(1) returns
-/// the token after Tok, etc.
-///
-/// NOTE: is a relatively expensive method, so it should not be used in common
-/// code paths if possible!
-///
-Token Preprocessor::LookAhead(unsigned N) {
- // FIXME: Optimize the case where multiple lookahead calls are used back to
- // back. Consider if the the parser contained (dynamically):
- // Lookahead(1); Lookahead(1); Lookahead(1)
- // This would return the same token 3 times, but would end up making lots of
- // token stream lexers to do it. To handle this common case, see if the top
- // of the lexer stack is a TokenStreamLexer with macro expansion disabled. If
- // so, see if it has 'N' tokens available in it. If so, just return the
- // token.
-
- // FIXME: Optimize the case when the parser does multiple nearby lookahead
- // calls. For example, consider:
- // Lookahead(0); Lookahead(1); Lookahead(2);
- // The previous optimization won't apply, and there won't be any space left in
- // the array that was previously new'd. To handle this, always round up the
- // size we new to a multiple of 16 tokens. If the previous buffer has space
- // left, we can just grow it. This means we only have to do the new 1/16th as
- // often.
-
- // Optimized LookAhead(0) case.
- if (N == 0)
- return LookNext();
-
- Token *LookaheadTokens = new Token[N+1];
-
- // Read N+1 tokens into LookaheadTokens. After this loop, Tok is the token
- // to return.
- Token Tok;
- unsigned NumTokens = 0;
- for (; N != ~0U; --N, ++NumTokens) {
- Lex(Tok);
- LookaheadTokens[NumTokens] = Tok;
-
- // If we got to EOF, don't lex past it. This will cause LookAhead to return
- // the EOF token.
- if (Tok.is(tok::eof))
- break;
- }
-
- // Okay, at this point, we have the token we want to return in Tok. However,
- // we read it and a bunch of other stuff (in LookaheadTokens) that we must
- // allow subsequent calls to 'Lex' to return. To do this, we push a new token
- // lexer onto the lexer stack with the tokens we read here. This passes
- // ownership of LookaheadTokens to EnterTokenStream.
- //
- // Note that we disable macro expansion of the tokens from this buffer, since
- // any macros have already been expanded, and the internal preprocessor state
- // may already read past new macros. Consider something like LookAhead(1) on
- // X
- // #define X 14
- // Y
- // The lookahead call should return 'Y', and the next Lex call should return
- // 'X' even though X -> 14 has already been entered as a macro.
- //
- EnterTokenStream(LookaheadTokens, NumTokens, true /*DisableExpansion*/,
- true /*OwnsTokens*/);
- return Tok;
-}
-
-/// PeekToken - Lexes one token into PeekedToken and pushes CurLexer,
-/// CurLexerToken into the IncludeMacroStack before setting them to null.
-void Preprocessor::PeekToken() {
- Lex(PeekedToken);
- // Cache the current Lexer, TokenLexer and set them both to null.
- // When Lex() is called, PeekedToken will be "consumed".
- IncludeMacroStack.push_back(IncludeStackInfo(CurLexer, CurDirLookup,
- CurTokenLexer));
- CurLexer = 0;
- CurTokenLexer = 0;
-}
-
-/// ConsumedPeekedToken - Called when Lex() is about to return the PeekedToken
-/// and have it "consumed".
-void Preprocessor::ConsumedPeekedToken() {
- assert(PeekedToken.getLocation().isValid() && "Confused Peeking?");
- // Restore CurLexer, TokenLexer.
- RemoveTopOfLexerStack();
- // Make PeekedToken invalid.
- PeekedToken.startToken();
-}
-
//===----------------------------------------------------------------------===//
// Methods for Entering and Callbacks for leaving various contexts
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index 631b836..33c94b6 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -68,6 +68,9 @@
InMacroArgs = false;
NumCachedTokenLexers = 0;
+ CacheTokens = false;
+ CachedLexPos = 0;
+
// "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
// This gets unpoisoned where it is allowed.
(Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
@@ -579,4 +582,3 @@
if (II.isExtensionToken() && Features.C99)
Diag(Identifier, diag::ext_token_used);
}
-