Fix a couple of bugs, add some new cool stuff.
1. Fix a todo in Parser::ParseTag, to recover better. On code like
that in test/Sema/decl-invalid.c it causes us to return a single
error instead of multiple.
2. Fix an error in Sema::ParseDeclarator, where it would crash if the
declarator didn't have an identifier. Instead, diagnose the problem.
3. Start adding infrastructure to track the range of locations covered
by a declspec or declarator. This is mostly implemented for declspec,
but could be improved, it is missing for declarator.
Thanks to Neil for pointing out this crash.
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@40482 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/Driver/PrintPreprocessedOutput.cpp b/Driver/PrintPreprocessedOutput.cpp
new file mode 100644
index 0000000..3eb11b7
--- /dev/null
+++ b/Driver/PrintPreprocessedOutput.cpp
@@ -0,0 +1,572 @@
+//===--- PrintPreprocessedOutput.cpp - Implement the -E mode --------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This code simply runs the preprocessor on the input file and prints out the
+// result. This is the traditional behavior of the -E option.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/Pragma.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Config/config.h"
+#include <cstdio>
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// Simple buffered I/O
+//===----------------------------------------------------------------------===//
+//
+// Empirically, iostream is over 30% slower than stdio for this workload, and
+// stdio itself isn't very well suited. The problem with stdio is use of
+// putchar_unlocked. We have many newline characters that need to be emitted,
+// but stdio needs to do extra checks to handle line buffering mode. These
+// extra checks make putchar_unlocked fall off its inlined code path, hitting
+// slow system code. In practice, using 'write' directly makes 'clang -E -P'
+// about 10% faster than using the stdio path on darwin.
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#else
+#define USE_STDIO 1
+#endif
+
+static char *OutBufStart = 0, *OutBufEnd, *OutBufCur;
+
+/// InitOutputBuffer - Initialize our output buffer.
+///
+static void InitOutputBuffer() {
+#ifndef USE_STDIO
+ OutBufStart = new char[64*1024];
+ OutBufEnd = OutBufStart+64*1024;
+ OutBufCur = OutBufStart;
+#endif
+}
+
+/// FlushBuffer - Write the accumulated bytes to the output stream.
+///
+static void FlushBuffer() {
+#ifndef USE_STDIO
+ write(STDOUT_FILENO, OutBufStart, OutBufCur-OutBufStart);
+ OutBufCur = OutBufStart;
+#endif
+}
+
+/// CleanupOutputBuffer - Finish up output.
+///
+static void CleanupOutputBuffer() {
+#ifndef USE_STDIO
+ FlushBuffer();
+ delete [] OutBufStart;
+#endif
+}
+
+static void OutputChar(char c) {
+#ifdef USE_STDIO
+ putchar_unlocked(c);
+#else
+ if (OutBufCur >= OutBufEnd)
+ FlushBuffer();
+ *OutBufCur++ = c;
+#endif
+}
+
+static void OutputString(const char *Ptr, unsigned Size) {
+#ifdef USE_STDIO
+ fwrite(Ptr, Size, 1, stdout);
+#else
+ if (OutBufCur+Size >= OutBufEnd)
+ FlushBuffer();
+
+ switch (Size) {
+ default:
+ memcpy(OutBufCur, Ptr, Size);
+ break;
+ case 3:
+ OutBufCur[2] = Ptr[2];
+ case 2:
+ OutBufCur[1] = Ptr[1];
+ case 1:
+ OutBufCur[0] = Ptr[0];
+ case 0:
+ break;
+ }
+ OutBufCur += Size;
+#endif
+}
+
+
+//===----------------------------------------------------------------------===//
+// Preprocessed token printer
+//===----------------------------------------------------------------------===//
+
+static llvm::cl::opt<bool>
+DisableLineMarkers("P", llvm::cl::desc("Disable linemarker output in -E mode"));
+static llvm::cl::opt<bool>
+EnableCommentOutput("C", llvm::cl::desc("Enable comment output in -E mode"));
+static llvm::cl::opt<bool>
+EnableMacroCommentOutput("CC",
+ llvm::cl::desc("Enable comment output in -E mode, "
+ "even from macro expansions"));
+
+namespace {
+class PrintPPOutputPPCallbacks : public PPCallbacks {
+ Preprocessor &PP;
+ unsigned CurLine;
+ bool EmittedTokensOnThisLine;
+ DirectoryLookup::DirType FileType;
+ llvm::SmallString<512> CurFilename;
+public:
+ PrintPPOutputPPCallbacks(Preprocessor &pp) : PP(pp) {
+ CurLine = 0;
+ CurFilename += "<uninit>";
+ EmittedTokensOnThisLine = false;
+ FileType = DirectoryLookup::NormalHeaderDir;
+ }
+
+ void SetEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; }
+ bool hasEmittedTokensOnThisLine() const { return EmittedTokensOnThisLine; }
+
+ virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
+ DirectoryLookup::DirType FileType);
+ virtual void Ident(SourceLocation Loc, const std::string &str);
+
+
+ void HandleFirstTokOnLine(Token &Tok);
+ void MoveToLine(SourceLocation Loc);
+ bool AvoidConcat(const Token &PrevTok, const Token &Tok);
+};
+}
+
+/// UToStr - Do itoa on the specified number, in-place in the specified buffer.
+/// endptr points to the end of the buffer.
+static char *UToStr(unsigned N, char *EndPtr) {
+ // Null terminate the buffer.
+ *--EndPtr = '\0';
+ if (N == 0) // Zero is a special case.
+ *--EndPtr = '0';
+ while (N) {
+ *--EndPtr = '0' + char(N % 10);
+ N /= 10;
+ }
+ return EndPtr;
+}
+
+
+/// MoveToLine - Move the output to the source line specified by the location
+/// object. We can do this by emitting some number of \n's, or be emitting a
+/// #line directive.
+void PrintPPOutputPPCallbacks::MoveToLine(SourceLocation Loc) {
+ if (DisableLineMarkers) {
+ if (EmittedTokensOnThisLine) {
+ OutputChar('\n');
+ EmittedTokensOnThisLine = false;
+ }
+ return;
+ }
+
+ unsigned LineNo = PP.getSourceManager().getLogicalLineNumber(Loc);
+
+ // If this line is "close enough" to the original line, just print newlines,
+ // otherwise print a #line directive.
+ if (LineNo-CurLine < 8) {
+ if (LineNo-CurLine == 1)
+ OutputChar('\n');
+ else {
+ const char *NewLines = "\n\n\n\n\n\n\n\n";
+ OutputString(NewLines, LineNo-CurLine);
+ CurLine = LineNo;
+ }
+ } else {
+ if (EmittedTokensOnThisLine) {
+ OutputChar('\n');
+ EmittedTokensOnThisLine = false;
+ }
+
+ CurLine = LineNo;
+
+ OutputChar('#');
+ OutputChar(' ');
+ char NumberBuffer[20];
+ const char *NumStr = UToStr(LineNo, NumberBuffer+20);
+ OutputString(NumStr, (NumberBuffer+20)-NumStr-1);
+ OutputChar(' ');
+ OutputChar('"');
+ OutputString(&CurFilename[0], CurFilename.size());
+ OutputChar('"');
+
+ if (FileType == DirectoryLookup::SystemHeaderDir)
+ OutputString(" 3", 2);
+ else if (FileType == DirectoryLookup::ExternCSystemHeaderDir)
+ OutputString(" 3 4", 4);
+ OutputChar('\n');
+ }
+}
+
+
+/// FileChanged - Whenever the preprocessor enters or exits a #include file
+/// it invokes this handler. Update our conception of the current source
+/// position.
+void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
+ FileChangeReason Reason,
+ DirectoryLookup::DirType FileType) {
+ if (DisableLineMarkers) return;
+
+ // Unless we are exiting a #include, make sure to skip ahead to the line the
+ // #include directive was at.
+ SourceManager &SourceMgr = PP.getSourceManager();
+ if (Reason == PPCallbacks::EnterFile) {
+ MoveToLine(SourceMgr.getIncludeLoc(Loc));
+ } else if (Reason == PPCallbacks::SystemHeaderPragma) {
+ MoveToLine(Loc);
+
+ // TODO GCC emits the # directive for this directive on the line AFTER the
+ // directive and emits a bunch of spaces that aren't needed. Emulate this
+ // strange behavior.
+ }
+
+ Loc = SourceMgr.getLogicalLoc(Loc);
+ CurLine = SourceMgr.getLineNumber(Loc);
+ CurFilename.clear();
+ CurFilename += SourceMgr.getSourceName(Loc);
+ Lexer::Stringify(CurFilename);
+ FileType = FileType;
+
+ if (EmittedTokensOnThisLine) {
+ OutputChar('\n');
+ EmittedTokensOnThisLine = false;
+ }
+
+ OutputChar('#');
+ OutputChar(' ');
+
+ char NumberBuffer[20];
+ const char *NumStr = UToStr(CurLine, NumberBuffer+20);
+ OutputString(NumStr, (NumberBuffer+20)-NumStr-1);
+ OutputChar(' ');
+ OutputChar('"');
+ OutputString(&CurFilename[0], CurFilename.size());
+ OutputChar('"');
+
+ switch (Reason) {
+ case PPCallbacks::EnterFile:
+ OutputString(" 1", 2);
+ break;
+ case PPCallbacks::ExitFile:
+ OutputString(" 2", 2);
+ break;
+ case PPCallbacks::SystemHeaderPragma: break;
+ case PPCallbacks::RenameFile: break;
+ }
+
+ if (FileType == DirectoryLookup::SystemHeaderDir)
+ OutputString(" 3", 2);
+ else if (FileType == DirectoryLookup::ExternCSystemHeaderDir)
+ OutputString(" 3 4", 4);
+
+ OutputChar('\n');
+}
+
+/// HandleIdent - Handle #ident directives when read by the preprocessor.
+///
+void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, const std::string &S) {
+ MoveToLine(Loc);
+
+ OutputString("#ident ", strlen("#ident "));
+ OutputString(&S[0], S.size());
+ EmittedTokensOnThisLine = true;
+}
+
+/// HandleFirstTokOnLine - When emitting a preprocessed file in -E mode, this
+/// is called for the first token on each new line.
+void PrintPPOutputPPCallbacks::HandleFirstTokOnLine(Token &Tok) {
+ // Figure out what line we went to and insert the appropriate number of
+ // newline characters.
+ MoveToLine(Tok.getLocation());
+
+ // Print out space characters so that the first token on a line is
+ // indented for easy reading.
+ const SourceManager &SourceMgr = PP.getSourceManager();
+ unsigned ColNo = SourceMgr.getLogicalColumnNumber(Tok.getLocation());
+
+ // This hack prevents stuff like:
+ // #define HASH #
+ // HASH define foo bar
+ // From having the # character end up at column 1, which makes it so it
+ // is not handled as a #define next time through the preprocessor if in
+ // -fpreprocessed mode.
+ if (ColNo <= 1 && Tok.getKind() == tok::hash)
+ OutputChar(' ');
+
+ // Otherwise, indent the appropriate number of spaces.
+ for (; ColNo > 1; --ColNo)
+ OutputChar(' ');
+}
+
+namespace {
+struct UnknownPragmaHandler : public PragmaHandler {
+ const char *Prefix;
+ PrintPPOutputPPCallbacks *Callbacks;
+
+ UnknownPragmaHandler(const char *prefix, PrintPPOutputPPCallbacks *callbacks)
+ : PragmaHandler(0), Prefix(prefix), Callbacks(callbacks) {}
+ virtual void HandlePragma(Preprocessor &PP, Token &PragmaTok) {
+ // Figure out what line we went to and insert the appropriate number of
+ // newline characters.
+ Callbacks->MoveToLine(PragmaTok.getLocation());
+ OutputString(Prefix, strlen(Prefix));
+
+ // Read and print all of the pragma tokens.
+ while (PragmaTok.getKind() != tok::eom) {
+ if (PragmaTok.hasLeadingSpace())
+ OutputChar(' ');
+ std::string TokSpell = PP.getSpelling(PragmaTok);
+ OutputString(&TokSpell[0], TokSpell.size());
+ PP.LexUnexpandedToken(PragmaTok);
+ }
+ OutputChar('\n');
+ }
+};
+} // end anonymous namespace
+
+
+enum AvoidConcatInfo {
+ /// By default, a token never needs to avoid concatenation. Most tokens (e.g.
+ /// ',', ')', etc) don't cause a problem when concatenated.
+ aci_never_avoid_concat = 0,
+
+ /// aci_custom_firstchar - AvoidConcat contains custom code to handle this
+ /// token's requirements, and it needs to know the first character of the
+ /// token.
+ aci_custom_firstchar = 1,
+
+ /// aci_custom - AvoidConcat contains custom code to handle this token's
+ /// requirements, but it doesn't need to know the first character of the
+ /// token.
+ aci_custom = 2,
+
+ /// aci_avoid_equal - Many tokens cannot be safely followed by an '='
+ /// character. For example, "<<" turns into "<<=" when followed by an =.
+ aci_avoid_equal = 4
+};
+
+/// This array contains information for each token on what action to take when
+/// avoiding concatenation of tokens in the AvoidConcat method.
+static char TokenInfo[tok::NUM_TOKENS];
+
+/// InitAvoidConcatTokenInfo - Tokens that must avoid concatenation should be
+/// marked by this function.
+static void InitAvoidConcatTokenInfo() {
+ // These tokens have custom code in AvoidConcat.
+ TokenInfo[tok::identifier ] |= aci_custom;
+ TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;
+ TokenInfo[tok::period ] |= aci_custom_firstchar;
+ TokenInfo[tok::amp ] |= aci_custom_firstchar;
+ TokenInfo[tok::plus ] |= aci_custom_firstchar;
+ TokenInfo[tok::minus ] |= aci_custom_firstchar;
+ TokenInfo[tok::slash ] |= aci_custom_firstchar;
+ TokenInfo[tok::less ] |= aci_custom_firstchar;
+ TokenInfo[tok::greater ] |= aci_custom_firstchar;
+ TokenInfo[tok::pipe ] |= aci_custom_firstchar;
+ TokenInfo[tok::percent ] |= aci_custom_firstchar;
+ TokenInfo[tok::colon ] |= aci_custom_firstchar;
+ TokenInfo[tok::hash ] |= aci_custom_firstchar;
+ TokenInfo[tok::arrow ] |= aci_custom_firstchar;
+
+ // These tokens change behavior if followed by an '='.
+ TokenInfo[tok::amp ] |= aci_avoid_equal; // &=
+ TokenInfo[tok::plus ] |= aci_avoid_equal; // +=
+ TokenInfo[tok::minus ] |= aci_avoid_equal; // -=
+ TokenInfo[tok::slash ] |= aci_avoid_equal; // /=
+ TokenInfo[tok::less ] |= aci_avoid_equal; // <=
+ TokenInfo[tok::greater ] |= aci_avoid_equal; // >=
+ TokenInfo[tok::pipe ] |= aci_avoid_equal; // |=
+ TokenInfo[tok::percent ] |= aci_avoid_equal; // %=
+ TokenInfo[tok::star ] |= aci_avoid_equal; // *=
+ TokenInfo[tok::exclaim ] |= aci_avoid_equal; // !=
+ TokenInfo[tok::lessless ] |= aci_avoid_equal; // <<=
+ TokenInfo[tok::greaterequal] |= aci_avoid_equal; // >>=
+ TokenInfo[tok::caret ] |= aci_avoid_equal; // ^=
+ TokenInfo[tok::equal ] |= aci_avoid_equal; // ==
+}
+
+/// AvoidConcat - If printing PrevTok immediately followed by Tok would cause
+/// the two individual tokens to be lexed as a single token, return true (which
+/// causes a space to be printed between them). This allows the output of -E
+/// mode to be lexed to the same token stream as lexing the input directly
+/// would.
+///
+/// This code must conservatively return true if it doesn't want to be 100%
+/// accurate. This will cause the output to include extra space characters, but
+/// the resulting output won't have incorrect concatenations going on. Examples
+/// include "..", which we print with a space between, because we don't want to
+/// track enough to tell "x.." from "...".
+bool PrintPPOutputPPCallbacks::AvoidConcat(const Token &PrevTok,
+ const Token &Tok) {
+ char Buffer[256];
+
+ tok::TokenKind PrevKind = PrevTok.getKind();
+ if (PrevTok.getIdentifierInfo()) // Language keyword or named operator.
+ PrevKind = tok::identifier;
+
+ // Look up information on when we should avoid concatenation with prevtok.
+ unsigned ConcatInfo = TokenInfo[PrevKind];
+
+ // If prevtok never causes a problem for anything after it, return quickly.
+ if (ConcatInfo == 0) return false;
+
+ if (ConcatInfo & aci_avoid_equal) {
+ // If the next token is '=' or '==', avoid concatenation.
+ if (Tok.getKind() == tok::equal ||
+ Tok.getKind() == tok::equalequal)
+ return true;
+ ConcatInfo &= ~aci_avoid_equal;
+ }
+
+ if (ConcatInfo == 0) return false;
+
+
+
+ // Basic algorithm: we look at the first character of the second token, and
+ // determine whether it, if appended to the first token, would form (or would
+ // contribute) to a larger token if concatenated.
+ char FirstChar = 0;
+ if (ConcatInfo & aci_custom) {
+ // If the token does not need to know the first character, don't get it.
+ } else if (IdentifierInfo *II = Tok.getIdentifierInfo()) {
+ // Avoid spelling identifiers, the most common form of token.
+ FirstChar = II->getName()[0];
+ } else if (!Tok.needsCleaning()) {
+ SourceManager &SrcMgr = PP.getSourceManager();
+ FirstChar =
+ *SrcMgr.getCharacterData(SrcMgr.getPhysicalLoc(Tok.getLocation()));
+ } else if (Tok.getLength() < 256) {
+ const char *TokPtr = Buffer;
+ PP.getSpelling(Tok, TokPtr);
+ FirstChar = TokPtr[0];
+ } else {
+ FirstChar = PP.getSpelling(Tok)[0];
+ }
+
+ switch (PrevKind) {
+ default: assert(0 && "InitAvoidConcatTokenInfo built wrong");
+ case tok::identifier: // id+id or id+number or id+L"foo".
+ if (Tok.getKind() == tok::numeric_constant || Tok.getIdentifierInfo() ||
+ Tok.getKind() == tok::wide_string_literal /* ||
+ Tok.getKind() == tok::wide_char_literal*/)
+ return true;
+ if (Tok.getKind() != tok::char_constant)
+ return false;
+
+ // FIXME: need a wide_char_constant!
+ if (!Tok.needsCleaning()) {
+ SourceManager &SrcMgr = PP.getSourceManager();
+ return *SrcMgr.getCharacterData(SrcMgr.getPhysicalLoc(Tok.getLocation()))
+ == 'L';
+ } else if (Tok.getLength() < 256) {
+ const char *TokPtr = Buffer;
+ PP.getSpelling(Tok, TokPtr);
+ return TokPtr[0] == 'L';
+ } else {
+ return PP.getSpelling(Tok)[0] == 'L';
+ }
+ case tok::numeric_constant:
+ return isalnum(FirstChar) || Tok.getKind() == tok::numeric_constant ||
+ FirstChar == '+' || FirstChar == '-' || FirstChar == '.';
+ case tok::period: // ..., .*, .1234
+ return FirstChar == '.' || FirstChar == '*' || isdigit(FirstChar);
+ case tok::amp: // &&
+ return FirstChar == '&';
+ case tok::plus: // ++
+ return FirstChar == '+';
+ case tok::minus: // --, ->, ->*
+ return FirstChar == '-' || FirstChar == '>';
+ case tok::slash: //, /*, //
+ return FirstChar == '*' || FirstChar == '/';
+ case tok::less: // <<, <<=, <:, <%
+ return FirstChar == '<' || FirstChar == ':' || FirstChar == '%';
+ case tok::greater: // >>, >>=
+ return FirstChar == '>';
+ case tok::pipe: // ||
+ return FirstChar == '|';
+ case tok::percent: // %>, %:
+ return FirstChar == '>' || FirstChar == ':';
+ case tok::colon: // ::, :>
+ return FirstChar == ':' || FirstChar == '>';
+ case tok::hash: // ##, #@, %:%:
+ return FirstChar == '#' || FirstChar == '@' || FirstChar == '%';
+ case tok::arrow: // ->*
+ return FirstChar == '*';
+ }
+}
+
+/// DoPrintPreprocessedInput - This implements -E mode.
+///
+void clang::DoPrintPreprocessedInput(unsigned MainFileID, Preprocessor &PP,
+ const LangOptions &Options) {
+ // Inform the preprocessor whether we want it to retain comments or not, due
+ // to -C or -CC.
+ PP.SetCommentRetentionState(EnableCommentOutput, EnableMacroCommentOutput);
+
+ InitOutputBuffer();
+ InitAvoidConcatTokenInfo();
+
+ Token Tok, PrevTok;
+ char Buffer[256];
+ PrintPPOutputPPCallbacks *Callbacks = new PrintPPOutputPPCallbacks(PP);
+ PP.setPPCallbacks(Callbacks);
+
+ PP.AddPragmaHandler(0, new UnknownPragmaHandler("#pragma", Callbacks));
+ PP.AddPragmaHandler("GCC", new UnknownPragmaHandler("#pragma GCC",Callbacks));
+
+ // After we have configured the preprocessor, enter the main file.
+
+ // Start parsing the specified input file.
+ PP.EnterSourceFile(MainFileID, 0, true);
+
+ do {
+ PrevTok = Tok;
+ PP.Lex(Tok);
+
+ // If this token is at the start of a line, emit newlines if needed.
+ if (Tok.isAtStartOfLine()) {
+ Callbacks->HandleFirstTokOnLine(Tok);
+ } else if (Tok.hasLeadingSpace() ||
+ // If we haven't emitted a token on this line yet, PrevTok isn't
+ // useful to look at and no concatenation could happen anyway.
+ (Callbacks->hasEmittedTokensOnThisLine() &&
+ // Don't print "-" next to "-", it would form "--".
+ Callbacks->AvoidConcat(PrevTok, Tok))) {
+ OutputChar(' ');
+ }
+
+ if (IdentifierInfo *II = Tok.getIdentifierInfo()) {
+ const char *Str = II->getName();
+ unsigned Len = Tok.needsCleaning() ? strlen(Str) : Tok.getLength();
+ OutputString(Str, Len);
+ } else if (Tok.getLength() < 256) {
+ const char *TokPtr = Buffer;
+ unsigned Len = PP.getSpelling(Tok, TokPtr);
+ OutputString(TokPtr, Len);
+ } else {
+ std::string S = PP.getSpelling(Tok);
+ OutputString(&S[0], S.size());
+ }
+ Callbacks->SetEmittedTokensOnThisLine();
+ } while (Tok.getKind() != tok::eof);
+ OutputChar('\n');
+
+ CleanupOutputBuffer();
+}
+