blob: 26bb3a90dac071c2ff8aa797d3e4544949333a1f [file] [log] [blame]
Chris Lattner22eb9722006-06-18 05:43:12 +00001//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner5b12ab82007-12-29 19:59:25 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Chris Lattner22eb9722006-06-18 05:43:12 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13//
Chris Lattner22eb9722006-06-18 05:43:12 +000014// Options to support:
15// -H - Print the name of each header file used.
Chris Lattner1630c3c2009-02-06 06:45:26 +000016// -d[DNI] - Dump various things.
Chris Lattner22eb9722006-06-18 05:43:12 +000017// -fworking-directory - #line's with preprocessor's working dir.
18// -fpreprocessed
19// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
20// -W*
21// -w
22//
23// Messages to emit:
24// "Multiple include guards may be useful for:\n"
25//
Chris Lattner22eb9722006-06-18 05:43:12 +000026//===----------------------------------------------------------------------===//
27
28#include "clang/Lex/Preprocessor.h"
Chris Lattnerd19564b2009-12-15 01:51:03 +000029#include "MacroArgs.h"
Douglas Gregor9882a5a2010-01-04 19:18:44 +000030#include "clang/Lex/ExternalPreprocessorSource.h"
Chris Lattner07b019a2006-10-22 07:28:56 +000031#include "clang/Lex/HeaderSearch.h"
Chris Lattner22eb9722006-06-18 05:43:12 +000032#include "clang/Lex/MacroInfo.h"
Chris Lattnerb8761832006-06-24 21:31:03 +000033#include "clang/Lex/Pragma.h"
Chris Lattner0b8cfc22006-06-28 06:49:17 +000034#include "clang/Lex/ScratchBuffer.h"
Chris Lattner60f36222009-01-29 05:15:15 +000035#include "clang/Lex/LexDiagnostic.h"
Chris Lattner22eb9722006-06-18 05:43:12 +000036#include "clang/Basic/SourceManager.h"
Ted Kremeneka5c2c272009-02-12 03:26:59 +000037#include "clang/Basic/FileManager.h"
Chris Lattner81278c62006-10-14 19:03:49 +000038#include "clang/Basic/TargetInfo.h"
Chris Lattner5cd83512008-10-05 20:40:30 +000039#include "llvm/ADT/APFloat.h"
Chris Lattner7a4af3b2006-07-26 06:26:52 +000040#include "llvm/ADT/SmallVector.h"
Chris Lattner8a7003c2007-07-16 06:48:38 +000041#include "llvm/Support/MemoryBuffer.h"
Benjamin Kramer89b422c2009-08-23 12:08:50 +000042#include "llvm/Support/raw_ostream.h"
Chris Lattnerc25d8a72009-03-02 22:20:04 +000043#include <cstdio>
Chris Lattner22eb9722006-06-18 05:43:12 +000044using namespace clang;
45
46//===----------------------------------------------------------------------===//
Douglas Gregor9882a5a2010-01-04 19:18:44 +000047ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
Chris Lattner22eb9722006-06-18 05:43:12 +000048
Chris Lattner02dffbd2006-10-14 07:50:21 +000049Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
Daniel Dunbar1b444192009-11-13 05:51:54 +000050 const TargetInfo &target, SourceManager &SM,
Ted Kremeneka705b042009-01-15 18:47:46 +000051 HeaderSearch &Headers,
Daniel Dunbar0c6c9302009-11-11 21:44:21 +000052 IdentifierInfoLookup* IILookup,
53 bool OwnsHeaders)
Chris Lattner83aba002009-03-13 21:17:43 +000054 : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
Douglas Gregor9882a5a2010-01-04 19:18:44 +000055 SourceMgr(SM), HeaderInfo(Headers), ExternalSource(0),
56 Identifiers(opts, IILookup), BuiltinInfo(Target), CodeCompletionFile(0),
57 CurPPLexer(0), CurDirLookup(0), Callbacks(0), MacroArgCache(0) {
Chris Lattner0b8cfc22006-06-28 06:49:17 +000058 ScratchBuf = new ScratchBuffer(SourceMgr);
Chris Lattner0af3ba12009-04-13 01:29:17 +000059 CounterValue = 0; // __COUNTER__ starts at 0.
Daniel Dunbar0c6c9302009-11-11 21:44:21 +000060 OwnsHeaderSearch = OwnsHeaders;
Mike Stump11289f42009-09-09 15:08:12 +000061
Chris Lattner22eb9722006-06-18 05:43:12 +000062 // Clear stats.
Chris Lattner59a9ebd2006-10-18 05:34:33 +000063 NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
Chris Lattner22eb9722006-06-18 05:43:12 +000064 NumIf = NumElse = NumEndif = 0;
Chris Lattner78186052006-07-09 00:45:31 +000065 NumEnteredSourceFiles = 0;
66 NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
Chris Lattner510ab612006-07-20 04:47:30 +000067 NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
Mike Stump11289f42009-09-09 15:08:12 +000068 MaxIncludeStackDepth = 0;
Chris Lattner22eb9722006-06-18 05:43:12 +000069 NumSkipped = 0;
Chris Lattnerb352e3e2006-11-21 06:17:10 +000070
71 // Default to discarding comments.
72 KeepComments = false;
73 KeepMacroComments = false;
Mike Stump11289f42009-09-09 15:08:12 +000074
Chris Lattner22eb9722006-06-18 05:43:12 +000075 // Macro expansion is enabled.
76 DisableMacroExpansion = false;
Chris Lattneree8760b2006-07-15 07:42:55 +000077 InMacroArgs = false;
Chris Lattner285c0c12008-03-09 02:26:03 +000078 NumCachedTokenLexers = 0;
Chris Lattner0c885f52006-06-21 06:50:18 +000079
Argyrios Kyrtzidisb3dd1e02008-08-10 13:15:22 +000080 CachedLexPos = 0;
81
Douglas Gregor9882a5a2010-01-04 19:18:44 +000082 // We haven't read anything from the external source.
83 ReadMacrosFromExternalSource = false;
84
Chris Lattner8ff71992006-07-06 05:17:39 +000085 // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
86 // This gets unpoisoned where it is allowed.
87 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
Mike Stump11289f42009-09-09 15:08:12 +000088
Chris Lattnerb8761832006-06-24 21:31:03 +000089 // Initialize the pragma handlers.
90 PragmaHandlers = new PragmaNamespace(0);
91 RegisterBuiltinPragmas();
Mike Stump11289f42009-09-09 15:08:12 +000092
Chris Lattner677757a2006-06-28 05:26:32 +000093 // Initialize builtin macros like __LINE__ and friends.
94 RegisterBuiltinMacros();
Chris Lattner22eb9722006-06-18 05:43:12 +000095}
96
97Preprocessor::~Preprocessor() {
Argyrios Kyrtzidis5d240d02008-08-23 12:12:06 +000098 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
99
Chris Lattner69772b02006-07-02 20:34:39 +0000100 while (!IncludeMacroStack.empty()) {
101 delete IncludeMacroStack.back().TheLexer;
Chris Lattner285c0c12008-03-09 02:26:03 +0000102 delete IncludeMacroStack.back().TheTokenLexer;
Chris Lattner69772b02006-07-02 20:34:39 +0000103 IncludeMacroStack.pop_back();
Chris Lattner22eb9722006-06-18 05:43:12 +0000104 }
Chris Lattnerc43ddc82007-10-07 08:44:20 +0000105
106 // Free any macro definitions.
107 for (llvm::DenseMap<IdentifierInfo*, MacroInfo*>::iterator I =
108 Macros.begin(), E = Macros.end(); I != E; ++I) {
Ted Kremenek6c7ea112008-12-15 19:56:42 +0000109 // We don't need to free the MacroInfo objects directly. These
110 // will be released when the BumpPtrAllocator 'BP' object gets
Chris Lattnerd2fa78d2009-12-14 22:02:43 +0000111 // destroyed. We still need to run the dtor, however, to free
Ted Kremenek8c3b8122009-01-19 07:45:44 +0000112 // memory alocated by MacroInfo.
Chris Lattner70946da2009-02-20 22:46:43 +0000113 I->second->Destroy(BP);
Chris Lattnerc43ddc82007-10-07 08:44:20 +0000114 I->first->setHasMacroDefinition(false);
115 }
Mike Stump11289f42009-09-09 15:08:12 +0000116
Chris Lattnerc02c4ab2007-07-15 00:25:26 +0000117 // Free any cached macro expanders.
Chris Lattner285c0c12008-03-09 02:26:03 +0000118 for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
119 delete TokenLexerCache[i];
Chris Lattnerd19564b2009-12-15 01:51:03 +0000120
121 // Free any cached MacroArgs.
122 for (MacroArgs *ArgList = MacroArgCache; ArgList; )
123 ArgList = ArgList->deallocate();
Mike Stump11289f42009-09-09 15:08:12 +0000124
Chris Lattnerb8761832006-06-24 21:31:03 +0000125 // Release pragma information.
126 delete PragmaHandlers;
Chris Lattner0b8cfc22006-06-28 06:49:17 +0000127
128 // Delete the scratch buffer info.
129 delete ScratchBuf;
Chris Lattner22192932008-03-14 06:07:05 +0000130
Daniel Dunbar0c6c9302009-11-11 21:44:21 +0000131 // Delete the header search info, if we own it.
132 if (OwnsHeaderSearch)
133 delete &HeaderInfo;
134
Chris Lattner22192932008-03-14 06:07:05 +0000135 delete Callbacks;
Chris Lattner22eb9722006-06-18 05:43:12 +0000136}
137
Ted Kremeneka5c2c272009-02-12 03:26:59 +0000138void Preprocessor::setPTHManager(PTHManager* pm) {
139 PTH.reset(pm);
Douglas Gregord2eb58a2009-10-16 18:18:30 +0000140 FileMgr.addStatCache(PTH->createStatCache());
Ted Kremeneka5c2c272009-02-12 03:26:59 +0000141}
142
Chris Lattner146762e2007-07-20 16:59:19 +0000143void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000144 llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
145 << getSpelling(Tok) << "'";
Mike Stump11289f42009-09-09 15:08:12 +0000146
Chris Lattnerd01e2912006-06-18 16:22:51 +0000147 if (!DumpFlags) return;
Mike Stump11289f42009-09-09 15:08:12 +0000148
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000149 llvm::errs() << "\t";
Chris Lattnerd01e2912006-06-18 16:22:51 +0000150 if (Tok.isAtStartOfLine())
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000151 llvm::errs() << " [StartOfLine]";
Chris Lattnerd01e2912006-06-18 16:22:51 +0000152 if (Tok.hasLeadingSpace())
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000153 llvm::errs() << " [LeadingSpace]";
Chris Lattner6e4bf522006-07-27 06:59:25 +0000154 if (Tok.isExpandDisabled())
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000155 llvm::errs() << " [ExpandDisabled]";
Chris Lattnerd01e2912006-06-18 16:22:51 +0000156 if (Tok.needsCleaning()) {
Chris Lattner50b497e2006-06-18 16:32:35 +0000157 const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000158 llvm::errs() << " [UnClean='" << std::string(Start, Start+Tok.getLength())
159 << "']";
Chris Lattnerd01e2912006-06-18 16:22:51 +0000160 }
Mike Stump11289f42009-09-09 15:08:12 +0000161
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000162 llvm::errs() << "\tLoc=<";
Chris Lattner615315f2007-12-09 20:31:55 +0000163 DumpLocation(Tok.getLocation());
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000164 llvm::errs() << ">";
Chris Lattner615315f2007-12-09 20:31:55 +0000165}
166
167void Preprocessor::DumpLocation(SourceLocation Loc) const {
Chris Lattnerf1ca7d32009-01-27 07:57:44 +0000168 Loc.dump(SourceMgr);
Chris Lattnerd01e2912006-06-18 16:22:51 +0000169}
170
171void Preprocessor::DumpMacro(const MacroInfo &MI) const {
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000172 llvm::errs() << "MACRO: ";
Chris Lattnerd01e2912006-06-18 16:22:51 +0000173 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
174 DumpToken(MI.getReplacementToken(i));
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000175 llvm::errs() << " ";
Chris Lattnerd01e2912006-06-18 16:22:51 +0000176 }
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000177 llvm::errs() << "\n";
Chris Lattnerd01e2912006-06-18 16:22:51 +0000178}
179
Chris Lattner22eb9722006-06-18 05:43:12 +0000180void Preprocessor::PrintStats() {
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000181 llvm::errs() << "\n*** Preprocessor Stats:\n";
182 llvm::errs() << NumDirectives << " directives found:\n";
183 llvm::errs() << " " << NumDefined << " #define.\n";
184 llvm::errs() << " " << NumUndefined << " #undef.\n";
185 llvm::errs() << " #include/#include_next/#import:\n";
186 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
187 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
188 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
189 llvm::errs() << " " << NumElse << " #else/#elif.\n";
190 llvm::errs() << " " << NumEndif << " #endif.\n";
191 llvm::errs() << " " << NumPragma << " #pragma.\n";
192 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
Chris Lattner22eb9722006-06-18 05:43:12 +0000193
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000194 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
Ted Kremeneka0a3e9b2008-01-14 16:44:48 +0000195 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
196 << NumFastMacroExpanded << " on the fast path.\n";
Benjamin Kramer89b422c2009-08-23 12:08:50 +0000197 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
Ted Kremeneka0a3e9b2008-01-14 16:44:48 +0000198 << " token paste (##) operations performed, "
199 << NumFastTokenPaste << " on the fast path.\n";
Chris Lattner22eb9722006-06-18 05:43:12 +0000200}
201
Douglas Gregor9882a5a2010-01-04 19:18:44 +0000202Preprocessor::macro_iterator
203Preprocessor::macro_begin(bool IncludeExternalMacros) const {
204 if (IncludeExternalMacros && ExternalSource &&
205 !ReadMacrosFromExternalSource) {
206 ReadMacrosFromExternalSource = true;
207 ExternalSource->ReadDefinedMacros();
208 }
209
210 return Macros.begin();
211}
212
213Preprocessor::macro_iterator
214Preprocessor::macro_end(bool IncludeExternalMacros) const {
215 if (IncludeExternalMacros && ExternalSource &&
216 !ReadMacrosFromExternalSource) {
217 ReadMacrosFromExternalSource = true;
218 ExternalSource->ReadDefinedMacros();
219 }
220
221 return Macros.end();
222}
223
Douglas Gregor53ad6b92009-12-02 06:49:09 +0000224bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
225 unsigned TruncateAtLine,
226 unsigned TruncateAtColumn) {
227 using llvm::MemoryBuffer;
228
229 CodeCompletionFile = File;
230
231 // Okay to clear out the code-completion point by passing NULL.
232 if (!CodeCompletionFile)
233 return false;
234
235 // Load the actual file's contents.
236 const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File);
237 if (!Buffer)
238 return true;
239
240 // Find the byte position of the truncation point.
241 const char *Position = Buffer->getBufferStart();
242 for (unsigned Line = 1; Line < TruncateAtLine; ++Line) {
243 for (; *Position; ++Position) {
244 if (*Position != '\r' && *Position != '\n')
245 continue;
246
247 // Eat \r\n or \n\r as a single line.
248 if ((Position[1] == '\r' || Position[1] == '\n') &&
249 Position[0] != Position[1])
250 ++Position;
251 ++Position;
252 break;
253 }
254 }
255
Douglas Gregor9291ab62009-12-08 21:45:46 +0000256 Position += TruncateAtColumn - 1;
Douglas Gregor53ad6b92009-12-02 06:49:09 +0000257
258 // Truncate the buffer.
Douglas Gregor9291ab62009-12-08 21:45:46 +0000259 if (Position < Buffer->getBufferEnd()) {
Douglas Gregor53ad6b92009-12-02 06:49:09 +0000260 MemoryBuffer *TruncatedBuffer
261 = MemoryBuffer::getMemBufferCopy(Buffer->getBufferStart(), Position,
262 Buffer->getBufferIdentifier());
263 SourceMgr.overrideFileContents(File, TruncatedBuffer);
264 }
265
266 return false;
267}
268
Douglas Gregor5f498832009-12-03 17:05:59 +0000269bool Preprocessor::isCodeCompletionFile(SourceLocation FileLoc) const {
Douglas Gregor53ad6b92009-12-02 06:49:09 +0000270 return CodeCompletionFile && FileLoc.isFileID() &&
271 SourceMgr.getFileEntryForID(SourceMgr.getFileID(FileLoc))
272 == CodeCompletionFile;
273}
274
Chris Lattner22eb9722006-06-18 05:43:12 +0000275//===----------------------------------------------------------------------===//
Chris Lattnerd01e2912006-06-18 16:22:51 +0000276// Token Spelling
277//===----------------------------------------------------------------------===//
278
Chris Lattnerd01e2912006-06-18 16:22:51 +0000279/// getSpelling() - Return the 'spelling' of this token. The spelling of a
280/// token are the characters used to represent the token in the source file
281/// after trigraph expansion and escaped-newline folding. In particular, this
282/// wants to get the true, uncanonicalized, spelling of things like digraphs
283/// UCNs, etc.
Daniel Dunbarbf410c62009-11-14 01:20:48 +0000284std::string Preprocessor::getSpelling(const Token &Tok,
285 const SourceManager &SourceMgr,
286 const LangOptions &Features) {
Chris Lattnerd01e2912006-06-18 16:22:51 +0000287 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
Ted Kremenek8d178f42009-01-27 00:01:05 +0000288
Chris Lattnerd01e2912006-06-18 16:22:51 +0000289 // If this token contains nothing interesting, return it directly.
Ted Kremenek8d178f42009-01-27 00:01:05 +0000290 const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation());
Chris Lattnerd01e2912006-06-18 16:22:51 +0000291 if (!Tok.needsCleaning())
292 return std::string(TokStart, TokStart+Tok.getLength());
Mike Stump11289f42009-09-09 15:08:12 +0000293
Chris Lattnerd01e2912006-06-18 16:22:51 +0000294 std::string Result;
295 Result.reserve(Tok.getLength());
Mike Stump11289f42009-09-09 15:08:12 +0000296
Chris Lattneref9eae12006-07-04 22:33:12 +0000297 // Otherwise, hard case, relex the characters into the string.
Chris Lattnerd01e2912006-06-18 16:22:51 +0000298 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
299 Ptr != End; ) {
300 unsigned CharSize;
301 Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
302 Ptr += CharSize;
303 }
304 assert(Result.size() != unsigned(Tok.getLength()) &&
305 "NeedsCleaning flag set on something that didn't need cleaning!");
306 return Result;
307}
308
Daniel Dunbarbf410c62009-11-14 01:20:48 +0000309/// getSpelling() - Return the 'spelling' of this token. The spelling of a
310/// token are the characters used to represent the token in the source file
311/// after trigraph expansion and escaped-newline folding. In particular, this
312/// wants to get the true, uncanonicalized, spelling of things like digraphs
313/// UCNs, etc.
314std::string Preprocessor::getSpelling(const Token &Tok) const {
315 return getSpelling(Tok, SourceMgr, Features);
316}
317
Chris Lattnerd01e2912006-06-18 16:22:51 +0000318/// getSpelling - This method is used to get the spelling of a token into a
319/// preallocated buffer, instead of as an std::string. The caller is required
320/// to allocate enough space for the token, which is guaranteed to be at least
321/// Tok.getLength() bytes long. The actual length of the token is returned.
Chris Lattneref9eae12006-07-04 22:33:12 +0000322///
323/// Note that this method may do two possible things: it may either fill in
324/// the buffer specified with characters, or it may *change the input pointer*
325/// to point to a constant buffer with the data already in it (avoiding a
326/// copy). The caller is not allowed to modify the returned buffer pointer
327/// if an internal buffer is returned.
Chris Lattner146762e2007-07-20 16:59:19 +0000328unsigned Preprocessor::getSpelling(const Token &Tok,
Chris Lattneref9eae12006-07-04 22:33:12 +0000329 const char *&Buffer) const {
Chris Lattnerd01e2912006-06-18 16:22:51 +0000330 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
Mike Stump11289f42009-09-09 15:08:12 +0000331
Chris Lattnerd3a15f72006-07-04 23:01:03 +0000332 // If this token is an identifier, just return the string from the identifier
333 // table, which is very quick.
334 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
Daniel Dunbar2c422dc92009-10-18 20:26:12 +0000335 Buffer = II->getNameStart();
Chris Lattner07ebf302009-01-05 19:44:41 +0000336 return II->getLength();
Chris Lattnerd3a15f72006-07-04 23:01:03 +0000337 }
Ted Kremenek884a5582009-01-08 02:47:16 +0000338
Chris Lattnerd3a15f72006-07-04 23:01:03 +0000339 // Otherwise, compute the start of the token in the input lexer buffer.
Chris Lattner5a7971e2009-01-26 19:29:26 +0000340 const char *TokStart = 0;
Mike Stump11289f42009-09-09 15:08:12 +0000341
Chris Lattner5a7971e2009-01-26 19:29:26 +0000342 if (Tok.isLiteral())
343 TokStart = Tok.getLiteralData();
Mike Stump11289f42009-09-09 15:08:12 +0000344
Chris Lattner5a7971e2009-01-26 19:29:26 +0000345 if (TokStart == 0)
346 TokStart = SourceMgr.getCharacterData(Tok.getLocation());
Chris Lattnerd01e2912006-06-18 16:22:51 +0000347
348 // If this token contains nothing interesting, return it directly.
349 if (!Tok.needsCleaning()) {
Chris Lattneref9eae12006-07-04 22:33:12 +0000350 Buffer = TokStart;
351 return Tok.getLength();
Chris Lattnerd01e2912006-06-18 16:22:51 +0000352 }
Mike Stump11289f42009-09-09 15:08:12 +0000353
Chris Lattnerd01e2912006-06-18 16:22:51 +0000354 // Otherwise, hard case, relex the characters into the string.
Chris Lattneref9eae12006-07-04 22:33:12 +0000355 char *OutBuf = const_cast<char*>(Buffer);
Chris Lattnerd01e2912006-06-18 16:22:51 +0000356 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
357 Ptr != End; ) {
358 unsigned CharSize;
359 *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
360 Ptr += CharSize;
361 }
362 assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
363 "NeedsCleaning flag set on something that didn't need cleaning!");
Mike Stump11289f42009-09-09 15:08:12 +0000364
Chris Lattnerd01e2912006-06-18 16:22:51 +0000365 return OutBuf-Buffer;
366}
367
Chris Lattnerb94ec7b2006-07-14 06:54:10 +0000368/// CreateString - Plop the specified string into a scratch buffer and return a
369/// location for it. If specified, the source location provides a source
370/// location for the token.
Chris Lattner5a7971e2009-01-26 19:29:26 +0000371void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
372 SourceLocation InstantiationLoc) {
373 Tok.setLength(Len);
Mike Stump11289f42009-09-09 15:08:12 +0000374
Chris Lattner5a7971e2009-01-26 19:29:26 +0000375 const char *DestPtr;
376 SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
Mike Stump11289f42009-09-09 15:08:12 +0000377
Chris Lattner5a7971e2009-01-26 19:29:26 +0000378 if (InstantiationLoc.isValid())
Chris Lattner9dc9c202009-02-15 20:52:18 +0000379 Loc = SourceMgr.createInstantiationLoc(Loc, InstantiationLoc,
380 InstantiationLoc, Len);
Chris Lattner5a7971e2009-01-26 19:29:26 +0000381 Tok.setLocation(Loc);
Mike Stump11289f42009-09-09 15:08:12 +0000382
Chris Lattner5a7971e2009-01-26 19:29:26 +0000383 // If this is a literal token, set the pointer data.
384 if (Tok.isLiteral())
385 Tok.setLiteralData(DestPtr);
Chris Lattnerb94ec7b2006-07-14 06:54:10 +0000386}
387
388
Chris Lattner8a7003c2007-07-16 06:48:38 +0000389/// AdvanceToTokenCharacter - Given a location that specifies the start of a
390/// token, return a new location that specifies a character within the token.
Mike Stump11289f42009-09-09 15:08:12 +0000391SourceLocation Preprocessor::AdvanceToTokenCharacter(SourceLocation TokStart,
Chris Lattner8a7003c2007-07-16 06:48:38 +0000392 unsigned CharNo) {
Chris Lattner8a425862009-01-16 07:36:28 +0000393 // Figure out how many physical characters away the specified instantiation
Chris Lattner8a7003c2007-07-16 06:48:38 +0000394 // character is. This needs to take into consideration newlines and
395 // trigraphs.
Chris Lattnerdc5c0552007-07-20 16:37:10 +0000396 const char *TokPtr = SourceMgr.getCharacterData(TokStart);
Mike Stump11289f42009-09-09 15:08:12 +0000397
Chris Lattner93017cc2009-04-18 22:28:58 +0000398 // If they request the first char of the token, we're trivially done.
399 if (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))
400 return TokStart;
Mike Stump11289f42009-09-09 15:08:12 +0000401
Chris Lattnerdc5c0552007-07-20 16:37:10 +0000402 unsigned PhysOffset = 0;
Mike Stump11289f42009-09-09 15:08:12 +0000403
Chris Lattner8a7003c2007-07-16 06:48:38 +0000404 // The usual case is that tokens don't contain anything interesting. Skip
405 // over the uninteresting characters. If a token only consists of simple
406 // chars, this method is extremely fast.
Chris Lattner93017cc2009-04-18 22:28:58 +0000407 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
408 if (CharNo == 0)
409 return TokStart.getFileLocWithOffset(PhysOffset);
Chris Lattnerdc5c0552007-07-20 16:37:10 +0000410 ++TokPtr, --CharNo, ++PhysOffset;
Chris Lattner93017cc2009-04-18 22:28:58 +0000411 }
Mike Stump11289f42009-09-09 15:08:12 +0000412
Chris Lattner8ddb5cf2009-01-17 07:57:25 +0000413 // If we have a character that may be a trigraph or escaped newline, use a
Chris Lattner8a7003c2007-07-16 06:48:38 +0000414 // lexer to parse it correctly.
Chris Lattner93017cc2009-04-18 22:28:58 +0000415 for (; CharNo; --CharNo) {
416 unsigned Size;
417 Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features);
418 TokPtr += Size;
419 PhysOffset += Size;
Chris Lattner8a7003c2007-07-16 06:48:38 +0000420 }
Mike Stump11289f42009-09-09 15:08:12 +0000421
Chris Lattner93017cc2009-04-18 22:28:58 +0000422 // Final detail: if we end up on an escaped newline, we want to return the
423 // location of the actual byte of the token. For example foo\<newline>bar
424 // advanced by 3 should return the location of b, not of \\. One compounding
425 // detail of this is that the escape may be made by a trigraph.
426 if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
427 PhysOffset = Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
Mike Stump11289f42009-09-09 15:08:12 +0000428
Chris Lattnerdc5c0552007-07-20 16:37:10 +0000429 return TokStart.getFileLocWithOffset(PhysOffset);
Chris Lattner8a7003c2007-07-16 06:48:38 +0000430}
431
Douglas Gregor96977da2009-02-27 17:53:17 +0000432/// \brief Computes the source location just past the end of the
433/// token at this source location.
434///
435/// This routine can be used to produce a source location that
436/// points just past the end of the token referenced by \p Loc, and
437/// is generally used when a diagnostic needs to point just after a
438/// token where it expected something different that it received. If
439/// the returned source location would not be meaningful (e.g., if
440/// it points into a macro), this routine returns an invalid
441/// source location.
442SourceLocation Preprocessor::getLocForEndOfToken(SourceLocation Loc) {
443 if (Loc.isInvalid() || !Loc.isFileID())
444 return SourceLocation();
445
Chris Lattner184e65d2009-04-14 23:22:57 +0000446 unsigned Len = Lexer::MeasureTokenLength(Loc, getSourceManager(), Features);
Douglas Gregor96977da2009-02-27 17:53:17 +0000447 return AdvanceToTokenCharacter(Loc, Len);
448}
449
450
Chris Lattner8a7003c2007-07-16 06:48:38 +0000451
Chris Lattner1f1b0db2007-10-09 22:10:18 +0000452//===----------------------------------------------------------------------===//
453// Preprocessor Initialization Methods
454//===----------------------------------------------------------------------===//
455
Chris Lattner1f1b0db2007-10-09 22:10:18 +0000456
457/// EnterMainSourceFile - Enter the specified FileID as the main source file,
Nate Begemanf7c3ff62008-01-07 04:01:26 +0000458/// which implicitly adds the builtin defines etc.
Ted Kremenek230bd912007-12-19 22:51:13 +0000459void Preprocessor::EnterMainSourceFile() {
Chris Lattner9ef847b2009-02-13 19:33:24 +0000460 // We do not allow the preprocessor to reenter the main file. Doing so will
461 // cause FileID's to accumulate information from both runs (e.g. #line
462 // information) and predefined macros aren't guaranteed to be set properly.
463 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
Chris Lattnerd32480d2009-01-17 06:22:33 +0000464 FileID MainFileID = SourceMgr.getMainFileID();
Mike Stump11289f42009-09-09 15:08:12 +0000465
Chris Lattner1f1b0db2007-10-09 22:10:18 +0000466 // Enter the main file source buffer.
Daniel Dunbar17766792009-12-06 09:19:12 +0000467 std::string ErrorStr;
468 bool Res = EnterSourceFile(MainFileID, 0, ErrorStr);
469 assert(!Res && "Entering main file should not fail!");
Mike Stump11289f42009-09-09 15:08:12 +0000470
Chris Lattner609d4132007-11-15 19:07:47 +0000471 // Tell the header info that the main file was entered. If the file is later
472 // #imported, it won't be re-entered.
Chris Lattnerd32480d2009-01-17 06:22:33 +0000473 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
Chris Lattner609d4132007-11-15 19:07:47 +0000474 HeaderInfo.IncrementIncludeCount(FE);
Mike Stump11289f42009-09-09 15:08:12 +0000475
Benjamin Kramerd77adb52009-12-31 15:33:09 +0000476 // Preprocess Predefines to populate the initial preprocessor state.
Mike Stump11289f42009-09-09 15:08:12 +0000477 llvm::MemoryBuffer *SB =
Benjamin Kramerd77adb52009-12-31 15:33:09 +0000478 llvm::MemoryBuffer::getMemBufferCopy(Predefines.data(),
479 Predefines.data() + Predefines.size(),
Chris Lattner1d1d80e2009-03-20 20:16:10 +0000480 "<built-in>");
Chris Lattner1f1b0db2007-10-09 22:10:18 +0000481 assert(SB && "Cannot fail to create predefined source buffer");
Chris Lattnerd32480d2009-01-17 06:22:33 +0000482 FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
483 assert(!FID.isInvalid() && "Could not create FileID for predefines?");
Mike Stump11289f42009-09-09 15:08:12 +0000484
Chris Lattner1f1b0db2007-10-09 22:10:18 +0000485 // Start parsing the predefines.
Daniel Dunbar17766792009-12-06 09:19:12 +0000486 Res = EnterSourceFile(FID, 0, ErrorStr);
487 assert(!Res && "Entering predefines should not fail!");
Chris Lattner1f1b0db2007-10-09 22:10:18 +0000488}
Chris Lattner8a7003c2007-07-16 06:48:38 +0000489
Chris Lattner677757a2006-06-28 05:26:32 +0000490
491//===----------------------------------------------------------------------===//
492// Lexer Event Handling.
493//===----------------------------------------------------------------------===//
494
Chris Lattnercefc7682006-07-08 08:28:12 +0000495/// LookUpIdentifierInfo - Given a tok::identifier token, look up the
496/// identifier information for the token and install it into the token.
Chris Lattner146762e2007-07-20 16:59:19 +0000497IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier,
Daniel Dunbar07dcd8b2009-11-05 01:53:52 +0000498 const char *BufPtr) const {
Chris Lattner98c1f7c2007-10-09 18:02:16 +0000499 assert(Identifier.is(tok::identifier) && "Not an identifier!");
Chris Lattnercefc7682006-07-08 08:28:12 +0000500 assert(Identifier.getIdentifierInfo() == 0 && "Identinfo already exists!");
Mike Stump11289f42009-09-09 15:08:12 +0000501
Chris Lattnercefc7682006-07-08 08:28:12 +0000502 // Look up this token, see if it is a macro, or if it is a language keyword.
503 IdentifierInfo *II;
504 if (BufPtr && !Identifier.needsCleaning()) {
505 // No cleaning needed, just use the characters from the lexed buffer.
Daniel Dunbarf539bfe2009-11-05 01:53:39 +0000506 II = getIdentifierInfo(llvm::StringRef(BufPtr, Identifier.getLength()));
Chris Lattnercefc7682006-07-08 08:28:12 +0000507 } else {
508 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
Chris Lattnerf9aba2c2007-07-13 17:10:38 +0000509 llvm::SmallVector<char, 64> IdentifierBuffer;
510 IdentifierBuffer.resize(Identifier.getLength());
511 const char *TmpBuf = &IdentifierBuffer[0];
Chris Lattnercefc7682006-07-08 08:28:12 +0000512 unsigned Size = getSpelling(Identifier, TmpBuf);
Daniel Dunbarf539bfe2009-11-05 01:53:39 +0000513 II = getIdentifierInfo(llvm::StringRef(TmpBuf, Size));
Chris Lattnercefc7682006-07-08 08:28:12 +0000514 }
Chris Lattner8c204872006-10-14 05:19:21 +0000515 Identifier.setIdentifierInfo(II);
Chris Lattnercefc7682006-07-08 08:28:12 +0000516 return II;
517}
518
519
Chris Lattner677757a2006-06-28 05:26:32 +0000520/// HandleIdentifier - This callback is invoked when the lexer reads an
521/// identifier. This callback looks up the identifier in the map and/or
522/// potentially macro expands it or turns it into a named token (like 'for').
Chris Lattnerad89ec02009-01-21 07:43:11 +0000523///
524/// Note that callers of this method are guarded by checking the
525/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
526/// IdentifierInfo methods that compute these properties will need to change to
527/// match.
Chris Lattner146762e2007-07-20 16:59:19 +0000528void Preprocessor::HandleIdentifier(Token &Identifier) {
Chris Lattner0f1f5052006-07-20 04:16:23 +0000529 assert(Identifier.getIdentifierInfo() &&
530 "Can't handle identifiers without identifier info!");
Mike Stump11289f42009-09-09 15:08:12 +0000531
Chris Lattnerc79f6fb2006-07-04 17:53:21 +0000532 IdentifierInfo &II = *Identifier.getIdentifierInfo();
Chris Lattner677757a2006-06-28 05:26:32 +0000533
534 // If this identifier was poisoned, and if it was not produced from a macro
535 // expansion, emit an error.
Ted Kremeneka2c3c8d2008-11-19 22:43:49 +0000536 if (II.isPoisoned() && CurPPLexer) {
Chris Lattner8ff71992006-07-06 05:17:39 +0000537 if (&II != Ident__VA_ARGS__) // We warn about __VA_ARGS__ with poisoning.
538 Diag(Identifier, diag::err_pp_used_poisoned_id);
539 else
540 Diag(Identifier, diag::ext_pp_bad_vaargs_use);
541 }
Mike Stump11289f42009-09-09 15:08:12 +0000542
Chris Lattner78186052006-07-09 00:45:31 +0000543 // If this is a macro to be expanded, do it.
Chris Lattnerc43ddc82007-10-07 08:44:20 +0000544 if (MacroInfo *MI = getMacroInfo(&II)) {
Chris Lattner6e4bf522006-07-27 06:59:25 +0000545 if (!DisableMacroExpansion && !Identifier.isExpandDisabled()) {
546 if (MI->isEnabled()) {
547 if (!HandleMacroExpandedIdentifier(Identifier, MI))
548 return;
549 } else {
550 // C99 6.10.3.4p2 says that a disabled macro may never again be
551 // expanded, even if it's in a context where it could be expanded in the
552 // future.
Chris Lattner146762e2007-07-20 16:59:19 +0000553 Identifier.setFlag(Token::DisableExpand);
Chris Lattner6e4bf522006-07-27 06:59:25 +0000554 }
555 }
Chris Lattner063400e2006-10-14 19:54:15 +0000556 }
Chris Lattner677757a2006-06-28 05:26:32 +0000557
Chris Lattner5b9f4892006-11-21 17:23:33 +0000558 // C++ 2.11p2: If this is an alternative representation of a C++ operator,
559 // then we act as if it is the actual operator and not the textual
560 // representation of it.
561 if (II.isCPlusPlusOperatorKeyword())
562 Identifier.setIdentifierInfo(0);
563
Chris Lattner677757a2006-06-28 05:26:32 +0000564 // If this is an extension token, diagnose its use.
Steve Naroffc84e8b72008-09-02 18:50:17 +0000565 // We avoid diagnosing tokens that originate from macro definitions.
Eli Friedman6bba2ad2009-04-28 03:59:15 +0000566 // FIXME: This warning is disabled in cases where it shouldn't be,
567 // like "#define TY typeof", "TY(1) x".
568 if (II.isExtensionToken() && !DisableMacroExpansion)
Chris Lattner53621a52007-06-13 20:44:40 +0000569 Diag(Identifier, diag::ext_token_used);
Chris Lattner677757a2006-06-28 05:26:32 +0000570}
Douglas Gregorc6d5edd2009-07-02 17:08:52 +0000571
572void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
573 assert(Handler && "NULL comment handler");
574 assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
575 CommentHandlers.end() && "Comment handler already registered");
576 CommentHandlers.push_back(Handler);
577}
578
579void Preprocessor::RemoveCommentHandler(CommentHandler *Handler) {
580 std::vector<CommentHandler *>::iterator Pos
581 = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
582 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
583 CommentHandlers.erase(Pos);
584}
585
586void Preprocessor::HandleComment(SourceRange Comment) {
587 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
588 HEnd = CommentHandlers.end();
589 H != HEnd; ++H)
590 (*H)->HandleComment(*this, Comment);
591}
592
593CommentHandler::~CommentHandler() { }