blob: 56ed7659249c0960dda89e5fd5efb82ad29d2611 [file] [log] [blame]
Reid Spencer5f016e22007-07-11 17:01:13 +00001//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner0bc735f2007-12-29 19:59:25 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Reid Spencer5f016e22007-07-11 17:01:13 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13//
14// Options to support:
15// -H - Print the name of each header file used.
Chris Lattnerf73903a2009-02-06 06:45:26 +000016// -d[DNI] - Dump various things.
Reid Spencer5f016e22007-07-11 17:01:13 +000017// -fworking-directory - #line's with preprocessor's working dir.
18// -fpreprocessed
19// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
20// -W*
21// -w
22//
23// Messages to emit:
24// "Multiple include guards may be useful for:\n"
25//
26//===----------------------------------------------------------------------===//
27
28#include "clang/Lex/Preprocessor.h"
Chris Lattner23f77e52009-12-15 01:51:03 +000029#include "MacroArgs.h"
Douglas Gregor88a35862010-01-04 19:18:44 +000030#include "clang/Lex/ExternalPreprocessorSource.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000031#include "clang/Lex/HeaderSearch.h"
32#include "clang/Lex/MacroInfo.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000033#include "clang/Lex/Pragma.h"
Douglas Gregor94dc8f62010-03-19 16:15:56 +000034#include "clang/Lex/PreprocessingRecord.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000035#include "clang/Lex/ScratchBuffer.h"
Chris Lattner500d3292009-01-29 05:15:15 +000036#include "clang/Lex/LexDiagnostic.h"
Douglas Gregorf44e8542010-08-24 19:08:16 +000037#include "clang/Lex/CodeCompletionHandler.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000038#include "clang/Basic/SourceManager.h"
Ted Kremenek337edcd2009-02-12 03:26:59 +000039#include "clang/Basic/FileManager.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000040#include "clang/Basic/TargetInfo.h"
Chris Lattner2db78dd2008-10-05 20:40:30 +000041#include "llvm/ADT/APFloat.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000042#include "llvm/ADT/SmallVector.h"
Chris Lattner97ba77c2007-07-16 06:48:38 +000043#include "llvm/Support/MemoryBuffer.h"
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +000044#include "llvm/Support/raw_ostream.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000045using namespace clang;
46
47//===----------------------------------------------------------------------===//
Douglas Gregor88a35862010-01-04 19:18:44 +000048ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
Reid Spencer5f016e22007-07-11 17:01:13 +000049
50Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
Daniel Dunbar444be732009-11-13 05:51:54 +000051 const TargetInfo &target, SourceManager &SM,
Ted Kremenek72b1b152009-01-15 18:47:46 +000052 HeaderSearch &Headers,
Daniel Dunbar5814e652009-11-11 21:44:21 +000053 IdentifierInfoLookup* IILookup,
54 bool OwnsHeaders)
Chris Lattner836040f2009-03-13 21:17:43 +000055 : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
Douglas Gregor88a35862010-01-04 19:18:44 +000056 SourceMgr(SM), HeaderInfo(Headers), ExternalSource(0),
Douglas Gregorf44e8542010-08-24 19:08:16 +000057 Identifiers(opts, IILookup), BuiltinInfo(Target), CodeComplete(0),
58 CodeCompletionFile(0), SkipMainFilePreamble(0, true), CurPPLexer(0),
59 CurDirLookup(0), Callbacks(0), MacroArgCache(0), Record(0) {
Reid Spencer5f016e22007-07-11 17:01:13 +000060 ScratchBuf = new ScratchBuffer(SourceMgr);
Chris Lattnerc1f9d822009-04-13 01:29:17 +000061 CounterValue = 0; // __COUNTER__ starts at 0.
Daniel Dunbar5814e652009-11-11 21:44:21 +000062 OwnsHeaderSearch = OwnsHeaders;
Mike Stump1eb44332009-09-09 15:08:12 +000063
Reid Spencer5f016e22007-07-11 17:01:13 +000064 // Clear stats.
65 NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
66 NumIf = NumElse = NumEndif = 0;
67 NumEnteredSourceFiles = 0;
68 NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
69 NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
Mike Stump1eb44332009-09-09 15:08:12 +000070 MaxIncludeStackDepth = 0;
Reid Spencer5f016e22007-07-11 17:01:13 +000071 NumSkipped = 0;
72
73 // Default to discarding comments.
74 KeepComments = false;
75 KeepMacroComments = false;
Mike Stump1eb44332009-09-09 15:08:12 +000076
Reid Spencer5f016e22007-07-11 17:01:13 +000077 // Macro expansion is enabled.
78 DisableMacroExpansion = false;
79 InMacroArgs = false;
Chris Lattner6cfe7592008-03-09 02:26:03 +000080 NumCachedTokenLexers = 0;
Reid Spencer5f016e22007-07-11 17:01:13 +000081
Argyrios Kyrtzidis03db1b32008-08-10 13:15:22 +000082 CachedLexPos = 0;
83
Douglas Gregor88a35862010-01-04 19:18:44 +000084 // We haven't read anything from the external source.
85 ReadMacrosFromExternalSource = false;
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +000086
Reid Spencer5f016e22007-07-11 17:01:13 +000087 // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
88 // This gets unpoisoned where it is allowed.
89 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
Mike Stump1eb44332009-09-09 15:08:12 +000090
Reid Spencer5f016e22007-07-11 17:01:13 +000091 // Initialize the pragma handlers.
Argyrios Kyrtzidis9b36c3f2010-07-13 09:07:17 +000092 PragmaHandlers = new PragmaNamespace(llvm::StringRef());
Reid Spencer5f016e22007-07-11 17:01:13 +000093 RegisterBuiltinPragmas();
Mike Stump1eb44332009-09-09 15:08:12 +000094
Reid Spencer5f016e22007-07-11 17:01:13 +000095 // Initialize builtin macros like __LINE__ and friends.
96 RegisterBuiltinMacros();
97}
98
99Preprocessor::~Preprocessor() {
Argyrios Kyrtzidis2174a4f2008-08-23 12:12:06 +0000100 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
101
Reid Spencer5f016e22007-07-11 17:01:13 +0000102 while (!IncludeMacroStack.empty()) {
103 delete IncludeMacroStack.back().TheLexer;
Chris Lattner6cfe7592008-03-09 02:26:03 +0000104 delete IncludeMacroStack.back().TheTokenLexer;
Reid Spencer5f016e22007-07-11 17:01:13 +0000105 IncludeMacroStack.pop_back();
106 }
Chris Lattnercc1a8752007-10-07 08:44:20 +0000107
108 // Free any macro definitions.
109 for (llvm::DenseMap<IdentifierInfo*, MacroInfo*>::iterator I =
110 Macros.begin(), E = Macros.end(); I != E; ++I) {
Ted Kremenek0ea76722008-12-15 19:56:42 +0000111 // We don't need to free the MacroInfo objects directly. These
112 // will be released when the BumpPtrAllocator 'BP' object gets
Chris Lattner2b9d6372009-12-14 22:02:43 +0000113 // destroyed. We still need to run the dtor, however, to free
Ted Kremenek9ee7d452009-01-19 07:45:44 +0000114 // memory alocated by MacroInfo.
Chris Lattner2c1ab902010-08-18 16:08:51 +0000115 I->second->Destroy();
Chris Lattnercc1a8752007-10-07 08:44:20 +0000116 I->first->setHasMacroDefinition(false);
117 }
Ted Kremenek83c1a6f2010-06-08 23:00:53 +0000118 for (std::vector<MacroInfo*>::iterator I = MICache.begin(),
119 E = MICache.end(); I != E; ++I) {
120 // We don't need to free the MacroInfo objects directly. These
121 // will be released when the BumpPtrAllocator 'BP' object gets
122 // destroyed. We still need to run the dtor, however, to free
123 // memory alocated by MacroInfo.
Chris Lattner2c1ab902010-08-18 16:08:51 +0000124 (*I)->Destroy();
Ted Kremenek83c1a6f2010-06-08 23:00:53 +0000125 }
Ted Kremeneke6a7dab2010-10-19 17:40:53 +0000126 for (llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> >::iterator I =
127 PragmaPushMacroInfo.begin(), E = PragmaPushMacroInfo.end(); I!=E; ++I){
128 for (std::vector<MacroInfo*>::iterator I2 = I->second.begin(), E2 = I->second.end();
129 I2 != E2; ++I2) {
130 (*I2)->Destroy();
131 }
132 }
Mike Stump1eb44332009-09-09 15:08:12 +0000133
Chris Lattner9594acf2007-07-15 00:25:26 +0000134 // Free any cached macro expanders.
Chris Lattner6cfe7592008-03-09 02:26:03 +0000135 for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
136 delete TokenLexerCache[i];
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000137
Chris Lattner23f77e52009-12-15 01:51:03 +0000138 // Free any cached MacroArgs.
139 for (MacroArgs *ArgList = MacroArgCache; ArgList; )
140 ArgList = ArgList->deallocate();
Mike Stump1eb44332009-09-09 15:08:12 +0000141
Reid Spencer5f016e22007-07-11 17:01:13 +0000142 // Release pragma information.
143 delete PragmaHandlers;
144
145 // Delete the scratch buffer info.
146 delete ScratchBuf;
Chris Lattnereb50ed82008-03-14 06:07:05 +0000147
Daniel Dunbar5814e652009-11-11 21:44:21 +0000148 // Delete the header search info, if we own it.
149 if (OwnsHeaderSearch)
150 delete &HeaderInfo;
151
Chris Lattnereb50ed82008-03-14 06:07:05 +0000152 delete Callbacks;
Reid Spencer5f016e22007-07-11 17:01:13 +0000153}
154
Ted Kremenek337edcd2009-02-12 03:26:59 +0000155void Preprocessor::setPTHManager(PTHManager* pm) {
156 PTH.reset(pm);
Douglas Gregor52e71082009-10-16 18:18:30 +0000157 FileMgr.addStatCache(PTH->createStatCache());
Ted Kremenek337edcd2009-02-12 03:26:59 +0000158}
159
Chris Lattnerd2177732007-07-20 16:59:19 +0000160void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000161 llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
162 << getSpelling(Tok) << "'";
Mike Stump1eb44332009-09-09 15:08:12 +0000163
Reid Spencer5f016e22007-07-11 17:01:13 +0000164 if (!DumpFlags) return;
Mike Stump1eb44332009-09-09 15:08:12 +0000165
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000166 llvm::errs() << "\t";
Reid Spencer5f016e22007-07-11 17:01:13 +0000167 if (Tok.isAtStartOfLine())
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000168 llvm::errs() << " [StartOfLine]";
Reid Spencer5f016e22007-07-11 17:01:13 +0000169 if (Tok.hasLeadingSpace())
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000170 llvm::errs() << " [LeadingSpace]";
Reid Spencer5f016e22007-07-11 17:01:13 +0000171 if (Tok.isExpandDisabled())
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000172 llvm::errs() << " [ExpandDisabled]";
Reid Spencer5f016e22007-07-11 17:01:13 +0000173 if (Tok.needsCleaning()) {
174 const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
Benjamin Kramer476d8b82010-08-11 14:47:12 +0000175 llvm::errs() << " [UnClean='" << llvm::StringRef(Start, Tok.getLength())
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000176 << "']";
Reid Spencer5f016e22007-07-11 17:01:13 +0000177 }
Mike Stump1eb44332009-09-09 15:08:12 +0000178
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000179 llvm::errs() << "\tLoc=<";
Chris Lattnerc3d8d572007-12-09 20:31:55 +0000180 DumpLocation(Tok.getLocation());
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000181 llvm::errs() << ">";
Chris Lattnerc3d8d572007-12-09 20:31:55 +0000182}
183
184void Preprocessor::DumpLocation(SourceLocation Loc) const {
Chris Lattnerb9c3f962009-01-27 07:57:44 +0000185 Loc.dump(SourceMgr);
Reid Spencer5f016e22007-07-11 17:01:13 +0000186}
187
188void Preprocessor::DumpMacro(const MacroInfo &MI) const {
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000189 llvm::errs() << "MACRO: ";
Reid Spencer5f016e22007-07-11 17:01:13 +0000190 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
191 DumpToken(MI.getReplacementToken(i));
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000192 llvm::errs() << " ";
Reid Spencer5f016e22007-07-11 17:01:13 +0000193 }
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000194 llvm::errs() << "\n";
Reid Spencer5f016e22007-07-11 17:01:13 +0000195}
196
197void Preprocessor::PrintStats() {
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000198 llvm::errs() << "\n*** Preprocessor Stats:\n";
199 llvm::errs() << NumDirectives << " directives found:\n";
200 llvm::errs() << " " << NumDefined << " #define.\n";
201 llvm::errs() << " " << NumUndefined << " #undef.\n";
202 llvm::errs() << " #include/#include_next/#import:\n";
203 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
204 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
205 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
206 llvm::errs() << " " << NumElse << " #else/#elif.\n";
207 llvm::errs() << " " << NumEndif << " #endif.\n";
208 llvm::errs() << " " << NumPragma << " #pragma.\n";
209 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
Reid Spencer5f016e22007-07-11 17:01:13 +0000210
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000211 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
Ted Kremenekbdd30c22008-01-14 16:44:48 +0000212 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
213 << NumFastMacroExpanded << " on the fast path.\n";
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000214 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
Ted Kremenekbdd30c22008-01-14 16:44:48 +0000215 << " token paste (##) operations performed, "
216 << NumFastTokenPaste << " on the fast path.\n";
Reid Spencer5f016e22007-07-11 17:01:13 +0000217}
218
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000219Preprocessor::macro_iterator
220Preprocessor::macro_begin(bool IncludeExternalMacros) const {
221 if (IncludeExternalMacros && ExternalSource &&
Douglas Gregor88a35862010-01-04 19:18:44 +0000222 !ReadMacrosFromExternalSource) {
223 ReadMacrosFromExternalSource = true;
224 ExternalSource->ReadDefinedMacros();
225 }
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000226
227 return Macros.begin();
Douglas Gregor88a35862010-01-04 19:18:44 +0000228}
229
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000230Preprocessor::macro_iterator
231Preprocessor::macro_end(bool IncludeExternalMacros) const {
232 if (IncludeExternalMacros && ExternalSource &&
Douglas Gregor88a35862010-01-04 19:18:44 +0000233 !ReadMacrosFromExternalSource) {
234 ReadMacrosFromExternalSource = true;
235 ExternalSource->ReadDefinedMacros();
236 }
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000237
238 return Macros.end();
Douglas Gregor88a35862010-01-04 19:18:44 +0000239}
240
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000241bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
242 unsigned TruncateAtLine,
Douglas Gregor29684422009-12-02 06:49:09 +0000243 unsigned TruncateAtColumn) {
244 using llvm::MemoryBuffer;
245
246 CodeCompletionFile = File;
247
248 // Okay to clear out the code-completion point by passing NULL.
249 if (!CodeCompletionFile)
250 return false;
251
252 // Load the actual file's contents.
Douglas Gregoraa38c3d2010-03-16 19:49:24 +0000253 bool Invalid = false;
254 const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
255 if (Invalid)
Douglas Gregor29684422009-12-02 06:49:09 +0000256 return true;
257
258 // Find the byte position of the truncation point.
259 const char *Position = Buffer->getBufferStart();
260 for (unsigned Line = 1; Line < TruncateAtLine; ++Line) {
261 for (; *Position; ++Position) {
262 if (*Position != '\r' && *Position != '\n')
263 continue;
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000264
Douglas Gregor29684422009-12-02 06:49:09 +0000265 // Eat \r\n or \n\r as a single line.
266 if ((Position[1] == '\r' || Position[1] == '\n') &&
267 Position[0] != Position[1])
268 ++Position;
269 ++Position;
270 break;
271 }
272 }
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000273
Douglas Gregorb760fe82009-12-08 21:45:46 +0000274 Position += TruncateAtColumn - 1;
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000275
Douglas Gregor29684422009-12-02 06:49:09 +0000276 // Truncate the buffer.
Douglas Gregorb760fe82009-12-08 21:45:46 +0000277 if (Position < Buffer->getBufferEnd()) {
Chris Lattnera0a270c2010-04-05 22:42:27 +0000278 llvm::StringRef Data(Buffer->getBufferStart(),
279 Position-Buffer->getBufferStart());
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000280 MemoryBuffer *TruncatedBuffer
Chris Lattnera0a270c2010-04-05 22:42:27 +0000281 = MemoryBuffer::getMemBufferCopy(Data, Buffer->getBufferIdentifier());
Douglas Gregor29684422009-12-02 06:49:09 +0000282 SourceMgr.overrideFileContents(File, TruncatedBuffer);
283 }
284
285 return false;
286}
287
Douglas Gregor109ae732009-12-03 17:05:59 +0000288bool Preprocessor::isCodeCompletionFile(SourceLocation FileLoc) const {
Douglas Gregor29684422009-12-02 06:49:09 +0000289 return CodeCompletionFile && FileLoc.isFileID() &&
290 SourceMgr.getFileEntryForID(SourceMgr.getFileID(FileLoc))
291 == CodeCompletionFile;
292}
293
Douglas Gregor55817af2010-08-25 17:04:25 +0000294void Preprocessor::CodeCompleteNaturalLanguage() {
295 SetCodeCompletionPoint(0, 0, 0);
296 getDiagnostics().setSuppressAllDiagnostics(true);
297 if (CodeComplete)
298 CodeComplete->CodeCompleteNaturalLanguage();
299}
300
Reid Spencer5f016e22007-07-11 17:01:13 +0000301//===----------------------------------------------------------------------===//
302// Token Spelling
303//===----------------------------------------------------------------------===//
304
Reid Spencer5f016e22007-07-11 17:01:13 +0000305/// getSpelling() - Return the 'spelling' of this token. The spelling of a
306/// token are the characters used to represent the token in the source file
307/// after trigraph expansion and escaped-newline folding. In particular, this
308/// wants to get the true, uncanonicalized, spelling of things like digraphs
309/// UCNs, etc.
Daniel Dunbar0ff10422009-11-14 01:20:48 +0000310std::string Preprocessor::getSpelling(const Token &Tok,
311 const SourceManager &SourceMgr,
Douglas Gregor50f6af72010-03-16 05:20:39 +0000312 const LangOptions &Features,
313 bool *Invalid) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000314 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
Ted Kremenek277faca2009-01-27 00:01:05 +0000315
Reid Spencer5f016e22007-07-11 17:01:13 +0000316 // If this token contains nothing interesting, return it directly.
Douglas Gregor50f6af72010-03-16 05:20:39 +0000317 bool CharDataInvalid = false;
318 const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(),
319 &CharDataInvalid);
320 if (Invalid)
321 *Invalid = CharDataInvalid;
322 if (CharDataInvalid)
323 return std::string();
324
Reid Spencer5f016e22007-07-11 17:01:13 +0000325 if (!Tok.needsCleaning())
326 return std::string(TokStart, TokStart+Tok.getLength());
Mike Stump1eb44332009-09-09 15:08:12 +0000327
Reid Spencer5f016e22007-07-11 17:01:13 +0000328 std::string Result;
329 Result.reserve(Tok.getLength());
Mike Stump1eb44332009-09-09 15:08:12 +0000330
Reid Spencer5f016e22007-07-11 17:01:13 +0000331 // Otherwise, hard case, relex the characters into the string.
332 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
333 Ptr != End; ) {
334 unsigned CharSize;
335 Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
336 Ptr += CharSize;
337 }
338 assert(Result.size() != unsigned(Tok.getLength()) &&
339 "NeedsCleaning flag set on something that didn't need cleaning!");
340 return Result;
341}
342
Daniel Dunbar0ff10422009-11-14 01:20:48 +0000343/// getSpelling() - Return the 'spelling' of this token. The spelling of a
344/// token are the characters used to represent the token in the source file
345/// after trigraph expansion and escaped-newline folding. In particular, this
346/// wants to get the true, uncanonicalized, spelling of things like digraphs
347/// UCNs, etc.
Douglas Gregor50f6af72010-03-16 05:20:39 +0000348std::string Preprocessor::getSpelling(const Token &Tok, bool *Invalid) const {
349 return getSpelling(Tok, SourceMgr, Features, Invalid);
Daniel Dunbar0ff10422009-11-14 01:20:48 +0000350}
351
Reid Spencer5f016e22007-07-11 17:01:13 +0000352/// getSpelling - This method is used to get the spelling of a token into a
353/// preallocated buffer, instead of as an std::string. The caller is required
354/// to allocate enough space for the token, which is guaranteed to be at least
355/// Tok.getLength() bytes long. The actual length of the token is returned.
356///
357/// Note that this method may do two possible things: it may either fill in
358/// the buffer specified with characters, or it may *change the input pointer*
359/// to point to a constant buffer with the data already in it (avoiding a
360/// copy). The caller is not allowed to modify the returned buffer pointer
361/// if an internal buffer is returned.
Sean Hunt6cf75022010-08-30 17:47:05 +0000362unsigned Preprocessor::getSpelling(const Token &Tok,
363 const char *&Buffer, bool *Invalid) const {
Reid Spencer5f016e22007-07-11 17:01:13 +0000364 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
Mike Stump1eb44332009-09-09 15:08:12 +0000365
Reid Spencer5f016e22007-07-11 17:01:13 +0000366 // If this token is an identifier, just return the string from the identifier
367 // table, which is very quick.
368 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
Sean Hunt6cf75022010-08-30 17:47:05 +0000369 Buffer = II->getNameStart();
370 return II->getLength();
Reid Spencer5f016e22007-07-11 17:01:13 +0000371 }
Ted Kremenekb70e3da2009-01-08 02:47:16 +0000372
Reid Spencer5f016e22007-07-11 17:01:13 +0000373 // Otherwise, compute the start of the token in the input lexer buffer.
Chris Lattner47246be2009-01-26 19:29:26 +0000374 const char *TokStart = 0;
Mike Stump1eb44332009-09-09 15:08:12 +0000375
Chris Lattner47246be2009-01-26 19:29:26 +0000376 if (Tok.isLiteral())
377 TokStart = Tok.getLiteralData();
Mike Stump1eb44332009-09-09 15:08:12 +0000378
Douglas Gregor50f6af72010-03-16 05:20:39 +0000379 if (TokStart == 0) {
380 bool CharDataInvalid = false;
381 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
382 if (Invalid)
383 *Invalid = CharDataInvalid;
384 if (CharDataInvalid) {
385 Buffer = "";
386 return 0;
387 }
388 }
Reid Spencer5f016e22007-07-11 17:01:13 +0000389
390 // If this token contains nothing interesting, return it directly.
Sean Hunt6cf75022010-08-30 17:47:05 +0000391 if (!Tok.needsCleaning()) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000392 Buffer = TokStart;
Sean Hunt6cf75022010-08-30 17:47:05 +0000393 return Tok.getLength();
Reid Spencer5f016e22007-07-11 17:01:13 +0000394 }
Mike Stump1eb44332009-09-09 15:08:12 +0000395
Reid Spencer5f016e22007-07-11 17:01:13 +0000396 // Otherwise, hard case, relex the characters into the string.
397 char *OutBuf = const_cast<char*>(Buffer);
Sean Hunt6cf75022010-08-30 17:47:05 +0000398 for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
Reid Spencer5f016e22007-07-11 17:01:13 +0000399 Ptr != End; ) {
400 unsigned CharSize;
401 *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
402 Ptr += CharSize;
403 }
Sean Hunt6cf75022010-08-30 17:47:05 +0000404 assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
Reid Spencer5f016e22007-07-11 17:01:13 +0000405 "NeedsCleaning flag set on something that didn't need cleaning!");
Mike Stump1eb44332009-09-09 15:08:12 +0000406
Reid Spencer5f016e22007-07-11 17:01:13 +0000407 return OutBuf-Buffer;
408}
409
Benjamin Kramer51f5fe32010-02-27 17:05:45 +0000410/// getSpelling - This method is used to get the spelling of a token into a
411/// SmallVector. Note that the returned StringRef may not point to the
412/// supplied buffer if a copy can be avoided.
413llvm::StringRef Preprocessor::getSpelling(const Token &Tok,
Douglas Gregor50f6af72010-03-16 05:20:39 +0000414 llvm::SmallVectorImpl<char> &Buffer,
415 bool *Invalid) const {
Benjamin Kramer51f5fe32010-02-27 17:05:45 +0000416 // Try the fast path.
417 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
418 return II->getName();
419
420 // Resize the buffer if we need to copy into it.
421 if (Tok.needsCleaning())
422 Buffer.resize(Tok.getLength());
423
424 const char *Ptr = Buffer.data();
Douglas Gregor50f6af72010-03-16 05:20:39 +0000425 unsigned Len = getSpelling(Tok, Ptr, Invalid);
Benjamin Kramer51f5fe32010-02-27 17:05:45 +0000426 return llvm::StringRef(Ptr, Len);
427}
428
Reid Spencer5f016e22007-07-11 17:01:13 +0000429/// CreateString - Plop the specified string into a scratch buffer and return a
430/// location for it. If specified, the source location provides a source
431/// location for the token.
Chris Lattner47246be2009-01-26 19:29:26 +0000432void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
433 SourceLocation InstantiationLoc) {
434 Tok.setLength(Len);
Mike Stump1eb44332009-09-09 15:08:12 +0000435
Chris Lattner47246be2009-01-26 19:29:26 +0000436 const char *DestPtr;
437 SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
Mike Stump1eb44332009-09-09 15:08:12 +0000438
Chris Lattner47246be2009-01-26 19:29:26 +0000439 if (InstantiationLoc.isValid())
Chris Lattnere7fb4842009-02-15 20:52:18 +0000440 Loc = SourceMgr.createInstantiationLoc(Loc, InstantiationLoc,
441 InstantiationLoc, Len);
Chris Lattner47246be2009-01-26 19:29:26 +0000442 Tok.setLocation(Loc);
Mike Stump1eb44332009-09-09 15:08:12 +0000443
Chris Lattner47246be2009-01-26 19:29:26 +0000444 // If this is a literal token, set the pointer data.
445 if (Tok.isLiteral())
446 Tok.setLiteralData(DestPtr);
Reid Spencer5f016e22007-07-11 17:01:13 +0000447}
448
449
Chris Lattner97ba77c2007-07-16 06:48:38 +0000450/// AdvanceToTokenCharacter - Given a location that specifies the start of a
451/// token, return a new location that specifies a character within the token.
Mike Stump1eb44332009-09-09 15:08:12 +0000452SourceLocation Preprocessor::AdvanceToTokenCharacter(SourceLocation TokStart,
Chris Lattner97ba77c2007-07-16 06:48:38 +0000453 unsigned CharNo) {
Chris Lattnerf7cf85b2009-01-16 07:36:28 +0000454 // Figure out how many physical characters away the specified instantiation
Chris Lattner97ba77c2007-07-16 06:48:38 +0000455 // character is. This needs to take into consideration newlines and
456 // trigraphs.
Douglas Gregora5430162010-03-16 20:46:42 +0000457 bool Invalid = false;
458 const char *TokPtr = SourceMgr.getCharacterData(TokStart, &Invalid);
Mike Stump1eb44332009-09-09 15:08:12 +0000459
Chris Lattner88e25242009-04-18 22:28:58 +0000460 // If they request the first char of the token, we're trivially done.
Douglas Gregora5430162010-03-16 20:46:42 +0000461 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
Chris Lattner88e25242009-04-18 22:28:58 +0000462 return TokStart;
Mike Stump1eb44332009-09-09 15:08:12 +0000463
Chris Lattner9dc1f532007-07-20 16:37:10 +0000464 unsigned PhysOffset = 0;
Mike Stump1eb44332009-09-09 15:08:12 +0000465
Chris Lattner97ba77c2007-07-16 06:48:38 +0000466 // The usual case is that tokens don't contain anything interesting. Skip
467 // over the uninteresting characters. If a token only consists of simple
468 // chars, this method is extremely fast.
Chris Lattner88e25242009-04-18 22:28:58 +0000469 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
470 if (CharNo == 0)
471 return TokStart.getFileLocWithOffset(PhysOffset);
Chris Lattner9dc1f532007-07-20 16:37:10 +0000472 ++TokPtr, --CharNo, ++PhysOffset;
Chris Lattner88e25242009-04-18 22:28:58 +0000473 }
Mike Stump1eb44332009-09-09 15:08:12 +0000474
Chris Lattner28c90ad2009-01-17 07:57:25 +0000475 // If we have a character that may be a trigraph or escaped newline, use a
Chris Lattner97ba77c2007-07-16 06:48:38 +0000476 // lexer to parse it correctly.
Chris Lattner88e25242009-04-18 22:28:58 +0000477 for (; CharNo; --CharNo) {
478 unsigned Size;
479 Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features);
480 TokPtr += Size;
481 PhysOffset += Size;
Chris Lattner97ba77c2007-07-16 06:48:38 +0000482 }
Mike Stump1eb44332009-09-09 15:08:12 +0000483
Chris Lattner88e25242009-04-18 22:28:58 +0000484 // Final detail: if we end up on an escaped newline, we want to return the
485 // location of the actual byte of the token. For example foo\<newline>bar
486 // advanced by 3 should return the location of b, not of \\. One compounding
487 // detail of this is that the escape may be made by a trigraph.
488 if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
Ted Kremenekc0178e92010-01-29 19:38:24 +0000489 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
Mike Stump1eb44332009-09-09 15:08:12 +0000490
Chris Lattner9dc1f532007-07-20 16:37:10 +0000491 return TokStart.getFileLocWithOffset(PhysOffset);
Chris Lattner97ba77c2007-07-16 06:48:38 +0000492}
493
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000494SourceLocation Preprocessor::getLocForEndOfToken(SourceLocation Loc,
Douglas Gregor33e9abd2010-01-22 19:49:59 +0000495 unsigned Offset) {
Douglas Gregorb2fb6de2009-02-27 17:53:17 +0000496 if (Loc.isInvalid() || !Loc.isFileID())
497 return SourceLocation();
498
Chris Lattner2c78b872009-04-14 23:22:57 +0000499 unsigned Len = Lexer::MeasureTokenLength(Loc, getSourceManager(), Features);
Douglas Gregor33e9abd2010-01-22 19:49:59 +0000500 if (Len > Offset)
501 Len = Len - Offset;
502 else
503 return Loc;
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000504
Douglas Gregorb2fb6de2009-02-27 17:53:17 +0000505 return AdvanceToTokenCharacter(Loc, Len);
506}
507
508
Chris Lattner97ba77c2007-07-16 06:48:38 +0000509
Chris Lattner53b0dab2007-10-09 22:10:18 +0000510//===----------------------------------------------------------------------===//
511// Preprocessor Initialization Methods
512//===----------------------------------------------------------------------===//
513
Chris Lattner53b0dab2007-10-09 22:10:18 +0000514
515/// EnterMainSourceFile - Enter the specified FileID as the main source file,
Nate Begeman6b616022008-01-07 04:01:26 +0000516/// which implicitly adds the builtin defines etc.
Chris Lattnere127a0d2010-04-20 20:35:58 +0000517void Preprocessor::EnterMainSourceFile() {
Chris Lattner05db4272009-02-13 19:33:24 +0000518 // We do not allow the preprocessor to reenter the main file. Doing so will
519 // cause FileID's to accumulate information from both runs (e.g. #line
520 // information) and predefined macros aren't guaranteed to be set properly.
521 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
Chris Lattner2b2453a2009-01-17 06:22:33 +0000522 FileID MainFileID = SourceMgr.getMainFileID();
Mike Stump1eb44332009-09-09 15:08:12 +0000523
Chris Lattner53b0dab2007-10-09 22:10:18 +0000524 // Enter the main file source buffer.
Chris Lattnere127a0d2010-04-20 20:35:58 +0000525 EnterSourceFile(MainFileID, 0, SourceLocation());
Mike Stump1eb44332009-09-09 15:08:12 +0000526
Douglas Gregorf4f6c9d2010-07-26 21:36:20 +0000527 // If we've been asked to skip bytes in the main file (e.g., as part of a
528 // precompiled preamble), do so now.
529 if (SkipMainFilePreamble.first > 0)
530 CurLexer->SkipBytes(SkipMainFilePreamble.first,
531 SkipMainFilePreamble.second);
532
Chris Lattnerb2832982007-11-15 19:07:47 +0000533 // Tell the header info that the main file was entered. If the file is later
534 // #imported, it won't be re-entered.
Chris Lattner2b2453a2009-01-17 06:22:33 +0000535 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
Chris Lattnerb2832982007-11-15 19:07:47 +0000536 HeaderInfo.IncrementIncludeCount(FE);
Mike Stump1eb44332009-09-09 15:08:12 +0000537
Benjamin Kramerffd6e392009-12-31 15:33:09 +0000538 // Preprocess Predefines to populate the initial preprocessor state.
Mike Stump1eb44332009-09-09 15:08:12 +0000539 llvm::MemoryBuffer *SB =
Chris Lattnera0a270c2010-04-05 22:42:27 +0000540 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
Douglas Gregor043266b2010-08-26 14:07:34 +0000541 assert(SB && "Cannot create predefined source buffer");
Chris Lattner2b2453a2009-01-17 06:22:33 +0000542 FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
543 assert(!FID.isInvalid() && "Could not create FileID for predefines?");
Mike Stump1eb44332009-09-09 15:08:12 +0000544
Chris Lattner53b0dab2007-10-09 22:10:18 +0000545 // Start parsing the predefines.
Chris Lattnere127a0d2010-04-20 20:35:58 +0000546 EnterSourceFile(FID, 0, SourceLocation());
Chris Lattner53b0dab2007-10-09 22:10:18 +0000547}
Chris Lattner97ba77c2007-07-16 06:48:38 +0000548
Daniel Dunbardbd82092010-03-23 05:09:10 +0000549void Preprocessor::EndSourceFile() {
550 // Notify the client that we reached the end of the source file.
551 if (Callbacks)
552 Callbacks->EndOfMainFile();
553}
Reid Spencer5f016e22007-07-11 17:01:13 +0000554
555//===----------------------------------------------------------------------===//
556// Lexer Event Handling.
557//===----------------------------------------------------------------------===//
558
559/// LookUpIdentifierInfo - Given a tok::identifier token, look up the
560/// identifier information for the token and install it into the token.
Chris Lattnerd2177732007-07-20 16:59:19 +0000561IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier,
Daniel Dunbarc3222092009-11-05 01:53:52 +0000562 const char *BufPtr) const {
Chris Lattner22f6bbc2007-10-09 18:02:16 +0000563 assert(Identifier.is(tok::identifier) && "Not an identifier!");
Reid Spencer5f016e22007-07-11 17:01:13 +0000564 assert(Identifier.getIdentifierInfo() == 0 && "Identinfo already exists!");
Mike Stump1eb44332009-09-09 15:08:12 +0000565
Reid Spencer5f016e22007-07-11 17:01:13 +0000566 // Look up this token, see if it is a macro, or if it is a language keyword.
567 IdentifierInfo *II;
568 if (BufPtr && !Identifier.needsCleaning()) {
569 // No cleaning needed, just use the characters from the lexed buffer.
Daniel Dunbar3da736c2009-11-05 01:53:39 +0000570 II = getIdentifierInfo(llvm::StringRef(BufPtr, Identifier.getLength()));
Reid Spencer5f016e22007-07-11 17:01:13 +0000571 } else {
572 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000573 llvm::SmallString<64> IdentifierBuffer;
Benjamin Kramerddeea562010-02-27 13:44:12 +0000574 llvm::StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
575 II = getIdentifierInfo(CleanedStr);
Reid Spencer5f016e22007-07-11 17:01:13 +0000576 }
577 Identifier.setIdentifierInfo(II);
578 return II;
579}
580
581
582/// HandleIdentifier - This callback is invoked when the lexer reads an
583/// identifier. This callback looks up the identifier in the map and/or
584/// potentially macro expands it or turns it into a named token (like 'for').
Chris Lattner6a170eb2009-01-21 07:43:11 +0000585///
586/// Note that callers of this method are guarded by checking the
587/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
588/// IdentifierInfo methods that compute these properties will need to change to
589/// match.
Chris Lattnerd2177732007-07-20 16:59:19 +0000590void Preprocessor::HandleIdentifier(Token &Identifier) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000591 assert(Identifier.getIdentifierInfo() &&
592 "Can't handle identifiers without identifier info!");
Mike Stump1eb44332009-09-09 15:08:12 +0000593
Reid Spencer5f016e22007-07-11 17:01:13 +0000594 IdentifierInfo &II = *Identifier.getIdentifierInfo();
595
596 // If this identifier was poisoned, and if it was not produced from a macro
597 // expansion, emit an error.
Ted Kremenek1a531572008-11-19 22:43:49 +0000598 if (II.isPoisoned() && CurPPLexer) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000599 if (&II != Ident__VA_ARGS__) // We warn about __VA_ARGS__ with poisoning.
600 Diag(Identifier, diag::err_pp_used_poisoned_id);
601 else
602 Diag(Identifier, diag::ext_pp_bad_vaargs_use);
603 }
Mike Stump1eb44332009-09-09 15:08:12 +0000604
Reid Spencer5f016e22007-07-11 17:01:13 +0000605 // If this is a macro to be expanded, do it.
Chris Lattnercc1a8752007-10-07 08:44:20 +0000606 if (MacroInfo *MI = getMacroInfo(&II)) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000607 if (!DisableMacroExpansion && !Identifier.isExpandDisabled()) {
608 if (MI->isEnabled()) {
609 if (!HandleMacroExpandedIdentifier(Identifier, MI))
610 return;
611 } else {
612 // C99 6.10.3.4p2 says that a disabled macro may never again be
613 // expanded, even if it's in a context where it could be expanded in the
614 // future.
Chris Lattnerd2177732007-07-20 16:59:19 +0000615 Identifier.setFlag(Token::DisableExpand);
Reid Spencer5f016e22007-07-11 17:01:13 +0000616 }
617 }
Reid Spencer5f016e22007-07-11 17:01:13 +0000618 }
619
620 // C++ 2.11p2: If this is an alternative representation of a C++ operator,
621 // then we act as if it is the actual operator and not the textual
622 // representation of it.
Fariborz Jahanianafbc6812010-09-03 17:33:04 +0000623 if (II.isCPlusPlusOperatorKeyword())
Reid Spencer5f016e22007-07-11 17:01:13 +0000624 Identifier.setIdentifierInfo(0);
625
Reid Spencer5f016e22007-07-11 17:01:13 +0000626 // If this is an extension token, diagnose its use.
Steve Naroffb4eaf9c2008-09-02 18:50:17 +0000627 // We avoid diagnosing tokens that originate from macro definitions.
Eli Friedman2962f4d2009-04-28 03:59:15 +0000628 // FIXME: This warning is disabled in cases where it shouldn't be,
629 // like "#define TY typeof", "TY(1) x".
630 if (II.isExtensionToken() && !DisableMacroExpansion)
Reid Spencer5f016e22007-07-11 17:01:13 +0000631 Diag(Identifier, diag::ext_token_used);
632}
Douglas Gregor2e222532009-07-02 17:08:52 +0000633
634void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
635 assert(Handler && "NULL comment handler");
636 assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
637 CommentHandlers.end() && "Comment handler already registered");
638 CommentHandlers.push_back(Handler);
639}
640
641void Preprocessor::RemoveCommentHandler(CommentHandler *Handler) {
642 std::vector<CommentHandler *>::iterator Pos
643 = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
644 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
645 CommentHandlers.erase(Pos);
646}
647
Chris Lattner046c2272010-01-18 22:35:47 +0000648bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
649 bool AnyPendingTokens = false;
Douglas Gregor2e222532009-07-02 17:08:52 +0000650 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
651 HEnd = CommentHandlers.end();
Chris Lattner046c2272010-01-18 22:35:47 +0000652 H != HEnd; ++H) {
653 if ((*H)->HandleComment(*this, Comment))
654 AnyPendingTokens = true;
655 }
656 if (!AnyPendingTokens || getCommentRetentionState())
657 return false;
658 Lex(result);
659 return true;
Douglas Gregor2e222532009-07-02 17:08:52 +0000660}
661
662CommentHandler::~CommentHandler() { }
Douglas Gregor94dc8f62010-03-19 16:15:56 +0000663
Douglas Gregorf44e8542010-08-24 19:08:16 +0000664CodeCompletionHandler::~CodeCompletionHandler() { }
665
Douglas Gregor94dc8f62010-03-19 16:15:56 +0000666void Preprocessor::createPreprocessingRecord() {
667 if (Record)
668 return;
669
Douglas Gregorb9e1b752010-03-19 17:12:43 +0000670 Record = new PreprocessingRecord;
671 addPPCallbacks(Record);
Douglas Gregor94dc8f62010-03-19 16:15:56 +0000672}