blob: b6653fc5752448abdc5f772d83a201bb2c257b77 [file] [log] [blame]
Reid Spencer5f016e22007-07-11 17:01:13 +00001//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattner0bc735f2007-12-29 19:59:25 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Reid Spencer5f016e22007-07-11 17:01:13 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the Preprocessor interface.
11//
12//===----------------------------------------------------------------------===//
13//
14// Options to support:
15// -H - Print the name of each header file used.
Chris Lattnerf73903a2009-02-06 06:45:26 +000016// -d[DNI] - Dump various things.
Reid Spencer5f016e22007-07-11 17:01:13 +000017// -fworking-directory - #line's with preprocessor's working dir.
18// -fpreprocessed
19// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
20// -W*
21// -w
22//
23// Messages to emit:
24// "Multiple include guards may be useful for:\n"
25//
26//===----------------------------------------------------------------------===//
27
28#include "clang/Lex/Preprocessor.h"
Chris Lattner23f77e52009-12-15 01:51:03 +000029#include "MacroArgs.h"
Douglas Gregor88a35862010-01-04 19:18:44 +000030#include "clang/Lex/ExternalPreprocessorSource.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000031#include "clang/Lex/HeaderSearch.h"
32#include "clang/Lex/MacroInfo.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000033#include "clang/Lex/Pragma.h"
Douglas Gregor94dc8f62010-03-19 16:15:56 +000034#include "clang/Lex/PreprocessingRecord.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000035#include "clang/Lex/ScratchBuffer.h"
Chris Lattner500d3292009-01-29 05:15:15 +000036#include "clang/Lex/LexDiagnostic.h"
Douglas Gregorf44e8542010-08-24 19:08:16 +000037#include "clang/Lex/CodeCompletionHandler.h"
Douglas Gregor6aa52ec2011-08-26 23:56:07 +000038#include "clang/Lex/ModuleLoader.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000039#include "clang/Basic/SourceManager.h"
Ted Kremenek337edcd2009-02-12 03:26:59 +000040#include "clang/Basic/FileManager.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000041#include "clang/Basic/TargetInfo.h"
Chris Lattner2db78dd2008-10-05 20:40:30 +000042#include "llvm/ADT/APFloat.h"
Benjamin Kramer8fe83e12012-02-04 13:45:25 +000043#include "llvm/ADT/SmallString.h"
Chris Lattner97ba77c2007-07-16 06:48:38 +000044#include "llvm/Support/MemoryBuffer.h"
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +000045#include "llvm/Support/raw_ostream.h"
Ted Kremenek67485092011-07-27 18:41:23 +000046#include "llvm/Support/Capacity.h"
Reid Spencer5f016e22007-07-11 17:01:13 +000047using namespace clang;
48
49//===----------------------------------------------------------------------===//
Douglas Gregor88a35862010-01-04 19:18:44 +000050ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
Reid Spencer5f016e22007-07-11 17:01:13 +000051
David Blaikied6471f72011-09-25 23:23:43 +000052Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
Douglas Gregor998b3d32011-09-01 23:39:15 +000053 const TargetInfo *target, SourceManager &SM,
Douglas Gregor6aa52ec2011-08-26 23:56:07 +000054 HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
Daniel Dunbar5814e652009-11-11 21:44:21 +000055 IdentifierInfoLookup* IILookup,
Douglas Gregor998b3d32011-09-01 23:39:15 +000056 bool OwnsHeaders,
57 bool DelayInitialization)
Chris Lattner836040f2009-03-13 21:17:43 +000058 : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
Douglas Gregor6aa52ec2011-08-26 23:56:07 +000059 SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
60 ExternalSource(0),
Douglas Gregor998b3d32011-09-01 23:39:15 +000061 Identifiers(opts, IILookup), CodeComplete(0),
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +000062 CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
63 SkipMainFilePreamble(0, true), CurPPLexer(0),
Zhongxing Xu7e7b5032011-09-26 03:37:43 +000064 CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0),
65 Record(0), MIChainHead(0), MICache(0)
Douglas Gregor998b3d32011-09-01 23:39:15 +000066{
Daniel Dunbar5814e652009-11-11 21:44:21 +000067 OwnsHeaderSearch = OwnsHeaders;
Douglas Gregor998b3d32011-09-01 23:39:15 +000068
69 if (!DelayInitialization) {
70 assert(Target && "Must provide target information for PP initialization");
71 Initialize(*Target);
John Wiegley28bbe4b2011-04-28 01:08:34 +000072 }
Reid Spencer5f016e22007-07-11 17:01:13 +000073}
74
75Preprocessor::~Preprocessor() {
Argyrios Kyrtzidis2174a4f2008-08-23 12:12:06 +000076 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
77
Reid Spencer5f016e22007-07-11 17:01:13 +000078 while (!IncludeMacroStack.empty()) {
79 delete IncludeMacroStack.back().TheLexer;
Chris Lattner6cfe7592008-03-09 02:26:03 +000080 delete IncludeMacroStack.back().TheTokenLexer;
Reid Spencer5f016e22007-07-11 17:01:13 +000081 IncludeMacroStack.pop_back();
82 }
Chris Lattnercc1a8752007-10-07 08:44:20 +000083
84 // Free any macro definitions.
Ted Kremenek2a6b03a2010-10-19 21:30:11 +000085 for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next)
Ted Kremenekaf8fa252010-10-19 18:16:54 +000086 I->MI.Destroy();
Mike Stump1eb44332009-09-09 15:08:12 +000087
Chris Lattner9594acf2007-07-15 00:25:26 +000088 // Free any cached macro expanders.
Chris Lattner6cfe7592008-03-09 02:26:03 +000089 for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
90 delete TokenLexerCache[i];
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +000091
Chris Lattner23f77e52009-12-15 01:51:03 +000092 // Free any cached MacroArgs.
93 for (MacroArgs *ArgList = MacroArgCache; ArgList; )
94 ArgList = ArgList->deallocate();
Mike Stump1eb44332009-09-09 15:08:12 +000095
Reid Spencer5f016e22007-07-11 17:01:13 +000096 // Release pragma information.
97 delete PragmaHandlers;
98
99 // Delete the scratch buffer info.
100 delete ScratchBuf;
Chris Lattnereb50ed82008-03-14 06:07:05 +0000101
Daniel Dunbar5814e652009-11-11 21:44:21 +0000102 // Delete the header search info, if we own it.
103 if (OwnsHeaderSearch)
104 delete &HeaderInfo;
105
Chris Lattnereb50ed82008-03-14 06:07:05 +0000106 delete Callbacks;
Reid Spencer5f016e22007-07-11 17:01:13 +0000107}
108
Douglas Gregor998b3d32011-09-01 23:39:15 +0000109void Preprocessor::Initialize(const TargetInfo &Target) {
110 assert((!this->Target || this->Target == &Target) &&
111 "Invalid override of target information");
112 this->Target = &Target;
113
114 // Initialize information about built-ins.
115 BuiltinInfo.InitializeTarget(Target);
116
117 ScratchBuf = new ScratchBuffer(SourceMgr);
118 CounterValue = 0; // __COUNTER__ starts at 0.
119
120 // Clear stats.
121 NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
122 NumIf = NumElse = NumEndif = 0;
123 NumEnteredSourceFiles = 0;
124 NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
125 NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
126 MaxIncludeStackDepth = 0;
127 NumSkipped = 0;
128
129 // Default to discarding comments.
130 KeepComments = false;
131 KeepMacroComments = false;
132 SuppressIncludeNotFoundError = false;
133
134 // Macro expansion is enabled.
135 DisableMacroExpansion = false;
136 InMacroArgs = false;
137 NumCachedTokenLexers = 0;
138
139 CachedLexPos = 0;
140
141 // We haven't read anything from the external source.
142 ReadMacrosFromExternalSource = false;
143
Douglas Gregor998b3d32011-09-01 23:39:15 +0000144 // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
145 // This gets unpoisoned where it is allowed.
146 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
147 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
148
149 // Initialize the pragma handlers.
150 PragmaHandlers = new PragmaNamespace(StringRef());
151 RegisterBuiltinPragmas();
152
153 // Initialize builtin macros like __LINE__ and friends.
154 RegisterBuiltinMacros();
155
156 if(Features.Borland) {
157 Ident__exception_info = getIdentifierInfo("_exception_info");
158 Ident___exception_info = getIdentifierInfo("__exception_info");
159 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
160 Ident__exception_code = getIdentifierInfo("_exception_code");
161 Ident___exception_code = getIdentifierInfo("__exception_code");
162 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
163 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
164 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
165 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
166 } else {
167 Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0;
168 Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0;
169 Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0;
Douglas Gregordc58aa72012-01-30 06:01:29 +0000170 }
171
172 HeaderInfo.setTarget(Target);
Douglas Gregor998b3d32011-09-01 23:39:15 +0000173}
174
Ted Kremenek337edcd2009-02-12 03:26:59 +0000175void Preprocessor::setPTHManager(PTHManager* pm) {
176 PTH.reset(pm);
Douglas Gregor52e71082009-10-16 18:18:30 +0000177 FileMgr.addStatCache(PTH->createStatCache());
Ted Kremenek337edcd2009-02-12 03:26:59 +0000178}
179
Chris Lattnerd2177732007-07-20 16:59:19 +0000180void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000181 llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
182 << getSpelling(Tok) << "'";
Mike Stump1eb44332009-09-09 15:08:12 +0000183
Reid Spencer5f016e22007-07-11 17:01:13 +0000184 if (!DumpFlags) return;
Mike Stump1eb44332009-09-09 15:08:12 +0000185
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000186 llvm::errs() << "\t";
Reid Spencer5f016e22007-07-11 17:01:13 +0000187 if (Tok.isAtStartOfLine())
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000188 llvm::errs() << " [StartOfLine]";
Reid Spencer5f016e22007-07-11 17:01:13 +0000189 if (Tok.hasLeadingSpace())
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000190 llvm::errs() << " [LeadingSpace]";
Reid Spencer5f016e22007-07-11 17:01:13 +0000191 if (Tok.isExpandDisabled())
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000192 llvm::errs() << " [ExpandDisabled]";
Reid Spencer5f016e22007-07-11 17:01:13 +0000193 if (Tok.needsCleaning()) {
194 const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
Chris Lattner5f9e2722011-07-23 10:55:15 +0000195 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000196 << "']";
Reid Spencer5f016e22007-07-11 17:01:13 +0000197 }
Mike Stump1eb44332009-09-09 15:08:12 +0000198
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000199 llvm::errs() << "\tLoc=<";
Chris Lattnerc3d8d572007-12-09 20:31:55 +0000200 DumpLocation(Tok.getLocation());
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000201 llvm::errs() << ">";
Chris Lattnerc3d8d572007-12-09 20:31:55 +0000202}
203
204void Preprocessor::DumpLocation(SourceLocation Loc) const {
Chris Lattnerb9c3f962009-01-27 07:57:44 +0000205 Loc.dump(SourceMgr);
Reid Spencer5f016e22007-07-11 17:01:13 +0000206}
207
208void Preprocessor::DumpMacro(const MacroInfo &MI) const {
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000209 llvm::errs() << "MACRO: ";
Reid Spencer5f016e22007-07-11 17:01:13 +0000210 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
211 DumpToken(MI.getReplacementToken(i));
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000212 llvm::errs() << " ";
Reid Spencer5f016e22007-07-11 17:01:13 +0000213 }
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000214 llvm::errs() << "\n";
Reid Spencer5f016e22007-07-11 17:01:13 +0000215}
216
217void Preprocessor::PrintStats() {
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000218 llvm::errs() << "\n*** Preprocessor Stats:\n";
219 llvm::errs() << NumDirectives << " directives found:\n";
220 llvm::errs() << " " << NumDefined << " #define.\n";
221 llvm::errs() << " " << NumUndefined << " #undef.\n";
222 llvm::errs() << " #include/#include_next/#import:\n";
223 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
224 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
225 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
226 llvm::errs() << " " << NumElse << " #else/#elif.\n";
227 llvm::errs() << " " << NumEndif << " #endif.\n";
228 llvm::errs() << " " << NumPragma << " #pragma.\n";
229 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
Reid Spencer5f016e22007-07-11 17:01:13 +0000230
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000231 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
Ted Kremenekbdd30c22008-01-14 16:44:48 +0000232 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
233 << NumFastMacroExpanded << " on the fast path.\n";
Benjamin Kramer6cb7c1a2009-08-23 12:08:50 +0000234 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
Ted Kremenekbdd30c22008-01-14 16:44:48 +0000235 << " token paste (##) operations performed, "
236 << NumFastTokenPaste << " on the fast path.\n";
Reid Spencer5f016e22007-07-11 17:01:13 +0000237}
238
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000239Preprocessor::macro_iterator
240Preprocessor::macro_begin(bool IncludeExternalMacros) const {
241 if (IncludeExternalMacros && ExternalSource &&
Douglas Gregor88a35862010-01-04 19:18:44 +0000242 !ReadMacrosFromExternalSource) {
243 ReadMacrosFromExternalSource = true;
244 ExternalSource->ReadDefinedMacros();
245 }
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000246
247 return Macros.begin();
Douglas Gregor88a35862010-01-04 19:18:44 +0000248}
249
Argyrios Kyrtzidisc5c5e922011-06-29 22:20:04 +0000250size_t Preprocessor::getTotalMemory() const {
Ted Kremenek91d1bd62011-07-26 21:17:24 +0000251 return BP.getTotalMemory()
Ted Kremenek67485092011-07-27 18:41:23 +0000252 + llvm::capacity_in_bytes(MacroExpandedTokens)
Ted Kremenek91d1bd62011-07-26 21:17:24 +0000253 + Predefines.capacity() /* Predefines buffer. */
Ted Kremenek67485092011-07-27 18:41:23 +0000254 + llvm::capacity_in_bytes(Macros)
255 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
256 + llvm::capacity_in_bytes(PoisonReasons)
257 + llvm::capacity_in_bytes(CommentHandlers);
Argyrios Kyrtzidisc5c5e922011-06-29 22:20:04 +0000258}
259
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000260Preprocessor::macro_iterator
261Preprocessor::macro_end(bool IncludeExternalMacros) const {
262 if (IncludeExternalMacros && ExternalSource &&
Douglas Gregor88a35862010-01-04 19:18:44 +0000263 !ReadMacrosFromExternalSource) {
264 ReadMacrosFromExternalSource = true;
265 ExternalSource->ReadDefinedMacros();
266 }
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000267
268 return Macros.end();
Douglas Gregor88a35862010-01-04 19:18:44 +0000269}
270
Douglas Gregord6aba062012-01-04 06:20:15 +0000271void Preprocessor::recomputeCurLexerKind() {
272 if (CurLexer)
273 CurLexerKind = CLK_Lexer;
274 else if (CurPTHLexer)
275 CurLexerKind = CLK_PTHLexer;
276 else if (CurTokenLexer)
277 CurLexerKind = CLK_TokenLexer;
278 else
279 CurLexerKind = CLK_CachingLexer;
280}
281
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000282bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +0000283 unsigned CompleteLine,
284 unsigned CompleteColumn) {
285 assert(File);
286 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
287 assert(!CodeCompletionFile && "Already set");
288
Douglas Gregor29684422009-12-02 06:49:09 +0000289 using llvm::MemoryBuffer;
290
Douglas Gregor29684422009-12-02 06:49:09 +0000291 // Load the actual file's contents.
Douglas Gregoraa38c3d2010-03-16 19:49:24 +0000292 bool Invalid = false;
293 const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
294 if (Invalid)
Douglas Gregor29684422009-12-02 06:49:09 +0000295 return true;
296
297 // Find the byte position of the truncation point.
298 const char *Position = Buffer->getBufferStart();
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +0000299 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
Douglas Gregor29684422009-12-02 06:49:09 +0000300 for (; *Position; ++Position) {
301 if (*Position != '\r' && *Position != '\n')
302 continue;
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000303
Douglas Gregor29684422009-12-02 06:49:09 +0000304 // Eat \r\n or \n\r as a single line.
305 if ((Position[1] == '\r' || Position[1] == '\n') &&
306 Position[0] != Position[1])
307 ++Position;
308 ++Position;
309 break;
310 }
311 }
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000312
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +0000313 Position += CompleteColumn - 1;
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000314
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +0000315 // Insert '\0' at the code-completion point.
Douglas Gregorb760fe82009-12-08 21:45:46 +0000316 if (Position < Buffer->getBufferEnd()) {
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +0000317 CodeCompletionFile = File;
318 CodeCompletionOffset = Position - Buffer->getBufferStart();
319
320 MemoryBuffer *NewBuffer =
321 MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
322 Buffer->getBufferIdentifier());
Benjamin Kramer41a50a92011-09-04 20:26:28 +0000323 char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +0000324 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
325 *NewPos = '\0';
326 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
327 SourceMgr.overrideFileContents(File, NewBuffer);
Douglas Gregor29684422009-12-02 06:49:09 +0000328 }
329
330 return false;
331}
332
Douglas Gregor55817af2010-08-25 17:04:25 +0000333void Preprocessor::CodeCompleteNaturalLanguage() {
Douglas Gregor55817af2010-08-25 17:04:25 +0000334 if (CodeComplete)
335 CodeComplete->CodeCompleteNaturalLanguage();
Argyrios Kyrtzidis7d100872011-09-04 03:32:15 +0000336 setCodeCompletionReached();
Douglas Gregor55817af2010-08-25 17:04:25 +0000337}
338
Benjamin Kramerfdd15602012-02-04 13:02:15 +0000339void Preprocessor::setCodeCompletionReached() {
340 assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
341 CodeCompletionReached = true;
342 // Silence any diagnostics that occur after we hit the code-completion.
343 getDiagnostics().setSuppressAllDiagnostics(true);
344}
345
346DiagnosticBuilder Preprocessor::Diag(SourceLocation Loc, unsigned DiagID) const{
347 return Diags->Report(Loc, DiagID);
348}
349
350DiagnosticBuilder Preprocessor::Diag(const Token &Tok, unsigned DiagID) const {
351 return Diags->Report(Tok.getLocation(), DiagID);
352}
353
Benjamin Kramer51f5fe32010-02-27 17:05:45 +0000354/// getSpelling - This method is used to get the spelling of a token into a
355/// SmallVector. Note that the returned StringRef may not point to the
356/// supplied buffer if a copy can be avoided.
Chris Lattner5f9e2722011-07-23 10:55:15 +0000357StringRef Preprocessor::getSpelling(const Token &Tok,
358 SmallVectorImpl<char> &Buffer,
Douglas Gregor50f6af72010-03-16 05:20:39 +0000359 bool *Invalid) const {
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000360 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
361 if (Tok.isNot(tok::raw_identifier)) {
362 // Try the fast path.
363 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
364 return II->getName();
365 }
Benjamin Kramer51f5fe32010-02-27 17:05:45 +0000366
367 // Resize the buffer if we need to copy into it.
368 if (Tok.needsCleaning())
369 Buffer.resize(Tok.getLength());
370
371 const char *Ptr = Buffer.data();
Douglas Gregor50f6af72010-03-16 05:20:39 +0000372 unsigned Len = getSpelling(Tok, Ptr, Invalid);
Chris Lattner5f9e2722011-07-23 10:55:15 +0000373 return StringRef(Ptr, Len);
Benjamin Kramer51f5fe32010-02-27 17:05:45 +0000374}
375
Reid Spencer5f016e22007-07-11 17:01:13 +0000376/// CreateString - Plop the specified string into a scratch buffer and return a
377/// location for it. If specified, the source location provides a source
378/// location for the token.
Chris Lattner47246be2009-01-26 19:29:26 +0000379void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
Abramo Bagnaraa08529c2011-10-03 18:39:03 +0000380 SourceLocation ExpansionLocStart,
381 SourceLocation ExpansionLocEnd) {
Chris Lattner47246be2009-01-26 19:29:26 +0000382 Tok.setLength(Len);
Mike Stump1eb44332009-09-09 15:08:12 +0000383
Chris Lattner47246be2009-01-26 19:29:26 +0000384 const char *DestPtr;
385 SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
Mike Stump1eb44332009-09-09 15:08:12 +0000386
Abramo Bagnaraa08529c2011-10-03 18:39:03 +0000387 if (ExpansionLocStart.isValid())
388 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
389 ExpansionLocEnd, Len);
Chris Lattner47246be2009-01-26 19:29:26 +0000390 Tok.setLocation(Loc);
Mike Stump1eb44332009-09-09 15:08:12 +0000391
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000392 // If this is a raw identifier or a literal token, set the pointer data.
393 if (Tok.is(tok::raw_identifier))
394 Tok.setRawIdentifierData(DestPtr);
395 else if (Tok.isLiteral())
Chris Lattner47246be2009-01-26 19:29:26 +0000396 Tok.setLiteralData(DestPtr);
Reid Spencer5f016e22007-07-11 17:01:13 +0000397}
398
Douglas Gregor90db2602011-12-02 01:47:07 +0000399Module *Preprocessor::getCurrentModule() {
400 if (getLangOptions().CurrentModule.empty())
401 return 0;
402
Douglas Gregore434ec72012-01-29 17:08:11 +0000403 return getHeaderSearchInfo().lookupModule(getLangOptions().CurrentModule);
Douglas Gregor90db2602011-12-02 01:47:07 +0000404}
Chris Lattner97ba77c2007-07-16 06:48:38 +0000405
Chris Lattner53b0dab2007-10-09 22:10:18 +0000406//===----------------------------------------------------------------------===//
407// Preprocessor Initialization Methods
408//===----------------------------------------------------------------------===//
409
Chris Lattner53b0dab2007-10-09 22:10:18 +0000410
411/// EnterMainSourceFile - Enter the specified FileID as the main source file,
Nate Begeman6b616022008-01-07 04:01:26 +0000412/// which implicitly adds the builtin defines etc.
Chris Lattnere127a0d2010-04-20 20:35:58 +0000413void Preprocessor::EnterMainSourceFile() {
Chris Lattner05db4272009-02-13 19:33:24 +0000414 // We do not allow the preprocessor to reenter the main file. Doing so will
415 // cause FileID's to accumulate information from both runs (e.g. #line
416 // information) and predefined macros aren't guaranteed to be set properly.
417 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
Chris Lattner2b2453a2009-01-17 06:22:33 +0000418 FileID MainFileID = SourceMgr.getMainFileID();
Mike Stump1eb44332009-09-09 15:08:12 +0000419
Argyrios Kyrtzidisb8c879a2012-01-05 21:36:25 +0000420 // If MainFileID is loaded it means we loaded an AST file, no need to enter
421 // a main file.
422 if (!SourceMgr.isLoadedFileID(MainFileID)) {
423 // Enter the main file source buffer.
424 EnterSourceFile(MainFileID, 0, SourceLocation());
Douglas Gregorf4f6c9d2010-07-26 21:36:20 +0000425
Argyrios Kyrtzidisb8c879a2012-01-05 21:36:25 +0000426 // If we've been asked to skip bytes in the main file (e.g., as part of a
427 // precompiled preamble), do so now.
428 if (SkipMainFilePreamble.first > 0)
429 CurLexer->SkipBytes(SkipMainFilePreamble.first,
430 SkipMainFilePreamble.second);
431
432 // Tell the header info that the main file was entered. If the file is later
433 // #imported, it won't be re-entered.
434 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
435 HeaderInfo.IncrementIncludeCount(FE);
436 }
Mike Stump1eb44332009-09-09 15:08:12 +0000437
Benjamin Kramerffd6e392009-12-31 15:33:09 +0000438 // Preprocess Predefines to populate the initial preprocessor state.
Mike Stump1eb44332009-09-09 15:08:12 +0000439 llvm::MemoryBuffer *SB =
Chris Lattnera0a270c2010-04-05 22:42:27 +0000440 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
Douglas Gregor043266b2010-08-26 14:07:34 +0000441 assert(SB && "Cannot create predefined source buffer");
Chris Lattner2b2453a2009-01-17 06:22:33 +0000442 FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
443 assert(!FID.isInvalid() && "Could not create FileID for predefines?");
Mike Stump1eb44332009-09-09 15:08:12 +0000444
Chris Lattner53b0dab2007-10-09 22:10:18 +0000445 // Start parsing the predefines.
Chris Lattnere127a0d2010-04-20 20:35:58 +0000446 EnterSourceFile(FID, 0, SourceLocation());
Chris Lattner53b0dab2007-10-09 22:10:18 +0000447}
Chris Lattner97ba77c2007-07-16 06:48:38 +0000448
Daniel Dunbardbd82092010-03-23 05:09:10 +0000449void Preprocessor::EndSourceFile() {
450 // Notify the client that we reached the end of the source file.
451 if (Callbacks)
452 Callbacks->EndOfMainFile();
453}
Reid Spencer5f016e22007-07-11 17:01:13 +0000454
455//===----------------------------------------------------------------------===//
456// Lexer Event Handling.
457//===----------------------------------------------------------------------===//
458
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000459/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
460/// identifier information for the token and install it into the token,
461/// updating the token kind accordingly.
462IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
463 assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!");
Mike Stump1eb44332009-09-09 15:08:12 +0000464
Reid Spencer5f016e22007-07-11 17:01:13 +0000465 // Look up this token, see if it is a macro, or if it is a language keyword.
466 IdentifierInfo *II;
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000467 if (!Identifier.needsCleaning()) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000468 // No cleaning needed, just use the characters from the lexed buffer.
Chris Lattner5f9e2722011-07-23 10:55:15 +0000469 II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(),
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000470 Identifier.getLength()));
Reid Spencer5f016e22007-07-11 17:01:13 +0000471 } else {
472 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
Kovarththanan Rajaratnam19357542010-03-13 10:17:05 +0000473 llvm::SmallString<64> IdentifierBuffer;
Chris Lattner5f9e2722011-07-23 10:55:15 +0000474 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
Benjamin Kramerddeea562010-02-27 13:44:12 +0000475 II = getIdentifierInfo(CleanedStr);
Reid Spencer5f016e22007-07-11 17:01:13 +0000476 }
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000477
478 // Update the token info (identifier info and appropriate token kind).
Reid Spencer5f016e22007-07-11 17:01:13 +0000479 Identifier.setIdentifierInfo(II);
Abramo Bagnarac4bf2b92010-12-22 08:23:18 +0000480 Identifier.setKind(II->getTokenID());
481
Reid Spencer5f016e22007-07-11 17:01:13 +0000482 return II;
483}
484
John Wiegley28bbe4b2011-04-28 01:08:34 +0000485void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
486 PoisonReasons[II] = DiagID;
487}
488
489void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
490 assert(Ident__exception_code && Ident__exception_info);
491 assert(Ident___exception_code && Ident___exception_info);
492 Ident__exception_code->setIsPoisoned(Poison);
493 Ident___exception_code->setIsPoisoned(Poison);
494 Ident_GetExceptionCode->setIsPoisoned(Poison);
495 Ident__exception_info->setIsPoisoned(Poison);
496 Ident___exception_info->setIsPoisoned(Poison);
497 Ident_GetExceptionInfo->setIsPoisoned(Poison);
498 Ident__abnormal_termination->setIsPoisoned(Poison);
499 Ident___abnormal_termination->setIsPoisoned(Poison);
500 Ident_AbnormalTermination->setIsPoisoned(Poison);
501}
502
503void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
504 assert(Identifier.getIdentifierInfo() &&
505 "Can't handle identifiers without identifier info!");
506 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
507 PoisonReasons.find(Identifier.getIdentifierInfo());
508 if(it == PoisonReasons.end())
509 Diag(Identifier, diag::err_pp_used_poisoned_id);
510 else
511 Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
512}
Reid Spencer5f016e22007-07-11 17:01:13 +0000513
514/// HandleIdentifier - This callback is invoked when the lexer reads an
515/// identifier. This callback looks up the identifier in the map and/or
516/// potentially macro expands it or turns it into a named token (like 'for').
Chris Lattner6a170eb2009-01-21 07:43:11 +0000517///
518/// Note that callers of this method are guarded by checking the
519/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
520/// IdentifierInfo methods that compute these properties will need to change to
521/// match.
Chris Lattnerd2177732007-07-20 16:59:19 +0000522void Preprocessor::HandleIdentifier(Token &Identifier) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000523 assert(Identifier.getIdentifierInfo() &&
524 "Can't handle identifiers without identifier info!");
Mike Stump1eb44332009-09-09 15:08:12 +0000525
Reid Spencer5f016e22007-07-11 17:01:13 +0000526 IdentifierInfo &II = *Identifier.getIdentifierInfo();
527
Douglas Gregoreee242f2011-10-27 09:33:13 +0000528 // If the information about this identifier is out of date, update it from
529 // the external source.
530 if (II.isOutOfDate()) {
531 ExternalSource->updateOutOfDateIdentifier(II);
532 Identifier.setKind(II.getTokenID());
533 }
534
Reid Spencer5f016e22007-07-11 17:01:13 +0000535 // If this identifier was poisoned, and if it was not produced from a macro
536 // expansion, emit an error.
Ted Kremenek1a531572008-11-19 22:43:49 +0000537 if (II.isPoisoned() && CurPPLexer) {
John Wiegley28bbe4b2011-04-28 01:08:34 +0000538 HandlePoisonedIdentifier(Identifier);
Reid Spencer5f016e22007-07-11 17:01:13 +0000539 }
Mike Stump1eb44332009-09-09 15:08:12 +0000540
Reid Spencer5f016e22007-07-11 17:01:13 +0000541 // If this is a macro to be expanded, do it.
Chris Lattnercc1a8752007-10-07 08:44:20 +0000542 if (MacroInfo *MI = getMacroInfo(&II)) {
Abramo Bagnara163ada82012-01-01 22:01:04 +0000543 if (!DisableMacroExpansion) {
544 if (Identifier.isExpandDisabled()) {
545 Diag(Identifier, diag::pp_disabled_macro_expansion);
546 } else if (MI->isEnabled()) {
Reid Spencer5f016e22007-07-11 17:01:13 +0000547 if (!HandleMacroExpandedIdentifier(Identifier, MI))
Douglas Gregor6be16fe2011-08-27 06:37:51 +0000548 return;
Reid Spencer5f016e22007-07-11 17:01:13 +0000549 } else {
550 // C99 6.10.3.4p2 says that a disabled macro may never again be
551 // expanded, even if it's in a context where it could be expanded in the
552 // future.
Chris Lattnerd2177732007-07-20 16:59:19 +0000553 Identifier.setFlag(Token::DisableExpand);
Abramo Bagnara163ada82012-01-01 22:01:04 +0000554 Diag(Identifier, diag::pp_disabled_macro_expansion);
Reid Spencer5f016e22007-07-11 17:01:13 +0000555 }
556 }
Reid Spencer5f016e22007-07-11 17:01:13 +0000557 }
558
Richard Smith98d86b92011-10-11 19:57:52 +0000559 // If this identifier is a keyword in C++11, produce a warning. Don't warn if
560 // we're not considering macro expansion, since this identifier might be the
561 // name of a macro.
562 // FIXME: This warning is disabled in cases where it shouldn't be, like
563 // "#define constexpr constexpr", "int constexpr;"
564 if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) {
565 Diag(Identifier, diag::warn_cxx11_keyword) << II.getName();
566 // Don't diagnose this keyword again in this translation unit.
567 II.setIsCXX11CompatKeyword(false);
568 }
569
Reid Spencer5f016e22007-07-11 17:01:13 +0000570 // C++ 2.11p2: If this is an alternative representation of a C++ operator,
571 // then we act as if it is the actual operator and not the textual
572 // representation of it.
Fariborz Jahanianafbc6812010-09-03 17:33:04 +0000573 if (II.isCPlusPlusOperatorKeyword())
Reid Spencer5f016e22007-07-11 17:01:13 +0000574 Identifier.setIdentifierInfo(0);
575
Reid Spencer5f016e22007-07-11 17:01:13 +0000576 // If this is an extension token, diagnose its use.
Steve Naroffb4eaf9c2008-09-02 18:50:17 +0000577 // We avoid diagnosing tokens that originate from macro definitions.
Eli Friedman2962f4d2009-04-28 03:59:15 +0000578 // FIXME: This warning is disabled in cases where it shouldn't be,
579 // like "#define TY typeof", "TY(1) x".
580 if (II.isExtensionToken() && !DisableMacroExpansion)
Reid Spencer5f016e22007-07-11 17:01:13 +0000581 Diag(Identifier, diag::ext_token_used);
Douglas Gregorb8db7cd2011-09-07 23:11:54 +0000582
Douglas Gregorc13a34b2012-01-03 19:32:59 +0000583 // If this is the 'import' contextual keyword, note that the next token
584 // indicates a module name.
Douglas Gregord6aba062012-01-04 06:20:15 +0000585 //
586 // Note that we do not treat 'import' as a contextual keyword when we're
587 // in a caching lexer, because caching lexers only get used in contexts where
588 // import declarations are disallowed.
589 if (II.isImport() && !InMacroArgs && !DisableMacroExpansion &&
590 getLangOptions().Modules && CurLexerKind != CLK_CachingLexer) {
Douglas Gregorb8db7cd2011-09-07 23:11:54 +0000591 ModuleImportLoc = Identifier.getLocation();
Douglas Gregorb514c792011-11-30 04:26:53 +0000592 ModuleImportPath.clear();
593 ModuleImportExpectsIdentifier = true;
Douglas Gregorb8db7cd2011-09-07 23:11:54 +0000594 CurLexerKind = CLK_LexAfterModuleImport;
595 }
Douglas Gregor6aa52ec2011-08-26 23:56:07 +0000596}
597
Douglas Gregorc13a34b2012-01-03 19:32:59 +0000598/// \brief Lex a token following the 'import' contextual keyword.
Douglas Gregor5948ae12012-01-03 18:04:46 +0000599///
Douglas Gregorb8db7cd2011-09-07 23:11:54 +0000600void Preprocessor::LexAfterModuleImport(Token &Result) {
601 // Figure out what kind of lexer we actually have.
Douglas Gregord6aba062012-01-04 06:20:15 +0000602 recomputeCurLexerKind();
Douglas Gregorb8db7cd2011-09-07 23:11:54 +0000603
604 // Lex the next token.
605 Lex(Result);
606
Douglas Gregor6aa52ec2011-08-26 23:56:07 +0000607 // The token sequence
608 //
Douglas Gregor5948ae12012-01-03 18:04:46 +0000609 // import identifier (. identifier)*
610 //
Douglas Gregorc13a34b2012-01-03 19:32:59 +0000611 // indicates a module import directive. We already saw the 'import'
612 // contextual keyword, so now we're looking for the identifiers.
Douglas Gregorb514c792011-11-30 04:26:53 +0000613 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
614 // We expected to see an identifier here, and we did; continue handling
615 // identifiers.
616 ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
617 Result.getLocation()));
618 ModuleImportExpectsIdentifier = false;
619 CurLexerKind = CLK_LexAfterModuleImport;
Douglas Gregor6aa52ec2011-08-26 23:56:07 +0000620 return;
Douglas Gregorb514c792011-11-30 04:26:53 +0000621 }
Douglas Gregor6aa52ec2011-08-26 23:56:07 +0000622
Douglas Gregorb514c792011-11-30 04:26:53 +0000623 // If we're expecting a '.' or a ';', and we got a '.', then wait until we
624 // see the next identifier.
625 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
626 ModuleImportExpectsIdentifier = true;
627 CurLexerKind = CLK_LexAfterModuleImport;
628 return;
629 }
630
631 // If we have a non-empty module path, load the named module.
632 if (!ModuleImportPath.empty())
Douglas Gregor5e356932011-12-01 17:11:21 +0000633 (void)TheModuleLoader.loadModule(ModuleImportLoc, ModuleImportPath,
Douglas Gregor93ebfa62011-12-02 23:42:12 +0000634 Module::MacrosVisible,
635 /*IsIncludeDirective=*/false);
Reid Spencer5f016e22007-07-11 17:01:13 +0000636}
Douglas Gregor2e222532009-07-02 17:08:52 +0000637
638void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
639 assert(Handler && "NULL comment handler");
640 assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
641 CommentHandlers.end() && "Comment handler already registered");
642 CommentHandlers.push_back(Handler);
643}
644
645void Preprocessor::RemoveCommentHandler(CommentHandler *Handler) {
646 std::vector<CommentHandler *>::iterator Pos
647 = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
648 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
649 CommentHandlers.erase(Pos);
650}
651
Chris Lattner046c2272010-01-18 22:35:47 +0000652bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
653 bool AnyPendingTokens = false;
Douglas Gregor2e222532009-07-02 17:08:52 +0000654 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
655 HEnd = CommentHandlers.end();
Chris Lattner046c2272010-01-18 22:35:47 +0000656 H != HEnd; ++H) {
657 if ((*H)->HandleComment(*this, Comment))
658 AnyPendingTokens = true;
659 }
660 if (!AnyPendingTokens || getCommentRetentionState())
661 return false;
662 Lex(result);
663 return true;
Douglas Gregor2e222532009-07-02 17:08:52 +0000664}
665
Douglas Gregor6aa52ec2011-08-26 23:56:07 +0000666ModuleLoader::~ModuleLoader() { }
667
Douglas Gregor2e222532009-07-02 17:08:52 +0000668CommentHandler::~CommentHandler() { }
Douglas Gregor94dc8f62010-03-19 16:15:56 +0000669
Douglas Gregorf44e8542010-08-24 19:08:16 +0000670CodeCompletionHandler::~CodeCompletionHandler() { }
671
Douglas Gregordca8ee82011-05-06 16:33:08 +0000672void Preprocessor::createPreprocessingRecord(
Chandler Carruth9e5bb852011-07-14 08:20:46 +0000673 bool IncludeNestedMacroExpansions) {
Douglas Gregor94dc8f62010-03-19 16:15:56 +0000674 if (Record)
675 return;
676
Argyrios Kyrtzidis2dbaca72011-09-19 20:40:25 +0000677 Record = new PreprocessingRecord(getSourceManager(),
678 IncludeNestedMacroExpansions);
Douglas Gregorb9e1b752010-03-19 17:12:43 +0000679 addPPCallbacks(Record);
Douglas Gregor94dc8f62010-03-19 16:15:56 +0000680}