Introduce basic support for loading a precompiled preamble while
reparsing an ASTUnit. When saving a preamble, create a buffer larger
than the actual file we're working with but fill everything from the
end of the preamble to the end of the file with spaces (so the lexer
will quickly skip them). When we load the file, create a buffer of the
same size, filling it with the file and then spaces. Then, instruct
the lexer to start lexing after the preamble, therefore continuing the
parse from the spot where the preamble left off.
It's now possible to perform a simple preamble build + parse (+
reparse) with ASTUnit. However, one has to disable a bunch of checking
in the PCH reader to do so. That part isn't committed; it will likely
be handled with some other kind of flag (e.g., -fno-validate-pch).
As part of this, fix some issues with null termination of the memory
buffers created for the preamble; we were trying to explicitly
NULL-terminate them, even though they were also getting implicitly
NULL terminated, leading to excess warnings about NULL characters in
source files.
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@109445 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index 77a4147..7ff3cdc 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -46,6 +46,20 @@
CleanTemporaryFiles();
if (!PreambleFile.empty())
PreambleFile.eraseFromDisk();
+
+ // Free the buffers associated with remapped files. We are required to
+ // perform this operation here because we explicitly request that the
+ // compiler instance *not* free these buffers for each invocation of the
+ // parser.
+ if (Invocation.get()) {
+ PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts();
+ for (PreprocessorOptions::remapped_file_buffer_iterator
+ FB = PPOpts.remapped_file_buffer_begin(),
+ FBEnd = PPOpts.remapped_file_buffer_end();
+ FB != FBEnd;
+ ++FB)
+ delete FB->second;
+ }
}
void ASTUnit::CleanTemporaryFiles() {
@@ -371,6 +385,17 @@
// Create the source manager.
Clang.setSourceManager(&getSourceManager());
+ // If the main file has been overridden due to the use of a preamble,
+ // make that override happen and introduce the preamble.
+ PreprocessorOptions &PreprocessorOpts = Clang.getPreprocessorOpts();
+ if (OverrideMainBuffer) {
+ PreprocessorOpts.addRemappedFile(OriginalSourceFile, OverrideMainBuffer);
+ PreprocessorOpts.PrecompiledPreambleBytes.first = Preamble.size();
+ PreprocessorOpts.PrecompiledPreambleBytes.second
+ = PreambleEndsAtStartOfLine;
+ PreprocessorOpts.ImplicitPCHInclude = PreambleFile.str();
+ }
+
llvm::OwningPtr<TopLevelDeclTrackerAction> Act;
Act.reset(new TopLevelDeclTrackerAction(*this));
if (!Act->BeginSourceFile(Clang, Clang.getFrontendOpts().Inputs[0].second,
@@ -388,6 +413,11 @@
Target.reset(Clang.takeTarget());
Act->EndSourceFile();
+
+ // Remove the overridden buffer we used for the preamble.
+ if (OverrideMainBuffer)
+ PreprocessorOpts.eraseRemappedFile(
+ PreprocessorOpts.remapped_file_buffer_end() - 1);
Clang.takeDiagnosticClient();
@@ -395,6 +425,11 @@
return false;
error:
+ // Remove the overridden buffer we used for the preamble.
+ if (OverrideMainBuffer)
+ PreprocessorOpts.eraseRemappedFile(
+ PreprocessorOpts.remapped_file_buffer_end() - 1);
+
Clang.takeSourceManager();
Clang.takeFileManager();
Clang.takeDiagnosticClient();
@@ -424,8 +459,10 @@
return P.str();
}
-/// \brief Compute the preamble for the main file, providing
-std::pair<llvm::MemoryBuffer *, unsigned>
+/// \brief Compute the preamble for the main file, providing the source buffer
+/// that corresponds to the main file along with a pair (bytes, start-of-line)
+/// that describes the preamble.
+std::pair<llvm::MemoryBuffer *, std::pair<unsigned, bool> >
ASTUnit::ComputePreamble(CompilerInvocation &Invocation, bool &CreatedBuffer) {
FrontendOptions &FrontendOpts = Invocation.getFrontendOpts();
PreprocessorOptions &PreprocessorOpts
@@ -455,7 +492,8 @@
Buffer = llvm::MemoryBuffer::getFile(M->second);
if (!Buffer)
- return std::make_pair((llvm::MemoryBuffer*)0, 0);
+ return std::make_pair((llvm::MemoryBuffer*)0,
+ std::make_pair(0, true));
CreatedBuffer = true;
// Remove this remapping. We've captured the buffer already.
@@ -495,7 +533,7 @@
if (!Buffer) {
Buffer = llvm::MemoryBuffer::getFile(FrontendOpts.Inputs[0].second);
if (!Buffer)
- return std::make_pair((llvm::MemoryBuffer*)0, 0);
+ return std::make_pair((llvm::MemoryBuffer*)0, std::make_pair(0, true));
CreatedBuffer = true;
}
@@ -512,9 +550,8 @@
memcpy(const_cast<char*>(Result->getBufferStart()),
Old->getBufferStart(), Old->getBufferSize());
memset(const_cast<char*>(Result->getBufferStart()) + Old->getBufferSize(),
- ' ', NewSize - Old->getBufferSize() - 2);
- const_cast<char*>(Result->getBufferEnd())[-2] = '\n';
- const_cast<char*>(Result->getBufferEnd())[-1] = 0;
+ ' ', NewSize - Old->getBufferSize() - 1);
+ const_cast<char*>(Result->getBufferEnd())[-1] = '\n';
if (DeleteOld)
delete Old;
@@ -542,10 +579,10 @@
= PreambleInvocation.getPreprocessorOpts();
bool CreatedPreambleBuffer = false;
- std::pair<llvm::MemoryBuffer *, unsigned> NewPreamble
+ std::pair<llvm::MemoryBuffer *, std::pair<unsigned, bool> > NewPreamble
= ComputePreamble(PreambleInvocation, CreatedPreambleBuffer);
- if (!NewPreamble.second) {
+ if (!NewPreamble.second.first) {
// We couldn't find a preamble in the main source. Clear out the current
// preamble, if we have one. It's obviously no good any more.
Preamble.clear();
@@ -564,10 +601,11 @@
// preamble now that we did before, and that there's enough space in
// the main-file buffer within the precompiled preamble to fit the
// new main file.
- if (Preamble.size() == NewPreamble.second &&
+ if (Preamble.size() == NewPreamble.second.first &&
+ PreambleEndsAtStartOfLine == NewPreamble.second.second &&
NewPreamble.first->getBufferSize() < PreambleReservedSize-2 &&
memcmp(&Preamble[0], NewPreamble.first->getBufferStart(),
- NewPreamble.second) == 0) {
+ NewPreamble.second.first) == 0) {
// The preamble has not changed. We may be able to re-use the precompiled
// preamble.
// FIXME: Check that none of the files used by the preamble have changed.
@@ -593,7 +631,7 @@
// grows.
PreambleReservedSize = NewPreamble.first->getBufferSize();
if (PreambleReservedSize < 4096)
- PreambleReservedSize = 8192;
+ PreambleReservedSize = 8191;
else
PreambleReservedSize *= 2;
@@ -603,14 +641,15 @@
memcpy(const_cast<char*>(PreambleBuffer->getBufferStart()),
NewPreamble.first->getBufferStart(), Preamble.size());
memset(const_cast<char*>(PreambleBuffer->getBufferStart()) + Preamble.size(),
- ' ', PreambleReservedSize - Preamble.size() - 2);
- const_cast<char*>(PreambleBuffer->getBufferEnd())[-1] = 0;
- const_cast<char*>(PreambleBuffer->getBufferEnd())[-2] = '\n';
+ ' ', PreambleReservedSize - Preamble.size() - 1);
+ const_cast<char*>(PreambleBuffer->getBufferEnd())[-1] = '\n';
// Save the preamble text for later; we'll need to compare against it for
// subsequent reparses.
Preamble.assign(NewPreamble.first->getBufferStart(),
- NewPreamble.first->getBufferStart() + NewPreamble.second);
+ NewPreamble.first->getBufferStart()
+ + NewPreamble.second.first);
+ PreambleEndsAtStartOfLine = NewPreamble.second.second;
// Remap the main source file to the preamble buffer.
llvm::sys::PathWithStatus MainFilePath(FrontendOpts.Inputs[0].second);
@@ -734,6 +773,7 @@
AST->CaptureDiagnostics = CaptureDiagnostics;
AST->OnlyLocalDecls = OnlyLocalDecls;
AST->Invocation.reset(CI);
+ CI->getPreprocessorOpts().RetainRemappedFileBuffers = true;
llvm::MemoryBuffer *OverrideMainBuffer = 0;
if (PrecompilePreamble)
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index a0c7c7c..68703c2 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -1353,6 +1353,23 @@
Opts.TokenCache = Opts.ImplicitPTHInclude;
Opts.UsePredefines = !Args.hasArg(OPT_undef);
Opts.DetailedRecord = Args.hasArg(OPT_detailed_preprocessing_record);
+
+ if (const Arg *A = Args.getLastArg(OPT_preamble_bytes_EQ)) {
+ llvm::StringRef Value(A->getValue(Args));
+ size_t Comma = Value.find(',');
+ unsigned Bytes = 0;
+ unsigned EndOfLine = 0;
+
+ if (Comma == llvm::StringRef::npos ||
+ Value.substr(0, Comma).getAsInteger(10, Bytes) ||
+ Value.substr(Comma + 1).getAsInteger(10, EndOfLine))
+ Diags.Report(diag::err_drv_preamble_format);
+ else {
+ Opts.PrecompiledPreambleBytes.first = Bytes;
+ Opts.PrecompiledPreambleBytes.second = (EndOfLine != 0);
+ }
+ }
+
// Add macros from the command line.
for (arg_iterator it = Args.filtered_begin(OPT_D, OPT_U),
ie = Args.filtered_end(); it != ie; ++it) {
diff --git a/lib/Frontend/FrontendActions.cpp b/lib/Frontend/FrontendActions.cpp
index 2c0132a..2d1287f 100644
--- a/lib/Frontend/FrontendActions.cpp
+++ b/lib/Frontend/FrontendActions.cpp
@@ -195,7 +195,7 @@
llvm::MemoryBuffer *Buffer = llvm::MemoryBuffer::getFile(getCurrentFile());
if (Buffer) {
- unsigned Preamble = Lexer::ComputePreamble(Buffer);
+ unsigned Preamble = Lexer::ComputePreamble(Buffer).first;
llvm::outs().write(Buffer->getBufferStart(), Preamble);
delete Buffer;
}
diff --git a/lib/Frontend/InitPreprocessor.cpp b/lib/Frontend/InitPreprocessor.cpp
index 15d804f..9990fc4 100644
--- a/lib/Frontend/InitPreprocessor.cpp
+++ b/lib/Frontend/InitPreprocessor.cpp
@@ -489,13 +489,15 @@
if (!FromFile) {
Diags.Report(diag::err_fe_remap_missing_from_file)
<< Remap->first;
- delete Remap->second;
+ if (!InitOpts.RetainRemappedFileBuffers)
+ delete Remap->second;
continue;
}
// Override the contents of the "from" file with the contents of
// the "to" file.
- SourceMgr.overrideFileContents(FromFile, Remap->second);
+ SourceMgr.overrideFileContents(FromFile, Remap->second,
+ InitOpts.RetainRemappedFileBuffers);
}
// Remap files in the source manager (with other files).
@@ -596,6 +598,10 @@
if (!PP.getLangOptions().AsmPreprocessor)
Builder.append("# 1 \"<built-in>\" 2");
+ // Instruct the preprocessor to skip the preamble.
+ PP.setSkipMainFilePreamble(InitOpts.PrecompiledPreambleBytes.first,
+ InitOpts.PrecompiledPreambleBytes.second);
+
// Copy PredefinedBuffer into the Preprocessor.
PP.setPredefines(Predefines.str());