Modules: Use hash of PCM content for SIGNATURE
Change ASTFileSignature from a random 32-bit number to the hash of the
PCM content.
- Move definition ASTFileSignature to Basic/Module.h so Module and
ASTSourceDescriptor can use it.
- Change the signature from uint64_t to std::array<uint32_t,5>.
- Stop using (saving/reading) the size and modification time of PCM
files when there is a valid SIGNATURE.
- Add UNHASHED_CONTROL_BLOCK, and use it to store the SIGNATURE record
and other records that shouldn't affect the hash. Because implicit
modules reuses the same file for multiple levels of -Werror, this
includes DIAGNOSTIC_OPTIONS and DIAG_PRAGMA_MAPPINGS.
This helps to solve a PCH + implicit Modules dependency issue: PCH files
are handled by the external build system, whereas implicit modules are
handled by internal compiler build system. This prevents invalidating a
PCH when the compiler overwrites a PCM file with the same content
(modulo the diagnostic differences).
Design and original patch by Manman Ren!
llvm-svn: 297655
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index cdc1cf5..b13a4e1 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -18,8 +18,8 @@
#include "clang/AST/ASTContext.h"
#include "clang/AST/ASTUnresolvedSet.h"
#include "clang/AST/Decl.h"
-#include "clang/AST/DeclContextInternals.h"
#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclContextInternals.h"
#include "clang/AST/DeclFriend.h"
#include "clang/AST/DeclTemplate.h"
#include "clang/AST/Expr.h"
@@ -33,8 +33,8 @@
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/FileSystemOptions.h"
-#include "clang/Basic/LangOptions.h"
#include "clang/Basic/LLVM.h"
+#include "clang/Basic/LangOptions.h"
#include "clang/Basic/Module.h"
#include "clang/Basic/ObjCRuntime.h"
#include "clang/Basic/SourceManager.h"
@@ -64,9 +64,9 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Bitcode/BitCodes.h"
#include "llvm/Bitcode/BitstreamWriter.h"
@@ -79,6 +79,7 @@
#include "llvm/Support/OnDiskHashTable.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
+#include "llvm/Support/SHA1.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
@@ -1016,7 +1017,6 @@
// Control Block.
BLOCK(CONTROL_BLOCK);
RECORD(METADATA);
- RECORD(SIGNATURE);
RECORD(MODULE_NAME);
RECORD(MODULE_DIRECTORY);
RECORD(MODULE_MAP_FILE);
@@ -1029,7 +1029,6 @@
BLOCK(OPTIONS_BLOCK);
RECORD(LANGUAGE_OPTIONS);
RECORD(TARGET_OPTIONS);
- RECORD(DIAGNOSTIC_OPTIONS);
RECORD(FILE_SYSTEM_OPTIONS);
RECORD(HEADER_SEARCH_OPTIONS);
RECORD(PREPROCESSOR_OPTIONS);
@@ -1065,7 +1064,6 @@
RECORD(UPDATE_VISIBLE);
RECORD(DECL_UPDATE_OFFSETS);
RECORD(DECL_UPDATES);
- RECORD(DIAG_PRAGMA_MAPPINGS);
RECORD(CUDA_SPECIAL_DECL_REFS);
RECORD(HEADER_SEARCH_TABLE);
RECORD(FP_PRAGMA_OPTIONS);
@@ -1258,6 +1256,11 @@
BLOCK(EXTENSION_BLOCK);
RECORD(EXTENSION_METADATA);
+ BLOCK(UNHASHED_CONTROL_BLOCK);
+ RECORD(SIGNATURE);
+ RECORD(DIAGNOSTIC_OPTIONS);
+ RECORD(DIAG_PRAGMA_MAPPINGS);
+
#undef RECORD
#undef BLOCK
Stream.ExitBlock();
@@ -1320,21 +1323,73 @@
return Filename + Pos;
}
-static ASTFileSignature getSignature() {
- while (true) {
- if (ASTFileSignature S = llvm::sys::Process::GetRandomNumber())
- return S;
- // Rely on GetRandomNumber to eventually return non-zero...
+ASTFileSignature ASTWriter::createSignature(StringRef Bytes) {
+ // Calculate the hash till start of UNHASHED_CONTROL_BLOCK.
+ llvm::SHA1 Hasher;
+ Hasher.update(ArrayRef<uint8_t>(Bytes.bytes_begin(), Bytes.size()));
+ auto Hash = Hasher.result();
+
+ // Convert to an array [5*i32].
+ ASTFileSignature Signature;
+ auto LShift = [&](unsigned char Val, unsigned Shift) {
+ return (uint32_t)Val << Shift;
+ };
+ for (int I = 0; I != 5; ++I)
+ Signature[I] = LShift(Hash[I * 4 + 0], 24) | LShift(Hash[I * 4 + 1], 16) |
+ LShift(Hash[I * 4 + 2], 8) | LShift(Hash[I * 4 + 3], 0);
+
+ return Signature;
+}
+
+ASTFileSignature ASTWriter::writeUnhashedControlBlock(Preprocessor &PP,
+ ASTContext &Context) {
+ // Flush first to prepare the PCM hash (signature).
+ Stream.FlushToWord();
+ auto StartOfUnhashedControl = Stream.GetCurrentBitNo() >> 3;
+
+ // Enter the block and prepare to write records.
+ RecordData Record;
+ Stream.EnterSubblock(UNHASHED_CONTROL_BLOCK_ID, 5);
+
+ // For implicit modules, write the hash of the PCM as its signature.
+ ASTFileSignature Signature;
+ if (WritingModule &&
+ PP.getHeaderSearchInfo().getHeaderSearchOpts().ModulesHashContent) {
+ Signature = createSignature(StringRef(Buffer.begin(), StartOfUnhashedControl));
+ Record.append(Signature.begin(), Signature.end());
+ Stream.EmitRecord(SIGNATURE, Record);
+ Record.clear();
}
+
+ // Diagnostic options.
+ const auto &Diags = Context.getDiagnostics();
+ const DiagnosticOptions &DiagOpts = Diags.getDiagnosticOptions();
+#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name);
+#define ENUM_DIAGOPT(Name, Type, Bits, Default) \
+ Record.push_back(static_cast<unsigned>(DiagOpts.get##Name()));
+#include "clang/Basic/DiagnosticOptions.def"
+ Record.push_back(DiagOpts.Warnings.size());
+ for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I)
+ AddString(DiagOpts.Warnings[I], Record);
+ Record.push_back(DiagOpts.Remarks.size());
+ for (unsigned I = 0, N = DiagOpts.Remarks.size(); I != N; ++I)
+ AddString(DiagOpts.Remarks[I], Record);
+ // Note: we don't serialize the log or serialization file names, because they
+ // are generally transient files and will almost always be overridden.
+ Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record);
+
+ // Write out the diagnostic/pragma mappings.
+ WritePragmaDiagnosticMappings(Diags, /* IsModule = */ WritingModule);
+
+ // Leave the options block.
+ Stream.ExitBlock();
+ return Signature;
}
/// \brief Write the control block.
-uint64_t ASTWriter::WriteControlBlock(Preprocessor &PP,
- ASTContext &Context,
- StringRef isysroot,
- const std::string &OutputFile) {
- ASTFileSignature Signature = 0;
-
+void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
+ StringRef isysroot,
+ const std::string &OutputFile) {
using namespace llvm;
Stream.EnterSubblock(CONTROL_BLOCK_ID, 5);
RecordData Record;
@@ -1362,15 +1417,6 @@
getClangFullRepositoryVersion());
}
if (WritingModule) {
- // For implicit modules we output a signature that we can use to ensure
- // duplicate module builds don't collide in the cache as their output order
- // is non-deterministic.
- // FIXME: Remove this when output is deterministic.
- if (Context.getLangOpts().ImplicitModules) {
- Signature = getSignature();
- RecordData::value_type Record[] = {Signature};
- Stream.EmitRecord(SIGNATURE, Record);
- }
// Module name
auto Abbrev = std::make_shared<BitCodeAbbrev>();
@@ -1443,9 +1489,15 @@
Record.push_back((unsigned)M.Kind); // FIXME: Stable encoding
AddSourceLocation(M.ImportLoc, Record);
- Record.push_back(M.File->getSize());
- Record.push_back(getTimestampForOutput(M.File));
- Record.push_back(M.Signature);
+
+ // If we have calculated signature, there is no need to store
+ // the size or timestamp.
+ Record.push_back(M.Signature ? 0 : M.File->getSize());
+ Record.push_back(M.Signature ? 0 : getTimestampForOutput(M.File));
+
+ for (auto I : M.Signature)
+ Record.push_back(I);
+
AddPath(M.FileName, Record);
}
Stream.EmitRecord(IMPORTS, Record);
@@ -1508,24 +1560,6 @@
}
Stream.EmitRecord(TARGET_OPTIONS, Record);
- // Diagnostic options.
- Record.clear();
- const DiagnosticOptions &DiagOpts
- = Context.getDiagnostics().getDiagnosticOptions();
-#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name);
-#define ENUM_DIAGOPT(Name, Type, Bits, Default) \
- Record.push_back(static_cast<unsigned>(DiagOpts.get##Name()));
-#include "clang/Basic/DiagnosticOptions.def"
- Record.push_back(DiagOpts.Warnings.size());
- for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I)
- AddString(DiagOpts.Warnings[I], Record);
- Record.push_back(DiagOpts.Remarks.size());
- for (unsigned I = 0, N = DiagOpts.Remarks.size(); I != N; ++I)
- AddString(DiagOpts.Remarks[I], Record);
- // Note: we don't serialize the log or serialization file names, because they
- // are generally transient files and will almost always be overridden.
- Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record);
-
// File system options.
Record.clear();
const FileSystemOptions &FSOpts =
@@ -1639,7 +1673,6 @@
PP.getHeaderSearchInfo().getHeaderSearchOpts(),
PP.getLangOpts().Modules);
Stream.ExitBlock();
- return Signature;
}
namespace {
@@ -4267,9 +4300,10 @@
}
ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream,
+ SmallVectorImpl<char> &Buffer,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool IncludeTimestamps)
- : Stream(Stream), IncludeTimestamps(IncludeTimestamps) {
+ : Stream(Stream), Buffer(Buffer), IncludeTimestamps(IncludeTimestamps) {
for (const auto &Ext : Extensions) {
if (auto Writer = Ext->createExtensionWriter(*this))
ModuleFileExtensionWriters.push_back(std::move(Writer));
@@ -4289,9 +4323,10 @@
return IncludeTimestamps ? E->getModificationTime() : 0;
}
-uint64_t ASTWriter::WriteAST(Sema &SemaRef, const std::string &OutputFile,
- Module *WritingModule, StringRef isysroot,
- bool hasErrors) {
+ASTFileSignature ASTWriter::WriteAST(Sema &SemaRef,
+ const std::string &OutputFile,
+ Module *WritingModule, StringRef isysroot,
+ bool hasErrors) {
WritingAST = true;
ASTHasCompilerErrors = hasErrors;
@@ -4327,9 +4362,9 @@
}
}
-uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
- const std::string &OutputFile,
- Module *WritingModule) {
+ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
+ const std::string &OutputFile,
+ Module *WritingModule) {
using namespace llvm;
bool isModule = WritingModule != nullptr;
@@ -4477,7 +4512,7 @@
}
// Write the control block
- uint64_t Signature = WriteControlBlock(PP, Context, isysroot, OutputFile);
+ WriteControlBlock(PP, Context, isysroot, OutputFile);
// Write the remaining AST contents.
Stream.EnterSubblock(AST_BLOCK_ID, 5);
@@ -4694,7 +4729,6 @@
WriteOpenCLExtensionTypes(SemaRef);
WriteOpenCLExtensionDecls(SemaRef);
WriteCUDAPragmas(SemaRef);
- WritePragmaDiagnosticMappings(Context.getDiagnostics(), isModule);
// If we're emitting a module, write out the submodule information.
if (WritingModule)
@@ -4823,7 +4857,7 @@
for (const auto &ExtWriter : ModuleFileExtensionWriters)
WriteModuleFileExtension(SemaRef, *ExtWriter);
- return Signature;
+ return writeUnhashedControlBlock(PP, Context);
}
void ASTWriter::WriteDeclUpdatesBlocks(RecordDataImpl &OffsetsRecord) {