Add a module Hash in the bitcode and the combined index, implementing a kind of "build-id"
This is intended to be used for ThinLTO incremental build.
Differential Revision: http://reviews.llvm.org/D18213
This is a recommit of r265095 after fixing the Windows issues.
From: Mehdi Amini <mehdi.amini@apple.com>
llvm-svn: 265111
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 93496fe..1840b60 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -5632,11 +5632,7 @@
}
continue;
- case BitstreamEntry::Record:
- // Once we find the last record of interest, skip the rest.
- if (VSTOffset > 0)
- Stream.skipRecord(Entry.ID);
- else {
+ case BitstreamEntry::Record: {
Record.clear();
auto BitCode = Stream.readRecord(Entry.ID, Record);
switch (BitCode) {
@@ -5650,6 +5646,25 @@
SourceFileName = ValueName.c_str();
break;
}
+ /// MODULE_CODE_HASH: [5*i32]
+ case bitc::MODULE_CODE_HASH: {
+ if (Record.size() != 5)
+ return error("Invalid hash length " + Twine(Record.size()).str());
+ if (!TheIndex)
+ break;
+ if (TheIndex->modulePaths().empty())
+ // Does not have any summary emitted.
+ break;
+ if (TheIndex->modulePaths().size() != 1)
+ return error("Don't expect multiple modules defined?");
+ auto &Hash = TheIndex->modulePaths().begin()->second.second;
+ int Pos = 0;
+ for (auto &Val : Record) {
+ assert(!(Val >> 32) && "Unexpected high bits set");
+ Hash[Pos++] = Val;
+ }
+ break;
+ }
/// MODULE_CODE_VSTOFFSET: [offset]
case bitc::MODULE_CODE_VSTOFFSET:
if (Record.size() < 1)
@@ -5761,7 +5776,7 @@
// module path string table entry with an empty (0) ID to take
// ownership.
FS->setModulePath(
- TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0));
+ TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)->first());
static int RefListStartIndex = 4;
int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs;
assert(Record.size() >= RefListStartIndex + NumRefs &&
@@ -5799,7 +5814,7 @@
std::unique_ptr<GlobalVarSummary> FS =
llvm::make_unique<GlobalVarSummary>(getDecodedLinkage(RawLinkage));
FS->setModulePath(
- TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0));
+ TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0)->first());
for (unsigned I = 2, E = Record.size(); I != E; ++I) {
unsigned RefValueId = Record[I];
uint64_t RefGUID = getGUIDFromValueId(RefValueId);
@@ -5887,6 +5902,7 @@
SmallVector<uint64_t, 64> Record;
SmallString<128> ModulePath;
+ ModulePathStringTableTy::iterator LastSeenModulePath;
while (1) {
BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
@@ -5907,14 +5923,32 @@
break;
case bitc::MST_CODE_ENTRY: {
// MST_ENTRY: [modid, namechar x N]
+ uint64_t ModuleId = Record[0];
+
if (convertToString(Record, 1, ModulePath))
return error("Invalid record");
- uint64_t ModuleId = Record[0];
- StringRef ModulePathInMap = TheIndex->addModulePath(ModulePath, ModuleId);
- ModuleIdMap[ModuleId] = ModulePathInMap;
+
+ LastSeenModulePath = TheIndex->addModulePath(ModulePath, ModuleId);
+ ModuleIdMap[ModuleId] = LastSeenModulePath->first();
+
ModulePath.clear();
break;
}
+ /// MST_CODE_HASH: [5*i32]
+ case bitc::MST_CODE_HASH: {
+ if (Record.size() != 5)
+ return error("Invalid hash length " + Twine(Record.size()).str());
+ if (LastSeenModulePath == TheIndex->modulePaths().end())
+ return error("Invalid hash that does not follow a module path");
+ int Pos = 0;
+ for (auto &Val : Record) {
+ assert(!(Val >> 32) && "Unexpected high bits set");
+ LastSeenModulePath->second.second[Pos++] = Val;
+ }
+ // Reset LastSeenModulePath to avoid overriding the hash unexpectedly.
+ LastSeenModulePath = TheIndex->modulePaths().end();
+ break;
+ }
}
}
llvm_unreachable("Exit infinite loop");
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index c151341..18fb7ad 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "ValueEnumerator.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
@@ -39,6 +40,7 @@
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/SHA1.h"
#include <cctype>
#include <map>
using namespace llvm;
@@ -2852,8 +2854,18 @@
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv);
- SmallVector<unsigned, 64> NameVals;
- for (const StringMapEntry<uint64_t> &MPSE : I.modulePaths()) {
+ // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_HASH));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+ unsigned AbbrevHash = Stream.EmitAbbrev(Abbv);
+
+ SmallVector<unsigned, 64> Vals;
+ for (const auto &MPSE : I.modulePaths()) {
StringEncoding Bits =
getStringEncoding(MPSE.getKey().data(), MPSE.getKey().size());
unsigned AbbrevToUse = Abbrev8Bit;
@@ -2862,14 +2874,29 @@
else if (Bits == SE_Fixed7)
AbbrevToUse = Abbrev7Bit;
- NameVals.push_back(MPSE.getValue());
+ Vals.push_back(MPSE.getValue().first);
for (const auto P : MPSE.getKey())
- NameVals.push_back((unsigned char)P);
+ Vals.push_back((unsigned char)P);
// Emit the finished record.
- Stream.EmitRecord(bitc::MST_CODE_ENTRY, NameVals, AbbrevToUse);
- NameVals.clear();
+ Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse);
+
+ Vals.clear();
+ // Emit an optional hash for the module now
+ auto &Hash = MPSE.getValue().second;
+ bool AllZero = true; // Detect if the hash is empty, and do not generate it
+ for (auto Val : Hash) {
+ if (Val)
+ AllZero = false;
+ Vals.push_back(Val);
+ }
+ if (!AllZero) {
+ // Emit the hash record.
+ Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash);
+ }
+
+ Vals.clear();
}
Stream.ExitBlock();
}
@@ -3177,11 +3204,36 @@
Stream.ExitBlock();
}
+static void writeModuleHash(BitstreamWriter &Stream,
+ SmallVectorImpl<char> &Buffer,
+ size_t BlockStartPos) {
+ // Emit the module's hash.
+ // MODULE_CODE_HASH: [5*i32]
+ SHA1 Hasher;
+ Hasher.update(ArrayRef<uint8_t>((uint8_t *)&Buffer[BlockStartPos],
+ Buffer.size() - BlockStartPos));
+ auto Hash = Hasher.result();
+ SmallVector<uint64_t, 20> Vals;
+ auto LShift = [&](unsigned char Val, unsigned Amount)
+ -> uint64_t { return ((uint64_t)Val) << Amount; };
+ for (int Pos = 0; Pos < 20; Pos += 4) {
+ uint32_t SubHash = LShift(Hash[Pos + 0], 24);
+ SubHash |= LShift(Hash[Pos + 1], 16) | LShift(Hash[Pos + 2], 8) |
+ (unsigned)(unsigned char)Hash[Pos + 3];
+ Vals.push_back(SubHash);
+ }
+
+ // Emit the finished record.
+ Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals);
+}
+
/// WriteModule - Emit the specified module to the bitstream.
static void WriteModule(const Module *M, BitstreamWriter &Stream,
bool ShouldPreserveUseListOrder,
- uint64_t BitcodeStartBit, bool EmitSummaryIndex) {
+ uint64_t BitcodeStartBit, bool EmitSummaryIndex,
+ bool GenerateHash, SmallVectorImpl<char> &Buffer) {
Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
+ size_t BlockStartPos = Buffer.size();
SmallVector<unsigned, 1> Vals;
unsigned CurVersion = 1;
@@ -3238,6 +3290,10 @@
WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream,
VSTOffsetPlaceholder, BitcodeStartBit, &FunctionIndex);
+ if (GenerateHash) {
+ writeModuleHash(Stream, Buffer, BlockStartPos);
+ }
+
Stream.ExitBlock();
}
@@ -3322,7 +3378,7 @@
/// stream.
void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
bool ShouldPreserveUseListOrder,
- bool EmitSummaryIndex) {
+ bool EmitSummaryIndex, bool GenerateHash) {
SmallVector<char, 0> Buffer;
Buffer.reserve(256*1024);
@@ -3348,7 +3404,7 @@
// Emit the module.
WriteModule(M, Stream, ShouldPreserveUseListOrder, BitcodeStartBit,
- EmitSummaryIndex);
+ EmitSummaryIndex, GenerateHash, Buffer);
}
if (TT.isOSDarwin() || TT.isOSBinFormatMachO())