[asan] change the coverage collection scheme so that we can easily emit coverage for the entire process as a single bit set, and if coverage_bitset=1 actually emit that bitset
llvm-svn: 224789
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc b/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
index 3e521d1..eb79f46 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc
@@ -12,14 +12,16 @@
//
// Compiler instrumentation:
// For every interesting basic block the compiler injects the following code:
-// if (Guard) {
+// if (Guard < 0) {
// __sanitizer_cov(&Guard);
// }
+// At the module start up time __sanitizer_cov_module_init sets the guards
+// to consecutive negative numbers (-1, -2, -3, ...).
// It's fine to call __sanitizer_cov more than once for a given block.
//
// Run-time:
// - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC).
-// and atomically set Guard to 1.
+// and atomically set Guard to -Guard.
// - __sanitizer_cov_dump: dump the coverage data to disk.
// For every module of the current process that has coverage data
// this will create a file module_name.PID.sancov. The file format is simple:
@@ -65,14 +67,16 @@
void BeforeFork();
void AfterFork(int child_pid);
void Extend(uptr npcs);
- void Add(uptr pc, u8 *guard);
+ void Add(uptr pc, u32 *guard);
void IndirCall(uptr caller, uptr callee, uptr callee_cache[],
uptr cache_size);
void DumpCallerCalleePairs();
void DumpTrace();
ALWAYS_INLINE
- void TraceBasicaBlock(uptr *cache);
+ void TraceBasicBlock(uptr *cache);
+
+ void InitializeGuards(s32 **guards, uptr n);
uptr *data();
uptr size();
@@ -149,12 +153,11 @@
pc_array = reinterpret_cast<uptr *>(
MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit"));
pc_fd = kInvalidFd;
+ atomic_store(&pc_array_index, 0, memory_order_relaxed);
if (common_flags()->coverage_direct) {
atomic_store(&pc_array_size, 0, memory_order_relaxed);
- atomic_store(&pc_array_index, 0, memory_order_relaxed);
} else {
atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed);
- atomic_store(&pc_array_index, 0, memory_order_relaxed);
}
cc_array = reinterpret_cast<uptr **>(MmapNoReserveOrDie(
@@ -230,15 +233,27 @@
atomic_store(&pc_array_size, size, memory_order_release);
}
+void CoverageData::InitializeGuards(s32 **guards, uptr n) {
+ for (uptr i = 0; i < n; i++) {
+ uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed);
+ *guards[i] = -static_cast<s32>(idx + 1);
+ }
+}
+
// Atomically add the pc to the vector. The atomically set the guard to 1.
// If the function is called more than once for a given PC it will
// be inserted multiple times, which is fine.
-void CoverageData::Add(uptr pc, u8 *guard) {
- // Set the guard.
- atomic_uint8_t *atomic_guard = reinterpret_cast<atomic_uint8_t*>(guard);
- atomic_store(atomic_guard, 1, memory_order_relaxed);
+void CoverageData::Add(uptr pc, u32 *guard) {
+ atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard);
+ s32 guard_value = atomic_load(atomic_guard, memory_order_relaxed);
+ if (guard_value >= 0) return;
+
+ atomic_store(atomic_guard, -guard_value, memory_order_relaxed);
if (!pc_array) return;
- uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed);
+
+ uptr idx = -guard_value - 1;
+ if (idx >= atomic_load(&pc_array_index, memory_order_acquire))
+ return; // May happen after fork when pc_array_index becomes 0.
CHECK_LT(idx * sizeof(uptr),
atomic_load(&pc_array_size, memory_order_acquire));
pc_array[idx] = pc;
@@ -338,18 +353,20 @@
// If packed = true and name == 0: <pid>.<sancov>.<packed>.
// If packed = true and name != 0: <name>.<sancov>.<packed> (name is
// user-supplied).
-static int CovOpenFile(bool packed, const char* name) {
+static int CovOpenFile(bool packed, const char *name,
+ const char *extension = "sancov") {
InternalScopedString path(kMaxPathLength);
if (!packed) {
CHECK(name);
- path.append("%s/%s.%zd.sancov", common_flags()->coverage_dir, name,
- internal_getpid());
+ path.append("%s/%s.%zd.%s", common_flags()->coverage_dir, name,
+ internal_getpid(), extension);
} else {
if (!name)
- path.append("%s/%zd.sancov.packed", common_flags()->coverage_dir,
- internal_getpid());
+ path.append("%s/%zd.%s.packed", common_flags()->coverage_dir,
+ internal_getpid(), extension);
else
- path.append("%s/%s.sancov.packed", common_flags()->coverage_dir, name);
+ path.append("%s/%s.%s.packed", common_flags()->coverage_dir, name,
+ extension);
}
uptr fd = OpenFile(path.data(), true);
if (internal_iserror(fd)) {
@@ -434,7 +451,7 @@
// Record the current PC into the event buffer.
// Every event is a u32 value (index in tr_pc_array_index) so we compute
// it once and then cache in the provided 'cache' storage.
-void CoverageData::TraceBasicaBlock(uptr *cache) {
+void CoverageData::TraceBasicBlock(uptr *cache) {
CHECK(common_flags()->coverage);
uptr idx = *cache;
if (!idx) {
@@ -450,12 +467,33 @@
tr_event_array_index++;
}
+static void CovDumpAsBitSet() {
+ if (!common_flags()->coverage_bitset) return;
+ if (!coverage_data.size()) return;
+ int fd = CovOpenFile(/* packed */false, "combined", "bitset-sancov");
+ if (fd < 0) return;
+ uptr n = coverage_data.size();
+ uptr n_set_bits = 0;
+ InternalScopedBuffer<char> out(n);
+ for (uptr i = 0; i < n; i++) {
+ uptr pc = coverage_data.data()[i];
+ out[i] = pc ? '1' : '0';
+ if (pc)
+ n_set_bits++;
+ }
+ internal_write(fd, out.data(), n);
+ internal_close(fd);
+ VReport(1, " CovDump: bitset of %zd bits written, %zd bits are set\n", n,
+ n_set_bits);
+}
+
// Dump the coverage on disk.
static void CovDump() {
if (!common_flags()->coverage || common_flags()->coverage_direct) return;
#if !SANITIZER_WINDOWS
if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
return;
+ CovDumpAsBitSet();
uptr size = coverage_data.size();
InternalMmapVector<u32> offsets(size);
uptr *vb = coverage_data.data();
@@ -539,7 +577,7 @@
} // namespace __sanitizer
extern "C" {
-SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u8 *guard) {
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) {
coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()),
guard);
}
@@ -552,8 +590,11 @@
SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() {
coverage_data.Init();
}
-SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(uptr npcs) {
- if (!common_flags()->coverage || !common_flags()->coverage_direct) return;
+SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(s32 **guards,
+ uptr npcs) {
+ coverage_data.InitializeGuards(guards, npcs);
+ if (!common_flags()->coverage || !common_flags()->coverage_direct)
+ return;
if (SANITIZER_ANDROID) {
// dlopen/dlclose interceptors do not work on Android, so we rely on
// Extend() calls to update .sancov.map.
@@ -572,10 +613,10 @@
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_cov_trace_func_enter(uptr *cache) {
- coverage_data.TraceBasicaBlock(cache);
+ coverage_data.TraceBasicBlock(cache);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_cov_trace_basic_block(uptr *cache) {
- coverage_data.TraceBasicaBlock(cache);
+ coverage_data.TraceBasicBlock(cache);
}
} // extern "C"
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_flags.cc b/compiler-rt/lib/sanitizer_common/sanitizer_flags.cc
index dd8ec7a..7eabb3a 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_flags.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_flags.cc
@@ -63,6 +63,7 @@
legacy_pthread_cond = false;
intercept_tls_get_addr = false;
coverage = false;
+ coverage_bitset = false;
coverage_direct = SANITIZER_ANDROID;
coverage_dir = ".";
full_address_space = false;
@@ -150,6 +151,9 @@
ParseFlag(str, &coverage, "coverage",
"If set, coverage information will be dumped at program shutdown (if the "
"coverage instrumentation was enabled at compile time).");
+ ParseFlag(str, &coverage_bitset, "coverage_bitset",
+ "If set (and if 'coverage' is set too), the coverage information "
+ "will also be dumped as a bitset to a separate file.");
ParseFlag(str, &coverage_direct, "coverage_direct",
"If set, coverage information will be dumped directly to a memory "
"mapped file. This way data is not lost even if the process is "
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_flags.h b/compiler-rt/lib/sanitizer_common/sanitizer_flags.h
index b9f2204..6d251b4 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_flags.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_flags.h
@@ -56,6 +56,7 @@
uptr mmap_limit_mb;
uptr hard_rss_limit_mb;
bool coverage;
+ bool coverage_bitset;
bool coverage_direct;
const char *coverage_dir;
bool full_address_space;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
index 97eef74..5e162d6 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
@@ -120,7 +120,7 @@
SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init();
- SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u8 *guard);
+ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_annotate_contiguous_container(const void *beg,
const void *end,
diff --git a/compiler-rt/test/asan/TestCases/Linux/coverage-fork.cc b/compiler-rt/test/asan/TestCases/Linux/coverage-fork.cc
index 38c2009..b9e981f 100644
--- a/compiler-rt/test/asan/TestCases/Linux/coverage-fork.cc
+++ b/compiler-rt/test/asan/TestCases/Linux/coverage-fork.cc
@@ -33,6 +33,7 @@
}
// CHECK-DAG: Child PID: [[ChildPID:[0-9]+]]
-// CHECK-DAG: [[ChildPID]].sancov: 1 PCs written
+// Coverage in the child (after fork,before exec) is not expected to work.
+// (disabled): [[ChildPID]].sancov: 1 PCs written
// CHECK-DAG: Parent PID: [[ParentPID:[0-9]+]]
// CHECK-DAG: [[ParentPID]].sancov: 3 PCs written
diff --git a/compiler-rt/test/asan/TestCases/Linux/coverage-levels.cc b/compiler-rt/test/asan/TestCases/Linux/coverage-levels.cc
index 748ef1f..d187d36 100644
--- a/compiler-rt/test/asan/TestCases/Linux/coverage-levels.cc
+++ b/compiler-rt/test/asan/TestCases/Linux/coverage-levels.cc
@@ -1,11 +1,14 @@
// Test various levels of coverage
//
// RUN: %clangxx_asan -O1 -fsanitize-coverage=1 %s -o %t
-// RUN: ASAN_OPTIONS=coverage=1:verbosity=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK1
+// RUN: ASAN_OPTIONS=coverage=1:coverage_bitset=1:verbosity=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK1
// RUN: %clangxx_asan -O1 -fsanitize-coverage=2 %s -o %t
-// RUN: ASAN_OPTIONS=coverage=1:verbosity=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
+// RUN: ASAN_OPTIONS=coverage=1:coverage_bitset=1:verbosity=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
// RUN: %clangxx_asan -O1 -fsanitize-coverage=3 %s -o %t
-// RUN: ASAN_OPTIONS=coverage=1:verbosity=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK3
+// RUN: ASAN_OPTIONS=coverage=1:coverage_bitset=1:verbosity=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK3
+
+// RUN: ASAN_OPTIONS=coverage=1:coverage_bitset=0:verbosity=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK3_NOBITSET
+// RUN: ASAN_OPTIONS=coverage=1:verbosity=1 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK3_NOBITSET
//
// REQUIRES: asan-64-bits
@@ -15,6 +18,10 @@
sink = 0;
}
+// CHECK1: CovDump: bitset of 1 bits written, 1 bits are set
// CHECK1: 1 PCs written
+// CHECK2: CovDump: bitset of 3 bits written, 2 bits are set
// CHECK2: 2 PCs written
+// CHECK3: CovDump: bitset of 4 bits written, 3 bits are set
// CHECK3: 3 PCs written
+// CHECK3_NOBITSET-NOT: bitset of
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 9293362..8600f91 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -10,10 +10,10 @@
// Coverage instrumentation that works with AddressSanitizer
// and potentially with other Sanitizers.
//
-// We create a Guard boolean variable with the same linkage
+// We create a Guard variable with the same linkage
// as the function and inject this code into the entry block (CoverageLevel=1)
// or all blocks (CoverageLevel>=2):
-// if (Guard) {
+// if (Guard < 0) {
// __sanitizer_cov(&Guard);
// }
// The accesses to Guard are atomic. The rest of the logic is
@@ -112,6 +112,8 @@
Type *IntptrTy;
LLVMContext *C;
+ SmallVector<Constant *, 16> Guards;
+
int CoverageLevel;
};
@@ -134,6 +136,8 @@
IntptrTy = Type::getIntNTy(*C, DLP->getDataLayout().getPointerSizeInBits());
Type *VoidTy = Type::getVoidTy(*C);
IRBuilder<> IRB(*C);
+ Type *Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
+ Type *Int32PtrPtrTy = PointerType::getUnqual(Int32PtrTy);
Function *CtorFunc =
Function::Create(FunctionType::get(VoidTy, false),
@@ -142,11 +146,12 @@
appendToGlobalCtors(M, CtorFunc, kSanCtorAndDtorPriority);
SanCovFunction = checkInterfaceFunction(
- M.getOrInsertFunction(kSanCovName, VoidTy, IRB.getInt8PtrTy(), nullptr));
+ M.getOrInsertFunction(kSanCovName, VoidTy, Int32PtrTy, nullptr));
SanCovIndirCallFunction = checkInterfaceFunction(M.getOrInsertFunction(
kSanCovIndirCallName, VoidTy, IntptrTy, IntptrTy, nullptr));
- SanCovModuleInit = checkInterfaceFunction(M.getOrInsertFunction(
- kSanCovModuleInitName, Type::getVoidTy(*C), IntptrTy, nullptr));
+ SanCovModuleInit = checkInterfaceFunction(
+ M.getOrInsertFunction(kSanCovModuleInitName, Type::getVoidTy(*C),
+ Int32PtrPtrTy, IntptrTy, nullptr));
SanCovModuleInit->setLinkage(Function::ExternalLinkage);
// We insert an empty inline asm after cov callbacks to avoid callback merge.
EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
@@ -163,9 +168,15 @@
for (auto &F : M)
runOnFunction(F);
+ ArrayType *ArrayOfInt32PtrTy = ArrayType::get(Int32PtrTy, Guards.size());
IRB.SetInsertPoint(CtorFunc->getEntryBlock().getTerminator());
- IRB.CreateCall(SanCovModuleInit,
- ConstantInt::get(IntptrTy, SanCovFunction->getNumUses()));
+ GlobalVariable *AllGuards = new GlobalVariable(
+ M, ArrayOfInt32PtrTy, false, GlobalVariable::InternalLinkage,
+ ConstantArray::get(ArrayOfInt32PtrTy, Guards), "");
+ assert(SanCovFunction->getNumUses() == Guards.size());
+ IRB.CreateCall2(SanCovModuleInit,
+ IRB.CreatePointerCast(AllGuards, Int32PtrPtrTy),
+ ConstantInt::get(IntptrTy, Guards.size()));
return true;
}
@@ -271,16 +282,16 @@
: IP->getDebugLoc();
IRBuilder<> IRB(IP);
IRB.SetCurrentDebugLocation(EntryLoc);
- Type *Int8Ty = IRB.getInt8Ty();
+ Type *Int32Ty = IRB.getInt32Ty();
GlobalVariable *Guard = new GlobalVariable(
- *F.getParent(), Int8Ty, false, GlobalValue::PrivateLinkage,
- Constant::getNullValue(Int8Ty), "__sancov_gen_cov_" + F.getName());
+ *F.getParent(), Int32Ty, false, GlobalValue::PrivateLinkage,
+ Constant::getNullValue(Int32Ty), "__sancov_gen_cov_" + F.getName());
LoadInst *Load = IRB.CreateLoad(Guard);
Load->setAtomic(Monotonic);
- Load->setAlignment(1);
+ Load->setAlignment(4);
Load->setMetadata(F.getParent()->getMDKindID("nosanitize"),
MDNode::get(*C, None));
- Value *Cmp = IRB.CreateICmpEQ(Constant::getNullValue(Int8Ty), Load);
+ Value *Cmp = IRB.CreateICmpSGE(Constant::getNullValue(Int32Ty), Load);
Instruction *Ins = SplitBlockAndInsertIfThen(
Cmp, IP, false, MDBuilder(*C).createBranchWeights(1, 100000));
IRB.SetInsertPoint(Ins);
@@ -288,6 +299,7 @@
// __sanitizer_cov gets the PC of the instruction using GET_CALLER_PC.
IRB.CreateCall(SanCovFunction, Guard);
IRB.CreateCall(EmptyAsm); // Avoids callback merge.
+ Guards.push_back(Guard); // Save the guard for later.
}
char SanitizerCoverageModule::ID = 0;
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
index 22151ba..9115fec 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage-dbg.ll
@@ -15,7 +15,7 @@
; and add sanitize_address to @_ZN1A1fEv
; Test that __sanitizer_cov call has !dbg pointing to the opening { of A::f().
-; CHECK: call void @__sanitizer_cov(i8*{{.*}}), !dbg [[A:!.*]]
+; CHECK: call void @__sanitizer_cov(i32*{{.*}}), !dbg [[A:!.*]]
; CHECK: [[A]] = !{i32 6, i32 0, !{{.*}}, null}
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
index 68e7fc0..9a05fa8 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage.ll
@@ -33,17 +33,17 @@
; CHECK0-NOT: call void @__sanitizer_cov_module_init(
; CHECK1-LABEL: define void @foo
-; CHECK1: %0 = load atomic i8* @__sancov_gen_cov_foo monotonic, align 1, !nosanitize
-; CHECK1: %1 = icmp eq i8 0, %0
+; CHECK1: %0 = load atomic i32* @__sancov_gen_cov_foo monotonic, align 4, !nosanitize
+; CHECK1: %1 = icmp sge i32 0, %0
; CHECK1: br i1 %1, label %2, label %3
-; CHECK1: call void @__sanitizer_cov(i8*{{.*}})
+; CHECK1: call void @__sanitizer_cov(i32*{{.*}})
; CHECK1: call void asm sideeffect "", ""()
; CHECK1-NOT: call void @__sanitizer_cov
; CHECK1: ret void
; CHECK1-LABEL: define internal void @sancov.module_ctor
; CHECK1-NOT: ret
-; CHECK1: call void @__sanitizer_cov_module_init(i64 2)
+; CHECK1: call void @__sanitizer_cov_module_init({{.*}}, i64 2)
; CHECK1: ret
@@ -59,7 +59,7 @@
; CHECK2-LABEL: define internal void @sancov.module_ctor
; CHECK2-NOT: ret
-; CHECK2: call void @__sanitizer_cov_module_init(i64 4)
+; CHECK2: call void @__sanitizer_cov_module_init({{.*}}, i64 4)
; CHECK2: ret
; CHECK3-LABEL: define void @foo
diff --git a/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll b/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
index f78d61f..54e0e6a 100644
--- a/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
+++ b/llvm/test/Instrumentation/SanitizerCoverage/coverage2-dbg.ll
@@ -17,9 +17,9 @@
; Check that __sanitizer_cov call has !dgb pointing to the beginning
; of appropriate basic blocks.
; CHECK-LABEL:_Z3fooPi
-; CHECK: call void @__sanitizer_cov(i8*{{.*}}), !dbg [[A:!.*]]
-; CHECK: call void @__sanitizer_cov(i8*{{.*}}), !dbg [[B:!.*]]
-; CHECK: call void @__sanitizer_cov(i8*{{.*}}), !dbg [[C:!.*]]
+; CHECK: call void @__sanitizer_cov(i32*{{.*}}), !dbg [[A:!.*]]
+; CHECK: call void @__sanitizer_cov(i32*{{.*}}), !dbg [[B:!.*]]
+; CHECK: call void @__sanitizer_cov(i32*{{.*}}), !dbg [[C:!.*]]
; CHECK: ret void
; CHECK: [[A]] = !{i32 1, i32 0, !{{.*}}, null}
; CHECK: [[B]] = !{i32 3, i32 5, !{{.*}}, null}