blob: 2a6b4d1223211f6952a719492bfc4485e773f01c [file] [log] [blame]
Bob Wilsona08e9ac2013-11-15 07:18:15 +00001//===-- sanitizer_coverage.cc ---------------------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Sanitizer Coverage.
11// This file implements run-time support for a poor man's coverage tool.
12//
13// Compiler instrumentation:
Kostya Serebryany714c67c2014-01-17 11:00:30 +000014// For every interesting basic block the compiler injects the following code:
Kostya Serebryany9fdeb372014-12-23 22:32:17 +000015// if (Guard < 0) {
Kostya Serebryany4cadd4a2014-11-24 18:49:53 +000016// __sanitizer_cov(&Guard);
Bob Wilsona08e9ac2013-11-15 07:18:15 +000017// }
Kostya Serebryany9fdeb372014-12-23 22:32:17 +000018// At the module start up time __sanitizer_cov_module_init sets the guards
19// to consecutive negative numbers (-1, -2, -3, ...).
Kostya Serebryany714c67c2014-01-17 11:00:30 +000020// It's fine to call __sanitizer_cov more than once for a given block.
Bob Wilsona08e9ac2013-11-15 07:18:15 +000021//
22// Run-time:
Kostya Serebryany714c67c2014-01-17 11:00:30 +000023// - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC).
Kostya Serebryany9fdeb372014-12-23 22:32:17 +000024// and atomically set Guard to -Guard.
Bob Wilsona08e9ac2013-11-15 07:18:15 +000025// - __sanitizer_cov_dump: dump the coverage data to disk.
26// For every module of the current process that has coverage data
27// this will create a file module_name.PID.sancov. The file format is simple:
28// it's just a sorted sequence of 4-byte offsets in the module.
29//
30// Eventually, this coverage implementation should be obsoleted by a more
31// powerful general purpose Clang/LLVM coverage instrumentation.
32// Consider this implementation as prototype.
33//
34// FIXME: support (or at least test with) dlclose.
35//===----------------------------------------------------------------------===//
36
37#include "sanitizer_allocator_internal.h"
38#include "sanitizer_common.h"
39#include "sanitizer_libc.h"
40#include "sanitizer_mutex.h"
41#include "sanitizer_procmaps.h"
Kostya Serebryany714c67c2014-01-17 11:00:30 +000042#include "sanitizer_stacktrace.h"
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +000043#include "sanitizer_symbolizer.h"
Bob Wilsona08e9ac2013-11-15 07:18:15 +000044#include "sanitizer_flags.h"
45
Kostya Serebryany183cb6e2014-11-14 23:15:55 +000046static atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once.
47
48static atomic_uintptr_t coverage_counter;
Bob Wilsona08e9ac2013-11-15 07:18:15 +000049
Kostya Serebryany8b530e12014-04-30 10:40:48 +000050// pc_array is the array containing the covered PCs.
Sergey Matveev6cb47a082014-05-19 12:53:03 +000051// To make the pc_array thread- and async-signal-safe it has to be large enough.
Kostya Serebryany8b530e12014-04-30 10:40:48 +000052// 128M counters "ought to be enough for anybody" (4M on 32-bit).
Evgeniy Stepanov567e5162014-05-27 12:37:52 +000053
54// With coverage_direct=1 in ASAN_OPTIONS, pc_array memory is mapped to a file.
55// In this mode, __sanitizer_cov_dump does nothing, and CovUpdateMapping()
56// dump current memory layout to another file.
Bob Wilsona08e9ac2013-11-15 07:18:15 +000057
Sergey Matveev6cb47a082014-05-19 12:53:03 +000058static bool cov_sandboxed = false;
59static int cov_fd = kInvalidFd;
60static unsigned int cov_max_block_size = 0;
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +000061static bool coverage_enabled = false;
62static const char *coverage_dir;
Sergey Matveev6cb47a082014-05-19 12:53:03 +000063
Bob Wilsona08e9ac2013-11-15 07:18:15 +000064namespace __sanitizer {
65
Evgeniy Stepanov567e5162014-05-27 12:37:52 +000066class CoverageData {
67 public:
68 void Init();
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +000069 void ReInit();
Evgeniy Stepanovfe181022014-06-04 12:13:54 +000070 void BeforeFork();
71 void AfterFork(int child_pid);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +000072 void Extend(uptr npcs);
Kostya Serebryany9fdeb372014-12-23 22:32:17 +000073 void Add(uptr pc, u32 *guard);
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +000074 void IndirCall(uptr caller, uptr callee, uptr callee_cache[],
75 uptr cache_size);
76 void DumpCallerCalleePairs();
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +000077 void DumpTrace();
78
79 ALWAYS_INLINE
Kostya Serebryanyd421db02015-01-03 00:54:43 +000080 void TraceBasicBlock(s32 *id);
Kostya Serebryany9fdeb372014-12-23 22:32:17 +000081
Kostya Serebryany77c5c1a2014-12-30 23:16:12 +000082 void InitializeGuardArray(s32 *guards);
Kostya Serebryanyaa185bf2014-12-30 19:29:28 +000083 void InitializeGuards(s32 *guards, uptr n);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +000084
85 uptr *data();
86 uptr size();
87
88 private:
89 // Maximal size pc array may ever grow.
90 // We MmapNoReserve this space to ensure that the array is contiguous.
Evgeniy Stepanov10308462014-12-26 13:54:11 +000091 static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 24, 1 << 27);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +000092 // The amount file mapping for the pc array is grown by.
93 static const uptr kPcArrayMmapSize = 64 * 1024;
94
95 // pc_array is allocated with MmapNoReserveOrDie and so it uses only as
96 // much RAM as it really needs.
97 uptr *pc_array;
98 // Index of the first available pc_array slot.
99 atomic_uintptr_t pc_array_index;
100 // Array size.
101 atomic_uintptr_t pc_array_size;
102 // Current file mapped size of the pc array.
103 uptr pc_array_mapped_size;
104 // Descriptor of the file mapped pc array.
105 int pc_fd;
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000106
Kostya Serebryany77c5c1a2014-12-30 23:16:12 +0000107 // Vector of coverage guard arrays, protected by mu.
108 InternalMmapVectorNoCtor<s32*> guard_array_vec;
109
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000110 // Caller-Callee (cc) array, size and current index.
111 static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24);
112 uptr **cc_array;
113 atomic_uintptr_t cc_array_index;
114 atomic_uintptr_t cc_array_size;
115
Kostya Serebryanyd421db02015-01-03 00:54:43 +0000116 // Tracing event array, size and current index.
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000117 // We record all events (basic block entries) in a global buffer of u32
Kostya Serebryanyd421db02015-01-03 00:54:43 +0000118 // values. Each such value is the index in pc_array (incremented by one).
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000119 // So far the tracing is highly experimental:
120 // - not thread-safe;
121 // - does not support long traces;
122 // - not tuned for performance.
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000123 static const uptr kTrEventArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 30);
124 u32 *tr_event_array;
125 uptr tr_event_array_size;
126 uptr tr_event_array_index;
127 static const uptr kTrPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27);
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000128
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000129 StaticSpinMutex mu;
130
Evgeniy Stepanovce984522014-06-03 15:27:15 +0000131 void DirectOpen();
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000132};
133
134static CoverageData coverage_data;
135
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000136void CovUpdateMapping(const char *path, uptr caller_pc = 0);
137
Evgeniy Stepanovce984522014-06-03 15:27:15 +0000138void CoverageData::DirectOpen() {
Alexey Samsonov4cc76cb2014-11-26 01:48:39 +0000139 InternalScopedString path(kMaxPathLength);
Evgeniy Stepanovfa5c0752014-05-29 14:33:16 +0000140 internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw",
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000141 coverage_dir, internal_getpid());
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000142 pc_fd = OpenFile(path.data(), true);
143 if (internal_iserror(pc_fd)) {
144 Report(" Coverage: failed to open %s for writing\n", path.data());
145 Die();
146 }
147
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000148 pc_array_mapped_size = 0;
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000149 CovUpdateMapping(coverage_dir);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000150}
151
152void CoverageData::Init() {
153 pc_array = reinterpret_cast<uptr *>(
154 MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit"));
Evgeniy Stepanovfe181022014-06-04 12:13:54 +0000155 pc_fd = kInvalidFd;
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000156 atomic_store(&pc_array_index, 0, memory_order_relaxed);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000157 if (common_flags()->coverage_direct) {
Evgeniy Stepanovce984522014-06-03 15:27:15 +0000158 atomic_store(&pc_array_size, 0, memory_order_relaxed);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000159 } else {
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000160 atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed);
161 }
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000162
163 cc_array = reinterpret_cast<uptr **>(MmapNoReserveOrDie(
164 sizeof(uptr *) * kCcArrayMaxSize, "CovInit::cc_array"));
165 atomic_store(&cc_array_size, kCcArrayMaxSize, memory_order_relaxed);
166 atomic_store(&cc_array_index, 0, memory_order_relaxed);
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000167
Kostya Serebryany0f53d9a2015-01-03 02:07:58 +0000168 // Allocate tr_event_array with a guard page at the end.
169 tr_event_array = reinterpret_cast<u32 *>(MmapNoReserveOrDie(
170 sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(),
171 "CovInit::tr_event_array"));
172 Mprotect(reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]),
173 GetMmapGranularity());
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000174 tr_event_array_size = kTrEventArrayMaxSize;
175 tr_event_array_index = 0;
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000176}
177
Kostya Serebryany77c5c1a2014-12-30 23:16:12 +0000178void CoverageData::InitializeGuardArray(s32 *guards) {
179 s32 n = guards[0];
180 for (s32 j = 1; j <= n; j++) {
181 uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed);
182 guards[j] = -static_cast<s32>(idx + 1);
183 }
184}
185
Evgeniy Stepanovfe181022014-06-04 12:13:54 +0000186void CoverageData::ReInit() {
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000187 if (pc_array) {
188 internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize);
189 pc_array = nullptr;
190 }
Evgeniy Stepanovfe181022014-06-04 12:13:54 +0000191 if (pc_fd != kInvalidFd) internal_close(pc_fd);
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000192 if (coverage_enabled) {
193 if (common_flags()->coverage_direct) {
194 // In memory-mapped mode we must extend the new file to the known array
195 // size.
196 uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
197 Init();
198 if (size) Extend(size);
199 if (coverage_enabled) CovUpdateMapping(coverage_dir);
200 } else {
201 Init();
202 }
Evgeniy Stepanovfe181022014-06-04 12:13:54 +0000203 }
Kostya Serebryany77c5c1a2014-12-30 23:16:12 +0000204 // Re-initialize the guards.
205 // We are single-threaded now, no need to grab any lock.
206 CHECK_EQ(atomic_load(&pc_array_index, memory_order_relaxed), 0);
207 for (uptr i = 0; i < guard_array_vec.size(); i++)
208 InitializeGuardArray(guard_array_vec[i]);
Evgeniy Stepanovfe181022014-06-04 12:13:54 +0000209}
210
211void CoverageData::BeforeFork() {
212 mu.Lock();
213}
214
215void CoverageData::AfterFork(int child_pid) {
216 // We are single-threaded so it's OK to release the lock early.
217 mu.Unlock();
218 if (child_pid == 0) ReInit();
219}
220
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000221// Extend coverage PC array to fit additional npcs elements.
222void CoverageData::Extend(uptr npcs) {
Evgeniy Stepanovce984522014-06-03 15:27:15 +0000223 if (!common_flags()->coverage_direct) return;
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000224 SpinMutexLock l(&mu);
225
226 uptr size = atomic_load(&pc_array_size, memory_order_relaxed);
227 size += npcs * sizeof(uptr);
228
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000229 if (coverage_enabled && size > pc_array_mapped_size) {
230 if (pc_fd == kInvalidFd) DirectOpen();
231 CHECK_NE(pc_fd, kInvalidFd);
232
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000233 uptr new_mapped_size = pc_array_mapped_size;
234 while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize;
Evgeniy Stepanovca9e0452014-12-24 13:57:11 +0000235 CHECK_LE(new_mapped_size, sizeof(uptr) * kPcArrayMaxSize);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000236
237 // Extend the file and map the new space at the end of pc_array.
238 uptr res = internal_ftruncate(pc_fd, new_mapped_size);
239 int err;
240 if (internal_iserror(res, &err)) {
241 Printf("failed to extend raw coverage file: %d\n", err);
242 Die();
243 }
Evgeniy Stepanovca9e0452014-12-24 13:57:11 +0000244
245 uptr next_map_base = ((uptr)pc_array) + pc_array_mapped_size;
246 void *p = MapWritableFileToMemory((void *)next_map_base,
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000247 new_mapped_size - pc_array_mapped_size,
248 pc_fd, pc_array_mapped_size);
Evgeniy Stepanovca9e0452014-12-24 13:57:11 +0000249 CHECK_EQ((uptr)p, next_map_base);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000250 pc_array_mapped_size = new_mapped_size;
251 }
252
253 atomic_store(&pc_array_size, size, memory_order_release);
254}
255
Kostya Serebryanyaa185bf2014-12-30 19:29:28 +0000256void CoverageData::InitializeGuards(s32 *guards, uptr n) {
257 // The array 'guards' has n+1 elements, we use the element zero
258 // to store 'n'.
259 CHECK_LT(n, 1 << 30);
260 guards[0] = static_cast<s32>(n);
Kostya Serebryany77c5c1a2014-12-30 23:16:12 +0000261 InitializeGuardArray(guards);
262 SpinMutexLock l(&mu);
263 guard_array_vec.push_back(guards);
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000264}
265
Kostya Serebryanyaa185bf2014-12-30 19:29:28 +0000266// If guard is negative, atomically set it to -guard and store the PC in
267// pc_array.
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000268void CoverageData::Add(uptr pc, u32 *guard) {
269 atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard);
270 s32 guard_value = atomic_load(atomic_guard, memory_order_relaxed);
271 if (guard_value >= 0) return;
272
273 atomic_store(atomic_guard, -guard_value, memory_order_relaxed);
Kostya Serebryany8b530e12014-04-30 10:40:48 +0000274 if (!pc_array) return;
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000275
276 uptr idx = -guard_value - 1;
277 if (idx >= atomic_load(&pc_array_index, memory_order_acquire))
278 return; // May happen after fork when pc_array_index becomes 0.
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000279 CHECK_LT(idx * sizeof(uptr),
280 atomic_load(&pc_array_size, memory_order_acquire));
Kostya Serebryany8b530e12014-04-30 10:40:48 +0000281 pc_array[idx] = pc;
Kostya Serebryany183cb6e2014-11-14 23:15:55 +0000282 atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed);
Kostya Serebryany8b530e12014-04-30 10:40:48 +0000283}
284
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000285// Registers a pair caller=>callee.
286// When a given caller is seen for the first time, the callee_cache is added
287// to the global array cc_array, callee_cache[0] is set to caller and
288// callee_cache[1] is set to cache_size.
289// Then we are trying to add callee to callee_cache [2,cache_size) if it is
290// not there yet.
291// If the cache is full we drop the callee (may want to fix this later).
292void CoverageData::IndirCall(uptr caller, uptr callee, uptr callee_cache[],
293 uptr cache_size) {
294 if (!cc_array) return;
295 atomic_uintptr_t *atomic_callee_cache =
296 reinterpret_cast<atomic_uintptr_t *>(callee_cache);
297 uptr zero = 0;
298 if (atomic_compare_exchange_strong(&atomic_callee_cache[0], &zero, caller,
299 memory_order_seq_cst)) {
300 uptr idx = atomic_fetch_add(&cc_array_index, 1, memory_order_relaxed);
301 CHECK_LT(idx * sizeof(uptr),
302 atomic_load(&cc_array_size, memory_order_acquire));
303 callee_cache[1] = cache_size;
304 cc_array[idx] = callee_cache;
305 }
306 CHECK_EQ(atomic_load(&atomic_callee_cache[0], memory_order_relaxed), caller);
307 for (uptr i = 2; i < cache_size; i++) {
308 uptr was = 0;
309 if (atomic_compare_exchange_strong(&atomic_callee_cache[i], &was, callee,
Kostya Serebryany183cb6e2014-11-14 23:15:55 +0000310 memory_order_seq_cst)) {
311 atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed);
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000312 return;
Kostya Serebryany183cb6e2014-11-14 23:15:55 +0000313 }
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000314 if (was == callee) // Already have this callee.
315 return;
316 }
317}
318
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000319uptr *CoverageData::data() {
320 return pc_array;
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000321}
322
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000323uptr CoverageData::size() {
324 return atomic_load(&pc_array_index, memory_order_relaxed);
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000325}
326
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000327// Block layout for packed file format: header, followed by module name (no
328// trailing zero), followed by data blob.
329struct CovHeader {
330 int pid;
331 unsigned int module_name_length;
332 unsigned int data_length;
333};
334
335static void CovWritePacked(int pid, const char *module, const void *blob,
336 unsigned int blob_size) {
Sergey Matveev83f91e72014-05-21 13:43:52 +0000337 if (cov_fd < 0) return;
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000338 unsigned module_name_length = internal_strlen(module);
339 CovHeader header = {pid, module_name_length, blob_size};
340
341 if (cov_max_block_size == 0) {
342 // Writing to a file. Just go ahead.
343 internal_write(cov_fd, &header, sizeof(header));
344 internal_write(cov_fd, module, module_name_length);
345 internal_write(cov_fd, blob, blob_size);
346 } else {
347 // Writing to a socket. We want to split the data into appropriately sized
348 // blocks.
349 InternalScopedBuffer<char> block(cov_max_block_size);
350 CHECK_EQ((uptr)block.data(), (uptr)(CovHeader *)block.data());
351 uptr header_size_with_module = sizeof(header) + module_name_length;
352 CHECK_LT(header_size_with_module, cov_max_block_size);
353 unsigned int max_payload_size =
354 cov_max_block_size - header_size_with_module;
355 char *block_pos = block.data();
356 internal_memcpy(block_pos, &header, sizeof(header));
357 block_pos += sizeof(header);
358 internal_memcpy(block_pos, module, module_name_length);
359 block_pos += module_name_length;
360 char *block_data_begin = block_pos;
Alexey Samsonov4925fd42014-11-13 22:40:59 +0000361 const char *blob_pos = (const char *)blob;
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000362 while (blob_size > 0) {
363 unsigned int payload_size = Min(blob_size, max_payload_size);
364 blob_size -= payload_size;
365 internal_memcpy(block_data_begin, blob_pos, payload_size);
366 blob_pos += payload_size;
367 ((CovHeader *)block.data())->data_length = payload_size;
368 internal_write(cov_fd, block.data(),
369 header_size_with_module + payload_size);
370 }
371 }
372}
373
Sergey Matveev83f91e72014-05-21 13:43:52 +0000374// If packed = false: <name>.<pid>.<sancov> (name = module name).
375// If packed = true and name == 0: <pid>.<sancov>.<packed>.
376// If packed = true and name != 0: <name>.<sancov>.<packed> (name is
377// user-supplied).
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000378static int CovOpenFile(bool packed, const char *name,
379 const char *extension = "sancov") {
Alexey Samsonov656c29b2014-12-02 22:20:11 +0000380 InternalScopedString path(kMaxPathLength);
Sergey Matveev83f91e72014-05-21 13:43:52 +0000381 if (!packed) {
382 CHECK(name);
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000383 path.append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(),
384 extension);
Sergey Matveev83f91e72014-05-21 13:43:52 +0000385 } else {
386 if (!name)
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000387 path.append("%s/%zd.%s.packed", coverage_dir, internal_getpid(),
Evgeniy Stepanovf8c7e252014-12-26 10:19:56 +0000388 extension);
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000389 else
390 path.append("%s/%s.%s.packed", coverage_dir, name, extension);
Sergey Matveev83f91e72014-05-21 13:43:52 +0000391 }
392 uptr fd = OpenFile(path.data(), true);
393 if (internal_iserror(fd)) {
394 Report(" SanitizerCoverage: failed to open %s for writing\n", path.data());
395 return -1;
396 }
397 return fd;
398}
399
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000400// Dump trace PCs and trace events into two separate files.
401void CoverageData::DumpTrace() {
402 uptr max_idx = tr_event_array_index;
403 if (!max_idx) return;
404 auto sym = Symbolizer::GetOrInit();
405 if (!sym)
406 return;
407 InternalScopedString out(32 << 20);
Kostya Serebryanyd421db02015-01-03 00:54:43 +0000408 for (uptr i = 0, n = size(); i < n; i++) {
409 const char *module_name = "<unknown>";
410 uptr module_address = 0;
411 sym->GetModuleNameAndOffsetForPC(pc_array[i], &module_name,
412 &module_address);
413 out.append("%s 0x%zx\n", module_name, module_address);
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000414 }
415 int fd = CovOpenFile(false, "trace-points");
416 if (fd < 0) return;
417 internal_write(fd, out.data(), out.length());
418 internal_close(fd);
419
420 fd = CovOpenFile(false, "trace-events");
421 if (fd < 0) return;
Kostya Serebryany0f53d9a2015-01-03 02:07:58 +0000422 for (uptr i = 0; i < max_idx; i++)
423 tr_event_array[i]--; // Fix the IDs.
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000424 internal_write(fd, tr_event_array, max_idx * sizeof(tr_event_array[0]));
425 internal_close(fd);
Kostya Serebryanyd421db02015-01-03 00:54:43 +0000426 VReport(1, " CovDump: Trace: %zd PCs written\n", size());
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000427 VReport(1, " CovDump: Trace: %zd Events written\n", tr_event_array_index);
428}
429
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000430// This function dumps the caller=>callee pairs into a file as a sequence of
431// lines like "module_name offset".
432void CoverageData::DumpCallerCalleePairs() {
433 uptr max_idx = atomic_load(&cc_array_index, memory_order_relaxed);
434 if (!max_idx) return;
435 auto sym = Symbolizer::GetOrInit();
436 if (!sym)
437 return;
Kostya Serebryany40aa4a22014-10-31 19:49:46 +0000438 InternalScopedString out(32 << 20);
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000439 uptr total = 0;
440 for (uptr i = 0; i < max_idx; i++) {
441 uptr *cc_cache = cc_array[i];
442 CHECK(cc_cache);
443 uptr caller = cc_cache[0];
444 uptr n_callees = cc_cache[1];
445 const char *caller_module_name = "<unknown>";
446 uptr caller_module_address = 0;
447 sym->GetModuleNameAndOffsetForPC(caller, &caller_module_name,
448 &caller_module_address);
449 for (uptr j = 2; j < n_callees; j++) {
450 uptr callee = cc_cache[j];
451 if (!callee) break;
452 total++;
453 const char *callee_module_name = "<unknown>";
454 uptr callee_module_address = 0;
455 sym->GetModuleNameAndOffsetForPC(callee, &callee_module_name,
456 &callee_module_address);
457 out.append("%s 0x%zx\n%s 0x%zx\n", caller_module_name,
458 caller_module_address, callee_module_name,
459 callee_module_address);
460 }
461 }
462 int fd = CovOpenFile(false, "caller-callee");
463 if (fd < 0) return;
464 internal_write(fd, out.data(), out.length());
465 internal_close(fd);
466 VReport(1, " CovDump: %zd caller-callee pairs written\n", total);
467}
468
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000469// Record the current PC into the event buffer.
470// Every event is a u32 value (index in tr_pc_array_index) so we compute
471// it once and then cache in the provided 'cache' storage.
Kostya Serebryany0f53d9a2015-01-03 02:07:58 +0000472//
473// This function will eventually be inlined by the compiler.
Kostya Serebryanyd421db02015-01-03 00:54:43 +0000474void CoverageData::TraceBasicBlock(s32 *id) {
Kostya Serebryany0f53d9a2015-01-03 02:07:58 +0000475 // CHECK(coverage_enabled);
476 // CHECK_LT(tr_event_array_index, tr_event_array_size);
477 //
478 // Will trap here if
479 // 1. coverage is not enabled at run-time.
480 // 2. The array tr_event_array is full.
481 tr_event_array[tr_event_array_index++] = static_cast<u32>(*id);
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000482}
483
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000484static void CovDumpAsBitSet() {
485 if (!common_flags()->coverage_bitset) return;
486 if (!coverage_data.size()) return;
487 int fd = CovOpenFile(/* packed */false, "combined", "bitset-sancov");
488 if (fd < 0) return;
489 uptr n = coverage_data.size();
490 uptr n_set_bits = 0;
491 InternalScopedBuffer<char> out(n);
492 for (uptr i = 0; i < n; i++) {
493 uptr pc = coverage_data.data()[i];
494 out[i] = pc ? '1' : '0';
495 if (pc)
496 n_set_bits++;
497 }
498 internal_write(fd, out.data(), n);
499 internal_close(fd);
500 VReport(1, " CovDump: bitset of %zd bits written, %zd bits are set\n", n,
501 n_set_bits);
502}
503
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000504// Dump the coverage on disk.
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000505static void CovDump() {
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000506 if (!coverage_enabled || common_flags()->coverage_direct) return;
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000507#if !SANITIZER_WINDOWS
Kostya Serebryany8b530e12014-04-30 10:40:48 +0000508 if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed))
509 return;
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000510 CovDumpAsBitSet();
Kostya Serebryanyd421db02015-01-03 00:54:43 +0000511 coverage_data.DumpTrace();
Kostya Serebryanya7ee2732014-12-30 19:55:04 +0000512 if (!common_flags()->coverage_pcs) return;
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000513 uptr size = coverage_data.size();
Kostya Serebryany8b530e12014-04-30 10:40:48 +0000514 InternalMmapVector<u32> offsets(size);
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000515 uptr *vb = coverage_data.data();
516 uptr *ve = vb + size;
517 SortArray(vb, size);
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000518 MemoryMappingLayout proc_maps(/*cache_enabled*/true);
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000519 uptr mb, me, off, prot;
Alexey Samsonov656c29b2014-12-02 22:20:11 +0000520 InternalScopedString module(kMaxPathLength);
521 InternalScopedString path(kMaxPathLength);
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000522 for (int i = 0;
523 proc_maps.Next(&mb, &me, &off, module.data(), module.size(), &prot);
524 i++) {
525 if ((prot & MemoryMappingLayout::kProtectionExecute) == 0)
526 continue;
Sergey Matveev76e02e92014-05-08 16:09:54 +0000527 while (vb < ve && *vb < mb) vb++;
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000528 if (vb >= ve) break;
Sergey Matveev76e02e92014-05-08 16:09:54 +0000529 if (*vb < me) {
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000530 offsets.clear();
531 const uptr *old_vb = vb;
532 CHECK_LE(off, *vb);
533 for (; vb < ve && *vb < me; vb++) {
534 uptr diff = *vb - (i ? mb : 0) + off;
535 CHECK_LE(diff, 0xffffffffU);
536 offsets.push_back(static_cast<u32>(diff));
537 }
Alexey Samsonov26ca05a2014-11-04 19:34:29 +0000538 const char *module_name = StripModuleName(module.data());
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000539 if (cov_sandboxed) {
Sergey Matveev83f91e72014-05-21 13:43:52 +0000540 if (cov_fd >= 0) {
541 CovWritePacked(internal_getpid(), module_name, offsets.data(),
542 offsets.size() * sizeof(u32));
543 VReport(1, " CovDump: %zd PCs written to packed file\n", vb - old_vb);
544 }
Evgeniy Stepanov8ab205f2014-02-12 15:29:22 +0000545 } else {
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000546 // One file per module per process.
Alexey Samsonov656c29b2014-12-02 22:20:11 +0000547 path.clear();
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000548 path.append("%s/%s.%zd.sancov", coverage_dir, module_name,
549 internal_getpid());
Sergey Matveev83f91e72014-05-21 13:43:52 +0000550 int fd = CovOpenFile(false /* packed */, module_name);
551 if (fd > 0) {
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000552 internal_write(fd, offsets.data(), offsets.size() * sizeof(u32));
553 internal_close(fd);
554 VReport(1, " CovDump: %s: %zd PCs written\n", path.data(),
555 vb - old_vb);
556 }
Evgeniy Stepanov8ab205f2014-02-12 15:29:22 +0000557 }
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000558 }
559 }
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000560 if (cov_fd >= 0)
561 internal_close(cov_fd);
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000562 coverage_data.DumpCallerCalleePairs();
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000563#endif // !SANITIZER_WINDOWS
564}
565
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000566void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
567 if (!args) return;
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000568 if (!coverage_enabled) return;
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000569 cov_sandboxed = args->coverage_sandboxed;
570 if (!cov_sandboxed) return;
571 cov_fd = args->coverage_fd;
572 cov_max_block_size = args->coverage_max_block_size;
573 if (cov_fd < 0)
574 // Pre-open the file now. The sandbox won't allow us to do it later.
Sergey Matveev83f91e72014-05-21 13:43:52 +0000575 cov_fd = CovOpenFile(true /* packed */, 0);
Sergey Matveev6cb47a082014-05-19 12:53:03 +0000576}
577
Sergey Matveev83f91e72014-05-21 13:43:52 +0000578int MaybeOpenCovFile(const char *name) {
579 CHECK(name);
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000580 if (!coverage_enabled) return -1;
Sergey Matveev83f91e72014-05-21 13:43:52 +0000581 return CovOpenFile(true /* packed */, name);
582}
Evgeniy Stepanovfe181022014-06-04 12:13:54 +0000583
584void CovBeforeFork() {
585 coverage_data.BeforeFork();
586}
587
588void CovAfterFork(int child_pid) {
589 coverage_data.AfterFork(child_pid);
590}
591
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000592void InitializeCoverage(bool enabled, const char *dir) {
593 coverage_enabled = enabled;
594 coverage_dir = dir;
595 if (enabled) coverage_data.Init();
596#if !SANITIZER_WINDOWS
597 if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump);
598#endif
599}
600
601void ReInitializeCoverage(bool enabled, const char *dir) {
602 coverage_enabled = enabled;
603 coverage_dir = dir;
604 coverage_data.ReInit();
605}
606
607void CoverageUpdateMapping() {
608 if (coverage_enabled)
609 CovUpdateMapping(coverage_dir);
610}
611
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000612} // namespace __sanitizer
613
614extern "C" {
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000615SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) {
Kostya Serebryany4cadd4a2014-11-24 18:49:53 +0000616 coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()),
617 guard);
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000618}
Kostya Serebryanyb6eae0c2014-10-31 17:19:11 +0000619SANITIZER_INTERFACE_ATTRIBUTE void
620__sanitizer_cov_indir_call16(uptr callee, uptr callee_cache16[]) {
621 coverage_data.IndirCall(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()),
622 callee, callee_cache16, 16);
623}
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000624SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() {
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000625 coverage_enabled = true;
626 coverage_dir = common_flags()->coverage_dir;
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000627 coverage_data.Init();
628}
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000629SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); }
Kostya Serebryanyaa185bf2014-12-30 19:29:28 +0000630SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(s32 *guards,
Kostya Serebryany9fdeb372014-12-23 22:32:17 +0000631 uptr npcs) {
632 coverage_data.InitializeGuards(guards, npcs);
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000633 if (!common_flags()->coverage_direct) return;
634 if (SANITIZER_ANDROID && coverage_enabled) {
Evgeniy Stepanov38c228a2014-06-05 14:38:53 +0000635 // dlopen/dlclose interceptors do not work on Android, so we rely on
636 // Extend() calls to update .sancov.map.
Evgeniy Stepanov05dc4be2014-12-26 12:32:32 +0000637 CovUpdateMapping(coverage_dir, GET_CALLER_PC());
Evgeniy Stepanov38c228a2014-06-05 14:38:53 +0000638 }
Evgeniy Stepanov567e5162014-05-27 12:37:52 +0000639 coverage_data.Extend(npcs);
640}
Sergey Matveev83f91e72014-05-21 13:43:52 +0000641SANITIZER_INTERFACE_ATTRIBUTE
642sptr __sanitizer_maybe_open_cov_file(const char *name) {
643 return MaybeOpenCovFile(name);
644}
Kostya Serebryany183cb6e2014-11-14 23:15:55 +0000645SANITIZER_INTERFACE_ATTRIBUTE
646uptr __sanitizer_get_total_unique_coverage() {
647 return atomic_load(&coverage_counter, memory_order_relaxed);
648}
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000649
650SANITIZER_INTERFACE_ATTRIBUTE
Kostya Serebryanyd421db02015-01-03 00:54:43 +0000651void __sanitizer_cov_trace_func_enter(s32 *id) {
652 coverage_data.TraceBasicBlock(id);
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000653}
654SANITIZER_INTERFACE_ATTRIBUTE
Kostya Serebryanyd421db02015-01-03 00:54:43 +0000655void __sanitizer_cov_trace_basic_block(s32 *id) {
656 coverage_data.TraceBasicBlock(id);
Kostya Serebryanyc9d251e2014-11-19 00:24:11 +0000657}
Bob Wilsona08e9ac2013-11-15 07:18:15 +0000658} // extern "C"