blob: 5fe57bbb53decafb3d75fd429436619f373a7e65 [file] [log] [blame]
Aaron Ballmanef116982015-01-29 16:58:29 +00001//===- FuzzerLoop.cpp - Fuzzer's main loop --------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9// Fuzzer's main loop.
10//===----------------------------------------------------------------------===//
11
12#include "FuzzerInternal.h"
Aaron Ballmanef116982015-01-29 16:58:29 +000013#include <algorithm>
Kostya Serebryanyd50a3ee2016-01-13 23:02:30 +000014#include <cstring>
15#include <memory>
Aaron Ballmanef116982015-01-29 16:58:29 +000016
Kostya Serebryany2a48c242015-11-13 01:54:40 +000017#if defined(__has_include)
Ivan Krasindf919102016-01-22 22:28:27 +000018#if __has_include(<sanitizer / coverage_interface.h>)
19#include <sanitizer/coverage_interface.h>
20#endif
Kostya Serebryany1bfd5832016-04-20 00:24:21 +000021#if __has_include(<sanitizer / lsan_interface.h>)
22#include <sanitizer/lsan_interface.h>
23#endif
Kostya Serebryany2a48c242015-11-13 01:54:40 +000024#endif
25
Benjamin Kramerd96b0c12016-03-18 14:19:19 +000026#define NO_SANITIZE_MEMORY
27#if defined(__has_feature)
28#if __has_feature(memory_sanitizer)
29#undef NO_SANITIZE_MEMORY
30#define NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))
31#endif
32#endif
33
Kostya Serebryanyd6edce92015-10-16 23:04:31 +000034extern "C" {
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000035// Re-declare some of the sanitizer functions as "weak" so that
Kostya Serebryany2a48c242015-11-13 01:54:40 +000036// libFuzzer can be linked w/o the sanitizers and sanitizer-coverage
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000037// (in which case it will complain at start-up time).
Kostya Serebryanyd6edce92015-10-16 23:04:31 +000038__attribute__((weak)) void __sanitizer_print_stack_trace();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +000039__attribute__((weak)) void __sanitizer_reset_coverage();
Kostya Serebryany94660b32015-10-23 18:37:58 +000040__attribute__((weak)) size_t __sanitizer_get_total_unique_caller_callee_pairs();
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000041__attribute__((weak)) size_t __sanitizer_get_total_unique_coverage();
Ivan Krasindf919102016-01-22 22:28:27 +000042__attribute__((weak)) void
43__sanitizer_set_death_callback(void (*callback)(void));
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000044__attribute__((weak)) size_t __sanitizer_get_number_of_counters();
Ivan Krasindf919102016-01-22 22:28:27 +000045__attribute__((weak)) uintptr_t
46__sanitizer_update_counter_bitset_and_clear_counters(uint8_t *bitset);
Mike Aizatsky8b11f872016-01-06 00:21:22 +000047__attribute__((weak)) uintptr_t
48__sanitizer_get_coverage_pc_buffer(uintptr_t **data);
Kostya Serebryany22cc5e22016-02-13 02:29:38 +000049
50__attribute__((weak)) size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size,
51 size_t MaxSize,
52 unsigned int Seed);
Kostya Serebryany1bfd5832016-04-20 00:24:21 +000053__attribute__((weak)) void __sanitizer_malloc_hook(void *ptr, size_t size);
54__attribute__((weak)) void __sanitizer_free_hook(void *ptr);
55__attribute__((weak)) void __lsan_enable();
56__attribute__((weak)) void __lsan_disable();
57__attribute__((weak)) int __lsan_do_recoverable_leak_check();
Kostya Serebryanyd6edce92015-10-16 23:04:31 +000058}
59
Aaron Ballmanef116982015-01-29 16:58:29 +000060namespace fuzzer {
Kostya Serebryanya9da9b42015-10-16 22:47:20 +000061static const size_t kMaxUnitSizeToPrint = 256;
Aaron Ballmanef116982015-01-29 16:58:29 +000062
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000063static void MissingWeakApiFunction(const char *FnName) {
64 Printf("ERROR: %s is not defined. Exiting.\n"
Ivan Krasindf919102016-01-22 22:28:27 +000065 "Did you use -fsanitize-coverage=... to build your code?\n",
66 FnName);
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000067 exit(1);
68}
69
70#define CHECK_WEAK_API_FUNCTION(fn) \
71 do { \
72 if (!fn) \
73 MissingWeakApiFunction(#fn); \
74 } while (false)
75
Kostya Serebryany52a788e2015-03-31 20:13:20 +000076// Only one Fuzzer per process.
77static Fuzzer *F;
78
Kostya Serebryany1deb0492016-02-13 06:24:18 +000079size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize) {
80 assert(F);
81 return F->GetMD().Mutate(Data, Size, MaxSize);
82}
83
Kostya Serebryany7ec0c562016-02-13 03:25:16 +000084Fuzzer::Fuzzer(UserCallback CB, MutationDispatcher &MD, FuzzingOptions Options)
85 : CB(CB), MD(MD), Options(Options) {
Kostya Serebryany52a788e2015-03-31 20:13:20 +000086 SetDeathCallback();
Kostya Serebryany22526252015-05-11 21:16:27 +000087 InitializeTraceState();
Kostya Serebryany52a788e2015-03-31 20:13:20 +000088 assert(!F);
89 F = this;
90}
Aaron Ballmanef116982015-01-29 16:58:29 +000091
92void Fuzzer::SetDeathCallback() {
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000093 CHECK_WEAK_API_FUNCTION(__sanitizer_set_death_callback);
Kostya Serebryany52a788e2015-03-31 20:13:20 +000094 __sanitizer_set_death_callback(StaticDeathCallback);
95}
96
Kostya Serebryany52a788e2015-03-31 20:13:20 +000097void Fuzzer::StaticDeathCallback() {
98 assert(F);
99 F->DeathCallback();
Aaron Ballmanef116982015-01-29 16:58:29 +0000100}
101
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000102void Fuzzer::DumpCurrentUnit(const char *Prefix) {
Kostya Serebryany98abb2c2016-01-13 23:46:01 +0000103 if (CurrentUnitSize <= kMaxUnitSizeToPrint) {
104 PrintHexArray(CurrentUnitData, CurrentUnitSize, "\n");
105 PrintASCII(CurrentUnitData, CurrentUnitSize, "\n");
Kostya Serebryanye95022a2015-10-09 04:03:14 +0000106 }
Kostya Serebryany98abb2c2016-01-13 23:46:01 +0000107 WriteUnitToFileWithPrefix(
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000108 {CurrentUnitData, CurrentUnitData + CurrentUnitSize}, Prefix);
109}
110
Benjamin Kramerd96b0c12016-03-18 14:19:19 +0000111NO_SANITIZE_MEMORY
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000112void Fuzzer::DeathCallback() {
113 if (!CurrentUnitSize) return;
114 Printf("DEATH:\n");
115 DumpCurrentUnit("crash-");
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000116 PrintFinalStats();
Aaron Ballmanef116982015-01-29 16:58:29 +0000117}
118
Kostya Serebryany52a788e2015-03-31 20:13:20 +0000119void Fuzzer::StaticAlarmCallback() {
120 assert(F);
121 F->AlarmCallback();
122}
123
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000124void Fuzzer::StaticCrashSignalCallback() {
125 assert(F);
126 F->CrashCallback();
127}
128
129void Fuzzer::StaticInterruptCallback() {
130 assert(F);
131 F->InterruptCallback();
132}
133
134void Fuzzer::CrashCallback() {
135 Printf("==%d== ERROR: libFuzzer: deadly signal\n", GetPid());
136 if (__sanitizer_print_stack_trace)
137 __sanitizer_print_stack_trace();
138 Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
139 " Combine libFuzzer with AddressSanitizer or similar for better "
140 "crash reports.\n");
141 Printf("SUMMARY: libFuzzer: deadly signal\n");
142 DumpCurrentUnit("crash-");
143 PrintFinalStats();
144 exit(Options.ErrorExitCode);
145}
146
147void Fuzzer::InterruptCallback() {
148 Printf("==%d== libFuzzer: run interrupted; exiting\n", GetPid());
149 PrintFinalStats();
Kostya Serebryanye483ed22016-03-03 22:36:37 +0000150 _Exit(0); // Stop right now, don't perform any at-exit actions.
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000151}
152
Benjamin Kramerd96b0c12016-03-18 14:19:19 +0000153NO_SANITIZE_MEMORY
Aaron Ballmanef116982015-01-29 16:58:29 +0000154void Fuzzer::AlarmCallback() {
Kostya Serebryany490bbd62015-05-19 22:12:57 +0000155 assert(Options.UnitTimeoutSec > 0);
Kostya Serebryanycfbcf902016-02-17 19:42:34 +0000156 if (!CurrentUnitSize)
157 return; // We have not started running units yet.
Aaron Ballmanef116982015-01-29 16:58:29 +0000158 size_t Seconds =
159 duration_cast<seconds>(system_clock::now() - UnitStartTime).count();
Ivan Krasindf919102016-01-22 22:28:27 +0000160 if (Seconds == 0)
161 return;
Kostya Serebryany490bbd62015-05-19 22:12:57 +0000162 if (Options.Verbosity >= 2)
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000163 Printf("AlarmCallback %zd\n", Seconds);
Kostya Serebryany490bbd62015-05-19 22:12:57 +0000164 if (Seconds >= (size_t)Options.UnitTimeoutSec) {
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000165 Printf("ALARM: working on the last Unit for %zd seconds\n", Seconds);
Kostya Serebryany316b5712015-05-26 20:57:47 +0000166 Printf(" and the timeout value is %d (use -timeout=N to change)\n",
167 Options.UnitTimeoutSec);
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000168 DumpCurrentUnit("timeout-");
Kostya Serebryanyd6edce92015-10-16 23:04:31 +0000169 Printf("==%d== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
170 Seconds);
171 if (__sanitizer_print_stack_trace)
172 __sanitizer_print_stack_trace();
173 Printf("SUMMARY: libFuzzer: timeout\n");
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000174 PrintFinalStats();
Kostya Serebryany31516732016-03-24 01:32:08 +0000175 _Exit(Options.TimeoutExitCode); // Stop right now.
Aaron Ballmanef116982015-01-29 16:58:29 +0000176 }
Aaron Ballmanef116982015-01-29 16:58:29 +0000177}
178
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000179void Fuzzer::PrintStats(const char *Where, const char *End) {
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000180 size_t ExecPerSec = execPerSec();
Mike Aizatskya9c23872015-11-12 04:38:40 +0000181 if (Options.OutputCSV) {
182 static bool csvHeaderPrinted = false;
183 if (!csvHeaderPrinted) {
184 csvHeaderPrinted = true;
185 Printf("runs,block_cov,bits,cc_cov,corpus,execs_per_sec,tbms,reason\n");
186 }
187 Printf("%zd,%zd,%zd,%zd,%zd,%zd,%zd,%s\n", TotalNumberOfRuns,
188 LastRecordedBlockCoverage, TotalBits(),
189 LastRecordedCallerCalleeCoverage, Corpus.size(), ExecPerSec,
190 TotalNumberOfExecutedTraceBasedMutations, Where);
191 }
192
193 if (!Options.Verbosity)
194 return;
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000195 Printf("#%zd\t%s", TotalNumberOfRuns, Where);
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000196 if (LastRecordedBlockCoverage)
197 Printf(" cov: %zd", LastRecordedBlockCoverage);
Kostya Serebryanyda63c1d2016-02-26 21:33:56 +0000198 if (LastRecordedPcMapSize)
199 Printf(" path: %zd", LastRecordedPcMapSize);
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000200 if (auto TB = TotalBits())
201 Printf(" bits: %zd", TB);
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000202 if (LastRecordedCallerCalleeCoverage)
203 Printf(" indir: %zd", LastRecordedCallerCalleeCoverage);
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000204 Printf(" units: %zd exec/s: %zd", Corpus.size(), ExecPerSec);
Kostya Serebryany12c78372015-08-12 01:55:37 +0000205 if (TotalNumberOfExecutedTraceBasedMutations)
206 Printf(" tbm: %zd", TotalNumberOfExecutedTraceBasedMutations);
207 Printf("%s", End);
Kostya Serebryany03db8b92015-03-30 22:44:03 +0000208}
209
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000210void Fuzzer::PrintFinalStats() {
211 if (!Options.PrintFinalStats) return;
212 size_t ExecPerSec = execPerSec();
213 Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns);
214 Printf("stat::average_exec_per_sec: %zd\n", ExecPerSec);
215 Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded);
216 Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds);
217 Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb());
218}
219
Kostya Serebryany64d24572016-03-12 01:57:04 +0000220size_t Fuzzer::MaxUnitSizeInCorpus() const {
221 size_t Res = 0;
222 for (auto &X : Corpus)
223 Res = std::max(Res, X.size());
224 return Res;
225}
226
227void Fuzzer::SetMaxLen(size_t MaxLen) {
228 assert(Options.MaxLen == 0); // Can only reset MaxLen from 0 to non-0.
229 assert(MaxLen);
230 Options.MaxLen = MaxLen;
231 Printf("INFO: -max_len is not provided, using %zd\n", Options.MaxLen);
232}
233
234
235void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
Ivan Krasindf919102016-01-22 22:28:27 +0000236 if (Options.OutputCorpus.empty())
237 return;
Kostya Serebryany1ac80552015-05-08 21:30:55 +0000238 std::vector<Unit> AdditionalCorpus;
239 ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
Kostya Serebryany64d24572016-03-12 01:57:04 +0000240 &EpochOfLastReadOfOutputCorpus, MaxSize);
Kostya Serebryany1ac80552015-05-08 21:30:55 +0000241 if (Corpus.empty()) {
242 Corpus = AdditionalCorpus;
243 return;
244 }
Ivan Krasindf919102016-01-22 22:28:27 +0000245 if (!Options.Reload)
246 return;
Kostya Serebryany490bbd62015-05-19 22:12:57 +0000247 if (Options.Verbosity >= 2)
Ivan Krasindf919102016-01-22 22:28:27 +0000248 Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
Kostya Serebryany1ac80552015-05-08 21:30:55 +0000249 for (auto &X : AdditionalCorpus) {
Kostya Serebryany64d24572016-03-12 01:57:04 +0000250 if (X.size() > MaxSize)
251 X.resize(MaxSize);
Kostya Serebryanycbb23342015-05-19 01:06:07 +0000252 if (UnitHashesAddedToCorpus.insert(Hash(X)).second) {
Kostya Serebryany98abb2c2016-01-13 23:46:01 +0000253 if (RunOne(X)) {
Kostya Serebryanycbb23342015-05-19 01:06:07 +0000254 Corpus.push_back(X);
Ivan Krasindf919102016-01-22 22:28:27 +0000255 UpdateCorpusDistribution();
Mike Aizatskya9c23872015-11-12 04:38:40 +0000256 PrintStats("RELOAD");
Kostya Serebryanycbb23342015-05-19 01:06:07 +0000257 }
Kostya Serebryany1ac80552015-05-08 21:30:55 +0000258 }
259 }
260}
261
Kostya Serebryany945761b2016-03-18 00:23:29 +0000262void Fuzzer::ShuffleCorpus(UnitVector *V) {
263 std::random_shuffle(V->begin(), V->end(), MD.GetRand());
264 if (Options.PreferSmall)
265 std::stable_sort(V->begin(), V->end(), [](const Unit &A, const Unit &B) {
266 return A.size() < B.size();
267 });
268}
269
Aaron Ballmanef116982015-01-29 16:58:29 +0000270void Fuzzer::ShuffleAndMinimize() {
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000271 PrintStats("READ ");
Aaron Ballmanef116982015-01-29 16:58:29 +0000272 std::vector<Unit> NewCorpus;
Kostya Serebryany945761b2016-03-18 00:23:29 +0000273 if (Options.ShuffleAtStartUp)
274 ShuffleCorpus(&Corpus);
275
276 for (const auto &U : Corpus) {
277 if (RunOne(U)) {
278 NewCorpus.push_back(U);
279 if (Options.Verbosity >= 2)
280 Printf("NEW0: %zd L %zd\n", LastRecordedBlockCoverage, U.size());
Aaron Ballmanef116982015-01-29 16:58:29 +0000281 }
282 }
283 Corpus = NewCorpus;
Ivan Krasindf919102016-01-22 22:28:27 +0000284 UpdateCorpusDistribution();
Kostya Serebryanycbb23342015-05-19 01:06:07 +0000285 for (auto &X : Corpus)
286 UnitHashesAddedToCorpus.insert(Hash(X));
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000287 PrintStats("INITED");
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000288 CheckForMemoryLeaks();
Aaron Ballmanef116982015-01-29 16:58:29 +0000289}
290
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000291bool Fuzzer::RunOne(const uint8_t *Data, size_t Size) {
Aaron Ballmanef116982015-01-29 16:58:29 +0000292 TotalNumberOfRuns++;
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000293
294 PrepareCoverageBeforeRun();
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000295 ExecuteCallback(Data, Size);
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000296 bool Res = CheckCoverageAfterRun();
297
Kostya Serebryany16901a92015-03-30 23:04:35 +0000298 auto UnitStopTime = system_clock::now();
299 auto TimeOfUnit =
300 duration_cast<seconds>(UnitStopTime - UnitStartTime).count();
Mike Aizatskya9c23872015-11-12 04:38:40 +0000301 if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) &&
302 secondsSinceProcessStartUp() >= 2)
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000303 PrintStats("pulse ");
Kostya Serebryany70926ae2015-08-05 21:43:48 +0000304 if (TimeOfUnit > TimeOfLongestUnitInSeconds &&
305 TimeOfUnit >= Options.ReportSlowUnits) {
Kostya Serebryany16901a92015-03-30 23:04:35 +0000306 TimeOfLongestUnitInSeconds = TimeOfUnit;
Kostya Serebryany70926ae2015-08-05 21:43:48 +0000307 Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000308 WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-");
Kostya Serebryany16901a92015-03-30 23:04:35 +0000309 }
310 return Res;
Kostya Serebryany2c1b33b2015-01-29 23:01:07 +0000311}
312
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000313void Fuzzer::RunOneAndUpdateCorpus(uint8_t *Data, size_t Size) {
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000314 if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
315 return;
Kostya Serebryanybc7c0ad2015-08-11 01:44:42 +0000316 if (Options.OnlyASCII)
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000317 ToASCII(Data, Size);
318 if (RunOne(Data, Size))
319 ReportNewCoverage({Data, Data + Size});
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000320}
321
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000322// Leak detection is expensive, so we first check if there were more mallocs
323// than frees (using the sanitizer malloc hooks) and only then try to call lsan.
324struct MallocFreeTracer {
325 void Start() {
326 Mallocs = 0;
327 Frees = 0;
328 }
329 // Returns true if there were more mallocs than frees.
330 bool Stop() { return Mallocs > Frees; }
331 size_t Mallocs;
332 size_t Frees;
333};
334
335static thread_local MallocFreeTracer AllocTracer;
336
337extern "C" {
338void __sanitizer_malloc_hook(void *ptr, size_t size) { AllocTracer.Mallocs++; }
339void __sanitizer_free_hook(void *ptr) { AllocTracer.Frees++; }
340} // extern "C"
341
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000342void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
Kostya Serebryanyebb932d2016-04-18 22:50:39 +0000343 UnitStartTime = system_clock::now();
Kostya Serebryanyd50a3ee2016-01-13 23:02:30 +0000344 // We copy the contents of Unit into a separate heap buffer
345 // so that we reliably find buffer overflows in it.
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000346 std::unique_ptr<uint8_t[]> DataCopy(new uint8_t[Size]);
347 memcpy(DataCopy.get(), Data, Size);
348 AssignTaintLabels(DataCopy.get(), Size);
349 CurrentUnitData = DataCopy.get();
350 CurrentUnitSize = Size;
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000351 AllocTracer.Start();
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000352 int Res = CB(DataCopy.get(), Size);
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000353 (void)Res;
354 HasMoreMallocsThanFrees = AllocTracer.Stop();
Kostya Serebryanyebb932d2016-04-18 22:50:39 +0000355 CurrentUnitSize = 0;
356 CurrentUnitData = nullptr;
Kostya Serebryany20bb5e72015-10-02 23:34:06 +0000357 assert(Res == 0);
Kostya Serebryany52a788e2015-03-31 20:13:20 +0000358}
359
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000360size_t Fuzzer::RecordBlockCoverage() {
Kostya Serebryany5eab74e2015-11-09 23:17:45 +0000361 CHECK_WEAK_API_FUNCTION(__sanitizer_get_total_unique_coverage);
Mike Aizatsky8b11f872016-01-06 00:21:22 +0000362 uintptr_t PrevCoverage = LastRecordedBlockCoverage;
363 LastRecordedBlockCoverage = __sanitizer_get_total_unique_coverage();
364
365 if (PrevCoverage == LastRecordedBlockCoverage || !Options.PrintNewCovPcs)
366 return LastRecordedBlockCoverage;
367
368 uintptr_t PrevBufferLen = LastCoveragePcBufferLen;
369 uintptr_t *CoverageBuf;
370 LastCoveragePcBufferLen = __sanitizer_get_coverage_pc_buffer(&CoverageBuf);
371 assert(CoverageBuf);
372 for (size_t i = PrevBufferLen; i < LastCoveragePcBufferLen; ++i) {
Mike Aizatskye313f8f2016-01-21 00:02:09 +0000373 Printf("%p\n", CoverageBuf[i]);
Mike Aizatsky8b11f872016-01-06 00:21:22 +0000374 }
375
376 return LastRecordedBlockCoverage;
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000377}
378
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000379size_t Fuzzer::RecordCallerCalleeCoverage() {
380 if (!Options.UseIndirCalls)
381 return 0;
Kostya Serebryany94660b32015-10-23 18:37:58 +0000382 if (!__sanitizer_get_total_unique_caller_callee_pairs)
383 return 0;
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000384 return LastRecordedCallerCalleeCoverage =
385 __sanitizer_get_total_unique_caller_callee_pairs();
386}
387
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000388void Fuzzer::PrepareCoverageBeforeRun() {
Kostya Serebryanybe5e0ed2015-03-03 23:27:02 +0000389 if (Options.UseCounters) {
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000390 size_t NumCounters = __sanitizer_get_number_of_counters();
Kostya Serebryanybe5e0ed2015-03-03 23:27:02 +0000391 CounterBitmap.resize(NumCounters);
392 __sanitizer_update_counter_bitset_and_clear_counters(0);
393 }
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000394 RecordBlockCoverage();
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000395 RecordCallerCalleeCoverage();
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000396}
397
398bool Fuzzer::CheckCoverageAfterRun() {
399 size_t OldCoverage = LastRecordedBlockCoverage;
400 size_t NewCoverage = RecordBlockCoverage();
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000401 size_t OldCallerCalleeCoverage = LastRecordedCallerCalleeCoverage;
402 size_t NewCallerCalleeCoverage = RecordCallerCalleeCoverage();
Kostya Serebryanybe5e0ed2015-03-03 23:27:02 +0000403 size_t NumNewBits = 0;
Kostya Serebryanyda63c1d2016-02-26 21:33:56 +0000404 size_t OldPcMapSize = LastRecordedPcMapSize;
405 PcMapMergeCurrentToCombined();
406 size_t NewPcMapSize = PcMapCombinedSize();
407 LastRecordedPcMapSize = NewPcMapSize;
408 if (NewPcMapSize > OldPcMapSize)
409 return true;
Kostya Serebryanybe5e0ed2015-03-03 23:27:02 +0000410 if (Options.UseCounters)
411 NumNewBits = __sanitizer_update_counter_bitset_and_clear_counters(
412 CounterBitmap.data());
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000413 return NewCoverage > OldCoverage ||
414 NewCallerCalleeCoverage > OldCallerCalleeCoverage || NumNewBits;
Aaron Ballmanef116982015-01-29 16:58:29 +0000415}
416
417void Fuzzer::WriteToOutputCorpus(const Unit &U) {
Ivan Krasindf919102016-01-22 22:28:27 +0000418 if (Options.OutputCorpus.empty())
419 return;
Aaron Ballmanef116982015-01-29 16:58:29 +0000420 std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U));
421 WriteToFile(U, Path);
422 if (Options.Verbosity >= 2)
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000423 Printf("Written to %s\n", Path.c_str());
Kostya Serebryanya9346c22015-09-02 19:08:08 +0000424 assert(!Options.OnlyASCII || IsASCII(U));
Aaron Ballmanef116982015-01-29 16:58:29 +0000425}
426
Kostya Serebryany2b7d2e92015-07-23 18:37:22 +0000427void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) {
Kostya Serebryanyb91c62b2015-10-16 22:41:47 +0000428 if (!Options.SaveArtifacts)
429 return;
Kostya Serebryanybd5d1cd2015-10-09 03:57:59 +0000430 std::string Path = Options.ArtifactPrefix + Prefix + Hash(U);
Kostya Serebryany2d0ef142015-11-25 21:40:46 +0000431 if (!Options.ExactArtifactPath.empty())
Ivan Krasindf919102016-01-22 22:28:27 +0000432 Path = Options.ExactArtifactPath; // Overrides ArtifactPrefix.
Aaron Ballmanef116982015-01-29 16:58:29 +0000433 WriteToFile(U, Path);
Kostya Serebryanybd5d1cd2015-10-09 03:57:59 +0000434 Printf("artifact_prefix='%s'; Test unit written to %s\n",
435 Options.ArtifactPrefix.c_str(), Path.c_str());
Kostya Serebryany9e48cda2015-12-04 22:29:39 +0000436 if (U.size() <= kMaxUnitSizeToPrint)
437 Printf("Base64: %s\n", Base64(U).c_str());
Aaron Ballmanef116982015-01-29 16:58:29 +0000438}
439
440void Fuzzer::SaveCorpus() {
Ivan Krasindf919102016-01-22 22:28:27 +0000441 if (Options.OutputCorpus.empty())
442 return;
Aaron Ballmanef116982015-01-29 16:58:29 +0000443 for (const auto &U : Corpus)
444 WriteToFile(U, DirPlusFile(Options.OutputCorpus, Hash(U)));
445 if (Options.Verbosity)
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000446 Printf("Written corpus of %zd files to %s\n", Corpus.size(),
447 Options.OutputCorpus.c_str());
Aaron Ballmanef116982015-01-29 16:58:29 +0000448}
449
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000450void Fuzzer::PrintStatusForNewUnit(const Unit &U) {
451 if (!Options.PrintNEW)
452 return;
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000453 PrintStats("NEW ", "");
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000454 if (Options.Verbosity) {
Kostya Serebryany14c50282015-12-19 01:09:49 +0000455 Printf(" L: %zd ", U.size());
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000456 MD.PrintMutationSequence();
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000457 Printf("\n");
Aaron Ballmanef116982015-01-29 16:58:29 +0000458 }
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000459}
460
461void Fuzzer::ReportNewCoverage(const Unit &U) {
462 Corpus.push_back(U);
Ivan Krasindf919102016-01-22 22:28:27 +0000463 UpdateCorpusDistribution();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000464 UnitHashesAddedToCorpus.insert(Hash(U));
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000465 MD.RecordSuccessfulMutationSequence();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000466 PrintStatusForNewUnit(U);
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000467 WriteToOutputCorpus(U);
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000468 NumberOfNewUnitsAdded++;
Aaron Ballmanef116982015-01-29 16:58:29 +0000469}
470
Kostya Serebryany945761b2016-03-18 00:23:29 +0000471// Finds minimal number of units in 'Extra' that add coverage to 'Initial'.
472// We do it by actually executing the units, sometimes more than once,
473// because we may be using different coverage-like signals and the only
474// common thing between them is that we can say "this unit found new stuff".
475UnitVector Fuzzer::FindExtraUnits(const UnitVector &Initial,
476 const UnitVector &Extra) {
477 UnitVector Res = Extra;
478 size_t OldSize = Res.size();
479 for (int Iter = 0; Iter < 10; Iter++) {
480 ShuffleCorpus(&Res);
481 ResetCoverage();
482
483 for (auto &U : Initial)
484 RunOne(U);
485
486 Corpus.clear();
487 for (auto &U : Res)
488 if (RunOne(U))
489 Corpus.push_back(U);
490
491 char Stat[7] = "MIN ";
492 Stat[3] = '0' + Iter;
493 PrintStats(Stat);
494
495 size_t NewSize = Corpus.size();
496 Res.swap(Corpus);
497
498 if (NewSize == OldSize)
499 break;
500 OldSize = NewSize;
501 }
502 return Res;
503}
504
Kostya Serebryany9cc3b0d2015-10-24 01:16:40 +0000505void Fuzzer::Merge(const std::vector<std::string> &Corpora) {
506 if (Corpora.size() <= 1) {
507 Printf("Merge requires two or more corpus dirs\n");
508 return;
509 }
Kostya Serebryany9cc3b0d2015-10-24 01:16:40 +0000510 std::vector<std::string> ExtraCorpora(Corpora.begin() + 1, Corpora.end());
511
Kostya Serebryany945761b2016-03-18 00:23:29 +0000512 assert(Options.MaxLen > 0);
513 UnitVector Initial, Extra;
514 ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, Options.MaxLen);
515 for (auto &C : ExtraCorpora)
516 ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, Options.MaxLen);
517
518 if (!Initial.empty()) {
519 Printf("=== Minimizing the initial corpus of %zd units\n", Initial.size());
520 Initial = FindExtraUnits({}, Initial);
Kostya Serebryany9cc3b0d2015-10-24 01:16:40 +0000521 }
Kostya Serebryany945761b2016-03-18 00:23:29 +0000522
523 Printf("=== Merging extra %zd units\n", Extra.size());
524 auto Res = FindExtraUnits(Initial, Extra);
525
526 for (auto &U: Res)
527 WriteToOutputCorpus(U);
528
529 Printf("=== Merge: written %zd units\n", Res.size());
Kostya Serebryany9cc3b0d2015-10-24 01:16:40 +0000530}
531
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000532// Tries to call lsan, and if there are leaks exits. We call this right after
533// the initial corpus was read because if there are leaky inputs in the corpus
534// further fuzzing will likely hit OOMs.
535void Fuzzer::CheckForMemoryLeaks() {
536 if (!Options.DetectLeaks) return;
537 if (!__lsan_do_recoverable_leak_check)
538 return;
539 if (__lsan_do_recoverable_leak_check()) {
540 Printf("==%d== ERROR: libFuzzer: initial corpus triggers memory leaks.\n"
541 "Exiting now. Use -detect_leaks=0 to disable leak detection here.\n"
542 "LeakSanitizer will still check for leaks at the process exit.\n",
543 GetPid());
544 PrintFinalStats();
545 _Exit(Options.ErrorExitCode);
546 }
547}
548
549// Tries detecting a memory leak on the particular input that we have just
550// executed before calling this function.
551void Fuzzer::TryDetectingAMemoryLeak(uint8_t *Data, size_t Size) {
552 if (!HasMoreMallocsThanFrees) return; // mallocs==frees, a leak is unlikely.
553 if (!Options.DetectLeaks) return;
554 if (!&__lsan_enable || !&__lsan_disable || !__lsan_do_recoverable_leak_check)
555 return; // No lsan.
556 // Run the target once again, but with lsan disabled so that if there is
557 // a real leak we do not report it twice.
558 __lsan_disable();
559 RunOneAndUpdateCorpus(Data, Size);
560 __lsan_enable();
561 if (!HasMoreMallocsThanFrees) return; // a leak is unlikely.
562 // Now perform the actual lsan pass. This is expensive and we must ensure
563 // we don't call it too often.
564 if (__lsan_do_recoverable_leak_check()) { // Leak is found, report it.
565 CurrentUnitData = Data;
566 CurrentUnitSize = Size;
567 DumpCurrentUnit("leak-");
568 PrintFinalStats();
569 _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on.
570 }
571}
572
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000573void Fuzzer::MutateAndTestOne() {
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000574 MD.StartMutationSequence();
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000575
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000576 auto &U = ChooseUnitToMutate();
577 MutateInPlaceHere.resize(Options.MaxLen);
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000578 size_t Size = U.size();
Mike Aizatsky94e29662016-04-08 23:32:24 +0000579 assert(Size <= Options.MaxLen && "Oversized Unit");
580 memcpy(MutateInPlaceHere.data(), U.data(), Size);
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000581
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000582 for (int i = 0; i < Options.MutateDepth; i++) {
Kostya Serebryany22cc5e22016-02-13 02:29:38 +0000583 size_t NewSize = 0;
584 if (LLVMFuzzerCustomMutator)
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000585 NewSize = LLVMFuzzerCustomMutator(MutateInPlaceHere.data(), Size,
586 Options.MaxLen, MD.GetRand().Rand());
Kostya Serebryany22cc5e22016-02-13 02:29:38 +0000587 else
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000588 NewSize = MD.Mutate(MutateInPlaceHere.data(), Size, Options.MaxLen);
Kostya Serebryany2ea204e2015-05-30 17:33:13 +0000589 assert(NewSize > 0 && "Mutator returned empty unit");
Kostya Serebryany64d24572016-03-12 01:57:04 +0000590 assert(NewSize <= Options.MaxLen &&
Kostya Serebryany2ea204e2015-05-30 17:33:13 +0000591 "Mutator return overisized unit");
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000592 Size = NewSize;
Kostya Serebryanyb65805a2016-01-09 03:08:58 +0000593 if (i == 0)
594 StartTraceRecording();
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000595 RunOneAndUpdateCorpus(MutateInPlaceHere.data(), Size);
Kostya Serebryanyb65805a2016-01-09 03:08:58 +0000596 StopTraceRecording();
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000597 TryDetectingAMemoryLeak(MutateInPlaceHere.data(), Size);
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000598 }
599}
600
Kostya Serebryanye6926212015-11-04 23:22:25 +0000601// Returns an index of random unit from the corpus to mutate.
602// Hypothesis: units added to the corpus last are more likely to be interesting.
Ivan Krasindf919102016-01-22 22:28:27 +0000603// This function gives more weight to the more recent units.
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000604size_t Fuzzer::ChooseUnitIdxToMutate() {
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000605 size_t Idx =
606 static_cast<size_t>(CorpusDistribution(MD.GetRand().Get_mt19937()));
Ivan Krasindf919102016-01-22 22:28:27 +0000607 assert(Idx < Corpus.size());
608 return Idx;
Kostya Serebryanye6926212015-11-04 23:22:25 +0000609}
610
Kostya Serebryany945761b2016-03-18 00:23:29 +0000611void Fuzzer::ResetCoverage() {
612 CHECK_WEAK_API_FUNCTION(__sanitizer_reset_coverage);
613 __sanitizer_reset_coverage();
614 CounterBitmap.clear();
615}
616
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000617// Experimental search heuristic: drilling.
618// - Read, shuffle, execute and minimize the corpus.
619// - Choose one random unit.
620// - Reset the coverage.
621// - Start fuzzing as if the chosen unit was the only element of the corpus.
622// - When done, reset the coverage again.
623// - Merge the newly created corpus into the original one.
624void Fuzzer::Drill() {
625 // The corpus is already read, shuffled, and minimized.
626 assert(!Corpus.empty());
Ivan Krasindf919102016-01-22 22:28:27 +0000627 Options.PrintNEW = false; // Don't print NEW status lines when drilling.
Kostya Serebryany7d211662015-09-04 00:12:11 +0000628
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000629 Unit U = ChooseUnitToMutate();
630
Kostya Serebryany945761b2016-03-18 00:23:29 +0000631 ResetCoverage();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000632
633 std::vector<Unit> SavedCorpus;
634 SavedCorpus.swap(Corpus);
635 Corpus.push_back(U);
Ivan Krasindf919102016-01-22 22:28:27 +0000636 UpdateCorpusDistribution();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000637 assert(Corpus.size() == 1);
638 RunOne(U);
639 PrintStats("DRILL ");
640 std::string SavedOutputCorpusPath; // Don't write new units while drilling.
641 SavedOutputCorpusPath.swap(Options.OutputCorpus);
642 Loop();
643
Kostya Serebryany945761b2016-03-18 00:23:29 +0000644 ResetCoverage();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000645
646 PrintStats("REINIT");
647 SavedOutputCorpusPath.swap(Options.OutputCorpus);
Kostya Serebryany98abb2c2016-01-13 23:46:01 +0000648 for (auto &U : SavedCorpus)
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000649 RunOne(U);
650 PrintStats("MERGE ");
651 Options.PrintNEW = true;
652 size_t NumMerged = 0;
653 for (auto &U : Corpus) {
654 if (RunOne(U)) {
655 PrintStatusForNewUnit(U);
656 NumMerged++;
657 WriteToOutputCorpus(U);
658 }
659 }
660 PrintStats("MERGED");
661 if (NumMerged && Options.Verbosity)
662 Printf("Drilling discovered %zd new units\n", NumMerged);
663}
664
665void Fuzzer::Loop() {
Kostya Serebryany8617aaa2015-12-05 02:09:22 +0000666 system_clock::time_point LastCorpusReload = system_clock::now();
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000667 if (Options.DoCrossOver)
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000668 MD.SetCorpus(&Corpus);
Kostya Serebryany468ed782015-09-08 17:30:35 +0000669 while (true) {
Kostya Serebryany8617aaa2015-12-05 02:09:22 +0000670 auto Now = system_clock::now();
671 if (duration_cast<seconds>(Now - LastCorpusReload).count()) {
Kostya Serebryany64d24572016-03-12 01:57:04 +0000672 RereadOutputCorpus(Options.MaxLen);
Kostya Serebryany8617aaa2015-12-05 02:09:22 +0000673 LastCorpusReload = Now;
674 }
Kostya Serebryanye6926212015-11-04 23:22:25 +0000675 if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
Mike Aizatskya9c23872015-11-12 04:38:40 +0000676 break;
Kostya Serebryanye6926212015-11-04 23:22:25 +0000677 if (Options.MaxTotalTimeSec > 0 &&
678 secondsSinceProcessStartUp() >
Ivan Krasindf919102016-01-22 22:28:27 +0000679 static_cast<size_t>(Options.MaxTotalTimeSec))
Mike Aizatskya9c23872015-11-12 04:38:40 +0000680 break;
Kostya Serebryanye6926212015-11-04 23:22:25 +0000681 // Perform several mutations and runs.
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000682 MutateAndTestOne();
Aaron Ballmanef116982015-01-29 16:58:29 +0000683 }
Mike Aizatskya9c23872015-11-12 04:38:40 +0000684
685 PrintStats("DONE ", "\n");
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000686 MD.PrintRecommendedDictionary();
Aaron Ballmanef116982015-01-29 16:58:29 +0000687}
688
Ivan Krasindf919102016-01-22 22:28:27 +0000689void Fuzzer::UpdateCorpusDistribution() {
690 size_t N = Corpus.size();
691 std::vector<double> Intervals(N + 1);
692 std::vector<double> Weights(N);
693 std::iota(Intervals.begin(), Intervals.end(), 0);
694 std::iota(Weights.begin(), Weights.end(), 1);
695 CorpusDistribution = std::piecewise_constant_distribution<double>(
696 Intervals.begin(), Intervals.end(), Weights.begin());
697}
698
699} // namespace fuzzer