blob: 3b0b339bf9c701f30b7845eba1090809caa596a2 [file] [log] [blame]
Aaron Ballmanef116982015-01-29 16:58:29 +00001//===- FuzzerLoop.cpp - Fuzzer's main loop --------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9// Fuzzer's main loop.
10//===----------------------------------------------------------------------===//
11
12#include "FuzzerInternal.h"
Aaron Ballmanef116982015-01-29 16:58:29 +000013#include <algorithm>
Kostya Serebryanyd50a3ee2016-01-13 23:02:30 +000014#include <cstring>
15#include <memory>
Aaron Ballmanef116982015-01-29 16:58:29 +000016
Kostya Serebryany2a48c242015-11-13 01:54:40 +000017#if defined(__has_include)
Ivan Krasindf919102016-01-22 22:28:27 +000018#if __has_include(<sanitizer / coverage_interface.h>)
19#include <sanitizer/coverage_interface.h>
20#endif
Kostya Serebryany1bfd5832016-04-20 00:24:21 +000021#if __has_include(<sanitizer / lsan_interface.h>)
22#include <sanitizer/lsan_interface.h>
23#endif
Kostya Serebryany2a48c242015-11-13 01:54:40 +000024#endif
25
Benjamin Kramerd96b0c12016-03-18 14:19:19 +000026#define NO_SANITIZE_MEMORY
27#if defined(__has_feature)
28#if __has_feature(memory_sanitizer)
29#undef NO_SANITIZE_MEMORY
30#define NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))
31#endif
32#endif
33
Kostya Serebryanyd6edce92015-10-16 23:04:31 +000034extern "C" {
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000035// Re-declare some of the sanitizer functions as "weak" so that
Kostya Serebryany2a48c242015-11-13 01:54:40 +000036// libFuzzer can be linked w/o the sanitizers and sanitizer-coverage
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000037// (in which case it will complain at start-up time).
Kostya Serebryanyd6edce92015-10-16 23:04:31 +000038__attribute__((weak)) void __sanitizer_print_stack_trace();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +000039__attribute__((weak)) void __sanitizer_reset_coverage();
Kostya Serebryany94660b32015-10-23 18:37:58 +000040__attribute__((weak)) size_t __sanitizer_get_total_unique_caller_callee_pairs();
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000041__attribute__((weak)) size_t __sanitizer_get_total_unique_coverage();
Ivan Krasindf919102016-01-22 22:28:27 +000042__attribute__((weak)) void
43__sanitizer_set_death_callback(void (*callback)(void));
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000044__attribute__((weak)) size_t __sanitizer_get_number_of_counters();
Ivan Krasindf919102016-01-22 22:28:27 +000045__attribute__((weak)) uintptr_t
46__sanitizer_update_counter_bitset_and_clear_counters(uint8_t *bitset);
Mike Aizatsky8b11f872016-01-06 00:21:22 +000047__attribute__((weak)) uintptr_t
48__sanitizer_get_coverage_pc_buffer(uintptr_t **data);
Kostya Serebryany22cc5e22016-02-13 02:29:38 +000049
50__attribute__((weak)) size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size,
51 size_t MaxSize,
52 unsigned int Seed);
Kostya Serebryany1bfd5832016-04-20 00:24:21 +000053__attribute__((weak)) void __sanitizer_malloc_hook(void *ptr, size_t size);
54__attribute__((weak)) void __sanitizer_free_hook(void *ptr);
55__attribute__((weak)) void __lsan_enable();
56__attribute__((weak)) void __lsan_disable();
57__attribute__((weak)) int __lsan_do_recoverable_leak_check();
Kostya Serebryanyd6edce92015-10-16 23:04:31 +000058}
59
Aaron Ballmanef116982015-01-29 16:58:29 +000060namespace fuzzer {
Kostya Serebryanya9da9b42015-10-16 22:47:20 +000061static const size_t kMaxUnitSizeToPrint = 256;
Aaron Ballmanef116982015-01-29 16:58:29 +000062
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000063static void MissingWeakApiFunction(const char *FnName) {
64 Printf("ERROR: %s is not defined. Exiting.\n"
Ivan Krasindf919102016-01-22 22:28:27 +000065 "Did you use -fsanitize-coverage=... to build your code?\n",
66 FnName);
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000067 exit(1);
68}
69
70#define CHECK_WEAK_API_FUNCTION(fn) \
71 do { \
72 if (!fn) \
73 MissingWeakApiFunction(#fn); \
74 } while (false)
75
Kostya Serebryany52a788e2015-03-31 20:13:20 +000076// Only one Fuzzer per process.
77static Fuzzer *F;
78
Kostya Serebryany1deb0492016-02-13 06:24:18 +000079size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize) {
80 assert(F);
81 return F->GetMD().Mutate(Data, Size, MaxSize);
82}
83
Kostya Serebryany7ec0c562016-02-13 03:25:16 +000084Fuzzer::Fuzzer(UserCallback CB, MutationDispatcher &MD, FuzzingOptions Options)
85 : CB(CB), MD(MD), Options(Options) {
Kostya Serebryany52a788e2015-03-31 20:13:20 +000086 SetDeathCallback();
Kostya Serebryany22526252015-05-11 21:16:27 +000087 InitializeTraceState();
Kostya Serebryany52a788e2015-03-31 20:13:20 +000088 assert(!F);
89 F = this;
90}
Aaron Ballmanef116982015-01-29 16:58:29 +000091
92void Fuzzer::SetDeathCallback() {
Kostya Serebryany5eab74e2015-11-09 23:17:45 +000093 CHECK_WEAK_API_FUNCTION(__sanitizer_set_death_callback);
Kostya Serebryany52a788e2015-03-31 20:13:20 +000094 __sanitizer_set_death_callback(StaticDeathCallback);
95}
96
Kostya Serebryany52a788e2015-03-31 20:13:20 +000097void Fuzzer::StaticDeathCallback() {
98 assert(F);
99 F->DeathCallback();
Aaron Ballmanef116982015-01-29 16:58:29 +0000100}
101
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000102void Fuzzer::DumpCurrentUnit(const char *Prefix) {
Kostya Serebryany98abb2c2016-01-13 23:46:01 +0000103 if (CurrentUnitSize <= kMaxUnitSizeToPrint) {
104 PrintHexArray(CurrentUnitData, CurrentUnitSize, "\n");
105 PrintASCII(CurrentUnitData, CurrentUnitSize, "\n");
Kostya Serebryanye95022a2015-10-09 04:03:14 +0000106 }
Kostya Serebryany98abb2c2016-01-13 23:46:01 +0000107 WriteUnitToFileWithPrefix(
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000108 {CurrentUnitData, CurrentUnitData + CurrentUnitSize}, Prefix);
109}
110
Benjamin Kramerd96b0c12016-03-18 14:19:19 +0000111NO_SANITIZE_MEMORY
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000112void Fuzzer::DeathCallback() {
113 if (!CurrentUnitSize) return;
114 Printf("DEATH:\n");
115 DumpCurrentUnit("crash-");
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000116 PrintFinalStats();
Aaron Ballmanef116982015-01-29 16:58:29 +0000117}
118
Kostya Serebryany52a788e2015-03-31 20:13:20 +0000119void Fuzzer::StaticAlarmCallback() {
120 assert(F);
121 F->AlarmCallback();
122}
123
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000124void Fuzzer::StaticCrashSignalCallback() {
125 assert(F);
126 F->CrashCallback();
127}
128
129void Fuzzer::StaticInterruptCallback() {
130 assert(F);
131 F->InterruptCallback();
132}
133
134void Fuzzer::CrashCallback() {
135 Printf("==%d== ERROR: libFuzzer: deadly signal\n", GetPid());
136 if (__sanitizer_print_stack_trace)
137 __sanitizer_print_stack_trace();
138 Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
139 " Combine libFuzzer with AddressSanitizer or similar for better "
140 "crash reports.\n");
141 Printf("SUMMARY: libFuzzer: deadly signal\n");
142 DumpCurrentUnit("crash-");
143 PrintFinalStats();
144 exit(Options.ErrorExitCode);
145}
146
147void Fuzzer::InterruptCallback() {
148 Printf("==%d== libFuzzer: run interrupted; exiting\n", GetPid());
149 PrintFinalStats();
Kostya Serebryanye483ed22016-03-03 22:36:37 +0000150 _Exit(0); // Stop right now, don't perform any at-exit actions.
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000151}
152
Benjamin Kramerd96b0c12016-03-18 14:19:19 +0000153NO_SANITIZE_MEMORY
Aaron Ballmanef116982015-01-29 16:58:29 +0000154void Fuzzer::AlarmCallback() {
Kostya Serebryany490bbd62015-05-19 22:12:57 +0000155 assert(Options.UnitTimeoutSec > 0);
Kostya Serebryanycfbcf902016-02-17 19:42:34 +0000156 if (!CurrentUnitSize)
157 return; // We have not started running units yet.
Aaron Ballmanef116982015-01-29 16:58:29 +0000158 size_t Seconds =
159 duration_cast<seconds>(system_clock::now() - UnitStartTime).count();
Ivan Krasindf919102016-01-22 22:28:27 +0000160 if (Seconds == 0)
161 return;
Kostya Serebryany490bbd62015-05-19 22:12:57 +0000162 if (Options.Verbosity >= 2)
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000163 Printf("AlarmCallback %zd\n", Seconds);
Kostya Serebryany490bbd62015-05-19 22:12:57 +0000164 if (Seconds >= (size_t)Options.UnitTimeoutSec) {
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000165 Printf("ALARM: working on the last Unit for %zd seconds\n", Seconds);
Kostya Serebryany316b5712015-05-26 20:57:47 +0000166 Printf(" and the timeout value is %d (use -timeout=N to change)\n",
167 Options.UnitTimeoutSec);
Kostya Serebryany228d5b12016-03-01 22:19:21 +0000168 DumpCurrentUnit("timeout-");
Kostya Serebryanyd6edce92015-10-16 23:04:31 +0000169 Printf("==%d== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
170 Seconds);
171 if (__sanitizer_print_stack_trace)
172 __sanitizer_print_stack_trace();
173 Printf("SUMMARY: libFuzzer: timeout\n");
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000174 PrintFinalStats();
Kostya Serebryany31516732016-03-24 01:32:08 +0000175 _Exit(Options.TimeoutExitCode); // Stop right now.
Aaron Ballmanef116982015-01-29 16:58:29 +0000176 }
Aaron Ballmanef116982015-01-29 16:58:29 +0000177}
178
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000179void Fuzzer::PrintStats(const char *Where, const char *End) {
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000180 size_t ExecPerSec = execPerSec();
Mike Aizatskya9c23872015-11-12 04:38:40 +0000181 if (Options.OutputCSV) {
182 static bool csvHeaderPrinted = false;
183 if (!csvHeaderPrinted) {
184 csvHeaderPrinted = true;
185 Printf("runs,block_cov,bits,cc_cov,corpus,execs_per_sec,tbms,reason\n");
186 }
Kostya Serebryany9ba19182016-04-25 19:41:45 +0000187 Printf("%zd,%zd,%zd,%zd,%zd,%zd,%s\n", TotalNumberOfRuns,
Mike Aizatskya9c23872015-11-12 04:38:40 +0000188 LastRecordedBlockCoverage, TotalBits(),
189 LastRecordedCallerCalleeCoverage, Corpus.size(), ExecPerSec,
Kostya Serebryany9ba19182016-04-25 19:41:45 +0000190 Where);
Mike Aizatskya9c23872015-11-12 04:38:40 +0000191 }
192
193 if (!Options.Verbosity)
194 return;
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000195 Printf("#%zd\t%s", TotalNumberOfRuns, Where);
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000196 if (LastRecordedBlockCoverage)
197 Printf(" cov: %zd", LastRecordedBlockCoverage);
Kostya Serebryanyda63c1d2016-02-26 21:33:56 +0000198 if (LastRecordedPcMapSize)
199 Printf(" path: %zd", LastRecordedPcMapSize);
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000200 if (auto TB = TotalBits())
201 Printf(" bits: %zd", TB);
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000202 if (LastRecordedCallerCalleeCoverage)
203 Printf(" indir: %zd", LastRecordedCallerCalleeCoverage);
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000204 Printf(" units: %zd exec/s: %zd", Corpus.size(), ExecPerSec);
Kostya Serebryany12c78372015-08-12 01:55:37 +0000205 Printf("%s", End);
Kostya Serebryany03db8b92015-03-30 22:44:03 +0000206}
207
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000208void Fuzzer::PrintFinalStats() {
209 if (!Options.PrintFinalStats) return;
210 size_t ExecPerSec = execPerSec();
211 Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns);
212 Printf("stat::average_exec_per_sec: %zd\n", ExecPerSec);
213 Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded);
214 Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds);
215 Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb());
216}
217
Kostya Serebryany64d24572016-03-12 01:57:04 +0000218size_t Fuzzer::MaxUnitSizeInCorpus() const {
219 size_t Res = 0;
220 for (auto &X : Corpus)
221 Res = std::max(Res, X.size());
222 return Res;
223}
224
225void Fuzzer::SetMaxLen(size_t MaxLen) {
226 assert(Options.MaxLen == 0); // Can only reset MaxLen from 0 to non-0.
227 assert(MaxLen);
228 Options.MaxLen = MaxLen;
229 Printf("INFO: -max_len is not provided, using %zd\n", Options.MaxLen);
230}
231
232
233void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
Ivan Krasindf919102016-01-22 22:28:27 +0000234 if (Options.OutputCorpus.empty())
235 return;
Kostya Serebryany1ac80552015-05-08 21:30:55 +0000236 std::vector<Unit> AdditionalCorpus;
237 ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
Kostya Serebryany64d24572016-03-12 01:57:04 +0000238 &EpochOfLastReadOfOutputCorpus, MaxSize);
Kostya Serebryany1ac80552015-05-08 21:30:55 +0000239 if (Corpus.empty()) {
240 Corpus = AdditionalCorpus;
241 return;
242 }
Ivan Krasindf919102016-01-22 22:28:27 +0000243 if (!Options.Reload)
244 return;
Kostya Serebryany490bbd62015-05-19 22:12:57 +0000245 if (Options.Verbosity >= 2)
Ivan Krasindf919102016-01-22 22:28:27 +0000246 Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
Kostya Serebryany1ac80552015-05-08 21:30:55 +0000247 for (auto &X : AdditionalCorpus) {
Kostya Serebryany64d24572016-03-12 01:57:04 +0000248 if (X.size() > MaxSize)
249 X.resize(MaxSize);
Kostya Serebryanycbb23342015-05-19 01:06:07 +0000250 if (UnitHashesAddedToCorpus.insert(Hash(X)).second) {
Kostya Serebryany98abb2c2016-01-13 23:46:01 +0000251 if (RunOne(X)) {
Kostya Serebryanycbb23342015-05-19 01:06:07 +0000252 Corpus.push_back(X);
Ivan Krasindf919102016-01-22 22:28:27 +0000253 UpdateCorpusDistribution();
Mike Aizatskya9c23872015-11-12 04:38:40 +0000254 PrintStats("RELOAD");
Kostya Serebryanycbb23342015-05-19 01:06:07 +0000255 }
Kostya Serebryany1ac80552015-05-08 21:30:55 +0000256 }
257 }
258}
259
Kostya Serebryany945761b2016-03-18 00:23:29 +0000260void Fuzzer::ShuffleCorpus(UnitVector *V) {
261 std::random_shuffle(V->begin(), V->end(), MD.GetRand());
262 if (Options.PreferSmall)
263 std::stable_sort(V->begin(), V->end(), [](const Unit &A, const Unit &B) {
264 return A.size() < B.size();
265 });
266}
267
Aaron Ballmanef116982015-01-29 16:58:29 +0000268void Fuzzer::ShuffleAndMinimize() {
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000269 PrintStats("READ ");
Aaron Ballmanef116982015-01-29 16:58:29 +0000270 std::vector<Unit> NewCorpus;
Kostya Serebryany945761b2016-03-18 00:23:29 +0000271 if (Options.ShuffleAtStartUp)
272 ShuffleCorpus(&Corpus);
273
274 for (const auto &U : Corpus) {
275 if (RunOne(U)) {
276 NewCorpus.push_back(U);
277 if (Options.Verbosity >= 2)
278 Printf("NEW0: %zd L %zd\n", LastRecordedBlockCoverage, U.size());
Aaron Ballmanef116982015-01-29 16:58:29 +0000279 }
280 }
281 Corpus = NewCorpus;
Ivan Krasindf919102016-01-22 22:28:27 +0000282 UpdateCorpusDistribution();
Kostya Serebryanycbb23342015-05-19 01:06:07 +0000283 for (auto &X : Corpus)
284 UnitHashesAddedToCorpus.insert(Hash(X));
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000285 PrintStats("INITED");
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000286 CheckForMemoryLeaks();
Aaron Ballmanef116982015-01-29 16:58:29 +0000287}
288
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000289bool Fuzzer::RunOne(const uint8_t *Data, size_t Size) {
Aaron Ballmanef116982015-01-29 16:58:29 +0000290 TotalNumberOfRuns++;
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000291
292 PrepareCoverageBeforeRun();
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000293 ExecuteCallback(Data, Size);
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000294 bool Res = CheckCoverageAfterRun();
295
Kostya Serebryany16901a92015-03-30 23:04:35 +0000296 auto UnitStopTime = system_clock::now();
297 auto TimeOfUnit =
298 duration_cast<seconds>(UnitStopTime - UnitStartTime).count();
Mike Aizatskya9c23872015-11-12 04:38:40 +0000299 if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) &&
300 secondsSinceProcessStartUp() >= 2)
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000301 PrintStats("pulse ");
Kostya Serebryany70926ae2015-08-05 21:43:48 +0000302 if (TimeOfUnit > TimeOfLongestUnitInSeconds &&
303 TimeOfUnit >= Options.ReportSlowUnits) {
Kostya Serebryany16901a92015-03-30 23:04:35 +0000304 TimeOfLongestUnitInSeconds = TimeOfUnit;
Kostya Serebryany70926ae2015-08-05 21:43:48 +0000305 Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000306 WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-");
Kostya Serebryany16901a92015-03-30 23:04:35 +0000307 }
308 return Res;
Kostya Serebryany2c1b33b2015-01-29 23:01:07 +0000309}
310
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000311void Fuzzer::RunOneAndUpdateCorpus(uint8_t *Data, size_t Size) {
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000312 if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
313 return;
Kostya Serebryanybc7c0ad2015-08-11 01:44:42 +0000314 if (Options.OnlyASCII)
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000315 ToASCII(Data, Size);
316 if (RunOne(Data, Size))
317 ReportNewCoverage({Data, Data + Size});
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000318}
319
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000320// Leak detection is expensive, so we first check if there were more mallocs
321// than frees (using the sanitizer malloc hooks) and only then try to call lsan.
322struct MallocFreeTracer {
323 void Start() {
324 Mallocs = 0;
325 Frees = 0;
326 }
327 // Returns true if there were more mallocs than frees.
328 bool Stop() { return Mallocs > Frees; }
329 size_t Mallocs;
330 size_t Frees;
331};
332
333static thread_local MallocFreeTracer AllocTracer;
334
335extern "C" {
336void __sanitizer_malloc_hook(void *ptr, size_t size) { AllocTracer.Mallocs++; }
337void __sanitizer_free_hook(void *ptr) { AllocTracer.Frees++; }
338} // extern "C"
339
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000340void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
Kostya Serebryanyebb932d2016-04-18 22:50:39 +0000341 UnitStartTime = system_clock::now();
Kostya Serebryanyd50a3ee2016-01-13 23:02:30 +0000342 // We copy the contents of Unit into a separate heap buffer
343 // so that we reliably find buffer overflows in it.
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000344 std::unique_ptr<uint8_t[]> DataCopy(new uint8_t[Size]);
345 memcpy(DataCopy.get(), Data, Size);
346 AssignTaintLabels(DataCopy.get(), Size);
347 CurrentUnitData = DataCopy.get();
348 CurrentUnitSize = Size;
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000349 AllocTracer.Start();
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000350 int Res = CB(DataCopy.get(), Size);
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000351 (void)Res;
352 HasMoreMallocsThanFrees = AllocTracer.Stop();
Kostya Serebryanyebb932d2016-04-18 22:50:39 +0000353 CurrentUnitSize = 0;
354 CurrentUnitData = nullptr;
Kostya Serebryany20bb5e72015-10-02 23:34:06 +0000355 assert(Res == 0);
Kostya Serebryany52a788e2015-03-31 20:13:20 +0000356}
357
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000358size_t Fuzzer::RecordBlockCoverage() {
Kostya Serebryany5eab74e2015-11-09 23:17:45 +0000359 CHECK_WEAK_API_FUNCTION(__sanitizer_get_total_unique_coverage);
Mike Aizatsky8b11f872016-01-06 00:21:22 +0000360 uintptr_t PrevCoverage = LastRecordedBlockCoverage;
361 LastRecordedBlockCoverage = __sanitizer_get_total_unique_coverage();
362
363 if (PrevCoverage == LastRecordedBlockCoverage || !Options.PrintNewCovPcs)
364 return LastRecordedBlockCoverage;
365
366 uintptr_t PrevBufferLen = LastCoveragePcBufferLen;
367 uintptr_t *CoverageBuf;
368 LastCoveragePcBufferLen = __sanitizer_get_coverage_pc_buffer(&CoverageBuf);
369 assert(CoverageBuf);
370 for (size_t i = PrevBufferLen; i < LastCoveragePcBufferLen; ++i) {
Mike Aizatskye313f8f2016-01-21 00:02:09 +0000371 Printf("%p\n", CoverageBuf[i]);
Mike Aizatsky8b11f872016-01-06 00:21:22 +0000372 }
373
374 return LastRecordedBlockCoverage;
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000375}
376
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000377size_t Fuzzer::RecordCallerCalleeCoverage() {
378 if (!Options.UseIndirCalls)
379 return 0;
Kostya Serebryany94660b32015-10-23 18:37:58 +0000380 if (!__sanitizer_get_total_unique_caller_callee_pairs)
381 return 0;
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000382 return LastRecordedCallerCalleeCoverage =
383 __sanitizer_get_total_unique_caller_callee_pairs();
384}
385
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000386void Fuzzer::PrepareCoverageBeforeRun() {
Kostya Serebryanybe5e0ed2015-03-03 23:27:02 +0000387 if (Options.UseCounters) {
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000388 size_t NumCounters = __sanitizer_get_number_of_counters();
Kostya Serebryanybe5e0ed2015-03-03 23:27:02 +0000389 CounterBitmap.resize(NumCounters);
390 __sanitizer_update_counter_bitset_and_clear_counters(0);
391 }
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000392 RecordBlockCoverage();
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000393 RecordCallerCalleeCoverage();
Kostya Serebryany007c9b22015-10-22 22:50:47 +0000394}
395
396bool Fuzzer::CheckCoverageAfterRun() {
397 size_t OldCoverage = LastRecordedBlockCoverage;
398 size_t NewCoverage = RecordBlockCoverage();
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000399 size_t OldCallerCalleeCoverage = LastRecordedCallerCalleeCoverage;
400 size_t NewCallerCalleeCoverage = RecordCallerCalleeCoverage();
Kostya Serebryanybe5e0ed2015-03-03 23:27:02 +0000401 size_t NumNewBits = 0;
Kostya Serebryanyda63c1d2016-02-26 21:33:56 +0000402 size_t OldPcMapSize = LastRecordedPcMapSize;
403 PcMapMergeCurrentToCombined();
404 size_t NewPcMapSize = PcMapCombinedSize();
405 LastRecordedPcMapSize = NewPcMapSize;
406 if (NewPcMapSize > OldPcMapSize)
407 return true;
Kostya Serebryanybe5e0ed2015-03-03 23:27:02 +0000408 if (Options.UseCounters)
409 NumNewBits = __sanitizer_update_counter_bitset_and_clear_counters(
410 CounterBitmap.data());
Kostya Serebryany2e9fca92015-10-22 23:55:39 +0000411 return NewCoverage > OldCoverage ||
412 NewCallerCalleeCoverage > OldCallerCalleeCoverage || NumNewBits;
Aaron Ballmanef116982015-01-29 16:58:29 +0000413}
414
415void Fuzzer::WriteToOutputCorpus(const Unit &U) {
Ivan Krasindf919102016-01-22 22:28:27 +0000416 if (Options.OutputCorpus.empty())
417 return;
Aaron Ballmanef116982015-01-29 16:58:29 +0000418 std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U));
419 WriteToFile(U, Path);
420 if (Options.Verbosity >= 2)
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000421 Printf("Written to %s\n", Path.c_str());
Kostya Serebryanya9346c22015-09-02 19:08:08 +0000422 assert(!Options.OnlyASCII || IsASCII(U));
Aaron Ballmanef116982015-01-29 16:58:29 +0000423}
424
Kostya Serebryany2b7d2e92015-07-23 18:37:22 +0000425void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) {
Kostya Serebryanyb91c62b2015-10-16 22:41:47 +0000426 if (!Options.SaveArtifacts)
427 return;
Kostya Serebryanybd5d1cd2015-10-09 03:57:59 +0000428 std::string Path = Options.ArtifactPrefix + Prefix + Hash(U);
Kostya Serebryany2d0ef142015-11-25 21:40:46 +0000429 if (!Options.ExactArtifactPath.empty())
Ivan Krasindf919102016-01-22 22:28:27 +0000430 Path = Options.ExactArtifactPath; // Overrides ArtifactPrefix.
Aaron Ballmanef116982015-01-29 16:58:29 +0000431 WriteToFile(U, Path);
Kostya Serebryanybd5d1cd2015-10-09 03:57:59 +0000432 Printf("artifact_prefix='%s'; Test unit written to %s\n",
433 Options.ArtifactPrefix.c_str(), Path.c_str());
Kostya Serebryany9e48cda2015-12-04 22:29:39 +0000434 if (U.size() <= kMaxUnitSizeToPrint)
435 Printf("Base64: %s\n", Base64(U).c_str());
Aaron Ballmanef116982015-01-29 16:58:29 +0000436}
437
438void Fuzzer::SaveCorpus() {
Ivan Krasindf919102016-01-22 22:28:27 +0000439 if (Options.OutputCorpus.empty())
440 return;
Aaron Ballmanef116982015-01-29 16:58:29 +0000441 for (const auto &U : Corpus)
442 WriteToFile(U, DirPlusFile(Options.OutputCorpus, Hash(U)));
443 if (Options.Verbosity)
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000444 Printf("Written corpus of %zd files to %s\n", Corpus.size(),
445 Options.OutputCorpus.c_str());
Aaron Ballmanef116982015-01-29 16:58:29 +0000446}
447
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000448void Fuzzer::PrintStatusForNewUnit(const Unit &U) {
449 if (!Options.PrintNEW)
450 return;
Kostya Serebryany09d2a5f2015-10-22 22:56:45 +0000451 PrintStats("NEW ", "");
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000452 if (Options.Verbosity) {
Kostya Serebryany14c50282015-12-19 01:09:49 +0000453 Printf(" L: %zd ", U.size());
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000454 MD.PrintMutationSequence();
Kostya Serebryany7c180ea2015-05-23 01:22:35 +0000455 Printf("\n");
Aaron Ballmanef116982015-01-29 16:58:29 +0000456 }
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000457}
458
459void Fuzzer::ReportNewCoverage(const Unit &U) {
460 Corpus.push_back(U);
Ivan Krasindf919102016-01-22 22:28:27 +0000461 UpdateCorpusDistribution();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000462 UnitHashesAddedToCorpus.insert(Hash(U));
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000463 MD.RecordSuccessfulMutationSequence();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000464 PrintStatusForNewUnit(U);
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000465 WriteToOutputCorpus(U);
Kostya Serebryany66ff0752016-02-26 22:42:23 +0000466 NumberOfNewUnitsAdded++;
Aaron Ballmanef116982015-01-29 16:58:29 +0000467}
468
Kostya Serebryany945761b2016-03-18 00:23:29 +0000469// Finds minimal number of units in 'Extra' that add coverage to 'Initial'.
470// We do it by actually executing the units, sometimes more than once,
471// because we may be using different coverage-like signals and the only
472// common thing between them is that we can say "this unit found new stuff".
473UnitVector Fuzzer::FindExtraUnits(const UnitVector &Initial,
474 const UnitVector &Extra) {
475 UnitVector Res = Extra;
476 size_t OldSize = Res.size();
477 for (int Iter = 0; Iter < 10; Iter++) {
478 ShuffleCorpus(&Res);
479 ResetCoverage();
480
481 for (auto &U : Initial)
482 RunOne(U);
483
484 Corpus.clear();
485 for (auto &U : Res)
486 if (RunOne(U))
487 Corpus.push_back(U);
488
489 char Stat[7] = "MIN ";
490 Stat[3] = '0' + Iter;
491 PrintStats(Stat);
492
493 size_t NewSize = Corpus.size();
494 Res.swap(Corpus);
495
496 if (NewSize == OldSize)
497 break;
498 OldSize = NewSize;
499 }
500 return Res;
501}
502
Kostya Serebryany9cc3b0d2015-10-24 01:16:40 +0000503void Fuzzer::Merge(const std::vector<std::string> &Corpora) {
504 if (Corpora.size() <= 1) {
505 Printf("Merge requires two or more corpus dirs\n");
506 return;
507 }
Kostya Serebryany9cc3b0d2015-10-24 01:16:40 +0000508 std::vector<std::string> ExtraCorpora(Corpora.begin() + 1, Corpora.end());
509
Kostya Serebryany945761b2016-03-18 00:23:29 +0000510 assert(Options.MaxLen > 0);
511 UnitVector Initial, Extra;
512 ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, Options.MaxLen);
513 for (auto &C : ExtraCorpora)
514 ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, Options.MaxLen);
515
516 if (!Initial.empty()) {
517 Printf("=== Minimizing the initial corpus of %zd units\n", Initial.size());
518 Initial = FindExtraUnits({}, Initial);
Kostya Serebryany9cc3b0d2015-10-24 01:16:40 +0000519 }
Kostya Serebryany945761b2016-03-18 00:23:29 +0000520
521 Printf("=== Merging extra %zd units\n", Extra.size());
522 auto Res = FindExtraUnits(Initial, Extra);
523
524 for (auto &U: Res)
525 WriteToOutputCorpus(U);
526
527 Printf("=== Merge: written %zd units\n", Res.size());
Kostya Serebryany9cc3b0d2015-10-24 01:16:40 +0000528}
529
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000530// Tries to call lsan, and if there are leaks exits. We call this right after
531// the initial corpus was read because if there are leaky inputs in the corpus
532// further fuzzing will likely hit OOMs.
533void Fuzzer::CheckForMemoryLeaks() {
534 if (!Options.DetectLeaks) return;
535 if (!__lsan_do_recoverable_leak_check)
536 return;
537 if (__lsan_do_recoverable_leak_check()) {
538 Printf("==%d== ERROR: libFuzzer: initial corpus triggers memory leaks.\n"
539 "Exiting now. Use -detect_leaks=0 to disable leak detection here.\n"
540 "LeakSanitizer will still check for leaks at the process exit.\n",
541 GetPid());
542 PrintFinalStats();
543 _Exit(Options.ErrorExitCode);
544 }
545}
546
547// Tries detecting a memory leak on the particular input that we have just
548// executed before calling this function.
549void Fuzzer::TryDetectingAMemoryLeak(uint8_t *Data, size_t Size) {
550 if (!HasMoreMallocsThanFrees) return; // mallocs==frees, a leak is unlikely.
551 if (!Options.DetectLeaks) return;
552 if (!&__lsan_enable || !&__lsan_disable || !__lsan_do_recoverable_leak_check)
553 return; // No lsan.
554 // Run the target once again, but with lsan disabled so that if there is
555 // a real leak we do not report it twice.
556 __lsan_disable();
557 RunOneAndUpdateCorpus(Data, Size);
558 __lsan_enable();
559 if (!HasMoreMallocsThanFrees) return; // a leak is unlikely.
Kostya Serebryany7018a1a2016-04-27 19:52:34 +0000560 if (NumberOfLeakDetectionAttempts++ > 1000) {
561 Options.DetectLeaks = false;
562 Printf("INFO: libFuzzer disabled leak detection after every mutation.\n"
563 " Most likely the target function accumulates allocated\n"
564 " memory in a global state w/o actually leaking it.\n"
565 " If LeakSanitizer is enabled in this process it will still\n"
566 " run on the process shutdown.\n");
567 return;
568 }
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000569 // Now perform the actual lsan pass. This is expensive and we must ensure
570 // we don't call it too often.
571 if (__lsan_do_recoverable_leak_check()) { // Leak is found, report it.
572 CurrentUnitData = Data;
573 CurrentUnitSize = Size;
574 DumpCurrentUnit("leak-");
575 PrintFinalStats();
576 _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on.
577 }
578}
579
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000580void Fuzzer::MutateAndTestOne() {
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000581 MD.StartMutationSequence();
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000582
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000583 auto &U = ChooseUnitToMutate();
584 MutateInPlaceHere.resize(Options.MaxLen);
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000585 size_t Size = U.size();
Mike Aizatsky94e29662016-04-08 23:32:24 +0000586 assert(Size <= Options.MaxLen && "Oversized Unit");
587 memcpy(MutateInPlaceHere.data(), U.data(), Size);
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000588
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000589 for (int i = 0; i < Options.MutateDepth; i++) {
Kostya Serebryany22cc5e22016-02-13 02:29:38 +0000590 size_t NewSize = 0;
591 if (LLVMFuzzerCustomMutator)
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000592 NewSize = LLVMFuzzerCustomMutator(MutateInPlaceHere.data(), Size,
593 Options.MaxLen, MD.GetRand().Rand());
Kostya Serebryany22cc5e22016-02-13 02:29:38 +0000594 else
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000595 NewSize = MD.Mutate(MutateInPlaceHere.data(), Size, Options.MaxLen);
Kostya Serebryany2ea204e2015-05-30 17:33:13 +0000596 assert(NewSize > 0 && "Mutator returned empty unit");
Kostya Serebryany64d24572016-03-12 01:57:04 +0000597 assert(NewSize <= Options.MaxLen &&
Kostya Serebryany2ea204e2015-05-30 17:33:13 +0000598 "Mutator return overisized unit");
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000599 Size = NewSize;
Kostya Serebryanyb65805a2016-01-09 03:08:58 +0000600 if (i == 0)
601 StartTraceRecording();
Kostya Serebryany8a5bef02016-02-13 17:56:51 +0000602 RunOneAndUpdateCorpus(MutateInPlaceHere.data(), Size);
Kostya Serebryanyb65805a2016-01-09 03:08:58 +0000603 StopTraceRecording();
Kostya Serebryany1bfd5832016-04-20 00:24:21 +0000604 TryDetectingAMemoryLeak(MutateInPlaceHere.data(), Size);
Kostya Serebryany7d470cf2015-05-07 18:32:29 +0000605 }
606}
607
Kostya Serebryanye6926212015-11-04 23:22:25 +0000608// Returns an index of random unit from the corpus to mutate.
609// Hypothesis: units added to the corpus last are more likely to be interesting.
Ivan Krasindf919102016-01-22 22:28:27 +0000610// This function gives more weight to the more recent units.
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000611size_t Fuzzer::ChooseUnitIdxToMutate() {
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000612 size_t Idx =
613 static_cast<size_t>(CorpusDistribution(MD.GetRand().Get_mt19937()));
Ivan Krasindf919102016-01-22 22:28:27 +0000614 assert(Idx < Corpus.size());
615 return Idx;
Kostya Serebryanye6926212015-11-04 23:22:25 +0000616}
617
Kostya Serebryany945761b2016-03-18 00:23:29 +0000618void Fuzzer::ResetCoverage() {
619 CHECK_WEAK_API_FUNCTION(__sanitizer_reset_coverage);
620 __sanitizer_reset_coverage();
621 CounterBitmap.clear();
622}
623
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000624// Experimental search heuristic: drilling.
625// - Read, shuffle, execute and minimize the corpus.
626// - Choose one random unit.
627// - Reset the coverage.
628// - Start fuzzing as if the chosen unit was the only element of the corpus.
629// - When done, reset the coverage again.
630// - Merge the newly created corpus into the original one.
631void Fuzzer::Drill() {
632 // The corpus is already read, shuffled, and minimized.
633 assert(!Corpus.empty());
Ivan Krasindf919102016-01-22 22:28:27 +0000634 Options.PrintNEW = false; // Don't print NEW status lines when drilling.
Kostya Serebryany7d211662015-09-04 00:12:11 +0000635
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000636 Unit U = ChooseUnitToMutate();
637
Kostya Serebryany945761b2016-03-18 00:23:29 +0000638 ResetCoverage();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000639
640 std::vector<Unit> SavedCorpus;
641 SavedCorpus.swap(Corpus);
642 Corpus.push_back(U);
Ivan Krasindf919102016-01-22 22:28:27 +0000643 UpdateCorpusDistribution();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000644 assert(Corpus.size() == 1);
645 RunOne(U);
646 PrintStats("DRILL ");
647 std::string SavedOutputCorpusPath; // Don't write new units while drilling.
648 SavedOutputCorpusPath.swap(Options.OutputCorpus);
649 Loop();
650
Kostya Serebryany945761b2016-03-18 00:23:29 +0000651 ResetCoverage();
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000652
653 PrintStats("REINIT");
654 SavedOutputCorpusPath.swap(Options.OutputCorpus);
Kostya Serebryany98abb2c2016-01-13 23:46:01 +0000655 for (auto &U : SavedCorpus)
Kostya Serebryanydc3135d2015-11-12 01:02:01 +0000656 RunOne(U);
657 PrintStats("MERGE ");
658 Options.PrintNEW = true;
659 size_t NumMerged = 0;
660 for (auto &U : Corpus) {
661 if (RunOne(U)) {
662 PrintStatusForNewUnit(U);
663 NumMerged++;
664 WriteToOutputCorpus(U);
665 }
666 }
667 PrintStats("MERGED");
668 if (NumMerged && Options.Verbosity)
669 Printf("Drilling discovered %zd new units\n", NumMerged);
670}
671
672void Fuzzer::Loop() {
Kostya Serebryany8617aaa2015-12-05 02:09:22 +0000673 system_clock::time_point LastCorpusReload = system_clock::now();
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000674 if (Options.DoCrossOver)
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000675 MD.SetCorpus(&Corpus);
Kostya Serebryany468ed782015-09-08 17:30:35 +0000676 while (true) {
Kostya Serebryany8617aaa2015-12-05 02:09:22 +0000677 auto Now = system_clock::now();
678 if (duration_cast<seconds>(Now - LastCorpusReload).count()) {
Kostya Serebryany64d24572016-03-12 01:57:04 +0000679 RereadOutputCorpus(Options.MaxLen);
Kostya Serebryany8617aaa2015-12-05 02:09:22 +0000680 LastCorpusReload = Now;
681 }
Kostya Serebryanye6926212015-11-04 23:22:25 +0000682 if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
Mike Aizatskya9c23872015-11-12 04:38:40 +0000683 break;
Kostya Serebryanye6926212015-11-04 23:22:25 +0000684 if (Options.MaxTotalTimeSec > 0 &&
685 secondsSinceProcessStartUp() >
Ivan Krasindf919102016-01-22 22:28:27 +0000686 static_cast<size_t>(Options.MaxTotalTimeSec))
Mike Aizatskya9c23872015-11-12 04:38:40 +0000687 break;
Kostya Serebryanye6926212015-11-04 23:22:25 +0000688 // Perform several mutations and runs.
Kostya Serebryany27ab2d72015-12-19 02:49:09 +0000689 MutateAndTestOne();
Aaron Ballmanef116982015-01-29 16:58:29 +0000690 }
Mike Aizatskya9c23872015-11-12 04:38:40 +0000691
692 PrintStats("DONE ", "\n");
Kostya Serebryany7ec0c562016-02-13 03:25:16 +0000693 MD.PrintRecommendedDictionary();
Aaron Ballmanef116982015-01-29 16:58:29 +0000694}
695
Ivan Krasindf919102016-01-22 22:28:27 +0000696void Fuzzer::UpdateCorpusDistribution() {
697 size_t N = Corpus.size();
698 std::vector<double> Intervals(N + 1);
699 std::vector<double> Weights(N);
700 std::iota(Intervals.begin(), Intervals.end(), 0);
701 std::iota(Weights.begin(), Weights.end(), 1);
702 CorpusDistribution = std::piecewise_constant_distribution<double>(
703 Intervals.begin(), Intervals.end(), Weights.begin());
704}
705
706} // namespace fuzzer