blob: 985c707c8e5e29c8aed0febb4c2c5a5e7b25e977 [file] [log] [blame]
robert.swiecki3bb518c2010-10-14 00:48:24 +00001/*
robert.swiecki@gmail.com3b630b42015-02-16 10:53:53 +00002 *
robert.swiecki@gmail.com90e99112015-02-15 02:05:14 +00003 * honggfuzz - fuzzing routines
4 * -----------------------------------------
robert.swiecki@gmail.com3b630b42015-02-16 10:53:53 +00005 *
robert.swiecki@gmail.com8531f692015-02-17 12:25:36 +00006 * Author:
7 * Robert Swiecki <swiecki@google.com>
8 * Felix Gröbert <groebert@google.com>
robert.swiecki@gmail.com3b630b42015-02-16 10:53:53 +00009 *
robert.swiecki@gmail.com772b33d2015-02-14 20:35:00 +000010 * Copyright 2010-2015 by Google Inc. All Rights Reserved.
robert.swiecki@gmail.com3b630b42015-02-16 10:53:53 +000011 *
12 * Licensed under the Apache License, Version 2.0 (the "License"); you may
13 * not use this file except in compliance with the License. You may obtain
robert.swiecki@gmail.com772b33d2015-02-14 20:35:00 +000014 * a copy of the License at
robert.swiecki@gmail.com3b630b42015-02-16 10:53:53 +000015 *
robert.swiecki@gmail.com772b33d2015-02-14 20:35:00 +000016 * http://www.apache.org/licenses/LICENSE-2.0
robert.swiecki@gmail.com3b630b42015-02-16 10:53:53 +000017 *
robert.swiecki@gmail.com772b33d2015-02-14 20:35:00 +000018 * Unless required by applicable law or agreed to in writing, software
19 * distributed under the License is distributed on an "AS IS" BASIS,
20 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
21 * implied. See the License for the specific language governing
22 * permissions and limitations under the License.
robert.swiecki@gmail.com3b630b42015-02-16 10:53:53 +000023 *
robert.swiecki@gmail.com772b33d2015-02-14 20:35:00 +000024 */
robert.swiecki3bb518c2010-10-14 00:48:24 +000025
robert.swiecki@gmail.comba85c3e2015-02-02 14:55:16 +000026#include "fuzz.h"
27
28#include <errno.h>
29#include <fcntl.h>
robert.swiecki@gmail.com90e99112015-02-15 02:05:14 +000030#include <inttypes.h>
Robert Swiecki2af83ec2017-06-05 23:54:22 +020031#include <libgen.h>
robert.swiecki@gmail.com882900b2015-02-11 13:56:22 +000032#include <pthread.h>
robert.swiecki@gmail.comba85c3e2015-02-02 14:55:16 +000033#include <signal.h>
34#include <stddef.h>
35#include <stdint.h>
36#include <stdio.h>
37#include <stdlib.h>
38#include <string.h>
Robert Swiecki10eeb0a2017-09-28 15:42:52 +020039#include <sys/mman.h>
robert.swiecki3bb518c2010-10-14 00:48:24 +000040#include <sys/param.h>
41#include <sys/stat.h>
robert.swiecki@gmail.comba85c3e2015-02-02 14:55:16 +000042#include <sys/time.h>
43#include <sys/types.h>
robert.swiecki3bb518c2010-10-14 00:48:24 +000044#include <time.h>
robert.swiecki@gmail.comba85c3e2015-02-02 14:55:16 +000045#include <unistd.h>
robert.swiecki3bb518c2010-10-14 00:48:24 +000046
Robert Swieckid0fa62c2017-09-28 18:11:05 +020047#include "arch.h"
48#include "honggfuzz.h"
49#include "input.h"
Robert Swiecki246af3e2018-01-05 14:56:32 +010050#include "libhfcommon/common.h"
51#include "libhfcommon/files.h"
52#include "libhfcommon/log.h"
53#include "libhfcommon/util.h"
robert.swiecki@gmail.com36700b52015-02-22 05:03:16 +000054#include "mangle.h"
robert.swiecki@gmail.come7190b92015-02-14 23:05:42 +000055#include "report.h"
Robert Swieckiec7b8452017-06-01 13:25:56 +020056#include "sancov.h"
57#include "sanitizers.h"
dobinedf9f8d2018-01-21 13:57:02 +010058#include "socketfuzzer.h"
Robert Swiecki56276192018-01-21 15:43:02 +010059#include "subproc.h"
robert.swiecki3bb518c2010-10-14 00:48:24 +000060
Robert Swiecki0dde76d2017-11-16 19:25:44 +010061static time_t termTimeStamp = 0;
62
63bool fuzz_isTerminating(void) {
64 if (ATOMIC_GET(termTimeStamp) != 0) {
Robert Swiecki35978ac2017-11-16 18:00:53 +010065 return true;
66 }
67 return false;
68}
69
Robert Swiecki0dde76d2017-11-16 19:25:44 +010070void fuzz_setTerminating(void) {
71 if (ATOMIC_GET(termTimeStamp) != 0) {
Robert Swiecki35978ac2017-11-16 18:00:53 +010072 return;
73 }
Robert Swiecki0dde76d2017-11-16 19:25:44 +010074 ATOMIC_SET(termTimeStamp, time(NULL));
75}
76
77bool fuzz_shouldTerminate() {
78 if (ATOMIC_GET(termTimeStamp) == 0) {
79 return false;
80 }
81 if ((time(NULL) - ATOMIC_GET(termTimeStamp)) > 5) {
82 return true;
83 }
84 return false;
Robert Swiecki35978ac2017-11-16 18:00:53 +010085}
86
Robert Swieckifb8a5b62018-01-14 05:16:59 +010087static fuzzState_t fuzz_getState(honggfuzz_t* hfuzz) {
88 return ATOMIC_GET(hfuzz->state);
Robert Swieckia7841da2017-02-24 17:27:06 +010089}
90
Robert Swieckiced3eba2017-12-15 15:33:03 +010091static bool fuzz_writeCovFile(const char* dir, const uint8_t* data, size_t len) {
92 char fname[PATH_MAX];
93
94 uint64_t crc64f = util_CRC64(data, len);
95 uint64_t crc64r = util_CRC64Rev(data, len);
96 snprintf(fname, sizeof(fname), "%s/%016" PRIx64 "%016" PRIx64 ".%08" PRIx32 ".honggfuzz.cov",
97 dir, crc64f, crc64r, (uint32_t)len);
98
Robert Swieckifff99812018-01-12 02:23:42 +010099 if (files_exists(fname)) {
Robert Swieckiced3eba2017-12-15 15:33:03 +0100100 LOG_D("File '%s' already exists in the output corpus directory '%s'", fname, dir);
101 return true;
102 }
103
104 LOG_D("Adding file '%s' to the corpus directory '%s'", fname, dir);
105
Robert Swieckifff99812018-01-12 02:23:42 +0100106 if (!files_writeBufToFile(fname, data, len, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC)) {
Robert Swieckiced3eba2017-12-15 15:33:03 +0100107 LOG_W("Couldn't write buffer to file '%s'", fname);
108 return false;
109 }
110
111 return true;
112}
113
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100114static void fuzz_addFileToFileQ(honggfuzz_t* hfuzz, const uint8_t* data, size_t len) {
Robert Swiecki36f7e512018-01-16 03:46:41 +0100115 ATOMIC_SET(hfuzz->timing.lastCovUpdate, time(NULL));
116
Robert Swiecki4e595fb2017-10-11 17:26:51 +0200117 struct dynfile_t* dynfile = (struct dynfile_t*)util_Malloc(sizeof(struct dynfile_t));
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100118 dynfile->size = len;
119 dynfile->data = (uint8_t*)util_Malloc(len);
120 memcpy(dynfile->data, data, len);
Robert Swiecki37498fd2017-03-12 21:12:54 +0100121
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100122 MX_SCOPED_RWLOCK_WRITE(&hfuzz->dynfileq_mutex);
123 TAILQ_INSERT_TAIL(&hfuzz->dynfileq, dynfile, pointers);
124 hfuzz->dynfileqCnt++;
Robert Swieckif3534bb2016-03-14 18:55:10 +0100125
dobinedf9f8d2018-01-21 13:57:02 +0100126 if (hfuzz->socketFuzzer) {
Robert Swiecki5eeb29b2018-01-21 16:07:06 +0100127 /* Don't add coverage data to files in socketFuzzer mode */
dobinedf9f8d2018-01-21 13:57:02 +0100128 return;
129 }
130
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100131 if (!fuzz_writeCovFile(hfuzz->io.covDirAll, data, len)) {
132 LOG_E("Couldn't save the coverage data to '%s'", hfuzz->io.covDirAll);
Robert Swieckiced3eba2017-12-15 15:33:03 +0100133 }
134
135 /* No need to add files to the new coverage dir, if this is just the dry-run phase */
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100136 if (fuzz_getState(hfuzz) == _HF_STATE_DYNAMIC_DRY_RUN || hfuzz->io.covDirNew == NULL) {
Jagger3c7e7ce2016-09-25 16:05:19 +0200137 return;
138 }
139
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100140 if (!fuzz_writeCovFile(hfuzz->io.covDirNew, data, len)) {
141 LOG_E("Couldn't save the new coverage data to '%s'", hfuzz->io.covDirNew);
142 }
143}
144
145static void fuzz_setDynamicMainState(run_t* run) {
146 /* All threads need to indicate willingness to switch to the DYNAMIC_MAIN state. Count them! */
147 static uint32_t cnt = 0;
148 ATOMIC_PRE_INC(cnt);
149
150 static pthread_mutex_t state_mutex = PTHREAD_MUTEX_INITIALIZER;
151 MX_SCOPED_LOCK(&state_mutex);
152
153 if (fuzz_getState(run->global) == _HF_STATE_DYNAMIC_MAIN) {
154 return;
155 }
156
157 for (;;) {
158 /* Check if all threads have already reported in for changing state */
159 if (ATOMIC_GET(cnt) == run->global->threads.threadsMax) {
160 break;
161 }
162 if (fuzz_isTerminating()) {
163 return;
164 }
165 usleep(1000 * 10); /* Check every 10ms */
166 }
167
168 LOG_I("Entering phase 2/2: Dynamic Main");
Robert Swiecki94d314c2018-02-07 21:23:00 +0100169 snprintf(run->origFileName, sizeof(run->origFileName), "[DYNAMIC]");
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100170 ATOMIC_SET(run->global->state, _HF_STATE_DYNAMIC_MAIN);
171
172 /*
173 * If the initial fuzzing yielded no useful coverage, just add a single 1-byte file to the
174 * dynamic corpus, so the dynamic phase doesn't fail because of lack of useful inputs
175 */
176 if (run->global->dynfileqCnt == 0) {
177 fuzz_addFileToFileQ(run->global, (const uint8_t*)"\0", 1U);
Robert Swieckif3534bb2016-03-14 18:55:10 +0100178 }
179}
180
Robert Swieckid50ed422017-11-13 23:32:26 +0100181static void fuzz_perfFeedback(run_t* run) {
Robert Swiecki78633d12017-11-13 23:24:55 +0100182 if (run->global->skipFeedbackOnTimeout && run->tmOutSignaled) {
Robert Swiecki53ec9e42017-02-15 20:34:27 +0100183 return;
184 }
185
Robert Swiecki0b566112017-10-17 17:39:07 +0200186 LOG_D("New file size: %zu, Perf feedback new/cur (instr,branch): %" PRIu64 "/%" PRIu64
187 "/%" PRIu64 "/%" PRIu64 ", BBcnt new/total: %" PRIu64 "/%" PRIu64,
Robert Swiecki78633d12017-11-13 23:24:55 +0100188 run->dynamicFileSz, run->linux.hwCnts.cpuInstrCnt, run->global->linux.hwCnts.cpuInstrCnt,
189 run->linux.hwCnts.cpuBranchCnt, run->global->linux.hwCnts.cpuBranchCnt,
190 run->linux.hwCnts.newBBCnt, run->global->linux.hwCnts.bbCnt);
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200191
Robert Swiecki78633d12017-11-13 23:24:55 +0100192 MX_SCOPED_LOCK(&run->global->feedback_mutex);
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200193
Robert Swieckif2da05a2018-01-12 03:01:09 +0100194 uint64_t softCntPc = 0;
195 uint64_t softCntEdge = 0;
196 uint64_t softCntCmp = 0;
Robert Swiecki78633d12017-11-13 23:24:55 +0100197 if (run->global->bbFd != -1) {
198 softCntPc = ATOMIC_GET(run->global->feedback->pidFeedbackPc[run->fuzzNo]);
199 ATOMIC_CLEAR(run->global->feedback->pidFeedbackPc[run->fuzzNo]);
200 softCntEdge = ATOMIC_GET(run->global->feedback->pidFeedbackEdge[run->fuzzNo]);
201 ATOMIC_CLEAR(run->global->feedback->pidFeedbackEdge[run->fuzzNo]);
202 softCntCmp = ATOMIC_GET(run->global->feedback->pidFeedbackCmp[run->fuzzNo]);
203 ATOMIC_CLEAR(run->global->feedback->pidFeedbackCmp[run->fuzzNo]);
Jagger251d0192016-08-24 00:54:04 +0200204 }
Jaggerb01aaae2016-08-20 03:35:38 +0200205
Robert Swiecki78633d12017-11-13 23:24:55 +0100206 int64_t diff0 = run->global->linux.hwCnts.cpuInstrCnt - run->linux.hwCnts.cpuInstrCnt;
207 int64_t diff1 = run->global->linux.hwCnts.cpuBranchCnt - run->linux.hwCnts.cpuBranchCnt;
Jagger302c2ea2016-09-07 03:54:43 +0200208
Robert Swiecki7b19fe52018-01-12 03:56:42 +0100209 /* Any increase in coverage (edge, pc, cmp, hw) counters forces adding input to the corpus */
Robert Swieckid50ed422017-11-13 23:32:26 +0100210 if (run->linux.hwCnts.newBBCnt > 0 || softCntPc > 0 || softCntEdge > 0 || softCntCmp > 0 ||
211 diff0 < 0 || diff1 < 0) {
Robert Swiecki92ec8d22016-11-21 01:10:18 +0100212 if (diff0 < 0) {
Robert Swiecki78633d12017-11-13 23:24:55 +0100213 run->global->linux.hwCnts.cpuInstrCnt = run->linux.hwCnts.cpuInstrCnt;
Robert Swiecki92ec8d22016-11-21 01:10:18 +0100214 }
215 if (diff1 < 0) {
Robert Swiecki78633d12017-11-13 23:24:55 +0100216 run->global->linux.hwCnts.cpuBranchCnt = run->linux.hwCnts.cpuBranchCnt;
Robert Swiecki92ec8d22016-11-21 01:10:18 +0100217 }
Robert Swiecki78633d12017-11-13 23:24:55 +0100218 run->global->linux.hwCnts.bbCnt += run->linux.hwCnts.newBBCnt;
219 run->global->linux.hwCnts.softCntPc += softCntPc;
220 run->global->linux.hwCnts.softCntEdge += softCntEdge;
221 run->global->linux.hwCnts.softCntCmp += softCntCmp;
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200222
Robert Swieckie60f3532017-12-17 20:25:20 +0100223 LOG_I("Size:%zu (i,b,hw,edge,ip,cmp): %" PRIu64 "/%" PRIu64 "/%" PRIu64 "/%" PRIu64
Robert Swiecki0b566112017-10-17 17:39:07 +0200224 "/%" PRIu64 "/%" PRIu64 ", Tot:%" PRIu64 "/%" PRIu64 "/%" PRIu64 "/%" PRIu64
225 "/%" PRIu64 "/%" PRIu64,
Robert Swieckie7294ca2017-11-11 02:46:32 +0100226 run->dynamicFileSz, run->linux.hwCnts.cpuInstrCnt, run->linux.hwCnts.cpuBranchCnt,
Robert Swieckie60f3532017-12-17 20:25:20 +0100227 run->linux.hwCnts.newBBCnt, softCntEdge, softCntPc, softCntCmp,
Robert Swiecki78633d12017-11-13 23:24:55 +0100228 run->global->linux.hwCnts.cpuInstrCnt, run->global->linux.hwCnts.cpuBranchCnt,
Robert Swieckie60f3532017-12-17 20:25:20 +0100229 run->global->linux.hwCnts.bbCnt, run->global->linux.hwCnts.softCntEdge,
230 run->global->linux.hwCnts.softCntPc, run->global->linux.hwCnts.softCntCmp);
Jagger395df022016-08-21 01:13:25 +0200231
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100232 fuzz_addFileToFileQ(run->global, run->dynamicFile, run->dynamicFileSz);
dobinedf9f8d2018-01-21 13:57:02 +0100233
Robert Swiecki56276192018-01-21 15:43:02 +0100234 if (run->global->socketFuzzer) {
dobinedf9f8d2018-01-21 13:57:02 +0100235 LOG_D("SocketFuzzer: fuzz: new BB (perf)");
236 fuzz_notifySocketFuzzerNewCov(run->global);
237 }
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200238 }
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200239}
240
Robert Swieckid50ed422017-11-13 23:32:26 +0100241static void fuzz_sanCovFeedback(run_t* run) {
Robert Swiecki78633d12017-11-13 23:24:55 +0100242 if (run->global->skipFeedbackOnTimeout && run->tmOutSignaled) {
Robert Swiecki53ec9e42017-02-15 20:34:27 +0100243 return;
244 }
245
Robert Swiecki0b566112017-10-17 17:39:07 +0200246 LOG_D("File size (Best/New): %zu, SanCov feedback (bb,dso): Best: [%" PRIu64 ",%" PRIu64
247 "] / New: [%" PRIu64 ",%" PRIu64 "], newBBs:%" PRIu64,
Robert Swiecki78633d12017-11-13 23:24:55 +0100248 run->dynamicFileSz, run->global->sanCovCnts.hitBBCnt, run->global->sanCovCnts.iDsoCnt,
Robert Swieckie7294ca2017-11-11 02:46:32 +0100249 run->sanCovCnts.hitBBCnt, run->sanCovCnts.iDsoCnt, run->sanCovCnts.newBBCnt);
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200250
Robert Swiecki78633d12017-11-13 23:24:55 +0100251 MX_SCOPED_LOCK(&run->global->feedback_mutex);
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200252
Robert Swiecki78633d12017-11-13 23:24:55 +0100253 int64_t diff0 = run->global->linux.hwCnts.cpuInstrCnt - run->linux.hwCnts.cpuInstrCnt;
254 int64_t diff1 = run->global->linux.hwCnts.cpuBranchCnt - run->linux.hwCnts.cpuBranchCnt;
Jaggerd5738372016-08-17 20:12:15 +0200255
Anestis Bechtsoudisa16f70f2016-01-03 13:03:21 +0200256 /*
257 * Keep mutated seed if:
Robert Swiecki142f9412016-03-14 19:22:01 +0100258 * a) Newly discovered (not met before) BBs
259 * b) More instrumented DSOs loaded
Robert Swiecki23ec02a2016-01-19 18:47:45 +0100260 *
Anestis Bechtsoudisb78cf602016-01-07 13:10:50 +0200261 * TODO: (a) method can significantly assist to further improvements in interesting areas
Anestis Bechtsoudisa16f70f2016-01-03 13:03:21 +0200262 * discovery if combined with seeds pool/queue support. If a runtime queue is maintained
263 * more interesting seeds can be saved between runs instead of instantly discarded
264 * based on current absolute elitism (only one mutated seed is promoted).
265 */
Jaggerd5738372016-08-17 20:12:15 +0200266
Robert Swieckid50ed422017-11-13 23:32:26 +0100267 bool newCov =
268 (run->sanCovCnts.newBBCnt > 0 || run->global->sanCovCnts.iDsoCnt < run->sanCovCnts.iDsoCnt);
Jaggerd5738372016-08-17 20:12:15 +0200269
Robert Swieckid158aac2016-11-01 23:14:12 +0100270 if (newCov || (diff0 < 0 || diff1 < 0)) {
Robert Swiecki0b566112017-10-17 17:39:07 +0200271 LOG_I("SanCov Update: fsize:%zu, newBBs:%" PRIu64 ", (Cur,New): %" PRIu64 "/%" PRIu64
272 ",%" PRIu64 "/%" PRIu64,
Robert Swiecki78633d12017-11-13 23:24:55 +0100273 run->dynamicFileSz, run->sanCovCnts.newBBCnt, run->global->sanCovCnts.hitBBCnt,
274 run->global->sanCovCnts.iDsoCnt, run->sanCovCnts.hitBBCnt, run->sanCovCnts.iDsoCnt);
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200275
Robert Swiecki78633d12017-11-13 23:24:55 +0100276 run->global->sanCovCnts.hitBBCnt += run->sanCovCnts.newBBCnt;
277 run->global->sanCovCnts.dsoCnt = run->sanCovCnts.dsoCnt;
278 run->global->sanCovCnts.iDsoCnt = run->sanCovCnts.iDsoCnt;
279 run->global->sanCovCnts.crashesCnt += run->sanCovCnts.crashesCnt;
280 run->global->sanCovCnts.newBBCnt = run->sanCovCnts.newBBCnt;
Anestis Bechtsoudisb78cf602016-01-07 13:10:50 +0200281
Robert Swiecki78633d12017-11-13 23:24:55 +0100282 if (run->global->sanCovCnts.totalBBCnt < run->sanCovCnts.totalBBCnt) {
Anestis Bechtsoudisa16f70f2016-01-03 13:03:21 +0200283 /* Keep only the max value (for dlopen cases) to measure total target coverage */
Robert Swiecki78633d12017-11-13 23:24:55 +0100284 run->global->sanCovCnts.totalBBCnt = run->sanCovCnts.totalBBCnt;
Anestis Bechtsoudisa16f70f2016-01-03 13:03:21 +0200285 }
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200286
Robert Swiecki78633d12017-11-13 23:24:55 +0100287 run->global->linux.hwCnts.cpuInstrCnt = run->linux.hwCnts.cpuInstrCnt;
288 run->global->linux.hwCnts.cpuBranchCnt = run->linux.hwCnts.cpuBranchCnt;
Jaggerd5738372016-08-17 20:12:15 +0200289
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100290 fuzz_addFileToFileQ(run->global, run->dynamicFile, run->dynamicFileSz);
dobinedf9f8d2018-01-21 13:57:02 +0100291
Robert Swiecki56276192018-01-21 15:43:02 +0100292 if (run->global->socketFuzzer) {
dobinedf9f8d2018-01-21 13:57:02 +0100293 LOG_D("SocketFuzzer: fuzz: new BB (cov)");
294 fuzz_notifySocketFuzzerNewCov(run->global);
295 }
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200296 }
Anestis Bechtsoudisbe0ac7b2015-12-26 15:38:47 +0200297}
298
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100299/* Return value indicates whether report file should be updated with the current verified crash */
Robert Swiecki9badb552018-01-12 01:42:08 +0100300static bool fuzz_runVerifier(run_t* run) {
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100301 if (!run->crashFileName[0] || !run->backtrace) {
302 return false;
303 }
304
Robert Swiecki9badb552018-01-12 01:42:08 +0100305 uint64_t backtrace = run->backtrace;
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100306
Robert Swiecki9badb552018-01-12 01:42:08 +0100307 char origCrashPath[PATH_MAX];
308 snprintf(origCrashPath, sizeof(origCrashPath), "%s", run->crashFileName);
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100309 /* Workspace is inherited, just append a extra suffix */
310 char verFile[PATH_MAX];
311 snprintf(verFile, sizeof(verFile), "%s.verified", origCrashPath);
312
313 if (files_exists(verFile)) {
Robert Swiecki965af7f2018-01-12 02:30:14 +0100314 LOG_D("Crash file to verify '%s' is already verified as '%s'", origCrashPath, verFile);
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100315 return false;
316 }
Robert Swiecki9badb552018-01-12 01:42:08 +0100317
318 for (int i = 0; i < _HF_VERIFIER_ITER; i++) {
Robert Swieckic4b573f2018-01-12 19:48:24 +0100319 LOG_I("Launching verifier for HASH: %" PRIx64 " (iteration: %d out of %d)", run->backtrace,
320 i + 1, _HF_VERIFIER_ITER);
Robert Swieckif2da05a2018-01-12 03:01:09 +0100321 run->timeStartedMillis = 0;
322 run->backtrace = 0;
323 run->access = 0;
Robert Swiecki9badb552018-01-12 01:42:08 +0100324 run->exception = 0;
Robert Swiecki9badb552018-01-12 01:42:08 +0100325 run->mainWorker = false;
Robert Swiecki9badb552018-01-12 01:42:08 +0100326
327 if (!subproc_Run(run)) {
328 LOG_F("subproc_Run()");
329 }
330
331 /* If stack hash doesn't match skip name tag and exit */
332 if (run->backtrace != backtrace) {
333 LOG_E("Verifier stack mismatch: (original) %" PRIx64 " != (new) %" PRIx64, backtrace,
334 run->backtrace);
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100335 run->backtrace = backtrace;
336 return true;
Robert Swiecki9badb552018-01-12 01:42:08 +0100337 }
Robert Swiecki17b37eb2018-01-12 13:47:12 +0100338
Robert Swieckic4b573f2018-01-12 19:48:24 +0100339 LOG_I("Verifier for HASH: %" PRIx64 " (iteration: %d, left: %d). MATCH!", run->backtrace,
340 i + 1, _HF_VERIFIER_ITER - i - 1);
Robert Swiecki9badb552018-01-12 01:42:08 +0100341 }
342
Robert Swiecki9badb552018-01-12 01:42:08 +0100343 /* Copy file with new suffix & remove original copy */
344 int fd = TEMP_FAILURE_RETRY(open(verFile, O_CREAT | O_EXCL | O_WRONLY, 0600));
345 if (fd == -1 && errno == EEXIST) {
346 LOG_I("It seems that '%s' already exists, skipping", verFile);
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100347 return false;
Robert Swiecki9badb552018-01-12 01:42:08 +0100348 }
349 if (fd == -1) {
350 PLOG_E("Couldn't create '%s'", verFile);
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100351 return true;
Robert Swiecki9badb552018-01-12 01:42:08 +0100352 }
Robert Swiecki3ab16642018-01-12 18:08:37 +0100353 defer {
354 close(fd);
355 };
Robert Swiecki965af7f2018-01-12 02:30:14 +0100356 if (!files_writeToFd(fd, run->dynamicFile, run->dynamicFileSz)) {
357 LOG_E("Couldn't save verified file as '%s'", verFile);
358 unlink(verFile);
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100359 return true;
Robert Swiecki9badb552018-01-12 01:42:08 +0100360 }
361
362 LOG_I("Verified crash for HASH: %" PRIx64 " and saved it as '%s'", backtrace, verFile);
363 ATOMIC_POST_INC(run->global->cnts.verifiedCrashesCnt);
Robert Swiecki9badb552018-01-12 01:42:08 +0100364
365 return true;
366}
367
Robert Swiecki3ab16642018-01-12 18:08:37 +0100368static bool fuzz_fetchInput(run_t* run) {
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100369 if (fuzz_getState(run->global) == _HF_STATE_DYNAMIC_DRY_RUN) {
Robert Swiecki2bad0b42018-01-13 04:00:18 +0100370 run->mutationsPerRun = 0U;
Robert Swiecki0f2c30a2018-01-13 14:03:39 +0100371 if (input_prepareStaticFile(run, /* rewind= */ false)) {
Robert Swiecki3ab16642018-01-12 18:08:37 +0100372 return true;
373 }
Robert Swiecki308ebac2018-01-13 03:59:22 +0100374 fuzz_setDynamicMainState(run);
Robert Swiecki2bad0b42018-01-13 04:00:18 +0100375 run->mutationsPerRun = run->global->mutationsPerRun;
Robert Swiecki3ab16642018-01-12 18:08:37 +0100376 }
377
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100378 if (fuzz_getState(run->global) == _HF_STATE_DYNAMIC_MAIN) {
Robert Swieckia9e34ed2018-01-17 00:31:56 +0100379 if (run->global->exe.externalCommand) {
380 if (!input_prepareExternalFile(run)) {
381 LOG_E("input_prepareFileExternally() failed");
382 return false;
383 }
Robert Swiecki0f2c30a2018-01-13 14:03:39 +0100384 } else if (!input_prepareDynamicInput(run)) {
385 LOG_E("input_prepareFileDynamically() failed");
Robert Swiecki3ab16642018-01-12 18:08:37 +0100386 return false;
387 }
388 }
389
Robert Swieckifb8a5b62018-01-14 05:16:59 +0100390 if (fuzz_getState(run->global) == _HF_STATE_STATIC) {
Robert Swieckia9e34ed2018-01-17 00:31:56 +0100391 if (run->global->exe.externalCommand) {
392 if (!input_prepareExternalFile(run)) {
393 LOG_E("input_prepareFileExternally() failed");
394 return false;
395 }
Robert Swiecki0f2c30a2018-01-13 14:03:39 +0100396 } else if (!input_prepareStaticFile(run, true /* rewind */)) {
397 LOG_E("input_prepareFile() failed");
Robert Swiecki3ab16642018-01-12 18:08:37 +0100398 return false;
399 }
400 }
401
Robert Swiecki0f2c30a2018-01-13 14:03:39 +0100402 if (run->global->exe.postExternalCommand && !input_postProcessFile(run)) {
403 LOG_E("input_postProcessFile() failed");
Robert Swiecki3ab16642018-01-12 18:08:37 +0100404 return false;
405 }
406
407 return true;
408}
409
Robert Swieckid50ed422017-11-13 23:32:26 +0100410static void fuzz_fuzzLoop(run_t* run) {
Robert Swieckie7294ca2017-11-11 02:46:32 +0100411 run->pid = 0;
Robert Swieckif2da05a2018-01-12 03:01:09 +0100412 run->timeStartedMillis = 0;
Robert Swieckie7294ca2017-11-11 02:46:32 +0100413 run->crashFileName[0] = '\0';
Robert Swieckif2da05a2018-01-12 03:01:09 +0100414 run->pc = 0;
415 run->backtrace = 0;
416 run->access = 0;
Robert Swieckie7294ca2017-11-11 02:46:32 +0100417 run->exception = 0;
418 run->report[0] = '\0';
419 run->mainWorker = true;
Robert Swiecki78633d12017-11-13 23:24:55 +0100420 run->mutationsPerRun = run->global->mutationsPerRun;
Robert Swieckie7294ca2017-11-11 02:46:32 +0100421 run->dynamicFileSz = 0;
Robert Swiecki1f1a2f92018-01-15 15:26:37 +0100422 run->dynamicFileCopyFd = -1,
Robert Swieckia96d78d2016-03-14 16:50:50 +0100423
Robert Swieckif2da05a2018-01-12 03:01:09 +0100424 run->sanCovCnts.hitBBCnt = 0;
425 run->sanCovCnts.totalBBCnt = 0;
426 run->sanCovCnts.dsoCnt = 0;
427 run->sanCovCnts.newBBCnt = 0;
428 run->sanCovCnts.crashesCnt = 0;
Robert Swieckia96d78d2016-03-14 16:50:50 +0100429
Robert Swieckif2da05a2018-01-12 03:01:09 +0100430 run->linux.hwCnts.cpuInstrCnt = 0;
431 run->linux.hwCnts.cpuBranchCnt = 0;
432 run->linux.hwCnts.bbCnt = 0;
433 run->linux.hwCnts.newBBCnt = 0;
Jagger190f0dc2015-09-05 16:41:22 +0200434
Robert Swiecki3ab16642018-01-12 18:08:37 +0100435 if (!fuzz_fetchInput(run)) {
436 LOG_F("Cound't prepare input for fuzzing");
Robert Swiecki92a31362017-02-24 16:21:40 +0100437 }
Robert Swiecki3ab16642018-01-12 18:08:37 +0100438 if (!subproc_Run(run)) {
439 LOG_F("Couldn't run fuzzed command");
Jagger190f0dc2015-09-05 16:41:22 +0200440 }
441
Robert Swiecki78633d12017-11-13 23:24:55 +0100442 if (run->global->dynFileMethod != _HF_DYNFILE_NONE) {
443 fuzz_perfFeedback(run);
Robert Swiecki53ec9e42017-02-15 20:34:27 +0100444 }
Robert Swiecki34c876f2018-01-25 17:19:11 +0100445 if (run->global->dynFileMethod & _HF_DYNFILE_SANCOV) {
Robert Swiecki78633d12017-11-13 23:24:55 +0100446 fuzz_sanCovFeedback(run);
Robert Swiecki0f937af2016-03-30 18:19:16 +0200447 }
Robert Swiecki28cc4cb2018-01-12 02:18:29 +0100448 if (run->global->useVerifier && !fuzz_runVerifier(run)) {
449 return;
Anestis Bechtsoudis5c86ebc2015-09-27 18:06:43 +0300450 }
Robert Swiecki78633d12017-11-13 23:24:55 +0100451 report_Report(run);
Jagger190f0dc2015-09-05 16:41:22 +0200452}
453
Robert Swiecki56276192018-01-21 15:43:02 +0100454static void fuzz_fuzzLoopSocket(run_t* run) {
dobinedf9f8d2018-01-21 13:57:02 +0100455 run->pid = 0;
456 run->timeStartedMillis = 0;
457 run->crashFileName[0] = '\0';
458 run->pc = 0;
459 run->backtrace = 0;
460 run->access = 0;
461 run->exception = 0;
462 run->report[0] = '\0';
463 run->mainWorker = true;
dobinedf9f8d2018-01-21 13:57:02 +0100464 run->mutationsPerRun = run->global->mutationsPerRun;
465 run->dynamicFileSz = 0;
466 run->dynamicFileCopyFd = -1,
467
468 run->sanCovCnts.hitBBCnt = 0;
469 run->sanCovCnts.totalBBCnt = 0;
470 run->sanCovCnts.dsoCnt = 0;
471 run->sanCovCnts.newBBCnt = 0;
472 run->sanCovCnts.crashesCnt = 0;
473
474 run->linux.hwCnts.cpuInstrCnt = 0;
475 run->linux.hwCnts.cpuBranchCnt = 0;
476 run->linux.hwCnts.bbCnt = 0;
477 run->linux.hwCnts.newBBCnt = 0;
478
479 LOG_I("------------------------------------------------------");
480
481 /* First iteration: Start target
482 Other iterations: re-start target, if necessary
483 subproc_Run() will decide by itself if a restart is necessary, via
484 subproc_New()
485 */
486 LOG_D("------[ 1: subproc_run");
487 if (!subproc_Run(run)) {
488 LOG_W("Couldn't run server");
489 }
490
491 /* Tell the external fuzzer to send data to target
492 The fuzzer will notify us when finished; block until then.
493 */
494 LOG_D("------[ 2: fetch input");
495 if (!fuzz_waitForExternalInput(run)) {
496 /* Fuzzer could not connect to target, and told us to
497 restart it. Do it on the next iteration. */
Robert Swiecki56276192018-01-21 15:43:02 +0100498 LOG_D("------[ 2.1: Target down, will restart it");
dobinedf9f8d2018-01-21 13:57:02 +0100499 run->hasCrashed = true;
500 return;
501 }
502
503 LOG_D("------[ 3: feedback");
504 if (run->global->dynFileMethod != _HF_DYNFILE_NONE) {
505 fuzz_perfFeedback(run);
506 }
Robert Swiecki34c876f2018-01-25 17:19:11 +0100507 if (run->global->dynFileMethod & _HF_DYNFILE_SANCOV) {
dobinedf9f8d2018-01-21 13:57:02 +0100508 fuzz_sanCovFeedback(run);
509 }
510 if (run->global->useVerifier && !fuzz_runVerifier(run)) {
511 return;
512 }
513
514 report_Report(run);
515
516 /* Try to identify if the target crashed.
517 This information will be used in the next iteration of
518 this loop, to restart the target if necessary.
519 The fuzzer will be also notified.
520
521 Crash identification does not need to work 100%, as the external fuzzer
522 can also detect timeouts on the target server, and will
523 notify us. */
524 LOG_D("------[ 4: reap child");
525 arch_reapChild(run);
526}
527
Robert Swieckid50ed422017-11-13 23:32:26 +0100528static void* fuzz_threadNew(void* arg) {
Robert Swiecki4e595fb2017-10-11 17:26:51 +0200529 honggfuzz_t* hfuzz = (honggfuzz_t*)arg;
Robert Swiecki66b65122017-11-11 02:55:55 +0100530 unsigned int fuzzNo = ATOMIC_POST_INC(hfuzz->threads.threadsActiveCnt);
Robert Swiecki0ec98112017-02-03 02:08:14 +0100531 LOG_I("Launched new fuzzing thread, no. #%" PRId32, fuzzNo);
Anestis Bechtsoudis02b99be2015-12-27 11:53:01 +0200532
Robert Swieckie7294ca2017-11-11 02:46:32 +0100533 run_t run = {
Robert Swiecki78633d12017-11-13 23:24:55 +0100534 .global = hfuzz,
Robert Swieckidecf14b2016-03-31 15:09:28 +0200535 .pid = 0,
536 .persistentPid = 0,
Robert Swieckibf8f8cc2017-11-09 00:42:50 +0100537 .dynfileqCurrent = NULL,
Robert Swiecki599dee12018-01-10 02:21:58 +0100538 .dynamicFile = NULL,
539 .dynamicFileFd = -1,
Jaggerfa3544a2016-08-30 02:55:55 +0200540 .fuzzNo = fuzzNo,
Jagger93253f72016-09-01 22:40:12 +0200541 .persistentSock = -1,
Robert Swiecki013bc9c2016-12-12 17:31:06 +0100542 .tmOutSignaled = false,
Robert Swiecki94d314c2018-02-07 21:23:00 +0100543 .origFileName = "[DYNAMIC]",
Robert Swiecki12800cd2016-03-31 15:38:10 +0200544
Robert Swieckib692f282016-08-25 16:21:08 +0200545 .linux.attachedPid = 0,
Robert Swieckidecf14b2016-03-31 15:09:28 +0200546 };
dobinedf9f8d2018-01-21 13:57:02 +0100547
Robert Swiecki5eeb29b2018-01-21 16:07:06 +0100548 /* Do not try to handle input files with socketfuzzer */
Robert Swiecki56276192018-01-21 15:43:02 +0100549 if (!hfuzz->socketFuzzer) {
550 if (!(run.dynamicFile = files_mapSharedMem(
Robert Swiecki2d1631a2018-02-02 12:56:08 +0100551 hfuzz->maxFileSz, &run.dynamicFileFd, "hfuzz-input", run.global->io.workDir))) {
dobinedf9f8d2018-01-21 13:57:02 +0100552 LOG_F("Couldn't create an input file of size: %zu", hfuzz->maxFileSz);
553 }
Robert Swiecki599dee12018-01-10 02:21:58 +0100554 }
Robert Swiecki41033812018-01-21 16:02:45 +0100555 defer {
556 if (run.dynamicFileFd != -1) {
557 close(run.dynamicFileFd);
558 }
559 };
Robert Swieckidecf14b2016-03-31 15:09:28 +0200560
Robert Swiecki78633d12017-11-13 23:24:55 +0100561 if (arch_archThreadInit(&run) == false) {
Robert Swiecki0f937af2016-03-30 18:19:16 +0200562 LOG_F("Could not initialize the thread");
563 }
564
Robert Swieckia96d78d2016-03-14 16:50:50 +0100565 for (;;) {
Anestis Bechtsoudis46ea10e2015-11-07 18:16:25 +0200566 /* Check if dry run mode with verifier enabled */
Robert Swiecki56276192018-01-21 15:43:02 +0100567 if (run.global->mutationsPerRun == 0U && run.global->useVerifier && !hfuzz->socketFuzzer) {
Robert Swiecki82c707c2017-11-14 16:36:23 +0100568 if (ATOMIC_POST_INC(run.global->cnts.mutationsCnt) >= run.global->io.fileCnt) {
Robert Swiecki78633d12017-11-13 23:24:55 +0100569 ATOMIC_POST_INC(run.global->threads.threadsFinished);
Robert Swiecki069b48f2017-05-31 01:00:08 +0200570 break;
Anestis Bechtsoudis46ea10e2015-11-07 18:16:25 +0200571 }
572 }
573 /* Check for max iterations limit if set */
Robert Swiecki2542dc02017-11-14 03:35:59 +0100574 else if ((ATOMIC_POST_INC(run.global->cnts.mutationsCnt) >= run.global->mutationsMax) &&
Robert Swieckid50ed422017-11-13 23:32:26 +0100575 run.global->mutationsMax) {
Robert Swiecki78633d12017-11-13 23:24:55 +0100576 ATOMIC_POST_INC(run.global->threads.threadsFinished);
Robert Swiecki069b48f2017-05-31 01:00:08 +0200577 break;
Robert Swiecki8d01b012017-02-19 15:48:11 +0100578 }
579
Robert Swieckicfa81142018-01-15 02:19:26 +0100580 input_setSize(&run, run.global->maxFileSz);
dobinedf9f8d2018-01-21 13:57:02 +0100581 if (hfuzz->socketFuzzer) {
582 fuzz_fuzzLoopSocket(&run);
583 } else {
584 fuzz_fuzzLoop(&run);
585 }
586
Robert Swiecki0dde76d2017-11-16 19:25:44 +0100587 if (fuzz_isTerminating()) {
Robert Swiecki069b48f2017-05-31 01:00:08 +0200588 break;
589 }
590
Robert Swiecki2542dc02017-11-14 03:35:59 +0100591 if (run.global->exitUponCrash && ATOMIC_GET(run.global->cnts.crashesCnt) > 0) {
Robert Swiecki069b48f2017-05-31 01:00:08 +0200592 LOG_I("Seen a crash. Terminating all fuzzing threads");
Robert Swiecki0dde76d2017-11-16 19:25:44 +0100593 fuzz_setTerminating();
Robert Swiecki069b48f2017-05-31 01:00:08 +0200594 break;
595 }
robert.swiecki@gmail.comd4dd4df2015-02-18 00:50:12 +0000596 }
Robert Swiecki069b48f2017-05-31 01:00:08 +0200597
598 LOG_I("Terminating thread no. #%" PRId32, fuzzNo);
Robert Swiecki78633d12017-11-13 23:24:55 +0100599 ATOMIC_POST_INC(run.global->threads.threadsFinished);
Robert Swiecki82c707c2017-11-14 16:36:23 +0100600 pthread_kill(run.global->threads.mainThread, SIGALRM);
Robert Swiecki069b48f2017-05-31 01:00:08 +0200601 return NULL;
robert.swiecki@gmail.com882900b2015-02-11 13:56:22 +0000602}
603
Robert Swieckid50ed422017-11-13 23:32:26 +0100604static void fuzz_runThread(honggfuzz_t* hfuzz, pthread_t* thread, void* (*thread_func)(void*)) {
robert.swiecki@gmail.comc8443142015-02-13 13:46:40 +0000605 pthread_attr_t attr;
robert.swiecki@gmail.comc8443142015-02-13 13:46:40 +0000606
robert.swiecki@gmail.com6f5c2392015-02-16 18:13:09 +0000607 pthread_attr_init(&attr);
Robert Swiecki33fb2842017-02-19 05:39:50 +0100608 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
robert.swiecki@gmail.com441089a2015-02-23 13:14:07 +0000609 pthread_attr_setstacksize(&attr, _HF_PTHREAD_STACKSIZE);
Robert Swiecki4e595fb2017-10-11 17:26:51 +0200610 pthread_attr_setguardsize(&attr, (size_t)sysconf(_SC_PAGESIZE));
robert.swiecki@gmail.comc8443142015-02-13 13:46:40 +0000611
Robert Swiecki4e595fb2017-10-11 17:26:51 +0200612 if (pthread_create(thread, &attr, thread_func, (void*)hfuzz) < 0) {
Robert Swieckic8c32db2015-10-09 18:06:22 +0200613 PLOG_F("Couldn't create a new thread");
robert.swiecki@gmail.comc8443142015-02-13 13:46:40 +0000614 }
615
Robert Swiecki33fb2842017-02-19 05:39:50 +0100616 pthread_attr_destroy(&attr);
617
robert.swiecki@gmail.comc8443142015-02-13 13:46:40 +0000618 return;
619}
620
Robert Swieckid50ed422017-11-13 23:32:26 +0100621void fuzz_threadsStart(honggfuzz_t* hfuzz, pthread_t* threads) {
robert.swiecki@gmail.com956276a2015-04-16 16:51:52 +0000622 if (!arch_archInit(hfuzz)) {
Robert Swieckic8c32db2015-10-09 18:06:22 +0200623 LOG_F("Couldn't prepare arch for fuzzing");
robert.swiecki@gmail.comef829fa2011-06-22 13:51:57 +0000624 }
Anestis Bechtsoudise5f09f82016-12-27 16:06:05 +0200625 if (!sanitizers_Init(hfuzz)) {
626 LOG_F("Couldn't prepare sanitizer options");
627 }
Jagger00265602016-03-10 02:36:27 +0100628 if (!sancov_Init(hfuzz)) {
629 LOG_F("Couldn't prepare sancov options");
630 }
robert.swiecki@gmail.comef829fa2011-06-22 13:51:57 +0000631
Robert Swiecki11cb5ad2018-01-21 15:54:24 +0100632 if (hfuzz->socketFuzzer) {
Robert Swiecki5eeb29b2018-01-21 16:07:06 +0100633 /* Don't do dry run with socketFuzzer */
Robert Swiecki41033812018-01-21 16:02:45 +0100634 LOG_I("Entering phase - Feedback Driven Mode (SocketFuzzer)");
dobinedf9f8d2018-01-21 13:57:02 +0100635 hfuzz->state = _HF_STATE_DYNAMIC_MAIN;
Robert Swiecki34c876f2018-01-25 17:19:11 +0100636 } else if (hfuzz->dynFileMethod != _HF_DYNFILE_NONE) {
Robert Swiecki41033812018-01-21 16:02:45 +0100637 LOG_I("Entering phase 1/2: Dry Run");
638 hfuzz->state = _HF_STATE_DYNAMIC_DRY_RUN;
Robert Swieckia96d78d2016-03-14 16:50:50 +0100639 } else {
Robert Swiecki41033812018-01-21 16:02:45 +0100640 LOG_I("Entering phase: Static");
641 hfuzz->state = _HF_STATE_STATIC;
Robert Swieckia96d78d2016-03-14 16:50:50 +0100642 }
643
Robert Swiecki66b65122017-11-11 02:55:55 +0100644 for (size_t i = 0; i < hfuzz->threads.threadsMax; i++) {
Robert Swiecki33fb2842017-02-19 05:39:50 +0100645 fuzz_runThread(hfuzz, &threads[i], fuzz_threadNew);
robert.swiecki3bb518c2010-10-14 00:48:24 +0000646 }
robert.swiecki3bb518c2010-10-14 00:48:24 +0000647}
Robert Swiecki33fb2842017-02-19 05:39:50 +0100648
Robert Swieckid50ed422017-11-13 23:32:26 +0100649void fuzz_threadsStop(honggfuzz_t* hfuzz, pthread_t* threads) {
Robert Swiecki66b65122017-11-11 02:55:55 +0100650 for (size_t i = 0; i < hfuzz->threads.threadsMax; i++) {
Robert Swiecki4e595fb2017-10-11 17:26:51 +0200651 void* retval;
Robert Swiecki33fb2842017-02-19 05:39:50 +0100652 if (pthread_join(threads[i], &retval) != 0) {
653 PLOG_F("Couldn't pthread_join() thread: %zu", i);
654 }
655 }
656 LOG_I("All threads done");
657}