blob: e205b3aa52ac9f9a693fd249c52de1b84f45afb8 [file] [log] [blame]
Jens Axboeebac4652005-12-08 15:25:21 +01001/*
2 * fio - the flexible io tester
3 *
4 * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
Jens Axboeaae22ca2006-09-05 10:46:22 +02005 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
Jens Axboeebac4652005-12-08 15:25:21 +01006 *
Jens Axboe8e9fe632006-10-24 15:11:09 +02007 * The license below covers all files distributed with fio unless otherwise
8 * noted in the file itself.
9 *
Jens Axboeebac4652005-12-08 15:25:21 +010010 * This program is free software; you can redistribute it and/or modify
Jens Axboe8e9fe632006-10-24 15:11:09 +020011 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
Jens Axboeebac4652005-12-08 15:25:21 +010013 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 *
23 */
Jens Axboeebac4652005-12-08 15:25:21 +010024#include <unistd.h>
25#include <fcntl.h>
26#include <string.h>
Bruce Cran9b836562011-01-08 19:49:54 +010027#include <limits.h>
Jens Axboeebac4652005-12-08 15:25:21 +010028#include <signal.h>
29#include <time.h>
Jens Axboedbe11252007-02-11 00:19:51 +010030#include <locale.h>
Jens Axboe36167d82007-02-18 05:41:31 +010031#include <assert.h>
Jens Axboea5b01f12010-11-11 09:19:49 +010032#include <time.h>
Jens Axboeebac4652005-12-08 15:25:21 +010033#include <sys/stat.h>
34#include <sys/wait.h>
35#include <sys/ipc.h>
36#include <sys/shm.h>
Jens Axboeebac4652005-12-08 15:25:21 +010037#include <sys/mman.h>
38
39#include "fio.h"
Jens Axboe210f43b2007-04-17 19:52:18 +020040#include "hash.h"
Jens Axboe2e5cdb12008-03-01 18:52:49 +010041#include "smalloc.h"
Jens Axboe4f5af7b2009-06-03 08:45:40 +020042#include "verify.h"
Jens Axboea917a8b2010-09-02 13:23:20 +020043#include "trim.h"
Jens Axboe7c9b1bc2009-06-03 09:25:57 +020044#include "diskutil.h"
Jens Axboea696fa22009-12-04 10:05:02 +010045#include "cgroup.h"
Jens Axboe07b32322010-03-05 09:48:44 +010046#include "profile.h"
Jens Axboed54a1442010-03-25 23:07:35 +010047#include "lib/rand.h"
Jens Axboeebac4652005-12-08 15:25:21 +010048
Jens Axboecfc99db2007-03-14 10:34:47 +010049unsigned long page_mask;
50unsigned long page_size;
Jens Axboed529ee12009-07-01 10:33:03 +020051
52#define PAGE_ALIGN(buf) \
Jens Axboe29d610e2007-02-06 13:25:33 +010053 (char *) (((unsigned long) (buf) + page_mask) & ~page_mask)
Jens Axboeebac4652005-12-08 15:25:21 +010054
55int groupid = 0;
56int thread_number = 0;
Jens Axboe9cedf162007-03-12 11:29:30 +010057int nr_process = 0;
58int nr_thread = 0;
Jens Axboeebac4652005-12-08 15:25:21 +010059int shm_id = 0;
Jens Axboe53cdc682006-10-18 11:50:58 +020060int temp_stall_ts;
Jens Axboed3eeeab2008-04-06 12:19:05 +020061unsigned long done_secs = 0;
Jens Axboeebac4652005-12-08 15:25:21 +010062
Jens Axboecdd18ad2008-02-27 18:58:00 +010063static struct fio_mutex *startup_mutex;
Fabio Checconid2bbb8d2009-05-18 12:54:50 +020064static struct fio_mutex *writeout_mutex;
Jens Axboe6ce15a32007-01-12 09:04:41 +010065static volatile int fio_abort;
Jens Axboe437c9b72007-02-13 18:20:37 +010066static int exit_value;
Jens Axboebe4ecfd2008-12-08 14:10:52 +010067static pthread_t gtod_thread;
Jens Axboe79857012011-01-14 14:14:58 +010068static pthread_t disk_util_thread;
Jens Axboedae53412009-12-04 19:54:18 +010069static struct flist_head *cgroup_list;
Jens Axboe6adb38a2009-12-07 08:01:26 +010070static char *cgroup_mnt;
Jens Axboeebac4652005-12-08 15:25:21 +010071
Jens Axboebb3884d2007-01-17 17:23:11 +110072struct io_log *agg_io_log[2];
73
Jens Axboeebac4652005-12-08 15:25:21 +010074#define TERMINATE_ALL (-1)
Jens Axboe75154842006-06-01 13:56:09 +020075#define JOB_START_TIMEOUT (5 * 1000)
Jens Axboeebac4652005-12-08 15:25:21 +010076
Jens Axboeb29ee5b2008-09-11 10:17:26 +020077void td_set_runstate(struct thread_data *td, int runstate)
Jens Axboe6ce15a32007-01-12 09:04:41 +010078{
Jens Axboed8fab1b2008-03-06 10:25:17 +010079 if (td->runstate == runstate)
80 return;
81
Jens Axboe5921e802008-05-30 15:02:38 +020082 dprint(FD_PROCESS, "pid=%d: runstate %d -> %d\n", (int) td->pid,
83 td->runstate, runstate);
Jens Axboe6ce15a32007-01-12 09:04:41 +010084 td->runstate = runstate;
85}
86
Jens Axboe390c40e2007-03-05 12:35:29 +010087static void terminate_threads(int group_id)
Jens Axboeebac4652005-12-08 15:25:21 +010088{
Jens Axboe34572e22006-10-20 09:33:18 +020089 struct thread_data *td;
Jens Axboeebac4652005-12-08 15:25:21 +010090 int i;
91
Jens Axboe6f88bcb2008-03-19 10:29:07 +010092 dprint(FD_PROCESS, "terminate group_id=%d\n", group_id);
93
Jens Axboe34572e22006-10-20 09:33:18 +020094 for_each_td(td, i) {
Jens Axboeebac4652005-12-08 15:25:21 +010095 if (group_id == TERMINATE_ALL || groupid == td->groupid) {
Jens Axboe6f88bcb2008-03-19 10:29:07 +010096 dprint(FD_PROCESS, "setting terminate on %s/%d\n",
Jens Axboe5921e802008-05-30 15:02:38 +020097 td->o.name, (int) td->pid);
Jens Axboead830ed2008-02-18 21:11:24 +010098 td->terminate = 1;
99 td->o.start_delay = 0;
100
Jens Axboe7a93ab12007-04-18 12:25:41 +0200101 /*
102 * if the thread is running, just let it exit
103 */
104 if (td->runstate < TD_RUNNING)
Bruce Cran03e20d62011-01-02 20:14:54 +0100105 kill(td->pid, SIGTERM);
Jens Axboef63f7f32008-03-01 15:25:24 +0100106 else {
107 struct ioengine_ops *ops = td->io_ops;
108
Bruce Cran03e20d62011-01-02 20:14:54 +0100109 if (ops && (ops->flags & FIO_SIGTERM))
110 kill(td->pid, SIGTERM);
Jens Axboef63f7f32008-03-01 15:25:24 +0100111 }
Jens Axboeebac4652005-12-08 15:25:21 +0100112 }
113 }
114}
115
Jens Axboe5f8f9722008-11-18 19:10:21 +0100116/*
117 * Happens on thread runs with ctrl-c, ignore our own SIGQUIT
118 */
119static void sig_quit(int sig)
120{
121}
122
Jens Axboe697a6062008-05-31 00:04:45 +0200123static void sig_int(int sig)
124{
125 if (threads) {
Jens Axboe4ceb30d2010-02-24 09:19:14 +0100126 log_info("\nfio: terminating on signal %d\n", sig);
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100127 fflush(stdout);
128 terminate_threads(TERMINATE_ALL);
Jens Axboeebac4652005-12-08 15:25:21 +0100129 }
130}
131
Jens Axboe79857012011-01-14 14:14:58 +0100132static void *disk_thread_main(void *data)
Jens Axboea5b01f12010-11-11 09:19:49 +0100133{
Jens Axboe79857012011-01-14 14:14:58 +0100134 fio_mutex_up(startup_mutex);
135
136 while (threads) {
137 usleep(DISK_UTIL_MSEC * 1000);
Jens Axboe9a793c22011-01-14 15:22:14 +0100138 if (!threads)
139 break;
Jens Axboe79857012011-01-14 14:14:58 +0100140 update_io_ticks();
141 print_thread_status();
142 }
143
144 return NULL;
Jens Axboea5b01f12010-11-11 09:19:49 +0100145}
146
Jens Axboe79857012011-01-14 14:14:58 +0100147static int create_disk_util_thread(void)
Jens Axboea5b01f12010-11-11 09:19:49 +0100148{
Jens Axboe79857012011-01-14 14:14:58 +0100149 int ret;
Jens Axboea5b01f12010-11-11 09:19:49 +0100150
Jens Axboe69204d62011-01-14 18:29:19 +0100151 ret = pthread_create(&disk_util_thread, NULL, disk_thread_main, NULL);
Jens Axboe79857012011-01-14 14:14:58 +0100152 if (ret) {
153 log_err("Can't create disk util thread: %s\n", strerror(ret));
154 return 1;
155 }
Jens Axboea5b01f12010-11-11 09:19:49 +0100156
Jens Axboe79857012011-01-14 14:14:58 +0100157 ret = pthread_detach(disk_util_thread);
158 if (ret) {
159 log_err("Can't detatch disk util thread: %s\n", strerror(ret));
160 return 1;
161 }
Bruce Cran03e20d62011-01-02 20:14:54 +0100162
Jens Axboe79857012011-01-14 14:14:58 +0100163 dprint(FD_MUTEX, "wait on startup_mutex\n");
164 fio_mutex_down(startup_mutex);
165 dprint(FD_MUTEX, "done waiting on startup_mutex\n");
166 return 0;
Jens Axboea5b01f12010-11-11 09:19:49 +0100167}
168
Jens Axboe697a6062008-05-31 00:04:45 +0200169static void set_sig_handlers(void)
170{
171 struct sigaction act;
172
173 memset(&act, 0, sizeof(act));
Jens Axboe697a6062008-05-31 00:04:45 +0200174 act.sa_handler = sig_int;
175 act.sa_flags = SA_RESTART;
176 sigaction(SIGINT, &act, NULL);
Jens Axboede79c912008-08-04 21:43:55 +0200177
178 memset(&act, 0, sizeof(act));
Jens Axboe5f8f9722008-11-18 19:10:21 +0100179 act.sa_handler = sig_quit;
180 act.sa_flags = SA_RESTART;
Bruce Cran03e20d62011-01-02 20:14:54 +0100181 sigaction(SIGTERM, &act, NULL);
Jens Axboe697a6062008-05-31 00:04:45 +0200182}
183
184/*
Jens Axboe906c8d72006-06-13 09:37:56 +0200185 * Check if we are above the minimum rate given.
186 */
Jens Axboe581e7142009-06-09 12:47:16 +0200187static int __check_min_rate(struct thread_data *td, struct timeval *now,
188 enum td_ddir ddir)
Jens Axboeebac4652005-12-08 15:25:21 +0100189{
Jens Axboe09042002007-02-23 10:56:22 +0100190 unsigned long long bytes = 0;
Jens Axboe4e991c22007-03-15 11:41:11 +0100191 unsigned long iops = 0;
Jens Axboeebac4652005-12-08 15:25:21 +0100192 unsigned long spent;
193 unsigned long rate;
Jens Axboe581e7142009-06-09 12:47:16 +0200194 unsigned int ratemin = 0;
195 unsigned int rate_iops = 0;
196 unsigned int rate_iops_min = 0;
Jens Axboeebac4652005-12-08 15:25:21 +0100197
Jens Axboeff58fce2010-08-25 12:02:08 +0200198 assert(ddir_rw(ddir));
199
Jens Axboed982d872009-06-10 06:35:48 +0200200 if (!td->o.ratemin[ddir] && !td->o.rate_iops_min[ddir])
201 return 0;
202
Jens Axboeebac4652005-12-08 15:25:21 +0100203 /*
204 * allow a 2 second settle period in the beginning
205 */
206 if (mtime_since(&td->start, now) < 2000)
207 return 0;
208
Jens Axboe581e7142009-06-09 12:47:16 +0200209 iops += td->io_blocks[ddir];
210 bytes += td->this_io_bytes[ddir];
211 ratemin += td->o.ratemin[ddir];
212 rate_iops += td->o.rate_iops[ddir];
213 rate_iops_min += td->o.rate_iops_min[ddir];
Jens Axboe09042002007-02-23 10:56:22 +0100214
Jens Axboeebac4652005-12-08 15:25:21 +0100215 /*
216 * if rate blocks is set, sample is running
217 */
Jens Axboe581e7142009-06-09 12:47:16 +0200218 if (td->rate_bytes[ddir] || td->rate_blocks[ddir]) {
219 spent = mtime_since(&td->lastrate[ddir], now);
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100220 if (spent < td->o.ratecycle)
Jens Axboeebac4652005-12-08 15:25:21 +0100221 return 0;
222
Jens Axboe581e7142009-06-09 12:47:16 +0200223 if (td->o.rate[ddir]) {
Jens Axboe4e991c22007-03-15 11:41:11 +0100224 /*
225 * check bandwidth specified rate
226 */
Jens Axboe581e7142009-06-09 12:47:16 +0200227 if (bytes < td->rate_bytes[ddir]) {
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100228 log_err("%s: min rate %u not met\n", td->o.name,
Jens Axboe581e7142009-06-09 12:47:16 +0200229 ratemin);
Jens Axboe413dd452007-02-23 09:26:09 +0100230 return 1;
Jens Axboe4e991c22007-03-15 11:41:11 +0100231 } else {
Jens Axboe581e7142009-06-09 12:47:16 +0200232 rate = ((bytes - td->rate_bytes[ddir]) * 1000) / spent;
233 if (rate < ratemin ||
234 bytes < td->rate_bytes[ddir]) {
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100235 log_err("%s: min rate %u not met, got"
Jens Axboeb22989b2009-07-17 22:29:23 +0200236 " %luKB/sec\n", td->o.name,
Jens Axboe581e7142009-06-09 12:47:16 +0200237 ratemin, rate);
Jens Axboe4e991c22007-03-15 11:41:11 +0100238 return 1;
239 }
240 }
241 } else {
242 /*
243 * checks iops specified rate
244 */
Jens Axboe581e7142009-06-09 12:47:16 +0200245 if (iops < rate_iops) {
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100246 log_err("%s: min iops rate %u not met\n",
Jens Axboe581e7142009-06-09 12:47:16 +0200247 td->o.name, rate_iops);
Jens Axboe4e991c22007-03-15 11:41:11 +0100248 return 1;
249 } else {
Jens Axboe581e7142009-06-09 12:47:16 +0200250 rate = ((iops - td->rate_blocks[ddir]) * 1000) / spent;
251 if (rate < rate_iops_min ||
252 iops < td->rate_blocks[ddir]) {
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100253 log_err("%s: min iops rate %u not met,"
254 " got %lu\n", td->o.name,
Jens Axboe581e7142009-06-09 12:47:16 +0200255 rate_iops_min, rate);
Jens Axboe4e991c22007-03-15 11:41:11 +0100256 }
Jens Axboe413dd452007-02-23 09:26:09 +0100257 }
Jens Axboeebac4652005-12-08 15:25:21 +0100258 }
259 }
260
Jens Axboe581e7142009-06-09 12:47:16 +0200261 td->rate_bytes[ddir] = bytes;
262 td->rate_blocks[ddir] = iops;
263 memcpy(&td->lastrate[ddir], now, sizeof(*now));
Jens Axboeebac4652005-12-08 15:25:21 +0100264 return 0;
265}
266
Jens Axboe581e7142009-06-09 12:47:16 +0200267static int check_min_rate(struct thread_data *td, struct timeval *now,
268 unsigned long *bytes_done)
269{
270 int ret = 0;
271
272 if (bytes_done[0])
273 ret |= __check_min_rate(td, now, 0);
274 if (bytes_done[1])
275 ret |= __check_min_rate(td, now, 1);
276
277 return ret;
278}
279
Jens Axboeebac4652005-12-08 15:25:21 +0100280static inline int runtime_exceeded(struct thread_data *td, struct timeval *t)
281{
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100282 if (!td->o.timeout)
Jens Axboeebac4652005-12-08 15:25:21 +0100283 return 0;
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100284 if (mtime_since(&td->epoch, t) >= td->o.timeout * 1000)
Jens Axboeebac4652005-12-08 15:25:21 +0100285 return 1;
286
287 return 0;
288}
289
Jens Axboe906c8d72006-06-13 09:37:56 +0200290/*
291 * When job exits, we can cancel the in-flight IO if we are using async
292 * io. Attempt to do so.
293 */
Jens Axboeebac4652005-12-08 15:25:21 +0100294static void cleanup_pending_aio(struct thread_data *td)
295{
Jens Axboe01743ee2008-06-02 12:19:19 +0200296 struct flist_head *entry, *n;
Jens Axboeebac4652005-12-08 15:25:21 +0100297 struct io_u *io_u;
298 int r;
299
300 /*
301 * get immediately available events, if any
302 */
Jens Axboe581e7142009-06-09 12:47:16 +0200303 r = io_u_queued_complete(td, 0, NULL);
Jens Axboeb2fdda42007-02-20 20:52:51 +0100304 if (r < 0)
305 return;
Jens Axboeebac4652005-12-08 15:25:21 +0100306
307 /*
308 * now cancel remaining active events
309 */
Jens Axboe2866c822006-10-09 15:57:48 +0200310 if (td->io_ops->cancel) {
Jens Axboe01743ee2008-06-02 12:19:19 +0200311 flist_for_each_safe(entry, n, &td->io_u_busylist) {
312 io_u = flist_entry(entry, struct io_u, list);
Jens Axboeebac4652005-12-08 15:25:21 +0100313
Jens Axboe0c6e7512007-02-22 11:19:39 +0100314 /*
315 * if the io_u isn't in flight, then that generally
316 * means someone leaked an io_u. complain but fix
317 * it up, so we don't stall here.
318 */
319 if ((io_u->flags & IO_U_F_FLIGHT) == 0) {
320 log_err("fio: non-busy IO on busy list\n");
Jens Axboeebac4652005-12-08 15:25:21 +0100321 put_io_u(td, io_u);
Jens Axboe0c6e7512007-02-22 11:19:39 +0100322 } else {
323 r = td->io_ops->cancel(td, io_u);
324 if (!r)
325 put_io_u(td, io_u);
326 }
Jens Axboeebac4652005-12-08 15:25:21 +0100327 }
328 }
329
Jens Axboe97601022007-02-18 12:47:29 +0100330 if (td->cur_depth)
Jens Axboe581e7142009-06-09 12:47:16 +0200331 r = io_u_queued_complete(td, td->cur_depth, NULL);
Jens Axboeebac4652005-12-08 15:25:21 +0100332}
333
Jens Axboe906c8d72006-06-13 09:37:56 +0200334/*
Jens Axboe858a3d42006-10-24 14:54:44 +0200335 * Helper to handle the final sync of a file. Works just like the normal
336 * io path, just does everything sync.
337 */
338static int fio_io_sync(struct thread_data *td, struct fio_file *f)
339{
340 struct io_u *io_u = __get_io_u(td);
Jens Axboe858a3d42006-10-24 14:54:44 +0200341 int ret;
342
343 if (!io_u)
344 return 1;
345
346 io_u->ddir = DDIR_SYNC;
347 io_u->file = f;
348
349 if (td_io_prep(td, io_u)) {
350 put_io_u(td, io_u);
351 return 1;
352 }
353
Jens Axboe755200a2007-02-19 13:08:12 +0100354requeue:
Jens Axboe858a3d42006-10-24 14:54:44 +0200355 ret = td_io_queue(td, io_u);
Jens Axboe36167d82007-02-18 05:41:31 +0100356 if (ret < 0) {
Jens Axboee1161c32007-02-22 19:36:48 +0100357 td_verror(td, io_u->error, "td_io_queue");
Jens Axboe858a3d42006-10-24 14:54:44 +0200358 put_io_u(td, io_u);
Jens Axboe858a3d42006-10-24 14:54:44 +0200359 return 1;
Jens Axboe36167d82007-02-18 05:41:31 +0100360 } else if (ret == FIO_Q_QUEUED) {
Jens Axboe581e7142009-06-09 12:47:16 +0200361 if (io_u_queued_complete(td, 1, NULL) < 0)
Jens Axboe36167d82007-02-18 05:41:31 +0100362 return 1;
Jens Axboe36167d82007-02-18 05:41:31 +0100363 } else if (ret == FIO_Q_COMPLETED) {
364 if (io_u->error) {
Jens Axboee1161c32007-02-22 19:36:48 +0100365 td_verror(td, io_u->error, "td_io_queue");
Jens Axboe36167d82007-02-18 05:41:31 +0100366 return 1;
367 }
Jens Axboe858a3d42006-10-24 14:54:44 +0200368
Jens Axboe581e7142009-06-09 12:47:16 +0200369 if (io_u_sync_complete(td, io_u, NULL) < 0)
Jens Axboeb2fdda42007-02-20 20:52:51 +0100370 return 1;
Jens Axboe755200a2007-02-19 13:08:12 +0100371 } else if (ret == FIO_Q_BUSY) {
372 if (td_io_commit(td))
373 return 1;
374 goto requeue;
Jens Axboe858a3d42006-10-24 14:54:44 +0200375 }
376
377 return 0;
378}
379
Jens Axboe993bf482008-11-14 13:04:53 +0100380static inline void update_tv_cache(struct thread_data *td)
381{
382 if ((++td->tv_cache_nr & td->tv_cache_mask) == td->tv_cache_mask)
383 fio_gettime(&td->tv_cache, NULL);
384}
385
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200386static int break_on_this_error(struct thread_data *td, int *retptr)
387{
388 int ret = *retptr;
389
390 if (ret < 0 || td->error) {
391 int err;
392
Jens Axboe1ec99ee2009-06-15 12:37:30 +0200393 if (!td->o.continue_on_error)
394 return 1;
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200395
396 if (ret < 0)
397 err = -ret;
398 else
399 err = td->error;
400
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200401 if (td_non_fatal_error(err)) {
402 /*
403 * Continue with the I/Os in case of
404 * a non fatal error.
405 */
Radha Ramachandran8a4f8a22009-07-02 08:30:49 +0200406 update_error_count(td, err);
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200407 td_clear_error(td);
408 *retptr = 0;
409 return 0;
Radha Ramachandran8a4f8a22009-07-02 08:30:49 +0200410 } else if (td->o.fill_device && err == ENOSPC) {
411 /*
412 * We expect to hit this error if
413 * fill_device option is set.
414 */
415 td_clear_error(td);
416 td->terminate = 1;
417 return 1;
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200418 } else {
419 /*
420 * Stop the I/O in case of a fatal
421 * error.
422 */
Radha Ramachandran8a4f8a22009-07-02 08:30:49 +0200423 update_error_count(td, err);
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200424 return 1;
425 }
426 }
427
428 return 0;
429}
430
Jens Axboe858a3d42006-10-24 14:54:44 +0200431/*
Jens Axboe34403fb2007-03-02 21:43:25 +0100432 * The main verify engine. Runs over the writes we previously submitted,
Jens Axboe906c8d72006-06-13 09:37:56 +0200433 * reads the blocks back in, and checks the crc/md5 of the data.
434 */
Jens Axboe1e97cce2006-12-05 11:44:16 +0100435static void do_verify(struct thread_data *td)
Jens Axboeebac4652005-12-08 15:25:21 +0100436{
Jens Axboe53cdc682006-10-18 11:50:58 +0200437 struct fio_file *f;
Jens Axboe36167d82007-02-18 05:41:31 +0100438 struct io_u *io_u;
Jens Axboeaf52b342007-03-13 10:07:47 +0100439 int ret, min_events;
440 unsigned int i;
Jens Axboee5b401d2006-10-18 16:03:40 +0200441
Jens Axboe008618a2010-03-18 19:29:45 +0100442 dprint(FD_VERIFY, "starting loop\n");
443
Jens Axboee5b401d2006-10-18 16:03:40 +0200444 /*
445 * sync io first and invalidate cache, to make sure we really
446 * read from disk.
447 */
448 for_each_file(td, f, i) {
Jens Axboed6aed792009-06-03 08:41:15 +0200449 if (!fio_file_open(f))
Jens Axboe3d7b4852007-03-13 14:50:28 +0100450 continue;
Jens Axboeb2fdda42007-02-20 20:52:51 +0100451 if (fio_io_sync(td, f))
452 break;
453 if (file_invalidate_cache(td, f))
454 break;
Jens Axboee5b401d2006-10-18 16:03:40 +0200455 }
Jens Axboeebac4652005-12-08 15:25:21 +0100456
Jens Axboeb2fdda42007-02-20 20:52:51 +0100457 if (td->error)
458 return;
459
Jens Axboeebac4652005-12-08 15:25:21 +0100460 td_set_runstate(td, TD_VERIFYING);
461
Jens Axboe36167d82007-02-18 05:41:31 +0100462 io_u = NULL;
463 while (!td->terminate) {
Jens Axboe49504212008-06-05 09:03:30 +0200464 int ret2, full;
Jens Axboe54517922007-03-05 10:06:06 +0100465
Jens Axboe993bf482008-11-14 13:04:53 +0100466 update_tv_cache(td);
467
468 if (runtime_exceeded(td, &td->tv_cache)) {
Jens Axboe90a87d42007-04-18 09:02:32 +0200469 td->terminate = 1;
Jens Axboe53cdc682006-10-18 11:50:58 +0200470 break;
Jens Axboe069c2912007-02-21 23:02:39 +0100471 }
Jens Axboe53cdc682006-10-18 11:50:58 +0200472
Radha Ramachandrandcb48302009-06-16 08:31:29 +0200473 io_u = __get_io_u(td);
474 if (!io_u)
475 break;
476
Jens Axboe069c2912007-02-21 23:02:39 +0100477 if (get_next_verify(td, io_u)) {
478 put_io_u(td, io_u);
Jens Axboeebac4652005-12-08 15:25:21 +0100479 break;
Jens Axboe069c2912007-02-21 23:02:39 +0100480 }
Jens Axboeebac4652005-12-08 15:25:21 +0100481
Jens Axboe069c2912007-02-21 23:02:39 +0100482 if (td_io_prep(td, io_u)) {
483 put_io_u(td, io_u);
Jens Axboe36167d82007-02-18 05:41:31 +0100484 break;
Jens Axboe069c2912007-02-21 23:02:39 +0100485 }
Jens Axboed7762cf2007-02-23 12:34:57 +0100486
Jens Axboee8462bd2009-07-06 12:59:04 +0200487 if (td->o.verify_async)
488 io_u->end_io = verify_io_u_async;
489 else
490 io_u->end_io = verify_io_u;
Jens Axboe36167d82007-02-18 05:41:31 +0100491
Jens Axboe11786802007-03-05 18:33:22 +0100492 ret = td_io_queue(td, io_u);
Jens Axboe36167d82007-02-18 05:41:31 +0100493 switch (ret) {
494 case FIO_Q_COMPLETED:
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200495 if (io_u->error) {
Jens Axboe22819ec2007-02-18 07:47:14 +0100496 ret = -io_u->error;
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200497 clear_io_u(td, io_u);
498 } else if (io_u->resid) {
Jens Axboe36167d82007-02-18 05:41:31 +0100499 int bytes = io_u->xfer_buflen - io_u->resid;
500
Jens Axboe9e9d1642007-03-12 09:37:46 +0100501 /*
502 * zero read, fail
503 */
504 if (!bytes) {
Jens Axboe41d9f0f2008-06-02 12:19:50 +0200505 td_verror(td, EIO, "full resid");
Jens Axboe9e9d1642007-03-12 09:37:46 +0100506 put_io_u(td, io_u);
507 break;
508 }
Jens Axboe8400d9b2007-03-28 11:10:09 +0200509
Jens Axboe36167d82007-02-18 05:41:31 +0100510 io_u->xfer_buflen = io_u->resid;
511 io_u->xfer_buf += bytes;
Jens Axboe8400d9b2007-03-28 11:10:09 +0200512 io_u->offset += bytes;
513
Jens Axboeff58fce2010-08-25 12:02:08 +0200514 if (ddir_rw(io_u->ddir))
515 td->ts.short_io_u[io_u->ddir]++;
Jens Axboe30061b92007-04-17 13:31:34 +0200516
Jens Axboe2b13e712011-01-19 14:04:16 -0700517 f = io_u->file;
Jens Axboed460eb32007-04-16 21:39:33 +0200518 if (io_u->offset == f->real_file_size)
Jens Axboe8400d9b2007-03-28 11:10:09 +0200519 goto sync_done;
520
Jens Axboe11786802007-03-05 18:33:22 +0100521 requeue_io_u(td, &io_u);
522 } else {
Jens Axboe8400d9b2007-03-28 11:10:09 +0200523sync_done:
Jens Axboe581e7142009-06-09 12:47:16 +0200524 ret = io_u_sync_complete(td, io_u, NULL);
Jens Axboe11786802007-03-05 18:33:22 +0100525 if (ret < 0)
526 break;
Jens Axboe36167d82007-02-18 05:41:31 +0100527 }
Jens Axboe36167d82007-02-18 05:41:31 +0100528 continue;
529 case FIO_Q_QUEUED:
530 break;
Jens Axboe755200a2007-02-19 13:08:12 +0100531 case FIO_Q_BUSY:
532 requeue_io_u(td, &io_u);
Jens Axboe54517922007-03-05 10:06:06 +0100533 ret2 = td_io_commit(td);
534 if (ret2 < 0)
535 ret = ret2;
Jens Axboe755200a2007-02-19 13:08:12 +0100536 break;
Jens Axboe36167d82007-02-18 05:41:31 +0100537 default:
538 assert(ret < 0);
Jens Axboee1161c32007-02-22 19:36:48 +0100539 td_verror(td, -ret, "td_io_queue");
Jens Axboeebac4652005-12-08 15:25:21 +0100540 break;
541 }
542
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200543 if (break_on_this_error(td, &ret))
Jens Axboeebac4652005-12-08 15:25:21 +0100544 break;
545
Jens Axboe3af6ef32007-02-18 06:57:43 +0100546 /*
547 * if we can queue more, do so. but check if there are
548 * completed io_u's first.
549 */
Jens Axboe49504212008-06-05 09:03:30 +0200550 full = queue_full(td) || ret == FIO_Q_BUSY;
551 if (full || !td->o.iodepth_batch_complete) {
Radha Ramachandran6a962762009-11-05 08:33:49 +0100552 min_events = min(td->o.iodepth_batch_complete,
553 td->cur_depth);
Jens Axboe49504212008-06-05 09:03:30 +0200554 if (full && !min_events)
Jens Axboe838bc702008-05-22 13:08:23 +0200555 min_events = 1;
Jens Axboee916b392007-02-20 14:37:26 +0100556
Jens Axboe49504212008-06-05 09:03:30 +0200557 do {
558 /*
559 * Reap required number of io units, if any,
560 * and do the verification on them through
561 * the callback handler
562 */
Jens Axboe581e7142009-06-09 12:47:16 +0200563 if (io_u_queued_complete(td, min_events, NULL) < 0) {
Jens Axboe49504212008-06-05 09:03:30 +0200564 ret = -1;
565 break;
566 }
567 } while (full && (td->cur_depth > td->o.iodepth_low));
568 }
569 if (ret < 0)
Jens Axboe36167d82007-02-18 05:41:31 +0100570 break;
571 }
572
Jens Axboec01c0392007-02-26 12:43:42 +0100573 if (!td->error) {
574 min_events = td->cur_depth;
575
576 if (min_events)
Jens Axboe581e7142009-06-09 12:47:16 +0200577 ret = io_u_queued_complete(td, min_events, NULL);
Jens Axboec01c0392007-02-26 12:43:42 +0100578 } else
Jens Axboeebac4652005-12-08 15:25:21 +0100579 cleanup_pending_aio(td);
580
581 td_set_runstate(td, TD_RUNNING);
Jens Axboe008618a2010-03-18 19:29:45 +0100582
583 dprint(FD_VERIFY, "exiting loop\n");
Jens Axboeebac4652005-12-08 15:25:21 +0100584}
585
Jens Axboe32cd46a2006-06-07 13:40:40 +0200586/*
Jens Axboe906c8d72006-06-13 09:37:56 +0200587 * Main IO worker function. It retrieves io_u's to process and queues
Jens Axboe32cd46a2006-06-07 13:40:40 +0200588 * and reaps them, checking for rate and errors along the way.
589 */
Jens Axboeebac4652005-12-08 15:25:21 +0100590static void do_io(struct thread_data *td)
591{
Jens Axboeaf52b342007-03-13 10:07:47 +0100592 unsigned int i;
593 int ret = 0;
Jens Axboeebac4652005-12-08 15:25:21 +0100594
Jens Axboeb29ee5b2008-09-11 10:17:26 +0200595 if (in_ramp_time(td))
596 td_set_runstate(td, TD_RAMP);
597 else
598 td_set_runstate(td, TD_RUNNING);
Jens Axboe5853e5a2006-06-08 14:41:05 +0200599
Glen Ogilvie457bf392009-11-03 21:52:36 +0100600 while ( (td->o.read_iolog_file && !flist_empty(&td->io_log_list)) ||
Jens Axboea917a8b2010-09-02 13:23:20 +0200601 (!flist_empty(&td->trim_list)) ||
Glen Ogilvie457bf392009-11-03 21:52:36 +0100602 ((td->this_io_bytes[0] + td->this_io_bytes[1]) < td->o.size) ) {
Jens Axboe97601022007-02-18 12:47:29 +0100603 struct timeval comp_time;
Jens Axboe581e7142009-06-09 12:47:16 +0200604 unsigned long bytes_done[2] = { 0, 0 };
Jens Axboe84585002006-10-19 20:26:22 +0200605 int min_evts = 0;
Jens Axboeebac4652005-12-08 15:25:21 +0100606 struct io_u *io_u;
Jens Axboe49504212008-06-05 09:03:30 +0200607 int ret2, full;
Jens Axboeebac4652005-12-08 15:25:21 +0100608
609 if (td->terminate)
610 break;
611
Jens Axboe993bf482008-11-14 13:04:53 +0100612 update_tv_cache(td);
Jens Axboe97601022007-02-18 12:47:29 +0100613
Jens Axboe993bf482008-11-14 13:04:53 +0100614 if (runtime_exceeded(td, &td->tv_cache)) {
Jens Axboe90a87d42007-04-18 09:02:32 +0200615 td->terminate = 1;
Jens Axboe97601022007-02-18 12:47:29 +0100616 break;
617 }
Jens Axboe36167d82007-02-18 05:41:31 +0100618
Radha Ramachandrandcb48302009-06-16 08:31:29 +0200619 io_u = get_io_u(td);
620 if (!io_u)
621 break;
622
Jens Axboeb67740d2007-07-26 12:22:30 +0200623 /*
624 * Add verification end_io handler, if asked to verify
625 * a previously written file.
626 */
Jens Axboe715d2b32009-09-04 20:53:53 +0200627 if (td->o.verify != VERIFY_NONE && io_u->ddir == DDIR_READ &&
628 !td_rw(td)) {
Jens Axboee8462bd2009-07-06 12:59:04 +0200629 if (td->o.verify_async)
630 io_u->end_io = verify_io_u_async;
631 else
632 io_u->end_io = verify_io_u;
Jens Axboed8fab1b2008-03-06 10:25:17 +0100633 td_set_runstate(td, TD_VERIFYING);
Jens Axboeb29ee5b2008-09-11 10:17:26 +0200634 } else if (in_ramp_time(td))
635 td_set_runstate(td, TD_RAMP);
636 else
Jens Axboed8fab1b2008-03-06 10:25:17 +0100637 td_set_runstate(td, TD_RUNNING);
Jens Axboeb67740d2007-07-26 12:22:30 +0200638
Jens Axboe11786802007-03-05 18:33:22 +0100639 ret = td_io_queue(td, io_u);
Jens Axboe36167d82007-02-18 05:41:31 +0100640 switch (ret) {
641 case FIO_Q_COMPLETED:
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200642 if (io_u->error) {
Jens Axboe54517922007-03-05 10:06:06 +0100643 ret = -io_u->error;
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200644 clear_io_u(td, io_u);
645 } else if (io_u->resid) {
Jens Axboe36167d82007-02-18 05:41:31 +0100646 int bytes = io_u->xfer_buflen - io_u->resid;
Jens Axboed460eb32007-04-16 21:39:33 +0200647 struct fio_file *f = io_u->file;
Jens Axboe36167d82007-02-18 05:41:31 +0100648
Jens Axboe9e9d1642007-03-12 09:37:46 +0100649 /*
650 * zero read, fail
651 */
652 if (!bytes) {
Jens Axboe41d9f0f2008-06-02 12:19:50 +0200653 td_verror(td, EIO, "full resid");
Jens Axboe9e9d1642007-03-12 09:37:46 +0100654 put_io_u(td, io_u);
655 break;
656 }
657
Jens Axboe36167d82007-02-18 05:41:31 +0100658 io_u->xfer_buflen = io_u->resid;
659 io_u->xfer_buf += bytes;
Jens Axboe5a7c5682007-03-28 11:07:20 +0200660 io_u->offset += bytes;
661
Jens Axboeff58fce2010-08-25 12:02:08 +0200662 if (ddir_rw(io_u->ddir))
663 td->ts.short_io_u[io_u->ddir]++;
Jens Axboe30061b92007-04-17 13:31:34 +0200664
Jens Axboed460eb32007-04-16 21:39:33 +0200665 if (io_u->offset == f->real_file_size)
Jens Axboe5a7c5682007-03-28 11:07:20 +0200666 goto sync_done;
667
Jens Axboe11786802007-03-05 18:33:22 +0100668 requeue_io_u(td, &io_u);
669 } else {
Jens Axboe5a7c5682007-03-28 11:07:20 +0200670sync_done:
Jens Axboe581e7142009-06-09 12:47:16 +0200671 if (__should_check_rate(td, 0) ||
672 __should_check_rate(td, 1))
Jens Axboe993bf482008-11-14 13:04:53 +0100673 fio_gettime(&comp_time, NULL);
674
Jens Axboe581e7142009-06-09 12:47:16 +0200675 ret = io_u_sync_complete(td, io_u, bytes_done);
676 if (ret < 0)
677 break;
Jens Axboe36167d82007-02-18 05:41:31 +0100678 }
Jens Axboe36167d82007-02-18 05:41:31 +0100679 break;
680 case FIO_Q_QUEUED:
Jens Axboe7e77dd02007-02-20 10:57:34 +0100681 /*
682 * if the engine doesn't have a commit hook,
683 * the io_u is really queued. if it does have such
684 * a hook, it has to call io_u_queued() itself.
685 */
686 if (td->io_ops->commit == NULL)
687 io_u_queued(td, io_u);
Jens Axboe36167d82007-02-18 05:41:31 +0100688 break;
Jens Axboe755200a2007-02-19 13:08:12 +0100689 case FIO_Q_BUSY:
690 requeue_io_u(td, &io_u);
Jens Axboe54517922007-03-05 10:06:06 +0100691 ret2 = td_io_commit(td);
692 if (ret2 < 0)
693 ret = ret2;
Jens Axboe755200a2007-02-19 13:08:12 +0100694 break;
Jens Axboe36167d82007-02-18 05:41:31 +0100695 default:
696 assert(ret < 0);
697 put_io_u(td, io_u);
698 break;
Jens Axboeebac4652005-12-08 15:25:21 +0100699 }
700
Radha Ramachandranf2bba182009-06-15 08:40:16 +0200701 if (break_on_this_error(td, &ret))
Jens Axboe36167d82007-02-18 05:41:31 +0100702 break;
703
Jens Axboe97601022007-02-18 12:47:29 +0100704 /*
705 * See if we need to complete some commands
706 */
Jens Axboe49504212008-06-05 09:03:30 +0200707 full = queue_full(td) || ret == FIO_Q_BUSY;
708 if (full || !td->o.iodepth_batch_complete) {
Radha Ramachandran6a962762009-11-05 08:33:49 +0100709 min_evts = min(td->o.iodepth_batch_complete,
710 td->cur_depth);
Jens Axboe49504212008-06-05 09:03:30 +0200711 if (full && !min_evts)
Jens Axboe36167d82007-02-18 05:41:31 +0100712 min_evts = 1;
Jens Axboe49504212008-06-05 09:03:30 +0200713
Jens Axboe581e7142009-06-09 12:47:16 +0200714 if (__should_check_rate(td, 0) ||
715 __should_check_rate(td, 1))
Jens Axboe993bf482008-11-14 13:04:53 +0100716 fio_gettime(&comp_time, NULL);
Jens Axboe49504212008-06-05 09:03:30 +0200717
718 do {
Jens Axboe581e7142009-06-09 12:47:16 +0200719 ret = io_u_queued_complete(td, min_evts, bytes_done);
720 if (ret < 0)
Jens Axboe49504212008-06-05 09:03:30 +0200721 break;
722
Jens Axboe49504212008-06-05 09:03:30 +0200723 } while (full && (td->cur_depth > td->o.iodepth_low));
Jens Axboeebac4652005-12-08 15:25:21 +0100724 }
725
Jens Axboe49504212008-06-05 09:03:30 +0200726 if (ret < 0)
727 break;
Jens Axboe581e7142009-06-09 12:47:16 +0200728 if (!(bytes_done[0] + bytes_done[1]))
Jens Axboe97601022007-02-18 12:47:29 +0100729 continue;
730
Jens Axboe581e7142009-06-09 12:47:16 +0200731 if (!in_ramp_time(td) && should_check_rate(td, bytes_done)) {
732 if (check_min_rate(td, &comp_time, bytes_done)) {
Jens Axboe721938a2008-09-10 09:46:16 +0200733 if (exitall_on_terminate)
734 terminate_threads(td->groupid);
735 td_verror(td, EIO, "check_min_rate");
736 break;
737 }
Jens Axboeebac4652005-12-08 15:25:21 +0100738 }
739
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100740 if (td->o.thinktime) {
Jens Axboe9c1f7432007-01-03 20:43:19 +0100741 unsigned long long b;
742
743 b = td->io_blocks[0] + td->io_blocks[1];
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100744 if (!(b % td->o.thinktime_blocks)) {
Jens Axboe48097d52007-02-17 06:30:44 +0100745 int left;
746
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100747 if (td->o.thinktime_spin)
Jens Axboe6dd6f2c2008-12-10 13:24:12 +0100748 usec_spin(td->o.thinktime_spin);
Jens Axboe48097d52007-02-17 06:30:44 +0100749
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100750 left = td->o.thinktime - td->o.thinktime_spin;
Jens Axboe48097d52007-02-17 06:30:44 +0100751 if (left)
752 usec_sleep(td, left);
753 }
Jens Axboe9c1f7432007-01-03 20:43:19 +0100754 }
Jens Axboeebac4652005-12-08 15:25:21 +0100755 }
756
Jens Axboed708ac52010-09-02 13:29:55 +0200757 if (td->trim_entries)
758 printf("trim entries %ld\n", td->trim_entries);
Jens Axboea917a8b2010-09-02 13:23:20 +0200759
Shawn Lewisaa31f1f2008-01-11 09:45:11 +0100760 if (td->o.fill_device && td->error == ENOSPC) {
761 td->error = 0;
762 td->terminate = 1;
763 }
Jens Axboe4d2413c2006-11-23 11:58:59 +0100764 if (!td->error) {
Jens Axboe3d7c3912007-02-19 13:16:12 +0100765 struct fio_file *f;
766
Jens Axboec01c0392007-02-26 12:43:42 +0100767 i = td->cur_depth;
768 if (i)
Jens Axboe581e7142009-06-09 12:47:16 +0200769 ret = io_u_queued_complete(td, i, NULL);
Jens Axboeebac4652005-12-08 15:25:21 +0100770
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100771 if (should_fsync(td) && td->o.end_fsync) {
Jens Axboe84585002006-10-19 20:26:22 +0200772 td_set_runstate(td, TD_FSYNCING);
Jens Axboe3d7b4852007-03-13 14:50:28 +0100773
774 for_each_file(td, f, i) {
Jens Axboed6aed792009-06-03 08:41:15 +0200775 if (!fio_file_open(f))
Jens Axboe3d7b4852007-03-13 14:50:28 +0100776 continue;
Jens Axboe858a3d42006-10-24 14:54:44 +0200777 fio_io_sync(td, f);
Jens Axboe3d7b4852007-03-13 14:50:28 +0100778 }
Jens Axboe84585002006-10-19 20:26:22 +0200779 }
Jens Axboec01c0392007-02-26 12:43:42 +0100780 } else
781 cleanup_pending_aio(td);
Jens Axboe2ba1c292008-02-01 13:16:38 +0100782
783 /*
784 * stop job if we failed doing any IO
785 */
786 if ((td->this_io_bytes[0] + td->this_io_bytes[1]) == 0)
787 td->done = 1;
Jens Axboeebac4652005-12-08 15:25:21 +0100788}
789
Jens Axboeebac4652005-12-08 15:25:21 +0100790static void cleanup_io_u(struct thread_data *td)
791{
Jens Axboe01743ee2008-06-02 12:19:19 +0200792 struct flist_head *entry, *n;
Jens Axboeebac4652005-12-08 15:25:21 +0100793 struct io_u *io_u;
794
Jens Axboe01743ee2008-06-02 12:19:19 +0200795 flist_for_each_safe(entry, n, &td->io_u_freelist) {
796 io_u = flist_entry(entry, struct io_u, list);
Jens Axboeebac4652005-12-08 15:25:21 +0100797
Jens Axboe01743ee2008-06-02 12:19:19 +0200798 flist_del(&io_u->list);
Jens Axboeebac4652005-12-08 15:25:21 +0100799 free(io_u);
800 }
801
Jens Axboe2f9ade32006-10-20 11:25:52 +0200802 free_io_mem(td);
Jens Axboeebac4652005-12-08 15:25:21 +0100803}
804
805static int init_io_u(struct thread_data *td)
806{
807 struct io_u *io_u;
Jens Axboea00735e2006-11-03 08:58:08 +0100808 unsigned int max_bs;
Jens Axboeeb7ccf32009-04-29 09:48:04 +0200809 int cl_align, i, max_units;
Jens Axboeebac4652005-12-08 15:25:21 +0100810 char *p;
811
Jens Axboe1e3c09a2008-02-04 10:48:13 +0100812 max_units = td->o.iodepth;
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100813 max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100814 td->orig_buffer_size = (unsigned long long) max_bs
815 * (unsigned long long) max_units;
Jens Axboe74b025b2006-12-19 15:18:14 +0100816
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100817 if (td->o.mem_type == MEM_SHMHUGE || td->o.mem_type == MEM_MMAPHUGE) {
818 unsigned long bs;
819
820 bs = td->orig_buffer_size + td->o.hugepage_size - 1;
821 td->orig_buffer_size = bs & ~(td->o.hugepage_size - 1);
822 }
Jens Axboeebac4652005-12-08 15:25:21 +0100823
Jens Axboec3990642007-07-19 10:17:09 +0200824 if (td->orig_buffer_size != (size_t) td->orig_buffer_size) {
825 log_err("fio: IO memory too large. Reduce max_bs or iodepth\n");
826 return 1;
827 }
828
Jens Axboe2f9ade32006-10-20 11:25:52 +0200829 if (allocate_io_mem(td))
830 return 1;
Jens Axboeebac4652005-12-08 15:25:21 +0100831
Jens Axboe1c04bf72010-09-07 13:28:17 +0200832 if (td->o.odirect || td->o.mem_align ||
833 (td->io_ops->flags & FIO_RAWIO))
Jens Axboed529ee12009-07-01 10:33:03 +0200834 p = PAGE_ALIGN(td->orig_buffer) + td->o.mem_align;
Jens Axboeb3f378e2007-07-20 12:39:22 +0200835 else
836 p = td->orig_buffer;
837
Jens Axboeeb7ccf32009-04-29 09:48:04 +0200838 cl_align = os_cache_line_size();
839
Jens Axboeebac4652005-12-08 15:25:21 +0100840 for (i = 0; i < max_units; i++) {
Jens Axboeeb7ccf32009-04-29 09:48:04 +0200841 void *ptr;
842
Jens Axboedb1cd902007-04-26 13:47:07 +0200843 if (td->terminate)
844 return 1;
Jens Axboeeb7ccf32009-04-29 09:48:04 +0200845
846 if (posix_memalign(&ptr, cl_align, sizeof(*io_u))) {
847 log_err("fio: posix_memalign=%s\n", strerror(errno));
848 break;
849 }
850
851 io_u = ptr;
Jens Axboeebac4652005-12-08 15:25:21 +0100852 memset(io_u, 0, sizeof(*io_u));
Jens Axboe01743ee2008-06-02 12:19:19 +0200853 INIT_FLIST_HEAD(&io_u->list);
Jens Axboed529ee12009-07-01 10:33:03 +0200854 dprint(FD_MEM, "io_u alloc %p, index %u\n", io_u, i);
Jens Axboeebac4652005-12-08 15:25:21 +0100855
Jens Axboeb4c5e1a2007-10-25 18:44:45 +0200856 if (!(td->io_ops->flags & FIO_NOIO)) {
857 io_u->buf = p + max_bs * i;
Jens Axboed529ee12009-07-01 10:33:03 +0200858 dprint(FD_MEM, "io_u %p, mem %p\n", io_u, io_u->buf);
Jens Axboee9459e52007-04-17 15:46:32 +0200859
Jens Axboe5973caf2008-05-21 19:52:35 +0200860 if (td_write(td) && !td->o.refill_buffers)
861 io_u_fill_buffer(td, io_u, max_bs);
Radha Ramachandrancbe8d752010-07-14 08:36:07 +0200862 else if (td_write(td) && td->o.verify_pattern_bytes) {
863 /*
864 * Fill the buffer with the pattern if we are
865 * going to be doing writes.
866 */
Jens Axboe7d9fb452011-01-11 22:16:49 +0100867 fill_pattern(td, io_u->buf, max_bs, io_u, 0, 0);
Radha Ramachandrancbe8d752010-07-14 08:36:07 +0200868 }
Jens Axboeb4c5e1a2007-10-25 18:44:45 +0200869 }
Jens Axboe6b9cea22006-10-30 17:00:24 +0100870
Jens Axboeb1ff3402006-02-17 11:35:58 +0100871 io_u->index = i;
Jens Axboe0c6e7512007-02-22 11:19:39 +0100872 io_u->flags = IO_U_F_FREE;
Jens Axboe01743ee2008-06-02 12:19:19 +0200873 flist_add(&io_u->list, &td->io_u_freelist);
Jens Axboeebac4652005-12-08 15:25:21 +0100874 }
875
876 return 0;
877}
878
Jens Axboeda867742006-06-06 10:39:10 -0700879static int switch_ioscheduler(struct thread_data *td)
880{
881 char tmp[256], tmp2[128];
882 FILE *f;
883 int ret;
884
Jens Axboeba0fbe12007-03-09 14:34:23 +0100885 if (td->io_ops->flags & FIO_DISKLESSIO)
Jens Axboef48b4672006-10-27 11:30:07 +0200886 return 0;
887
Jens Axboeda867742006-06-06 10:39:10 -0700888 sprintf(tmp, "%s/queue/scheduler", td->sysfs_root);
889
890 f = fopen(tmp, "r+");
891 if (!f) {
Jens Axboecbf5c1d2007-04-16 12:56:00 +0200892 if (errno == ENOENT) {
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100893 log_err("fio: os or kernel doesn't support IO scheduler"
894 " switching\n");
Jens Axboecbf5c1d2007-04-16 12:56:00 +0200895 return 0;
896 }
897 td_verror(td, errno, "fopen iosched");
Jens Axboeda867742006-06-06 10:39:10 -0700898 return 1;
899 }
900
901 /*
902 * Set io scheduler.
903 */
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100904 ret = fwrite(td->o.ioscheduler, strlen(td->o.ioscheduler), 1, f);
Jens Axboeda867742006-06-06 10:39:10 -0700905 if (ferror(f) || ret != 1) {
Jens Axboee1161c32007-02-22 19:36:48 +0100906 td_verror(td, errno, "fwrite");
Jens Axboeda867742006-06-06 10:39:10 -0700907 fclose(f);
908 return 1;
909 }
910
911 rewind(f);
912
913 /*
914 * Read back and check that the selected scheduler is now the default.
915 */
916 ret = fread(tmp, 1, sizeof(tmp), f);
917 if (ferror(f) || ret < 0) {
Jens Axboee1161c32007-02-22 19:36:48 +0100918 td_verror(td, errno, "fread");
Jens Axboeda867742006-06-06 10:39:10 -0700919 fclose(f);
920 return 1;
921 }
922
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100923 sprintf(tmp2, "[%s]", td->o.ioscheduler);
Jens Axboeda867742006-06-06 10:39:10 -0700924 if (!strstr(tmp, tmp2)) {
Jens Axboe2dc1bbe2007-03-15 15:01:33 +0100925 log_err("fio: io scheduler %s not found\n", td->o.ioscheduler);
Jens Axboee1161c32007-02-22 19:36:48 +0100926 td_verror(td, EINVAL, "iosched_switch");
Jens Axboeda867742006-06-06 10:39:10 -0700927 fclose(f);
928 return 1;
929 }
930
931 fclose(f);
932 return 0;
933}
934
Jens Axboe492158c2007-05-24 10:32:47 +0200935static int keep_running(struct thread_data *td)
936{
937 unsigned long long io_done;
938
Jens Axboe20e354e2007-07-23 14:36:16 +0200939 if (td->done)
940 return 0;
Jens Axboe492158c2007-05-24 10:32:47 +0200941 if (td->o.time_based)
942 return 1;
943 if (td->o.loops) {
944 td->o.loops--;
945 return 1;
946 }
947
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100948 io_done = td->io_bytes[DDIR_READ] + td->io_bytes[DDIR_WRITE]
949 + td->io_skip_bytes;
Jens Axboe492158c2007-05-24 10:32:47 +0200950 if (io_done < td->o.size)
951 return 1;
952
953 return 0;
954}
955
Jens Axboeb29ee5b2008-09-11 10:17:26 +0200956static void reset_io_counters(struct thread_data *td)
Jens Axboeebac4652005-12-08 15:25:21 +0100957{
Jens Axboe756867b2007-03-06 15:19:24 +0100958 td->ts.stat_io_bytes[0] = td->ts.stat_io_bytes[1] = 0;
Jens Axboeebac4652005-12-08 15:25:21 +0100959 td->this_io_bytes[0] = td->this_io_bytes[1] = 0;
Jens Axboe20dc95c2005-12-09 10:29:35 +0100960 td->zone_bytes = 0;
Jens Axboe581e7142009-06-09 12:47:16 +0200961 td->rate_bytes[0] = td->rate_bytes[1] = 0;
962 td->rate_blocks[0] = td->rate_blocks[1] = 0;
Jens Axboeebac4652005-12-08 15:25:21 +0100963
Jens Axboec1324df2007-02-19 19:06:41 +0100964 td->last_was_sync = 0;
965
Jens Axboe2ba1c292008-02-01 13:16:38 +0100966 /*
967 * reset file done count if we are to start over
968 */
969 if (td->o.time_based || td->o.loops)
Jens Axboe48f5abd2007-07-20 13:25:04 +0200970 td->nr_done_files = 0;
Jens Axboe5bfc35d2009-04-07 13:28:12 +0200971
972 /*
973 * Set the same seed to get repeatable runs
974 */
975 td_fill_rand_seeds(td);
Jens Axboeb29ee5b2008-09-11 10:17:26 +0200976}
977
978void reset_all_stats(struct thread_data *td)
979{
980 struct timeval tv;
981 int i;
982
983 reset_io_counters(td);
984
985 for (i = 0; i < 2; i++) {
986 td->io_bytes[i] = 0;
987 td->io_blocks[i] = 0;
988 td->io_issues[i] = 0;
989 td->ts.total_io_u[i] = 0;
990 }
Bruce Cran9b836562011-01-08 19:49:54 +0100991
Jens Axboeb29ee5b2008-09-11 10:17:26 +0200992 fio_gettime(&tv, NULL);
Signed-off-by Steven Prattcda99fa2010-12-14 08:31:36 +0100993 td->ts.runtime[0] = 0;
994 td->ts.runtime[1] = 0;
Jens Axboeb29ee5b2008-09-11 10:17:26 +0200995 memcpy(&td->epoch, &tv, sizeof(tv));
996 memcpy(&td->start, &tv, sizeof(tv));
997}
998
Jens Axboedf9c26b2009-03-05 10:13:58 +0100999static void clear_io_state(struct thread_data *td)
Jens Axboeb29ee5b2008-09-11 10:17:26 +02001000{
1001 struct fio_file *f;
1002 unsigned int i;
Jens Axboeb29ee5b2008-09-11 10:17:26 +02001003
1004 reset_io_counters(td);
Jens Axboe281f9b62007-05-23 09:34:42 +02001005
Jens Axboe24ffd2c2008-03-01 15:47:08 +01001006 close_files(td);
Jens Axboedf9c26b2009-03-05 10:13:58 +01001007 for_each_file(td, f, i)
Jens Axboed6aed792009-06-03 08:41:15 +02001008 fio_file_clear_done(f);
Jens Axboeebac4652005-12-08 15:25:21 +01001009}
1010
Jens Axboe398e8c42009-05-20 12:10:02 +02001011static int exec_string(const char *string)
1012{
1013 int ret, newlen = strlen(string) + 1 + 8;
1014 char *str;
1015
1016 str = malloc(newlen);
1017 sprintf(str, "sh -c %s", string);
1018
1019 ret = system(str);
1020 if (ret == -1)
1021 log_err("fio: exec of cmd <%s> failed\n", str);
1022
1023 free(str);
1024 return ret;
1025}
1026
Jens Axboe906c8d72006-06-13 09:37:56 +02001027/*
1028 * Entry point for the thread based jobs. The process based jobs end up
1029 * here as well, after a little setup.
1030 */
Jens Axboeebac4652005-12-08 15:25:21 +01001031static void *thread_main(void *data)
1032{
Signed-off-by Steven Prattcda99fa2010-12-14 08:31:36 +01001033 unsigned long long elapsed;
Jens Axboeebac4652005-12-08 15:25:21 +01001034 struct thread_data *td = data;
Jens Axboee8462bd2009-07-06 12:59:04 +02001035 pthread_condattr_t attr;
Jens Axboea978ba62007-03-08 14:29:03 +01001036 int clear_state;
Jens Axboeebac4652005-12-08 15:25:21 +01001037
Jens Axboe2dc1bbe2007-03-15 15:01:33 +01001038 if (!td->o.use_thread)
Jens Axboeebac4652005-12-08 15:25:21 +01001039 setsid();
1040
1041 td->pid = getpid();
1042
Jens Axboe5921e802008-05-30 15:02:38 +02001043 dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid);
Jens Axboeee56ad52008-02-01 10:30:20 +01001044
Jens Axboe01743ee2008-06-02 12:19:19 +02001045 INIT_FLIST_HEAD(&td->io_u_freelist);
1046 INIT_FLIST_HEAD(&td->io_u_busylist);
1047 INIT_FLIST_HEAD(&td->io_u_requeues);
1048 INIT_FLIST_HEAD(&td->io_log_list);
1049 INIT_FLIST_HEAD(&td->io_hist_list);
Jens Axboee8462bd2009-07-06 12:59:04 +02001050 INIT_FLIST_HEAD(&td->verify_list);
Jens Axboe0d29de82010-09-01 13:54:15 +02001051 INIT_FLIST_HEAD(&td->trim_list);
Jens Axboee8462bd2009-07-06 12:59:04 +02001052 pthread_mutex_init(&td->io_u_lock, NULL);
Jens Axboebb5d7d02007-03-27 10:21:25 +02001053 td->io_hist_tree = RB_ROOT;
Jens Axboeaea47d42006-05-26 19:27:29 +02001054
Jens Axboee8462bd2009-07-06 12:59:04 +02001055 pthread_condattr_init(&attr);
1056 pthread_cond_init(&td->verify_cond, &attr);
1057 pthread_cond_init(&td->free_cond, &attr);
1058
Jens Axboe75154842006-06-01 13:56:09 +02001059 td_set_runstate(td, TD_INITIALIZED);
Jens Axboe29adda32009-01-05 19:06:39 +01001060 dprint(FD_MUTEX, "up startup_mutex\n");
Jens Axboecdd18ad2008-02-27 18:58:00 +01001061 fio_mutex_up(startup_mutex);
Jens Axboe29adda32009-01-05 19:06:39 +01001062 dprint(FD_MUTEX, "wait on td->mutex\n");
Jens Axboecdd18ad2008-02-27 18:58:00 +01001063 fio_mutex_down(td->mutex);
Jens Axboe29adda32009-01-05 19:06:39 +01001064 dprint(FD_MUTEX, "done waiting on td->mutex\n");
Jens Axboeebac4652005-12-08 15:25:21 +01001065
Jens Axboe37f56872007-03-09 12:42:35 +01001066 /*
Jens Axboecdd18ad2008-02-27 18:58:00 +01001067 * the ->mutex mutex is now no longer used, close it to avoid
Jens Axboe37f56872007-03-09 12:42:35 +01001068 * eating a file descriptor
1069 */
Jens Axboecdd18ad2008-02-27 18:58:00 +01001070 fio_mutex_remove(td->mutex);
Jens Axboe37f56872007-03-09 12:42:35 +01001071
Stefan Hajnoczidfc6e752011-01-07 20:58:05 +00001072 /*
1073 * A new gid requires privilege, so we need to do this before setting
1074 * the uid.
1075 */
Jens Axboee0b0d892009-12-08 10:10:14 +01001076 if (td->o.gid != -1U && setgid(td->o.gid)) {
1077 td_verror(td, errno, "setgid");
1078 goto err;
1079 }
Stefan Hajnoczidfc6e752011-01-07 20:58:05 +00001080 if (td->o.uid != -1U && setuid(td->o.uid)) {
1081 td_verror(td, errno, "setuid");
1082 goto err;
1083 }
Jens Axboee0b0d892009-12-08 10:10:14 +01001084
Jens Axboed84f8d42007-05-16 08:47:46 +02001085 /*
1086 * May alter parameters that init_io_u() will use, so we need to
1087 * do this first.
1088 */
1089 if (init_iolog(td))
1090 goto err;
1091
Jens Axboedb1cd902007-04-26 13:47:07 +02001092 if (init_io_u(td))
1093 goto err;
1094
Jens Axboee8462bd2009-07-06 12:59:04 +02001095 if (td->o.verify_async && verify_async_init(td))
1096 goto err;
1097
1098 if (td->o.cpumask_set && fio_setaffinity(td->pid, td->o.cpumask) == -1) {
Jens Axboedb1cd902007-04-26 13:47:07 +02001099 td_verror(td, errno, "cpu_set_affinity");
1100 goto err;
1101 }
1102
Jens Axboef4844702008-12-09 08:04:29 +01001103 /*
1104 * If we have a gettimeofday() thread, make sure we exclude that
1105 * thread from this job
1106 */
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001107 if (td->o.gtod_cpu) {
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001108 fio_cpu_clear(&td->o.cpumask, td->o.gtod_cpu);
Jens Axboee8462bd2009-07-06 12:59:04 +02001109 if (fio_setaffinity(td->pid, td->o.cpumask) == -1) {
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001110 td_verror(td, errno, "cpu_set_affinity");
1111 goto err;
1112 }
1113 }
1114
Jens Axboeac684782007-11-08 08:29:07 +01001115 if (td->ioprio_set) {
Jens Axboedb1cd902007-04-26 13:47:07 +02001116 if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) {
1117 td_verror(td, errno, "ioprio_set");
1118 goto err;
1119 }
1120 }
1121
Jens Axboe6adb38a2009-12-07 08:01:26 +01001122 if (td->o.cgroup_weight && cgroup_setup(td, cgroup_list, &cgroup_mnt))
Jens Axboea696fa22009-12-04 10:05:02 +01001123 goto err;
1124
Jens Axboedb1cd902007-04-26 13:47:07 +02001125 if (nice(td->o.nice) == -1) {
1126 td_verror(td, errno, "nice");
1127 goto err;
1128 }
1129
1130 if (td->o.ioscheduler && switch_ioscheduler(td))
1131 goto err;
1132
Jens Axboe2dc1bbe2007-03-15 15:01:33 +01001133 if (!td->o.create_serialize && setup_files(td))
Jens Axboeebac4652005-12-08 15:25:21 +01001134 goto err;
Jens Axboeb5af8292007-03-08 12:43:13 +01001135
1136 if (td_io_init(td))
Jens Axboe21972cd2006-11-23 12:39:16 +01001137 goto err;
Jens Axboeebac4652005-12-08 15:25:21 +01001138
Jens Axboe68727072007-03-15 20:49:25 +01001139 if (init_random_map(td))
1140 goto err;
1141
Jens Axboe2dc1bbe2007-03-15 15:01:33 +01001142 if (td->o.exec_prerun) {
Jens Axboe398e8c42009-05-20 12:10:02 +02001143 if (exec_string(td->o.exec_prerun))
Jens Axboe69cfd7e2007-02-07 13:58:53 +01001144 goto err;
1145 }
Jens Axboe4e0ba8a2006-06-06 09:36:28 +02001146
Zhang, Yanminafad68f2009-05-20 11:30:55 +02001147 if (td->o.pre_read) {
1148 if (pre_read_files(td) < 0)
1149 goto err;
1150 }
1151
Jens Axboe69008992006-11-24 13:12:56 +01001152 fio_gettime(&td->epoch, NULL);
Jens Axboe756867b2007-03-06 15:19:24 +01001153 getrusage(RUSAGE_SELF, &td->ts.ru_start);
Jens Axboe69008992006-11-24 13:12:56 +01001154
Jens Axboea978ba62007-03-08 14:29:03 +01001155 clear_state = 0;
Jens Axboe492158c2007-05-24 10:32:47 +02001156 while (keep_running(td)) {
Jens Axboe02bcaa82006-11-24 10:42:00 +01001157 fio_gettime(&td->start, NULL);
Jens Axboec5d3d792009-07-31 11:20:52 +02001158 memcpy(&td->ts.stat_sample_time[0], &td->start,
1159 sizeof(td->start));
1160 memcpy(&td->ts.stat_sample_time[1], &td->start,
1161 sizeof(td->start));
Jens Axboe993bf482008-11-14 13:04:53 +01001162 memcpy(&td->tv_cache, &td->start, sizeof(td->start));
Jens Axboeebac4652005-12-08 15:25:21 +01001163
Jens Axboecf2dd8d2009-06-09 12:58:37 +02001164 if (td->o.ratemin[0] || td->o.ratemin[1])
Jens Axboe5ec10ea2008-03-06 15:42:00 +01001165 memcpy(&td->lastrate, &td->ts.stat_sample_time,
1166 sizeof(td->lastrate));
Jens Axboeebac4652005-12-08 15:25:21 +01001167
Jens Axboedf9c26b2009-03-05 10:13:58 +01001168 if (clear_state)
1169 clear_io_state(td);
Jens Axboea978ba62007-03-08 14:29:03 +01001170
Jens Axboeebac4652005-12-08 15:25:21 +01001171 prune_io_piece_log(td);
1172
Jens Axboeba0fbe12007-03-09 14:34:23 +01001173 do_io(td);
Jens Axboeebac4652005-12-08 15:25:21 +01001174
Jens Axboea978ba62007-03-08 14:29:03 +01001175 clear_state = 1;
1176
Jens Axboe38d77ca2007-03-21 14:20:20 +01001177 if (td_read(td) && td->io_bytes[DDIR_READ]) {
Jens Axboe0a4957c2009-06-03 10:30:33 +02001178 elapsed = utime_since_now(&td->start);
Signed-off-by Steven Prattcda99fa2010-12-14 08:31:36 +01001179 td->ts.runtime[DDIR_READ] += elapsed;
Jens Axboe38d77ca2007-03-21 14:20:20 +01001180 }
1181 if (td_write(td) && td->io_bytes[DDIR_WRITE]) {
Jens Axboe0a4957c2009-06-03 10:30:33 +02001182 elapsed = utime_since_now(&td->start);
Signed-off-by Steven Prattcda99fa2010-12-14 08:31:36 +01001183 td->ts.runtime[DDIR_WRITE] += elapsed;
Jens Axboe38d77ca2007-03-21 14:20:20 +01001184 }
Jens Axboe5ec10ea2008-03-06 15:42:00 +01001185
Jens Axboeebac4652005-12-08 15:25:21 +01001186 if (td->error || td->terminate)
1187 break;
1188
Shawn Lewise84c73a2007-08-02 22:19:32 +02001189 if (!td->o.do_verify ||
1190 td->o.verify == VERIFY_NONE ||
Jens Axboeb67740d2007-07-26 12:22:30 +02001191 (td->io_ops->flags & FIO_UNIDIR))
Jens Axboeebac4652005-12-08 15:25:21 +01001192 continue;
1193
Jens Axboedf9c26b2009-03-05 10:13:58 +01001194 clear_io_state(td);
Jens Axboea978ba62007-03-08 14:29:03 +01001195
Jens Axboe02bcaa82006-11-24 10:42:00 +01001196 fio_gettime(&td->start, NULL);
Jens Axboeebac4652005-12-08 15:25:21 +01001197
1198 do_verify(td);
1199
Signed-off-by Steven Prattcda99fa2010-12-14 08:31:36 +01001200 td->ts.runtime[DDIR_READ] += utime_since_now(&td->start);
Jens Axboeebac4652005-12-08 15:25:21 +01001201
1202 if (td->error || td->terminate)
1203 break;
1204 }
1205
Jens Axboe36dff962006-11-24 13:29:20 +01001206 update_rusage_stat(td);
Signed-off-by Steven Prattcda99fa2010-12-14 08:31:36 +01001207 td->ts.runtime[0] = (td->ts.runtime[0] + 999) / 1000;
1208 td->ts.runtime[1] = (td->ts.runtime[1] + 999) / 1000;
Jens Axboe756867b2007-03-06 15:19:24 +01001209 td->ts.total_run_time = mtime_since_now(&td->epoch);
1210 td->ts.io_bytes[0] = td->io_bytes[0];
1211 td->ts.io_bytes[1] = td->io_bytes[1];
Jens Axboe69008992006-11-24 13:12:56 +01001212
Fabio Checconid2bbb8d2009-05-18 12:54:50 +02001213 fio_mutex_down(writeout_mutex);
Jens Axboee3cedca2008-11-19 19:57:52 +01001214 if (td->ts.bw_log) {
Jens Axboef4844702008-12-09 08:04:29 +01001215 if (td->o.bw_log_file) {
1216 finish_log_named(td, td->ts.bw_log,
1217 td->o.bw_log_file, "bw");
1218 } else
Jens Axboee3cedca2008-11-19 19:57:52 +01001219 finish_log(td, td->ts.bw_log, "bw");
1220 }
Jens Axboe02af0982010-06-24 09:59:34 +02001221 if (td->ts.lat_log) {
1222 if (td->o.lat_log_file) {
1223 finish_log_named(td, td->ts.lat_log,
1224 td->o.lat_log_file, "lat");
1225 } else
1226 finish_log(td, td->ts.lat_log, "lat");
1227 }
Jens Axboee3cedca2008-11-19 19:57:52 +01001228 if (td->ts.slat_log) {
Jens Axboef4844702008-12-09 08:04:29 +01001229 if (td->o.lat_log_file) {
1230 finish_log_named(td, td->ts.slat_log,
Radha Ramachandran1ca95732009-04-29 07:23:26 +02001231 td->o.lat_log_file, "slat");
Jens Axboef4844702008-12-09 08:04:29 +01001232 } else
Jens Axboee3cedca2008-11-19 19:57:52 +01001233 finish_log(td, td->ts.slat_log, "slat");
1234 }
1235 if (td->ts.clat_log) {
Jens Axboef4844702008-12-09 08:04:29 +01001236 if (td->o.lat_log_file) {
1237 finish_log_named(td, td->ts.clat_log,
1238 td->o.lat_log_file, "clat");
1239 } else
Jens Axboee3cedca2008-11-19 19:57:52 +01001240 finish_log(td, td->ts.clat_log, "clat");
1241 }
Fabio Checconid2bbb8d2009-05-18 12:54:50 +02001242 fio_mutex_up(writeout_mutex);
Jens Axboe398e8c42009-05-20 12:10:02 +02001243 if (td->o.exec_postrun)
1244 exec_string(td->o.exec_postrun);
Jens Axboeebac4652005-12-08 15:25:21 +01001245
1246 if (exitall_on_terminate)
Jens Axboe390c40e2007-03-05 12:35:29 +01001247 terminate_threads(td->groupid);
Jens Axboeebac4652005-12-08 15:25:21 +01001248
1249err:
Jens Axboe5bf13a52007-02-17 01:51:47 +01001250 if (td->error)
Jens Axboe4ceb30d2010-02-24 09:19:14 +01001251 log_info("fio: pid=%d, err=%d/%s\n", (int) td->pid, td->error,
Jens Axboe5ec10ea2008-03-06 15:42:00 +01001252 td->verror);
Jens Axboeb4707ed2009-11-04 12:51:17 +01001253
1254 if (td->o.verify_async)
1255 verify_async_exit(td);
1256
Jens Axboe24ffd2c2008-03-01 15:47:08 +01001257 close_and_free_files(td);
Jens Axboe2866c822006-10-09 15:57:48 +02001258 close_ioengine(td);
Jens Axboeebac4652005-12-08 15:25:21 +01001259 cleanup_io_u(td);
Jens Axboe6adb38a2009-12-07 08:01:26 +01001260 cgroup_shutdown(td, &cgroup_mnt);
Jens Axboef29b25a2007-07-23 08:56:43 +02001261
Jens Axboed2ce18b2008-12-12 20:51:40 +01001262 if (td->o.cpumask_set) {
1263 int ret = fio_cpuset_exit(&td->o.cpumask);
1264
1265 td_verror(td, ret, "fio_cpuset_exit");
1266 }
Jens Axboe4e78e402008-12-12 20:40:27 +01001267
Jens Axboef29b25a2007-07-23 08:56:43 +02001268 /*
1269 * do this very late, it will log file closing as well
1270 */
1271 if (td->o.write_iolog_file)
1272 write_iolog_close(td);
1273
Jens Axboed23bb322007-03-23 15:57:56 +01001274 options_mem_free(td);
Jens Axboeebac4652005-12-08 15:25:21 +01001275 td_set_runstate(td, TD_EXITED);
Jens Axboe43d76802007-02-21 16:35:29 +01001276 return (void *) (unsigned long) td->error;
Jens Axboeebac4652005-12-08 15:25:21 +01001277}
1278
Jens Axboe906c8d72006-06-13 09:37:56 +02001279/*
1280 * We cannot pass the td data into a forked process, so attach the td and
1281 * pass it to the thread worker.
1282 */
Jens Axboea6418142007-02-17 04:47:18 +01001283static int fork_main(int shmid, int offset)
Jens Axboeebac4652005-12-08 15:25:21 +01001284{
1285 struct thread_data *td;
Jens Axboea6418142007-02-17 04:47:18 +01001286 void *data, *ret;
Jens Axboeebac4652005-12-08 15:25:21 +01001287
1288 data = shmat(shmid, NULL, 0);
1289 if (data == (void *) -1) {
Jens Axboea6418142007-02-17 04:47:18 +01001290 int __err = errno;
1291
Jens Axboeebac4652005-12-08 15:25:21 +01001292 perror("shmat");
Jens Axboea6418142007-02-17 04:47:18 +01001293 return __err;
Jens Axboeebac4652005-12-08 15:25:21 +01001294 }
1295
1296 td = data + offset * sizeof(struct thread_data);
Jens Axboea6418142007-02-17 04:47:18 +01001297 ret = thread_main(td);
Jens Axboeebac4652005-12-08 15:25:21 +01001298 shmdt(data);
Jens Axboe43d76802007-02-21 16:35:29 +01001299 return (int) (unsigned long) ret;
Jens Axboeebac4652005-12-08 15:25:21 +01001300}
1301
Jens Axboe906c8d72006-06-13 09:37:56 +02001302/*
Jens Axboe906c8d72006-06-13 09:37:56 +02001303 * Run over the job map and reap the threads that have exited, if any.
1304 */
Jens Axboeebac4652005-12-08 15:25:21 +01001305static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
1306{
Jens Axboe34572e22006-10-20 09:33:18 +02001307 struct thread_data *td;
Jens Axboe1f809d12007-10-25 18:34:02 +02001308 int i, cputhreads, realthreads, pending, status, ret;
Jens Axboeebac4652005-12-08 15:25:21 +01001309
1310 /*
1311 * reap exited threads (TD_EXITED -> TD_REAPED)
1312 */
Jens Axboe1f809d12007-10-25 18:34:02 +02001313 realthreads = pending = cputhreads = 0;
Jens Axboe34572e22006-10-20 09:33:18 +02001314 for_each_td(td, i) {
Jens Axboe3707f452007-02-22 12:26:57 +01001315 int flags = 0;
Jens Axboea2f77c92007-02-22 11:53:00 +01001316
Jens Axboe84585002006-10-19 20:26:22 +02001317 /*
1318 * ->io_ops is NULL for a thread that has closed its
1319 * io engine
1320 */
Jens Axboeba0fbe12007-03-09 14:34:23 +01001321 if (td->io_ops && !strcmp(td->io_ops->name, "cpuio"))
Jens Axboeb990b5c2006-09-14 09:48:22 +02001322 cputhreads++;
Jens Axboe1f809d12007-10-25 18:34:02 +02001323 else
1324 realthreads++;
Jens Axboeb990b5c2006-09-14 09:48:22 +02001325
Aaron Carrolla1dedc12008-02-20 09:14:12 +01001326 if (!td->pid) {
1327 pending++;
1328 continue;
1329 }
1330 if (td->runstate == TD_REAPED)
Jens Axboea2f77c92007-02-22 11:53:00 +01001331 continue;
Jens Axboe2dc1bbe2007-03-15 15:01:33 +01001332 if (td->o.use_thread) {
Jens Axboe3707f452007-02-22 12:26:57 +01001333 if (td->runstate == TD_EXITED) {
1334 td_set_runstate(td, TD_REAPED);
1335 goto reaped;
1336 }
1337 continue;
1338 }
Jens Axboea2f77c92007-02-22 11:53:00 +01001339
1340 flags = WNOHANG;
1341 if (td->runstate == TD_EXITED)
1342 flags = 0;
1343
1344 /*
1345 * check if someone quit or got killed in an unusual way
1346 */
1347 ret = waitpid(td->pid, &status, flags);
Jens Axboe3707f452007-02-22 12:26:57 +01001348 if (ret < 0) {
Jens Axboea2f77c92007-02-22 11:53:00 +01001349 if (errno == ECHILD) {
Jens Axboe5921e802008-05-30 15:02:38 +02001350 log_err("fio: pid=%d disappeared %d\n",
1351 (int) td->pid, td->runstate);
Jens Axboea2f77c92007-02-22 11:53:00 +01001352 td_set_runstate(td, TD_REAPED);
1353 goto reaped;
1354 }
1355 perror("waitpid");
1356 } else if (ret == td->pid) {
1357 if (WIFSIGNALED(status)) {
Jens Axboefab6aa72007-02-17 06:19:24 +01001358 int sig = WTERMSIG(status);
1359
Bruce Cran03e20d62011-01-02 20:14:54 +01001360 if (sig != SIGTERM)
Jens Axboe5ec10ea2008-03-06 15:42:00 +01001361 log_err("fio: pid=%d, got signal=%d\n",
Jens Axboe5921e802008-05-30 15:02:38 +02001362 (int) td->pid, sig);
Jens Axboefab6aa72007-02-17 06:19:24 +01001363 td_set_runstate(td, TD_REAPED);
1364 goto reaped;
1365 }
Jens Axboea2f77c92007-02-22 11:53:00 +01001366 if (WIFEXITED(status)) {
1367 if (WEXITSTATUS(status) && !td->error)
1368 td->error = WEXITSTATUS(status);
Jens Axboefab6aa72007-02-17 06:19:24 +01001369
Jens Axboea2f77c92007-02-22 11:53:00 +01001370 td_set_runstate(td, TD_REAPED);
1371 goto reaped;
Jens Axboea6418142007-02-17 04:47:18 +01001372 }
1373 }
Jens Axboeebac4652005-12-08 15:25:21 +01001374
Jens Axboea2f77c92007-02-22 11:53:00 +01001375 /*
1376 * thread is not dead, continue
1377 */
gurudas paie48676b2007-04-11 12:44:27 +02001378 pending++;
Jens Axboea2f77c92007-02-22 11:53:00 +01001379 continue;
Jens Axboefab6aa72007-02-17 06:19:24 +01001380reaped:
Jens Axboeebac4652005-12-08 15:25:21 +01001381 (*nr_running)--;
Jens Axboe581e7142009-06-09 12:47:16 +02001382 (*m_rate) -= (td->o.ratemin[0] + td->o.ratemin[1]);
1383 (*t_rate) -= (td->o.rate[0] + td->o.rate[1]);
Jens Axboe6f88bcb2008-03-19 10:29:07 +01001384 if (!td->pid)
1385 pending--;
Jens Axboea2f77c92007-02-22 11:53:00 +01001386
1387 if (td->error)
1388 exit_value++;
Jens Axboed3eeeab2008-04-06 12:19:05 +02001389
1390 done_secs += mtime_since_now(&td->epoch) / 1000;
Jens Axboeebac4652005-12-08 15:25:21 +01001391 }
Jens Axboeb990b5c2006-09-14 09:48:22 +02001392
Jens Axboe1f809d12007-10-25 18:34:02 +02001393 if (*nr_running == cputhreads && !pending && realthreads)
Jens Axboe390c40e2007-03-05 12:35:29 +01001394 terminate_threads(TERMINATE_ALL);
Jens Axboeebac4652005-12-08 15:25:21 +01001395}
1396
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001397static void *gtod_thread_main(void *data)
1398{
1399 fio_mutex_up(startup_mutex);
1400
1401 /*
1402 * As long as we have jobs around, update the clock. It would be nice
1403 * to have some way of NOT hammering that CPU with gettimeofday(),
1404 * but I'm not sure what to use outside of a simple CPU nop to relax
1405 * it - we don't want to lose precision.
1406 */
1407 while (threads) {
1408 fio_gtod_update();
1409 nop;
1410 }
1411
1412 return NULL;
1413}
1414
1415static int fio_start_gtod_thread(void)
1416{
bart Van Assche304a47c2010-08-01 21:31:26 +02001417 pthread_attr_t attr;
Jens Axboef42c1532009-01-05 19:07:13 +01001418 int ret;
1419
bart Van Assche304a47c2010-08-01 21:31:26 +02001420 pthread_attr_init(&attr);
1421 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
1422 ret = pthread_create(&gtod_thread, &attr, gtod_thread_main, NULL);
1423 pthread_attr_destroy(&attr);
Jens Axboef42c1532009-01-05 19:07:13 +01001424 if (ret) {
1425 log_err("Can't create gtod thread: %s\n", strerror(ret));
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001426 return 1;
1427 }
Jens Axboef42c1532009-01-05 19:07:13 +01001428
1429 ret = pthread_detach(gtod_thread);
1430 if (ret) {
1431 log_err("Can't detatch gtod thread: %s\n", strerror(ret));
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001432 return 1;
1433 }
1434
Jens Axboe29adda32009-01-05 19:06:39 +01001435 dprint(FD_MUTEX, "wait on startup_mutex\n");
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001436 fio_mutex_down(startup_mutex);
Jens Axboe29adda32009-01-05 19:06:39 +01001437 dprint(FD_MUTEX, "done waiting on startup_mutex\n");
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001438 return 0;
1439}
1440
Jens Axboe906c8d72006-06-13 09:37:56 +02001441/*
1442 * Main function for kicking off and reaping jobs, as needed.
1443 */
Jens Axboeebac4652005-12-08 15:25:21 +01001444static void run_threads(void)
1445{
Jens Axboeebac4652005-12-08 15:25:21 +01001446 struct thread_data *td;
1447 unsigned long spent;
1448 int i, todo, nr_running, m_rate, t_rate, nr_started;
Jens Axboefcb6ade2006-05-31 12:14:35 +02001449
Jens Axboe2f9ade32006-10-20 11:25:52 +02001450 if (fio_pin_memory())
1451 return;
Jens Axboeebac4652005-12-08 15:25:21 +01001452
Jens Axboebe4ecfd2008-12-08 14:10:52 +01001453 if (fio_gtod_offload && fio_start_gtod_thread())
1454 return;
1455
Jens Axboec6ae0a52006-06-12 11:04:44 +02001456 if (!terse_output) {
Jens Axboe4ceb30d2010-02-24 09:19:14 +01001457 log_info("Starting ");
Jens Axboe9cedf162007-03-12 11:29:30 +01001458 if (nr_thread)
Jens Axboe4ceb30d2010-02-24 09:19:14 +01001459 log_info("%d thread%s", nr_thread,
Jens Axboe5ec10ea2008-03-06 15:42:00 +01001460 nr_thread > 1 ? "s" : "");
Jens Axboe9cedf162007-03-12 11:29:30 +01001461 if (nr_process) {
1462 if (nr_thread)
1463 printf(" and ");
Jens Axboe4ceb30d2010-02-24 09:19:14 +01001464 log_info("%d process%s", nr_process,
Jens Axboe5ec10ea2008-03-06 15:42:00 +01001465 nr_process > 1 ? "es" : "");
Jens Axboe9cedf162007-03-12 11:29:30 +01001466 }
Jens Axboe4ceb30d2010-02-24 09:19:14 +01001467 log_info("\n");
Jens Axboec6ae0a52006-06-12 11:04:44 +02001468 fflush(stdout);
1469 }
Jens Axboec04f7ec2006-05-31 10:13:16 +02001470
Jens Axboe697a6062008-05-31 00:04:45 +02001471 set_sig_handlers();
Jens Axboe4efa9702006-05-31 11:56:49 +02001472
Jens Axboeebac4652005-12-08 15:25:21 +01001473 todo = thread_number;
1474 nr_running = 0;
1475 nr_started = 0;
1476 m_rate = t_rate = 0;
1477
Jens Axboe34572e22006-10-20 09:33:18 +02001478 for_each_td(td, i) {
Jens Axboe263e5292006-10-18 15:37:01 +02001479 print_status_init(td->thread_number - 1);
Jens Axboeebac4652005-12-08 15:25:21 +01001480
Jens Axboe2dc1bbe2007-03-15 15:01:33 +01001481 if (!td->o.create_serialize) {
Jens Axboe380cf262007-01-11 14:01:27 +01001482 init_disk_util(td);
Jens Axboeebac4652005-12-08 15:25:21 +01001483 continue;
Jens Axboe380cf262007-01-11 14:01:27 +01001484 }
Jens Axboeebac4652005-12-08 15:25:21 +01001485
1486 /*
1487 * do file setup here so it happens sequentially,
1488 * we don't want X number of threads getting their
1489 * client data interspersed on disk
1490 */
Jens Axboe53cdc682006-10-18 11:50:58 +02001491 if (setup_files(td)) {
Jens Axboe5bf13a52007-02-17 01:51:47 +01001492 exit_value++;
1493 if (td->error)
Jens Axboe5921e802008-05-30 15:02:38 +02001494 log_err("fio: pid=%d, err=%d/%s\n",
1495 (int) td->pid, td->error, td->verror);
Jens Axboeebac4652005-12-08 15:25:21 +01001496 td_set_runstate(td, TD_REAPED);
1497 todo--;
Jens Axboec69e8872008-03-01 18:50:48 +01001498 } else {
1499 struct fio_file *f;
Jens Axboe2b13e712011-01-19 14:04:16 -07001500 unsigned int j;
Jens Axboec69e8872008-03-01 18:50:48 +01001501
1502 /*
1503 * for sharing to work, each job must always open
1504 * its own files. so close them, if we opened them
1505 * for creation
1506 */
Jens Axboe2b13e712011-01-19 14:04:16 -07001507 for_each_file(td, f, j) {
Jens Axboe22a57ba2009-06-09 14:34:35 +02001508 if (fio_file_open(f))
1509 td_io_close_file(td, f);
Jens Axboe22a57ba2009-06-09 14:34:35 +02001510 }
Jens Axboe5ec10ea2008-03-06 15:42:00 +01001511 }
Jens Axboe380cf262007-01-11 14:01:27 +01001512
1513 init_disk_util(td);
Jens Axboeebac4652005-12-08 15:25:21 +01001514 }
1515
Jens Axboea2f77c92007-02-22 11:53:00 +01001516 set_genesis_time();
1517
Jens Axboeebac4652005-12-08 15:25:21 +01001518 while (todo) {
Jens Axboe75154842006-06-01 13:56:09 +02001519 struct thread_data *map[MAX_JOBS];
1520 struct timeval this_start;
1521 int this_jobs = 0, left;
1522
Jens Axboeebac4652005-12-08 15:25:21 +01001523 /*
1524 * create threads (TD_NOT_CREATED -> TD_CREATED)
1525 */
Jens Axboe34572e22006-10-20 09:33:18 +02001526 for_each_td(td, i) {
Jens Axboeebac4652005-12-08 15:25:21 +01001527 if (td->runstate != TD_NOT_CREATED)
1528 continue;
1529
1530 /*
1531 * never got a chance to start, killed by other
1532 * thread for some reason
1533 */
1534 if (td->terminate) {
1535 todo--;
1536 continue;
1537 }
1538
Jens Axboe2dc1bbe2007-03-15 15:01:33 +01001539 if (td->o.start_delay) {
Jens Axboe263e5292006-10-18 15:37:01 +02001540 spent = mtime_since_genesis();
Jens Axboeebac4652005-12-08 15:25:21 +01001541
Jens Axboe2dc1bbe2007-03-15 15:01:33 +01001542 if (td->o.start_delay * 1000 > spent)
Jens Axboeebac4652005-12-08 15:25:21 +01001543 continue;
1544 }
1545
Jens Axboe6f88bcb2008-03-19 10:29:07 +01001546 if (td->o.stonewall && (nr_started || nr_running)) {
1547 dprint(FD_PROCESS, "%s: stonewall wait\n",
1548 td->o.name);
Jens Axboeebac4652005-12-08 15:25:21 +01001549 break;
Jens Axboe6f88bcb2008-03-19 10:29:07 +01001550 }
Jens Axboeebac4652005-12-08 15:25:21 +01001551
Jens Axboe75154842006-06-01 13:56:09 +02001552 /*
1553 * Set state to created. Thread will transition
1554 * to TD_INITIALIZED when it's done setting up.
1555 */
Jens Axboeebac4652005-12-08 15:25:21 +01001556 td_set_runstate(td, TD_CREATED);
Jens Axboe75154842006-06-01 13:56:09 +02001557 map[this_jobs++] = td;
Jens Axboeebac4652005-12-08 15:25:21 +01001558 nr_started++;
1559
Jens Axboe2dc1bbe2007-03-15 15:01:33 +01001560 if (td->o.use_thread) {
Jens Axboef42c1532009-01-05 19:07:13 +01001561 int ret;
1562
Jens Axboeee56ad52008-02-01 10:30:20 +01001563 dprint(FD_PROCESS, "will pthread_create\n");
Jens Axboef42c1532009-01-05 19:07:13 +01001564 ret = pthread_create(&td->thread, NULL,
1565 thread_main, td);
1566 if (ret) {
1567 log_err("pthread_create: %s\n",
1568 strerror(ret));
Jens Axboeebac4652005-12-08 15:25:21 +01001569 nr_started--;
Jens Axboec8bb6fa2007-03-12 11:03:04 +01001570 break;
Jens Axboeebac4652005-12-08 15:25:21 +01001571 }
Jens Axboef42c1532009-01-05 19:07:13 +01001572 ret = pthread_detach(td->thread);
1573 if (ret)
1574 log_err("pthread_detach: %s",
1575 strerror(ret));
Jens Axboeebac4652005-12-08 15:25:21 +01001576 } else {
Jens Axboe5e1d3062008-05-23 11:55:53 +02001577 pid_t pid;
Jens Axboeee56ad52008-02-01 10:30:20 +01001578 dprint(FD_PROCESS, "will fork\n");
Jens Axboe5e1d3062008-05-23 11:55:53 +02001579 pid = fork();
1580 if (!pid) {
Jens Axboea6418142007-02-17 04:47:18 +01001581 int ret = fork_main(shm_id, i);
1582
Jens Axboe5e1d3062008-05-23 11:55:53 +02001583 _exit(ret);
1584 } else if (i == fio_debug_jobno)
1585 *fio_debug_jobp = pid;
Jens Axboeebac4652005-12-08 15:25:21 +01001586 }
Jens Axboe29adda32009-01-05 19:06:39 +01001587 dprint(FD_MUTEX, "wait on startup_mutex\n");
Jens Axboe656b1392009-07-01 23:02:10 +02001588 if (fio_mutex_down_timeout(startup_mutex, 10)) {
1589 log_err("fio: job startup hung? exiting.\n");
1590 terminate_threads(TERMINATE_ALL);
1591 fio_abort = 1;
1592 nr_started--;
1593 break;
1594 }
Jens Axboe29adda32009-01-05 19:06:39 +01001595 dprint(FD_MUTEX, "done waiting on startup_mutex\n");
Jens Axboeebac4652005-12-08 15:25:21 +01001596 }
1597
1598 /*
Jens Axboe75154842006-06-01 13:56:09 +02001599 * Wait for the started threads to transition to
1600 * TD_INITIALIZED.
Jens Axboeebac4652005-12-08 15:25:21 +01001601 */
Jens Axboe02bcaa82006-11-24 10:42:00 +01001602 fio_gettime(&this_start, NULL);
Jens Axboe75154842006-06-01 13:56:09 +02001603 left = this_jobs;
Jens Axboe6ce15a32007-01-12 09:04:41 +01001604 while (left && !fio_abort) {
Jens Axboe75154842006-06-01 13:56:09 +02001605 if (mtime_since_now(&this_start) > JOB_START_TIMEOUT)
1606 break;
1607
1608 usleep(100000);
1609
1610 for (i = 0; i < this_jobs; i++) {
1611 td = map[i];
1612 if (!td)
1613 continue;
Jens Axboeb6f4d882006-06-02 10:32:51 +02001614 if (td->runstate == TD_INITIALIZED) {
Jens Axboe75154842006-06-01 13:56:09 +02001615 map[i] = NULL;
1616 left--;
Jens Axboeb6f4d882006-06-02 10:32:51 +02001617 } else if (td->runstate >= TD_EXITED) {
1618 map[i] = NULL;
1619 left--;
1620 todo--;
1621 nr_running++; /* work-around... */
Jens Axboe75154842006-06-01 13:56:09 +02001622 }
1623 }
1624 }
1625
1626 if (left) {
Jens Axboe3b70d7e2006-06-08 21:48:46 +02001627 log_err("fio: %d jobs failed to start\n", left);
Jens Axboe75154842006-06-01 13:56:09 +02001628 for (i = 0; i < this_jobs; i++) {
1629 td = map[i];
1630 if (!td)
1631 continue;
1632 kill(td->pid, SIGTERM);
1633 }
1634 break;
1635 }
1636
1637 /*
Jens Axboeb6f4d882006-06-02 10:32:51 +02001638 * start created threads (TD_INITIALIZED -> TD_RUNNING).
Jens Axboe75154842006-06-01 13:56:09 +02001639 */
Jens Axboe34572e22006-10-20 09:33:18 +02001640 for_each_td(td, i) {
Jens Axboe75154842006-06-01 13:56:09 +02001641 if (td->runstate != TD_INITIALIZED)
Jens Axboeebac4652005-12-08 15:25:21 +01001642 continue;
1643
Jens Axboeb29ee5b2008-09-11 10:17:26 +02001644 if (in_ramp_time(td))
1645 td_set_runstate(td, TD_RAMP);
1646 else
1647 td_set_runstate(td, TD_RUNNING);
Jens Axboeebac4652005-12-08 15:25:21 +01001648 nr_running++;
1649 nr_started--;
Jens Axboe581e7142009-06-09 12:47:16 +02001650 m_rate += td->o.ratemin[0] + td->o.ratemin[1];
1651 t_rate += td->o.rate[0] + td->o.rate[1];
Jens Axboe75154842006-06-01 13:56:09 +02001652 todo--;
Jens Axboecdd18ad2008-02-27 18:58:00 +01001653 fio_mutex_up(td->mutex);
Jens Axboeebac4652005-12-08 15:25:21 +01001654 }
1655
1656 reap_threads(&nr_running, &t_rate, &m_rate);
1657
1658 if (todo)
1659 usleep(100000);
1660 }
1661
1662 while (nr_running) {
1663 reap_threads(&nr_running, &t_rate, &m_rate);
1664 usleep(10000);
1665 }
1666
1667 update_io_ticks();
Jens Axboe2f9ade32006-10-20 11:25:52 +02001668 fio_unpin_memory();
Jens Axboeebac4652005-12-08 15:25:21 +01001669}
1670
Jens Axboeebac4652005-12-08 15:25:21 +01001671int main(int argc, char *argv[])
1672{
Jens Axboe29d610e2007-02-06 13:25:33 +01001673 long ps;
1674
Jens Axboe2e5cdb12008-03-01 18:52:49 +01001675 sinit();
Jens Axboed54a1442010-03-25 23:07:35 +01001676 init_rand(&__fio_rand_state);
Jens Axboe2e5cdb12008-03-01 18:52:49 +01001677
Jens Axboedbe11252007-02-11 00:19:51 +01001678 /*
1679 * We need locale for number printing, if it isn't set then just
1680 * go with the US format.
1681 */
1682 if (!getenv("LC_NUMERIC"))
1683 setlocale(LC_NUMERIC, "en_US");
1684
Jens Axboe29d610e2007-02-06 13:25:33 +01001685 ps = sysconf(_SC_PAGESIZE);
1686 if (ps < 0) {
1687 log_err("Failed to get page size\n");
1688 return 1;
1689 }
1690
Jens Axboecfc99db2007-03-14 10:34:47 +01001691 page_size = ps;
Jens Axboe29d610e2007-02-06 13:25:33 +01001692 page_mask = ps - 1;
1693
Jens Axboe74929ac2009-08-05 11:42:37 +02001694 fio_keywords_init();
1695
1696 if (parse_options(argc, argv))
1697 return 1;
1698
Jens Axboe07b32322010-03-05 09:48:44 +01001699 if (exec_profile && load_profile(exec_profile))
1700 return 1;
1701
Jens Axboe74929ac2009-08-05 11:42:37 +02001702 if (!thread_number)
1703 return 0;
1704
Jens Axboebb3884d2007-01-17 17:23:11 +11001705 if (write_bw_log) {
1706 setup_log(&agg_io_log[DDIR_READ]);
1707 setup_log(&agg_io_log[DDIR_WRITE]);
1708 }
1709
Jens Axboecdd18ad2008-02-27 18:58:00 +01001710 startup_mutex = fio_mutex_init(0);
Bruce Cran03e20d62011-01-02 20:14:54 +01001711 if (startup_mutex == NULL)
1712 return 1;
Fabio Checconid2bbb8d2009-05-18 12:54:50 +02001713 writeout_mutex = fio_mutex_init(1);
Bruce Cran03e20d62011-01-02 20:14:54 +01001714 if (writeout_mutex == NULL)
1715 return 1;
Jens Axboe07739b52007-03-08 20:25:46 +01001716
Jens Axboea2f77c92007-02-22 11:53:00 +01001717 set_genesis_time();
Jens Axboe79857012011-01-14 14:14:58 +01001718 create_disk_util_thread();
Jens Axboeebac4652005-12-08 15:25:21 +01001719
Jens Axboedae53412009-12-04 19:54:18 +01001720 cgroup_list = smalloc(sizeof(*cgroup_list));
1721 INIT_FLIST_HEAD(cgroup_list);
1722
Jens Axboeebac4652005-12-08 15:25:21 +01001723 run_threads();
Jens Axboe6ce15a32007-01-12 09:04:41 +01001724
Jens Axboebb3884d2007-01-17 17:23:11 +11001725 if (!fio_abort) {
Jens Axboe6ce15a32007-01-12 09:04:41 +01001726 show_run_stats();
Jens Axboebb3884d2007-01-17 17:23:11 +11001727 if (write_bw_log) {
Jens Axboe5ec10ea2008-03-06 15:42:00 +01001728 __finish_log(agg_io_log[DDIR_READ], "agg-read_bw.log");
1729 __finish_log(agg_io_log[DDIR_WRITE],
1730 "agg-write_bw.log");
Jens Axboebb3884d2007-01-17 17:23:11 +11001731 }
1732 }
Jens Axboeebac4652005-12-08 15:25:21 +01001733
Jens Axboedae53412009-12-04 19:54:18 +01001734 cgroup_kill(cgroup_list);
1735 sfree(cgroup_list);
Jens Axboe61eb3132010-03-31 20:05:59 +02001736 sfree(cgroup_mnt);
Jens Axboe39f22022009-12-04 11:29:12 +01001737
Jens Axboecdd18ad2008-02-27 18:58:00 +01001738 fio_mutex_remove(startup_mutex);
Fabio Checconid2bbb8d2009-05-18 12:54:50 +02001739 fio_mutex_remove(writeout_mutex);
Jens Axboe437c9b72007-02-13 18:20:37 +01001740 return exit_value;
Jens Axboeebac4652005-12-08 15:25:21 +01001741}