blob: 53e57b4ffe4a6bcc51907c54ac25ca2531c73d1a [file] [log] [blame]
Jens Axboe10ba5352006-10-20 11:39:27 +02001#include <unistd.h>
2#include <fcntl.h>
3#include <string.h>
4#include <signal.h>
5#include <time.h>
Jens Axboe0c6e7512007-02-22 11:19:39 +01006#include <assert.h>
Jens Axboe10ba5352006-10-20 11:39:27 +02007
8#include "fio.h"
9#include "os.h"
10
Jens Axboe5945b9b2007-02-22 20:33:01 +010011/*
12 * Change this define to play with the timeout handling
13 */
14#undef FIO_USE_TIMEOUT
15
Jens Axboe97601022007-02-18 12:47:29 +010016struct io_completion_data {
17 int nr; /* input */
Jens Axboe97601022007-02-18 12:47:29 +010018
19 int error; /* output */
20 unsigned long bytes_done[2]; /* output */
21 struct timeval time; /* output */
22};
23
Jens Axboe10ba5352006-10-20 11:39:27 +020024/*
25 * The ->file_map[] contains a map of blocks we have or have not done io
26 * to yet. Used to make sure we cover the entire range in a fair fashion.
27 */
28static int random_map_free(struct thread_data *td, struct fio_file *f,
29 unsigned long long block)
30{
31 unsigned int idx = RAND_MAP_IDX(td, f, block);
32 unsigned int bit = RAND_MAP_BIT(td, f, block);
33
34 return (f->file_map[idx] & (1UL << bit)) == 0;
35}
36
37/*
Jens Axboedf415582006-10-20 11:41:03 +020038 * Mark a given offset as used in the map.
39 */
40static void mark_random_map(struct thread_data *td, struct fio_file *f,
41 struct io_u *io_u)
42{
Jens Axboeafdbe582007-02-10 19:01:57 +010043 unsigned int min_bs = td->rw_min_bs;
Jens Axboea00735e2006-11-03 08:58:08 +010044 unsigned long long block;
45 unsigned int blocks;
Jens Axboec685b5b2007-02-10 20:02:28 +010046 unsigned int nr_blocks;
Jens Axboedf415582006-10-20 11:41:03 +020047
Jens Axboea00735e2006-11-03 08:58:08 +010048 block = io_u->offset / (unsigned long long) min_bs;
49 blocks = 0;
Jens Axboec685b5b2007-02-10 20:02:28 +010050 nr_blocks = (io_u->buflen + min_bs - 1) / min_bs;
51
52 while (blocks < nr_blocks) {
Jens Axboedf415582006-10-20 11:41:03 +020053 unsigned int idx, bit;
54
55 if (!random_map_free(td, f, block))
56 break;
57
58 idx = RAND_MAP_IDX(td, f, block);
59 bit = RAND_MAP_BIT(td, f, block);
60
Jens Axboe0032bf92007-02-13 17:39:56 +010061 fio_assert(td, idx < f->num_maps);
Jens Axboedf415582006-10-20 11:41:03 +020062
63 f->file_map[idx] |= (1UL << bit);
64 block++;
65 blocks++;
66 }
67
Jens Axboea00735e2006-11-03 08:58:08 +010068 if ((blocks * min_bs) < io_u->buflen)
69 io_u->buflen = blocks * min_bs;
Jens Axboedf415582006-10-20 11:41:03 +020070}
71
72/*
Jens Axboe10ba5352006-10-20 11:39:27 +020073 * Return the next free block in the map.
74 */
75static int get_next_free_block(struct thread_data *td, struct fio_file *f,
76 unsigned long long *b)
77{
78 int i;
79
Jens Axboec685b5b2007-02-10 20:02:28 +010080 i = f->last_free_lookup;
81 *b = (i * BLOCKS_PER_MAP);
Jens Axboeee884702006-12-20 10:58:07 +010082 while ((*b) * td->rw_min_bs < f->real_file_size) {
Jens Axboe10ba5352006-10-20 11:39:27 +020083 if (f->file_map[i] != -1UL) {
84 *b += ffz(f->file_map[i]);
Jens Axboec685b5b2007-02-10 20:02:28 +010085 f->last_free_lookup = i;
Jens Axboe10ba5352006-10-20 11:39:27 +020086 return 0;
87 }
88
89 *b += BLOCKS_PER_MAP;
90 i++;
91 }
92
93 return 1;
94}
95
96/*
97 * For random io, generate a random new block and see if it's used. Repeat
98 * until we find a free one. For sequential io, just return the end of
99 * the last io issued.
100 */
101static int get_next_offset(struct thread_data *td, struct fio_file *f,
Jens Axboec685b5b2007-02-10 20:02:28 +0100102 struct io_u *io_u)
Jens Axboe10ba5352006-10-20 11:39:27 +0200103{
Jens Axboec685b5b2007-02-10 20:02:28 +0100104 const int ddir = io_u->ddir;
Jens Axboe10ba5352006-10-20 11:39:27 +0200105 unsigned long long b, rb;
106 long r;
107
Jens Axboe413dd452007-02-23 09:26:09 +0100108 if (td_random(td)) {
Jens Axboef6971252006-11-15 10:00:31 +0100109 unsigned long long max_blocks = f->file_size / td->min_bs[ddir];
Jens Axboec685b5b2007-02-10 20:02:28 +0100110 int loops = 5;
Jens Axboe10ba5352006-10-20 11:39:27 +0200111
112 do {
113 r = os_random_long(&td->random_state);
114 b = ((max_blocks - 1) * r / (unsigned long long) (RAND_MAX+1.0));
Jens Axboebb8895e2006-10-30 15:14:48 +0100115 if (td->norandommap)
116 break;
Jens Axboea00735e2006-11-03 08:58:08 +0100117 rb = b + (f->file_offset / td->min_bs[ddir]);
Jens Axboe10ba5352006-10-20 11:39:27 +0200118 loops--;
119 } while (!random_map_free(td, f, rb) && loops);
120
Jens Axboebca4ed42007-02-12 05:13:23 +0100121 /*
122 * if we failed to retrieve a truly random offset within
123 * the loops assigned, see if there are free ones left at all
124 */
125 if (!loops && get_next_free_block(td, f, &b))
126 return 1;
Jens Axboe10ba5352006-10-20 11:39:27 +0200127 } else
Jens Axboea00735e2006-11-03 08:58:08 +0100128 b = f->last_pos / td->min_bs[ddir];
Jens Axboe10ba5352006-10-20 11:39:27 +0200129
Jens Axboec685b5b2007-02-10 20:02:28 +0100130 io_u->offset = (b * td->min_bs[ddir]) + f->file_offset;
Jens Axboebca4ed42007-02-12 05:13:23 +0100131 if (io_u->offset >= f->real_file_size)
Jens Axboe10ba5352006-10-20 11:39:27 +0200132 return 1;
133
134 return 0;
135}
136
Jens Axboebca4ed42007-02-12 05:13:23 +0100137static unsigned int get_next_buflen(struct thread_data *td, struct fio_file *f,
138 struct io_u *io_u)
Jens Axboe10ba5352006-10-20 11:39:27 +0200139{
Jens Axboebca4ed42007-02-12 05:13:23 +0100140 const int ddir = io_u->ddir;
Jens Axboe10ba5352006-10-20 11:39:27 +0200141 unsigned int buflen;
142 long r;
143
Jens Axboea00735e2006-11-03 08:58:08 +0100144 if (td->min_bs[ddir] == td->max_bs[ddir])
145 buflen = td->min_bs[ddir];
Jens Axboe10ba5352006-10-20 11:39:27 +0200146 else {
147 r = os_random_long(&td->bsrange_state);
Jens Axboe1e97cce2006-12-05 11:44:16 +0100148 buflen = (unsigned int) (1 + (double) (td->max_bs[ddir] - 1) * r / (RAND_MAX + 1.0));
Jens Axboe690adba2006-10-30 15:25:09 +0100149 if (!td->bs_unaligned)
Jens Axboea00735e2006-11-03 08:58:08 +0100150 buflen = (buflen + td->min_bs[ddir] - 1) & ~(td->min_bs[ddir] - 1);
Jens Axboe10ba5352006-10-20 11:39:27 +0200151 }
152
Jens Axboebca4ed42007-02-12 05:13:23 +0100153 while (buflen + io_u->offset > f->real_file_size) {
Jens Axboebca4ed42007-02-12 05:13:23 +0100154 if (buflen == td->min_bs[ddir])
155 return 0;
156
157 buflen = td->min_bs[ddir];
Jens Axboe10ba5352006-10-20 11:39:27 +0200158 }
159
160 return buflen;
161}
162
163/*
164 * Return the data direction for the next io_u. If the job is a
165 * mixed read/write workload, check the rwmix cycle and switch if
166 * necessary.
167 */
Jens Axboe1e97cce2006-12-05 11:44:16 +0100168static enum fio_ddir get_rw_ddir(struct thread_data *td)
Jens Axboe10ba5352006-10-20 11:39:27 +0200169{
170 if (td_rw(td)) {
171 struct timeval now;
172 unsigned long elapsed;
173
Jens Axboe02bcaa82006-11-24 10:42:00 +0100174 fio_gettime(&now, NULL);
Jens Axboe10ba5352006-10-20 11:39:27 +0200175 elapsed = mtime_since_now(&td->rwmix_switch);
176
177 /*
178 * Check if it's time to seed a new data direction.
179 */
180 if (elapsed >= td->rwmixcycle) {
181 unsigned int v;
182 long r;
183
184 r = os_random_long(&td->rwmix_state);
185 v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0)));
186 if (v < td->rwmixread)
187 td->rwmix_ddir = DDIR_READ;
188 else
189 td->rwmix_ddir = DDIR_WRITE;
190 memcpy(&td->rwmix_switch, &now, sizeof(now));
191 }
192 return td->rwmix_ddir;
193 } else if (td_read(td))
194 return DDIR_READ;
195 else
196 return DDIR_WRITE;
197}
198
Jens Axboe10ba5352006-10-20 11:39:27 +0200199void put_io_u(struct thread_data *td, struct io_u *io_u)
200{
Jens Axboe0c6e7512007-02-22 11:19:39 +0100201 assert((io_u->flags & IO_U_F_FREE) == 0);
202 io_u->flags |= IO_U_F_FREE;
203
Jens Axboe10ba5352006-10-20 11:39:27 +0200204 io_u->file = NULL;
205 list_del(&io_u->list);
206 list_add(&io_u->list, &td->io_u_freelist);
207 td->cur_depth--;
208}
209
Jens Axboe755200a2007-02-19 13:08:12 +0100210void requeue_io_u(struct thread_data *td, struct io_u **io_u)
211{
212 struct io_u *__io_u = *io_u;
213
214 list_del(&__io_u->list);
215 list_add_tail(&__io_u->list, &td->io_u_requeues);
216 td->cur_depth--;
217 *io_u = NULL;
218}
219
Jens Axboe10ba5352006-10-20 11:39:27 +0200220static int fill_io_u(struct thread_data *td, struct fio_file *f,
221 struct io_u *io_u)
222{
223 /*
224 * If using an iolog, grab next piece if any available.
225 */
226 if (td->read_iolog)
227 return read_iolog_get(td, io_u);
228
229 /*
Jens Axboe87dc1ab2006-10-24 14:41:26 +0200230 * see if it's time to sync
231 */
Jens Axboe755200a2007-02-19 13:08:12 +0100232 if (td->fsync_blocks && !(td->io_issues[DDIR_WRITE] % td->fsync_blocks)
233 && td->io_issues[DDIR_WRITE] && should_fsync(td)) {
Jens Axboe87dc1ab2006-10-24 14:41:26 +0200234 io_u->ddir = DDIR_SYNC;
235 io_u->file = f;
236 return 0;
237 }
238
Jens Axboea00735e2006-11-03 08:58:08 +0100239 io_u->ddir = get_rw_ddir(td);
240
Jens Axboe87dc1ab2006-10-24 14:41:26 +0200241 /*
Jens Axboec685b5b2007-02-10 20:02:28 +0100242 * No log, let the seq/rand engine retrieve the next buflen and
243 * position.
Jens Axboe10ba5352006-10-20 11:39:27 +0200244 */
Jens Axboebca4ed42007-02-12 05:13:23 +0100245 if (get_next_offset(td, f, io_u))
246 return 1;
Jens Axboec685b5b2007-02-10 20:02:28 +0100247
Jens Axboebca4ed42007-02-12 05:13:23 +0100248 io_u->buflen = get_next_buflen(td, f, io_u);
249 if (!io_u->buflen)
250 return 1;
Jens Axboe10ba5352006-10-20 11:39:27 +0200251
Jens Axboebca4ed42007-02-12 05:13:23 +0100252 /*
253 * mark entry before potentially trimming io_u
254 */
Jens Axboe413dd452007-02-23 09:26:09 +0100255 if (!td->read_iolog && td_random(td) && !td->norandommap)
Jens Axboebca4ed42007-02-12 05:13:23 +0100256 mark_random_map(td, f, io_u);
Jens Axboe10ba5352006-10-20 11:39:27 +0200257
Jens Axboebca4ed42007-02-12 05:13:23 +0100258 /*
259 * If using a write iolog, store this entry.
260 */
261 if (td->write_iolog_file)
262 write_iolog_put(td, io_u);
263
264 io_u->file = f;
265 return 0;
Jens Axboe10ba5352006-10-20 11:39:27 +0200266}
267
Jens Axboe71619dc2007-01-13 23:56:33 +0100268static void io_u_mark_depth(struct thread_data *td)
269{
270 int index = 0;
271
272 switch (td->cur_depth) {
273 default:
274 index++;
275 case 32 ... 63:
276 index++;
277 case 16 ... 31:
278 index++;
279 case 8 ... 15:
280 index++;
281 case 4 ... 7:
282 index++;
283 case 2 ... 3:
284 index++;
285 case 1:
286 break;
287 }
288
289 td->io_u_map[index]++;
290 td->total_io_u++;
291}
292
Jens Axboeec118302007-02-17 04:38:20 +0100293static void io_u_mark_latency(struct thread_data *td, unsigned long msec)
294{
295 int index = 0;
296
297 switch (msec) {
298 default:
299 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100300 case 1000 ... 1999:
Jens Axboeec118302007-02-17 04:38:20 +0100301 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100302 case 750 ... 999:
Jens Axboeec118302007-02-17 04:38:20 +0100303 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100304 case 500 ... 749:
Jens Axboeec118302007-02-17 04:38:20 +0100305 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100306 case 250 ... 499:
Jens Axboeec118302007-02-17 04:38:20 +0100307 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100308 case 100 ... 249:
Jens Axboeec118302007-02-17 04:38:20 +0100309 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100310 case 50 ... 99:
Jens Axboeec118302007-02-17 04:38:20 +0100311 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100312 case 20 ... 49:
Jens Axboeec118302007-02-17 04:38:20 +0100313 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100314 case 10 ... 19:
Jens Axboeec118302007-02-17 04:38:20 +0100315 index++;
Jens Axboe8abdce62007-02-21 10:22:55 +0100316 case 4 ... 9:
Jens Axboeec118302007-02-17 04:38:20 +0100317 index++;
318 case 2 ... 3:
319 index++;
320 case 0 ... 1:
321 break;
322 }
323
324 td->io_u_lat[index]++;
325}
326
Jens Axboe0aabe162007-02-23 08:45:55 +0100327/*
328 * Get next file to service by choosing one at random
329 */
330static struct fio_file *get_next_file_rand(struct thread_data *td)
331{
332 long r = os_random_long(&td->next_file_state);
333 unsigned int fileno;
334 struct fio_file *f;
335
336 do {
337 fileno = (unsigned int) ((double) (td->nr_files - 1) * r / (RAND_MAX + 1.0));
338 f = &td->files[fileno];
339 if (f->fd != -1)
340 return f;
341 } while (1);
342}
343
344/*
345 * Get next file to service by doing round robin between all available ones
346 */
347static struct fio_file *get_next_file_rr(struct thread_data *td)
Jens Axboe3d7c3912007-02-19 13:16:12 +0100348{
349 unsigned int old_next_file = td->next_file;
350 struct fio_file *f;
351
352 do {
353 f = &td->files[td->next_file];
354
355 td->next_file++;
356 if (td->next_file >= td->nr_files)
357 td->next_file = 0;
358
359 if (f->fd != -1)
360 break;
361
362 f = NULL;
363 } while (td->next_file != old_next_file);
364
365 return f;
366}
367
Jens Axboe10ba5352006-10-20 11:39:27 +0200368struct io_u *__get_io_u(struct thread_data *td)
369{
370 struct io_u *io_u = NULL;
371
Jens Axboe755200a2007-02-19 13:08:12 +0100372 if (!list_empty(&td->io_u_requeues))
373 io_u = list_entry(td->io_u_requeues.next, struct io_u, list);
374 else if (!queue_full(td)) {
Jens Axboe10ba5352006-10-20 11:39:27 +0200375 io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
376
Jens Axboe6040dab2006-10-24 19:38:15 +0200377 io_u->buflen = 0;
Jens Axboe10ba5352006-10-20 11:39:27 +0200378 io_u->resid = 0;
Jens Axboe755200a2007-02-19 13:08:12 +0100379 io_u->file = NULL;
Jens Axboed7762cf2007-02-23 12:34:57 +0100380 io_u->end_io = NULL;
Jens Axboe755200a2007-02-19 13:08:12 +0100381 }
382
383 if (io_u) {
Jens Axboe0c6e7512007-02-22 11:19:39 +0100384 assert(io_u->flags & IO_U_F_FREE);
385 io_u->flags &= ~IO_U_F_FREE;
386
Jens Axboe755200a2007-02-19 13:08:12 +0100387 io_u->error = 0;
Jens Axboe10ba5352006-10-20 11:39:27 +0200388 list_del(&io_u->list);
389 list_add(&io_u->list, &td->io_u_busylist);
390 td->cur_depth++;
Jens Axboe71619dc2007-01-13 23:56:33 +0100391 io_u_mark_depth(td);
Jens Axboe10ba5352006-10-20 11:39:27 +0200392 }
393
394 return io_u;
395}
396
397/*
398 * Return an io_u to be processed. Gets a buflen and offset, sets direction,
399 * etc. The returned io_u is fully ready to be prepped and submitted.
400 */
Jens Axboe3d7c3912007-02-19 13:16:12 +0100401struct io_u *get_io_u(struct thread_data *td)
Jens Axboe10ba5352006-10-20 11:39:27 +0200402{
Jens Axboe3d7c3912007-02-19 13:16:12 +0100403 struct fio_file *f;
Jens Axboe10ba5352006-10-20 11:39:27 +0200404 struct io_u *io_u;
405
406 io_u = __get_io_u(td);
407 if (!io_u)
408 return NULL;
409
Jens Axboe755200a2007-02-19 13:08:12 +0100410 /*
411 * from a requeue, io_u already setup
412 */
413 if (io_u->file)
Jens Axboe77f392b2007-02-19 20:13:09 +0100414 goto out;
Jens Axboe755200a2007-02-19 13:08:12 +0100415
Jens Axboe0aabe162007-02-23 08:45:55 +0100416 if (td->file_service_type == FIO_FSERVICE_RR)
417 f = get_next_file_rr(td);
418 else
419 f = get_next_file_rand(td);
420
Jens Axboe3d7c3912007-02-19 13:16:12 +0100421 if (!f) {
422 put_io_u(td, io_u);
423 return NULL;
424 }
425
426 io_u->file = f;
427
Jens Axboe10ba5352006-10-20 11:39:27 +0200428 if (td->zone_bytes >= td->zone_size) {
429 td->zone_bytes = 0;
430 f->last_pos += td->zone_skip;
431 }
432
433 if (fill_io_u(td, f, io_u)) {
434 put_io_u(td, io_u);
435 return NULL;
436 }
437
Jens Axboeee884702006-12-20 10:58:07 +0100438 if (io_u->buflen + io_u->offset > f->real_file_size) {
Jens Axboe10ba5352006-10-20 11:39:27 +0200439 if (td->io_ops->flags & FIO_RAWIO) {
440 put_io_u(td, io_u);
441 return NULL;
442 }
443
Jens Axboeee884702006-12-20 10:58:07 +0100444 io_u->buflen = f->real_file_size - io_u->offset;
Jens Axboe10ba5352006-10-20 11:39:27 +0200445 }
446
Jens Axboe87dc1ab2006-10-24 14:41:26 +0200447 if (io_u->ddir != DDIR_SYNC) {
448 if (!io_u->buflen) {
449 put_io_u(td, io_u);
450 return NULL;
451 }
452
Jens Axboe36167d82007-02-18 05:41:31 +0100453 f->last_pos = io_u->offset + io_u->buflen;
Jens Axboe87dc1ab2006-10-24 14:41:26 +0200454
455 if (td->verify != VERIFY_NONE)
456 populate_verify_io_u(td, io_u);
Jens Axboe10ba5352006-10-20 11:39:27 +0200457 }
458
Jens Axboe165faf12007-02-07 11:30:37 +0100459 /*
460 * Set io data pointers.
461 */
Jens Axboe77f392b2007-02-19 20:13:09 +0100462out:
Jens Axboecec6b552007-02-06 20:15:38 +0100463 io_u->xfer_buf = io_u->buf;
464 io_u->xfer_buflen = io_u->buflen;
Jens Axboe165faf12007-02-07 11:30:37 +0100465
Jens Axboe36167d82007-02-18 05:41:31 +0100466 if (td_io_prep(td, io_u)) {
467 put_io_u(td, io_u);
468 return NULL;
469 }
470
Jens Axboe02bcaa82006-11-24 10:42:00 +0100471 fio_gettime(&io_u->start_time, NULL);
Jens Axboe10ba5352006-10-20 11:39:27 +0200472 return io_u;
473}
474
Jens Axboe97601022007-02-18 12:47:29 +0100475static void io_completed(struct thread_data *td, struct io_u *io_u,
476 struct io_completion_data *icd)
Jens Axboe10ba5352006-10-20 11:39:27 +0200477{
Jens Axboe10ba5352006-10-20 11:39:27 +0200478 unsigned long msec;
479
Jens Axboe0c6e7512007-02-22 11:19:39 +0100480 assert(io_u->flags & IO_U_F_FLIGHT);
481 io_u->flags &= ~IO_U_F_FLIGHT;
482
Jens Axboe87dc1ab2006-10-24 14:41:26 +0200483 if (io_u->ddir == DDIR_SYNC) {
484 td->last_was_sync = 1;
485 return;
486 }
487
488 td->last_was_sync = 0;
489
Jens Axboe10ba5352006-10-20 11:39:27 +0200490 if (!io_u->error) {
491 unsigned int bytes = io_u->buflen - io_u->resid;
Jens Axboe1e97cce2006-12-05 11:44:16 +0100492 const enum fio_ddir idx = io_u->ddir;
Jens Axboe3af6ef32007-02-18 06:57:43 +0100493 int ret;
Jens Axboe10ba5352006-10-20 11:39:27 +0200494
495 td->io_blocks[idx]++;
496 td->io_bytes[idx] += bytes;
497 td->zone_bytes += bytes;
498 td->this_io_bytes[idx] += bytes;
499
Jens Axboe02bcaa82006-11-24 10:42:00 +0100500 io_u->file->last_completed_pos = io_u->offset + io_u->buflen;
501
502 msec = mtime_since(&io_u->issue_time, &icd->time);
Jens Axboe10ba5352006-10-20 11:39:27 +0200503
504 add_clat_sample(td, idx, msec);
Jens Axboe02bcaa82006-11-24 10:42:00 +0100505 add_bw_sample(td, idx, &icd->time);
Jens Axboeec118302007-02-17 04:38:20 +0100506 io_u_mark_latency(td, msec);
Jens Axboe10ba5352006-10-20 11:39:27 +0200507
508 if ((td_rw(td) || td_write(td)) && idx == DDIR_WRITE)
509 log_io_piece(td, io_u);
510
511 icd->bytes_done[idx] += bytes;
Jens Axboe3af6ef32007-02-18 06:57:43 +0100512
Jens Axboed7762cf2007-02-23 12:34:57 +0100513 if (io_u->end_io) {
514 ret = io_u->end_io(io_u);
Jens Axboe3af6ef32007-02-18 06:57:43 +0100515 if (ret && !icd->error)
516 icd->error = ret;
517 }
Jens Axboe10ba5352006-10-20 11:39:27 +0200518 } else
519 icd->error = io_u->error;
520}
521
Jens Axboed7762cf2007-02-23 12:34:57 +0100522static void init_icd(struct io_completion_data *icd, int nr)
Jens Axboe36167d82007-02-18 05:41:31 +0100523{
524 fio_gettime(&icd->time, NULL);
525
Jens Axboe3af6ef32007-02-18 06:57:43 +0100526 icd->nr = nr;
527
Jens Axboe36167d82007-02-18 05:41:31 +0100528 icd->error = 0;
529 icd->bytes_done[0] = icd->bytes_done[1] = 0;
530}
531
Jens Axboe97601022007-02-18 12:47:29 +0100532static void ios_completed(struct thread_data *td,
533 struct io_completion_data *icd)
Jens Axboe10ba5352006-10-20 11:39:27 +0200534{
535 struct io_u *io_u;
536 int i;
537
Jens Axboe10ba5352006-10-20 11:39:27 +0200538 for (i = 0; i < icd->nr; i++) {
539 io_u = td->io_ops->event(td, i);
540
541 io_completed(td, io_u, icd);
542 put_io_u(td, io_u);
543 }
544}
Jens Axboe97601022007-02-18 12:47:29 +0100545
Jens Axboee7e6cfb2007-02-20 10:58:34 +0100546/*
547 * Complete a single io_u for the sync engines.
548 */
Jens Axboed7762cf2007-02-23 12:34:57 +0100549long io_u_sync_complete(struct thread_data *td, struct io_u *io_u)
Jens Axboe97601022007-02-18 12:47:29 +0100550{
551 struct io_completion_data icd;
552
Jens Axboed7762cf2007-02-23 12:34:57 +0100553 init_icd(&icd, 1);
Jens Axboe97601022007-02-18 12:47:29 +0100554 io_completed(td, io_u, &icd);
555 put_io_u(td, io_u);
556
557 if (!icd.error)
558 return icd.bytes_done[0] + icd.bytes_done[1];
559
Jens Axboe97601022007-02-18 12:47:29 +0100560 return -1;
561}
562
Jens Axboee7e6cfb2007-02-20 10:58:34 +0100563/*
564 * Called to complete min_events number of io for the async engines.
565 */
Jens Axboed7762cf2007-02-23 12:34:57 +0100566long io_u_queued_complete(struct thread_data *td, int min_events)
Jens Axboe97601022007-02-18 12:47:29 +0100567{
Jens Axboe97601022007-02-18 12:47:29 +0100568 struct io_completion_data icd;
Jens Axboe00de55e2007-02-20 10:45:57 +0100569 struct timespec *tvp = NULL;
Jens Axboe97601022007-02-18 12:47:29 +0100570 int ret;
571
Jens Axboe755200a2007-02-19 13:08:12 +0100572 if (min_events > 0) {
Jens Axboe755200a2007-02-19 13:08:12 +0100573 ret = td_io_commit(td);
574 if (ret < 0) {
Jens Axboee1161c32007-02-22 19:36:48 +0100575 td_verror(td, -ret, "td_io_commit");
Jens Axboe755200a2007-02-19 13:08:12 +0100576 return ret;
577 }
Jens Axboe00de55e2007-02-20 10:45:57 +0100578 } else {
579 struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, };
580
581 tvp = &ts;
Jens Axboe755200a2007-02-19 13:08:12 +0100582 }
Jens Axboe97601022007-02-18 12:47:29 +0100583
Jens Axboe00de55e2007-02-20 10:45:57 +0100584 ret = td_io_getevents(td, min_events, td->cur_depth, tvp);
Jens Axboe97601022007-02-18 12:47:29 +0100585 if (ret < 0) {
Jens Axboee1161c32007-02-22 19:36:48 +0100586 td_verror(td, -ret, "td_io_getevents");
Jens Axboe97601022007-02-18 12:47:29 +0100587 return ret;
588 } else if (!ret)
589 return ret;
590
Jens Axboed7762cf2007-02-23 12:34:57 +0100591 init_icd(&icd, ret);
Jens Axboe97601022007-02-18 12:47:29 +0100592 ios_completed(td, &icd);
593 if (!icd.error)
594 return icd.bytes_done[0] + icd.bytes_done[1];
595
Jens Axboe97601022007-02-18 12:47:29 +0100596 return -1;
597}
Jens Axboe7e77dd02007-02-20 10:57:34 +0100598
599/*
600 * Call when io_u is really queued, to update the submission latency.
601 */
602void io_u_queued(struct thread_data *td, struct io_u *io_u)
603{
604 unsigned long slat_time;
605
606 slat_time = mtime_since(&io_u->start_time, &io_u->issue_time);
607 add_slat_sample(td, io_u->ddir, slat_time);
608}
Jens Axboe433afcb2007-02-22 10:39:01 +0100609
Jens Axboe55bc9722007-02-22 11:30:05 +0100610#ifdef FIO_USE_TIMEOUT
Jens Axboe433afcb2007-02-22 10:39:01 +0100611void io_u_set_timeout(struct thread_data *td)
612{
613 assert(td->cur_depth);
614
615 td->timer.it_interval.tv_sec = 0;
616 td->timer.it_interval.tv_usec = 0;
617 td->timer.it_value.tv_sec = IO_U_TIMEOUT + IO_U_TIMEOUT_INC;
618 td->timer.it_value.tv_usec = 0;
619 setitimer(ITIMER_REAL, &td->timer, NULL);
620 fio_gettime(&td->timeout_end, NULL);
621}
Jens Axboe5945b9b2007-02-22 20:33:01 +0100622
623static void io_u_dump(struct io_u *io_u)
624{
625 unsigned long t_start = mtime_since_now(&io_u->start_time);
626 unsigned long t_issue = mtime_since_now(&io_u->issue_time);
627
628 log_err("io_u=%p, t_start=%lu, t_issue=%lu\n", io_u, t_start, t_issue);
629 log_err(" buf=%p/%p, len=%lu/%lu, offset=%llu\n", io_u->buf, io_u->xfer_buf, io_u->buflen, io_u->xfer_buflen, io_u->offset);
630 log_err(" ddir=%d, fname=%s\n", io_u->ddir, io_u->file->file_name);
631}
Jens Axboe55bc9722007-02-22 11:30:05 +0100632#else
633void io_u_set_timeout(struct thread_data fio_unused *td)
634{
635}
636#endif
Jens Axboe433afcb2007-02-22 10:39:01 +0100637
Jens Axboe55bc9722007-02-22 11:30:05 +0100638#ifdef FIO_USE_TIMEOUT
Jens Axboe433afcb2007-02-22 10:39:01 +0100639static void io_u_timeout_handler(int fio_unused sig)
640{
641 struct thread_data *td, *__td;
642 pid_t pid = getpid();
Jens Axboe5945b9b2007-02-22 20:33:01 +0100643 struct list_head *entry;
644 struct io_u *io_u;
Jens Axboe433afcb2007-02-22 10:39:01 +0100645 int i;
646
647 log_err("fio: io_u timeout\n");
648
649 /*
650 * TLS would be nice...
651 */
652 td = NULL;
653 for_each_td(__td, i) {
654 if (__td->pid == pid) {
655 td = __td;
656 break;
657 }
658 }
659
660 if (!td) {
661 log_err("fio: io_u timeout, can't find job\n");
662 exit(1);
663 }
664
665 if (!td->cur_depth) {
666 log_err("fio: timeout without pending work?\n");
667 return;
668 }
669
670 log_err("fio: io_u timeout: job=%s, pid=%d\n", td->name, td->pid);
Jens Axboe5945b9b2007-02-22 20:33:01 +0100671
672 list_for_each(entry, &td->io_u_busylist) {
673 io_u = list_entry(entry, struct io_u, list);
674
675 io_u_dump(io_u);
676 }
677
678 td_verror(td, ETIMEDOUT, "io_u timeout");
Jens Axboe433afcb2007-02-22 10:39:01 +0100679 exit(1);
680}
Jens Axboe55bc9722007-02-22 11:30:05 +0100681#endif
Jens Axboe433afcb2007-02-22 10:39:01 +0100682
683void io_u_init_timeout(void)
684{
Jens Axboe55bc9722007-02-22 11:30:05 +0100685#ifdef FIO_USE_TIMEOUT
Jens Axboe433afcb2007-02-22 10:39:01 +0100686 signal(SIGALRM, io_u_timeout_handler);
Jens Axboe55bc9722007-02-22 11:30:05 +0100687#endif
Jens Axboe433afcb2007-02-22 10:39:01 +0100688}