blob: 9e88d6154546afdd0a960a5ece2b723abe0406b8 [file] [log] [blame]
Jens Axboee29d1b72006-10-18 15:43:15 +02001/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
Jens Axboe97af62c2007-05-22 11:12:13 +02007#include <assert.h>
Jens Axboee8462bd2009-07-06 12:59:04 +02008#include <pthread.h>
Jens Axboe79402a12011-01-14 12:41:15 +01009#include <libgen.h>
Jens Axboee29d1b72006-10-18 15:43:15 +020010
11#include "fio.h"
Jens Axboe4f5af7b2009-06-03 08:45:40 +020012#include "verify.h"
Jens Axboe0d29de82010-09-01 13:54:15 +020013#include "trim.h"
Jens Axboe637ef8d2010-06-21 20:39:38 +020014#include "lib/rand.h"
Jens Axboe51aa2da2013-01-21 10:55:02 -070015#include "lib/hweight.h"
Jens Axboee29d1b72006-10-18 15:43:15 +020016
Jens Axboeeef6eea2007-07-30 12:27:58 +020017#include "crc/md5.h"
18#include "crc/crc64.h"
19#include "crc/crc32.h"
Jens Axboebac39e02008-06-11 20:46:19 +020020#include "crc/crc32c.h"
Jens Axboeeef6eea2007-07-30 12:27:58 +020021#include "crc/crc16.h"
22#include "crc/crc7.h"
23#include "crc/sha256.h"
24#include "crc/sha512.h"
Jens Axboe7c353ce2009-08-09 22:40:33 +020025#include "crc/sha1.h"
Jens Axboecd14cc12007-07-30 10:59:33 +020026
Jens Axboe7d9fb452011-01-11 22:16:49 +010027static void populate_hdr(struct thread_data *td, struct io_u *io_u,
28 struct verify_header *hdr, unsigned int header_num,
29 unsigned int header_len);
30
31void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u, unsigned long seed, int use_seed)
Jens Axboe90059d62007-07-30 09:33:12 +020032{
33 switch (td->o.verify_pattern_bytes) {
34 case 0:
Jens Axboebd6f78b2008-02-01 20:27:52 +010035 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
Jens Axboe7d9fb452011-01-11 22:16:49 +010036 if (use_seed)
37 __fill_random_buf(p, len, seed);
38 else
Jens Axboe3545a102011-08-31 15:20:15 -060039 io_u->rand_seed = fill_random_buf(&td->buf_state, p, len);
Jens Axboe90059d62007-07-30 09:33:12 +020040 break;
41 case 1:
Radha Ramachandran10e8a7b2010-07-14 17:39:05 -060042 if (io_u->buf_filled_len >= len) {
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020043 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len);
44 return;
45 }
Jens Axboebd6f78b2008-02-01 20:27:52 +010046 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
Radha Ramachandran0e92f872009-10-27 20:14:27 +010047 memset(p, td->o.verify_pattern[0], len);
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020048 io_u->buf_filled_len = len;
Jens Axboe90059d62007-07-30 09:33:12 +020049 break;
Radha Ramachandran0e92f872009-10-27 20:14:27 +010050 default: {
51 unsigned int i = 0, size = 0;
Jens Axboe90059d62007-07-30 09:33:12 +020052 unsigned char *b = p;
53
Radha Ramachandran10e8a7b2010-07-14 17:39:05 -060054 if (io_u->buf_filled_len >= len) {
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020055 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
56 td->o.verify_pattern_bytes, len);
57 return;
58 }
Jens Axboe81f03662012-02-02 09:20:09 +010059
Jens Axboebd6f78b2008-02-01 20:27:52 +010060 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
61 td->o.verify_pattern_bytes, len);
62
Jens Axboe90059d62007-07-30 09:33:12 +020063 while (i < len) {
Radha Ramachandran0e92f872009-10-27 20:14:27 +010064 size = td->o.verify_pattern_bytes;
65 if (size > (len - i))
66 size = len - i;
67 memcpy(b+i, td->o.verify_pattern, size);
68 i += size;
Jens Axboe90059d62007-07-30 09:33:12 +020069 }
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020070 io_u->buf_filled_len = len;
Jens Axboe90059d62007-07-30 09:33:12 +020071 break;
72 }
73 }
74}
75
Jens Axboecfbcd0d2011-01-14 14:49:20 +010076static unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u)
77{
78 unsigned int hdr_inc;
79
80 hdr_inc = io_u->buflen;
Jens Axboe8a99fdf2012-03-06 19:24:49 +010081 if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen)
Jens Axboecfbcd0d2011-01-14 14:49:20 +010082 hdr_inc = td->o.verify_interval;
83
84 return hdr_inc;
85}
86
Jens Axboec50ca7b2011-01-12 08:31:54 +010087static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u,
88 unsigned long seed, int use_seed)
89{
90 unsigned int hdr_inc, header_num;
91 struct verify_header *hdr;
92 void *p = io_u->buf;
93
94 fill_pattern(td, p, io_u->buflen, io_u, seed, use_seed);
95
Jens Axboecfbcd0d2011-01-14 14:49:20 +010096 hdr_inc = get_hdr_inc(td, io_u);
Jens Axboec50ca7b2011-01-12 08:31:54 +010097 header_num = 0;
98 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
99 hdr = p;
100 populate_hdr(td, io_u, hdr, header_num, hdr_inc);
101 header_num++;
102 }
103}
104
Jens Axboe4764aec2007-08-23 09:03:38 +0200105static void memswp(void *buf1, void *buf2, unsigned int len)
Shawn Lewis546a9142007-07-28 21:11:37 +0200106{
Shawn Lewisdee6de72007-08-02 22:17:52 +0200107 char swap[200];
108
109 assert(len <= sizeof(swap));
Jens Axboe90059d62007-07-30 09:33:12 +0200110
Shawn Lewis546a9142007-07-28 21:11:37 +0200111 memcpy(&swap, buf1, len);
112 memcpy(buf1, buf2, len);
113 memcpy(buf2, &swap, len);
114}
115
Jens Axboee29d1b72006-10-18 15:43:15 +0200116static void hexdump(void *buffer, int len)
117{
118 unsigned char *p = buffer;
119 int i;
120
121 for (i = 0; i < len; i++)
Jens Axboebfacf382010-06-18 14:29:52 +0200122 log_err("%02x", p[i]);
123 log_err("\n");
Jens Axboee29d1b72006-10-18 15:43:15 +0200124}
125
Jens Axboed9f2caf2007-07-28 21:22:03 +0200126/*
Jens Axboe87677832007-07-30 12:08:14 +0200127 * Prepare for seperation of verify_header and checksum header
128 */
Jens Axboe546dfd92007-07-30 12:23:05 +0200129static inline unsigned int __hdr_size(int verify_type)
Jens Axboe87677832007-07-30 12:08:14 +0200130{
Bruce Cran03e20d62011-01-02 20:14:54 +0100131 unsigned int len = 0;
Jens Axboe87677832007-07-30 12:08:14 +0200132
Jens Axboe546dfd92007-07-30 12:23:05 +0200133 switch (verify_type) {
134 case VERIFY_NONE:
135 case VERIFY_NULL:
136 len = 0;
137 break;
138 case VERIFY_MD5:
139 len = sizeof(struct vhdr_md5);
140 break;
141 case VERIFY_CRC64:
142 len = sizeof(struct vhdr_crc64);
143 break;
Jens Axboebac39e02008-06-11 20:46:19 +0200144 case VERIFY_CRC32C:
Jens Axboe546dfd92007-07-30 12:23:05 +0200145 case VERIFY_CRC32:
Jens Axboe38455912008-08-04 15:35:26 +0200146 case VERIFY_CRC32C_INTEL:
Jens Axboe546dfd92007-07-30 12:23:05 +0200147 len = sizeof(struct vhdr_crc32);
148 break;
149 case VERIFY_CRC16:
150 len = sizeof(struct vhdr_crc16);
151 break;
152 case VERIFY_CRC7:
153 len = sizeof(struct vhdr_crc7);
154 break;
155 case VERIFY_SHA256:
156 len = sizeof(struct vhdr_sha256);
157 break;
158 case VERIFY_SHA512:
159 len = sizeof(struct vhdr_sha512);
160 break;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200161 case VERIFY_META:
162 len = sizeof(struct vhdr_meta);
163 break;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200164 case VERIFY_SHA1:
165 len = sizeof(struct vhdr_sha1);
166 break;
Jens Axboe92bf48d2011-01-14 21:20:42 +0100167 case VERIFY_PATTERN:
168 len = 0;
169 break;
Jens Axboe546dfd92007-07-30 12:23:05 +0200170 default:
171 log_err("fio: unknown verify header!\n");
172 assert(0);
173 }
174
175 return len + sizeof(struct verify_header);
Jens Axboe87677832007-07-30 12:08:14 +0200176}
177
178static inline unsigned int hdr_size(struct verify_header *hdr)
179{
Jens Axboe546dfd92007-07-30 12:23:05 +0200180 return __hdr_size(hdr->verify_type);
181}
182
183static void *hdr_priv(struct verify_header *hdr)
184{
185 void *priv = hdr;
186
187 return priv + sizeof(struct verify_header);
Jens Axboe87677832007-07-30 12:08:14 +0200188}
189
190/*
Jens Axboe936216f2010-06-18 13:44:11 +0200191 * Verify container, pass info to verify handlers and allow them to
192 * pass info back in case of error
193 */
194struct vcont {
195 /*
196 * Input
197 */
198 struct io_u *io_u;
199 unsigned int hdr_num;
Jens Axboe7d9fb452011-01-11 22:16:49 +0100200 struct thread_data *td;
Jens Axboe936216f2010-06-18 13:44:11 +0200201
202 /*
203 * Output, only valid in case of error
204 */
Jens Axboebfacf382010-06-18 14:29:52 +0200205 const char *name;
206 void *good_crc;
207 void *bad_crc;
Jens Axboe936216f2010-06-18 13:44:11 +0200208 unsigned int crc_len;
209};
210
Jens Axboec50ca7b2011-01-12 08:31:54 +0100211static void dump_buf(char *buf, unsigned int len, unsigned long long offset,
212 const char *type, struct fio_file *f)
Jens Axboe7d9fb452011-01-11 22:16:49 +0100213{
Jens Axboe6f260b32011-01-13 18:57:54 +0100214 char *ptr, fname[256];
Jens Axboe7d9fb452011-01-11 22:16:49 +0100215 int ret, fd;
216
Jens Axboe6f260b32011-01-13 18:57:54 +0100217 ptr = strdup(f->file_name);
218 strcpy(fname, basename(ptr));
Jens Axboec50ca7b2011-01-12 08:31:54 +0100219
220 sprintf(fname + strlen(fname), ".%llu.%s", offset, type);
Jens Axboe7d9fb452011-01-11 22:16:49 +0100221
222 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
223 if (fd < 0) {
224 perror("open verify buf file");
225 return;
226 }
227
228 while (len) {
229 ret = write(fd, buf, len);
230 if (!ret)
231 break;
232 else if (ret < 0) {
233 perror("write verify buf file");
234 break;
235 }
236 len -= ret;
237 buf += ret;
238 }
239
240 close(fd);
Jens Axboec50ca7b2011-01-12 08:31:54 +0100241 log_err(" %s data dumped as %s\n", type, fname);
Jens Axboe6f260b32011-01-13 18:57:54 +0100242 free(ptr);
Jens Axboe7d9fb452011-01-11 22:16:49 +0100243}
244
Jens Axboec50ca7b2011-01-12 08:31:54 +0100245/*
246 * Dump the contents of the read block and re-generate the correct data
247 * and dump that too.
248 */
Jens Axboe7d9fb452011-01-11 22:16:49 +0100249static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc)
250{
251 struct thread_data *td = vc->td;
252 struct io_u *io_u = vc->io_u;
253 unsigned long hdr_offset;
Jens Axboe7d9fb452011-01-11 22:16:49 +0100254 struct io_u dummy;
Jens Axboec50ca7b2011-01-12 08:31:54 +0100255 void *buf;
Jens Axboe7d9fb452011-01-11 22:16:49 +0100256
Steven Lang0dce9bc2011-10-25 22:41:05 +0200257 if (!td->o.verify_dump)
258 return;
259
Jens Axboec50ca7b2011-01-12 08:31:54 +0100260 /*
261 * Dump the contents we just read off disk
262 */
Jens Axboe7d9fb452011-01-11 22:16:49 +0100263 hdr_offset = vc->hdr_num * hdr->len;
264
Jens Axboec50ca7b2011-01-12 08:31:54 +0100265 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset,
266 "received", vc->io_u->file);
Jens Axboe7d9fb452011-01-11 22:16:49 +0100267
Jens Axboec50ca7b2011-01-12 08:31:54 +0100268 /*
269 * Allocate a new buf and re-generate the original data
270 */
271 buf = malloc(io_u->buflen);
Jens Axboe7d9fb452011-01-11 22:16:49 +0100272 dummy = *io_u;
Jens Axboec50ca7b2011-01-12 08:31:54 +0100273 dummy.buf = buf;
Jens Axboe4aae38a2011-01-12 08:59:12 +0100274 dummy.rand_seed = hdr->rand_seed;
Jens Axboecfbcd0d2011-01-14 14:49:20 +0100275 dummy.buf_filled_len = 0;
Josef Bacik0d81daf2013-07-08 20:35:01 -0400276 dummy.buflen = io_u->buflen;
Jens Axboe7d9fb452011-01-11 22:16:49 +0100277
Jens Axboec50ca7b2011-01-12 08:31:54 +0100278 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1);
Jens Axboe7d9fb452011-01-11 22:16:49 +0100279
Jens Axboec50ca7b2011-01-12 08:31:54 +0100280 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset,
281 "expected", vc->io_u->file);
Jens Axboe7d9fb452011-01-11 22:16:49 +0100282 free(buf);
283}
284
Jens Axboebfacf382010-06-18 14:29:52 +0200285static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
286{
287 unsigned long long offset;
288
289 offset = vc->io_u->offset;
290 offset += vc->hdr_num * hdr->len;
Jens Axboe32c17ad2010-06-18 14:32:21 +0200291 log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
292 vc->name, vc->io_u->file->file_name, offset, hdr->len);
Jens Axboebfacf382010-06-18 14:29:52 +0200293
294 if (vc->good_crc && vc->bad_crc) {
295 log_err(" Expected CRC: ");
296 hexdump(vc->good_crc, vc->crc_len);
297 log_err(" Received CRC: ");
298 hexdump(vc->bad_crc, vc->crc_len);
299 }
Jens Axboe7d9fb452011-01-11 22:16:49 +0100300
301 dump_verify_buffers(hdr, vc);
Jens Axboebfacf382010-06-18 14:29:52 +0200302}
303
Jens Axboe936216f2010-06-18 13:44:11 +0200304/*
Jens Axboed9f2caf2007-07-28 21:22:03 +0200305 * Return data area 'header_num'
306 */
Jens Axboe936216f2010-06-18 13:44:11 +0200307static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
Jens Axboed9f2caf2007-07-28 21:22:03 +0200308{
Jens Axboe936216f2010-06-18 13:44:11 +0200309 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
Jens Axboed9f2caf2007-07-28 21:22:03 +0200310}
311
Jens Axboe92bf48d2011-01-14 21:20:42 +0100312static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc)
313{
314 struct thread_data *td = vc->td;
315 struct io_u *io_u = vc->io_u;
316 char *buf, *pattern;
Jens Axboe2b13e712011-01-19 14:04:16 -0700317 unsigned int header_size = __hdr_size(td->o.verify);
Steven Lang9a2a86d2012-02-07 09:42:59 +0100318 unsigned int len, mod, i, size, pattern_size;
Jens Axboe92bf48d2011-01-14 21:20:42 +0100319
320 pattern = td->o.verify_pattern;
Steven Lang9a2a86d2012-02-07 09:42:59 +0100321 pattern_size = td->o.verify_pattern_bytes;
322 if (pattern_size <= 1)
323 pattern_size = MAX_PATTERN_SIZE;
Jens Axboe2b13e712011-01-19 14:04:16 -0700324 buf = (void *) hdr + header_size;
325 len = get_hdr_inc(td, io_u) - header_size;
Steven Lang9a2a86d2012-02-07 09:42:59 +0100326 mod = header_size % pattern_size;
Jens Axboe92bf48d2011-01-14 21:20:42 +0100327
Steven Lang9a2a86d2012-02-07 09:42:59 +0100328 for (i = 0; i < len; i += size) {
329 size = pattern_size - mod;
330 if (size > (len - i))
331 size = len - i;
332 if (memcmp(buf + i, pattern + mod, size))
Jens Axboee4ad68b2012-02-23 08:35:04 +0100333 /* Let the slow compare find the first mismatch byte. */
Steven Lang9a2a86d2012-02-07 09:42:59 +0100334 break;
335 mod = 0;
336 }
337
338 for (; i < len; i++) {
Jens Axboe92bf48d2011-01-14 21:20:42 +0100339 if (buf[i] != pattern[mod]) {
340 unsigned int bits;
341
342 bits = hweight8(buf[i] ^ pattern[mod]);
343 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
344 buf[i], pattern[mod], bits);
345 log_err("fio: bad pattern block offset %u\n", i);
346 dump_verify_buffers(hdr, vc);
347 return EILSEQ;
348 }
349 mod++;
350 if (mod == td->o.verify_pattern_bytes)
351 mod = 0;
352 }
353
354 return 0;
355}
356
357static int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc)
358{
359 struct thread_data *td = vc->td;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200360 struct vhdr_meta *vh = hdr_priv(hdr);
Jens Axboe936216f2010-06-18 13:44:11 +0200361 struct io_u *io_u = vc->io_u;
Jens Axboe92bf48d2011-01-14 21:20:42 +0100362 int ret = EILSEQ;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200363
Jens Axboebd6f78b2008-02-01 20:27:52 +0100364 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
365
Jens Axboebfacf382010-06-18 14:29:52 +0200366 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
Jens Axboe92bf48d2011-01-14 21:20:42 +0100367 ret = 0;
368
369 if (td->o.verify_pattern_bytes)
370 ret |= verify_io_u_pattern(hdr, vc);
371
372 if (!ret)
Jens Axboebfacf382010-06-18 14:29:52 +0200373 return 0;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200374
Jens Axboebfacf382010-06-18 14:29:52 +0200375 vc->name = "meta";
376 log_verify_failure(hdr, vc);
Jens Axboe92bf48d2011-01-14 21:20:42 +0100377 return ret;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200378}
379
Jens Axboe936216f2010-06-18 13:44:11 +0200380static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
Jens Axboecd14cc12007-07-30 10:59:33 +0200381{
Jens Axboe936216f2010-06-18 13:44:11 +0200382 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200383 struct vhdr_sha512 *vh = hdr_priv(hdr);
Jens Axboecd14cc12007-07-30 10:59:33 +0200384 uint8_t sha512[128];
Jens Axboe25dfa842012-02-29 10:01:34 +0100385 struct fio_sha512_ctx sha512_ctx = {
Jens Axboecd14cc12007-07-30 10:59:33 +0200386 .buf = sha512,
387 };
388
Jens Axboe936216f2010-06-18 13:44:11 +0200389 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100390
Jens Axboe25dfa842012-02-29 10:01:34 +0100391 fio_sha512_init(&sha512_ctx);
392 fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
Jens Axboecd14cc12007-07-30 10:59:33 +0200393
Jens Axboebfacf382010-06-18 14:29:52 +0200394 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
395 return 0;
Jens Axboecd14cc12007-07-30 10:59:33 +0200396
Jens Axboebfacf382010-06-18 14:29:52 +0200397 vc->name = "sha512";
398 vc->good_crc = vh->sha512;
399 vc->bad_crc = sha512_ctx.buf;
400 vc->crc_len = sizeof(vh->sha512);
401 log_verify_failure(hdr, vc);
402 return EILSEQ;
Jens Axboecd14cc12007-07-30 10:59:33 +0200403}
404
Jens Axboe936216f2010-06-18 13:44:11 +0200405static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
Jens Axboecd14cc12007-07-30 10:59:33 +0200406{
Jens Axboe936216f2010-06-18 13:44:11 +0200407 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200408 struct vhdr_sha256 *vh = hdr_priv(hdr);
Jens Axboebc77f562010-02-23 10:36:21 +0100409 uint8_t sha256[64];
Jens Axboe25dfa842012-02-29 10:01:34 +0100410 struct fio_sha256_ctx sha256_ctx = {
Jens Axboecd14cc12007-07-30 10:59:33 +0200411 .buf = sha256,
412 };
413
Jens Axboe936216f2010-06-18 13:44:11 +0200414 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100415
Jens Axboe25dfa842012-02-29 10:01:34 +0100416 fio_sha256_init(&sha256_ctx);
417 fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
Jens Axboecd14cc12007-07-30 10:59:33 +0200418
Jens Axboebfacf382010-06-18 14:29:52 +0200419 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
420 return 0;
Jens Axboecd14cc12007-07-30 10:59:33 +0200421
Jens Axboebfacf382010-06-18 14:29:52 +0200422 vc->name = "sha256";
423 vc->good_crc = vh->sha256;
424 vc->bad_crc = sha256_ctx.buf;
425 vc->crc_len = sizeof(vh->sha256);
426 log_verify_failure(hdr, vc);
427 return EILSEQ;
Jens Axboecd14cc12007-07-30 10:59:33 +0200428}
429
Jens Axboe936216f2010-06-18 13:44:11 +0200430static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
Jens Axboe7c353ce2009-08-09 22:40:33 +0200431{
Jens Axboe936216f2010-06-18 13:44:11 +0200432 void *p = io_u_verify_off(hdr, vc);
Jens Axboe7c353ce2009-08-09 22:40:33 +0200433 struct vhdr_sha1 *vh = hdr_priv(hdr);
434 uint32_t sha1[5];
Jens Axboe25dfa842012-02-29 10:01:34 +0100435 struct fio_sha1_ctx sha1_ctx = {
Jens Axboe7c353ce2009-08-09 22:40:33 +0200436 .H = sha1,
437 };
438
Jens Axboe936216f2010-06-18 13:44:11 +0200439 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboe7c353ce2009-08-09 22:40:33 +0200440
Jens Axboe25dfa842012-02-29 10:01:34 +0100441 fio_sha1_init(&sha1_ctx);
442 fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
Jens Axboe7c353ce2009-08-09 22:40:33 +0200443
Jens Axboebfacf382010-06-18 14:29:52 +0200444 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
445 return 0;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200446
Jens Axboebfacf382010-06-18 14:29:52 +0200447 vc->name = "sha1";
448 vc->good_crc = vh->sha1;
449 vc->bad_crc = sha1_ctx.H;
450 vc->crc_len = sizeof(vh->sha1);
451 log_verify_failure(hdr, vc);
452 return EILSEQ;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200453}
454
Jens Axboe936216f2010-06-18 13:44:11 +0200455static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
Jens Axboe1e154bd2007-07-27 09:52:40 +0200456{
Jens Axboe936216f2010-06-18 13:44:11 +0200457 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200458 struct vhdr_crc7 *vh = hdr_priv(hdr);
Jens Axboe1e154bd2007-07-27 09:52:40 +0200459 unsigned char c;
460
Jens Axboe936216f2010-06-18 13:44:11 +0200461 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100462
Jens Axboe25dfa842012-02-29 10:01:34 +0100463 c = fio_crc7(p, hdr->len - hdr_size(hdr));
Jens Axboe1e154bd2007-07-27 09:52:40 +0200464
Jens Axboebfacf382010-06-18 14:29:52 +0200465 if (c == vh->crc7)
466 return 0;
Jens Axboe1e154bd2007-07-27 09:52:40 +0200467
Jens Axboebfacf382010-06-18 14:29:52 +0200468 vc->name = "crc7";
469 vc->good_crc = &vh->crc7;
470 vc->bad_crc = &c;
471 vc->crc_len = 1;
472 log_verify_failure(hdr, vc);
473 return EILSEQ;
Jens Axboe1e154bd2007-07-27 09:52:40 +0200474}
475
Jens Axboe936216f2010-06-18 13:44:11 +0200476static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
Jens Axboe969f7ed2007-07-27 09:07:17 +0200477{
Jens Axboe936216f2010-06-18 13:44:11 +0200478 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200479 struct vhdr_crc16 *vh = hdr_priv(hdr);
Jens Axboe969f7ed2007-07-27 09:07:17 +0200480 unsigned short c;
481
Jens Axboe936216f2010-06-18 13:44:11 +0200482 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100483
Jens Axboe25dfa842012-02-29 10:01:34 +0100484 c = fio_crc16(p, hdr->len - hdr_size(hdr));
Jens Axboe969f7ed2007-07-27 09:07:17 +0200485
Jens Axboebfacf382010-06-18 14:29:52 +0200486 if (c == vh->crc16)
487 return 0;
Jens Axboe969f7ed2007-07-27 09:07:17 +0200488
Jens Axboebfacf382010-06-18 14:29:52 +0200489 vc->name = "crc16";
490 vc->good_crc = &vh->crc16;
491 vc->bad_crc = &c;
492 vc->crc_len = 2;
493 log_verify_failure(hdr, vc);
494 return EILSEQ;
Jens Axboe969f7ed2007-07-27 09:07:17 +0200495}
496
Jens Axboe936216f2010-06-18 13:44:11 +0200497static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
Jens Axboed77a7af2007-07-27 15:35:06 +0200498{
Jens Axboe936216f2010-06-18 13:44:11 +0200499 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200500 struct vhdr_crc64 *vh = hdr_priv(hdr);
Jens Axboed77a7af2007-07-27 15:35:06 +0200501 unsigned long long c;
502
Jens Axboe936216f2010-06-18 13:44:11 +0200503 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100504
Jens Axboe25dfa842012-02-29 10:01:34 +0100505 c = fio_crc64(p, hdr->len - hdr_size(hdr));
Jens Axboed77a7af2007-07-27 15:35:06 +0200506
Jens Axboebfacf382010-06-18 14:29:52 +0200507 if (c == vh->crc64)
508 return 0;
Jens Axboed77a7af2007-07-27 15:35:06 +0200509
Jens Axboebfacf382010-06-18 14:29:52 +0200510 vc->name = "crc64";
511 vc->good_crc = &vh->crc64;
512 vc->bad_crc = &c;
513 vc->crc_len = 8;
514 log_verify_failure(hdr, vc);
515 return EILSEQ;
Jens Axboed77a7af2007-07-27 15:35:06 +0200516}
517
Jens Axboe936216f2010-06-18 13:44:11 +0200518static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
Jens Axboee29d1b72006-10-18 15:43:15 +0200519{
Jens Axboe936216f2010-06-18 13:44:11 +0200520 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200521 struct vhdr_crc32 *vh = hdr_priv(hdr);
522 uint32_t c;
Jens Axboee29d1b72006-10-18 15:43:15 +0200523
Jens Axboe936216f2010-06-18 13:44:11 +0200524 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100525
Jens Axboe25dfa842012-02-29 10:01:34 +0100526 c = fio_crc32(p, hdr->len - hdr_size(hdr));
Jens Axboee29d1b72006-10-18 15:43:15 +0200527
Jens Axboebfacf382010-06-18 14:29:52 +0200528 if (c == vh->crc32)
529 return 0;
Jens Axboee29d1b72006-10-18 15:43:15 +0200530
Jens Axboebfacf382010-06-18 14:29:52 +0200531 vc->name = "crc32";
532 vc->good_crc = &vh->crc32;
533 vc->bad_crc = &c;
534 vc->crc_len = 4;
535 log_verify_failure(hdr, vc);
536 return EILSEQ;
Jens Axboee29d1b72006-10-18 15:43:15 +0200537}
538
Jens Axboe936216f2010-06-18 13:44:11 +0200539static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
Jens Axboebac39e02008-06-11 20:46:19 +0200540{
Jens Axboe936216f2010-06-18 13:44:11 +0200541 void *p = io_u_verify_off(hdr, vc);
Jens Axboebac39e02008-06-11 20:46:19 +0200542 struct vhdr_crc32 *vh = hdr_priv(hdr);
543 uint32_t c;
544
Jens Axboe936216f2010-06-18 13:44:11 +0200545 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebac39e02008-06-11 20:46:19 +0200546
Jens Axboe25dfa842012-02-29 10:01:34 +0100547 c = fio_crc32c(p, hdr->len - hdr_size(hdr));
Jens Axboebac39e02008-06-11 20:46:19 +0200548
Jens Axboebfacf382010-06-18 14:29:52 +0200549 if (c == vh->crc32)
550 return 0;
Jens Axboebac39e02008-06-11 20:46:19 +0200551
Jens Axboebfacf382010-06-18 14:29:52 +0200552 vc->name = "crc32c";
553 vc->good_crc = &vh->crc32;
554 vc->bad_crc = &c;
555 vc->crc_len = 4;
556 log_verify_failure(hdr, vc);
557 return EILSEQ;
Jens Axboebac39e02008-06-11 20:46:19 +0200558}
559
Jens Axboe936216f2010-06-18 13:44:11 +0200560static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
Jens Axboee29d1b72006-10-18 15:43:15 +0200561{
Jens Axboe936216f2010-06-18 13:44:11 +0200562 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200563 struct vhdr_md5 *vh = hdr_priv(hdr);
Jens Axboe8c432322007-07-27 13:16:24 +0200564 uint32_t hash[MD5_HASH_WORDS];
Jens Axboe25dfa842012-02-29 10:01:34 +0100565 struct fio_md5_ctx md5_ctx = {
Jens Axboe8c432322007-07-27 13:16:24 +0200566 .hash = hash,
567 };
Jens Axboee29d1b72006-10-18 15:43:15 +0200568
Jens Axboe936216f2010-06-18 13:44:11 +0200569 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100570
Jens Axboe25dfa842012-02-29 10:01:34 +0100571 fio_md5_init(&md5_ctx);
572 fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
Jens Axboee29d1b72006-10-18 15:43:15 +0200573
Jens Axboebfacf382010-06-18 14:29:52 +0200574 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
575 return 0;
Jens Axboee29d1b72006-10-18 15:43:15 +0200576
Jens Axboebfacf382010-06-18 14:29:52 +0200577 vc->name = "md5";
578 vc->good_crc = vh->md5_digest;
579 vc->bad_crc = md5_ctx.hash;
580 vc->crc_len = sizeof(hash);
581 log_verify_failure(hdr, vc);
582 return EILSEQ;
Jens Axboee29d1b72006-10-18 15:43:15 +0200583}
584
Jens Axboee8462bd2009-07-06 12:59:04 +0200585/*
586 * Push IO verification to a separate thread
587 */
588int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
589{
590 if (io_u->file)
591 put_file_log(td, io_u->file);
592
Jens Axboee8462bd2009-07-06 12:59:04 +0200593 pthread_mutex_lock(&td->io_u_lock);
Steven Langd7ee2a72011-10-26 09:46:50 +0200594
Radha Ramachandran0c412142009-11-03 21:45:31 +0100595 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
596 td->cur_depth--;
597 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH;
598 }
Jens Axboe2ae0b202013-05-28 14:16:55 +0200599 flist_add_tail(&io_u->verify_list, &td->verify_list);
Jens Axboe2ecc1b52009-11-04 20:58:09 +0100600 io_u->flags |= IO_U_F_FREE_DEF;
Jens Axboee8462bd2009-07-06 12:59:04 +0200601 pthread_mutex_unlock(&td->io_u_lock);
602
603 pthread_cond_signal(&td->verify_cond);
Jens Axboee8462bd2009-07-06 12:59:04 +0200604 return 0;
605}
606
Jens Axboe0d29de82010-09-01 13:54:15 +0200607static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u)
608{
609 static char zero_buf[1024];
610 unsigned int this_len, len;
611 int ret = 0;
612 void *p;
613
614 if (!td->o.trim_zero)
615 return 0;
616
617 len = io_u->buflen;
618 p = io_u->buf;
619 do {
620 this_len = sizeof(zero_buf);
621 if (this_len > len)
622 this_len = len;
623 if (memcmp(p, zero_buf, this_len)) {
624 ret = EILSEQ;
625 break;
626 }
627 len -= this_len;
628 p += this_len;
629 } while (len);
630
631 if (!ret)
632 return 0;
633
Jens Axboea917a8b2010-09-02 13:23:20 +0200634 log_err("trim: verify failed at file %s offset %llu, length %lu"
635 ", block offset %lu\n",
636 io_u->file->file_name, io_u->offset, io_u->buflen,
Jens Axboe2f681242010-10-21 08:15:59 +0200637 (unsigned long) (p - io_u->buf));
Jens Axboe0d29de82010-09-01 13:54:15 +0200638 return ret;
639}
640
Jens Axboe0ae2c6e2012-03-06 17:46:44 +0100641static int verify_header(struct io_u *io_u, struct verify_header *hdr)
Jens Axboef65d1c22012-02-22 20:11:57 +0100642{
643 void *p = hdr;
644 uint32_t crc;
645
Jens Axboeae38c0d2012-02-23 10:31:07 +0100646 if (hdr->magic != FIO_HDR_MAGIC)
647 return 0;
Jens Axboe0ae2c6e2012-03-06 17:46:44 +0100648 if (hdr->len > io_u->buflen) {
649 log_err("fio: verify header exceeds buffer length (%u > %lu)\n", hdr->len, io_u->buflen);
650 return 0;
651 }
Jens Axboeae38c0d2012-02-23 10:31:07 +0100652
Jens Axboe25dfa842012-02-29 10:01:34 +0100653 crc = fio_crc32c(p, offsetof(struct verify_header, crc32));
Jens Axboef65d1c22012-02-22 20:11:57 +0100654 if (crc == hdr->crc32)
655 return 1;
656
657 log_err("fio: verify header crc %x, calculated %x\n", hdr->crc32, crc);
658 return 0;
659}
660
Jens Axboe36690c92007-03-26 10:23:34 +0200661int verify_io_u(struct thread_data *td, struct io_u *io_u)
Jens Axboee29d1b72006-10-18 15:43:15 +0200662{
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200663 struct verify_header *hdr;
Jens Axboe2b13e712011-01-19 14:04:16 -0700664 unsigned int header_size, hdr_inc, hdr_num = 0;
Jens Axboe95646102007-07-28 21:17:50 +0200665 void *p;
Jens Axboee29d1b72006-10-18 15:43:15 +0200666 int ret;
667
Shawn Lewis1dcc0492007-07-27 08:02:45 +0200668 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
Jens Axboe36690c92007-03-26 10:23:34 +0200669 return 0;
Jens Axboe0d29de82010-09-01 13:54:15 +0200670 if (io_u->flags & IO_U_F_TRIMMED) {
671 ret = verify_trimmed_io_u(td, io_u);
672 goto done;
673 }
Jens Axboe36690c92007-03-26 10:23:34 +0200674
Jens Axboecfbcd0d2011-01-14 14:49:20 +0100675 hdr_inc = get_hdr_inc(td, io_u);
Jens Axboee29d1b72006-10-18 15:43:15 +0200676
Jens Axboea12a3b42007-08-09 10:20:54 +0200677 ret = 0;
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100678 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
679 p += hdr_inc, hdr_num++) {
Jens Axboebfacf382010-06-18 14:29:52 +0200680 struct vcont vc = {
681 .io_u = io_u,
682 .hdr_num = hdr_num,
Jens Axboe7d9fb452011-01-11 22:16:49 +0100683 .td = td,
Jens Axboebfacf382010-06-18 14:29:52 +0200684 };
Jens Axboef00b2102012-11-30 09:39:02 +0100685 unsigned int verify_type;
Jens Axboe936216f2010-06-18 13:44:11 +0200686
Jens Axboef3e6cb92010-06-18 14:48:43 +0200687 if (ret && td->o.verify_fatal)
Jens Axboea12a3b42007-08-09 10:20:54 +0200688 break;
Jens Axboef3e6cb92010-06-18 14:48:43 +0200689
Jens Axboe2b13e712011-01-19 14:04:16 -0700690 header_size = __hdr_size(td->o.verify);
Jens Axboea59e1702007-07-30 08:53:27 +0200691 if (td->o.verify_offset)
Jens Axboe2b13e712011-01-19 14:04:16 -0700692 memswp(p, p + td->o.verify_offset, header_size);
Jens Axboe95646102007-07-28 21:17:50 +0200693 hdr = p;
Jens Axboee29d1b72006-10-18 15:43:15 +0200694
Jens Axboe0ae2c6e2012-03-06 17:46:44 +0100695 if (!verify_header(io_u, hdr)) {
Jens Axboeae38c0d2012-02-23 10:31:07 +0100696 log_err("verify: bad magic header %x, wanted %x at "
697 "file %s offset %llu, length %u\n",
Jens Axboed3a173a2012-02-23 08:23:18 +0100698 hdr->magic, FIO_HDR_MAGIC,
Jens Axboec9b44032010-06-18 14:46:06 +0200699 io_u->file->file_name,
700 io_u->offset + hdr_num * hdr->len, hdr->len);
Jens Axboe9fd18962009-05-19 10:35:38 +0200701 return EILSEQ;
Jens Axboe3f199b02007-08-09 10:16:31 +0200702 }
703
Jens Axboef00b2102012-11-30 09:39:02 +0100704 if (td->o.verify != VERIFY_NONE)
705 verify_type = td->o.verify;
706 else
707 verify_type = hdr->verify_type;
708
709 switch (verify_type) {
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200710 case VERIFY_MD5:
Jens Axboe936216f2010-06-18 13:44:11 +0200711 ret = verify_io_u_md5(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200712 break;
713 case VERIFY_CRC64:
Jens Axboe936216f2010-06-18 13:44:11 +0200714 ret = verify_io_u_crc64(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200715 break;
Jens Axboebac39e02008-06-11 20:46:19 +0200716 case VERIFY_CRC32C:
Jens Axboe38455912008-08-04 15:35:26 +0200717 case VERIFY_CRC32C_INTEL:
Jens Axboe936216f2010-06-18 13:44:11 +0200718 ret = verify_io_u_crc32c(hdr, &vc);
Jens Axboebac39e02008-06-11 20:46:19 +0200719 break;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200720 case VERIFY_CRC32:
Jens Axboe936216f2010-06-18 13:44:11 +0200721 ret = verify_io_u_crc32(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200722 break;
723 case VERIFY_CRC16:
Jens Axboe936216f2010-06-18 13:44:11 +0200724 ret = verify_io_u_crc16(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200725 break;
726 case VERIFY_CRC7:
Jens Axboe936216f2010-06-18 13:44:11 +0200727 ret = verify_io_u_crc7(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200728 break;
Jens Axboecd14cc12007-07-30 10:59:33 +0200729 case VERIFY_SHA256:
Jens Axboe936216f2010-06-18 13:44:11 +0200730 ret = verify_io_u_sha256(hdr, &vc);
Jens Axboecd14cc12007-07-30 10:59:33 +0200731 break;
732 case VERIFY_SHA512:
Jens Axboe936216f2010-06-18 13:44:11 +0200733 ret = verify_io_u_sha512(hdr, &vc);
Jens Axboecd14cc12007-07-30 10:59:33 +0200734 break;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200735 case VERIFY_META:
Jens Axboe92bf48d2011-01-14 21:20:42 +0100736 ret = verify_io_u_meta(hdr, &vc);
Shawn Lewis7437ee82007-08-02 21:05:58 +0200737 break;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200738 case VERIFY_SHA1:
Jens Axboe936216f2010-06-18 13:44:11 +0200739 ret = verify_io_u_sha1(hdr, &vc);
Jens Axboe7c353ce2009-08-09 22:40:33 +0200740 break;
Jens Axboe92bf48d2011-01-14 21:20:42 +0100741 case VERIFY_PATTERN:
742 ret = verify_io_u_pattern(hdr, &vc);
743 break;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200744 default:
745 log_err("Bad verify type %u\n", hdr->verify_type);
Jens Axboed16d4e02007-09-06 16:16:44 +0200746 ret = EINVAL;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200747 }
Jens Axboef00b2102012-11-30 09:39:02 +0100748
749 if (ret && verify_type != hdr->verify_type)
750 log_err("fio: verify type mismatch (%u media, %u given)\n",
751 hdr->verify_type, verify_type);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200752 }
Jens Axboea7dfe862007-03-12 10:05:08 +0100753
Jens Axboe0d29de82010-09-01 13:54:15 +0200754done:
Jens Axboef3e6cb92010-06-18 14:48:43 +0200755 if (ret && td->o.verify_fatal)
756 td->terminate = 1;
757
Jens Axboea12a3b42007-08-09 10:20:54 +0200758 return ret;
Jens Axboee29d1b72006-10-18 15:43:15 +0200759}
760
Shawn Lewis7437ee82007-08-02 21:05:58 +0200761static void fill_meta(struct verify_header *hdr, struct thread_data *td,
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100762 struct io_u *io_u, unsigned int header_num)
Shawn Lewis7437ee82007-08-02 21:05:58 +0200763{
764 struct vhdr_meta *vh = hdr_priv(hdr);
765
766 vh->thread = td->thread_number;
767
768 vh->time_sec = io_u->start_time.tv_sec;
769 vh->time_usec = io_u->start_time.tv_usec;
770
771 vh->numberio = td->io_issues[DDIR_WRITE];
772
773 vh->offset = io_u->offset + header_num * td->o.verify_interval;
774}
775
Jens Axboecd14cc12007-07-30 10:59:33 +0200776static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
777{
Jens Axboe546dfd92007-07-30 12:23:05 +0200778 struct vhdr_sha512 *vh = hdr_priv(hdr);
Jens Axboe25dfa842012-02-29 10:01:34 +0100779 struct fio_sha512_ctx sha512_ctx = {
Jens Axboe546dfd92007-07-30 12:23:05 +0200780 .buf = vh->sha512,
Jens Axboecd14cc12007-07-30 10:59:33 +0200781 };
782
Jens Axboe25dfa842012-02-29 10:01:34 +0100783 fio_sha512_init(&sha512_ctx);
784 fio_sha512_update(&sha512_ctx, p, len);
Jens Axboecd14cc12007-07-30 10:59:33 +0200785}
786
787static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
788{
Jens Axboe546dfd92007-07-30 12:23:05 +0200789 struct vhdr_sha256 *vh = hdr_priv(hdr);
Jens Axboe25dfa842012-02-29 10:01:34 +0100790 struct fio_sha256_ctx sha256_ctx = {
Jens Axboe546dfd92007-07-30 12:23:05 +0200791 .buf = vh->sha256,
Jens Axboecd14cc12007-07-30 10:59:33 +0200792 };
793
Jens Axboe25dfa842012-02-29 10:01:34 +0100794 fio_sha256_init(&sha256_ctx);
795 fio_sha256_update(&sha256_ctx, p, len);
Jens Axboecd14cc12007-07-30 10:59:33 +0200796}
797
Jens Axboe7c353ce2009-08-09 22:40:33 +0200798static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
799{
800 struct vhdr_sha1 *vh = hdr_priv(hdr);
Jens Axboe25dfa842012-02-29 10:01:34 +0100801 struct fio_sha1_ctx sha1_ctx = {
Jens Axboe7c353ce2009-08-09 22:40:33 +0200802 .H = vh->sha1,
803 };
804
Jens Axboe25dfa842012-02-29 10:01:34 +0100805 fio_sha1_init(&sha1_ctx);
806 fio_sha1_update(&sha1_ctx, p, len);
Jens Axboe7c353ce2009-08-09 22:40:33 +0200807}
808
Jens Axboe1e154bd2007-07-27 09:52:40 +0200809static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
810{
Jens Axboe546dfd92007-07-30 12:23:05 +0200811 struct vhdr_crc7 *vh = hdr_priv(hdr);
812
Jens Axboe25dfa842012-02-29 10:01:34 +0100813 vh->crc7 = fio_crc7(p, len);
Jens Axboe1e154bd2007-07-27 09:52:40 +0200814}
815
Jens Axboe969f7ed2007-07-27 09:07:17 +0200816static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
817{
Jens Axboe546dfd92007-07-30 12:23:05 +0200818 struct vhdr_crc16 *vh = hdr_priv(hdr);
819
Jens Axboe25dfa842012-02-29 10:01:34 +0100820 vh->crc16 = fio_crc16(p, len);
Jens Axboe969f7ed2007-07-27 09:07:17 +0200821}
822
Jens Axboee29d1b72006-10-18 15:43:15 +0200823static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
824{
Jens Axboe546dfd92007-07-30 12:23:05 +0200825 struct vhdr_crc32 *vh = hdr_priv(hdr);
826
Jens Axboe25dfa842012-02-29 10:01:34 +0100827 vh->crc32 = fio_crc32(p, len);
Jens Axboee29d1b72006-10-18 15:43:15 +0200828}
829
Jens Axboebac39e02008-06-11 20:46:19 +0200830static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
831{
832 struct vhdr_crc32 *vh = hdr_priv(hdr);
833
Jens Axboe25dfa842012-02-29 10:01:34 +0100834 vh->crc32 = fio_crc32c(p, len);
Jens Axboebac39e02008-06-11 20:46:19 +0200835}
836
Jens Axboed77a7af2007-07-27 15:35:06 +0200837static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
838{
Jens Axboe546dfd92007-07-30 12:23:05 +0200839 struct vhdr_crc64 *vh = hdr_priv(hdr);
840
Jens Axboe25dfa842012-02-29 10:01:34 +0100841 vh->crc64 = fio_crc64(p, len);
Jens Axboed77a7af2007-07-27 15:35:06 +0200842}
843
Jens Axboee29d1b72006-10-18 15:43:15 +0200844static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
845{
Jens Axboe546dfd92007-07-30 12:23:05 +0200846 struct vhdr_md5 *vh = hdr_priv(hdr);
Jens Axboe25dfa842012-02-29 10:01:34 +0100847 struct fio_md5_ctx md5_ctx = {
Jens Axboe546dfd92007-07-30 12:23:05 +0200848 .hash = (uint32_t *) vh->md5_digest,
Jens Axboe8c432322007-07-27 13:16:24 +0200849 };
Jens Axboee29d1b72006-10-18 15:43:15 +0200850
Jens Axboe25dfa842012-02-29 10:01:34 +0100851 fio_md5_init(&md5_ctx);
852 fio_md5_update(&md5_ctx, p, len);
Jens Axboee29d1b72006-10-18 15:43:15 +0200853}
854
Jens Axboe7d9fb452011-01-11 22:16:49 +0100855static void populate_hdr(struct thread_data *td, struct io_u *io_u,
856 struct verify_header *hdr, unsigned int header_num,
857 unsigned int header_len)
858{
859 unsigned int data_len;
860 void *data, *p;
861
862 p = (void *) hdr;
863
Jens Axboed3a173a2012-02-23 08:23:18 +0100864 hdr->magic = FIO_HDR_MAGIC;
Jens Axboe7d9fb452011-01-11 22:16:49 +0100865 hdr->verify_type = td->o.verify;
Jens Axboed3a173a2012-02-23 08:23:18 +0100866 hdr->len = header_len;
Jens Axboe7d9fb452011-01-11 22:16:49 +0100867 hdr->rand_seed = io_u->rand_seed;
Jens Axboe25dfa842012-02-29 10:01:34 +0100868 hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32));
Jens Axboef65d1c22012-02-22 20:11:57 +0100869
Jens Axboe7d9fb452011-01-11 22:16:49 +0100870 data_len = header_len - hdr_size(hdr);
871
872 data = p + hdr_size(hdr);
873 switch (td->o.verify) {
874 case VERIFY_MD5:
875 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
876 io_u, hdr->len);
877 fill_md5(hdr, data, data_len);
878 break;
879 case VERIFY_CRC64:
880 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
881 io_u, hdr->len);
882 fill_crc64(hdr, data, data_len);
883 break;
884 case VERIFY_CRC32C:
885 case VERIFY_CRC32C_INTEL:
886 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
887 io_u, hdr->len);
888 fill_crc32c(hdr, data, data_len);
889 break;
890 case VERIFY_CRC32:
891 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
892 io_u, hdr->len);
893 fill_crc32(hdr, data, data_len);
894 break;
895 case VERIFY_CRC16:
896 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
897 io_u, hdr->len);
898 fill_crc16(hdr, data, data_len);
899 break;
900 case VERIFY_CRC7:
901 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
902 io_u, hdr->len);
903 fill_crc7(hdr, data, data_len);
904 break;
905 case VERIFY_SHA256:
906 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
907 io_u, hdr->len);
908 fill_sha256(hdr, data, data_len);
909 break;
910 case VERIFY_SHA512:
911 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
912 io_u, hdr->len);
913 fill_sha512(hdr, data, data_len);
914 break;
915 case VERIFY_META:
916 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
917 io_u, hdr->len);
918 fill_meta(hdr, td, io_u, header_num);
919 break;
920 case VERIFY_SHA1:
921 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
922 io_u, hdr->len);
923 fill_sha1(hdr, data, data_len);
924 break;
Jens Axboe92bf48d2011-01-14 21:20:42 +0100925 case VERIFY_PATTERN:
926 /* nothing to do here */
927 break;
Jens Axboe7d9fb452011-01-11 22:16:49 +0100928 default:
929 log_err("fio: bad verify type: %d\n", td->o.verify);
930 assert(0);
931 }
932 if (td->o.verify_offset)
933 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
934}
935
Jens Axboee29d1b72006-10-18 15:43:15 +0200936/*
937 * fill body of io_u->buf with random data and add a header with the
Jens Axboec50ca7b2011-01-12 08:31:54 +0100938 * checksum of choice
Jens Axboee29d1b72006-10-18 15:43:15 +0200939 */
940void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
941{
Jens Axboe9cc3d152007-03-26 10:32:30 +0200942 if (td->o.verify == VERIFY_NULL)
943 return;
944
Jens Axboec50ca7b2011-01-12 08:31:54 +0100945 fill_pattern_headers(td, io_u, 0, 0);
Jens Axboee29d1b72006-10-18 15:43:15 +0200946}
947
948int get_next_verify(struct thread_data *td, struct io_u *io_u)
949{
Jens Axboe8de8f042007-03-27 10:36:12 +0200950 struct io_piece *ipo = NULL;
Jens Axboee29d1b72006-10-18 15:43:15 +0200951
Jens Axboed2d7fa52007-02-19 13:21:35 +0100952 /*
953 * this io_u is from a requeue, we already filled the offsets
954 */
955 if (io_u->file)
956 return 0;
957
Jens Axboe8de8f042007-03-27 10:36:12 +0200958 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
959 struct rb_node *n = rb_first(&td->io_hist_tree);
960
Jens Axboe4b878982007-03-26 09:32:22 +0200961 ipo = rb_entry(n, struct io_piece, rb_node);
Jens Axboe4b878982007-03-26 09:32:22 +0200962 rb_erase(n, &td->io_hist_tree);
Jens Axboea917a8b2010-09-02 13:23:20 +0200963 assert(ipo->flags & IP_F_ONRB);
964 ipo->flags &= ~IP_F_ONRB;
Jens Axboe01743ee2008-06-02 12:19:19 +0200965 } else if (!flist_empty(&td->io_hist_list)) {
966 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
967 flist_del(&ipo->list);
Jens Axboea917a8b2010-09-02 13:23:20 +0200968 assert(ipo->flags & IP_F_ONLIST);
969 ipo->flags &= ~IP_F_ONLIST;
Jens Axboe8de8f042007-03-27 10:36:12 +0200970 }
Jens Axboee29d1b72006-10-18 15:43:15 +0200971
Jens Axboe8de8f042007-03-27 10:36:12 +0200972 if (ipo) {
Jens Axboe0d29de82010-09-01 13:54:15 +0200973 td->io_hist_len--;
974
Jens Axboee29d1b72006-10-18 15:43:15 +0200975 io_u->offset = ipo->offset;
976 io_u->buflen = ipo->len;
Jens Axboe36167d82007-02-18 05:41:31 +0100977 io_u->file = ipo->file;
Jens Axboe82af2a72012-03-13 13:45:58 +0100978 io_u->flags |= IO_U_F_VER_LIST;
Jens Axboe97af62c2007-05-22 11:12:13 +0200979
Jens Axboe0d29de82010-09-01 13:54:15 +0200980 if (ipo->flags & IP_F_TRIMMED)
981 io_u->flags |= IO_U_F_TRIMMED;
982
Jens Axboed6aed792009-06-03 08:41:15 +0200983 if (!fio_file_open(io_u->file)) {
Jens Axboe97af62c2007-05-22 11:12:13 +0200984 int r = td_io_open_file(td, io_u->file);
985
Jens Axboebd6f78b2008-02-01 20:27:52 +0100986 if (r) {
987 dprint(FD_VERIFY, "failed file %s open\n",
988 io_u->file->file_name);
Jens Axboe97af62c2007-05-22 11:12:13 +0200989 return 1;
Jens Axboebd6f78b2008-02-01 20:27:52 +0100990 }
Jens Axboe97af62c2007-05-22 11:12:13 +0200991 }
992
993 get_file(ipo->file);
Jens Axboed6aed792009-06-03 08:41:15 +0200994 assert(fio_file_open(io_u->file));
Jens Axboee29d1b72006-10-18 15:43:15 +0200995 io_u->ddir = DDIR_READ;
Jens Axboe36167d82007-02-18 05:41:31 +0100996 io_u->xfer_buf = io_u->buf;
997 io_u->xfer_buflen = io_u->buflen;
Jens Axboe0d29de82010-09-01 13:54:15 +0200998
999 remove_trim_entry(td, ipo);
Jens Axboee29d1b72006-10-18 15:43:15 +02001000 free(ipo);
Jens Axboebd6f78b2008-02-01 20:27:52 +01001001 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
Jens Axboee29d1b72006-10-18 15:43:15 +02001002 return 0;
1003 }
1004
Jens Axboebd6f78b2008-02-01 20:27:52 +01001005 dprint(FD_VERIFY, "get_next_verify: empty\n");
Jens Axboee29d1b72006-10-18 15:43:15 +02001006 return 1;
1007}
Jens Axboee8462bd2009-07-06 12:59:04 +02001008
Jens Axboedc5bfbb2013-04-10 22:23:40 +02001009void fio_verify_init(struct thread_data *td)
1010{
1011 if (td->o.verify == VERIFY_CRC32C_INTEL ||
1012 td->o.verify == VERIFY_CRC32C) {
1013 crc32c_intel_probe();
1014 }
1015}
1016
Jens Axboee8462bd2009-07-06 12:59:04 +02001017static void *verify_async_thread(void *data)
1018{
1019 struct thread_data *td = data;
1020 struct io_u *io_u;
1021 int ret = 0;
1022
1023 if (td->o.verify_cpumask_set &&
1024 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
1025 log_err("fio: failed setting verify thread affinity\n");
1026 goto done;
1027 }
1028
1029 do {
Jens Axboee53ab272009-07-06 13:43:46 +02001030 FLIST_HEAD(list);
1031
Jens Axboee8462bd2009-07-06 12:59:04 +02001032 read_barrier();
1033 if (td->verify_thread_exit)
1034 break;
1035
1036 pthread_mutex_lock(&td->io_u_lock);
1037
1038 while (flist_empty(&td->verify_list) &&
1039 !td->verify_thread_exit) {
Jens Axboeb36e2982009-07-06 14:12:52 +02001040 ret = pthread_cond_wait(&td->verify_cond,
1041 &td->io_u_lock);
Jens Axboee8462bd2009-07-06 12:59:04 +02001042 if (ret) {
1043 pthread_mutex_unlock(&td->io_u_lock);
1044 break;
1045 }
1046 }
1047
Jens Axboee53ab272009-07-06 13:43:46 +02001048 flist_splice_init(&td->verify_list, &list);
Jens Axboee8462bd2009-07-06 12:59:04 +02001049 pthread_mutex_unlock(&td->io_u_lock);
1050
Jens Axboee53ab272009-07-06 13:43:46 +02001051 if (flist_empty(&list))
1052 continue;
1053
1054 while (!flist_empty(&list)) {
Jens Axboe2ae0b202013-05-28 14:16:55 +02001055 io_u = flist_entry(list.next, struct io_u, verify_list);
1056 flist_del(&io_u->verify_list);
Jens Axboee53ab272009-07-06 13:43:46 +02001057
Jens Axboed561f2a2009-07-06 13:51:05 +02001058 ret = verify_io_u(td, io_u);
Jens Axboee53ab272009-07-06 13:43:46 +02001059 put_io_u(td, io_u);
Jens Axboed561f2a2009-07-06 13:51:05 +02001060 if (!ret)
1061 continue;
Dmitry Monakhov8b28bd42012-09-23 15:46:09 +04001062 if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) {
Jens Axboed561f2a2009-07-06 13:51:05 +02001063 update_error_count(td, ret);
1064 td_clear_error(td);
1065 ret = 0;
1066 }
Jens Axboee53ab272009-07-06 13:43:46 +02001067 }
Jens Axboee8462bd2009-07-06 12:59:04 +02001068 } while (!ret);
1069
Jens Axboed561f2a2009-07-06 13:51:05 +02001070 if (ret) {
1071 td_verror(td, ret, "async_verify");
Jens Axboef3e6cb92010-06-18 14:48:43 +02001072 if (td->o.verify_fatal)
1073 td->terminate = 1;
Jens Axboed561f2a2009-07-06 13:51:05 +02001074 }
1075
Jens Axboee8462bd2009-07-06 12:59:04 +02001076done:
1077 pthread_mutex_lock(&td->io_u_lock);
1078 td->nr_verify_threads--;
1079 pthread_mutex_unlock(&td->io_u_lock);
1080
1081 pthread_cond_signal(&td->free_cond);
1082 return NULL;
1083}
1084
1085int verify_async_init(struct thread_data *td)
1086{
1087 int i, ret;
bart Van Assche304a47c2010-08-01 21:31:26 +02001088 pthread_attr_t attr;
1089
1090 pthread_attr_init(&attr);
1091 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
Jens Axboee8462bd2009-07-06 12:59:04 +02001092
1093 td->verify_thread_exit = 0;
1094
1095 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
1096 for (i = 0; i < td->o.verify_async; i++) {
bart Van Assche304a47c2010-08-01 21:31:26 +02001097 ret = pthread_create(&td->verify_threads[i], &attr,
Jens Axboee8462bd2009-07-06 12:59:04 +02001098 verify_async_thread, td);
1099 if (ret) {
1100 log_err("fio: async verify creation failed: %s\n",
1101 strerror(ret));
1102 break;
1103 }
1104 ret = pthread_detach(td->verify_threads[i]);
1105 if (ret) {
1106 log_err("fio: async verify thread detach failed: %s\n",
1107 strerror(ret));
1108 break;
1109 }
1110 td->nr_verify_threads++;
1111 }
1112
bart Van Assche304a47c2010-08-01 21:31:26 +02001113 pthread_attr_destroy(&attr);
1114
Jens Axboee8462bd2009-07-06 12:59:04 +02001115 if (i != td->o.verify_async) {
Jens Axboee40823b2009-07-06 14:28:21 +02001116 log_err("fio: only %d verify threads started, exiting\n", i);
Jens Axboee8462bd2009-07-06 12:59:04 +02001117 td->verify_thread_exit = 1;
1118 write_barrier();
1119 pthread_cond_broadcast(&td->verify_cond);
1120 return 1;
1121 }
1122
1123 return 0;
1124}
1125
1126void verify_async_exit(struct thread_data *td)
1127{
1128 td->verify_thread_exit = 1;
1129 write_barrier();
1130 pthread_cond_broadcast(&td->verify_cond);
1131
1132 pthread_mutex_lock(&td->io_u_lock);
1133
1134 while (td->nr_verify_threads)
1135 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
1136
1137 pthread_mutex_unlock(&td->io_u_lock);
1138 free(td->verify_threads);
1139 td->verify_threads = NULL;
1140}