blob: ea1a911555558f503d706f100989f5b0da848439 [file] [log] [blame]
Jens Axboee29d1b72006-10-18 15:43:15 +02001/*
2 * IO verification helpers
3 */
4#include <unistd.h>
5#include <fcntl.h>
6#include <string.h>
Jens Axboe97af62c2007-05-22 11:12:13 +02007#include <assert.h>
Jens Axboee8462bd2009-07-06 12:59:04 +02008#include <pthread.h>
Jens Axboee29d1b72006-10-18 15:43:15 +02009
10#include "fio.h"
Jens Axboe4f5af7b2009-06-03 08:45:40 +020011#include "verify.h"
Jens Axboee8462bd2009-07-06 12:59:04 +020012#include "smalloc.h"
Jens Axboe0d29de82010-09-01 13:54:15 +020013#include "trim.h"
Jens Axboe637ef8d2010-06-21 20:39:38 +020014#include "lib/rand.h"
Jens Axboee29d1b72006-10-18 15:43:15 +020015
Jens Axboeeef6eea2007-07-30 12:27:58 +020016#include "crc/md5.h"
17#include "crc/crc64.h"
18#include "crc/crc32.h"
Jens Axboebac39e02008-06-11 20:46:19 +020019#include "crc/crc32c.h"
Jens Axboeeef6eea2007-07-30 12:27:58 +020020#include "crc/crc16.h"
21#include "crc/crc7.h"
22#include "crc/sha256.h"
23#include "crc/sha512.h"
Jens Axboe7c353ce2009-08-09 22:40:33 +020024#include "crc/sha1.h"
Jens Axboecd14cc12007-07-30 10:59:33 +020025
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020026void fill_pattern(struct thread_data *td, void *p, unsigned int len, struct io_u *io_u)
Jens Axboe90059d62007-07-30 09:33:12 +020027{
28 switch (td->o.verify_pattern_bytes) {
29 case 0:
Jens Axboebd6f78b2008-02-01 20:27:52 +010030 dprint(FD_VERIFY, "fill random bytes len=%u\n", len);
Jens Axboe637ef8d2010-06-21 20:39:38 +020031 fill_random_buf(p, len);
Jens Axboe90059d62007-07-30 09:33:12 +020032 break;
33 case 1:
Radha Ramachandran10e8a7b2010-07-14 17:39:05 -060034 if (io_u->buf_filled_len >= len) {
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020035 dprint(FD_VERIFY, "using already filled verify pattern b=0 len=%u\n", len);
36 return;
37 }
Jens Axboebd6f78b2008-02-01 20:27:52 +010038 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len);
Radha Ramachandran0e92f872009-10-27 20:14:27 +010039 memset(p, td->o.verify_pattern[0], len);
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020040 io_u->buf_filled_len = len;
Jens Axboe90059d62007-07-30 09:33:12 +020041 break;
Radha Ramachandran0e92f872009-10-27 20:14:27 +010042 default: {
43 unsigned int i = 0, size = 0;
Jens Axboe90059d62007-07-30 09:33:12 +020044 unsigned char *b = p;
45
Radha Ramachandran10e8a7b2010-07-14 17:39:05 -060046 if (io_u->buf_filled_len >= len) {
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020047 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n",
48 td->o.verify_pattern_bytes, len);
49 return;
50 }
Jens Axboebd6f78b2008-02-01 20:27:52 +010051 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n",
52 td->o.verify_pattern_bytes, len);
53
Jens Axboe90059d62007-07-30 09:33:12 +020054 while (i < len) {
Radha Ramachandran0e92f872009-10-27 20:14:27 +010055 size = td->o.verify_pattern_bytes;
56 if (size > (len - i))
57 size = len - i;
58 memcpy(b+i, td->o.verify_pattern, size);
59 i += size;
Jens Axboe90059d62007-07-30 09:33:12 +020060 }
Radha Ramachandrancbe8d752010-07-14 08:36:07 +020061 io_u->buf_filled_len = len;
Jens Axboe90059d62007-07-30 09:33:12 +020062 break;
63 }
64 }
65}
66
Jens Axboe4764aec2007-08-23 09:03:38 +020067static void memswp(void *buf1, void *buf2, unsigned int len)
Shawn Lewis546a9142007-07-28 21:11:37 +020068{
Shawn Lewisdee6de72007-08-02 22:17:52 +020069 char swap[200];
70
71 assert(len <= sizeof(swap));
Jens Axboe90059d62007-07-30 09:33:12 +020072
Shawn Lewis546a9142007-07-28 21:11:37 +020073 memcpy(&swap, buf1, len);
74 memcpy(buf1, buf2, len);
75 memcpy(buf2, &swap, len);
76}
77
Jens Axboee29d1b72006-10-18 15:43:15 +020078static void hexdump(void *buffer, int len)
79{
80 unsigned char *p = buffer;
81 int i;
82
83 for (i = 0; i < len; i++)
Jens Axboebfacf382010-06-18 14:29:52 +020084 log_err("%02x", p[i]);
85 log_err("\n");
Jens Axboee29d1b72006-10-18 15:43:15 +020086}
87
Jens Axboed9f2caf2007-07-28 21:22:03 +020088/*
Jens Axboe87677832007-07-30 12:08:14 +020089 * Prepare for seperation of verify_header and checksum header
90 */
Jens Axboe546dfd92007-07-30 12:23:05 +020091static inline unsigned int __hdr_size(int verify_type)
Jens Axboe87677832007-07-30 12:08:14 +020092{
Jens Axboe5921e802008-05-30 15:02:38 +020093 unsigned int len = len;
Jens Axboe87677832007-07-30 12:08:14 +020094
Jens Axboe546dfd92007-07-30 12:23:05 +020095 switch (verify_type) {
96 case VERIFY_NONE:
97 case VERIFY_NULL:
98 len = 0;
99 break;
100 case VERIFY_MD5:
101 len = sizeof(struct vhdr_md5);
102 break;
103 case VERIFY_CRC64:
104 len = sizeof(struct vhdr_crc64);
105 break;
Jens Axboebac39e02008-06-11 20:46:19 +0200106 case VERIFY_CRC32C:
Jens Axboe546dfd92007-07-30 12:23:05 +0200107 case VERIFY_CRC32:
Jens Axboe38455912008-08-04 15:35:26 +0200108 case VERIFY_CRC32C_INTEL:
Jens Axboe546dfd92007-07-30 12:23:05 +0200109 len = sizeof(struct vhdr_crc32);
110 break;
111 case VERIFY_CRC16:
112 len = sizeof(struct vhdr_crc16);
113 break;
114 case VERIFY_CRC7:
115 len = sizeof(struct vhdr_crc7);
116 break;
117 case VERIFY_SHA256:
118 len = sizeof(struct vhdr_sha256);
119 break;
120 case VERIFY_SHA512:
121 len = sizeof(struct vhdr_sha512);
122 break;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200123 case VERIFY_META:
124 len = sizeof(struct vhdr_meta);
125 break;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200126 case VERIFY_SHA1:
127 len = sizeof(struct vhdr_sha1);
128 break;
Jens Axboe546dfd92007-07-30 12:23:05 +0200129 default:
130 log_err("fio: unknown verify header!\n");
131 assert(0);
132 }
133
134 return len + sizeof(struct verify_header);
Jens Axboe87677832007-07-30 12:08:14 +0200135}
136
137static inline unsigned int hdr_size(struct verify_header *hdr)
138{
Jens Axboe546dfd92007-07-30 12:23:05 +0200139 return __hdr_size(hdr->verify_type);
140}
141
142static void *hdr_priv(struct verify_header *hdr)
143{
144 void *priv = hdr;
145
146 return priv + sizeof(struct verify_header);
Jens Axboe87677832007-07-30 12:08:14 +0200147}
148
149/*
Jens Axboe936216f2010-06-18 13:44:11 +0200150 * Verify container, pass info to verify handlers and allow them to
151 * pass info back in case of error
152 */
153struct vcont {
154 /*
155 * Input
156 */
157 struct io_u *io_u;
158 unsigned int hdr_num;
159
160 /*
161 * Output, only valid in case of error
162 */
Jens Axboebfacf382010-06-18 14:29:52 +0200163 const char *name;
164 void *good_crc;
165 void *bad_crc;
Jens Axboe936216f2010-06-18 13:44:11 +0200166 unsigned int crc_len;
167};
168
Jens Axboebfacf382010-06-18 14:29:52 +0200169static void log_verify_failure(struct verify_header *hdr, struct vcont *vc)
170{
171 unsigned long long offset;
172
173 offset = vc->io_u->offset;
174 offset += vc->hdr_num * hdr->len;
Jens Axboe32c17ad2010-06-18 14:32:21 +0200175 log_err("%.8s: verify failed at file %s offset %llu, length %u\n",
176 vc->name, vc->io_u->file->file_name, offset, hdr->len);
Jens Axboebfacf382010-06-18 14:29:52 +0200177
178 if (vc->good_crc && vc->bad_crc) {
179 log_err(" Expected CRC: ");
180 hexdump(vc->good_crc, vc->crc_len);
181 log_err(" Received CRC: ");
182 hexdump(vc->bad_crc, vc->crc_len);
183 }
184}
185
Jens Axboe936216f2010-06-18 13:44:11 +0200186/*
Jens Axboed9f2caf2007-07-28 21:22:03 +0200187 * Return data area 'header_num'
188 */
Jens Axboe936216f2010-06-18 13:44:11 +0200189static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc)
Jens Axboed9f2caf2007-07-28 21:22:03 +0200190{
Jens Axboe936216f2010-06-18 13:44:11 +0200191 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr);
Jens Axboed9f2caf2007-07-28 21:22:03 +0200192}
193
Shawn Lewis7437ee82007-08-02 21:05:58 +0200194static int verify_io_u_meta(struct verify_header *hdr, struct thread_data *td,
Jens Axboe936216f2010-06-18 13:44:11 +0200195 struct vcont *vc)
Shawn Lewis7437ee82007-08-02 21:05:58 +0200196{
197 struct vhdr_meta *vh = hdr_priv(hdr);
Jens Axboe936216f2010-06-18 13:44:11 +0200198 struct io_u *io_u = vc->io_u;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200199
Jens Axboebd6f78b2008-02-01 20:27:52 +0100200 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len);
201
Jens Axboebfacf382010-06-18 14:29:52 +0200202 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval)
203 return 0;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200204
Jens Axboebfacf382010-06-18 14:29:52 +0200205 vc->name = "meta";
206 log_verify_failure(hdr, vc);
207 return EILSEQ;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200208}
209
Jens Axboe936216f2010-06-18 13:44:11 +0200210static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc)
Jens Axboecd14cc12007-07-30 10:59:33 +0200211{
Jens Axboe936216f2010-06-18 13:44:11 +0200212 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200213 struct vhdr_sha512 *vh = hdr_priv(hdr);
Jens Axboecd14cc12007-07-30 10:59:33 +0200214 uint8_t sha512[128];
215 struct sha512_ctx sha512_ctx = {
216 .buf = sha512,
217 };
218
Jens Axboe936216f2010-06-18 13:44:11 +0200219 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100220
Jens Axboecd14cc12007-07-30 10:59:33 +0200221 sha512_init(&sha512_ctx);
Jens Axboe87677832007-07-30 12:08:14 +0200222 sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr));
Jens Axboecd14cc12007-07-30 10:59:33 +0200223
Jens Axboebfacf382010-06-18 14:29:52 +0200224 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512)))
225 return 0;
Jens Axboecd14cc12007-07-30 10:59:33 +0200226
Jens Axboebfacf382010-06-18 14:29:52 +0200227 vc->name = "sha512";
228 vc->good_crc = vh->sha512;
229 vc->bad_crc = sha512_ctx.buf;
230 vc->crc_len = sizeof(vh->sha512);
231 log_verify_failure(hdr, vc);
232 return EILSEQ;
Jens Axboecd14cc12007-07-30 10:59:33 +0200233}
234
Jens Axboe936216f2010-06-18 13:44:11 +0200235static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc)
Jens Axboecd14cc12007-07-30 10:59:33 +0200236{
Jens Axboe936216f2010-06-18 13:44:11 +0200237 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200238 struct vhdr_sha256 *vh = hdr_priv(hdr);
Jens Axboebc77f562010-02-23 10:36:21 +0100239 uint8_t sha256[64];
Jens Axboecd14cc12007-07-30 10:59:33 +0200240 struct sha256_ctx sha256_ctx = {
241 .buf = sha256,
242 };
243
Jens Axboe936216f2010-06-18 13:44:11 +0200244 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100245
Jens Axboecd14cc12007-07-30 10:59:33 +0200246 sha256_init(&sha256_ctx);
Jens Axboe87677832007-07-30 12:08:14 +0200247 sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr));
Jens Axboecd14cc12007-07-30 10:59:33 +0200248
Jens Axboebfacf382010-06-18 14:29:52 +0200249 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256)))
250 return 0;
Jens Axboecd14cc12007-07-30 10:59:33 +0200251
Jens Axboebfacf382010-06-18 14:29:52 +0200252 vc->name = "sha256";
253 vc->good_crc = vh->sha256;
254 vc->bad_crc = sha256_ctx.buf;
255 vc->crc_len = sizeof(vh->sha256);
256 log_verify_failure(hdr, vc);
257 return EILSEQ;
Jens Axboecd14cc12007-07-30 10:59:33 +0200258}
259
Jens Axboe936216f2010-06-18 13:44:11 +0200260static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc)
Jens Axboe7c353ce2009-08-09 22:40:33 +0200261{
Jens Axboe936216f2010-06-18 13:44:11 +0200262 void *p = io_u_verify_off(hdr, vc);
Jens Axboe7c353ce2009-08-09 22:40:33 +0200263 struct vhdr_sha1 *vh = hdr_priv(hdr);
264 uint32_t sha1[5];
265 struct sha1_ctx sha1_ctx = {
266 .H = sha1,
267 };
268
Jens Axboe936216f2010-06-18 13:44:11 +0200269 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboe7c353ce2009-08-09 22:40:33 +0200270
271 sha1_init(&sha1_ctx);
272 sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr));
273
Jens Axboebfacf382010-06-18 14:29:52 +0200274 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1)))
275 return 0;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200276
Jens Axboebfacf382010-06-18 14:29:52 +0200277 vc->name = "sha1";
278 vc->good_crc = vh->sha1;
279 vc->bad_crc = sha1_ctx.H;
280 vc->crc_len = sizeof(vh->sha1);
281 log_verify_failure(hdr, vc);
282 return EILSEQ;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200283}
284
Jens Axboe936216f2010-06-18 13:44:11 +0200285static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc)
Jens Axboe1e154bd2007-07-27 09:52:40 +0200286{
Jens Axboe936216f2010-06-18 13:44:11 +0200287 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200288 struct vhdr_crc7 *vh = hdr_priv(hdr);
Jens Axboe1e154bd2007-07-27 09:52:40 +0200289 unsigned char c;
290
Jens Axboe936216f2010-06-18 13:44:11 +0200291 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100292
Jens Axboe87677832007-07-30 12:08:14 +0200293 c = crc7(p, hdr->len - hdr_size(hdr));
Jens Axboe1e154bd2007-07-27 09:52:40 +0200294
Jens Axboebfacf382010-06-18 14:29:52 +0200295 if (c == vh->crc7)
296 return 0;
Jens Axboe1e154bd2007-07-27 09:52:40 +0200297
Jens Axboebfacf382010-06-18 14:29:52 +0200298 vc->name = "crc7";
299 vc->good_crc = &vh->crc7;
300 vc->bad_crc = &c;
301 vc->crc_len = 1;
302 log_verify_failure(hdr, vc);
303 return EILSEQ;
Jens Axboe1e154bd2007-07-27 09:52:40 +0200304}
305
Jens Axboe936216f2010-06-18 13:44:11 +0200306static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc)
Jens Axboe969f7ed2007-07-27 09:07:17 +0200307{
Jens Axboe936216f2010-06-18 13:44:11 +0200308 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200309 struct vhdr_crc16 *vh = hdr_priv(hdr);
Jens Axboe969f7ed2007-07-27 09:07:17 +0200310 unsigned short c;
311
Jens Axboe936216f2010-06-18 13:44:11 +0200312 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100313
Jens Axboe87677832007-07-30 12:08:14 +0200314 c = crc16(p, hdr->len - hdr_size(hdr));
Jens Axboe969f7ed2007-07-27 09:07:17 +0200315
Jens Axboebfacf382010-06-18 14:29:52 +0200316 if (c == vh->crc16)
317 return 0;
Jens Axboe969f7ed2007-07-27 09:07:17 +0200318
Jens Axboebfacf382010-06-18 14:29:52 +0200319 vc->name = "crc16";
320 vc->good_crc = &vh->crc16;
321 vc->bad_crc = &c;
322 vc->crc_len = 2;
323 log_verify_failure(hdr, vc);
324 return EILSEQ;
Jens Axboe969f7ed2007-07-27 09:07:17 +0200325}
326
Jens Axboe936216f2010-06-18 13:44:11 +0200327static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc)
Jens Axboed77a7af2007-07-27 15:35:06 +0200328{
Jens Axboe936216f2010-06-18 13:44:11 +0200329 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200330 struct vhdr_crc64 *vh = hdr_priv(hdr);
Jens Axboed77a7af2007-07-27 15:35:06 +0200331 unsigned long long c;
332
Jens Axboe936216f2010-06-18 13:44:11 +0200333 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100334
Jens Axboe87677832007-07-30 12:08:14 +0200335 c = crc64(p, hdr->len - hdr_size(hdr));
Jens Axboed77a7af2007-07-27 15:35:06 +0200336
Jens Axboebfacf382010-06-18 14:29:52 +0200337 if (c == vh->crc64)
338 return 0;
Jens Axboed77a7af2007-07-27 15:35:06 +0200339
Jens Axboebfacf382010-06-18 14:29:52 +0200340 vc->name = "crc64";
341 vc->good_crc = &vh->crc64;
342 vc->bad_crc = &c;
343 vc->crc_len = 8;
344 log_verify_failure(hdr, vc);
345 return EILSEQ;
Jens Axboed77a7af2007-07-27 15:35:06 +0200346}
347
Jens Axboe936216f2010-06-18 13:44:11 +0200348static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc)
Jens Axboee29d1b72006-10-18 15:43:15 +0200349{
Jens Axboe936216f2010-06-18 13:44:11 +0200350 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200351 struct vhdr_crc32 *vh = hdr_priv(hdr);
352 uint32_t c;
Jens Axboee29d1b72006-10-18 15:43:15 +0200353
Jens Axboe936216f2010-06-18 13:44:11 +0200354 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100355
Jens Axboe87677832007-07-30 12:08:14 +0200356 c = crc32(p, hdr->len - hdr_size(hdr));
Jens Axboee29d1b72006-10-18 15:43:15 +0200357
Jens Axboebfacf382010-06-18 14:29:52 +0200358 if (c == vh->crc32)
359 return 0;
Jens Axboee29d1b72006-10-18 15:43:15 +0200360
Jens Axboebfacf382010-06-18 14:29:52 +0200361 vc->name = "crc32";
362 vc->good_crc = &vh->crc32;
363 vc->bad_crc = &c;
364 vc->crc_len = 4;
365 log_verify_failure(hdr, vc);
366 return EILSEQ;
Jens Axboee29d1b72006-10-18 15:43:15 +0200367}
368
Jens Axboe936216f2010-06-18 13:44:11 +0200369static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc)
Jens Axboebac39e02008-06-11 20:46:19 +0200370{
Jens Axboe936216f2010-06-18 13:44:11 +0200371 void *p = io_u_verify_off(hdr, vc);
Jens Axboebac39e02008-06-11 20:46:19 +0200372 struct vhdr_crc32 *vh = hdr_priv(hdr);
373 uint32_t c;
374
Jens Axboe936216f2010-06-18 13:44:11 +0200375 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebac39e02008-06-11 20:46:19 +0200376
Jens Axboe38455912008-08-04 15:35:26 +0200377 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
378 c = crc32c_intel(p, hdr->len - hdr_size(hdr));
379 else
380 c = crc32c(p, hdr->len - hdr_size(hdr));
Jens Axboebac39e02008-06-11 20:46:19 +0200381
Jens Axboebfacf382010-06-18 14:29:52 +0200382 if (c == vh->crc32)
383 return 0;
Jens Axboebac39e02008-06-11 20:46:19 +0200384
Jens Axboebfacf382010-06-18 14:29:52 +0200385 vc->name = "crc32c";
386 vc->good_crc = &vh->crc32;
387 vc->bad_crc = &c;
388 vc->crc_len = 4;
389 log_verify_failure(hdr, vc);
390 return EILSEQ;
Jens Axboebac39e02008-06-11 20:46:19 +0200391}
392
Jens Axboe936216f2010-06-18 13:44:11 +0200393static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc)
Jens Axboee29d1b72006-10-18 15:43:15 +0200394{
Jens Axboe936216f2010-06-18 13:44:11 +0200395 void *p = io_u_verify_off(hdr, vc);
Jens Axboe546dfd92007-07-30 12:23:05 +0200396 struct vhdr_md5 *vh = hdr_priv(hdr);
Jens Axboe8c432322007-07-27 13:16:24 +0200397 uint32_t hash[MD5_HASH_WORDS];
398 struct md5_ctx md5_ctx = {
399 .hash = hash,
400 };
Jens Axboee29d1b72006-10-18 15:43:15 +0200401
Jens Axboe936216f2010-06-18 13:44:11 +0200402 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100403
Jens Axboe61f821f2007-07-30 10:18:06 +0200404 md5_init(&md5_ctx);
Jens Axboe87677832007-07-30 12:08:14 +0200405 md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr));
Jens Axboee29d1b72006-10-18 15:43:15 +0200406
Jens Axboebfacf382010-06-18 14:29:52 +0200407 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash)))
408 return 0;
Jens Axboee29d1b72006-10-18 15:43:15 +0200409
Jens Axboebfacf382010-06-18 14:29:52 +0200410 vc->name = "md5";
411 vc->good_crc = vh->md5_digest;
412 vc->bad_crc = md5_ctx.hash;
413 vc->crc_len = sizeof(hash);
414 log_verify_failure(hdr, vc);
415 return EILSEQ;
Jens Axboee29d1b72006-10-18 15:43:15 +0200416}
417
Jens Axboe3f199b02007-08-09 10:16:31 +0200418static unsigned int hweight8(unsigned int w)
419{
420 unsigned int res = w - ((w >> 1) & 0x55);
421
422 res = (res & 0x33) + ((res >> 2) & 0x33);
423 return (res + (res >> 4)) & 0x0F;
424}
425
Radha Ramachandran0e92f872009-10-27 20:14:27 +0100426int verify_io_u_pattern(char *pattern, unsigned long pattern_size,
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100427 char *buf, unsigned int len, unsigned int mod)
Shawn Lewisa944e332007-08-02 22:18:29 +0200428{
429 unsigned int i;
Shawn Lewisa944e332007-08-02 22:18:29 +0200430
431 for (i = 0; i < len; i++) {
Radha Ramachandran0e92f872009-10-27 20:14:27 +0100432 if (buf[i] != pattern[mod]) {
Jens Axboe3f199b02007-08-09 10:16:31 +0200433 unsigned int bits;
434
Radha Ramachandran0e92f872009-10-27 20:14:27 +0100435 bits = hweight8(buf[i] ^ pattern[mod]);
Jens Axboe3f199b02007-08-09 10:16:31 +0200436 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n",
Radha Ramachandran0e92f872009-10-27 20:14:27 +0100437 buf[i], pattern[mod], bits);
Jens Axboe3f199b02007-08-09 10:16:31 +0200438 log_err("fio: bad pattern block offset %u\n", i);
Jens Axboe9fd18962009-05-19 10:35:38 +0200439 return EILSEQ;
Jens Axboe3f199b02007-08-09 10:16:31 +0200440 }
Shawn Lewisa944e332007-08-02 22:18:29 +0200441 mod++;
442 if (mod == pattern_size)
443 mod = 0;
444 }
445
446 return 0;
447}
448
Jens Axboee8462bd2009-07-06 12:59:04 +0200449/*
450 * Push IO verification to a separate thread
451 */
452int verify_io_u_async(struct thread_data *td, struct io_u *io_u)
453{
454 if (io_u->file)
455 put_file_log(td, io_u->file);
456
457 io_u->file = NULL;
458
459 pthread_mutex_lock(&td->io_u_lock);
Radha Ramachandran0c412142009-11-03 21:45:31 +0100460
461 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) {
462 td->cur_depth--;
463 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH;
464 }
Jens Axboee8462bd2009-07-06 12:59:04 +0200465 flist_del(&io_u->list);
466 flist_add_tail(&io_u->list, &td->verify_list);
Jens Axboe2ecc1b52009-11-04 20:58:09 +0100467 io_u->flags |= IO_U_F_FREE_DEF;
Jens Axboee8462bd2009-07-06 12:59:04 +0200468 pthread_mutex_unlock(&td->io_u_lock);
469
470 pthread_cond_signal(&td->verify_cond);
Jens Axboee8462bd2009-07-06 12:59:04 +0200471 return 0;
472}
473
Jens Axboe0d29de82010-09-01 13:54:15 +0200474static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u)
475{
476 static char zero_buf[1024];
477 unsigned int this_len, len;
478 int ret = 0;
479 void *p;
480
481 if (!td->o.trim_zero)
482 return 0;
483
484 len = io_u->buflen;
485 p = io_u->buf;
486 do {
487 this_len = sizeof(zero_buf);
488 if (this_len > len)
489 this_len = len;
490 if (memcmp(p, zero_buf, this_len)) {
491 ret = EILSEQ;
492 break;
493 }
494 len -= this_len;
495 p += this_len;
496 } while (len);
497
498 if (!ret)
499 return 0;
500
Jens Axboea917a8b2010-09-02 13:23:20 +0200501 log_err("trim: verify failed at file %s offset %llu, length %lu"
502 ", block offset %lu\n",
503 io_u->file->file_name, io_u->offset, io_u->buflen,
Jens Axboe2f681242010-10-21 08:15:59 +0200504 (unsigned long) (p - io_u->buf));
Jens Axboe0d29de82010-09-01 13:54:15 +0200505 return ret;
506}
507
Jens Axboe36690c92007-03-26 10:23:34 +0200508int verify_io_u(struct thread_data *td, struct io_u *io_u)
Jens Axboee29d1b72006-10-18 15:43:15 +0200509{
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200510 struct verify_header *hdr;
Shawn Lewisa944e332007-08-02 22:18:29 +0200511 unsigned int hdr_size, hdr_inc, hdr_num = 0;
Jens Axboe95646102007-07-28 21:17:50 +0200512 void *p;
Jens Axboee29d1b72006-10-18 15:43:15 +0200513 int ret;
514
Shawn Lewis1dcc0492007-07-27 08:02:45 +0200515 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
Jens Axboe36690c92007-03-26 10:23:34 +0200516 return 0;
Jens Axboe0d29de82010-09-01 13:54:15 +0200517 if (io_u->flags & IO_U_F_TRIMMED) {
518 ret = verify_trimmed_io_u(td, io_u);
519 goto done;
520 }
Jens Axboe36690c92007-03-26 10:23:34 +0200521
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200522 hdr_inc = io_u->buflen;
Jens Axboea59e1702007-07-30 08:53:27 +0200523 if (td->o.verify_interval)
524 hdr_inc = td->o.verify_interval;
Jens Axboee29d1b72006-10-18 15:43:15 +0200525
Jens Axboea12a3b42007-08-09 10:20:54 +0200526 ret = 0;
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100527 for (p = io_u->buf; p < io_u->buf + io_u->buflen;
528 p += hdr_inc, hdr_num++) {
Jens Axboebfacf382010-06-18 14:29:52 +0200529 struct vcont vc = {
530 .io_u = io_u,
531 .hdr_num = hdr_num,
532 };
Jens Axboe936216f2010-06-18 13:44:11 +0200533
Jens Axboef3e6cb92010-06-18 14:48:43 +0200534 if (ret && td->o.verify_fatal)
Jens Axboea12a3b42007-08-09 10:20:54 +0200535 break;
Jens Axboef3e6cb92010-06-18 14:48:43 +0200536
Shawn Lewisa944e332007-08-02 22:18:29 +0200537 hdr_size = __hdr_size(td->o.verify);
Jens Axboea59e1702007-07-30 08:53:27 +0200538 if (td->o.verify_offset)
Shawn Lewisa944e332007-08-02 22:18:29 +0200539 memswp(p, p + td->o.verify_offset, hdr_size);
Jens Axboe95646102007-07-28 21:17:50 +0200540 hdr = p;
Jens Axboee29d1b72006-10-18 15:43:15 +0200541
Jens Axboe3f199b02007-08-09 10:16:31 +0200542 if (hdr->fio_magic != FIO_HDR_MAGIC) {
Jens Axboec9b44032010-06-18 14:46:06 +0200543 log_err("verify: bad magic header %x, wanted %x at file %s offset %llu, length %u\n",
544 hdr->fio_magic, FIO_HDR_MAGIC,
545 io_u->file->file_name,
546 io_u->offset + hdr_num * hdr->len, hdr->len);
Jens Axboe9fd18962009-05-19 10:35:38 +0200547 return EILSEQ;
Jens Axboe3f199b02007-08-09 10:16:31 +0200548 }
549
Shawn Lewise28218f2008-01-16 11:01:33 +0100550 if (td->o.verify_pattern_bytes) {
Jens Axboebd6f78b2008-02-01 20:27:52 +0100551 dprint(FD_VERIFY, "pattern verify io_u %p, len %u\n",
552 io_u, hdr->len);
Shawn Lewise28218f2008-01-16 11:01:33 +0100553 ret = verify_io_u_pattern(td->o.verify_pattern,
Radha Ramachandran0e92f872009-10-27 20:14:27 +0100554 td->o.verify_pattern_bytes,
555 p + hdr_size,
556 hdr_inc - hdr_size,
557 hdr_size % td->o.verify_pattern_bytes);
Jens Axboec9b44032010-06-18 14:46:06 +0200558
559 if (ret) {
560 log_err("pattern: verify failed at file %s offset %llu, length %u\n",
561 io_u->file->file_name,
562 io_u->offset + hdr_num * hdr->len,
563 hdr->len);
564 }
565
Radha Ramachandrane92d3d72009-07-27 12:27:55 +0200566 /*
567 * Also verify the meta data, if applicable
568 */
569 if (hdr->verify_type == VERIFY_META)
Jens Axboe936216f2010-06-18 13:44:11 +0200570 ret |= verify_io_u_meta(hdr, td, &vc);
Shawn Lewise28218f2008-01-16 11:01:33 +0100571 continue;
572 }
573
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200574 switch (hdr->verify_type) {
575 case VERIFY_MD5:
Jens Axboe936216f2010-06-18 13:44:11 +0200576 ret = verify_io_u_md5(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200577 break;
578 case VERIFY_CRC64:
Jens Axboe936216f2010-06-18 13:44:11 +0200579 ret = verify_io_u_crc64(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200580 break;
Jens Axboebac39e02008-06-11 20:46:19 +0200581 case VERIFY_CRC32C:
Jens Axboe38455912008-08-04 15:35:26 +0200582 case VERIFY_CRC32C_INTEL:
Jens Axboe936216f2010-06-18 13:44:11 +0200583 ret = verify_io_u_crc32c(hdr, &vc);
Jens Axboebac39e02008-06-11 20:46:19 +0200584 break;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200585 case VERIFY_CRC32:
Jens Axboe936216f2010-06-18 13:44:11 +0200586 ret = verify_io_u_crc32(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200587 break;
588 case VERIFY_CRC16:
Jens Axboe936216f2010-06-18 13:44:11 +0200589 ret = verify_io_u_crc16(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200590 break;
591 case VERIFY_CRC7:
Jens Axboe936216f2010-06-18 13:44:11 +0200592 ret = verify_io_u_crc7(hdr, &vc);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200593 break;
Jens Axboecd14cc12007-07-30 10:59:33 +0200594 case VERIFY_SHA256:
Jens Axboe936216f2010-06-18 13:44:11 +0200595 ret = verify_io_u_sha256(hdr, &vc);
Jens Axboecd14cc12007-07-30 10:59:33 +0200596 break;
597 case VERIFY_SHA512:
Jens Axboe936216f2010-06-18 13:44:11 +0200598 ret = verify_io_u_sha512(hdr, &vc);
Jens Axboecd14cc12007-07-30 10:59:33 +0200599 break;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200600 case VERIFY_META:
Jens Axboe936216f2010-06-18 13:44:11 +0200601 ret = verify_io_u_meta(hdr, td, &vc);
Shawn Lewis7437ee82007-08-02 21:05:58 +0200602 break;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200603 case VERIFY_SHA1:
Jens Axboe936216f2010-06-18 13:44:11 +0200604 ret = verify_io_u_sha1(hdr, &vc);
Jens Axboe7c353ce2009-08-09 22:40:33 +0200605 break;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200606 default:
607 log_err("Bad verify type %u\n", hdr->verify_type);
Jens Axboed16d4e02007-09-06 16:16:44 +0200608 ret = EINVAL;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200609 }
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200610 }
Jens Axboea7dfe862007-03-12 10:05:08 +0100611
Jens Axboe0d29de82010-09-01 13:54:15 +0200612done:
Jens Axboef3e6cb92010-06-18 14:48:43 +0200613 if (ret && td->o.verify_fatal)
614 td->terminate = 1;
615
Jens Axboea12a3b42007-08-09 10:20:54 +0200616 return ret;
Jens Axboee29d1b72006-10-18 15:43:15 +0200617}
618
Shawn Lewis7437ee82007-08-02 21:05:58 +0200619static void fill_meta(struct verify_header *hdr, struct thread_data *td,
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100620 struct io_u *io_u, unsigned int header_num)
Shawn Lewis7437ee82007-08-02 21:05:58 +0200621{
622 struct vhdr_meta *vh = hdr_priv(hdr);
623
624 vh->thread = td->thread_number;
625
626 vh->time_sec = io_u->start_time.tv_sec;
627 vh->time_usec = io_u->start_time.tv_usec;
628
629 vh->numberio = td->io_issues[DDIR_WRITE];
630
631 vh->offset = io_u->offset + header_num * td->o.verify_interval;
632}
633
Jens Axboecd14cc12007-07-30 10:59:33 +0200634static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len)
635{
Jens Axboe546dfd92007-07-30 12:23:05 +0200636 struct vhdr_sha512 *vh = hdr_priv(hdr);
Jens Axboecd14cc12007-07-30 10:59:33 +0200637 struct sha512_ctx sha512_ctx = {
Jens Axboe546dfd92007-07-30 12:23:05 +0200638 .buf = vh->sha512,
Jens Axboecd14cc12007-07-30 10:59:33 +0200639 };
640
641 sha512_init(&sha512_ctx);
642 sha512_update(&sha512_ctx, p, len);
643}
644
645static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len)
646{
Jens Axboe546dfd92007-07-30 12:23:05 +0200647 struct vhdr_sha256 *vh = hdr_priv(hdr);
Jens Axboecd14cc12007-07-30 10:59:33 +0200648 struct sha256_ctx sha256_ctx = {
Jens Axboe546dfd92007-07-30 12:23:05 +0200649 .buf = vh->sha256,
Jens Axboecd14cc12007-07-30 10:59:33 +0200650 };
651
652 sha256_init(&sha256_ctx);
653 sha256_update(&sha256_ctx, p, len);
654}
655
Jens Axboe7c353ce2009-08-09 22:40:33 +0200656static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len)
657{
658 struct vhdr_sha1 *vh = hdr_priv(hdr);
659 struct sha1_ctx sha1_ctx = {
660 .H = vh->sha1,
661 };
662
663 sha1_init(&sha1_ctx);
664 sha1_update(&sha1_ctx, p, len);
665}
666
Jens Axboe1e154bd2007-07-27 09:52:40 +0200667static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len)
668{
Jens Axboe546dfd92007-07-30 12:23:05 +0200669 struct vhdr_crc7 *vh = hdr_priv(hdr);
670
671 vh->crc7 = crc7(p, len);
Jens Axboe1e154bd2007-07-27 09:52:40 +0200672}
673
Jens Axboe969f7ed2007-07-27 09:07:17 +0200674static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len)
675{
Jens Axboe546dfd92007-07-30 12:23:05 +0200676 struct vhdr_crc16 *vh = hdr_priv(hdr);
677
678 vh->crc16 = crc16(p, len);
Jens Axboe969f7ed2007-07-27 09:07:17 +0200679}
680
Jens Axboee29d1b72006-10-18 15:43:15 +0200681static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
682{
Jens Axboe546dfd92007-07-30 12:23:05 +0200683 struct vhdr_crc32 *vh = hdr_priv(hdr);
684
685 vh->crc32 = crc32(p, len);
Jens Axboee29d1b72006-10-18 15:43:15 +0200686}
687
Jens Axboebac39e02008-06-11 20:46:19 +0200688static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len)
689{
690 struct vhdr_crc32 *vh = hdr_priv(hdr);
691
Jens Axboe38455912008-08-04 15:35:26 +0200692 if (hdr->verify_type == VERIFY_CRC32C_INTEL)
693 vh->crc32 = crc32c_intel(p, len);
694 else
695 vh->crc32 = crc32c(p, len);
Jens Axboebac39e02008-06-11 20:46:19 +0200696}
697
Jens Axboed77a7af2007-07-27 15:35:06 +0200698static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len)
699{
Jens Axboe546dfd92007-07-30 12:23:05 +0200700 struct vhdr_crc64 *vh = hdr_priv(hdr);
701
702 vh->crc64 = crc64(p, len);
Jens Axboed77a7af2007-07-27 15:35:06 +0200703}
704
Jens Axboee29d1b72006-10-18 15:43:15 +0200705static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
706{
Jens Axboe546dfd92007-07-30 12:23:05 +0200707 struct vhdr_md5 *vh = hdr_priv(hdr);
Jens Axboe8c432322007-07-27 13:16:24 +0200708 struct md5_ctx md5_ctx = {
Jens Axboe546dfd92007-07-30 12:23:05 +0200709 .hash = (uint32_t *) vh->md5_digest,
Jens Axboe8c432322007-07-27 13:16:24 +0200710 };
Jens Axboee29d1b72006-10-18 15:43:15 +0200711
Jens Axboe61f821f2007-07-30 10:18:06 +0200712 md5_init(&md5_ctx);
Jens Axboee29d1b72006-10-18 15:43:15 +0200713 md5_update(&md5_ctx, p, len);
Jens Axboee29d1b72006-10-18 15:43:15 +0200714}
715
716/*
717 * fill body of io_u->buf with random data and add a header with the
718 * crc32 or md5 sum of that data.
719 */
720void populate_verify_io_u(struct thread_data *td, struct io_u *io_u)
721{
Jens Axboebaefa9b2007-07-27 12:57:25 +0200722 struct verify_header *hdr;
Jens Axboe95646102007-07-28 21:17:50 +0200723 void *p = io_u->buf, *data;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200724 unsigned int hdr_inc, data_len, header_num = 0;
Jens Axboee29d1b72006-10-18 15:43:15 +0200725
Jens Axboe9cc3d152007-03-26 10:32:30 +0200726 if (td->o.verify == VERIFY_NULL)
727 return;
728
Radha Ramachandrancbe8d752010-07-14 08:36:07 +0200729 fill_pattern(td, p, io_u->buflen, io_u);
Jens Axboebaefa9b2007-07-27 12:57:25 +0200730
Shawn Lewis546a9142007-07-28 21:11:37 +0200731 hdr_inc = io_u->buflen;
Jens Axboea59e1702007-07-30 08:53:27 +0200732 if (td->o.verify_interval)
733 hdr_inc = td->o.verify_interval;
Shawn Lewis546a9142007-07-28 21:11:37 +0200734
Jens Axboe5ec10ea2008-03-06 15:42:00 +0100735 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) {
Jens Axboe95646102007-07-28 21:17:50 +0200736 hdr = p;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200737
738 hdr->fio_magic = FIO_HDR_MAGIC;
739 hdr->verify_type = td->o.verify;
Shawn Lewis546a9142007-07-28 21:11:37 +0200740 hdr->len = hdr_inc;
Jens Axboe87677832007-07-30 12:08:14 +0200741 data_len = hdr_inc - hdr_size(hdr);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200742
Jens Axboe87677832007-07-30 12:08:14 +0200743 data = p + hdr_size(hdr);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200744 switch (td->o.verify) {
745 case VERIFY_MD5:
Jens Axboebd6f78b2008-02-01 20:27:52 +0100746 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n",
747 io_u, hdr->len);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200748 fill_md5(hdr, data, data_len);
749 break;
750 case VERIFY_CRC64:
Jens Axboebd6f78b2008-02-01 20:27:52 +0100751 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n",
752 io_u, hdr->len);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200753 fill_crc64(hdr, data, data_len);
754 break;
Jens Axboebac39e02008-06-11 20:46:19 +0200755 case VERIFY_CRC32C:
Jens Axboe38455912008-08-04 15:35:26 +0200756 case VERIFY_CRC32C_INTEL:
Jens Axboebac39e02008-06-11 20:46:19 +0200757 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n",
758 io_u, hdr->len);
759 fill_crc32c(hdr, data, data_len);
760 break;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200761 case VERIFY_CRC32:
Jens Axboebd6f78b2008-02-01 20:27:52 +0100762 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n",
763 io_u, hdr->len);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200764 fill_crc32(hdr, data, data_len);
765 break;
766 case VERIFY_CRC16:
Jens Axboebd6f78b2008-02-01 20:27:52 +0100767 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n",
768 io_u, hdr->len);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200769 fill_crc16(hdr, data, data_len);
770 break;
771 case VERIFY_CRC7:
Jens Axboebd6f78b2008-02-01 20:27:52 +0100772 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n",
773 io_u, hdr->len);
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200774 fill_crc7(hdr, data, data_len);
775 break;
Jens Axboecd14cc12007-07-30 10:59:33 +0200776 case VERIFY_SHA256:
Jens Axboebd6f78b2008-02-01 20:27:52 +0100777 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n",
778 io_u, hdr->len);
Jens Axboecd14cc12007-07-30 10:59:33 +0200779 fill_sha256(hdr, data, data_len);
780 break;
781 case VERIFY_SHA512:
Jens Axboebd6f78b2008-02-01 20:27:52 +0100782 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n",
783 io_u, hdr->len);
Jens Axboecd14cc12007-07-30 10:59:33 +0200784 fill_sha512(hdr, data, data_len);
785 break;
Shawn Lewis7437ee82007-08-02 21:05:58 +0200786 case VERIFY_META:
Jens Axboebd6f78b2008-02-01 20:27:52 +0100787 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n",
788 io_u, hdr->len);
Shawn Lewis7437ee82007-08-02 21:05:58 +0200789 fill_meta(hdr, td, io_u, header_num);
790 break;
Jens Axboe7c353ce2009-08-09 22:40:33 +0200791 case VERIFY_SHA1:
792 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n",
793 io_u, hdr->len);
794 fill_sha1(hdr, data, data_len);
795 break;
Shawn Lewis3f9f4e22007-07-28 21:10:37 +0200796 default:
797 log_err("fio: bad verify type: %d\n", td->o.verify);
798 assert(0);
799 }
Jens Axboea59e1702007-07-30 08:53:27 +0200800 if (td->o.verify_offset)
Jens Axboe87677832007-07-30 12:08:14 +0200801 memswp(p, p + td->o.verify_offset, hdr_size(hdr));
Shawn Lewis7437ee82007-08-02 21:05:58 +0200802 header_num++;
Jens Axboee29d1b72006-10-18 15:43:15 +0200803 }
Jens Axboee29d1b72006-10-18 15:43:15 +0200804}
805
806int get_next_verify(struct thread_data *td, struct io_u *io_u)
807{
Jens Axboe8de8f042007-03-27 10:36:12 +0200808 struct io_piece *ipo = NULL;
Jens Axboee29d1b72006-10-18 15:43:15 +0200809
Jens Axboed2d7fa52007-02-19 13:21:35 +0100810 /*
811 * this io_u is from a requeue, we already filled the offsets
812 */
813 if (io_u->file)
814 return 0;
815
Jens Axboe8de8f042007-03-27 10:36:12 +0200816 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) {
817 struct rb_node *n = rb_first(&td->io_hist_tree);
818
Jens Axboe4b878982007-03-26 09:32:22 +0200819 ipo = rb_entry(n, struct io_piece, rb_node);
Jens Axboe4b878982007-03-26 09:32:22 +0200820 rb_erase(n, &td->io_hist_tree);
Jens Axboea917a8b2010-09-02 13:23:20 +0200821 assert(ipo->flags & IP_F_ONRB);
822 ipo->flags &= ~IP_F_ONRB;
Jens Axboe01743ee2008-06-02 12:19:19 +0200823 } else if (!flist_empty(&td->io_hist_list)) {
824 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
825 flist_del(&ipo->list);
Jens Axboea917a8b2010-09-02 13:23:20 +0200826 assert(ipo->flags & IP_F_ONLIST);
827 ipo->flags &= ~IP_F_ONLIST;
Jens Axboe8de8f042007-03-27 10:36:12 +0200828 }
Jens Axboee29d1b72006-10-18 15:43:15 +0200829
Jens Axboe8de8f042007-03-27 10:36:12 +0200830 if (ipo) {
Jens Axboe0d29de82010-09-01 13:54:15 +0200831 td->io_hist_len--;
832
Jens Axboee29d1b72006-10-18 15:43:15 +0200833 io_u->offset = ipo->offset;
834 io_u->buflen = ipo->len;
Jens Axboe36167d82007-02-18 05:41:31 +0100835 io_u->file = ipo->file;
Jens Axboe97af62c2007-05-22 11:12:13 +0200836
Jens Axboe0d29de82010-09-01 13:54:15 +0200837 if (ipo->flags & IP_F_TRIMMED)
838 io_u->flags |= IO_U_F_TRIMMED;
839
Jens Axboed6aed792009-06-03 08:41:15 +0200840 if (!fio_file_open(io_u->file)) {
Jens Axboe97af62c2007-05-22 11:12:13 +0200841 int r = td_io_open_file(td, io_u->file);
842
Jens Axboebd6f78b2008-02-01 20:27:52 +0100843 if (r) {
844 dprint(FD_VERIFY, "failed file %s open\n",
845 io_u->file->file_name);
Jens Axboe97af62c2007-05-22 11:12:13 +0200846 return 1;
Jens Axboebd6f78b2008-02-01 20:27:52 +0100847 }
Jens Axboe97af62c2007-05-22 11:12:13 +0200848 }
849
850 get_file(ipo->file);
Jens Axboed6aed792009-06-03 08:41:15 +0200851 assert(fio_file_open(io_u->file));
Jens Axboee29d1b72006-10-18 15:43:15 +0200852 io_u->ddir = DDIR_READ;
Jens Axboe36167d82007-02-18 05:41:31 +0100853 io_u->xfer_buf = io_u->buf;
854 io_u->xfer_buflen = io_u->buflen;
Jens Axboe0d29de82010-09-01 13:54:15 +0200855
856 remove_trim_entry(td, ipo);
Jens Axboee29d1b72006-10-18 15:43:15 +0200857 free(ipo);
Jens Axboebd6f78b2008-02-01 20:27:52 +0100858 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u);
Jens Axboee29d1b72006-10-18 15:43:15 +0200859 return 0;
860 }
861
Jens Axboebd6f78b2008-02-01 20:27:52 +0100862 dprint(FD_VERIFY, "get_next_verify: empty\n");
Jens Axboee29d1b72006-10-18 15:43:15 +0200863 return 1;
864}
Jens Axboee8462bd2009-07-06 12:59:04 +0200865
866static void *verify_async_thread(void *data)
867{
868 struct thread_data *td = data;
869 struct io_u *io_u;
870 int ret = 0;
871
872 if (td->o.verify_cpumask_set &&
873 fio_setaffinity(td->pid, td->o.verify_cpumask)) {
874 log_err("fio: failed setting verify thread affinity\n");
875 goto done;
876 }
877
878 do {
Jens Axboee53ab272009-07-06 13:43:46 +0200879 FLIST_HEAD(list);
880
Jens Axboee8462bd2009-07-06 12:59:04 +0200881 read_barrier();
882 if (td->verify_thread_exit)
883 break;
884
885 pthread_mutex_lock(&td->io_u_lock);
886
887 while (flist_empty(&td->verify_list) &&
888 !td->verify_thread_exit) {
Jens Axboeb36e2982009-07-06 14:12:52 +0200889 ret = pthread_cond_wait(&td->verify_cond,
890 &td->io_u_lock);
Jens Axboee8462bd2009-07-06 12:59:04 +0200891 if (ret) {
892 pthread_mutex_unlock(&td->io_u_lock);
893 break;
894 }
895 }
896
Jens Axboee53ab272009-07-06 13:43:46 +0200897 flist_splice_init(&td->verify_list, &list);
Jens Axboee8462bd2009-07-06 12:59:04 +0200898 pthread_mutex_unlock(&td->io_u_lock);
899
Jens Axboee53ab272009-07-06 13:43:46 +0200900 if (flist_empty(&list))
901 continue;
902
903 while (!flist_empty(&list)) {
904 io_u = flist_entry(list.next, struct io_u, list);
905 flist_del_init(&io_u->list);
906
Jens Axboed561f2a2009-07-06 13:51:05 +0200907 ret = verify_io_u(td, io_u);
Jens Axboee53ab272009-07-06 13:43:46 +0200908 put_io_u(td, io_u);
Jens Axboed561f2a2009-07-06 13:51:05 +0200909 if (!ret)
910 continue;
911 if (td->o.continue_on_error &&
912 td_non_fatal_error(ret)) {
913 update_error_count(td, ret);
914 td_clear_error(td);
915 ret = 0;
916 }
Jens Axboee53ab272009-07-06 13:43:46 +0200917 }
Jens Axboee8462bd2009-07-06 12:59:04 +0200918 } while (!ret);
919
Jens Axboed561f2a2009-07-06 13:51:05 +0200920 if (ret) {
921 td_verror(td, ret, "async_verify");
Jens Axboef3e6cb92010-06-18 14:48:43 +0200922 if (td->o.verify_fatal)
923 td->terminate = 1;
Jens Axboed561f2a2009-07-06 13:51:05 +0200924 }
925
Jens Axboee8462bd2009-07-06 12:59:04 +0200926done:
927 pthread_mutex_lock(&td->io_u_lock);
928 td->nr_verify_threads--;
929 pthread_mutex_unlock(&td->io_u_lock);
930
931 pthread_cond_signal(&td->free_cond);
932 return NULL;
933}
934
935int verify_async_init(struct thread_data *td)
936{
937 int i, ret;
bart Van Assche304a47c2010-08-01 21:31:26 +0200938 pthread_attr_t attr;
939
940 pthread_attr_init(&attr);
941 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN);
Jens Axboee8462bd2009-07-06 12:59:04 +0200942
943 td->verify_thread_exit = 0;
944
945 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async);
946 for (i = 0; i < td->o.verify_async; i++) {
bart Van Assche304a47c2010-08-01 21:31:26 +0200947 ret = pthread_create(&td->verify_threads[i], &attr,
Jens Axboee8462bd2009-07-06 12:59:04 +0200948 verify_async_thread, td);
949 if (ret) {
950 log_err("fio: async verify creation failed: %s\n",
951 strerror(ret));
952 break;
953 }
954 ret = pthread_detach(td->verify_threads[i]);
955 if (ret) {
956 log_err("fio: async verify thread detach failed: %s\n",
957 strerror(ret));
958 break;
959 }
960 td->nr_verify_threads++;
961 }
962
bart Van Assche304a47c2010-08-01 21:31:26 +0200963 pthread_attr_destroy(&attr);
964
Jens Axboee8462bd2009-07-06 12:59:04 +0200965 if (i != td->o.verify_async) {
Jens Axboee40823b2009-07-06 14:28:21 +0200966 log_err("fio: only %d verify threads started, exiting\n", i);
Jens Axboee8462bd2009-07-06 12:59:04 +0200967 td->verify_thread_exit = 1;
968 write_barrier();
969 pthread_cond_broadcast(&td->verify_cond);
970 return 1;
971 }
972
973 return 0;
974}
975
976void verify_async_exit(struct thread_data *td)
977{
978 td->verify_thread_exit = 1;
979 write_barrier();
980 pthread_cond_broadcast(&td->verify_cond);
981
982 pthread_mutex_lock(&td->io_u_lock);
983
984 while (td->nr_verify_threads)
985 pthread_cond_wait(&td->free_cond, &td->io_u_lock);
986
987 pthread_mutex_unlock(&td->io_u_lock);
988 free(td->verify_threads);
989 td->verify_threads = NULL;
990}