Allow explicit setting of a number of files
We currently only allow filename=foo for one file, add the possibility
to specify any number of files by seperating with a colon.
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
diff --git a/HOWTO b/HOWTO
index f9e3fc7..5df2607 100644
--- a/HOWTO
+++ b/HOWTO
@@ -205,7 +205,11 @@
files between threads in a job or several jobs, specify
a filename for each of them to override the default. If
the ioengine used is 'net', the filename is the host and
- port to connect to in the format of =host:port.
+ port to connect to in the format of =host:port. If the
+ ioengine is file based, you can specify a number of files
+ by seperating the names with a ':' colon. So if you wanted
+ a job to open /dev/sda and /dev/sdb as the two working files,
+ you would use filename=/dev/sda:/dev/sdb
rw=str Type of io pattern. Accepted values are:
diff --git a/engines/cpu.c b/engines/cpu.c
index 7a084e0..b6f9075 100644
--- a/engines/cpu.c
+++ b/engines/cpu.c
@@ -10,7 +10,7 @@
static int fio_cpuio_setup(struct thread_data fio_unused *td)
{
struct fio_file *f;
- int i;
+ unsigned int i;
td->total_file_size = -1;
td->io_size = td->total_file_size;
diff --git a/engines/net.c b/engines/net.c
index 2e136f3..c2f45e5 100644
--- a/engines/net.c
+++ b/engines/net.c
@@ -223,8 +223,9 @@
unsigned short port;
struct fio_file *f;
char host[64], buf[128];
+ unsigned int i;
char *sep;
- int ret, i;
+ int ret;
if (!td->total_file_size) {
log_err("fio: need size= set\n");
diff --git a/engines/null.c b/engines/null.c
index 67ac453..42a780a 100644
--- a/engines/null.c
+++ b/engines/null.c
@@ -61,7 +61,7 @@
static int fio_null_setup(struct thread_data *td)
{
struct fio_file *f;
- int i;
+ unsigned int i;
if (!td->total_file_size) {
log_err("fio: need size= set\n");
diff --git a/engines/sg.c b/engines/sg.c
index 3a6a248..98ecbd1 100644
--- a/engines/sg.c
+++ b/engines/sg.c
@@ -69,8 +69,9 @@
*/
struct fio_file *f = &td->files[0];
struct sgio_data *sd = td->io_ops->data;
- int left = max, ret, events, i, r = 0;
+ int left = max, ret, r = 0;
void *buf = sd->sgbuf;
+ unsigned int i, events;
/*
* Fill in the file descriptors
@@ -191,7 +192,7 @@
{
struct fio_file *f = io_u->file;
- if (td->filetype == FIO_TYPE_BD)
+ if (f->filetype == FIO_TYPE_BD)
return fio_sgio_ioctl_doio(td, f, io_u);
return fio_sgio_rw_doio(f, io_u, sync);
@@ -347,12 +348,12 @@
struct sgio_data *sd = td->io_ops->data;
unsigned int bs;
- if (td->filetype == FIO_TYPE_BD) {
+ if (f->filetype == FIO_TYPE_BD) {
if (ioctl(f->fd, BLKSSZGET, &bs) < 0) {
td_verror(td, errno, "ioctl");
return 1;
}
- } else if (td->filetype == FIO_TYPE_CHAR) {
+ } else if (f->filetype == FIO_TYPE_CHAR) {
int version, ret;
if (ioctl(f->fd, SG_GET_VERSION_NUM, &version) < 0) {
@@ -370,7 +371,7 @@
sd->bs = bs;
- if (td->filetype == FIO_TYPE_BD) {
+ if (f->filetype == FIO_TYPE_BD) {
td->io_ops->getevents = NULL;
td->io_ops->event = NULL;
}
diff --git a/filesetup.c b/filesetup.c
index 48b034a..948cecd 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -15,7 +15,7 @@
{
struct stat st;
- if (td->filetype != FIO_TYPE_FILE ||
+ if (f->filetype != FIO_TYPE_FILE ||
(td->io_ops->flags & FIO_DISKLESSIO))
return 0;
@@ -98,7 +98,8 @@
static int create_files(struct thread_data *td)
{
struct fio_file *f;
- int i, err, need_create, can_extend;
+ int err, need_create, can_extend;
+ unsigned int i;
for_each_file(td, f, i)
f->file_size = td->total_file_size / td->nr_files;
@@ -111,17 +112,20 @@
return 0;
need_create = 0;
- if (td->filetype == FIO_TYPE_FILE) {
- for_each_file(td, f, i) {
- int file_there = !file_ok(td, f);
+ for_each_file(td, f, i) {
+ int file_there;
- if (file_there && td_write(td) && !td->overwrite) {
- unlink(f->file_name);
- file_there = 0;
- }
+ if (f->filetype != FIO_TYPE_FILE)
+ continue;
- need_create += !file_there;
+ file_there = !file_ok(td, f);
+
+ if (file_there && td_write(td) && !td->overwrite) {
+ unlink(f->file_name);
+ file_there = 0;
}
+
+ need_create += !file_there;
}
if (!need_create)
@@ -204,9 +208,9 @@
{
int ret = 0;
- if (td->filetype == FIO_TYPE_FILE)
+ if (f->filetype == FIO_TYPE_FILE)
ret = file_size(td, f);
- else if (td->filetype == FIO_TYPE_BD)
+ else if (f->filetype == FIO_TYPE_BD)
ret = bdev_size(td, f);
else
f->real_file_size = -1;
@@ -234,11 +238,11 @@
*/
if (f->mmap)
ret = madvise(f->mmap, f->file_size, MADV_DONTNEED);
- else if (td->filetype == FIO_TYPE_FILE) {
+ else if (f->filetype == FIO_TYPE_FILE) {
ret = fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_DONTNEED);
- } else if (td->filetype == FIO_TYPE_BD) {
+ } else if (f->filetype == FIO_TYPE_BD) {
ret = blockdev_invalidate_cache(f->fd);
- } else if (td->filetype == FIO_TYPE_CHAR)
+ } else if (f->filetype == FIO_TYPE_CHAR)
ret = 0;
if (ret < 0) {
@@ -267,12 +271,12 @@
if (td_write(td) || td_rw(td)) {
flags |= O_RDWR;
- if (td->filetype == FIO_TYPE_FILE)
+ if (f->filetype == FIO_TYPE_FILE)
flags |= O_CREAT;
f->fd = open(f->file_name, flags, 0600);
} else {
- if (td->filetype == FIO_TYPE_CHAR)
+ if (f->filetype == FIO_TYPE_CHAR)
flags |= O_RDWR;
else
flags |= O_RDONLY;
@@ -316,7 +320,8 @@
int open_files(struct thread_data *td)
{
struct fio_file *f;
- int i, err = 0;
+ unsigned int i;
+ int err = 0;
for_each_file(td, f, i) {
err = td_io_open_file(td, f);
@@ -339,7 +344,8 @@
int setup_files(struct thread_data *td)
{
struct fio_file *f;
- int err, i;
+ unsigned int i;
+ int err;
/*
* if ioengine defines a setup() method, it's responsible for
@@ -385,13 +391,12 @@
void close_files(struct thread_data *td)
{
struct fio_file *f;
- int i;
+ unsigned int i;
for_each_file(td, f, i) {
if (!td->filename && f->unlink &&
- td->filetype == FIO_TYPE_FILE) {
+ f->filetype == FIO_TYPE_FILE) {
unlink(f->file_name);
- free(f->file_name);
f->file_name = NULL;
}
@@ -406,3 +411,35 @@
td->files = NULL;
td->nr_files = 0;
}
+
+static void get_file_type(struct thread_data *td, struct fio_file *f)
+{
+ struct stat sb;
+
+ f->filetype = FIO_TYPE_FILE;
+
+ if (!lstat(td->filename, &sb)) {
+ if (S_ISBLK(sb.st_mode))
+ f->filetype = FIO_TYPE_BD;
+ else if (S_ISCHR(sb.st_mode))
+ f->filetype = FIO_TYPE_CHAR;
+ }
+}
+
+void add_file(struct thread_data *td, const char *fname)
+{
+ int cur_files = td->open_files;
+ struct fio_file *f;
+
+ td->files = realloc(td->files, (cur_files + 1) * sizeof(*f));
+
+ f = &td->files[cur_files];
+ memset(f, 0, sizeof(*f));
+ f->fd = -1;
+ f->file_name = fname;
+
+ get_file_type(td, f);
+
+ td->open_files++;
+ td->nr_uniq_files = td->open_files;
+}
diff --git a/fio.c b/fio.c
index 6222a41..4d628b9 100644
--- a/fio.c
+++ b/fio.c
@@ -255,7 +255,8 @@
{
struct fio_file *f;
struct io_u *io_u;
- int ret, i, min_events;
+ int ret, min_events;
+ unsigned int i;
/*
* sync io first and invalidate cache, to make sure we really
@@ -379,7 +380,8 @@
{
struct timeval s;
unsigned long usec;
- int i, ret = 0;
+ unsigned int i;
+ int ret = 0;
td_set_runstate(td, TD_RUNNING);
@@ -650,7 +652,8 @@
static int clear_io_state(struct thread_data *td)
{
struct fio_file *f;
- int i, ret;
+ unsigned int i;
+ int ret;
td->ts.stat_io_bytes[0] = td->ts.stat_io_bytes[1] = 0;
td->this_io_bytes[0] = td->this_io_bytes[1] = 0;
diff --git a/fio.h b/fio.h
index 0ca23cc..3b902a8 100644
--- a/fio.h
+++ b/fio.h
@@ -223,6 +223,8 @@
* this structure holds state information for a single file.
*/
struct fio_file {
+ enum fio_filetype filetype;
+
/*
* A file may not be a file descriptor, let the io engine decide
*/
@@ -230,7 +232,7 @@
unsigned long file_data;
int fd;
};
- char *file_name;
+ const char *file_name;
void *mmap;
unsigned long long file_size;
unsigned long long real_file_size;
@@ -314,7 +316,6 @@
int thread_number;
int groupid;
struct thread_stat ts;
- enum fio_filetype filetype;
struct fio_file *files;
unsigned int nr_files;
unsigned int nr_open_files;
@@ -648,6 +649,7 @@
extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_file *);
extern int __must_check generic_open_file(struct thread_data *, struct fio_file *);
extern void generic_close_file(struct thread_data *, struct fio_file *);
+extern void add_file(struct thread_data *, const char *);
/*
* ETA/status stuff
@@ -764,7 +766,7 @@
#define for_each_td(td, i) \
for ((i) = 0, (td) = &threads[0]; (i) < (int) thread_number; (i)++, (td)++)
#define for_each_file(td, f, i) \
- for ((i) = 0, (f) = &(td)->files[0]; (i) < (int) (td)->open_files; (i)++, (f)++)
+ for ((i) = 0, (f) = &(td)->files[0]; (i) < (td)->open_files; (i)++, (f)++)
#define fio_assert(td, cond) do { \
if (!(cond)) { \
diff --git a/init.c b/init.c
index 624d284..f3c756e 100644
--- a/init.c
+++ b/init.c
@@ -31,6 +31,8 @@
static int str_exitall_cb(void);
static int str_cpumask_cb(void *, unsigned int *);
static int str_fst_cb(void *, const char *);
+static int str_filename_cb(void *, const char *);
+static int str_directory_cb(void *, const char *);
#define __stringify_1(x) #x
#define __stringify(x) __stringify_1(x)
@@ -55,13 +57,15 @@
.name = "directory",
.type = FIO_OPT_STR_STORE,
.off1 = td_var_offset(directory),
+ .cb = str_directory_cb,
.help = "Directory to store files in",
},
{
.name = "filename",
.type = FIO_OPT_STR_STORE,
.off1 = td_var_offset(filename),
- .help = "Force the use of a specific file",
+ .cb = str_filename_cb,
+ .help = "File(s) to use for the workload",
},
{
.name = "rw",
@@ -722,12 +726,6 @@
log_err("fio: bs_unaligned may not work with raw io\n");
/*
- * O_DIRECT and char doesn't mix, clear that flag if necessary.
- */
- if (td->filetype == FIO_TYPE_CHAR && td->odirect)
- td->odirect = 0;
-
- /*
* thinktime_spin must be less than thinktime
*/
if (td->thinktime_spin > td->thinktime)
@@ -802,11 +800,11 @@
{
const char *ddir_str[] = { NULL, "read", "write", "rw", NULL,
"randread", "randwrite", "randrw" };
- struct stat sb;
- int numjobs, i;
+ unsigned int i;
struct fio_file *f;
const char *engine;
- int fn_given;
+ char fname[PATH_MAX];
+ int numjobs;
/*
* the def_thread is just for options, it's not a real job
@@ -829,65 +827,23 @@
if (td->odirect)
td->io_ops->flags |= FIO_RAWIO;
- fn_given = (long) td->filename;
- if (!td->filename)
+ if (!td->filename) {
td->filename = strdup(jobname);
- td->filetype = FIO_TYPE_FILE;
- if (!lstat(td->filename, &sb)) {
- if (S_ISBLK(sb.st_mode))
- td->filetype = FIO_TYPE_BD;
- else if (S_ISCHR(sb.st_mode))
- td->filetype = FIO_TYPE_CHAR;
+ for (i = 0; i < td->nr_files; i++) {
+ sprintf(fname, "%s.%d.%d", td->filename, td->thread_number, i);
+ add_file(td, fname);
+ }
}
fixup_options(td);
- if (fn_given)
- td->nr_uniq_files = 1;
- else
- td->nr_uniq_files = td->open_files;
-
- if (td->filetype == FIO_TYPE_FILE) {
- char tmp[PATH_MAX];
- int len = 0;
-
- if (td->directory && td->directory[0] != '\0') {
- if (lstat(td->directory, &sb) < 0) {
- log_err("fio: %s is not a directory\n", td->directory);
- td_verror(td, errno, "lstat");
- return 1;
- }
- if (!S_ISDIR(sb.st_mode)) {
- log_err("fio: %s is not a directory\n", td->directory);
- return 1;
- }
- len = sprintf(tmp, "%s/", td->directory);
- }
-
- td->files = malloc(sizeof(struct fio_file) * td->open_files);
-
- for_each_file(td, f, i) {
- memset(f, 0, sizeof(*f));
- f->fd = -1;
-
- if (fn_given)
- sprintf(tmp + len, "%s", td->filename);
- else
- sprintf(tmp + len, "%s.%d.%d", td->filename, td->thread_number, i);
- f->file_name = strdup(tmp);
- }
- } else {
- td->open_files = td->nr_files = 1;
- td->files = malloc(sizeof(struct fio_file));
- f = &td->files[0];
-
- memset(f, 0, sizeof(*f));
- f->fd = -1;
- f->file_name = strdup(td->filename);
- }
-
for_each_file(td, f, i) {
+ if (td->directory && f->filetype == FIO_TYPE_FILE) {
+ sprintf(fname, "%s/%s", td->directory, f->file_name);
+ f->file_name = strdup(fname);
+ }
+
f->file_size = td->total_file_size / td->nr_files;
f->file_offset = td->start_offset;
}
@@ -979,8 +935,9 @@
int init_random_state(struct thread_data *td)
{
unsigned long seeds[5];
- int fd, num_maps, blocks, i;
+ int fd, num_maps, blocks;
struct fio_file *f;
+ unsigned int i;
if (td->io_ops->flags & FIO_DISKLESSIO)
return 0;
@@ -1141,6 +1098,36 @@
return 0;
}
+static int str_filename_cb(void *data, const char *input)
+{
+ struct thread_data *td = data;
+ char *fname, *str;
+
+ str = strdup(input);
+ while ((fname = strsep(&str, ":")) != NULL)
+ add_file(td, fname);
+
+ return 0;
+}
+
+static int str_directory_cb(void *data, const char fio_unused *str)
+{
+ struct thread_data *td = data;
+ struct stat sb;
+
+ if (lstat(td->directory, &sb) < 0) {
+ log_err("fio: %s is not a directory\n", td->directory);
+ td_verror(td, errno, "lstat");
+ return 1;
+ }
+ if (!S_ISDIR(sb.st_mode)) {
+ log_err("fio: %s is not a directory\n", td->directory);
+ return 1;
+ }
+
+ return 0;
+}
+
/*
* This is our [ini] type file parser.
*/
diff --git a/parse.c b/parse.c
index 21fb3c2..e09b1bb 100644
--- a/parse.c
+++ b/parse.c
@@ -257,10 +257,20 @@
}
break;
}
- case FIO_OPT_STR_STORE:
+ case FIO_OPT_STR_STORE: {
+ fio_opt_str_fn *fn = o->cb;
+
cp = td_var(data, o->off1);
*cp = strdup(ptr);
+ if (fn) {
+ ret = fn(data, ptr);
+ if (ret) {
+ free(*cp);
+ *cp = NULL;
+ }
+ }
break;
+ }
case FIO_OPT_RANGE: {
char tmp[128];
char *p1, *p2;
@@ -579,7 +589,7 @@
}
if (!o->cb && !o->off1)
fprintf(stderr, "Option %s: neither cb nor offset given\n", o->name);
- if (o->type == FIO_OPT_STR)
+ if (o->type == FIO_OPT_STR || o->type == FIO_OPT_STR_STORE)
continue;
if (o->cb && (o->off1 || o->off2 || o->off3 || o->off4))
fprintf(stderr, "Option %s: both cb and offset given\n", o->name);