Merge branch 'master' of ssh://git.kernel.dk/data/git/fio
diff --git a/HOWTO b/HOWTO
index 3d92293..66ebebe 100644
--- a/HOWTO
+++ b/HOWTO
@@ -482,6 +482,12 @@
fio doesn't track potential block rewrites which may alter
the calculated checksum for that block.
+softrandommap See norandommap. If fio runs with the random block map enabled
+ and it fails to allocate the map, if this option is set it
+ will continue without a random block map. As coverage will
+ not be as complete as with random maps, this option is
+ disabled by default.
+
nice=int Run the job with the given nice value. See man nice(2).
prio=int Set the io priority value of this job. Linux limits us to
diff --git a/README b/README
index 7d6262e..db8e672 100644
--- a/README
+++ b/README
@@ -76,6 +76,7 @@
--eta=when When ETA estimate should be printed
May be "always", "never" or "auto"
--section=name Only run specified section in job file
+ --alloc-size=kb Set smalloc pool to this size in kb (def 1024)
Any parameters following the options will be assumed to be job files,
@@ -114,6 +115,12 @@
option only applies to job sections, the reserved 'global' section is
always parsed and taken into account.
+Fio has an internal allocator for shared memory called smalloc. It
+allocates shared structures from this pool. The pool defaults to 1024k
+in size, and can grow to 32 pools. If running large jobs with randommap
+enabled it can run out of memory, in which case the --alloc-size switch
+is handy for starting with a larger pool size.
+
Job file
--------
diff --git a/filesetup.c b/filesetup.c
index 98479fd..4cffd76 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -33,12 +33,14 @@
* does that for operations involving reads, or for writes
* where overwrite is set
*/
- if (td_read(td) || (td_write(td) && td->o.overwrite))
+ if (td_read(td) || (td_write(td) && td->o.overwrite) ||
+ (td_write(td) && td->io_ops->flags & FIO_NOEXTEND))
new_layout = 1;
if (td_write(td) && !td->o.overwrite)
unlink_file = 1;
if (unlink_file || new_layout) {
+ dprint(FD_FILE, "layout unlink %s\n", f->file_name);
if ((unlink(f->file_name) < 0) && (errno != ENOENT)) {
td_verror(td, errno, "unlink");
return 1;
@@ -97,9 +99,10 @@
}
}
- if (td->terminate)
+ if (td->terminate) {
+ dprint(FD_FILE, "terminate unlink %s\n", f->file_name);
unlink(f->file_name);
- else if (td->o.create_fsync) {
+ } else if (td->o.create_fsync) {
if (fsync(f->fd) < 0) {
td_verror(td, errno, "fsync");
goto err;
@@ -597,13 +600,21 @@
num_maps = (blocks + BLOCKS_PER_MAP - 1) /
(unsigned long long) BLOCKS_PER_MAP;
f->file_map = smalloc(num_maps * sizeof(long));
- if (!f->file_map) {
+ if (f->file_map) {
+ f->num_maps = num_maps;
+ continue;
+ }
+ if (!td->o.softrandommap) {
log_err("fio: failed allocating random map. If running"
" a large number of jobs, try the 'norandommap'"
- " option\n");
+ " option or set 'softrandommap'. Or give"
+ " a larger --alloc-size to fio.\n");
return 1;
}
- f->num_maps = num_maps;
+
+ log_info("fio: file %s failed allocating random map. Running "
+ "job without.\n", f->file_name);
+ f->num_maps = 0;
}
return 0;
@@ -626,8 +637,10 @@
dprint(FD_FILE, "close files\n");
for_each_file(td, f, i) {
- if (td->o.unlink && f->filetype == FIO_TYPE_FILE)
+ if (td->o.unlink && f->filetype == FIO_TYPE_FILE) {
+ dprint(FD_FILE, "free unlink %s\n", f->file_name);
unlink(f->file_name);
+ }
td_io_close_file(td, f);
diff --git a/fio.c b/fio.c
index 1997bb7..231a581 100644
--- a/fio.c
+++ b/fio.c
@@ -74,10 +74,12 @@
struct thread_data *td;
int i;
+ dprint(FD_PROCESS, "terminate group_id=%d\n", group_id);
+
for_each_td(td, i) {
if (group_id == TERMINATE_ALL || groupid == td->groupid) {
- dprint(FD_PROCESS, "setting terminate on %d\n",
- td->pid);
+ dprint(FD_PROCESS, "setting terminate on %s/%d\n",
+ td->o.name, td->pid);
td->terminate = 1;
td->o.start_delay = 0;
@@ -1111,7 +1113,8 @@
(*nr_running)--;
(*m_rate) -= td->o.ratemin;
(*t_rate) -= td->o.rate;
- pending--;
+ if (!td->pid)
+ pending--;
if (td->error)
exit_value++;
@@ -1222,8 +1225,11 @@
continue;
}
- if (td->o.stonewall && (nr_started || nr_running))
+ if (td->o.stonewall && (nr_started || nr_running)) {
+ dprint(FD_PROCESS, "%s: stonewall wait\n",
+ td->o.name);
break;
+ }
/*
* Set state to created. Thread will transition
diff --git a/fio.h b/fio.h
index 34f02d6..b2930f1 100644
--- a/fio.h
+++ b/fio.h
@@ -446,6 +446,7 @@
unsigned int write_lat_log;
unsigned int write_bw_log;
unsigned int norandommap;
+ unsigned int softrandommap;
unsigned int bs_unaligned;
unsigned int fsync_on_close;
@@ -691,6 +692,7 @@
#define td_write(td) ((td)->o.td_ddir & TD_DDIR_WRITE)
#define td_rw(td) (((td)->o.td_ddir & TD_DDIR_RW) == TD_DDIR_RW)
#define td_random(td) ((td)->o.td_ddir & TD_DDIR_RAND)
+#define file_randommap(td, f) (!(td)->o.norandommap && (f)->file_map)
static inline void fio_ro_check(struct thread_data *td, struct io_u *io_u)
{
diff --git a/init.c b/init.c
index effe63c..1dfaa1f 100644
--- a/init.c
+++ b/init.c
@@ -119,6 +119,11 @@
.val = 'x',
},
{
+ .name = "alloc-size",
+ .has_arg = required_argument,
+ .val = 'a',
+ },
+ {
.name = NULL,
},
};
@@ -826,6 +831,8 @@
printf("\t--readonly\tTurn on safety read-only checks, preventing"
" writes\n");
printf("\t--section=name\tOnly run specified section in job file\n");
+ printf("\t--alloc-size=kb\tSet smalloc pool to this size in kb"
+ " (def 1024)\n");
}
#ifdef FIO_INC_DEBUG
@@ -897,6 +904,9 @@
while ((c = getopt_long_only(argc, argv, "", l_opts, &lidx)) != -1) {
switch (c) {
+ case 'a':
+ smalloc_pool_size = atoi(optarg);
+ break;
case 't':
def_timeout = atoi(optarg);
break;
diff --git a/io_u.c b/io_u.c
index 1e60eee..7a23d2c 100644
--- a/io_u.c
+++ b/io_u.c
@@ -130,7 +130,7 @@
/*
* if we are not maintaining a random map, we are done.
*/
- if (td->o.norandommap)
+ if (!file_randommap(td, f))
return 0;
/*
@@ -416,7 +416,7 @@
/*
* mark entry before potentially trimming io_u
*/
- if (td_random(td) && !td->o.norandommap)
+ if (td_random(td) && file_randommap(td, io_u->file))
mark_random_map(td, io_u);
/*
diff --git a/options.c b/options.c
index f05b786..ead31ac 100644
--- a/options.c
+++ b/options.c
@@ -5,6 +5,7 @@
#include <string.h>
#include <getopt.h>
#include <assert.h>
+#include <libgen.h>
#include "fio.h"
#include "parse.h"
@@ -269,6 +270,30 @@
return 0;
}
+static int check_dir(struct thread_data *td, char *fname)
+{
+ char file[PATH_MAX], *dir;
+ struct stat sb;
+
+ strcpy(file, fname);
+ dir = dirname(file);
+
+ if (lstat(dir, &sb) < 0) {
+ int ret = errno;
+
+ log_err("fio: %s is not a directory\n", dir);
+ td_verror(td, ret, "lstat");
+ return 1;
+ }
+
+ if (!S_ISDIR(sb.st_mode)) {
+ log_err("fio: %s is not a directory\n", dir);
+ return 1;
+ }
+
+ return 0;
+}
+
static int str_filename_cb(void *data, const char *input)
{
struct thread_data *td = data;
@@ -285,6 +310,10 @@
while ((fname = strsep(&str, ":")) != NULL) {
if (!strlen(fname))
break;
+ if (check_dir(td, fname)) {
+ free(p);
+ return 1;
+ }
add_file(td, fname);
td->o.nr_files++;
}
@@ -299,8 +328,10 @@
struct stat sb;
if (lstat(td->o.directory, &sb) < 0) {
+ int ret = errno;
+
log_err("fio: %s is not a directory\n", td->o.directory);
- td_verror(td, errno, "lstat");
+ td_verror(td, ret, "lstat");
return 1;
}
if (!S_ISDIR(sb.st_mode)) {
@@ -638,6 +669,14 @@
.parent = "rw",
},
{
+ .name = "softrandommap",
+ .type = FIO_OPT_BOOL,
+ .off1 = td_var_offset(softrandommap),
+ .help = "Allow randommap to fail and continue witout",
+ .parent = "norandommap",
+ .def = "0",
+ },
+ {
.name = "nrfiles",
.type = FIO_OPT_INT,
.off1 = td_var_offset(nr_files),
diff --git a/smalloc.c b/smalloc.c
index b7502dc..5baf5a4 100644
--- a/smalloc.c
+++ b/smalloc.c
@@ -16,11 +16,13 @@
#undef ENABLE_RESIZE /* define to enable pool resizing */
#define MP_SAFE /* define to made allocator thread safe */
-#define INITIAL_SIZE 65536 /* new pool size */
+#define INITIAL_SIZE 1048576 /* new pool size */
#define MAX_POOLS 32 /* maximum number of pools to setup */
+unsigned int smalloc_pool_size = INITIAL_SIZE;
+
#ifdef ENABLE_RESIZE
-#define MAX_SIZE 8 * INITIAL_SIZE
+#define MAX_SIZE 8 * smalloc_pool_size
static unsigned int resize_error;
#endif
@@ -229,7 +231,7 @@
if (fd < 0)
goto out_close;
- pool->size = INITIAL_SIZE;
+ pool->size = smalloc_pool_size;
if (ftruncate(fd, pool->size) < 0)
goto out_unlink;
diff --git a/smalloc.h b/smalloc.h
index d5d4557..6905c6a 100644
--- a/smalloc.h
+++ b/smalloc.h
@@ -7,4 +7,6 @@
extern void sinit(void);
extern void scleanup(void);
+extern unsigned int smalloc_pool_size;
+
#endif