Merge branch 'master' of ssh://git.kernel.dk/data/git/fio
diff --git a/HOWTO b/HOWTO
index 3d92293..66ebebe 100644
--- a/HOWTO
+++ b/HOWTO
@@ -482,6 +482,12 @@
 		fio doesn't track potential block rewrites which may alter
 		the calculated checksum for that block.
 
+softrandommap	See norandommap. If fio runs with the random block map enabled
+		and it fails to allocate the map, if this option is set it
+		will continue without a random block map. As coverage will
+		not be as complete as with random maps, this option is
+		disabled by default.
+
 nice=int	Run the job with the given nice value. See man nice(2).
 
 prio=int	Set the io priority value of this job. Linux limits us to
diff --git a/README b/README
index 7d6262e..db8e672 100644
--- a/README
+++ b/README
@@ -76,6 +76,7 @@
 	--eta=when	When ETA estimate should be printed
 			May be "always", "never" or "auto"
 	--section=name	Only run specified section in job file
+	--alloc-size=kb	Set smalloc pool to this size in kb (def 1024)
 
 
 Any parameters following the options will be assumed to be job files,
@@ -114,6 +115,12 @@
 option only applies to job sections, the reserved 'global' section is
 always parsed and taken into account.
 
+Fio has an internal allocator for shared memory called smalloc. It
+allocates shared structures from this pool. The pool defaults to 1024k
+in size, and can grow to 32 pools. If running large jobs with randommap
+enabled it can run out of memory, in which case the --alloc-size switch
+is handy for starting with a larger pool size.
+
 
 Job file
 --------
diff --git a/filesetup.c b/filesetup.c
index 98479fd..4cffd76 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -33,12 +33,14 @@
 	 * does that for operations involving reads, or for writes
 	 * where overwrite is set
 	 */
-	if (td_read(td) || (td_write(td) && td->o.overwrite))
+	if (td_read(td) || (td_write(td) && td->o.overwrite) ||
+	    (td_write(td) && td->io_ops->flags & FIO_NOEXTEND))
 		new_layout = 1;
 	if (td_write(td) && !td->o.overwrite)
 		unlink_file = 1;
 
 	if (unlink_file || new_layout) {
+		dprint(FD_FILE, "layout unlink %s\n", f->file_name);
 		if ((unlink(f->file_name) < 0) && (errno != ENOENT)) {
 			td_verror(td, errno, "unlink");
 			return 1;
@@ -97,9 +99,10 @@
 		}
 	}
 
-	if (td->terminate)
+	if (td->terminate) {
+		dprint(FD_FILE, "terminate unlink %s\n", f->file_name);
 		unlink(f->file_name);
-	else if (td->o.create_fsync) {
+	} else if (td->o.create_fsync) {
 		if (fsync(f->fd) < 0) {
 			td_verror(td, errno, "fsync");
 			goto err;
@@ -597,13 +600,21 @@
 		num_maps = (blocks + BLOCKS_PER_MAP - 1) /
 				(unsigned long long) BLOCKS_PER_MAP;
 		f->file_map = smalloc(num_maps * sizeof(long));
-		if (!f->file_map) {
+		if (f->file_map) {
+			f->num_maps = num_maps;
+			continue;
+		}
+		if (!td->o.softrandommap) {
 			log_err("fio: failed allocating random map. If running"
 				" a large number of jobs, try the 'norandommap'"
-				" option\n");
+				" option or set 'softrandommap'. Or give"
+				" a larger --alloc-size to fio.\n");
 			return 1;
 		}
-		f->num_maps = num_maps;
+
+		log_info("fio: file %s failed allocating random map. Running "
+			 "job without.\n", f->file_name);
+		f->num_maps = 0;
 	}
 
 	return 0;
@@ -626,8 +637,10 @@
 	dprint(FD_FILE, "close files\n");
 
 	for_each_file(td, f, i) {
-		if (td->o.unlink && f->filetype == FIO_TYPE_FILE)
+		if (td->o.unlink && f->filetype == FIO_TYPE_FILE) {
+			dprint(FD_FILE, "free unlink %s\n", f->file_name);
 			unlink(f->file_name);
+		}
 
 		td_io_close_file(td, f);
 
diff --git a/fio.c b/fio.c
index 1997bb7..231a581 100644
--- a/fio.c
+++ b/fio.c
@@ -74,10 +74,12 @@
 	struct thread_data *td;
 	int i;
 
+	dprint(FD_PROCESS, "terminate group_id=%d\n", group_id);
+
 	for_each_td(td, i) {
 		if (group_id == TERMINATE_ALL || groupid == td->groupid) {
-			dprint(FD_PROCESS, "setting terminate on %d\n",
-								td->pid);
+			dprint(FD_PROCESS, "setting terminate on %s/%d\n",
+							td->o.name, td->pid);
 			td->terminate = 1;
 			td->o.start_delay = 0;
 
@@ -1111,7 +1113,8 @@
 		(*nr_running)--;
 		(*m_rate) -= td->o.ratemin;
 		(*t_rate) -= td->o.rate;
-		pending--;
+		if (!td->pid)
+			pending--;
 
 		if (td->error)
 			exit_value++;
@@ -1222,8 +1225,11 @@
 					continue;
 			}
 
-			if (td->o.stonewall && (nr_started || nr_running))
+			if (td->o.stonewall && (nr_started || nr_running)) {
+				dprint(FD_PROCESS, "%s: stonewall wait\n",
+							td->o.name);
 				break;
+			}
 
 			/*
 			 * Set state to created. Thread will transition
diff --git a/fio.h b/fio.h
index 34f02d6..b2930f1 100644
--- a/fio.h
+++ b/fio.h
@@ -446,6 +446,7 @@
 	unsigned int write_lat_log;
 	unsigned int write_bw_log;
 	unsigned int norandommap;
+	unsigned int softrandommap;
 	unsigned int bs_unaligned;
 	unsigned int fsync_on_close;
 
@@ -691,6 +692,7 @@
 #define td_write(td)		((td)->o.td_ddir & TD_DDIR_WRITE)
 #define td_rw(td)		(((td)->o.td_ddir & TD_DDIR_RW) == TD_DDIR_RW)
 #define td_random(td)		((td)->o.td_ddir & TD_DDIR_RAND)
+#define file_randommap(td, f)	(!(td)->o.norandommap && (f)->file_map)
 
 static inline void fio_ro_check(struct thread_data *td, struct io_u *io_u)
 {
diff --git a/init.c b/init.c
index effe63c..1dfaa1f 100644
--- a/init.c
+++ b/init.c
@@ -119,6 +119,11 @@
 		.val		= 'x',
 	},
 	{
+		.name		= "alloc-size",
+		.has_arg	= required_argument,
+		.val		= 'a',
+	},
+	{
 		.name		= NULL,
 	},
 };
@@ -826,6 +831,8 @@
 	printf("\t--readonly\tTurn on safety read-only checks, preventing"
 		" writes\n");
 	printf("\t--section=name\tOnly run specified section in job file\n");
+	printf("\t--alloc-size=kb\tSet smalloc pool to this size in kb"
+		" (def 1024)\n");
 }
 
 #ifdef FIO_INC_DEBUG
@@ -897,6 +904,9 @@
 
 	while ((c = getopt_long_only(argc, argv, "", l_opts, &lidx)) != -1) {
 		switch (c) {
+		case 'a':
+			smalloc_pool_size = atoi(optarg);
+			break;
 		case 't':
 			def_timeout = atoi(optarg);
 			break;
diff --git a/io_u.c b/io_u.c
index 1e60eee..7a23d2c 100644
--- a/io_u.c
+++ b/io_u.c
@@ -130,7 +130,7 @@
 		/*
 		 * if we are not maintaining a random map, we are done.
 		 */
-		if (td->o.norandommap)
+		if (!file_randommap(td, f))
 			return 0;
 
 		/*
@@ -416,7 +416,7 @@
 	/*
 	 * mark entry before potentially trimming io_u
 	 */
-	if (td_random(td) && !td->o.norandommap)
+	if (td_random(td) && file_randommap(td, io_u->file))
 		mark_random_map(td, io_u);
 
 	/*
diff --git a/options.c b/options.c
index f05b786..ead31ac 100644
--- a/options.c
+++ b/options.c
@@ -5,6 +5,7 @@
 #include <string.h>
 #include <getopt.h>
 #include <assert.h>
+#include <libgen.h>
 
 #include "fio.h"
 #include "parse.h"
@@ -269,6 +270,30 @@
 	return 0;
 }
 
+static int check_dir(struct thread_data *td, char *fname)
+{
+	char file[PATH_MAX], *dir;
+	struct stat sb;
+
+	strcpy(file, fname);
+	dir = dirname(file);
+
+	if (lstat(dir, &sb) < 0) {
+		int ret = errno;
+
+		log_err("fio: %s is not a directory\n", dir);
+		td_verror(td, ret, "lstat");
+		return 1;
+	}
+
+	if (!S_ISDIR(sb.st_mode)) {
+		log_err("fio: %s is not a directory\n", dir);
+		return 1;
+	}
+
+	return 0;
+}
+
 static int str_filename_cb(void *data, const char *input)
 {
 	struct thread_data *td = data;
@@ -285,6 +310,10 @@
 	while ((fname = strsep(&str, ":")) != NULL) {
 		if (!strlen(fname))
 			break;
+		if (check_dir(td, fname)) {
+			free(p);
+			return 1;
+		}
 		add_file(td, fname);
 		td->o.nr_files++;
 	}
@@ -299,8 +328,10 @@
 	struct stat sb;
 
 	if (lstat(td->o.directory, &sb) < 0) {
+		int ret = errno;
+
 		log_err("fio: %s is not a directory\n", td->o.directory);
-		td_verror(td, errno, "lstat");
+		td_verror(td, ret, "lstat");
 		return 1;
 	}
 	if (!S_ISDIR(sb.st_mode)) {
@@ -638,6 +669,14 @@
 		.parent = "rw",
 	},
 	{
+		.name	= "softrandommap",
+		.type	= FIO_OPT_BOOL,
+		.off1	= td_var_offset(softrandommap),
+		.help	= "Allow randommap to fail and continue witout",
+		.parent	= "norandommap",
+		.def	= "0",
+	},
+	{
 		.name	= "nrfiles",
 		.type	= FIO_OPT_INT,
 		.off1	= td_var_offset(nr_files),
diff --git a/smalloc.c b/smalloc.c
index b7502dc..5baf5a4 100644
--- a/smalloc.c
+++ b/smalloc.c
@@ -16,11 +16,13 @@
 #undef ENABLE_RESIZE		/* define to enable pool resizing */
 #define MP_SAFE			/* define to made allocator thread safe */
 
-#define INITIAL_SIZE	65536	/* new pool size */
+#define INITIAL_SIZE	1048576	/* new pool size */
 #define MAX_POOLS	32	/* maximum number of pools to setup */
 
+unsigned int smalloc_pool_size = INITIAL_SIZE;
+
 #ifdef ENABLE_RESIZE
-#define MAX_SIZE	8 * INITIAL_SIZE
+#define MAX_SIZE	8 * smalloc_pool_size
 static unsigned int resize_error;
 #endif
 
@@ -229,7 +231,7 @@
 	if (fd < 0)
 		goto out_close;
 
-	pool->size = INITIAL_SIZE;
+	pool->size = smalloc_pool_size;
 	if (ftruncate(fd, pool->size) < 0)
 		goto out_unlink;
 
diff --git a/smalloc.h b/smalloc.h
index d5d4557..6905c6a 100644
--- a/smalloc.h
+++ b/smalloc.h
@@ -7,4 +7,6 @@
 extern void sinit(void);
 extern void scleanup(void);
 
+extern unsigned int smalloc_pool_size;
+
 #endif