Add job number specific dumping and fix atexit() error

The threads/process jobs should use _exit() so they don't run
the main atexit() function.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
diff --git a/README b/README
index ea8f5ef..53cc7eb 100644
--- a/README
+++ b/README
@@ -116,6 +116,7 @@
 	random		Dump info related to random offset generation
 	parse		Dump info related to option matching and parsing
 	diskutil	Dump info related to disk utilization updates
+	job:x		Dump info only related to job number x
 	? or help	Show available debug options.
 
 You can specify as many as you want, eg --debug=file,mem will enable
diff --git a/debug.h b/debug.h
index 22577a8..883628d 100644
--- a/debug.h
+++ b/debug.h
@@ -14,6 +14,7 @@
 	FD_RANDOM,
 	FD_PARSE,
 	FD_DISKUTIL,
+	FD_JOB,
 	FD_DEBUG_MAX,
 };
 
@@ -21,17 +22,24 @@
 struct debug_level {
 	const char *name;
 	unsigned long shift;
+	unsigned int jobno;
 };
 extern struct debug_level debug_levels[];
 
 extern unsigned long fio_debug;
+extern unsigned int fio_debug_jobno, *fio_debug_jobp;
 
 #define dprint(type, str, args...)				\
 	do {							\
+		pid_t pid = getpid();				\
 		assert(type < FD_DEBUG_MAX);			\
 		if ((((1 << type)) & fio_debug) == 0)		\
 			break;					\
+		if (fio_debug_jobp && *fio_debug_jobp != -1U	\
+		    && pid != *fio_debug_jobp)			\
+			break;					\
 		log_info("%-8s ", debug_levels[(type)].name);	\
+		log_info("%-5u ", pid);				\
 		log_info(str, ##args);				\
 	} while (0)
 
diff --git a/filehash.c b/filehash.c
index b459220..54cea99 100644
--- a/filehash.c
+++ b/filehash.c
@@ -79,6 +79,23 @@
 	return alias;
 }
 
+void file_hash_exit(void)
+{
+	unsigned int i, has_entries = 0;
+
+	fio_mutex_down(hash_lock);
+	for (i = 0; i < HASH_BUCKETS; i++)
+		has_entries += !list_empty(&file_hash[i]);
+	fio_mutex_up(hash_lock);
+
+	if (has_entries)
+		log_err("fio: file hash not empty on exit\n");
+
+	file_hash = NULL;
+	fio_mutex_remove(hash_lock);
+	hash_lock = NULL;
+}
+
 void file_hash_init(void *ptr)
 {
 	unsigned int i;
diff --git a/filehash.h b/filehash.h
index a89a91f..993943a 100644
--- a/filehash.h
+++ b/filehash.h
@@ -4,6 +4,7 @@
 extern unsigned int file_hash_size;
 
 extern void file_hash_init(void *);
+extern void file_hash_exit(void);
 extern struct fio_file *lookup_file_hash(const char *);
 extern struct fio_file *add_file_hash(struct fio_file *);
 extern void remove_file_hash(struct fio_file *);
diff --git a/fio.c b/fio.c
index 90bad97..c17ea3c 100644
--- a/fio.c
+++ b/fio.c
@@ -1233,12 +1233,15 @@
 				if (pthread_detach(td->thread) < 0)
 					perror("pthread_detach");
 			} else {
+				pid_t pid;
 				dprint(FD_PROCESS, "will fork\n");
-				if (!fork()) {
+				pid = fork();
+				if (!pid) {
 					int ret = fork_main(shm_id, i);
 
-					exit(ret);
-				}
+					_exit(ret);
+				} else if (i == fio_debug_jobno)
+					*fio_debug_jobp = pid;
 			}
 			fio_mutex_down(startup_mutex);
 		}
diff --git a/init.c b/init.c
index 2a3fcee..3469584 100644
--- a/init.c
+++ b/init.c
@@ -47,6 +47,8 @@
 static int prev_group_jobs;
 
 unsigned long fio_debug = 0;
+unsigned int fio_debug_jobno = -1;
+unsigned int *fio_debug_jobp = NULL;
 
 /*
  * Command line options. These will contain the above, plus a few
@@ -760,8 +762,12 @@
 	struct shmid_ds sbuf;
 
 	if (threads) {
-		shmdt((void *) threads);
+		void *tp = threads;
+
 		threads = NULL;
+		file_hash_exit();
+		fio_debug_jobp = NULL;
+		shmdt(tp);
 		shmctl(shm_id, IPC_RMID, &sbuf);
 	}
 
@@ -786,6 +792,7 @@
 		size_t size = max_jobs * sizeof(struct thread_data);
 
 		size += file_hash_size;
+		size += sizeof(unsigned int);
 
 		shm_id = shmget(0, size, IPC_CREAT | 0600);
 		if (shm_id != -1)
@@ -809,6 +816,8 @@
 
 	memset(threads, 0, max_jobs * sizeof(struct thread_data));
 	hash = (void *) threads + max_jobs * sizeof(struct thread_data);
+	fio_debug_jobp = (void *) hash + file_hash_size;
+	*fio_debug_jobp = -1;
 	file_hash_init(hash);
 	atexit(free_shm);
 	return 0;
@@ -849,6 +858,7 @@
 	{ .name = "random",	.shift = FD_RANDOM },
 	{ .name = "parse",	.shift = FD_PARSE },
 	{ .name = "diskutil",	.shift = FD_DISKUTIL },
+	{ .name = "job",	.shift = FD_JOB },
 	{ },
 };
 
@@ -869,22 +879,38 @@
 		}
 		log_info("all\n");
 		return 1;
-	} else if (!strcmp(string, "all")) {
-		fio_debug = ~0UL;
-		return 0;
 	}
 
 	while ((opt = strsep(&p, ",")) != NULL) {
 		int found = 0;
 
+		if (!strncmp(opt, "all", 3)) {
+			log_info("fio: set all debug options\n");
+			fio_debug = ~0UL;
+			continue;
+		}
+
 		for (i = 0; debug_levels[i].name; i++) {
 			dl = &debug_levels[i];
-			if (!strncmp(opt, dl->name, strlen(opt))) {
+			found = !strncmp(opt, dl->name, strlen(dl->name));
+			if (!found)
+				continue;
+
+			if (dl->shift == FD_JOB) {
+				opt = strchr(opt, ':');
+				if (!opt) {
+					log_err("fio: missing job number\n");
+					break;
+				}
+				opt++;
+				fio_debug_jobno = atoi(opt);
+				log_info("fio: set debug jobno %d\n",
+							fio_debug_jobno);
+			} else {
 				log_info("fio: set debug option %s\n", opt);
-				found = 1;
 				fio_debug |= (1UL << dl->shift);
-				break;
 			}
+			break;
 		}
 
 		if (!found)