cpus_allowed_policy fixups

- Make 'split' roundrobin the CPUs, if we have more threads
  than CPUs in the set.

- Properly use a CPU index, don't assume a sequential set of
  CPUs.

Signed-off-by: Jens Axboe <axboe@fb.com>
diff --git a/options.c b/options.c
index c1a8f32..4ff4c9b 100644
--- a/options.c
+++ b/options.c
@@ -394,16 +394,23 @@
 }
 
 #ifdef FIO_HAVE_CPU_AFFINITY
-int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu)
+int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu_index)
 {
+	unsigned int i, index, cpus_in_mask;
 	const long max_cpu = cpus_online();
-	unsigned int i;
 
+	cpus_in_mask = fio_cpu_count(mask);
+	cpu_index = cpu_index % cpus_in_mask;
+
+	index = 0;
 	for (i = 0; i < max_cpu; i++) {
-		if (cpu != i) {
-			fio_cpu_clear(mask, i);
+		if (!fio_cpu_isset(mask, i))
 			continue;
-		}
+
+		if (cpu_index != index)
+			fio_cpu_clear(mask, i);
+
+		index++;
 	}
 
 	return fio_cpu_count(mask);
diff --git a/os/os-freebsd.h b/os/os-freebsd.h
index 402792a..e35c835 100644
--- a/os/os-freebsd.h
+++ b/os/os-freebsd.h
@@ -32,6 +32,7 @@
 
 #define fio_cpu_clear(mask, cpu)        (void) CPU_CLR((cpu), (mask))
 #define fio_cpu_set(mask, cpu)          (void) CPU_SET((cpu), (mask))
+#define fio_cpu_isset(mask, cpu)	CPU_ISSET((cpu), (mask))
 #define fio_cpu_count(maks)		CPU_COUNT((mask))
 
 static inline int fio_cpuset_init(os_cpu_mask_t *mask)
diff --git a/os/os-linux.h b/os/os-linux.h
index 3ed8c2e..ef80ce2 100644
--- a/os/os-linux.h
+++ b/os/os-linux.h
@@ -61,6 +61,7 @@
 
 #define fio_cpu_clear(mask, cpu)	(void) CPU_CLR((cpu), (mask))
 #define fio_cpu_set(mask, cpu)		(void) CPU_SET((cpu), (mask))
+#define fio_cpu_isset(mask, cpu)	CPU_ISSET((cpu), (mask))
 #define fio_cpu_count(maks)		CPU_COUNT((mask))
 
 static inline int fio_cpuset_init(os_cpu_mask_t *mask)
diff --git a/os/os-solaris.h b/os/os-solaris.h
index 7a0a3f0..c8896b8 100644
--- a/os/os-solaris.h
+++ b/os/os-solaris.h
@@ -103,6 +103,31 @@
 #define fio_cpu_clear(mask, cpu)	pset_assign(PS_NONE, (cpu), NULL)
 #define fio_cpu_set(mask, cpu)		pset_assign(*(mask), (cpu), NULL)
 
+static inline int fio_cpu_isset(os_cpu_mask_t *mask, int cpu)
+{
+	const unsigned int max_cpus = cpus_online();
+	processorid_t *cpus;
+	int i, ret;
+
+	cpus = malloc(sizeof(*cpus) * max_cpus);
+
+	if (pset_info(*mask, NULL, &num_cpus, cpus) < 0) {
+		free(cpus);
+		return 0;
+	}
+
+	ret = 0;
+	for (i = 0; i < max_cpus; i++) {
+		if (cpus[i] == cpu) {
+			ret = 1;
+			break;
+		}
+	}
+
+	free(cpus);
+	return ret;
+}
+
 static inline int fio_cpuset_init(os_cpu_mask_t *mask)
 {
 	if (pset_create(mask) < 0)
diff --git a/os/os-windows.h b/os/os-windows.h
index 243edc6..49f9606 100644
--- a/os/os-windows.h
+++ b/os/os-windows.h
@@ -215,6 +215,11 @@
 	*mask |= 1 << cpu;
 }
 
+static inline int fio_cpu_isset(os_cpu_mask_t *mask, int cpu)
+{
+	return (*mask & (1U << cpu));
+}
+
 static inline int fio_cpu_count(os_cpu_mask_t *mask)
 {
 	return hweight64(*mask);