Merge branch 'master' into gfio
Conflicts:
Makefile
Signed-off-by: Jens Axboe <axboe@kernel.dk>
diff --git a/.gitignore b/.gitignore
index 2457d65..3993a30 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,9 @@
-fio
-*.o
*.d
-.depend
-cscope.out
+*.o
+/.depend
+/FIO-VERSION-FILE
+/config-host.h
+/config-host.mak
+/config.log
+/cscope.out
+/fio
diff --git a/Makefile b/Makefile
index 4aa05fa..1755981 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,3 @@
-DEBUGFLAGS = -D_FORTIFY_SOURCE=2 -DFIO_INC_DEBUG
-CPPFLAGS= -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 $(DEBUGFLAGS)
-OPTFLAGS= -O3 -g -ffast-math $(EXTFLAGS)
-CFLAGS = -std=gnu99 -Wwrite-strings -Wall $(OPTFLAGS)
-LIBS = -lm -lz $(EXTLIBS)
-PROGS = fio
-SCRIPTS = fio_generate_plots
-
ifneq ($(wildcard config-host.mak),)
all:
include config-host.mak
@@ -20,11 +12,19 @@
include config-host.mak
endif
+DEBUGFLAGS = -D_FORTIFY_SOURCE=2 -DFIO_INC_DEBUG
+CPPFLAGS= -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 $(DEBUGFLAGS)
+OPTFLAGS= -O3 -g -ffast-math
+CFLAGS = -std=gnu99 -Wwrite-strings -Wall $(OPTFLAGS) $(EXTFLAGS) $(BUILD_CFLAGS)
+LIBS += -lm $(EXTLIBS)
+PROGS = fio
+SCRIPTS = fio_generate_plots
+
ifdef CONFIG_GFIO
PROGS += gfio
endif
-SOURCE := gettime.c ioengines.c init.c stat.c log.c time.c filesetup.c \
+SOURCE := gettime.c fio.c ioengines.c init.c stat.c log.c time.c filesetup.c \
eta.c verify.c memory.c io_u.c parse.c mutex.c options.c \
lib/rbtree.c smalloc.c filehash.c profile.c debug.c lib/rand.c \
lib/num2str.c lib/ieee754.c $(wildcard crc/*.c) engines/cpu.c \
diff --git a/backend.c b/backend.c
index 600f5ce..022122a 100644
--- a/backend.c
+++ b/backend.c
@@ -406,6 +406,15 @@
return 0;
}
+static void check_update_rusage(struct thread_data *td)
+{
+ if (td->update_rusage) {
+ td->update_rusage = 0;
+ update_rusage_stat(td);
+ fio_mutex_up(td->rusage_sem);
+ }
+}
+
/*
* The main verify engine. Runs over the writes we previously submitted,
* reads the blocks back in, and checks the crc/md5 of the data.
@@ -433,6 +442,8 @@
break;
}
+ check_update_rusage(td);
+
if (td->error)
return;
@@ -444,6 +455,7 @@
int ret2, full;
update_tv_cache(td);
+ check_update_rusage(td);
if (runtime_exceeded(td, &td->tv_cache)) {
__update_tv_cache(td);
@@ -597,6 +609,8 @@
break;
}
+ check_update_rusage(td);
+
if (!td->error) {
min_events = td->cur_depth;
@@ -652,6 +666,8 @@
int ret2, full;
enum fio_ddir ddir;
+ check_update_rusage(td);
+
if (td->terminate || td->done)
break;
@@ -816,6 +832,8 @@
}
}
+ check_update_rusage(td);
+
if (td->trim_entries)
log_err("fio: %d trim entries leaked?\n", td->trim_entries);
@@ -884,8 +902,7 @@
char *p;
max_units = td->o.iodepth;
- max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
- max_bs = max(td->o.max_bs[DDIR_TRIM], max_bs);
+ max_bs = td_max_bs(td);
min_write = td->o.min_bs[DDIR_WRITE];
td->orig_buffer_size = (unsigned long long) max_bs
* (unsigned long long) max_units;
@@ -1042,7 +1059,7 @@
* are done.
*/
diff = td->o.size - ddir_rw_sum(td->io_bytes);
- if (diff < td->o.rw_min_bs)
+ if (diff < td_max_bs(td))
return 0;
return 1;
@@ -1392,6 +1409,9 @@
if (td->o.write_iolog_file)
write_iolog_close(td);
+ fio_mutex_remove(td->rusage_sem);
+ td->rusage_sem = NULL;
+
td_set_runstate(td, TD_EXITED);
return (void *) (uintptr_t) td->error;
}
@@ -1645,6 +1665,9 @@
init_disk_util(td);
+ td->rusage_sem = fio_mutex_init(FIO_MUTEX_LOCKED);
+ td->update_rusage = 0;
+
/*
* Set state to created. Thread will transition
* to TD_INITIALIZED when it's done setting up.
diff --git a/configure b/configure
index 222befe..5dd683c 100755
--- a/configure
+++ b/configure
@@ -38,8 +38,8 @@
}
# Default CFLAGS
-CFLAGS="-D_GNU_SOURCE"
-EXTFLAGS="-include config-host.h"
+CFLAGS="-D_GNU_SOURCE -include config-host.h"
+BUILD_CFLAGS=""
# Print a helpful header at the top of config.log
echo "# FIO configure log $(date)" >> config.log
@@ -236,7 +236,7 @@
output_sym "CONFIG_SCHED_IDLE"
output_sym "CONFIG_TCP_NODELAY"
echo "CC=$CC" >> $config_host_mak
- echo "EXTFLAGS=$CFLAGS -include config-host.h -D_GNU_SOURCE" >> $config_host_mak
+ echo "BUILD_CFLAGS=$CFLAGS -include config-host.h -D_GNU_SOURCE" >> $config_host_mak
exit 0
;;
esac
@@ -816,6 +816,24 @@
echo "libnuma $libnuma"
##########################################
+# libnuma 2.x version API
+if test "$libnuma" = "yes" ; then
+libnuma_v2="no"
+cat > $TMPC << EOF
+#include <numa.h>
+int main(int argc, char **argv)
+{
+ struct bitmask *mask = numa_parse_nodestring(NULL);
+ return 0;
+}
+EOF
+if compile_prog "" "" "libnuma api"; then
+ libnuma_v2="yes"
+fi
+echo "libnuma v2 $libnuma_v2"
+fi
+
+##########################################
# strsep() probe
strsep="no"
cat > $TMPC << EOF
@@ -1047,7 +1065,7 @@
if test "$sfaa" = "yes" ; then
output_sym "CONFIG_SFAA"
fi
-if test "$libverbs" = "yes" -o "rdmacm" = "yes" ; then
+if test "$libverbs" = "yes" -a "rdmacm" = "yes" ; then
output_sym "CONFIG_RDMA"
fi
if test "$clock_gettime" = "yes" ; then
@@ -1094,7 +1112,7 @@
if test "$fusion_aw" = "yes" ; then
output_sym "CONFIG_FUSION_AW"
fi
-if test "$libnuma" = "yes" ; then
+if test "$libnuma_v2" = "yes" ; then
output_sym "CONFIG_LIBNUMA"
fi
if test "$solaris_aio" = "yes" ; then
@@ -1122,4 +1140,4 @@
echo "LIBS+=$LIBS" >> $config_host_mak
echo "CFLAGS+=$CFLAGS" >> $config_host_mak
echo "CC=$cc" >> $config_host_mak
-echo "EXTFLAGS=$EXTFLAGS $CFLAGS" >> $config_host_mak
+echo "BUILD_CFLAGS=$BUILD_CFLAGS $CFLAGS" >> $config_host_mak
diff --git a/engines/posixaio.c b/engines/posixaio.c
index a943e5b..1858e52 100644
--- a/engines/posixaio.c
+++ b/engines/posixaio.c
@@ -98,7 +98,7 @@
struct flist_head *entry;
struct timespec start;
int have_timeout = 0;
- int suspend_entries = 0;
+ int suspend_entries;
unsigned int r;
if (t && !fill_timespec(&start))
@@ -107,8 +107,9 @@
memset(&start, 0, sizeof(start));
r = 0;
- memset(suspend_list, 0, sizeof(*suspend_list));
restart:
+ memset(suspend_list, 0, sizeof(*suspend_list));
+ suspend_entries = 0;
flist_for_each(entry, &td->io_u_busylist) {
struct io_u *io_u = flist_entry(entry, struct io_u, list);
int err;
diff --git a/fio.h b/fio.h
index 05406cb..1cb44e6 100644
--- a/fio.h
+++ b/fio.h
@@ -113,6 +113,8 @@
uint64_t stat_io_blocks[DDIR_RWDIR_CNT];
struct timeval iops_sample_time;
+ volatile int update_rusage;
+ struct fio_mutex *rusage_sem;
struct rusage ru_start;
struct rusage ru_end;
@@ -568,6 +570,14 @@
return ret;
}
+static inline unsigned int td_max_bs(struct thread_data *td)
+{
+ unsigned int max_bs;
+
+ max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
+ return max(td->o.max_bs[DDIR_TRIM], max_bs);
+}
+
static inline int is_power_of_2(unsigned int val)
{
return (val != 0 && ((val & (val - 1)) == 0));
diff --git a/ioengines.c b/ioengines.c
index f4eae04..93e7631 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -475,7 +475,9 @@
fio_file_set_closing(f);
disk_util_dec(f->du);
- unlock_file_all(td, f);
+
+ if (td->o.file_lock_mode != FILE_LOCK_NONE)
+ unlock_file_all(td, f);
return put_file(td, f);
}
diff --git a/lib/axmap.c b/lib/axmap.c
index eeb32d4..c9f3a4f 100644
--- a/lib/axmap.c
+++ b/lib/axmap.c
@@ -189,7 +189,6 @@
struct axmap_set_data {
unsigned int nr_bits;
unsigned int set_bits;
- unsigned int fail_ok;
};
static unsigned long bit_masks[] = {
@@ -229,10 +228,8 @@
* Mask off any potential overlap, only sets contig regions
*/
overlap = al->map[offset] & mask;
- if (overlap == mask) {
- assert(data->fail_ok);
+ if (overlap == mask)
return 1;
- }
while (overlap) {
unsigned long clear_mask = ~(1UL << ffz(~overlap));
@@ -273,14 +270,14 @@
axmap_handler(axmap, bit_nr, axmap_set_fn, data);
set_bits += data->set_bits;
- if (data->set_bits != (BLOCKS_PER_UNIT - nr_bits))
+ if (!data->set_bits ||
+ data->set_bits != (BLOCKS_PER_UNIT - nr_bits))
break;
nr_bits -= data->set_bits;
bit_nr += data->set_bits;
data->nr_bits = nr_bits;
- data->fail_ok = 1;
}
data->set_bits = set_bits;
@@ -295,10 +292,27 @@
unsigned int axmap_set_nr(struct axmap *axmap, uint64_t bit_nr, unsigned int nr_bits)
{
- struct axmap_set_data data = { .nr_bits = nr_bits, };
+ unsigned int set_bits = 0;
- __axmap_set(axmap, bit_nr, &data);
- return data.set_bits;
+ do {
+ struct axmap_set_data data = { .nr_bits = nr_bits, };
+ unsigned int max_bits, this_set;
+
+ max_bits = BLOCKS_PER_UNIT - (bit_nr & BLOCKS_PER_UNIT_MASK);
+ if (max_bits < nr_bits)
+ data.nr_bits = max_bits;
+
+ this_set = data.nr_bits;
+ __axmap_set(axmap, bit_nr, &data);
+ set_bits += data.set_bits;
+ if (data.set_bits != this_set)
+ break;
+
+ nr_bits -= data.set_bits;
+ bit_nr += data.set_bits;
+ } while (nr_bits);
+
+ return set_bits;
}
static int axmap_isset_fn(struct axmap_level *al, unsigned long offset,
diff --git a/mutex.c b/mutex.c
index 332e9f9..e1fbb60 100644
--- a/mutex.c
+++ b/mutex.c
@@ -7,6 +7,7 @@
#include <errno.h>
#include <pthread.h>
#include <sys/mman.h>
+#include <assert.h>
#include "fio.h"
#include "log.h"
@@ -19,6 +20,7 @@
void fio_mutex_remove(struct fio_mutex *mutex)
{
+ assert(mutex->magic == FIO_MUTEX_MAGIC);
pthread_cond_destroy(&mutex->cond);
munmap((void *) mutex, sizeof(*mutex));
}
@@ -40,6 +42,7 @@
}
mutex->value = value;
+ mutex->magic = FIO_MUTEX_MAGIC;
ret = pthread_mutexattr_init(&attr);
if (ret) {
@@ -92,6 +95,8 @@
struct timespec t;
int ret = 0;
+ assert(mutex->magic == FIO_MUTEX_MAGIC);
+
gettimeofday(&tv_s, NULL);
t.tv_sec = tv_s.tv_sec + seconds;
t.tv_nsec = tv_s.tv_usec * 1000;
@@ -122,6 +127,8 @@
void fio_mutex_down(struct fio_mutex *mutex)
{
+ assert(mutex->magic == FIO_MUTEX_MAGIC);
+
pthread_mutex_lock(&mutex->lock);
while (!mutex->value) {
@@ -136,6 +143,8 @@
void fio_mutex_up(struct fio_mutex *mutex)
{
+ assert(mutex->magic == FIO_MUTEX_MAGIC);
+
pthread_mutex_lock(&mutex->lock);
read_barrier();
if (!mutex->value && mutex->waiters)
@@ -146,27 +155,32 @@
void fio_rwlock_write(struct fio_rwlock *lock)
{
+ assert(lock->magic == FIO_RWLOCK_MAGIC);
pthread_rwlock_wrlock(&lock->lock);
}
void fio_rwlock_read(struct fio_rwlock *lock)
{
+ assert(lock->magic == FIO_RWLOCK_MAGIC);
pthread_rwlock_rdlock(&lock->lock);
}
void fio_rwlock_unlock(struct fio_rwlock *lock)
{
+ assert(lock->magic == FIO_RWLOCK_MAGIC);
pthread_rwlock_unlock(&lock->lock);
}
void fio_rwlock_remove(struct fio_rwlock *lock)
{
+ assert(lock->magic == FIO_RWLOCK_MAGIC);
munmap((void *) lock, sizeof(*lock));
}
struct fio_rwlock *fio_rwlock_init(void)
{
struct fio_rwlock *lock;
+ pthread_rwlockattr_t attr;
int ret;
lock = (void *) mmap(NULL, sizeof(struct fio_rwlock),
@@ -178,13 +192,35 @@
goto err;
}
- ret = pthread_rwlock_init(&lock->lock, NULL);
+ lock->magic = FIO_RWLOCK_MAGIC;
+
+ ret = pthread_rwlockattr_init(&attr);
if (ret) {
- log_err("pthread_rwlock_init: %s\n", strerror(ret));
+ log_err("pthread_rwlockattr_init: %s\n", strerror(ret));
goto err;
}
+#ifdef FIO_HAVE_PSHARED_MUTEX
+ ret = pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
+ if (ret) {
+ log_err("pthread_rwlockattr_setpshared: %s\n", strerror(ret));
+ goto destroy_attr;
+ }
+
+ ret = pthread_rwlock_init(&lock->lock, &attr);
+#else
+ ret = pthread_rwlock_init(&lock->lock, NULL);
+#endif
+
+ if (ret) {
+ log_err("pthread_rwlock_init: %s\n", strerror(ret));
+ goto destroy_attr;
+ }
+
+ pthread_rwlockattr_destroy(&attr);
return lock;
+destroy_attr:
+ pthread_rwlockattr_destroy(&attr);
err:
if (lock)
fio_rwlock_remove(lock);
diff --git a/mutex.h b/mutex.h
index 49a66e3..4f3486d 100644
--- a/mutex.h
+++ b/mutex.h
@@ -3,15 +3,20 @@
#include <pthread.h>
+#define FIO_MUTEX_MAGIC 0x4d555445U
+#define FIO_RWLOCK_MAGIC 0x52574c4fU
+
struct fio_mutex {
pthread_mutex_t lock;
pthread_cond_t cond;
int value;
int waiters;
+ int magic;
};
struct fio_rwlock {
pthread_rwlock_t lock;
+ int magic;
};
enum {
diff --git a/parse.c b/parse.c
index 9250133..a701a5b 100644
--- a/parse.c
+++ b/parse.c
@@ -139,6 +139,19 @@
}
}
+static int is_separator(char c)
+{
+ switch (c) {
+ case ':':
+ case '-':
+ case ',':
+ case '/':
+ return 1;
+ default:
+ return 0;
+ }
+}
+
static unsigned long long __get_mult_bytes(const char *p, void *data,
int *percent)
{
@@ -152,8 +165,13 @@
c = strdup(p);
- for (i = 0; i < strlen(c); i++)
+ for (i = 0; i < strlen(c); i++) {
c[i] = tolower(c[i]);
+ if (is_separator(c[i])) {
+ c[i] = '\0';
+ break;
+ }
+ }
if (!strncmp("pib", c, 3)) {
pow = 5;
diff --git a/stat.c b/stat.c
index c3d3c4a..402a73f 100644
--- a/stat.c
+++ b/stat.c
@@ -734,7 +734,7 @@
if (ovals)
free(ovals);
- if (!calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) {
+ if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) {
if (rs->agg[ddir]) {
p_of_agg = mean * 100 / (double) rs->agg[ddir];
if (p_of_agg > 100.0)
@@ -1350,13 +1350,21 @@
if (td_trim(td) && td->io_bytes[DDIR_TRIM])
td->ts.runtime[DDIR_TRIM] += rt[i];
- update_rusage_stat(td);
+ td->update_rusage = 1;
td->ts.io_bytes[DDIR_READ] = td->io_bytes[DDIR_READ];
td->ts.io_bytes[DDIR_WRITE] = td->io_bytes[DDIR_WRITE];
td->ts.io_bytes[DDIR_TRIM] = td->io_bytes[DDIR_TRIM];
td->ts.total_run_time = mtime_since(&td->epoch, &tv);
}
+ for_each_td(td, i) {
+ if (td->rusage_sem) {
+ td->update_rusage = 1;
+ fio_mutex_down(td->rusage_sem);
+ }
+ td->update_rusage = 0;
+ }
+
show_run_stats();
for_each_td(td, i) {
diff --git a/t/axmap.c b/t/axmap.c
index 7ab500f..57c585b 100644
--- a/t/axmap.c
+++ b/t/axmap.c
@@ -18,12 +18,111 @@
free(ptr);
}
-int main(int argc, char *argv[])
+static int test_regular(size_t size, int seed)
{
struct fio_lfsr lfsr;
- size_t osize, size = (1UL << 28) - 200;
struct axmap *map;
+ size_t osize;
uint64_t ff;
+ int err;
+
+ printf("Using %llu entries...", (unsigned long long) size);
+ fflush(stdout);
+
+ lfsr_init(&lfsr, size, seed, seed & 0xF);
+ map = axmap_new(size);
+ osize = size;
+ err = 0;
+
+ while (size--) {
+ uint64_t val;
+
+ if (lfsr_next(&lfsr, &val, osize)) {
+ printf("lfsr: short loop\n");
+ err = 1;
+ break;
+ }
+ if (axmap_isset(map, val)) {
+ printf("bit already set\n");
+ err = 1;
+ break;
+ }
+ axmap_set(map, val);
+ if (!axmap_isset(map, val)) {
+ printf("bit not set\n");
+ err = 1;
+ break;
+ }
+ }
+
+ if (err)
+ return err;
+
+ ff = axmap_next_free(map, osize);
+ if (ff != (uint64_t) -1ULL) {
+ printf("axmap_next_free broken: got %llu\n", (unsigned long long) ff);
+ return 1;
+ }
+
+ printf("pass!\n");
+ axmap_free(map);
+ return 0;
+}
+
+static int test_multi(size_t size, unsigned int bit_off)
+{
+ unsigned int map_size = size;
+ struct axmap *map;
+ uint64_t val = bit_off;
+ int i, err;
+
+ printf("Test multi %llu entries %u offset...", (unsigned long long) size, bit_off);
+ fflush(stdout);
+
+ map = axmap_new(map_size);
+ while (val + 128 <= map_size) {
+ err = 0;
+ for (i = val; i < val + 128; i++) {
+ if (axmap_isset(map, val + i)) {
+ printf("bit already set\n");
+ err = 1;
+ break;
+ }
+ }
+
+ if (err)
+ break;
+
+ err = axmap_set_nr(map, val, 128);
+ if (err != 128) {
+ printf("only set %u bits\n", err);
+ break;
+ }
+
+ err = 0;
+ for (i = 0; i < 128; i++) {
+ if (!axmap_isset(map, val + i)) {
+ printf("bit not set: %llu\n", (unsigned long long) val + i);
+ err = 1;
+ break;
+ }
+ }
+
+ val += 128;
+ if (err)
+ break;
+ }
+
+ if (!err)
+ printf("pass!\n");
+
+ axmap_free(map);
+ return err;
+}
+
+int main(int argc, char *argv[])
+{
+ size_t size = (1UL << 23) - 200;
int seed = 1;
if (argc > 1) {
@@ -32,35 +131,12 @@
seed = strtoul(argv[2], NULL, 10);
}
- printf("Using %llu entries\n", (unsigned long long) size);
-
- lfsr_init(&lfsr, size, seed, seed & 0xF);
- map = axmap_new(size);
- osize = size;
-
- while (size--) {
- uint64_t val;
-
- if (lfsr_next(&lfsr, &val, osize)) {
- printf("lfsr: short loop\n");
- break;
- }
- if (axmap_isset(map, val)) {
- printf("bit already set\n");
- break;
- }
- axmap_set(map, val);
- if (!axmap_isset(map, val)) {
- printf("bit not set\n");
- break;
- }
- }
-
- ff = axmap_next_free(map, osize);
- if (ff != (uint64_t) -1ULL) {
- printf("axmap_next_free broken: got %llu\n", (unsigned long long) ff);
+ if (test_regular(size, seed))
return 1;
- }
+ if (test_multi(size, 0))
+ return 2;
+ if (test_multi(size, 17))
+ return 3;
return 0;
}