Add support for file appends
Add option file_append (bool). If set, fio will append to a file
instead of operate within the size of it. This is similar to
setting offset= to the size of each file.
Signed-off-by: Jens Axboe <axboe@fb.com>
diff --git a/HOWTO b/HOWTO
index 0f9c0fc..7e77c40 100644
--- a/HOWTO
+++ b/HOWTO
@@ -435,6 +435,11 @@
and limited to 'size' in total (if that is given). If not
given, each created file is the same size.
+file_append=bool Perform IO after the end of the file. Normally fio will
+ operate within the size of a file. If this option is set, then
+ fio will append to the file instead. This has identical
+ behavior to setting offset to the size of a file.
+
fill_device=bool
fill_fs=bool Sets size to something really large and waits for ENOSPC (no
space left on device) as the terminating condition. Only makes
diff --git a/cconv.c b/cconv.c
index 357a784..2843a71 100644
--- a/cconv.c
+++ b/cconv.c
@@ -79,6 +79,7 @@
o->size = le64_to_cpu(top->size);
o->size_percent = le32_to_cpu(top->size_percent);
o->fill_device = le32_to_cpu(top->fill_device);
+ o->file_append = le32_to_cpu(top->file_append);
o->file_size_low = le64_to_cpu(top->file_size_low);
o->file_size_high = le64_to_cpu(top->file_size_high);
o->start_offset = le64_to_cpu(top->start_offset);
@@ -280,6 +281,7 @@
top->iodepth_batch_complete = cpu_to_le32(o->iodepth_batch_complete);
top->size_percent = cpu_to_le32(o->size_percent);
top->fill_device = cpu_to_le32(o->fill_device);
+ top->file_append = cpu_to_le32(o->file_append);
top->ratecycle = cpu_to_le32(o->ratecycle);
top->nr_files = cpu_to_le32(o->nr_files);
top->open_files = cpu_to_le32(o->open_files);
diff --git a/file.h b/file.h
index c929d1d..add7773 100644
--- a/file.h
+++ b/file.h
@@ -159,7 +159,7 @@
struct thread_data;
extern void close_files(struct thread_data *);
extern void close_and_free_files(struct thread_data *);
-extern uint64_t get_start_offset(struct thread_data *);
+extern uint64_t get_start_offset(struct thread_data *, struct fio_file *);
extern int __must_check setup_files(struct thread_data *);
extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_file *);
extern int __must_check generic_open_file(struct thread_data *, struct fio_file *);
diff --git a/filesetup.c b/filesetup.c
index 4bfa470..c3c0fc4 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -722,8 +722,13 @@
return ret;
}
-uint64_t get_start_offset(struct thread_data *td)
+uint64_t get_start_offset(struct thread_data *td, struct fio_file *f)
{
+ struct thread_options *o = &td->o;
+
+ if (o->file_append && f->filetype == FIO_TYPE_FILE)
+ return f->real_file_size;
+
return td->o.start_offset +
(td->thread_number - 1) * td->o.offset_increment;
}
@@ -810,7 +815,7 @@
extend_size = total_size = 0;
need_extend = 0;
for_each_file(td, f, i) {
- f->file_offset = get_start_offset(td);
+ f->file_offset = get_start_offset(td, f);
if (!o->file_size_low) {
/*
diff --git a/fio.1 b/fio.1
index 20f0c09..fee5427 100644
--- a/fio.1
+++ b/fio.1
@@ -386,6 +386,12 @@
that is given). If \fBfilesize\fR is not specified, each created file is the
same size.
.TP
+.BI file_append \fR=\fPbool
+Perform IO after the end of the file. Normally fio will operate within the
+size of a file. If this option is set, then fio will append to the file
+instead. This has identical behavior to setting \fRoffset\fP to the size
+of a file.
+.TP
.BI blocksize \fR=\fPint[,int] "\fR,\fB bs" \fR=\fPint[,int]
Block size for I/O units. Default: 4k. Values for reads, writes, and trims
can be specified separately in the format \fIread\fR,\fIwrite\fR,\fItrim\fR
diff --git a/fio.h b/fio.h
index 52f1def..befdce3 100644
--- a/fio.h
+++ b/fio.h
@@ -71,6 +71,7 @@
TD_F_SCRAMBLE_BUFFERS = 16,
TD_F_VER_NONE = 32,
TD_F_PROFILE_OPS = 64,
+ TD_F_COMPRESS = 128,
};
enum {
diff --git a/init.c b/init.c
index 0a872c0..73ec9eb 100644
--- a/init.c
+++ b/init.c
@@ -856,11 +856,6 @@
return 0;
}
-static int compression_enabled(struct thread_options *o)
-{
- return o->compress_percentage || o->compress_chunk;
-}
-
static void init_flags(struct thread_data *td)
{
struct thread_options *o = &td->o;
@@ -873,14 +868,8 @@
td->flags |= TD_F_READ_IOLOG;
if (o->refill_buffers)
td->flags |= TD_F_REFILL_BUFFERS;
-
- /*
- * Don't scramble buffers if we set any of the compression
- * settings
- */
- if (o->scramble_buffers && !compression_enabled(o))
+ if (o->scramble_buffers)
td->flags |= TD_F_SCRAMBLE_BUFFERS;
-
if (o->verify != VERIFY_NONE)
td->flags |= TD_F_VER_NONE;
}
diff --git a/io_u.c b/io_u.c
index 0b86d9f..2f6aecf 100644
--- a/io_u.c
+++ b/io_u.c
@@ -273,7 +273,7 @@
{
assert(ddir_rw(ddir));
- if (f->last_pos >= f->io_size + get_start_offset(td) && td->o.time_based)
+ if (f->last_pos >= f->io_size + get_start_offset(td, f) && td->o.time_based)
f->last_pos = f->last_pos - f->io_size;
if (f->last_pos < f->real_file_size) {
@@ -415,7 +415,7 @@
{
struct fio_file *f = io_u->file;
- return io_u->offset + buflen <= f->io_size + get_start_offset(td);
+ return io_u->offset + buflen <= f->io_size + get_start_offset(td, f);
}
static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u,
@@ -1490,7 +1490,8 @@
if (td->flags & TD_F_REFILL_BUFFERS) {
io_u_fill_buffer(td, io_u,
io_u->xfer_buflen, io_u->xfer_buflen);
- } else if (td->flags & TD_F_SCRAMBLE_BUFFERS)
+ } else if ((td->flags & TD_F_SCRAMBLE_BUFFERS) &&
+ !(td->flags & TD_F_COMPRESS))
do_scramble = 1;
if (td->flags & TD_F_VER_NONE) {
populate_verify_io_u(td, io_u);
diff --git a/options.c b/options.c
index 4ff4c9b..4a54c98 100644
--- a/options.c
+++ b/options.c
@@ -1002,6 +1002,15 @@
return ret;
}
+static int str_buffer_compress_cb(void *data, unsigned long long *il)
+{
+ struct thread_data *td = data;
+
+ td->flags |= TD_F_COMPRESS;
+ td->o.compress_percentage = *il;
+ return 0;
+}
+
static int str_verify_pattern_cb(void *data, const char *input)
{
struct thread_data *td = data;
@@ -1606,6 +1615,16 @@
.group = FIO_OPT_G_INVALID,
},
{
+ .name = "file_append",
+ .lname = "File append",
+ .type = FIO_OPT_BOOL,
+ .off1 = td_var_offset(file_append),
+ .help = "IO will start at the end of the file(s)",
+ .def = "0",
+ .category = FIO_OPT_C_FILE,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
.name = "offset",
.lname = "IO offset",
.alias = "fileoffset",
@@ -3118,7 +3137,7 @@
.name = "buffer_compress_percentage",
.lname = "Buffer compression percentage",
.type = FIO_OPT_INT,
- .off1 = td_var_offset(compress_percentage),
+ .cb = str_buffer_compress_cb,
.maxval = 100,
.minval = 0,
.help = "How compressible the buffer is (approximately)",
diff --git a/thread_options.h b/thread_options.h
index bacd86b..4642120 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -54,6 +54,7 @@
unsigned long long size;
unsigned int size_percent;
unsigned int fill_device;
+ unsigned int file_append;
unsigned long long file_size_low;
unsigned long long file_size_high;
unsigned long long start_offset;
@@ -281,6 +282,7 @@
uint64_t size;
uint32_t size_percent;
uint32_t fill_device;
+ uint32_t file_append;
uint64_t file_size_low;
uint64_t file_size_high;
uint64_t start_offset;