Change IO engine queuing

Instead of always pretending to be async, let the IO engines
return FIO_Q_COMPLETED or FIO_Q_QUEUED to signal async or
sync completions regardless of their nature. This cleans up
the queuing model quite a bit.

Also fixed a verification error spotted while doing this
transformation.

The main intent of this is to allow queuing more than 1 piece
of IO at the time, that will come in a later changeset.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
diff --git a/engines/libaio.c b/engines/libaio.c
index c2f47d8..ba8c49d 100644
--- a/engines/libaio.c
+++ b/engines/libaio.c
@@ -18,7 +18,6 @@
 struct libaio_data {
 	io_context_t aio_ctx;
 	struct io_event *aio_events;
-	struct io_u *sync_io_u;
 };
 
 static int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u)
@@ -41,13 +40,6 @@
 {
 	struct libaio_data *ld = td->io_ops->data;
 
-	if (ld->sync_io_u) {
-		struct io_u *ret = ld->sync_io_u;
-
-		ld->sync_io_u = NULL;
-		return ret;
-	}
-
 	return ev_to_iou(ld->aio_events + event);
 }
 
@@ -57,9 +49,6 @@
 	struct libaio_data *ld = td->io_ops->data;
 	long r;
 
-	if (ld->sync_io_u)
-		return 1;
-
 	do {
 		r = io_getevents(ld->aio_ctx, min, max, ld->aio_events, t);
 		if (r >= min)
@@ -88,7 +77,7 @@
 	do {
 		ret = io_submit(ld->aio_ctx, 1, &iocb);
 		if (ret == 1)
-			break;
+			return FIO_Q_QUEUED;
 		else if (ret == -EAGAIN || !ret)
 			usleep(100);
 		else if (ret == -EINTR)
@@ -103,10 +92,8 @@
 			 */
 			if (fsync(io_u->file->fd) < 0)
 				ret = errno;
-			else {
-				ret = 1;
-				ld->sync_io_u = io_u;
-			}
+			else
+				ret = FIO_Q_COMPLETED;
 			break;
 		} else
 			break;
@@ -116,10 +103,10 @@
 		io_u->resid = io_u->xfer_buflen;
 		io_u->error = -ret;
 		td_verror(td, io_u->error);
-		return 1;
+		return FIO_Q_COMPLETED;
 	}
 
-	return 0;
+	return ret;
 }
 
 static int fio_libaio_cancel(struct thread_data *td, struct io_u *io_u)