cfq-iosched: implement slower async initiate and queue ramp up
This slowly ramps up the async queue depth based on the time
passed since the sync IO, and doesn't allow async at all until
a sync slice period has passed.
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
diff --git a/block/blk-core.c b/block/blk-core.c
index ddaaea4..a8c7fbe 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -2492,6 +2492,14 @@
}
EXPORT_SYMBOL(kblockd_schedule_work);
+int kblockd_schedule_delayed_work(struct request_queue *q,
+ struct delayed_work *work,
+ unsigned long delay)
+{
+ return queue_delayed_work(kblockd_workqueue, work, delay);
+}
+EXPORT_SYMBOL(kblockd_schedule_delayed_work);
+
int __init blk_dev_init(void)
{
BUILD_BUG_ON(__REQ_NR_BITS > 8 *
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 70b48ea..fce8a74 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -150,7 +150,7 @@
* idle window management
*/
struct timer_list idle_slice_timer;
- struct work_struct unplug_work;
+ struct delayed_work unplug_work;
struct cfq_queue *active_queue;
struct cfq_io_context *active_cic;
@@ -268,11 +268,13 @@
* scheduler run of queue, if there are requests pending and no one in the
* driver that will restart queueing
*/
-static inline void cfq_schedule_dispatch(struct cfq_data *cfqd)
+static inline void cfq_schedule_dispatch(struct cfq_data *cfqd,
+ unsigned long delay)
{
if (cfqd->busy_queues) {
cfq_log(cfqd, "schedule dispatch");
- kblockd_schedule_work(cfqd->queue, &cfqd->unplug_work);
+ kblockd_schedule_delayed_work(cfqd->queue, &cfqd->unplug_work,
+ delay);
}
}
@@ -1316,8 +1318,6 @@
* Does this cfqq already have too much IO in flight?
*/
if (cfqq->dispatched >= max_dispatch) {
- unsigned long load_at = cfqd->last_end_sync_rq + cfq_slice_sync;
-
/*
* idle queue must always only have a single IO in flight
*/
@@ -1331,20 +1331,36 @@
return 0;
/*
- * If a sync request has completed recently, don't overload
- * the dispatch queue yet with async requests.
+ * Sole queue user, allow bigger slice
*/
- if (cfqd->cfq_desktop && !cfq_cfqq_sync(cfqq)
- && time_before(jiffies, load_at))
- return 0;
+ max_dispatch *= 4;
+ }
+
+ /*
+ * Async queues must wait a bit before being allowed dispatch.
+ * We also ramp up the dispatch depth gradually for async IO,
+ * based on the last sync IO we serviced
+ */
+ if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_desktop) {
+ unsigned long last_sync = jiffies - cfqd->last_end_sync_rq;
+ unsigned int depth;
/*
- * we are the only queue, allow up to 4 times of 'quantum'
+ * must wait a bit longer
*/
- if (cfqq->dispatched >= 4 * max_dispatch)
+ if (last_sync < cfq_slice_sync) {
+ cfq_schedule_dispatch(cfqd, cfq_slice_sync - last_sync);
return 0;
+ }
+
+ depth = last_sync / cfq_slice_sync;
+ if (depth < max_dispatch)
+ max_dispatch = depth;
}
+ if (cfqq->dispatched >= max_dispatch)
+ return 0;
+
/*
* Dispatch a request from this cfqq
*/
@@ -1389,7 +1405,7 @@
if (unlikely(cfqd->active_queue == cfqq)) {
__cfq_slice_expired(cfqd, cfqq, 0);
- cfq_schedule_dispatch(cfqd);
+ cfq_schedule_dispatch(cfqd, 0);
}
kmem_cache_free(cfq_pool, cfqq);
@@ -1484,7 +1500,7 @@
{
if (unlikely(cfqq == cfqd->active_queue)) {
__cfq_slice_expired(cfqd, cfqq, 0);
- cfq_schedule_dispatch(cfqd);
+ cfq_schedule_dispatch(cfqd, 0);
}
cfq_put_queue(cfqq);
@@ -2201,7 +2217,7 @@
}
if (!rq_in_driver(cfqd))
- cfq_schedule_dispatch(cfqd);
+ cfq_schedule_dispatch(cfqd, 0);
}
/*
@@ -2331,7 +2347,7 @@
if (cic)
put_io_context(cic->ioc);
- cfq_schedule_dispatch(cfqd);
+ cfq_schedule_dispatch(cfqd, 0);
spin_unlock_irqrestore(q->queue_lock, flags);
cfq_log(cfqd, "set_request fail");
return 1;
@@ -2340,7 +2356,7 @@
static void cfq_kick_queue(struct work_struct *work)
{
struct cfq_data *cfqd =
- container_of(work, struct cfq_data, unplug_work);
+ container_of(work, struct cfq_data, unplug_work.work);
struct request_queue *q = cfqd->queue;
spin_lock_irq(q->queue_lock);
@@ -2394,7 +2410,7 @@
expire:
cfq_slice_expired(cfqd, timed_out);
out_kick:
- cfq_schedule_dispatch(cfqd);
+ cfq_schedule_dispatch(cfqd, 0);
out_cont:
spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
}
@@ -2402,7 +2418,7 @@
static void cfq_shutdown_timer_wq(struct cfq_data *cfqd)
{
del_timer_sync(&cfqd->idle_slice_timer);
- cancel_work_sync(&cfqd->unplug_work);
+ cancel_delayed_work_sync(&cfqd->unplug_work);
}
static void cfq_put_async_queues(struct cfq_data *cfqd)
@@ -2484,7 +2500,7 @@
cfqd->idle_slice_timer.function = cfq_idle_slice_timer;
cfqd->idle_slice_timer.data = (unsigned long) cfqd;
- INIT_WORK(&cfqd->unplug_work, cfq_kick_queue);
+ INIT_DELAYED_WORK(&cfqd->unplug_work, cfq_kick_queue);
cfqd->cfq_quantum = cfq_quantum;
cfqd->cfq_fifo_expire[0] = cfq_fifo_expire[0];
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 1a03b71..a732393 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1147,7 +1147,11 @@
}
struct work_struct;
+struct delayed_work;
int kblockd_schedule_work(struct request_queue *q, struct work_struct *work);
+int kblockd_schedule_delayed_work(struct request_queue *q,
+ struct delayed_work *work,
+ unsigned long delay);
#define MODULE_ALIAS_BLOCKDEV(major,minor) \
MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))