blk-mq: add ->init_request and ->exit_request methods
The current blk_mq_init_commands/blk_mq_free_commands interface has a
two problems:
1) Because only the constructor is passed to blk_mq_init_commands there
is no easy way to clean up when a comman initialization failed. The
current code simply leaks the allocations done in the constructor.
2) There is no good place to call blk_mq_free_commands: before
blk_cleanup_queue there is no guarantee that all outstanding
commands have completed, so we can't free them yet. After
blk_cleanup_queue the queue has usually been freed. This can be
worked around by grabbing an unconditional reference before calling
blk_cleanup_queue and dropping it after blk_mq_free_commands is
done, although that's not exatly pretty and driver writers are
guaranteed to get it wrong sooner or later.
Both issues are easily fixed by making the request constructor and
destructor normal blk_mq_ops methods.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@fb.com>
diff --git a/block/blk-mq.c b/block/blk-mq.c
index e644fee..48d2d84 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1031,75 +1031,21 @@
blk_mq_put_ctx(ctx);
}
-static int blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx,
- int (*init)(void *, struct blk_mq_hw_ctx *,
- struct request *, unsigned int),
- void *data)
-{
- unsigned int i;
- int ret = 0;
-
- for (i = 0; i < hctx->queue_depth; i++) {
- struct request *rq = hctx->rqs[i];
-
- ret = init(data, hctx, rq, i);
- if (ret)
- break;
- }
-
- return ret;
-}
-
-int blk_mq_init_commands(struct request_queue *q,
- int (*init)(void *, struct blk_mq_hw_ctx *,
- struct request *, unsigned int),
- void *data)
-{
- struct blk_mq_hw_ctx *hctx;
- unsigned int i;
- int ret = 0;
-
- queue_for_each_hw_ctx(q, hctx, i) {
- ret = blk_mq_init_hw_commands(hctx, init, data);
- if (ret)
- break;
- }
-
- return ret;
-}
-EXPORT_SYMBOL(blk_mq_init_commands);
-
-static void blk_mq_free_hw_commands(struct blk_mq_hw_ctx *hctx,
- void (*free)(void *, struct blk_mq_hw_ctx *,
- struct request *, unsigned int),
- void *data)
-{
- unsigned int i;
-
- for (i = 0; i < hctx->queue_depth; i++) {
- struct request *rq = hctx->rqs[i];
-
- free(data, hctx, rq, i);
- }
-}
-
-void blk_mq_free_commands(struct request_queue *q,
- void (*free)(void *, struct blk_mq_hw_ctx *,
- struct request *, unsigned int),
- void *data)
-{
- struct blk_mq_hw_ctx *hctx;
- unsigned int i;
-
- queue_for_each_hw_ctx(q, hctx, i)
- blk_mq_free_hw_commands(hctx, free, data);
-}
-EXPORT_SYMBOL(blk_mq_free_commands);
-
-static void blk_mq_free_rq_map(struct blk_mq_hw_ctx *hctx)
+static void blk_mq_free_rq_map(struct blk_mq_hw_ctx *hctx, void *driver_data)
{
struct page *page;
+ if (hctx->rqs && hctx->queue->mq_ops->exit_request) {
+ int i;
+
+ for (i = 0; i < hctx->queue_depth; i++) {
+ if (!hctx->rqs[i])
+ continue;
+ hctx->queue->mq_ops->exit_request(driver_data, hctx,
+ hctx->rqs[i], i);
+ }
+ }
+
while (!list_empty(&hctx->page_list)) {
page = list_first_entry(&hctx->page_list, struct page, lru);
list_del_init(&page->lru);
@@ -1123,10 +1069,12 @@
}
static int blk_mq_init_rq_map(struct blk_mq_hw_ctx *hctx,
- unsigned int reserved_tags, int node)
+ struct blk_mq_reg *reg, void *driver_data, int node)
{
+ unsigned int reserved_tags = reg->reserved_tags;
unsigned int i, j, entries_per_page, max_order = 4;
size_t rq_size, left;
+ int error;
INIT_LIST_HEAD(&hctx->page_list);
@@ -1175,14 +1123,23 @@
for (j = 0; j < to_do; j++) {
hctx->rqs[i] = p;
blk_rq_init(hctx->queue, hctx->rqs[i]);
+ if (reg->ops->init_request) {
+ error = reg->ops->init_request(driver_data,
+ hctx, hctx->rqs[i], i);
+ if (error)
+ goto err_rq_map;
+ }
+
p += rq_size;
i++;
}
}
- if (i < (reserved_tags + BLK_MQ_TAG_MIN))
+ if (i < (reserved_tags + BLK_MQ_TAG_MIN)) {
+ error = -ENOMEM;
goto err_rq_map;
- else if (i != hctx->queue_depth) {
+ }
+ if (i != hctx->queue_depth) {
hctx->queue_depth = i;
pr_warn("%s: queue depth set to %u because of low memory\n",
__func__, i);
@@ -1190,12 +1147,14 @@
hctx->tags = blk_mq_init_tags(hctx->queue_depth, reserved_tags, node);
if (!hctx->tags) {
-err_rq_map:
- blk_mq_free_rq_map(hctx);
- return -ENOMEM;
+ error = -ENOMEM;
+ goto err_rq_map;
}
return 0;
+err_rq_map:
+ blk_mq_free_rq_map(hctx, driver_data);
+ return error;
}
static int blk_mq_init_hw_queues(struct request_queue *q,
@@ -1228,7 +1187,7 @@
blk_mq_hctx_notify, hctx);
blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
- if (blk_mq_init_rq_map(hctx, reg->reserved_tags, node))
+ if (blk_mq_init_rq_map(hctx, reg, driver_data, node))
break;
/*
@@ -1268,7 +1227,7 @@
reg->ops->exit_hctx(hctx, j);
blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
- blk_mq_free_rq_map(hctx);
+ blk_mq_free_rq_map(hctx, driver_data);
kfree(hctx->ctxs);
}
@@ -1455,7 +1414,7 @@
queue_for_each_hw_ctx(q, hctx, i) {
kfree(hctx->ctx_map);
kfree(hctx->ctxs);
- blk_mq_free_rq_map(hctx);
+ blk_mq_free_rq_map(hctx, q->queuedata);
blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
if (q->mq_ops->exit_hctx)
q->mq_ops->exit_hctx(hctx, i);