blk-mq: fix schedule from atomic context
blk_mq_put_ctx() has to be called before io_schedule() in
bt_get().
This patch fixes the problem by taking similar approach from
percpu_ida allocation for the situation.
Signed-off-by: Ming Lei <tom.leiming@gmail.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index d90c4ae..1aab39f 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -221,8 +221,10 @@
return bs;
}
-static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx,
- unsigned int *last_tag, gfp_t gfp)
+static int bt_get(struct blk_mq_alloc_data *data,
+ struct blk_mq_bitmap_tags *bt,
+ struct blk_mq_hw_ctx *hctx,
+ unsigned int *last_tag)
{
struct bt_wait_state *bs;
DEFINE_WAIT(wait);
@@ -232,7 +234,7 @@
if (tag != -1)
return tag;
- if (!(gfp & __GFP_WAIT))
+ if (!(data->gfp & __GFP_WAIT))
return -1;
bs = bt_wait_ptr(bt, hctx);
@@ -249,50 +251,62 @@
if (was_empty)
atomic_set(&bs->wait_cnt, bt->wake_cnt);
+ blk_mq_put_ctx(data->ctx);
+
io_schedule();
+
+ data->ctx = blk_mq_get_ctx(data->q);
+ data->hctx = data->q->mq_ops->map_queue(data->q,
+ data->ctx->cpu);
+ if (data->reserved) {
+ bt = &data->hctx->tags->breserved_tags;
+ } else {
+ last_tag = &data->ctx->last_tag;
+ hctx = data->hctx;
+ bt = &hctx->tags->bitmap_tags;
+ }
+ finish_wait(&bs->wait, &wait);
+ bs = bt_wait_ptr(bt, hctx);
} while (1);
finish_wait(&bs->wait, &wait);
return tag;
}
-static unsigned int __blk_mq_get_tag(struct blk_mq_tags *tags,
- struct blk_mq_hw_ctx *hctx,
- unsigned int *last_tag, gfp_t gfp)
+static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
int tag;
- tag = bt_get(&tags->bitmap_tags, hctx, last_tag, gfp);
+ tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,
+ &data->ctx->last_tag);
if (tag >= 0)
- return tag + tags->nr_reserved_tags;
+ return tag + data->hctx->tags->nr_reserved_tags;
return BLK_MQ_TAG_FAIL;
}
-static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_tags *tags,
- gfp_t gfp)
+static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
{
int tag, zero = 0;
- if (unlikely(!tags->nr_reserved_tags)) {
+ if (unlikely(!data->hctx->tags->nr_reserved_tags)) {
WARN_ON_ONCE(1);
return BLK_MQ_TAG_FAIL;
}
- tag = bt_get(&tags->breserved_tags, NULL, &zero, gfp);
+ tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero);
if (tag < 0)
return BLK_MQ_TAG_FAIL;
return tag;
}
-unsigned int blk_mq_get_tag(struct blk_mq_hw_ctx *hctx, unsigned int *last_tag,
- gfp_t gfp, bool reserved)
+unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
- if (!reserved)
- return __blk_mq_get_tag(hctx->tags, hctx, last_tag, gfp);
+ if (!data->reserved)
+ return __blk_mq_get_tag(data);
- return __blk_mq_get_reserved_tag(hctx->tags, gfp);
+ return __blk_mq_get_reserved_tag(data);
}
static struct bt_wait_state *bt_wake_ptr(struct blk_mq_bitmap_tags *bt)