blob: bb66c96850b18cb419b0e44aab1894169352f9af [file] [log] [blame]
Jens Axboe75bb4622014-05-28 10:15:41 -06001/*
2 * Block multiqueue core code
3 *
4 * Copyright (C) 2013-2014 Jens Axboe
5 * Copyright (C) 2013-2014 Christoph Hellwig
6 */
Jens Axboe320ae512013-10-24 09:20:05 +01007#include <linux/kernel.h>
8#include <linux/module.h>
9#include <linux/backing-dev.h>
10#include <linux/bio.h>
11#include <linux/blkdev.h>
Catalin Marinasf75782e2015-09-14 18:16:02 +010012#include <linux/kmemleak.h>
Jens Axboe320ae512013-10-24 09:20:05 +010013#include <linux/mm.h>
14#include <linux/init.h>
15#include <linux/slab.h>
16#include <linux/workqueue.h>
17#include <linux/smp.h>
18#include <linux/llist.h>
19#include <linux/list_sort.h>
20#include <linux/cpu.h>
21#include <linux/cache.h>
22#include <linux/sched/sysctl.h>
Ingo Molnar105ab3d2017-02-01 16:36:40 +010023#include <linux/sched/topology.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010024#include <linux/sched/signal.h>
Jens Axboe320ae512013-10-24 09:20:05 +010025#include <linux/delay.h>
Jens Axboeaedcd722014-09-17 08:27:03 -060026#include <linux/crash_dump.h>
Jens Axboe88c7b2b2016-08-25 08:07:30 -060027#include <linux/prefetch.h>
Jens Axboe320ae512013-10-24 09:20:05 +010028
29#include <trace/events/block.h>
30
31#include <linux/blk-mq.h>
32#include "blk.h"
33#include "blk-mq.h"
Omar Sandoval9c1051a2017-05-04 08:17:21 -060034#include "blk-mq-debugfs.h"
Jens Axboe320ae512013-10-24 09:20:05 +010035#include "blk-mq-tag.h"
Jens Axboecf43e6b2016-11-07 21:32:37 -070036#include "blk-stat.h"
Jens Axboe87760e52016-11-09 12:38:14 -070037#include "blk-wbt.h"
Jens Axboebd166ef2017-01-17 06:03:22 -070038#include "blk-mq-sched.h"
Jens Axboe320ae512013-10-24 09:20:05 +010039
40static DEFINE_MUTEX(all_q_mutex);
41static LIST_HEAD(all_q_list);
42
Omar Sandoval34dbad52017-03-21 08:56:08 -070043static void blk_mq_poll_stats_start(struct request_queue *q);
44static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
Jens Axboe2719aa22017-05-03 11:08:14 -060045static void __blk_mq_stop_hw_queues(struct request_queue *q, bool sync);
Omar Sandoval34dbad52017-03-21 08:56:08 -070046
Stephen Bates720b8cc2017-04-07 06:24:03 -060047static int blk_mq_poll_stats_bkt(const struct request *rq)
48{
49 int ddir, bytes, bucket;
50
Jens Axboe99c749a2017-04-21 07:55:42 -060051 ddir = rq_data_dir(rq);
Stephen Bates720b8cc2017-04-07 06:24:03 -060052 bytes = blk_rq_bytes(rq);
53
54 bucket = ddir + 2*(ilog2(bytes) - 9);
55
56 if (bucket < 0)
57 return -1;
58 else if (bucket >= BLK_MQ_POLL_STATS_BKTS)
59 return ddir + BLK_MQ_POLL_STATS_BKTS - 2;
60
61 return bucket;
62}
63
Jens Axboe320ae512013-10-24 09:20:05 +010064/*
65 * Check if any of the ctx's have pending work in this hardware queue
66 */
Jens Axboe50e1dab2017-01-26 14:42:34 -070067bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
Jens Axboe320ae512013-10-24 09:20:05 +010068{
Jens Axboebd166ef2017-01-17 06:03:22 -070069 return sbitmap_any_bit_set(&hctx->ctx_map) ||
70 !list_empty_careful(&hctx->dispatch) ||
71 blk_mq_sched_has_work(hctx);
Jens Axboe320ae512013-10-24 09:20:05 +010072}
73
74/*
75 * Mark this ctx as having pending work in this hardware queue
76 */
77static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
78 struct blk_mq_ctx *ctx)
79{
Omar Sandoval88459642016-09-17 08:38:44 -060080 if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
81 sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
Jens Axboe1429d7c2014-05-19 09:23:55 -060082}
83
84static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
85 struct blk_mq_ctx *ctx)
86{
Omar Sandoval88459642016-09-17 08:38:44 -060087 sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
Jens Axboe320ae512013-10-24 09:20:05 +010088}
89
Ming Lei1671d522017-03-27 20:06:57 +080090void blk_freeze_queue_start(struct request_queue *q)
Ming Lei43a5e4e2013-12-26 21:31:35 +080091{
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +020092 int freeze_depth;
Tejun Heocddd5d12014-08-16 08:02:24 -040093
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +020094 freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
95 if (freeze_depth == 1) {
Dan Williams3ef28e82015-10-21 13:20:12 -040096 percpu_ref_kill(&q->q_usage_counter);
Mike Snitzerb94ec292015-03-11 23:56:38 -040097 blk_mq_run_hw_queues(q, false);
Tejun Heocddd5d12014-08-16 08:02:24 -040098 }
Tejun Heof3af0202014-11-04 13:52:27 -050099}
Ming Lei1671d522017-03-27 20:06:57 +0800100EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
Tejun Heof3af0202014-11-04 13:52:27 -0500101
Keith Busch6bae3632017-03-01 14:22:10 -0500102void blk_mq_freeze_queue_wait(struct request_queue *q)
Tejun Heof3af0202014-11-04 13:52:27 -0500103{
Dan Williams3ef28e82015-10-21 13:20:12 -0400104 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
Ming Lei43a5e4e2013-12-26 21:31:35 +0800105}
Keith Busch6bae3632017-03-01 14:22:10 -0500106EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
Ming Lei43a5e4e2013-12-26 21:31:35 +0800107
Keith Buschf91328c2017-03-01 14:22:11 -0500108int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
109 unsigned long timeout)
110{
111 return wait_event_timeout(q->mq_freeze_wq,
112 percpu_ref_is_zero(&q->q_usage_counter),
113 timeout);
114}
115EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
Jens Axboe320ae512013-10-24 09:20:05 +0100116
Tejun Heof3af0202014-11-04 13:52:27 -0500117/*
118 * Guarantee no request is in use, so we can change any data structure of
119 * the queue afterward.
120 */
Dan Williams3ef28e82015-10-21 13:20:12 -0400121void blk_freeze_queue(struct request_queue *q)
Tejun Heof3af0202014-11-04 13:52:27 -0500122{
Dan Williams3ef28e82015-10-21 13:20:12 -0400123 /*
124 * In the !blk_mq case we are only calling this to kill the
125 * q_usage_counter, otherwise this increases the freeze depth
126 * and waits for it to return to zero. For this reason there is
127 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
128 * exported to drivers as the only user for unfreeze is blk_mq.
129 */
Ming Lei1671d522017-03-27 20:06:57 +0800130 blk_freeze_queue_start(q);
Tejun Heof3af0202014-11-04 13:52:27 -0500131 blk_mq_freeze_queue_wait(q);
132}
Dan Williams3ef28e82015-10-21 13:20:12 -0400133
134void blk_mq_freeze_queue(struct request_queue *q)
135{
136 /*
137 * ...just an alias to keep freeze and unfreeze actions balanced
138 * in the blk_mq_* namespace
139 */
140 blk_freeze_queue(q);
141}
Jens Axboec761d962015-01-02 15:05:12 -0700142EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
Tejun Heof3af0202014-11-04 13:52:27 -0500143
Keith Buschb4c6a022014-12-19 17:54:14 -0700144void blk_mq_unfreeze_queue(struct request_queue *q)
Jens Axboe320ae512013-10-24 09:20:05 +0100145{
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +0200146 int freeze_depth;
Jens Axboe320ae512013-10-24 09:20:05 +0100147
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +0200148 freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
149 WARN_ON_ONCE(freeze_depth < 0);
150 if (!freeze_depth) {
Dan Williams3ef28e82015-10-21 13:20:12 -0400151 percpu_ref_reinit(&q->q_usage_counter);
Jens Axboe320ae512013-10-24 09:20:05 +0100152 wake_up_all(&q->mq_freeze_wq);
Tejun Heoadd703f2014-07-01 10:34:38 -0600153 }
Jens Axboe320ae512013-10-24 09:20:05 +0100154}
Keith Buschb4c6a022014-12-19 17:54:14 -0700155EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
Jens Axboe320ae512013-10-24 09:20:05 +0100156
Bart Van Assche6a83e742016-11-02 10:09:51 -0600157/**
158 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
159 * @q: request queue.
160 *
161 * Note: this function does not prevent that the struct request end_io()
162 * callback function is invoked. Additionally, it is not prevented that
163 * new queue_rq() calls occur unless the queue has been stopped first.
164 */
165void blk_mq_quiesce_queue(struct request_queue *q)
166{
167 struct blk_mq_hw_ctx *hctx;
168 unsigned int i;
169 bool rcu = false;
170
Jens Axboe2719aa22017-05-03 11:08:14 -0600171 __blk_mq_stop_hw_queues(q, true);
Bart Van Assche6a83e742016-11-02 10:09:51 -0600172
173 queue_for_each_hw_ctx(q, hctx, i) {
174 if (hctx->flags & BLK_MQ_F_BLOCKING)
175 synchronize_srcu(&hctx->queue_rq_srcu);
176 else
177 rcu = true;
178 }
179 if (rcu)
180 synchronize_rcu();
181}
182EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
183
Jens Axboeaed3ea92014-12-22 14:04:42 -0700184void blk_mq_wake_waiters(struct request_queue *q)
185{
186 struct blk_mq_hw_ctx *hctx;
187 unsigned int i;
188
189 queue_for_each_hw_ctx(q, hctx, i)
190 if (blk_mq_hw_queue_mapped(hctx))
191 blk_mq_tag_wakeup_all(hctx->tags, true);
Keith Busch3fd59402015-01-08 08:53:56 -0700192
193 /*
194 * If we are called because the queue has now been marked as
195 * dying, we need to ensure that processes currently waiting on
196 * the queue are notified as well.
197 */
198 wake_up_all(&q->mq_freeze_wq);
Jens Axboeaed3ea92014-12-22 14:04:42 -0700199}
200
Jens Axboe320ae512013-10-24 09:20:05 +0100201bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
202{
203 return blk_mq_has_free_tags(hctx->tags);
204}
205EXPORT_SYMBOL(blk_mq_can_queue);
206
Jens Axboe2c3ad662016-12-14 14:34:47 -0700207void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
208 struct request *rq, unsigned int op)
Jens Axboe320ae512013-10-24 09:20:05 +0100209{
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200210 INIT_LIST_HEAD(&rq->queuelist);
211 /* csd/requeue_work/fifo_time is initialized before use */
212 rq->q = q;
Jens Axboe320ae512013-10-24 09:20:05 +0100213 rq->mq_ctx = ctx;
Christoph Hellwigef295ec2016-10-28 08:48:16 -0600214 rq->cmd_flags = op;
Christoph Hellwige8064022016-10-20 15:12:13 +0200215 if (blk_queue_io_stat(q))
216 rq->rq_flags |= RQF_IO_STAT;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200217 /* do not touch atomic flags, it needs atomic ops against the timer */
218 rq->cpu = -1;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200219 INIT_HLIST_NODE(&rq->hash);
220 RB_CLEAR_NODE(&rq->rb_node);
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200221 rq->rq_disk = NULL;
222 rq->part = NULL;
Jens Axboe3ee32372014-06-09 09:36:53 -0600223 rq->start_time = jiffies;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200224#ifdef CONFIG_BLK_CGROUP
225 rq->rl = NULL;
Ming Lei0fec08b2014-01-03 10:00:08 -0700226 set_start_time_ns(rq);
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200227 rq->io_start_time_ns = 0;
228#endif
229 rq->nr_phys_segments = 0;
230#if defined(CONFIG_BLK_DEV_INTEGRITY)
231 rq->nr_integrity_segments = 0;
232#endif
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200233 rq->special = NULL;
234 /* tag was already set */
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200235 rq->extra_len = 0;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200236
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200237 INIT_LIST_HEAD(&rq->timeout_list);
Jens Axboef6be4fb2014-06-06 11:03:48 -0600238 rq->timeout = 0;
239
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200240 rq->end_io = NULL;
241 rq->end_io_data = NULL;
242 rq->next_rq = NULL;
243
Christoph Hellwigef295ec2016-10-28 08:48:16 -0600244 ctx->rq_dispatched[op_is_sync(op)]++;
Jens Axboe320ae512013-10-24 09:20:05 +0100245}
Jens Axboe2c3ad662016-12-14 14:34:47 -0700246EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init);
Jens Axboe320ae512013-10-24 09:20:05 +0100247
Jens Axboe2c3ad662016-12-14 14:34:47 -0700248struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
249 unsigned int op)
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200250{
251 struct request *rq;
252 unsigned int tag;
253
Ming Leicb96a422014-06-01 00:43:37 +0800254 tag = blk_mq_get_tag(data);
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200255 if (tag != BLK_MQ_TAG_FAIL) {
Jens Axboebd166ef2017-01-17 06:03:22 -0700256 struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
257
258 rq = tags->static_rqs[tag];
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200259
Jens Axboebd166ef2017-01-17 06:03:22 -0700260 if (data->flags & BLK_MQ_REQ_INTERNAL) {
261 rq->tag = -1;
262 rq->internal_tag = tag;
263 } else {
Jens Axboe200e86b2017-01-25 08:11:38 -0700264 if (blk_mq_tag_busy(data->hctx)) {
265 rq->rq_flags = RQF_MQ_INFLIGHT;
266 atomic_inc(&data->hctx->nr_active);
267 }
Jens Axboebd166ef2017-01-17 06:03:22 -0700268 rq->tag = tag;
269 rq->internal_tag = -1;
Omar Sandoval562bef42017-02-27 09:47:55 -0800270 data->hctx->tags->rqs[rq->tag] = rq;
Jens Axboebd166ef2017-01-17 06:03:22 -0700271 }
272
Christoph Hellwigef295ec2016-10-28 08:48:16 -0600273 blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200274 return rq;
275 }
276
277 return NULL;
278}
Jens Axboe2c3ad662016-12-14 14:34:47 -0700279EXPORT_SYMBOL_GPL(__blk_mq_alloc_request);
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200280
Christoph Hellwig6f3b0e82015-11-26 09:13:05 +0100281struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
282 unsigned int flags)
Jens Axboe320ae512013-10-24 09:20:05 +0100283{
Jens Axboe5a797e02017-01-26 12:22:11 -0700284 struct blk_mq_alloc_data alloc_data = { .flags = flags };
Jens Axboebd166ef2017-01-17 06:03:22 -0700285 struct request *rq;
Joe Lawrencea492f072014-08-28 08:15:21 -0600286 int ret;
Jens Axboe320ae512013-10-24 09:20:05 +0100287
Christoph Hellwig6f3b0e82015-11-26 09:13:05 +0100288 ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
Joe Lawrencea492f072014-08-28 08:15:21 -0600289 if (ret)
290 return ERR_PTR(ret);
Jens Axboe320ae512013-10-24 09:20:05 +0100291
Jens Axboebd166ef2017-01-17 06:03:22 -0700292 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
Jens Axboe841bac22016-09-21 10:08:43 -0600293
Jens Axboebd166ef2017-01-17 06:03:22 -0700294 blk_mq_put_ctx(alloc_data.ctx);
295 blk_queue_exit(q);
296
297 if (!rq)
Joe Lawrencea492f072014-08-28 08:15:21 -0600298 return ERR_PTR(-EWOULDBLOCK);
Christoph Hellwig0c4de0f2016-07-19 11:31:50 +0200299
300 rq->__data_len = 0;
301 rq->__sector = (sector_t) -1;
302 rq->bio = rq->biotail = NULL;
Jens Axboe320ae512013-10-24 09:20:05 +0100303 return rq;
304}
Jens Axboe4bb659b2014-05-09 09:36:49 -0600305EXPORT_SYMBOL(blk_mq_alloc_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100306
Ming Lin1f5bd332016-06-13 16:45:21 +0200307struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
308 unsigned int flags, unsigned int hctx_idx)
309{
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800310 struct blk_mq_alloc_data alloc_data = { .flags = flags };
Ming Lin1f5bd332016-06-13 16:45:21 +0200311 struct request *rq;
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800312 unsigned int cpu;
Ming Lin1f5bd332016-06-13 16:45:21 +0200313 int ret;
314
315 /*
316 * If the tag allocator sleeps we could get an allocation for a
317 * different hardware context. No need to complicate the low level
318 * allocator for this for the rare use case of a command tied to
319 * a specific queue.
320 */
321 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
322 return ERR_PTR(-EINVAL);
323
324 if (hctx_idx >= q->nr_hw_queues)
325 return ERR_PTR(-EIO);
326
327 ret = blk_queue_enter(q, true);
328 if (ret)
329 return ERR_PTR(ret);
330
Christoph Hellwigc8712c62016-09-23 10:25:48 -0600331 /*
332 * Check if the hardware context is actually mapped to anything.
333 * If not tell the caller that it should skip this queue.
334 */
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800335 alloc_data.hctx = q->queue_hw_ctx[hctx_idx];
336 if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) {
337 blk_queue_exit(q);
338 return ERR_PTR(-EXDEV);
Christoph Hellwigc8712c62016-09-23 10:25:48 -0600339 }
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800340 cpu = cpumask_first(alloc_data.hctx->cpumask);
341 alloc_data.ctx = __blk_mq_get_ctx(q, cpu);
Ming Lin1f5bd332016-06-13 16:45:21 +0200342
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800343 rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
344
Omar Sandoval6d2809d2017-02-27 10:28:27 -0800345 blk_queue_exit(q);
346
347 if (!rq)
348 return ERR_PTR(-EWOULDBLOCK);
Ming Lin1f5bd332016-06-13 16:45:21 +0200349
350 return rq;
351}
352EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
353
Jens Axboebd166ef2017-01-17 06:03:22 -0700354void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
355 struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100356{
Jens Axboebd166ef2017-01-17 06:03:22 -0700357 const int sched_tag = rq->internal_tag;
Jens Axboe320ae512013-10-24 09:20:05 +0100358 struct request_queue *q = rq->q;
359
Christoph Hellwige8064022016-10-20 15:12:13 +0200360 if (rq->rq_flags & RQF_MQ_INFLIGHT)
Jens Axboe0d2602c2014-05-13 15:10:52 -0600361 atomic_dec(&hctx->nr_active);
Jens Axboe87760e52016-11-09 12:38:14 -0700362
363 wbt_done(q->rq_wb, &rq->issue_stat);
Christoph Hellwige8064022016-10-20 15:12:13 +0200364 rq->rq_flags = 0;
Jens Axboe0d2602c2014-05-13 15:10:52 -0600365
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200366 clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
Jens Axboe06426ad2016-11-14 13:01:59 -0700367 clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
Jens Axboebd166ef2017-01-17 06:03:22 -0700368 if (rq->tag != -1)
369 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
370 if (sched_tag != -1)
Omar Sandovalc05f8522017-04-14 01:00:01 -0700371 blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
Bart Van Assche6d8c6c02017-04-07 12:40:09 -0600372 blk_mq_sched_restart(hctx);
Dan Williams3ef28e82015-10-21 13:20:12 -0400373 blk_queue_exit(q);
Jens Axboe320ae512013-10-24 09:20:05 +0100374}
375
Jens Axboebd166ef2017-01-17 06:03:22 -0700376static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
Jens Axboe16a3c2a2016-12-15 14:27:46 -0700377 struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100378{
379 struct blk_mq_ctx *ctx = rq->mq_ctx;
Jens Axboe7c7f2f22014-11-17 10:41:57 -0700380
381 ctx->rq_completed[rq_is_sync(rq)]++;
Jens Axboebd166ef2017-01-17 06:03:22 -0700382 __blk_mq_finish_request(hctx, ctx, rq);
383}
384
385void blk_mq_finish_request(struct request *rq)
386{
387 blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
Jens Axboe7c7f2f22014-11-17 10:41:57 -0700388}
Omar Sandoval5b727272017-04-14 01:00:00 -0700389EXPORT_SYMBOL_GPL(blk_mq_finish_request);
Jens Axboe7c7f2f22014-11-17 10:41:57 -0700390
391void blk_mq_free_request(struct request *rq)
392{
Jens Axboebd166ef2017-01-17 06:03:22 -0700393 blk_mq_sched_put_request(rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100394}
Jens Axboe1a3b5952014-11-17 10:40:48 -0700395EXPORT_SYMBOL_GPL(blk_mq_free_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100396
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700397inline void __blk_mq_end_request(struct request *rq, int error)
Jens Axboe320ae512013-10-24 09:20:05 +0100398{
Ming Lei0d11e6a2013-12-05 10:50:39 -0700399 blk_account_io_done(rq);
400
Christoph Hellwig91b63632014-04-16 09:44:53 +0200401 if (rq->end_io) {
Jens Axboe87760e52016-11-09 12:38:14 -0700402 wbt_done(rq->q->rq_wb, &rq->issue_stat);
Jens Axboe320ae512013-10-24 09:20:05 +0100403 rq->end_io(rq, error);
Christoph Hellwig91b63632014-04-16 09:44:53 +0200404 } else {
405 if (unlikely(blk_bidi_rq(rq)))
406 blk_mq_free_request(rq->next_rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100407 blk_mq_free_request(rq);
Christoph Hellwig91b63632014-04-16 09:44:53 +0200408 }
Jens Axboe320ae512013-10-24 09:20:05 +0100409}
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700410EXPORT_SYMBOL(__blk_mq_end_request);
Christoph Hellwig63151a42014-04-16 09:44:52 +0200411
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700412void blk_mq_end_request(struct request *rq, int error)
Christoph Hellwig63151a42014-04-16 09:44:52 +0200413{
414 if (blk_update_request(rq, error, blk_rq_bytes(rq)))
415 BUG();
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700416 __blk_mq_end_request(rq, error);
Christoph Hellwig63151a42014-04-16 09:44:52 +0200417}
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700418EXPORT_SYMBOL(blk_mq_end_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100419
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800420static void __blk_mq_complete_request_remote(void *data)
Jens Axboe320ae512013-10-24 09:20:05 +0100421{
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800422 struct request *rq = data;
Jens Axboe320ae512013-10-24 09:20:05 +0100423
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800424 rq->q->softirq_done_fn(rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100425}
426
Christoph Hellwig453f8342017-04-20 16:03:10 +0200427static void __blk_mq_complete_request(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100428{
429 struct blk_mq_ctx *ctx = rq->mq_ctx;
Christoph Hellwig38535202014-04-25 02:32:53 -0700430 bool shared = false;
Jens Axboe320ae512013-10-24 09:20:05 +0100431 int cpu;
432
Christoph Hellwig453f8342017-04-20 16:03:10 +0200433 if (rq->internal_tag != -1)
434 blk_mq_sched_completed_request(rq);
435 if (rq->rq_flags & RQF_STATS) {
436 blk_mq_poll_stats_start(rq->q);
437 blk_stat_add(rq);
438 }
439
Christoph Hellwig38535202014-04-25 02:32:53 -0700440 if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800441 rq->q->softirq_done_fn(rq);
442 return;
443 }
Jens Axboe320ae512013-10-24 09:20:05 +0100444
445 cpu = get_cpu();
Christoph Hellwig38535202014-04-25 02:32:53 -0700446 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
447 shared = cpus_share_cache(cpu, ctx->cpu);
448
449 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800450 rq->csd.func = __blk_mq_complete_request_remote;
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800451 rq->csd.info = rq;
452 rq->csd.flags = 0;
Frederic Weisbeckerc46fff22014-02-24 16:40:02 +0100453 smp_call_function_single_async(ctx->cpu, &rq->csd);
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800454 } else {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800455 rq->q->softirq_done_fn(rq);
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800456 }
Jens Axboe320ae512013-10-24 09:20:05 +0100457 put_cpu();
458}
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800459
460/**
461 * blk_mq_complete_request - end I/O on a request
462 * @rq: the request being processed
463 *
464 * Description:
465 * Ends all I/O on a request. It does not handle partial completions.
466 * The actual completion happens out-of-order, through a IPI handler.
467 **/
Christoph Hellwig08e00292017-04-20 16:03:09 +0200468void blk_mq_complete_request(struct request *rq)
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800469{
Jens Axboe95f09682014-05-27 17:46:48 -0600470 struct request_queue *q = rq->q;
471
472 if (unlikely(blk_should_fake_timeout(q)))
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800473 return;
Christoph Hellwig08e00292017-04-20 16:03:09 +0200474 if (!blk_mark_rq_complete(rq))
Jens Axboeed851862014-05-30 21:20:50 -0600475 __blk_mq_complete_request(rq);
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800476}
477EXPORT_SYMBOL(blk_mq_complete_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100478
Keith Busch973c0192015-01-07 18:55:43 -0700479int blk_mq_request_started(struct request *rq)
480{
481 return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
482}
483EXPORT_SYMBOL_GPL(blk_mq_request_started);
484
Christoph Hellwige2490072014-09-13 16:40:09 -0700485void blk_mq_start_request(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100486{
487 struct request_queue *q = rq->q;
488
Jens Axboebd166ef2017-01-17 06:03:22 -0700489 blk_mq_sched_started_request(rq);
490
Jens Axboe320ae512013-10-24 09:20:05 +0100491 trace_block_rq_issue(q, rq);
492
Jens Axboecf43e6b2016-11-07 21:32:37 -0700493 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
Shaohua Li88eeca42017-03-27 15:19:41 -0700494 blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq));
Jens Axboecf43e6b2016-11-07 21:32:37 -0700495 rq->rq_flags |= RQF_STATS;
Jens Axboe87760e52016-11-09 12:38:14 -0700496 wbt_issue(q->rq_wb, &rq->issue_stat);
Jens Axboecf43e6b2016-11-07 21:32:37 -0700497 }
498
Ming Lei2b8393b2014-06-10 00:16:41 +0800499 blk_add_timer(rq);
Jens Axboe87ee7b12014-04-24 08:51:47 -0600500
501 /*
Jens Axboe538b7532014-09-16 10:37:37 -0600502 * Ensure that ->deadline is visible before set the started
503 * flag and clear the completed flag.
504 */
505 smp_mb__before_atomic();
506
507 /*
Jens Axboe87ee7b12014-04-24 08:51:47 -0600508 * Mark us as started and clear complete. Complete might have been
509 * set if requeue raced with timeout, which then marked it as
510 * complete. So be sure to clear complete again when we start
511 * the request, otherwise we'll ignore the completion event.
512 */
Jens Axboe4b570522014-05-29 11:00:11 -0600513 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
514 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
515 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
516 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
Christoph Hellwig49f5baa2014-02-11 08:27:14 -0800517
518 if (q->dma_drain_size && blk_rq_bytes(rq)) {
519 /*
520 * Make sure space for the drain appears. We know we can do
521 * this because max_hw_segments has been adjusted to be one
522 * fewer than the device can handle.
523 */
524 rq->nr_phys_segments++;
525 }
Jens Axboe320ae512013-10-24 09:20:05 +0100526}
Christoph Hellwige2490072014-09-13 16:40:09 -0700527EXPORT_SYMBOL(blk_mq_start_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100528
Ming Leid9d149a2017-03-27 20:06:55 +0800529/*
530 * When we reach here because queue is busy, REQ_ATOM_COMPLETE
Jens Axboe48b99c92017-03-29 11:10:34 -0600531 * flag isn't set yet, so there may be race with timeout handler,
Ming Leid9d149a2017-03-27 20:06:55 +0800532 * but given rq->deadline is just set in .queue_rq() under
533 * this situation, the race won't be possible in reality because
534 * rq->timeout should be set as big enough to cover the window
535 * between blk_mq_start_request() called from .queue_rq() and
536 * clearing REQ_ATOM_STARTED here.
537 */
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200538static void __blk_mq_requeue_request(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100539{
540 struct request_queue *q = rq->q;
541
542 trace_block_rq_requeue(q, rq);
Jens Axboe87760e52016-11-09 12:38:14 -0700543 wbt_requeue(q->rq_wb, &rq->issue_stat);
Jens Axboebd166ef2017-01-17 06:03:22 -0700544 blk_mq_sched_requeue_request(rq);
Christoph Hellwig49f5baa2014-02-11 08:27:14 -0800545
Christoph Hellwige2490072014-09-13 16:40:09 -0700546 if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
547 if (q->dma_drain_size && blk_rq_bytes(rq))
548 rq->nr_phys_segments--;
549 }
Jens Axboe320ae512013-10-24 09:20:05 +0100550}
551
Bart Van Assche2b053ac2016-10-28 17:21:41 -0700552void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200553{
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200554 __blk_mq_requeue_request(rq);
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200555
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200556 BUG_ON(blk_queued_rq(rq));
Bart Van Assche2b053ac2016-10-28 17:21:41 -0700557 blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200558}
559EXPORT_SYMBOL(blk_mq_requeue_request);
560
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600561static void blk_mq_requeue_work(struct work_struct *work)
562{
563 struct request_queue *q =
Mike Snitzer28494502016-09-14 13:28:30 -0400564 container_of(work, struct request_queue, requeue_work.work);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600565 LIST_HEAD(rq_list);
566 struct request *rq, *next;
567 unsigned long flags;
568
569 spin_lock_irqsave(&q->requeue_lock, flags);
570 list_splice_init(&q->requeue_list, &rq_list);
571 spin_unlock_irqrestore(&q->requeue_lock, flags);
572
573 list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
Christoph Hellwige8064022016-10-20 15:12:13 +0200574 if (!(rq->rq_flags & RQF_SOFTBARRIER))
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600575 continue;
576
Christoph Hellwige8064022016-10-20 15:12:13 +0200577 rq->rq_flags &= ~RQF_SOFTBARRIER;
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600578 list_del_init(&rq->queuelist);
Jens Axboebd6737f2017-01-27 01:00:47 -0700579 blk_mq_sched_insert_request(rq, true, false, false, true);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600580 }
581
582 while (!list_empty(&rq_list)) {
583 rq = list_entry(rq_list.next, struct request, queuelist);
584 list_del_init(&rq->queuelist);
Jens Axboebd6737f2017-01-27 01:00:47 -0700585 blk_mq_sched_insert_request(rq, false, false, false, true);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600586 }
587
Bart Van Assche52d7f1b2016-10-28 17:20:32 -0700588 blk_mq_run_hw_queues(q, false);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600589}
590
Bart Van Assche2b053ac2016-10-28 17:21:41 -0700591void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
592 bool kick_requeue_list)
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600593{
594 struct request_queue *q = rq->q;
595 unsigned long flags;
596
597 /*
598 * We abuse this flag that is otherwise used by the I/O scheduler to
599 * request head insertation from the workqueue.
600 */
Christoph Hellwige8064022016-10-20 15:12:13 +0200601 BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600602
603 spin_lock_irqsave(&q->requeue_lock, flags);
604 if (at_head) {
Christoph Hellwige8064022016-10-20 15:12:13 +0200605 rq->rq_flags |= RQF_SOFTBARRIER;
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600606 list_add(&rq->queuelist, &q->requeue_list);
607 } else {
608 list_add_tail(&rq->queuelist, &q->requeue_list);
609 }
610 spin_unlock_irqrestore(&q->requeue_lock, flags);
Bart Van Assche2b053ac2016-10-28 17:21:41 -0700611
612 if (kick_requeue_list)
613 blk_mq_kick_requeue_list(q);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600614}
615EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
616
617void blk_mq_kick_requeue_list(struct request_queue *q)
618{
Mike Snitzer28494502016-09-14 13:28:30 -0400619 kblockd_schedule_delayed_work(&q->requeue_work, 0);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600620}
621EXPORT_SYMBOL(blk_mq_kick_requeue_list);
622
Mike Snitzer28494502016-09-14 13:28:30 -0400623void blk_mq_delay_kick_requeue_list(struct request_queue *q,
624 unsigned long msecs)
625{
626 kblockd_schedule_delayed_work(&q->requeue_work,
627 msecs_to_jiffies(msecs));
628}
629EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
630
Jens Axboe0e62f512014-06-04 10:23:49 -0600631struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
632{
Jens Axboe88c7b2b2016-08-25 08:07:30 -0600633 if (tag < tags->nr_tags) {
634 prefetch(tags->rqs[tag]);
Hannes Reinecke4ee86ba2016-03-15 12:03:28 -0700635 return tags->rqs[tag];
Jens Axboe88c7b2b2016-08-25 08:07:30 -0600636 }
Hannes Reinecke4ee86ba2016-03-15 12:03:28 -0700637
638 return NULL;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600639}
640EXPORT_SYMBOL(blk_mq_tag_to_rq);
641
Jens Axboe320ae512013-10-24 09:20:05 +0100642struct blk_mq_timeout_data {
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700643 unsigned long next;
644 unsigned int next_set;
Jens Axboe320ae512013-10-24 09:20:05 +0100645};
646
Christoph Hellwig90415832014-09-22 10:21:48 -0600647void blk_mq_rq_timed_out(struct request *req, bool reserved)
Jens Axboe320ae512013-10-24 09:20:05 +0100648{
Jens Axboef8a5b122016-12-13 09:24:51 -0700649 const struct blk_mq_ops *ops = req->q->mq_ops;
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700650 enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
Jens Axboe87ee7b12014-04-24 08:51:47 -0600651
652 /*
653 * We know that complete is set at this point. If STARTED isn't set
654 * anymore, then the request isn't active and the "timeout" should
655 * just be ignored. This can happen due to the bitflag ordering.
656 * Timeout first checks if STARTED is set, and if it is, assumes
657 * the request is active. But if we race with completion, then
Jens Axboe48b99c92017-03-29 11:10:34 -0600658 * both flags will get cleared. So check here again, and ignore
Jens Axboe87ee7b12014-04-24 08:51:47 -0600659 * a timeout event with a request that isn't active.
660 */
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700661 if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
662 return;
Jens Axboe87ee7b12014-04-24 08:51:47 -0600663
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700664 if (ops->timeout)
Christoph Hellwig0152fb62014-09-13 16:40:13 -0700665 ret = ops->timeout(req, reserved);
Jens Axboe87ee7b12014-04-24 08:51:47 -0600666
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700667 switch (ret) {
668 case BLK_EH_HANDLED:
669 __blk_mq_complete_request(req);
670 break;
671 case BLK_EH_RESET_TIMER:
672 blk_add_timer(req);
673 blk_clear_rq_complete(req);
674 break;
675 case BLK_EH_NOT_HANDLED:
676 break;
677 default:
678 printk(KERN_ERR "block: bad eh return: %d\n", ret);
679 break;
680 }
Jens Axboe87ee7b12014-04-24 08:51:47 -0600681}
Keith Busch5b3f25f2015-01-07 18:55:46 -0700682
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700683static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
684 struct request *rq, void *priv, bool reserved)
Jens Axboe320ae512013-10-24 09:20:05 +0100685{
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700686 struct blk_mq_timeout_data *data = priv;
687
Ming Lei95a49602017-03-22 10:14:43 +0800688 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700689 return;
Jens Axboe320ae512013-10-24 09:20:05 +0100690
Ming Leid9d149a2017-03-27 20:06:55 +0800691 /*
692 * The rq being checked may have been freed and reallocated
693 * out already here, we avoid this race by checking rq->deadline
694 * and REQ_ATOM_COMPLETE flag together:
695 *
696 * - if rq->deadline is observed as new value because of
697 * reusing, the rq won't be timed out because of timing.
698 * - if rq->deadline is observed as previous value,
699 * REQ_ATOM_COMPLETE flag won't be cleared in reuse path
700 * because we put a barrier between setting rq->deadline
701 * and clearing the flag in blk_mq_start_request(), so
702 * this rq won't be timed out too.
703 */
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700704 if (time_after_eq(jiffies, rq->deadline)) {
705 if (!blk_mark_rq_complete(rq))
Christoph Hellwig0152fb62014-09-13 16:40:13 -0700706 blk_mq_rq_timed_out(rq, reserved);
Christoph Hellwig46f92d42014-09-13 16:40:12 -0700707 } else if (!data->next_set || time_after(data->next, rq->deadline)) {
708 data->next = rq->deadline;
709 data->next_set = 1;
710 }
Jens Axboe320ae512013-10-24 09:20:05 +0100711}
712
Christoph Hellwig287922eb2015-10-30 20:57:30 +0800713static void blk_mq_timeout_work(struct work_struct *work)
Jens Axboe320ae512013-10-24 09:20:05 +0100714{
Christoph Hellwig287922eb2015-10-30 20:57:30 +0800715 struct request_queue *q =
716 container_of(work, struct request_queue, timeout_work);
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700717 struct blk_mq_timeout_data data = {
718 .next = 0,
719 .next_set = 0,
720 };
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700721 int i;
Jens Axboe320ae512013-10-24 09:20:05 +0100722
Gabriel Krisman Bertazi71f79fb2016-08-01 08:23:39 -0600723 /* A deadlock might occur if a request is stuck requiring a
724 * timeout at the same time a queue freeze is waiting
725 * completion, since the timeout code would not be able to
726 * acquire the queue reference here.
727 *
728 * That's why we don't use blk_queue_enter here; instead, we use
729 * percpu_ref_tryget directly, because we need to be able to
730 * obtain a reference even in the short window between the queue
731 * starting to freeze, by dropping the first reference in
Ming Lei1671d522017-03-27 20:06:57 +0800732 * blk_freeze_queue_start, and the moment the last request is
Gabriel Krisman Bertazi71f79fb2016-08-01 08:23:39 -0600733 * consumed, marked by the instant q_usage_counter reaches
734 * zero.
735 */
736 if (!percpu_ref_tryget(&q->q_usage_counter))
Christoph Hellwig287922eb2015-10-30 20:57:30 +0800737 return;
738
Christoph Hellwig0bf6cd52015-09-27 21:01:51 +0200739 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
Jens Axboe320ae512013-10-24 09:20:05 +0100740
Christoph Hellwig81481eb2014-09-13 16:40:11 -0700741 if (data.next_set) {
742 data.next = blk_rq_timeout(round_jiffies_up(data.next));
743 mod_timer(&q->timeout, data.next);
Jens Axboe0d2602c2014-05-13 15:10:52 -0600744 } else {
Christoph Hellwig0bf6cd52015-09-27 21:01:51 +0200745 struct blk_mq_hw_ctx *hctx;
746
Ming Leif054b562015-04-21 10:00:19 +0800747 queue_for_each_hw_ctx(q, hctx, i) {
748 /* the hctx may be unmapped, so check it here */
749 if (blk_mq_hw_queue_mapped(hctx))
750 blk_mq_tag_idle(hctx);
751 }
Jens Axboe0d2602c2014-05-13 15:10:52 -0600752 }
Christoph Hellwig287922eb2015-10-30 20:57:30 +0800753 blk_queue_exit(q);
Jens Axboe320ae512013-10-24 09:20:05 +0100754}
755
756/*
757 * Reverse check our software queue for entries that we could potentially
758 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
759 * too much time checking for merges.
760 */
761static bool blk_mq_attempt_merge(struct request_queue *q,
762 struct blk_mq_ctx *ctx, struct bio *bio)
763{
764 struct request *rq;
765 int checked = 8;
766
767 list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100768 bool merged = false;
Jens Axboe320ae512013-10-24 09:20:05 +0100769
770 if (!checked--)
771 break;
772
773 if (!blk_rq_merge_ok(rq, bio))
774 continue;
775
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100776 switch (blk_try_merge(rq, bio)) {
777 case ELEVATOR_BACK_MERGE:
778 if (blk_mq_sched_allow_merge(q, rq, bio))
779 merged = bio_attempt_back_merge(q, rq, bio);
780 break;
781 case ELEVATOR_FRONT_MERGE:
782 if (blk_mq_sched_allow_merge(q, rq, bio))
783 merged = bio_attempt_front_merge(q, rq, bio);
784 break;
Christoph Hellwig1e739732017-02-08 14:46:49 +0100785 case ELEVATOR_DISCARD_MERGE:
786 merged = bio_attempt_discard_merge(q, rq, bio);
787 break;
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100788 default:
Jens Axboebd166ef2017-01-17 06:03:22 -0700789 continue;
Jens Axboe320ae512013-10-24 09:20:05 +0100790 }
Christoph Hellwig34fe7c02017-02-08 14:46:48 +0100791
792 if (merged)
793 ctx->rq_merged++;
794 return merged;
Jens Axboe320ae512013-10-24 09:20:05 +0100795 }
796
797 return false;
798}
799
Omar Sandoval88459642016-09-17 08:38:44 -0600800struct flush_busy_ctx_data {
801 struct blk_mq_hw_ctx *hctx;
802 struct list_head *list;
803};
804
805static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
806{
807 struct flush_busy_ctx_data *flush_data = data;
808 struct blk_mq_hw_ctx *hctx = flush_data->hctx;
809 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
810
811 sbitmap_clear_bit(sb, bitnr);
812 spin_lock(&ctx->lock);
813 list_splice_tail_init(&ctx->rq_list, flush_data->list);
814 spin_unlock(&ctx->lock);
815 return true;
816}
817
Jens Axboe320ae512013-10-24 09:20:05 +0100818/*
Jens Axboe1429d7c2014-05-19 09:23:55 -0600819 * Process software queues that have been marked busy, splicing them
820 * to the for-dispatch
821 */
Jens Axboe2c3ad662016-12-14 14:34:47 -0700822void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
Jens Axboe1429d7c2014-05-19 09:23:55 -0600823{
Omar Sandoval88459642016-09-17 08:38:44 -0600824 struct flush_busy_ctx_data data = {
825 .hctx = hctx,
826 .list = list,
827 };
Jens Axboe1429d7c2014-05-19 09:23:55 -0600828
Omar Sandoval88459642016-09-17 08:38:44 -0600829 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
Jens Axboe1429d7c2014-05-19 09:23:55 -0600830}
Jens Axboe2c3ad662016-12-14 14:34:47 -0700831EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
Jens Axboe1429d7c2014-05-19 09:23:55 -0600832
Jens Axboe703fd1c2016-09-16 13:59:14 -0600833static inline unsigned int queued_to_index(unsigned int queued)
834{
835 if (!queued)
836 return 0;
Jens Axboe1429d7c2014-05-19 09:23:55 -0600837
Jens Axboe703fd1c2016-09-16 13:59:14 -0600838 return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
Jens Axboe1429d7c2014-05-19 09:23:55 -0600839}
840
Jens Axboebd6737f2017-01-27 01:00:47 -0700841bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
842 bool wait)
Jens Axboebd166ef2017-01-17 06:03:22 -0700843{
844 struct blk_mq_alloc_data data = {
845 .q = rq->q,
Jens Axboebd166ef2017-01-17 06:03:22 -0700846 .hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
847 .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
848 };
849
Jens Axboe5feeacd2017-04-20 17:23:13 -0600850 might_sleep_if(wait);
851
Omar Sandoval81380ca2017-04-07 08:56:26 -0600852 if (rq->tag != -1)
853 goto done;
Jens Axboebd166ef2017-01-17 06:03:22 -0700854
Sagi Grimberg415b8062017-02-27 10:04:39 -0700855 if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
856 data.flags |= BLK_MQ_REQ_RESERVED;
857
Jens Axboebd166ef2017-01-17 06:03:22 -0700858 rq->tag = blk_mq_get_tag(&data);
859 if (rq->tag >= 0) {
Jens Axboe200e86b2017-01-25 08:11:38 -0700860 if (blk_mq_tag_busy(data.hctx)) {
861 rq->rq_flags |= RQF_MQ_INFLIGHT;
862 atomic_inc(&data.hctx->nr_active);
863 }
Jens Axboebd166ef2017-01-17 06:03:22 -0700864 data.hctx->tags->rqs[rq->tag] = rq;
Jens Axboebd166ef2017-01-17 06:03:22 -0700865 }
866
Omar Sandoval81380ca2017-04-07 08:56:26 -0600867done:
868 if (hctx)
869 *hctx = data.hctx;
870 return rq->tag != -1;
Jens Axboebd166ef2017-01-17 06:03:22 -0700871}
872
Jens Axboe113285b2017-03-02 13:26:04 -0700873static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
874 struct request *rq)
Jens Axboe99cf1dc2017-01-26 12:32:32 -0700875{
Jens Axboe99cf1dc2017-01-26 12:32:32 -0700876 blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
877 rq->tag = -1;
878
879 if (rq->rq_flags & RQF_MQ_INFLIGHT) {
880 rq->rq_flags &= ~RQF_MQ_INFLIGHT;
881 atomic_dec(&hctx->nr_active);
882 }
883}
884
Jens Axboe113285b2017-03-02 13:26:04 -0700885static void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx,
886 struct request *rq)
887{
888 if (rq->tag == -1 || rq->internal_tag == -1)
889 return;
890
891 __blk_mq_put_driver_tag(hctx, rq);
892}
893
894static void blk_mq_put_driver_tag(struct request *rq)
895{
896 struct blk_mq_hw_ctx *hctx;
897
898 if (rq->tag == -1 || rq->internal_tag == -1)
899 return;
900
901 hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu);
902 __blk_mq_put_driver_tag(hctx, rq);
903}
904
Jens Axboebd166ef2017-01-17 06:03:22 -0700905/*
906 * If we fail getting a driver tag because all the driver tags are already
907 * assigned and on the dispatch list, BUT the first entry does not have a
908 * tag, then we could deadlock. For that case, move entries with assigned
909 * driver tags to the front, leaving the set of tagged requests in the
910 * same order, and the untagged set in the same order.
911 */
912static bool reorder_tags_to_front(struct list_head *list)
913{
914 struct request *rq, *tmp, *first = NULL;
915
916 list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
917 if (rq == first)
918 break;
919 if (rq->tag != -1) {
920 list_move(&rq->queuelist, list);
921 if (!first)
922 first = rq;
923 }
924 }
925
926 return first != NULL;
927}
928
Omar Sandovalda55f2c2017-02-22 10:58:29 -0800929static int blk_mq_dispatch_wake(wait_queue_t *wait, unsigned mode, int flags,
930 void *key)
931{
932 struct blk_mq_hw_ctx *hctx;
933
934 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
935
936 list_del(&wait->task_list);
937 clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state);
938 blk_mq_run_hw_queue(hctx, true);
939 return 1;
940}
941
942static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx)
943{
944 struct sbq_wait_state *ws;
945
946 /*
947 * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait.
948 * The thread which wins the race to grab this bit adds the hardware
949 * queue to the wait queue.
950 */
951 if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) ||
952 test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state))
953 return false;
954
955 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
956 ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx);
957
958 /*
959 * As soon as this returns, it's no longer safe to fiddle with
960 * hctx->dispatch_wait, since a completion can wake up the wait queue
961 * and unlock the bit.
962 */
963 add_wait_queue(&ws->wait, &hctx->dispatch_wait);
964 return true;
965}
966
Omar Sandoval81380ca2017-04-07 08:56:26 -0600967bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list)
Jens Axboef04c3df2016-12-07 08:41:17 -0700968{
Omar Sandoval81380ca2017-04-07 08:56:26 -0600969 struct blk_mq_hw_ctx *hctx;
Jens Axboef04c3df2016-12-07 08:41:17 -0700970 struct request *rq;
Jens Axboe93efe982017-03-24 12:04:19 -0600971 int errors, queued, ret = BLK_MQ_RQ_QUEUE_OK;
Jens Axboef04c3df2016-12-07 08:41:17 -0700972
Omar Sandoval81380ca2017-04-07 08:56:26 -0600973 if (list_empty(list))
974 return false;
975
Jens Axboef04c3df2016-12-07 08:41:17 -0700976 /*
Jens Axboef04c3df2016-12-07 08:41:17 -0700977 * Now process all the entries, sending them to the driver.
978 */
Jens Axboe93efe982017-03-24 12:04:19 -0600979 errors = queued = 0;
Omar Sandoval81380ca2017-04-07 08:56:26 -0600980 do {
Jens Axboef04c3df2016-12-07 08:41:17 -0700981 struct blk_mq_queue_data bd;
982
983 rq = list_first_entry(list, struct request, queuelist);
Jens Axboebd166ef2017-01-17 06:03:22 -0700984 if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
985 if (!queued && reorder_tags_to_front(list))
986 continue;
Jens Axboe3c782d62017-01-26 12:50:36 -0700987
988 /*
Omar Sandovalda55f2c2017-02-22 10:58:29 -0800989 * The initial allocation attempt failed, so we need to
990 * rerun the hardware queue when a tag is freed.
Jens Axboe3c782d62017-01-26 12:50:36 -0700991 */
Omar Sandoval807b1042017-04-05 12:01:35 -0700992 if (!blk_mq_dispatch_wait_add(hctx))
Jens Axboe3c782d62017-01-26 12:50:36 -0700993 break;
Omar Sandoval807b1042017-04-05 12:01:35 -0700994
995 /*
996 * It's possible that a tag was freed in the window
997 * between the allocation failure and adding the
998 * hardware queue to the wait queue.
999 */
1000 if (!blk_mq_get_driver_tag(rq, &hctx, false))
1001 break;
Jens Axboebd166ef2017-01-17 06:03:22 -07001002 }
Omar Sandovalda55f2c2017-02-22 10:58:29 -08001003
Jens Axboef04c3df2016-12-07 08:41:17 -07001004 list_del_init(&rq->queuelist);
1005
1006 bd.rq = rq;
Jens Axboe113285b2017-03-02 13:26:04 -07001007
1008 /*
1009 * Flag last if we have no more requests, or if we have more
1010 * but can't assign a driver tag to it.
1011 */
1012 if (list_empty(list))
1013 bd.last = true;
1014 else {
1015 struct request *nxt;
1016
1017 nxt = list_first_entry(list, struct request, queuelist);
1018 bd.last = !blk_mq_get_driver_tag(nxt, NULL, false);
1019 }
Jens Axboef04c3df2016-12-07 08:41:17 -07001020
1021 ret = q->mq_ops->queue_rq(hctx, &bd);
1022 switch (ret) {
1023 case BLK_MQ_RQ_QUEUE_OK:
1024 queued++;
1025 break;
1026 case BLK_MQ_RQ_QUEUE_BUSY:
Jens Axboe113285b2017-03-02 13:26:04 -07001027 blk_mq_put_driver_tag_hctx(hctx, rq);
Jens Axboef04c3df2016-12-07 08:41:17 -07001028 list_add(&rq->queuelist, list);
1029 __blk_mq_requeue_request(rq);
1030 break;
1031 default:
1032 pr_err("blk-mq: bad return on queue: %d\n", ret);
1033 case BLK_MQ_RQ_QUEUE_ERROR:
Jens Axboe93efe982017-03-24 12:04:19 -06001034 errors++;
Christoph Hellwigcaf7df12017-04-20 16:03:16 +02001035 blk_mq_end_request(rq, -EIO);
Jens Axboef04c3df2016-12-07 08:41:17 -07001036 break;
1037 }
1038
1039 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
1040 break;
Omar Sandoval81380ca2017-04-07 08:56:26 -06001041 } while (!list_empty(list));
Jens Axboef04c3df2016-12-07 08:41:17 -07001042
1043 hctx->dispatched[queued_to_index(queued)]++;
1044
1045 /*
1046 * Any items that need requeuing? Stuff them into hctx->dispatch,
1047 * that is where we will continue on next queue run.
1048 */
1049 if (!list_empty(list)) {
Jens Axboe113285b2017-03-02 13:26:04 -07001050 /*
Bart Van Assche710c7852017-04-07 11:16:51 -07001051 * If an I/O scheduler has been configured and we got a driver
1052 * tag for the next request already, free it again.
Jens Axboe113285b2017-03-02 13:26:04 -07001053 */
1054 rq = list_first_entry(list, struct request, queuelist);
1055 blk_mq_put_driver_tag(rq);
1056
Jens Axboef04c3df2016-12-07 08:41:17 -07001057 spin_lock(&hctx->lock);
Jens Axboec13660a2017-01-26 12:40:07 -07001058 list_splice_init(list, &hctx->dispatch);
Jens Axboef04c3df2016-12-07 08:41:17 -07001059 spin_unlock(&hctx->lock);
1060
1061 /*
Bart Van Assche710c7852017-04-07 11:16:51 -07001062 * If SCHED_RESTART was set by the caller of this function and
1063 * it is no longer set that means that it was cleared by another
1064 * thread and hence that a queue rerun is needed.
Jens Axboef04c3df2016-12-07 08:41:17 -07001065 *
Bart Van Assche710c7852017-04-07 11:16:51 -07001066 * If TAG_WAITING is set that means that an I/O scheduler has
1067 * been configured and another thread is waiting for a driver
1068 * tag. To guarantee fairness, do not rerun this hardware queue
1069 * but let the other thread grab the driver tag.
Jens Axboebd166ef2017-01-17 06:03:22 -07001070 *
Bart Van Assche710c7852017-04-07 11:16:51 -07001071 * If no I/O scheduler has been configured it is possible that
1072 * the hardware queue got stopped and restarted before requests
1073 * were pushed back onto the dispatch list. Rerun the queue to
1074 * avoid starvation. Notes:
1075 * - blk_mq_run_hw_queue() checks whether or not a queue has
1076 * been stopped before rerunning a queue.
1077 * - Some but not all block drivers stop a queue before
1078 * returning BLK_MQ_RQ_QUEUE_BUSY. Two exceptions are scsi-mq
1079 * and dm-rq.
Jens Axboebd166ef2017-01-17 06:03:22 -07001080 */
Omar Sandovalda55f2c2017-02-22 10:58:29 -08001081 if (!blk_mq_sched_needs_restart(hctx) &&
1082 !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state))
Jens Axboebd166ef2017-01-17 06:03:22 -07001083 blk_mq_run_hw_queue(hctx, true);
Jens Axboef04c3df2016-12-07 08:41:17 -07001084 }
1085
Jens Axboe93efe982017-03-24 12:04:19 -06001086 return (queued + errors) != 0;
Jens Axboef04c3df2016-12-07 08:41:17 -07001087}
1088
Bart Van Assche6a83e742016-11-02 10:09:51 -06001089static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
1090{
1091 int srcu_idx;
1092
1093 WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
1094 cpu_online(hctx->next_cpu));
1095
1096 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
1097 rcu_read_lock();
Jens Axboebd166ef2017-01-17 06:03:22 -07001098 blk_mq_sched_dispatch_requests(hctx);
Bart Van Assche6a83e742016-11-02 10:09:51 -06001099 rcu_read_unlock();
1100 } else {
Jens Axboebf4907c2017-03-30 12:30:39 -06001101 might_sleep();
1102
Bart Van Assche6a83e742016-11-02 10:09:51 -06001103 srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
Jens Axboebd166ef2017-01-17 06:03:22 -07001104 blk_mq_sched_dispatch_requests(hctx);
Bart Van Assche6a83e742016-11-02 10:09:51 -06001105 srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
1106 }
1107}
1108
Jens Axboe506e9312014-05-07 10:26:44 -06001109/*
1110 * It'd be great if the workqueue API had a way to pass
1111 * in a mask and had some smarts for more clever placement.
1112 * For now we just round-robin here, switching for every
1113 * BLK_MQ_CPU_WORK_BATCH queued items.
1114 */
1115static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
1116{
Christoph Hellwigb657d7e2014-11-24 09:27:23 +01001117 if (hctx->queue->nr_hw_queues == 1)
1118 return WORK_CPU_UNBOUND;
Jens Axboe506e9312014-05-07 10:26:44 -06001119
1120 if (--hctx->next_cpu_batch <= 0) {
Gabriel Krisman Bertazic02ebfd2016-09-28 00:24:24 -03001121 int next_cpu;
Jens Axboe506e9312014-05-07 10:26:44 -06001122
1123 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
1124 if (next_cpu >= nr_cpu_ids)
1125 next_cpu = cpumask_first(hctx->cpumask);
1126
1127 hctx->next_cpu = next_cpu;
1128 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1129 }
1130
Christoph Hellwigb657d7e2014-11-24 09:27:23 +01001131 return hctx->next_cpu;
Jens Axboe506e9312014-05-07 10:26:44 -06001132}
1133
Bart Van Assche7587a5a2017-04-07 11:16:52 -07001134static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
1135 unsigned long msecs)
Jens Axboe320ae512013-10-24 09:20:05 +01001136{
Bart Van Assche5d1b25c2016-10-28 17:19:15 -07001137 if (unlikely(blk_mq_hctx_stopped(hctx) ||
1138 !blk_mq_hw_queue_mapped(hctx)))
Jens Axboe320ae512013-10-24 09:20:05 +01001139 return;
1140
Jens Axboe1b792f22016-09-21 10:12:13 -06001141 if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
Paolo Bonzini2a90d4a2014-11-07 23:04:00 +01001142 int cpu = get_cpu();
1143 if (cpumask_test_cpu(cpu, hctx->cpumask)) {
Paolo Bonzini398205b2014-11-07 23:03:59 +01001144 __blk_mq_run_hw_queue(hctx);
Paolo Bonzini2a90d4a2014-11-07 23:04:00 +01001145 put_cpu();
Paolo Bonzini398205b2014-11-07 23:03:59 +01001146 return;
1147 }
Jens Axboee4043dc2014-04-09 10:18:23 -06001148
Paolo Bonzini2a90d4a2014-11-07 23:04:00 +01001149 put_cpu();
Jens Axboee4043dc2014-04-09 10:18:23 -06001150 }
Paolo Bonzini398205b2014-11-07 23:03:59 +01001151
Jens Axboe9f993732017-04-10 09:54:54 -06001152 kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
1153 &hctx->run_work,
1154 msecs_to_jiffies(msecs));
Bart Van Assche7587a5a2017-04-07 11:16:52 -07001155}
1156
1157void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1158{
1159 __blk_mq_delay_run_hw_queue(hctx, true, msecs);
1160}
1161EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
1162
1163void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1164{
1165 __blk_mq_delay_run_hw_queue(hctx, async, 0);
Jens Axboe320ae512013-10-24 09:20:05 +01001166}
Omar Sandoval5b727272017-04-14 01:00:00 -07001167EXPORT_SYMBOL(blk_mq_run_hw_queue);
Jens Axboe320ae512013-10-24 09:20:05 +01001168
Mike Snitzerb94ec292015-03-11 23:56:38 -04001169void blk_mq_run_hw_queues(struct request_queue *q, bool async)
Jens Axboe320ae512013-10-24 09:20:05 +01001170{
1171 struct blk_mq_hw_ctx *hctx;
1172 int i;
1173
1174 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboebd166ef2017-01-17 06:03:22 -07001175 if (!blk_mq_hctx_has_pending(hctx) ||
Bart Van Assche5d1b25c2016-10-28 17:19:15 -07001176 blk_mq_hctx_stopped(hctx))
Jens Axboe320ae512013-10-24 09:20:05 +01001177 continue;
1178
Mike Snitzerb94ec292015-03-11 23:56:38 -04001179 blk_mq_run_hw_queue(hctx, async);
Jens Axboe320ae512013-10-24 09:20:05 +01001180 }
1181}
Mike Snitzerb94ec292015-03-11 23:56:38 -04001182EXPORT_SYMBOL(blk_mq_run_hw_queues);
Jens Axboe320ae512013-10-24 09:20:05 +01001183
Bart Van Asschefd001442016-10-28 17:19:37 -07001184/**
1185 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
1186 * @q: request queue.
1187 *
1188 * The caller is responsible for serializing this function against
1189 * blk_mq_{start,stop}_hw_queue().
1190 */
1191bool blk_mq_queue_stopped(struct request_queue *q)
1192{
1193 struct blk_mq_hw_ctx *hctx;
1194 int i;
1195
1196 queue_for_each_hw_ctx(q, hctx, i)
1197 if (blk_mq_hctx_stopped(hctx))
1198 return true;
1199
1200 return false;
1201}
1202EXPORT_SYMBOL(blk_mq_queue_stopped);
1203
Jens Axboe2719aa22017-05-03 11:08:14 -06001204static void __blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx, bool sync)
1205{
1206 if (sync)
1207 cancel_delayed_work_sync(&hctx->run_work);
1208 else
1209 cancel_delayed_work(&hctx->run_work);
1210
1211 set_bit(BLK_MQ_S_STOPPED, &hctx->state);
1212}
1213
Jens Axboe320ae512013-10-24 09:20:05 +01001214void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
1215{
Jens Axboe2719aa22017-05-03 11:08:14 -06001216 __blk_mq_stop_hw_queue(hctx, false);
Jens Axboe320ae512013-10-24 09:20:05 +01001217}
1218EXPORT_SYMBOL(blk_mq_stop_hw_queue);
1219
Colin Ian Kingebd76852017-05-08 11:44:40 +01001220static void __blk_mq_stop_hw_queues(struct request_queue *q, bool sync)
Christoph Hellwig280d45f2013-10-25 14:45:58 +01001221{
1222 struct blk_mq_hw_ctx *hctx;
1223 int i;
1224
1225 queue_for_each_hw_ctx(q, hctx, i)
Jens Axboe2719aa22017-05-03 11:08:14 -06001226 __blk_mq_stop_hw_queue(hctx, sync);
1227}
1228
1229void blk_mq_stop_hw_queues(struct request_queue *q)
1230{
1231 __blk_mq_stop_hw_queues(q, false);
Christoph Hellwig280d45f2013-10-25 14:45:58 +01001232}
1233EXPORT_SYMBOL(blk_mq_stop_hw_queues);
1234
Jens Axboe320ae512013-10-24 09:20:05 +01001235void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
1236{
1237 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
Jens Axboee4043dc2014-04-09 10:18:23 -06001238
Jens Axboe0ffbce82014-06-25 08:22:34 -06001239 blk_mq_run_hw_queue(hctx, false);
Jens Axboe320ae512013-10-24 09:20:05 +01001240}
1241EXPORT_SYMBOL(blk_mq_start_hw_queue);
1242
Christoph Hellwig2f268552014-04-16 09:44:56 +02001243void blk_mq_start_hw_queues(struct request_queue *q)
1244{
1245 struct blk_mq_hw_ctx *hctx;
1246 int i;
1247
1248 queue_for_each_hw_ctx(q, hctx, i)
1249 blk_mq_start_hw_queue(hctx);
1250}
1251EXPORT_SYMBOL(blk_mq_start_hw_queues);
1252
Jens Axboeae911c52016-12-08 13:19:30 -07001253void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1254{
1255 if (!blk_mq_hctx_stopped(hctx))
1256 return;
1257
1258 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1259 blk_mq_run_hw_queue(hctx, async);
1260}
1261EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
1262
Christoph Hellwig1b4a3252014-04-16 09:44:54 +02001263void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
Jens Axboe320ae512013-10-24 09:20:05 +01001264{
1265 struct blk_mq_hw_ctx *hctx;
1266 int i;
1267
Jens Axboeae911c52016-12-08 13:19:30 -07001268 queue_for_each_hw_ctx(q, hctx, i)
1269 blk_mq_start_stopped_hw_queue(hctx, async);
Jens Axboe320ae512013-10-24 09:20:05 +01001270}
1271EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
1272
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001273static void blk_mq_run_work_fn(struct work_struct *work)
Jens Axboe320ae512013-10-24 09:20:05 +01001274{
1275 struct blk_mq_hw_ctx *hctx;
1276
Jens Axboe9f993732017-04-10 09:54:54 -06001277 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
Jens Axboe21c6e932017-04-10 09:54:56 -06001278
1279 /*
1280 * If we are stopped, don't run the queue. The exception is if
1281 * BLK_MQ_S_START_ON_RUN is set. For that case, we auto-clear
1282 * the STOPPED bit and run it.
1283 */
1284 if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) {
1285 if (!test_bit(BLK_MQ_S_START_ON_RUN, &hctx->state))
1286 return;
1287
1288 clear_bit(BLK_MQ_S_START_ON_RUN, &hctx->state);
1289 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1290 }
Jens Axboee4043dc2014-04-09 10:18:23 -06001291
Jens Axboe320ae512013-10-24 09:20:05 +01001292 __blk_mq_run_hw_queue(hctx);
1293}
1294
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001295
1296void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1297{
Ming Lei19c66e52014-12-03 19:38:04 +08001298 if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
1299 return;
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001300
Jens Axboe21c6e932017-04-10 09:54:56 -06001301 /*
1302 * Stop the hw queue, then modify currently delayed work.
1303 * This should prevent us from running the queue prematurely.
1304 * Mark the queue as auto-clearing STOPPED when it runs.
1305 */
Jens Axboe7e79dad2017-01-19 07:58:59 -07001306 blk_mq_stop_hw_queue(hctx);
Jens Axboe21c6e932017-04-10 09:54:56 -06001307 set_bit(BLK_MQ_S_START_ON_RUN, &hctx->state);
1308 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
1309 &hctx->run_work,
1310 msecs_to_jiffies(msecs));
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001311}
1312EXPORT_SYMBOL(blk_mq_delay_queue);
1313
Ming Leicfd0c552015-10-20 23:13:57 +08001314static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
Ming Leicfd0c552015-10-20 23:13:57 +08001315 struct request *rq,
1316 bool at_head)
Jens Axboe320ae512013-10-24 09:20:05 +01001317{
Jens Axboee57690f2016-08-24 15:34:35 -06001318 struct blk_mq_ctx *ctx = rq->mq_ctx;
1319
Jens Axboe01b983c2013-11-19 18:59:10 -07001320 trace_block_rq_insert(hctx->queue, rq);
1321
Christoph Hellwig72a0a362014-02-07 10:22:36 -08001322 if (at_head)
1323 list_add(&rq->queuelist, &ctx->rq_list);
1324 else
1325 list_add_tail(&rq->queuelist, &ctx->rq_list);
Ming Leicfd0c552015-10-20 23:13:57 +08001326}
Jens Axboe4bb659b2014-05-09 09:36:49 -06001327
Jens Axboe2c3ad662016-12-14 14:34:47 -07001328void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
1329 bool at_head)
Ming Leicfd0c552015-10-20 23:13:57 +08001330{
1331 struct blk_mq_ctx *ctx = rq->mq_ctx;
1332
Jens Axboee57690f2016-08-24 15:34:35 -06001333 __blk_mq_insert_req_list(hctx, rq, at_head);
Jens Axboe320ae512013-10-24 09:20:05 +01001334 blk_mq_hctx_mark_pending(hctx, ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001335}
1336
Jens Axboebd166ef2017-01-17 06:03:22 -07001337void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
1338 struct list_head *list)
Jens Axboe320ae512013-10-24 09:20:05 +01001339
1340{
Jens Axboe320ae512013-10-24 09:20:05 +01001341 /*
1342 * preemption doesn't flush plug list, so it's possible ctx->cpu is
1343 * offline now
1344 */
1345 spin_lock(&ctx->lock);
1346 while (!list_empty(list)) {
1347 struct request *rq;
1348
1349 rq = list_first_entry(list, struct request, queuelist);
Jens Axboee57690f2016-08-24 15:34:35 -06001350 BUG_ON(rq->mq_ctx != ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001351 list_del_init(&rq->queuelist);
Jens Axboee57690f2016-08-24 15:34:35 -06001352 __blk_mq_insert_req_list(hctx, rq, false);
Jens Axboe320ae512013-10-24 09:20:05 +01001353 }
Ming Leicfd0c552015-10-20 23:13:57 +08001354 blk_mq_hctx_mark_pending(hctx, ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001355 spin_unlock(&ctx->lock);
Jens Axboe320ae512013-10-24 09:20:05 +01001356}
1357
1358static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1359{
1360 struct request *rqa = container_of(a, struct request, queuelist);
1361 struct request *rqb = container_of(b, struct request, queuelist);
1362
1363 return !(rqa->mq_ctx < rqb->mq_ctx ||
1364 (rqa->mq_ctx == rqb->mq_ctx &&
1365 blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1366}
1367
1368void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1369{
1370 struct blk_mq_ctx *this_ctx;
1371 struct request_queue *this_q;
1372 struct request *rq;
1373 LIST_HEAD(list);
1374 LIST_HEAD(ctx_list);
1375 unsigned int depth;
1376
1377 list_splice_init(&plug->mq_list, &list);
1378
1379 list_sort(NULL, &list, plug_ctx_cmp);
1380
1381 this_q = NULL;
1382 this_ctx = NULL;
1383 depth = 0;
1384
1385 while (!list_empty(&list)) {
1386 rq = list_entry_rq(list.next);
1387 list_del_init(&rq->queuelist);
1388 BUG_ON(!rq->q);
1389 if (rq->mq_ctx != this_ctx) {
1390 if (this_ctx) {
Jens Axboebd166ef2017-01-17 06:03:22 -07001391 trace_block_unplug(this_q, depth, from_schedule);
1392 blk_mq_sched_insert_requests(this_q, this_ctx,
1393 &ctx_list,
1394 from_schedule);
Jens Axboe320ae512013-10-24 09:20:05 +01001395 }
1396
1397 this_ctx = rq->mq_ctx;
1398 this_q = rq->q;
1399 depth = 0;
1400 }
1401
1402 depth++;
1403 list_add_tail(&rq->queuelist, &ctx_list);
1404 }
1405
1406 /*
1407 * If 'this_ctx' is set, we know we have entries to complete
1408 * on 'ctx_list'. Do those.
1409 */
1410 if (this_ctx) {
Jens Axboebd166ef2017-01-17 06:03:22 -07001411 trace_block_unplug(this_q, depth, from_schedule);
1412 blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
1413 from_schedule);
Jens Axboe320ae512013-10-24 09:20:05 +01001414 }
1415}
1416
1417static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1418{
Bart Van Asscheda8d7f02017-04-19 14:01:24 -07001419 blk_init_request_from_bio(rq, bio);
Jens Axboe4b570522014-05-29 11:00:11 -06001420
Jens Axboe6e85eaf2016-12-02 20:00:14 -07001421 blk_account_io_start(rq, true);
Jens Axboe320ae512013-10-24 09:20:05 +01001422}
1423
Jens Axboe274a5842014-08-15 12:44:08 -06001424static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
1425{
1426 return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
1427 !blk_queue_nomerges(hctx->queue);
1428}
1429
Jens Axboe07068d52014-05-22 10:40:51 -06001430static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1431 struct blk_mq_ctx *ctx,
1432 struct request *rq, struct bio *bio)
1433{
Ming Leie18378a2015-10-20 23:13:54 +08001434 if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) {
Jens Axboe07068d52014-05-22 10:40:51 -06001435 blk_mq_bio_to_request(rq, bio);
1436 spin_lock(&ctx->lock);
1437insert_rq:
1438 __blk_mq_insert_request(hctx, rq, false);
1439 spin_unlock(&ctx->lock);
1440 return false;
1441 } else {
Jens Axboe274a5842014-08-15 12:44:08 -06001442 struct request_queue *q = hctx->queue;
1443
Jens Axboe07068d52014-05-22 10:40:51 -06001444 spin_lock(&ctx->lock);
1445 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1446 blk_mq_bio_to_request(rq, bio);
1447 goto insert_rq;
1448 }
1449
1450 spin_unlock(&ctx->lock);
Jens Axboebd166ef2017-01-17 06:03:22 -07001451 __blk_mq_finish_request(hctx, ctx, rq);
Jens Axboe07068d52014-05-22 10:40:51 -06001452 return true;
1453 }
1454}
1455
Jens Axboefd2d3322017-01-12 10:04:45 -07001456static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
1457{
Jens Axboebd166ef2017-01-17 06:03:22 -07001458 if (rq->tag != -1)
1459 return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);
1460
1461 return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
Jens Axboefd2d3322017-01-12 10:04:45 -07001462}
1463
Ming Leid964f042017-06-06 23:22:00 +08001464static void __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
1465 struct request *rq,
1466 blk_qc_t *cookie, bool may_sleep)
Shaohua Lif984df12015-05-08 10:51:32 -07001467{
Shaohua Lif984df12015-05-08 10:51:32 -07001468 struct request_queue *q = rq->q;
Shaohua Lif984df12015-05-08 10:51:32 -07001469 struct blk_mq_queue_data bd = {
1470 .rq = rq,
Omar Sandovald945a362017-04-05 12:01:36 -07001471 .last = true,
Shaohua Lif984df12015-05-08 10:51:32 -07001472 };
Jens Axboebd166ef2017-01-17 06:03:22 -07001473 blk_qc_t new_cookie;
1474 int ret;
Ming Leid964f042017-06-06 23:22:00 +08001475 bool run_queue = true;
1476
1477 if (blk_mq_hctx_stopped(hctx)) {
1478 run_queue = false;
1479 goto insert;
1480 }
Shaohua Lif984df12015-05-08 10:51:32 -07001481
Jens Axboebd166ef2017-01-17 06:03:22 -07001482 if (q->elevator)
Bart Van Assche2253efc2016-10-28 17:20:02 -07001483 goto insert;
1484
Ming Leid964f042017-06-06 23:22:00 +08001485 if (!blk_mq_get_driver_tag(rq, NULL, false))
Jens Axboebd166ef2017-01-17 06:03:22 -07001486 goto insert;
1487
1488 new_cookie = request_to_qc_t(hctx, rq);
1489
Shaohua Lif984df12015-05-08 10:51:32 -07001490 /*
1491 * For OK queue, we are done. For error, kill it. Any other
1492 * error (busy), just add it to our list as we previously
1493 * would have done
1494 */
1495 ret = q->mq_ops->queue_rq(hctx, &bd);
Jens Axboe7b371632015-11-05 10:41:40 -07001496 if (ret == BLK_MQ_RQ_QUEUE_OK) {
1497 *cookie = new_cookie;
Bart Van Assche2253efc2016-10-28 17:20:02 -07001498 return;
Shaohua Lif984df12015-05-08 10:51:32 -07001499 }
Jens Axboe7b371632015-11-05 10:41:40 -07001500
Jens Axboe7b371632015-11-05 10:41:40 -07001501 if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1502 *cookie = BLK_QC_T_NONE;
Christoph Hellwigcaf7df12017-04-20 16:03:16 +02001503 blk_mq_end_request(rq, -EIO);
Bart Van Assche2253efc2016-10-28 17:20:02 -07001504 return;
Jens Axboe7b371632015-11-05 10:41:40 -07001505 }
1506
Josef Bacikb58e1762017-03-28 16:37:52 -04001507 __blk_mq_requeue_request(rq);
Bart Van Assche2253efc2016-10-28 17:20:02 -07001508insert:
Ming Leid964f042017-06-06 23:22:00 +08001509 blk_mq_sched_insert_request(rq, false, run_queue, false, may_sleep);
Shaohua Lif984df12015-05-08 10:51:32 -07001510}
1511
Christoph Hellwig5eb61262017-03-22 15:01:51 -04001512static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
1513 struct request *rq, blk_qc_t *cookie)
1514{
1515 if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
1516 rcu_read_lock();
Ming Leid964f042017-06-06 23:22:00 +08001517 __blk_mq_try_issue_directly(hctx, rq, cookie, false);
Christoph Hellwig5eb61262017-03-22 15:01:51 -04001518 rcu_read_unlock();
1519 } else {
Jens Axboebf4907c2017-03-30 12:30:39 -06001520 unsigned int srcu_idx;
1521
1522 might_sleep();
1523
1524 srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
Ming Leid964f042017-06-06 23:22:00 +08001525 __blk_mq_try_issue_directly(hctx, rq, cookie, true);
Christoph Hellwig5eb61262017-03-22 15:01:51 -04001526 srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
1527 }
1528}
1529
Jens Axboedece1632015-11-05 10:41:16 -07001530static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
Jens Axboe07068d52014-05-22 10:40:51 -06001531{
Christoph Hellwigef295ec2016-10-28 08:48:16 -06001532 const int is_sync = op_is_sync(bio->bi_opf);
Christoph Hellwigf73f44e2017-01-27 08:30:47 -07001533 const int is_flush_fua = op_is_flush(bio->bi_opf);
Jens Axboe5a797e02017-01-26 12:22:11 -07001534 struct blk_mq_alloc_data data = { .flags = 0 };
Jens Axboe07068d52014-05-22 10:40:51 -06001535 struct request *rq;
Christoph Hellwig5eb61262017-03-22 15:01:51 -04001536 unsigned int request_count = 0;
Shaohua Lif984df12015-05-08 10:51:32 -07001537 struct blk_plug *plug;
Shaohua Li5b3f3412015-05-08 10:51:33 -07001538 struct request *same_queue_rq = NULL;
Jens Axboe7b371632015-11-05 10:41:40 -07001539 blk_qc_t cookie;
Jens Axboe87760e52016-11-09 12:38:14 -07001540 unsigned int wb_acct;
Jens Axboe07068d52014-05-22 10:40:51 -06001541
1542 blk_queue_bounce(q, &bio);
1543
Wen Xiongf36ea502017-05-10 08:54:11 -05001544 blk_queue_split(q, &bio, q->bio_split);
1545
Jens Axboe07068d52014-05-22 10:40:51 -06001546 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001547 bio_io_error(bio);
Jens Axboedece1632015-11-05 10:41:16 -07001548 return BLK_QC_T_NONE;
Jens Axboe07068d52014-05-22 10:40:51 -06001549 }
1550
Omar Sandoval87c279e2016-06-01 22:18:48 -07001551 if (!is_flush_fua && !blk_queue_nomerges(q) &&
1552 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
1553 return BLK_QC_T_NONE;
Shaohua Lif984df12015-05-08 10:51:32 -07001554
Jens Axboebd166ef2017-01-17 06:03:22 -07001555 if (blk_mq_sched_bio_merge(q, bio))
1556 return BLK_QC_T_NONE;
1557
Jens Axboe87760e52016-11-09 12:38:14 -07001558 wb_acct = wbt_wait(q->rq_wb, bio, NULL);
1559
Jens Axboebd166ef2017-01-17 06:03:22 -07001560 trace_block_getrq(q, bio, bio->bi_opf);
1561
1562 rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
Jens Axboe87760e52016-11-09 12:38:14 -07001563 if (unlikely(!rq)) {
1564 __wbt_done(q->rq_wb, wb_acct);
Jens Axboedece1632015-11-05 10:41:16 -07001565 return BLK_QC_T_NONE;
Jens Axboe87760e52016-11-09 12:38:14 -07001566 }
1567
1568 wbt_track(&rq->issue_stat, wb_acct);
Jens Axboe07068d52014-05-22 10:40:51 -06001569
Jens Axboefd2d3322017-01-12 10:04:45 -07001570 cookie = request_to_qc_t(data.hctx, rq);
Jens Axboe07068d52014-05-22 10:40:51 -06001571
Shaohua Lif984df12015-05-08 10:51:32 -07001572 plug = current->plug;
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001573 if (unlikely(is_flush_fua)) {
Shaohua Lif984df12015-05-08 10:51:32 -07001574 blk_mq_put_ctx(data.ctx);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001575 blk_mq_bio_to_request(rq, bio);
1576 if (q->elevator) {
1577 blk_mq_sched_insert_request(rq, false, true, true,
1578 true);
Bart Van Assche6a83e742016-11-02 10:09:51 -06001579 } else {
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001580 blk_insert_flush(rq);
1581 blk_mq_run_hw_queue(data.hctx, true);
Bart Van Assche6a83e742016-11-02 10:09:51 -06001582 }
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001583 } else if (plug && q->nr_hw_queues == 1) {
Shaohua Li600271d2016-11-03 17:03:54 -07001584 struct request *last = NULL;
1585
Jens Axboeb00c53e2017-04-20 16:40:36 -06001586 blk_mq_put_ctx(data.ctx);
Jeff Moyere6c44382015-05-08 10:51:30 -07001587 blk_mq_bio_to_request(rq, bio);
Ming Lei0a6219a2016-11-16 18:07:05 +08001588
1589 /*
1590 * @request_count may become stale because of schedule
1591 * out, so check the list again.
1592 */
1593 if (list_empty(&plug->mq_list))
1594 request_count = 0;
Christoph Hellwig254d2592017-03-22 15:01:50 -04001595 else if (blk_queue_nomerges(q))
1596 request_count = blk_plug_queued_count(q);
1597
Ming Lei676d0602015-10-20 23:13:56 +08001598 if (!request_count)
Jeff Moyere6c44382015-05-08 10:51:30 -07001599 trace_block_plug(q);
Shaohua Li600271d2016-11-03 17:03:54 -07001600 else
1601 last = list_entry_rq(plug->mq_list.prev);
Jens Axboeb094f892015-11-20 20:29:45 -07001602
Shaohua Li600271d2016-11-03 17:03:54 -07001603 if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
1604 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
Jeff Moyere6c44382015-05-08 10:51:30 -07001605 blk_flush_plug_list(plug, false);
1606 trace_block_plug(q);
Jens Axboe320ae512013-10-24 09:20:05 +01001607 }
Jens Axboeb094f892015-11-20 20:29:45 -07001608
Jeff Moyere6c44382015-05-08 10:51:30 -07001609 list_add_tail(&rq->queuelist, &plug->mq_list);
Christoph Hellwig22997222017-03-22 15:01:52 -04001610 } else if (plug && !blk_queue_nomerges(q)) {
Jens Axboe320ae512013-10-24 09:20:05 +01001611 blk_mq_bio_to_request(rq, bio);
Jens Axboe320ae512013-10-24 09:20:05 +01001612
Jens Axboe320ae512013-10-24 09:20:05 +01001613 /*
1614 * We do limited plugging. If the bio can be merged, do that.
1615 * Otherwise the existing request in the plug list will be
1616 * issued. So the plug list will have one request at most
Christoph Hellwig22997222017-03-22 15:01:52 -04001617 * The plug list might get flushed before this. If that happens,
1618 * the plug list is empty, and same_queue_rq is invalid.
Jens Axboe320ae512013-10-24 09:20:05 +01001619 */
Christoph Hellwig22997222017-03-22 15:01:52 -04001620 if (list_empty(&plug->mq_list))
1621 same_queue_rq = NULL;
1622 if (same_queue_rq)
1623 list_del_init(&same_queue_rq->queuelist);
1624 list_add_tail(&rq->queuelist, &plug->mq_list);
1625
Jens Axboebf4907c2017-03-30 12:30:39 -06001626 blk_mq_put_ctx(data.ctx);
1627
Ming Leidad7a3b2017-06-06 23:21:59 +08001628 if (same_queue_rq) {
1629 data.hctx = blk_mq_map_queue(q,
1630 same_queue_rq->mq_ctx->cpu);
Christoph Hellwig22997222017-03-22 15:01:52 -04001631 blk_mq_try_issue_directly(data.hctx, same_queue_rq,
1632 &cookie);
Ming Leidad7a3b2017-06-06 23:21:59 +08001633 }
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001634 } else if (q->nr_hw_queues > 1 && is_sync) {
Jens Axboebd166ef2017-01-17 06:03:22 -07001635 blk_mq_put_ctx(data.ctx);
1636 blk_mq_bio_to_request(rq, bio);
Christoph Hellwig22997222017-03-22 15:01:52 -04001637 blk_mq_try_issue_directly(data.hctx, rq, &cookie);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001638 } else if (q->elevator) {
Jens Axboeb00c53e2017-04-20 16:40:36 -06001639 blk_mq_put_ctx(data.ctx);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001640 blk_mq_bio_to_request(rq, bio);
1641 blk_mq_sched_insert_request(rq, false, true, true, true);
Jens Axboeb00c53e2017-04-20 16:40:36 -06001642 } else if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1643 blk_mq_put_ctx(data.ctx);
Christoph Hellwiga4d907b2017-03-22 15:01:53 -04001644 blk_mq_run_hw_queue(data.hctx, true);
Bart Van Asscheabc25a62017-04-21 12:00:40 -06001645 } else
1646 blk_mq_put_ctx(data.ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001647
Jens Axboe7b371632015-11-05 10:41:40 -07001648 return cookie;
Jens Axboe320ae512013-10-24 09:20:05 +01001649}
1650
Jens Axboecc71a6f2017-01-11 14:29:56 -07001651void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
1652 unsigned int hctx_idx)
Jens Axboe320ae512013-10-24 09:20:05 +01001653{
1654 struct page *page;
1655
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001656 if (tags->rqs && set->ops->exit_request) {
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001657 int i;
1658
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001659 for (i = 0; i < tags->nr_tags; i++) {
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001660 struct request *rq = tags->static_rqs[i];
1661
1662 if (!rq)
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001663 continue;
Christoph Hellwigd6296d32017-05-01 10:19:08 -06001664 set->ops->exit_request(set, rq, hctx_idx);
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001665 tags->static_rqs[i] = NULL;
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001666 }
1667 }
1668
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001669 while (!list_empty(&tags->page_list)) {
1670 page = list_first_entry(&tags->page_list, struct page, lru);
Dave Hansen67534712014-01-08 20:17:46 -07001671 list_del_init(&page->lru);
Catalin Marinasf75782e2015-09-14 18:16:02 +01001672 /*
1673 * Remove kmemleak object previously allocated in
1674 * blk_mq_init_rq_map().
1675 */
1676 kmemleak_free(page_address(page));
Jens Axboe320ae512013-10-24 09:20:05 +01001677 __free_pages(page, page->private);
1678 }
Jens Axboecc71a6f2017-01-11 14:29:56 -07001679}
Jens Axboe320ae512013-10-24 09:20:05 +01001680
Jens Axboecc71a6f2017-01-11 14:29:56 -07001681void blk_mq_free_rq_map(struct blk_mq_tags *tags)
1682{
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001683 kfree(tags->rqs);
Jens Axboecc71a6f2017-01-11 14:29:56 -07001684 tags->rqs = NULL;
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001685 kfree(tags->static_rqs);
1686 tags->static_rqs = NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01001687
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001688 blk_mq_free_tags(tags);
Jens Axboe320ae512013-10-24 09:20:05 +01001689}
1690
Jens Axboecc71a6f2017-01-11 14:29:56 -07001691struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
1692 unsigned int hctx_idx,
1693 unsigned int nr_tags,
1694 unsigned int reserved_tags)
Jens Axboe320ae512013-10-24 09:20:05 +01001695{
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001696 struct blk_mq_tags *tags;
Shaohua Li59f082e2017-02-01 09:53:14 -08001697 int node;
Jens Axboe320ae512013-10-24 09:20:05 +01001698
Shaohua Li59f082e2017-02-01 09:53:14 -08001699 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
1700 if (node == NUMA_NO_NODE)
1701 node = set->numa_node;
1702
1703 tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
Shaohua Li24391c02015-01-23 14:18:00 -07001704 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001705 if (!tags)
1706 return NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01001707
Jens Axboecc71a6f2017-01-11 14:29:56 -07001708 tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
Gabriel Krisman Bertazi36e1f3d12016-12-06 13:31:44 -02001709 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
Shaohua Li59f082e2017-02-01 09:53:14 -08001710 node);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001711 if (!tags->rqs) {
1712 blk_mq_free_tags(tags);
1713 return NULL;
1714 }
Jens Axboe320ae512013-10-24 09:20:05 +01001715
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001716 tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1717 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
Shaohua Li59f082e2017-02-01 09:53:14 -08001718 node);
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001719 if (!tags->static_rqs) {
1720 kfree(tags->rqs);
1721 blk_mq_free_tags(tags);
1722 return NULL;
1723 }
1724
Jens Axboecc71a6f2017-01-11 14:29:56 -07001725 return tags;
1726}
1727
1728static size_t order_to_size(unsigned int order)
1729{
1730 return (size_t)PAGE_SIZE << order;
1731}
1732
1733int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
1734 unsigned int hctx_idx, unsigned int depth)
1735{
1736 unsigned int i, j, entries_per_page, max_order = 4;
1737 size_t rq_size, left;
Shaohua Li59f082e2017-02-01 09:53:14 -08001738 int node;
1739
1740 node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
1741 if (node == NUMA_NO_NODE)
1742 node = set->numa_node;
Jens Axboecc71a6f2017-01-11 14:29:56 -07001743
1744 INIT_LIST_HEAD(&tags->page_list);
1745
Jens Axboe320ae512013-10-24 09:20:05 +01001746 /*
1747 * rq_size is the size of the request plus driver payload, rounded
1748 * to the cacheline size
1749 */
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001750 rq_size = round_up(sizeof(struct request) + set->cmd_size,
Jens Axboe320ae512013-10-24 09:20:05 +01001751 cache_line_size());
Jens Axboecc71a6f2017-01-11 14:29:56 -07001752 left = rq_size * depth;
Jens Axboe320ae512013-10-24 09:20:05 +01001753
Jens Axboecc71a6f2017-01-11 14:29:56 -07001754 for (i = 0; i < depth; ) {
Jens Axboe320ae512013-10-24 09:20:05 +01001755 int this_order = max_order;
1756 struct page *page;
1757 int to_do;
1758 void *p;
1759
Bartlomiej Zolnierkiewiczb3a834b2016-05-16 09:54:47 -06001760 while (this_order && left < order_to_size(this_order - 1))
Jens Axboe320ae512013-10-24 09:20:05 +01001761 this_order--;
1762
1763 do {
Shaohua Li59f082e2017-02-01 09:53:14 -08001764 page = alloc_pages_node(node,
Gabriel Krisman Bertazi36e1f3d12016-12-06 13:31:44 -02001765 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
Jens Axboea5164402014-09-10 09:02:03 -06001766 this_order);
Jens Axboe320ae512013-10-24 09:20:05 +01001767 if (page)
1768 break;
1769 if (!this_order--)
1770 break;
1771 if (order_to_size(this_order) < rq_size)
1772 break;
1773 } while (1);
1774
1775 if (!page)
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001776 goto fail;
Jens Axboe320ae512013-10-24 09:20:05 +01001777
1778 page->private = this_order;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001779 list_add_tail(&page->lru, &tags->page_list);
Jens Axboe320ae512013-10-24 09:20:05 +01001780
1781 p = page_address(page);
Catalin Marinasf75782e2015-09-14 18:16:02 +01001782 /*
1783 * Allow kmemleak to scan these pages as they contain pointers
1784 * to additional allocations like via ops->init_request().
1785 */
Gabriel Krisman Bertazi36e1f3d12016-12-06 13:31:44 -02001786 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
Jens Axboe320ae512013-10-24 09:20:05 +01001787 entries_per_page = order_to_size(this_order) / rq_size;
Jens Axboecc71a6f2017-01-11 14:29:56 -07001788 to_do = min(entries_per_page, depth - i);
Jens Axboe320ae512013-10-24 09:20:05 +01001789 left -= to_do * rq_size;
1790 for (j = 0; j < to_do; j++) {
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001791 struct request *rq = p;
1792
1793 tags->static_rqs[i] = rq;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001794 if (set->ops->init_request) {
Christoph Hellwigd6296d32017-05-01 10:19:08 -06001795 if (set->ops->init_request(set, rq, hctx_idx,
Shaohua Li59f082e2017-02-01 09:53:14 -08001796 node)) {
Jens Axboe2af8cbe2017-01-13 14:39:30 -07001797 tags->static_rqs[i] = NULL;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001798 goto fail;
Jens Axboea5164402014-09-10 09:02:03 -06001799 }
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001800 }
1801
Jens Axboe320ae512013-10-24 09:20:05 +01001802 p += rq_size;
1803 i++;
1804 }
1805 }
Jens Axboecc71a6f2017-01-11 14:29:56 -07001806 return 0;
Jens Axboe320ae512013-10-24 09:20:05 +01001807
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001808fail:
Jens Axboecc71a6f2017-01-11 14:29:56 -07001809 blk_mq_free_rqs(set, tags, hctx_idx);
1810 return -ENOMEM;
Jens Axboe320ae512013-10-24 09:20:05 +01001811}
1812
Jens Axboee57690f2016-08-24 15:34:35 -06001813/*
1814 * 'cpu' is going away. splice any existing rq_list entries from this
1815 * software queue to the hw queue dispatch list, and ensure that it
1816 * gets run.
1817 */
Thomas Gleixner9467f852016-09-22 08:05:17 -06001818static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
Jens Axboe484b4062014-05-21 14:01:15 -06001819{
Thomas Gleixner9467f852016-09-22 08:05:17 -06001820 struct blk_mq_hw_ctx *hctx;
Jens Axboe484b4062014-05-21 14:01:15 -06001821 struct blk_mq_ctx *ctx;
1822 LIST_HEAD(tmp);
1823
Thomas Gleixner9467f852016-09-22 08:05:17 -06001824 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
Jens Axboee57690f2016-08-24 15:34:35 -06001825 ctx = __blk_mq_get_ctx(hctx->queue, cpu);
Jens Axboe484b4062014-05-21 14:01:15 -06001826
1827 spin_lock(&ctx->lock);
1828 if (!list_empty(&ctx->rq_list)) {
1829 list_splice_init(&ctx->rq_list, &tmp);
1830 blk_mq_hctx_clear_pending(hctx, ctx);
1831 }
1832 spin_unlock(&ctx->lock);
1833
1834 if (list_empty(&tmp))
Thomas Gleixner9467f852016-09-22 08:05:17 -06001835 return 0;
Jens Axboe484b4062014-05-21 14:01:15 -06001836
Jens Axboee57690f2016-08-24 15:34:35 -06001837 spin_lock(&hctx->lock);
1838 list_splice_tail_init(&tmp, &hctx->dispatch);
1839 spin_unlock(&hctx->lock);
Jens Axboe484b4062014-05-21 14:01:15 -06001840
1841 blk_mq_run_hw_queue(hctx, true);
Thomas Gleixner9467f852016-09-22 08:05:17 -06001842 return 0;
Jens Axboe484b4062014-05-21 14:01:15 -06001843}
1844
Thomas Gleixner9467f852016-09-22 08:05:17 -06001845static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
Jens Axboe484b4062014-05-21 14:01:15 -06001846{
Thomas Gleixner9467f852016-09-22 08:05:17 -06001847 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
1848 &hctx->cpuhp_dead);
Jens Axboe484b4062014-05-21 14:01:15 -06001849}
1850
Ming Leic3b4afc2015-06-04 22:25:04 +08001851/* hctx->ctxs will be freed in queue's release handler */
Ming Lei08e98fc2014-09-25 23:23:38 +08001852static void blk_mq_exit_hctx(struct request_queue *q,
1853 struct blk_mq_tag_set *set,
1854 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1855{
Omar Sandoval9c1051a2017-05-04 08:17:21 -06001856 blk_mq_debugfs_unregister_hctx(hctx);
1857
Ming Lei08e98fc2014-09-25 23:23:38 +08001858 blk_mq_tag_idle(hctx);
1859
Ming Leif70ced02014-09-25 23:23:47 +08001860 if (set->ops->exit_request)
Christoph Hellwigd6296d32017-05-01 10:19:08 -06001861 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
Ming Leif70ced02014-09-25 23:23:47 +08001862
Omar Sandoval93252632017-04-05 12:01:31 -07001863 blk_mq_sched_exit_hctx(q, hctx, hctx_idx);
1864
Ming Lei08e98fc2014-09-25 23:23:38 +08001865 if (set->ops->exit_hctx)
1866 set->ops->exit_hctx(hctx, hctx_idx);
1867
Bart Van Assche6a83e742016-11-02 10:09:51 -06001868 if (hctx->flags & BLK_MQ_F_BLOCKING)
1869 cleanup_srcu_struct(&hctx->queue_rq_srcu);
1870
Thomas Gleixner9467f852016-09-22 08:05:17 -06001871 blk_mq_remove_cpuhp(hctx);
Ming Leif70ced02014-09-25 23:23:47 +08001872 blk_free_flush_queue(hctx->fq);
Omar Sandoval88459642016-09-17 08:38:44 -06001873 sbitmap_free(&hctx->ctx_map);
Ming Lei08e98fc2014-09-25 23:23:38 +08001874}
1875
Ming Lei624dbe42014-05-27 23:35:13 +08001876static void blk_mq_exit_hw_queues(struct request_queue *q,
1877 struct blk_mq_tag_set *set, int nr_queue)
1878{
1879 struct blk_mq_hw_ctx *hctx;
1880 unsigned int i;
1881
1882 queue_for_each_hw_ctx(q, hctx, i) {
1883 if (i == nr_queue)
1884 break;
Ming Lei08e98fc2014-09-25 23:23:38 +08001885 blk_mq_exit_hctx(q, set, hctx, i);
Ming Lei624dbe42014-05-27 23:35:13 +08001886 }
Ming Lei624dbe42014-05-27 23:35:13 +08001887}
1888
Ming Lei08e98fc2014-09-25 23:23:38 +08001889static int blk_mq_init_hctx(struct request_queue *q,
1890 struct blk_mq_tag_set *set,
1891 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1892{
1893 int node;
1894
1895 node = hctx->numa_node;
1896 if (node == NUMA_NO_NODE)
1897 node = hctx->numa_node = set->numa_node;
1898
Jens Axboe9f993732017-04-10 09:54:54 -06001899 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
Ming Lei08e98fc2014-09-25 23:23:38 +08001900 spin_lock_init(&hctx->lock);
1901 INIT_LIST_HEAD(&hctx->dispatch);
1902 hctx->queue = q;
1903 hctx->queue_num = hctx_idx;
Jeff Moyer2404e602015-11-03 10:40:06 -05001904 hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
Ming Lei08e98fc2014-09-25 23:23:38 +08001905
Thomas Gleixner9467f852016-09-22 08:05:17 -06001906 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
Ming Lei08e98fc2014-09-25 23:23:38 +08001907
1908 hctx->tags = set->tags[hctx_idx];
1909
1910 /*
1911 * Allocate space for all possible cpus to avoid allocation at
1912 * runtime
1913 */
1914 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1915 GFP_KERNEL, node);
1916 if (!hctx->ctxs)
1917 goto unregister_cpu_notifier;
1918
Omar Sandoval88459642016-09-17 08:38:44 -06001919 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
1920 node))
Ming Lei08e98fc2014-09-25 23:23:38 +08001921 goto free_ctxs;
1922
1923 hctx->nr_ctx = 0;
1924
1925 if (set->ops->init_hctx &&
1926 set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
1927 goto free_bitmap;
1928
Omar Sandoval93252632017-04-05 12:01:31 -07001929 if (blk_mq_sched_init_hctx(q, hctx, hctx_idx))
1930 goto exit_hctx;
1931
Ming Leif70ced02014-09-25 23:23:47 +08001932 hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
1933 if (!hctx->fq)
Omar Sandoval93252632017-04-05 12:01:31 -07001934 goto sched_exit_hctx;
Ming Leif70ced02014-09-25 23:23:47 +08001935
1936 if (set->ops->init_request &&
Christoph Hellwigd6296d32017-05-01 10:19:08 -06001937 set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx,
1938 node))
Ming Leif70ced02014-09-25 23:23:47 +08001939 goto free_fq;
1940
Bart Van Assche6a83e742016-11-02 10:09:51 -06001941 if (hctx->flags & BLK_MQ_F_BLOCKING)
1942 init_srcu_struct(&hctx->queue_rq_srcu);
1943
Omar Sandoval9c1051a2017-05-04 08:17:21 -06001944 blk_mq_debugfs_register_hctx(q, hctx);
1945
Ming Lei08e98fc2014-09-25 23:23:38 +08001946 return 0;
1947
Ming Leif70ced02014-09-25 23:23:47 +08001948 free_fq:
1949 kfree(hctx->fq);
Omar Sandoval93252632017-04-05 12:01:31 -07001950 sched_exit_hctx:
1951 blk_mq_sched_exit_hctx(q, hctx, hctx_idx);
Ming Leif70ced02014-09-25 23:23:47 +08001952 exit_hctx:
1953 if (set->ops->exit_hctx)
1954 set->ops->exit_hctx(hctx, hctx_idx);
Ming Lei08e98fc2014-09-25 23:23:38 +08001955 free_bitmap:
Omar Sandoval88459642016-09-17 08:38:44 -06001956 sbitmap_free(&hctx->ctx_map);
Ming Lei08e98fc2014-09-25 23:23:38 +08001957 free_ctxs:
1958 kfree(hctx->ctxs);
1959 unregister_cpu_notifier:
Thomas Gleixner9467f852016-09-22 08:05:17 -06001960 blk_mq_remove_cpuhp(hctx);
Ming Lei08e98fc2014-09-25 23:23:38 +08001961 return -1;
1962}
1963
Jens Axboe320ae512013-10-24 09:20:05 +01001964static void blk_mq_init_cpu_queues(struct request_queue *q,
1965 unsigned int nr_hw_queues)
1966{
1967 unsigned int i;
1968
1969 for_each_possible_cpu(i) {
1970 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1971 struct blk_mq_hw_ctx *hctx;
1972
Jens Axboe320ae512013-10-24 09:20:05 +01001973 __ctx->cpu = i;
1974 spin_lock_init(&__ctx->lock);
1975 INIT_LIST_HEAD(&__ctx->rq_list);
1976 __ctx->queue = q;
1977
1978 /* If the cpu isn't online, the cpu is mapped to first hctx */
Jens Axboe320ae512013-10-24 09:20:05 +01001979 if (!cpu_online(i))
1980 continue;
1981
Christoph Hellwig7d7e0f92016-09-14 16:18:54 +02001982 hctx = blk_mq_map_queue(q, i);
Jens Axboee4043dc2014-04-09 10:18:23 -06001983
Jens Axboe320ae512013-10-24 09:20:05 +01001984 /*
1985 * Set local node, IFF we have more than one hw queue. If
1986 * not, we remain on the home node of the device
1987 */
1988 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
Raghavendra K Tbffed452015-12-02 16:59:05 +05301989 hctx->numa_node = local_memory_node(cpu_to_node(i));
Jens Axboe320ae512013-10-24 09:20:05 +01001990 }
1991}
1992
Jens Axboecc71a6f2017-01-11 14:29:56 -07001993static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
1994{
1995 int ret = 0;
1996
1997 set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
1998 set->queue_depth, set->reserved_tags);
1999 if (!set->tags[hctx_idx])
2000 return false;
2001
2002 ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
2003 set->queue_depth);
2004 if (!ret)
2005 return true;
2006
2007 blk_mq_free_rq_map(set->tags[hctx_idx]);
2008 set->tags[hctx_idx] = NULL;
2009 return false;
2010}
2011
2012static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
2013 unsigned int hctx_idx)
2014{
Jens Axboebd166ef2017-01-17 06:03:22 -07002015 if (set->tags[hctx_idx]) {
2016 blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2017 blk_mq_free_rq_map(set->tags[hctx_idx]);
2018 set->tags[hctx_idx] = NULL;
2019 }
Jens Axboecc71a6f2017-01-11 14:29:56 -07002020}
2021
Akinobu Mita57783222015-09-27 02:09:23 +09002022static void blk_mq_map_swqueue(struct request_queue *q,
2023 const struct cpumask *online_mask)
Jens Axboe320ae512013-10-24 09:20:05 +01002024{
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002025 unsigned int i, hctx_idx;
Jens Axboe320ae512013-10-24 09:20:05 +01002026 struct blk_mq_hw_ctx *hctx;
2027 struct blk_mq_ctx *ctx;
Ming Lei2a34c082015-04-21 10:00:20 +08002028 struct blk_mq_tag_set *set = q->tag_set;
Jens Axboe320ae512013-10-24 09:20:05 +01002029
Akinobu Mita60de0742015-09-27 02:09:25 +09002030 /*
2031 * Avoid others reading imcomplete hctx->cpumask through sysfs
2032 */
2033 mutex_lock(&q->sysfs_lock);
2034
Jens Axboe320ae512013-10-24 09:20:05 +01002035 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboee4043dc2014-04-09 10:18:23 -06002036 cpumask_clear(hctx->cpumask);
Jens Axboe320ae512013-10-24 09:20:05 +01002037 hctx->nr_ctx = 0;
2038 }
2039
2040 /*
2041 * Map software to hardware queues
2042 */
Thomas Gleixner897bb0c2016-03-19 11:30:33 +01002043 for_each_possible_cpu(i) {
Jens Axboe320ae512013-10-24 09:20:05 +01002044 /* If the cpu isn't online, the cpu is mapped to first hctx */
Akinobu Mita57783222015-09-27 02:09:23 +09002045 if (!cpumask_test_cpu(i, online_mask))
Jens Axboee4043dc2014-04-09 10:18:23 -06002046 continue;
2047
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002048 hctx_idx = q->mq_map[i];
2049 /* unmapped hw queue can be remapped after CPU topo changed */
Jens Axboecc71a6f2017-01-11 14:29:56 -07002050 if (!set->tags[hctx_idx] &&
2051 !__blk_mq_alloc_rq_map(set, hctx_idx)) {
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002052 /*
2053 * If tags initialization fail for some hctx,
2054 * that hctx won't be brought online. In this
2055 * case, remap the current ctx to hctx[0] which
2056 * is guaranteed to always have tags allocated
2057 */
Jens Axboecc71a6f2017-01-11 14:29:56 -07002058 q->mq_map[i] = 0;
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002059 }
2060
Thomas Gleixner897bb0c2016-03-19 11:30:33 +01002061 ctx = per_cpu_ptr(q->queue_ctx, i);
Christoph Hellwig7d7e0f92016-09-14 16:18:54 +02002062 hctx = blk_mq_map_queue(q, i);
Keith Busch868f2f02015-12-17 17:08:14 -07002063
Jens Axboee4043dc2014-04-09 10:18:23 -06002064 cpumask_set_cpu(i, hctx->cpumask);
Jens Axboe320ae512013-10-24 09:20:05 +01002065 ctx->index_hw = hctx->nr_ctx;
2066 hctx->ctxs[hctx->nr_ctx++] = ctx;
2067 }
Jens Axboe506e9312014-05-07 10:26:44 -06002068
Akinobu Mita60de0742015-09-27 02:09:25 +09002069 mutex_unlock(&q->sysfs_lock);
2070
Jens Axboe506e9312014-05-07 10:26:44 -06002071 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboe484b4062014-05-21 14:01:15 -06002072 /*
Jens Axboea68aafa2014-08-15 13:19:15 -06002073 * If no software queues are mapped to this hardware queue,
2074 * disable it and free the request entries.
Jens Axboe484b4062014-05-21 14:01:15 -06002075 */
2076 if (!hctx->nr_ctx) {
Gabriel Krisman Bertazid1b1cea2016-12-14 18:48:36 -02002077 /* Never unmap queue 0. We need it as a
2078 * fallback in case of a new remap fails
2079 * allocation
2080 */
Jens Axboecc71a6f2017-01-11 14:29:56 -07002081 if (i && set->tags[i])
2082 blk_mq_free_map_and_requests(set, i);
2083
Ming Lei2a34c082015-04-21 10:00:20 +08002084 hctx->tags = NULL;
Jens Axboe484b4062014-05-21 14:01:15 -06002085 continue;
2086 }
2087
Ming Lei2a34c082015-04-21 10:00:20 +08002088 hctx->tags = set->tags[i];
2089 WARN_ON(!hctx->tags);
2090
Jens Axboe484b4062014-05-21 14:01:15 -06002091 /*
Chong Yuan889fa312015-04-15 11:39:29 -06002092 * Set the map size to the number of mapped software queues.
2093 * This is more accurate and more efficient than looping
2094 * over all possibly mapped software queues.
2095 */
Omar Sandoval88459642016-09-17 08:38:44 -06002096 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
Chong Yuan889fa312015-04-15 11:39:29 -06002097
2098 /*
Jens Axboe484b4062014-05-21 14:01:15 -06002099 * Initialize batch roundrobin counts
2100 */
Jens Axboe506e9312014-05-07 10:26:44 -06002101 hctx->next_cpu = cpumask_first(hctx->cpumask);
2102 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2103 }
Jens Axboe320ae512013-10-24 09:20:05 +01002104}
2105
Jeff Moyer2404e602015-11-03 10:40:06 -05002106static void queue_set_hctx_shared(struct request_queue *q, bool shared)
Jens Axboe0d2602c2014-05-13 15:10:52 -06002107{
2108 struct blk_mq_hw_ctx *hctx;
Jens Axboe0d2602c2014-05-13 15:10:52 -06002109 int i;
2110
Jeff Moyer2404e602015-11-03 10:40:06 -05002111 queue_for_each_hw_ctx(q, hctx, i) {
2112 if (shared)
2113 hctx->flags |= BLK_MQ_F_TAG_SHARED;
2114 else
2115 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
2116 }
2117}
2118
2119static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared)
2120{
2121 struct request_queue *q;
Jens Axboe0d2602c2014-05-13 15:10:52 -06002122
Bart Van Assche705cda92017-04-07 11:16:49 -07002123 lockdep_assert_held(&set->tag_list_lock);
2124
Jens Axboe0d2602c2014-05-13 15:10:52 -06002125 list_for_each_entry(q, &set->tag_list, tag_set_list) {
2126 blk_mq_freeze_queue(q);
Jeff Moyer2404e602015-11-03 10:40:06 -05002127 queue_set_hctx_shared(q, shared);
Jens Axboe0d2602c2014-05-13 15:10:52 -06002128 blk_mq_unfreeze_queue(q);
2129 }
2130}
2131
2132static void blk_mq_del_queue_tag_set(struct request_queue *q)
2133{
2134 struct blk_mq_tag_set *set = q->tag_set;
2135
Jens Axboe0d2602c2014-05-13 15:10:52 -06002136 mutex_lock(&set->tag_list_lock);
Bart Van Assche705cda92017-04-07 11:16:49 -07002137 list_del_rcu(&q->tag_set_list);
2138 INIT_LIST_HEAD(&q->tag_set_list);
Jeff Moyer2404e602015-11-03 10:40:06 -05002139 if (list_is_singular(&set->tag_list)) {
2140 /* just transitioned to unshared */
2141 set->flags &= ~BLK_MQ_F_TAG_SHARED;
2142 /* update existing queue */
2143 blk_mq_update_tag_set_depth(set, false);
2144 }
Jens Axboe0d2602c2014-05-13 15:10:52 -06002145 mutex_unlock(&set->tag_list_lock);
Bart Van Assche705cda92017-04-07 11:16:49 -07002146
2147 synchronize_rcu();
Jens Axboe0d2602c2014-05-13 15:10:52 -06002148}
2149
2150static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
2151 struct request_queue *q)
2152{
2153 q->tag_set = set;
2154
2155 mutex_lock(&set->tag_list_lock);
Jeff Moyer2404e602015-11-03 10:40:06 -05002156
2157 /* Check to see if we're transitioning to shared (from 1 to 2 queues). */
2158 if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
2159 set->flags |= BLK_MQ_F_TAG_SHARED;
2160 /* update existing queue */
2161 blk_mq_update_tag_set_depth(set, true);
2162 }
2163 if (set->flags & BLK_MQ_F_TAG_SHARED)
2164 queue_set_hctx_shared(q, true);
Bart Van Assche705cda92017-04-07 11:16:49 -07002165 list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
Jeff Moyer2404e602015-11-03 10:40:06 -05002166
Jens Axboe0d2602c2014-05-13 15:10:52 -06002167 mutex_unlock(&set->tag_list_lock);
2168}
2169
Ming Leie09aae72015-01-29 20:17:27 +08002170/*
2171 * It is the actual release handler for mq, but we do it from
2172 * request queue's release handler for avoiding use-after-free
2173 * and headache because q->mq_kobj shouldn't have been introduced,
2174 * but we can't group ctx/kctx kobj without it.
2175 */
2176void blk_mq_release(struct request_queue *q)
2177{
2178 struct blk_mq_hw_ctx *hctx;
2179 unsigned int i;
2180
2181 /* hctx kobj stays in hctx */
Ming Leic3b4afc2015-06-04 22:25:04 +08002182 queue_for_each_hw_ctx(q, hctx, i) {
2183 if (!hctx)
2184 continue;
Ming Lei6c8b2322017-02-22 18:14:01 +08002185 kobject_put(&hctx->kobj);
Ming Leic3b4afc2015-06-04 22:25:04 +08002186 }
Ming Leie09aae72015-01-29 20:17:27 +08002187
Akinobu Mitaa723bab2015-09-27 02:09:21 +09002188 q->mq_map = NULL;
2189
Ming Leie09aae72015-01-29 20:17:27 +08002190 kfree(q->queue_hw_ctx);
2191
Ming Lei7ea5fe32017-02-22 18:14:00 +08002192 /*
2193 * release .mq_kobj and sw queue's kobject now because
2194 * both share lifetime with request queue.
2195 */
2196 blk_mq_sysfs_deinit(q);
2197
Ming Leie09aae72015-01-29 20:17:27 +08002198 free_percpu(q->queue_ctx);
2199}
2200
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002201struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
Jens Axboe320ae512013-10-24 09:20:05 +01002202{
Mike Snitzerb62c21b2015-03-12 23:56:02 -04002203 struct request_queue *uninit_q, *q;
2204
2205 uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
2206 if (!uninit_q)
2207 return ERR_PTR(-ENOMEM);
2208
2209 q = blk_mq_init_allocated_queue(set, uninit_q);
2210 if (IS_ERR(q))
2211 blk_cleanup_queue(uninit_q);
2212
2213 return q;
2214}
2215EXPORT_SYMBOL(blk_mq_init_queue);
2216
Keith Busch868f2f02015-12-17 17:08:14 -07002217static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
2218 struct request_queue *q)
Mike Snitzerb62c21b2015-03-12 23:56:02 -04002219{
Keith Busch868f2f02015-12-17 17:08:14 -07002220 int i, j;
2221 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
Jens Axboe320ae512013-10-24 09:20:05 +01002222
Keith Busch868f2f02015-12-17 17:08:14 -07002223 blk_mq_sysfs_unregister(q);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002224 for (i = 0; i < set->nr_hw_queues; i++) {
Keith Busch868f2f02015-12-17 17:08:14 -07002225 int node;
Jens Axboef14bbe72014-05-27 12:06:53 -06002226
Keith Busch868f2f02015-12-17 17:08:14 -07002227 if (hctxs[i])
2228 continue;
2229
2230 node = blk_mq_hw_queue_to_node(q->mq_map, i);
Christoph Hellwigcdef54d2014-05-28 18:11:06 +02002231 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
2232 GFP_KERNEL, node);
Jens Axboe320ae512013-10-24 09:20:05 +01002233 if (!hctxs[i])
Keith Busch868f2f02015-12-17 17:08:14 -07002234 break;
Jens Axboe320ae512013-10-24 09:20:05 +01002235
Jens Axboea86073e2014-10-13 15:41:54 -06002236 if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
Keith Busch868f2f02015-12-17 17:08:14 -07002237 node)) {
2238 kfree(hctxs[i]);
2239 hctxs[i] = NULL;
2240 break;
2241 }
Jens Axboee4043dc2014-04-09 10:18:23 -06002242
Jens Axboe0d2602c2014-05-13 15:10:52 -06002243 atomic_set(&hctxs[i]->nr_active, 0);
Jens Axboef14bbe72014-05-27 12:06:53 -06002244 hctxs[i]->numa_node = node;
Jens Axboe320ae512013-10-24 09:20:05 +01002245 hctxs[i]->queue_num = i;
Keith Busch868f2f02015-12-17 17:08:14 -07002246
2247 if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
2248 free_cpumask_var(hctxs[i]->cpumask);
2249 kfree(hctxs[i]);
2250 hctxs[i] = NULL;
2251 break;
2252 }
2253 blk_mq_hctx_kobj_init(hctxs[i]);
Jens Axboe320ae512013-10-24 09:20:05 +01002254 }
Keith Busch868f2f02015-12-17 17:08:14 -07002255 for (j = i; j < q->nr_hw_queues; j++) {
2256 struct blk_mq_hw_ctx *hctx = hctxs[j];
2257
2258 if (hctx) {
Jens Axboecc71a6f2017-01-11 14:29:56 -07002259 if (hctx->tags)
2260 blk_mq_free_map_and_requests(set, j);
Keith Busch868f2f02015-12-17 17:08:14 -07002261 blk_mq_exit_hctx(q, set, hctx, j);
Keith Busch868f2f02015-12-17 17:08:14 -07002262 kobject_put(&hctx->kobj);
Keith Busch868f2f02015-12-17 17:08:14 -07002263 hctxs[j] = NULL;
2264
2265 }
2266 }
2267 q->nr_hw_queues = i;
2268 blk_mq_sysfs_register(q);
2269}
2270
2271struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
2272 struct request_queue *q)
2273{
Ming Lei66841672016-02-12 15:27:00 +08002274 /* mark the queue as mq asap */
2275 q->mq_ops = set->ops;
2276
Omar Sandoval34dbad52017-03-21 08:56:08 -07002277 q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
Stephen Bates720b8cc2017-04-07 06:24:03 -06002278 blk_mq_poll_stats_bkt,
2279 BLK_MQ_POLL_STATS_BKTS, q);
Omar Sandoval34dbad52017-03-21 08:56:08 -07002280 if (!q->poll_cb)
2281 goto err_exit;
2282
Keith Busch868f2f02015-12-17 17:08:14 -07002283 q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
2284 if (!q->queue_ctx)
Ming Linc7de5722016-05-25 23:23:27 -07002285 goto err_exit;
Keith Busch868f2f02015-12-17 17:08:14 -07002286
Ming Lei737f98c2017-02-22 18:13:59 +08002287 /* init q->mq_kobj and sw queues' kobjects */
2288 blk_mq_sysfs_init(q);
2289
Keith Busch868f2f02015-12-17 17:08:14 -07002290 q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
2291 GFP_KERNEL, set->numa_node);
2292 if (!q->queue_hw_ctx)
2293 goto err_percpu;
2294
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002295 q->mq_map = set->mq_map;
Keith Busch868f2f02015-12-17 17:08:14 -07002296
2297 blk_mq_realloc_hw_ctxs(set, q);
2298 if (!q->nr_hw_queues)
2299 goto err_hctxs;
Jens Axboe320ae512013-10-24 09:20:05 +01002300
Christoph Hellwig287922eb2015-10-30 20:57:30 +08002301 INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
Ming Leie56f6982015-07-16 19:53:22 +08002302 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
Jens Axboe320ae512013-10-24 09:20:05 +01002303
2304 q->nr_queues = nr_cpu_ids;
Jens Axboe320ae512013-10-24 09:20:05 +01002305
Jens Axboe94eddfb2013-11-19 09:25:07 -07002306 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
Jens Axboe320ae512013-10-24 09:20:05 +01002307
Jens Axboe05f1dd52014-05-29 09:53:32 -06002308 if (!(set->flags & BLK_MQ_F_SG_MERGE))
2309 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
2310
Christoph Hellwig1be036e2014-02-07 10:22:39 -08002311 q->sg_reserved_size = INT_MAX;
2312
Mike Snitzer28494502016-09-14 13:28:30 -04002313 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -06002314 INIT_LIST_HEAD(&q->requeue_list);
2315 spin_lock_init(&q->requeue_lock);
2316
Christoph Hellwig254d2592017-03-22 15:01:50 -04002317 blk_queue_make_request(q, blk_mq_make_request);
Jens Axboe07068d52014-05-22 10:40:51 -06002318
Jens Axboeeba71762014-05-20 15:17:27 -06002319 /*
2320 * Do this after blk_queue_make_request() overrides it...
2321 */
2322 q->nr_requests = set->queue_depth;
2323
Jens Axboe64f1c212016-11-14 13:03:03 -07002324 /*
2325 * Default to classic polling
2326 */
2327 q->poll_nsec = -1;
2328
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002329 if (set->ops->complete)
2330 blk_queue_softirq_done(q, set->ops->complete);
Christoph Hellwig30a91cb2014-02-10 03:24:38 -08002331
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002332 blk_mq_init_cpu_queues(q, set->nr_hw_queues);
Jens Axboe320ae512013-10-24 09:20:05 +01002333
Peter Zijlstraeabe0652017-05-04 15:05:26 +02002334 get_online_cpus();
Wanpeng Li51d638b2017-05-07 00:14:22 -07002335 mutex_lock(&all_q_mutex);
Akinobu Mita4593fdb2015-09-27 02:09:20 +09002336
Jens Axboe320ae512013-10-24 09:20:05 +01002337 list_add_tail(&q->all_q_node, &all_q_list);
Jens Axboe0d2602c2014-05-13 15:10:52 -06002338 blk_mq_add_queue_tag_set(set, q);
Akinobu Mita57783222015-09-27 02:09:23 +09002339 blk_mq_map_swqueue(q, cpu_online_mask);
Jens Axboe484b4062014-05-21 14:01:15 -06002340
Peter Zijlstraeabe0652017-05-04 15:05:26 +02002341 mutex_unlock(&all_q_mutex);
Wanpeng Li51d638b2017-05-07 00:14:22 -07002342 put_online_cpus();
Akinobu Mita4593fdb2015-09-27 02:09:20 +09002343
Jens Axboed3484992017-01-13 14:43:58 -07002344 if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
2345 int ret;
2346
2347 ret = blk_mq_sched_init(q);
2348 if (ret)
2349 return ERR_PTR(ret);
2350 }
2351
Jens Axboe320ae512013-10-24 09:20:05 +01002352 return q;
Christoph Hellwig18741982014-02-10 09:29:00 -07002353
Jens Axboe320ae512013-10-24 09:20:05 +01002354err_hctxs:
Keith Busch868f2f02015-12-17 17:08:14 -07002355 kfree(q->queue_hw_ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01002356err_percpu:
Keith Busch868f2f02015-12-17 17:08:14 -07002357 free_percpu(q->queue_ctx);
Ming Linc7de5722016-05-25 23:23:27 -07002358err_exit:
2359 q->mq_ops = NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01002360 return ERR_PTR(-ENOMEM);
2361}
Mike Snitzerb62c21b2015-03-12 23:56:02 -04002362EXPORT_SYMBOL(blk_mq_init_allocated_queue);
Jens Axboe320ae512013-10-24 09:20:05 +01002363
2364void blk_mq_free_queue(struct request_queue *q)
2365{
Ming Lei624dbe42014-05-27 23:35:13 +08002366 struct blk_mq_tag_set *set = q->tag_set;
Jens Axboe320ae512013-10-24 09:20:05 +01002367
Akinobu Mita0e626362015-09-27 02:09:22 +09002368 mutex_lock(&all_q_mutex);
2369 list_del_init(&q->all_q_node);
2370 mutex_unlock(&all_q_mutex);
2371
Jens Axboe0d2602c2014-05-13 15:10:52 -06002372 blk_mq_del_queue_tag_set(q);
2373
Ming Lei624dbe42014-05-27 23:35:13 +08002374 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
Jens Axboe320ae512013-10-24 09:20:05 +01002375}
Jens Axboe320ae512013-10-24 09:20:05 +01002376
2377/* Basically redo blk_mq_init_queue with queue frozen */
Akinobu Mita57783222015-09-27 02:09:23 +09002378static void blk_mq_queue_reinit(struct request_queue *q,
2379 const struct cpumask *online_mask)
Jens Axboe320ae512013-10-24 09:20:05 +01002380{
Christoph Hellwig4ecd4fe2015-05-07 09:38:13 +02002381 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
Jens Axboe320ae512013-10-24 09:20:05 +01002382
Omar Sandoval9c1051a2017-05-04 08:17:21 -06002383 blk_mq_debugfs_unregister_hctxs(q);
Jens Axboe67aec142014-05-30 08:25:36 -06002384 blk_mq_sysfs_unregister(q);
2385
Jens Axboe320ae512013-10-24 09:20:05 +01002386 /*
2387 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
2388 * we should change hctx numa_node according to new topology (this
2389 * involves free and re-allocate memory, worthy doing?)
2390 */
2391
Akinobu Mita57783222015-09-27 02:09:23 +09002392 blk_mq_map_swqueue(q, online_mask);
Jens Axboe320ae512013-10-24 09:20:05 +01002393
Jens Axboe67aec142014-05-30 08:25:36 -06002394 blk_mq_sysfs_register(q);
Omar Sandoval9c1051a2017-05-04 08:17:21 -06002395 blk_mq_debugfs_register_hctxs(q);
Jens Axboe320ae512013-10-24 09:20:05 +01002396}
2397
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002398/*
2399 * New online cpumask which is going to be set in this hotplug event.
2400 * Declare this cpumasks as global as cpu-hotplug operation is invoked
2401 * one-by-one and dynamically allocating this could result in a failure.
2402 */
2403static struct cpumask cpuhp_online_new;
2404
2405static void blk_mq_queue_reinit_work(void)
Jens Axboe320ae512013-10-24 09:20:05 +01002406{
2407 struct request_queue *q;
Jens Axboe320ae512013-10-24 09:20:05 +01002408
2409 mutex_lock(&all_q_mutex);
Tejun Heof3af0202014-11-04 13:52:27 -05002410 /*
2411 * We need to freeze and reinit all existing queues. Freezing
2412 * involves synchronous wait for an RCU grace period and doing it
2413 * one by one may take a long time. Start freezing all queues in
2414 * one swoop and then wait for the completions so that freezing can
2415 * take place in parallel.
2416 */
2417 list_for_each_entry(q, &all_q_list, all_q_node)
Ming Lei1671d522017-03-27 20:06:57 +08002418 blk_freeze_queue_start(q);
Gabriel Krisman Bertazi415d3da2016-11-28 15:01:48 -02002419 list_for_each_entry(q, &all_q_list, all_q_node)
Tejun Heof3af0202014-11-04 13:52:27 -05002420 blk_mq_freeze_queue_wait(q);
2421
Jens Axboe320ae512013-10-24 09:20:05 +01002422 list_for_each_entry(q, &all_q_list, all_q_node)
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002423 blk_mq_queue_reinit(q, &cpuhp_online_new);
Tejun Heof3af0202014-11-04 13:52:27 -05002424
2425 list_for_each_entry(q, &all_q_list, all_q_node)
2426 blk_mq_unfreeze_queue(q);
2427
Jens Axboe320ae512013-10-24 09:20:05 +01002428 mutex_unlock(&all_q_mutex);
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002429}
2430
2431static int blk_mq_queue_reinit_dead(unsigned int cpu)
2432{
Sebastian Andrzej Siewior97a32862016-09-23 15:02:38 +02002433 cpumask_copy(&cpuhp_online_new, cpu_online_mask);
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002434 blk_mq_queue_reinit_work();
2435 return 0;
2436}
2437
2438/*
2439 * Before hotadded cpu starts handling requests, new mappings must be
2440 * established. Otherwise, these requests in hw queue might never be
2441 * dispatched.
2442 *
2443 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
2444 * for CPU0, and ctx1 for CPU1).
2445 *
2446 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
2447 * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
2448 *
Jens Axboe2c3ad662016-12-14 14:34:47 -07002449 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
2450 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
2451 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
2452 * ignored.
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002453 */
2454static int blk_mq_queue_reinit_prepare(unsigned int cpu)
2455{
2456 cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2457 cpumask_set_cpu(cpu, &cpuhp_online_new);
2458 blk_mq_queue_reinit_work();
2459 return 0;
Jens Axboe320ae512013-10-24 09:20:05 +01002460}
2461
Jens Axboea5164402014-09-10 09:02:03 -06002462static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2463{
2464 int i;
2465
Jens Axboecc71a6f2017-01-11 14:29:56 -07002466 for (i = 0; i < set->nr_hw_queues; i++)
2467 if (!__blk_mq_alloc_rq_map(set, i))
Jens Axboea5164402014-09-10 09:02:03 -06002468 goto out_unwind;
Jens Axboea5164402014-09-10 09:02:03 -06002469
2470 return 0;
2471
2472out_unwind:
2473 while (--i >= 0)
Jens Axboecc71a6f2017-01-11 14:29:56 -07002474 blk_mq_free_rq_map(set->tags[i]);
Jens Axboea5164402014-09-10 09:02:03 -06002475
Jens Axboea5164402014-09-10 09:02:03 -06002476 return -ENOMEM;
2477}
2478
2479/*
2480 * Allocate the request maps associated with this tag_set. Note that this
2481 * may reduce the depth asked for, if memory is tight. set->queue_depth
2482 * will be updated to reflect the allocated depth.
2483 */
2484static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2485{
2486 unsigned int depth;
2487 int err;
2488
2489 depth = set->queue_depth;
2490 do {
2491 err = __blk_mq_alloc_rq_maps(set);
2492 if (!err)
2493 break;
2494
2495 set->queue_depth >>= 1;
2496 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
2497 err = -ENOMEM;
2498 break;
2499 }
2500 } while (set->queue_depth);
2501
2502 if (!set->queue_depth || err) {
2503 pr_err("blk-mq: failed to allocate request map\n");
2504 return -ENOMEM;
2505 }
2506
2507 if (depth != set->queue_depth)
2508 pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
2509 depth, set->queue_depth);
2510
2511 return 0;
2512}
2513
Omar Sandovalebe8bdd2017-04-07 08:53:11 -06002514static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
2515{
2516 if (set->ops->map_queues)
2517 return set->ops->map_queues(set);
2518 else
2519 return blk_mq_map_queues(set);
2520}
2521
Jens Axboea4391c62014-06-05 15:21:56 -06002522/*
2523 * Alloc a tag set to be associated with one or more request queues.
2524 * May fail with EINVAL for various error conditions. May adjust the
2525 * requested depth down, if if it too large. In that case, the set
2526 * value will be stored in set->queue_depth.
2527 */
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002528int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
2529{
Christoph Hellwigda695ba2016-09-14 16:18:55 +02002530 int ret;
2531
Bart Van Assche205fb5f2014-10-30 14:45:11 +01002532 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
2533
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002534 if (!set->nr_hw_queues)
2535 return -EINVAL;
Jens Axboea4391c62014-06-05 15:21:56 -06002536 if (!set->queue_depth)
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002537 return -EINVAL;
2538 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
2539 return -EINVAL;
2540
Christoph Hellwig7d7e0f92016-09-14 16:18:54 +02002541 if (!set->ops->queue_rq)
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002542 return -EINVAL;
2543
Jens Axboea4391c62014-06-05 15:21:56 -06002544 if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
2545 pr_info("blk-mq: reduced tag depth to %u\n",
2546 BLK_MQ_MAX_DEPTH);
2547 set->queue_depth = BLK_MQ_MAX_DEPTH;
2548 }
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002549
Shaohua Li6637fad2014-11-30 16:00:58 -08002550 /*
2551 * If a crashdump is active, then we are potentially in a very
2552 * memory constrained environment. Limit us to 1 queue and
2553 * 64 tags to prevent using too much memory.
2554 */
2555 if (is_kdump_kernel()) {
2556 set->nr_hw_queues = 1;
2557 set->queue_depth = min(64U, set->queue_depth);
2558 }
Keith Busch868f2f02015-12-17 17:08:14 -07002559 /*
2560 * There is no use for more h/w queues than cpus.
2561 */
2562 if (set->nr_hw_queues > nr_cpu_ids)
2563 set->nr_hw_queues = nr_cpu_ids;
Shaohua Li6637fad2014-11-30 16:00:58 -08002564
Keith Busch868f2f02015-12-17 17:08:14 -07002565 set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002566 GFP_KERNEL, set->numa_node);
2567 if (!set->tags)
Jens Axboea5164402014-09-10 09:02:03 -06002568 return -ENOMEM;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002569
Christoph Hellwigda695ba2016-09-14 16:18:55 +02002570 ret = -ENOMEM;
2571 set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
2572 GFP_KERNEL, set->numa_node);
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002573 if (!set->mq_map)
2574 goto out_free_tags;
2575
Omar Sandovalebe8bdd2017-04-07 08:53:11 -06002576 ret = blk_mq_update_queue_map(set);
Christoph Hellwigda695ba2016-09-14 16:18:55 +02002577 if (ret)
2578 goto out_free_mq_map;
2579
2580 ret = blk_mq_alloc_rq_maps(set);
2581 if (ret)
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002582 goto out_free_mq_map;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002583
Jens Axboe0d2602c2014-05-13 15:10:52 -06002584 mutex_init(&set->tag_list_lock);
2585 INIT_LIST_HEAD(&set->tag_list);
2586
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002587 return 0;
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002588
2589out_free_mq_map:
2590 kfree(set->mq_map);
2591 set->mq_map = NULL;
2592out_free_tags:
Robert Elliott5676e7b2014-09-02 11:38:44 -05002593 kfree(set->tags);
2594 set->tags = NULL;
Christoph Hellwigda695ba2016-09-14 16:18:55 +02002595 return ret;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002596}
2597EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2598
2599void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2600{
2601 int i;
2602
Jens Axboecc71a6f2017-01-11 14:29:56 -07002603 for (i = 0; i < nr_cpu_ids; i++)
2604 blk_mq_free_map_and_requests(set, i);
Jens Axboe484b4062014-05-21 14:01:15 -06002605
Christoph Hellwigbdd17e72016-09-14 16:18:53 +02002606 kfree(set->mq_map);
2607 set->mq_map = NULL;
2608
Ming Lei981bd182014-04-24 00:07:34 +08002609 kfree(set->tags);
Robert Elliott5676e7b2014-09-02 11:38:44 -05002610 set->tags = NULL;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002611}
2612EXPORT_SYMBOL(blk_mq_free_tag_set);
2613
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002614int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2615{
2616 struct blk_mq_tag_set *set = q->tag_set;
2617 struct blk_mq_hw_ctx *hctx;
2618 int i, ret;
2619
Jens Axboebd166ef2017-01-17 06:03:22 -07002620 if (!set)
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002621 return -EINVAL;
2622
Jens Axboe70f36b62017-01-19 10:59:07 -07002623 blk_mq_freeze_queue(q);
Jens Axboe70f36b62017-01-19 10:59:07 -07002624
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002625 ret = 0;
2626 queue_for_each_hw_ctx(q, hctx, i) {
Keith Busche9137d42016-02-18 14:56:35 -07002627 if (!hctx->tags)
2628 continue;
Jens Axboebd166ef2017-01-17 06:03:22 -07002629 /*
2630 * If we're using an MQ scheduler, just update the scheduler
2631 * queue depth. This is similar to what the old code would do.
2632 */
Jens Axboe70f36b62017-01-19 10:59:07 -07002633 if (!hctx->sched_tags) {
2634 ret = blk_mq_tag_update_depth(hctx, &hctx->tags,
2635 min(nr, set->queue_depth),
2636 false);
2637 } else {
2638 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
2639 nr, true);
2640 }
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002641 if (ret)
2642 break;
2643 }
2644
2645 if (!ret)
2646 q->nr_requests = nr;
2647
Jens Axboe70f36b62017-01-19 10:59:07 -07002648 blk_mq_unfreeze_queue(q);
Jens Axboe70f36b62017-01-19 10:59:07 -07002649
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002650 return ret;
2651}
2652
Keith Busche4dc2b32017-05-30 14:39:11 -04002653static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
2654 int nr_hw_queues)
Keith Busch868f2f02015-12-17 17:08:14 -07002655{
2656 struct request_queue *q;
2657
Bart Van Assche705cda92017-04-07 11:16:49 -07002658 lockdep_assert_held(&set->tag_list_lock);
2659
Keith Busch868f2f02015-12-17 17:08:14 -07002660 if (nr_hw_queues > nr_cpu_ids)
2661 nr_hw_queues = nr_cpu_ids;
2662 if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
2663 return;
2664
2665 list_for_each_entry(q, &set->tag_list, tag_set_list)
2666 blk_mq_freeze_queue(q);
2667
2668 set->nr_hw_queues = nr_hw_queues;
Omar Sandovalebe8bdd2017-04-07 08:53:11 -06002669 blk_mq_update_queue_map(set);
Keith Busch868f2f02015-12-17 17:08:14 -07002670 list_for_each_entry(q, &set->tag_list, tag_set_list) {
2671 blk_mq_realloc_hw_ctxs(set, q);
Keith Busch868f2f02015-12-17 17:08:14 -07002672 blk_mq_queue_reinit(q, cpu_online_mask);
2673 }
2674
2675 list_for_each_entry(q, &set->tag_list, tag_set_list)
2676 blk_mq_unfreeze_queue(q);
2677}
Keith Busche4dc2b32017-05-30 14:39:11 -04002678
2679void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
2680{
2681 mutex_lock(&set->tag_list_lock);
2682 __blk_mq_update_nr_hw_queues(set, nr_hw_queues);
2683 mutex_unlock(&set->tag_list_lock);
2684}
Keith Busch868f2f02015-12-17 17:08:14 -07002685EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
2686
Omar Sandoval34dbad52017-03-21 08:56:08 -07002687/* Enable polling stats and return whether they were already enabled. */
2688static bool blk_poll_stats_enable(struct request_queue *q)
2689{
2690 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
2691 test_and_set_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
2692 return true;
2693 blk_stat_add_callback(q, q->poll_cb);
2694 return false;
2695}
2696
2697static void blk_mq_poll_stats_start(struct request_queue *q)
2698{
2699 /*
2700 * We don't arm the callback if polling stats are not enabled or the
2701 * callback is already active.
2702 */
2703 if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
2704 blk_stat_is_active(q->poll_cb))
2705 return;
2706
2707 blk_stat_activate_msecs(q->poll_cb, 100);
2708}
2709
2710static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
2711{
2712 struct request_queue *q = cb->data;
Stephen Bates720b8cc2017-04-07 06:24:03 -06002713 int bucket;
Omar Sandoval34dbad52017-03-21 08:56:08 -07002714
Stephen Bates720b8cc2017-04-07 06:24:03 -06002715 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
2716 if (cb->stat[bucket].nr_samples)
2717 q->poll_stat[bucket] = cb->stat[bucket];
2718 }
Omar Sandoval34dbad52017-03-21 08:56:08 -07002719}
2720
Jens Axboe64f1c212016-11-14 13:03:03 -07002721static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
2722 struct blk_mq_hw_ctx *hctx,
2723 struct request *rq)
2724{
Jens Axboe64f1c212016-11-14 13:03:03 -07002725 unsigned long ret = 0;
Stephen Bates720b8cc2017-04-07 06:24:03 -06002726 int bucket;
Jens Axboe64f1c212016-11-14 13:03:03 -07002727
2728 /*
2729 * If stats collection isn't on, don't sleep but turn it on for
2730 * future users
2731 */
Omar Sandoval34dbad52017-03-21 08:56:08 -07002732 if (!blk_poll_stats_enable(q))
Jens Axboe64f1c212016-11-14 13:03:03 -07002733 return 0;
2734
2735 /*
Jens Axboe64f1c212016-11-14 13:03:03 -07002736 * As an optimistic guess, use half of the mean service time
2737 * for this type of request. We can (and should) make this smarter.
2738 * For instance, if the completion latencies are tight, we can
2739 * get closer than just half the mean. This is especially
2740 * important on devices where the completion latencies are longer
Stephen Bates720b8cc2017-04-07 06:24:03 -06002741 * than ~10 usec. We do use the stats for the relevant IO size
2742 * if available which does lead to better estimates.
Jens Axboe64f1c212016-11-14 13:03:03 -07002743 */
Stephen Bates720b8cc2017-04-07 06:24:03 -06002744 bucket = blk_mq_poll_stats_bkt(rq);
2745 if (bucket < 0)
2746 return ret;
2747
2748 if (q->poll_stat[bucket].nr_samples)
2749 ret = (q->poll_stat[bucket].mean + 1) / 2;
Jens Axboe64f1c212016-11-14 13:03:03 -07002750
2751 return ret;
2752}
2753
Jens Axboe06426ad2016-11-14 13:01:59 -07002754static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
Jens Axboe64f1c212016-11-14 13:03:03 -07002755 struct blk_mq_hw_ctx *hctx,
Jens Axboe06426ad2016-11-14 13:01:59 -07002756 struct request *rq)
2757{
2758 struct hrtimer_sleeper hs;
2759 enum hrtimer_mode mode;
Jens Axboe64f1c212016-11-14 13:03:03 -07002760 unsigned int nsecs;
Jens Axboe06426ad2016-11-14 13:01:59 -07002761 ktime_t kt;
2762
Jens Axboe64f1c212016-11-14 13:03:03 -07002763 if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
2764 return false;
2765
2766 /*
2767 * poll_nsec can be:
2768 *
2769 * -1: don't ever hybrid sleep
2770 * 0: use half of prev avg
2771 * >0: use this specific value
2772 */
2773 if (q->poll_nsec == -1)
2774 return false;
2775 else if (q->poll_nsec > 0)
2776 nsecs = q->poll_nsec;
2777 else
2778 nsecs = blk_mq_poll_nsecs(q, hctx, rq);
2779
2780 if (!nsecs)
Jens Axboe06426ad2016-11-14 13:01:59 -07002781 return false;
2782
2783 set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
2784
2785 /*
2786 * This will be replaced with the stats tracking code, using
2787 * 'avg_completion_time / 2' as the pre-sleep target.
2788 */
Thomas Gleixner8b0e1952016-12-25 12:30:41 +01002789 kt = nsecs;
Jens Axboe06426ad2016-11-14 13:01:59 -07002790
2791 mode = HRTIMER_MODE_REL;
2792 hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
2793 hrtimer_set_expires(&hs.timer, kt);
2794
2795 hrtimer_init_sleeper(&hs, current);
2796 do {
2797 if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
2798 break;
2799 set_current_state(TASK_UNINTERRUPTIBLE);
2800 hrtimer_start_expires(&hs.timer, mode);
2801 if (hs.task)
2802 io_schedule();
2803 hrtimer_cancel(&hs.timer);
2804 mode = HRTIMER_MODE_ABS;
2805 } while (hs.task && !signal_pending(current));
2806
2807 __set_current_state(TASK_RUNNING);
2808 destroy_hrtimer_on_stack(&hs.timer);
2809 return true;
2810}
2811
Jens Axboebbd7bb72016-11-04 09:34:34 -06002812static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
2813{
2814 struct request_queue *q = hctx->queue;
2815 long state;
2816
Jens Axboe06426ad2016-11-14 13:01:59 -07002817 /*
2818 * If we sleep, have the caller restart the poll loop to reset
2819 * the state. Like for the other success return cases, the
2820 * caller is responsible for checking if the IO completed. If
2821 * the IO isn't complete, we'll get called again and will go
2822 * straight to the busy poll loop.
2823 */
Jens Axboe64f1c212016-11-14 13:03:03 -07002824 if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
Jens Axboe06426ad2016-11-14 13:01:59 -07002825 return true;
2826
Jens Axboebbd7bb72016-11-04 09:34:34 -06002827 hctx->poll_considered++;
2828
2829 state = current->state;
2830 while (!need_resched()) {
2831 int ret;
2832
2833 hctx->poll_invoked++;
2834
2835 ret = q->mq_ops->poll(hctx, rq->tag);
2836 if (ret > 0) {
2837 hctx->poll_success++;
2838 set_current_state(TASK_RUNNING);
2839 return true;
2840 }
2841
2842 if (signal_pending_state(state, current))
2843 set_current_state(TASK_RUNNING);
2844
2845 if (current->state == TASK_RUNNING)
2846 return true;
2847 if (ret < 0)
2848 break;
2849 cpu_relax();
2850 }
2851
2852 return false;
2853}
2854
2855bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
2856{
2857 struct blk_mq_hw_ctx *hctx;
2858 struct blk_plug *plug;
2859 struct request *rq;
2860
2861 if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
2862 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
2863 return false;
2864
2865 plug = current->plug;
2866 if (plug)
2867 blk_flush_plug_list(plug, false);
2868
2869 hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
Jens Axboebd166ef2017-01-17 06:03:22 -07002870 if (!blk_qc_t_is_internal(cookie))
2871 rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
Jens Axboe3a07bb12017-04-20 14:53:28 -06002872 else {
Jens Axboebd166ef2017-01-17 06:03:22 -07002873 rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
Jens Axboe3a07bb12017-04-20 14:53:28 -06002874 /*
2875 * With scheduling, if the request has completed, we'll
2876 * get a NULL return here, as we clear the sched tag when
2877 * that happens. The request still remains valid, like always,
2878 * so we should be safe with just the NULL check.
2879 */
2880 if (!rq)
2881 return false;
2882 }
Jens Axboebbd7bb72016-11-04 09:34:34 -06002883
2884 return __blk_mq_poll(hctx, rq);
2885}
2886EXPORT_SYMBOL_GPL(blk_mq_poll);
2887
Jens Axboe676141e2014-03-20 13:29:18 -06002888void blk_mq_disable_hotplug(void)
2889{
2890 mutex_lock(&all_q_mutex);
2891}
2892
2893void blk_mq_enable_hotplug(void)
2894{
2895 mutex_unlock(&all_q_mutex);
2896}
2897
Jens Axboe320ae512013-10-24 09:20:05 +01002898static int __init blk_mq_init(void)
2899{
Thomas Gleixner9467f852016-09-22 08:05:17 -06002900 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
2901 blk_mq_hctx_notify_dead);
Jens Axboe320ae512013-10-24 09:20:05 +01002902
Sebastian Andrzej Siewior65d52912016-09-22 08:05:19 -06002903 cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
2904 blk_mq_queue_reinit_prepare,
2905 blk_mq_queue_reinit_dead);
Jens Axboe320ae512013-10-24 09:20:05 +01002906 return 0;
2907}
2908subsys_initcall(blk_mq_init);