blob: 535ab2e13d2e74ff13e85ce28b869b2b0eae3b74 [file] [log] [blame]
Jens Axboe320ae512013-10-24 09:20:05 +01001#ifndef BLK_MQ_H
2#define BLK_MQ_H
3
4#include <linux/blkdev.h>
Omar Sandoval88459642016-09-17 08:38:44 -06005#include <linux/sbitmap.h>
Jens Axboe320ae512013-10-24 09:20:05 +01006
7struct blk_mq_tags;
Ming Leif70ced02014-09-25 23:23:47 +08008struct blk_flush_queue;
Jens Axboe320ae512013-10-24 09:20:05 +01009
Jens Axboe320ae512013-10-24 09:20:05 +010010struct blk_mq_hw_ctx {
11 struct {
12 spinlock_t lock;
13 struct list_head dispatch;
Jens Axboe8d354f12016-08-25 08:00:28 -060014 unsigned long state; /* BLK_MQ_S_* flags */
Jens Axboe320ae512013-10-24 09:20:05 +010015 } ____cacheline_aligned_in_smp;
16
Jens Axboe27489a32016-08-24 15:54:25 -060017 struct work_struct run_work;
Jens Axboee4043dc2014-04-09 10:18:23 -060018 cpumask_var_t cpumask;
Jens Axboe506e9312014-05-07 10:26:44 -060019 int next_cpu;
20 int next_cpu_batch;
Jens Axboe320ae512013-10-24 09:20:05 +010021
22 unsigned long flags; /* BLK_MQ_F_* flags */
23
24 struct request_queue *queue;
Ming Leif70ced02014-09-25 23:23:47 +080025 struct blk_flush_queue *fq;
Jens Axboe320ae512013-10-24 09:20:05 +010026
27 void *driver_data;
28
Omar Sandoval88459642016-09-17 08:38:44 -060029 struct sbitmap ctx_map;
Jens Axboe1429d7c2014-05-19 09:23:55 -060030
Jens Axboe4bb659b2014-05-09 09:36:49 -060031 struct blk_mq_ctx **ctxs;
Jens Axboe8d354f12016-08-25 08:00:28 -060032 unsigned int nr_ctx;
Jens Axboe4bb659b2014-05-09 09:36:49 -060033
Alexander Gordeev8537b122014-06-17 22:12:35 -070034 atomic_t wait_index;
Jens Axboe320ae512013-10-24 09:20:05 +010035
Jens Axboe320ae512013-10-24 09:20:05 +010036 struct blk_mq_tags *tags;
37
38 unsigned long queued;
39 unsigned long run;
Jens Axboe8d354f12016-08-25 08:00:28 -060040#define BLK_MQ_MAX_DISPATCH_ORDER 7
Jens Axboe320ae512013-10-24 09:20:05 +010041 unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER];
42
Jens Axboe320ae512013-10-24 09:20:05 +010043 unsigned int numa_node;
Jens Axboe17ded322015-01-07 10:44:04 -070044 unsigned int queue_num;
Jens Axboe320ae512013-10-24 09:20:05 +010045
Jens Axboe0d2602c2014-05-13 15:10:52 -060046 atomic_t nr_active;
47
Jens Axboe8d354f12016-08-25 08:00:28 -060048 struct delayed_work delay_work;
49
Thomas Gleixner9467f852016-09-22 08:05:17 -060050 struct hlist_node cpuhp_dead;
Jens Axboe320ae512013-10-24 09:20:05 +010051 struct kobject kobj;
Jens Axboe05229be2015-11-05 10:44:55 -070052
Stephen Bates6e219352016-09-13 12:23:15 -060053 unsigned long poll_considered;
Jens Axboe05229be2015-11-05 10:44:55 -070054 unsigned long poll_invoked;
55 unsigned long poll_success;
Jens Axboe320ae512013-10-24 09:20:05 +010056};
57
Christoph Hellwig24d2f902014-04-15 14:14:00 -060058struct blk_mq_tag_set {
Christoph Hellwigbdd17e72016-09-14 16:18:53 +020059 unsigned int *mq_map;
Jens Axboe320ae512013-10-24 09:20:05 +010060 struct blk_mq_ops *ops;
61 unsigned int nr_hw_queues;
Jens Axboee3a2b3f2014-05-20 11:49:02 -060062 unsigned int queue_depth; /* max hw supported */
Jens Axboe320ae512013-10-24 09:20:05 +010063 unsigned int reserved_tags;
64 unsigned int cmd_size; /* per-request extra data */
65 int numa_node;
66 unsigned int timeout;
67 unsigned int flags; /* BLK_MQ_F_* */
Christoph Hellwig24d2f902014-04-15 14:14:00 -060068 void *driver_data;
69
70 struct blk_mq_tags **tags;
Jens Axboe0d2602c2014-05-13 15:10:52 -060071
72 struct mutex tag_list_lock;
73 struct list_head tag_list;
Jens Axboe320ae512013-10-24 09:20:05 +010074};
75
Jens Axboe74c45052014-10-29 11:14:52 -060076struct blk_mq_queue_data {
77 struct request *rq;
78 struct list_head *list;
79 bool last;
80};
81
82typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *);
Christoph Hellwig0152fb62014-09-13 16:40:13 -070083typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool);
Jens Axboe320ae512013-10-24 09:20:05 +010084typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int);
85typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int);
Christoph Hellwig24d2f902014-04-15 14:14:00 -060086typedef int (init_request_fn)(void *, struct request *, unsigned int,
87 unsigned int, unsigned int);
88typedef void (exit_request_fn)(void *, struct request *, unsigned int,
89 unsigned int);
Sagi Grimberg486cf982016-07-06 21:55:48 +090090typedef int (reinit_request_fn)(void *, struct request *);
Jens Axboe320ae512013-10-24 09:20:05 +010091
Christoph Hellwig81481eb2014-09-13 16:40:11 -070092typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,
93 bool);
Keith Buschf26cdc82015-06-01 09:29:53 -060094typedef void (busy_tag_iter_fn)(struct request *, void *, bool);
Jens Axboe05229be2015-11-05 10:44:55 -070095typedef int (poll_fn)(struct blk_mq_hw_ctx *, unsigned int);
Christoph Hellwigda695ba2016-09-14 16:18:55 +020096typedef int (map_queues_fn)(struct blk_mq_tag_set *set);
Jens Axboe05229be2015-11-05 10:44:55 -070097
Christoph Hellwig81481eb2014-09-13 16:40:11 -070098
Jens Axboe320ae512013-10-24 09:20:05 +010099struct blk_mq_ops {
100 /*
101 * Queue request
102 */
103 queue_rq_fn *queue_rq;
104
105 /*
Jens Axboe320ae512013-10-24 09:20:05 +0100106 * Called on request timeout
107 */
Christoph Hellwig0152fb62014-09-13 16:40:13 -0700108 timeout_fn *timeout;
Jens Axboe320ae512013-10-24 09:20:05 +0100109
Jens Axboe05229be2015-11-05 10:44:55 -0700110 /*
111 * Called to poll for completion of a specific tag.
112 */
113 poll_fn *poll;
114
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800115 softirq_done_fn *complete;
116
Jens Axboe320ae512013-10-24 09:20:05 +0100117 /*
Jens Axboe320ae512013-10-24 09:20:05 +0100118 * Called when the block layer side of a hardware queue has been
119 * set up, allowing the driver to allocate/init matching structures.
120 * Ditto for exit/teardown.
121 */
122 init_hctx_fn *init_hctx;
123 exit_hctx_fn *exit_hctx;
Christoph Hellwige9b267d2014-04-15 13:59:10 -0600124
125 /*
126 * Called for every command allocated by the block layer to allow
127 * the driver to set up driver specific data.
Ming Leif70ced02014-09-25 23:23:47 +0800128 *
129 * Tag greater than or equal to queue_depth is for setting up
130 * flush request.
131 *
Christoph Hellwige9b267d2014-04-15 13:59:10 -0600132 * Ditto for exit/teardown.
133 */
134 init_request_fn *init_request;
135 exit_request_fn *exit_request;
Sagi Grimberg486cf982016-07-06 21:55:48 +0900136 reinit_request_fn *reinit_request;
Christoph Hellwigda695ba2016-09-14 16:18:55 +0200137
138 map_queues_fn *map_queues;
Jens Axboe320ae512013-10-24 09:20:05 +0100139};
140
141enum {
142 BLK_MQ_RQ_QUEUE_OK = 0, /* queued fine */
143 BLK_MQ_RQ_QUEUE_BUSY = 1, /* requeue IO for later */
144 BLK_MQ_RQ_QUEUE_ERROR = 2, /* end IO with error */
145
146 BLK_MQ_F_SHOULD_MERGE = 1 << 0,
Jens Axboe8a58d1f2014-08-15 12:38:41 -0600147 BLK_MQ_F_TAG_SHARED = 1 << 1,
148 BLK_MQ_F_SG_MERGE = 1 << 2,
Jens Axboee167dfb2014-10-29 11:18:26 -0600149 BLK_MQ_F_DEFER_ISSUE = 1 << 4,
Jens Axboe1b792f22016-09-21 10:12:13 -0600150 BLK_MQ_F_BLOCKING = 1 << 5,
Shaohua Li24391c02015-01-23 14:18:00 -0700151 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
152 BLK_MQ_F_ALLOC_POLICY_BITS = 1,
Jens Axboe320ae512013-10-24 09:20:05 +0100153
Jens Axboe5d12f902014-03-19 15:25:02 -0600154 BLK_MQ_S_STOPPED = 0,
Jens Axboe0d2602c2014-05-13 15:10:52 -0600155 BLK_MQ_S_TAG_ACTIVE = 1,
Jens Axboe320ae512013-10-24 09:20:05 +0100156
Jens Axboea4391c62014-06-05 15:21:56 -0600157 BLK_MQ_MAX_DEPTH = 10240,
Jens Axboe506e9312014-05-07 10:26:44 -0600158
159 BLK_MQ_CPU_WORK_BATCH = 8,
Jens Axboe320ae512013-10-24 09:20:05 +0100160};
Shaohua Li24391c02015-01-23 14:18:00 -0700161#define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \
162 ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \
163 ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1))
164#define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \
165 ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \
166 << BLK_MQ_F_ALLOC_POLICY_START_BIT)
Jens Axboe320ae512013-10-24 09:20:05 +0100167
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600168struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *);
Mike Snitzerb62c21b2015-03-12 23:56:02 -0400169struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
170 struct request_queue *q);
Matias Bjørlingb21d5b32016-09-16 14:25:06 +0200171int blk_mq_register_dev(struct device *, struct request_queue *);
172void blk_mq_unregister_dev(struct device *, struct request_queue *);
Jens Axboe320ae512013-10-24 09:20:05 +0100173
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600174int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set);
175void blk_mq_free_tag_set(struct blk_mq_tag_set *set);
176
Jens Axboe320ae512013-10-24 09:20:05 +0100177void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
178
Christoph Hellwigfeb71da2014-02-20 15:32:37 -0800179void blk_mq_insert_request(struct request *, bool, bool, bool);
Jens Axboe320ae512013-10-24 09:20:05 +0100180void blk_mq_free_request(struct request *rq);
Jens Axboe7c7f2f22014-11-17 10:41:57 -0700181void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *, struct request *rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100182bool blk_mq_can_queue(struct blk_mq_hw_ctx *);
Christoph Hellwig6f3b0e82015-11-26 09:13:05 +0100183
184enum {
185 BLK_MQ_REQ_NOWAIT = (1 << 0), /* return when out of requests */
186 BLK_MQ_REQ_RESERVED = (1 << 1), /* allocate from reserved pool */
187};
188
Christoph Hellwig4ce01dd2014-05-27 20:59:46 +0200189struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
Christoph Hellwig6f3b0e82015-11-26 09:13:05 +0100190 unsigned int flags);
Ming Lin1f5bd332016-06-13 16:45:21 +0200191struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int op,
192 unsigned int flags, unsigned int hctx_idx);
Jens Axboe0e62f512014-06-04 10:23:49 -0600193struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag);
Jens Axboe320ae512013-10-24 09:20:05 +0100194
Bart Van Assche205fb5f2014-10-30 14:45:11 +0100195enum {
196 BLK_MQ_UNIQUE_TAG_BITS = 16,
197 BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1,
198};
199
200u32 blk_mq_unique_tag(struct request *rq);
201
202static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag)
203{
204 return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS;
205}
206
207static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag)
208{
209 return unique_tag & BLK_MQ_UNIQUE_TAG_MASK;
210}
211
Jens Axboe320ae512013-10-24 09:20:05 +0100212
Keith Busch973c0192015-01-07 18:55:43 -0700213int blk_mq_request_started(struct request *rq);
Christoph Hellwige2490072014-09-13 16:40:09 -0700214void blk_mq_start_request(struct request *rq);
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700215void blk_mq_end_request(struct request *rq, int error);
216void __blk_mq_end_request(struct request *rq, int error);
Jens Axboe320ae512013-10-24 09:20:05 +0100217
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200218void blk_mq_requeue_request(struct request *rq);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600219void blk_mq_add_to_requeue_list(struct request *rq, bool at_head);
Keith Buschc68ed592015-01-07 18:55:44 -0700220void blk_mq_cancel_requeue_work(struct request_queue *q);
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600221void blk_mq_kick_requeue_list(struct request_queue *q);
Mike Snitzer28494502016-09-14 13:28:30 -0400222void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs);
Jens Axboe1885b242015-01-07 18:55:45 -0700223void blk_mq_abort_requeue_list(struct request_queue *q);
Christoph Hellwigf4829a92015-09-27 21:01:50 +0200224void blk_mq_complete_request(struct request *rq, int error);
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800225
Jens Axboe320ae512013-10-24 09:20:05 +0100226void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
227void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
Christoph Hellwig280d45f2013-10-25 14:45:58 +0100228void blk_mq_stop_hw_queues(struct request_queue *q);
Christoph Hellwig2f268552014-04-16 09:44:56 +0200229void blk_mq_start_hw_queues(struct request_queue *q);
Christoph Hellwig1b4a3252014-04-16 09:44:54 +0200230void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
Mike Snitzerb94ec292015-03-11 23:56:38 -0400231void blk_mq_run_hw_queues(struct request_queue *q, bool async);
Christoph Hellwig70f4db62014-04-16 10:48:08 -0600232void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
Sagi Grimberge0489482016-03-10 13:58:46 +0200233void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
234 busy_tag_iter_fn *fn, void *priv);
Jens Axboec761d962015-01-02 15:05:12 -0700235void blk_mq_freeze_queue(struct request_queue *q);
Keith Buschb4c6a022014-12-19 17:54:14 -0700236void blk_mq_unfreeze_queue(struct request_queue *q);
237void blk_mq_freeze_queue_start(struct request_queue *q);
Sagi Grimberg486cf982016-07-06 21:55:48 +0900238int blk_mq_reinit_tagset(struct blk_mq_tag_set *set);
Jens Axboe320ae512013-10-24 09:20:05 +0100239
Keith Busch868f2f02015-12-17 17:08:14 -0700240void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues);
241
Jens Axboe320ae512013-10-24 09:20:05 +0100242/*
243 * Driver command data is immediately after the request. So subtract request
Jens Axboe2963e3f2015-04-09 15:54:05 -0600244 * size to get back to the original request, add request size to get the PDU.
Jens Axboe320ae512013-10-24 09:20:05 +0100245 */
246static inline struct request *blk_mq_rq_from_pdu(void *pdu)
247{
248 return pdu - sizeof(struct request);
249}
250static inline void *blk_mq_rq_to_pdu(struct request *rq)
251{
Jens Axboe2963e3f2015-04-09 15:54:05 -0600252 return rq + 1;
Jens Axboe320ae512013-10-24 09:20:05 +0100253}
254
Jens Axboe320ae512013-10-24 09:20:05 +0100255#define queue_for_each_hw_ctx(q, hctx, i) \
Jose Alonso0d0b7d42014-01-28 08:09:46 -0700256 for ((i) = 0; (i) < (q)->nr_hw_queues && \
257 ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++)
Jens Axboe320ae512013-10-24 09:20:05 +0100258
Jens Axboe320ae512013-10-24 09:20:05 +0100259#define hctx_for_each_ctx(hctx, ctx, i) \
Jose Alonso0d0b7d42014-01-28 08:09:46 -0700260 for ((i) = 0; (i) < (hctx)->nr_ctx && \
261 ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++)
Jens Axboe320ae512013-10-24 09:20:05 +0100262
Jens Axboe320ae512013-10-24 09:20:05 +0100263#endif