blob: 8001e812018bbbc0c4360dacd6dc71e8029ab068 [file] [log] [blame]
Jens Axboef2298c02013-10-25 11:52:25 +01001#include <linux/module.h>
Matias Bjørlingfc1bc352013-12-21 00:11:01 +01002
Jens Axboef2298c02013-10-25 11:52:25 +01003#include <linux/moduleparam.h>
4#include <linux/sched.h>
5#include <linux/fs.h>
6#include <linux/blkdev.h>
7#include <linux/init.h>
8#include <linux/slab.h>
9#include <linux/blk-mq.h>
10#include <linux/hrtimer.h>
11
12struct nullb_cmd {
13 struct list_head list;
14 struct llist_node ll_list;
15 struct call_single_data csd;
16 struct request *rq;
17 struct bio *bio;
18 unsigned int tag;
19 struct nullb_queue *nq;
20};
21
22struct nullb_queue {
23 unsigned long *tag_map;
24 wait_queue_head_t wait;
25 unsigned int queue_depth;
26
27 struct nullb_cmd *cmds;
28};
29
30struct nullb {
31 struct list_head list;
32 unsigned int index;
33 struct request_queue *q;
34 struct gendisk *disk;
Christoph Hellwig24d2f902014-04-15 14:14:00 -060035 struct blk_mq_tag_set tag_set;
Jens Axboef2298c02013-10-25 11:52:25 +010036 struct hrtimer timer;
37 unsigned int queue_depth;
38 spinlock_t lock;
39
40 struct nullb_queue *queues;
41 unsigned int nr_queues;
42};
43
44static LIST_HEAD(nullb_list);
45static struct mutex lock;
46static int null_major;
47static int nullb_indexes;
48
49struct completion_queue {
50 struct llist_head list;
51 struct hrtimer timer;
52};
53
54/*
55 * These are per-cpu for now, they will need to be configured by the
56 * complete_queues parameter and appropriately mapped.
57 */
58static DEFINE_PER_CPU(struct completion_queue, completion_queues);
59
60enum {
61 NULL_IRQ_NONE = 0,
62 NULL_IRQ_SOFTIRQ = 1,
63 NULL_IRQ_TIMER = 2,
Christoph Hellwigce2c3502014-02-10 03:24:40 -080064};
Jens Axboef2298c02013-10-25 11:52:25 +010065
Christoph Hellwigce2c3502014-02-10 03:24:40 -080066enum {
Jens Axboef2298c02013-10-25 11:52:25 +010067 NULL_Q_BIO = 0,
68 NULL_Q_RQ = 1,
69 NULL_Q_MQ = 2,
70};
71
Matias Bjorling2d263a782013-12-18 13:41:43 +010072static int submit_queues;
Jens Axboef2298c02013-10-25 11:52:25 +010073module_param(submit_queues, int, S_IRUGO);
74MODULE_PARM_DESC(submit_queues, "Number of submission queues");
75
76static int home_node = NUMA_NO_NODE;
77module_param(home_node, int, S_IRUGO);
78MODULE_PARM_DESC(home_node, "Home node for the device");
79
80static int queue_mode = NULL_Q_MQ;
81module_param(queue_mode, int, S_IRUGO);
Mike Snitzer54ae81c2014-06-11 17:13:50 -040082MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
Jens Axboef2298c02013-10-25 11:52:25 +010083
84static int gb = 250;
85module_param(gb, int, S_IRUGO);
86MODULE_PARM_DESC(gb, "Size in GB");
87
88static int bs = 512;
89module_param(bs, int, S_IRUGO);
90MODULE_PARM_DESC(bs, "Block size (in bytes)");
91
92static int nr_devices = 2;
93module_param(nr_devices, int, S_IRUGO);
94MODULE_PARM_DESC(nr_devices, "Number of devices to register");
95
96static int irqmode = NULL_IRQ_SOFTIRQ;
97module_param(irqmode, int, S_IRUGO);
98MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
99
100static int completion_nsec = 10000;
101module_param(completion_nsec, int, S_IRUGO);
102MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
103
104static int hw_queue_depth = 64;
105module_param(hw_queue_depth, int, S_IRUGO);
106MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
107
Matias Bjørling20005242013-12-21 00:11:00 +0100108static bool use_per_node_hctx = false;
Jens Axboef2298c02013-10-25 11:52:25 +0100109module_param(use_per_node_hctx, bool, S_IRUGO);
Matias Bjørling20005242013-12-21 00:11:00 +0100110MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
Jens Axboef2298c02013-10-25 11:52:25 +0100111
112static void put_tag(struct nullb_queue *nq, unsigned int tag)
113{
114 clear_bit_unlock(tag, nq->tag_map);
115
116 if (waitqueue_active(&nq->wait))
117 wake_up(&nq->wait);
118}
119
120static unsigned int get_tag(struct nullb_queue *nq)
121{
122 unsigned int tag;
123
124 do {
125 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
126 if (tag >= nq->queue_depth)
127 return -1U;
128 } while (test_and_set_bit_lock(tag, nq->tag_map));
129
130 return tag;
131}
132
133static void free_cmd(struct nullb_cmd *cmd)
134{
135 put_tag(cmd->nq, cmd->tag);
136}
137
138static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
139{
140 struct nullb_cmd *cmd;
141 unsigned int tag;
142
143 tag = get_tag(nq);
144 if (tag != -1U) {
145 cmd = &nq->cmds[tag];
146 cmd->tag = tag;
147 cmd->nq = nq;
148 return cmd;
149 }
150
151 return NULL;
152}
153
154static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
155{
156 struct nullb_cmd *cmd;
157 DEFINE_WAIT(wait);
158
159 cmd = __alloc_cmd(nq);
160 if (cmd || !can_wait)
161 return cmd;
162
163 do {
164 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
165 cmd = __alloc_cmd(nq);
166 if (cmd)
167 break;
168
169 io_schedule();
170 } while (1);
171
172 finish_wait(&nq->wait, &wait);
173 return cmd;
174}
175
176static void end_cmd(struct nullb_cmd *cmd)
177{
Christoph Hellwigce2c3502014-02-10 03:24:40 -0800178 switch (queue_mode) {
179 case NULL_Q_MQ:
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700180 blk_mq_end_request(cmd->rq, 0);
Christoph Hellwigce2c3502014-02-10 03:24:40 -0800181 return;
182 case NULL_Q_RQ:
183 INIT_LIST_HEAD(&cmd->rq->queuelist);
184 blk_end_request_all(cmd->rq, 0);
185 break;
186 case NULL_Q_BIO:
Jens Axboef2298c02013-10-25 11:52:25 +0100187 bio_endio(cmd->bio, 0);
Christoph Hellwigce2c3502014-02-10 03:24:40 -0800188 break;
189 }
Jens Axboef2298c02013-10-25 11:52:25 +0100190
Christoph Hellwigce2c3502014-02-10 03:24:40 -0800191 free_cmd(cmd);
Jens Axboef2298c02013-10-25 11:52:25 +0100192}
193
194static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
195{
196 struct completion_queue *cq;
197 struct llist_node *entry;
198 struct nullb_cmd *cmd;
199
200 cq = &per_cpu(completion_queues, smp_processor_id());
201
202 while ((entry = llist_del_all(&cq->list)) != NULL) {
Shlomo Pongratzd7790b92014-02-06 18:33:17 +0200203 entry = llist_reverse_order(entry);
Jens Axboef2298c02013-10-25 11:52:25 +0100204 do {
205 cmd = container_of(entry, struct nullb_cmd, ll_list);
Jens Axboef2298c02013-10-25 11:52:25 +0100206 entry = entry->next;
Ming Leifc276912014-05-01 15:12:36 +0800207 end_cmd(cmd);
Jens Axboef2298c02013-10-25 11:52:25 +0100208 } while (entry);
209 }
210
211 return HRTIMER_NORESTART;
212}
213
214static void null_cmd_end_timer(struct nullb_cmd *cmd)
215{
216 struct completion_queue *cq = &per_cpu(completion_queues, get_cpu());
217
218 cmd->ll_list.next = NULL;
219 if (llist_add(&cmd->ll_list, &cq->list)) {
220 ktime_t kt = ktime_set(0, completion_nsec);
221
222 hrtimer_start(&cq->timer, kt, HRTIMER_MODE_REL);
223 }
224
225 put_cpu();
226}
227
228static void null_softirq_done_fn(struct request *rq)
229{
Jens Axboed891fa72014-06-16 11:40:25 -0600230 if (queue_mode == NULL_Q_MQ)
231 end_cmd(blk_mq_rq_to_pdu(rq));
232 else
233 end_cmd(rq->special);
Jens Axboef2298c02013-10-25 11:52:25 +0100234}
235
Jens Axboef2298c02013-10-25 11:52:25 +0100236static inline void null_handle_cmd(struct nullb_cmd *cmd)
237{
238 /* Complete IO by inline, softirq or timer */
239 switch (irqmode) {
Christoph Hellwigce2c3502014-02-10 03:24:40 -0800240 case NULL_IRQ_SOFTIRQ:
241 switch (queue_mode) {
242 case NULL_Q_MQ:
243 blk_mq_complete_request(cmd->rq);
244 break;
245 case NULL_Q_RQ:
246 blk_complete_request(cmd->rq);
247 break;
248 case NULL_Q_BIO:
249 /*
250 * XXX: no proper submitting cpu information available.
251 */
252 end_cmd(cmd);
253 break;
254 }
255 break;
Jens Axboef2298c02013-10-25 11:52:25 +0100256 case NULL_IRQ_NONE:
257 end_cmd(cmd);
258 break;
Jens Axboef2298c02013-10-25 11:52:25 +0100259 case NULL_IRQ_TIMER:
260 null_cmd_end_timer(cmd);
261 break;
262 }
263}
264
265static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
266{
267 int index = 0;
268
269 if (nullb->nr_queues != 1)
270 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
271
272 return &nullb->queues[index];
273}
274
275static void null_queue_bio(struct request_queue *q, struct bio *bio)
276{
277 struct nullb *nullb = q->queuedata;
278 struct nullb_queue *nq = nullb_to_queue(nullb);
279 struct nullb_cmd *cmd;
280
281 cmd = alloc_cmd(nq, 1);
282 cmd->bio = bio;
283
284 null_handle_cmd(cmd);
285}
286
287static int null_rq_prep_fn(struct request_queue *q, struct request *req)
288{
289 struct nullb *nullb = q->queuedata;
290 struct nullb_queue *nq = nullb_to_queue(nullb);
291 struct nullb_cmd *cmd;
292
293 cmd = alloc_cmd(nq, 0);
294 if (cmd) {
295 cmd->rq = req;
296 req->special = cmd;
297 return BLKPREP_OK;
298 }
299
300 return BLKPREP_DEFER;
301}
302
303static void null_request_fn(struct request_queue *q)
304{
305 struct request *rq;
306
307 while ((rq = blk_fetch_request(q)) != NULL) {
308 struct nullb_cmd *cmd = rq->special;
309
310 spin_unlock_irq(q->queue_lock);
311 null_handle_cmd(cmd);
312 spin_lock_irq(q->queue_lock);
313 }
314}
315
Christoph Hellwigbf572292014-09-13 16:40:08 -0700316static int null_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq,
317 bool last)
Jens Axboef2298c02013-10-25 11:52:25 +0100318{
Christoph Hellwig9d74e252014-04-14 10:30:07 +0200319 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
Jens Axboef2298c02013-10-25 11:52:25 +0100320
321 cmd->rq = rq;
322 cmd->nq = hctx->driver_data;
323
Christoph Hellwige2490072014-09-13 16:40:09 -0700324 blk_mq_start_request(rq);
325
Jens Axboef2298c02013-10-25 11:52:25 +0100326 null_handle_cmd(cmd);
327 return BLK_MQ_RQ_QUEUE_OK;
328}
329
Matias Bjorling2d263a782013-12-18 13:41:43 +0100330static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
331{
332 BUG_ON(!nullb);
333 BUG_ON(!nq);
334
335 init_waitqueue_head(&nq->wait);
336 nq->queue_depth = nullb->queue_depth;
337}
338
Jens Axboef2298c02013-10-25 11:52:25 +0100339static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
340 unsigned int index)
341{
342 struct nullb *nullb = data;
343 struct nullb_queue *nq = &nullb->queues[index];
344
Jens Axboef2298c02013-10-25 11:52:25 +0100345 hctx->driver_data = nq;
Matias Bjorling2d263a782013-12-18 13:41:43 +0100346 null_init_queue(nullb, nq);
347 nullb->nr_queues++;
Jens Axboef2298c02013-10-25 11:52:25 +0100348
349 return 0;
350}
351
352static struct blk_mq_ops null_mq_ops = {
353 .queue_rq = null_queue_rq,
354 .map_queue = blk_mq_map_queue,
355 .init_hctx = null_init_hctx,
Christoph Hellwigce2c3502014-02-10 03:24:40 -0800356 .complete = null_softirq_done_fn,
Jens Axboef2298c02013-10-25 11:52:25 +0100357};
358
359static void null_del_dev(struct nullb *nullb)
360{
361 list_del_init(&nullb->list);
362
363 del_gendisk(nullb->disk);
Ming Lei518d00b2013-12-26 21:31:37 +0800364 blk_cleanup_queue(nullb->q);
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600365 if (queue_mode == NULL_Q_MQ)
366 blk_mq_free_tag_set(&nullb->tag_set);
Jens Axboef2298c02013-10-25 11:52:25 +0100367 put_disk(nullb->disk);
368 kfree(nullb);
369}
370
371static int null_open(struct block_device *bdev, fmode_t mode)
372{
373 return 0;
374}
375
376static void null_release(struct gendisk *disk, fmode_t mode)
377{
378}
379
380static const struct block_device_operations null_fops = {
381 .owner = THIS_MODULE,
382 .open = null_open,
383 .release = null_release,
384};
385
386static int setup_commands(struct nullb_queue *nq)
387{
388 struct nullb_cmd *cmd;
389 int i, tag_size;
390
391 nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL);
392 if (!nq->cmds)
Matias Bjorling2d263a782013-12-18 13:41:43 +0100393 return -ENOMEM;
Jens Axboef2298c02013-10-25 11:52:25 +0100394
395 tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
396 nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL);
397 if (!nq->tag_map) {
398 kfree(nq->cmds);
Matias Bjorling2d263a782013-12-18 13:41:43 +0100399 return -ENOMEM;
Jens Axboef2298c02013-10-25 11:52:25 +0100400 }
401
402 for (i = 0; i < nq->queue_depth; i++) {
403 cmd = &nq->cmds[i];
404 INIT_LIST_HEAD(&cmd->list);
405 cmd->ll_list.next = NULL;
406 cmd->tag = -1U;
407 }
408
409 return 0;
410}
411
412static void cleanup_queue(struct nullb_queue *nq)
413{
414 kfree(nq->tag_map);
415 kfree(nq->cmds);
416}
417
418static void cleanup_queues(struct nullb *nullb)
419{
420 int i;
421
422 for (i = 0; i < nullb->nr_queues; i++)
423 cleanup_queue(&nullb->queues[i]);
424
425 kfree(nullb->queues);
426}
427
428static int setup_queues(struct nullb *nullb)
429{
Matias Bjorling2d263a782013-12-18 13:41:43 +0100430 nullb->queues = kzalloc(submit_queues * sizeof(struct nullb_queue),
431 GFP_KERNEL);
Jens Axboef2298c02013-10-25 11:52:25 +0100432 if (!nullb->queues)
Matias Bjorling2d263a782013-12-18 13:41:43 +0100433 return -ENOMEM;
Jens Axboef2298c02013-10-25 11:52:25 +0100434
435 nullb->nr_queues = 0;
436 nullb->queue_depth = hw_queue_depth;
437
Matias Bjorling2d263a782013-12-18 13:41:43 +0100438 return 0;
439}
440
441static int init_driver_queues(struct nullb *nullb)
442{
443 struct nullb_queue *nq;
444 int i, ret = 0;
Jens Axboef2298c02013-10-25 11:52:25 +0100445
446 for (i = 0; i < submit_queues; i++) {
447 nq = &nullb->queues[i];
Matias Bjorling2d263a782013-12-18 13:41:43 +0100448
449 null_init_queue(nullb, nq);
450
451 ret = setup_commands(nq);
452 if (ret)
Jan Kara31f96902014-10-22 15:34:21 +0200453 return ret;
Jens Axboef2298c02013-10-25 11:52:25 +0100454 nullb->nr_queues++;
455 }
Matias Bjorling2d263a782013-12-18 13:41:43 +0100456 return 0;
Jens Axboef2298c02013-10-25 11:52:25 +0100457}
458
459static int null_add_dev(void)
460{
461 struct gendisk *disk;
462 struct nullb *nullb;
463 sector_t size;
Robert Elliottdc501dc2014-09-02 11:38:49 -0500464 int rv;
Jens Axboef2298c02013-10-25 11:52:25 +0100465
466 nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, home_node);
Robert Elliottdc501dc2014-09-02 11:38:49 -0500467 if (!nullb) {
468 rv = -ENOMEM;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600469 goto out;
Robert Elliottdc501dc2014-09-02 11:38:49 -0500470 }
Jens Axboef2298c02013-10-25 11:52:25 +0100471
472 spin_lock_init(&nullb->lock);
473
Matias Bjorling57053d82013-12-10 16:50:38 +0100474 if (queue_mode == NULL_Q_MQ && use_per_node_hctx)
475 submit_queues = nr_online_nodes;
476
Robert Elliottdc501dc2014-09-02 11:38:49 -0500477 rv = setup_queues(nullb);
478 if (rv)
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600479 goto out_free_nullb;
Jens Axboef2298c02013-10-25 11:52:25 +0100480
481 if (queue_mode == NULL_Q_MQ) {
Christoph Hellwigcdef54d2014-05-28 18:11:06 +0200482 nullb->tag_set.ops = &null_mq_ops;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600483 nullb->tag_set.nr_hw_queues = submit_queues;
484 nullb->tag_set.queue_depth = hw_queue_depth;
485 nullb->tag_set.numa_node = home_node;
486 nullb->tag_set.cmd_size = sizeof(struct nullb_cmd);
487 nullb->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
488 nullb->tag_set.driver_data = nullb;
Jens Axboef2298c02013-10-25 11:52:25 +0100489
Robert Elliottdc501dc2014-09-02 11:38:49 -0500490 rv = blk_mq_alloc_tag_set(&nullb->tag_set);
491 if (rv)
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600492 goto out_cleanup_queues;
Jens Axboef2298c02013-10-25 11:52:25 +0100493
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600494 nullb->q = blk_mq_init_queue(&nullb->tag_set);
Robert Elliottdc501dc2014-09-02 11:38:49 -0500495 if (!nullb->q) {
496 rv = -ENOMEM;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600497 goto out_cleanup_tags;
Robert Elliottdc501dc2014-09-02 11:38:49 -0500498 }
Jens Axboef2298c02013-10-25 11:52:25 +0100499 } else if (queue_mode == NULL_Q_BIO) {
500 nullb->q = blk_alloc_queue_node(GFP_KERNEL, home_node);
Robert Elliottdc501dc2014-09-02 11:38:49 -0500501 if (!nullb->q) {
502 rv = -ENOMEM;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600503 goto out_cleanup_queues;
Robert Elliottdc501dc2014-09-02 11:38:49 -0500504 }
Jens Axboef2298c02013-10-25 11:52:25 +0100505 blk_queue_make_request(nullb->q, null_queue_bio);
Jan Kara31f96902014-10-22 15:34:21 +0200506 rv = init_driver_queues(nullb);
507 if (rv)
508 goto out_cleanup_blk_queue;
Jens Axboef2298c02013-10-25 11:52:25 +0100509 } else {
510 nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, home_node);
Robert Elliottdc501dc2014-09-02 11:38:49 -0500511 if (!nullb->q) {
512 rv = -ENOMEM;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600513 goto out_cleanup_queues;
Robert Elliottdc501dc2014-09-02 11:38:49 -0500514 }
Jens Axboef2298c02013-10-25 11:52:25 +0100515 blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600516 blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
Jan Kara31f96902014-10-22 15:34:21 +0200517 rv = init_driver_queues(nullb);
518 if (rv)
519 goto out_cleanup_blk_queue;
Jens Axboef2298c02013-10-25 11:52:25 +0100520 }
521
Jens Axboef2298c02013-10-25 11:52:25 +0100522 nullb->q->queuedata = nullb;
523 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q);
Mike Snitzerb277da02014-10-04 10:55:32 -0600524 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, nullb->q);
Jens Axboef2298c02013-10-25 11:52:25 +0100525
526 disk = nullb->disk = alloc_disk_node(1, home_node);
Robert Elliottdc501dc2014-09-02 11:38:49 -0500527 if (!disk) {
528 rv = -ENOMEM;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600529 goto out_cleanup_blk_queue;
Robert Elliottdc501dc2014-09-02 11:38:49 -0500530 }
Jens Axboef2298c02013-10-25 11:52:25 +0100531
532 mutex_lock(&lock);
533 list_add_tail(&nullb->list, &nullb_list);
534 nullb->index = nullb_indexes++;
535 mutex_unlock(&lock);
536
537 blk_queue_logical_block_size(nullb->q, bs);
538 blk_queue_physical_block_size(nullb->q, bs);
539
540 size = gb * 1024 * 1024 * 1024ULL;
541 sector_div(size, bs);
542 set_capacity(disk, size);
543
544 disk->flags |= GENHD_FL_EXT_DEVT;
545 disk->major = null_major;
546 disk->first_minor = nullb->index;
547 disk->fops = &null_fops;
548 disk->private_data = nullb;
549 disk->queue = nullb->q;
550 sprintf(disk->disk_name, "nullb%d", nullb->index);
551 add_disk(disk);
552 return 0;
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600553
554out_cleanup_blk_queue:
555 blk_cleanup_queue(nullb->q);
556out_cleanup_tags:
557 if (queue_mode == NULL_Q_MQ)
558 blk_mq_free_tag_set(&nullb->tag_set);
559out_cleanup_queues:
560 cleanup_queues(nullb);
561out_free_nullb:
562 kfree(nullb);
563out:
Robert Elliottdc501dc2014-09-02 11:38:49 -0500564 return rv;
Jens Axboef2298c02013-10-25 11:52:25 +0100565}
566
567static int __init null_init(void)
568{
569 unsigned int i;
570
Raghavendra K T9967d8a2014-01-21 16:59:59 +0530571 if (bs > PAGE_SIZE) {
572 pr_warn("null_blk: invalid block size\n");
573 pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
574 bs = PAGE_SIZE;
575 }
Jens Axboef2298c02013-10-25 11:52:25 +0100576
Matias Bjorlingd15ee6b2013-12-18 13:41:44 +0100577 if (queue_mode == NULL_Q_MQ && use_per_node_hctx) {
Matias Bjørlingfc1bc352013-12-21 00:11:01 +0100578 if (submit_queues < nr_online_nodes) {
Matias Bjorlingd15ee6b2013-12-18 13:41:44 +0100579 pr_warn("null_blk: submit_queues param is set to %u.",
580 nr_online_nodes);
Matias Bjørlingfc1bc352013-12-21 00:11:01 +0100581 submit_queues = nr_online_nodes;
582 }
Matias Bjorlingd15ee6b2013-12-18 13:41:44 +0100583 } else if (submit_queues > nr_cpu_ids)
Jens Axboef2298c02013-10-25 11:52:25 +0100584 submit_queues = nr_cpu_ids;
585 else if (!submit_queues)
586 submit_queues = 1;
587
588 mutex_init(&lock);
589
590 /* Initialize a separate list for each CPU for issuing softirqs */
591 for_each_possible_cpu(i) {
592 struct completion_queue *cq = &per_cpu(completion_queues, i);
593
594 init_llist_head(&cq->list);
595
596 if (irqmode != NULL_IRQ_TIMER)
597 continue;
598
599 hrtimer_init(&cq->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
600 cq->timer.function = null_cmd_timer_expired;
601 }
602
603 null_major = register_blkdev(0, "nullb");
604 if (null_major < 0)
605 return null_major;
606
607 for (i = 0; i < nr_devices; i++) {
608 if (null_add_dev()) {
609 unregister_blkdev(null_major, "nullb");
610 return -EINVAL;
611 }
612 }
613
614 pr_info("null: module loaded\n");
615 return 0;
616}
617
618static void __exit null_exit(void)
619{
620 struct nullb *nullb;
621
622 unregister_blkdev(null_major, "nullb");
623
624 mutex_lock(&lock);
625 while (!list_empty(&nullb_list)) {
626 nullb = list_entry(nullb_list.next, struct nullb, list);
627 null_del_dev(nullb);
628 }
629 mutex_unlock(&lock);
630}
631
632module_init(null_init);
633module_exit(null_exit);
634
635MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>");
636MODULE_LICENSE("GPL");