blob: e9b9c83928325fc708b72df751d51c08bf7bde82 [file] [log] [blame]
Sebastian Ottf30664e2012-08-28 16:50:38 +02001/*
2 * Block driver for s390 storage class memory.
3 *
4 * Copyright IBM Corp. 2012
5 * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
6 */
7
8#define KMSG_COMPONENT "scm_block"
9#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
11#include <linux/interrupt.h>
12#include <linux/spinlock.h>
13#include <linux/module.h>
14#include <linux/blkdev.h>
15#include <linux/genhd.h>
16#include <linux/slab.h>
17#include <linux/list.h>
18#include <asm/eadm.h>
19#include "scm_blk.h"
20
21debug_info_t *scm_debug;
22static int scm_major;
23static DEFINE_SPINLOCK(list_lock);
24static LIST_HEAD(inactive_requests);
25static unsigned int nr_requests = 64;
26static atomic_t nr_devices = ATOMIC_INIT(0);
27module_param(nr_requests, uint, S_IRUGO);
28MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
29
30MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
31MODULE_LICENSE("GPL");
32MODULE_ALIAS("scm:scmdev*");
33
34static void __scm_free_rq(struct scm_request *scmrq)
35{
36 struct aob_rq_header *aobrq = to_aobrq(scmrq);
37
38 free_page((unsigned long) scmrq->aob);
39 free_page((unsigned long) scmrq->aidaw);
Sebastian Ott0d804b22012-08-28 16:51:19 +020040 __scm_free_rq_cluster(scmrq);
Sebastian Ottf30664e2012-08-28 16:50:38 +020041 kfree(aobrq);
42}
43
44static void scm_free_rqs(void)
45{
46 struct list_head *iter, *safe;
47 struct scm_request *scmrq;
48
49 spin_lock_irq(&list_lock);
50 list_for_each_safe(iter, safe, &inactive_requests) {
51 scmrq = list_entry(iter, struct scm_request, list);
52 list_del(&scmrq->list);
53 __scm_free_rq(scmrq);
54 }
55 spin_unlock_irq(&list_lock);
56}
57
58static int __scm_alloc_rq(void)
59{
60 struct aob_rq_header *aobrq;
61 struct scm_request *scmrq;
62
63 aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
64 if (!aobrq)
65 return -ENOMEM;
66
67 scmrq = (void *) aobrq->data;
68 scmrq->aidaw = (void *) get_zeroed_page(GFP_DMA);
69 scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
70 if (!scmrq->aob || !scmrq->aidaw) {
71 __scm_free_rq(scmrq);
72 return -ENOMEM;
73 }
Sebastian Ott0d804b22012-08-28 16:51:19 +020074
75 if (__scm_alloc_rq_cluster(scmrq)) {
76 __scm_free_rq(scmrq);
77 return -ENOMEM;
78 }
79
Sebastian Ottf30664e2012-08-28 16:50:38 +020080 INIT_LIST_HEAD(&scmrq->list);
81 spin_lock_irq(&list_lock);
82 list_add(&scmrq->list, &inactive_requests);
83 spin_unlock_irq(&list_lock);
84
85 return 0;
86}
87
88static int scm_alloc_rqs(unsigned int nrqs)
89{
90 int ret = 0;
91
92 while (nrqs-- && !ret)
93 ret = __scm_alloc_rq();
94
95 return ret;
96}
97
98static struct scm_request *scm_request_fetch(void)
99{
100 struct scm_request *scmrq = NULL;
101
102 spin_lock(&list_lock);
103 if (list_empty(&inactive_requests))
104 goto out;
105 scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
106 list_del(&scmrq->list);
107out:
108 spin_unlock(&list_lock);
109 return scmrq;
110}
111
112static void scm_request_done(struct scm_request *scmrq)
113{
114 unsigned long flags;
115
116 spin_lock_irqsave(&list_lock, flags);
117 list_add(&scmrq->list, &inactive_requests);
118 spin_unlock_irqrestore(&list_lock, flags);
119}
120
121static int scm_open(struct block_device *blkdev, fmode_t mode)
122{
123 return scm_get_ref();
124}
125
126static int scm_release(struct gendisk *gendisk, fmode_t mode)
127{
128 scm_put_ref();
129 return 0;
130}
131
132static const struct block_device_operations scm_blk_devops = {
133 .owner = THIS_MODULE,
134 .open = scm_open,
135 .release = scm_release,
136};
137
Sebastian Ott4fa3c012013-02-28 12:07:48 +0100138static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
139{
140 return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
141}
142
Sebastian Ottf30664e2012-08-28 16:50:38 +0200143static void scm_request_prepare(struct scm_request *scmrq)
144{
145 struct scm_blk_dev *bdev = scmrq->bdev;
146 struct scm_device *scmdev = bdev->gendisk->private_data;
147 struct aidaw *aidaw = scmrq->aidaw;
148 struct msb *msb = &scmrq->aob->msb[0];
149 struct req_iterator iter;
150 struct bio_vec *bv;
151
152 msb->bs = MSB_BS_4K;
153 scmrq->aob->request.msb_count = 1;
154 msb->scm_addr = scmdev->address +
155 ((u64) blk_rq_pos(scmrq->request) << 9);
156 msb->oc = (rq_data_dir(scmrq->request) == READ) ?
157 MSB_OC_READ : MSB_OC_WRITE;
158 msb->flags |= MSB_FLAG_IDA;
159 msb->data_addr = (u64) aidaw;
160
161 rq_for_each_segment(bv, scmrq->request, iter) {
162 WARN_ON(bv->bv_offset);
163 msb->blk_count += bv->bv_len >> 12;
164 aidaw->data_addr = (u64) page_address(bv->bv_page);
165 aidaw++;
166 }
167}
168
169static inline void scm_request_init(struct scm_blk_dev *bdev,
170 struct scm_request *scmrq,
171 struct request *req)
172{
173 struct aob_rq_header *aobrq = to_aobrq(scmrq);
174 struct aob *aob = scmrq->aob;
175
176 memset(aob, 0, sizeof(*aob));
177 memset(scmrq->aidaw, 0, PAGE_SIZE);
178 aobrq->scmdev = bdev->scmdev;
179 aob->request.cmd_code = ARQB_CMD_MOVE;
180 aob->request.data = (u64) aobrq;
181 scmrq->request = req;
182 scmrq->bdev = bdev;
183 scmrq->retries = 4;
184 scmrq->error = 0;
Sebastian Ott0d804b22012-08-28 16:51:19 +0200185 scm_request_cluster_init(scmrq);
Sebastian Ottf30664e2012-08-28 16:50:38 +0200186}
187
188static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
189{
190 if (atomic_read(&bdev->queued_reqs)) {
191 /* Queue restart is triggered by the next interrupt. */
192 return;
193 }
194 blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
195}
196
Sebastian Ott0d804b22012-08-28 16:51:19 +0200197void scm_request_requeue(struct scm_request *scmrq)
Sebastian Ottf30664e2012-08-28 16:50:38 +0200198{
199 struct scm_blk_dev *bdev = scmrq->bdev;
200
Sebastian Ott0d804b22012-08-28 16:51:19 +0200201 scm_release_cluster(scmrq);
Sebastian Ottf30664e2012-08-28 16:50:38 +0200202 blk_requeue_request(bdev->rq, scmrq->request);
Sebastian Ott8360cb52013-02-28 12:07:27 +0100203 atomic_dec(&bdev->queued_reqs);
Sebastian Ottf30664e2012-08-28 16:50:38 +0200204 scm_request_done(scmrq);
205 scm_ensure_queue_restart(bdev);
206}
207
Sebastian Ott0d804b22012-08-28 16:51:19 +0200208void scm_request_finish(struct scm_request *scmrq)
Sebastian Ottf30664e2012-08-28 16:50:38 +0200209{
Sebastian Ott8360cb52013-02-28 12:07:27 +0100210 struct scm_blk_dev *bdev = scmrq->bdev;
211
Sebastian Ott0d804b22012-08-28 16:51:19 +0200212 scm_release_cluster(scmrq);
Sebastian Ottf30664e2012-08-28 16:50:38 +0200213 blk_end_request_all(scmrq->request, scmrq->error);
Sebastian Ott8360cb52013-02-28 12:07:27 +0100214 atomic_dec(&bdev->queued_reqs);
Sebastian Ottf30664e2012-08-28 16:50:38 +0200215 scm_request_done(scmrq);
216}
217
218static void scm_blk_request(struct request_queue *rq)
219{
220 struct scm_device *scmdev = rq->queuedata;
221 struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
222 struct scm_request *scmrq;
223 struct request *req;
224 int ret;
225
226 while ((req = blk_peek_request(rq))) {
227 if (req->cmd_type != REQ_TYPE_FS)
228 continue;
229
Sebastian Ott4fa3c012013-02-28 12:07:48 +0100230 if (!scm_permit_request(bdev, req)) {
231 scm_ensure_queue_restart(bdev);
232 return;
233 }
Sebastian Ottf30664e2012-08-28 16:50:38 +0200234 scmrq = scm_request_fetch();
235 if (!scmrq) {
236 SCM_LOG(5, "no request");
237 scm_ensure_queue_restart(bdev);
238 return;
239 }
240 scm_request_init(bdev, scmrq, req);
Sebastian Ott0d804b22012-08-28 16:51:19 +0200241 if (!scm_reserve_cluster(scmrq)) {
242 SCM_LOG(5, "cluster busy");
243 scm_request_done(scmrq);
244 return;
245 }
246 if (scm_need_cluster_request(scmrq)) {
Sebastian Ott8360cb52013-02-28 12:07:27 +0100247 atomic_inc(&bdev->queued_reqs);
Sebastian Ott0d804b22012-08-28 16:51:19 +0200248 blk_start_request(req);
249 scm_initiate_cluster_request(scmrq);
250 return;
251 }
Sebastian Ottf30664e2012-08-28 16:50:38 +0200252 scm_request_prepare(scmrq);
Sebastian Ott8360cb52013-02-28 12:07:27 +0100253 atomic_inc(&bdev->queued_reqs);
Sebastian Ottf30664e2012-08-28 16:50:38 +0200254 blk_start_request(req);
255
256 ret = scm_start_aob(scmrq->aob);
257 if (ret) {
258 SCM_LOG(5, "no subchannel");
259 scm_request_requeue(scmrq);
260 return;
261 }
Sebastian Ottf30664e2012-08-28 16:50:38 +0200262 }
263}
264
265static void __scmrq_log_error(struct scm_request *scmrq)
266{
267 struct aob *aob = scmrq->aob;
268
269 if (scmrq->error == -ETIMEDOUT)
270 SCM_LOG(1, "Request timeout");
271 else {
272 SCM_LOG(1, "Request error");
273 SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
274 }
275 if (scmrq->retries)
276 SCM_LOG(1, "Retry request");
277 else
278 pr_err("An I/O operation to SCM failed with rc=%d\n",
279 scmrq->error);
280}
281
282void scm_blk_irq(struct scm_device *scmdev, void *data, int error)
283{
284 struct scm_request *scmrq = data;
285 struct scm_blk_dev *bdev = scmrq->bdev;
286
287 scmrq->error = error;
288 if (error)
289 __scmrq_log_error(scmrq);
290
291 spin_lock(&bdev->lock);
292 list_add_tail(&scmrq->list, &bdev->finished_requests);
293 spin_unlock(&bdev->lock);
294 tasklet_hi_schedule(&bdev->tasklet);
295}
296
Sebastian Ott4fa3c012013-02-28 12:07:48 +0100297static void scm_blk_handle_error(struct scm_request *scmrq)
298{
299 struct scm_blk_dev *bdev = scmrq->bdev;
300 unsigned long flags;
301
302 if (scmrq->error != -EIO)
303 goto restart;
304
305 /* For -EIO the response block is valid. */
306 switch (scmrq->aob->response.eqc) {
307 case EQC_WR_PROHIBIT:
308 spin_lock_irqsave(&bdev->lock, flags);
309 if (bdev->state != SCM_WR_PROHIBIT)
Sebastian Ott3bff6032013-03-18 16:01:30 +0100310 pr_info("%lx: Write access to the SCM increment is suspended\n",
Sebastian Ott4fa3c012013-02-28 12:07:48 +0100311 (unsigned long) bdev->scmdev->address);
312 bdev->state = SCM_WR_PROHIBIT;
313 spin_unlock_irqrestore(&bdev->lock, flags);
314 goto requeue;
315 default:
316 break;
317 }
318
319restart:
320 if (!scm_start_aob(scmrq->aob))
321 return;
322
323requeue:
324 spin_lock_irqsave(&bdev->rq_lock, flags);
325 scm_request_requeue(scmrq);
326 spin_unlock_irqrestore(&bdev->rq_lock, flags);
327}
328
Sebastian Ottf30664e2012-08-28 16:50:38 +0200329static void scm_blk_tasklet(struct scm_blk_dev *bdev)
330{
331 struct scm_request *scmrq;
332 unsigned long flags;
333
334 spin_lock_irqsave(&bdev->lock, flags);
335 while (!list_empty(&bdev->finished_requests)) {
336 scmrq = list_first_entry(&bdev->finished_requests,
337 struct scm_request, list);
338 list_del(&scmrq->list);
339 spin_unlock_irqrestore(&bdev->lock, flags);
340
341 if (scmrq->error && scmrq->retries-- > 0) {
Sebastian Ott4fa3c012013-02-28 12:07:48 +0100342 scm_blk_handle_error(scmrq);
343
Sebastian Ottf30664e2012-08-28 16:50:38 +0200344 /* Request restarted or requeued, handle next. */
345 spin_lock_irqsave(&bdev->lock, flags);
346 continue;
347 }
Sebastian Ott0d804b22012-08-28 16:51:19 +0200348
349 if (scm_test_cluster_request(scmrq)) {
350 scm_cluster_request_irq(scmrq);
351 spin_lock_irqsave(&bdev->lock, flags);
352 continue;
353 }
354
Sebastian Ottf30664e2012-08-28 16:50:38 +0200355 scm_request_finish(scmrq);
Sebastian Ottf30664e2012-08-28 16:50:38 +0200356 spin_lock_irqsave(&bdev->lock, flags);
357 }
358 spin_unlock_irqrestore(&bdev->lock, flags);
359 /* Look out for more requests. */
360 blk_run_queue(bdev->rq);
361}
362
363int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
364{
365 struct request_queue *rq;
366 int len, ret = -ENOMEM;
367 unsigned int devindex, nr_max_blk;
368
369 devindex = atomic_inc_return(&nr_devices) - 1;
370 /* scma..scmz + scmaa..scmzz */
371 if (devindex > 701) {
372 ret = -ENODEV;
373 goto out;
374 }
375
376 bdev->scmdev = scmdev;
Sebastian Ott4fa3c012013-02-28 12:07:48 +0100377 bdev->state = SCM_OPER;
Sebastian Ottf30664e2012-08-28 16:50:38 +0200378 spin_lock_init(&bdev->rq_lock);
379 spin_lock_init(&bdev->lock);
380 INIT_LIST_HEAD(&bdev->finished_requests);
381 atomic_set(&bdev->queued_reqs, 0);
382 tasklet_init(&bdev->tasklet,
383 (void (*)(unsigned long)) scm_blk_tasklet,
384 (unsigned long) bdev);
385
386 rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
387 if (!rq)
388 goto out;
389
390 bdev->rq = rq;
391 nr_max_blk = min(scmdev->nr_max_block,
392 (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
393
394 blk_queue_logical_block_size(rq, 1 << 12);
395 blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
396 blk_queue_max_segments(rq, nr_max_blk);
397 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
Sebastian Ott0d804b22012-08-28 16:51:19 +0200398 scm_blk_dev_cluster_setup(bdev);
Sebastian Ottf30664e2012-08-28 16:50:38 +0200399
400 bdev->gendisk = alloc_disk(SCM_NR_PARTS);
401 if (!bdev->gendisk)
402 goto out_queue;
403
404 rq->queuedata = scmdev;
405 bdev->gendisk->driverfs_dev = &scmdev->dev;
406 bdev->gendisk->private_data = scmdev;
407 bdev->gendisk->fops = &scm_blk_devops;
408 bdev->gendisk->queue = rq;
409 bdev->gendisk->major = scm_major;
410 bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
411
412 len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
413 if (devindex > 25) {
414 len += snprintf(bdev->gendisk->disk_name + len,
415 DISK_NAME_LEN - len, "%c",
416 'a' + (devindex / 26) - 1);
417 devindex = devindex % 26;
418 }
419 snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
420 'a' + devindex);
421
422 /* 512 byte sectors */
423 set_capacity(bdev->gendisk, scmdev->size >> 9);
424 add_disk(bdev->gendisk);
425 return 0;
426
427out_queue:
428 blk_cleanup_queue(rq);
429out:
430 atomic_dec(&nr_devices);
431 return ret;
432}
433
434void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
435{
436 tasklet_kill(&bdev->tasklet);
437 del_gendisk(bdev->gendisk);
438 blk_cleanup_queue(bdev->gendisk->queue);
439 put_disk(bdev->gendisk);
440}
441
Sebastian Ott4fa3c012013-02-28 12:07:48 +0100442void scm_blk_set_available(struct scm_blk_dev *bdev)
443{
444 unsigned long flags;
445
446 spin_lock_irqsave(&bdev->lock, flags);
447 if (bdev->state == SCM_WR_PROHIBIT)
Sebastian Ott3bff6032013-03-18 16:01:30 +0100448 pr_info("%lx: Write access to the SCM increment is restored\n",
Sebastian Ott4fa3c012013-02-28 12:07:48 +0100449 (unsigned long) bdev->scmdev->address);
450 bdev->state = SCM_OPER;
451 spin_unlock_irqrestore(&bdev->lock, flags);
452}
453
Sebastian Ottf30664e2012-08-28 16:50:38 +0200454static int __init scm_blk_init(void)
455{
Sebastian Ott0d804b22012-08-28 16:51:19 +0200456 int ret = -EINVAL;
457
458 if (!scm_cluster_size_valid())
459 goto out;
Sebastian Ottf30664e2012-08-28 16:50:38 +0200460
461 ret = register_blkdev(0, "scm");
462 if (ret < 0)
463 goto out;
464
465 scm_major = ret;
Wei Yongjun94f98522013-03-20 13:40:54 +0100466 ret = scm_alloc_rqs(nr_requests);
467 if (ret)
Sebastian Ottf30664e2012-08-28 16:50:38 +0200468 goto out_unreg;
469
470 scm_debug = debug_register("scm_log", 16, 1, 16);
Wei Yongjun94f98522013-03-20 13:40:54 +0100471 if (!scm_debug) {
472 ret = -ENOMEM;
Sebastian Ottf30664e2012-08-28 16:50:38 +0200473 goto out_free;
Wei Yongjun94f98522013-03-20 13:40:54 +0100474 }
Sebastian Ottf30664e2012-08-28 16:50:38 +0200475
476 debug_register_view(scm_debug, &debug_hex_ascii_view);
477 debug_set_level(scm_debug, 2);
478
479 ret = scm_drv_init();
480 if (ret)
481 goto out_dbf;
482
483 return ret;
484
485out_dbf:
486 debug_unregister(scm_debug);
487out_free:
488 scm_free_rqs();
489out_unreg:
490 unregister_blkdev(scm_major, "scm");
491out:
492 return ret;
493}
494module_init(scm_blk_init);
495
496static void __exit scm_blk_cleanup(void)
497{
498 scm_drv_cleanup();
499 debug_unregister(scm_debug);
500 scm_free_rqs();
501 unregister_blkdev(scm_major, "scm");
502}
503module_exit(scm_blk_cleanup);