blob: 372369b1cc2068c286964883638c2b3a9f2c11c4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07003 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
9#include "dm-bio-list.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +010010#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
12#include <linux/init.h>
13#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080014#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/moduleparam.h>
16#include <linux/blkpg.h>
17#include <linux/bio.h>
18#include <linux/buffer_head.h>
19#include <linux/mempool.h>
20#include <linux/slab.h>
21#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080022#include <linux/hdreg.h>
Jens Axboe2056a782006-03-23 20:00:26 +010023#include <linux/blktrace_api.h>
Milan Brozaa129a22006-10-03 01:15:15 -070024#include <linux/smp_lock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Alasdair G Kergon72d94862006-06-26 00:27:35 -070026#define DM_MSG_PREFIX "core"
27
Linus Torvalds1da177e2005-04-16 15:20:36 -070028static const char *_name = DM_NAME;
29
30static unsigned int major = 0;
31static unsigned int _major = 0;
32
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070033static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070034/*
35 * One of these is allocated per bio.
36 */
37struct dm_io {
38 struct mapped_device *md;
39 int error;
40 struct bio *bio;
41 atomic_t io_count;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080042 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070043};
44
45/*
46 * One of these is allocated per target within a bio. Hopefully
47 * this will be simplified out one day.
48 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +010049struct dm_target_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 struct dm_io *io;
51 struct dm_target *ti;
52 union map_info info;
53};
54
55union map_info *dm_get_mapinfo(struct bio *bio)
56{
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070057 if (bio && bio->bi_private)
Alasdair G Kergon028867a2007-07-12 17:26:32 +010058 return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070059 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070060}
61
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070062#define MINOR_ALLOCED ((void *)-1)
63
Linus Torvalds1da177e2005-04-16 15:20:36 -070064/*
65 * Bits for the md->flags field.
66 */
67#define DMF_BLOCK_IO 0
68#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080069#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -070070#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070071#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080072#define DMF_NOFLUSH_SUSPENDING 5
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Milan Broz304f3f62008-02-08 02:11:17 +000074/*
75 * Work processed by per-device workqueue.
76 */
77struct dm_wq_req {
78 enum {
79 DM_WQ_FLUSH_ALL,
80 DM_WQ_FLUSH_DEFERRED,
81 } type;
82 struct work_struct work;
83 struct mapped_device *md;
84 void *context;
85};
86
Linus Torvalds1da177e2005-04-16 15:20:36 -070087struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -070088 struct rw_semaphore io_lock;
Daniel Walkere61290a2008-02-08 02:10:08 +000089 struct mutex suspend_lock;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080090 spinlock_t pushback_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -070091 rwlock_t map_lock;
92 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070093 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070094
95 unsigned long flags;
96
Jens Axboe165125e2007-07-24 09:28:11 +020097 struct request_queue *queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -080099 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100
101 void *interface_ptr;
102
103 /*
104 * A list of ios that arrived while we were suspended.
105 */
106 atomic_t pending;
107 wait_queue_head_t wait;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800108 struct bio_list deferred;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800109 struct bio_list pushback;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110
111 /*
Milan Broz304f3f62008-02-08 02:11:17 +0000112 * Processing queue (flush/barriers)
113 */
114 struct workqueue_struct *wq;
115
116 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 * The current mapping.
118 */
119 struct dm_table *map;
120
121 /*
122 * io objects are allocated from here.
123 */
124 mempool_t *io_pool;
125 mempool_t *tio_pool;
126
Stefan Bader9faf4002006-10-03 01:15:41 -0700127 struct bio_set *bs;
128
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129 /*
130 * Event handling.
131 */
132 atomic_t event_nr;
133 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100134 atomic_t uevent_seq;
135 struct list_head uevent_list;
136 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
138 /*
139 * freeze/thaw support require holding onto a super block
140 */
141 struct super_block *frozen_sb;
Alasdair G Kergone39e2e92006-01-06 00:20:05 -0800142 struct block_device *suspended_bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800143
144 /* forced geometry settings */
145 struct hd_geometry geometry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146};
147
148#define MIN_IOS 256
Christoph Lametere18b8902006-12-06 20:33:20 -0800149static struct kmem_cache *_io_cache;
150static struct kmem_cache *_tio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152static int __init local_init(void)
153{
154 int r;
155
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100157 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 if (!_io_cache)
159 return -ENOMEM;
160
161 /* allocate a slab for the target ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100162 _tio_cache = KMEM_CACHE(dm_target_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 if (!_tio_cache) {
164 kmem_cache_destroy(_io_cache);
165 return -ENOMEM;
166 }
167
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100168 r = dm_uevent_init();
169 if (r) {
170 kmem_cache_destroy(_tio_cache);
171 kmem_cache_destroy(_io_cache);
172 return r;
173 }
174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 _major = major;
176 r = register_blkdev(_major, _name);
177 if (r < 0) {
178 kmem_cache_destroy(_tio_cache);
179 kmem_cache_destroy(_io_cache);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100180 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 return r;
182 }
183
184 if (!_major)
185 _major = r;
186
187 return 0;
188}
189
190static void local_exit(void)
191{
192 kmem_cache_destroy(_tio_cache);
193 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700194 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100195 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
197 _major = 0;
198
199 DMINFO("cleaned up");
200}
201
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000202static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203 local_init,
204 dm_target_init,
205 dm_linear_init,
206 dm_stripe_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100207 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 dm_interface_init,
209};
210
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000211static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 local_exit,
213 dm_target_exit,
214 dm_linear_exit,
215 dm_stripe_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100216 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 dm_interface_exit,
218};
219
220static int __init dm_init(void)
221{
222 const int count = ARRAY_SIZE(_inits);
223
224 int r, i;
225
226 for (i = 0; i < count; i++) {
227 r = _inits[i]();
228 if (r)
229 goto bad;
230 }
231
232 return 0;
233
234 bad:
235 while (i--)
236 _exits[i]();
237
238 return r;
239}
240
241static void __exit dm_exit(void)
242{
243 int i = ARRAY_SIZE(_exits);
244
245 while (i--)
246 _exits[i]();
247}
248
249/*
250 * Block device functions
251 */
252static int dm_blk_open(struct inode *inode, struct file *file)
253{
254 struct mapped_device *md;
255
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700256 spin_lock(&_minor_lock);
257
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 md = inode->i_bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700259 if (!md)
260 goto out;
261
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700262 if (test_bit(DMF_FREEING, &md->flags) ||
263 test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700264 md = NULL;
265 goto out;
266 }
267
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700269 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700270
271out:
272 spin_unlock(&_minor_lock);
273
274 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275}
276
277static int dm_blk_close(struct inode *inode, struct file *file)
278{
279 struct mapped_device *md;
280
281 md = inode->i_bdev->bd_disk->private_data;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700282 atomic_dec(&md->open_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 dm_put(md);
284 return 0;
285}
286
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700287int dm_open_count(struct mapped_device *md)
288{
289 return atomic_read(&md->open_count);
290}
291
292/*
293 * Guarantees nothing is using the device before it's deleted.
294 */
295int dm_lock_for_deletion(struct mapped_device *md)
296{
297 int r = 0;
298
299 spin_lock(&_minor_lock);
300
301 if (dm_open_count(md))
302 r = -EBUSY;
303 else
304 set_bit(DMF_DELETING, &md->flags);
305
306 spin_unlock(&_minor_lock);
307
308 return r;
309}
310
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800311static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
312{
313 struct mapped_device *md = bdev->bd_disk->private_data;
314
315 return dm_get_geometry(md, geo);
316}
317
Milan Brozaa129a22006-10-03 01:15:15 -0700318static int dm_blk_ioctl(struct inode *inode, struct file *file,
319 unsigned int cmd, unsigned long arg)
320{
321 struct mapped_device *md;
322 struct dm_table *map;
323 struct dm_target *tgt;
324 int r = -ENOTTY;
325
326 /* We don't really need this lock, but we do need 'inode'. */
327 unlock_kernel();
328
329 md = inode->i_bdev->bd_disk->private_data;
330
331 map = dm_get_table(md);
332
333 if (!map || !dm_table_get_size(map))
334 goto out;
335
336 /* We only support devices that have a single target */
337 if (dm_table_get_num_targets(map) != 1)
338 goto out;
339
340 tgt = dm_table_get_target(map, 0);
341
342 if (dm_suspended(md)) {
343 r = -EAGAIN;
344 goto out;
345 }
346
347 if (tgt->type->ioctl)
348 r = tgt->type->ioctl(tgt, inode, file, cmd, arg);
349
350out:
351 dm_table_put(map);
352
353 lock_kernel();
354 return r;
355}
356
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100357static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358{
359 return mempool_alloc(md->io_pool, GFP_NOIO);
360}
361
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100362static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363{
364 mempool_free(io, md->io_pool);
365}
366
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100367static struct dm_target_io *alloc_tio(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368{
369 return mempool_alloc(md->tio_pool, GFP_NOIO);
370}
371
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100372static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373{
374 mempool_free(tio, md->tio_pool);
375}
376
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800377static void start_io_acct(struct dm_io *io)
378{
379 struct mapped_device *md = io->md;
380
381 io->start_time = jiffies;
382
383 preempt_disable();
384 disk_round_stats(dm_disk(md));
385 preempt_enable();
386 dm_disk(md)->in_flight = atomic_inc_return(&md->pending);
387}
388
389static int end_io_acct(struct dm_io *io)
390{
391 struct mapped_device *md = io->md;
392 struct bio *bio = io->bio;
393 unsigned long duration = jiffies - io->start_time;
394 int pending;
395 int rw = bio_data_dir(bio);
396
397 preempt_disable();
398 disk_round_stats(dm_disk(md));
399 preempt_enable();
400 dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending);
401
402 disk_stat_add(dm_disk(md), ticks[rw], duration);
403
404 return !pending;
405}
406
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407/*
408 * Add the bio to the list of deferred io.
409 */
410static int queue_io(struct mapped_device *md, struct bio *bio)
411{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700412 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
414 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700415 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 return 1;
417 }
418
419 bio_list_add(&md->deferred, bio);
420
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700421 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 return 0; /* deferred successfully */
423}
424
425/*
426 * Everyone (including functions in this file), should use this
427 * function to access the md->map field, and make sure they call
428 * dm_table_put() when finished.
429 */
430struct dm_table *dm_get_table(struct mapped_device *md)
431{
432 struct dm_table *t;
433
434 read_lock(&md->map_lock);
435 t = md->map;
436 if (t)
437 dm_table_get(t);
438 read_unlock(&md->map_lock);
439
440 return t;
441}
442
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800443/*
444 * Get the geometry associated with a dm device
445 */
446int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
447{
448 *geo = md->geometry;
449
450 return 0;
451}
452
453/*
454 * Set the geometry of a device.
455 */
456int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
457{
458 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
459
460 if (geo->start > sz) {
461 DMWARN("Start sector is beyond the geometry limits.");
462 return -EINVAL;
463 }
464
465 md->geometry = *geo;
466
467 return 0;
468}
469
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470/*-----------------------------------------------------------------
471 * CRUD START:
472 * A more elegant soln is in the works that uses the queue
473 * merge fn, unfortunately there are a couple of changes to
474 * the block layer that I want to make for this. So in the
475 * interests of getting something for people to use I give
476 * you this clearly demarcated crap.
477 *---------------------------------------------------------------*/
478
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800479static int __noflush_suspending(struct mapped_device *md)
480{
481 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
482}
483
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484/*
485 * Decrements the number of outstanding ios that a bio has been
486 * cloned into, completing the original io if necc.
487 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800488static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800490 unsigned long flags;
491
492 /* Push-back supersedes any I/O errors */
493 if (error && !(io->error > 0 && __noflush_suspending(io->md)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 io->error = error;
495
496 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800497 if (io->error == DM_ENDIO_REQUEUE) {
498 /*
499 * Target requested pushing back the I/O.
500 * This must be handled before the sleeper on
501 * suspend queue merges the pushback list.
502 */
503 spin_lock_irqsave(&io->md->pushback_lock, flags);
504 if (__noflush_suspending(io->md))
505 bio_list_add(&io->md->pushback, io->bio);
506 else
507 /* noflush suspend was interrupted. */
508 io->error = -EIO;
509 spin_unlock_irqrestore(&io->md->pushback_lock, flags);
510 }
511
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800512 if (end_io_acct(io))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 /* nudge anyone waiting on suspend queue */
514 wake_up(&io->md->wait);
515
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800516 if (io->error != DM_ENDIO_REQUEUE) {
517 blk_add_trace_bio(io->md->queue, io->bio,
518 BLK_TA_COMPLETE);
Jens Axboe2056a782006-03-23 20:00:26 +0100519
NeilBrown6712ecf2007-09-27 12:47:43 +0200520 bio_endio(io->bio, io->error);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800521 }
522
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 free_io(io->md, io);
524 }
525}
526
NeilBrown6712ecf2007-09-27 12:47:43 +0200527static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528{
529 int r = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100530 struct dm_target_io *tio = bio->bi_private;
Stefan Bader9faf4002006-10-03 01:15:41 -0700531 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 dm_endio_fn endio = tio->ti->type->end_io;
533
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
535 error = -EIO;
536
537 if (endio) {
538 r = endio(tio->ti, bio, error, &tio->info);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800539 if (r < 0 || r == DM_ENDIO_REQUEUE)
540 /*
541 * error and requeue request are handled
542 * in dec_pending().
543 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800545 else if (r == DM_ENDIO_INCOMPLETE)
546 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200547 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800548 else if (r) {
549 DMWARN("unimplemented target endio return value: %d", r);
550 BUG();
551 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 }
553
Stefan Bader9faf4002006-10-03 01:15:41 -0700554 dec_pending(tio->io, error);
555
556 /*
557 * Store md for cleanup instead of tio which is about to get freed.
558 */
559 bio->bi_private = md->bs;
560
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 bio_put(bio);
Stefan Bader9faf4002006-10-03 01:15:41 -0700562 free_tio(md, tio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563}
564
565static sector_t max_io_len(struct mapped_device *md,
566 sector_t sector, struct dm_target *ti)
567{
568 sector_t offset = sector - ti->begin;
569 sector_t len = ti->len - offset;
570
571 /*
572 * Does the target need to split even further ?
573 */
574 if (ti->split_io) {
575 sector_t boundary;
576 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
577 - offset;
578 if (len > boundary)
579 len = boundary;
580 }
581
582 return len;
583}
584
585static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100586 struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587{
588 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100589 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -0700590 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591
592 /*
593 * Sanity checks.
594 */
595 BUG_ON(!clone->bi_size);
596
597 clone->bi_end_io = clone_endio;
598 clone->bi_private = tio;
599
600 /*
601 * Map the clone. If r == 0 we don't need to do
602 * anything, the target has assumed ownership of
603 * this io.
604 */
605 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100606 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 r = ti->type->map(ti, clone, &tio->info);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800608 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100610
Alasdair G Kergon17b2f662006-06-26 00:27:33 -0700611 blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
Alan D. Brunellec7149d62007-08-07 15:30:23 +0200612 tio->io->bio->bi_bdev->bd_dev,
613 clone->bi_sector, sector);
Jens Axboe2056a782006-03-23 20:00:26 +0100614
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800616 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
617 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -0700618 md = tio->io->md;
619 dec_pending(tio->io, r);
620 /*
621 * Store bio_set for cleanup.
622 */
623 clone->bi_private = md->bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 bio_put(clone);
Stefan Bader9faf4002006-10-03 01:15:41 -0700625 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800626 } else if (r) {
627 DMWARN("unimplemented target map return value: %d", r);
628 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 }
630}
631
632struct clone_info {
633 struct mapped_device *md;
634 struct dm_table *map;
635 struct bio *bio;
636 struct dm_io *io;
637 sector_t sector;
638 sector_t sector_count;
639 unsigned short idx;
640};
641
Peter Osterlund36763472005-09-06 15:16:42 -0700642static void dm_bio_destructor(struct bio *bio)
643{
Stefan Bader9faf4002006-10-03 01:15:41 -0700644 struct bio_set *bs = bio->bi_private;
645
646 bio_free(bio, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700647}
648
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649/*
650 * Creates a little bio that is just does part of a bvec.
651 */
652static struct bio *split_bvec(struct bio *bio, sector_t sector,
653 unsigned short idx, unsigned int offset,
Stefan Bader9faf4002006-10-03 01:15:41 -0700654 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655{
656 struct bio *clone;
657 struct bio_vec *bv = bio->bi_io_vec + idx;
658
Stefan Bader9faf4002006-10-03 01:15:41 -0700659 clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700660 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 *clone->bi_io_vec = *bv;
662
663 clone->bi_sector = sector;
664 clone->bi_bdev = bio->bi_bdev;
665 clone->bi_rw = bio->bi_rw;
666 clone->bi_vcnt = 1;
667 clone->bi_size = to_bytes(len);
668 clone->bi_io_vec->bv_offset = offset;
669 clone->bi_io_vec->bv_len = clone->bi_size;
670
671 return clone;
672}
673
674/*
675 * Creates a bio that consists of range of complete bvecs.
676 */
677static struct bio *clone_bio(struct bio *bio, sector_t sector,
678 unsigned short idx, unsigned short bv_count,
Stefan Bader9faf4002006-10-03 01:15:41 -0700679 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680{
681 struct bio *clone;
682
Stefan Bader9faf4002006-10-03 01:15:41 -0700683 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
684 __bio_clone(clone, bio);
685 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 clone->bi_sector = sector;
687 clone->bi_idx = idx;
688 clone->bi_vcnt = idx + bv_count;
689 clone->bi_size = to_bytes(len);
690 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
691
692 return clone;
693}
694
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000695static int __clone_and_map(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696{
697 struct bio *clone, *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000698 struct dm_target *ti;
699 sector_t len = 0, max;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100700 struct dm_target_io *tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000702 ti = dm_table_find_target(ci->map, ci->sector);
703 if (!dm_target_is_valid(ti))
704 return -EIO;
705
706 max = max_io_len(ci->md, ci->sector, ti);
707
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 /*
709 * Allocate a target io object.
710 */
711 tio = alloc_tio(ci->md);
712 tio->io = ci->io;
713 tio->ti = ti;
714 memset(&tio->info, 0, sizeof(tio->info));
715
716 if (ci->sector_count <= max) {
717 /*
718 * Optimise for the simple case where we can do all of
719 * the remaining io with a single clone.
720 */
721 clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700722 bio->bi_vcnt - ci->idx, ci->sector_count,
723 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 __map_bio(ti, clone, tio);
725 ci->sector_count = 0;
726
727 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
728 /*
729 * There are some bvecs that don't span targets.
730 * Do as many of these as possible.
731 */
732 int i;
733 sector_t remaining = max;
734 sector_t bv_len;
735
736 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
737 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
738
739 if (bv_len > remaining)
740 break;
741
742 remaining -= bv_len;
743 len += bv_len;
744 }
745
Stefan Bader9faf4002006-10-03 01:15:41 -0700746 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
747 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 __map_bio(ti, clone, tio);
749
750 ci->sector += len;
751 ci->sector_count -= len;
752 ci->idx = i;
753
754 } else {
755 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800756 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 */
758 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800759 sector_t remaining = to_sector(bv->bv_len);
760 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800762 do {
763 if (offset) {
764 ti = dm_table_find_target(ci->map, ci->sector);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000765 if (!dm_target_is_valid(ti))
766 return -EIO;
767
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800768 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800770 tio = alloc_tio(ci->md);
771 tio->io = ci->io;
772 tio->ti = ti;
773 memset(&tio->info, 0, sizeof(tio->info));
774 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800776 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800778 clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700779 bv->bv_offset + offset, len,
780 ci->md->bs);
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800781
782 __map_bio(ti, clone, tio);
783
784 ci->sector += len;
785 ci->sector_count -= len;
786 offset += to_bytes(len);
787 } while (remaining -= len);
788
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 ci->idx++;
790 }
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000791
792 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793}
794
795/*
796 * Split the bio into several clones.
797 */
Milan Broz9e4e5f82007-10-19 22:38:53 +0100798static int __split_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799{
800 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000801 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
803 ci.map = dm_get_table(md);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100804 if (unlikely(!ci.map))
805 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806
807 ci.md = md;
808 ci.bio = bio;
809 ci.io = alloc_io(md);
810 ci.io->error = 0;
811 atomic_set(&ci.io->io_count, 1);
812 ci.io->bio = bio;
813 ci.io->md = md;
814 ci.sector = bio->bi_sector;
815 ci.sector_count = bio_sectors(bio);
816 ci.idx = bio->bi_idx;
817
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800818 start_io_acct(ci.io);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000819 while (ci.sector_count && !error)
820 error = __clone_and_map(&ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821
822 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000823 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824 dm_table_put(ci.map);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100825
826 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827}
828/*-----------------------------------------------------------------
829 * CRUD END
830 *---------------------------------------------------------------*/
831
832/*
833 * The request function that just remaps the bio built up by
834 * dm_merge_bvec.
835 */
Jens Axboe165125e2007-07-24 09:28:11 +0200836static int dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837{
Milan Broz9e4e5f82007-10-19 22:38:53 +0100838 int r = -EIO;
Kevin Corry12f03a42006-02-01 03:04:52 -0800839 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 struct mapped_device *md = q->queuedata;
841
Stefan Bader07a83c42007-07-12 17:28:33 +0100842 /*
843 * There is no use in forwarding any barrier request since we can't
844 * guarantee it is (or can be) handled by the targets correctly.
845 */
846 if (unlikely(bio_barrier(bio))) {
NeilBrown6712ecf2007-09-27 12:47:43 +0200847 bio_endio(bio, -EOPNOTSUPP);
Stefan Bader07a83c42007-07-12 17:28:33 +0100848 return 0;
849 }
850
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700851 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852
Kevin Corry12f03a42006-02-01 03:04:52 -0800853 disk_stat_inc(dm_disk(md), ios[rw]);
854 disk_stat_add(dm_disk(md), sectors[rw], bio_sectors(bio));
855
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 /*
857 * If we're suspended we have to queue
858 * this io for later.
859 */
860 while (test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700861 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862
Milan Broz9e4e5f82007-10-19 22:38:53 +0100863 if (bio_rw(bio) != READA)
864 r = queue_io(md, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865
Milan Broz9e4e5f82007-10-19 22:38:53 +0100866 if (r <= 0)
867 goto out_req;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868
869 /*
870 * We're in a while loop, because someone could suspend
871 * before we get to the following read lock.
872 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700873 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 }
875
Milan Broz9e4e5f82007-10-19 22:38:53 +0100876 r = __split_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700877 up_read(&md->io_lock);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100878
879out_req:
880 if (r < 0)
881 bio_io_error(bio);
882
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 return 0;
884}
885
Jens Axboe165125e2007-07-24 09:28:11 +0200886static void dm_unplug_all(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887{
888 struct mapped_device *md = q->queuedata;
889 struct dm_table *map = dm_get_table(md);
890
891 if (map) {
892 dm_table_unplug_all(map);
893 dm_table_put(map);
894 }
895}
896
897static int dm_any_congested(void *congested_data, int bdi_bits)
898{
899 int r;
900 struct mapped_device *md = (struct mapped_device *) congested_data;
901 struct dm_table *map = dm_get_table(md);
902
903 if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
904 r = bdi_bits;
905 else
906 r = dm_table_any_congested(map, bdi_bits);
907
908 dm_table_put(map);
909 return r;
910}
911
912/*-----------------------------------------------------------------
913 * An IDR is used to keep track of allocated minor numbers.
914 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915static DEFINE_IDR(_minor_idr);
916
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -0700917static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700919 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700921 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922}
923
924/*
925 * See if the device with a specific minor # is free.
926 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +0100927static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928{
929 int r, m;
930
931 if (minor >= (1 << MINORBITS))
932 return -EINVAL;
933
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700934 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
935 if (!r)
936 return -ENOMEM;
937
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700938 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939
940 if (idr_find(&_minor_idr, minor)) {
941 r = -EBUSY;
942 goto out;
943 }
944
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700945 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700946 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948
949 if (m != minor) {
950 idr_remove(&_minor_idr, m);
951 r = -EBUSY;
952 goto out;
953 }
954
955out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700956 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957 return r;
958}
959
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +0100960static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -0700962 int r, m;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700965 if (!r)
966 return -ENOMEM;
967
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700968 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700970 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +0100971 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973
974 if (m >= (1 << MINORBITS)) {
975 idr_remove(&_minor_idr, m);
976 r = -ENOSPC;
977 goto out;
978 }
979
980 *minor = m;
981
982out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700983 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984 return r;
985}
986
987static struct block_device_operations dm_blk_dops;
988
989/*
990 * Allocate and initialise a blank device with a given minor.
991 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -0700992static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993{
994 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +0100995 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700996 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997
998 if (!md) {
999 DMWARN("unable to allocate device, out of memory.");
1000 return NULL;
1001 }
1002
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001003 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001004 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001005
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001007 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001008 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001009 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001010 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001012 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001014 init_rwsem(&md->io_lock);
Daniel Walkere61290a2008-02-08 02:10:08 +00001015 mutex_init(&md->suspend_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001016 spin_lock_init(&md->pushback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 rwlock_init(&md->map_lock);
1018 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001019 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001021 atomic_set(&md->uevent_seq, 0);
1022 INIT_LIST_HEAD(&md->uevent_list);
1023 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024
1025 md->queue = blk_alloc_queue(GFP_KERNEL);
1026 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001027 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028
1029 md->queue->queuedata = md;
1030 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1031 md->queue->backing_dev_info.congested_data = md;
1032 blk_queue_make_request(md->queue, dm_request);
Jens Axboedaef2652006-01-10 10:48:02 +01001033 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034 md->queue->unplug_fn = dm_unplug_all;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035
Matthew Dobson93d23412006-03-26 01:37:50 -08001036 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
Kiyoshi Ueda74859362006-12-08 02:41:02 -08001037 if (!md->io_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001038 goto bad_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039
Matthew Dobson93d23412006-03-26 01:37:50 -08001040 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041 if (!md->tio_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001042 goto bad_tio_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043
Jens Axboe59725112007-04-02 10:06:42 +02001044 md->bs = bioset_create(16, 16);
Stefan Bader9faf4002006-10-03 01:15:41 -07001045 if (!md->bs)
1046 goto bad_no_bioset;
1047
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 md->disk = alloc_disk(1);
1049 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001050 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001052 atomic_set(&md->pending, 0);
1053 init_waitqueue_head(&md->wait);
1054 init_waitqueue_head(&md->eventq);
1055
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 md->disk->major = _major;
1057 md->disk->first_minor = minor;
1058 md->disk->fops = &dm_blk_dops;
1059 md->disk->queue = md->queue;
1060 md->disk->private_data = md;
1061 sprintf(md->disk->disk_name, "dm-%d", minor);
1062 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001063 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064
Milan Broz304f3f62008-02-08 02:11:17 +00001065 md->wq = create_singlethread_workqueue("kdmflush");
1066 if (!md->wq)
1067 goto bad_thread;
1068
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001069 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001070 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001071 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001072 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001073
1074 BUG_ON(old_md != MINOR_ALLOCED);
1075
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076 return md;
1077
Milan Broz304f3f62008-02-08 02:11:17 +00001078bad_thread:
1079 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001080bad_disk:
Stefan Bader9faf4002006-10-03 01:15:41 -07001081 bioset_free(md->bs);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001082bad_no_bioset:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 mempool_destroy(md->tio_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001084bad_tio_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 mempool_destroy(md->io_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001086bad_io_pool:
Al Viro1312f402006-03-12 11:02:03 -05001087 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001088bad_queue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001090bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001091 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001092bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 kfree(md);
1094 return NULL;
1095}
1096
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001097static void unlock_fs(struct mapped_device *md);
1098
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099static void free_dev(struct mapped_device *md)
1100{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001101 int minor = md->disk->first_minor;
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001102
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001103 if (md->suspended_bdev) {
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001104 unlock_fs(md);
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001105 bdput(md->suspended_bdev);
1106 }
Milan Broz304f3f62008-02-08 02:11:17 +00001107 destroy_workqueue(md->wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 mempool_destroy(md->tio_pool);
1109 mempool_destroy(md->io_pool);
Stefan Bader9faf4002006-10-03 01:15:41 -07001110 bioset_free(md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001112 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001113
1114 spin_lock(&_minor_lock);
1115 md->disk->private_data = NULL;
1116 spin_unlock(&_minor_lock);
1117
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05001119 blk_cleanup_queue(md->queue);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001120 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121 kfree(md);
1122}
1123
1124/*
1125 * Bind a table to the device.
1126 */
1127static void event_callback(void *context)
1128{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001129 unsigned long flags;
1130 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131 struct mapped_device *md = (struct mapped_device *) context;
1132
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001133 spin_lock_irqsave(&md->uevent_lock, flags);
1134 list_splice_init(&md->uevent_list, &uevents);
1135 spin_unlock_irqrestore(&md->uevent_lock, flags);
1136
Kay Sieversedfaa7c2007-05-21 22:08:01 +02001137 dm_send_uevents(&uevents, &md->disk->dev.kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001138
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139 atomic_inc(&md->event_nr);
1140 wake_up(&md->eventq);
1141}
1142
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001143static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144{
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001145 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001147 mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001148 i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001149 mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150}
1151
1152static int __bind(struct mapped_device *md, struct dm_table *t)
1153{
Jens Axboe165125e2007-07-24 09:28:11 +02001154 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 sector_t size;
1156
1157 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001158
1159 /*
1160 * Wipe any geometry if the size of the table changed.
1161 */
1162 if (size != get_capacity(md->disk))
1163 memset(&md->geometry, 0, sizeof(md->geometry));
1164
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001165 if (md->suspended_bdev)
1166 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001167 if (size == 0)
1168 return 0;
1169
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170 dm_table_get(t);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001171 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001172
1173 write_lock(&md->map_lock);
1174 md->map = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 dm_table_set_restrictions(t, q);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001176 write_unlock(&md->map_lock);
1177
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 return 0;
1179}
1180
1181static void __unbind(struct mapped_device *md)
1182{
1183 struct dm_table *map = md->map;
1184
1185 if (!map)
1186 return;
1187
1188 dm_table_event_callback(map, NULL, NULL);
1189 write_lock(&md->map_lock);
1190 md->map = NULL;
1191 write_unlock(&md->map_lock);
1192 dm_table_put(map);
1193}
1194
1195/*
1196 * Constructor for a new device.
1197 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001198int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199{
1200 struct mapped_device *md;
1201
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001202 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203 if (!md)
1204 return -ENXIO;
1205
1206 *result = md;
1207 return 0;
1208}
1209
David Teigland637842c2006-01-06 00:20:00 -08001210static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211{
1212 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 unsigned minor = MINOR(dev);
1214
1215 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1216 return NULL;
1217
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001218 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219
1220 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001221 if (md && (md == MINOR_ALLOCED ||
1222 (dm_disk(md)->first_minor != minor) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07001223 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08001224 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001225 goto out;
1226 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001228out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001229 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
David Teigland637842c2006-01-06 00:20:00 -08001231 return md;
1232}
1233
David Teiglandd229a952006-01-06 00:20:01 -08001234struct mapped_device *dm_get_md(dev_t dev)
1235{
1236 struct mapped_device *md = dm_find_md(dev);
1237
1238 if (md)
1239 dm_get(md);
1240
1241 return md;
1242}
1243
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001244void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08001245{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001246 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247}
1248
1249void dm_set_mdptr(struct mapped_device *md, void *ptr)
1250{
1251 md->interface_ptr = ptr;
1252}
1253
1254void dm_get(struct mapped_device *md)
1255{
1256 atomic_inc(&md->holders);
1257}
1258
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001259const char *dm_device_name(struct mapped_device *md)
1260{
1261 return md->name;
1262}
1263EXPORT_SYMBOL_GPL(dm_device_name);
1264
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265void dm_put(struct mapped_device *md)
1266{
Mike Anderson1134e5a2006-03-27 01:17:54 -08001267 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001269 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1270
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001271 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08001272 map = dm_get_table(md);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001273 idr_replace(&_minor_idr, MINOR_ALLOCED, dm_disk(md)->first_minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001274 set_bit(DMF_FREEING, &md->flags);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001275 spin_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001276 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277 dm_table_presuspend_targets(map);
1278 dm_table_postsuspend_targets(map);
1279 }
1280 __unbind(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08001281 dm_table_put(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282 free_dev(md);
1283 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284}
Edward Goggin79eb8852007-05-09 02:32:56 -07001285EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286
Milan Broz46125c12008-02-08 02:10:30 +00001287static int dm_wait_for_completion(struct mapped_device *md)
1288{
1289 int r = 0;
1290
1291 while (1) {
1292 set_current_state(TASK_INTERRUPTIBLE);
1293
1294 smp_mb();
1295 if (!atomic_read(&md->pending))
1296 break;
1297
1298 if (signal_pending(current)) {
1299 r = -EINTR;
1300 break;
1301 }
1302
1303 io_schedule();
1304 }
1305 set_current_state(TASK_RUNNING);
1306
1307 return r;
1308}
1309
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310/*
1311 * Process the deferred bios
1312 */
Milan Broz6d6f10d2008-02-08 02:10:22 +00001313static void __flush_deferred_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314{
Milan Broz6d6f10d2008-02-08 02:10:22 +00001315 struct bio *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316
Milan Broz6d6f10d2008-02-08 02:10:22 +00001317 while ((c = bio_list_pop(&md->deferred))) {
Milan Broz9e4e5f82007-10-19 22:38:53 +01001318 if (__split_bio(md, c))
1319 bio_io_error(c);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320 }
Milan Broz73d410c2008-02-08 02:10:25 +00001321
1322 clear_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323}
1324
Milan Broz6d6f10d2008-02-08 02:10:22 +00001325static void __merge_pushback_list(struct mapped_device *md)
1326{
1327 unsigned long flags;
1328
1329 spin_lock_irqsave(&md->pushback_lock, flags);
1330 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1331 bio_list_merge_head(&md->deferred, &md->pushback);
1332 bio_list_init(&md->pushback);
1333 spin_unlock_irqrestore(&md->pushback_lock, flags);
1334}
1335
Milan Broz304f3f62008-02-08 02:11:17 +00001336static void dm_wq_work(struct work_struct *work)
1337{
1338 struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
1339 struct mapped_device *md = req->md;
1340
1341 down_write(&md->io_lock);
1342 switch (req->type) {
1343 case DM_WQ_FLUSH_ALL:
1344 __merge_pushback_list(md);
1345 /* pass through */
1346 case DM_WQ_FLUSH_DEFERRED:
1347 __flush_deferred_io(md);
1348 break;
1349 default:
1350 DMERR("dm_wq_work: unrecognised work type %d", req->type);
1351 BUG();
1352 }
1353 up_write(&md->io_lock);
1354}
1355
1356static void dm_wq_queue(struct mapped_device *md, int type, void *context,
1357 struct dm_wq_req *req)
1358{
1359 req->type = type;
1360 req->md = md;
1361 req->context = context;
1362 INIT_WORK(&req->work, dm_wq_work);
1363 queue_work(md->wq, &req->work);
1364}
1365
1366static void dm_queue_flush(struct mapped_device *md, int type, void *context)
1367{
1368 struct dm_wq_req req;
1369
1370 dm_wq_queue(md, type, context, &req);
1371 flush_workqueue(md->wq);
1372}
1373
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374/*
1375 * Swap in a new table (destroying old one).
1376 */
1377int dm_swap_table(struct mapped_device *md, struct dm_table *table)
1378{
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001379 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380
Daniel Walkere61290a2008-02-08 02:10:08 +00001381 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382
1383 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001384 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001385 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001387 /* without bdev, the device size cannot be changed */
1388 if (!md->suspended_bdev)
1389 if (get_capacity(md->disk) != dm_table_get_size(table))
1390 goto out;
1391
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392 __unbind(md);
1393 r = __bind(md, table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001394
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001395out:
Daniel Walkere61290a2008-02-08 02:10:08 +00001396 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001397 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001398}
1399
1400/*
1401 * Functions to lock and unlock any filesystem running on the
1402 * device.
1403 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001404static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001406 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407
1408 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001409
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001410 md->frozen_sb = freeze_bdev(md->suspended_bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001411 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001412 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001413 md->frozen_sb = NULL;
1414 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001415 }
1416
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001417 set_bit(DMF_FROZEN, &md->flags);
1418
Linus Torvalds1da177e2005-04-16 15:20:36 -07001419 /* don't bdput right now, we don't want the bdev
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001420 * to go away while it is locked.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 */
1422 return 0;
1423}
1424
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001425static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001427 if (!test_bit(DMF_FROZEN, &md->flags))
1428 return;
1429
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001430 thaw_bdev(md->suspended_bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001432 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433}
1434
1435/*
1436 * We need to be able to change a mapping table under a mounted
1437 * filesystem. For example we might want to move some data in
1438 * the background. Before the table can be swapped with
1439 * dm_bind_table, dm_suspend must be called to flush any in
1440 * flight bios and ensure that any further io gets deferred.
1441 */
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001442int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001444 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445 DECLARE_WAITQUEUE(wait, current);
Milan Broz46125c12008-02-08 02:10:30 +00001446 int r = 0;
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001447 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001448 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001449
Daniel Walkere61290a2008-02-08 02:10:08 +00001450 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001451
Milan Broz73d410c2008-02-08 02:10:25 +00001452 if (dm_suspended(md)) {
1453 r = -EINVAL;
Alasdair G Kergond2874832006-11-08 17:44:43 -08001454 goto out_unlock;
Milan Broz73d410c2008-02-08 02:10:25 +00001455 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456
1457 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001459 /*
1460 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
1461 * This flag is cleared before dm_suspend returns.
1462 */
1463 if (noflush)
1464 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1465
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001466 /* This does not get reverted if there's an error later. */
1467 dm_table_presuspend_targets(map);
1468
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001469 /* bdget() can stall if the pending I/Os are not flushed */
1470 if (!noflush) {
1471 md->suspended_bdev = bdget_disk(md->disk, 0);
1472 if (!md->suspended_bdev) {
1473 DMWARN("bdget failed in dm_suspend");
1474 r = -ENOMEM;
1475 goto flush_and_out;
1476 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001477
Milan Broz6d6f10d2008-02-08 02:10:22 +00001478 /*
1479 * Flush I/O to the device. noflush supersedes do_lockfs,
1480 * because lock_fs() needs to flush I/Os.
1481 */
1482 if (do_lockfs) {
1483 r = lock_fs(md);
1484 if (r)
1485 goto out;
1486 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001487 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488
1489 /*
Alasdair G Kergon354e0072005-05-05 16:16:05 -07001490 * First we set the BLOCK_IO flag so no more ios will be mapped.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001492 down_write(&md->io_lock);
1493 set_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495 add_wait_queue(&md->wait, &wait);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001496 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497
1498 /* unplug */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001499 if (map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501
1502 /*
Milan Broz46125c12008-02-08 02:10:30 +00001503 * Wait for the already-mapped ios to complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504 */
Milan Broz46125c12008-02-08 02:10:30 +00001505 r = dm_wait_for_completion(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001507 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508 remove_wait_queue(&md->wait, &wait);
1509
Milan Broz6d6f10d2008-02-08 02:10:22 +00001510 if (noflush)
1511 __merge_pushback_list(md);
Milan Broz94d63512008-02-08 02:10:27 +00001512 up_write(&md->io_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001513
Linus Torvalds1da177e2005-04-16 15:20:36 -07001514 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00001515 if (r < 0) {
Milan Broz304f3f62008-02-08 02:11:17 +00001516 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Milan Broz73d410c2008-02-08 02:10:25 +00001517
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001518 unlock_fs(md);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001519 goto out; /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001520 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001521
1522 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523
1524 set_bit(DMF_SUSPENDED, &md->flags);
1525
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001526flush_and_out:
Milan Broz304f3f62008-02-08 02:11:17 +00001527 if (r && noflush)
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001528 /*
1529 * Because there may be already I/Os in the pushback list,
1530 * flush them before return.
1531 */
Milan Broz304f3f62008-02-08 02:11:17 +00001532 dm_queue_flush(md, DM_WQ_FLUSH_ALL, NULL);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001533
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001534out:
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001535 if (r && md->suspended_bdev) {
1536 bdput(md->suspended_bdev);
1537 md->suspended_bdev = NULL;
1538 }
1539
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540 dm_table_put(map);
Alasdair G Kergond2874832006-11-08 17:44:43 -08001541
1542out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00001543 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001544 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545}
1546
1547int dm_resume(struct mapped_device *md)
1548{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001549 int r = -EINVAL;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001550 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551
Daniel Walkere61290a2008-02-08 02:10:08 +00001552 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001553 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001554 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001555
1556 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001557 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001558 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559
Milan Broz8757b772006-10-03 01:15:36 -07001560 r = dm_table_resume_targets(map);
1561 if (r)
1562 goto out;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001563
Milan Broz304f3f62008-02-08 02:11:17 +00001564 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001565
1566 unlock_fs(md);
1567
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001568 if (md->suspended_bdev) {
1569 bdput(md->suspended_bdev);
1570 md->suspended_bdev = NULL;
1571 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001572
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001573 clear_bit(DMF_SUSPENDED, &md->flags);
1574
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001576
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001577 dm_kobject_uevent(md);
Hannes Reinecke8560ed62006-10-03 01:15:35 -07001578
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001579 r = 0;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001580
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001581out:
1582 dm_table_put(map);
Daniel Walkere61290a2008-02-08 02:10:08 +00001583 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001584
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001585 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586}
1587
1588/*-----------------------------------------------------------------
1589 * Event notification.
1590 *---------------------------------------------------------------*/
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001591void dm_kobject_uevent(struct mapped_device *md)
1592{
Kay Sieversedfaa7c2007-05-21 22:08:01 +02001593 kobject_uevent(&md->disk->dev.kobj, KOBJ_CHANGE);
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001594}
1595
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001596uint32_t dm_next_uevent_seq(struct mapped_device *md)
1597{
1598 return atomic_add_return(1, &md->uevent_seq);
1599}
1600
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601uint32_t dm_get_event_nr(struct mapped_device *md)
1602{
1603 return atomic_read(&md->event_nr);
1604}
1605
1606int dm_wait_event(struct mapped_device *md, int event_nr)
1607{
1608 return wait_event_interruptible(md->eventq,
1609 (event_nr != atomic_read(&md->event_nr)));
1610}
1611
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001612void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
1613{
1614 unsigned long flags;
1615
1616 spin_lock_irqsave(&md->uevent_lock, flags);
1617 list_add(elist, &md->uevent_list);
1618 spin_unlock_irqrestore(&md->uevent_lock, flags);
1619}
1620
Linus Torvalds1da177e2005-04-16 15:20:36 -07001621/*
1622 * The gendisk is only valid as long as you have a reference
1623 * count on 'md'.
1624 */
1625struct gendisk *dm_disk(struct mapped_device *md)
1626{
1627 return md->disk;
1628}
1629
1630int dm_suspended(struct mapped_device *md)
1631{
1632 return test_bit(DMF_SUSPENDED, &md->flags);
1633}
1634
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001635int dm_noflush_suspending(struct dm_target *ti)
1636{
1637 struct mapped_device *md = dm_table_get_md(ti->table);
1638 int r = __noflush_suspending(md);
1639
1640 dm_put(md);
1641
1642 return r;
1643}
1644EXPORT_SYMBOL_GPL(dm_noflush_suspending);
1645
Linus Torvalds1da177e2005-04-16 15:20:36 -07001646static struct block_device_operations dm_blk_dops = {
1647 .open = dm_blk_open,
1648 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07001649 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001650 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 .owner = THIS_MODULE
1652};
1653
1654EXPORT_SYMBOL(dm_get_mapinfo);
1655
1656/*
1657 * module hooks
1658 */
1659module_init(dm_init);
1660module_exit(dm_exit);
1661
1662module_param(major, uint, 0);
1663MODULE_PARM_DESC(major, "The major number of the device mapper");
1664MODULE_DESCRIPTION(DM_NAME " driver");
1665MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1666MODULE_LICENSE("GPL");