blob: bca448e118786a78a5e07b54f3cb3a7e3aca85d5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07003 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
9#include "dm-bio-list.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +010010#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
12#include <linux/init.h>
13#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080014#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/moduleparam.h>
16#include <linux/blkpg.h>
17#include <linux/bio.h>
18#include <linux/buffer_head.h>
19#include <linux/mempool.h>
20#include <linux/slab.h>
21#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080022#include <linux/hdreg.h>
Jens Axboe2056a782006-03-23 20:00:26 +010023#include <linux/blktrace_api.h>
Milan Brozaa129a22006-10-03 01:15:15 -070024#include <linux/smp_lock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Alasdair G Kergon72d94862006-06-26 00:27:35 -070026#define DM_MSG_PREFIX "core"
27
Linus Torvalds1da177e2005-04-16 15:20:36 -070028static const char *_name = DM_NAME;
29
30static unsigned int major = 0;
31static unsigned int _major = 0;
32
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070033static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070034/*
35 * One of these is allocated per bio.
36 */
37struct dm_io {
38 struct mapped_device *md;
39 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010041 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080042 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070043};
44
45/*
46 * One of these is allocated per target within a bio. Hopefully
47 * this will be simplified out one day.
48 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +010049struct dm_target_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 struct dm_io *io;
51 struct dm_target *ti;
52 union map_info info;
53};
54
55union map_info *dm_get_mapinfo(struct bio *bio)
56{
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070057 if (bio && bio->bi_private)
Alasdair G Kergon028867a2007-07-12 17:26:32 +010058 return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070059 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070060}
61
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070062#define MINOR_ALLOCED ((void *)-1)
63
Linus Torvalds1da177e2005-04-16 15:20:36 -070064/*
65 * Bits for the md->flags field.
66 */
67#define DMF_BLOCK_IO 0
68#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080069#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -070070#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070071#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080072#define DMF_NOFLUSH_SUSPENDING 5
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Milan Broz304f3f62008-02-08 02:11:17 +000074/*
75 * Work processed by per-device workqueue.
76 */
77struct dm_wq_req {
78 enum {
79 DM_WQ_FLUSH_ALL,
80 DM_WQ_FLUSH_DEFERRED,
81 } type;
82 struct work_struct work;
83 struct mapped_device *md;
84 void *context;
85};
86
Linus Torvalds1da177e2005-04-16 15:20:36 -070087struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -070088 struct rw_semaphore io_lock;
Daniel Walkere61290a2008-02-08 02:10:08 +000089 struct mutex suspend_lock;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080090 spinlock_t pushback_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -070091 rwlock_t map_lock;
92 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070093 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070094
95 unsigned long flags;
96
Jens Axboe165125e2007-07-24 09:28:11 +020097 struct request_queue *queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -080099 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100
101 void *interface_ptr;
102
103 /*
104 * A list of ios that arrived while we were suspended.
105 */
106 atomic_t pending;
107 wait_queue_head_t wait;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800108 struct bio_list deferred;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800109 struct bio_list pushback;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110
111 /*
Milan Broz304f3f62008-02-08 02:11:17 +0000112 * Processing queue (flush/barriers)
113 */
114 struct workqueue_struct *wq;
115
116 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 * The current mapping.
118 */
119 struct dm_table *map;
120
121 /*
122 * io objects are allocated from here.
123 */
124 mempool_t *io_pool;
125 mempool_t *tio_pool;
126
Stefan Bader9faf4002006-10-03 01:15:41 -0700127 struct bio_set *bs;
128
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129 /*
130 * Event handling.
131 */
132 atomic_t event_nr;
133 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100134 atomic_t uevent_seq;
135 struct list_head uevent_list;
136 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
138 /*
139 * freeze/thaw support require holding onto a super block
140 */
141 struct super_block *frozen_sb;
Alasdair G Kergone39e2e92006-01-06 00:20:05 -0800142 struct block_device *suspended_bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800143
144 /* forced geometry settings */
145 struct hd_geometry geometry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146};
147
148#define MIN_IOS 256
Christoph Lametere18b8902006-12-06 20:33:20 -0800149static struct kmem_cache *_io_cache;
150static struct kmem_cache *_tio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152static int __init local_init(void)
153{
154 int r;
155
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100157 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 if (!_io_cache)
159 return -ENOMEM;
160
161 /* allocate a slab for the target ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100162 _tio_cache = KMEM_CACHE(dm_target_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 if (!_tio_cache) {
164 kmem_cache_destroy(_io_cache);
165 return -ENOMEM;
166 }
167
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100168 r = dm_uevent_init();
169 if (r) {
170 kmem_cache_destroy(_tio_cache);
171 kmem_cache_destroy(_io_cache);
172 return r;
173 }
174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 _major = major;
176 r = register_blkdev(_major, _name);
177 if (r < 0) {
178 kmem_cache_destroy(_tio_cache);
179 kmem_cache_destroy(_io_cache);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100180 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 return r;
182 }
183
184 if (!_major)
185 _major = r;
186
187 return 0;
188}
189
190static void local_exit(void)
191{
192 kmem_cache_destroy(_tio_cache);
193 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700194 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100195 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
197 _major = 0;
198
199 DMINFO("cleaned up");
200}
201
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000202static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203 local_init,
204 dm_target_init,
205 dm_linear_init,
206 dm_stripe_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100207 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 dm_interface_init,
209};
210
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000211static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 local_exit,
213 dm_target_exit,
214 dm_linear_exit,
215 dm_stripe_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100216 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 dm_interface_exit,
218};
219
220static int __init dm_init(void)
221{
222 const int count = ARRAY_SIZE(_inits);
223
224 int r, i;
225
226 for (i = 0; i < count; i++) {
227 r = _inits[i]();
228 if (r)
229 goto bad;
230 }
231
232 return 0;
233
234 bad:
235 while (i--)
236 _exits[i]();
237
238 return r;
239}
240
241static void __exit dm_exit(void)
242{
243 int i = ARRAY_SIZE(_exits);
244
245 while (i--)
246 _exits[i]();
247}
248
249/*
250 * Block device functions
251 */
252static int dm_blk_open(struct inode *inode, struct file *file)
253{
254 struct mapped_device *md;
255
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700256 spin_lock(&_minor_lock);
257
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 md = inode->i_bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700259 if (!md)
260 goto out;
261
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700262 if (test_bit(DMF_FREEING, &md->flags) ||
263 test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700264 md = NULL;
265 goto out;
266 }
267
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700269 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700270
271out:
272 spin_unlock(&_minor_lock);
273
274 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275}
276
277static int dm_blk_close(struct inode *inode, struct file *file)
278{
279 struct mapped_device *md;
280
281 md = inode->i_bdev->bd_disk->private_data;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700282 atomic_dec(&md->open_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 dm_put(md);
284 return 0;
285}
286
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700287int dm_open_count(struct mapped_device *md)
288{
289 return atomic_read(&md->open_count);
290}
291
292/*
293 * Guarantees nothing is using the device before it's deleted.
294 */
295int dm_lock_for_deletion(struct mapped_device *md)
296{
297 int r = 0;
298
299 spin_lock(&_minor_lock);
300
301 if (dm_open_count(md))
302 r = -EBUSY;
303 else
304 set_bit(DMF_DELETING, &md->flags);
305
306 spin_unlock(&_minor_lock);
307
308 return r;
309}
310
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800311static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
312{
313 struct mapped_device *md = bdev->bd_disk->private_data;
314
315 return dm_get_geometry(md, geo);
316}
317
Milan Brozaa129a22006-10-03 01:15:15 -0700318static int dm_blk_ioctl(struct inode *inode, struct file *file,
319 unsigned int cmd, unsigned long arg)
320{
321 struct mapped_device *md;
322 struct dm_table *map;
323 struct dm_target *tgt;
324 int r = -ENOTTY;
325
326 /* We don't really need this lock, but we do need 'inode'. */
327 unlock_kernel();
328
329 md = inode->i_bdev->bd_disk->private_data;
330
331 map = dm_get_table(md);
332
333 if (!map || !dm_table_get_size(map))
334 goto out;
335
336 /* We only support devices that have a single target */
337 if (dm_table_get_num_targets(map) != 1)
338 goto out;
339
340 tgt = dm_table_get_target(map, 0);
341
342 if (dm_suspended(md)) {
343 r = -EAGAIN;
344 goto out;
345 }
346
347 if (tgt->type->ioctl)
348 r = tgt->type->ioctl(tgt, inode, file, cmd, arg);
349
350out:
351 dm_table_put(map);
352
353 lock_kernel();
354 return r;
355}
356
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100357static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358{
359 return mempool_alloc(md->io_pool, GFP_NOIO);
360}
361
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100362static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363{
364 mempool_free(io, md->io_pool);
365}
366
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100367static struct dm_target_io *alloc_tio(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368{
369 return mempool_alloc(md->tio_pool, GFP_NOIO);
370}
371
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100372static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373{
374 mempool_free(tio, md->tio_pool);
375}
376
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800377static void start_io_acct(struct dm_io *io)
378{
379 struct mapped_device *md = io->md;
380
381 io->start_time = jiffies;
382
383 preempt_disable();
384 disk_round_stats(dm_disk(md));
385 preempt_enable();
386 dm_disk(md)->in_flight = atomic_inc_return(&md->pending);
387}
388
389static int end_io_acct(struct dm_io *io)
390{
391 struct mapped_device *md = io->md;
392 struct bio *bio = io->bio;
393 unsigned long duration = jiffies - io->start_time;
394 int pending;
395 int rw = bio_data_dir(bio);
396
397 preempt_disable();
398 disk_round_stats(dm_disk(md));
399 preempt_enable();
400 dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending);
401
402 disk_stat_add(dm_disk(md), ticks[rw], duration);
403
404 return !pending;
405}
406
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407/*
408 * Add the bio to the list of deferred io.
409 */
410static int queue_io(struct mapped_device *md, struct bio *bio)
411{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700412 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
414 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700415 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 return 1;
417 }
418
419 bio_list_add(&md->deferred, bio);
420
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700421 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 return 0; /* deferred successfully */
423}
424
425/*
426 * Everyone (including functions in this file), should use this
427 * function to access the md->map field, and make sure they call
428 * dm_table_put() when finished.
429 */
430struct dm_table *dm_get_table(struct mapped_device *md)
431{
432 struct dm_table *t;
433
434 read_lock(&md->map_lock);
435 t = md->map;
436 if (t)
437 dm_table_get(t);
438 read_unlock(&md->map_lock);
439
440 return t;
441}
442
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800443/*
444 * Get the geometry associated with a dm device
445 */
446int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
447{
448 *geo = md->geometry;
449
450 return 0;
451}
452
453/*
454 * Set the geometry of a device.
455 */
456int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
457{
458 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
459
460 if (geo->start > sz) {
461 DMWARN("Start sector is beyond the geometry limits.");
462 return -EINVAL;
463 }
464
465 md->geometry = *geo;
466
467 return 0;
468}
469
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470/*-----------------------------------------------------------------
471 * CRUD START:
472 * A more elegant soln is in the works that uses the queue
473 * merge fn, unfortunately there are a couple of changes to
474 * the block layer that I want to make for this. So in the
475 * interests of getting something for people to use I give
476 * you this clearly demarcated crap.
477 *---------------------------------------------------------------*/
478
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800479static int __noflush_suspending(struct mapped_device *md)
480{
481 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
482}
483
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484/*
485 * Decrements the number of outstanding ios that a bio has been
486 * cloned into, completing the original io if necc.
487 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800488static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800490 unsigned long flags;
491
492 /* Push-back supersedes any I/O errors */
493 if (error && !(io->error > 0 && __noflush_suspending(io->md)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 io->error = error;
495
496 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800497 if (io->error == DM_ENDIO_REQUEUE) {
498 /*
499 * Target requested pushing back the I/O.
500 * This must be handled before the sleeper on
501 * suspend queue merges the pushback list.
502 */
503 spin_lock_irqsave(&io->md->pushback_lock, flags);
504 if (__noflush_suspending(io->md))
505 bio_list_add(&io->md->pushback, io->bio);
506 else
507 /* noflush suspend was interrupted. */
508 io->error = -EIO;
509 spin_unlock_irqrestore(&io->md->pushback_lock, flags);
510 }
511
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800512 if (end_io_acct(io))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 /* nudge anyone waiting on suspend queue */
514 wake_up(&io->md->wait);
515
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800516 if (io->error != DM_ENDIO_REQUEUE) {
517 blk_add_trace_bio(io->md->queue, io->bio,
518 BLK_TA_COMPLETE);
Jens Axboe2056a782006-03-23 20:00:26 +0100519
NeilBrown6712ecf2007-09-27 12:47:43 +0200520 bio_endio(io->bio, io->error);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800521 }
522
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 free_io(io->md, io);
524 }
525}
526
NeilBrown6712ecf2007-09-27 12:47:43 +0200527static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528{
529 int r = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100530 struct dm_target_io *tio = bio->bi_private;
Stefan Bader9faf4002006-10-03 01:15:41 -0700531 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 dm_endio_fn endio = tio->ti->type->end_io;
533
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
535 error = -EIO;
536
537 if (endio) {
538 r = endio(tio->ti, bio, error, &tio->info);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800539 if (r < 0 || r == DM_ENDIO_REQUEUE)
540 /*
541 * error and requeue request are handled
542 * in dec_pending().
543 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800545 else if (r == DM_ENDIO_INCOMPLETE)
546 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200547 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800548 else if (r) {
549 DMWARN("unimplemented target endio return value: %d", r);
550 BUG();
551 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 }
553
Stefan Bader9faf4002006-10-03 01:15:41 -0700554 dec_pending(tio->io, error);
555
556 /*
557 * Store md for cleanup instead of tio which is about to get freed.
558 */
559 bio->bi_private = md->bs;
560
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 bio_put(bio);
Stefan Bader9faf4002006-10-03 01:15:41 -0700562 free_tio(md, tio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563}
564
565static sector_t max_io_len(struct mapped_device *md,
566 sector_t sector, struct dm_target *ti)
567{
568 sector_t offset = sector - ti->begin;
569 sector_t len = ti->len - offset;
570
571 /*
572 * Does the target need to split even further ?
573 */
574 if (ti->split_io) {
575 sector_t boundary;
576 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
577 - offset;
578 if (len > boundary)
579 len = boundary;
580 }
581
582 return len;
583}
584
585static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100586 struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587{
588 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100589 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -0700590 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591
592 /*
593 * Sanity checks.
594 */
595 BUG_ON(!clone->bi_size);
596
597 clone->bi_end_io = clone_endio;
598 clone->bi_private = tio;
599
600 /*
601 * Map the clone. If r == 0 we don't need to do
602 * anything, the target has assumed ownership of
603 * this io.
604 */
605 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100606 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 r = ti->type->map(ti, clone, &tio->info);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800608 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100610
Alasdair G Kergon17b2f662006-06-26 00:27:33 -0700611 blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
Alan D. Brunellec7149d62007-08-07 15:30:23 +0200612 tio->io->bio->bi_bdev->bd_dev,
613 clone->bi_sector, sector);
Jens Axboe2056a782006-03-23 20:00:26 +0100614
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800616 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
617 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -0700618 md = tio->io->md;
619 dec_pending(tio->io, r);
620 /*
621 * Store bio_set for cleanup.
622 */
623 clone->bi_private = md->bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 bio_put(clone);
Stefan Bader9faf4002006-10-03 01:15:41 -0700625 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800626 } else if (r) {
627 DMWARN("unimplemented target map return value: %d", r);
628 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 }
630}
631
632struct clone_info {
633 struct mapped_device *md;
634 struct dm_table *map;
635 struct bio *bio;
636 struct dm_io *io;
637 sector_t sector;
638 sector_t sector_count;
639 unsigned short idx;
640};
641
Peter Osterlund36763472005-09-06 15:16:42 -0700642static void dm_bio_destructor(struct bio *bio)
643{
Stefan Bader9faf4002006-10-03 01:15:41 -0700644 struct bio_set *bs = bio->bi_private;
645
646 bio_free(bio, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700647}
648
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649/*
650 * Creates a little bio that is just does part of a bvec.
651 */
652static struct bio *split_bvec(struct bio *bio, sector_t sector,
653 unsigned short idx, unsigned int offset,
Stefan Bader9faf4002006-10-03 01:15:41 -0700654 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655{
656 struct bio *clone;
657 struct bio_vec *bv = bio->bi_io_vec + idx;
658
Stefan Bader9faf4002006-10-03 01:15:41 -0700659 clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700660 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 *clone->bi_io_vec = *bv;
662
663 clone->bi_sector = sector;
664 clone->bi_bdev = bio->bi_bdev;
665 clone->bi_rw = bio->bi_rw;
666 clone->bi_vcnt = 1;
667 clone->bi_size = to_bytes(len);
668 clone->bi_io_vec->bv_offset = offset;
669 clone->bi_io_vec->bv_len = clone->bi_size;
670
671 return clone;
672}
673
674/*
675 * Creates a bio that consists of range of complete bvecs.
676 */
677static struct bio *clone_bio(struct bio *bio, sector_t sector,
678 unsigned short idx, unsigned short bv_count,
Stefan Bader9faf4002006-10-03 01:15:41 -0700679 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680{
681 struct bio *clone;
682
Stefan Bader9faf4002006-10-03 01:15:41 -0700683 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
684 __bio_clone(clone, bio);
685 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 clone->bi_sector = sector;
687 clone->bi_idx = idx;
688 clone->bi_vcnt = idx + bv_count;
689 clone->bi_size = to_bytes(len);
690 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
691
692 return clone;
693}
694
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000695static int __clone_and_map(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700696{
697 struct bio *clone, *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000698 struct dm_target *ti;
699 sector_t len = 0, max;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100700 struct dm_target_io *tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000702 ti = dm_table_find_target(ci->map, ci->sector);
703 if (!dm_target_is_valid(ti))
704 return -EIO;
705
706 max = max_io_len(ci->md, ci->sector, ti);
707
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 /*
709 * Allocate a target io object.
710 */
711 tio = alloc_tio(ci->md);
712 tio->io = ci->io;
713 tio->ti = ti;
714 memset(&tio->info, 0, sizeof(tio->info));
715
716 if (ci->sector_count <= max) {
717 /*
718 * Optimise for the simple case where we can do all of
719 * the remaining io with a single clone.
720 */
721 clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700722 bio->bi_vcnt - ci->idx, ci->sector_count,
723 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 __map_bio(ti, clone, tio);
725 ci->sector_count = 0;
726
727 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
728 /*
729 * There are some bvecs that don't span targets.
730 * Do as many of these as possible.
731 */
732 int i;
733 sector_t remaining = max;
734 sector_t bv_len;
735
736 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
737 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
738
739 if (bv_len > remaining)
740 break;
741
742 remaining -= bv_len;
743 len += bv_len;
744 }
745
Stefan Bader9faf4002006-10-03 01:15:41 -0700746 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
747 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 __map_bio(ti, clone, tio);
749
750 ci->sector += len;
751 ci->sector_count -= len;
752 ci->idx = i;
753
754 } else {
755 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800756 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 */
758 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800759 sector_t remaining = to_sector(bv->bv_len);
760 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800762 do {
763 if (offset) {
764 ti = dm_table_find_target(ci->map, ci->sector);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000765 if (!dm_target_is_valid(ti))
766 return -EIO;
767
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800768 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800770 tio = alloc_tio(ci->md);
771 tio->io = ci->io;
772 tio->ti = ti;
773 memset(&tio->info, 0, sizeof(tio->info));
774 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800776 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800778 clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700779 bv->bv_offset + offset, len,
780 ci->md->bs);
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800781
782 __map_bio(ti, clone, tio);
783
784 ci->sector += len;
785 ci->sector_count -= len;
786 offset += to_bytes(len);
787 } while (remaining -= len);
788
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 ci->idx++;
790 }
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000791
792 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793}
794
795/*
796 * Split the bio into several clones.
797 */
Milan Broz9e4e5f82007-10-19 22:38:53 +0100798static int __split_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799{
800 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000801 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
803 ci.map = dm_get_table(md);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100804 if (unlikely(!ci.map))
805 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806
807 ci.md = md;
808 ci.bio = bio;
809 ci.io = alloc_io(md);
810 ci.io->error = 0;
811 atomic_set(&ci.io->io_count, 1);
812 ci.io->bio = bio;
813 ci.io->md = md;
814 ci.sector = bio->bi_sector;
815 ci.sector_count = bio_sectors(bio);
816 ci.idx = bio->bi_idx;
817
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800818 start_io_acct(ci.io);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000819 while (ci.sector_count && !error)
820 error = __clone_and_map(&ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821
822 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000823 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700824 dm_table_put(ci.map);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100825
826 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827}
828/*-----------------------------------------------------------------
829 * CRUD END
830 *---------------------------------------------------------------*/
831
Milan Brozf6fccb12008-07-21 12:00:37 +0100832static int dm_merge_bvec(struct request_queue *q,
833 struct bvec_merge_data *bvm,
834 struct bio_vec *biovec)
835{
836 struct mapped_device *md = q->queuedata;
837 struct dm_table *map = dm_get_table(md);
838 struct dm_target *ti;
839 sector_t max_sectors;
840 int max_size;
841
842 if (unlikely(!map))
843 return 0;
844
845 ti = dm_table_find_target(map, bvm->bi_sector);
846
847 /*
848 * Find maximum amount of I/O that won't need splitting
849 */
850 max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
851 (sector_t) BIO_MAX_SECTORS);
852 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
853 if (max_size < 0)
854 max_size = 0;
855
856 /*
857 * merge_bvec_fn() returns number of bytes
858 * it can accept at this offset
859 * max is precomputed maximal io size
860 */
861 if (max_size && ti->type->merge)
862 max_size = ti->type->merge(ti, bvm, biovec, max_size);
863
864 /*
865 * Always allow an entire first page
866 */
867 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
868 max_size = biovec->bv_len;
869
870 dm_table_put(map);
871
872 return max_size;
873}
874
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875/*
876 * The request function that just remaps the bio built up by
877 * dm_merge_bvec.
878 */
Jens Axboe165125e2007-07-24 09:28:11 +0200879static int dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880{
Milan Broz9e4e5f82007-10-19 22:38:53 +0100881 int r = -EIO;
Kevin Corry12f03a42006-02-01 03:04:52 -0800882 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 struct mapped_device *md = q->queuedata;
884
Stefan Bader07a83c42007-07-12 17:28:33 +0100885 /*
886 * There is no use in forwarding any barrier request since we can't
887 * guarantee it is (or can be) handled by the targets correctly.
888 */
889 if (unlikely(bio_barrier(bio))) {
NeilBrown6712ecf2007-09-27 12:47:43 +0200890 bio_endio(bio, -EOPNOTSUPP);
Stefan Bader07a83c42007-07-12 17:28:33 +0100891 return 0;
892 }
893
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700894 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895
Kevin Corry12f03a42006-02-01 03:04:52 -0800896 disk_stat_inc(dm_disk(md), ios[rw]);
897 disk_stat_add(dm_disk(md), sectors[rw], bio_sectors(bio));
898
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899 /*
900 * If we're suspended we have to queue
901 * this io for later.
902 */
903 while (test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700904 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
Milan Broz9e4e5f82007-10-19 22:38:53 +0100906 if (bio_rw(bio) != READA)
907 r = queue_io(md, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908
Milan Broz9e4e5f82007-10-19 22:38:53 +0100909 if (r <= 0)
910 goto out_req;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911
912 /*
913 * We're in a while loop, because someone could suspend
914 * before we get to the following read lock.
915 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700916 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 }
918
Milan Broz9e4e5f82007-10-19 22:38:53 +0100919 r = __split_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700920 up_read(&md->io_lock);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100921
922out_req:
923 if (r < 0)
924 bio_io_error(bio);
925
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 return 0;
927}
928
Jens Axboe165125e2007-07-24 09:28:11 +0200929static void dm_unplug_all(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930{
931 struct mapped_device *md = q->queuedata;
932 struct dm_table *map = dm_get_table(md);
933
934 if (map) {
935 dm_table_unplug_all(map);
936 dm_table_put(map);
937 }
938}
939
940static int dm_any_congested(void *congested_data, int bdi_bits)
941{
942 int r;
943 struct mapped_device *md = (struct mapped_device *) congested_data;
944 struct dm_table *map = dm_get_table(md);
945
946 if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
947 r = bdi_bits;
948 else
949 r = dm_table_any_congested(map, bdi_bits);
950
951 dm_table_put(map);
952 return r;
953}
954
955/*-----------------------------------------------------------------
956 * An IDR is used to keep track of allocated minor numbers.
957 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958static DEFINE_IDR(_minor_idr);
959
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -0700960static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700962 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700964 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965}
966
967/*
968 * See if the device with a specific minor # is free.
969 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +0100970static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971{
972 int r, m;
973
974 if (minor >= (1 << MINORBITS))
975 return -EINVAL;
976
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700977 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
978 if (!r)
979 return -ENOMEM;
980
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700981 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982
983 if (idr_find(&_minor_idr, minor)) {
984 r = -EBUSY;
985 goto out;
986 }
987
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700988 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700989 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991
992 if (m != minor) {
993 idr_remove(&_minor_idr, m);
994 r = -EBUSY;
995 goto out;
996 }
997
998out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700999 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000 return r;
1001}
1002
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001003static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001005 int r, m;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001008 if (!r)
1009 return -ENOMEM;
1010
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001011 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001013 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001014 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016
1017 if (m >= (1 << MINORBITS)) {
1018 idr_remove(&_minor_idr, m);
1019 r = -ENOSPC;
1020 goto out;
1021 }
1022
1023 *minor = m;
1024
1025out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001026 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 return r;
1028}
1029
1030static struct block_device_operations dm_blk_dops;
1031
1032/*
1033 * Allocate and initialise a blank device with a given minor.
1034 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001035static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036{
1037 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001038 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001039 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040
1041 if (!md) {
1042 DMWARN("unable to allocate device, out of memory.");
1043 return NULL;
1044 }
1045
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001046 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001047 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001048
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001050 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001051 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001052 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001053 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001055 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001057 init_rwsem(&md->io_lock);
Daniel Walkere61290a2008-02-08 02:10:08 +00001058 mutex_init(&md->suspend_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001059 spin_lock_init(&md->pushback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 rwlock_init(&md->map_lock);
1061 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001062 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001064 atomic_set(&md->uevent_seq, 0);
1065 INIT_LIST_HEAD(&md->uevent_list);
1066 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067
1068 md->queue = blk_alloc_queue(GFP_KERNEL);
1069 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001070 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001071
1072 md->queue->queuedata = md;
1073 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1074 md->queue->backing_dev_info.congested_data = md;
1075 blk_queue_make_request(md->queue, dm_request);
Jens Axboedaef2652006-01-10 10:48:02 +01001076 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 md->queue->unplug_fn = dm_unplug_all;
Milan Brozf6fccb12008-07-21 12:00:37 +01001078 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079
Matthew Dobson93d23412006-03-26 01:37:50 -08001080 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
Kiyoshi Ueda74859362006-12-08 02:41:02 -08001081 if (!md->io_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001082 goto bad_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083
Matthew Dobson93d23412006-03-26 01:37:50 -08001084 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 if (!md->tio_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001086 goto bad_tio_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087
Jens Axboe59725112007-04-02 10:06:42 +02001088 md->bs = bioset_create(16, 16);
Stefan Bader9faf4002006-10-03 01:15:41 -07001089 if (!md->bs)
1090 goto bad_no_bioset;
1091
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092 md->disk = alloc_disk(1);
1093 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001094 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001096 atomic_set(&md->pending, 0);
1097 init_waitqueue_head(&md->wait);
1098 init_waitqueue_head(&md->eventq);
1099
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100 md->disk->major = _major;
1101 md->disk->first_minor = minor;
1102 md->disk->fops = &dm_blk_dops;
1103 md->disk->queue = md->queue;
1104 md->disk->private_data = md;
1105 sprintf(md->disk->disk_name, "dm-%d", minor);
1106 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001107 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
Milan Broz304f3f62008-02-08 02:11:17 +00001109 md->wq = create_singlethread_workqueue("kdmflush");
1110 if (!md->wq)
1111 goto bad_thread;
1112
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001113 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001114 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001115 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001116 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001117
1118 BUG_ON(old_md != MINOR_ALLOCED);
1119
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120 return md;
1121
Milan Broz304f3f62008-02-08 02:11:17 +00001122bad_thread:
1123 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001124bad_disk:
Stefan Bader9faf4002006-10-03 01:15:41 -07001125 bioset_free(md->bs);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001126bad_no_bioset:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 mempool_destroy(md->tio_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001128bad_tio_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129 mempool_destroy(md->io_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001130bad_io_pool:
Al Viro1312f402006-03-12 11:02:03 -05001131 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001132bad_queue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001134bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001135 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001136bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 kfree(md);
1138 return NULL;
1139}
1140
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001141static void unlock_fs(struct mapped_device *md);
1142
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143static void free_dev(struct mapped_device *md)
1144{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001145 int minor = md->disk->first_minor;
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001146
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001147 if (md->suspended_bdev) {
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001148 unlock_fs(md);
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001149 bdput(md->suspended_bdev);
1150 }
Milan Broz304f3f62008-02-08 02:11:17 +00001151 destroy_workqueue(md->wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 mempool_destroy(md->tio_pool);
1153 mempool_destroy(md->io_pool);
Stefan Bader9faf4002006-10-03 01:15:41 -07001154 bioset_free(md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001156 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001157
1158 spin_lock(&_minor_lock);
1159 md->disk->private_data = NULL;
1160 spin_unlock(&_minor_lock);
1161
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05001163 blk_cleanup_queue(md->queue);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001164 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165 kfree(md);
1166}
1167
1168/*
1169 * Bind a table to the device.
1170 */
1171static void event_callback(void *context)
1172{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001173 unsigned long flags;
1174 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175 struct mapped_device *md = (struct mapped_device *) context;
1176
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001177 spin_lock_irqsave(&md->uevent_lock, flags);
1178 list_splice_init(&md->uevent_list, &uevents);
1179 spin_unlock_irqrestore(&md->uevent_lock, flags);
1180
Kay Sieversedfaa7c2007-05-21 22:08:01 +02001181 dm_send_uevents(&uevents, &md->disk->dev.kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001182
Linus Torvalds1da177e2005-04-16 15:20:36 -07001183 atomic_inc(&md->event_nr);
1184 wake_up(&md->eventq);
1185}
1186
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001187static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188{
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001189 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001191 mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001192 i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001193 mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194}
1195
1196static int __bind(struct mapped_device *md, struct dm_table *t)
1197{
Jens Axboe165125e2007-07-24 09:28:11 +02001198 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 sector_t size;
1200
1201 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001202
1203 /*
1204 * Wipe any geometry if the size of the table changed.
1205 */
1206 if (size != get_capacity(md->disk))
1207 memset(&md->geometry, 0, sizeof(md->geometry));
1208
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001209 if (md->suspended_bdev)
1210 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 if (size == 0)
1212 return 0;
1213
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 dm_table_get(t);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001215 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001216
1217 write_lock(&md->map_lock);
1218 md->map = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001219 dm_table_set_restrictions(t, q);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001220 write_unlock(&md->map_lock);
1221
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222 return 0;
1223}
1224
1225static void __unbind(struct mapped_device *md)
1226{
1227 struct dm_table *map = md->map;
1228
1229 if (!map)
1230 return;
1231
1232 dm_table_event_callback(map, NULL, NULL);
1233 write_lock(&md->map_lock);
1234 md->map = NULL;
1235 write_unlock(&md->map_lock);
1236 dm_table_put(map);
1237}
1238
1239/*
1240 * Constructor for a new device.
1241 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001242int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243{
1244 struct mapped_device *md;
1245
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001246 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247 if (!md)
1248 return -ENXIO;
1249
1250 *result = md;
1251 return 0;
1252}
1253
David Teigland637842c2006-01-06 00:20:00 -08001254static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255{
1256 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001257 unsigned minor = MINOR(dev);
1258
1259 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1260 return NULL;
1261
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001262 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263
1264 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001265 if (md && (md == MINOR_ALLOCED ||
1266 (dm_disk(md)->first_minor != minor) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07001267 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08001268 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001269 goto out;
1270 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001272out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001273 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274
David Teigland637842c2006-01-06 00:20:00 -08001275 return md;
1276}
1277
David Teiglandd229a952006-01-06 00:20:01 -08001278struct mapped_device *dm_get_md(dev_t dev)
1279{
1280 struct mapped_device *md = dm_find_md(dev);
1281
1282 if (md)
1283 dm_get(md);
1284
1285 return md;
1286}
1287
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001288void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08001289{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001290 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291}
1292
1293void dm_set_mdptr(struct mapped_device *md, void *ptr)
1294{
1295 md->interface_ptr = ptr;
1296}
1297
1298void dm_get(struct mapped_device *md)
1299{
1300 atomic_inc(&md->holders);
1301}
1302
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001303const char *dm_device_name(struct mapped_device *md)
1304{
1305 return md->name;
1306}
1307EXPORT_SYMBOL_GPL(dm_device_name);
1308
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309void dm_put(struct mapped_device *md)
1310{
Mike Anderson1134e5a2006-03-27 01:17:54 -08001311 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001313 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1314
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001315 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08001316 map = dm_get_table(md);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001317 idr_replace(&_minor_idr, MINOR_ALLOCED, dm_disk(md)->first_minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001318 set_bit(DMF_FREEING, &md->flags);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001319 spin_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001320 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321 dm_table_presuspend_targets(map);
1322 dm_table_postsuspend_targets(map);
1323 }
1324 __unbind(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08001325 dm_table_put(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001326 free_dev(md);
1327 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328}
Edward Goggin79eb8852007-05-09 02:32:56 -07001329EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330
Milan Broz46125c12008-02-08 02:10:30 +00001331static int dm_wait_for_completion(struct mapped_device *md)
1332{
1333 int r = 0;
1334
1335 while (1) {
1336 set_current_state(TASK_INTERRUPTIBLE);
1337
1338 smp_mb();
1339 if (!atomic_read(&md->pending))
1340 break;
1341
1342 if (signal_pending(current)) {
1343 r = -EINTR;
1344 break;
1345 }
1346
1347 io_schedule();
1348 }
1349 set_current_state(TASK_RUNNING);
1350
1351 return r;
1352}
1353
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354/*
1355 * Process the deferred bios
1356 */
Milan Broz6d6f10d2008-02-08 02:10:22 +00001357static void __flush_deferred_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358{
Milan Broz6d6f10d2008-02-08 02:10:22 +00001359 struct bio *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360
Milan Broz6d6f10d2008-02-08 02:10:22 +00001361 while ((c = bio_list_pop(&md->deferred))) {
Milan Broz9e4e5f82007-10-19 22:38:53 +01001362 if (__split_bio(md, c))
1363 bio_io_error(c);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364 }
Milan Broz73d410c2008-02-08 02:10:25 +00001365
1366 clear_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367}
1368
Milan Broz6d6f10d2008-02-08 02:10:22 +00001369static void __merge_pushback_list(struct mapped_device *md)
1370{
1371 unsigned long flags;
1372
1373 spin_lock_irqsave(&md->pushback_lock, flags);
1374 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1375 bio_list_merge_head(&md->deferred, &md->pushback);
1376 bio_list_init(&md->pushback);
1377 spin_unlock_irqrestore(&md->pushback_lock, flags);
1378}
1379
Milan Broz304f3f62008-02-08 02:11:17 +00001380static void dm_wq_work(struct work_struct *work)
1381{
1382 struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
1383 struct mapped_device *md = req->md;
1384
1385 down_write(&md->io_lock);
1386 switch (req->type) {
1387 case DM_WQ_FLUSH_ALL:
1388 __merge_pushback_list(md);
1389 /* pass through */
1390 case DM_WQ_FLUSH_DEFERRED:
1391 __flush_deferred_io(md);
1392 break;
1393 default:
1394 DMERR("dm_wq_work: unrecognised work type %d", req->type);
1395 BUG();
1396 }
1397 up_write(&md->io_lock);
1398}
1399
1400static void dm_wq_queue(struct mapped_device *md, int type, void *context,
1401 struct dm_wq_req *req)
1402{
1403 req->type = type;
1404 req->md = md;
1405 req->context = context;
1406 INIT_WORK(&req->work, dm_wq_work);
1407 queue_work(md->wq, &req->work);
1408}
1409
1410static void dm_queue_flush(struct mapped_device *md, int type, void *context)
1411{
1412 struct dm_wq_req req;
1413
1414 dm_wq_queue(md, type, context, &req);
1415 flush_workqueue(md->wq);
1416}
1417
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418/*
1419 * Swap in a new table (destroying old one).
1420 */
1421int dm_swap_table(struct mapped_device *md, struct dm_table *table)
1422{
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001423 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424
Daniel Walkere61290a2008-02-08 02:10:08 +00001425 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426
1427 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001428 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001429 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001431 /* without bdev, the device size cannot be changed */
1432 if (!md->suspended_bdev)
1433 if (get_capacity(md->disk) != dm_table_get_size(table))
1434 goto out;
1435
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 __unbind(md);
1437 r = __bind(md, table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001439out:
Daniel Walkere61290a2008-02-08 02:10:08 +00001440 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001441 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442}
1443
1444/*
1445 * Functions to lock and unlock any filesystem running on the
1446 * device.
1447 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001448static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001449{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001450 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451
1452 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001453
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001454 md->frozen_sb = freeze_bdev(md->suspended_bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001455 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001456 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001457 md->frozen_sb = NULL;
1458 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001459 }
1460
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001461 set_bit(DMF_FROZEN, &md->flags);
1462
Linus Torvalds1da177e2005-04-16 15:20:36 -07001463 /* don't bdput right now, we don't want the bdev
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001464 * to go away while it is locked.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465 */
1466 return 0;
1467}
1468
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001469static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001471 if (!test_bit(DMF_FROZEN, &md->flags))
1472 return;
1473
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001474 thaw_bdev(md->suspended_bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001476 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477}
1478
1479/*
1480 * We need to be able to change a mapping table under a mounted
1481 * filesystem. For example we might want to move some data in
1482 * the background. Before the table can be swapped with
1483 * dm_bind_table, dm_suspend must be called to flush any in
1484 * flight bios and ensure that any further io gets deferred.
1485 */
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001486int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001488 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 DECLARE_WAITQUEUE(wait, current);
Milan Broz46125c12008-02-08 02:10:30 +00001490 int r = 0;
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001491 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001492 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001493
Daniel Walkere61290a2008-02-08 02:10:08 +00001494 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001495
Milan Broz73d410c2008-02-08 02:10:25 +00001496 if (dm_suspended(md)) {
1497 r = -EINVAL;
Alasdair G Kergond2874832006-11-08 17:44:43 -08001498 goto out_unlock;
Milan Broz73d410c2008-02-08 02:10:25 +00001499 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001500
1501 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001503 /*
1504 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
1505 * This flag is cleared before dm_suspend returns.
1506 */
1507 if (noflush)
1508 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1509
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001510 /* This does not get reverted if there's an error later. */
1511 dm_table_presuspend_targets(map);
1512
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001513 /* bdget() can stall if the pending I/Os are not flushed */
1514 if (!noflush) {
1515 md->suspended_bdev = bdget_disk(md->disk, 0);
1516 if (!md->suspended_bdev) {
1517 DMWARN("bdget failed in dm_suspend");
1518 r = -ENOMEM;
1519 goto flush_and_out;
1520 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001521
Milan Broz6d6f10d2008-02-08 02:10:22 +00001522 /*
1523 * Flush I/O to the device. noflush supersedes do_lockfs,
1524 * because lock_fs() needs to flush I/Os.
1525 */
1526 if (do_lockfs) {
1527 r = lock_fs(md);
1528 if (r)
1529 goto out;
1530 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001531 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001532
1533 /*
Alasdair G Kergon354e0072005-05-05 16:16:05 -07001534 * First we set the BLOCK_IO flag so no more ios will be mapped.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001536 down_write(&md->io_lock);
1537 set_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539 add_wait_queue(&md->wait, &wait);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001540 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541
1542 /* unplug */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001543 if (map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545
1546 /*
Milan Broz46125c12008-02-08 02:10:30 +00001547 * Wait for the already-mapped ios to complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548 */
Milan Broz46125c12008-02-08 02:10:30 +00001549 r = dm_wait_for_completion(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001551 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552 remove_wait_queue(&md->wait, &wait);
1553
Milan Broz6d6f10d2008-02-08 02:10:22 +00001554 if (noflush)
1555 __merge_pushback_list(md);
Milan Broz94d63512008-02-08 02:10:27 +00001556 up_write(&md->io_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001557
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00001559 if (r < 0) {
Milan Broz304f3f62008-02-08 02:11:17 +00001560 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Milan Broz73d410c2008-02-08 02:10:25 +00001561
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001562 unlock_fs(md);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001563 goto out; /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001564 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001565
1566 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567
1568 set_bit(DMF_SUSPENDED, &md->flags);
1569
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001570flush_and_out:
Milan Broz304f3f62008-02-08 02:11:17 +00001571 if (r && noflush)
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001572 /*
1573 * Because there may be already I/Os in the pushback list,
1574 * flush them before return.
1575 */
Milan Broz304f3f62008-02-08 02:11:17 +00001576 dm_queue_flush(md, DM_WQ_FLUSH_ALL, NULL);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001577
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001578out:
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001579 if (r && md->suspended_bdev) {
1580 bdput(md->suspended_bdev);
1581 md->suspended_bdev = NULL;
1582 }
1583
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584 dm_table_put(map);
Alasdair G Kergond2874832006-11-08 17:44:43 -08001585
1586out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00001587 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001588 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589}
1590
1591int dm_resume(struct mapped_device *md)
1592{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001593 int r = -EINVAL;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001594 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595
Daniel Walkere61290a2008-02-08 02:10:08 +00001596 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001597 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001598 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001599
1600 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001601 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001602 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001603
Milan Broz8757b772006-10-03 01:15:36 -07001604 r = dm_table_resume_targets(map);
1605 if (r)
1606 goto out;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001607
Milan Broz304f3f62008-02-08 02:11:17 +00001608 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001609
1610 unlock_fs(md);
1611
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001612 if (md->suspended_bdev) {
1613 bdput(md->suspended_bdev);
1614 md->suspended_bdev = NULL;
1615 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001616
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001617 clear_bit(DMF_SUSPENDED, &md->flags);
1618
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001620
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001621 dm_kobject_uevent(md);
Hannes Reinecke8560ed62006-10-03 01:15:35 -07001622
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001623 r = 0;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001624
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001625out:
1626 dm_table_put(map);
Daniel Walkere61290a2008-02-08 02:10:08 +00001627 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001628
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001629 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630}
1631
1632/*-----------------------------------------------------------------
1633 * Event notification.
1634 *---------------------------------------------------------------*/
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001635void dm_kobject_uevent(struct mapped_device *md)
1636{
Kay Sieversedfaa7c2007-05-21 22:08:01 +02001637 kobject_uevent(&md->disk->dev.kobj, KOBJ_CHANGE);
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001638}
1639
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001640uint32_t dm_next_uevent_seq(struct mapped_device *md)
1641{
1642 return atomic_add_return(1, &md->uevent_seq);
1643}
1644
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645uint32_t dm_get_event_nr(struct mapped_device *md)
1646{
1647 return atomic_read(&md->event_nr);
1648}
1649
1650int dm_wait_event(struct mapped_device *md, int event_nr)
1651{
1652 return wait_event_interruptible(md->eventq,
1653 (event_nr != atomic_read(&md->event_nr)));
1654}
1655
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001656void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
1657{
1658 unsigned long flags;
1659
1660 spin_lock_irqsave(&md->uevent_lock, flags);
1661 list_add(elist, &md->uevent_list);
1662 spin_unlock_irqrestore(&md->uevent_lock, flags);
1663}
1664
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665/*
1666 * The gendisk is only valid as long as you have a reference
1667 * count on 'md'.
1668 */
1669struct gendisk *dm_disk(struct mapped_device *md)
1670{
1671 return md->disk;
1672}
1673
1674int dm_suspended(struct mapped_device *md)
1675{
1676 return test_bit(DMF_SUSPENDED, &md->flags);
1677}
1678
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001679int dm_noflush_suspending(struct dm_target *ti)
1680{
1681 struct mapped_device *md = dm_table_get_md(ti->table);
1682 int r = __noflush_suspending(md);
1683
1684 dm_put(md);
1685
1686 return r;
1687}
1688EXPORT_SYMBOL_GPL(dm_noflush_suspending);
1689
Linus Torvalds1da177e2005-04-16 15:20:36 -07001690static struct block_device_operations dm_blk_dops = {
1691 .open = dm_blk_open,
1692 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07001693 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001694 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695 .owner = THIS_MODULE
1696};
1697
1698EXPORT_SYMBOL(dm_get_mapinfo);
1699
1700/*
1701 * module hooks
1702 */
1703module_init(dm_init);
1704module_exit(dm_exit);
1705
1706module_param(major, uint, 0);
1707MODULE_PARM_DESC(major, "The major number of the device mapper");
1708MODULE_DESCRIPTION(DM_NAME " driver");
1709MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1710MODULE_LICENSE("GPL");