blob: 87d8ca1121e2217e31b0b908e01a19e8ef74fbc0 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
9#include "dm-bio-list.h"
10
11#include <linux/init.h>
12#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080013#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/moduleparam.h>
15#include <linux/blkpg.h>
16#include <linux/bio.h>
17#include <linux/buffer_head.h>
18#include <linux/mempool.h>
19#include <linux/slab.h>
20#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080021#include <linux/hdreg.h>
Jens Axboe2056a782006-03-23 20:00:26 +010022#include <linux/blktrace_api.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
24static const char *_name = DM_NAME;
25
26static unsigned int major = 0;
27static unsigned int _major = 0;
28
29/*
30 * One of these is allocated per bio.
31 */
32struct dm_io {
33 struct mapped_device *md;
34 int error;
35 struct bio *bio;
36 atomic_t io_count;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080037 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070038};
39
40/*
41 * One of these is allocated per target within a bio. Hopefully
42 * this will be simplified out one day.
43 */
44struct target_io {
45 struct dm_io *io;
46 struct dm_target *ti;
47 union map_info info;
48};
49
50union map_info *dm_get_mapinfo(struct bio *bio)
51{
52 if (bio && bio->bi_private)
53 return &((struct target_io *)bio->bi_private)->info;
54 return NULL;
55}
56
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070057#define MINOR_ALLOCED ((void *)-1)
58
Linus Torvalds1da177e2005-04-16 15:20:36 -070059/*
60 * Bits for the md->flags field.
61 */
62#define DMF_BLOCK_IO 0
63#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080064#define DMF_FROZEN 2
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
66struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -070067 struct rw_semaphore io_lock;
68 struct semaphore suspend_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 rwlock_t map_lock;
70 atomic_t holders;
71
72 unsigned long flags;
73
74 request_queue_t *queue;
75 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -080076 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78 void *interface_ptr;
79
80 /*
81 * A list of ios that arrived while we were suspended.
82 */
83 atomic_t pending;
84 wait_queue_head_t wait;
85 struct bio_list deferred;
86
87 /*
88 * The current mapping.
89 */
90 struct dm_table *map;
91
92 /*
93 * io objects are allocated from here.
94 */
95 mempool_t *io_pool;
96 mempool_t *tio_pool;
97
98 /*
99 * Event handling.
100 */
101 atomic_t event_nr;
102 wait_queue_head_t eventq;
103
104 /*
105 * freeze/thaw support require holding onto a super block
106 */
107 struct super_block *frozen_sb;
Alasdair G Kergone39e2e92006-01-06 00:20:05 -0800108 struct block_device *suspended_bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800109
110 /* forced geometry settings */
111 struct hd_geometry geometry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112};
113
114#define MIN_IOS 256
115static kmem_cache_t *_io_cache;
116static kmem_cache_t *_tio_cache;
117
118static struct bio_set *dm_set;
119
120static int __init local_init(void)
121{
122 int r;
123
124 dm_set = bioset_create(16, 16, 4);
125 if (!dm_set)
126 return -ENOMEM;
127
128 /* allocate a slab for the dm_ios */
129 _io_cache = kmem_cache_create("dm_io",
130 sizeof(struct dm_io), 0, 0, NULL, NULL);
131 if (!_io_cache)
132 return -ENOMEM;
133
134 /* allocate a slab for the target ios */
135 _tio_cache = kmem_cache_create("dm_tio", sizeof(struct target_io),
136 0, 0, NULL, NULL);
137 if (!_tio_cache) {
138 kmem_cache_destroy(_io_cache);
139 return -ENOMEM;
140 }
141
142 _major = major;
143 r = register_blkdev(_major, _name);
144 if (r < 0) {
145 kmem_cache_destroy(_tio_cache);
146 kmem_cache_destroy(_io_cache);
147 return r;
148 }
149
150 if (!_major)
151 _major = r;
152
153 return 0;
154}
155
156static void local_exit(void)
157{
158 kmem_cache_destroy(_tio_cache);
159 kmem_cache_destroy(_io_cache);
160
161 bioset_free(dm_set);
162
163 if (unregister_blkdev(_major, _name) < 0)
164 DMERR("devfs_unregister_blkdev failed");
165
166 _major = 0;
167
168 DMINFO("cleaned up");
169}
170
171int (*_inits[])(void) __initdata = {
172 local_init,
173 dm_target_init,
174 dm_linear_init,
175 dm_stripe_init,
176 dm_interface_init,
177};
178
179void (*_exits[])(void) = {
180 local_exit,
181 dm_target_exit,
182 dm_linear_exit,
183 dm_stripe_exit,
184 dm_interface_exit,
185};
186
187static int __init dm_init(void)
188{
189 const int count = ARRAY_SIZE(_inits);
190
191 int r, i;
192
193 for (i = 0; i < count; i++) {
194 r = _inits[i]();
195 if (r)
196 goto bad;
197 }
198
199 return 0;
200
201 bad:
202 while (i--)
203 _exits[i]();
204
205 return r;
206}
207
208static void __exit dm_exit(void)
209{
210 int i = ARRAY_SIZE(_exits);
211
212 while (i--)
213 _exits[i]();
214}
215
216/*
217 * Block device functions
218 */
219static int dm_blk_open(struct inode *inode, struct file *file)
220{
221 struct mapped_device *md;
222
223 md = inode->i_bdev->bd_disk->private_data;
224 dm_get(md);
225 return 0;
226}
227
228static int dm_blk_close(struct inode *inode, struct file *file)
229{
230 struct mapped_device *md;
231
232 md = inode->i_bdev->bd_disk->private_data;
233 dm_put(md);
234 return 0;
235}
236
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800237static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
238{
239 struct mapped_device *md = bdev->bd_disk->private_data;
240
241 return dm_get_geometry(md, geo);
242}
243
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244static inline struct dm_io *alloc_io(struct mapped_device *md)
245{
246 return mempool_alloc(md->io_pool, GFP_NOIO);
247}
248
249static inline void free_io(struct mapped_device *md, struct dm_io *io)
250{
251 mempool_free(io, md->io_pool);
252}
253
254static inline struct target_io *alloc_tio(struct mapped_device *md)
255{
256 return mempool_alloc(md->tio_pool, GFP_NOIO);
257}
258
259static inline void free_tio(struct mapped_device *md, struct target_io *tio)
260{
261 mempool_free(tio, md->tio_pool);
262}
263
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800264static void start_io_acct(struct dm_io *io)
265{
266 struct mapped_device *md = io->md;
267
268 io->start_time = jiffies;
269
270 preempt_disable();
271 disk_round_stats(dm_disk(md));
272 preempt_enable();
273 dm_disk(md)->in_flight = atomic_inc_return(&md->pending);
274}
275
276static int end_io_acct(struct dm_io *io)
277{
278 struct mapped_device *md = io->md;
279 struct bio *bio = io->bio;
280 unsigned long duration = jiffies - io->start_time;
281 int pending;
282 int rw = bio_data_dir(bio);
283
284 preempt_disable();
285 disk_round_stats(dm_disk(md));
286 preempt_enable();
287 dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending);
288
289 disk_stat_add(dm_disk(md), ticks[rw], duration);
290
291 return !pending;
292}
293
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294/*
295 * Add the bio to the list of deferred io.
296 */
297static int queue_io(struct mapped_device *md, struct bio *bio)
298{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700299 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300
301 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700302 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 return 1;
304 }
305
306 bio_list_add(&md->deferred, bio);
307
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700308 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 return 0; /* deferred successfully */
310}
311
312/*
313 * Everyone (including functions in this file), should use this
314 * function to access the md->map field, and make sure they call
315 * dm_table_put() when finished.
316 */
317struct dm_table *dm_get_table(struct mapped_device *md)
318{
319 struct dm_table *t;
320
321 read_lock(&md->map_lock);
322 t = md->map;
323 if (t)
324 dm_table_get(t);
325 read_unlock(&md->map_lock);
326
327 return t;
328}
329
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800330/*
331 * Get the geometry associated with a dm device
332 */
333int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
334{
335 *geo = md->geometry;
336
337 return 0;
338}
339
340/*
341 * Set the geometry of a device.
342 */
343int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
344{
345 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
346
347 if (geo->start > sz) {
348 DMWARN("Start sector is beyond the geometry limits.");
349 return -EINVAL;
350 }
351
352 md->geometry = *geo;
353
354 return 0;
355}
356
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357/*-----------------------------------------------------------------
358 * CRUD START:
359 * A more elegant soln is in the works that uses the queue
360 * merge fn, unfortunately there are a couple of changes to
361 * the block layer that I want to make for this. So in the
362 * interests of getting something for people to use I give
363 * you this clearly demarcated crap.
364 *---------------------------------------------------------------*/
365
366/*
367 * Decrements the number of outstanding ios that a bio has been
368 * cloned into, completing the original io if necc.
369 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800370static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371{
372 if (error)
373 io->error = error;
374
375 if (atomic_dec_and_test(&io->io_count)) {
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800376 if (end_io_acct(io))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 /* nudge anyone waiting on suspend queue */
378 wake_up(&io->md->wait);
379
Jens Axboe2056a782006-03-23 20:00:26 +0100380 blk_add_trace_bio(io->md->queue, io->bio, BLK_TA_COMPLETE);
381
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 bio_endio(io->bio, io->bio->bi_size, io->error);
383 free_io(io->md, io);
384 }
385}
386
387static int clone_endio(struct bio *bio, unsigned int done, int error)
388{
389 int r = 0;
390 struct target_io *tio = bio->bi_private;
391 struct dm_io *io = tio->io;
392 dm_endio_fn endio = tio->ti->type->end_io;
393
394 if (bio->bi_size)
395 return 1;
396
397 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
398 error = -EIO;
399
400 if (endio) {
401 r = endio(tio->ti, bio, error, &tio->info);
402 if (r < 0)
403 error = r;
404
405 else if (r > 0)
406 /* the target wants another shot at the io */
407 return 1;
408 }
409
410 free_tio(io->md, tio);
411 dec_pending(io, error);
412 bio_put(bio);
413 return r;
414}
415
416static sector_t max_io_len(struct mapped_device *md,
417 sector_t sector, struct dm_target *ti)
418{
419 sector_t offset = sector - ti->begin;
420 sector_t len = ti->len - offset;
421
422 /*
423 * Does the target need to split even further ?
424 */
425 if (ti->split_io) {
426 sector_t boundary;
427 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
428 - offset;
429 if (len > boundary)
430 len = boundary;
431 }
432
433 return len;
434}
435
436static void __map_bio(struct dm_target *ti, struct bio *clone,
437 struct target_io *tio)
438{
439 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100440 sector_t sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441
442 /*
443 * Sanity checks.
444 */
445 BUG_ON(!clone->bi_size);
446
447 clone->bi_end_io = clone_endio;
448 clone->bi_private = tio;
449
450 /*
451 * Map the clone. If r == 0 we don't need to do
452 * anything, the target has assumed ownership of
453 * this io.
454 */
455 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100456 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457 r = ti->type->map(ti, clone, &tio->info);
Jens Axboe2056a782006-03-23 20:00:26 +0100458 if (r > 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100460
461 blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
462 tio->io->bio->bi_bdev->bd_dev, sector,
463 clone->bi_sector);
464
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 generic_make_request(clone);
Jens Axboe2056a782006-03-23 20:00:26 +0100466 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467
468 else if (r < 0) {
469 /* error the io and bail out */
470 struct dm_io *io = tio->io;
471 free_tio(tio->io->md, tio);
Alasdair G Kergonf6a80ea2005-07-12 15:53:01 -0700472 dec_pending(io, r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 bio_put(clone);
474 }
475}
476
477struct clone_info {
478 struct mapped_device *md;
479 struct dm_table *map;
480 struct bio *bio;
481 struct dm_io *io;
482 sector_t sector;
483 sector_t sector_count;
484 unsigned short idx;
485};
486
Peter Osterlund36763472005-09-06 15:16:42 -0700487static void dm_bio_destructor(struct bio *bio)
488{
489 bio_free(bio, dm_set);
490}
491
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492/*
493 * Creates a little bio that is just does part of a bvec.
494 */
495static struct bio *split_bvec(struct bio *bio, sector_t sector,
496 unsigned short idx, unsigned int offset,
497 unsigned int len)
498{
499 struct bio *clone;
500 struct bio_vec *bv = bio->bi_io_vec + idx;
501
502 clone = bio_alloc_bioset(GFP_NOIO, 1, dm_set);
Peter Osterlund36763472005-09-06 15:16:42 -0700503 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 *clone->bi_io_vec = *bv;
505
506 clone->bi_sector = sector;
507 clone->bi_bdev = bio->bi_bdev;
508 clone->bi_rw = bio->bi_rw;
509 clone->bi_vcnt = 1;
510 clone->bi_size = to_bytes(len);
511 clone->bi_io_vec->bv_offset = offset;
512 clone->bi_io_vec->bv_len = clone->bi_size;
513
514 return clone;
515}
516
517/*
518 * Creates a bio that consists of range of complete bvecs.
519 */
520static struct bio *clone_bio(struct bio *bio, sector_t sector,
521 unsigned short idx, unsigned short bv_count,
522 unsigned int len)
523{
524 struct bio *clone;
525
526 clone = bio_clone(bio, GFP_NOIO);
527 clone->bi_sector = sector;
528 clone->bi_idx = idx;
529 clone->bi_vcnt = idx + bv_count;
530 clone->bi_size = to_bytes(len);
531 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
532
533 return clone;
534}
535
536static void __clone_and_map(struct clone_info *ci)
537{
538 struct bio *clone, *bio = ci->bio;
539 struct dm_target *ti = dm_table_find_target(ci->map, ci->sector);
540 sector_t len = 0, max = max_io_len(ci->md, ci->sector, ti);
541 struct target_io *tio;
542
543 /*
544 * Allocate a target io object.
545 */
546 tio = alloc_tio(ci->md);
547 tio->io = ci->io;
548 tio->ti = ti;
549 memset(&tio->info, 0, sizeof(tio->info));
550
551 if (ci->sector_count <= max) {
552 /*
553 * Optimise for the simple case where we can do all of
554 * the remaining io with a single clone.
555 */
556 clone = clone_bio(bio, ci->sector, ci->idx,
557 bio->bi_vcnt - ci->idx, ci->sector_count);
558 __map_bio(ti, clone, tio);
559 ci->sector_count = 0;
560
561 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
562 /*
563 * There are some bvecs that don't span targets.
564 * Do as many of these as possible.
565 */
566 int i;
567 sector_t remaining = max;
568 sector_t bv_len;
569
570 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
571 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
572
573 if (bv_len > remaining)
574 break;
575
576 remaining -= bv_len;
577 len += bv_len;
578 }
579
580 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len);
581 __map_bio(ti, clone, tio);
582
583 ci->sector += len;
584 ci->sector_count -= len;
585 ci->idx = i;
586
587 } else {
588 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800589 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 */
591 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800592 sector_t remaining = to_sector(bv->bv_len);
593 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800595 do {
596 if (offset) {
597 ti = dm_table_find_target(ci->map, ci->sector);
598 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800600 tio = alloc_tio(ci->md);
601 tio->io = ci->io;
602 tio->ti = ti;
603 memset(&tio->info, 0, sizeof(tio->info));
604 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800606 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800608 clone = split_bvec(bio, ci->sector, ci->idx,
609 bv->bv_offset + offset, len);
610
611 __map_bio(ti, clone, tio);
612
613 ci->sector += len;
614 ci->sector_count -= len;
615 offset += to_bytes(len);
616 } while (remaining -= len);
617
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 ci->idx++;
619 }
620}
621
622/*
623 * Split the bio into several clones.
624 */
625static void __split_bio(struct mapped_device *md, struct bio *bio)
626{
627 struct clone_info ci;
628
629 ci.map = dm_get_table(md);
630 if (!ci.map) {
631 bio_io_error(bio, bio->bi_size);
632 return;
633 }
634
635 ci.md = md;
636 ci.bio = bio;
637 ci.io = alloc_io(md);
638 ci.io->error = 0;
639 atomic_set(&ci.io->io_count, 1);
640 ci.io->bio = bio;
641 ci.io->md = md;
642 ci.sector = bio->bi_sector;
643 ci.sector_count = bio_sectors(bio);
644 ci.idx = bio->bi_idx;
645
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800646 start_io_acct(ci.io);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 while (ci.sector_count)
648 __clone_and_map(&ci);
649
650 /* drop the extra reference count */
651 dec_pending(ci.io, 0);
652 dm_table_put(ci.map);
653}
654/*-----------------------------------------------------------------
655 * CRUD END
656 *---------------------------------------------------------------*/
657
658/*
659 * The request function that just remaps the bio built up by
660 * dm_merge_bvec.
661 */
662static int dm_request(request_queue_t *q, struct bio *bio)
663{
664 int r;
Kevin Corry12f03a42006-02-01 03:04:52 -0800665 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 struct mapped_device *md = q->queuedata;
667
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700668 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669
Kevin Corry12f03a42006-02-01 03:04:52 -0800670 disk_stat_inc(dm_disk(md), ios[rw]);
671 disk_stat_add(dm_disk(md), sectors[rw], bio_sectors(bio));
672
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 /*
674 * If we're suspended we have to queue
675 * this io for later.
676 */
677 while (test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700678 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679
680 if (bio_rw(bio) == READA) {
681 bio_io_error(bio, bio->bi_size);
682 return 0;
683 }
684
685 r = queue_io(md, bio);
686 if (r < 0) {
687 bio_io_error(bio, bio->bi_size);
688 return 0;
689
690 } else if (r == 0)
691 return 0; /* deferred successfully */
692
693 /*
694 * We're in a while loop, because someone could suspend
695 * before we get to the following read lock.
696 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700697 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 }
699
700 __split_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700701 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 return 0;
703}
704
705static int dm_flush_all(request_queue_t *q, struct gendisk *disk,
706 sector_t *error_sector)
707{
708 struct mapped_device *md = q->queuedata;
709 struct dm_table *map = dm_get_table(md);
710 int ret = -ENXIO;
711
712 if (map) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -0700713 ret = dm_table_flush_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714 dm_table_put(map);
715 }
716
717 return ret;
718}
719
720static void dm_unplug_all(request_queue_t *q)
721{
722 struct mapped_device *md = q->queuedata;
723 struct dm_table *map = dm_get_table(md);
724
725 if (map) {
726 dm_table_unplug_all(map);
727 dm_table_put(map);
728 }
729}
730
731static int dm_any_congested(void *congested_data, int bdi_bits)
732{
733 int r;
734 struct mapped_device *md = (struct mapped_device *) congested_data;
735 struct dm_table *map = dm_get_table(md);
736
737 if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
738 r = bdi_bits;
739 else
740 r = dm_table_any_congested(map, bdi_bits);
741
742 dm_table_put(map);
743 return r;
744}
745
746/*-----------------------------------------------------------------
747 * An IDR is used to keep track of allocated minor numbers.
748 *---------------------------------------------------------------*/
Arjan van de Ven48c9c272006-03-27 01:18:20 -0800749static DEFINE_MUTEX(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750static DEFINE_IDR(_minor_idr);
751
752static void free_minor(unsigned int minor)
753{
Arjan van de Ven48c9c272006-03-27 01:18:20 -0800754 mutex_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755 idr_remove(&_minor_idr, minor);
Arjan van de Ven48c9c272006-03-27 01:18:20 -0800756 mutex_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757}
758
759/*
760 * See if the device with a specific minor # is free.
761 */
762static int specific_minor(struct mapped_device *md, unsigned int minor)
763{
764 int r, m;
765
766 if (minor >= (1 << MINORBITS))
767 return -EINVAL;
768
Arjan van de Ven48c9c272006-03-27 01:18:20 -0800769 mutex_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770
771 if (idr_find(&_minor_idr, minor)) {
772 r = -EBUSY;
773 goto out;
774 }
775
776 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
777 if (!r) {
778 r = -ENOMEM;
779 goto out;
780 }
781
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700782 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783 if (r) {
784 goto out;
785 }
786
787 if (m != minor) {
788 idr_remove(&_minor_idr, m);
789 r = -EBUSY;
790 goto out;
791 }
792
793out:
Arjan van de Ven48c9c272006-03-27 01:18:20 -0800794 mutex_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 return r;
796}
797
798static int next_free_minor(struct mapped_device *md, unsigned int *minor)
799{
800 int r;
801 unsigned int m;
802
Arjan van de Ven48c9c272006-03-27 01:18:20 -0800803 mutex_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804
805 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
806 if (!r) {
807 r = -ENOMEM;
808 goto out;
809 }
810
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700811 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 if (r) {
813 goto out;
814 }
815
816 if (m >= (1 << MINORBITS)) {
817 idr_remove(&_minor_idr, m);
818 r = -ENOSPC;
819 goto out;
820 }
821
822 *minor = m;
823
824out:
Arjan van de Ven48c9c272006-03-27 01:18:20 -0800825 mutex_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 return r;
827}
828
829static struct block_device_operations dm_blk_dops;
830
831/*
832 * Allocate and initialise a blank device with a given minor.
833 */
834static struct mapped_device *alloc_dev(unsigned int minor, int persistent)
835{
836 int r;
837 struct mapped_device *md = kmalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700838 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839
840 if (!md) {
841 DMWARN("unable to allocate device, out of memory.");
842 return NULL;
843 }
844
845 /* get a minor number for the dev */
846 r = persistent ? specific_minor(md, minor) : next_free_minor(md, &minor);
847 if (r < 0)
848 goto bad1;
849
850 memset(md, 0, sizeof(*md));
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700851 init_rwsem(&md->io_lock);
852 init_MUTEX(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853 rwlock_init(&md->map_lock);
854 atomic_set(&md->holders, 1);
855 atomic_set(&md->event_nr, 0);
856
857 md->queue = blk_alloc_queue(GFP_KERNEL);
858 if (!md->queue)
859 goto bad1;
860
861 md->queue->queuedata = md;
862 md->queue->backing_dev_info.congested_fn = dm_any_congested;
863 md->queue->backing_dev_info.congested_data = md;
864 blk_queue_make_request(md->queue, dm_request);
Jens Axboedaef2652006-01-10 10:48:02 +0100865 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866 md->queue->unplug_fn = dm_unplug_all;
867 md->queue->issue_flush_fn = dm_flush_all;
868
Matthew Dobson93d23412006-03-26 01:37:50 -0800869 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 if (!md->io_pool)
871 goto bad2;
872
Matthew Dobson93d23412006-03-26 01:37:50 -0800873 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 if (!md->tio_pool)
875 goto bad3;
876
877 md->disk = alloc_disk(1);
878 if (!md->disk)
879 goto bad4;
880
881 md->disk->major = _major;
882 md->disk->first_minor = minor;
883 md->disk->fops = &dm_blk_dops;
884 md->disk->queue = md->queue;
885 md->disk->private_data = md;
886 sprintf(md->disk->disk_name, "dm-%d", minor);
887 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -0800888 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889
890 atomic_set(&md->pending, 0);
891 init_waitqueue_head(&md->wait);
892 init_waitqueue_head(&md->eventq);
893
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700894 /* Populate the mapping, nobody knows we exist yet */
895 mutex_lock(&_minor_lock);
896 old_md = idr_replace(&_minor_idr, md, minor);
897 mutex_unlock(&_minor_lock);
898
899 BUG_ON(old_md != MINOR_ALLOCED);
900
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 return md;
902
903 bad4:
904 mempool_destroy(md->tio_pool);
905 bad3:
906 mempool_destroy(md->io_pool);
907 bad2:
Al Viro1312f402006-03-12 11:02:03 -0500908 blk_cleanup_queue(md->queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 free_minor(minor);
910 bad1:
911 kfree(md);
912 return NULL;
913}
914
915static void free_dev(struct mapped_device *md)
916{
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -0800917 unsigned int minor = md->disk->first_minor;
918
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -0800919 if (md->suspended_bdev) {
920 thaw_bdev(md->suspended_bdev, NULL);
921 bdput(md->suspended_bdev);
922 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923 mempool_destroy(md->tio_pool);
924 mempool_destroy(md->io_pool);
925 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -0800926 free_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -0500928 blk_cleanup_queue(md->queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 kfree(md);
930}
931
932/*
933 * Bind a table to the device.
934 */
935static void event_callback(void *context)
936{
937 struct mapped_device *md = (struct mapped_device *) context;
938
939 atomic_inc(&md->event_nr);
940 wake_up(&md->eventq);
941}
942
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -0700943static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944{
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -0700945 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946
Jes Sorensen1b1dcc12006-01-09 15:59:24 -0800947 mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -0800948 i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Jes Sorensen1b1dcc12006-01-09 15:59:24 -0800949 mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950}
951
952static int __bind(struct mapped_device *md, struct dm_table *t)
953{
954 request_queue_t *q = md->queue;
955 sector_t size;
956
957 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800958
959 /*
960 * Wipe any geometry if the size of the table changed.
961 */
962 if (size != get_capacity(md->disk))
963 memset(&md->geometry, 0, sizeof(md->geometry));
964
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -0700965 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966 if (size == 0)
967 return 0;
968
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 dm_table_get(t);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -0700970 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700971
972 write_lock(&md->map_lock);
973 md->map = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974 dm_table_set_restrictions(t, q);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700975 write_unlock(&md->map_lock);
976
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 return 0;
978}
979
980static void __unbind(struct mapped_device *md)
981{
982 struct dm_table *map = md->map;
983
984 if (!map)
985 return;
986
987 dm_table_event_callback(map, NULL, NULL);
988 write_lock(&md->map_lock);
989 md->map = NULL;
990 write_unlock(&md->map_lock);
991 dm_table_put(map);
992}
993
994/*
995 * Constructor for a new device.
996 */
997static int create_aux(unsigned int minor, int persistent,
998 struct mapped_device **result)
999{
1000 struct mapped_device *md;
1001
1002 md = alloc_dev(minor, persistent);
1003 if (!md)
1004 return -ENXIO;
1005
1006 *result = md;
1007 return 0;
1008}
1009
1010int dm_create(struct mapped_device **result)
1011{
1012 return create_aux(0, 0, result);
1013}
1014
1015int dm_create_with_minor(unsigned int minor, struct mapped_device **result)
1016{
1017 return create_aux(minor, 1, result);
1018}
1019
David Teigland637842c2006-01-06 00:20:00 -08001020static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021{
1022 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023 unsigned minor = MINOR(dev);
1024
1025 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1026 return NULL;
1027
Arjan van de Ven48c9c272006-03-27 01:18:20 -08001028 mutex_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029
1030 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001031 if (md && (md == MINOR_ALLOCED || (dm_disk(md)->first_minor != minor)))
David Teigland637842c2006-01-06 00:20:00 -08001032 md = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033
Arjan van de Ven48c9c272006-03-27 01:18:20 -08001034 mutex_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035
David Teigland637842c2006-01-06 00:20:00 -08001036 return md;
1037}
1038
David Teiglandd229a952006-01-06 00:20:01 -08001039struct mapped_device *dm_get_md(dev_t dev)
1040{
1041 struct mapped_device *md = dm_find_md(dev);
1042
1043 if (md)
1044 dm_get(md);
1045
1046 return md;
1047}
1048
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001049void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08001050{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001051 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052}
1053
1054void dm_set_mdptr(struct mapped_device *md, void *ptr)
1055{
1056 md->interface_ptr = ptr;
1057}
1058
1059void dm_get(struct mapped_device *md)
1060{
1061 atomic_inc(&md->holders);
1062}
1063
1064void dm_put(struct mapped_device *md)
1065{
Mike Anderson1134e5a2006-03-27 01:17:54 -08001066 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067
1068 if (atomic_dec_and_test(&md->holders)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08001069 map = dm_get_table(md);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001070 mutex_lock(&_minor_lock);
1071 idr_replace(&_minor_idr, MINOR_ALLOCED, dm_disk(md)->first_minor);
1072 mutex_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001073 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 dm_table_presuspend_targets(map);
1075 dm_table_postsuspend_targets(map);
1076 }
1077 __unbind(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08001078 dm_table_put(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079 free_dev(md);
1080 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081}
1082
1083/*
1084 * Process the deferred bios
1085 */
1086static void __flush_deferred_io(struct mapped_device *md, struct bio *c)
1087{
1088 struct bio *n;
1089
1090 while (c) {
1091 n = c->bi_next;
1092 c->bi_next = NULL;
1093 __split_bio(md, c);
1094 c = n;
1095 }
1096}
1097
1098/*
1099 * Swap in a new table (destroying old one).
1100 */
1101int dm_swap_table(struct mapped_device *md, struct dm_table *table)
1102{
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001103 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001105 down(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106
1107 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001108 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001109 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
1111 __unbind(md);
1112 r = __bind(md, table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001114out:
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001115 up(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001116 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117}
1118
1119/*
1120 * Functions to lock and unlock any filesystem running on the
1121 * device.
1122 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001123static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001125 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126
1127 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001128
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001129 md->frozen_sb = freeze_bdev(md->suspended_bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001130 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001131 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001132 md->frozen_sb = NULL;
1133 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001134 }
1135
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001136 set_bit(DMF_FROZEN, &md->flags);
1137
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 /* don't bdput right now, we don't want the bdev
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001139 * to go away while it is locked.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140 */
1141 return 0;
1142}
1143
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001144static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001146 if (!test_bit(DMF_FROZEN, &md->flags))
1147 return;
1148
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001149 thaw_bdev(md->suspended_bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001151 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152}
1153
1154/*
1155 * We need to be able to change a mapping table under a mounted
1156 * filesystem. For example we might want to move some data in
1157 * the background. Before the table can be swapped with
1158 * dm_bind_table, dm_suspend must be called to flush any in
1159 * flight bios and ensure that any further io gets deferred.
1160 */
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001161int dm_suspend(struct mapped_device *md, int do_lockfs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001163 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164 DECLARE_WAITQUEUE(wait, current);
Jun'ichi Nomura1ecac7f2006-03-27 01:17:51 -08001165 struct bio *def;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001166 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001167
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001168 down(&md->suspend_lock);
1169
1170 if (dm_suspended(md))
1171 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172
1173 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001175 /* This does not get reverted if there's an error later. */
1176 dm_table_presuspend_targets(map);
1177
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001178 md->suspended_bdev = bdget_disk(md->disk, 0);
1179 if (!md->suspended_bdev) {
1180 DMWARN("bdget failed in dm_suspend");
1181 r = -ENOMEM;
1182 goto out;
1183 }
1184
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001185 /* Flush I/O to the device. */
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001186 if (do_lockfs) {
1187 r = lock_fs(md);
1188 if (r)
1189 goto out;
1190 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001191
1192 /*
Alasdair G Kergon354e0072005-05-05 16:16:05 -07001193 * First we set the BLOCK_IO flag so no more ios will be mapped.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001195 down_write(&md->io_lock);
1196 set_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198 add_wait_queue(&md->wait, &wait);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001199 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200
1201 /* unplug */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001202 if (map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204
1205 /*
1206 * Then we wait for the already mapped ios to
1207 * complete.
1208 */
1209 while (1) {
1210 set_current_state(TASK_INTERRUPTIBLE);
1211
1212 if (!atomic_read(&md->pending) || signal_pending(current))
1213 break;
1214
1215 io_schedule();
1216 }
1217 set_current_state(TASK_RUNNING);
1218
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001219 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 remove_wait_queue(&md->wait, &wait);
1221
1222 /* were we interrupted ? */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001223 r = -EINTR;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001224 if (atomic_read(&md->pending)) {
Jun'ichi Nomura1ecac7f2006-03-27 01:17:51 -08001225 clear_bit(DMF_BLOCK_IO, &md->flags);
1226 def = bio_list_get(&md->deferred);
1227 __flush_deferred_io(md, def);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001228 up_write(&md->io_lock);
1229 unlock_fs(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001230 goto out;
1231 }
1232 up_write(&md->io_lock);
1233
1234 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235
1236 set_bit(DMF_SUSPENDED, &md->flags);
1237
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001238 r = 0;
1239
1240out:
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001241 if (r && md->suspended_bdev) {
1242 bdput(md->suspended_bdev);
1243 md->suspended_bdev = NULL;
1244 }
1245
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246 dm_table_put(map);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001247 up(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001248 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249}
1250
1251int dm_resume(struct mapped_device *md)
1252{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001253 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254 struct bio *def;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001255 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001257 down(&md->suspend_lock);
1258 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001259 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001260
1261 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001262 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001263 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264
1265 dm_table_resume_targets(map);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001266
1267 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 clear_bit(DMF_BLOCK_IO, &md->flags);
1269
1270 def = bio_list_get(&md->deferred);
1271 __flush_deferred_io(md, def);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001272 up_write(&md->io_lock);
1273
1274 unlock_fs(md);
1275
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001276 bdput(md->suspended_bdev);
1277 md->suspended_bdev = NULL;
1278
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001279 clear_bit(DMF_SUSPENDED, &md->flags);
1280
Linus Torvalds1da177e2005-04-16 15:20:36 -07001281 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001283 r = 0;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001284
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001285out:
1286 dm_table_put(map);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001287 up(&md->suspend_lock);
1288
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001289 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290}
1291
1292/*-----------------------------------------------------------------
1293 * Event notification.
1294 *---------------------------------------------------------------*/
1295uint32_t dm_get_event_nr(struct mapped_device *md)
1296{
1297 return atomic_read(&md->event_nr);
1298}
1299
1300int dm_wait_event(struct mapped_device *md, int event_nr)
1301{
1302 return wait_event_interruptible(md->eventq,
1303 (event_nr != atomic_read(&md->event_nr)));
1304}
1305
1306/*
1307 * The gendisk is only valid as long as you have a reference
1308 * count on 'md'.
1309 */
1310struct gendisk *dm_disk(struct mapped_device *md)
1311{
1312 return md->disk;
1313}
1314
1315int dm_suspended(struct mapped_device *md)
1316{
1317 return test_bit(DMF_SUSPENDED, &md->flags);
1318}
1319
1320static struct block_device_operations dm_blk_dops = {
1321 .open = dm_blk_open,
1322 .release = dm_blk_close,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001323 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 .owner = THIS_MODULE
1325};
1326
1327EXPORT_SYMBOL(dm_get_mapinfo);
1328
1329/*
1330 * module hooks
1331 */
1332module_init(dm_init);
1333module_exit(dm_exit);
1334
1335module_param(major, uint, 0);
1336MODULE_PARM_DESC(major, "The major number of the device mapper");
1337MODULE_DESCRIPTION(DM_NAME " driver");
1338MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1339MODULE_LICENSE("GPL");