blob: 59806e67a1758fc09a150f0abe3db33d81d25725 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Milan Broz784aae72009-01-06 03:05:12 +00003 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +01009#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070010
11#include <linux/init.h>
12#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080013#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/moduleparam.h>
15#include <linux/blkpg.h>
16#include <linux/bio.h>
17#include <linux/buffer_head.h>
18#include <linux/mempool.h>
19#include <linux/slab.h>
20#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080021#include <linux/hdreg.h>
Li Zefan55782132009-06-09 13:43:05 +080022
23#include <trace/events/block.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Alasdair G Kergon72d94862006-06-26 00:27:35 -070025#define DM_MSG_PREFIX "core"
26
Linus Torvalds1da177e2005-04-16 15:20:36 -070027static const char *_name = DM_NAME;
28
29static unsigned int major = 0;
30static unsigned int _major = 0;
31
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070032static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070033/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000034 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070035 * One of these is allocated per bio.
36 */
37struct dm_io {
38 struct mapped_device *md;
39 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010041 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080042 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070043};
44
45/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000046 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 * One of these is allocated per target within a bio. Hopefully
48 * this will be simplified out one day.
49 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +010050struct dm_target_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 struct dm_io *io;
52 struct dm_target *ti;
53 union map_info info;
54};
55
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000056/*
57 * For request-based dm.
58 * One of these is allocated per request.
59 */
60struct dm_rq_target_io {
61 struct mapped_device *md;
62 struct dm_target *ti;
63 struct request *orig, clone;
64 int error;
65 union map_info info;
66};
67
68/*
69 * For request-based dm.
70 * One of these is allocated per bio.
71 */
72struct dm_rq_clone_bio_info {
73 struct bio *orig;
74 struct request *rq;
75};
76
Linus Torvalds1da177e2005-04-16 15:20:36 -070077union map_info *dm_get_mapinfo(struct bio *bio)
78{
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070079 if (bio && bio->bi_private)
Alasdair G Kergon028867a2007-07-12 17:26:32 +010080 return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070081 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070082}
83
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070084#define MINOR_ALLOCED ((void *)-1)
85
Linus Torvalds1da177e2005-04-16 15:20:36 -070086/*
87 * Bits for the md->flags field.
88 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +010089#define DMF_BLOCK_IO_FOR_SUSPEND 0
Linus Torvalds1da177e2005-04-16 15:20:36 -070090#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080091#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -070092#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070093#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080094#define DMF_NOFLUSH_SUSPENDING 5
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +010095#define DMF_QUEUE_IO_TO_THREAD 6
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Milan Broz304f3f62008-02-08 02:11:17 +000097/*
98 * Work processed by per-device workqueue.
99 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700101 struct rw_semaphore io_lock;
Daniel Walkere61290a2008-02-08 02:10:08 +0000102 struct mutex suspend_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 rwlock_t map_lock;
104 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700105 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
107 unsigned long flags;
108
Jens Axboe165125e2007-07-24 09:28:11 +0200109 struct request_queue *queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -0800111 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
113 void *interface_ptr;
114
115 /*
116 * A list of ios that arrived while we were suspended.
117 */
118 atomic_t pending;
119 wait_queue_head_t wait;
Mikulas Patocka53d59142009-04-02 19:55:37 +0100120 struct work_struct work;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800121 struct bio_list deferred;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100122 spinlock_t deferred_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123
124 /*
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100125 * An error from the barrier request currently being processed.
126 */
127 int barrier_error;
128
129 /*
Milan Broz304f3f62008-02-08 02:11:17 +0000130 * Processing queue (flush/barriers)
131 */
132 struct workqueue_struct *wq;
133
134 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 * The current mapping.
136 */
137 struct dm_table *map;
138
139 /*
140 * io objects are allocated from here.
141 */
142 mempool_t *io_pool;
143 mempool_t *tio_pool;
144
Stefan Bader9faf4002006-10-03 01:15:41 -0700145 struct bio_set *bs;
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 /*
148 * Event handling.
149 */
150 atomic_t event_nr;
151 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100152 atomic_t uevent_seq;
153 struct list_head uevent_list;
154 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155
156 /*
157 * freeze/thaw support require holding onto a super block
158 */
159 struct super_block *frozen_sb;
Alasdair G Kergone39e2e92006-01-06 00:20:05 -0800160 struct block_device *suspended_bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800161
162 /* forced geometry settings */
163 struct hd_geometry geometry;
Milan Broz784aae72009-01-06 03:05:12 +0000164
165 /* sysfs handle */
166 struct kobject kobj;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167};
168
169#define MIN_IOS 256
Christoph Lametere18b8902006-12-06 20:33:20 -0800170static struct kmem_cache *_io_cache;
171static struct kmem_cache *_tio_cache;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000172static struct kmem_cache *_rq_tio_cache;
173static struct kmem_cache *_rq_bio_info_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175static int __init local_init(void)
176{
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100177 int r = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100180 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 if (!_io_cache)
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100182 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183
184 /* allocate a slab for the target ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100185 _tio_cache = KMEM_CACHE(dm_target_io, 0);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100186 if (!_tio_cache)
187 goto out_free_io_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000189 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
190 if (!_rq_tio_cache)
191 goto out_free_tio_cache;
192
193 _rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
194 if (!_rq_bio_info_cache)
195 goto out_free_rq_tio_cache;
196
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100197 r = dm_uevent_init();
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100198 if (r)
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000199 goto out_free_rq_bio_info_cache;
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100200
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 _major = major;
202 r = register_blkdev(_major, _name);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100203 if (r < 0)
204 goto out_uevent_exit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205
206 if (!_major)
207 _major = r;
208
209 return 0;
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100210
211out_uevent_exit:
212 dm_uevent_exit();
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000213out_free_rq_bio_info_cache:
214 kmem_cache_destroy(_rq_bio_info_cache);
215out_free_rq_tio_cache:
216 kmem_cache_destroy(_rq_tio_cache);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100217out_free_tio_cache:
218 kmem_cache_destroy(_tio_cache);
219out_free_io_cache:
220 kmem_cache_destroy(_io_cache);
221
222 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223}
224
225static void local_exit(void)
226{
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000227 kmem_cache_destroy(_rq_bio_info_cache);
228 kmem_cache_destroy(_rq_tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 kmem_cache_destroy(_tio_cache);
230 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700231 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100232 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
234 _major = 0;
235
236 DMINFO("cleaned up");
237}
238
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000239static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 local_init,
241 dm_target_init,
242 dm_linear_init,
243 dm_stripe_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100244 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 dm_interface_init,
246};
247
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000248static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 local_exit,
250 dm_target_exit,
251 dm_linear_exit,
252 dm_stripe_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100253 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 dm_interface_exit,
255};
256
257static int __init dm_init(void)
258{
259 const int count = ARRAY_SIZE(_inits);
260
261 int r, i;
262
263 for (i = 0; i < count; i++) {
264 r = _inits[i]();
265 if (r)
266 goto bad;
267 }
268
269 return 0;
270
271 bad:
272 while (i--)
273 _exits[i]();
274
275 return r;
276}
277
278static void __exit dm_exit(void)
279{
280 int i = ARRAY_SIZE(_exits);
281
282 while (i--)
283 _exits[i]();
284}
285
286/*
287 * Block device functions
288 */
Al Virofe5f9f22008-03-02 10:29:31 -0500289static int dm_blk_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290{
291 struct mapped_device *md;
292
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700293 spin_lock(&_minor_lock);
294
Al Virofe5f9f22008-03-02 10:29:31 -0500295 md = bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700296 if (!md)
297 goto out;
298
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700299 if (test_bit(DMF_FREEING, &md->flags) ||
300 test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700301 md = NULL;
302 goto out;
303 }
304
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700306 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700307
308out:
309 spin_unlock(&_minor_lock);
310
311 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312}
313
Al Virofe5f9f22008-03-02 10:29:31 -0500314static int dm_blk_close(struct gendisk *disk, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315{
Al Virofe5f9f22008-03-02 10:29:31 -0500316 struct mapped_device *md = disk->private_data;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700317 atomic_dec(&md->open_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 dm_put(md);
319 return 0;
320}
321
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700322int dm_open_count(struct mapped_device *md)
323{
324 return atomic_read(&md->open_count);
325}
326
327/*
328 * Guarantees nothing is using the device before it's deleted.
329 */
330int dm_lock_for_deletion(struct mapped_device *md)
331{
332 int r = 0;
333
334 spin_lock(&_minor_lock);
335
336 if (dm_open_count(md))
337 r = -EBUSY;
338 else
339 set_bit(DMF_DELETING, &md->flags);
340
341 spin_unlock(&_minor_lock);
342
343 return r;
344}
345
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800346static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
347{
348 struct mapped_device *md = bdev->bd_disk->private_data;
349
350 return dm_get_geometry(md, geo);
351}
352
Al Virofe5f9f22008-03-02 10:29:31 -0500353static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
Milan Brozaa129a22006-10-03 01:15:15 -0700354 unsigned int cmd, unsigned long arg)
355{
Al Virofe5f9f22008-03-02 10:29:31 -0500356 struct mapped_device *md = bdev->bd_disk->private_data;
357 struct dm_table *map = dm_get_table(md);
Milan Brozaa129a22006-10-03 01:15:15 -0700358 struct dm_target *tgt;
359 int r = -ENOTTY;
360
Milan Brozaa129a22006-10-03 01:15:15 -0700361 if (!map || !dm_table_get_size(map))
362 goto out;
363
364 /* We only support devices that have a single target */
365 if (dm_table_get_num_targets(map) != 1)
366 goto out;
367
368 tgt = dm_table_get_target(map, 0);
369
370 if (dm_suspended(md)) {
371 r = -EAGAIN;
372 goto out;
373 }
374
375 if (tgt->type->ioctl)
Al Viro647b3d02007-08-28 22:15:59 -0400376 r = tgt->type->ioctl(tgt, cmd, arg);
Milan Brozaa129a22006-10-03 01:15:15 -0700377
378out:
379 dm_table_put(map);
380
Milan Brozaa129a22006-10-03 01:15:15 -0700381 return r;
382}
383
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100384static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385{
386 return mempool_alloc(md->io_pool, GFP_NOIO);
387}
388
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100389static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390{
391 mempool_free(io, md->io_pool);
392}
393
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100394static struct dm_target_io *alloc_tio(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395{
396 return mempool_alloc(md->tio_pool, GFP_NOIO);
397}
398
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100399static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400{
401 mempool_free(tio, md->tio_pool);
402}
403
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800404static void start_io_acct(struct dm_io *io)
405{
406 struct mapped_device *md = io->md;
Tejun Heoc9959052008-08-25 19:47:21 +0900407 int cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800408
409 io->start_time = jiffies;
410
Tejun Heo074a7ac2008-08-25 19:56:14 +0900411 cpu = part_stat_lock();
412 part_round_stats(cpu, &dm_disk(md)->part0);
413 part_stat_unlock();
414 dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800415}
416
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000417static void end_io_acct(struct dm_io *io)
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800418{
419 struct mapped_device *md = io->md;
420 struct bio *bio = io->bio;
421 unsigned long duration = jiffies - io->start_time;
Tejun Heoc9959052008-08-25 19:47:21 +0900422 int pending, cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800423 int rw = bio_data_dir(bio);
424
Tejun Heo074a7ac2008-08-25 19:56:14 +0900425 cpu = part_stat_lock();
426 part_round_stats(cpu, &dm_disk(md)->part0);
427 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
428 part_stat_unlock();
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800429
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100430 /*
431 * After this is decremented the bio must not be touched if it is
432 * a barrier.
433 */
Tejun Heo074a7ac2008-08-25 19:56:14 +0900434 dm_disk(md)->part0.in_flight = pending =
435 atomic_dec_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800436
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000437 /* nudge anyone waiting on suspend queue */
438 if (!pending)
439 wake_up(&md->wait);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800440}
441
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442/*
443 * Add the bio to the list of deferred io.
444 */
Mikulas Patocka92c63902009-04-09 00:27:15 +0100445static void queue_io(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700447 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448
Mikulas Patocka022c2612009-04-02 19:55:39 +0100449 spin_lock_irq(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 bio_list_add(&md->deferred, bio);
Mikulas Patocka022c2612009-04-02 19:55:39 +0100451 spin_unlock_irq(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452
Mikulas Patocka92c63902009-04-09 00:27:15 +0100453 if (!test_and_set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags))
454 queue_work(md->wq, &md->work);
455
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700456 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457}
458
459/*
460 * Everyone (including functions in this file), should use this
461 * function to access the md->map field, and make sure they call
462 * dm_table_put() when finished.
463 */
464struct dm_table *dm_get_table(struct mapped_device *md)
465{
466 struct dm_table *t;
467
468 read_lock(&md->map_lock);
469 t = md->map;
470 if (t)
471 dm_table_get(t);
472 read_unlock(&md->map_lock);
473
474 return t;
475}
476
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800477/*
478 * Get the geometry associated with a dm device
479 */
480int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
481{
482 *geo = md->geometry;
483
484 return 0;
485}
486
487/*
488 * Set the geometry of a device.
489 */
490int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
491{
492 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
493
494 if (geo->start > sz) {
495 DMWARN("Start sector is beyond the geometry limits.");
496 return -EINVAL;
497 }
498
499 md->geometry = *geo;
500
501 return 0;
502}
503
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504/*-----------------------------------------------------------------
505 * CRUD START:
506 * A more elegant soln is in the works that uses the queue
507 * merge fn, unfortunately there are a couple of changes to
508 * the block layer that I want to make for this. So in the
509 * interests of getting something for people to use I give
510 * you this clearly demarcated crap.
511 *---------------------------------------------------------------*/
512
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800513static int __noflush_suspending(struct mapped_device *md)
514{
515 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
516}
517
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518/*
519 * Decrements the number of outstanding ios that a bio has been
520 * cloned into, completing the original io if necc.
521 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800522static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800524 unsigned long flags;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000525 int io_error;
526 struct bio *bio;
527 struct mapped_device *md = io->md;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800528
529 /* Push-back supersedes any I/O errors */
Milan Brozb35f8ca2009-03-16 17:44:36 +0000530 if (error && !(io->error > 0 && __noflush_suspending(md)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 io->error = error;
532
533 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800534 if (io->error == DM_ENDIO_REQUEUE) {
535 /*
536 * Target requested pushing back the I/O.
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800537 */
Mikulas Patocka022c2612009-04-02 19:55:39 +0100538 spin_lock_irqsave(&md->deferred_lock, flags);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000539 if (__noflush_suspending(md))
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100540 bio_list_add_head(&md->deferred, io->bio);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800541 else
542 /* noflush suspend was interrupted. */
543 io->error = -EIO;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100544 spin_unlock_irqrestore(&md->deferred_lock, flags);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800545 }
546
Milan Brozb35f8ca2009-03-16 17:44:36 +0000547 io_error = io->error;
548 bio = io->bio;
Jens Axboe2056a782006-03-23 20:00:26 +0100549
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100550 if (bio_barrier(bio)) {
551 /*
552 * There can be just one barrier request so we use
553 * a per-device variable for error reporting.
554 * Note that you can't touch the bio after end_io_acct
555 */
556 md->barrier_error = io_error;
557 end_io_acct(io);
558 } else {
559 end_io_acct(io);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000560
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100561 if (io_error != DM_ENDIO_REQUEUE) {
562 trace_block_bio_complete(md->queue, bio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000563
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100564 bio_endio(bio, io_error);
565 }
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800566 }
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100567
568 free_io(md, io);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 }
570}
571
NeilBrown6712ecf2007-09-27 12:47:43 +0200572static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573{
574 int r = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100575 struct dm_target_io *tio = bio->bi_private;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000576 struct dm_io *io = tio->io;
Stefan Bader9faf4002006-10-03 01:15:41 -0700577 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 dm_endio_fn endio = tio->ti->type->end_io;
579
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
581 error = -EIO;
582
583 if (endio) {
584 r = endio(tio->ti, bio, error, &tio->info);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800585 if (r < 0 || r == DM_ENDIO_REQUEUE)
586 /*
587 * error and requeue request are handled
588 * in dec_pending().
589 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800591 else if (r == DM_ENDIO_INCOMPLETE)
592 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200593 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800594 else if (r) {
595 DMWARN("unimplemented target endio return value: %d", r);
596 BUG();
597 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 }
599
Stefan Bader9faf4002006-10-03 01:15:41 -0700600 /*
601 * Store md for cleanup instead of tio which is about to get freed.
602 */
603 bio->bi_private = md->bs;
604
Stefan Bader9faf4002006-10-03 01:15:41 -0700605 free_tio(md, tio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000606 bio_put(bio);
607 dec_pending(io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608}
609
610static sector_t max_io_len(struct mapped_device *md,
611 sector_t sector, struct dm_target *ti)
612{
613 sector_t offset = sector - ti->begin;
614 sector_t len = ti->len - offset;
615
616 /*
617 * Does the target need to split even further ?
618 */
619 if (ti->split_io) {
620 sector_t boundary;
621 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
622 - offset;
623 if (len > boundary)
624 len = boundary;
625 }
626
627 return len;
628}
629
630static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100631 struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632{
633 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100634 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -0700635 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636
637 /*
638 * Sanity checks.
639 */
640 BUG_ON(!clone->bi_size);
641
642 clone->bi_end_io = clone_endio;
643 clone->bi_private = tio;
644
645 /*
646 * Map the clone. If r == 0 we don't need to do
647 * anything, the target has assumed ownership of
648 * this io.
649 */
650 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100651 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652 r = ti->type->map(ti, clone, &tio->info);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800653 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100655
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100656 trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
Alan D. Brunelle22a7c312009-05-04 16:35:08 -0400657 tio->io->bio->bi_bdev->bd_dev, sector);
Jens Axboe2056a782006-03-23 20:00:26 +0100658
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800660 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
661 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -0700662 md = tio->io->md;
663 dec_pending(tio->io, r);
664 /*
665 * Store bio_set for cleanup.
666 */
667 clone->bi_private = md->bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 bio_put(clone);
Stefan Bader9faf4002006-10-03 01:15:41 -0700669 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800670 } else if (r) {
671 DMWARN("unimplemented target map return value: %d", r);
672 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 }
674}
675
676struct clone_info {
677 struct mapped_device *md;
678 struct dm_table *map;
679 struct bio *bio;
680 struct dm_io *io;
681 sector_t sector;
682 sector_t sector_count;
683 unsigned short idx;
684};
685
Peter Osterlund36763472005-09-06 15:16:42 -0700686static void dm_bio_destructor(struct bio *bio)
687{
Stefan Bader9faf4002006-10-03 01:15:41 -0700688 struct bio_set *bs = bio->bi_private;
689
690 bio_free(bio, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700691}
692
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693/*
694 * Creates a little bio that is just does part of a bvec.
695 */
696static struct bio *split_bvec(struct bio *bio, sector_t sector,
697 unsigned short idx, unsigned int offset,
Stefan Bader9faf4002006-10-03 01:15:41 -0700698 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699{
700 struct bio *clone;
701 struct bio_vec *bv = bio->bi_io_vec + idx;
702
Stefan Bader9faf4002006-10-03 01:15:41 -0700703 clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700704 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 *clone->bi_io_vec = *bv;
706
707 clone->bi_sector = sector;
708 clone->bi_bdev = bio->bi_bdev;
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100709 clone->bi_rw = bio->bi_rw & ~(1 << BIO_RW_BARRIER);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 clone->bi_vcnt = 1;
711 clone->bi_size = to_bytes(len);
712 clone->bi_io_vec->bv_offset = offset;
713 clone->bi_io_vec->bv_len = clone->bi_size;
Martin K. Petersenf3e1d262008-10-21 17:45:04 +0100714 clone->bi_flags |= 1 << BIO_CLONED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715
Martin K. Petersen9c470082009-04-09 00:27:12 +0100716 if (bio_integrity(bio)) {
717 bio_integrity_clone(clone, bio, GFP_NOIO);
718 bio_integrity_trim(clone,
719 bio_sector_offset(bio, idx, offset), len);
720 }
721
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 return clone;
723}
724
725/*
726 * Creates a bio that consists of range of complete bvecs.
727 */
728static struct bio *clone_bio(struct bio *bio, sector_t sector,
729 unsigned short idx, unsigned short bv_count,
Stefan Bader9faf4002006-10-03 01:15:41 -0700730 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731{
732 struct bio *clone;
733
Stefan Bader9faf4002006-10-03 01:15:41 -0700734 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
735 __bio_clone(clone, bio);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100736 clone->bi_rw &= ~(1 << BIO_RW_BARRIER);
Stefan Bader9faf4002006-10-03 01:15:41 -0700737 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 clone->bi_sector = sector;
739 clone->bi_idx = idx;
740 clone->bi_vcnt = idx + bv_count;
741 clone->bi_size = to_bytes(len);
742 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
743
Martin K. Petersen9c470082009-04-09 00:27:12 +0100744 if (bio_integrity(bio)) {
745 bio_integrity_clone(clone, bio, GFP_NOIO);
746
747 if (idx != bio->bi_idx || clone->bi_size < bio->bi_size)
748 bio_integrity_trim(clone,
749 bio_sector_offset(bio, idx, 0), len);
750 }
751
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 return clone;
753}
754
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000755static int __clone_and_map(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756{
757 struct bio *clone, *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000758 struct dm_target *ti;
759 sector_t len = 0, max;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100760 struct dm_target_io *tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000762 ti = dm_table_find_target(ci->map, ci->sector);
763 if (!dm_target_is_valid(ti))
764 return -EIO;
765
766 max = max_io_len(ci->md, ci->sector, ti);
767
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768 /*
769 * Allocate a target io object.
770 */
771 tio = alloc_tio(ci->md);
772 tio->io = ci->io;
773 tio->ti = ti;
774 memset(&tio->info, 0, sizeof(tio->info));
775
776 if (ci->sector_count <= max) {
777 /*
778 * Optimise for the simple case where we can do all of
779 * the remaining io with a single clone.
780 */
781 clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700782 bio->bi_vcnt - ci->idx, ci->sector_count,
783 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 __map_bio(ti, clone, tio);
785 ci->sector_count = 0;
786
787 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
788 /*
789 * There are some bvecs that don't span targets.
790 * Do as many of these as possible.
791 */
792 int i;
793 sector_t remaining = max;
794 sector_t bv_len;
795
796 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
797 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
798
799 if (bv_len > remaining)
800 break;
801
802 remaining -= bv_len;
803 len += bv_len;
804 }
805
Stefan Bader9faf4002006-10-03 01:15:41 -0700806 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
807 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808 __map_bio(ti, clone, tio);
809
810 ci->sector += len;
811 ci->sector_count -= len;
812 ci->idx = i;
813
814 } else {
815 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800816 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817 */
818 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800819 sector_t remaining = to_sector(bv->bv_len);
820 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800822 do {
823 if (offset) {
824 ti = dm_table_find_target(ci->map, ci->sector);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000825 if (!dm_target_is_valid(ti))
826 return -EIO;
827
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800828 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800830 tio = alloc_tio(ci->md);
831 tio->io = ci->io;
832 tio->ti = ti;
833 memset(&tio->info, 0, sizeof(tio->info));
834 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800836 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800838 clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700839 bv->bv_offset + offset, len,
840 ci->md->bs);
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800841
842 __map_bio(ti, clone, tio);
843
844 ci->sector += len;
845 ci->sector_count -= len;
846 offset += to_bytes(len);
847 } while (remaining -= len);
848
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849 ci->idx++;
850 }
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000851
852 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853}
854
855/*
Mikulas Patocka8a53c282009-04-02 19:55:37 +0100856 * Split the bio into several clones and submit it to targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 */
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100858static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859{
860 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000861 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862
863 ci.map = dm_get_table(md);
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100864 if (unlikely(!ci.map)) {
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100865 if (!bio_barrier(bio))
866 bio_io_error(bio);
867 else
868 md->barrier_error = -EIO;
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100869 return;
870 }
Mikulas Patocka692d0eb2009-04-09 00:27:13 +0100871
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872 ci.md = md;
873 ci.bio = bio;
874 ci.io = alloc_io(md);
875 ci.io->error = 0;
876 atomic_set(&ci.io->io_count, 1);
877 ci.io->bio = bio;
878 ci.io->md = md;
879 ci.sector = bio->bi_sector;
880 ci.sector_count = bio_sectors(bio);
881 ci.idx = bio->bi_idx;
882
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800883 start_io_acct(ci.io);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000884 while (ci.sector_count && !error)
885 error = __clone_and_map(&ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886
887 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000888 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 dm_table_put(ci.map);
890}
891/*-----------------------------------------------------------------
892 * CRUD END
893 *---------------------------------------------------------------*/
894
Milan Brozf6fccb12008-07-21 12:00:37 +0100895static int dm_merge_bvec(struct request_queue *q,
896 struct bvec_merge_data *bvm,
897 struct bio_vec *biovec)
898{
899 struct mapped_device *md = q->queuedata;
900 struct dm_table *map = dm_get_table(md);
901 struct dm_target *ti;
902 sector_t max_sectors;
Mikulas Patocka50371082008-10-01 14:39:17 +0100903 int max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +0100904
905 if (unlikely(!map))
Mikulas Patocka50371082008-10-01 14:39:17 +0100906 goto out;
Milan Brozf6fccb12008-07-21 12:00:37 +0100907
908 ti = dm_table_find_target(map, bvm->bi_sector);
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100909 if (!dm_target_is_valid(ti))
910 goto out_table;
Milan Brozf6fccb12008-07-21 12:00:37 +0100911
912 /*
913 * Find maximum amount of I/O that won't need splitting
914 */
915 max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
916 (sector_t) BIO_MAX_SECTORS);
917 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
918 if (max_size < 0)
919 max_size = 0;
920
921 /*
922 * merge_bvec_fn() returns number of bytes
923 * it can accept at this offset
924 * max is precomputed maximal io size
925 */
926 if (max_size && ti->type->merge)
927 max_size = ti->type->merge(ti, bvm, biovec, max_size);
Mikulas Patocka8cbeb672009-06-22 10:12:14 +0100928 /*
929 * If the target doesn't support merge method and some of the devices
930 * provided their merge_bvec method (we know this by looking at
931 * queue_max_hw_sectors), then we can't allow bios with multiple vector
932 * entries. So always set max_size to 0, and the code below allows
933 * just one page.
934 */
935 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
936
937 max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +0100938
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100939out_table:
Mikulas Patocka50371082008-10-01 14:39:17 +0100940 dm_table_put(map);
941
942out:
Milan Brozf6fccb12008-07-21 12:00:37 +0100943 /*
944 * Always allow an entire first page
945 */
946 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
947 max_size = biovec->bv_len;
948
Milan Brozf6fccb12008-07-21 12:00:37 +0100949 return max_size;
950}
951
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952/*
953 * The request function that just remaps the bio built up by
954 * dm_merge_bvec.
955 */
Jens Axboe165125e2007-07-24 09:28:11 +0200956static int dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957{
Kevin Corry12f03a42006-02-01 03:04:52 -0800958 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 struct mapped_device *md = q->queuedata;
Tejun Heoc9959052008-08-25 19:47:21 +0900960 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700962 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963
Tejun Heo074a7ac2008-08-25 19:56:14 +0900964 cpu = part_stat_lock();
965 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
966 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
967 part_stat_unlock();
Kevin Corry12f03a42006-02-01 03:04:52 -0800968
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 /*
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +0100970 * If we're suspended or the thread is processing barriers
971 * we have to queue this io for later.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 */
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100973 if (unlikely(test_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags)) ||
974 unlikely(bio_barrier(bio))) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700975 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976
Alasdair G Kergon54d9a1b2009-04-09 00:27:14 +0100977 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) &&
978 bio_rw(bio) == READA) {
979 bio_io_error(bio);
980 return 0;
981 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982
Mikulas Patocka92c63902009-04-09 00:27:15 +0100983 queue_io(md, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984
Mikulas Patocka92c63902009-04-09 00:27:15 +0100985 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 }
987
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100988 __split_and_process_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700989 up_read(&md->io_lock);
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100990 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991}
992
Jens Axboe165125e2007-07-24 09:28:11 +0200993static void dm_unplug_all(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994{
995 struct mapped_device *md = q->queuedata;
996 struct dm_table *map = dm_get_table(md);
997
998 if (map) {
999 dm_table_unplug_all(map);
1000 dm_table_put(map);
1001 }
1002}
1003
1004static int dm_any_congested(void *congested_data, int bdi_bits)
1005{
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001006 int r = bdi_bits;
1007 struct mapped_device *md = congested_data;
1008 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001010 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001011 map = dm_get_table(md);
1012 if (map) {
1013 r = dm_table_any_congested(map, bdi_bits);
1014 dm_table_put(map);
1015 }
1016 }
1017
Linus Torvalds1da177e2005-04-16 15:20:36 -07001018 return r;
1019}
1020
1021/*-----------------------------------------------------------------
1022 * An IDR is used to keep track of allocated minor numbers.
1023 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024static DEFINE_IDR(_minor_idr);
1025
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001026static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001028 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001030 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031}
1032
1033/*
1034 * See if the device with a specific minor # is free.
1035 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001036static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037{
1038 int r, m;
1039
1040 if (minor >= (1 << MINORBITS))
1041 return -EINVAL;
1042
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001043 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
1044 if (!r)
1045 return -ENOMEM;
1046
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001047 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048
1049 if (idr_find(&_minor_idr, minor)) {
1050 r = -EBUSY;
1051 goto out;
1052 }
1053
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001054 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001055 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057
1058 if (m != minor) {
1059 idr_remove(&_minor_idr, m);
1060 r = -EBUSY;
1061 goto out;
1062 }
1063
1064out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001065 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001066 return r;
1067}
1068
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001069static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001071 int r, m;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001074 if (!r)
1075 return -ENOMEM;
1076
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001077 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001079 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001080 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082
1083 if (m >= (1 << MINORBITS)) {
1084 idr_remove(&_minor_idr, m);
1085 r = -ENOSPC;
1086 goto out;
1087 }
1088
1089 *minor = m;
1090
1091out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001092 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093 return r;
1094}
1095
1096static struct block_device_operations dm_blk_dops;
1097
Mikulas Patocka53d59142009-04-02 19:55:37 +01001098static void dm_wq_work(struct work_struct *work);
1099
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100/*
1101 * Allocate and initialise a blank device with a given minor.
1102 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001103static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104{
1105 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001106 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001107 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
1109 if (!md) {
1110 DMWARN("unable to allocate device, out of memory.");
1111 return NULL;
1112 }
1113
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001114 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001115 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001116
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001118 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001119 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001120 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001121 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001123 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001125 init_rwsem(&md->io_lock);
Daniel Walkere61290a2008-02-08 02:10:08 +00001126 mutex_init(&md->suspend_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001127 spin_lock_init(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128 rwlock_init(&md->map_lock);
1129 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001130 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001132 atomic_set(&md->uevent_seq, 0);
1133 INIT_LIST_HEAD(&md->uevent_list);
1134 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135
1136 md->queue = blk_alloc_queue(GFP_KERNEL);
1137 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001138 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139
1140 md->queue->queuedata = md;
1141 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1142 md->queue->backing_dev_info.congested_data = md;
1143 blk_queue_make_request(md->queue, dm_request);
Mikulas Patocka99360b42009-04-02 19:55:39 +01001144 blk_queue_ordered(md->queue, QUEUE_ORDERED_DRAIN, NULL);
Jens Axboedaef2652006-01-10 10:48:02 +01001145 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146 md->queue->unplug_fn = dm_unplug_all;
Milan Brozf6fccb12008-07-21 12:00:37 +01001147 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148
Matthew Dobson93d23412006-03-26 01:37:50 -08001149 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
Kiyoshi Ueda74859362006-12-08 02:41:02 -08001150 if (!md->io_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001151 goto bad_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152
Matthew Dobson93d23412006-03-26 01:37:50 -08001153 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154 if (!md->tio_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001155 goto bad_tio_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156
Jens Axboebb799ca2008-12-10 15:35:05 +01001157 md->bs = bioset_create(16, 0);
Stefan Bader9faf4002006-10-03 01:15:41 -07001158 if (!md->bs)
1159 goto bad_no_bioset;
1160
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 md->disk = alloc_disk(1);
1162 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001163 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001165 atomic_set(&md->pending, 0);
1166 init_waitqueue_head(&md->wait);
Mikulas Patocka53d59142009-04-02 19:55:37 +01001167 INIT_WORK(&md->work, dm_wq_work);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001168 init_waitqueue_head(&md->eventq);
1169
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170 md->disk->major = _major;
1171 md->disk->first_minor = minor;
1172 md->disk->fops = &dm_blk_dops;
1173 md->disk->queue = md->queue;
1174 md->disk->private_data = md;
1175 sprintf(md->disk->disk_name, "dm-%d", minor);
1176 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001177 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178
Milan Broz304f3f62008-02-08 02:11:17 +00001179 md->wq = create_singlethread_workqueue("kdmflush");
1180 if (!md->wq)
1181 goto bad_thread;
1182
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001183 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001184 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001185 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001186 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001187
1188 BUG_ON(old_md != MINOR_ALLOCED);
1189
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190 return md;
1191
Milan Broz304f3f62008-02-08 02:11:17 +00001192bad_thread:
1193 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001194bad_disk:
Stefan Bader9faf4002006-10-03 01:15:41 -07001195 bioset_free(md->bs);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001196bad_no_bioset:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 mempool_destroy(md->tio_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001198bad_tio_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 mempool_destroy(md->io_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001200bad_io_pool:
Al Viro1312f402006-03-12 11:02:03 -05001201 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001202bad_queue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001204bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001205 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001206bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207 kfree(md);
1208 return NULL;
1209}
1210
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001211static void unlock_fs(struct mapped_device *md);
1212
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213static void free_dev(struct mapped_device *md)
1214{
Tejun Heof331c022008-09-03 09:01:48 +02001215 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001216
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001217 if (md->suspended_bdev) {
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001218 unlock_fs(md);
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001219 bdput(md->suspended_bdev);
1220 }
Milan Broz304f3f62008-02-08 02:11:17 +00001221 destroy_workqueue(md->wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222 mempool_destroy(md->tio_pool);
1223 mempool_destroy(md->io_pool);
Stefan Bader9faf4002006-10-03 01:15:41 -07001224 bioset_free(md->bs);
Martin K. Petersen9c470082009-04-09 00:27:12 +01001225 blk_integrity_unregister(md->disk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001227 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001228
1229 spin_lock(&_minor_lock);
1230 md->disk->private_data = NULL;
1231 spin_unlock(&_minor_lock);
1232
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05001234 blk_cleanup_queue(md->queue);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001235 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236 kfree(md);
1237}
1238
1239/*
1240 * Bind a table to the device.
1241 */
1242static void event_callback(void *context)
1243{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001244 unsigned long flags;
1245 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246 struct mapped_device *md = (struct mapped_device *) context;
1247
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001248 spin_lock_irqsave(&md->uevent_lock, flags);
1249 list_splice_init(&md->uevent_list, &uevents);
1250 spin_unlock_irqrestore(&md->uevent_lock, flags);
1251
Tejun Heoed9e1982008-08-25 19:56:05 +09001252 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001253
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254 atomic_inc(&md->event_nr);
1255 wake_up(&md->eventq);
1256}
1257
Alasdair G Kergon4e901882005-07-28 21:15:59 -07001258static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259{
Alasdair G Kergon4e901882005-07-28 21:15:59 -07001260 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001261
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001262 mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001263 i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001264 mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265}
1266
1267static int __bind(struct mapped_device *md, struct dm_table *t)
1268{
Jens Axboe165125e2007-07-24 09:28:11 +02001269 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 sector_t size;
1271
1272 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001273
1274 /*
1275 * Wipe any geometry if the size of the table changed.
1276 */
1277 if (size != get_capacity(md->disk))
1278 memset(&md->geometry, 0, sizeof(md->geometry));
1279
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001280 if (md->suspended_bdev)
1281 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282
Mikulas Patockad5816872009-01-06 03:05:10 +00001283 if (!size) {
1284 dm_table_destroy(t);
1285 return 0;
1286 }
1287
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001288 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001289
1290 write_lock(&md->map_lock);
1291 md->map = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 dm_table_set_restrictions(t, q);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001293 write_unlock(&md->map_lock);
1294
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295 return 0;
1296}
1297
1298static void __unbind(struct mapped_device *md)
1299{
1300 struct dm_table *map = md->map;
1301
1302 if (!map)
1303 return;
1304
1305 dm_table_event_callback(map, NULL, NULL);
1306 write_lock(&md->map_lock);
1307 md->map = NULL;
1308 write_unlock(&md->map_lock);
Mikulas Patockad5816872009-01-06 03:05:10 +00001309 dm_table_destroy(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310}
1311
1312/*
1313 * Constructor for a new device.
1314 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001315int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316{
1317 struct mapped_device *md;
1318
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001319 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320 if (!md)
1321 return -ENXIO;
1322
Milan Broz784aae72009-01-06 03:05:12 +00001323 dm_sysfs_init(md);
1324
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325 *result = md;
1326 return 0;
1327}
1328
David Teigland637842c2006-01-06 00:20:00 -08001329static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330{
1331 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001332 unsigned minor = MINOR(dev);
1333
1334 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1335 return NULL;
1336
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001337 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338
1339 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001340 if (md && (md == MINOR_ALLOCED ||
Tejun Heof331c022008-09-03 09:01:48 +02001341 (MINOR(disk_devt(dm_disk(md))) != minor) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07001342 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08001343 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001344 goto out;
1345 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001347out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001348 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349
David Teigland637842c2006-01-06 00:20:00 -08001350 return md;
1351}
1352
David Teiglandd229a952006-01-06 00:20:01 -08001353struct mapped_device *dm_get_md(dev_t dev)
1354{
1355 struct mapped_device *md = dm_find_md(dev);
1356
1357 if (md)
1358 dm_get(md);
1359
1360 return md;
1361}
1362
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001363void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08001364{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001365 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366}
1367
1368void dm_set_mdptr(struct mapped_device *md, void *ptr)
1369{
1370 md->interface_ptr = ptr;
1371}
1372
1373void dm_get(struct mapped_device *md)
1374{
1375 atomic_inc(&md->holders);
1376}
1377
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001378const char *dm_device_name(struct mapped_device *md)
1379{
1380 return md->name;
1381}
1382EXPORT_SYMBOL_GPL(dm_device_name);
1383
Linus Torvalds1da177e2005-04-16 15:20:36 -07001384void dm_put(struct mapped_device *md)
1385{
Mike Anderson1134e5a2006-03-27 01:17:54 -08001386 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001388 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1389
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001390 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08001391 map = dm_get_table(md);
Tejun Heof331c022008-09-03 09:01:48 +02001392 idr_replace(&_minor_idr, MINOR_ALLOCED,
1393 MINOR(disk_devt(dm_disk(md))));
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001394 set_bit(DMF_FREEING, &md->flags);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001395 spin_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001396 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 dm_table_presuspend_targets(map);
1398 dm_table_postsuspend_targets(map);
1399 }
Milan Broz784aae72009-01-06 03:05:12 +00001400 dm_sysfs_exit(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08001401 dm_table_put(map);
Mikulas Patockaa1b51e92009-01-06 03:04:53 +00001402 __unbind(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403 free_dev(md);
1404 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405}
Edward Goggin79eb8852007-05-09 02:32:56 -07001406EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407
Mikulas Patocka401600d2009-04-02 19:55:38 +01001408static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
Milan Broz46125c12008-02-08 02:10:30 +00001409{
1410 int r = 0;
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01001411 DECLARE_WAITQUEUE(wait, current);
1412
1413 dm_unplug_all(md->queue);
1414
1415 add_wait_queue(&md->wait, &wait);
Milan Broz46125c12008-02-08 02:10:30 +00001416
1417 while (1) {
Mikulas Patocka401600d2009-04-02 19:55:38 +01001418 set_current_state(interruptible);
Milan Broz46125c12008-02-08 02:10:30 +00001419
1420 smp_mb();
1421 if (!atomic_read(&md->pending))
1422 break;
1423
Mikulas Patocka401600d2009-04-02 19:55:38 +01001424 if (interruptible == TASK_INTERRUPTIBLE &&
1425 signal_pending(current)) {
Milan Broz46125c12008-02-08 02:10:30 +00001426 r = -EINTR;
1427 break;
1428 }
1429
1430 io_schedule();
1431 }
1432 set_current_state(TASK_RUNNING);
1433
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01001434 remove_wait_queue(&md->wait, &wait);
1435
Milan Broz46125c12008-02-08 02:10:30 +00001436 return r;
1437}
1438
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001439static int dm_flush(struct mapped_device *md)
1440{
1441 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
1442 return 0;
1443}
1444
1445static void process_barrier(struct mapped_device *md, struct bio *bio)
1446{
1447 int error = dm_flush(md);
1448
1449 if (unlikely(error)) {
1450 bio_endio(bio, error);
1451 return;
1452 }
1453 if (bio_empty_barrier(bio)) {
1454 bio_endio(bio, 0);
1455 return;
1456 }
1457
1458 __split_and_process_bio(md, bio);
1459
1460 error = dm_flush(md);
1461
1462 if (!error && md->barrier_error)
1463 error = md->barrier_error;
1464
1465 if (md->barrier_error != DM_ENDIO_REQUEUE)
1466 bio_endio(bio, error);
1467}
1468
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469/*
1470 * Process the deferred bios
1471 */
Mikulas Patockaef208582009-04-02 19:55:38 +01001472static void dm_wq_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473{
Mikulas Patockaef208582009-04-02 19:55:38 +01001474 struct mapped_device *md = container_of(work, struct mapped_device,
1475 work);
Milan Broz6d6f10d2008-02-08 02:10:22 +00001476 struct bio *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477
Mikulas Patockaef208582009-04-02 19:55:38 +01001478 down_write(&md->io_lock);
1479
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001480 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01001481 spin_lock_irq(&md->deferred_lock);
1482 c = bio_list_pop(&md->deferred);
1483 spin_unlock_irq(&md->deferred_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001484
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01001485 if (!c) {
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001486 clear_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01001487 break;
1488 }
1489
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001490 up_write(&md->io_lock);
1491
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001492 if (bio_barrier(c))
1493 process_barrier(md, c);
1494 else
1495 __split_and_process_bio(md, c);
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001496
1497 down_write(&md->io_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001498 }
Milan Broz73d410c2008-02-08 02:10:25 +00001499
Mikulas Patockaef208582009-04-02 19:55:38 +01001500 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501}
1502
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01001503static void dm_queue_flush(struct mapped_device *md)
Milan Broz304f3f62008-02-08 02:11:17 +00001504{
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001505 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
1506 smp_mb__after_clear_bit();
Mikulas Patocka53d59142009-04-02 19:55:37 +01001507 queue_work(md->wq, &md->work);
Milan Broz304f3f62008-02-08 02:11:17 +00001508}
1509
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510/*
1511 * Swap in a new table (destroying old one).
1512 */
1513int dm_swap_table(struct mapped_device *md, struct dm_table *table)
1514{
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001515 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516
Daniel Walkere61290a2008-02-08 02:10:08 +00001517 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518
1519 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001520 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001521 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001523 /* without bdev, the device size cannot be changed */
1524 if (!md->suspended_bdev)
1525 if (get_capacity(md->disk) != dm_table_get_size(table))
1526 goto out;
1527
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528 __unbind(md);
1529 r = __bind(md, table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001531out:
Daniel Walkere61290a2008-02-08 02:10:08 +00001532 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001533 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534}
1535
1536/*
1537 * Functions to lock and unlock any filesystem running on the
1538 * device.
1539 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001540static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001542 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543
1544 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001545
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001546 md->frozen_sb = freeze_bdev(md->suspended_bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001547 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001548 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001549 md->frozen_sb = NULL;
1550 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001551 }
1552
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001553 set_bit(DMF_FROZEN, &md->flags);
1554
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555 /* don't bdput right now, we don't want the bdev
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001556 * to go away while it is locked.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 */
1558 return 0;
1559}
1560
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001561static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001563 if (!test_bit(DMF_FROZEN, &md->flags))
1564 return;
1565
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001566 thaw_bdev(md->suspended_bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001568 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569}
1570
1571/*
1572 * We need to be able to change a mapping table under a mounted
1573 * filesystem. For example we might want to move some data in
1574 * the background. Before the table can be swapped with
1575 * dm_bind_table, dm_suspend must be called to flush any in
1576 * flight bios and ensure that any further io gets deferred.
1577 */
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001578int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001579{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001580 struct dm_table *map = NULL;
Milan Broz46125c12008-02-08 02:10:30 +00001581 int r = 0;
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001582 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001583 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584
Daniel Walkere61290a2008-02-08 02:10:08 +00001585 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001586
Milan Broz73d410c2008-02-08 02:10:25 +00001587 if (dm_suspended(md)) {
1588 r = -EINVAL;
Alasdair G Kergond2874832006-11-08 17:44:43 -08001589 goto out_unlock;
Milan Broz73d410c2008-02-08 02:10:25 +00001590 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001591
1592 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001594 /*
1595 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
1596 * This flag is cleared before dm_suspend returns.
1597 */
1598 if (noflush)
1599 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1600
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001601 /* This does not get reverted if there's an error later. */
1602 dm_table_presuspend_targets(map);
1603
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001604 /* bdget() can stall if the pending I/Os are not flushed */
1605 if (!noflush) {
1606 md->suspended_bdev = bdget_disk(md->disk, 0);
1607 if (!md->suspended_bdev) {
1608 DMWARN("bdget failed in dm_suspend");
1609 r = -ENOMEM;
Kiyoshi Uedaf431d962008-10-21 17:45:07 +01001610 goto out;
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001611 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001612
Milan Broz6d6f10d2008-02-08 02:10:22 +00001613 /*
1614 * Flush I/O to the device. noflush supersedes do_lockfs,
1615 * because lock_fs() needs to flush I/Os.
1616 */
1617 if (do_lockfs) {
1618 r = lock_fs(md);
1619 if (r)
1620 goto out;
1621 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001622 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623
1624 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001625 * Here we must make sure that no processes are submitting requests
1626 * to target drivers i.e. no one may be executing
1627 * __split_and_process_bio. This is called from dm_request and
1628 * dm_wq_work.
1629 *
1630 * To get all processes out of __split_and_process_bio in dm_request,
1631 * we take the write lock. To prevent any process from reentering
1632 * __split_and_process_bio from dm_request, we set
1633 * DMF_QUEUE_IO_TO_THREAD.
1634 *
1635 * To quiesce the thread (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND
1636 * and call flush_workqueue(md->wq). flush_workqueue will wait until
1637 * dm_wq_work exits and DMF_BLOCK_IO_FOR_SUSPEND will prevent any
1638 * further calls to __split_and_process_bio from dm_wq_work.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001640 down_write(&md->io_lock);
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001641 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
1642 set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001643 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001644
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001645 flush_workqueue(md->wq);
1646
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001648 * At this point no more requests are entering target request routines.
1649 * We call dm_wait_for_completion to wait for all existing requests
1650 * to finish.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651 */
Mikulas Patocka401600d2009-04-02 19:55:38 +01001652 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001654 down_write(&md->io_lock);
Milan Broz6d6f10d2008-02-08 02:10:22 +00001655 if (noflush)
Mikulas Patocka022c2612009-04-02 19:55:39 +01001656 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
Milan Broz94d63512008-02-08 02:10:27 +00001657 up_write(&md->io_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001658
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00001660 if (r < 0) {
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01001661 dm_queue_flush(md);
Milan Broz73d410c2008-02-08 02:10:25 +00001662
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001663 unlock_fs(md);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001664 goto out; /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001665 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001666
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001667 /*
1668 * If dm_wait_for_completion returned 0, the device is completely
1669 * quiescent now. There is no request-processing activity. All new
1670 * requests are being added to md->deferred list.
1671 */
1672
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001673 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001674
1675 set_bit(DMF_SUSPENDED, &md->flags);
1676
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001677out:
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001678 if (r && md->suspended_bdev) {
1679 bdput(md->suspended_bdev);
1680 md->suspended_bdev = NULL;
1681 }
1682
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683 dm_table_put(map);
Alasdair G Kergond2874832006-11-08 17:44:43 -08001684
1685out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00001686 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001687 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001688}
1689
1690int dm_resume(struct mapped_device *md)
1691{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001692 int r = -EINVAL;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001693 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694
Daniel Walkere61290a2008-02-08 02:10:08 +00001695 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001696 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001697 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001698
1699 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001700 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001701 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702
Milan Broz8757b772006-10-03 01:15:36 -07001703 r = dm_table_resume_targets(map);
1704 if (r)
1705 goto out;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001706
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01001707 dm_queue_flush(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001708
1709 unlock_fs(md);
1710
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001711 if (md->suspended_bdev) {
1712 bdput(md->suspended_bdev);
1713 md->suspended_bdev = NULL;
1714 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001715
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001716 clear_bit(DMF_SUSPENDED, &md->flags);
1717
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001719
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001720 dm_kobject_uevent(md);
Hannes Reinecke8560ed62006-10-03 01:15:35 -07001721
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001722 r = 0;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001723
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001724out:
1725 dm_table_put(map);
Daniel Walkere61290a2008-02-08 02:10:08 +00001726 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001727
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001728 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729}
1730
1731/*-----------------------------------------------------------------
1732 * Event notification.
1733 *---------------------------------------------------------------*/
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001734void dm_kobject_uevent(struct mapped_device *md)
1735{
Tejun Heoed9e1982008-08-25 19:56:05 +09001736 kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001737}
1738
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001739uint32_t dm_next_uevent_seq(struct mapped_device *md)
1740{
1741 return atomic_add_return(1, &md->uevent_seq);
1742}
1743
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744uint32_t dm_get_event_nr(struct mapped_device *md)
1745{
1746 return atomic_read(&md->event_nr);
1747}
1748
1749int dm_wait_event(struct mapped_device *md, int event_nr)
1750{
1751 return wait_event_interruptible(md->eventq,
1752 (event_nr != atomic_read(&md->event_nr)));
1753}
1754
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001755void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
1756{
1757 unsigned long flags;
1758
1759 spin_lock_irqsave(&md->uevent_lock, flags);
1760 list_add(elist, &md->uevent_list);
1761 spin_unlock_irqrestore(&md->uevent_lock, flags);
1762}
1763
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764/*
1765 * The gendisk is only valid as long as you have a reference
1766 * count on 'md'.
1767 */
1768struct gendisk *dm_disk(struct mapped_device *md)
1769{
1770 return md->disk;
1771}
1772
Milan Broz784aae72009-01-06 03:05:12 +00001773struct kobject *dm_kobject(struct mapped_device *md)
1774{
1775 return &md->kobj;
1776}
1777
1778/*
1779 * struct mapped_device should not be exported outside of dm.c
1780 * so use this check to verify that kobj is part of md structure
1781 */
1782struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
1783{
1784 struct mapped_device *md;
1785
1786 md = container_of(kobj, struct mapped_device, kobj);
1787 if (&md->kobj != kobj)
1788 return NULL;
1789
Milan Broz4d89b7b2009-06-22 10:12:11 +01001790 if (test_bit(DMF_FREEING, &md->flags) ||
1791 test_bit(DMF_DELETING, &md->flags))
1792 return NULL;
1793
Milan Broz784aae72009-01-06 03:05:12 +00001794 dm_get(md);
1795 return md;
1796}
1797
Linus Torvalds1da177e2005-04-16 15:20:36 -07001798int dm_suspended(struct mapped_device *md)
1799{
1800 return test_bit(DMF_SUSPENDED, &md->flags);
1801}
1802
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001803int dm_noflush_suspending(struct dm_target *ti)
1804{
1805 struct mapped_device *md = dm_table_get_md(ti->table);
1806 int r = __noflush_suspending(md);
1807
1808 dm_put(md);
1809
1810 return r;
1811}
1812EXPORT_SYMBOL_GPL(dm_noflush_suspending);
1813
Linus Torvalds1da177e2005-04-16 15:20:36 -07001814static struct block_device_operations dm_blk_dops = {
1815 .open = dm_blk_open,
1816 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07001817 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001818 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001819 .owner = THIS_MODULE
1820};
1821
1822EXPORT_SYMBOL(dm_get_mapinfo);
1823
1824/*
1825 * module hooks
1826 */
1827module_init(dm_init);
1828module_exit(dm_exit);
1829
1830module_param(major, uint, 0);
1831MODULE_PARM_DESC(major, "The major number of the device mapper");
1832MODULE_DESCRIPTION(DM_NAME " driver");
1833MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1834MODULE_LICENSE("GPL");