blob: 910bce85f443ec01b654ccf11f5618619c54bcbd [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Milan Broz784aae72009-01-06 03:05:12 +00003 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +01009#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070010
11#include <linux/init.h>
12#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080013#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/moduleparam.h>
15#include <linux/blkpg.h>
16#include <linux/bio.h>
17#include <linux/buffer_head.h>
18#include <linux/mempool.h>
19#include <linux/slab.h>
20#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080021#include <linux/hdreg.h>
Li Zefan55782132009-06-09 13:43:05 +080022
23#include <trace/events/block.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Alasdair G Kergon72d94862006-06-26 00:27:35 -070025#define DM_MSG_PREFIX "core"
26
Linus Torvalds1da177e2005-04-16 15:20:36 -070027static const char *_name = DM_NAME;
28
29static unsigned int major = 0;
30static unsigned int _major = 0;
31
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070032static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070033/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000034 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070035 * One of these is allocated per bio.
36 */
37struct dm_io {
38 struct mapped_device *md;
39 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010041 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080042 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070043};
44
45/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000046 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 * One of these is allocated per target within a bio. Hopefully
48 * this will be simplified out one day.
49 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +010050struct dm_target_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 struct dm_io *io;
52 struct dm_target *ti;
53 union map_info info;
54};
55
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000056/*
57 * For request-based dm.
58 * One of these is allocated per request.
59 */
60struct dm_rq_target_io {
61 struct mapped_device *md;
62 struct dm_target *ti;
63 struct request *orig, clone;
64 int error;
65 union map_info info;
66};
67
68/*
69 * For request-based dm.
70 * One of these is allocated per bio.
71 */
72struct dm_rq_clone_bio_info {
73 struct bio *orig;
74 struct request *rq;
75};
76
Linus Torvalds1da177e2005-04-16 15:20:36 -070077union map_info *dm_get_mapinfo(struct bio *bio)
78{
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070079 if (bio && bio->bi_private)
Alasdair G Kergon028867a2007-07-12 17:26:32 +010080 return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070081 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070082}
83
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070084#define MINOR_ALLOCED ((void *)-1)
85
Linus Torvalds1da177e2005-04-16 15:20:36 -070086/*
87 * Bits for the md->flags field.
88 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +010089#define DMF_BLOCK_IO_FOR_SUSPEND 0
Linus Torvalds1da177e2005-04-16 15:20:36 -070090#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080091#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -070092#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070093#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080094#define DMF_NOFLUSH_SUSPENDING 5
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +010095#define DMF_QUEUE_IO_TO_THREAD 6
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Milan Broz304f3f62008-02-08 02:11:17 +000097/*
98 * Work processed by per-device workqueue.
99 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700101 struct rw_semaphore io_lock;
Daniel Walkere61290a2008-02-08 02:10:08 +0000102 struct mutex suspend_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 rwlock_t map_lock;
104 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700105 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
107 unsigned long flags;
108
Jens Axboe165125e2007-07-24 09:28:11 +0200109 struct request_queue *queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -0800111 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
113 void *interface_ptr;
114
115 /*
116 * A list of ios that arrived while we were suspended.
117 */
118 atomic_t pending;
119 wait_queue_head_t wait;
Mikulas Patocka53d59142009-04-02 19:55:37 +0100120 struct work_struct work;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800121 struct bio_list deferred;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100122 spinlock_t deferred_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123
124 /*
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100125 * An error from the barrier request currently being processed.
126 */
127 int barrier_error;
128
129 /*
Milan Broz304f3f62008-02-08 02:11:17 +0000130 * Processing queue (flush/barriers)
131 */
132 struct workqueue_struct *wq;
133
134 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 * The current mapping.
136 */
137 struct dm_table *map;
138
139 /*
140 * io objects are allocated from here.
141 */
142 mempool_t *io_pool;
143 mempool_t *tio_pool;
144
Stefan Bader9faf4002006-10-03 01:15:41 -0700145 struct bio_set *bs;
146
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 /*
148 * Event handling.
149 */
150 atomic_t event_nr;
151 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100152 atomic_t uevent_seq;
153 struct list_head uevent_list;
154 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155
156 /*
157 * freeze/thaw support require holding onto a super block
158 */
159 struct super_block *frozen_sb;
Mikulas Patockadb8fef42009-06-22 10:12:15 +0100160 struct block_device *bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800161
162 /* forced geometry settings */
163 struct hd_geometry geometry;
Milan Broz784aae72009-01-06 03:05:12 +0000164
165 /* sysfs handle */
166 struct kobject kobj;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167};
168
169#define MIN_IOS 256
Christoph Lametere18b8902006-12-06 20:33:20 -0800170static struct kmem_cache *_io_cache;
171static struct kmem_cache *_tio_cache;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000172static struct kmem_cache *_rq_tio_cache;
173static struct kmem_cache *_rq_bio_info_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175static int __init local_init(void)
176{
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100177 int r = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100180 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 if (!_io_cache)
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100182 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183
184 /* allocate a slab for the target ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100185 _tio_cache = KMEM_CACHE(dm_target_io, 0);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100186 if (!_tio_cache)
187 goto out_free_io_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000189 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
190 if (!_rq_tio_cache)
191 goto out_free_tio_cache;
192
193 _rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
194 if (!_rq_bio_info_cache)
195 goto out_free_rq_tio_cache;
196
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100197 r = dm_uevent_init();
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100198 if (r)
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000199 goto out_free_rq_bio_info_cache;
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100200
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 _major = major;
202 r = register_blkdev(_major, _name);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100203 if (r < 0)
204 goto out_uevent_exit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205
206 if (!_major)
207 _major = r;
208
209 return 0;
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100210
211out_uevent_exit:
212 dm_uevent_exit();
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000213out_free_rq_bio_info_cache:
214 kmem_cache_destroy(_rq_bio_info_cache);
215out_free_rq_tio_cache:
216 kmem_cache_destroy(_rq_tio_cache);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100217out_free_tio_cache:
218 kmem_cache_destroy(_tio_cache);
219out_free_io_cache:
220 kmem_cache_destroy(_io_cache);
221
222 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223}
224
225static void local_exit(void)
226{
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000227 kmem_cache_destroy(_rq_bio_info_cache);
228 kmem_cache_destroy(_rq_tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 kmem_cache_destroy(_tio_cache);
230 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700231 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100232 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
234 _major = 0;
235
236 DMINFO("cleaned up");
237}
238
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000239static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 local_init,
241 dm_target_init,
242 dm_linear_init,
243 dm_stripe_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100244 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 dm_interface_init,
246};
247
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000248static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 local_exit,
250 dm_target_exit,
251 dm_linear_exit,
252 dm_stripe_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100253 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 dm_interface_exit,
255};
256
257static int __init dm_init(void)
258{
259 const int count = ARRAY_SIZE(_inits);
260
261 int r, i;
262
263 for (i = 0; i < count; i++) {
264 r = _inits[i]();
265 if (r)
266 goto bad;
267 }
268
269 return 0;
270
271 bad:
272 while (i--)
273 _exits[i]();
274
275 return r;
276}
277
278static void __exit dm_exit(void)
279{
280 int i = ARRAY_SIZE(_exits);
281
282 while (i--)
283 _exits[i]();
284}
285
286/*
287 * Block device functions
288 */
Al Virofe5f9f22008-03-02 10:29:31 -0500289static int dm_blk_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290{
291 struct mapped_device *md;
292
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700293 spin_lock(&_minor_lock);
294
Al Virofe5f9f22008-03-02 10:29:31 -0500295 md = bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700296 if (!md)
297 goto out;
298
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700299 if (test_bit(DMF_FREEING, &md->flags) ||
300 test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700301 md = NULL;
302 goto out;
303 }
304
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700306 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700307
308out:
309 spin_unlock(&_minor_lock);
310
311 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312}
313
Al Virofe5f9f22008-03-02 10:29:31 -0500314static int dm_blk_close(struct gendisk *disk, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315{
Al Virofe5f9f22008-03-02 10:29:31 -0500316 struct mapped_device *md = disk->private_data;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700317 atomic_dec(&md->open_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 dm_put(md);
319 return 0;
320}
321
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700322int dm_open_count(struct mapped_device *md)
323{
324 return atomic_read(&md->open_count);
325}
326
327/*
328 * Guarantees nothing is using the device before it's deleted.
329 */
330int dm_lock_for_deletion(struct mapped_device *md)
331{
332 int r = 0;
333
334 spin_lock(&_minor_lock);
335
336 if (dm_open_count(md))
337 r = -EBUSY;
338 else
339 set_bit(DMF_DELETING, &md->flags);
340
341 spin_unlock(&_minor_lock);
342
343 return r;
344}
345
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800346static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
347{
348 struct mapped_device *md = bdev->bd_disk->private_data;
349
350 return dm_get_geometry(md, geo);
351}
352
Al Virofe5f9f22008-03-02 10:29:31 -0500353static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
Milan Brozaa129a22006-10-03 01:15:15 -0700354 unsigned int cmd, unsigned long arg)
355{
Al Virofe5f9f22008-03-02 10:29:31 -0500356 struct mapped_device *md = bdev->bd_disk->private_data;
357 struct dm_table *map = dm_get_table(md);
Milan Brozaa129a22006-10-03 01:15:15 -0700358 struct dm_target *tgt;
359 int r = -ENOTTY;
360
Milan Brozaa129a22006-10-03 01:15:15 -0700361 if (!map || !dm_table_get_size(map))
362 goto out;
363
364 /* We only support devices that have a single target */
365 if (dm_table_get_num_targets(map) != 1)
366 goto out;
367
368 tgt = dm_table_get_target(map, 0);
369
370 if (dm_suspended(md)) {
371 r = -EAGAIN;
372 goto out;
373 }
374
375 if (tgt->type->ioctl)
Al Viro647b3d02007-08-28 22:15:59 -0400376 r = tgt->type->ioctl(tgt, cmd, arg);
Milan Brozaa129a22006-10-03 01:15:15 -0700377
378out:
379 dm_table_put(map);
380
Milan Brozaa129a22006-10-03 01:15:15 -0700381 return r;
382}
383
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100384static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385{
386 return mempool_alloc(md->io_pool, GFP_NOIO);
387}
388
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100389static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390{
391 mempool_free(io, md->io_pool);
392}
393
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100394static struct dm_target_io *alloc_tio(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395{
396 return mempool_alloc(md->tio_pool, GFP_NOIO);
397}
398
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100399static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400{
401 mempool_free(tio, md->tio_pool);
402}
403
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800404static void start_io_acct(struct dm_io *io)
405{
406 struct mapped_device *md = io->md;
Tejun Heoc9959052008-08-25 19:47:21 +0900407 int cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800408
409 io->start_time = jiffies;
410
Tejun Heo074a7ac2008-08-25 19:56:14 +0900411 cpu = part_stat_lock();
412 part_round_stats(cpu, &dm_disk(md)->part0);
413 part_stat_unlock();
414 dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800415}
416
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000417static void end_io_acct(struct dm_io *io)
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800418{
419 struct mapped_device *md = io->md;
420 struct bio *bio = io->bio;
421 unsigned long duration = jiffies - io->start_time;
Tejun Heoc9959052008-08-25 19:47:21 +0900422 int pending, cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800423 int rw = bio_data_dir(bio);
424
Tejun Heo074a7ac2008-08-25 19:56:14 +0900425 cpu = part_stat_lock();
426 part_round_stats(cpu, &dm_disk(md)->part0);
427 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
428 part_stat_unlock();
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800429
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100430 /*
431 * After this is decremented the bio must not be touched if it is
432 * a barrier.
433 */
Tejun Heo074a7ac2008-08-25 19:56:14 +0900434 dm_disk(md)->part0.in_flight = pending =
435 atomic_dec_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800436
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000437 /* nudge anyone waiting on suspend queue */
438 if (!pending)
439 wake_up(&md->wait);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800440}
441
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442/*
443 * Add the bio to the list of deferred io.
444 */
Mikulas Patocka92c63902009-04-09 00:27:15 +0100445static void queue_io(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700447 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448
Mikulas Patocka022c2612009-04-02 19:55:39 +0100449 spin_lock_irq(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 bio_list_add(&md->deferred, bio);
Mikulas Patocka022c2612009-04-02 19:55:39 +0100451 spin_unlock_irq(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452
Mikulas Patocka92c63902009-04-09 00:27:15 +0100453 if (!test_and_set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags))
454 queue_work(md->wq, &md->work);
455
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700456 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457}
458
459/*
460 * Everyone (including functions in this file), should use this
461 * function to access the md->map field, and make sure they call
462 * dm_table_put() when finished.
463 */
464struct dm_table *dm_get_table(struct mapped_device *md)
465{
466 struct dm_table *t;
467
468 read_lock(&md->map_lock);
469 t = md->map;
470 if (t)
471 dm_table_get(t);
472 read_unlock(&md->map_lock);
473
474 return t;
475}
476
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800477/*
478 * Get the geometry associated with a dm device
479 */
480int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
481{
482 *geo = md->geometry;
483
484 return 0;
485}
486
487/*
488 * Set the geometry of a device.
489 */
490int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
491{
492 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
493
494 if (geo->start > sz) {
495 DMWARN("Start sector is beyond the geometry limits.");
496 return -EINVAL;
497 }
498
499 md->geometry = *geo;
500
501 return 0;
502}
503
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504/*-----------------------------------------------------------------
505 * CRUD START:
506 * A more elegant soln is in the works that uses the queue
507 * merge fn, unfortunately there are a couple of changes to
508 * the block layer that I want to make for this. So in the
509 * interests of getting something for people to use I give
510 * you this clearly demarcated crap.
511 *---------------------------------------------------------------*/
512
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800513static int __noflush_suspending(struct mapped_device *md)
514{
515 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
516}
517
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518/*
519 * Decrements the number of outstanding ios that a bio has been
520 * cloned into, completing the original io if necc.
521 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800522static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800524 unsigned long flags;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000525 int io_error;
526 struct bio *bio;
527 struct mapped_device *md = io->md;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800528
529 /* Push-back supersedes any I/O errors */
Milan Brozb35f8ca2009-03-16 17:44:36 +0000530 if (error && !(io->error > 0 && __noflush_suspending(md)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 io->error = error;
532
533 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800534 if (io->error == DM_ENDIO_REQUEUE) {
535 /*
536 * Target requested pushing back the I/O.
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800537 */
Mikulas Patocka022c2612009-04-02 19:55:39 +0100538 spin_lock_irqsave(&md->deferred_lock, flags);
Mikulas Patocka2761e952009-06-22 10:12:18 +0100539 if (__noflush_suspending(md)) {
540 if (!bio_barrier(io->bio))
541 bio_list_add_head(&md->deferred,
542 io->bio);
543 } else
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800544 /* noflush suspend was interrupted. */
545 io->error = -EIO;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100546 spin_unlock_irqrestore(&md->deferred_lock, flags);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800547 }
548
Milan Brozb35f8ca2009-03-16 17:44:36 +0000549 io_error = io->error;
550 bio = io->bio;
Jens Axboe2056a782006-03-23 20:00:26 +0100551
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100552 if (bio_barrier(bio)) {
553 /*
554 * There can be just one barrier request so we use
555 * a per-device variable for error reporting.
556 * Note that you can't touch the bio after end_io_acct
557 */
558 md->barrier_error = io_error;
559 end_io_acct(io);
560 } else {
561 end_io_acct(io);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000562
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100563 if (io_error != DM_ENDIO_REQUEUE) {
564 trace_block_bio_complete(md->queue, bio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000565
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100566 bio_endio(bio, io_error);
567 }
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800568 }
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100569
570 free_io(md, io);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 }
572}
573
NeilBrown6712ecf2007-09-27 12:47:43 +0200574static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575{
576 int r = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100577 struct dm_target_io *tio = bio->bi_private;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000578 struct dm_io *io = tio->io;
Stefan Bader9faf4002006-10-03 01:15:41 -0700579 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 dm_endio_fn endio = tio->ti->type->end_io;
581
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
583 error = -EIO;
584
585 if (endio) {
586 r = endio(tio->ti, bio, error, &tio->info);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800587 if (r < 0 || r == DM_ENDIO_REQUEUE)
588 /*
589 * error and requeue request are handled
590 * in dec_pending().
591 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800593 else if (r == DM_ENDIO_INCOMPLETE)
594 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200595 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800596 else if (r) {
597 DMWARN("unimplemented target endio return value: %d", r);
598 BUG();
599 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 }
601
Stefan Bader9faf4002006-10-03 01:15:41 -0700602 /*
603 * Store md for cleanup instead of tio which is about to get freed.
604 */
605 bio->bi_private = md->bs;
606
Stefan Bader9faf4002006-10-03 01:15:41 -0700607 free_tio(md, tio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000608 bio_put(bio);
609 dec_pending(io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610}
611
612static sector_t max_io_len(struct mapped_device *md,
613 sector_t sector, struct dm_target *ti)
614{
615 sector_t offset = sector - ti->begin;
616 sector_t len = ti->len - offset;
617
618 /*
619 * Does the target need to split even further ?
620 */
621 if (ti->split_io) {
622 sector_t boundary;
623 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
624 - offset;
625 if (len > boundary)
626 len = boundary;
627 }
628
629 return len;
630}
631
632static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100633 struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634{
635 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100636 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -0700637 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
639 /*
640 * Sanity checks.
641 */
642 BUG_ON(!clone->bi_size);
643
644 clone->bi_end_io = clone_endio;
645 clone->bi_private = tio;
646
647 /*
648 * Map the clone. If r == 0 we don't need to do
649 * anything, the target has assumed ownership of
650 * this io.
651 */
652 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100653 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 r = ti->type->map(ti, clone, &tio->info);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800655 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100657
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100658 trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
Alan D. Brunelle22a7c312009-05-04 16:35:08 -0400659 tio->io->bio->bi_bdev->bd_dev, sector);
Jens Axboe2056a782006-03-23 20:00:26 +0100660
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800662 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
663 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -0700664 md = tio->io->md;
665 dec_pending(tio->io, r);
666 /*
667 * Store bio_set for cleanup.
668 */
669 clone->bi_private = md->bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670 bio_put(clone);
Stefan Bader9faf4002006-10-03 01:15:41 -0700671 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800672 } else if (r) {
673 DMWARN("unimplemented target map return value: %d", r);
674 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 }
676}
677
678struct clone_info {
679 struct mapped_device *md;
680 struct dm_table *map;
681 struct bio *bio;
682 struct dm_io *io;
683 sector_t sector;
684 sector_t sector_count;
685 unsigned short idx;
686};
687
Peter Osterlund36763472005-09-06 15:16:42 -0700688static void dm_bio_destructor(struct bio *bio)
689{
Stefan Bader9faf4002006-10-03 01:15:41 -0700690 struct bio_set *bs = bio->bi_private;
691
692 bio_free(bio, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700693}
694
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695/*
696 * Creates a little bio that is just does part of a bvec.
697 */
698static struct bio *split_bvec(struct bio *bio, sector_t sector,
699 unsigned short idx, unsigned int offset,
Stefan Bader9faf4002006-10-03 01:15:41 -0700700 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701{
702 struct bio *clone;
703 struct bio_vec *bv = bio->bi_io_vec + idx;
704
Stefan Bader9faf4002006-10-03 01:15:41 -0700705 clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700706 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 *clone->bi_io_vec = *bv;
708
709 clone->bi_sector = sector;
710 clone->bi_bdev = bio->bi_bdev;
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100711 clone->bi_rw = bio->bi_rw & ~(1 << BIO_RW_BARRIER);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 clone->bi_vcnt = 1;
713 clone->bi_size = to_bytes(len);
714 clone->bi_io_vec->bv_offset = offset;
715 clone->bi_io_vec->bv_len = clone->bi_size;
Martin K. Petersenf3e1d262008-10-21 17:45:04 +0100716 clone->bi_flags |= 1 << BIO_CLONED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717
Martin K. Petersen9c470082009-04-09 00:27:12 +0100718 if (bio_integrity(bio)) {
719 bio_integrity_clone(clone, bio, GFP_NOIO);
720 bio_integrity_trim(clone,
721 bio_sector_offset(bio, idx, offset), len);
722 }
723
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 return clone;
725}
726
727/*
728 * Creates a bio that consists of range of complete bvecs.
729 */
730static struct bio *clone_bio(struct bio *bio, sector_t sector,
731 unsigned short idx, unsigned short bv_count,
Stefan Bader9faf4002006-10-03 01:15:41 -0700732 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733{
734 struct bio *clone;
735
Stefan Bader9faf4002006-10-03 01:15:41 -0700736 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
737 __bio_clone(clone, bio);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100738 clone->bi_rw &= ~(1 << BIO_RW_BARRIER);
Stefan Bader9faf4002006-10-03 01:15:41 -0700739 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740 clone->bi_sector = sector;
741 clone->bi_idx = idx;
742 clone->bi_vcnt = idx + bv_count;
743 clone->bi_size = to_bytes(len);
744 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
745
Martin K. Petersen9c470082009-04-09 00:27:12 +0100746 if (bio_integrity(bio)) {
747 bio_integrity_clone(clone, bio, GFP_NOIO);
748
749 if (idx != bio->bi_idx || clone->bi_size < bio->bi_size)
750 bio_integrity_trim(clone,
751 bio_sector_offset(bio, idx, 0), len);
752 }
753
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 return clone;
755}
756
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000757static int __clone_and_map(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758{
759 struct bio *clone, *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000760 struct dm_target *ti;
761 sector_t len = 0, max;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100762 struct dm_target_io *tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000764 ti = dm_table_find_target(ci->map, ci->sector);
765 if (!dm_target_is_valid(ti))
766 return -EIO;
767
768 max = max_io_len(ci->md, ci->sector, ti);
769
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770 /*
771 * Allocate a target io object.
772 */
773 tio = alloc_tio(ci->md);
774 tio->io = ci->io;
775 tio->ti = ti;
776 memset(&tio->info, 0, sizeof(tio->info));
777
778 if (ci->sector_count <= max) {
779 /*
780 * Optimise for the simple case where we can do all of
781 * the remaining io with a single clone.
782 */
783 clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700784 bio->bi_vcnt - ci->idx, ci->sector_count,
785 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 __map_bio(ti, clone, tio);
787 ci->sector_count = 0;
788
789 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
790 /*
791 * There are some bvecs that don't span targets.
792 * Do as many of these as possible.
793 */
794 int i;
795 sector_t remaining = max;
796 sector_t bv_len;
797
798 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
799 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
800
801 if (bv_len > remaining)
802 break;
803
804 remaining -= bv_len;
805 len += bv_len;
806 }
807
Stefan Bader9faf4002006-10-03 01:15:41 -0700808 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
809 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810 __map_bio(ti, clone, tio);
811
812 ci->sector += len;
813 ci->sector_count -= len;
814 ci->idx = i;
815
816 } else {
817 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800818 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819 */
820 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800821 sector_t remaining = to_sector(bv->bv_len);
822 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800824 do {
825 if (offset) {
826 ti = dm_table_find_target(ci->map, ci->sector);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000827 if (!dm_target_is_valid(ti))
828 return -EIO;
829
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800830 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800832 tio = alloc_tio(ci->md);
833 tio->io = ci->io;
834 tio->ti = ti;
835 memset(&tio->info, 0, sizeof(tio->info));
836 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800838 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800840 clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700841 bv->bv_offset + offset, len,
842 ci->md->bs);
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800843
844 __map_bio(ti, clone, tio);
845
846 ci->sector += len;
847 ci->sector_count -= len;
848 offset += to_bytes(len);
849 } while (remaining -= len);
850
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 ci->idx++;
852 }
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000853
854 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855}
856
857/*
Mikulas Patocka8a53c282009-04-02 19:55:37 +0100858 * Split the bio into several clones and submit it to targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 */
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100860static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861{
862 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000863 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700864
865 ci.map = dm_get_table(md);
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100866 if (unlikely(!ci.map)) {
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100867 if (!bio_barrier(bio))
868 bio_io_error(bio);
869 else
870 md->barrier_error = -EIO;
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100871 return;
872 }
Mikulas Patocka692d0eb2009-04-09 00:27:13 +0100873
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 ci.md = md;
875 ci.bio = bio;
876 ci.io = alloc_io(md);
877 ci.io->error = 0;
878 atomic_set(&ci.io->io_count, 1);
879 ci.io->bio = bio;
880 ci.io->md = md;
881 ci.sector = bio->bi_sector;
882 ci.sector_count = bio_sectors(bio);
883 ci.idx = bio->bi_idx;
884
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800885 start_io_acct(ci.io);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000886 while (ci.sector_count && !error)
887 error = __clone_and_map(&ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888
889 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000890 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891 dm_table_put(ci.map);
892}
893/*-----------------------------------------------------------------
894 * CRUD END
895 *---------------------------------------------------------------*/
896
Milan Brozf6fccb12008-07-21 12:00:37 +0100897static int dm_merge_bvec(struct request_queue *q,
898 struct bvec_merge_data *bvm,
899 struct bio_vec *biovec)
900{
901 struct mapped_device *md = q->queuedata;
902 struct dm_table *map = dm_get_table(md);
903 struct dm_target *ti;
904 sector_t max_sectors;
Mikulas Patocka50371082008-10-01 14:39:17 +0100905 int max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +0100906
907 if (unlikely(!map))
Mikulas Patocka50371082008-10-01 14:39:17 +0100908 goto out;
Milan Brozf6fccb12008-07-21 12:00:37 +0100909
910 ti = dm_table_find_target(map, bvm->bi_sector);
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100911 if (!dm_target_is_valid(ti))
912 goto out_table;
Milan Brozf6fccb12008-07-21 12:00:37 +0100913
914 /*
915 * Find maximum amount of I/O that won't need splitting
916 */
917 max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
918 (sector_t) BIO_MAX_SECTORS);
919 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
920 if (max_size < 0)
921 max_size = 0;
922
923 /*
924 * merge_bvec_fn() returns number of bytes
925 * it can accept at this offset
926 * max is precomputed maximal io size
927 */
928 if (max_size && ti->type->merge)
929 max_size = ti->type->merge(ti, bvm, biovec, max_size);
Mikulas Patocka8cbeb672009-06-22 10:12:14 +0100930 /*
931 * If the target doesn't support merge method and some of the devices
932 * provided their merge_bvec method (we know this by looking at
933 * queue_max_hw_sectors), then we can't allow bios with multiple vector
934 * entries. So always set max_size to 0, and the code below allows
935 * just one page.
936 */
937 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
938
939 max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +0100940
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100941out_table:
Mikulas Patocka50371082008-10-01 14:39:17 +0100942 dm_table_put(map);
943
944out:
Milan Brozf6fccb12008-07-21 12:00:37 +0100945 /*
946 * Always allow an entire first page
947 */
948 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
949 max_size = biovec->bv_len;
950
Milan Brozf6fccb12008-07-21 12:00:37 +0100951 return max_size;
952}
953
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954/*
955 * The request function that just remaps the bio built up by
956 * dm_merge_bvec.
957 */
Jens Axboe165125e2007-07-24 09:28:11 +0200958static int dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959{
Kevin Corry12f03a42006-02-01 03:04:52 -0800960 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961 struct mapped_device *md = q->queuedata;
Tejun Heoc9959052008-08-25 19:47:21 +0900962 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700964 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965
Tejun Heo074a7ac2008-08-25 19:56:14 +0900966 cpu = part_stat_lock();
967 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
968 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
969 part_stat_unlock();
Kevin Corry12f03a42006-02-01 03:04:52 -0800970
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971 /*
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +0100972 * If we're suspended or the thread is processing barriers
973 * we have to queue this io for later.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974 */
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100975 if (unlikely(test_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags)) ||
976 unlikely(bio_barrier(bio))) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700977 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978
Alasdair G Kergon54d9a1b2009-04-09 00:27:14 +0100979 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) &&
980 bio_rw(bio) == READA) {
981 bio_io_error(bio);
982 return 0;
983 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984
Mikulas Patocka92c63902009-04-09 00:27:15 +0100985 queue_io(md, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986
Mikulas Patocka92c63902009-04-09 00:27:15 +0100987 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 }
989
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100990 __split_and_process_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700991 up_read(&md->io_lock);
Mikulas Patockaf0b9a452009-04-02 19:55:38 +0100992 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993}
994
Jens Axboe165125e2007-07-24 09:28:11 +0200995static void dm_unplug_all(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996{
997 struct mapped_device *md = q->queuedata;
998 struct dm_table *map = dm_get_table(md);
999
1000 if (map) {
1001 dm_table_unplug_all(map);
1002 dm_table_put(map);
1003 }
1004}
1005
1006static int dm_any_congested(void *congested_data, int bdi_bits)
1007{
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001008 int r = bdi_bits;
1009 struct mapped_device *md = congested_data;
1010 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001012 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001013 map = dm_get_table(md);
1014 if (map) {
1015 r = dm_table_any_congested(map, bdi_bits);
1016 dm_table_put(map);
1017 }
1018 }
1019
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 return r;
1021}
1022
1023/*-----------------------------------------------------------------
1024 * An IDR is used to keep track of allocated minor numbers.
1025 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026static DEFINE_IDR(_minor_idr);
1027
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001028static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001030 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001032 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033}
1034
1035/*
1036 * See if the device with a specific minor # is free.
1037 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001038static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039{
1040 int r, m;
1041
1042 if (minor >= (1 << MINORBITS))
1043 return -EINVAL;
1044
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001045 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
1046 if (!r)
1047 return -ENOMEM;
1048
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001049 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050
1051 if (idr_find(&_minor_idr, minor)) {
1052 r = -EBUSY;
1053 goto out;
1054 }
1055
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001056 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001057 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059
1060 if (m != minor) {
1061 idr_remove(&_minor_idr, m);
1062 r = -EBUSY;
1063 goto out;
1064 }
1065
1066out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001067 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001068 return r;
1069}
1070
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001071static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001073 int r, m;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001076 if (!r)
1077 return -ENOMEM;
1078
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001079 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001081 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001082 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084
1085 if (m >= (1 << MINORBITS)) {
1086 idr_remove(&_minor_idr, m);
1087 r = -ENOSPC;
1088 goto out;
1089 }
1090
1091 *minor = m;
1092
1093out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001094 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095 return r;
1096}
1097
1098static struct block_device_operations dm_blk_dops;
1099
Mikulas Patocka53d59142009-04-02 19:55:37 +01001100static void dm_wq_work(struct work_struct *work);
1101
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102/*
1103 * Allocate and initialise a blank device with a given minor.
1104 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001105static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106{
1107 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001108 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001109 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110
1111 if (!md) {
1112 DMWARN("unable to allocate device, out of memory.");
1113 return NULL;
1114 }
1115
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001116 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001117 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001118
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001120 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001121 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001122 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001123 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001125 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001127 init_rwsem(&md->io_lock);
Daniel Walkere61290a2008-02-08 02:10:08 +00001128 mutex_init(&md->suspend_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001129 spin_lock_init(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130 rwlock_init(&md->map_lock);
1131 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001132 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001134 atomic_set(&md->uevent_seq, 0);
1135 INIT_LIST_HEAD(&md->uevent_list);
1136 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137
1138 md->queue = blk_alloc_queue(GFP_KERNEL);
1139 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001140 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001141
1142 md->queue->queuedata = md;
1143 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1144 md->queue->backing_dev_info.congested_data = md;
1145 blk_queue_make_request(md->queue, dm_request);
Mikulas Patocka99360b42009-04-02 19:55:39 +01001146 blk_queue_ordered(md->queue, QUEUE_ORDERED_DRAIN, NULL);
Jens Axboedaef2652006-01-10 10:48:02 +01001147 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148 md->queue->unplug_fn = dm_unplug_all;
Milan Brozf6fccb12008-07-21 12:00:37 +01001149 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150
Matthew Dobson93d23412006-03-26 01:37:50 -08001151 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
Kiyoshi Ueda74859362006-12-08 02:41:02 -08001152 if (!md->io_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001153 goto bad_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154
Matthew Dobson93d23412006-03-26 01:37:50 -08001155 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001156 if (!md->tio_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001157 goto bad_tio_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001158
Jens Axboebb799ca2008-12-10 15:35:05 +01001159 md->bs = bioset_create(16, 0);
Stefan Bader9faf4002006-10-03 01:15:41 -07001160 if (!md->bs)
1161 goto bad_no_bioset;
1162
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163 md->disk = alloc_disk(1);
1164 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001165 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001167 atomic_set(&md->pending, 0);
1168 init_waitqueue_head(&md->wait);
Mikulas Patocka53d59142009-04-02 19:55:37 +01001169 INIT_WORK(&md->work, dm_wq_work);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001170 init_waitqueue_head(&md->eventq);
1171
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172 md->disk->major = _major;
1173 md->disk->first_minor = minor;
1174 md->disk->fops = &dm_blk_dops;
1175 md->disk->queue = md->queue;
1176 md->disk->private_data = md;
1177 sprintf(md->disk->disk_name, "dm-%d", minor);
1178 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001179 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001180
Milan Broz304f3f62008-02-08 02:11:17 +00001181 md->wq = create_singlethread_workqueue("kdmflush");
1182 if (!md->wq)
1183 goto bad_thread;
1184
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001185 md->bdev = bdget_disk(md->disk, 0);
1186 if (!md->bdev)
1187 goto bad_bdev;
1188
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001189 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001190 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001191 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001192 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001193
1194 BUG_ON(old_md != MINOR_ALLOCED);
1195
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 return md;
1197
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001198bad_bdev:
1199 destroy_workqueue(md->wq);
Milan Broz304f3f62008-02-08 02:11:17 +00001200bad_thread:
1201 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001202bad_disk:
Stefan Bader9faf4002006-10-03 01:15:41 -07001203 bioset_free(md->bs);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001204bad_no_bioset:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205 mempool_destroy(md->tio_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001206bad_tio_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207 mempool_destroy(md->io_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001208bad_io_pool:
Al Viro1312f402006-03-12 11:02:03 -05001209 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001210bad_queue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001212bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001213 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001214bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215 kfree(md);
1216 return NULL;
1217}
1218
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001219static void unlock_fs(struct mapped_device *md);
1220
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221static void free_dev(struct mapped_device *md)
1222{
Tejun Heof331c022008-09-03 09:01:48 +02001223 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001224
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001225 unlock_fs(md);
1226 bdput(md->bdev);
Milan Broz304f3f62008-02-08 02:11:17 +00001227 destroy_workqueue(md->wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 mempool_destroy(md->tio_pool);
1229 mempool_destroy(md->io_pool);
Stefan Bader9faf4002006-10-03 01:15:41 -07001230 bioset_free(md->bs);
Martin K. Petersen9c470082009-04-09 00:27:12 +01001231 blk_integrity_unregister(md->disk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001233 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001234
1235 spin_lock(&_minor_lock);
1236 md->disk->private_data = NULL;
1237 spin_unlock(&_minor_lock);
1238
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05001240 blk_cleanup_queue(md->queue);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001241 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242 kfree(md);
1243}
1244
1245/*
1246 * Bind a table to the device.
1247 */
1248static void event_callback(void *context)
1249{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001250 unsigned long flags;
1251 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252 struct mapped_device *md = (struct mapped_device *) context;
1253
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001254 spin_lock_irqsave(&md->uevent_lock, flags);
1255 list_splice_init(&md->uevent_list, &uevents);
1256 spin_unlock_irqrestore(&md->uevent_lock, flags);
1257
Tejun Heoed9e1982008-08-25 19:56:05 +09001258 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001259
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260 atomic_inc(&md->event_nr);
1261 wake_up(&md->eventq);
1262}
1263
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001264static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265{
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001266 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001267
Mikulas Patockadb8fef42009-06-22 10:12:15 +01001268 mutex_lock(&md->bdev->bd_inode->i_mutex);
1269 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1270 mutex_unlock(&md->bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001271}
1272
1273static int __bind(struct mapped_device *md, struct dm_table *t)
1274{
Jens Axboe165125e2007-07-24 09:28:11 +02001275 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 sector_t size;
1277
1278 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001279
1280 /*
1281 * Wipe any geometry if the size of the table changed.
1282 */
1283 if (size != get_capacity(md->disk))
1284 memset(&md->geometry, 0, sizeof(md->geometry));
1285
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001286 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287
Mikulas Patockad5816872009-01-06 03:05:10 +00001288 if (!size) {
1289 dm_table_destroy(t);
1290 return 0;
1291 }
1292
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001293 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001294
1295 write_lock(&md->map_lock);
1296 md->map = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297 dm_table_set_restrictions(t, q);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001298 write_unlock(&md->map_lock);
1299
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 return 0;
1301}
1302
1303static void __unbind(struct mapped_device *md)
1304{
1305 struct dm_table *map = md->map;
1306
1307 if (!map)
1308 return;
1309
1310 dm_table_event_callback(map, NULL, NULL);
1311 write_lock(&md->map_lock);
1312 md->map = NULL;
1313 write_unlock(&md->map_lock);
Mikulas Patockad5816872009-01-06 03:05:10 +00001314 dm_table_destroy(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315}
1316
1317/*
1318 * Constructor for a new device.
1319 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001320int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321{
1322 struct mapped_device *md;
1323
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001324 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325 if (!md)
1326 return -ENXIO;
1327
Milan Broz784aae72009-01-06 03:05:12 +00001328 dm_sysfs_init(md);
1329
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330 *result = md;
1331 return 0;
1332}
1333
David Teigland637842c2006-01-06 00:20:00 -08001334static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335{
1336 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001337 unsigned minor = MINOR(dev);
1338
1339 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1340 return NULL;
1341
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001342 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343
1344 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001345 if (md && (md == MINOR_ALLOCED ||
Tejun Heof331c022008-09-03 09:01:48 +02001346 (MINOR(disk_devt(dm_disk(md))) != minor) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07001347 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08001348 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001349 goto out;
1350 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001352out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001353 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354
David Teigland637842c2006-01-06 00:20:00 -08001355 return md;
1356}
1357
David Teiglandd229a952006-01-06 00:20:01 -08001358struct mapped_device *dm_get_md(dev_t dev)
1359{
1360 struct mapped_device *md = dm_find_md(dev);
1361
1362 if (md)
1363 dm_get(md);
1364
1365 return md;
1366}
1367
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001368void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08001369{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001370 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371}
1372
1373void dm_set_mdptr(struct mapped_device *md, void *ptr)
1374{
1375 md->interface_ptr = ptr;
1376}
1377
1378void dm_get(struct mapped_device *md)
1379{
1380 atomic_inc(&md->holders);
1381}
1382
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001383const char *dm_device_name(struct mapped_device *md)
1384{
1385 return md->name;
1386}
1387EXPORT_SYMBOL_GPL(dm_device_name);
1388
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389void dm_put(struct mapped_device *md)
1390{
Mike Anderson1134e5a2006-03-27 01:17:54 -08001391 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001393 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1394
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001395 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08001396 map = dm_get_table(md);
Tejun Heof331c022008-09-03 09:01:48 +02001397 idr_replace(&_minor_idr, MINOR_ALLOCED,
1398 MINOR(disk_devt(dm_disk(md))));
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001399 set_bit(DMF_FREEING, &md->flags);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001400 spin_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001401 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402 dm_table_presuspend_targets(map);
1403 dm_table_postsuspend_targets(map);
1404 }
Milan Broz784aae72009-01-06 03:05:12 +00001405 dm_sysfs_exit(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08001406 dm_table_put(map);
Mikulas Patockaa1b51e92009-01-06 03:04:53 +00001407 __unbind(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408 free_dev(md);
1409 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410}
Edward Goggin79eb8852007-05-09 02:32:56 -07001411EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001412
Mikulas Patocka401600d2009-04-02 19:55:38 +01001413static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
Milan Broz46125c12008-02-08 02:10:30 +00001414{
1415 int r = 0;
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01001416 DECLARE_WAITQUEUE(wait, current);
1417
1418 dm_unplug_all(md->queue);
1419
1420 add_wait_queue(&md->wait, &wait);
Milan Broz46125c12008-02-08 02:10:30 +00001421
1422 while (1) {
Mikulas Patocka401600d2009-04-02 19:55:38 +01001423 set_current_state(interruptible);
Milan Broz46125c12008-02-08 02:10:30 +00001424
1425 smp_mb();
1426 if (!atomic_read(&md->pending))
1427 break;
1428
Mikulas Patocka401600d2009-04-02 19:55:38 +01001429 if (interruptible == TASK_INTERRUPTIBLE &&
1430 signal_pending(current)) {
Milan Broz46125c12008-02-08 02:10:30 +00001431 r = -EINTR;
1432 break;
1433 }
1434
1435 io_schedule();
1436 }
1437 set_current_state(TASK_RUNNING);
1438
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01001439 remove_wait_queue(&md->wait, &wait);
1440
Milan Broz46125c12008-02-08 02:10:30 +00001441 return r;
1442}
1443
Mikulas Patocka531fe962009-06-22 10:12:17 +01001444static void dm_flush(struct mapped_device *md)
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001445{
1446 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001447}
1448
1449static void process_barrier(struct mapped_device *md, struct bio *bio)
1450{
Mikulas Patocka531fe962009-06-22 10:12:17 +01001451 dm_flush(md);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001452
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001453 if (bio_empty_barrier(bio)) {
1454 bio_endio(bio, 0);
1455 return;
1456 }
1457
1458 __split_and_process_bio(md, bio);
Mikulas Patocka531fe962009-06-22 10:12:17 +01001459 dm_flush(md);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001460
1461 if (md->barrier_error != DM_ENDIO_REQUEUE)
Mikulas Patocka531fe962009-06-22 10:12:17 +01001462 bio_endio(bio, md->barrier_error);
Mikulas Patocka2761e952009-06-22 10:12:18 +01001463 else {
1464 spin_lock_irq(&md->deferred_lock);
1465 bio_list_add_head(&md->deferred, bio);
1466 spin_unlock_irq(&md->deferred_lock);
1467 }
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001468}
1469
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470/*
1471 * Process the deferred bios
1472 */
Mikulas Patockaef208582009-04-02 19:55:38 +01001473static void dm_wq_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474{
Mikulas Patockaef208582009-04-02 19:55:38 +01001475 struct mapped_device *md = container_of(work, struct mapped_device,
1476 work);
Milan Broz6d6f10d2008-02-08 02:10:22 +00001477 struct bio *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478
Mikulas Patockaef208582009-04-02 19:55:38 +01001479 down_write(&md->io_lock);
1480
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001481 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01001482 spin_lock_irq(&md->deferred_lock);
1483 c = bio_list_pop(&md->deferred);
1484 spin_unlock_irq(&md->deferred_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001485
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01001486 if (!c) {
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001487 clear_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01001488 break;
1489 }
1490
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001491 up_write(&md->io_lock);
1492
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001493 if (bio_barrier(c))
1494 process_barrier(md, c);
1495 else
1496 __split_and_process_bio(md, c);
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001497
1498 down_write(&md->io_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001499 }
Milan Broz73d410c2008-02-08 02:10:25 +00001500
Mikulas Patockaef208582009-04-02 19:55:38 +01001501 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502}
1503
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01001504static void dm_queue_flush(struct mapped_device *md)
Milan Broz304f3f62008-02-08 02:11:17 +00001505{
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001506 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
1507 smp_mb__after_clear_bit();
Mikulas Patocka53d59142009-04-02 19:55:37 +01001508 queue_work(md->wq, &md->work);
Milan Broz304f3f62008-02-08 02:11:17 +00001509}
1510
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511/*
1512 * Swap in a new table (destroying old one).
1513 */
1514int dm_swap_table(struct mapped_device *md, struct dm_table *table)
1515{
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001516 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001517
Daniel Walkere61290a2008-02-08 02:10:08 +00001518 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519
1520 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001521 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001522 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523
1524 __unbind(md);
1525 r = __bind(md, table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001527out:
Daniel Walkere61290a2008-02-08 02:10:08 +00001528 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001529 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530}
1531
1532/*
1533 * Functions to lock and unlock any filesystem running on the
1534 * device.
1535 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001536static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001538 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539
1540 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001541
Mikulas Patockadb8fef42009-06-22 10:12:15 +01001542 md->frozen_sb = freeze_bdev(md->bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001543 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001544 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001545 md->frozen_sb = NULL;
1546 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001547 }
1548
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001549 set_bit(DMF_FROZEN, &md->flags);
1550
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551 return 0;
1552}
1553
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001554static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001556 if (!test_bit(DMF_FROZEN, &md->flags))
1557 return;
1558
Mikulas Patockadb8fef42009-06-22 10:12:15 +01001559 thaw_bdev(md->bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001561 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562}
1563
1564/*
1565 * We need to be able to change a mapping table under a mounted
1566 * filesystem. For example we might want to move some data in
1567 * the background. Before the table can be swapped with
1568 * dm_bind_table, dm_suspend must be called to flush any in
1569 * flight bios and ensure that any further io gets deferred.
1570 */
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001571int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001573 struct dm_table *map = NULL;
Milan Broz46125c12008-02-08 02:10:30 +00001574 int r = 0;
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001575 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001576 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577
Daniel Walkere61290a2008-02-08 02:10:08 +00001578 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001579
Milan Broz73d410c2008-02-08 02:10:25 +00001580 if (dm_suspended(md)) {
1581 r = -EINVAL;
Alasdair G Kergond2874832006-11-08 17:44:43 -08001582 goto out_unlock;
Milan Broz73d410c2008-02-08 02:10:25 +00001583 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584
1585 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001587 /*
1588 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
1589 * This flag is cleared before dm_suspend returns.
1590 */
1591 if (noflush)
1592 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1593
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001594 /* This does not get reverted if there's an error later. */
1595 dm_table_presuspend_targets(map);
1596
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001597 /*
1598 * Flush I/O to the device. noflush supersedes do_lockfs,
1599 * because lock_fs() needs to flush I/Os.
1600 */
1601 if (!noflush && do_lockfs) {
1602 r = lock_fs(md);
1603 if (r)
Kiyoshi Uedaf431d962008-10-21 17:45:07 +01001604 goto out;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001605 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606
1607 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001608 * Here we must make sure that no processes are submitting requests
1609 * to target drivers i.e. no one may be executing
1610 * __split_and_process_bio. This is called from dm_request and
1611 * dm_wq_work.
1612 *
1613 * To get all processes out of __split_and_process_bio in dm_request,
1614 * we take the write lock. To prevent any process from reentering
1615 * __split_and_process_bio from dm_request, we set
1616 * DMF_QUEUE_IO_TO_THREAD.
1617 *
1618 * To quiesce the thread (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND
1619 * and call flush_workqueue(md->wq). flush_workqueue will wait until
1620 * dm_wq_work exits and DMF_BLOCK_IO_FOR_SUSPEND will prevent any
1621 * further calls to __split_and_process_bio from dm_wq_work.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001623 down_write(&md->io_lock);
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001624 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
1625 set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001626 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001628 flush_workqueue(md->wq);
1629
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001631 * At this point no more requests are entering target request routines.
1632 * We call dm_wait_for_completion to wait for all existing requests
1633 * to finish.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001634 */
Mikulas Patocka401600d2009-04-02 19:55:38 +01001635 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001636
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001637 down_write(&md->io_lock);
Milan Broz6d6f10d2008-02-08 02:10:22 +00001638 if (noflush)
Mikulas Patocka022c2612009-04-02 19:55:39 +01001639 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
Milan Broz94d63512008-02-08 02:10:27 +00001640 up_write(&md->io_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001641
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00001643 if (r < 0) {
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01001644 dm_queue_flush(md);
Milan Broz73d410c2008-02-08 02:10:25 +00001645
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001646 unlock_fs(md);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001647 goto out; /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001648 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001649
Mikulas Patocka3b00b202009-04-09 00:27:15 +01001650 /*
1651 * If dm_wait_for_completion returned 0, the device is completely
1652 * quiescent now. There is no request-processing activity. All new
1653 * requests are being added to md->deferred list.
1654 */
1655
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001656 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657
1658 set_bit(DMF_SUSPENDED, &md->flags);
1659
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001660out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661 dm_table_put(map);
Alasdair G Kergond2874832006-11-08 17:44:43 -08001662
1663out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00001664 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001665 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001666}
1667
1668int dm_resume(struct mapped_device *md)
1669{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001670 int r = -EINVAL;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001671 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672
Daniel Walkere61290a2008-02-08 02:10:08 +00001673 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001674 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001675 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001676
1677 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001678 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001679 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001680
Milan Broz8757b772006-10-03 01:15:36 -07001681 r = dm_table_resume_targets(map);
1682 if (r)
1683 goto out;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001684
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01001685 dm_queue_flush(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001686
1687 unlock_fs(md);
1688
1689 clear_bit(DMF_SUSPENDED, &md->flags);
1690
Linus Torvalds1da177e2005-04-16 15:20:36 -07001691 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001692
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001693 dm_kobject_uevent(md);
Hannes Reinecke8560ed62006-10-03 01:15:35 -07001694
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001695 r = 0;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001696
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001697out:
1698 dm_table_put(map);
Daniel Walkere61290a2008-02-08 02:10:08 +00001699 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001700
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001701 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702}
1703
1704/*-----------------------------------------------------------------
1705 * Event notification.
1706 *---------------------------------------------------------------*/
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001707void dm_kobject_uevent(struct mapped_device *md)
1708{
Tejun Heoed9e1982008-08-25 19:56:05 +09001709 kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001710}
1711
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001712uint32_t dm_next_uevent_seq(struct mapped_device *md)
1713{
1714 return atomic_add_return(1, &md->uevent_seq);
1715}
1716
Linus Torvalds1da177e2005-04-16 15:20:36 -07001717uint32_t dm_get_event_nr(struct mapped_device *md)
1718{
1719 return atomic_read(&md->event_nr);
1720}
1721
1722int dm_wait_event(struct mapped_device *md, int event_nr)
1723{
1724 return wait_event_interruptible(md->eventq,
1725 (event_nr != atomic_read(&md->event_nr)));
1726}
1727
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001728void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
1729{
1730 unsigned long flags;
1731
1732 spin_lock_irqsave(&md->uevent_lock, flags);
1733 list_add(elist, &md->uevent_list);
1734 spin_unlock_irqrestore(&md->uevent_lock, flags);
1735}
1736
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737/*
1738 * The gendisk is only valid as long as you have a reference
1739 * count on 'md'.
1740 */
1741struct gendisk *dm_disk(struct mapped_device *md)
1742{
1743 return md->disk;
1744}
1745
Milan Broz784aae72009-01-06 03:05:12 +00001746struct kobject *dm_kobject(struct mapped_device *md)
1747{
1748 return &md->kobj;
1749}
1750
1751/*
1752 * struct mapped_device should not be exported outside of dm.c
1753 * so use this check to verify that kobj is part of md structure
1754 */
1755struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
1756{
1757 struct mapped_device *md;
1758
1759 md = container_of(kobj, struct mapped_device, kobj);
1760 if (&md->kobj != kobj)
1761 return NULL;
1762
Milan Broz4d89b7b2009-06-22 10:12:11 +01001763 if (test_bit(DMF_FREEING, &md->flags) ||
1764 test_bit(DMF_DELETING, &md->flags))
1765 return NULL;
1766
Milan Broz784aae72009-01-06 03:05:12 +00001767 dm_get(md);
1768 return md;
1769}
1770
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771int dm_suspended(struct mapped_device *md)
1772{
1773 return test_bit(DMF_SUSPENDED, &md->flags);
1774}
1775
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001776int dm_noflush_suspending(struct dm_target *ti)
1777{
1778 struct mapped_device *md = dm_table_get_md(ti->table);
1779 int r = __noflush_suspending(md);
1780
1781 dm_put(md);
1782
1783 return r;
1784}
1785EXPORT_SYMBOL_GPL(dm_noflush_suspending);
1786
Linus Torvalds1da177e2005-04-16 15:20:36 -07001787static struct block_device_operations dm_blk_dops = {
1788 .open = dm_blk_open,
1789 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07001790 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001791 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792 .owner = THIS_MODULE
1793};
1794
1795EXPORT_SYMBOL(dm_get_mapinfo);
1796
1797/*
1798 * module hooks
1799 */
1800module_init(dm_init);
1801module_exit(dm_exit);
1802
1803module_param(major, uint, 0);
1804MODULE_PARM_DESC(major, "The major number of the device mapper");
1805MODULE_DESCRIPTION(DM_NAME " driver");
1806MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1807MODULE_LICENSE("GPL");