blob: 8d40f27cce894e4a3c489a7d84d321fb39b363d6 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Milan Broz784aae72009-01-06 03:05:12 +00003 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
9#include "dm-bio-list.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +010010#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
12#include <linux/init.h>
13#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080014#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/moduleparam.h>
16#include <linux/blkpg.h>
17#include <linux/bio.h>
18#include <linux/buffer_head.h>
19#include <linux/mempool.h>
20#include <linux/slab.h>
21#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080022#include <linux/hdreg.h>
Jens Axboe2056a782006-03-23 20:00:26 +010023#include <linux/blktrace_api.h>
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +010024#include <trace/block.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Alasdair G Kergon72d94862006-06-26 00:27:35 -070026#define DM_MSG_PREFIX "core"
27
Linus Torvalds1da177e2005-04-16 15:20:36 -070028static const char *_name = DM_NAME;
29
30static unsigned int major = 0;
31static unsigned int _major = 0;
32
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070033static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070034/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000035 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * One of these is allocated per bio.
37 */
38struct dm_io {
39 struct mapped_device *md;
40 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -070041 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010042 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080043 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070044};
45
46/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000047 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 * One of these is allocated per target within a bio. Hopefully
49 * this will be simplified out one day.
50 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +010051struct dm_target_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -070052 struct dm_io *io;
53 struct dm_target *ti;
54 union map_info info;
55};
56
Ingo Molnar0bfc2452008-11-26 11:59:56 +010057DEFINE_TRACE(block_bio_complete);
58
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000059/*
60 * For request-based dm.
61 * One of these is allocated per request.
62 */
63struct dm_rq_target_io {
64 struct mapped_device *md;
65 struct dm_target *ti;
66 struct request *orig, clone;
67 int error;
68 union map_info info;
69};
70
71/*
72 * For request-based dm.
73 * One of these is allocated per bio.
74 */
75struct dm_rq_clone_bio_info {
76 struct bio *orig;
77 struct request *rq;
78};
79
Linus Torvalds1da177e2005-04-16 15:20:36 -070080union map_info *dm_get_mapinfo(struct bio *bio)
81{
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070082 if (bio && bio->bi_private)
Alasdair G Kergon028867a2007-07-12 17:26:32 +010083 return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070084 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070085}
86
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070087#define MINOR_ALLOCED ((void *)-1)
88
Linus Torvalds1da177e2005-04-16 15:20:36 -070089/*
90 * Bits for the md->flags field.
91 */
92#define DMF_BLOCK_IO 0
93#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080094#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -070095#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070096#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080097#define DMF_NOFLUSH_SUSPENDING 5
Linus Torvalds1da177e2005-04-16 15:20:36 -070098
Milan Broz304f3f62008-02-08 02:11:17 +000099/*
100 * Work processed by per-device workqueue.
101 */
102struct dm_wq_req {
103 enum {
Milan Broz304f3f62008-02-08 02:11:17 +0000104 DM_WQ_FLUSH_DEFERRED,
105 } type;
106 struct work_struct work;
107 struct mapped_device *md;
108 void *context;
109};
110
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700112 struct rw_semaphore io_lock;
Daniel Walkere61290a2008-02-08 02:10:08 +0000113 struct mutex suspend_lock;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800114 spinlock_t pushback_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 rwlock_t map_lock;
116 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700117 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118
119 unsigned long flags;
120
Jens Axboe165125e2007-07-24 09:28:11 +0200121 struct request_queue *queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -0800123 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124
125 void *interface_ptr;
126
127 /*
128 * A list of ios that arrived while we were suspended.
129 */
130 atomic_t pending;
131 wait_queue_head_t wait;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800132 struct bio_list deferred;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800133 struct bio_list pushback;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
135 /*
Milan Broz304f3f62008-02-08 02:11:17 +0000136 * Processing queue (flush/barriers)
137 */
138 struct workqueue_struct *wq;
139
140 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141 * The current mapping.
142 */
143 struct dm_table *map;
144
145 /*
146 * io objects are allocated from here.
147 */
148 mempool_t *io_pool;
149 mempool_t *tio_pool;
150
Stefan Bader9faf4002006-10-03 01:15:41 -0700151 struct bio_set *bs;
152
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 /*
154 * Event handling.
155 */
156 atomic_t event_nr;
157 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100158 atomic_t uevent_seq;
159 struct list_head uevent_list;
160 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161
162 /*
163 * freeze/thaw support require holding onto a super block
164 */
165 struct super_block *frozen_sb;
Alasdair G Kergone39e2e92006-01-06 00:20:05 -0800166 struct block_device *suspended_bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800167
168 /* forced geometry settings */
169 struct hd_geometry geometry;
Milan Broz784aae72009-01-06 03:05:12 +0000170
171 /* sysfs handle */
172 struct kobject kobj;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173};
174
175#define MIN_IOS 256
Christoph Lametere18b8902006-12-06 20:33:20 -0800176static struct kmem_cache *_io_cache;
177static struct kmem_cache *_tio_cache;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000178static struct kmem_cache *_rq_tio_cache;
179static struct kmem_cache *_rq_bio_info_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181static int __init local_init(void)
182{
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100183 int r = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100186 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 if (!_io_cache)
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100188 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189
190 /* allocate a slab for the target ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100191 _tio_cache = KMEM_CACHE(dm_target_io, 0);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100192 if (!_tio_cache)
193 goto out_free_io_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000195 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
196 if (!_rq_tio_cache)
197 goto out_free_tio_cache;
198
199 _rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
200 if (!_rq_bio_info_cache)
201 goto out_free_rq_tio_cache;
202
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100203 r = dm_uevent_init();
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100204 if (r)
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000205 goto out_free_rq_bio_info_cache;
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100206
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 _major = major;
208 r = register_blkdev(_major, _name);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100209 if (r < 0)
210 goto out_uevent_exit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211
212 if (!_major)
213 _major = r;
214
215 return 0;
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100216
217out_uevent_exit:
218 dm_uevent_exit();
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000219out_free_rq_bio_info_cache:
220 kmem_cache_destroy(_rq_bio_info_cache);
221out_free_rq_tio_cache:
222 kmem_cache_destroy(_rq_tio_cache);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100223out_free_tio_cache:
224 kmem_cache_destroy(_tio_cache);
225out_free_io_cache:
226 kmem_cache_destroy(_io_cache);
227
228 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229}
230
231static void local_exit(void)
232{
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000233 kmem_cache_destroy(_rq_bio_info_cache);
234 kmem_cache_destroy(_rq_tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 kmem_cache_destroy(_tio_cache);
236 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700237 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100238 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239
240 _major = 0;
241
242 DMINFO("cleaned up");
243}
244
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000245static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246 local_init,
247 dm_target_init,
248 dm_linear_init,
249 dm_stripe_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100250 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 dm_interface_init,
252};
253
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000254static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 local_exit,
256 dm_target_exit,
257 dm_linear_exit,
258 dm_stripe_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100259 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 dm_interface_exit,
261};
262
263static int __init dm_init(void)
264{
265 const int count = ARRAY_SIZE(_inits);
266
267 int r, i;
268
269 for (i = 0; i < count; i++) {
270 r = _inits[i]();
271 if (r)
272 goto bad;
273 }
274
275 return 0;
276
277 bad:
278 while (i--)
279 _exits[i]();
280
281 return r;
282}
283
284static void __exit dm_exit(void)
285{
286 int i = ARRAY_SIZE(_exits);
287
288 while (i--)
289 _exits[i]();
290}
291
292/*
293 * Block device functions
294 */
Al Virofe5f9f22008-03-02 10:29:31 -0500295static int dm_blk_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296{
297 struct mapped_device *md;
298
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700299 spin_lock(&_minor_lock);
300
Al Virofe5f9f22008-03-02 10:29:31 -0500301 md = bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700302 if (!md)
303 goto out;
304
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700305 if (test_bit(DMF_FREEING, &md->flags) ||
306 test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700307 md = NULL;
308 goto out;
309 }
310
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700312 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700313
314out:
315 spin_unlock(&_minor_lock);
316
317 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318}
319
Al Virofe5f9f22008-03-02 10:29:31 -0500320static int dm_blk_close(struct gendisk *disk, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321{
Al Virofe5f9f22008-03-02 10:29:31 -0500322 struct mapped_device *md = disk->private_data;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700323 atomic_dec(&md->open_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 dm_put(md);
325 return 0;
326}
327
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700328int dm_open_count(struct mapped_device *md)
329{
330 return atomic_read(&md->open_count);
331}
332
333/*
334 * Guarantees nothing is using the device before it's deleted.
335 */
336int dm_lock_for_deletion(struct mapped_device *md)
337{
338 int r = 0;
339
340 spin_lock(&_minor_lock);
341
342 if (dm_open_count(md))
343 r = -EBUSY;
344 else
345 set_bit(DMF_DELETING, &md->flags);
346
347 spin_unlock(&_minor_lock);
348
349 return r;
350}
351
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800352static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
353{
354 struct mapped_device *md = bdev->bd_disk->private_data;
355
356 return dm_get_geometry(md, geo);
357}
358
Al Virofe5f9f22008-03-02 10:29:31 -0500359static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
Milan Brozaa129a22006-10-03 01:15:15 -0700360 unsigned int cmd, unsigned long arg)
361{
Al Virofe5f9f22008-03-02 10:29:31 -0500362 struct mapped_device *md = bdev->bd_disk->private_data;
363 struct dm_table *map = dm_get_table(md);
Milan Brozaa129a22006-10-03 01:15:15 -0700364 struct dm_target *tgt;
365 int r = -ENOTTY;
366
Milan Brozaa129a22006-10-03 01:15:15 -0700367 if (!map || !dm_table_get_size(map))
368 goto out;
369
370 /* We only support devices that have a single target */
371 if (dm_table_get_num_targets(map) != 1)
372 goto out;
373
374 tgt = dm_table_get_target(map, 0);
375
376 if (dm_suspended(md)) {
377 r = -EAGAIN;
378 goto out;
379 }
380
381 if (tgt->type->ioctl)
Al Viro647b3d02007-08-28 22:15:59 -0400382 r = tgt->type->ioctl(tgt, cmd, arg);
Milan Brozaa129a22006-10-03 01:15:15 -0700383
384out:
385 dm_table_put(map);
386
Milan Brozaa129a22006-10-03 01:15:15 -0700387 return r;
388}
389
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100390static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391{
392 return mempool_alloc(md->io_pool, GFP_NOIO);
393}
394
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100395static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396{
397 mempool_free(io, md->io_pool);
398}
399
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100400static struct dm_target_io *alloc_tio(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401{
402 return mempool_alloc(md->tio_pool, GFP_NOIO);
403}
404
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100405static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406{
407 mempool_free(tio, md->tio_pool);
408}
409
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800410static void start_io_acct(struct dm_io *io)
411{
412 struct mapped_device *md = io->md;
Tejun Heoc9959052008-08-25 19:47:21 +0900413 int cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800414
415 io->start_time = jiffies;
416
Tejun Heo074a7ac2008-08-25 19:56:14 +0900417 cpu = part_stat_lock();
418 part_round_stats(cpu, &dm_disk(md)->part0);
419 part_stat_unlock();
420 dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800421}
422
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000423static void end_io_acct(struct dm_io *io)
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800424{
425 struct mapped_device *md = io->md;
426 struct bio *bio = io->bio;
427 unsigned long duration = jiffies - io->start_time;
Tejun Heoc9959052008-08-25 19:47:21 +0900428 int pending, cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800429 int rw = bio_data_dir(bio);
430
Tejun Heo074a7ac2008-08-25 19:56:14 +0900431 cpu = part_stat_lock();
432 part_round_stats(cpu, &dm_disk(md)->part0);
433 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
434 part_stat_unlock();
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800435
Tejun Heo074a7ac2008-08-25 19:56:14 +0900436 dm_disk(md)->part0.in_flight = pending =
437 atomic_dec_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800438
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000439 /* nudge anyone waiting on suspend queue */
440 if (!pending)
441 wake_up(&md->wait);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800442}
443
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444/*
445 * Add the bio to the list of deferred io.
446 */
447static int queue_io(struct mapped_device *md, struct bio *bio)
448{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700449 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450
451 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700452 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 return 1;
454 }
455
456 bio_list_add(&md->deferred, bio);
457
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700458 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 return 0; /* deferred successfully */
460}
461
462/*
463 * Everyone (including functions in this file), should use this
464 * function to access the md->map field, and make sure they call
465 * dm_table_put() when finished.
466 */
467struct dm_table *dm_get_table(struct mapped_device *md)
468{
469 struct dm_table *t;
470
471 read_lock(&md->map_lock);
472 t = md->map;
473 if (t)
474 dm_table_get(t);
475 read_unlock(&md->map_lock);
476
477 return t;
478}
479
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800480/*
481 * Get the geometry associated with a dm device
482 */
483int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
484{
485 *geo = md->geometry;
486
487 return 0;
488}
489
490/*
491 * Set the geometry of a device.
492 */
493int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
494{
495 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
496
497 if (geo->start > sz) {
498 DMWARN("Start sector is beyond the geometry limits.");
499 return -EINVAL;
500 }
501
502 md->geometry = *geo;
503
504 return 0;
505}
506
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507/*-----------------------------------------------------------------
508 * CRUD START:
509 * A more elegant soln is in the works that uses the queue
510 * merge fn, unfortunately there are a couple of changes to
511 * the block layer that I want to make for this. So in the
512 * interests of getting something for people to use I give
513 * you this clearly demarcated crap.
514 *---------------------------------------------------------------*/
515
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800516static int __noflush_suspending(struct mapped_device *md)
517{
518 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
519}
520
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521/*
522 * Decrements the number of outstanding ios that a bio has been
523 * cloned into, completing the original io if necc.
524 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800525static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800527 unsigned long flags;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000528 int io_error;
529 struct bio *bio;
530 struct mapped_device *md = io->md;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800531
532 /* Push-back supersedes any I/O errors */
Milan Brozb35f8ca2009-03-16 17:44:36 +0000533 if (error && !(io->error > 0 && __noflush_suspending(md)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 io->error = error;
535
536 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800537 if (io->error == DM_ENDIO_REQUEUE) {
538 /*
539 * Target requested pushing back the I/O.
540 * This must be handled before the sleeper on
541 * suspend queue merges the pushback list.
542 */
Milan Brozb35f8ca2009-03-16 17:44:36 +0000543 spin_lock_irqsave(&md->pushback_lock, flags);
544 if (__noflush_suspending(md))
545 bio_list_add(&md->pushback, io->bio);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800546 else
547 /* noflush suspend was interrupted. */
548 io->error = -EIO;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000549 spin_unlock_irqrestore(&md->pushback_lock, flags);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800550 }
551
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000552 end_io_acct(io);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553
Milan Brozb35f8ca2009-03-16 17:44:36 +0000554 io_error = io->error;
555 bio = io->bio;
Jens Axboe2056a782006-03-23 20:00:26 +0100556
Milan Brozb35f8ca2009-03-16 17:44:36 +0000557 free_io(md, io);
558
559 if (io_error != DM_ENDIO_REQUEUE) {
560 trace_block_bio_complete(md->queue, bio);
561
562 bio_endio(bio, io_error);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800563 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 }
565}
566
NeilBrown6712ecf2007-09-27 12:47:43 +0200567static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568{
569 int r = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100570 struct dm_target_io *tio = bio->bi_private;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000571 struct dm_io *io = tio->io;
Stefan Bader9faf4002006-10-03 01:15:41 -0700572 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 dm_endio_fn endio = tio->ti->type->end_io;
574
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
576 error = -EIO;
577
578 if (endio) {
579 r = endio(tio->ti, bio, error, &tio->info);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800580 if (r < 0 || r == DM_ENDIO_REQUEUE)
581 /*
582 * error and requeue request are handled
583 * in dec_pending().
584 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800586 else if (r == DM_ENDIO_INCOMPLETE)
587 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200588 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800589 else if (r) {
590 DMWARN("unimplemented target endio return value: %d", r);
591 BUG();
592 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 }
594
Stefan Bader9faf4002006-10-03 01:15:41 -0700595 /*
596 * Store md for cleanup instead of tio which is about to get freed.
597 */
598 bio->bi_private = md->bs;
599
Stefan Bader9faf4002006-10-03 01:15:41 -0700600 free_tio(md, tio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000601 bio_put(bio);
602 dec_pending(io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603}
604
605static sector_t max_io_len(struct mapped_device *md,
606 sector_t sector, struct dm_target *ti)
607{
608 sector_t offset = sector - ti->begin;
609 sector_t len = ti->len - offset;
610
611 /*
612 * Does the target need to split even further ?
613 */
614 if (ti->split_io) {
615 sector_t boundary;
616 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
617 - offset;
618 if (len > boundary)
619 len = boundary;
620 }
621
622 return len;
623}
624
625static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100626 struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627{
628 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100629 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -0700630 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631
632 /*
633 * Sanity checks.
634 */
635 BUG_ON(!clone->bi_size);
636
637 clone->bi_end_io = clone_endio;
638 clone->bi_private = tio;
639
640 /*
641 * Map the clone. If r == 0 we don't need to do
642 * anything, the target has assumed ownership of
643 * this io.
644 */
645 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100646 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 r = ti->type->map(ti, clone, &tio->info);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800648 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100650
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100651 trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
Alan D. Brunellec7149d62007-08-07 15:30:23 +0200652 tio->io->bio->bi_bdev->bd_dev,
653 clone->bi_sector, sector);
Jens Axboe2056a782006-03-23 20:00:26 +0100654
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800656 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
657 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -0700658 md = tio->io->md;
659 dec_pending(tio->io, r);
660 /*
661 * Store bio_set for cleanup.
662 */
663 clone->bi_private = md->bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664 bio_put(clone);
Stefan Bader9faf4002006-10-03 01:15:41 -0700665 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800666 } else if (r) {
667 DMWARN("unimplemented target map return value: %d", r);
668 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669 }
670}
671
672struct clone_info {
673 struct mapped_device *md;
674 struct dm_table *map;
675 struct bio *bio;
676 struct dm_io *io;
677 sector_t sector;
678 sector_t sector_count;
679 unsigned short idx;
680};
681
Peter Osterlund36763472005-09-06 15:16:42 -0700682static void dm_bio_destructor(struct bio *bio)
683{
Stefan Bader9faf4002006-10-03 01:15:41 -0700684 struct bio_set *bs = bio->bi_private;
685
686 bio_free(bio, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700687}
688
Linus Torvalds1da177e2005-04-16 15:20:36 -0700689/*
690 * Creates a little bio that is just does part of a bvec.
691 */
692static struct bio *split_bvec(struct bio *bio, sector_t sector,
693 unsigned short idx, unsigned int offset,
Stefan Bader9faf4002006-10-03 01:15:41 -0700694 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695{
696 struct bio *clone;
697 struct bio_vec *bv = bio->bi_io_vec + idx;
698
Stefan Bader9faf4002006-10-03 01:15:41 -0700699 clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700700 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701 *clone->bi_io_vec = *bv;
702
703 clone->bi_sector = sector;
704 clone->bi_bdev = bio->bi_bdev;
705 clone->bi_rw = bio->bi_rw;
706 clone->bi_vcnt = 1;
707 clone->bi_size = to_bytes(len);
708 clone->bi_io_vec->bv_offset = offset;
709 clone->bi_io_vec->bv_len = clone->bi_size;
Martin K. Petersenf3e1d262008-10-21 17:45:04 +0100710 clone->bi_flags |= 1 << BIO_CLONED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711
712 return clone;
713}
714
715/*
716 * Creates a bio that consists of range of complete bvecs.
717 */
718static struct bio *clone_bio(struct bio *bio, sector_t sector,
719 unsigned short idx, unsigned short bv_count,
Stefan Bader9faf4002006-10-03 01:15:41 -0700720 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721{
722 struct bio *clone;
723
Stefan Bader9faf4002006-10-03 01:15:41 -0700724 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
725 __bio_clone(clone, bio);
726 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727 clone->bi_sector = sector;
728 clone->bi_idx = idx;
729 clone->bi_vcnt = idx + bv_count;
730 clone->bi_size = to_bytes(len);
731 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
732
733 return clone;
734}
735
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000736static int __clone_and_map(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737{
738 struct bio *clone, *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000739 struct dm_target *ti;
740 sector_t len = 0, max;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100741 struct dm_target_io *tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000743 ti = dm_table_find_target(ci->map, ci->sector);
744 if (!dm_target_is_valid(ti))
745 return -EIO;
746
747 max = max_io_len(ci->md, ci->sector, ti);
748
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 /*
750 * Allocate a target io object.
751 */
752 tio = alloc_tio(ci->md);
753 tio->io = ci->io;
754 tio->ti = ti;
755 memset(&tio->info, 0, sizeof(tio->info));
756
757 if (ci->sector_count <= max) {
758 /*
759 * Optimise for the simple case where we can do all of
760 * the remaining io with a single clone.
761 */
762 clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700763 bio->bi_vcnt - ci->idx, ci->sector_count,
764 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765 __map_bio(ti, clone, tio);
766 ci->sector_count = 0;
767
768 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
769 /*
770 * There are some bvecs that don't span targets.
771 * Do as many of these as possible.
772 */
773 int i;
774 sector_t remaining = max;
775 sector_t bv_len;
776
777 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
778 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
779
780 if (bv_len > remaining)
781 break;
782
783 remaining -= bv_len;
784 len += bv_len;
785 }
786
Stefan Bader9faf4002006-10-03 01:15:41 -0700787 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
788 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 __map_bio(ti, clone, tio);
790
791 ci->sector += len;
792 ci->sector_count -= len;
793 ci->idx = i;
794
795 } else {
796 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800797 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798 */
799 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800800 sector_t remaining = to_sector(bv->bv_len);
801 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800803 do {
804 if (offset) {
805 ti = dm_table_find_target(ci->map, ci->sector);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000806 if (!dm_target_is_valid(ti))
807 return -EIO;
808
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800809 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700810
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800811 tio = alloc_tio(ci->md);
812 tio->io = ci->io;
813 tio->ti = ti;
814 memset(&tio->info, 0, sizeof(tio->info));
815 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800817 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800819 clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700820 bv->bv_offset + offset, len,
821 ci->md->bs);
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800822
823 __map_bio(ti, clone, tio);
824
825 ci->sector += len;
826 ci->sector_count -= len;
827 offset += to_bytes(len);
828 } while (remaining -= len);
829
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830 ci->idx++;
831 }
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000832
833 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834}
835
836/*
837 * Split the bio into several clones.
838 */
Milan Broz9e4e5f82007-10-19 22:38:53 +0100839static int __split_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840{
841 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000842 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843
844 ci.map = dm_get_table(md);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100845 if (unlikely(!ci.map))
846 return -EIO;
Andi Kleenab4c142482009-01-06 03:05:09 +0000847 if (unlikely(bio_barrier(bio) && !dm_table_barrier_ok(ci.map))) {
848 dm_table_put(ci.map);
849 bio_endio(bio, -EOPNOTSUPP);
850 return 0;
851 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852 ci.md = md;
853 ci.bio = bio;
854 ci.io = alloc_io(md);
855 ci.io->error = 0;
856 atomic_set(&ci.io->io_count, 1);
857 ci.io->bio = bio;
858 ci.io->md = md;
859 ci.sector = bio->bi_sector;
860 ci.sector_count = bio_sectors(bio);
861 ci.idx = bio->bi_idx;
862
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800863 start_io_acct(ci.io);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000864 while (ci.sector_count && !error)
865 error = __clone_and_map(&ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866
867 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000868 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869 dm_table_put(ci.map);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100870
871 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872}
873/*-----------------------------------------------------------------
874 * CRUD END
875 *---------------------------------------------------------------*/
876
Milan Brozf6fccb12008-07-21 12:00:37 +0100877static int dm_merge_bvec(struct request_queue *q,
878 struct bvec_merge_data *bvm,
879 struct bio_vec *biovec)
880{
881 struct mapped_device *md = q->queuedata;
882 struct dm_table *map = dm_get_table(md);
883 struct dm_target *ti;
884 sector_t max_sectors;
Mikulas Patocka50371082008-10-01 14:39:17 +0100885 int max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +0100886
887 if (unlikely(!map))
Mikulas Patocka50371082008-10-01 14:39:17 +0100888 goto out;
Milan Brozf6fccb12008-07-21 12:00:37 +0100889
890 ti = dm_table_find_target(map, bvm->bi_sector);
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100891 if (!dm_target_is_valid(ti))
892 goto out_table;
Milan Brozf6fccb12008-07-21 12:00:37 +0100893
894 /*
895 * Find maximum amount of I/O that won't need splitting
896 */
897 max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
898 (sector_t) BIO_MAX_SECTORS);
899 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
900 if (max_size < 0)
901 max_size = 0;
902
903 /*
904 * merge_bvec_fn() returns number of bytes
905 * it can accept at this offset
906 * max is precomputed maximal io size
907 */
908 if (max_size && ti->type->merge)
909 max_size = ti->type->merge(ti, bvm, biovec, max_size);
910
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100911out_table:
Mikulas Patocka50371082008-10-01 14:39:17 +0100912 dm_table_put(map);
913
914out:
Milan Brozf6fccb12008-07-21 12:00:37 +0100915 /*
916 * Always allow an entire first page
917 */
918 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
919 max_size = biovec->bv_len;
920
Milan Brozf6fccb12008-07-21 12:00:37 +0100921 return max_size;
922}
923
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924/*
925 * The request function that just remaps the bio built up by
926 * dm_merge_bvec.
927 */
Jens Axboe165125e2007-07-24 09:28:11 +0200928static int dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929{
Milan Broz9e4e5f82007-10-19 22:38:53 +0100930 int r = -EIO;
Kevin Corry12f03a42006-02-01 03:04:52 -0800931 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932 struct mapped_device *md = q->queuedata;
Tejun Heoc9959052008-08-25 19:47:21 +0900933 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700935 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700936
Tejun Heo074a7ac2008-08-25 19:56:14 +0900937 cpu = part_stat_lock();
938 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
939 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
940 part_stat_unlock();
Kevin Corry12f03a42006-02-01 03:04:52 -0800941
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942 /*
943 * If we're suspended we have to queue
944 * this io for later.
945 */
946 while (test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700947 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948
Milan Broz9e4e5f82007-10-19 22:38:53 +0100949 if (bio_rw(bio) != READA)
950 r = queue_io(md, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951
Milan Broz9e4e5f82007-10-19 22:38:53 +0100952 if (r <= 0)
953 goto out_req;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954
955 /*
956 * We're in a while loop, because someone could suspend
957 * before we get to the following read lock.
958 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700959 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960 }
961
Milan Broz9e4e5f82007-10-19 22:38:53 +0100962 r = __split_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700963 up_read(&md->io_lock);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100964
965out_req:
966 if (r < 0)
967 bio_io_error(bio);
968
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 return 0;
970}
971
Jens Axboe165125e2007-07-24 09:28:11 +0200972static void dm_unplug_all(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973{
974 struct mapped_device *md = q->queuedata;
975 struct dm_table *map = dm_get_table(md);
976
977 if (map) {
978 dm_table_unplug_all(map);
979 dm_table_put(map);
980 }
981}
982
983static int dm_any_congested(void *congested_data, int bdi_bits)
984{
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +0000985 int r = bdi_bits;
986 struct mapped_device *md = congested_data;
987 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +0000989 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
990 map = dm_get_table(md);
991 if (map) {
992 r = dm_table_any_congested(map, bdi_bits);
993 dm_table_put(map);
994 }
995 }
996
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997 return r;
998}
999
1000/*-----------------------------------------------------------------
1001 * An IDR is used to keep track of allocated minor numbers.
1002 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003static DEFINE_IDR(_minor_idr);
1004
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001005static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001007 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001009 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010}
1011
1012/*
1013 * See if the device with a specific minor # is free.
1014 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001015static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016{
1017 int r, m;
1018
1019 if (minor >= (1 << MINORBITS))
1020 return -EINVAL;
1021
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001022 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
1023 if (!r)
1024 return -ENOMEM;
1025
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001026 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027
1028 if (idr_find(&_minor_idr, minor)) {
1029 r = -EBUSY;
1030 goto out;
1031 }
1032
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001033 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001034 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036
1037 if (m != minor) {
1038 idr_remove(&_minor_idr, m);
1039 r = -EBUSY;
1040 goto out;
1041 }
1042
1043out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001044 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 return r;
1046}
1047
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001048static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001050 int r, m;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001053 if (!r)
1054 return -ENOMEM;
1055
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001056 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001058 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001059 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061
1062 if (m >= (1 << MINORBITS)) {
1063 idr_remove(&_minor_idr, m);
1064 r = -ENOSPC;
1065 goto out;
1066 }
1067
1068 *minor = m;
1069
1070out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001071 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072 return r;
1073}
1074
1075static struct block_device_operations dm_blk_dops;
1076
1077/*
1078 * Allocate and initialise a blank device with a given minor.
1079 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001080static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081{
1082 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001083 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001084 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085
1086 if (!md) {
1087 DMWARN("unable to allocate device, out of memory.");
1088 return NULL;
1089 }
1090
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001091 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001092 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001093
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001095 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001096 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001097 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001098 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001100 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001101
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001102 init_rwsem(&md->io_lock);
Daniel Walkere61290a2008-02-08 02:10:08 +00001103 mutex_init(&md->suspend_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001104 spin_lock_init(&md->pushback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105 rwlock_init(&md->map_lock);
1106 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001107 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001109 atomic_set(&md->uevent_seq, 0);
1110 INIT_LIST_HEAD(&md->uevent_list);
1111 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112
1113 md->queue = blk_alloc_queue(GFP_KERNEL);
1114 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001115 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116
1117 md->queue->queuedata = md;
1118 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1119 md->queue->backing_dev_info.congested_data = md;
1120 blk_queue_make_request(md->queue, dm_request);
Jens Axboedaef2652006-01-10 10:48:02 +01001121 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122 md->queue->unplug_fn = dm_unplug_all;
Milan Brozf6fccb12008-07-21 12:00:37 +01001123 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124
Matthew Dobson93d23412006-03-26 01:37:50 -08001125 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
Kiyoshi Ueda74859362006-12-08 02:41:02 -08001126 if (!md->io_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001127 goto bad_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128
Matthew Dobson93d23412006-03-26 01:37:50 -08001129 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130 if (!md->tio_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001131 goto bad_tio_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132
Jens Axboebb799ca2008-12-10 15:35:05 +01001133 md->bs = bioset_create(16, 0);
Stefan Bader9faf4002006-10-03 01:15:41 -07001134 if (!md->bs)
1135 goto bad_no_bioset;
1136
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 md->disk = alloc_disk(1);
1138 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001139 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001141 atomic_set(&md->pending, 0);
1142 init_waitqueue_head(&md->wait);
1143 init_waitqueue_head(&md->eventq);
1144
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 md->disk->major = _major;
1146 md->disk->first_minor = minor;
1147 md->disk->fops = &dm_blk_dops;
1148 md->disk->queue = md->queue;
1149 md->disk->private_data = md;
1150 sprintf(md->disk->disk_name, "dm-%d", minor);
1151 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001152 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153
Milan Broz304f3f62008-02-08 02:11:17 +00001154 md->wq = create_singlethread_workqueue("kdmflush");
1155 if (!md->wq)
1156 goto bad_thread;
1157
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001158 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001159 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001160 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001161 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001162
1163 BUG_ON(old_md != MINOR_ALLOCED);
1164
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165 return md;
1166
Milan Broz304f3f62008-02-08 02:11:17 +00001167bad_thread:
1168 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001169bad_disk:
Stefan Bader9faf4002006-10-03 01:15:41 -07001170 bioset_free(md->bs);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001171bad_no_bioset:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172 mempool_destroy(md->tio_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001173bad_tio_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 mempool_destroy(md->io_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001175bad_io_pool:
Al Viro1312f402006-03-12 11:02:03 -05001176 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001177bad_queue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001179bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001180 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001181bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182 kfree(md);
1183 return NULL;
1184}
1185
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001186static void unlock_fs(struct mapped_device *md);
1187
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188static void free_dev(struct mapped_device *md)
1189{
Tejun Heof331c022008-09-03 09:01:48 +02001190 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001191
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001192 if (md->suspended_bdev) {
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001193 unlock_fs(md);
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001194 bdput(md->suspended_bdev);
1195 }
Milan Broz304f3f62008-02-08 02:11:17 +00001196 destroy_workqueue(md->wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 mempool_destroy(md->tio_pool);
1198 mempool_destroy(md->io_pool);
Stefan Bader9faf4002006-10-03 01:15:41 -07001199 bioset_free(md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001201 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001202
1203 spin_lock(&_minor_lock);
1204 md->disk->private_data = NULL;
1205 spin_unlock(&_minor_lock);
1206
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05001208 blk_cleanup_queue(md->queue);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001209 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210 kfree(md);
1211}
1212
1213/*
1214 * Bind a table to the device.
1215 */
1216static void event_callback(void *context)
1217{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001218 unsigned long flags;
1219 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 struct mapped_device *md = (struct mapped_device *) context;
1221
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001222 spin_lock_irqsave(&md->uevent_lock, flags);
1223 list_splice_init(&md->uevent_list, &uevents);
1224 spin_unlock_irqrestore(&md->uevent_lock, flags);
1225
Tejun Heoed9e1982008-08-25 19:56:05 +09001226 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001227
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 atomic_inc(&md->event_nr);
1229 wake_up(&md->eventq);
1230}
1231
Alasdair G Kergon4e901882005-07-28 21:15:59 -07001232static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233{
Alasdair G Kergon4e901882005-07-28 21:15:59 -07001234 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001236 mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001237 i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001238 mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239}
1240
1241static int __bind(struct mapped_device *md, struct dm_table *t)
1242{
Jens Axboe165125e2007-07-24 09:28:11 +02001243 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244 sector_t size;
1245
1246 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001247
1248 /*
1249 * Wipe any geometry if the size of the table changed.
1250 */
1251 if (size != get_capacity(md->disk))
1252 memset(&md->geometry, 0, sizeof(md->geometry));
1253
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001254 if (md->suspended_bdev)
1255 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256
Mikulas Patockad5816872009-01-06 03:05:10 +00001257 if (!size) {
1258 dm_table_destroy(t);
1259 return 0;
1260 }
1261
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001262 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001263
1264 write_lock(&md->map_lock);
1265 md->map = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266 dm_table_set_restrictions(t, q);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001267 write_unlock(&md->map_lock);
1268
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 return 0;
1270}
1271
1272static void __unbind(struct mapped_device *md)
1273{
1274 struct dm_table *map = md->map;
1275
1276 if (!map)
1277 return;
1278
1279 dm_table_event_callback(map, NULL, NULL);
1280 write_lock(&md->map_lock);
1281 md->map = NULL;
1282 write_unlock(&md->map_lock);
Mikulas Patockad5816872009-01-06 03:05:10 +00001283 dm_table_destroy(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284}
1285
1286/*
1287 * Constructor for a new device.
1288 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001289int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290{
1291 struct mapped_device *md;
1292
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001293 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294 if (!md)
1295 return -ENXIO;
1296
Milan Broz784aae72009-01-06 03:05:12 +00001297 dm_sysfs_init(md);
1298
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 *result = md;
1300 return 0;
1301}
1302
David Teigland637842c2006-01-06 00:20:00 -08001303static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304{
1305 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306 unsigned minor = MINOR(dev);
1307
1308 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1309 return NULL;
1310
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001311 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312
1313 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001314 if (md && (md == MINOR_ALLOCED ||
Tejun Heof331c022008-09-03 09:01:48 +02001315 (MINOR(disk_devt(dm_disk(md))) != minor) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07001316 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08001317 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001318 goto out;
1319 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001321out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001322 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323
David Teigland637842c2006-01-06 00:20:00 -08001324 return md;
1325}
1326
David Teiglandd229a952006-01-06 00:20:01 -08001327struct mapped_device *dm_get_md(dev_t dev)
1328{
1329 struct mapped_device *md = dm_find_md(dev);
1330
1331 if (md)
1332 dm_get(md);
1333
1334 return md;
1335}
1336
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001337void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08001338{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001339 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340}
1341
1342void dm_set_mdptr(struct mapped_device *md, void *ptr)
1343{
1344 md->interface_ptr = ptr;
1345}
1346
1347void dm_get(struct mapped_device *md)
1348{
1349 atomic_inc(&md->holders);
1350}
1351
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001352const char *dm_device_name(struct mapped_device *md)
1353{
1354 return md->name;
1355}
1356EXPORT_SYMBOL_GPL(dm_device_name);
1357
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358void dm_put(struct mapped_device *md)
1359{
Mike Anderson1134e5a2006-03-27 01:17:54 -08001360 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001362 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1363
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001364 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08001365 map = dm_get_table(md);
Tejun Heof331c022008-09-03 09:01:48 +02001366 idr_replace(&_minor_idr, MINOR_ALLOCED,
1367 MINOR(disk_devt(dm_disk(md))));
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001368 set_bit(DMF_FREEING, &md->flags);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001369 spin_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001370 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371 dm_table_presuspend_targets(map);
1372 dm_table_postsuspend_targets(map);
1373 }
Milan Broz784aae72009-01-06 03:05:12 +00001374 dm_sysfs_exit(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08001375 dm_table_put(map);
Mikulas Patockaa1b51e92009-01-06 03:04:53 +00001376 __unbind(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377 free_dev(md);
1378 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379}
Edward Goggin79eb8852007-05-09 02:32:56 -07001380EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381
Milan Broz46125c12008-02-08 02:10:30 +00001382static int dm_wait_for_completion(struct mapped_device *md)
1383{
1384 int r = 0;
1385
1386 while (1) {
1387 set_current_state(TASK_INTERRUPTIBLE);
1388
1389 smp_mb();
1390 if (!atomic_read(&md->pending))
1391 break;
1392
1393 if (signal_pending(current)) {
1394 r = -EINTR;
1395 break;
1396 }
1397
1398 io_schedule();
1399 }
1400 set_current_state(TASK_RUNNING);
1401
1402 return r;
1403}
1404
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405/*
1406 * Process the deferred bios
1407 */
Milan Broz6d6f10d2008-02-08 02:10:22 +00001408static void __flush_deferred_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409{
Milan Broz6d6f10d2008-02-08 02:10:22 +00001410 struct bio *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001411
Milan Broz6d6f10d2008-02-08 02:10:22 +00001412 while ((c = bio_list_pop(&md->deferred))) {
Milan Broz9e4e5f82007-10-19 22:38:53 +01001413 if (__split_bio(md, c))
1414 bio_io_error(c);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415 }
Milan Broz73d410c2008-02-08 02:10:25 +00001416
1417 clear_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418}
1419
Milan Broz6d6f10d2008-02-08 02:10:22 +00001420static void __merge_pushback_list(struct mapped_device *md)
1421{
1422 unsigned long flags;
1423
1424 spin_lock_irqsave(&md->pushback_lock, flags);
1425 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1426 bio_list_merge_head(&md->deferred, &md->pushback);
1427 bio_list_init(&md->pushback);
1428 spin_unlock_irqrestore(&md->pushback_lock, flags);
1429}
1430
Milan Broz304f3f62008-02-08 02:11:17 +00001431static void dm_wq_work(struct work_struct *work)
1432{
1433 struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
1434 struct mapped_device *md = req->md;
1435
1436 down_write(&md->io_lock);
1437 switch (req->type) {
Milan Broz304f3f62008-02-08 02:11:17 +00001438 case DM_WQ_FLUSH_DEFERRED:
1439 __flush_deferred_io(md);
1440 break;
1441 default:
1442 DMERR("dm_wq_work: unrecognised work type %d", req->type);
1443 BUG();
1444 }
1445 up_write(&md->io_lock);
1446}
1447
1448static void dm_wq_queue(struct mapped_device *md, int type, void *context,
1449 struct dm_wq_req *req)
1450{
1451 req->type = type;
1452 req->md = md;
1453 req->context = context;
1454 INIT_WORK(&req->work, dm_wq_work);
1455 queue_work(md->wq, &req->work);
1456}
1457
1458static void dm_queue_flush(struct mapped_device *md, int type, void *context)
1459{
1460 struct dm_wq_req req;
1461
1462 dm_wq_queue(md, type, context, &req);
1463 flush_workqueue(md->wq);
1464}
1465
Linus Torvalds1da177e2005-04-16 15:20:36 -07001466/*
1467 * Swap in a new table (destroying old one).
1468 */
1469int dm_swap_table(struct mapped_device *md, struct dm_table *table)
1470{
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001471 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472
Daniel Walkere61290a2008-02-08 02:10:08 +00001473 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474
1475 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001476 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001477 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001479 /* without bdev, the device size cannot be changed */
1480 if (!md->suspended_bdev)
1481 if (get_capacity(md->disk) != dm_table_get_size(table))
1482 goto out;
1483
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 __unbind(md);
1485 r = __bind(md, table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001487out:
Daniel Walkere61290a2008-02-08 02:10:08 +00001488 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001489 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490}
1491
1492/*
1493 * Functions to lock and unlock any filesystem running on the
1494 * device.
1495 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001496static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001498 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499
1500 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001501
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001502 md->frozen_sb = freeze_bdev(md->suspended_bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001503 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001504 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001505 md->frozen_sb = NULL;
1506 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001507 }
1508
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001509 set_bit(DMF_FROZEN, &md->flags);
1510
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 /* don't bdput right now, we don't want the bdev
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001512 * to go away while it is locked.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513 */
1514 return 0;
1515}
1516
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001517static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001519 if (!test_bit(DMF_FROZEN, &md->flags))
1520 return;
1521
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001522 thaw_bdev(md->suspended_bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001524 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525}
1526
1527/*
1528 * We need to be able to change a mapping table under a mounted
1529 * filesystem. For example we might want to move some data in
1530 * the background. Before the table can be swapped with
1531 * dm_bind_table, dm_suspend must be called to flush any in
1532 * flight bios and ensure that any further io gets deferred.
1533 */
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001534int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001535{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001536 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537 DECLARE_WAITQUEUE(wait, current);
Milan Broz46125c12008-02-08 02:10:30 +00001538 int r = 0;
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001539 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001540 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541
Daniel Walkere61290a2008-02-08 02:10:08 +00001542 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001543
Milan Broz73d410c2008-02-08 02:10:25 +00001544 if (dm_suspended(md)) {
1545 r = -EINVAL;
Alasdair G Kergond2874832006-11-08 17:44:43 -08001546 goto out_unlock;
Milan Broz73d410c2008-02-08 02:10:25 +00001547 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548
1549 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001551 /*
1552 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
1553 * This flag is cleared before dm_suspend returns.
1554 */
1555 if (noflush)
1556 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1557
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001558 /* This does not get reverted if there's an error later. */
1559 dm_table_presuspend_targets(map);
1560
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001561 /* bdget() can stall if the pending I/Os are not flushed */
1562 if (!noflush) {
1563 md->suspended_bdev = bdget_disk(md->disk, 0);
1564 if (!md->suspended_bdev) {
1565 DMWARN("bdget failed in dm_suspend");
1566 r = -ENOMEM;
Kiyoshi Uedaf431d962008-10-21 17:45:07 +01001567 goto out;
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001568 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001569
Milan Broz6d6f10d2008-02-08 02:10:22 +00001570 /*
1571 * Flush I/O to the device. noflush supersedes do_lockfs,
1572 * because lock_fs() needs to flush I/Os.
1573 */
1574 if (do_lockfs) {
1575 r = lock_fs(md);
1576 if (r)
1577 goto out;
1578 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001579 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580
1581 /*
Alasdair G Kergon354e0072005-05-05 16:16:05 -07001582 * First we set the BLOCK_IO flag so no more ios will be mapped.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001584 down_write(&md->io_lock);
1585 set_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586
Linus Torvalds1da177e2005-04-16 15:20:36 -07001587 add_wait_queue(&md->wait, &wait);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001588 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589
1590 /* unplug */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001591 if (map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593
1594 /*
Milan Broz46125c12008-02-08 02:10:30 +00001595 * Wait for the already-mapped ios to complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596 */
Milan Broz46125c12008-02-08 02:10:30 +00001597 r = dm_wait_for_completion(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001599 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001600 remove_wait_queue(&md->wait, &wait);
1601
Milan Broz6d6f10d2008-02-08 02:10:22 +00001602 if (noflush)
1603 __merge_pushback_list(md);
Milan Broz94d63512008-02-08 02:10:27 +00001604 up_write(&md->io_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001605
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00001607 if (r < 0) {
Milan Broz304f3f62008-02-08 02:11:17 +00001608 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Milan Broz73d410c2008-02-08 02:10:25 +00001609
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001610 unlock_fs(md);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001611 goto out; /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001612 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001613
1614 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615
1616 set_bit(DMF_SUSPENDED, &md->flags);
1617
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001618out:
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001619 if (r && md->suspended_bdev) {
1620 bdput(md->suspended_bdev);
1621 md->suspended_bdev = NULL;
1622 }
1623
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624 dm_table_put(map);
Alasdair G Kergond2874832006-11-08 17:44:43 -08001625
1626out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00001627 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001628 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629}
1630
1631int dm_resume(struct mapped_device *md)
1632{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001633 int r = -EINVAL;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001634 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635
Daniel Walkere61290a2008-02-08 02:10:08 +00001636 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001637 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001638 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001639
1640 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001641 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001642 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643
Milan Broz8757b772006-10-03 01:15:36 -07001644 r = dm_table_resume_targets(map);
1645 if (r)
1646 goto out;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001647
Milan Broz304f3f62008-02-08 02:11:17 +00001648 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001649
1650 unlock_fs(md);
1651
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001652 if (md->suspended_bdev) {
1653 bdput(md->suspended_bdev);
1654 md->suspended_bdev = NULL;
1655 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001656
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001657 clear_bit(DMF_SUSPENDED, &md->flags);
1658
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001661 dm_kobject_uevent(md);
Hannes Reinecke8560ed62006-10-03 01:15:35 -07001662
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001663 r = 0;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001664
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001665out:
1666 dm_table_put(map);
Daniel Walkere61290a2008-02-08 02:10:08 +00001667 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001668
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001669 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670}
1671
1672/*-----------------------------------------------------------------
1673 * Event notification.
1674 *---------------------------------------------------------------*/
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001675void dm_kobject_uevent(struct mapped_device *md)
1676{
Tejun Heoed9e1982008-08-25 19:56:05 +09001677 kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001678}
1679
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001680uint32_t dm_next_uevent_seq(struct mapped_device *md)
1681{
1682 return atomic_add_return(1, &md->uevent_seq);
1683}
1684
Linus Torvalds1da177e2005-04-16 15:20:36 -07001685uint32_t dm_get_event_nr(struct mapped_device *md)
1686{
1687 return atomic_read(&md->event_nr);
1688}
1689
1690int dm_wait_event(struct mapped_device *md, int event_nr)
1691{
1692 return wait_event_interruptible(md->eventq,
1693 (event_nr != atomic_read(&md->event_nr)));
1694}
1695
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001696void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
1697{
1698 unsigned long flags;
1699
1700 spin_lock_irqsave(&md->uevent_lock, flags);
1701 list_add(elist, &md->uevent_list);
1702 spin_unlock_irqrestore(&md->uevent_lock, flags);
1703}
1704
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705/*
1706 * The gendisk is only valid as long as you have a reference
1707 * count on 'md'.
1708 */
1709struct gendisk *dm_disk(struct mapped_device *md)
1710{
1711 return md->disk;
1712}
1713
Milan Broz784aae72009-01-06 03:05:12 +00001714struct kobject *dm_kobject(struct mapped_device *md)
1715{
1716 return &md->kobj;
1717}
1718
1719/*
1720 * struct mapped_device should not be exported outside of dm.c
1721 * so use this check to verify that kobj is part of md structure
1722 */
1723struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
1724{
1725 struct mapped_device *md;
1726
1727 md = container_of(kobj, struct mapped_device, kobj);
1728 if (&md->kobj != kobj)
1729 return NULL;
1730
1731 dm_get(md);
1732 return md;
1733}
1734
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735int dm_suspended(struct mapped_device *md)
1736{
1737 return test_bit(DMF_SUSPENDED, &md->flags);
1738}
1739
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001740int dm_noflush_suspending(struct dm_target *ti)
1741{
1742 struct mapped_device *md = dm_table_get_md(ti->table);
1743 int r = __noflush_suspending(md);
1744
1745 dm_put(md);
1746
1747 return r;
1748}
1749EXPORT_SYMBOL_GPL(dm_noflush_suspending);
1750
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751static struct block_device_operations dm_blk_dops = {
1752 .open = dm_blk_open,
1753 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07001754 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001755 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001756 .owner = THIS_MODULE
1757};
1758
1759EXPORT_SYMBOL(dm_get_mapinfo);
1760
1761/*
1762 * module hooks
1763 */
1764module_init(dm_init);
1765module_exit(dm_exit);
1766
1767module_param(major, uint, 0);
1768MODULE_PARM_DESC(major, "The major number of the device mapper");
1769MODULE_DESCRIPTION(DM_NAME " driver");
1770MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1771MODULE_LICENSE("GPL");