blob: dc48c2585feb9442b902a6634bfd1870cd315b68 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07003 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
9#include "dm-bio-list.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +010010#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
12#include <linux/init.h>
13#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080014#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/moduleparam.h>
16#include <linux/blkpg.h>
17#include <linux/bio.h>
18#include <linux/buffer_head.h>
19#include <linux/mempool.h>
20#include <linux/slab.h>
21#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080022#include <linux/hdreg.h>
Jens Axboe2056a782006-03-23 20:00:26 +010023#include <linux/blktrace_api.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Alasdair G Kergon72d94862006-06-26 00:27:35 -070025#define DM_MSG_PREFIX "core"
26
Linus Torvalds1da177e2005-04-16 15:20:36 -070027static const char *_name = DM_NAME;
28
29static unsigned int major = 0;
30static unsigned int _major = 0;
31
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070032static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070033/*
34 * One of these is allocated per bio.
35 */
36struct dm_io {
37 struct mapped_device *md;
38 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -070039 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010040 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080041 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070042};
43
44/*
45 * One of these is allocated per target within a bio. Hopefully
46 * this will be simplified out one day.
47 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +010048struct dm_target_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 struct dm_io *io;
50 struct dm_target *ti;
51 union map_info info;
52};
53
54union map_info *dm_get_mapinfo(struct bio *bio)
55{
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070056 if (bio && bio->bi_private)
Alasdair G Kergon028867a2007-07-12 17:26:32 +010057 return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070058 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070059}
60
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070061#define MINOR_ALLOCED ((void *)-1)
62
Linus Torvalds1da177e2005-04-16 15:20:36 -070063/*
64 * Bits for the md->flags field.
65 */
66#define DMF_BLOCK_IO 0
67#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080068#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -070069#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070070#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080071#define DMF_NOFLUSH_SUSPENDING 5
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
Milan Broz304f3f62008-02-08 02:11:17 +000073/*
74 * Work processed by per-device workqueue.
75 */
76struct dm_wq_req {
77 enum {
78 DM_WQ_FLUSH_ALL,
79 DM_WQ_FLUSH_DEFERRED,
80 } type;
81 struct work_struct work;
82 struct mapped_device *md;
83 void *context;
84};
85
Linus Torvalds1da177e2005-04-16 15:20:36 -070086struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -070087 struct rw_semaphore io_lock;
Daniel Walkere61290a2008-02-08 02:10:08 +000088 struct mutex suspend_lock;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080089 spinlock_t pushback_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 rwlock_t map_lock;
91 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070092 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070093
94 unsigned long flags;
95
Jens Axboe165125e2007-07-24 09:28:11 +020096 struct request_queue *queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -080098 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -070099
100 void *interface_ptr;
101
102 /*
103 * A list of ios that arrived while we were suspended.
104 */
105 atomic_t pending;
106 wait_queue_head_t wait;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800107 struct bio_list deferred;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800108 struct bio_list pushback;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110 /*
Milan Broz304f3f62008-02-08 02:11:17 +0000111 * Processing queue (flush/barriers)
112 */
113 struct workqueue_struct *wq;
114
115 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 * The current mapping.
117 */
118 struct dm_table *map;
119
120 /*
121 * io objects are allocated from here.
122 */
123 mempool_t *io_pool;
124 mempool_t *tio_pool;
125
Stefan Bader9faf4002006-10-03 01:15:41 -0700126 struct bio_set *bs;
127
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 /*
129 * Event handling.
130 */
131 atomic_t event_nr;
132 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100133 atomic_t uevent_seq;
134 struct list_head uevent_list;
135 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
137 /*
138 * freeze/thaw support require holding onto a super block
139 */
140 struct super_block *frozen_sb;
Alasdair G Kergone39e2e92006-01-06 00:20:05 -0800141 struct block_device *suspended_bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800142
143 /* forced geometry settings */
144 struct hd_geometry geometry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145};
146
147#define MIN_IOS 256
Christoph Lametere18b8902006-12-06 20:33:20 -0800148static struct kmem_cache *_io_cache;
149static struct kmem_cache *_tio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151static int __init local_init(void)
152{
153 int r;
154
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100156 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 if (!_io_cache)
158 return -ENOMEM;
159
160 /* allocate a slab for the target ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100161 _tio_cache = KMEM_CACHE(dm_target_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 if (!_tio_cache) {
163 kmem_cache_destroy(_io_cache);
164 return -ENOMEM;
165 }
166
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100167 r = dm_uevent_init();
168 if (r) {
169 kmem_cache_destroy(_tio_cache);
170 kmem_cache_destroy(_io_cache);
171 return r;
172 }
173
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 _major = major;
175 r = register_blkdev(_major, _name);
176 if (r < 0) {
177 kmem_cache_destroy(_tio_cache);
178 kmem_cache_destroy(_io_cache);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100179 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 return r;
181 }
182
183 if (!_major)
184 _major = r;
185
186 return 0;
187}
188
189static void local_exit(void)
190{
191 kmem_cache_destroy(_tio_cache);
192 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700193 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100194 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
196 _major = 0;
197
198 DMINFO("cleaned up");
199}
200
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000201static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 local_init,
203 dm_target_init,
204 dm_linear_init,
205 dm_stripe_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100206 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 dm_interface_init,
208};
209
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000210static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 local_exit,
212 dm_target_exit,
213 dm_linear_exit,
214 dm_stripe_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100215 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 dm_interface_exit,
217};
218
219static int __init dm_init(void)
220{
221 const int count = ARRAY_SIZE(_inits);
222
223 int r, i;
224
225 for (i = 0; i < count; i++) {
226 r = _inits[i]();
227 if (r)
228 goto bad;
229 }
230
231 return 0;
232
233 bad:
234 while (i--)
235 _exits[i]();
236
237 return r;
238}
239
240static void __exit dm_exit(void)
241{
242 int i = ARRAY_SIZE(_exits);
243
244 while (i--)
245 _exits[i]();
246}
247
248/*
249 * Block device functions
250 */
Al Virofe5f9f22008-03-02 10:29:31 -0500251static int dm_blk_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252{
253 struct mapped_device *md;
254
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700255 spin_lock(&_minor_lock);
256
Al Virofe5f9f22008-03-02 10:29:31 -0500257 md = bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700258 if (!md)
259 goto out;
260
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700261 if (test_bit(DMF_FREEING, &md->flags) ||
262 test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700263 md = NULL;
264 goto out;
265 }
266
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700268 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700269
270out:
271 spin_unlock(&_minor_lock);
272
273 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274}
275
Al Virofe5f9f22008-03-02 10:29:31 -0500276static int dm_blk_close(struct gendisk *disk, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277{
Al Virofe5f9f22008-03-02 10:29:31 -0500278 struct mapped_device *md = disk->private_data;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700279 atomic_dec(&md->open_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 dm_put(md);
281 return 0;
282}
283
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700284int dm_open_count(struct mapped_device *md)
285{
286 return atomic_read(&md->open_count);
287}
288
289/*
290 * Guarantees nothing is using the device before it's deleted.
291 */
292int dm_lock_for_deletion(struct mapped_device *md)
293{
294 int r = 0;
295
296 spin_lock(&_minor_lock);
297
298 if (dm_open_count(md))
299 r = -EBUSY;
300 else
301 set_bit(DMF_DELETING, &md->flags);
302
303 spin_unlock(&_minor_lock);
304
305 return r;
306}
307
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800308static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
309{
310 struct mapped_device *md = bdev->bd_disk->private_data;
311
312 return dm_get_geometry(md, geo);
313}
314
Al Virofe5f9f22008-03-02 10:29:31 -0500315static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
Milan Brozaa129a22006-10-03 01:15:15 -0700316 unsigned int cmd, unsigned long arg)
317{
Al Virofe5f9f22008-03-02 10:29:31 -0500318 struct mapped_device *md = bdev->bd_disk->private_data;
319 struct dm_table *map = dm_get_table(md);
Milan Brozaa129a22006-10-03 01:15:15 -0700320 struct dm_target *tgt;
321 int r = -ENOTTY;
322
Milan Brozaa129a22006-10-03 01:15:15 -0700323 if (!map || !dm_table_get_size(map))
324 goto out;
325
326 /* We only support devices that have a single target */
327 if (dm_table_get_num_targets(map) != 1)
328 goto out;
329
330 tgt = dm_table_get_target(map, 0);
331
332 if (dm_suspended(md)) {
333 r = -EAGAIN;
334 goto out;
335 }
336
337 if (tgt->type->ioctl)
Al Viro647b3d02007-08-28 22:15:59 -0400338 r = tgt->type->ioctl(tgt, cmd, arg);
Milan Brozaa129a22006-10-03 01:15:15 -0700339
340out:
341 dm_table_put(map);
342
Milan Brozaa129a22006-10-03 01:15:15 -0700343 return r;
344}
345
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100346static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347{
348 return mempool_alloc(md->io_pool, GFP_NOIO);
349}
350
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100351static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352{
353 mempool_free(io, md->io_pool);
354}
355
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100356static struct dm_target_io *alloc_tio(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357{
358 return mempool_alloc(md->tio_pool, GFP_NOIO);
359}
360
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100361static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362{
363 mempool_free(tio, md->tio_pool);
364}
365
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800366static void start_io_acct(struct dm_io *io)
367{
368 struct mapped_device *md = io->md;
Tejun Heoc9959052008-08-25 19:47:21 +0900369 int cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800370
371 io->start_time = jiffies;
372
Tejun Heo074a7ac2008-08-25 19:56:14 +0900373 cpu = part_stat_lock();
374 part_round_stats(cpu, &dm_disk(md)->part0);
375 part_stat_unlock();
376 dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800377}
378
379static int end_io_acct(struct dm_io *io)
380{
381 struct mapped_device *md = io->md;
382 struct bio *bio = io->bio;
383 unsigned long duration = jiffies - io->start_time;
Tejun Heoc9959052008-08-25 19:47:21 +0900384 int pending, cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800385 int rw = bio_data_dir(bio);
386
Tejun Heo074a7ac2008-08-25 19:56:14 +0900387 cpu = part_stat_lock();
388 part_round_stats(cpu, &dm_disk(md)->part0);
389 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
390 part_stat_unlock();
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800391
Tejun Heo074a7ac2008-08-25 19:56:14 +0900392 dm_disk(md)->part0.in_flight = pending =
393 atomic_dec_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800394
395 return !pending;
396}
397
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398/*
399 * Add the bio to the list of deferred io.
400 */
401static int queue_io(struct mapped_device *md, struct bio *bio)
402{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700403 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404
405 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700406 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 return 1;
408 }
409
410 bio_list_add(&md->deferred, bio);
411
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700412 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 return 0; /* deferred successfully */
414}
415
416/*
417 * Everyone (including functions in this file), should use this
418 * function to access the md->map field, and make sure they call
419 * dm_table_put() when finished.
420 */
421struct dm_table *dm_get_table(struct mapped_device *md)
422{
423 struct dm_table *t;
424
425 read_lock(&md->map_lock);
426 t = md->map;
427 if (t)
428 dm_table_get(t);
429 read_unlock(&md->map_lock);
430
431 return t;
432}
433
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800434/*
435 * Get the geometry associated with a dm device
436 */
437int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
438{
439 *geo = md->geometry;
440
441 return 0;
442}
443
444/*
445 * Set the geometry of a device.
446 */
447int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
448{
449 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
450
451 if (geo->start > sz) {
452 DMWARN("Start sector is beyond the geometry limits.");
453 return -EINVAL;
454 }
455
456 md->geometry = *geo;
457
458 return 0;
459}
460
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461/*-----------------------------------------------------------------
462 * CRUD START:
463 * A more elegant soln is in the works that uses the queue
464 * merge fn, unfortunately there are a couple of changes to
465 * the block layer that I want to make for this. So in the
466 * interests of getting something for people to use I give
467 * you this clearly demarcated crap.
468 *---------------------------------------------------------------*/
469
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800470static int __noflush_suspending(struct mapped_device *md)
471{
472 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
473}
474
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475/*
476 * Decrements the number of outstanding ios that a bio has been
477 * cloned into, completing the original io if necc.
478 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800479static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800481 unsigned long flags;
482
483 /* Push-back supersedes any I/O errors */
484 if (error && !(io->error > 0 && __noflush_suspending(io->md)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700485 io->error = error;
486
487 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800488 if (io->error == DM_ENDIO_REQUEUE) {
489 /*
490 * Target requested pushing back the I/O.
491 * This must be handled before the sleeper on
492 * suspend queue merges the pushback list.
493 */
494 spin_lock_irqsave(&io->md->pushback_lock, flags);
495 if (__noflush_suspending(io->md))
496 bio_list_add(&io->md->pushback, io->bio);
497 else
498 /* noflush suspend was interrupted. */
499 io->error = -EIO;
500 spin_unlock_irqrestore(&io->md->pushback_lock, flags);
501 }
502
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800503 if (end_io_acct(io))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 /* nudge anyone waiting on suspend queue */
505 wake_up(&io->md->wait);
506
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800507 if (io->error != DM_ENDIO_REQUEUE) {
508 blk_add_trace_bio(io->md->queue, io->bio,
509 BLK_TA_COMPLETE);
Jens Axboe2056a782006-03-23 20:00:26 +0100510
NeilBrown6712ecf2007-09-27 12:47:43 +0200511 bio_endio(io->bio, io->error);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800512 }
513
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 free_io(io->md, io);
515 }
516}
517
NeilBrown6712ecf2007-09-27 12:47:43 +0200518static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519{
520 int r = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100521 struct dm_target_io *tio = bio->bi_private;
Stefan Bader9faf4002006-10-03 01:15:41 -0700522 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 dm_endio_fn endio = tio->ti->type->end_io;
524
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
526 error = -EIO;
527
528 if (endio) {
529 r = endio(tio->ti, bio, error, &tio->info);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800530 if (r < 0 || r == DM_ENDIO_REQUEUE)
531 /*
532 * error and requeue request are handled
533 * in dec_pending().
534 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800536 else if (r == DM_ENDIO_INCOMPLETE)
537 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200538 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800539 else if (r) {
540 DMWARN("unimplemented target endio return value: %d", r);
541 BUG();
542 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 }
544
Stefan Bader9faf4002006-10-03 01:15:41 -0700545 dec_pending(tio->io, error);
546
547 /*
548 * Store md for cleanup instead of tio which is about to get freed.
549 */
550 bio->bi_private = md->bs;
551
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 bio_put(bio);
Stefan Bader9faf4002006-10-03 01:15:41 -0700553 free_tio(md, tio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554}
555
556static sector_t max_io_len(struct mapped_device *md,
557 sector_t sector, struct dm_target *ti)
558{
559 sector_t offset = sector - ti->begin;
560 sector_t len = ti->len - offset;
561
562 /*
563 * Does the target need to split even further ?
564 */
565 if (ti->split_io) {
566 sector_t boundary;
567 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
568 - offset;
569 if (len > boundary)
570 len = boundary;
571 }
572
573 return len;
574}
575
576static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100577 struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578{
579 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100580 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -0700581 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582
583 /*
584 * Sanity checks.
585 */
586 BUG_ON(!clone->bi_size);
587
588 clone->bi_end_io = clone_endio;
589 clone->bi_private = tio;
590
591 /*
592 * Map the clone. If r == 0 we don't need to do
593 * anything, the target has assumed ownership of
594 * this io.
595 */
596 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100597 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 r = ti->type->map(ti, clone, &tio->info);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800599 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100601
Alasdair G Kergon17b2f662006-06-26 00:27:33 -0700602 blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
Alan D. Brunellec7149d62007-08-07 15:30:23 +0200603 tio->io->bio->bi_bdev->bd_dev,
604 clone->bi_sector, sector);
Jens Axboe2056a782006-03-23 20:00:26 +0100605
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800607 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
608 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -0700609 md = tio->io->md;
610 dec_pending(tio->io, r);
611 /*
612 * Store bio_set for cleanup.
613 */
614 clone->bi_private = md->bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 bio_put(clone);
Stefan Bader9faf4002006-10-03 01:15:41 -0700616 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800617 } else if (r) {
618 DMWARN("unimplemented target map return value: %d", r);
619 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 }
621}
622
623struct clone_info {
624 struct mapped_device *md;
625 struct dm_table *map;
626 struct bio *bio;
627 struct dm_io *io;
628 sector_t sector;
629 sector_t sector_count;
630 unsigned short idx;
631};
632
Peter Osterlund36763472005-09-06 15:16:42 -0700633static void dm_bio_destructor(struct bio *bio)
634{
Stefan Bader9faf4002006-10-03 01:15:41 -0700635 struct bio_set *bs = bio->bi_private;
636
637 bio_free(bio, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700638}
639
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640/*
641 * Creates a little bio that is just does part of a bvec.
642 */
643static struct bio *split_bvec(struct bio *bio, sector_t sector,
644 unsigned short idx, unsigned int offset,
Stefan Bader9faf4002006-10-03 01:15:41 -0700645 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646{
647 struct bio *clone;
648 struct bio_vec *bv = bio->bi_io_vec + idx;
649
Stefan Bader9faf4002006-10-03 01:15:41 -0700650 clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700651 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652 *clone->bi_io_vec = *bv;
653
654 clone->bi_sector = sector;
655 clone->bi_bdev = bio->bi_bdev;
656 clone->bi_rw = bio->bi_rw;
657 clone->bi_vcnt = 1;
658 clone->bi_size = to_bytes(len);
659 clone->bi_io_vec->bv_offset = offset;
660 clone->bi_io_vec->bv_len = clone->bi_size;
661
662 return clone;
663}
664
665/*
666 * Creates a bio that consists of range of complete bvecs.
667 */
668static struct bio *clone_bio(struct bio *bio, sector_t sector,
669 unsigned short idx, unsigned short bv_count,
Stefan Bader9faf4002006-10-03 01:15:41 -0700670 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671{
672 struct bio *clone;
673
Stefan Bader9faf4002006-10-03 01:15:41 -0700674 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
675 __bio_clone(clone, bio);
676 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 clone->bi_sector = sector;
678 clone->bi_idx = idx;
679 clone->bi_vcnt = idx + bv_count;
680 clone->bi_size = to_bytes(len);
681 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
682
683 return clone;
684}
685
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000686static int __clone_and_map(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687{
688 struct bio *clone, *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000689 struct dm_target *ti;
690 sector_t len = 0, max;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100691 struct dm_target_io *tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000693 ti = dm_table_find_target(ci->map, ci->sector);
694 if (!dm_target_is_valid(ti))
695 return -EIO;
696
697 max = max_io_len(ci->md, ci->sector, ti);
698
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 /*
700 * Allocate a target io object.
701 */
702 tio = alloc_tio(ci->md);
703 tio->io = ci->io;
704 tio->ti = ti;
705 memset(&tio->info, 0, sizeof(tio->info));
706
707 if (ci->sector_count <= max) {
708 /*
709 * Optimise for the simple case where we can do all of
710 * the remaining io with a single clone.
711 */
712 clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700713 bio->bi_vcnt - ci->idx, ci->sector_count,
714 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 __map_bio(ti, clone, tio);
716 ci->sector_count = 0;
717
718 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
719 /*
720 * There are some bvecs that don't span targets.
721 * Do as many of these as possible.
722 */
723 int i;
724 sector_t remaining = max;
725 sector_t bv_len;
726
727 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
728 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
729
730 if (bv_len > remaining)
731 break;
732
733 remaining -= bv_len;
734 len += bv_len;
735 }
736
Stefan Bader9faf4002006-10-03 01:15:41 -0700737 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
738 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 __map_bio(ti, clone, tio);
740
741 ci->sector += len;
742 ci->sector_count -= len;
743 ci->idx = i;
744
745 } else {
746 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800747 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 */
749 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800750 sector_t remaining = to_sector(bv->bv_len);
751 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800753 do {
754 if (offset) {
755 ti = dm_table_find_target(ci->map, ci->sector);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000756 if (!dm_target_is_valid(ti))
757 return -EIO;
758
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800759 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800761 tio = alloc_tio(ci->md);
762 tio->io = ci->io;
763 tio->ti = ti;
764 memset(&tio->info, 0, sizeof(tio->info));
765 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800767 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800769 clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700770 bv->bv_offset + offset, len,
771 ci->md->bs);
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800772
773 __map_bio(ti, clone, tio);
774
775 ci->sector += len;
776 ci->sector_count -= len;
777 offset += to_bytes(len);
778 } while (remaining -= len);
779
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 ci->idx++;
781 }
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000782
783 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784}
785
786/*
787 * Split the bio into several clones.
788 */
Milan Broz9e4e5f82007-10-19 22:38:53 +0100789static int __split_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790{
791 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000792 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793
794 ci.map = dm_get_table(md);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100795 if (unlikely(!ci.map))
796 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
798 ci.md = md;
799 ci.bio = bio;
800 ci.io = alloc_io(md);
801 ci.io->error = 0;
802 atomic_set(&ci.io->io_count, 1);
803 ci.io->bio = bio;
804 ci.io->md = md;
805 ci.sector = bio->bi_sector;
806 ci.sector_count = bio_sectors(bio);
807 ci.idx = bio->bi_idx;
808
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800809 start_io_acct(ci.io);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000810 while (ci.sector_count && !error)
811 error = __clone_and_map(&ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
813 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000814 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 dm_table_put(ci.map);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100816
817 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818}
819/*-----------------------------------------------------------------
820 * CRUD END
821 *---------------------------------------------------------------*/
822
Milan Brozf6fccb12008-07-21 12:00:37 +0100823static int dm_merge_bvec(struct request_queue *q,
824 struct bvec_merge_data *bvm,
825 struct bio_vec *biovec)
826{
827 struct mapped_device *md = q->queuedata;
828 struct dm_table *map = dm_get_table(md);
829 struct dm_target *ti;
830 sector_t max_sectors;
Mikulas Patocka50371082008-10-01 14:39:17 +0100831 int max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +0100832
833 if (unlikely(!map))
Mikulas Patocka50371082008-10-01 14:39:17 +0100834 goto out;
Milan Brozf6fccb12008-07-21 12:00:37 +0100835
836 ti = dm_table_find_target(map, bvm->bi_sector);
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100837 if (!dm_target_is_valid(ti))
838 goto out_table;
Milan Brozf6fccb12008-07-21 12:00:37 +0100839
840 /*
841 * Find maximum amount of I/O that won't need splitting
842 */
843 max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
844 (sector_t) BIO_MAX_SECTORS);
845 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
846 if (max_size < 0)
847 max_size = 0;
848
849 /*
850 * merge_bvec_fn() returns number of bytes
851 * it can accept at this offset
852 * max is precomputed maximal io size
853 */
854 if (max_size && ti->type->merge)
855 max_size = ti->type->merge(ti, bvm, biovec, max_size);
856
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100857out_table:
Mikulas Patocka50371082008-10-01 14:39:17 +0100858 dm_table_put(map);
859
860out:
Milan Brozf6fccb12008-07-21 12:00:37 +0100861 /*
862 * Always allow an entire first page
863 */
864 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
865 max_size = biovec->bv_len;
866
Milan Brozf6fccb12008-07-21 12:00:37 +0100867 return max_size;
868}
869
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870/*
871 * The request function that just remaps the bio built up by
872 * dm_merge_bvec.
873 */
Jens Axboe165125e2007-07-24 09:28:11 +0200874static int dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875{
Milan Broz9e4e5f82007-10-19 22:38:53 +0100876 int r = -EIO;
Kevin Corry12f03a42006-02-01 03:04:52 -0800877 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 struct mapped_device *md = q->queuedata;
Tejun Heoc9959052008-08-25 19:47:21 +0900879 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880
Stefan Bader07a83c42007-07-12 17:28:33 +0100881 /*
882 * There is no use in forwarding any barrier request since we can't
883 * guarantee it is (or can be) handled by the targets correctly.
884 */
885 if (unlikely(bio_barrier(bio))) {
NeilBrown6712ecf2007-09-27 12:47:43 +0200886 bio_endio(bio, -EOPNOTSUPP);
Stefan Bader07a83c42007-07-12 17:28:33 +0100887 return 0;
888 }
889
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700890 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891
Tejun Heo074a7ac2008-08-25 19:56:14 +0900892 cpu = part_stat_lock();
893 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
894 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
895 part_stat_unlock();
Kevin Corry12f03a42006-02-01 03:04:52 -0800896
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 /*
898 * If we're suspended we have to queue
899 * this io for later.
900 */
901 while (test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700902 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903
Milan Broz9e4e5f82007-10-19 22:38:53 +0100904 if (bio_rw(bio) != READA)
905 r = queue_io(md, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906
Milan Broz9e4e5f82007-10-19 22:38:53 +0100907 if (r <= 0)
908 goto out_req;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909
910 /*
911 * We're in a while loop, because someone could suspend
912 * before we get to the following read lock.
913 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700914 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915 }
916
Milan Broz9e4e5f82007-10-19 22:38:53 +0100917 r = __split_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700918 up_read(&md->io_lock);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100919
920out_req:
921 if (r < 0)
922 bio_io_error(bio);
923
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 return 0;
925}
926
Jens Axboe165125e2007-07-24 09:28:11 +0200927static void dm_unplug_all(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928{
929 struct mapped_device *md = q->queuedata;
930 struct dm_table *map = dm_get_table(md);
931
932 if (map) {
933 dm_table_unplug_all(map);
934 dm_table_put(map);
935 }
936}
937
938static int dm_any_congested(void *congested_data, int bdi_bits)
939{
940 int r;
941 struct mapped_device *md = (struct mapped_device *) congested_data;
942 struct dm_table *map = dm_get_table(md);
943
944 if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
945 r = bdi_bits;
946 else
947 r = dm_table_any_congested(map, bdi_bits);
948
949 dm_table_put(map);
950 return r;
951}
952
953/*-----------------------------------------------------------------
954 * An IDR is used to keep track of allocated minor numbers.
955 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956static DEFINE_IDR(_minor_idr);
957
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -0700958static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700960 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700962 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700963}
964
965/*
966 * See if the device with a specific minor # is free.
967 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +0100968static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969{
970 int r, m;
971
972 if (minor >= (1 << MINORBITS))
973 return -EINVAL;
974
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700975 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
976 if (!r)
977 return -ENOMEM;
978
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700979 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980
981 if (idr_find(&_minor_idr, minor)) {
982 r = -EBUSY;
983 goto out;
984 }
985
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700986 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700987 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989
990 if (m != minor) {
991 idr_remove(&_minor_idr, m);
992 r = -EBUSY;
993 goto out;
994 }
995
996out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700997 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 return r;
999}
1000
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001001static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001003 int r, m;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001006 if (!r)
1007 return -ENOMEM;
1008
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001009 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001011 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001012 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001014
1015 if (m >= (1 << MINORBITS)) {
1016 idr_remove(&_minor_idr, m);
1017 r = -ENOSPC;
1018 goto out;
1019 }
1020
1021 *minor = m;
1022
1023out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001024 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025 return r;
1026}
1027
1028static struct block_device_operations dm_blk_dops;
1029
1030/*
1031 * Allocate and initialise a blank device with a given minor.
1032 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001033static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034{
1035 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001036 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001037 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038
1039 if (!md) {
1040 DMWARN("unable to allocate device, out of memory.");
1041 return NULL;
1042 }
1043
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001044 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001045 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001046
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001048 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001049 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001050 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001051 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001053 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001055 init_rwsem(&md->io_lock);
Daniel Walkere61290a2008-02-08 02:10:08 +00001056 mutex_init(&md->suspend_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001057 spin_lock_init(&md->pushback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 rwlock_init(&md->map_lock);
1059 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001060 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001062 atomic_set(&md->uevent_seq, 0);
1063 INIT_LIST_HEAD(&md->uevent_list);
1064 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065
1066 md->queue = blk_alloc_queue(GFP_KERNEL);
1067 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001068 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069
1070 md->queue->queuedata = md;
1071 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1072 md->queue->backing_dev_info.congested_data = md;
1073 blk_queue_make_request(md->queue, dm_request);
Jens Axboedaef2652006-01-10 10:48:02 +01001074 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075 md->queue->unplug_fn = dm_unplug_all;
Milan Brozf6fccb12008-07-21 12:00:37 +01001076 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077
Matthew Dobson93d23412006-03-26 01:37:50 -08001078 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
Kiyoshi Ueda74859362006-12-08 02:41:02 -08001079 if (!md->io_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001080 goto bad_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081
Matthew Dobson93d23412006-03-26 01:37:50 -08001082 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 if (!md->tio_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001084 goto bad_tio_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085
Jens Axboe59725112007-04-02 10:06:42 +02001086 md->bs = bioset_create(16, 16);
Stefan Bader9faf4002006-10-03 01:15:41 -07001087 if (!md->bs)
1088 goto bad_no_bioset;
1089
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090 md->disk = alloc_disk(1);
1091 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001092 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001094 atomic_set(&md->pending, 0);
1095 init_waitqueue_head(&md->wait);
1096 init_waitqueue_head(&md->eventq);
1097
Linus Torvalds1da177e2005-04-16 15:20:36 -07001098 md->disk->major = _major;
1099 md->disk->first_minor = minor;
1100 md->disk->fops = &dm_blk_dops;
1101 md->disk->queue = md->queue;
1102 md->disk->private_data = md;
1103 sprintf(md->disk->disk_name, "dm-%d", minor);
1104 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001105 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106
Milan Broz304f3f62008-02-08 02:11:17 +00001107 md->wq = create_singlethread_workqueue("kdmflush");
1108 if (!md->wq)
1109 goto bad_thread;
1110
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001111 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001112 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001113 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001114 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001115
1116 BUG_ON(old_md != MINOR_ALLOCED);
1117
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118 return md;
1119
Milan Broz304f3f62008-02-08 02:11:17 +00001120bad_thread:
1121 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001122bad_disk:
Stefan Bader9faf4002006-10-03 01:15:41 -07001123 bioset_free(md->bs);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001124bad_no_bioset:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125 mempool_destroy(md->tio_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001126bad_tio_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127 mempool_destroy(md->io_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001128bad_io_pool:
Al Viro1312f402006-03-12 11:02:03 -05001129 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001130bad_queue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001132bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001133 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001134bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135 kfree(md);
1136 return NULL;
1137}
1138
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001139static void unlock_fs(struct mapped_device *md);
1140
Linus Torvalds1da177e2005-04-16 15:20:36 -07001141static void free_dev(struct mapped_device *md)
1142{
Tejun Heof331c022008-09-03 09:01:48 +02001143 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001144
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001145 if (md->suspended_bdev) {
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001146 unlock_fs(md);
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001147 bdput(md->suspended_bdev);
1148 }
Milan Broz304f3f62008-02-08 02:11:17 +00001149 destroy_workqueue(md->wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150 mempool_destroy(md->tio_pool);
1151 mempool_destroy(md->io_pool);
Stefan Bader9faf4002006-10-03 01:15:41 -07001152 bioset_free(md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001154 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001155
1156 spin_lock(&_minor_lock);
1157 md->disk->private_data = NULL;
1158 spin_unlock(&_minor_lock);
1159
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05001161 blk_cleanup_queue(md->queue);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001162 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163 kfree(md);
1164}
1165
1166/*
1167 * Bind a table to the device.
1168 */
1169static void event_callback(void *context)
1170{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001171 unsigned long flags;
1172 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173 struct mapped_device *md = (struct mapped_device *) context;
1174
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001175 spin_lock_irqsave(&md->uevent_lock, flags);
1176 list_splice_init(&md->uevent_list, &uevents);
1177 spin_unlock_irqrestore(&md->uevent_lock, flags);
1178
Tejun Heoed9e1982008-08-25 19:56:05 +09001179 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001180
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 atomic_inc(&md->event_nr);
1182 wake_up(&md->eventq);
1183}
1184
Alasdair G Kergon4e901882005-07-28 21:15:59 -07001185static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186{
Alasdair G Kergon4e901882005-07-28 21:15:59 -07001187 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001189 mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001190 i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001191 mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192}
1193
1194static int __bind(struct mapped_device *md, struct dm_table *t)
1195{
Jens Axboe165125e2007-07-24 09:28:11 +02001196 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 sector_t size;
1198
1199 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001200
1201 /*
1202 * Wipe any geometry if the size of the table changed.
1203 */
1204 if (size != get_capacity(md->disk))
1205 memset(&md->geometry, 0, sizeof(md->geometry));
1206
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001207 if (md->suspended_bdev)
1208 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209 if (size == 0)
1210 return 0;
1211
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212 dm_table_get(t);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001213 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001214
1215 write_lock(&md->map_lock);
1216 md->map = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217 dm_table_set_restrictions(t, q);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001218 write_unlock(&md->map_lock);
1219
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 return 0;
1221}
1222
1223static void __unbind(struct mapped_device *md)
1224{
1225 struct dm_table *map = md->map;
1226
1227 if (!map)
1228 return;
1229
1230 dm_table_event_callback(map, NULL, NULL);
1231 write_lock(&md->map_lock);
1232 md->map = NULL;
1233 write_unlock(&md->map_lock);
1234 dm_table_put(map);
1235}
1236
1237/*
1238 * Constructor for a new device.
1239 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001240int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241{
1242 struct mapped_device *md;
1243
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001244 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001245 if (!md)
1246 return -ENXIO;
1247
1248 *result = md;
1249 return 0;
1250}
1251
David Teigland637842c2006-01-06 00:20:00 -08001252static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253{
1254 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255 unsigned minor = MINOR(dev);
1256
1257 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1258 return NULL;
1259
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001260 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001261
1262 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001263 if (md && (md == MINOR_ALLOCED ||
Tejun Heof331c022008-09-03 09:01:48 +02001264 (MINOR(disk_devt(dm_disk(md))) != minor) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07001265 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08001266 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001267 goto out;
1268 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001270out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001271 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272
David Teigland637842c2006-01-06 00:20:00 -08001273 return md;
1274}
1275
David Teiglandd229a952006-01-06 00:20:01 -08001276struct mapped_device *dm_get_md(dev_t dev)
1277{
1278 struct mapped_device *md = dm_find_md(dev);
1279
1280 if (md)
1281 dm_get(md);
1282
1283 return md;
1284}
1285
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001286void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08001287{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001288 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001289}
1290
1291void dm_set_mdptr(struct mapped_device *md, void *ptr)
1292{
1293 md->interface_ptr = ptr;
1294}
1295
1296void dm_get(struct mapped_device *md)
1297{
1298 atomic_inc(&md->holders);
1299}
1300
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001301const char *dm_device_name(struct mapped_device *md)
1302{
1303 return md->name;
1304}
1305EXPORT_SYMBOL_GPL(dm_device_name);
1306
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307void dm_put(struct mapped_device *md)
1308{
Mike Anderson1134e5a2006-03-27 01:17:54 -08001309 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001311 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1312
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001313 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08001314 map = dm_get_table(md);
Tejun Heof331c022008-09-03 09:01:48 +02001315 idr_replace(&_minor_idr, MINOR_ALLOCED,
1316 MINOR(disk_devt(dm_disk(md))));
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001317 set_bit(DMF_FREEING, &md->flags);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001318 spin_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001319 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320 dm_table_presuspend_targets(map);
1321 dm_table_postsuspend_targets(map);
1322 }
1323 __unbind(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08001324 dm_table_put(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325 free_dev(md);
1326 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327}
Edward Goggin79eb8852007-05-09 02:32:56 -07001328EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001329
Milan Broz46125c12008-02-08 02:10:30 +00001330static int dm_wait_for_completion(struct mapped_device *md)
1331{
1332 int r = 0;
1333
1334 while (1) {
1335 set_current_state(TASK_INTERRUPTIBLE);
1336
1337 smp_mb();
1338 if (!atomic_read(&md->pending))
1339 break;
1340
1341 if (signal_pending(current)) {
1342 r = -EINTR;
1343 break;
1344 }
1345
1346 io_schedule();
1347 }
1348 set_current_state(TASK_RUNNING);
1349
1350 return r;
1351}
1352
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353/*
1354 * Process the deferred bios
1355 */
Milan Broz6d6f10d2008-02-08 02:10:22 +00001356static void __flush_deferred_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357{
Milan Broz6d6f10d2008-02-08 02:10:22 +00001358 struct bio *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359
Milan Broz6d6f10d2008-02-08 02:10:22 +00001360 while ((c = bio_list_pop(&md->deferred))) {
Milan Broz9e4e5f82007-10-19 22:38:53 +01001361 if (__split_bio(md, c))
1362 bio_io_error(c);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001363 }
Milan Broz73d410c2008-02-08 02:10:25 +00001364
1365 clear_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001366}
1367
Milan Broz6d6f10d2008-02-08 02:10:22 +00001368static void __merge_pushback_list(struct mapped_device *md)
1369{
1370 unsigned long flags;
1371
1372 spin_lock_irqsave(&md->pushback_lock, flags);
1373 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1374 bio_list_merge_head(&md->deferred, &md->pushback);
1375 bio_list_init(&md->pushback);
1376 spin_unlock_irqrestore(&md->pushback_lock, flags);
1377}
1378
Milan Broz304f3f62008-02-08 02:11:17 +00001379static void dm_wq_work(struct work_struct *work)
1380{
1381 struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
1382 struct mapped_device *md = req->md;
1383
1384 down_write(&md->io_lock);
1385 switch (req->type) {
1386 case DM_WQ_FLUSH_ALL:
1387 __merge_pushback_list(md);
1388 /* pass through */
1389 case DM_WQ_FLUSH_DEFERRED:
1390 __flush_deferred_io(md);
1391 break;
1392 default:
1393 DMERR("dm_wq_work: unrecognised work type %d", req->type);
1394 BUG();
1395 }
1396 up_write(&md->io_lock);
1397}
1398
1399static void dm_wq_queue(struct mapped_device *md, int type, void *context,
1400 struct dm_wq_req *req)
1401{
1402 req->type = type;
1403 req->md = md;
1404 req->context = context;
1405 INIT_WORK(&req->work, dm_wq_work);
1406 queue_work(md->wq, &req->work);
1407}
1408
1409static void dm_queue_flush(struct mapped_device *md, int type, void *context)
1410{
1411 struct dm_wq_req req;
1412
1413 dm_wq_queue(md, type, context, &req);
1414 flush_workqueue(md->wq);
1415}
1416
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417/*
1418 * Swap in a new table (destroying old one).
1419 */
1420int dm_swap_table(struct mapped_device *md, struct dm_table *table)
1421{
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001422 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423
Daniel Walkere61290a2008-02-08 02:10:08 +00001424 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425
1426 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001427 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001428 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001430 /* without bdev, the device size cannot be changed */
1431 if (!md->suspended_bdev)
1432 if (get_capacity(md->disk) != dm_table_get_size(table))
1433 goto out;
1434
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435 __unbind(md);
1436 r = __bind(md, table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001437
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001438out:
Daniel Walkere61290a2008-02-08 02:10:08 +00001439 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001440 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441}
1442
1443/*
1444 * Functions to lock and unlock any filesystem running on the
1445 * device.
1446 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001447static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001449 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450
1451 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001452
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001453 md->frozen_sb = freeze_bdev(md->suspended_bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001454 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001455 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001456 md->frozen_sb = NULL;
1457 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001458 }
1459
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001460 set_bit(DMF_FROZEN, &md->flags);
1461
Linus Torvalds1da177e2005-04-16 15:20:36 -07001462 /* don't bdput right now, we don't want the bdev
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001463 * to go away while it is locked.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001464 */
1465 return 0;
1466}
1467
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001468static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001470 if (!test_bit(DMF_FROZEN, &md->flags))
1471 return;
1472
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001473 thaw_bdev(md->suspended_bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001475 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001476}
1477
1478/*
1479 * We need to be able to change a mapping table under a mounted
1480 * filesystem. For example we might want to move some data in
1481 * the background. Before the table can be swapped with
1482 * dm_bind_table, dm_suspend must be called to flush any in
1483 * flight bios and ensure that any further io gets deferred.
1484 */
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001485int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001487 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488 DECLARE_WAITQUEUE(wait, current);
Milan Broz46125c12008-02-08 02:10:30 +00001489 int r = 0;
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001490 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001491 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492
Daniel Walkere61290a2008-02-08 02:10:08 +00001493 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001494
Milan Broz73d410c2008-02-08 02:10:25 +00001495 if (dm_suspended(md)) {
1496 r = -EINVAL;
Alasdair G Kergond2874832006-11-08 17:44:43 -08001497 goto out_unlock;
Milan Broz73d410c2008-02-08 02:10:25 +00001498 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499
1500 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001502 /*
1503 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
1504 * This flag is cleared before dm_suspend returns.
1505 */
1506 if (noflush)
1507 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1508
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001509 /* This does not get reverted if there's an error later. */
1510 dm_table_presuspend_targets(map);
1511
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001512 /* bdget() can stall if the pending I/Os are not flushed */
1513 if (!noflush) {
1514 md->suspended_bdev = bdget_disk(md->disk, 0);
1515 if (!md->suspended_bdev) {
1516 DMWARN("bdget failed in dm_suspend");
1517 r = -ENOMEM;
1518 goto flush_and_out;
1519 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001520
Milan Broz6d6f10d2008-02-08 02:10:22 +00001521 /*
1522 * Flush I/O to the device. noflush supersedes do_lockfs,
1523 * because lock_fs() needs to flush I/Os.
1524 */
1525 if (do_lockfs) {
1526 r = lock_fs(md);
1527 if (r)
1528 goto out;
1529 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001530 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531
1532 /*
Alasdair G Kergon354e0072005-05-05 16:16:05 -07001533 * First we set the BLOCK_IO flag so no more ios will be mapped.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001535 down_write(&md->io_lock);
1536 set_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 add_wait_queue(&md->wait, &wait);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001539 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001540
1541 /* unplug */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001542 if (map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001543 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544
1545 /*
Milan Broz46125c12008-02-08 02:10:30 +00001546 * Wait for the already-mapped ios to complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001547 */
Milan Broz46125c12008-02-08 02:10:30 +00001548 r = dm_wait_for_completion(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001550 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551 remove_wait_queue(&md->wait, &wait);
1552
Milan Broz6d6f10d2008-02-08 02:10:22 +00001553 if (noflush)
1554 __merge_pushback_list(md);
Milan Broz94d63512008-02-08 02:10:27 +00001555 up_write(&md->io_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001556
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00001558 if (r < 0) {
Milan Broz304f3f62008-02-08 02:11:17 +00001559 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Milan Broz73d410c2008-02-08 02:10:25 +00001560
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001561 unlock_fs(md);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001562 goto out; /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001563 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001564
1565 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001566
1567 set_bit(DMF_SUSPENDED, &md->flags);
1568
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001569flush_and_out:
Milan Broz304f3f62008-02-08 02:11:17 +00001570 if (r && noflush)
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001571 /*
1572 * Because there may be already I/Os in the pushback list,
1573 * flush them before return.
1574 */
Milan Broz304f3f62008-02-08 02:11:17 +00001575 dm_queue_flush(md, DM_WQ_FLUSH_ALL, NULL);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001576
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001577out:
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001578 if (r && md->suspended_bdev) {
1579 bdput(md->suspended_bdev);
1580 md->suspended_bdev = NULL;
1581 }
1582
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 dm_table_put(map);
Alasdair G Kergond2874832006-11-08 17:44:43 -08001584
1585out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00001586 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001587 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588}
1589
1590int dm_resume(struct mapped_device *md)
1591{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001592 int r = -EINVAL;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001593 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594
Daniel Walkere61290a2008-02-08 02:10:08 +00001595 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001596 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001597 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001598
1599 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001600 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001601 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001602
Milan Broz8757b772006-10-03 01:15:36 -07001603 r = dm_table_resume_targets(map);
1604 if (r)
1605 goto out;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001606
Milan Broz304f3f62008-02-08 02:11:17 +00001607 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001608
1609 unlock_fs(md);
1610
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001611 if (md->suspended_bdev) {
1612 bdput(md->suspended_bdev);
1613 md->suspended_bdev = NULL;
1614 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001615
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001616 clear_bit(DMF_SUSPENDED, &md->flags);
1617
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001619
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001620 dm_kobject_uevent(md);
Hannes Reinecke8560ed62006-10-03 01:15:35 -07001621
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001622 r = 0;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001623
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001624out:
1625 dm_table_put(map);
Daniel Walkere61290a2008-02-08 02:10:08 +00001626 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001627
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001628 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629}
1630
1631/*-----------------------------------------------------------------
1632 * Event notification.
1633 *---------------------------------------------------------------*/
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001634void dm_kobject_uevent(struct mapped_device *md)
1635{
Tejun Heoed9e1982008-08-25 19:56:05 +09001636 kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001637}
1638
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001639uint32_t dm_next_uevent_seq(struct mapped_device *md)
1640{
1641 return atomic_add_return(1, &md->uevent_seq);
1642}
1643
Linus Torvalds1da177e2005-04-16 15:20:36 -07001644uint32_t dm_get_event_nr(struct mapped_device *md)
1645{
1646 return atomic_read(&md->event_nr);
1647}
1648
1649int dm_wait_event(struct mapped_device *md, int event_nr)
1650{
1651 return wait_event_interruptible(md->eventq,
1652 (event_nr != atomic_read(&md->event_nr)));
1653}
1654
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001655void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
1656{
1657 unsigned long flags;
1658
1659 spin_lock_irqsave(&md->uevent_lock, flags);
1660 list_add(elist, &md->uevent_list);
1661 spin_unlock_irqrestore(&md->uevent_lock, flags);
1662}
1663
Linus Torvalds1da177e2005-04-16 15:20:36 -07001664/*
1665 * The gendisk is only valid as long as you have a reference
1666 * count on 'md'.
1667 */
1668struct gendisk *dm_disk(struct mapped_device *md)
1669{
1670 return md->disk;
1671}
1672
1673int dm_suspended(struct mapped_device *md)
1674{
1675 return test_bit(DMF_SUSPENDED, &md->flags);
1676}
1677
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001678int dm_noflush_suspending(struct dm_target *ti)
1679{
1680 struct mapped_device *md = dm_table_get_md(ti->table);
1681 int r = __noflush_suspending(md);
1682
1683 dm_put(md);
1684
1685 return r;
1686}
1687EXPORT_SYMBOL_GPL(dm_noflush_suspending);
1688
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689static struct block_device_operations dm_blk_dops = {
Al Virofe5f9f22008-03-02 10:29:31 -05001690 .open = dm_blk_open,
1691 .release = dm_blk_close,
1692 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001693 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001694 .owner = THIS_MODULE
1695};
1696
1697EXPORT_SYMBOL(dm_get_mapinfo);
1698
1699/*
1700 * module hooks
1701 */
1702module_init(dm_init);
1703module_exit(dm_exit);
1704
1705module_param(major, uint, 0);
1706MODULE_PARM_DESC(major, "The major number of the device mapper");
1707MODULE_DESCRIPTION(DM_NAME " driver");
1708MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1709MODULE_LICENSE("GPL");