blob: 327de03a5bdfed7f11b0904abb72f3eac65fc92d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07003 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
9#include "dm-bio-list.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +010010#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070011
12#include <linux/init.h>
13#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080014#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/moduleparam.h>
16#include <linux/blkpg.h>
17#include <linux/bio.h>
18#include <linux/buffer_head.h>
19#include <linux/mempool.h>
20#include <linux/slab.h>
21#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080022#include <linux/hdreg.h>
Jens Axboe2056a782006-03-23 20:00:26 +010023#include <linux/blktrace_api.h>
Milan Brozaa129a22006-10-03 01:15:15 -070024#include <linux/smp_lock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Alasdair G Kergon72d94862006-06-26 00:27:35 -070026#define DM_MSG_PREFIX "core"
27
Linus Torvalds1da177e2005-04-16 15:20:36 -070028static const char *_name = DM_NAME;
29
30static unsigned int major = 0;
31static unsigned int _major = 0;
32
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070033static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070034/*
35 * One of these is allocated per bio.
36 */
37struct dm_io {
38 struct mapped_device *md;
39 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010041 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080042 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070043};
44
45/*
46 * One of these is allocated per target within a bio. Hopefully
47 * this will be simplified out one day.
48 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +010049struct dm_target_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 struct dm_io *io;
51 struct dm_target *ti;
52 union map_info info;
53};
54
55union map_info *dm_get_mapinfo(struct bio *bio)
56{
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070057 if (bio && bio->bi_private)
Alasdair G Kergon028867a2007-07-12 17:26:32 +010058 return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070059 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070060}
61
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070062#define MINOR_ALLOCED ((void *)-1)
63
Linus Torvalds1da177e2005-04-16 15:20:36 -070064/*
65 * Bits for the md->flags field.
66 */
67#define DMF_BLOCK_IO 0
68#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -080069#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -070070#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070071#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080072#define DMF_NOFLUSH_SUSPENDING 5
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Milan Broz304f3f62008-02-08 02:11:17 +000074/*
75 * Work processed by per-device workqueue.
76 */
77struct dm_wq_req {
78 enum {
79 DM_WQ_FLUSH_ALL,
80 DM_WQ_FLUSH_DEFERRED,
81 } type;
82 struct work_struct work;
83 struct mapped_device *md;
84 void *context;
85};
86
Linus Torvalds1da177e2005-04-16 15:20:36 -070087struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -070088 struct rw_semaphore io_lock;
Daniel Walkere61290a2008-02-08 02:10:08 +000089 struct mutex suspend_lock;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -080090 spinlock_t pushback_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -070091 rwlock_t map_lock;
92 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -070093 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070094
95 unsigned long flags;
96
Jens Axboe165125e2007-07-24 09:28:11 +020097 struct request_queue *queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -080099 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100
101 void *interface_ptr;
102
103 /*
104 * A list of ios that arrived while we were suspended.
105 */
106 atomic_t pending;
107 wait_queue_head_t wait;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800108 struct bio_list deferred;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800109 struct bio_list pushback;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110
111 /*
Milan Broz304f3f62008-02-08 02:11:17 +0000112 * Processing queue (flush/barriers)
113 */
114 struct workqueue_struct *wq;
115
116 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 * The current mapping.
118 */
119 struct dm_table *map;
120
121 /*
122 * io objects are allocated from here.
123 */
124 mempool_t *io_pool;
125 mempool_t *tio_pool;
126
Stefan Bader9faf4002006-10-03 01:15:41 -0700127 struct bio_set *bs;
128
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129 /*
130 * Event handling.
131 */
132 atomic_t event_nr;
133 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100134 atomic_t uevent_seq;
135 struct list_head uevent_list;
136 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
138 /*
139 * freeze/thaw support require holding onto a super block
140 */
141 struct super_block *frozen_sb;
Alasdair G Kergone39e2e92006-01-06 00:20:05 -0800142 struct block_device *suspended_bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800143
144 /* forced geometry settings */
145 struct hd_geometry geometry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146};
147
148#define MIN_IOS 256
Christoph Lametere18b8902006-12-06 20:33:20 -0800149static struct kmem_cache *_io_cache;
150static struct kmem_cache *_tio_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152static int __init local_init(void)
153{
154 int r;
155
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100157 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 if (!_io_cache)
159 return -ENOMEM;
160
161 /* allocate a slab for the target ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100162 _tio_cache = KMEM_CACHE(dm_target_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 if (!_tio_cache) {
164 kmem_cache_destroy(_io_cache);
165 return -ENOMEM;
166 }
167
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100168 r = dm_uevent_init();
169 if (r) {
170 kmem_cache_destroy(_tio_cache);
171 kmem_cache_destroy(_io_cache);
172 return r;
173 }
174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 _major = major;
176 r = register_blkdev(_major, _name);
177 if (r < 0) {
178 kmem_cache_destroy(_tio_cache);
179 kmem_cache_destroy(_io_cache);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100180 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 return r;
182 }
183
184 if (!_major)
185 _major = r;
186
187 return 0;
188}
189
190static void local_exit(void)
191{
192 kmem_cache_destroy(_tio_cache);
193 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700194 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100195 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
197 _major = 0;
198
199 DMINFO("cleaned up");
200}
201
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000202static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203 local_init,
204 dm_target_init,
205 dm_linear_init,
206 dm_stripe_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100207 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 dm_interface_init,
209};
210
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000211static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 local_exit,
213 dm_target_exit,
214 dm_linear_exit,
215 dm_stripe_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100216 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 dm_interface_exit,
218};
219
220static int __init dm_init(void)
221{
222 const int count = ARRAY_SIZE(_inits);
223
224 int r, i;
225
226 for (i = 0; i < count; i++) {
227 r = _inits[i]();
228 if (r)
229 goto bad;
230 }
231
232 return 0;
233
234 bad:
235 while (i--)
236 _exits[i]();
237
238 return r;
239}
240
241static void __exit dm_exit(void)
242{
243 int i = ARRAY_SIZE(_exits);
244
245 while (i--)
246 _exits[i]();
247}
248
249/*
250 * Block device functions
251 */
252static int dm_blk_open(struct inode *inode, struct file *file)
253{
254 struct mapped_device *md;
255
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700256 spin_lock(&_minor_lock);
257
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 md = inode->i_bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700259 if (!md)
260 goto out;
261
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700262 if (test_bit(DMF_FREEING, &md->flags) ||
263 test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700264 md = NULL;
265 goto out;
266 }
267
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700269 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700270
271out:
272 spin_unlock(&_minor_lock);
273
274 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275}
276
277static int dm_blk_close(struct inode *inode, struct file *file)
278{
279 struct mapped_device *md;
280
281 md = inode->i_bdev->bd_disk->private_data;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700282 atomic_dec(&md->open_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 dm_put(md);
284 return 0;
285}
286
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700287int dm_open_count(struct mapped_device *md)
288{
289 return atomic_read(&md->open_count);
290}
291
292/*
293 * Guarantees nothing is using the device before it's deleted.
294 */
295int dm_lock_for_deletion(struct mapped_device *md)
296{
297 int r = 0;
298
299 spin_lock(&_minor_lock);
300
301 if (dm_open_count(md))
302 r = -EBUSY;
303 else
304 set_bit(DMF_DELETING, &md->flags);
305
306 spin_unlock(&_minor_lock);
307
308 return r;
309}
310
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800311static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
312{
313 struct mapped_device *md = bdev->bd_disk->private_data;
314
315 return dm_get_geometry(md, geo);
316}
317
Milan Brozaa129a22006-10-03 01:15:15 -0700318static int dm_blk_ioctl(struct inode *inode, struct file *file,
319 unsigned int cmd, unsigned long arg)
320{
321 struct mapped_device *md;
322 struct dm_table *map;
323 struct dm_target *tgt;
324 int r = -ENOTTY;
325
326 /* We don't really need this lock, but we do need 'inode'. */
327 unlock_kernel();
328
329 md = inode->i_bdev->bd_disk->private_data;
330
331 map = dm_get_table(md);
332
333 if (!map || !dm_table_get_size(map))
334 goto out;
335
336 /* We only support devices that have a single target */
337 if (dm_table_get_num_targets(map) != 1)
338 goto out;
339
340 tgt = dm_table_get_target(map, 0);
341
342 if (dm_suspended(md)) {
343 r = -EAGAIN;
344 goto out;
345 }
346
347 if (tgt->type->ioctl)
348 r = tgt->type->ioctl(tgt, inode, file, cmd, arg);
349
350out:
351 dm_table_put(map);
352
353 lock_kernel();
354 return r;
355}
356
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100357static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358{
359 return mempool_alloc(md->io_pool, GFP_NOIO);
360}
361
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100362static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363{
364 mempool_free(io, md->io_pool);
365}
366
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100367static struct dm_target_io *alloc_tio(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368{
369 return mempool_alloc(md->tio_pool, GFP_NOIO);
370}
371
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100372static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373{
374 mempool_free(tio, md->tio_pool);
375}
376
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800377static void start_io_acct(struct dm_io *io)
378{
379 struct mapped_device *md = io->md;
Tejun Heoc9959052008-08-25 19:47:21 +0900380 int cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800381
382 io->start_time = jiffies;
383
Tejun Heo074a7ac2008-08-25 19:56:14 +0900384 cpu = part_stat_lock();
385 part_round_stats(cpu, &dm_disk(md)->part0);
386 part_stat_unlock();
387 dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800388}
389
390static int end_io_acct(struct dm_io *io)
391{
392 struct mapped_device *md = io->md;
393 struct bio *bio = io->bio;
394 unsigned long duration = jiffies - io->start_time;
Tejun Heoc9959052008-08-25 19:47:21 +0900395 int pending, cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800396 int rw = bio_data_dir(bio);
397
Tejun Heo074a7ac2008-08-25 19:56:14 +0900398 cpu = part_stat_lock();
399 part_round_stats(cpu, &dm_disk(md)->part0);
400 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
401 part_stat_unlock();
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800402
Tejun Heo074a7ac2008-08-25 19:56:14 +0900403 dm_disk(md)->part0.in_flight = pending =
404 atomic_dec_return(&md->pending);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800405
406 return !pending;
407}
408
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409/*
410 * Add the bio to the list of deferred io.
411 */
412static int queue_io(struct mapped_device *md, struct bio *bio)
413{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700414 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415
416 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700417 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 return 1;
419 }
420
421 bio_list_add(&md->deferred, bio);
422
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700423 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 return 0; /* deferred successfully */
425}
426
427/*
428 * Everyone (including functions in this file), should use this
429 * function to access the md->map field, and make sure they call
430 * dm_table_put() when finished.
431 */
432struct dm_table *dm_get_table(struct mapped_device *md)
433{
434 struct dm_table *t;
435
436 read_lock(&md->map_lock);
437 t = md->map;
438 if (t)
439 dm_table_get(t);
440 read_unlock(&md->map_lock);
441
442 return t;
443}
444
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800445/*
446 * Get the geometry associated with a dm device
447 */
448int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
449{
450 *geo = md->geometry;
451
452 return 0;
453}
454
455/*
456 * Set the geometry of a device.
457 */
458int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
459{
460 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
461
462 if (geo->start > sz) {
463 DMWARN("Start sector is beyond the geometry limits.");
464 return -EINVAL;
465 }
466
467 md->geometry = *geo;
468
469 return 0;
470}
471
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472/*-----------------------------------------------------------------
473 * CRUD START:
474 * A more elegant soln is in the works that uses the queue
475 * merge fn, unfortunately there are a couple of changes to
476 * the block layer that I want to make for this. So in the
477 * interests of getting something for people to use I give
478 * you this clearly demarcated crap.
479 *---------------------------------------------------------------*/
480
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800481static int __noflush_suspending(struct mapped_device *md)
482{
483 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
484}
485
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486/*
487 * Decrements the number of outstanding ios that a bio has been
488 * cloned into, completing the original io if necc.
489 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800490static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800492 unsigned long flags;
493
494 /* Push-back supersedes any I/O errors */
495 if (error && !(io->error > 0 && __noflush_suspending(io->md)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 io->error = error;
497
498 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800499 if (io->error == DM_ENDIO_REQUEUE) {
500 /*
501 * Target requested pushing back the I/O.
502 * This must be handled before the sleeper on
503 * suspend queue merges the pushback list.
504 */
505 spin_lock_irqsave(&io->md->pushback_lock, flags);
506 if (__noflush_suspending(io->md))
507 bio_list_add(&io->md->pushback, io->bio);
508 else
509 /* noflush suspend was interrupted. */
510 io->error = -EIO;
511 spin_unlock_irqrestore(&io->md->pushback_lock, flags);
512 }
513
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800514 if (end_io_acct(io))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 /* nudge anyone waiting on suspend queue */
516 wake_up(&io->md->wait);
517
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800518 if (io->error != DM_ENDIO_REQUEUE) {
519 blk_add_trace_bio(io->md->queue, io->bio,
520 BLK_TA_COMPLETE);
Jens Axboe2056a782006-03-23 20:00:26 +0100521
NeilBrown6712ecf2007-09-27 12:47:43 +0200522 bio_endio(io->bio, io->error);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800523 }
524
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 free_io(io->md, io);
526 }
527}
528
NeilBrown6712ecf2007-09-27 12:47:43 +0200529static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530{
531 int r = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100532 struct dm_target_io *tio = bio->bi_private;
Stefan Bader9faf4002006-10-03 01:15:41 -0700533 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 dm_endio_fn endio = tio->ti->type->end_io;
535
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
537 error = -EIO;
538
539 if (endio) {
540 r = endio(tio->ti, bio, error, &tio->info);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800541 if (r < 0 || r == DM_ENDIO_REQUEUE)
542 /*
543 * error and requeue request are handled
544 * in dec_pending().
545 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800547 else if (r == DM_ENDIO_INCOMPLETE)
548 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200549 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800550 else if (r) {
551 DMWARN("unimplemented target endio return value: %d", r);
552 BUG();
553 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 }
555
Stefan Bader9faf4002006-10-03 01:15:41 -0700556 dec_pending(tio->io, error);
557
558 /*
559 * Store md for cleanup instead of tio which is about to get freed.
560 */
561 bio->bi_private = md->bs;
562
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 bio_put(bio);
Stefan Bader9faf4002006-10-03 01:15:41 -0700564 free_tio(md, tio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565}
566
567static sector_t max_io_len(struct mapped_device *md,
568 sector_t sector, struct dm_target *ti)
569{
570 sector_t offset = sector - ti->begin;
571 sector_t len = ti->len - offset;
572
573 /*
574 * Does the target need to split even further ?
575 */
576 if (ti->split_io) {
577 sector_t boundary;
578 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
579 - offset;
580 if (len > boundary)
581 len = boundary;
582 }
583
584 return len;
585}
586
587static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100588 struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589{
590 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100591 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -0700592 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593
594 /*
595 * Sanity checks.
596 */
597 BUG_ON(!clone->bi_size);
598
599 clone->bi_end_io = clone_endio;
600 clone->bi_private = tio;
601
602 /*
603 * Map the clone. If r == 0 we don't need to do
604 * anything, the target has assumed ownership of
605 * this io.
606 */
607 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100608 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 r = ti->type->map(ti, clone, &tio->info);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800610 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100612
Alasdair G Kergon17b2f662006-06-26 00:27:33 -0700613 blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
Alan D. Brunellec7149d62007-08-07 15:30:23 +0200614 tio->io->bio->bi_bdev->bd_dev,
615 clone->bi_sector, sector);
Jens Axboe2056a782006-03-23 20:00:26 +0100616
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800618 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
619 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -0700620 md = tio->io->md;
621 dec_pending(tio->io, r);
622 /*
623 * Store bio_set for cleanup.
624 */
625 clone->bi_private = md->bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 bio_put(clone);
Stefan Bader9faf4002006-10-03 01:15:41 -0700627 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800628 } else if (r) {
629 DMWARN("unimplemented target map return value: %d", r);
630 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 }
632}
633
634struct clone_info {
635 struct mapped_device *md;
636 struct dm_table *map;
637 struct bio *bio;
638 struct dm_io *io;
639 sector_t sector;
640 sector_t sector_count;
641 unsigned short idx;
642};
643
Peter Osterlund36763472005-09-06 15:16:42 -0700644static void dm_bio_destructor(struct bio *bio)
645{
Stefan Bader9faf4002006-10-03 01:15:41 -0700646 struct bio_set *bs = bio->bi_private;
647
648 bio_free(bio, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700649}
650
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651/*
652 * Creates a little bio that is just does part of a bvec.
653 */
654static struct bio *split_bvec(struct bio *bio, sector_t sector,
655 unsigned short idx, unsigned int offset,
Stefan Bader9faf4002006-10-03 01:15:41 -0700656 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657{
658 struct bio *clone;
659 struct bio_vec *bv = bio->bi_io_vec + idx;
660
Stefan Bader9faf4002006-10-03 01:15:41 -0700661 clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund36763472005-09-06 15:16:42 -0700662 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 *clone->bi_io_vec = *bv;
664
665 clone->bi_sector = sector;
666 clone->bi_bdev = bio->bi_bdev;
667 clone->bi_rw = bio->bi_rw;
668 clone->bi_vcnt = 1;
669 clone->bi_size = to_bytes(len);
670 clone->bi_io_vec->bv_offset = offset;
671 clone->bi_io_vec->bv_len = clone->bi_size;
672
673 return clone;
674}
675
676/*
677 * Creates a bio that consists of range of complete bvecs.
678 */
679static struct bio *clone_bio(struct bio *bio, sector_t sector,
680 unsigned short idx, unsigned short bv_count,
Stefan Bader9faf4002006-10-03 01:15:41 -0700681 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682{
683 struct bio *clone;
684
Stefan Bader9faf4002006-10-03 01:15:41 -0700685 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
686 __bio_clone(clone, bio);
687 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 clone->bi_sector = sector;
689 clone->bi_idx = idx;
690 clone->bi_vcnt = idx + bv_count;
691 clone->bi_size = to_bytes(len);
692 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
693
694 return clone;
695}
696
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000697static int __clone_and_map(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698{
699 struct bio *clone, *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000700 struct dm_target *ti;
701 sector_t len = 0, max;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100702 struct dm_target_io *tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000704 ti = dm_table_find_target(ci->map, ci->sector);
705 if (!dm_target_is_valid(ti))
706 return -EIO;
707
708 max = max_io_len(ci->md, ci->sector, ti);
709
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710 /*
711 * Allocate a target io object.
712 */
713 tio = alloc_tio(ci->md);
714 tio->io = ci->io;
715 tio->ti = ti;
716 memset(&tio->info, 0, sizeof(tio->info));
717
718 if (ci->sector_count <= max) {
719 /*
720 * Optimise for the simple case where we can do all of
721 * the remaining io with a single clone.
722 */
723 clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700724 bio->bi_vcnt - ci->idx, ci->sector_count,
725 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 __map_bio(ti, clone, tio);
727 ci->sector_count = 0;
728
729 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
730 /*
731 * There are some bvecs that don't span targets.
732 * Do as many of these as possible.
733 */
734 int i;
735 sector_t remaining = max;
736 sector_t bv_len;
737
738 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
739 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
740
741 if (bv_len > remaining)
742 break;
743
744 remaining -= bv_len;
745 len += bv_len;
746 }
747
Stefan Bader9faf4002006-10-03 01:15:41 -0700748 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
749 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 __map_bio(ti, clone, tio);
751
752 ci->sector += len;
753 ci->sector_count -= len;
754 ci->idx = i;
755
756 } else {
757 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800758 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 */
760 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800761 sector_t remaining = to_sector(bv->bv_len);
762 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800764 do {
765 if (offset) {
766 ti = dm_table_find_target(ci->map, ci->sector);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000767 if (!dm_target_is_valid(ti))
768 return -EIO;
769
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800770 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800772 tio = alloc_tio(ci->md);
773 tio->io = ci->io;
774 tio->ti = ti;
775 memset(&tio->info, 0, sizeof(tio->info));
776 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800778 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800780 clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -0700781 bv->bv_offset + offset, len,
782 ci->md->bs);
Alasdair G Kergond2044a92006-03-22 00:07:42 -0800783
784 __map_bio(ti, clone, tio);
785
786 ci->sector += len;
787 ci->sector_count -= len;
788 offset += to_bytes(len);
789 } while (remaining -= len);
790
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 ci->idx++;
792 }
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000793
794 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795}
796
797/*
798 * Split the bio into several clones.
799 */
Milan Broz9e4e5f82007-10-19 22:38:53 +0100800static int __split_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801{
802 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000803 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804
805 ci.map = dm_get_table(md);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100806 if (unlikely(!ci.map))
807 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808
809 ci.md = md;
810 ci.bio = bio;
811 ci.io = alloc_io(md);
812 ci.io->error = 0;
813 atomic_set(&ci.io->io_count, 1);
814 ci.io->bio = bio;
815 ci.io->md = md;
816 ci.sector = bio->bi_sector;
817 ci.sector_count = bio_sectors(bio);
818 ci.idx = bio->bi_idx;
819
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800820 start_io_acct(ci.io);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000821 while (ci.sector_count && !error)
822 error = __clone_and_map(&ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823
824 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +0000825 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 dm_table_put(ci.map);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100827
828 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829}
830/*-----------------------------------------------------------------
831 * CRUD END
832 *---------------------------------------------------------------*/
833
Milan Brozf6fccb12008-07-21 12:00:37 +0100834static int dm_merge_bvec(struct request_queue *q,
835 struct bvec_merge_data *bvm,
836 struct bio_vec *biovec)
837{
838 struct mapped_device *md = q->queuedata;
839 struct dm_table *map = dm_get_table(md);
840 struct dm_target *ti;
841 sector_t max_sectors;
Mikulas Patocka50371082008-10-01 14:39:17 +0100842 int max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +0100843
844 if (unlikely(!map))
Mikulas Patocka50371082008-10-01 14:39:17 +0100845 goto out;
Milan Brozf6fccb12008-07-21 12:00:37 +0100846
847 ti = dm_table_find_target(map, bvm->bi_sector);
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100848 if (!dm_target_is_valid(ti))
849 goto out_table;
Milan Brozf6fccb12008-07-21 12:00:37 +0100850
851 /*
852 * Find maximum amount of I/O that won't need splitting
853 */
854 max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
855 (sector_t) BIO_MAX_SECTORS);
856 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
857 if (max_size < 0)
858 max_size = 0;
859
860 /*
861 * merge_bvec_fn() returns number of bytes
862 * it can accept at this offset
863 * max is precomputed maximal io size
864 */
865 if (max_size && ti->type->merge)
866 max_size = ti->type->merge(ti, bvm, biovec, max_size);
867
Mikulas Patockab01cd5a2008-10-01 14:39:24 +0100868out_table:
Mikulas Patocka50371082008-10-01 14:39:17 +0100869 dm_table_put(map);
870
871out:
Milan Brozf6fccb12008-07-21 12:00:37 +0100872 /*
873 * Always allow an entire first page
874 */
875 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
876 max_size = biovec->bv_len;
877
Milan Brozf6fccb12008-07-21 12:00:37 +0100878 return max_size;
879}
880
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881/*
882 * The request function that just remaps the bio built up by
883 * dm_merge_bvec.
884 */
Jens Axboe165125e2007-07-24 09:28:11 +0200885static int dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886{
Milan Broz9e4e5f82007-10-19 22:38:53 +0100887 int r = -EIO;
Kevin Corry12f03a42006-02-01 03:04:52 -0800888 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 struct mapped_device *md = q->queuedata;
Tejun Heoc9959052008-08-25 19:47:21 +0900890 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891
Stefan Bader07a83c42007-07-12 17:28:33 +0100892 /*
893 * There is no use in forwarding any barrier request since we can't
894 * guarantee it is (or can be) handled by the targets correctly.
895 */
896 if (unlikely(bio_barrier(bio))) {
NeilBrown6712ecf2007-09-27 12:47:43 +0200897 bio_endio(bio, -EOPNOTSUPP);
Stefan Bader07a83c42007-07-12 17:28:33 +0100898 return 0;
899 }
900
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700901 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902
Tejun Heo074a7ac2008-08-25 19:56:14 +0900903 cpu = part_stat_lock();
904 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
905 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
906 part_stat_unlock();
Kevin Corry12f03a42006-02-01 03:04:52 -0800907
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908 /*
909 * If we're suspended we have to queue
910 * this io for later.
911 */
912 while (test_bit(DMF_BLOCK_IO, &md->flags)) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700913 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914
Milan Broz9e4e5f82007-10-19 22:38:53 +0100915 if (bio_rw(bio) != READA)
916 r = queue_io(md, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917
Milan Broz9e4e5f82007-10-19 22:38:53 +0100918 if (r <= 0)
919 goto out_req;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920
921 /*
922 * We're in a while loop, because someone could suspend
923 * before we get to the following read lock.
924 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700925 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926 }
927
Milan Broz9e4e5f82007-10-19 22:38:53 +0100928 r = __split_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700929 up_read(&md->io_lock);
Milan Broz9e4e5f82007-10-19 22:38:53 +0100930
931out_req:
932 if (r < 0)
933 bio_io_error(bio);
934
Linus Torvalds1da177e2005-04-16 15:20:36 -0700935 return 0;
936}
937
Jens Axboe165125e2007-07-24 09:28:11 +0200938static void dm_unplug_all(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939{
940 struct mapped_device *md = q->queuedata;
941 struct dm_table *map = dm_get_table(md);
942
943 if (map) {
944 dm_table_unplug_all(map);
945 dm_table_put(map);
946 }
947}
948
949static int dm_any_congested(void *congested_data, int bdi_bits)
950{
951 int r;
952 struct mapped_device *md = (struct mapped_device *) congested_data;
953 struct dm_table *map = dm_get_table(md);
954
955 if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
956 r = bdi_bits;
957 else
958 r = dm_table_any_congested(map, bdi_bits);
959
960 dm_table_put(map);
961 return r;
962}
963
964/*-----------------------------------------------------------------
965 * An IDR is used to keep track of allocated minor numbers.
966 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967static DEFINE_IDR(_minor_idr);
968
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -0700969static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700971 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700973 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974}
975
976/*
977 * See if the device with a specific minor # is free.
978 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +0100979static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980{
981 int r, m;
982
983 if (minor >= (1 << MINORBITS))
984 return -EINVAL;
985
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700986 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
987 if (!r)
988 return -ENOMEM;
989
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -0700990 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991
992 if (idr_find(&_minor_idr, minor)) {
993 r = -EBUSY;
994 goto out;
995 }
996
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -0700997 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney62f75c22006-06-26 00:27:21 -0700998 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000
1001 if (m != minor) {
1002 idr_remove(&_minor_idr, m);
1003 r = -EBUSY;
1004 goto out;
1005 }
1006
1007out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001008 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009 return r;
1010}
1011
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001012static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001014 int r, m;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001017 if (!r)
1018 return -ENOMEM;
1019
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001020 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001021
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001022 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001023 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025
1026 if (m >= (1 << MINORBITS)) {
1027 idr_remove(&_minor_idr, m);
1028 r = -ENOSPC;
1029 goto out;
1030 }
1031
1032 *minor = m;
1033
1034out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001035 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 return r;
1037}
1038
1039static struct block_device_operations dm_blk_dops;
1040
1041/*
1042 * Allocate and initialise a blank device with a given minor.
1043 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001044static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045{
1046 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001047 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001048 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049
1050 if (!md) {
1051 DMWARN("unable to allocate device, out of memory.");
1052 return NULL;
1053 }
1054
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001055 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001056 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001057
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001059 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001060 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001061 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001062 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001064 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001066 init_rwsem(&md->io_lock);
Daniel Walkere61290a2008-02-08 02:10:08 +00001067 mutex_init(&md->suspend_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001068 spin_lock_init(&md->pushback_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 rwlock_init(&md->map_lock);
1070 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001071 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001073 atomic_set(&md->uevent_seq, 0);
1074 INIT_LIST_HEAD(&md->uevent_list);
1075 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076
1077 md->queue = blk_alloc_queue(GFP_KERNEL);
1078 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001079 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080
1081 md->queue->queuedata = md;
1082 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1083 md->queue->backing_dev_info.congested_data = md;
1084 blk_queue_make_request(md->queue, dm_request);
Jens Axboedaef2652006-01-10 10:48:02 +01001085 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001086 md->queue->unplug_fn = dm_unplug_all;
Milan Brozf6fccb12008-07-21 12:00:37 +01001087 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088
Matthew Dobson93d23412006-03-26 01:37:50 -08001089 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
Kiyoshi Ueda74859362006-12-08 02:41:02 -08001090 if (!md->io_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001091 goto bad_io_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092
Matthew Dobson93d23412006-03-26 01:37:50 -08001093 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 if (!md->tio_pool)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001095 goto bad_tio_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096
Jens Axboe59725112007-04-02 10:06:42 +02001097 md->bs = bioset_create(16, 16);
Stefan Bader9faf4002006-10-03 01:15:41 -07001098 if (!md->bs)
1099 goto bad_no_bioset;
1100
Linus Torvalds1da177e2005-04-16 15:20:36 -07001101 md->disk = alloc_disk(1);
1102 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001103 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001105 atomic_set(&md->pending, 0);
1106 init_waitqueue_head(&md->wait);
1107 init_waitqueue_head(&md->eventq);
1108
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 md->disk->major = _major;
1110 md->disk->first_minor = minor;
1111 md->disk->fops = &dm_blk_dops;
1112 md->disk->queue = md->queue;
1113 md->disk->private_data = md;
1114 sprintf(md->disk->disk_name, "dm-%d", minor);
1115 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001116 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117
Milan Broz304f3f62008-02-08 02:11:17 +00001118 md->wq = create_singlethread_workqueue("kdmflush");
1119 if (!md->wq)
1120 goto bad_thread;
1121
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001122 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001123 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001124 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001125 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001126
1127 BUG_ON(old_md != MINOR_ALLOCED);
1128
Linus Torvalds1da177e2005-04-16 15:20:36 -07001129 return md;
1130
Milan Broz304f3f62008-02-08 02:11:17 +00001131bad_thread:
1132 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001133bad_disk:
Stefan Bader9faf4002006-10-03 01:15:41 -07001134 bioset_free(md->bs);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001135bad_no_bioset:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136 mempool_destroy(md->tio_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001137bad_tio_pool:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 mempool_destroy(md->io_pool);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001139bad_io_pool:
Al Viro1312f402006-03-12 11:02:03 -05001140 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001141bad_queue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001143bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001144 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001145bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146 kfree(md);
1147 return NULL;
1148}
1149
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001150static void unlock_fs(struct mapped_device *md);
1151
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152static void free_dev(struct mapped_device *md)
1153{
Tejun Heof331c022008-09-03 09:01:48 +02001154 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001155
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001156 if (md->suspended_bdev) {
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001157 unlock_fs(md);
Jun'ichi Nomurad9dde592006-02-24 13:04:24 -08001158 bdput(md->suspended_bdev);
1159 }
Milan Broz304f3f62008-02-08 02:11:17 +00001160 destroy_workqueue(md->wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 mempool_destroy(md->tio_pool);
1162 mempool_destroy(md->io_pool);
Stefan Bader9faf4002006-10-03 01:15:41 -07001163 bioset_free(md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001165 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001166
1167 spin_lock(&_minor_lock);
1168 md->disk->private_data = NULL;
1169 spin_unlock(&_minor_lock);
1170
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05001172 blk_cleanup_queue(md->queue);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001173 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174 kfree(md);
1175}
1176
1177/*
1178 * Bind a table to the device.
1179 */
1180static void event_callback(void *context)
1181{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001182 unsigned long flags;
1183 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184 struct mapped_device *md = (struct mapped_device *) context;
1185
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001186 spin_lock_irqsave(&md->uevent_lock, flags);
1187 list_splice_init(&md->uevent_list, &uevents);
1188 spin_unlock_irqrestore(&md->uevent_lock, flags);
1189
Tejun Heoed9e1982008-08-25 19:56:05 +09001190 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001191
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 atomic_inc(&md->event_nr);
1193 wake_up(&md->eventq);
1194}
1195
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001196static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197{
Alasdair G Kergon4e90188be2005-07-28 21:15:59 -07001198 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001200 mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001201 i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
Jes Sorensen1b1dcc12006-01-09 15:59:24 -08001202 mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203}
1204
1205static int __bind(struct mapped_device *md, struct dm_table *t)
1206{
Jens Axboe165125e2007-07-24 09:28:11 +02001207 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208 sector_t size;
1209
1210 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001211
1212 /*
1213 * Wipe any geometry if the size of the table changed.
1214 */
1215 if (size != get_capacity(md->disk))
1216 memset(&md->geometry, 0, sizeof(md->geometry));
1217
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001218 if (md->suspended_bdev)
1219 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220 if (size == 0)
1221 return 0;
1222
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223 dm_table_get(t);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001224 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001225
1226 write_lock(&md->map_lock);
1227 md->map = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 dm_table_set_restrictions(t, q);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001229 write_unlock(&md->map_lock);
1230
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231 return 0;
1232}
1233
1234static void __unbind(struct mapped_device *md)
1235{
1236 struct dm_table *map = md->map;
1237
1238 if (!map)
1239 return;
1240
1241 dm_table_event_callback(map, NULL, NULL);
1242 write_lock(&md->map_lock);
1243 md->map = NULL;
1244 write_unlock(&md->map_lock);
1245 dm_table_put(map);
1246}
1247
1248/*
1249 * Constructor for a new device.
1250 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001251int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252{
1253 struct mapped_device *md;
1254
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001255 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 if (!md)
1257 return -ENXIO;
1258
1259 *result = md;
1260 return 0;
1261}
1262
David Teigland637842c2006-01-06 00:20:00 -08001263static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001264{
1265 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266 unsigned minor = MINOR(dev);
1267
1268 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1269 return NULL;
1270
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001271 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272
1273 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001274 if (md && (md == MINOR_ALLOCED ||
Tejun Heof331c022008-09-03 09:01:48 +02001275 (MINOR(disk_devt(dm_disk(md))) != minor) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07001276 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08001277 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001278 goto out;
1279 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001280
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001281out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001282 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283
David Teigland637842c2006-01-06 00:20:00 -08001284 return md;
1285}
1286
David Teiglandd229a952006-01-06 00:20:01 -08001287struct mapped_device *dm_get_md(dev_t dev)
1288{
1289 struct mapped_device *md = dm_find_md(dev);
1290
1291 if (md)
1292 dm_get(md);
1293
1294 return md;
1295}
1296
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001297void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08001298{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08001299 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300}
1301
1302void dm_set_mdptr(struct mapped_device *md, void *ptr)
1303{
1304 md->interface_ptr = ptr;
1305}
1306
1307void dm_get(struct mapped_device *md)
1308{
1309 atomic_inc(&md->holders);
1310}
1311
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001312const char *dm_device_name(struct mapped_device *md)
1313{
1314 return md->name;
1315}
1316EXPORT_SYMBOL_GPL(dm_device_name);
1317
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318void dm_put(struct mapped_device *md)
1319{
Mike Anderson1134e5a2006-03-27 01:17:54 -08001320 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001322 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1323
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001324 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08001325 map = dm_get_table(md);
Tejun Heof331c022008-09-03 09:01:48 +02001326 idr_replace(&_minor_idr, MINOR_ALLOCED,
1327 MINOR(disk_devt(dm_disk(md))));
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001328 set_bit(DMF_FREEING, &md->flags);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001329 spin_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001330 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331 dm_table_presuspend_targets(map);
1332 dm_table_postsuspend_targets(map);
1333 }
1334 __unbind(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08001335 dm_table_put(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 free_dev(md);
1337 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338}
Edward Goggin79eb8852007-05-09 02:32:56 -07001339EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340
Milan Broz46125c12008-02-08 02:10:30 +00001341static int dm_wait_for_completion(struct mapped_device *md)
1342{
1343 int r = 0;
1344
1345 while (1) {
1346 set_current_state(TASK_INTERRUPTIBLE);
1347
1348 smp_mb();
1349 if (!atomic_read(&md->pending))
1350 break;
1351
1352 if (signal_pending(current)) {
1353 r = -EINTR;
1354 break;
1355 }
1356
1357 io_schedule();
1358 }
1359 set_current_state(TASK_RUNNING);
1360
1361 return r;
1362}
1363
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364/*
1365 * Process the deferred bios
1366 */
Milan Broz6d6f10d2008-02-08 02:10:22 +00001367static void __flush_deferred_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001368{
Milan Broz6d6f10d2008-02-08 02:10:22 +00001369 struct bio *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370
Milan Broz6d6f10d2008-02-08 02:10:22 +00001371 while ((c = bio_list_pop(&md->deferred))) {
Milan Broz9e4e5f82007-10-19 22:38:53 +01001372 if (__split_bio(md, c))
1373 bio_io_error(c);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374 }
Milan Broz73d410c2008-02-08 02:10:25 +00001375
1376 clear_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001377}
1378
Milan Broz6d6f10d2008-02-08 02:10:22 +00001379static void __merge_pushback_list(struct mapped_device *md)
1380{
1381 unsigned long flags;
1382
1383 spin_lock_irqsave(&md->pushback_lock, flags);
1384 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1385 bio_list_merge_head(&md->deferred, &md->pushback);
1386 bio_list_init(&md->pushback);
1387 spin_unlock_irqrestore(&md->pushback_lock, flags);
1388}
1389
Milan Broz304f3f62008-02-08 02:11:17 +00001390static void dm_wq_work(struct work_struct *work)
1391{
1392 struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
1393 struct mapped_device *md = req->md;
1394
1395 down_write(&md->io_lock);
1396 switch (req->type) {
1397 case DM_WQ_FLUSH_ALL:
1398 __merge_pushback_list(md);
1399 /* pass through */
1400 case DM_WQ_FLUSH_DEFERRED:
1401 __flush_deferred_io(md);
1402 break;
1403 default:
1404 DMERR("dm_wq_work: unrecognised work type %d", req->type);
1405 BUG();
1406 }
1407 up_write(&md->io_lock);
1408}
1409
1410static void dm_wq_queue(struct mapped_device *md, int type, void *context,
1411 struct dm_wq_req *req)
1412{
1413 req->type = type;
1414 req->md = md;
1415 req->context = context;
1416 INIT_WORK(&req->work, dm_wq_work);
1417 queue_work(md->wq, &req->work);
1418}
1419
1420static void dm_queue_flush(struct mapped_device *md, int type, void *context)
1421{
1422 struct dm_wq_req req;
1423
1424 dm_wq_queue(md, type, context, &req);
1425 flush_workqueue(md->wq);
1426}
1427
Linus Torvalds1da177e2005-04-16 15:20:36 -07001428/*
1429 * Swap in a new table (destroying old one).
1430 */
1431int dm_swap_table(struct mapped_device *md, struct dm_table *table)
1432{
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001433 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434
Daniel Walkere61290a2008-02-08 02:10:08 +00001435 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436
1437 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001438 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001439 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001440
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001441 /* without bdev, the device size cannot be changed */
1442 if (!md->suspended_bdev)
1443 if (get_capacity(md->disk) != dm_table_get_size(table))
1444 goto out;
1445
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446 __unbind(md);
1447 r = __bind(md, table);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001449out:
Daniel Walkere61290a2008-02-08 02:10:08 +00001450 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07001451 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452}
1453
1454/*
1455 * Functions to lock and unlock any filesystem running on the
1456 * device.
1457 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001458static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001460 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461
1462 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001463
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001464 md->frozen_sb = freeze_bdev(md->suspended_bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001465 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001466 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001467 md->frozen_sb = NULL;
1468 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07001469 }
1470
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001471 set_bit(DMF_FROZEN, &md->flags);
1472
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473 /* don't bdput right now, we don't want the bdev
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001474 * to go away while it is locked.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001475 */
1476 return 0;
1477}
1478
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001479static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001481 if (!test_bit(DMF_FROZEN, &md->flags))
1482 return;
1483
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001484 thaw_bdev(md->suspended_bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001485 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001486 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487}
1488
1489/*
1490 * We need to be able to change a mapping table under a mounted
1491 * filesystem. For example we might want to move some data in
1492 * the background. Before the table can be swapped with
1493 * dm_bind_table, dm_suspend must be called to flush any in
1494 * flight bios and ensure that any further io gets deferred.
1495 */
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001496int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001498 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499 DECLARE_WAITQUEUE(wait, current);
Milan Broz46125c12008-02-08 02:10:30 +00001500 int r = 0;
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08001501 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001502 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503
Daniel Walkere61290a2008-02-08 02:10:08 +00001504 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001505
Milan Broz73d410c2008-02-08 02:10:25 +00001506 if (dm_suspended(md)) {
1507 r = -EINVAL;
Alasdair G Kergond2874832006-11-08 17:44:43 -08001508 goto out_unlock;
Milan Broz73d410c2008-02-08 02:10:25 +00001509 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510
1511 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001513 /*
1514 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
1515 * This flag is cleared before dm_suspend returns.
1516 */
1517 if (noflush)
1518 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1519
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001520 /* This does not get reverted if there's an error later. */
1521 dm_table_presuspend_targets(map);
1522
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001523 /* bdget() can stall if the pending I/Os are not flushed */
1524 if (!noflush) {
1525 md->suspended_bdev = bdget_disk(md->disk, 0);
1526 if (!md->suspended_bdev) {
1527 DMWARN("bdget failed in dm_suspend");
1528 r = -ENOMEM;
1529 goto flush_and_out;
1530 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001531
Milan Broz6d6f10d2008-02-08 02:10:22 +00001532 /*
1533 * Flush I/O to the device. noflush supersedes do_lockfs,
1534 * because lock_fs() needs to flush I/Os.
1535 */
1536 if (do_lockfs) {
1537 r = lock_fs(md);
1538 if (r)
1539 goto out;
1540 }
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08001541 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542
1543 /*
Alasdair G Kergon354e0072005-05-05 16:16:05 -07001544 * First we set the BLOCK_IO flag so no more ios will be mapped.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001546 down_write(&md->io_lock);
1547 set_bit(DMF_BLOCK_IO, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549 add_wait_queue(&md->wait, &wait);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001550 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551
1552 /* unplug */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001553 if (map)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555
1556 /*
Milan Broz46125c12008-02-08 02:10:30 +00001557 * Wait for the already-mapped ios to complete.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001558 */
Milan Broz46125c12008-02-08 02:10:30 +00001559 r = dm_wait_for_completion(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001561 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562 remove_wait_queue(&md->wait, &wait);
1563
Milan Broz6d6f10d2008-02-08 02:10:22 +00001564 if (noflush)
1565 __merge_pushback_list(md);
Milan Broz94d63512008-02-08 02:10:27 +00001566 up_write(&md->io_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001567
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00001569 if (r < 0) {
Milan Broz304f3f62008-02-08 02:11:17 +00001570 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Milan Broz73d410c2008-02-08 02:10:25 +00001571
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001572 unlock_fs(md);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001573 goto out; /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001574 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001575
1576 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577
1578 set_bit(DMF_SUSPENDED, &md->flags);
1579
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001580flush_and_out:
Milan Broz304f3f62008-02-08 02:11:17 +00001581 if (r && noflush)
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001582 /*
1583 * Because there may be already I/Os in the pushback list,
1584 * flush them before return.
1585 */
Milan Broz304f3f62008-02-08 02:11:17 +00001586 dm_queue_flush(md, DM_WQ_FLUSH_ALL, NULL);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001587
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001588out:
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001589 if (r && md->suspended_bdev) {
1590 bdput(md->suspended_bdev);
1591 md->suspended_bdev = NULL;
1592 }
1593
Linus Torvalds1da177e2005-04-16 15:20:36 -07001594 dm_table_put(map);
Alasdair G Kergond2874832006-11-08 17:44:43 -08001595
1596out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00001597 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001598 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599}
1600
1601int dm_resume(struct mapped_device *md)
1602{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001603 int r = -EINVAL;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001604 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605
Daniel Walkere61290a2008-02-08 02:10:08 +00001606 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001607 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001608 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001609
1610 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001611 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001612 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613
Milan Broz8757b772006-10-03 01:15:36 -07001614 r = dm_table_resume_targets(map);
1615 if (r)
1616 goto out;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001617
Milan Broz304f3f62008-02-08 02:11:17 +00001618 dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001619
1620 unlock_fs(md);
1621
Jun'ichi Nomurabfa152f2007-01-26 00:57:07 -08001622 if (md->suspended_bdev) {
1623 bdput(md->suspended_bdev);
1624 md->suspended_bdev = NULL;
1625 }
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08001626
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001627 clear_bit(DMF_SUSPENDED, &md->flags);
1628
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629 dm_table_unplug_all(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001631 dm_kobject_uevent(md);
Hannes Reinecke8560ed62006-10-03 01:15:35 -07001632
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001633 r = 0;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001634
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001635out:
1636 dm_table_put(map);
Daniel Walkere61290a2008-02-08 02:10:08 +00001637 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001638
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001639 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640}
1641
1642/*-----------------------------------------------------------------
1643 * Event notification.
1644 *---------------------------------------------------------------*/
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001645void dm_kobject_uevent(struct mapped_device *md)
1646{
Tejun Heoed9e1982008-08-25 19:56:05 +09001647 kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
Alasdair G Kergon69267a32007-12-13 14:15:57 +00001648}
1649
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001650uint32_t dm_next_uevent_seq(struct mapped_device *md)
1651{
1652 return atomic_add_return(1, &md->uevent_seq);
1653}
1654
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655uint32_t dm_get_event_nr(struct mapped_device *md)
1656{
1657 return atomic_read(&md->event_nr);
1658}
1659
1660int dm_wait_event(struct mapped_device *md, int event_nr)
1661{
1662 return wait_event_interruptible(md->eventq,
1663 (event_nr != atomic_read(&md->event_nr)));
1664}
1665
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001666void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
1667{
1668 unsigned long flags;
1669
1670 spin_lock_irqsave(&md->uevent_lock, flags);
1671 list_add(elist, &md->uevent_list);
1672 spin_unlock_irqrestore(&md->uevent_lock, flags);
1673}
1674
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675/*
1676 * The gendisk is only valid as long as you have a reference
1677 * count on 'md'.
1678 */
1679struct gendisk *dm_disk(struct mapped_device *md)
1680{
1681 return md->disk;
1682}
1683
1684int dm_suspended(struct mapped_device *md)
1685{
1686 return test_bit(DMF_SUSPENDED, &md->flags);
1687}
1688
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08001689int dm_noflush_suspending(struct dm_target *ti)
1690{
1691 struct mapped_device *md = dm_table_get_md(ti->table);
1692 int r = __noflush_suspending(md);
1693
1694 dm_put(md);
1695
1696 return r;
1697}
1698EXPORT_SYMBOL_GPL(dm_noflush_suspending);
1699
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700static struct block_device_operations dm_blk_dops = {
1701 .open = dm_blk_open,
1702 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07001703 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001704 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705 .owner = THIS_MODULE
1706};
1707
1708EXPORT_SYMBOL(dm_get_mapinfo);
1709
1710/*
1711 * module hooks
1712 */
1713module_init(dm_init);
1714module_exit(dm_exit);
1715
1716module_param(major, uint, 0);
1717MODULE_PARM_DESC(major, "The major number of the device mapper");
1718MODULE_DESCRIPTION(DM_NAME " driver");
1719MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1720MODULE_LICENSE("GPL");