blob: 376f1ab48a245fcdce521bd55a7ceb109f7f8c9e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
Milan Broz784aae72009-01-06 03:05:12 +00003 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
Mike Anderson51e5b2b2007-10-19 22:48:00 +01009#include "dm-uevent.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070010
11#include <linux/init.h>
12#include <linux/module.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080013#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/moduleparam.h>
15#include <linux/blkpg.h>
16#include <linux/bio.h>
17#include <linux/buffer_head.h>
18#include <linux/mempool.h>
19#include <linux/slab.h>
20#include <linux/idr.h>
Darrick J. Wong3ac51e72006-03-27 01:17:54 -080021#include <linux/hdreg.h>
Li Zefan55782132009-06-09 13:43:05 +080022
23#include <trace/events/block.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Alasdair G Kergon72d94862006-06-26 00:27:35 -070025#define DM_MSG_PREFIX "core"
26
Milan Broz60935eb2009-06-22 10:12:30 +010027/*
28 * Cookies are numeric values sent with CHANGE and REMOVE
29 * uevents while resuming, removing or renaming the device.
30 */
31#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
32#define DM_COOKIE_LENGTH 24
33
Linus Torvalds1da177e2005-04-16 15:20:36 -070034static const char *_name = DM_NAME;
35
36static unsigned int major = 0;
37static unsigned int _major = 0;
38
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -070039static DEFINE_SPINLOCK(_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070040/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000041 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 * One of these is allocated per bio.
43 */
44struct dm_io {
45 struct mapped_device *md;
46 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 atomic_t io_count;
Richard Kennedy6ae2fa62008-07-21 12:00:28 +010048 struct bio *bio;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -080049 unsigned long start_time;
Linus Torvalds1da177e2005-04-16 15:20:36 -070050};
51
52/*
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000053 * For bio-based dm.
Linus Torvalds1da177e2005-04-16 15:20:36 -070054 * One of these is allocated per target within a bio. Hopefully
55 * this will be simplified out one day.
56 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +010057struct dm_target_io {
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 struct dm_io *io;
59 struct dm_target *ti;
60 union map_info info;
61};
62
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000063/*
64 * For request-based dm.
65 * One of these is allocated per request.
66 */
67struct dm_rq_target_io {
68 struct mapped_device *md;
69 struct dm_target *ti;
70 struct request *orig, clone;
71 int error;
72 union map_info info;
73};
74
75/*
76 * For request-based dm.
77 * One of these is allocated per bio.
78 */
79struct dm_rq_clone_bio_info {
80 struct bio *orig;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +010081 struct dm_rq_target_io *tio;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +000082};
83
Linus Torvalds1da177e2005-04-16 15:20:36 -070084union map_info *dm_get_mapinfo(struct bio *bio)
85{
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070086 if (bio && bio->bi_private)
Alasdair G Kergon028867a2007-07-12 17:26:32 +010087 return &((struct dm_target_io *)bio->bi_private)->info;
Alasdair G Kergon17b2f662006-06-26 00:27:33 -070088 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070089}
90
Kiyoshi Uedacec47e32009-06-22 10:12:35 +010091union map_info *dm_get_rq_mapinfo(struct request *rq)
92{
93 if (rq && rq->end_io_data)
94 return &((struct dm_rq_target_io *)rq->end_io_data)->info;
95 return NULL;
96}
97EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo);
98
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -070099#define MINOR_ALLOCED ((void *)-1)
100
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101/*
102 * Bits for the md->flags field.
103 */
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +0100104#define DMF_BLOCK_IO_FOR_SUSPEND 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105#define DMF_SUSPENDED 1
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -0800106#define DMF_FROZEN 2
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700107#define DMF_FREEING 3
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700108#define DMF_DELETING 4
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800109#define DMF_NOFLUSH_SUSPENDING 5
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +0100110#define DMF_QUEUE_IO_TO_THREAD 6
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111
Milan Broz304f3f62008-02-08 02:11:17 +0000112/*
113 * Work processed by per-device workqueue.
114 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115struct mapped_device {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700116 struct rw_semaphore io_lock;
Daniel Walkere61290a2008-02-08 02:10:08 +0000117 struct mutex suspend_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 rwlock_t map_lock;
119 atomic_t holders;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700120 atomic_t open_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121
122 unsigned long flags;
123
Jens Axboe165125e2007-07-24 09:28:11 +0200124 struct request_queue *queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125 struct gendisk *disk;
Mike Anderson7e51f252006-03-27 01:17:52 -0800126 char name[16];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127
128 void *interface_ptr;
129
130 /*
131 * A list of ios that arrived while we were suspended.
132 */
Nikanth Karthikesana9327ca2009-09-11 09:18:54 +0200133 atomic_t pending[2];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 wait_queue_head_t wait;
Mikulas Patocka53d59142009-04-02 19:55:37 +0100135 struct work_struct work;
Kiyoshi Ueda74859362006-12-08 02:41:02 -0800136 struct bio_list deferred;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100137 spinlock_t deferred_lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
139 /*
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100140 * An error from the barrier request currently being processed.
141 */
142 int barrier_error;
143
144 /*
Milan Broz304f3f62008-02-08 02:11:17 +0000145 * Processing queue (flush/barriers)
146 */
147 struct workqueue_struct *wq;
148
149 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 * The current mapping.
151 */
152 struct dm_table *map;
153
154 /*
155 * io objects are allocated from here.
156 */
157 mempool_t *io_pool;
158 mempool_t *tio_pool;
159
Stefan Bader9faf4002006-10-03 01:15:41 -0700160 struct bio_set *bs;
161
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 /*
163 * Event handling.
164 */
165 atomic_t event_nr;
166 wait_queue_head_t eventq;
Mike Anderson7a8c3d32007-10-19 22:48:01 +0100167 atomic_t uevent_seq;
168 struct list_head uevent_list;
169 spinlock_t uevent_lock; /* Protect access to uevent_list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
171 /*
172 * freeze/thaw support require holding onto a super block
173 */
174 struct super_block *frozen_sb;
Mikulas Patockadb8fef42009-06-22 10:12:15 +0100175 struct block_device *bdev;
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800176
177 /* forced geometry settings */
178 struct hd_geometry geometry;
Milan Broz784aae72009-01-06 03:05:12 +0000179
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100180 /* marker of flush suspend for request-based dm */
181 struct request suspend_rq;
182
183 /* For saving the address of __make_request for request based dm */
184 make_request_fn *saved_make_request_fn;
185
Milan Broz784aae72009-01-06 03:05:12 +0000186 /* sysfs handle */
187 struct kobject kobj;
Mikulas Patocka52b1fd52009-06-22 10:12:21 +0100188
189 /* zero-length barrier that will be cloned and submitted to targets */
190 struct bio barrier_bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191};
192
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +0100193/*
194 * For mempools pre-allocation at the table loading time.
195 */
196struct dm_md_mempools {
197 mempool_t *io_pool;
198 mempool_t *tio_pool;
199 struct bio_set *bs;
200};
201
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202#define MIN_IOS 256
Christoph Lametere18b8902006-12-06 20:33:20 -0800203static struct kmem_cache *_io_cache;
204static struct kmem_cache *_tio_cache;
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000205static struct kmem_cache *_rq_tio_cache;
206static struct kmem_cache *_rq_bio_info_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208static int __init local_init(void)
209{
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100210 int r = -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 /* allocate a slab for the dm_ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100213 _io_cache = KMEM_CACHE(dm_io, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214 if (!_io_cache)
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100215 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217 /* allocate a slab for the target ios */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100218 _tio_cache = KMEM_CACHE(dm_target_io, 0);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100219 if (!_tio_cache)
220 goto out_free_io_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000222 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
223 if (!_rq_tio_cache)
224 goto out_free_tio_cache;
225
226 _rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
227 if (!_rq_bio_info_cache)
228 goto out_free_rq_tio_cache;
229
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100230 r = dm_uevent_init();
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100231 if (r)
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000232 goto out_free_rq_bio_info_cache;
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100233
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 _major = major;
235 r = register_blkdev(_major, _name);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100236 if (r < 0)
237 goto out_uevent_exit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
239 if (!_major)
240 _major = r;
241
242 return 0;
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100243
244out_uevent_exit:
245 dm_uevent_exit();
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000246out_free_rq_bio_info_cache:
247 kmem_cache_destroy(_rq_bio_info_cache);
248out_free_rq_tio_cache:
249 kmem_cache_destroy(_rq_tio_cache);
Kiyoshi Ueda51157b42008-10-21 17:45:08 +0100250out_free_tio_cache:
251 kmem_cache_destroy(_tio_cache);
252out_free_io_cache:
253 kmem_cache_destroy(_io_cache);
254
255 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256}
257
258static void local_exit(void)
259{
Kiyoshi Ueda8fbf26a2009-01-06 03:05:06 +0000260 kmem_cache_destroy(_rq_bio_info_cache);
261 kmem_cache_destroy(_rq_tio_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 kmem_cache_destroy(_tio_cache);
263 kmem_cache_destroy(_io_cache);
Akinobu Mita00d59402007-07-17 04:03:46 -0700264 unregister_blkdev(_major, _name);
Mike Anderson51e5b2b2007-10-19 22:48:00 +0100265 dm_uevent_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
267 _major = 0;
268
269 DMINFO("cleaned up");
270}
271
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000272static int (*_inits[])(void) __initdata = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 local_init,
274 dm_target_init,
275 dm_linear_init,
276 dm_stripe_init,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100277 dm_kcopyd_init,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 dm_interface_init,
279};
280
Alasdair G Kergonb9249e52008-02-08 02:09:51 +0000281static void (*_exits[])(void) = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 local_exit,
283 dm_target_exit,
284 dm_linear_exit,
285 dm_stripe_exit,
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100286 dm_kcopyd_exit,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 dm_interface_exit,
288};
289
290static int __init dm_init(void)
291{
292 const int count = ARRAY_SIZE(_inits);
293
294 int r, i;
295
296 for (i = 0; i < count; i++) {
297 r = _inits[i]();
298 if (r)
299 goto bad;
300 }
301
302 return 0;
303
304 bad:
305 while (i--)
306 _exits[i]();
307
308 return r;
309}
310
311static void __exit dm_exit(void)
312{
313 int i = ARRAY_SIZE(_exits);
314
315 while (i--)
316 _exits[i]();
317}
318
319/*
320 * Block device functions
321 */
Al Virofe5f9f22008-03-02 10:29:31 -0500322static int dm_blk_open(struct block_device *bdev, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323{
324 struct mapped_device *md;
325
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700326 spin_lock(&_minor_lock);
327
Al Virofe5f9f22008-03-02 10:29:31 -0500328 md = bdev->bd_disk->private_data;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700329 if (!md)
330 goto out;
331
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700332 if (test_bit(DMF_FREEING, &md->flags) ||
333 test_bit(DMF_DELETING, &md->flags)) {
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700334 md = NULL;
335 goto out;
336 }
337
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 dm_get(md);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700339 atomic_inc(&md->open_count);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -0700340
341out:
342 spin_unlock(&_minor_lock);
343
344 return md ? 0 : -ENXIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345}
346
Al Virofe5f9f22008-03-02 10:29:31 -0500347static int dm_blk_close(struct gendisk *disk, fmode_t mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348{
Al Virofe5f9f22008-03-02 10:29:31 -0500349 struct mapped_device *md = disk->private_data;
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700350 atomic_dec(&md->open_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 dm_put(md);
352 return 0;
353}
354
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -0700355int dm_open_count(struct mapped_device *md)
356{
357 return atomic_read(&md->open_count);
358}
359
360/*
361 * Guarantees nothing is using the device before it's deleted.
362 */
363int dm_lock_for_deletion(struct mapped_device *md)
364{
365 int r = 0;
366
367 spin_lock(&_minor_lock);
368
369 if (dm_open_count(md))
370 r = -EBUSY;
371 else
372 set_bit(DMF_DELETING, &md->flags);
373
374 spin_unlock(&_minor_lock);
375
376 return r;
377}
378
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800379static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
380{
381 struct mapped_device *md = bdev->bd_disk->private_data;
382
383 return dm_get_geometry(md, geo);
384}
385
Al Virofe5f9f22008-03-02 10:29:31 -0500386static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
Milan Brozaa129a22006-10-03 01:15:15 -0700387 unsigned int cmd, unsigned long arg)
388{
Al Virofe5f9f22008-03-02 10:29:31 -0500389 struct mapped_device *md = bdev->bd_disk->private_data;
390 struct dm_table *map = dm_get_table(md);
Milan Brozaa129a22006-10-03 01:15:15 -0700391 struct dm_target *tgt;
392 int r = -ENOTTY;
393
Milan Brozaa129a22006-10-03 01:15:15 -0700394 if (!map || !dm_table_get_size(map))
395 goto out;
396
397 /* We only support devices that have a single target */
398 if (dm_table_get_num_targets(map) != 1)
399 goto out;
400
401 tgt = dm_table_get_target(map, 0);
402
403 if (dm_suspended(md)) {
404 r = -EAGAIN;
405 goto out;
406 }
407
408 if (tgt->type->ioctl)
Al Viro647b3d02007-08-28 22:15:59 -0400409 r = tgt->type->ioctl(tgt, cmd, arg);
Milan Brozaa129a22006-10-03 01:15:15 -0700410
411out:
412 dm_table_put(map);
413
Milan Brozaa129a22006-10-03 01:15:15 -0700414 return r;
415}
416
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100417static struct dm_io *alloc_io(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418{
419 return mempool_alloc(md->io_pool, GFP_NOIO);
420}
421
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100422static void free_io(struct mapped_device *md, struct dm_io *io)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423{
424 mempool_free(io, md->io_pool);
425}
426
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100427static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428{
429 mempool_free(tio, md->tio_pool);
430}
431
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100432static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md)
433{
434 return mempool_alloc(md->tio_pool, GFP_ATOMIC);
435}
436
437static void free_rq_tio(struct dm_rq_target_io *tio)
438{
439 mempool_free(tio, tio->md->tio_pool);
440}
441
442static struct dm_rq_clone_bio_info *alloc_bio_info(struct mapped_device *md)
443{
444 return mempool_alloc(md->io_pool, GFP_ATOMIC);
445}
446
447static void free_bio_info(struct dm_rq_clone_bio_info *info)
448{
449 mempool_free(info, info->tio->md->io_pool);
450}
451
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800452static void start_io_acct(struct dm_io *io)
453{
454 struct mapped_device *md = io->md;
Tejun Heoc9959052008-08-25 19:47:21 +0900455 int cpu;
Nikanth Karthikesana9327ca2009-09-11 09:18:54 +0200456 int rw = bio_data_dir(io->bio);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800457
458 io->start_time = jiffies;
459
Tejun Heo074a7ac2008-08-25 19:56:14 +0900460 cpu = part_stat_lock();
461 part_round_stats(cpu, &dm_disk(md)->part0);
462 part_stat_unlock();
Nikanth Karthikesana9327ca2009-09-11 09:18:54 +0200463 dm_disk(md)->part0.in_flight[rw] = atomic_inc_return(&md->pending[rw]);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800464}
465
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000466static void end_io_acct(struct dm_io *io)
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800467{
468 struct mapped_device *md = io->md;
469 struct bio *bio = io->bio;
470 unsigned long duration = jiffies - io->start_time;
Tejun Heoc9959052008-08-25 19:47:21 +0900471 int pending, cpu;
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800472 int rw = bio_data_dir(bio);
473
Tejun Heo074a7ac2008-08-25 19:56:14 +0900474 cpu = part_stat_lock();
475 part_round_stats(cpu, &dm_disk(md)->part0);
476 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
477 part_stat_unlock();
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800478
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100479 /*
480 * After this is decremented the bio must not be touched if it is
481 * a barrier.
482 */
Nikanth Karthikesana9327ca2009-09-11 09:18:54 +0200483 dm_disk(md)->part0.in_flight[rw] = pending =
484 atomic_dec_return(&md->pending[rw]);
485 pending += atomic_read(&md->pending[rw^0x1]);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800486
Mikulas Patockad221d2e2008-11-13 23:39:10 +0000487 /* nudge anyone waiting on suspend queue */
488 if (!pending)
489 wake_up(&md->wait);
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800490}
491
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492/*
493 * Add the bio to the list of deferred io.
494 */
Mikulas Patocka92c63902009-04-09 00:27:15 +0100495static void queue_io(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700497 down_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
Mikulas Patocka022c2612009-04-02 19:55:39 +0100499 spin_lock_irq(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 bio_list_add(&md->deferred, bio);
Mikulas Patocka022c2612009-04-02 19:55:39 +0100501 spin_unlock_irq(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
Mikulas Patocka92c63902009-04-09 00:27:15 +0100503 if (!test_and_set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags))
504 queue_work(md->wq, &md->work);
505
Alasdair G Kergon2ca33102005-07-28 21:16:00 -0700506 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507}
508
509/*
510 * Everyone (including functions in this file), should use this
511 * function to access the md->map field, and make sure they call
512 * dm_table_put() when finished.
513 */
514struct dm_table *dm_get_table(struct mapped_device *md)
515{
516 struct dm_table *t;
Kiyoshi Ueda523d9292009-06-22 10:12:37 +0100517 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518
Kiyoshi Ueda523d9292009-06-22 10:12:37 +0100519 read_lock_irqsave(&md->map_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 t = md->map;
521 if (t)
522 dm_table_get(t);
Kiyoshi Ueda523d9292009-06-22 10:12:37 +0100523 read_unlock_irqrestore(&md->map_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524
525 return t;
526}
527
Darrick J. Wong3ac51e72006-03-27 01:17:54 -0800528/*
529 * Get the geometry associated with a dm device
530 */
531int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
532{
533 *geo = md->geometry;
534
535 return 0;
536}
537
538/*
539 * Set the geometry of a device.
540 */
541int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
542{
543 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
544
545 if (geo->start > sz) {
546 DMWARN("Start sector is beyond the geometry limits.");
547 return -EINVAL;
548 }
549
550 md->geometry = *geo;
551
552 return 0;
553}
554
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555/*-----------------------------------------------------------------
556 * CRUD START:
557 * A more elegant soln is in the works that uses the queue
558 * merge fn, unfortunately there are a couple of changes to
559 * the block layer that I want to make for this. So in the
560 * interests of getting something for people to use I give
561 * you this clearly demarcated crap.
562 *---------------------------------------------------------------*/
563
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800564static int __noflush_suspending(struct mapped_device *md)
565{
566 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
567}
568
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569/*
570 * Decrements the number of outstanding ios that a bio has been
571 * cloned into, completing the original io if necc.
572 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800573static void dec_pending(struct dm_io *io, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574{
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800575 unsigned long flags;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000576 int io_error;
577 struct bio *bio;
578 struct mapped_device *md = io->md;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800579
580 /* Push-back supersedes any I/O errors */
Milan Brozb35f8ca2009-03-16 17:44:36 +0000581 if (error && !(io->error > 0 && __noflush_suspending(md)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 io->error = error;
583
584 if (atomic_dec_and_test(&io->io_count)) {
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800585 if (io->error == DM_ENDIO_REQUEUE) {
586 /*
587 * Target requested pushing back the I/O.
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800588 */
Mikulas Patocka022c2612009-04-02 19:55:39 +0100589 spin_lock_irqsave(&md->deferred_lock, flags);
Mikulas Patocka2761e952009-06-22 10:12:18 +0100590 if (__noflush_suspending(md)) {
Jens Axboe1f98a132009-09-11 14:32:04 +0200591 if (!bio_rw_flagged(io->bio, BIO_RW_BARRIER))
Mikulas Patocka2761e952009-06-22 10:12:18 +0100592 bio_list_add_head(&md->deferred,
593 io->bio);
594 } else
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800595 /* noflush suspend was interrupted. */
596 io->error = -EIO;
Mikulas Patocka022c2612009-04-02 19:55:39 +0100597 spin_unlock_irqrestore(&md->deferred_lock, flags);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800598 }
599
Milan Brozb35f8ca2009-03-16 17:44:36 +0000600 io_error = io->error;
601 bio = io->bio;
Jens Axboe2056a782006-03-23 20:00:26 +0100602
Jens Axboe1f98a132009-09-11 14:32:04 +0200603 if (bio_rw_flagged(bio, BIO_RW_BARRIER)) {
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100604 /*
605 * There can be just one barrier request so we use
606 * a per-device variable for error reporting.
607 * Note that you can't touch the bio after end_io_acct
608 */
Mikulas Patockafdb95722009-06-22 10:12:19 +0100609 if (!md->barrier_error && io_error != -EOPNOTSUPP)
Mikulas Patocka5aa27812009-06-22 10:12:18 +0100610 md->barrier_error = io_error;
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100611 end_io_acct(io);
612 } else {
613 end_io_acct(io);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000614
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100615 if (io_error != DM_ENDIO_REQUEUE) {
616 trace_block_bio_complete(md->queue, bio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000617
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100618 bio_endio(bio, io_error);
619 }
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800620 }
Mikulas Patockaaf7e4662009-04-09 00:27:16 +0100621
622 free_io(md, io);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 }
624}
625
NeilBrown6712ecf2007-09-27 12:47:43 +0200626static void clone_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627{
628 int r = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100629 struct dm_target_io *tio = bio->bi_private;
Milan Brozb35f8ca2009-03-16 17:44:36 +0000630 struct dm_io *io = tio->io;
Stefan Bader9faf4002006-10-03 01:15:41 -0700631 struct mapped_device *md = tio->io->md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 dm_endio_fn endio = tio->ti->type->end_io;
633
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
635 error = -EIO;
636
637 if (endio) {
638 r = endio(tio->ti, bio, error, &tio->info);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800639 if (r < 0 || r == DM_ENDIO_REQUEUE)
640 /*
641 * error and requeue request are handled
642 * in dec_pending().
643 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 error = r;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800645 else if (r == DM_ENDIO_INCOMPLETE)
646 /* The target will handle the io */
NeilBrown6712ecf2007-09-27 12:47:43 +0200647 return;
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800648 else if (r) {
649 DMWARN("unimplemented target endio return value: %d", r);
650 BUG();
651 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652 }
653
Stefan Bader9faf4002006-10-03 01:15:41 -0700654 /*
655 * Store md for cleanup instead of tio which is about to get freed.
656 */
657 bio->bi_private = md->bs;
658
Stefan Bader9faf4002006-10-03 01:15:41 -0700659 free_tio(md, tio);
Milan Brozb35f8ca2009-03-16 17:44:36 +0000660 bio_put(bio);
661 dec_pending(io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662}
663
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100664/*
665 * Partial completion handling for request-based dm
666 */
667static void end_clone_bio(struct bio *clone, int error)
668{
669 struct dm_rq_clone_bio_info *info = clone->bi_private;
670 struct dm_rq_target_io *tio = info->tio;
671 struct bio *bio = info->orig;
672 unsigned int nr_bytes = info->orig->bi_size;
673
674 bio_put(clone);
675
676 if (tio->error)
677 /*
678 * An error has already been detected on the request.
679 * Once error occurred, just let clone->end_io() handle
680 * the remainder.
681 */
682 return;
683 else if (error) {
684 /*
685 * Don't notice the error to the upper layer yet.
686 * The error handling decision is made by the target driver,
687 * when the request is completed.
688 */
689 tio->error = error;
690 return;
691 }
692
693 /*
694 * I/O for the bio successfully completed.
695 * Notice the data completion to the upper layer.
696 */
697
698 /*
699 * bios are processed from the head of the list.
700 * So the completing bio should always be rq->bio.
701 * If it's not, something wrong is happening.
702 */
703 if (tio->orig->bio != bio)
704 DMERR("bio completion is going in the middle of the request");
705
706 /*
707 * Update the original request.
708 * Do not use blk_end_request() here, because it may complete
709 * the original request before the clone, and break the ordering.
710 */
711 blk_update_request(tio->orig, 0, nr_bytes);
712}
713
714/*
715 * Don't touch any member of the md after calling this function because
716 * the md may be freed in dm_put() at the end of this function.
717 * Or do dm_get() before calling this function and dm_put() later.
718 */
719static void rq_completed(struct mapped_device *md, int run_queue)
720{
721 int wakeup_waiters = 0;
722 struct request_queue *q = md->queue;
723 unsigned long flags;
724
725 spin_lock_irqsave(q->queue_lock, flags);
726 if (!queue_in_flight(q))
727 wakeup_waiters = 1;
728 spin_unlock_irqrestore(q->queue_lock, flags);
729
730 /* nudge anyone waiting on suspend queue */
731 if (wakeup_waiters)
732 wake_up(&md->wait);
733
734 if (run_queue)
735 blk_run_queue(q);
736
737 /*
738 * dm_put() must be at the end of this function. See the comment above
739 */
740 dm_put(md);
741}
742
Kiyoshi Uedaa77e28c2009-09-04 20:40:16 +0100743static void free_rq_clone(struct request *clone)
744{
745 struct dm_rq_target_io *tio = clone->end_io_data;
746
747 blk_rq_unprep_clone(clone);
748 free_rq_tio(tio);
749}
750
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100751static void dm_unprep_request(struct request *rq)
752{
753 struct request *clone = rq->special;
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100754
755 rq->special = NULL;
756 rq->cmd_flags &= ~REQ_DONTPREP;
757
Kiyoshi Uedaa77e28c2009-09-04 20:40:16 +0100758 free_rq_clone(clone);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100759}
760
761/*
762 * Requeue the original request of a clone.
763 */
764void dm_requeue_unmapped_request(struct request *clone)
765{
766 struct dm_rq_target_io *tio = clone->end_io_data;
767 struct mapped_device *md = tio->md;
768 struct request *rq = tio->orig;
769 struct request_queue *q = rq->q;
770 unsigned long flags;
771
772 dm_unprep_request(rq);
773
774 spin_lock_irqsave(q->queue_lock, flags);
775 if (elv_queue_empty(q))
776 blk_plug_device(q);
777 blk_requeue_request(q, rq);
778 spin_unlock_irqrestore(q->queue_lock, flags);
779
780 rq_completed(md, 0);
781}
782EXPORT_SYMBOL_GPL(dm_requeue_unmapped_request);
783
784static void __stop_queue(struct request_queue *q)
785{
786 blk_stop_queue(q);
787}
788
789static void stop_queue(struct request_queue *q)
790{
791 unsigned long flags;
792
793 spin_lock_irqsave(q->queue_lock, flags);
794 __stop_queue(q);
795 spin_unlock_irqrestore(q->queue_lock, flags);
796}
797
798static void __start_queue(struct request_queue *q)
799{
800 if (blk_queue_stopped(q))
801 blk_start_queue(q);
802}
803
804static void start_queue(struct request_queue *q)
805{
806 unsigned long flags;
807
808 spin_lock_irqsave(q->queue_lock, flags);
809 __start_queue(q);
810 spin_unlock_irqrestore(q->queue_lock, flags);
811}
812
813/*
814 * Complete the clone and the original request.
815 * Must be called without queue lock.
816 */
817static void dm_end_request(struct request *clone, int error)
818{
819 struct dm_rq_target_io *tio = clone->end_io_data;
820 struct mapped_device *md = tio->md;
821 struct request *rq = tio->orig;
822
823 if (blk_pc_request(rq)) {
824 rq->errors = clone->errors;
825 rq->resid_len = clone->resid_len;
826
827 if (rq->sense)
828 /*
829 * We are using the sense buffer of the original
830 * request.
831 * So setting the length of the sense data is enough.
832 */
833 rq->sense_len = clone->sense_len;
834 }
835
Kiyoshi Uedaa77e28c2009-09-04 20:40:16 +0100836 free_rq_clone(clone);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +0100837
838 blk_end_request_all(rq, error);
839
840 rq_completed(md, 1);
841}
842
843/*
844 * Request completion handler for request-based dm
845 */
846static void dm_softirq_done(struct request *rq)
847{
848 struct request *clone = rq->completion_data;
849 struct dm_rq_target_io *tio = clone->end_io_data;
850 dm_request_endio_fn rq_end_io = tio->ti->type->rq_end_io;
851 int error = tio->error;
852
853 if (!(rq->cmd_flags & REQ_FAILED) && rq_end_io)
854 error = rq_end_io(tio->ti, clone, error, &tio->info);
855
856 if (error <= 0)
857 /* The target wants to complete the I/O */
858 dm_end_request(clone, error);
859 else if (error == DM_ENDIO_INCOMPLETE)
860 /* The target will handle the I/O */
861 return;
862 else if (error == DM_ENDIO_REQUEUE)
863 /* The target wants to requeue the I/O */
864 dm_requeue_unmapped_request(clone);
865 else {
866 DMWARN("unimplemented target endio return value: %d", error);
867 BUG();
868 }
869}
870
871/*
872 * Complete the clone and the original request with the error status
873 * through softirq context.
874 */
875static void dm_complete_request(struct request *clone, int error)
876{
877 struct dm_rq_target_io *tio = clone->end_io_data;
878 struct request *rq = tio->orig;
879
880 tio->error = error;
881 rq->completion_data = clone;
882 blk_complete_request(rq);
883}
884
885/*
886 * Complete the not-mapped clone and the original request with the error status
887 * through softirq context.
888 * Target's rq_end_io() function isn't called.
889 * This may be used when the target's map_rq() function fails.
890 */
891void dm_kill_unmapped_request(struct request *clone, int error)
892{
893 struct dm_rq_target_io *tio = clone->end_io_data;
894 struct request *rq = tio->orig;
895
896 rq->cmd_flags |= REQ_FAILED;
897 dm_complete_request(clone, error);
898}
899EXPORT_SYMBOL_GPL(dm_kill_unmapped_request);
900
901/*
902 * Called with the queue lock held
903 */
904static void end_clone_request(struct request *clone, int error)
905{
906 /*
907 * For just cleaning up the information of the queue in which
908 * the clone was dispatched.
909 * The clone is *NOT* freed actually here because it is alloced from
910 * dm own mempool and REQ_ALLOCED isn't set in clone->cmd_flags.
911 */
912 __blk_put_request(clone->q, clone);
913
914 /*
915 * Actual request completion is done in a softirq context which doesn't
916 * hold the queue lock. Otherwise, deadlock could occur because:
917 * - another request may be submitted by the upper level driver
918 * of the stacking during the completion
919 * - the submission which requires queue lock may be done
920 * against this queue
921 */
922 dm_complete_request(clone, error);
923}
924
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925static sector_t max_io_len(struct mapped_device *md,
926 sector_t sector, struct dm_target *ti)
927{
928 sector_t offset = sector - ti->begin;
929 sector_t len = ti->len - offset;
930
931 /*
932 * Does the target need to split even further ?
933 */
934 if (ti->split_io) {
935 sector_t boundary;
936 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
937 - offset;
938 if (len > boundary)
939 len = boundary;
940 }
941
942 return len;
943}
944
945static void __map_bio(struct dm_target *ti, struct bio *clone,
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100946 struct dm_target_io *tio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947{
948 int r;
Jens Axboe2056a782006-03-23 20:00:26 +0100949 sector_t sector;
Stefan Bader9faf4002006-10-03 01:15:41 -0700950 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952 clone->bi_end_io = clone_endio;
953 clone->bi_private = tio;
954
955 /*
956 * Map the clone. If r == 0 we don't need to do
957 * anything, the target has assumed ownership of
958 * this io.
959 */
960 atomic_inc(&tio->io->io_count);
Jens Axboe2056a782006-03-23 20:00:26 +0100961 sector = clone->bi_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 r = ti->type->map(ti, clone, &tio->info);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800963 if (r == DM_MAPIO_REMAPPED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 /* the bio has been remapped so dispatch it */
Jens Axboe2056a782006-03-23 20:00:26 +0100965
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100966 trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
Alan D. Brunelle22a7c312009-05-04 16:35:08 -0400967 tio->io->bio->bi_bdev->bd_dev, sector);
Jens Axboe2056a782006-03-23 20:00:26 +0100968
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 generic_make_request(clone);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -0800970 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
971 /* error the io and bail out, or requeue it if needed */
Stefan Bader9faf4002006-10-03 01:15:41 -0700972 md = tio->io->md;
973 dec_pending(tio->io, r);
974 /*
975 * Store bio_set for cleanup.
976 */
977 clone->bi_private = md->bs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 bio_put(clone);
Stefan Bader9faf4002006-10-03 01:15:41 -0700979 free_tio(md, tio);
Kiyoshi Ueda45cbcd72006-12-08 02:41:05 -0800980 } else if (r) {
981 DMWARN("unimplemented target map return value: %d", r);
982 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 }
984}
985
986struct clone_info {
987 struct mapped_device *md;
988 struct dm_table *map;
989 struct bio *bio;
990 struct dm_io *io;
991 sector_t sector;
992 sector_t sector_count;
993 unsigned short idx;
994};
995
Peter Osterlund36763472005-09-06 15:16:42 -0700996static void dm_bio_destructor(struct bio *bio)
997{
Stefan Bader9faf4002006-10-03 01:15:41 -0700998 struct bio_set *bs = bio->bi_private;
999
1000 bio_free(bio, bs);
Peter Osterlund36763472005-09-06 15:16:42 -07001001}
1002
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003/*
1004 * Creates a little bio that is just does part of a bvec.
1005 */
1006static struct bio *split_bvec(struct bio *bio, sector_t sector,
1007 unsigned short idx, unsigned int offset,
Stefan Bader9faf4002006-10-03 01:15:41 -07001008 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009{
1010 struct bio *clone;
1011 struct bio_vec *bv = bio->bi_io_vec + idx;
1012
Stefan Bader9faf4002006-10-03 01:15:41 -07001013 clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
Peter Osterlund36763472005-09-06 15:16:42 -07001014 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 *clone->bi_io_vec = *bv;
1016
1017 clone->bi_sector = sector;
1018 clone->bi_bdev = bio->bi_bdev;
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001019 clone->bi_rw = bio->bi_rw & ~(1 << BIO_RW_BARRIER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 clone->bi_vcnt = 1;
1021 clone->bi_size = to_bytes(len);
1022 clone->bi_io_vec->bv_offset = offset;
1023 clone->bi_io_vec->bv_len = clone->bi_size;
Martin K. Petersenf3e1d262008-10-21 17:45:04 +01001024 clone->bi_flags |= 1 << BIO_CLONED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025
Martin K. Petersen9c470082009-04-09 00:27:12 +01001026 if (bio_integrity(bio)) {
Martin K. Petersen7878cba2009-06-26 15:37:49 +02001027 bio_integrity_clone(clone, bio, GFP_NOIO, bs);
Martin K. Petersen9c470082009-04-09 00:27:12 +01001028 bio_integrity_trim(clone,
1029 bio_sector_offset(bio, idx, offset), len);
1030 }
1031
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032 return clone;
1033}
1034
1035/*
1036 * Creates a bio that consists of range of complete bvecs.
1037 */
1038static struct bio *clone_bio(struct bio *bio, sector_t sector,
1039 unsigned short idx, unsigned short bv_count,
Stefan Bader9faf4002006-10-03 01:15:41 -07001040 unsigned int len, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041{
1042 struct bio *clone;
1043
Stefan Bader9faf4002006-10-03 01:15:41 -07001044 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
1045 __bio_clone(clone, bio);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001046 clone->bi_rw &= ~(1 << BIO_RW_BARRIER);
Stefan Bader9faf4002006-10-03 01:15:41 -07001047 clone->bi_destructor = dm_bio_destructor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048 clone->bi_sector = sector;
1049 clone->bi_idx = idx;
1050 clone->bi_vcnt = idx + bv_count;
1051 clone->bi_size = to_bytes(len);
1052 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
1053
Martin K. Petersen9c470082009-04-09 00:27:12 +01001054 if (bio_integrity(bio)) {
Martin K. Petersen7878cba2009-06-26 15:37:49 +02001055 bio_integrity_clone(clone, bio, GFP_NOIO, bs);
Martin K. Petersen9c470082009-04-09 00:27:12 +01001056
1057 if (idx != bio->bi_idx || clone->bi_size < bio->bi_size)
1058 bio_integrity_trim(clone,
1059 bio_sector_offset(bio, idx, 0), len);
1060 }
1061
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 return clone;
1063}
1064
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001065static struct dm_target_io *alloc_tio(struct clone_info *ci,
1066 struct dm_target *ti)
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001067{
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001068 struct dm_target_io *tio = mempool_alloc(ci->md->tio_pool, GFP_NOIO);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001069
1070 tio->io = ci->io;
1071 tio->ti = ti;
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001072 memset(&tio->info, 0, sizeof(tio->info));
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001073
1074 return tio;
1075}
1076
1077static void __flush_target(struct clone_info *ci, struct dm_target *ti,
1078 unsigned flush_nr)
1079{
1080 struct dm_target_io *tio = alloc_tio(ci, ti);
1081 struct bio *clone;
1082
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001083 tio->info.flush_request = flush_nr;
1084
1085 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
1086 __bio_clone(clone, ci->bio);
1087 clone->bi_destructor = dm_bio_destructor;
1088
1089 __map_bio(ti, clone, tio);
1090}
1091
1092static int __clone_and_map_empty_barrier(struct clone_info *ci)
1093{
1094 unsigned target_nr = 0, flush_nr;
1095 struct dm_target *ti;
1096
1097 while ((ti = dm_table_get_target(ci->map, target_nr++)))
1098 for (flush_nr = 0; flush_nr < ti->num_flush_requests;
1099 flush_nr++)
1100 __flush_target(ci, ti, flush_nr);
1101
1102 ci->sector_count = 0;
1103
1104 return 0;
1105}
1106
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001107static int __clone_and_map(struct clone_info *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108{
1109 struct bio *clone, *bio = ci->bio;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001110 struct dm_target *ti;
1111 sector_t len = 0, max;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001112 struct dm_target_io *tio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001114 if (unlikely(bio_empty_barrier(bio)))
1115 return __clone_and_map_empty_barrier(ci);
1116
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001117 ti = dm_table_find_target(ci->map, ci->sector);
1118 if (!dm_target_is_valid(ti))
1119 return -EIO;
1120
1121 max = max_io_len(ci->md, ci->sector, ti);
1122
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123 /*
1124 * Allocate a target io object.
1125 */
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001126 tio = alloc_tio(ci, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127
1128 if (ci->sector_count <= max) {
1129 /*
1130 * Optimise for the simple case where we can do all of
1131 * the remaining io with a single clone.
1132 */
1133 clone = clone_bio(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -07001134 bio->bi_vcnt - ci->idx, ci->sector_count,
1135 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136 __map_bio(ti, clone, tio);
1137 ci->sector_count = 0;
1138
1139 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
1140 /*
1141 * There are some bvecs that don't span targets.
1142 * Do as many of these as possible.
1143 */
1144 int i;
1145 sector_t remaining = max;
1146 sector_t bv_len;
1147
1148 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
1149 bv_len = to_sector(bio->bi_io_vec[i].bv_len);
1150
1151 if (bv_len > remaining)
1152 break;
1153
1154 remaining -= bv_len;
1155 len += bv_len;
1156 }
1157
Stefan Bader9faf4002006-10-03 01:15:41 -07001158 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
1159 ci->md->bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160 __map_bio(ti, clone, tio);
1161
1162 ci->sector += len;
1163 ci->sector_count -= len;
1164 ci->idx = i;
1165
1166 } else {
1167 /*
Alasdair G Kergond2044a92006-03-22 00:07:42 -08001168 * Handle a bvec that must be split between two or more targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001169 */
1170 struct bio_vec *bv = bio->bi_io_vec + ci->idx;
Alasdair G Kergond2044a92006-03-22 00:07:42 -08001171 sector_t remaining = to_sector(bv->bv_len);
1172 unsigned int offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173
Alasdair G Kergond2044a92006-03-22 00:07:42 -08001174 do {
1175 if (offset) {
1176 ti = dm_table_find_target(ci->map, ci->sector);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001177 if (!dm_target_is_valid(ti))
1178 return -EIO;
1179
Alasdair G Kergond2044a92006-03-22 00:07:42 -08001180 max = max_io_len(ci->md, ci->sector, ti);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181
Alasdair G Kergon9015df22009-06-22 10:12:21 +01001182 tio = alloc_tio(ci, ti);
Alasdair G Kergond2044a92006-03-22 00:07:42 -08001183 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184
Alasdair G Kergond2044a92006-03-22 00:07:42 -08001185 len = min(remaining, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186
Alasdair G Kergond2044a92006-03-22 00:07:42 -08001187 clone = split_bvec(bio, ci->sector, ci->idx,
Stefan Bader9faf4002006-10-03 01:15:41 -07001188 bv->bv_offset + offset, len,
1189 ci->md->bs);
Alasdair G Kergond2044a92006-03-22 00:07:42 -08001190
1191 __map_bio(ti, clone, tio);
1192
1193 ci->sector += len;
1194 ci->sector_count -= len;
1195 offset += to_bytes(len);
1196 } while (remaining -= len);
1197
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198 ci->idx++;
1199 }
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001200
1201 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202}
1203
1204/*
Mikulas Patocka8a53c282009-04-02 19:55:37 +01001205 * Split the bio into several clones and submit it to targets.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206 */
Mikulas Patockaf0b9a452009-04-02 19:55:38 +01001207static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208{
1209 struct clone_info ci;
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001210 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211
1212 ci.map = dm_get_table(md);
Mikulas Patockaf0b9a452009-04-02 19:55:38 +01001213 if (unlikely(!ci.map)) {
Jens Axboe1f98a132009-09-11 14:32:04 +02001214 if (!bio_rw_flagged(bio, BIO_RW_BARRIER))
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001215 bio_io_error(bio);
1216 else
Mikulas Patocka5aa27812009-06-22 10:12:18 +01001217 if (!md->barrier_error)
1218 md->barrier_error = -EIO;
Mikulas Patockaf0b9a452009-04-02 19:55:38 +01001219 return;
1220 }
Mikulas Patocka692d0eb2009-04-09 00:27:13 +01001221
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222 ci.md = md;
1223 ci.bio = bio;
1224 ci.io = alloc_io(md);
1225 ci.io->error = 0;
1226 atomic_set(&ci.io->io_count, 1);
1227 ci.io->bio = bio;
1228 ci.io->md = md;
1229 ci.sector = bio->bi_sector;
1230 ci.sector_count = bio_sectors(bio);
Mikulas Patockaf9ab94c2009-06-22 10:12:20 +01001231 if (unlikely(bio_empty_barrier(bio)))
1232 ci.sector_count = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233 ci.idx = bio->bi_idx;
1234
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -08001235 start_io_acct(ci.io);
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001236 while (ci.sector_count && !error)
1237 error = __clone_and_map(&ci);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238
1239 /* drop the extra reference count */
Jun'ichi Nomura512875b2007-12-13 14:15:25 +00001240 dec_pending(ci.io, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 dm_table_put(ci.map);
1242}
1243/*-----------------------------------------------------------------
1244 * CRUD END
1245 *---------------------------------------------------------------*/
1246
Milan Brozf6fccb12008-07-21 12:00:37 +01001247static int dm_merge_bvec(struct request_queue *q,
1248 struct bvec_merge_data *bvm,
1249 struct bio_vec *biovec)
1250{
1251 struct mapped_device *md = q->queuedata;
1252 struct dm_table *map = dm_get_table(md);
1253 struct dm_target *ti;
1254 sector_t max_sectors;
Mikulas Patocka50371082008-10-01 14:39:17 +01001255 int max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +01001256
1257 if (unlikely(!map))
Mikulas Patocka50371082008-10-01 14:39:17 +01001258 goto out;
Milan Brozf6fccb12008-07-21 12:00:37 +01001259
1260 ti = dm_table_find_target(map, bvm->bi_sector);
Mikulas Patockab01cd5a2008-10-01 14:39:24 +01001261 if (!dm_target_is_valid(ti))
1262 goto out_table;
Milan Brozf6fccb12008-07-21 12:00:37 +01001263
1264 /*
1265 * Find maximum amount of I/O that won't need splitting
1266 */
1267 max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
1268 (sector_t) BIO_MAX_SECTORS);
1269 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1270 if (max_size < 0)
1271 max_size = 0;
1272
1273 /*
1274 * merge_bvec_fn() returns number of bytes
1275 * it can accept at this offset
1276 * max is precomputed maximal io size
1277 */
1278 if (max_size && ti->type->merge)
1279 max_size = ti->type->merge(ti, bvm, biovec, max_size);
Mikulas Patocka8cbeb672009-06-22 10:12:14 +01001280 /*
1281 * If the target doesn't support merge method and some of the devices
1282 * provided their merge_bvec method (we know this by looking at
1283 * queue_max_hw_sectors), then we can't allow bios with multiple vector
1284 * entries. So always set max_size to 0, and the code below allows
1285 * just one page.
1286 */
1287 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
1288
1289 max_size = 0;
Milan Brozf6fccb12008-07-21 12:00:37 +01001290
Mikulas Patockab01cd5a2008-10-01 14:39:24 +01001291out_table:
Mikulas Patocka50371082008-10-01 14:39:17 +01001292 dm_table_put(map);
1293
1294out:
Milan Brozf6fccb12008-07-21 12:00:37 +01001295 /*
1296 * Always allow an entire first page
1297 */
1298 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1299 max_size = biovec->bv_len;
1300
Milan Brozf6fccb12008-07-21 12:00:37 +01001301 return max_size;
1302}
1303
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304/*
1305 * The request function that just remaps the bio built up by
1306 * dm_merge_bvec.
1307 */
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001308static int _dm_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309{
Kevin Corry12f03a42006-02-01 03:04:52 -08001310 int rw = bio_data_dir(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 struct mapped_device *md = q->queuedata;
Tejun Heoc9959052008-08-25 19:47:21 +09001312 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001314 down_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315
Tejun Heo074a7ac2008-08-25 19:56:14 +09001316 cpu = part_stat_lock();
1317 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
1318 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
1319 part_stat_unlock();
Kevin Corry12f03a42006-02-01 03:04:52 -08001320
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321 /*
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001322 * If we're suspended or the thread is processing barriers
1323 * we have to queue this io for later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 */
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01001325 if (unlikely(test_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags)) ||
Jens Axboe1f98a132009-09-11 14:32:04 +02001326 unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER))) {
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001327 up_read(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328
Alasdair G Kergon54d9a1b2009-04-09 00:27:14 +01001329 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) &&
1330 bio_rw(bio) == READA) {
1331 bio_io_error(bio);
1332 return 0;
1333 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001334
Mikulas Patocka92c63902009-04-09 00:27:15 +01001335 queue_io(md, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336
Mikulas Patocka92c63902009-04-09 00:27:15 +01001337 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338 }
1339
Mikulas Patockaf0b9a452009-04-02 19:55:38 +01001340 __split_and_process_bio(md, bio);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001341 up_read(&md->io_lock);
Mikulas Patockaf0b9a452009-04-02 19:55:38 +01001342 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343}
1344
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001345static int dm_make_request(struct request_queue *q, struct bio *bio)
1346{
1347 struct mapped_device *md = q->queuedata;
1348
Jens Axboe1f98a132009-09-11 14:32:04 +02001349 if (unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER))) {
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001350 bio_endio(bio, -EOPNOTSUPP);
1351 return 0;
1352 }
1353
1354 return md->saved_make_request_fn(q, bio); /* call __make_request() */
1355}
1356
1357static int dm_request_based(struct mapped_device *md)
1358{
1359 return blk_queue_stackable(md->queue);
1360}
1361
1362static int dm_request(struct request_queue *q, struct bio *bio)
1363{
1364 struct mapped_device *md = q->queuedata;
1365
1366 if (dm_request_based(md))
1367 return dm_make_request(q, bio);
1368
1369 return _dm_request(q, bio);
1370}
1371
1372void dm_dispatch_request(struct request *rq)
1373{
1374 int r;
1375
1376 if (blk_queue_io_stat(rq->q))
1377 rq->cmd_flags |= REQ_IO_STAT;
1378
1379 rq->start_time = jiffies;
1380 r = blk_insert_cloned_request(rq->q, rq);
1381 if (r)
1382 dm_complete_request(rq, r);
1383}
1384EXPORT_SYMBOL_GPL(dm_dispatch_request);
1385
1386static void dm_rq_bio_destructor(struct bio *bio)
1387{
1388 struct dm_rq_clone_bio_info *info = bio->bi_private;
1389 struct mapped_device *md = info->tio->md;
1390
1391 free_bio_info(info);
1392 bio_free(bio, md->bs);
1393}
1394
1395static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
1396 void *data)
1397{
1398 struct dm_rq_target_io *tio = data;
1399 struct mapped_device *md = tio->md;
1400 struct dm_rq_clone_bio_info *info = alloc_bio_info(md);
1401
1402 if (!info)
1403 return -ENOMEM;
1404
1405 info->orig = bio_orig;
1406 info->tio = tio;
1407 bio->bi_end_io = end_clone_bio;
1408 bio->bi_private = info;
1409 bio->bi_destructor = dm_rq_bio_destructor;
1410
1411 return 0;
1412}
1413
1414static int setup_clone(struct request *clone, struct request *rq,
1415 struct dm_rq_target_io *tio)
1416{
1417 int r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,
1418 dm_rq_bio_constructor, tio);
1419
1420 if (r)
1421 return r;
1422
1423 clone->cmd = rq->cmd;
1424 clone->cmd_len = rq->cmd_len;
1425 clone->sense = rq->sense;
1426 clone->buffer = rq->buffer;
1427 clone->end_io = end_clone_request;
1428 clone->end_io_data = tio;
1429
1430 return 0;
1431}
1432
1433static int dm_rq_flush_suspending(struct mapped_device *md)
1434{
1435 return !md->suspend_rq.special;
1436}
1437
1438/*
1439 * Called with the queue lock held.
1440 */
1441static int dm_prep_fn(struct request_queue *q, struct request *rq)
1442{
1443 struct mapped_device *md = q->queuedata;
1444 struct dm_rq_target_io *tio;
1445 struct request *clone;
1446
1447 if (unlikely(rq == &md->suspend_rq)) {
1448 if (dm_rq_flush_suspending(md))
1449 return BLKPREP_OK;
1450 else
1451 /* The flush suspend was interrupted */
1452 return BLKPREP_KILL;
1453 }
1454
1455 if (unlikely(rq->special)) {
1456 DMWARN("Already has something in rq->special.");
1457 return BLKPREP_KILL;
1458 }
1459
1460 tio = alloc_rq_tio(md); /* Only one for each original request */
1461 if (!tio)
1462 /* -ENOMEM */
1463 return BLKPREP_DEFER;
1464
1465 tio->md = md;
1466 tio->ti = NULL;
1467 tio->orig = rq;
1468 tio->error = 0;
1469 memset(&tio->info, 0, sizeof(tio->info));
1470
1471 clone = &tio->clone;
1472 if (setup_clone(clone, rq, tio)) {
1473 /* -ENOMEM */
1474 free_rq_tio(tio);
1475 return BLKPREP_DEFER;
1476 }
1477
1478 rq->special = clone;
1479 rq->cmd_flags |= REQ_DONTPREP;
1480
1481 return BLKPREP_OK;
1482}
1483
1484static void map_request(struct dm_target *ti, struct request *rq,
1485 struct mapped_device *md)
1486{
1487 int r;
1488 struct request *clone = rq->special;
1489 struct dm_rq_target_io *tio = clone->end_io_data;
1490
1491 /*
1492 * Hold the md reference here for the in-flight I/O.
1493 * We can't rely on the reference count by device opener,
1494 * because the device may be closed during the request completion
1495 * when all bios are completed.
1496 * See the comment in rq_completed() too.
1497 */
1498 dm_get(md);
1499
1500 tio->ti = ti;
1501 r = ti->type->map_rq(ti, clone, &tio->info);
1502 switch (r) {
1503 case DM_MAPIO_SUBMITTED:
1504 /* The target has taken the I/O to submit by itself later */
1505 break;
1506 case DM_MAPIO_REMAPPED:
1507 /* The target has remapped the I/O so dispatch it */
1508 dm_dispatch_request(clone);
1509 break;
1510 case DM_MAPIO_REQUEUE:
1511 /* The target wants to requeue the I/O */
1512 dm_requeue_unmapped_request(clone);
1513 break;
1514 default:
1515 if (r > 0) {
1516 DMWARN("unimplemented target map return value: %d", r);
1517 BUG();
1518 }
1519
1520 /* The target wants to complete the I/O */
1521 dm_kill_unmapped_request(clone, r);
1522 break;
1523 }
1524}
1525
1526/*
1527 * q->request_fn for request-based dm.
1528 * Called with the queue lock held.
1529 */
1530static void dm_request_fn(struct request_queue *q)
1531{
1532 struct mapped_device *md = q->queuedata;
1533 struct dm_table *map = dm_get_table(md);
1534 struct dm_target *ti;
1535 struct request *rq;
1536
1537 /*
1538 * For noflush suspend, check blk_queue_stopped() to immediately
1539 * quit I/O dispatching.
1540 */
1541 while (!blk_queue_plugged(q) && !blk_queue_stopped(q)) {
1542 rq = blk_peek_request(q);
1543 if (!rq)
1544 goto plug_and_out;
1545
1546 if (unlikely(rq == &md->suspend_rq)) { /* Flush suspend maker */
1547 if (queue_in_flight(q))
1548 /* Not quiet yet. Wait more */
1549 goto plug_and_out;
1550
1551 /* This device should be quiet now */
1552 __stop_queue(q);
1553 blk_start_request(rq);
1554 __blk_end_request_all(rq, 0);
1555 wake_up(&md->wait);
1556 goto out;
1557 }
1558
1559 ti = dm_table_find_target(map, blk_rq_pos(rq));
1560 if (ti->type->busy && ti->type->busy(ti))
1561 goto plug_and_out;
1562
1563 blk_start_request(rq);
1564 spin_unlock(q->queue_lock);
1565 map_request(ti, rq, md);
1566 spin_lock_irq(q->queue_lock);
1567 }
1568
1569 goto out;
1570
1571plug_and_out:
1572 if (!elv_queue_empty(q))
1573 /* Some requests still remain, retry later */
1574 blk_plug_device(q);
1575
1576out:
1577 dm_table_put(map);
1578
1579 return;
1580}
1581
1582int dm_underlying_device_busy(struct request_queue *q)
1583{
1584 return blk_lld_busy(q);
1585}
1586EXPORT_SYMBOL_GPL(dm_underlying_device_busy);
1587
1588static int dm_lld_busy(struct request_queue *q)
1589{
1590 int r;
1591 struct mapped_device *md = q->queuedata;
1592 struct dm_table *map = dm_get_table(md);
1593
1594 if (!map || test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))
1595 r = 1;
1596 else
1597 r = dm_table_any_busy_target(map);
1598
1599 dm_table_put(map);
1600
1601 return r;
1602}
1603
Jens Axboe165125e2007-07-24 09:28:11 +02001604static void dm_unplug_all(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605{
1606 struct mapped_device *md = q->queuedata;
1607 struct dm_table *map = dm_get_table(md);
1608
1609 if (map) {
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001610 if (dm_request_based(md))
1611 generic_unplug_device(q);
1612
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613 dm_table_unplug_all(map);
1614 dm_table_put(map);
1615 }
1616}
1617
1618static int dm_any_congested(void *congested_data, int bdi_bits)
1619{
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001620 int r = bdi_bits;
1621 struct mapped_device *md = congested_data;
1622 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001623
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01001624 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001625 map = dm_get_table(md);
1626 if (map) {
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01001627 /*
1628 * Request-based dm cares about only own queue for
1629 * the query about congestion status of request_queue
1630 */
1631 if (dm_request_based(md))
1632 r = md->queue->backing_dev_info.state &
1633 bdi_bits;
1634 else
1635 r = dm_table_any_congested(map, bdi_bits);
1636
Chandra Seetharaman8a57dfc2008-11-13 23:39:14 +00001637 dm_table_put(map);
1638 }
1639 }
1640
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641 return r;
1642}
1643
1644/*-----------------------------------------------------------------
1645 * An IDR is used to keep track of allocated minor numbers.
1646 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647static DEFINE_IDR(_minor_idr);
1648
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001649static void free_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650{
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001651 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 idr_remove(&_minor_idr, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001653 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654}
1655
1656/*
1657 * See if the device with a specific minor # is free.
1658 */
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001659static int specific_minor(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660{
1661 int r, m;
1662
1663 if (minor >= (1 << MINORBITS))
1664 return -EINVAL;
1665
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001666 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
1667 if (!r)
1668 return -ENOMEM;
1669
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001670 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671
1672 if (idr_find(&_minor_idr, minor)) {
1673 r = -EBUSY;
1674 goto out;
1675 }
1676
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001677 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001678 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001679 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001680
1681 if (m != minor) {
1682 idr_remove(&_minor_idr, m);
1683 r = -EBUSY;
1684 goto out;
1685 }
1686
1687out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001688 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689 return r;
1690}
1691
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001692static int next_free_minor(int *minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693{
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001694 int r, m;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695
Linus Torvalds1da177e2005-04-16 15:20:36 -07001696 r = idr_pre_get(&_minor_idr, GFP_KERNEL);
Jeff Mahoney62f75c22006-06-26 00:27:21 -07001697 if (!r)
1698 return -ENOMEM;
1699
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001700 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001701
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001702 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001703 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705
1706 if (m >= (1 << MINORBITS)) {
1707 idr_remove(&_minor_idr, m);
1708 r = -ENOSPC;
1709 goto out;
1710 }
1711
1712 *minor = m;
1713
1714out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001715 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716 return r;
1717}
1718
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -07001719static const struct block_device_operations dm_blk_dops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001720
Mikulas Patocka53d59142009-04-02 19:55:37 +01001721static void dm_wq_work(struct work_struct *work);
1722
Linus Torvalds1da177e2005-04-16 15:20:36 -07001723/*
1724 * Allocate and initialise a blank device with a given minor.
1725 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001726static struct mapped_device *alloc_dev(int minor)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001727{
1728 int r;
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001729 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001730 void *old_md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001731
1732 if (!md) {
1733 DMWARN("unable to allocate device, out of memory.");
1734 return NULL;
1735 }
1736
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001737 if (!try_module_get(THIS_MODULE))
Milan Broz6ed7ade2008-02-08 02:10:19 +00001738 goto bad_module_get;
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001739
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 /* get a minor number for the dev */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001741 if (minor == DM_ANY_MINOR)
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001742 r = next_free_minor(&minor);
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001743 else
Frederik Deweerdtcf13ab82008-04-24 22:10:59 +01001744 r = specific_minor(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001745 if (r < 0)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001746 goto bad_minor;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001747
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001748 init_rwsem(&md->io_lock);
Daniel Walkere61290a2008-02-08 02:10:08 +00001749 mutex_init(&md->suspend_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01001750 spin_lock_init(&md->deferred_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751 rwlock_init(&md->map_lock);
1752 atomic_set(&md->holders, 1);
Alasdair G Kergon5c6bd752006-06-26 00:27:34 -07001753 atomic_set(&md->open_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001754 atomic_set(&md->event_nr, 0);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001755 atomic_set(&md->uevent_seq, 0);
1756 INIT_LIST_HEAD(&md->uevent_list);
1757 spin_lock_init(&md->uevent_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001759 md->queue = blk_init_queue(dm_request_fn, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001760 if (!md->queue)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001761 goto bad_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001762
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001763 /*
1764 * Request-based dm devices cannot be stacked on top of bio-based dm
1765 * devices. The type of this dm device has not been decided yet,
1766 * although we initialized the queue using blk_init_queue().
1767 * The type is decided at the first table loading time.
1768 * To prevent problematic device stacking, clear the queue flag
1769 * for request stacking support until then.
1770 *
1771 * This queue is new, so no concurrency on the queue_flags.
1772 */
1773 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
1774 md->saved_make_request_fn = md->queue->make_request_fn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001775 md->queue->queuedata = md;
1776 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1777 md->queue->backing_dev_info.congested_data = md;
1778 blk_queue_make_request(md->queue, dm_request);
Jens Axboedaef2652006-01-10 10:48:02 +01001779 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001780 md->queue->unplug_fn = dm_unplug_all;
Milan Brozf6fccb12008-07-21 12:00:37 +01001781 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001782 blk_queue_softirq_done(md->queue, dm_softirq_done);
1783 blk_queue_prep_rq(md->queue, dm_prep_fn);
1784 blk_queue_lld_busy(md->queue, dm_lld_busy);
Stefan Bader9faf4002006-10-03 01:15:41 -07001785
Linus Torvalds1da177e2005-04-16 15:20:36 -07001786 md->disk = alloc_disk(1);
1787 if (!md->disk)
Milan Broz6ed7ade2008-02-08 02:10:19 +00001788 goto bad_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001789
Nikanth Karthikesana9327ca2009-09-11 09:18:54 +02001790 atomic_set(&md->pending[0], 0);
1791 atomic_set(&md->pending[1], 0);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001792 init_waitqueue_head(&md->wait);
Mikulas Patocka53d59142009-04-02 19:55:37 +01001793 INIT_WORK(&md->work, dm_wq_work);
Jeff Mahoneyf0b04112006-06-26 00:27:25 -07001794 init_waitqueue_head(&md->eventq);
1795
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 md->disk->major = _major;
1797 md->disk->first_minor = minor;
1798 md->disk->fops = &dm_blk_dops;
1799 md->disk->queue = md->queue;
1800 md->disk->private_data = md;
1801 sprintf(md->disk->disk_name, "dm-%d", minor);
1802 add_disk(md->disk);
Mike Anderson7e51f252006-03-27 01:17:52 -08001803 format_dev_t(md->name, MKDEV(_major, minor));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001804
Milan Broz304f3f62008-02-08 02:11:17 +00001805 md->wq = create_singlethread_workqueue("kdmflush");
1806 if (!md->wq)
1807 goto bad_thread;
1808
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001809 md->bdev = bdget_disk(md->disk, 0);
1810 if (!md->bdev)
1811 goto bad_bdev;
1812
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001813 /* Populate the mapping, nobody knows we exist yet */
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001814 spin_lock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001815 old_md = idr_replace(&_minor_idr, md, minor);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07001816 spin_unlock(&_minor_lock);
Jeff Mahoneyba61fdd2006-06-26 00:27:21 -07001817
1818 BUG_ON(old_md != MINOR_ALLOCED);
1819
Linus Torvalds1da177e2005-04-16 15:20:36 -07001820 return md;
1821
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001822bad_bdev:
1823 destroy_workqueue(md->wq);
Milan Broz304f3f62008-02-08 02:11:17 +00001824bad_thread:
1825 put_disk(md->disk);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001826bad_disk:
Al Viro1312f402006-03-12 11:02:03 -05001827 blk_cleanup_queue(md->queue);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001828bad_queue:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829 free_minor(minor);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001830bad_minor:
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001831 module_put(THIS_MODULE);
Milan Broz6ed7ade2008-02-08 02:10:19 +00001832bad_module_get:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833 kfree(md);
1834 return NULL;
1835}
1836
Jun'ichi Nomuraae9da832007-10-19 22:38:43 +01001837static void unlock_fs(struct mapped_device *md);
1838
Linus Torvalds1da177e2005-04-16 15:20:36 -07001839static void free_dev(struct mapped_device *md)
1840{
Tejun Heof331c022008-09-03 09:01:48 +02001841 int minor = MINOR(disk_devt(md->disk));
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001842
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001843 unlock_fs(md);
1844 bdput(md->bdev);
Milan Broz304f3f62008-02-08 02:11:17 +00001845 destroy_workqueue(md->wq);
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001846 if (md->tio_pool)
1847 mempool_destroy(md->tio_pool);
1848 if (md->io_pool)
1849 mempool_destroy(md->io_pool);
1850 if (md->bs)
1851 bioset_free(md->bs);
Martin K. Petersen9c470082009-04-09 00:27:12 +01001852 blk_integrity_unregister(md->disk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853 del_gendisk(md->disk);
Jun'ichi Nomura63d94e42006-02-24 13:04:25 -08001854 free_minor(minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07001855
1856 spin_lock(&_minor_lock);
1857 md->disk->private_data = NULL;
1858 spin_unlock(&_minor_lock);
1859
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860 put_disk(md->disk);
Al Viro1312f402006-03-12 11:02:03 -05001861 blk_cleanup_queue(md->queue);
Jeff Mahoney10da4f72006-06-26 00:27:25 -07001862 module_put(THIS_MODULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863 kfree(md);
1864}
1865
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001866static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
1867{
1868 struct dm_md_mempools *p;
1869
1870 if (md->io_pool && md->tio_pool && md->bs)
1871 /* the md already has necessary mempools */
1872 goto out;
1873
1874 p = dm_table_get_md_mempools(t);
1875 BUG_ON(!p || md->io_pool || md->tio_pool || md->bs);
1876
1877 md->io_pool = p->io_pool;
1878 p->io_pool = NULL;
1879 md->tio_pool = p->tio_pool;
1880 p->tio_pool = NULL;
1881 md->bs = p->bs;
1882 p->bs = NULL;
1883
1884out:
1885 /* mempool bind completed, now no need any mempools in the table */
1886 dm_table_free_md_mempools(t);
1887}
1888
Linus Torvalds1da177e2005-04-16 15:20:36 -07001889/*
1890 * Bind a table to the device.
1891 */
1892static void event_callback(void *context)
1893{
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001894 unsigned long flags;
1895 LIST_HEAD(uevents);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001896 struct mapped_device *md = (struct mapped_device *) context;
1897
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001898 spin_lock_irqsave(&md->uevent_lock, flags);
1899 list_splice_init(&md->uevent_list, &uevents);
1900 spin_unlock_irqrestore(&md->uevent_lock, flags);
1901
Tejun Heoed9e1982008-08-25 19:56:05 +09001902 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
Mike Anderson7a8c3d32007-10-19 22:48:01 +01001903
Linus Torvalds1da177e2005-04-16 15:20:36 -07001904 atomic_inc(&md->event_nr);
1905 wake_up(&md->eventq);
1906}
1907
Alasdair G Kergon4e901882005-07-28 21:15:59 -07001908static void __set_size(struct mapped_device *md, sector_t size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001909{
Alasdair G Kergon4e901882005-07-28 21:15:59 -07001910 set_capacity(md->disk, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001911
Mikulas Patockadb8fef42009-06-22 10:12:15 +01001912 mutex_lock(&md->bdev->bd_inode->i_mutex);
1913 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1914 mutex_unlock(&md->bdev->bd_inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001915}
1916
Mike Snitzer754c5fc2009-06-22 10:12:34 +01001917static int __bind(struct mapped_device *md, struct dm_table *t,
1918 struct queue_limits *limits)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001919{
Jens Axboe165125e2007-07-24 09:28:11 +02001920 struct request_queue *q = md->queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921 sector_t size;
Kiyoshi Ueda523d9292009-06-22 10:12:37 +01001922 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001923
1924 size = dm_table_get_size(t);
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08001925
1926 /*
1927 * Wipe any geometry if the size of the table changed.
1928 */
1929 if (size != get_capacity(md->disk))
1930 memset(&md->geometry, 0, sizeof(md->geometry));
1931
Mikulas Patocka32a926d2009-06-22 10:12:17 +01001932 __set_size(md, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001933
Mikulas Patockad5816872009-01-06 03:05:10 +00001934 if (!size) {
1935 dm_table_destroy(t);
1936 return 0;
1937 }
1938
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07001939 dm_table_event_callback(t, event_callback, md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001940
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01001941 /*
1942 * The queue hasn't been stopped yet, if the old table type wasn't
1943 * for request-based during suspension. So stop it to prevent
1944 * I/O mapping before resume.
1945 * This must be done before setting the queue restrictions,
1946 * because request-based dm may be run just after the setting.
1947 */
1948 if (dm_table_request_based(t) && !blk_queue_stopped(q))
1949 stop_queue(q);
1950
1951 __bind_mempools(md, t);
1952
Kiyoshi Ueda523d9292009-06-22 10:12:37 +01001953 write_lock_irqsave(&md->map_lock, flags);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001954 md->map = t;
Mike Snitzer754c5fc2009-06-22 10:12:34 +01001955 dm_table_set_restrictions(t, q, limits);
Kiyoshi Ueda523d9292009-06-22 10:12:37 +01001956 write_unlock_irqrestore(&md->map_lock, flags);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07001957
Linus Torvalds1da177e2005-04-16 15:20:36 -07001958 return 0;
1959}
1960
1961static void __unbind(struct mapped_device *md)
1962{
1963 struct dm_table *map = md->map;
Kiyoshi Ueda523d9292009-06-22 10:12:37 +01001964 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001965
1966 if (!map)
1967 return;
1968
1969 dm_table_event_callback(map, NULL, NULL);
Kiyoshi Ueda523d9292009-06-22 10:12:37 +01001970 write_lock_irqsave(&md->map_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001971 md->map = NULL;
Kiyoshi Ueda523d9292009-06-22 10:12:37 +01001972 write_unlock_irqrestore(&md->map_lock, flags);
Mikulas Patockad5816872009-01-06 03:05:10 +00001973 dm_table_destroy(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974}
1975
1976/*
1977 * Constructor for a new device.
1978 */
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001979int dm_create(int minor, struct mapped_device **result)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001980{
1981 struct mapped_device *md;
1982
Alasdair G Kergon2b06cff2006-06-26 00:27:32 -07001983 md = alloc_dev(minor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001984 if (!md)
1985 return -ENXIO;
1986
Milan Broz784aae72009-01-06 03:05:12 +00001987 dm_sysfs_init(md);
1988
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989 *result = md;
1990 return 0;
1991}
1992
David Teigland637842c2006-01-06 00:20:00 -08001993static struct mapped_device *dm_find_md(dev_t dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001994{
1995 struct mapped_device *md;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001996 unsigned minor = MINOR(dev);
1997
1998 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1999 return NULL;
2000
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002001 spin_lock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002002
2003 md = idr_find(&_minor_idr, minor);
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002004 if (md && (md == MINOR_ALLOCED ||
Tejun Heof331c022008-09-03 09:01:48 +02002005 (MINOR(disk_devt(dm_disk(md))) != minor) ||
Alasdair G Kergon17b2f662006-06-26 00:27:33 -07002006 test_bit(DMF_FREEING, &md->flags))) {
David Teigland637842c2006-01-06 00:20:00 -08002007 md = NULL;
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002008 goto out;
2009 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002010
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002011out:
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002012 spin_unlock(&_minor_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002013
David Teigland637842c2006-01-06 00:20:00 -08002014 return md;
2015}
2016
David Teiglandd229a952006-01-06 00:20:01 -08002017struct mapped_device *dm_get_md(dev_t dev)
2018{
2019 struct mapped_device *md = dm_find_md(dev);
2020
2021 if (md)
2022 dm_get(md);
2023
2024 return md;
2025}
2026
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08002027void *dm_get_mdptr(struct mapped_device *md)
David Teigland637842c2006-01-06 00:20:00 -08002028{
Alasdair G Kergon9ade92a2006-03-27 01:17:53 -08002029 return md->interface_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002030}
2031
2032void dm_set_mdptr(struct mapped_device *md, void *ptr)
2033{
2034 md->interface_ptr = ptr;
2035}
2036
2037void dm_get(struct mapped_device *md)
2038{
2039 atomic_inc(&md->holders);
2040}
2041
Alasdair G Kergon72d94862006-06-26 00:27:35 -07002042const char *dm_device_name(struct mapped_device *md)
2043{
2044 return md->name;
2045}
2046EXPORT_SYMBOL_GPL(dm_device_name);
2047
Linus Torvalds1da177e2005-04-16 15:20:36 -07002048void dm_put(struct mapped_device *md)
2049{
Mike Anderson1134e5a2006-03-27 01:17:54 -08002050 struct dm_table *map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002051
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002052 BUG_ON(test_bit(DMF_FREEING, &md->flags));
2053
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002054 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
Mike Anderson1134e5a2006-03-27 01:17:54 -08002055 map = dm_get_table(md);
Tejun Heof331c022008-09-03 09:01:48 +02002056 idr_replace(&_minor_idr, MINOR_ALLOCED,
2057 MINOR(disk_devt(dm_disk(md))));
Jeff Mahoneyfba9f902006-06-26 00:27:23 -07002058 set_bit(DMF_FREEING, &md->flags);
Jeff Mahoneyf32c10b2006-06-26 00:27:22 -07002059 spin_unlock(&_minor_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002060 if (!dm_suspended(md)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002061 dm_table_presuspend_targets(map);
2062 dm_table_postsuspend_targets(map);
2063 }
Milan Broz784aae72009-01-06 03:05:12 +00002064 dm_sysfs_exit(md);
Mike Anderson1134e5a2006-03-27 01:17:54 -08002065 dm_table_put(map);
Mikulas Patockaa1b51e92009-01-06 03:04:53 +00002066 __unbind(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067 free_dev(md);
2068 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002069}
Edward Goggin79eb8852007-05-09 02:32:56 -07002070EXPORT_SYMBOL_GPL(dm_put);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002071
Mikulas Patocka401600d2009-04-02 19:55:38 +01002072static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
Milan Broz46125c12008-02-08 02:10:30 +00002073{
2074 int r = 0;
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01002075 DECLARE_WAITQUEUE(wait, current);
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002076 struct request_queue *q = md->queue;
2077 unsigned long flags;
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01002078
2079 dm_unplug_all(md->queue);
2080
2081 add_wait_queue(&md->wait, &wait);
Milan Broz46125c12008-02-08 02:10:30 +00002082
2083 while (1) {
Mikulas Patocka401600d2009-04-02 19:55:38 +01002084 set_current_state(interruptible);
Milan Broz46125c12008-02-08 02:10:30 +00002085
2086 smp_mb();
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002087 if (dm_request_based(md)) {
2088 spin_lock_irqsave(q->queue_lock, flags);
2089 if (!queue_in_flight(q) && blk_queue_stopped(q)) {
2090 spin_unlock_irqrestore(q->queue_lock, flags);
2091 break;
2092 }
2093 spin_unlock_irqrestore(q->queue_lock, flags);
Nikanth Karthikesana9327ca2009-09-11 09:18:54 +02002094 } else if (!atomic_read(&md->pending[0]) &&
2095 !atomic_read(&md->pending[1]))
Milan Broz46125c12008-02-08 02:10:30 +00002096 break;
2097
Mikulas Patocka401600d2009-04-02 19:55:38 +01002098 if (interruptible == TASK_INTERRUPTIBLE &&
2099 signal_pending(current)) {
Milan Broz46125c12008-02-08 02:10:30 +00002100 r = -EINTR;
2101 break;
2102 }
2103
2104 io_schedule();
2105 }
2106 set_current_state(TASK_RUNNING);
2107
Mikulas Patockab44ebeb2009-04-02 19:55:39 +01002108 remove_wait_queue(&md->wait, &wait);
2109
Milan Broz46125c12008-02-08 02:10:30 +00002110 return r;
2111}
2112
Mikulas Patocka531fe962009-06-22 10:12:17 +01002113static void dm_flush(struct mapped_device *md)
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01002114{
2115 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
Mikulas Patocka52b1fd52009-06-22 10:12:21 +01002116
2117 bio_init(&md->barrier_bio);
2118 md->barrier_bio.bi_bdev = md->bdev;
2119 md->barrier_bio.bi_rw = WRITE_BARRIER;
2120 __split_and_process_bio(md, &md->barrier_bio);
2121
2122 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01002123}
2124
2125static void process_barrier(struct mapped_device *md, struct bio *bio)
2126{
Mikulas Patocka5aa27812009-06-22 10:12:18 +01002127 md->barrier_error = 0;
2128
Mikulas Patocka531fe962009-06-22 10:12:17 +01002129 dm_flush(md);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01002130
Mikulas Patocka5aa27812009-06-22 10:12:18 +01002131 if (!bio_empty_barrier(bio)) {
2132 __split_and_process_bio(md, bio);
2133 dm_flush(md);
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01002134 }
2135
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01002136 if (md->barrier_error != DM_ENDIO_REQUEUE)
Mikulas Patocka531fe962009-06-22 10:12:17 +01002137 bio_endio(bio, md->barrier_error);
Mikulas Patocka2761e952009-06-22 10:12:18 +01002138 else {
2139 spin_lock_irq(&md->deferred_lock);
2140 bio_list_add_head(&md->deferred, bio);
2141 spin_unlock_irq(&md->deferred_lock);
2142 }
Mikulas Patockaaf7e4662009-04-09 00:27:16 +01002143}
2144
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145/*
2146 * Process the deferred bios
2147 */
Mikulas Patockaef208582009-04-02 19:55:38 +01002148static void dm_wq_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002149{
Mikulas Patockaef208582009-04-02 19:55:38 +01002150 struct mapped_device *md = container_of(work, struct mapped_device,
2151 work);
Milan Broz6d6f10d2008-02-08 02:10:22 +00002152 struct bio *c;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002153
Mikulas Patockaef208582009-04-02 19:55:38 +01002154 down_write(&md->io_lock);
2155
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002156 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002157 spin_lock_irq(&md->deferred_lock);
2158 c = bio_list_pop(&md->deferred);
2159 spin_unlock_irq(&md->deferred_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002160
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002161 if (!c) {
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01002162 clear_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
Alasdair G Kergondf12ee92009-04-09 00:27:13 +01002163 break;
2164 }
2165
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002166 up_write(&md->io_lock);
2167
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002168 if (dm_request_based(md))
2169 generic_make_request(c);
2170 else {
Jens Axboe1f98a132009-09-11 14:32:04 +02002171 if (bio_rw_flagged(c, BIO_RW_BARRIER))
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002172 process_barrier(md, c);
2173 else
2174 __split_and_process_bio(md, c);
2175 }
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002176
2177 down_write(&md->io_lock);
Mikulas Patocka022c2612009-04-02 19:55:39 +01002178 }
Milan Broz73d410c2008-02-08 02:10:25 +00002179
Mikulas Patockaef208582009-04-02 19:55:38 +01002180 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002181}
2182
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002183static void dm_queue_flush(struct mapped_device *md)
Milan Broz304f3f62008-02-08 02:11:17 +00002184{
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002185 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2186 smp_mb__after_clear_bit();
Mikulas Patocka53d59142009-04-02 19:55:37 +01002187 queue_work(md->wq, &md->work);
Milan Broz304f3f62008-02-08 02:11:17 +00002188}
2189
Linus Torvalds1da177e2005-04-16 15:20:36 -07002190/*
2191 * Swap in a new table (destroying old one).
2192 */
2193int dm_swap_table(struct mapped_device *md, struct dm_table *table)
2194{
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002195 struct queue_limits limits;
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002196 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002197
Daniel Walkere61290a2008-02-08 02:10:08 +00002198 mutex_lock(&md->suspend_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002199
2200 /* device must be suspended */
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002201 if (!dm_suspended(md))
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002202 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002204 r = dm_calculate_queue_limits(table, &limits);
2205 if (r)
2206 goto out;
2207
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002208 /* cannot change the device type, once a table is bound */
2209 if (md->map &&
2210 (dm_table_get_type(md->map) != dm_table_get_type(table))) {
2211 DMWARN("can't change the device type after a table is bound");
2212 goto out;
2213 }
2214
Linus Torvalds1da177e2005-04-16 15:20:36 -07002215 __unbind(md);
Mike Snitzer754c5fc2009-06-22 10:12:34 +01002216 r = __bind(md, table, &limits);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002217
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002218out:
Daniel Walkere61290a2008-02-08 02:10:08 +00002219 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon93c534a2005-07-12 15:53:05 -07002220 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002221}
2222
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002223static void dm_rq_invalidate_suspend_marker(struct mapped_device *md)
2224{
2225 md->suspend_rq.special = (void *)0x1;
2226}
2227
2228static void dm_rq_abort_suspend(struct mapped_device *md, int noflush)
2229{
2230 struct request_queue *q = md->queue;
2231 unsigned long flags;
2232
2233 spin_lock_irqsave(q->queue_lock, flags);
2234 if (!noflush)
2235 dm_rq_invalidate_suspend_marker(md);
2236 __start_queue(q);
2237 spin_unlock_irqrestore(q->queue_lock, flags);
2238}
2239
2240static void dm_rq_start_suspend(struct mapped_device *md, int noflush)
2241{
2242 struct request *rq = &md->suspend_rq;
2243 struct request_queue *q = md->queue;
2244
2245 if (noflush)
2246 stop_queue(q);
2247 else {
2248 blk_rq_init(q, rq);
2249 blk_insert_request(q, rq, 0, NULL);
2250 }
2251}
2252
2253static int dm_rq_suspend_available(struct mapped_device *md, int noflush)
2254{
2255 int r = 1;
2256 struct request *rq = &md->suspend_rq;
2257 struct request_queue *q = md->queue;
2258 unsigned long flags;
2259
2260 if (noflush)
2261 return r;
2262
2263 /* The marker must be protected by queue lock if it is in use */
2264 spin_lock_irqsave(q->queue_lock, flags);
2265 if (unlikely(rq->ref_count)) {
2266 /*
2267 * This can happen, when the previous flush suspend was
2268 * interrupted, the marker is still in the queue and
2269 * this flush suspend has been invoked, because we don't
2270 * remove the marker at the time of suspend interruption.
2271 * We have only one marker per mapped_device, so we can't
2272 * start another flush suspend while it is in use.
2273 */
2274 BUG_ON(!rq->special); /* The marker should be invalidated */
2275 DMWARN("Invalidating the previous flush suspend is still in"
2276 " progress. Please retry later.");
2277 r = 0;
2278 }
2279 spin_unlock_irqrestore(q->queue_lock, flags);
2280
2281 return r;
2282}
2283
Linus Torvalds1da177e2005-04-16 15:20:36 -07002284/*
2285 * Functions to lock and unlock any filesystem running on the
2286 * device.
2287 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002288static int lock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002289{
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08002290 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002291
2292 WARN_ON(md->frozen_sb);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002293
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002294 md->frozen_sb = freeze_bdev(md->bdev);
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002295 if (IS_ERR(md->frozen_sb)) {
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002296 r = PTR_ERR(md->frozen_sb);
Alasdair G Kergone39e2e92006-01-06 00:20:05 -08002297 md->frozen_sb = NULL;
2298 return r;
Alasdair G Kergondfbe03f2005-05-05 16:16:04 -07002299 }
2300
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002301 set_bit(DMF_FROZEN, &md->flags);
2302
Linus Torvalds1da177e2005-04-16 15:20:36 -07002303 return 0;
2304}
2305
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002306static void unlock_fs(struct mapped_device *md)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002307{
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002308 if (!test_bit(DMF_FROZEN, &md->flags))
2309 return;
2310
Mikulas Patockadb8fef42009-06-22 10:12:15 +01002311 thaw_bdev(md->bdev, md->frozen_sb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002312 md->frozen_sb = NULL;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002313 clear_bit(DMF_FROZEN, &md->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002314}
2315
2316/*
2317 * We need to be able to change a mapping table under a mounted
2318 * filesystem. For example we might want to move some data in
2319 * the background. Before the table can be swapped with
2320 * dm_bind_table, dm_suspend must be called to flush any in
2321 * flight bios and ensure that any further io gets deferred.
2322 */
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002323/*
2324 * Suspend mechanism in request-based dm.
2325 *
2326 * After the suspend starts, further incoming requests are kept in
2327 * the request_queue and deferred.
2328 * Remaining requests in the request_queue at the start of suspend are flushed
2329 * if it is flush suspend.
2330 * The suspend completes when the following conditions have been satisfied,
2331 * so wait for it:
2332 * 1. q->in_flight is 0 (which means no in_flight request)
2333 * 2. queue has been stopped (which means no request dispatching)
2334 *
2335 *
2336 * Noflush suspend
2337 * ---------------
2338 * Noflush suspend doesn't need to dispatch remaining requests.
2339 * So stop the queue immediately. Then, wait for all in_flight requests
2340 * to be completed or requeued.
2341 *
2342 * To abort noflush suspend, start the queue.
2343 *
2344 *
2345 * Flush suspend
2346 * -------------
2347 * Flush suspend needs to dispatch remaining requests. So stop the queue
2348 * after the remaining requests are completed. (Requeued request must be also
2349 * re-dispatched and completed. Until then, we can't stop the queue.)
2350 *
2351 * During flushing the remaining requests, further incoming requests are also
2352 * inserted to the same queue. To distinguish which requests are to be
2353 * flushed, we insert a marker request to the queue at the time of starting
2354 * flush suspend, like a barrier.
2355 * The dispatching is blocked when the marker is found on the top of the queue.
2356 * And the queue is stopped when all in_flight requests are completed, since
2357 * that means the remaining requests are completely flushed.
2358 * Then, the marker is removed from the queue.
2359 *
2360 * To abort flush suspend, we also need to take care of the marker, not only
2361 * starting the queue.
2362 * We don't remove the marker forcibly from the queue since it's against
2363 * the block-layer manner. Instead, we put a invalidated mark on the marker.
2364 * When the invalidated marker is found on the top of the queue, it is
2365 * immediately removed from the queue, so it doesn't block dispatching.
2366 * Because we have only one marker per mapped_device, we can't start another
2367 * flush suspend until the invalidated marker is removed from the queue.
2368 * So fail and return with -EBUSY in such a case.
2369 */
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08002370int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002371{
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002372 struct dm_table *map = NULL;
Milan Broz46125c12008-02-08 02:10:30 +00002373 int r = 0;
Kiyoshi Uedaa3d77d32006-12-08 02:41:04 -08002374 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002375 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002376
Daniel Walkere61290a2008-02-08 02:10:08 +00002377 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002378
Milan Broz73d410c2008-02-08 02:10:25 +00002379 if (dm_suspended(md)) {
2380 r = -EINVAL;
Alasdair G Kergond2874832006-11-08 17:44:43 -08002381 goto out_unlock;
Milan Broz73d410c2008-02-08 02:10:25 +00002382 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002383
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002384 if (dm_request_based(md) && !dm_rq_suspend_available(md, noflush)) {
2385 r = -EBUSY;
2386 goto out_unlock;
2387 }
2388
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389 map = dm_get_table(md);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002390
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002391 /*
2392 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2393 * This flag is cleared before dm_suspend returns.
2394 */
2395 if (noflush)
2396 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2397
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002398 /* This does not get reverted if there's an error later. */
2399 dm_table_presuspend_targets(map);
2400
Mikulas Patocka32a926d2009-06-22 10:12:17 +01002401 /*
2402 * Flush I/O to the device. noflush supersedes do_lockfs,
2403 * because lock_fs() needs to flush I/Os.
2404 */
2405 if (!noflush && do_lockfs) {
2406 r = lock_fs(md);
2407 if (r)
Kiyoshi Uedaf431d962008-10-21 17:45:07 +01002408 goto out;
Alasdair G Kergonaa8d7c22006-01-06 00:20:06 -08002409 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002410
2411 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002412 * Here we must make sure that no processes are submitting requests
2413 * to target drivers i.e. no one may be executing
2414 * __split_and_process_bio. This is called from dm_request and
2415 * dm_wq_work.
2416 *
2417 * To get all processes out of __split_and_process_bio in dm_request,
2418 * we take the write lock. To prevent any process from reentering
2419 * __split_and_process_bio from dm_request, we set
2420 * DMF_QUEUE_IO_TO_THREAD.
2421 *
2422 * To quiesce the thread (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND
2423 * and call flush_workqueue(md->wq). flush_workqueue will wait until
2424 * dm_wq_work exits and DMF_BLOCK_IO_FOR_SUSPEND will prevent any
2425 * further calls to __split_and_process_bio from dm_wq_work.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002426 */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002427 down_write(&md->io_lock);
Alasdair G Kergon1eb787e2009-04-09 00:27:14 +01002428 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2429 set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002430 up_write(&md->io_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002431
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002432 flush_workqueue(md->wq);
2433
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002434 if (dm_request_based(md))
2435 dm_rq_start_suspend(md, noflush);
2436
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437 /*
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002438 * At this point no more requests are entering target request routines.
2439 * We call dm_wait_for_completion to wait for all existing requests
2440 * to finish.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002441 */
Mikulas Patocka401600d2009-04-02 19:55:38 +01002442 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002443
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002444 down_write(&md->io_lock);
Milan Broz6d6f10d2008-02-08 02:10:22 +00002445 if (noflush)
Mikulas Patocka022c2612009-04-02 19:55:39 +01002446 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
Milan Broz94d63512008-02-08 02:10:27 +00002447 up_write(&md->io_lock);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002448
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449 /* were we interrupted ? */
Milan Broz46125c12008-02-08 02:10:30 +00002450 if (r < 0) {
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002451 dm_queue_flush(md);
Milan Broz73d410c2008-02-08 02:10:25 +00002452
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002453 if (dm_request_based(md))
2454 dm_rq_abort_suspend(md, noflush);
2455
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002456 unlock_fs(md);
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002457 goto out; /* pushback list is already flushed, so skip flush */
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002458 }
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002459
Mikulas Patocka3b00b202009-04-09 00:27:15 +01002460 /*
2461 * If dm_wait_for_completion returned 0, the device is completely
2462 * quiescent now. There is no request-processing activity. All new
2463 * requests are being added to md->deferred list.
2464 */
2465
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002466 dm_table_postsuspend_targets(map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002467
2468 set_bit(DMF_SUSPENDED, &md->flags);
2469
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002470out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002471 dm_table_put(map);
Alasdair G Kergond2874832006-11-08 17:44:43 -08002472
2473out_unlock:
Daniel Walkere61290a2008-02-08 02:10:08 +00002474 mutex_unlock(&md->suspend_lock);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002475 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002476}
2477
2478int dm_resume(struct mapped_device *md)
2479{
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002480 int r = -EINVAL;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002481 struct dm_table *map = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002482
Daniel Walkere61290a2008-02-08 02:10:08 +00002483 mutex_lock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002484 if (!dm_suspended(md))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002485 goto out;
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002486
2487 map = dm_get_table(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002488 if (!map || !dm_table_get_size(map))
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002489 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490
Milan Broz8757b772006-10-03 01:15:36 -07002491 r = dm_table_resume_targets(map);
2492 if (r)
2493 goto out;
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002494
Mikulas Patocka9a1fb462009-04-02 19:55:36 +01002495 dm_queue_flush(md);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002496
Kiyoshi Uedacec47e32009-06-22 10:12:35 +01002497 /*
2498 * Flushing deferred I/Os must be done after targets are resumed
2499 * so that mapping of targets can work correctly.
2500 * Request-based dm is queueing the deferred I/Os in its request_queue.
2501 */
2502 if (dm_request_based(md))
2503 start_queue(md->queue);
2504
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002505 unlock_fs(md);
2506
2507 clear_bit(DMF_SUSPENDED, &md->flags);
2508
Linus Torvalds1da177e2005-04-16 15:20:36 -07002509 dm_table_unplug_all(map);
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002510 r = 0;
2511out:
2512 dm_table_put(map);
Daniel Walkere61290a2008-02-08 02:10:08 +00002513 mutex_unlock(&md->suspend_lock);
Alasdair G Kergon2ca33102005-07-28 21:16:00 -07002514
Alasdair G Kergoncf222b32005-07-28 21:15:57 -07002515 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002516}
2517
2518/*-----------------------------------------------------------------
2519 * Event notification.
2520 *---------------------------------------------------------------*/
Milan Broz60935eb2009-06-22 10:12:30 +01002521void dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2522 unsigned cookie)
Alasdair G Kergon69267a32007-12-13 14:15:57 +00002523{
Milan Broz60935eb2009-06-22 10:12:30 +01002524 char udev_cookie[DM_COOKIE_LENGTH];
2525 char *envp[] = { udev_cookie, NULL };
2526
2527 if (!cookie)
2528 kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2529 else {
2530 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2531 DM_COOKIE_ENV_VAR_NAME, cookie);
2532 kobject_uevent_env(&disk_to_dev(md->disk)->kobj, action, envp);
2533 }
Alasdair G Kergon69267a32007-12-13 14:15:57 +00002534}
2535
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002536uint32_t dm_next_uevent_seq(struct mapped_device *md)
2537{
2538 return atomic_add_return(1, &md->uevent_seq);
2539}
2540
Linus Torvalds1da177e2005-04-16 15:20:36 -07002541uint32_t dm_get_event_nr(struct mapped_device *md)
2542{
2543 return atomic_read(&md->event_nr);
2544}
2545
2546int dm_wait_event(struct mapped_device *md, int event_nr)
2547{
2548 return wait_event_interruptible(md->eventq,
2549 (event_nr != atomic_read(&md->event_nr)));
2550}
2551
Mike Anderson7a8c3d32007-10-19 22:48:01 +01002552void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2553{
2554 unsigned long flags;
2555
2556 spin_lock_irqsave(&md->uevent_lock, flags);
2557 list_add(elist, &md->uevent_list);
2558 spin_unlock_irqrestore(&md->uevent_lock, flags);
2559}
2560
Linus Torvalds1da177e2005-04-16 15:20:36 -07002561/*
2562 * The gendisk is only valid as long as you have a reference
2563 * count on 'md'.
2564 */
2565struct gendisk *dm_disk(struct mapped_device *md)
2566{
2567 return md->disk;
2568}
2569
Milan Broz784aae72009-01-06 03:05:12 +00002570struct kobject *dm_kobject(struct mapped_device *md)
2571{
2572 return &md->kobj;
2573}
2574
2575/*
2576 * struct mapped_device should not be exported outside of dm.c
2577 * so use this check to verify that kobj is part of md structure
2578 */
2579struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2580{
2581 struct mapped_device *md;
2582
2583 md = container_of(kobj, struct mapped_device, kobj);
2584 if (&md->kobj != kobj)
2585 return NULL;
2586
Milan Broz4d89b7b2009-06-22 10:12:11 +01002587 if (test_bit(DMF_FREEING, &md->flags) ||
2588 test_bit(DMF_DELETING, &md->flags))
2589 return NULL;
2590
Milan Broz784aae72009-01-06 03:05:12 +00002591 dm_get(md);
2592 return md;
2593}
2594
Linus Torvalds1da177e2005-04-16 15:20:36 -07002595int dm_suspended(struct mapped_device *md)
2596{
2597 return test_bit(DMF_SUSPENDED, &md->flags);
2598}
2599
Kiyoshi Ueda2e93ccc2006-12-08 02:41:09 -08002600int dm_noflush_suspending(struct dm_target *ti)
2601{
2602 struct mapped_device *md = dm_table_get_md(ti->table);
2603 int r = __noflush_suspending(md);
2604
2605 dm_put(md);
2606
2607 return r;
2608}
2609EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2610
Kiyoshi Uedae6ee8c02009-06-22 10:12:36 +01002611struct dm_md_mempools *dm_alloc_md_mempools(unsigned type)
2612{
2613 struct dm_md_mempools *pools = kmalloc(sizeof(*pools), GFP_KERNEL);
2614
2615 if (!pools)
2616 return NULL;
2617
2618 pools->io_pool = (type == DM_TYPE_BIO_BASED) ?
2619 mempool_create_slab_pool(MIN_IOS, _io_cache) :
2620 mempool_create_slab_pool(MIN_IOS, _rq_bio_info_cache);
2621 if (!pools->io_pool)
2622 goto free_pools_and_out;
2623
2624 pools->tio_pool = (type == DM_TYPE_BIO_BASED) ?
2625 mempool_create_slab_pool(MIN_IOS, _tio_cache) :
2626 mempool_create_slab_pool(MIN_IOS, _rq_tio_cache);
2627 if (!pools->tio_pool)
2628 goto free_io_pool_and_out;
2629
2630 pools->bs = (type == DM_TYPE_BIO_BASED) ?
2631 bioset_create(16, 0) : bioset_create(MIN_IOS, 0);
2632 if (!pools->bs)
2633 goto free_tio_pool_and_out;
2634
2635 return pools;
2636
2637free_tio_pool_and_out:
2638 mempool_destroy(pools->tio_pool);
2639
2640free_io_pool_and_out:
2641 mempool_destroy(pools->io_pool);
2642
2643free_pools_and_out:
2644 kfree(pools);
2645
2646 return NULL;
2647}
2648
2649void dm_free_md_mempools(struct dm_md_mempools *pools)
2650{
2651 if (!pools)
2652 return;
2653
2654 if (pools->io_pool)
2655 mempool_destroy(pools->io_pool);
2656
2657 if (pools->tio_pool)
2658 mempool_destroy(pools->tio_pool);
2659
2660 if (pools->bs)
2661 bioset_free(pools->bs);
2662
2663 kfree(pools);
2664}
2665
Alexey Dobriyan83d5cde2009-09-21 17:01:13 -07002666static const struct block_device_operations dm_blk_dops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002667 .open = dm_blk_open,
2668 .release = dm_blk_close,
Milan Brozaa129a22006-10-03 01:15:15 -07002669 .ioctl = dm_blk_ioctl,
Darrick J. Wong3ac51e72006-03-27 01:17:54 -08002670 .getgeo = dm_blk_getgeo,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002671 .owner = THIS_MODULE
2672};
2673
2674EXPORT_SYMBOL(dm_get_mapinfo);
2675
2676/*
2677 * module hooks
2678 */
2679module_init(dm_init);
2680module_exit(dm_exit);
2681
2682module_param(major, uint, 0);
2683MODULE_PARM_DESC(major, "The major number of the device mapper");
2684MODULE_DESCRIPTION(DM_NAME " driver");
2685MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2686MODULE_LICENSE("GPL");