blob: 6f758870fc19cf0e66db3055dab36ab96e038fee [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * dm-snapshot.c
3 *
4 * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
5 *
6 * This file is released under the GPL.
7 */
8
9#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/device-mapper.h>
Mikulas Patocka90fa1522009-01-06 03:04:54 +000011#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/fs.h>
13#include <linux/init.h>
14#include <linux/kdev_t.h>
15#include <linux/list.h>
16#include <linux/mempool.h>
17#include <linux/module.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
vignesh babu6f3c3f02007-10-19 22:38:44 +010020#include <linux/log2.h>
Alasdair G Kergona765e202008-04-24 22:02:01 +010021#include <linux/dm-kcopyd.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
Jonathan Brassowaea53d92009-01-06 03:05:15 +000023#include "dm-exception-store.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
Alasdair G Kergon72d94862006-06-26 00:27:35 -070025#define DM_MSG_PREFIX "snapshots"
26
Mikulas Patockad698aa42009-12-10 23:52:30 +000027static const char dm_snapshot_merge_target_name[] = "snapshot-merge";
28
29#define dm_target_is_snapshot_merge(ti) \
30 ((ti)->type->name == dm_snapshot_merge_target_name)
31
Linus Torvalds1da177e2005-04-16 15:20:36 -070032/*
Mikulas Patockacd45daf2008-07-21 12:00:32 +010033 * The size of the mempool used to track chunks in use.
34 */
35#define MIN_IOS 256
36
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010037#define DM_TRACKED_CHUNK_HASH_SIZE 16
38#define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \
39 (DM_TRACKED_CHUNK_HASH_SIZE - 1))
40
Jon Brassow191437a2009-12-10 23:52:10 +000041struct dm_exception_table {
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010042 uint32_t hash_mask;
43 unsigned hash_shift;
44 struct list_head *table;
45};
46
47struct dm_snapshot {
48 struct rw_semaphore lock;
49
50 struct dm_dev *origin;
Mike Snitzerfc56f6f2009-12-10 23:52:12 +000051 struct dm_dev *cow;
52
53 struct dm_target *ti;
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010054
55 /* List of snapshots per Origin */
56 struct list_head list;
57
Mike Snitzerd8ddb1c2009-12-10 23:52:35 +000058 /*
59 * You can't use a snapshot if this is 0 (e.g. if full).
60 * A snapshot-merge target never clears this.
61 */
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010062 int valid;
63
64 /* Origin writes don't trigger exceptions until this is set */
65 int active;
66
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010067 atomic_t pending_exceptions_count;
68
Mike Snitzer924e6002010-03-06 02:32:33 +000069 mempool_t *pending_pool;
70
Jon Brassow191437a2009-12-10 23:52:10 +000071 struct dm_exception_table pending;
72 struct dm_exception_table complete;
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010073
74 /*
75 * pe_lock protects all pending_exception operations and access
76 * as well as the snapshot_bios list.
77 */
78 spinlock_t pe_lock;
79
Mike Snitzer924e6002010-03-06 02:32:33 +000080 /* Chunks with outstanding reads */
81 spinlock_t tracked_chunk_lock;
82 mempool_t *tracked_chunk_pool;
83 struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
84
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010085 /* The on disk metadata handler */
86 struct dm_exception_store *store;
87
88 struct dm_kcopyd_client *kcopyd_client;
89
Mike Snitzer924e6002010-03-06 02:32:33 +000090 /* Wait for events based on state_bits */
91 unsigned long state_bits;
92
93 /* Range of chunks currently being merged. */
94 chunk_t first_merging_chunk;
95 int num_merging_chunks;
Mikulas Patocka1e03f972009-12-10 23:52:32 +000096
Mike Snitzerd8ddb1c2009-12-10 23:52:35 +000097 /*
98 * The merge operation failed if this flag is set.
99 * Failure modes are handled as follows:
100 * - I/O error reading the header
101 * => don't load the target; abort.
102 * - Header does not have "valid" flag set
103 * => use the origin; forget about the snapshot.
104 * - I/O error when reading exceptions
105 * => don't load the target; abort.
106 * (We can't use the intermediate origin state.)
107 * - I/O error while merging
108 * => stop merging; set merge_failed; process I/O normally.
109 */
110 int merge_failed;
111
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000112 /*
113 * Incoming bios that overlap with chunks being merged must wait
114 * for them to be committed.
115 */
116 struct bio_list bios_queued_during_merge;
Jonathan Brassowccc45ea2009-04-02 19:55:34 +0100117};
118
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000119/*
120 * state_bits:
121 * RUNNING_MERGE - Merge operation is in progress.
122 * SHUTDOWN_MERGE - Set to signal that merge needs to be stopped;
123 * cleared afterwards.
124 */
125#define RUNNING_MERGE 0
126#define SHUTDOWN_MERGE 1
127
Mikulas Patockac2411042010-08-12 04:13:51 +0100128struct dm_dev *dm_snap_origin(struct dm_snapshot *s)
129{
130 return s->origin;
131}
132EXPORT_SYMBOL(dm_snap_origin);
133
Mike Snitzerfc56f6f2009-12-10 23:52:12 +0000134struct dm_dev *dm_snap_cow(struct dm_snapshot *s)
135{
136 return s->cow;
137}
138EXPORT_SYMBOL(dm_snap_cow);
139
Jonathan Brassowccc45ea2009-04-02 19:55:34 +0100140static sector_t chunk_to_sector(struct dm_exception_store *store,
141 chunk_t chunk)
142{
143 return chunk << store->chunk_shift;
144}
145
146static int bdev_equal(struct block_device *lhs, struct block_device *rhs)
147{
148 /*
149 * There is only ever one instance of a particular block
150 * device so we can compare pointers safely.
151 */
152 return lhs == rhs;
153}
154
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100155struct dm_snap_pending_exception {
Jon Brassow1d4989c2009-12-10 23:52:10 +0000156 struct dm_exception e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157
158 /*
159 * Origin buffers waiting for this to complete are held
160 * in a bio list
161 */
162 struct bio_list origin_bios;
163 struct bio_list snapshot_bios;
164
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 /* Pointer back to snapshot context */
166 struct dm_snapshot *snap;
167
168 /*
169 * 1 indicates the exception has already been sent to
170 * kcopyd.
171 */
172 int started;
Mikulas Patockaa6e50b42011-08-02 12:32:04 +0100173
174 /*
175 * For writing a complete chunk, bypassing the copy.
176 */
177 struct bio *full_bio;
178 bio_end_io_t *full_bio_end_io;
179 void *full_bio_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180};
181
182/*
183 * Hash table mapping origin volumes to lists of snapshots and
184 * a lock to protect it
185 */
Christoph Lametere18b8902006-12-06 20:33:20 -0800186static struct kmem_cache *exception_cache;
187static struct kmem_cache *pending_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100189struct dm_snap_tracked_chunk {
190 struct hlist_node node;
191 chunk_t chunk;
192};
193
194static struct kmem_cache *tracked_chunk_cache;
195
196static struct dm_snap_tracked_chunk *track_chunk(struct dm_snapshot *s,
197 chunk_t chunk)
198{
199 struct dm_snap_tracked_chunk *c = mempool_alloc(s->tracked_chunk_pool,
200 GFP_NOIO);
201 unsigned long flags;
202
203 c->chunk = chunk;
204
205 spin_lock_irqsave(&s->tracked_chunk_lock, flags);
206 hlist_add_head(&c->node,
207 &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);
208 spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
209
210 return c;
211}
212
213static void stop_tracking_chunk(struct dm_snapshot *s,
214 struct dm_snap_tracked_chunk *c)
215{
216 unsigned long flags;
217
218 spin_lock_irqsave(&s->tracked_chunk_lock, flags);
219 hlist_del(&c->node);
220 spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
221
222 mempool_free(c, s->tracked_chunk_pool);
223}
224
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100225static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)
226{
227 struct dm_snap_tracked_chunk *c;
228 struct hlist_node *hn;
229 int found = 0;
230
231 spin_lock_irq(&s->tracked_chunk_lock);
232
233 hlist_for_each_entry(c, hn,
234 &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) {
235 if (c->chunk == chunk) {
236 found = 1;
237 break;
238 }
239 }
240
241 spin_unlock_irq(&s->tracked_chunk_lock);
242
243 return found;
244}
245
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246/*
Mike Snitzer615d1eb2009-12-10 23:52:29 +0000247 * This conflicting I/O is extremely improbable in the caller,
248 * so msleep(1) is sufficient and there is no need for a wait queue.
249 */
250static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk)
251{
252 while (__chunk_is_tracked(s, chunk))
253 msleep(1);
254}
255
256/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 * One of these per registered origin, held in the snapshot_origins hash
258 */
259struct origin {
260 /* The origin device */
261 struct block_device *bdev;
262
263 struct list_head hash_list;
264
265 /* List of snapshots for this origin */
266 struct list_head snapshots;
267};
268
269/*
270 * Size of the hash table for origin volumes. If we make this
271 * the size of the minors list then it should be nearly perfect
272 */
273#define ORIGIN_HASH_SIZE 256
274#define ORIGIN_MASK 0xFF
275static struct list_head *_origins;
276static struct rw_semaphore _origins_lock;
277
Mikulas Patocka73dfd072009-12-10 23:52:34 +0000278static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done);
279static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock);
280static uint64_t _pending_exceptions_done_count;
281
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282static int init_origin_hash(void)
283{
284 int i;
285
286 _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head),
287 GFP_KERNEL);
288 if (!_origins) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700289 DMERR("unable to allocate memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 return -ENOMEM;
291 }
292
293 for (i = 0; i < ORIGIN_HASH_SIZE; i++)
294 INIT_LIST_HEAD(_origins + i);
295 init_rwsem(&_origins_lock);
296
297 return 0;
298}
299
300static void exit_origin_hash(void)
301{
302 kfree(_origins);
303}
304
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100305static unsigned origin_hash(struct block_device *bdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306{
307 return bdev->bd_dev & ORIGIN_MASK;
308}
309
310static struct origin *__lookup_origin(struct block_device *origin)
311{
312 struct list_head *ol;
313 struct origin *o;
314
315 ol = &_origins[origin_hash(origin)];
316 list_for_each_entry (o, ol, hash_list)
317 if (bdev_equal(o->bdev, origin))
318 return o;
319
320 return NULL;
321}
322
323static void __insert_origin(struct origin *o)
324{
325 struct list_head *sl = &_origins[origin_hash(o->bdev)];
326 list_add_tail(&o->hash_list, sl);
327}
328
329/*
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000330 * _origins_lock must be held when calling this function.
331 * Returns number of snapshots registered using the supplied cow device, plus:
332 * snap_src - a snapshot suitable for use as a source of exception handover
333 * snap_dest - a snapshot capable of receiving exception handover.
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +0000334 * snap_merge - an existing snapshot-merge target linked to the same origin.
335 * There can be at most one snapshot-merge target. The parameter is optional.
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000336 *
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +0000337 * Possible return values and states of snap_src and snap_dest.
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000338 * 0: NULL, NULL - first new snapshot
339 * 1: snap_src, NULL - normal snapshot
340 * 2: snap_src, snap_dest - waiting for handover
341 * 2: snap_src, NULL - handed over, waiting for old to be deleted
342 * 1: NULL, snap_dest - source got destroyed without handover
343 */
344static int __find_snapshots_sharing_cow(struct dm_snapshot *snap,
345 struct dm_snapshot **snap_src,
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +0000346 struct dm_snapshot **snap_dest,
347 struct dm_snapshot **snap_merge)
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000348{
349 struct dm_snapshot *s;
350 struct origin *o;
351 int count = 0;
352 int active;
353
354 o = __lookup_origin(snap->origin->bdev);
355 if (!o)
356 goto out;
357
358 list_for_each_entry(s, &o->snapshots, list) {
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +0000359 if (dm_target_is_snapshot_merge(s->ti) && snap_merge)
360 *snap_merge = s;
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000361 if (!bdev_equal(s->cow->bdev, snap->cow->bdev))
362 continue;
363
364 down_read(&s->lock);
365 active = s->active;
366 up_read(&s->lock);
367
368 if (active) {
369 if (snap_src)
370 *snap_src = s;
371 } else if (snap_dest)
372 *snap_dest = s;
373
374 count++;
375 }
376
377out:
378 return count;
379}
380
381/*
382 * On success, returns 1 if this snapshot is a handover destination,
383 * otherwise returns 0.
384 */
385static int __validate_exception_handover(struct dm_snapshot *snap)
386{
387 struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +0000388 struct dm_snapshot *snap_merge = NULL;
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000389
390 /* Does snapshot need exceptions handed over to it? */
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +0000391 if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest,
392 &snap_merge) == 2) ||
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000393 snap_dest) {
394 snap->ti->error = "Snapshot cow pairing for exception "
395 "table handover failed";
396 return -EINVAL;
397 }
398
399 /*
400 * If no snap_src was found, snap cannot become a handover
401 * destination.
402 */
403 if (!snap_src)
404 return 0;
405
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +0000406 /*
407 * Non-snapshot-merge handover?
408 */
409 if (!dm_target_is_snapshot_merge(snap->ti))
410 return 1;
411
412 /*
413 * Do not allow more than one merging snapshot.
414 */
415 if (snap_merge) {
416 snap->ti->error = "A snapshot is already merging.";
417 return -EINVAL;
418 }
419
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000420 if (!snap_src->store->type->prepare_merge ||
421 !snap_src->store->type->commit_merge) {
422 snap->ti->error = "Snapshot exception store does not "
423 "support snapshot-merge.";
424 return -EINVAL;
425 }
426
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000427 return 1;
428}
429
430static void __insert_snapshot(struct origin *o, struct dm_snapshot *s)
431{
432 struct dm_snapshot *l;
433
434 /* Sort the list according to chunk size, largest-first smallest-last */
435 list_for_each_entry(l, &o->snapshots, list)
436 if (l->store->chunk_size < s->store->chunk_size)
437 break;
438 list_add_tail(&s->list, &l->list);
439}
440
441/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 * Make a note of the snapshot and its origin so we can look it
443 * up when the origin has a write on it.
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000444 *
445 * Also validate snapshot exception store handovers.
446 * On success, returns 1 if this registration is a handover destination,
447 * otherwise returns 0.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 */
449static int register_snapshot(struct dm_snapshot *snap)
450{
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000451 struct origin *o, *new_o = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 struct block_device *bdev = snap->origin->bdev;
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000453 int r = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000455 new_o = kmalloc(sizeof(*new_o), GFP_KERNEL);
456 if (!new_o)
457 return -ENOMEM;
458
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 down_write(&_origins_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000461 r = __validate_exception_handover(snap);
462 if (r < 0) {
463 kfree(new_o);
464 goto out;
465 }
466
467 o = __lookup_origin(bdev);
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000468 if (o)
469 kfree(new_o);
470 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 /* New origin */
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000472 o = new_o;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473
474 /* Initialise the struct */
475 INIT_LIST_HEAD(&o->snapshots);
476 o->bdev = bdev;
477
478 __insert_origin(o);
479 }
480
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000481 __insert_snapshot(o, snap);
482
483out:
484 up_write(&_origins_lock);
485
486 return r;
487}
488
489/*
490 * Move snapshot to correct place in list according to chunk size.
491 */
492static void reregister_snapshot(struct dm_snapshot *s)
493{
494 struct block_device *bdev = s->origin->bdev;
495
496 down_write(&_origins_lock);
497
498 list_del(&s->list);
499 __insert_snapshot(__lookup_origin(bdev), s);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500
501 up_write(&_origins_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502}
503
504static void unregister_snapshot(struct dm_snapshot *s)
505{
506 struct origin *o;
507
508 down_write(&_origins_lock);
509 o = __lookup_origin(s->origin->bdev);
510
511 list_del(&s->list);
Mike Snitzerc1f0c182009-12-10 23:52:24 +0000512 if (o && list_empty(&o->snapshots)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 list_del(&o->hash_list);
514 kfree(o);
515 }
516
517 up_write(&_origins_lock);
518}
519
520/*
521 * Implementation of the exception hash tables.
Milan Brozd74f81f2008-02-08 02:11:27 +0000522 * The lowest hash_shift bits of the chunk number are ignored, allowing
523 * some consecutive chunks to be grouped together.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 */
Jon Brassow3510cb92009-12-10 23:52:11 +0000525static int dm_exception_table_init(struct dm_exception_table *et,
526 uint32_t size, unsigned hash_shift)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527{
528 unsigned int i;
529
Milan Brozd74f81f2008-02-08 02:11:27 +0000530 et->hash_shift = hash_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 et->hash_mask = size - 1;
532 et->table = dm_vcalloc(size, sizeof(struct list_head));
533 if (!et->table)
534 return -ENOMEM;
535
536 for (i = 0; i < size; i++)
537 INIT_LIST_HEAD(et->table + i);
538
539 return 0;
540}
541
Jon Brassow3510cb92009-12-10 23:52:11 +0000542static void dm_exception_table_exit(struct dm_exception_table *et,
543 struct kmem_cache *mem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544{
545 struct list_head *slot;
Jon Brassow1d4989c2009-12-10 23:52:10 +0000546 struct dm_exception *ex, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 int i, size;
548
549 size = et->hash_mask + 1;
550 for (i = 0; i < size; i++) {
551 slot = et->table + i;
552
553 list_for_each_entry_safe (ex, next, slot, hash_list)
554 kmem_cache_free(mem, ex);
555 }
556
557 vfree(et->table);
558}
559
Jon Brassow191437a2009-12-10 23:52:10 +0000560static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561{
Milan Brozd74f81f2008-02-08 02:11:27 +0000562 return (chunk >> et->hash_shift) & et->hash_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563}
564
Jon Brassow3510cb92009-12-10 23:52:11 +0000565static void dm_remove_exception(struct dm_exception *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566{
567 list_del(&e->hash_list);
568}
569
570/*
571 * Return the exception data for a sector, or NULL if not
572 * remapped.
573 */
Jon Brassow3510cb92009-12-10 23:52:11 +0000574static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et,
575 chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576{
577 struct list_head *slot;
Jon Brassow1d4989c2009-12-10 23:52:10 +0000578 struct dm_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579
580 slot = &et->table[exception_hash(et, chunk)];
581 list_for_each_entry (e, slot, hash_list)
Milan Brozd74f81f2008-02-08 02:11:27 +0000582 if (chunk >= e->old_chunk &&
583 chunk <= e->old_chunk + dm_consecutive_chunk_count(e))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 return e;
585
586 return NULL;
587}
588
Jon Brassow3510cb92009-12-10 23:52:11 +0000589static struct dm_exception *alloc_completed_exception(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590{
Jon Brassow1d4989c2009-12-10 23:52:10 +0000591 struct dm_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592
593 e = kmem_cache_alloc(exception_cache, GFP_NOIO);
594 if (!e)
595 e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
596
597 return e;
598}
599
Jon Brassow3510cb92009-12-10 23:52:11 +0000600static void free_completed_exception(struct dm_exception *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601{
602 kmem_cache_free(exception_cache, e);
603}
604
Mikulas Patocka92e86812008-07-21 12:00:35 +0100605static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606{
Mikulas Patocka92e86812008-07-21 12:00:35 +0100607 struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool,
608 GFP_NOIO);
609
Mikulas Patocka879129d22008-10-30 13:33:16 +0000610 atomic_inc(&s->pending_exceptions_count);
Mikulas Patocka92e86812008-07-21 12:00:35 +0100611 pe->snap = s;
612
613 return pe;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614}
615
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100616static void free_pending_exception(struct dm_snap_pending_exception *pe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617{
Mikulas Patocka879129d22008-10-30 13:33:16 +0000618 struct dm_snapshot *s = pe->snap;
619
620 mempool_free(pe, s->pending_pool);
621 smp_mb__before_atomic_dec();
622 atomic_dec(&s->pending_exceptions_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623}
624
Jon Brassow3510cb92009-12-10 23:52:11 +0000625static void dm_insert_exception(struct dm_exception_table *eh,
626 struct dm_exception *new_e)
Milan Brozd74f81f2008-02-08 02:11:27 +0000627{
Milan Brozd74f81f2008-02-08 02:11:27 +0000628 struct list_head *l;
Jon Brassow1d4989c2009-12-10 23:52:10 +0000629 struct dm_exception *e = NULL;
Milan Brozd74f81f2008-02-08 02:11:27 +0000630
631 l = &eh->table[exception_hash(eh, new_e->old_chunk)];
632
633 /* Add immediately if this table doesn't support consecutive chunks */
634 if (!eh->hash_shift)
635 goto out;
636
637 /* List is ordered by old_chunk */
638 list_for_each_entry_reverse(e, l, hash_list) {
639 /* Insert after an existing chunk? */
640 if (new_e->old_chunk == (e->old_chunk +
641 dm_consecutive_chunk_count(e) + 1) &&
642 new_e->new_chunk == (dm_chunk_number(e->new_chunk) +
643 dm_consecutive_chunk_count(e) + 1)) {
644 dm_consecutive_chunk_count_inc(e);
Jon Brassow3510cb92009-12-10 23:52:11 +0000645 free_completed_exception(new_e);
Milan Brozd74f81f2008-02-08 02:11:27 +0000646 return;
647 }
648
649 /* Insert before an existing chunk? */
650 if (new_e->old_chunk == (e->old_chunk - 1) &&
651 new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) {
652 dm_consecutive_chunk_count_inc(e);
653 e->old_chunk--;
654 e->new_chunk--;
Jon Brassow3510cb92009-12-10 23:52:11 +0000655 free_completed_exception(new_e);
Milan Brozd74f81f2008-02-08 02:11:27 +0000656 return;
657 }
658
659 if (new_e->old_chunk > e->old_chunk)
660 break;
661 }
662
663out:
664 list_add(&new_e->hash_list, e ? &e->hash_list : l);
665}
666
Jonathan Brassowa159c1a2009-01-06 03:05:19 +0000667/*
668 * Callback used by the exception stores to load exceptions when
669 * initialising.
670 */
671static int dm_add_exception(void *context, chunk_t old, chunk_t new)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672{
Jonathan Brassowa159c1a2009-01-06 03:05:19 +0000673 struct dm_snapshot *s = context;
Jon Brassow1d4989c2009-12-10 23:52:10 +0000674 struct dm_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675
Jon Brassow3510cb92009-12-10 23:52:11 +0000676 e = alloc_completed_exception();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 if (!e)
678 return -ENOMEM;
679
680 e->old_chunk = old;
Milan Brozd74f81f2008-02-08 02:11:27 +0000681
682 /* Consecutive_count is implicitly initialised to zero */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 e->new_chunk = new;
Milan Brozd74f81f2008-02-08 02:11:27 +0000684
Jon Brassow3510cb92009-12-10 23:52:11 +0000685 dm_insert_exception(&s->complete, e);
Milan Brozd74f81f2008-02-08 02:11:27 +0000686
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 return 0;
688}
689
Mikulas Patocka7e201b32009-12-10 23:52:08 +0000690/*
691 * Return a minimum chunk size of all snapshots that have the specified origin.
692 * Return zero if the origin has no snapshots.
693 */
694static sector_t __minimum_chunk_size(struct origin *o)
695{
696 struct dm_snapshot *snap;
697 unsigned chunk_size = 0;
698
699 if (o)
700 list_for_each_entry(snap, &o->snapshots, list)
701 chunk_size = min_not_zero(chunk_size,
702 snap->store->chunk_size);
703
704 return chunk_size;
705}
706
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707/*
708 * Hard coded magic.
709 */
710static int calc_max_buckets(void)
711{
712 /* use a fixed size of 2MB */
713 unsigned long mem = 2 * 1024 * 1024;
714 mem /= sizeof(struct list_head);
715
716 return mem;
717}
718
719/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 * Allocate room for a suitable hash table.
721 */
Jonathan Brassowfee19982009-04-02 19:55:34 +0100722static int init_hash_tables(struct dm_snapshot *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723{
724 sector_t hash_size, cow_dev_size, origin_dev_size, max_buckets;
725
726 /*
727 * Calculate based on the size of the original volume or
728 * the COW volume...
729 */
Mike Snitzerfc56f6f2009-12-10 23:52:12 +0000730 cow_dev_size = get_dev_size(s->cow->bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 origin_dev_size = get_dev_size(s->origin->bdev);
732 max_buckets = calc_max_buckets();
733
Jonathan Brassowfee19982009-04-02 19:55:34 +0100734 hash_size = min(origin_dev_size, cow_dev_size) >> s->store->chunk_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 hash_size = min(hash_size, max_buckets);
736
Mikulas Patocka8e87b9b2009-12-10 23:51:54 +0000737 if (hash_size < 64)
738 hash_size = 64;
Robert P. J. Day8defd832008-02-08 02:10:06 +0000739 hash_size = rounddown_pow_of_two(hash_size);
Jon Brassow3510cb92009-12-10 23:52:11 +0000740 if (dm_exception_table_init(&s->complete, hash_size,
741 DM_CHUNK_CONSECUTIVE_BITS))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 return -ENOMEM;
743
744 /*
745 * Allocate hash table for in-flight exceptions
746 * Make this smaller than the real hash table
747 */
748 hash_size >>= 3;
749 if (hash_size < 64)
750 hash_size = 64;
751
Jon Brassow3510cb92009-12-10 23:52:11 +0000752 if (dm_exception_table_init(&s->pending, hash_size, 0)) {
753 dm_exception_table_exit(&s->complete, exception_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 return -ENOMEM;
755 }
756
757 return 0;
758}
759
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000760static void merge_shutdown(struct dm_snapshot *s)
761{
762 clear_bit_unlock(RUNNING_MERGE, &s->state_bits);
763 smp_mb__after_clear_bit();
764 wake_up_bit(&s->state_bits, RUNNING_MERGE);
765}
766
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000767static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)
768{
769 s->first_merging_chunk = 0;
770 s->num_merging_chunks = 0;
771
772 return bio_list_get(&s->bios_queued_during_merge);
773}
774
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000775/*
776 * Remove one chunk from the index of completed exceptions.
777 */
778static int __remove_single_exception_chunk(struct dm_snapshot *s,
779 chunk_t old_chunk)
780{
781 struct dm_exception *e;
782
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000783 e = dm_lookup_exception(&s->complete, old_chunk);
784 if (!e) {
785 DMERR("Corruption detected: exception for block %llu is "
786 "on disk but not in memory",
787 (unsigned long long)old_chunk);
788 return -EINVAL;
789 }
790
791 /*
792 * If this is the only chunk using this exception, remove exception.
793 */
794 if (!dm_consecutive_chunk_count(e)) {
795 dm_remove_exception(e);
796 free_completed_exception(e);
797 return 0;
798 }
799
800 /*
801 * The chunk may be either at the beginning or the end of a
802 * group of consecutive chunks - never in the middle. We are
803 * removing chunks in the opposite order to that in which they
804 * were added, so this should always be true.
805 * Decrement the consecutive chunk counter and adjust the
806 * starting point if necessary.
807 */
808 if (old_chunk == e->old_chunk) {
809 e->old_chunk++;
810 e->new_chunk++;
811 } else if (old_chunk != e->old_chunk +
812 dm_consecutive_chunk_count(e)) {
813 DMERR("Attempt to merge block %llu from the "
814 "middle of a chunk range [%llu - %llu]",
815 (unsigned long long)old_chunk,
816 (unsigned long long)e->old_chunk,
817 (unsigned long long)
818 e->old_chunk + dm_consecutive_chunk_count(e));
819 return -EINVAL;
820 }
821
822 dm_consecutive_chunk_count_dec(e);
823
824 return 0;
825}
826
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000827static void flush_bios(struct bio *bio);
828
829static int remove_single_exception_chunk(struct dm_snapshot *s)
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000830{
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000831 struct bio *b = NULL;
832 int r;
833 chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1;
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000834
835 down_write(&s->lock);
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000836
837 /*
838 * Process chunks (and associated exceptions) in reverse order
839 * so that dm_consecutive_chunk_count_dec() accounting works.
840 */
841 do {
842 r = __remove_single_exception_chunk(s, old_chunk);
843 if (r)
844 goto out;
845 } while (old_chunk-- > s->first_merging_chunk);
846
847 b = __release_queued_bios_after_merge(s);
848
849out:
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000850 up_write(&s->lock);
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000851 if (b)
852 flush_bios(b);
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000853
854 return r;
855}
856
Mikulas Patocka73dfd072009-12-10 23:52:34 +0000857static int origin_write_extent(struct dm_snapshot *merging_snap,
858 sector_t sector, unsigned chunk_size);
859
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000860static void merge_callback(int read_err, unsigned long write_err,
861 void *context);
862
Mikulas Patocka73dfd072009-12-10 23:52:34 +0000863static uint64_t read_pending_exceptions_done_count(void)
864{
865 uint64_t pending_exceptions_done;
866
867 spin_lock(&_pending_exceptions_done_spinlock);
868 pending_exceptions_done = _pending_exceptions_done_count;
869 spin_unlock(&_pending_exceptions_done_spinlock);
870
871 return pending_exceptions_done;
872}
873
874static void increment_pending_exceptions_done_count(void)
875{
876 spin_lock(&_pending_exceptions_done_spinlock);
877 _pending_exceptions_done_count++;
878 spin_unlock(&_pending_exceptions_done_spinlock);
879
880 wake_up_all(&_pending_exceptions_done);
881}
882
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000883static void snapshot_merge_next_chunks(struct dm_snapshot *s)
884{
Mike Snitzer8a2d5282009-12-10 23:52:34 +0000885 int i, linear_chunks;
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000886 chunk_t old_chunk, new_chunk;
887 struct dm_io_region src, dest;
Mike Snitzer8a2d5282009-12-10 23:52:34 +0000888 sector_t io_size;
Mikulas Patocka73dfd072009-12-10 23:52:34 +0000889 uint64_t previous_count;
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000890
891 BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits));
892 if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits)))
893 goto shut;
894
895 /*
896 * valid flag never changes during merge, so no lock required.
897 */
898 if (!s->valid) {
899 DMERR("Snapshot is invalid: can't merge");
900 goto shut;
901 }
902
Mike Snitzer8a2d5282009-12-10 23:52:34 +0000903 linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk,
904 &new_chunk);
905 if (linear_chunks <= 0) {
Mike Snitzerd8ddb1c2009-12-10 23:52:35 +0000906 if (linear_chunks < 0) {
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000907 DMERR("Read error in exception store: "
908 "shutting down merge");
Mike Snitzerd8ddb1c2009-12-10 23:52:35 +0000909 down_write(&s->lock);
910 s->merge_failed = 1;
911 up_write(&s->lock);
912 }
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000913 goto shut;
914 }
915
Mike Snitzer8a2d5282009-12-10 23:52:34 +0000916 /* Adjust old_chunk and new_chunk to reflect start of linear region */
917 old_chunk = old_chunk + 1 - linear_chunks;
918 new_chunk = new_chunk + 1 - linear_chunks;
919
920 /*
921 * Use one (potentially large) I/O to copy all 'linear_chunks'
922 * from the exception store to the origin
923 */
924 io_size = linear_chunks * s->store->chunk_size;
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000925
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000926 dest.bdev = s->origin->bdev;
927 dest.sector = chunk_to_sector(s->store, old_chunk);
Mike Snitzer8a2d5282009-12-10 23:52:34 +0000928 dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector);
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000929
930 src.bdev = s->cow->bdev;
931 src.sector = chunk_to_sector(s->store, new_chunk);
932 src.count = dest.count;
933
Mikulas Patocka73dfd072009-12-10 23:52:34 +0000934 /*
935 * Reallocate any exceptions needed in other snapshots then
936 * wait for the pending exceptions to complete.
937 * Each time any pending exception (globally on the system)
938 * completes we are woken and repeat the process to find out
939 * if we can proceed. While this may not seem a particularly
940 * efficient algorithm, it is not expected to have any
941 * significant impact on performance.
942 */
943 previous_count = read_pending_exceptions_done_count();
Mike Snitzer8a2d5282009-12-10 23:52:34 +0000944 while (origin_write_extent(s, dest.sector, io_size)) {
Mikulas Patocka73dfd072009-12-10 23:52:34 +0000945 wait_event(_pending_exceptions_done,
946 (read_pending_exceptions_done_count() !=
947 previous_count));
948 /* Retry after the wait, until all exceptions are done. */
949 previous_count = read_pending_exceptions_done_count();
950 }
951
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000952 down_write(&s->lock);
953 s->first_merging_chunk = old_chunk;
Mike Snitzer8a2d5282009-12-10 23:52:34 +0000954 s->num_merging_chunks = linear_chunks;
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000955 up_write(&s->lock);
956
Mike Snitzer8a2d5282009-12-10 23:52:34 +0000957 /* Wait until writes to all 'linear_chunks' drain */
958 for (i = 0; i < linear_chunks; i++)
959 __check_for_conflicting_io(s, old_chunk + i);
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000960
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000961 dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s);
962 return;
963
964shut:
965 merge_shutdown(s);
966}
967
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000968static void error_bios(struct bio *bio);
969
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000970static void merge_callback(int read_err, unsigned long write_err, void *context)
971{
972 struct dm_snapshot *s = context;
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000973 struct bio *b = NULL;
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000974
975 if (read_err || write_err) {
976 if (read_err)
977 DMERR("Read error: shutting down merge.");
978 else
979 DMERR("Write error: shutting down merge.");
980 goto shut;
981 }
982
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000983 if (s->store->type->commit_merge(s->store,
984 s->num_merging_chunks) < 0) {
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000985 DMERR("Write error in exception store: shutting down merge");
986 goto shut;
987 }
988
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000989 if (remove_single_exception_chunk(s) < 0)
990 goto shut;
991
Mikulas Patocka1e03f972009-12-10 23:52:32 +0000992 snapshot_merge_next_chunks(s);
993
994 return;
995
996shut:
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000997 down_write(&s->lock);
Mike Snitzerd8ddb1c2009-12-10 23:52:35 +0000998 s->merge_failed = 1;
Mikulas Patocka9fe862542009-12-10 23:52:33 +0000999 b = __release_queued_bios_after_merge(s);
1000 up_write(&s->lock);
1001 error_bios(b);
1002
Mikulas Patocka1e03f972009-12-10 23:52:32 +00001003 merge_shutdown(s);
1004}
1005
1006static void start_merge(struct dm_snapshot *s)
1007{
1008 if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits))
1009 snapshot_merge_next_chunks(s);
1010}
1011
1012static int wait_schedule(void *ptr)
1013{
1014 schedule();
1015
1016 return 0;
1017}
1018
1019/*
1020 * Stop the merging process and wait until it finishes.
1021 */
1022static void stop_merge(struct dm_snapshot *s)
1023{
1024 set_bit(SHUTDOWN_MERGE, &s->state_bits);
1025 wait_on_bit(&s->state_bits, RUNNING_MERGE, wait_schedule,
1026 TASK_UNINTERRUPTIBLE);
1027 clear_bit(SHUTDOWN_MERGE, &s->state_bits);
1028}
1029
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
1032 */
1033static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1034{
1035 struct dm_snapshot *s;
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001036 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 int r = -EINVAL;
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001038 char *origin_path, *cow_path;
Mike Snitzer10b81062009-12-10 23:52:31 +00001039 unsigned args_used, num_flush_requests = 1;
1040 fmode_t origin_mode = FMODE_READ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -07001042 if (argc != 4) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001043 ti->error = "requires exactly 4 arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044 r = -EINVAL;
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001045 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046 }
1047
Mike Snitzer10b81062009-12-10 23:52:31 +00001048 if (dm_target_is_snapshot_merge(ti)) {
1049 num_flush_requests = 2;
1050 origin_mode = FMODE_WRITE;
1051 }
1052
Linus Torvalds1da177e2005-04-16 15:20:36 -07001053 s = kmalloc(sizeof(*s), GFP_KERNEL);
Jonathan Brassowfee19982009-04-02 19:55:34 +01001054 if (!s) {
Jonathan Brassowa2d2b032011-08-02 12:32:03 +01001055 ti->error = "Cannot allocate private snapshot structure";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 r = -ENOMEM;
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001057 goto bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 }
1059
Mikulas Patockac2411042010-08-12 04:13:51 +01001060 origin_path = argv[0];
1061 argv++;
1062 argc--;
1063
1064 r = dm_get_device(ti, origin_path, origin_mode, &s->origin);
1065 if (r) {
1066 ti->error = "Cannot get origin device";
1067 goto bad_origin;
1068 }
1069
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001070 cow_path = argv[0];
1071 argv++;
1072 argc--;
1073
Milan Broz024d37e2011-03-24 13:52:14 +00001074 r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow);
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001075 if (r) {
1076 ti->error = "Cannot get COW device";
1077 goto bad_cow;
1078 }
1079
1080 r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store);
1081 if (r) {
1082 ti->error = "Couldn't create exception store";
1083 r = -EINVAL;
1084 goto bad_store;
1085 }
1086
1087 argv += args_used;
1088 argc -= args_used;
1089
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001090 s->ti = ti;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091 s->valid = 1;
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001092 s->active = 0;
Mikulas Patocka879129d22008-10-30 13:33:16 +00001093 atomic_set(&s->pending_exceptions_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 init_rwsem(&s->lock);
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001095 INIT_LIST_HEAD(&s->list);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001096 spin_lock_init(&s->pe_lock);
Mikulas Patocka1e03f972009-12-10 23:52:32 +00001097 s->state_bits = 0;
Mike Snitzerd8ddb1c2009-12-10 23:52:35 +00001098 s->merge_failed = 0;
Mikulas Patocka9fe862542009-12-10 23:52:33 +00001099 s->first_merging_chunk = 0;
1100 s->num_merging_chunks = 0;
1101 bio_list_init(&s->bios_queued_during_merge);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102
1103 /* Allocate hash table for COW data */
Jonathan Brassowfee19982009-04-02 19:55:34 +01001104 if (init_hash_tables(s)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105 ti->error = "Unable to allocate hash table space";
1106 r = -ENOMEM;
Jonathan Brassowfee19982009-04-02 19:55:34 +01001107 goto bad_hash_tables;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 }
1109
Mikulas Patockafa34ce72011-05-29 13:03:13 +01001110 s->kcopyd_client = dm_kcopyd_client_create();
1111 if (IS_ERR(s->kcopyd_client)) {
1112 r = PTR_ERR(s->kcopyd_client);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113 ti->error = "Could not create kcopyd client";
Jonathan Brassowfee19982009-04-02 19:55:34 +01001114 goto bad_kcopyd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 }
1116
Mikulas Patocka92e86812008-07-21 12:00:35 +01001117 s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache);
1118 if (!s->pending_pool) {
1119 ti->error = "Could not allocate mempool for pending exceptions";
Jonathan Brassowfee19982009-04-02 19:55:34 +01001120 goto bad_pending_pool;
Mikulas Patocka92e86812008-07-21 12:00:35 +01001121 }
1122
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001123 s->tracked_chunk_pool = mempool_create_slab_pool(MIN_IOS,
1124 tracked_chunk_cache);
1125 if (!s->tracked_chunk_pool) {
1126 ti->error = "Could not allocate tracked_chunk mempool for "
1127 "tracking reads";
Mikulas Patocka92e86812008-07-21 12:00:35 +01001128 goto bad_tracked_chunk_pool;
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001129 }
1130
1131 for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
1132 INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
1133
1134 spin_lock_init(&s->tracked_chunk_lock);
1135
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001136 ti->private = s;
Mike Snitzer10b81062009-12-10 23:52:31 +00001137 ti->num_flush_requests = num_flush_requests;
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001138
1139 /* Add snapshot to the list of snapshots for this origin */
1140 /* Exceptions aren't triggered till snapshot_resume() is called */
1141 r = register_snapshot(s);
1142 if (r == -ENOMEM) {
1143 ti->error = "Snapshot origin struct allocation failed";
1144 goto bad_load_and_register;
1145 } else if (r < 0) {
1146 /* invalid handover, register_snapshot has set ti->error */
1147 goto bad_load_and_register;
1148 }
1149
1150 /*
1151 * Metadata must only be loaded into one table at once, so skip this
1152 * if metadata will be handed over during resume.
1153 * Chunk size will be set during the handover - set it to zero to
1154 * ensure it's ignored.
1155 */
1156 if (r > 0) {
1157 s->store->chunk_size = 0;
1158 return 0;
1159 }
1160
Jonathan Brassow493df712009-04-02 19:55:31 +01001161 r = s->store->type->read_metadata(s->store, dm_add_exception,
1162 (void *)s);
Milan Broz07641472007-07-12 17:28:13 +01001163 if (r < 0) {
Mark McLoughlinf9cea4f2006-10-03 01:15:25 -07001164 ti->error = "Failed to read snapshot metadata";
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001165 goto bad_read_metadata;
Milan Broz07641472007-07-12 17:28:13 +01001166 } else if (r > 0) {
1167 s->valid = 0;
1168 DMWARN("Snapshot is marked invalid.");
Mark McLoughlinf9cea4f2006-10-03 01:15:25 -07001169 }
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001170
Mikulas Patocka3f2412d2009-10-16 23:18:16 +01001171 if (!s->store->chunk_size) {
1172 ti->error = "Chunk size not set";
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001173 goto bad_read_metadata;
Mikulas Patocka3f2412d2009-10-16 23:18:16 +01001174 }
Jonathan Brassowd0216842009-04-02 19:55:32 +01001175 ti->split_io = s->store->chunk_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176
1177 return 0;
1178
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001179bad_read_metadata:
1180 unregister_snapshot(s);
1181
Jonathan Brassowfee19982009-04-02 19:55:34 +01001182bad_load_and_register:
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001183 mempool_destroy(s->tracked_chunk_pool);
1184
Jonathan Brassowfee19982009-04-02 19:55:34 +01001185bad_tracked_chunk_pool:
Mikulas Patocka92e86812008-07-21 12:00:35 +01001186 mempool_destroy(s->pending_pool);
1187
Jonathan Brassowfee19982009-04-02 19:55:34 +01001188bad_pending_pool:
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +01001189 dm_kcopyd_client_destroy(s->kcopyd_client);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001190
Jonathan Brassowfee19982009-04-02 19:55:34 +01001191bad_kcopyd:
Jon Brassow3510cb92009-12-10 23:52:11 +00001192 dm_exception_table_exit(&s->pending, pending_cache);
1193 dm_exception_table_exit(&s->complete, exception_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194
Jonathan Brassowfee19982009-04-02 19:55:34 +01001195bad_hash_tables:
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001196 dm_exception_store_destroy(s->store);
1197
1198bad_store:
1199 dm_put_device(ti, s->cow);
1200
1201bad_cow:
Mikulas Patockac2411042010-08-12 04:13:51 +01001202 dm_put_device(ti, s->origin);
1203
1204bad_origin:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205 kfree(s);
1206
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001207bad:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208 return r;
1209}
1210
Milan Broz31c93a02006-12-08 02:41:11 -08001211static void __free_exceptions(struct dm_snapshot *s)
1212{
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +01001213 dm_kcopyd_client_destroy(s->kcopyd_client);
Milan Broz31c93a02006-12-08 02:41:11 -08001214 s->kcopyd_client = NULL;
1215
Jon Brassow3510cb92009-12-10 23:52:11 +00001216 dm_exception_table_exit(&s->pending, pending_cache);
1217 dm_exception_table_exit(&s->complete, exception_cache);
Milan Broz31c93a02006-12-08 02:41:11 -08001218}
1219
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001220static void __handover_exceptions(struct dm_snapshot *snap_src,
1221 struct dm_snapshot *snap_dest)
1222{
1223 union {
1224 struct dm_exception_table table_swap;
1225 struct dm_exception_store *store_swap;
1226 } u;
1227
1228 /*
1229 * Swap all snapshot context information between the two instances.
1230 */
1231 u.table_swap = snap_dest->complete;
1232 snap_dest->complete = snap_src->complete;
1233 snap_src->complete = u.table_swap;
1234
1235 u.store_swap = snap_dest->store;
1236 snap_dest->store = snap_src->store;
1237 snap_src->store = u.store_swap;
1238
1239 snap_dest->store->snap = snap_dest;
1240 snap_src->store->snap = snap_src;
1241
1242 snap_dest->ti->split_io = snap_dest->store->chunk_size;
1243 snap_dest->valid = snap_src->valid;
1244
1245 /*
1246 * Set source invalid to ensure it receives no further I/O.
1247 */
1248 snap_src->valid = 0;
1249}
1250
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251static void snapshot_dtr(struct dm_target *ti)
1252{
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001253#ifdef CONFIG_DM_DEBUG
1254 int i;
1255#endif
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001256 struct dm_snapshot *s = ti->private;
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001257 struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001259 down_read(&_origins_lock);
1260 /* Check whether exception handover must be cancelled */
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +00001261 (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001262 if (snap_src && snap_dest && (s == snap_src)) {
1263 down_write(&snap_dest->lock);
1264 snap_dest->valid = 0;
1265 up_write(&snap_dest->lock);
1266 DMERR("Cancelling snapshot handover.");
1267 }
1268 up_read(&_origins_lock);
1269
Mikulas Patocka1e03f972009-12-10 23:52:32 +00001270 if (dm_target_is_snapshot_merge(ti))
1271 stop_merge(s);
1272
Alasdair G Kergon138728dc2006-03-27 01:17:50 -08001273 /* Prevent further origin writes from using this snapshot. */
1274 /* After this returns there can be no new kcopyd jobs. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275 unregister_snapshot(s);
1276
Mikulas Patocka879129d22008-10-30 13:33:16 +00001277 while (atomic_read(&s->pending_exceptions_count))
Mikulas Patocka90fa1522009-01-06 03:04:54 +00001278 msleep(1);
Mikulas Patocka879129d22008-10-30 13:33:16 +00001279 /*
1280 * Ensure instructions in mempool_destroy aren't reordered
1281 * before atomic_read.
1282 */
1283 smp_mb();
1284
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001285#ifdef CONFIG_DM_DEBUG
1286 for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
1287 BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));
1288#endif
1289
1290 mempool_destroy(s->tracked_chunk_pool);
1291
Milan Broz31c93a02006-12-08 02:41:11 -08001292 __free_exceptions(s);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293
Mikulas Patocka92e86812008-07-21 12:00:35 +01001294 mempool_destroy(s->pending_pool);
1295
Jonathan Brassowfee19982009-04-02 19:55:34 +01001296 dm_exception_store_destroy(s->store);
Alasdair G Kergon138728dc2006-03-27 01:17:50 -08001297
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001298 dm_put_device(ti, s->cow);
1299
Mikulas Patockac2411042010-08-12 04:13:51 +01001300 dm_put_device(ti, s->origin);
1301
Linus Torvalds1da177e2005-04-16 15:20:36 -07001302 kfree(s);
1303}
1304
1305/*
1306 * Flush a list of buffers.
1307 */
1308static void flush_bios(struct bio *bio)
1309{
1310 struct bio *n;
1311
1312 while (bio) {
1313 n = bio->bi_next;
1314 bio->bi_next = NULL;
1315 generic_make_request(bio);
1316 bio = n;
1317 }
1318}
1319
Mikulas Patocka515ad662009-12-10 23:52:30 +00001320static int do_origin(struct dm_dev *origin, struct bio *bio);
1321
1322/*
1323 * Flush a list of buffers.
1324 */
1325static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio)
1326{
1327 struct bio *n;
1328 int r;
1329
1330 while (bio) {
1331 n = bio->bi_next;
1332 bio->bi_next = NULL;
1333 r = do_origin(s->origin, bio);
1334 if (r == DM_MAPIO_REMAPPED)
1335 generic_make_request(bio);
1336 bio = n;
1337 }
1338}
1339
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340/*
1341 * Error a list of buffers.
1342 */
1343static void error_bios(struct bio *bio)
1344{
1345 struct bio *n;
1346
1347 while (bio) {
1348 n = bio->bi_next;
1349 bio->bi_next = NULL;
NeilBrown6712ecf2007-09-27 12:47:43 +02001350 bio_io_error(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351 bio = n;
1352 }
1353}
1354
Alasdair G Kergon695368a2006-10-03 01:15:31 -07001355static void __invalidate_snapshot(struct dm_snapshot *s, int err)
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001356{
1357 if (!s->valid)
1358 return;
1359
1360 if (err == -EIO)
1361 DMERR("Invalidating snapshot: Error reading/writing.");
1362 else if (err == -ENOMEM)
1363 DMERR("Invalidating snapshot: Unable to allocate exception.");
1364
Jonathan Brassow493df712009-04-02 19:55:31 +01001365 if (s->store->type->drop_snapshot)
1366 s->store->type->drop_snapshot(s->store);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001367
1368 s->valid = 0;
1369
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001370 dm_table_event(s->ti->table);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001371}
1372
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001373static void pending_complete(struct dm_snap_pending_exception *pe, int success)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374{
Jon Brassow1d4989c2009-12-10 23:52:10 +00001375 struct dm_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 struct dm_snapshot *s = pe->snap;
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001377 struct bio *origin_bios = NULL;
1378 struct bio *snapshot_bios = NULL;
Mikulas Patockaa6e50b42011-08-02 12:32:04 +01001379 struct bio *full_bio = NULL;
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001380 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001382 if (!success) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383 /* Read/write error - snapshot is unusable */
1384 down_write(&s->lock);
Alasdair G Kergon695368a2006-10-03 01:15:31 -07001385 __invalidate_snapshot(s, -EIO);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001386 error = 1;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001387 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388 }
1389
Jon Brassow3510cb92009-12-10 23:52:11 +00001390 e = alloc_completed_exception();
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001391 if (!e) {
1392 down_write(&s->lock);
Alasdair G Kergon695368a2006-10-03 01:15:31 -07001393 __invalidate_snapshot(s, -ENOMEM);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001394 error = 1;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001395 goto out;
1396 }
1397 *e = pe->e;
1398
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001399 down_write(&s->lock);
1400 if (!s->valid) {
Jon Brassow3510cb92009-12-10 23:52:11 +00001401 free_completed_exception(e);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001402 error = 1;
1403 goto out;
1404 }
1405
Mike Snitzer615d1eb2009-12-10 23:52:29 +00001406 /* Check for conflicting reads */
1407 __check_for_conflicting_io(s, pe->e.old_chunk);
Mikulas Patockaa8d41b52008-07-21 12:00:34 +01001408
1409 /*
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001410 * Add a proper exception, and remove the
1411 * in-flight exception from the list.
1412 */
Jon Brassow3510cb92009-12-10 23:52:11 +00001413 dm_insert_exception(&s->complete, e);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001414
Jonathan Brassowa2d2b032011-08-02 12:32:03 +01001415out:
Jon Brassow3510cb92009-12-10 23:52:11 +00001416 dm_remove_exception(&pe->e);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001417 snapshot_bios = bio_list_get(&pe->snapshot_bios);
Mikulas Patocka515ad662009-12-10 23:52:30 +00001418 origin_bios = bio_list_get(&pe->origin_bios);
Mikulas Patockaa6e50b42011-08-02 12:32:04 +01001419 full_bio = pe->full_bio;
1420 if (full_bio) {
1421 full_bio->bi_end_io = pe->full_bio_end_io;
1422 full_bio->bi_private = pe->full_bio_private;
1423 }
Mikulas Patocka515ad662009-12-10 23:52:30 +00001424 free_pending_exception(pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425
Mikulas Patocka73dfd072009-12-10 23:52:34 +00001426 increment_pending_exceptions_done_count();
1427
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001428 up_write(&s->lock);
1429
1430 /* Submit any pending write bios */
Mikulas Patockaa6e50b42011-08-02 12:32:04 +01001431 if (error) {
1432 if (full_bio)
1433 bio_io_error(full_bio);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001434 error_bios(snapshot_bios);
Mikulas Patockaa6e50b42011-08-02 12:32:04 +01001435 } else {
1436 if (full_bio)
1437 bio_endio(full_bio, 0);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001438 flush_bios(snapshot_bios);
Mikulas Patockaa6e50b42011-08-02 12:32:04 +01001439 }
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -07001440
Mikulas Patocka515ad662009-12-10 23:52:30 +00001441 retry_origin_bios(s, origin_bios);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442}
1443
1444static void commit_callback(void *context, int success)
1445{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001446 struct dm_snap_pending_exception *pe = context;
1447
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448 pending_complete(pe, success);
1449}
1450
1451/*
1452 * Called when the copy I/O has finished. kcopyd actually runs
1453 * this code so don't block.
1454 */
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -07001455static void copy_callback(int read_err, unsigned long write_err, void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001456{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001457 struct dm_snap_pending_exception *pe = context;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001458 struct dm_snapshot *s = pe->snap;
1459
1460 if (read_err || write_err)
1461 pending_complete(pe, 0);
1462
1463 else
1464 /* Update the metadata if we are persistent */
Jonathan Brassow493df712009-04-02 19:55:31 +01001465 s->store->type->commit_exception(s->store, &pe->e,
1466 commit_callback, pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001467}
1468
1469/*
1470 * Dispatches the copy operation to kcopyd.
1471 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001472static void start_copy(struct dm_snap_pending_exception *pe)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001473{
1474 struct dm_snapshot *s = pe->snap;
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +01001475 struct dm_io_region src, dest;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001476 struct block_device *bdev = s->origin->bdev;
1477 sector_t dev_size;
1478
1479 dev_size = get_dev_size(bdev);
1480
1481 src.bdev = bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +01001482 src.sector = chunk_to_sector(s->store, pe->e.old_chunk);
Mikulas Patockadf96eee2009-10-16 23:18:17 +01001483 src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001485 dest.bdev = s->cow->bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +01001486 dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001487 dest.count = src.count;
1488
1489 /* Hand over to kcopyd */
Jonathan Brassowa2d2b032011-08-02 12:32:03 +01001490 dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491}
1492
Mikulas Patockaa6e50b42011-08-02 12:32:04 +01001493static void full_bio_end_io(struct bio *bio, int error)
1494{
1495 void *callback_data = bio->bi_private;
1496
1497 dm_kcopyd_do_callback(callback_data, 0, error ? 1 : 0);
1498}
1499
1500static void start_full_bio(struct dm_snap_pending_exception *pe,
1501 struct bio *bio)
1502{
1503 struct dm_snapshot *s = pe->snap;
1504 void *callback_data;
1505
1506 pe->full_bio = bio;
1507 pe->full_bio_end_io = bio->bi_end_io;
1508 pe->full_bio_private = bio->bi_private;
1509
1510 callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client,
1511 copy_callback, pe);
1512
1513 bio->bi_end_io = full_bio_end_io;
1514 bio->bi_private = callback_data;
1515
1516 generic_make_request(bio);
1517}
1518
Mikulas Patocka29138082009-04-02 19:55:25 +01001519static struct dm_snap_pending_exception *
1520__lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)
1521{
Jon Brassow3510cb92009-12-10 23:52:11 +00001522 struct dm_exception *e = dm_lookup_exception(&s->pending, chunk);
Mikulas Patocka29138082009-04-02 19:55:25 +01001523
1524 if (!e)
1525 return NULL;
1526
1527 return container_of(e, struct dm_snap_pending_exception, e);
1528}
1529
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530/*
1531 * Looks to see if this snapshot already has a pending exception
1532 * for this chunk, otherwise it allocates a new one and inserts
1533 * it into the pending table.
1534 *
1535 * NOTE: a write lock must be held on snap->lock before calling
1536 * this.
1537 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001538static struct dm_snap_pending_exception *
Mikulas Patockac6621392009-04-02 19:55:25 +01001539__find_pending_exception(struct dm_snapshot *s,
1540 struct dm_snap_pending_exception *pe, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541{
Mikulas Patockac6621392009-04-02 19:55:25 +01001542 struct dm_snap_pending_exception *pe2;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001543
Mikulas Patocka29138082009-04-02 19:55:25 +01001544 pe2 = __lookup_pending_exception(s, chunk);
1545 if (pe2) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001546 free_pending_exception(pe);
Mikulas Patocka29138082009-04-02 19:55:25 +01001547 return pe2;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001548 }
1549
1550 pe->e.old_chunk = chunk;
1551 bio_list_init(&pe->origin_bios);
1552 bio_list_init(&pe->snapshot_bios);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001553 pe->started = 0;
Mikulas Patockaa6e50b42011-08-02 12:32:04 +01001554 pe->full_bio = NULL;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001555
Jonathan Brassow493df712009-04-02 19:55:31 +01001556 if (s->store->type->prepare_exception(s->store, &pe->e)) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001557 free_pending_exception(pe);
1558 return NULL;
1559 }
1560
Jon Brassow3510cb92009-12-10 23:52:11 +00001561 dm_insert_exception(&s->pending, &pe->e);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001562
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563 return pe;
1564}
1565
Jon Brassow1d4989c2009-12-10 23:52:10 +00001566static void remap_exception(struct dm_snapshot *s, struct dm_exception *e,
Milan Brozd74f81f2008-02-08 02:11:27 +00001567 struct bio *bio, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001568{
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001569 bio->bi_bdev = s->cow->bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +01001570 bio->bi_sector = chunk_to_sector(s->store,
1571 dm_chunk_number(e->new_chunk) +
1572 (chunk - e->old_chunk)) +
1573 (bio->bi_sector &
1574 s->store->chunk_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575}
1576
1577static int snapshot_map(struct dm_target *ti, struct bio *bio,
1578 union map_info *map_context)
1579{
Jon Brassow1d4989c2009-12-10 23:52:10 +00001580 struct dm_exception *e;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001581 struct dm_snapshot *s = ti->private;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001582 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 chunk_t chunk;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001584 struct dm_snap_pending_exception *pe = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585
Tejun Heod87f4c12010-09-03 11:56:19 +02001586 if (bio->bi_rw & REQ_FLUSH) {
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001587 bio->bi_bdev = s->cow->bdev;
Mikulas Patocka494b3ee2009-06-22 10:12:25 +01001588 return DM_MAPIO_REMAPPED;
1589 }
1590
Jonathan Brassow71fab002009-04-02 19:55:33 +01001591 chunk = sector_to_chunk(s->store, bio->bi_sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001592
1593 /* Full snapshots are not usable */
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001594 /* To get here the table must be live so s->active is always set. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595 if (!s->valid)
Alasdair G Kergonf6a80ea2005-07-12 15:53:01 -07001596 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001598 /* FIXME: should only take write lock if we need
1599 * to copy an exception */
1600 down_write(&s->lock);
1601
1602 if (!s->valid) {
1603 r = -EIO;
1604 goto out_unlock;
1605 }
1606
1607 /* If the block is already remapped - use that, else remap it */
Jon Brassow3510cb92009-12-10 23:52:11 +00001608 e = dm_lookup_exception(&s->complete, chunk);
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001609 if (e) {
Milan Brozd74f81f2008-02-08 02:11:27 +00001610 remap_exception(s, e, bio, chunk);
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001611 goto out_unlock;
1612 }
1613
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614 /*
1615 * Write to snapshot - higher level takes care of RW/RO
1616 * flags so we should only get this if we are
1617 * writeable.
1618 */
1619 if (bio_rw(bio) == WRITE) {
Mikulas Patocka29138082009-04-02 19:55:25 +01001620 pe = __lookup_pending_exception(s, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001621 if (!pe) {
Mikulas Patockac6621392009-04-02 19:55:25 +01001622 up_write(&s->lock);
1623 pe = alloc_pending_exception(s);
1624 down_write(&s->lock);
1625
1626 if (!s->valid) {
1627 free_pending_exception(pe);
1628 r = -EIO;
1629 goto out_unlock;
1630 }
1631
Jon Brassow3510cb92009-12-10 23:52:11 +00001632 e = dm_lookup_exception(&s->complete, chunk);
Mikulas Patocka35bf6592009-04-02 19:55:26 +01001633 if (e) {
1634 free_pending_exception(pe);
1635 remap_exception(s, e, bio, chunk);
1636 goto out_unlock;
1637 }
1638
Mikulas Patockac6621392009-04-02 19:55:25 +01001639 pe = __find_pending_exception(s, pe, chunk);
Mikulas Patocka29138082009-04-02 19:55:25 +01001640 if (!pe) {
1641 __invalidate_snapshot(s, -ENOMEM);
1642 r = -EIO;
1643 goto out_unlock;
1644 }
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001645 }
1646
Milan Brozd74f81f2008-02-08 02:11:27 +00001647 remap_exception(s, &pe->e, bio, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001648
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001649 r = DM_MAPIO_SUBMITTED;
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001650
Mikulas Patockaa6e50b42011-08-02 12:32:04 +01001651 if (!pe->started &&
1652 bio->bi_size == (s->store->chunk_size << SECTOR_SHIFT)) {
1653 pe->started = 1;
1654 up_write(&s->lock);
1655 start_full_bio(pe, bio);
1656 goto out;
1657 }
1658
1659 bio_list_add(&pe->snapshot_bios, bio);
1660
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001661 if (!pe->started) {
1662 /* this is protected by snap->lock */
1663 pe->started = 1;
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001664 up_write(&s->lock);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001665 start_copy(pe);
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001666 goto out;
1667 }
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001668 } else {
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001669 bio->bi_bdev = s->origin->bdev;
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001670 map_context->ptr = track_chunk(s, chunk);
1671 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672
Jonathan Brassowa2d2b032011-08-02 12:32:03 +01001673out_unlock:
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001674 up_write(&s->lock);
Jonathan Brassowa2d2b032011-08-02 12:32:03 +01001675out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 return r;
1677}
1678
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001679/*
1680 * A snapshot-merge target behaves like a combination of a snapshot
1681 * target and a snapshot-origin target. It only generates new
1682 * exceptions in other snapshots and not in the one that is being
1683 * merged.
1684 *
1685 * For each chunk, if there is an existing exception, it is used to
1686 * redirect I/O to the cow device. Otherwise I/O is sent to the origin,
1687 * which in turn might generate exceptions in other snapshots.
Mikulas Patocka9fe862542009-12-10 23:52:33 +00001688 * If merging is currently taking place on the chunk in question, the
1689 * I/O is deferred by adding it to s->bios_queued_during_merge.
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001690 */
1691static int snapshot_merge_map(struct dm_target *ti, struct bio *bio,
1692 union map_info *map_context)
1693{
1694 struct dm_exception *e;
1695 struct dm_snapshot *s = ti->private;
1696 int r = DM_MAPIO_REMAPPED;
1697 chunk_t chunk;
1698
Tejun Heod87f4c12010-09-03 11:56:19 +02001699 if (bio->bi_rw & REQ_FLUSH) {
Mike Snitzer57cba5d2010-08-12 04:14:04 +01001700 if (!map_context->target_request_nr)
Mike Snitzer10b81062009-12-10 23:52:31 +00001701 bio->bi_bdev = s->origin->bdev;
1702 else
1703 bio->bi_bdev = s->cow->bdev;
1704 map_context->ptr = NULL;
1705 return DM_MAPIO_REMAPPED;
1706 }
1707
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001708 chunk = sector_to_chunk(s->store, bio->bi_sector);
1709
Mikulas Patocka9fe862542009-12-10 23:52:33 +00001710 down_write(&s->lock);
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001711
Mikulas Patockad2fdb772009-12-10 23:52:36 +00001712 /* Full merging snapshots are redirected to the origin */
1713 if (!s->valid)
1714 goto redirect_to_origin;
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001715
1716 /* If the block is already remapped - use that */
1717 e = dm_lookup_exception(&s->complete, chunk);
1718 if (e) {
Mikulas Patocka9fe862542009-12-10 23:52:33 +00001719 /* Queue writes overlapping with chunks being merged */
1720 if (bio_rw(bio) == WRITE &&
1721 chunk >= s->first_merging_chunk &&
1722 chunk < (s->first_merging_chunk +
1723 s->num_merging_chunks)) {
1724 bio->bi_bdev = s->origin->bdev;
1725 bio_list_add(&s->bios_queued_during_merge, bio);
1726 r = DM_MAPIO_SUBMITTED;
1727 goto out_unlock;
1728 }
Mikulas Patocka17aa0332009-12-10 23:52:33 +00001729
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001730 remap_exception(s, e, bio, chunk);
Mikulas Patocka17aa0332009-12-10 23:52:33 +00001731
1732 if (bio_rw(bio) == WRITE)
1733 map_context->ptr = track_chunk(s, chunk);
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001734 goto out_unlock;
1735 }
1736
Mikulas Patockad2fdb772009-12-10 23:52:36 +00001737redirect_to_origin:
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001738 bio->bi_bdev = s->origin->bdev;
1739
1740 if (bio_rw(bio) == WRITE) {
Mikulas Patocka9fe862542009-12-10 23:52:33 +00001741 up_write(&s->lock);
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001742 return do_origin(s->origin, bio);
1743 }
1744
1745out_unlock:
Mikulas Patocka9fe862542009-12-10 23:52:33 +00001746 up_write(&s->lock);
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001747
1748 return r;
1749}
1750
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001751static int snapshot_end_io(struct dm_target *ti, struct bio *bio,
1752 int error, union map_info *map_context)
1753{
1754 struct dm_snapshot *s = ti->private;
1755 struct dm_snap_tracked_chunk *c = map_context->ptr;
1756
1757 if (c)
1758 stop_tracking_chunk(s, c);
1759
1760 return 0;
1761}
1762
Mikulas Patocka1e03f972009-12-10 23:52:32 +00001763static void snapshot_merge_presuspend(struct dm_target *ti)
1764{
1765 struct dm_snapshot *s = ti->private;
1766
1767 stop_merge(s);
1768}
1769
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001770static int snapshot_preresume(struct dm_target *ti)
1771{
1772 int r = 0;
1773 struct dm_snapshot *s = ti->private;
1774 struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
1775
1776 down_read(&_origins_lock);
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +00001777 (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001778 if (snap_src && snap_dest) {
1779 down_read(&snap_src->lock);
1780 if (s == snap_src) {
1781 DMERR("Unable to resume snapshot source until "
1782 "handover completes.");
1783 r = -EINVAL;
Mike Snitzerb83b2f22011-01-13 19:59:59 +00001784 } else if (!dm_suspended(snap_src->ti)) {
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001785 DMERR("Unable to perform snapshot handover until "
1786 "source is suspended.");
1787 r = -EINVAL;
1788 }
1789 up_read(&snap_src->lock);
1790 }
1791 up_read(&_origins_lock);
1792
1793 return r;
1794}
1795
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796static void snapshot_resume(struct dm_target *ti)
1797{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001798 struct dm_snapshot *s = ti->private;
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001799 struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
1800
1801 down_read(&_origins_lock);
Mikulas Patocka9d3b15c2009-12-10 23:52:32 +00001802 (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
Mike Snitzerc1f0c182009-12-10 23:52:24 +00001803 if (snap_src && snap_dest) {
1804 down_write(&snap_src->lock);
1805 down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING);
1806 __handover_exceptions(snap_src, snap_dest);
1807 up_write(&snap_dest->lock);
1808 up_write(&snap_src->lock);
1809 }
1810 up_read(&_origins_lock);
1811
1812 /* Now we have correct chunk size, reregister */
1813 reregister_snapshot(s);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001814
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001815 down_write(&s->lock);
1816 s->active = 1;
1817 up_write(&s->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001818}
1819
Mikulas Patocka1e03f972009-12-10 23:52:32 +00001820static sector_t get_origin_minimum_chunksize(struct block_device *bdev)
1821{
1822 sector_t min_chunksize;
1823
1824 down_read(&_origins_lock);
1825 min_chunksize = __minimum_chunk_size(__lookup_origin(bdev));
1826 up_read(&_origins_lock);
1827
1828 return min_chunksize;
1829}
1830
1831static void snapshot_merge_resume(struct dm_target *ti)
1832{
1833 struct dm_snapshot *s = ti->private;
1834
1835 /*
1836 * Handover exceptions from existing snapshot.
1837 */
1838 snapshot_resume(ti);
1839
1840 /*
1841 * snapshot-merge acts as an origin, so set ti->split_io
1842 */
1843 ti->split_io = get_origin_minimum_chunksize(s->origin->bdev);
1844
1845 start_merge(s);
1846}
1847
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848static int snapshot_status(struct dm_target *ti, status_type_t type,
1849 char *result, unsigned int maxlen)
1850{
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001851 unsigned sz = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001852 struct dm_snapshot *snap = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853
1854 switch (type) {
1855 case STATUSTYPE_INFO:
Mikulas Patocka94e765722009-12-10 23:51:53 +00001856
1857 down_write(&snap->lock);
1858
Linus Torvalds1da177e2005-04-16 15:20:36 -07001859 if (!snap->valid)
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001860 DMEMIT("Invalid");
Mike Snitzerd8ddb1c2009-12-10 23:52:35 +00001861 else if (snap->merge_failed)
1862 DMEMIT("Merge failed");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863 else {
Mike Snitzer985903b2009-12-10 23:52:11 +00001864 if (snap->store->type->usage) {
1865 sector_t total_sectors, sectors_allocated,
1866 metadata_sectors;
1867 snap->store->type->usage(snap->store,
1868 &total_sectors,
1869 &sectors_allocated,
1870 &metadata_sectors);
1871 DMEMIT("%llu/%llu %llu",
1872 (unsigned long long)sectors_allocated,
1873 (unsigned long long)total_sectors,
1874 (unsigned long long)metadata_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001875 }
1876 else
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001877 DMEMIT("Unknown");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001878 }
Mikulas Patocka94e765722009-12-10 23:51:53 +00001879
1880 up_write(&snap->lock);
1881
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882 break;
1883
1884 case STATUSTYPE_TABLE:
1885 /*
1886 * kdevname returns a static pointer so we need
1887 * to make private copies if the output is to
1888 * make sense.
1889 */
Mike Snitzerfc56f6f2009-12-10 23:52:12 +00001890 DMEMIT("%s %s", snap->origin->name, snap->cow->name);
Jonathan Brassow1e302a92009-04-02 19:55:35 +01001891 snap->store->type->status(snap->store, type, result + sz,
1892 maxlen - sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001893 break;
1894 }
1895
1896 return 0;
1897}
1898
Mike Snitzer8811f462009-09-04 20:40:19 +01001899static int snapshot_iterate_devices(struct dm_target *ti,
1900 iterate_devices_callout_fn fn, void *data)
1901{
1902 struct dm_snapshot *snap = ti->private;
Mikulas Patocka1e5554c2010-08-12 04:13:50 +01001903 int r;
Mike Snitzer8811f462009-09-04 20:40:19 +01001904
Mikulas Patocka1e5554c2010-08-12 04:13:50 +01001905 r = fn(ti, snap->origin, 0, ti->len, data);
1906
1907 if (!r)
1908 r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data);
1909
1910 return r;
Mike Snitzer8811f462009-09-04 20:40:19 +01001911}
1912
1913
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914/*-----------------------------------------------------------------
1915 * Origin methods
1916 *---------------------------------------------------------------*/
Mikulas Patocka9eaae8f2009-12-10 23:52:28 +00001917
1918/*
1919 * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any
1920 * supplied bio was ignored. The caller may submit it immediately.
1921 * (No remapping actually occurs as the origin is always a direct linear
1922 * map.)
1923 *
1924 * If further exceptions are required, DM_MAPIO_SUBMITTED is returned
1925 * and any supplied bio is added to a list to be submitted once all
1926 * the necessary exceptions exist.
1927 */
1928static int __origin_write(struct list_head *snapshots, sector_t sector,
1929 struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001930{
Mikulas Patocka515ad662009-12-10 23:52:30 +00001931 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001932 struct dm_snapshot *snap;
Jon Brassow1d4989c2009-12-10 23:52:10 +00001933 struct dm_exception *e;
Mikulas Patocka515ad662009-12-10 23:52:30 +00001934 struct dm_snap_pending_exception *pe;
1935 struct dm_snap_pending_exception *pe_to_start_now = NULL;
1936 struct dm_snap_pending_exception *pe_to_start_last = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001937 chunk_t chunk;
1938
1939 /* Do all the snapshots on this origin */
1940 list_for_each_entry (snap, snapshots, list) {
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00001941 /*
1942 * Don't make new exceptions in a merging snapshot
1943 * because it has effectively been deleted
1944 */
1945 if (dm_target_is_snapshot_merge(snap->ti))
1946 continue;
1947
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001948 down_write(&snap->lock);
1949
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001950 /* Only deal with valid and active snapshots */
1951 if (!snap->valid || !snap->active)
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001952 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001953
Alasdair G Kergond5e404c2005-07-12 15:53:05 -07001954 /* Nothing to do if writing beyond end of snapshot */
Mikulas Patocka9eaae8f2009-12-10 23:52:28 +00001955 if (sector >= dm_table_get_size(snap->ti->table))
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001956 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957
1958 /*
1959 * Remember, different snapshots can have
1960 * different chunk sizes.
1961 */
Mikulas Patocka9eaae8f2009-12-10 23:52:28 +00001962 chunk = sector_to_chunk(snap->store, sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001963
1964 /*
1965 * Check exception table to see if block
1966 * is already remapped in this snapshot
1967 * and trigger an exception if not.
1968 */
Jon Brassow3510cb92009-12-10 23:52:11 +00001969 e = dm_lookup_exception(&snap->complete, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001970 if (e)
1971 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001972
Mikulas Patocka29138082009-04-02 19:55:25 +01001973 pe = __lookup_pending_exception(snap, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001974 if (!pe) {
Mikulas Patockac6621392009-04-02 19:55:25 +01001975 up_write(&snap->lock);
1976 pe = alloc_pending_exception(snap);
1977 down_write(&snap->lock);
1978
1979 if (!snap->valid) {
1980 free_pending_exception(pe);
1981 goto next_snapshot;
1982 }
1983
Jon Brassow3510cb92009-12-10 23:52:11 +00001984 e = dm_lookup_exception(&snap->complete, chunk);
Mikulas Patocka35bf6592009-04-02 19:55:26 +01001985 if (e) {
1986 free_pending_exception(pe);
1987 goto next_snapshot;
1988 }
1989
Mikulas Patockac6621392009-04-02 19:55:25 +01001990 pe = __find_pending_exception(snap, pe, chunk);
Mikulas Patocka29138082009-04-02 19:55:25 +01001991 if (!pe) {
1992 __invalidate_snapshot(snap, -ENOMEM);
1993 goto next_snapshot;
1994 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001995 }
1996
Mikulas Patocka515ad662009-12-10 23:52:30 +00001997 r = DM_MAPIO_SUBMITTED;
1998
1999 /*
2000 * If an origin bio was supplied, queue it to wait for the
2001 * completion of this exception, and start this one last,
2002 * at the end of the function.
2003 */
2004 if (bio) {
2005 bio_list_add(&pe->origin_bios, bio);
2006 bio = NULL;
2007
2008 if (!pe->started) {
2009 pe->started = 1;
2010 pe_to_start_last = pe;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08002011 }
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08002012 }
2013
2014 if (!pe->started) {
2015 pe->started = 1;
Mikulas Patocka515ad662009-12-10 23:52:30 +00002016 pe_to_start_now = pe;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08002017 }
2018
Jonathan Brassowa2d2b032011-08-02 12:32:03 +01002019next_snapshot:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002020 up_write(&snap->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002021
Mikulas Patocka515ad662009-12-10 23:52:30 +00002022 if (pe_to_start_now) {
2023 start_copy(pe_to_start_now);
2024 pe_to_start_now = NULL;
2025 }
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08002026 }
2027
Linus Torvalds1da177e2005-04-16 15:20:36 -07002028 /*
Mikulas Patocka515ad662009-12-10 23:52:30 +00002029 * Submit the exception against which the bio is queued last,
2030 * to give the other exceptions a head start.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031 */
Mikulas Patocka515ad662009-12-10 23:52:30 +00002032 if (pe_to_start_last)
2033 start_copy(pe_to_start_last);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002034
2035 return r;
2036}
2037
2038/*
2039 * Called on a write from the origin driver.
2040 */
2041static int do_origin(struct dm_dev *origin, struct bio *bio)
2042{
2043 struct origin *o;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08002044 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002045
2046 down_read(&_origins_lock);
2047 o = __lookup_origin(origin->bdev);
2048 if (o)
Mikulas Patocka9eaae8f2009-12-10 23:52:28 +00002049 r = __origin_write(&o->snapshots, bio->bi_sector, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050 up_read(&_origins_lock);
2051
2052 return r;
2053}
2054
2055/*
Mikulas Patocka73dfd072009-12-10 23:52:34 +00002056 * Trigger exceptions in all non-merging snapshots.
2057 *
2058 * The chunk size of the merging snapshot may be larger than the chunk
2059 * size of some other snapshot so we may need to reallocate multiple
2060 * chunks in other snapshots.
2061 *
2062 * We scan all the overlapping exceptions in the other snapshots.
2063 * Returns 1 if anything was reallocated and must be waited for,
2064 * otherwise returns 0.
2065 *
2066 * size must be a multiple of merging_snap's chunk_size.
2067 */
2068static int origin_write_extent(struct dm_snapshot *merging_snap,
2069 sector_t sector, unsigned size)
2070{
2071 int must_wait = 0;
2072 sector_t n;
2073 struct origin *o;
2074
2075 /*
2076 * The origin's __minimum_chunk_size() got stored in split_io
2077 * by snapshot_merge_resume().
2078 */
2079 down_read(&_origins_lock);
2080 o = __lookup_origin(merging_snap->origin->bdev);
2081 for (n = 0; n < size; n += merging_snap->ti->split_io)
2082 if (__origin_write(&o->snapshots, sector + n, NULL) ==
2083 DM_MAPIO_SUBMITTED)
2084 must_wait = 1;
2085 up_read(&_origins_lock);
2086
2087 return must_wait;
2088}
2089
2090/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091 * Origin: maps a linear range of a device, with hooks for snapshotting.
2092 */
2093
2094/*
2095 * Construct an origin mapping: <dev_path>
2096 * The context for an origin is merely a 'struct dm_dev *'
2097 * pointing to the real device.
2098 */
2099static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2100{
2101 int r;
2102 struct dm_dev *dev;
2103
2104 if (argc != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07002105 ti->error = "origin: incorrect number of arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -07002106 return -EINVAL;
2107 }
2108
Nikanth Karthikesan8215d6e2010-03-06 02:32:27 +00002109 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002110 if (r) {
2111 ti->error = "Cannot get target device";
2112 return r;
2113 }
2114
2115 ti->private = dev;
Mikulas Patocka494b3ee2009-06-22 10:12:25 +01002116 ti->num_flush_requests = 1;
2117
Linus Torvalds1da177e2005-04-16 15:20:36 -07002118 return 0;
2119}
2120
2121static void origin_dtr(struct dm_target *ti)
2122{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01002123 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002124 dm_put_device(ti, dev);
2125}
2126
2127static int origin_map(struct dm_target *ti, struct bio *bio,
2128 union map_info *map_context)
2129{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01002130 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002131 bio->bi_bdev = dev->bdev;
2132
Tejun Heod87f4c12010-09-03 11:56:19 +02002133 if (bio->bi_rw & REQ_FLUSH)
Mikulas Patocka494b3ee2009-06-22 10:12:25 +01002134 return DM_MAPIO_REMAPPED;
2135
Linus Torvalds1da177e2005-04-16 15:20:36 -07002136 /* Only tell snapshots if this is a write */
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08002137 return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002138}
2139
Linus Torvalds1da177e2005-04-16 15:20:36 -07002140/*
2141 * Set the target "split_io" field to the minimum of all the snapshots'
2142 * chunk sizes.
2143 */
2144static void origin_resume(struct dm_target *ti)
2145{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01002146 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002147
Mikulas Patocka1e03f972009-12-10 23:52:32 +00002148 ti->split_io = get_origin_minimum_chunksize(dev->bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002149}
2150
2151static int origin_status(struct dm_target *ti, status_type_t type, char *result,
2152 unsigned int maxlen)
2153{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01002154 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002155
2156 switch (type) {
2157 case STATUSTYPE_INFO:
2158 result[0] = '\0';
2159 break;
2160
2161 case STATUSTYPE_TABLE:
2162 snprintf(result, maxlen, "%s", dev->name);
2163 break;
2164 }
2165
2166 return 0;
2167}
2168
Mikulas Patockab1d55522010-08-12 04:14:02 +01002169static int origin_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2170 struct bio_vec *biovec, int max_size)
2171{
2172 struct dm_dev *dev = ti->private;
2173 struct request_queue *q = bdev_get_queue(dev->bdev);
2174
2175 if (!q->merge_bvec_fn)
2176 return max_size;
2177
2178 bvm->bi_bdev = dev->bdev;
2179 bvm->bi_sector = bvm->bi_sector;
2180
2181 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2182}
2183
Mike Snitzer8811f462009-09-04 20:40:19 +01002184static int origin_iterate_devices(struct dm_target *ti,
2185 iterate_devices_callout_fn fn, void *data)
2186{
2187 struct dm_dev *dev = ti->private;
2188
2189 return fn(ti, dev, 0, ti->len, data);
2190}
2191
Linus Torvalds1da177e2005-04-16 15:20:36 -07002192static struct target_type origin_target = {
2193 .name = "snapshot-origin",
Mike Snitzerb83b2f22011-01-13 19:59:59 +00002194 .version = {1, 7, 1},
Linus Torvalds1da177e2005-04-16 15:20:36 -07002195 .module = THIS_MODULE,
2196 .ctr = origin_ctr,
2197 .dtr = origin_dtr,
2198 .map = origin_map,
2199 .resume = origin_resume,
2200 .status = origin_status,
Mikulas Patockab1d55522010-08-12 04:14:02 +01002201 .merge = origin_merge,
Mike Snitzer8811f462009-09-04 20:40:19 +01002202 .iterate_devices = origin_iterate_devices,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203};
2204
2205static struct target_type snapshot_target = {
2206 .name = "snapshot",
Mike Snitzerb83b2f22011-01-13 19:59:59 +00002207 .version = {1, 10, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07002208 .module = THIS_MODULE,
2209 .ctr = snapshot_ctr,
2210 .dtr = snapshot_dtr,
2211 .map = snapshot_map,
Mikulas Patockacd45daf2008-07-21 12:00:32 +01002212 .end_io = snapshot_end_io,
Mike Snitzerc1f0c182009-12-10 23:52:24 +00002213 .preresume = snapshot_preresume,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214 .resume = snapshot_resume,
2215 .status = snapshot_status,
Mike Snitzer8811f462009-09-04 20:40:19 +01002216 .iterate_devices = snapshot_iterate_devices,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002217};
2218
Mikulas Patockad698aa42009-12-10 23:52:30 +00002219static struct target_type merge_target = {
2220 .name = dm_snapshot_merge_target_name,
Mike Snitzerb83b2f22011-01-13 19:59:59 +00002221 .version = {1, 1, 0},
Mikulas Patockad698aa42009-12-10 23:52:30 +00002222 .module = THIS_MODULE,
2223 .ctr = snapshot_ctr,
2224 .dtr = snapshot_dtr,
Mikulas Patocka3452c2a2009-12-10 23:52:31 +00002225 .map = snapshot_merge_map,
Mikulas Patockad698aa42009-12-10 23:52:30 +00002226 .end_io = snapshot_end_io,
Mikulas Patocka1e03f972009-12-10 23:52:32 +00002227 .presuspend = snapshot_merge_presuspend,
Mikulas Patockad698aa42009-12-10 23:52:30 +00002228 .preresume = snapshot_preresume,
Mikulas Patocka1e03f972009-12-10 23:52:32 +00002229 .resume = snapshot_merge_resume,
Mikulas Patockad698aa42009-12-10 23:52:30 +00002230 .status = snapshot_status,
2231 .iterate_devices = snapshot_iterate_devices,
2232};
2233
Linus Torvalds1da177e2005-04-16 15:20:36 -07002234static int __init dm_snapshot_init(void)
2235{
2236 int r;
2237
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00002238 r = dm_exception_store_init();
2239 if (r) {
2240 DMERR("Failed to initialize exception stores");
2241 return r;
2242 }
2243
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244 r = dm_register_target(&snapshot_target);
Mikulas Patockad698aa42009-12-10 23:52:30 +00002245 if (r < 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002246 DMERR("snapshot target register failed %d", r);
Jonathan Brassow034a1862009-10-16 23:18:14 +01002247 goto bad_register_snapshot_target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002248 }
2249
2250 r = dm_register_target(&origin_target);
2251 if (r < 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07002252 DMERR("Origin target register failed %d", r);
Mikulas Patockad698aa42009-12-10 23:52:30 +00002253 goto bad_register_origin_target;
2254 }
2255
2256 r = dm_register_target(&merge_target);
2257 if (r < 0) {
2258 DMERR("Merge target register failed %d", r);
2259 goto bad_register_merge_target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002260 }
2261
2262 r = init_origin_hash();
2263 if (r) {
2264 DMERR("init_origin_hash failed.");
Mikulas Patockad698aa42009-12-10 23:52:30 +00002265 goto bad_origin_hash;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002266 }
2267
Jon Brassow1d4989c2009-12-10 23:52:10 +00002268 exception_cache = KMEM_CACHE(dm_exception, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002269 if (!exception_cache) {
2270 DMERR("Couldn't create exception cache.");
2271 r = -ENOMEM;
Mikulas Patockad698aa42009-12-10 23:52:30 +00002272 goto bad_exception_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002273 }
2274
Alasdair G Kergon028867a2007-07-12 17:26:32 +01002275 pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002276 if (!pending_cache) {
2277 DMERR("Couldn't create pending cache.");
2278 r = -ENOMEM;
Mikulas Patockad698aa42009-12-10 23:52:30 +00002279 goto bad_pending_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002280 }
2281
Mikulas Patockacd45daf2008-07-21 12:00:32 +01002282 tracked_chunk_cache = KMEM_CACHE(dm_snap_tracked_chunk, 0);
2283 if (!tracked_chunk_cache) {
2284 DMERR("Couldn't create cache to track chunks in use.");
2285 r = -ENOMEM;
Mikulas Patockad698aa42009-12-10 23:52:30 +00002286 goto bad_tracked_chunk_cache;
Mikulas Patockacd45daf2008-07-21 12:00:32 +01002287 }
2288
Linus Torvalds1da177e2005-04-16 15:20:36 -07002289 return 0;
2290
Mikulas Patockad698aa42009-12-10 23:52:30 +00002291bad_tracked_chunk_cache:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002292 kmem_cache_destroy(pending_cache);
Mikulas Patockad698aa42009-12-10 23:52:30 +00002293bad_pending_cache:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002294 kmem_cache_destroy(exception_cache);
Mikulas Patockad698aa42009-12-10 23:52:30 +00002295bad_exception_cache:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002296 exit_origin_hash();
Mikulas Patockad698aa42009-12-10 23:52:30 +00002297bad_origin_hash:
2298 dm_unregister_target(&merge_target);
2299bad_register_merge_target:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002300 dm_unregister_target(&origin_target);
Mikulas Patockad698aa42009-12-10 23:52:30 +00002301bad_register_origin_target:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302 dm_unregister_target(&snapshot_target);
Jonathan Brassow034a1862009-10-16 23:18:14 +01002303bad_register_snapshot_target:
2304 dm_exception_store_exit();
Mikulas Patockad698aa42009-12-10 23:52:30 +00002305
Linus Torvalds1da177e2005-04-16 15:20:36 -07002306 return r;
2307}
2308
2309static void __exit dm_snapshot_exit(void)
2310{
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00002311 dm_unregister_target(&snapshot_target);
2312 dm_unregister_target(&origin_target);
Mikulas Patockad698aa42009-12-10 23:52:30 +00002313 dm_unregister_target(&merge_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002314
2315 exit_origin_hash();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002316 kmem_cache_destroy(pending_cache);
2317 kmem_cache_destroy(exception_cache);
Mikulas Patockacd45daf2008-07-21 12:00:32 +01002318 kmem_cache_destroy(tracked_chunk_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00002319
2320 dm_exception_store_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321}
2322
2323/* Module hooks */
2324module_init(dm_snapshot_init);
2325module_exit(dm_snapshot_exit);
2326
2327MODULE_DESCRIPTION(DM_NAME " snapshot target");
2328MODULE_AUTHOR("Joe Thornber");
2329MODULE_LICENSE("GPL");