blob: 57f1bf7f3b7a1d4ffd18f51563a2d5c52b0693ad [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * dm-snapshot.c
3 *
4 * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
5 *
6 * This file is released under the GPL.
7 */
8
9#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/device-mapper.h>
Mikulas Patocka90fa1522009-01-06 03:04:54 +000011#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/fs.h>
13#include <linux/init.h>
14#include <linux/kdev_t.h>
15#include <linux/list.h>
16#include <linux/mempool.h>
17#include <linux/module.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
vignesh babu6f3c3f02007-10-19 22:38:44 +010020#include <linux/log2.h>
Alasdair G Kergona765e202008-04-24 22:02:01 +010021#include <linux/dm-kcopyd.h>
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010022#include <linux/workqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
Jonathan Brassowaea53d92009-01-06 03:05:15 +000024#include "dm-exception-store.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Alasdair G Kergon72d94862006-06-26 00:27:35 -070026#define DM_MSG_PREFIX "snapshots"
27
Linus Torvalds1da177e2005-04-16 15:20:36 -070028/*
29 * The percentage increment we will wake up users at
30 */
31#define WAKE_UP_PERCENT 5
32
33/*
34 * kcopyd priority of snapshot operations
35 */
36#define SNAPSHOT_COPY_PRIORITY 2
37
38/*
Milan Broz8ee27672008-04-24 21:42:36 +010039 * Reserve 1MB for each snapshot initially (with minimum of 1 page).
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 */
Milan Broz8ee27672008-04-24 21:42:36 +010041#define SNAPSHOT_PAGES (((1UL << 20) >> PAGE_SHIFT) ? : 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
Mikulas Patockacd45daf2008-07-21 12:00:32 +010043/*
44 * The size of the mempool used to track chunks in use.
45 */
46#define MIN_IOS 256
47
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010048#define DM_TRACKED_CHUNK_HASH_SIZE 16
49#define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \
50 (DM_TRACKED_CHUNK_HASH_SIZE - 1))
51
52struct exception_table {
53 uint32_t hash_mask;
54 unsigned hash_shift;
55 struct list_head *table;
56};
57
58struct dm_snapshot {
59 struct rw_semaphore lock;
60
61 struct dm_dev *origin;
62
63 /* List of snapshots per Origin */
64 struct list_head list;
65
66 /* You can't use a snapshot if this is 0 (e.g. if full) */
67 int valid;
68
69 /* Origin writes don't trigger exceptions until this is set */
70 int active;
71
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010072 mempool_t *pending_pool;
73
74 atomic_t pending_exceptions_count;
75
76 struct exception_table pending;
77 struct exception_table complete;
78
79 /*
80 * pe_lock protects all pending_exception operations and access
81 * as well as the snapshot_bios list.
82 */
83 spinlock_t pe_lock;
84
85 /* The on disk metadata handler */
86 struct dm_exception_store *store;
87
88 struct dm_kcopyd_client *kcopyd_client;
89
90 /* Queue of snapshot writes for ksnapd to flush */
91 struct bio_list queued_bios;
92 struct work_struct queued_bios_work;
93
94 /* Chunks with outstanding reads */
95 mempool_t *tracked_chunk_pool;
96 spinlock_t tracked_chunk_lock;
97 struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
98};
99
Adrian Bunkc642f9e2006-12-08 02:41:13 -0800100static struct workqueue_struct *ksnapd;
David Howellsc4028952006-11-22 14:57:56 +0000101static void flush_queued_bios(struct work_struct *work);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700102
Jonathan Brassowccc45ea2009-04-02 19:55:34 +0100103static sector_t chunk_to_sector(struct dm_exception_store *store,
104 chunk_t chunk)
105{
106 return chunk << store->chunk_shift;
107}
108
109static int bdev_equal(struct block_device *lhs, struct block_device *rhs)
110{
111 /*
112 * There is only ever one instance of a particular block
113 * device so we can compare pointers safely.
114 */
115 return lhs == rhs;
116}
117
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100118struct dm_snap_pending_exception {
119 struct dm_snap_exception e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120
121 /*
122 * Origin buffers waiting for this to complete are held
123 * in a bio list
124 */
125 struct bio_list origin_bios;
126 struct bio_list snapshot_bios;
127
128 /*
Alasdair G Kergoneccf0812006-03-27 01:17:42 -0800129 * Short-term queue of pending exceptions prior to submission.
130 */
131 struct list_head list;
132
133 /*
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -0800134 * The primary pending_exception is the one that holds
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700135 * the ref_count and the list of origin_bios for a
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -0800136 * group of pending_exceptions. It is always last to get freed.
137 * These fields get set up when writing to the origin.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100139 struct dm_snap_pending_exception *primary_pe;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -0800140
141 /*
142 * Number of pending_exceptions processing this chunk.
143 * When this drops to zero we must complete the origin bios.
144 * If incrementing or decrementing this, hold pe->snap->lock for
145 * the sibling concerned and not pe->primary_pe->snap->lock unless
146 * they are the same.
147 */
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700148 atomic_t ref_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
150 /* Pointer back to snapshot context */
151 struct dm_snapshot *snap;
152
153 /*
154 * 1 indicates the exception has already been sent to
155 * kcopyd.
156 */
157 int started;
158};
159
160/*
161 * Hash table mapping origin volumes to lists of snapshots and
162 * a lock to protect it
163 */
Christoph Lametere18b8902006-12-06 20:33:20 -0800164static struct kmem_cache *exception_cache;
165static struct kmem_cache *pending_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100167struct dm_snap_tracked_chunk {
168 struct hlist_node node;
169 chunk_t chunk;
170};
171
172static struct kmem_cache *tracked_chunk_cache;
173
174static struct dm_snap_tracked_chunk *track_chunk(struct dm_snapshot *s,
175 chunk_t chunk)
176{
177 struct dm_snap_tracked_chunk *c = mempool_alloc(s->tracked_chunk_pool,
178 GFP_NOIO);
179 unsigned long flags;
180
181 c->chunk = chunk;
182
183 spin_lock_irqsave(&s->tracked_chunk_lock, flags);
184 hlist_add_head(&c->node,
185 &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);
186 spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
187
188 return c;
189}
190
191static void stop_tracking_chunk(struct dm_snapshot *s,
192 struct dm_snap_tracked_chunk *c)
193{
194 unsigned long flags;
195
196 spin_lock_irqsave(&s->tracked_chunk_lock, flags);
197 hlist_del(&c->node);
198 spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
199
200 mempool_free(c, s->tracked_chunk_pool);
201}
202
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100203static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)
204{
205 struct dm_snap_tracked_chunk *c;
206 struct hlist_node *hn;
207 int found = 0;
208
209 spin_lock_irq(&s->tracked_chunk_lock);
210
211 hlist_for_each_entry(c, hn,
212 &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) {
213 if (c->chunk == chunk) {
214 found = 1;
215 break;
216 }
217 }
218
219 spin_unlock_irq(&s->tracked_chunk_lock);
220
221 return found;
222}
223
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224/*
225 * One of these per registered origin, held in the snapshot_origins hash
226 */
227struct origin {
228 /* The origin device */
229 struct block_device *bdev;
230
231 struct list_head hash_list;
232
233 /* List of snapshots for this origin */
234 struct list_head snapshots;
235};
236
237/*
238 * Size of the hash table for origin volumes. If we make this
239 * the size of the minors list then it should be nearly perfect
240 */
241#define ORIGIN_HASH_SIZE 256
242#define ORIGIN_MASK 0xFF
243static struct list_head *_origins;
244static struct rw_semaphore _origins_lock;
245
246static int init_origin_hash(void)
247{
248 int i;
249
250 _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head),
251 GFP_KERNEL);
252 if (!_origins) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700253 DMERR("unable to allocate memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 return -ENOMEM;
255 }
256
257 for (i = 0; i < ORIGIN_HASH_SIZE; i++)
258 INIT_LIST_HEAD(_origins + i);
259 init_rwsem(&_origins_lock);
260
261 return 0;
262}
263
264static void exit_origin_hash(void)
265{
266 kfree(_origins);
267}
268
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100269static unsigned origin_hash(struct block_device *bdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270{
271 return bdev->bd_dev & ORIGIN_MASK;
272}
273
274static struct origin *__lookup_origin(struct block_device *origin)
275{
276 struct list_head *ol;
277 struct origin *o;
278
279 ol = &_origins[origin_hash(origin)];
280 list_for_each_entry (o, ol, hash_list)
281 if (bdev_equal(o->bdev, origin))
282 return o;
283
284 return NULL;
285}
286
287static void __insert_origin(struct origin *o)
288{
289 struct list_head *sl = &_origins[origin_hash(o->bdev)];
290 list_add_tail(&o->hash_list, sl);
291}
292
293/*
294 * Make a note of the snapshot and its origin so we can look it
295 * up when the origin has a write on it.
296 */
297static int register_snapshot(struct dm_snapshot *snap)
298{
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000299 struct origin *o, *new_o;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 struct block_device *bdev = snap->origin->bdev;
301
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000302 new_o = kmalloc(sizeof(*new_o), GFP_KERNEL);
303 if (!new_o)
304 return -ENOMEM;
305
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 down_write(&_origins_lock);
307 o = __lookup_origin(bdev);
308
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000309 if (o)
310 kfree(new_o);
311 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 /* New origin */
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000313 o = new_o;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
315 /* Initialise the struct */
316 INIT_LIST_HEAD(&o->snapshots);
317 o->bdev = bdev;
318
319 __insert_origin(o);
320 }
321
322 list_add_tail(&snap->list, &o->snapshots);
323
324 up_write(&_origins_lock);
325 return 0;
326}
327
328static void unregister_snapshot(struct dm_snapshot *s)
329{
330 struct origin *o;
331
332 down_write(&_origins_lock);
333 o = __lookup_origin(s->origin->bdev);
334
335 list_del(&s->list);
336 if (list_empty(&o->snapshots)) {
337 list_del(&o->hash_list);
338 kfree(o);
339 }
340
341 up_write(&_origins_lock);
342}
343
344/*
345 * Implementation of the exception hash tables.
Milan Brozd74f81f2008-02-08 02:11:27 +0000346 * The lowest hash_shift bits of the chunk number are ignored, allowing
347 * some consecutive chunks to be grouped together.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 */
Milan Brozd74f81f2008-02-08 02:11:27 +0000349static int init_exception_table(struct exception_table *et, uint32_t size,
350 unsigned hash_shift)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351{
352 unsigned int i;
353
Milan Brozd74f81f2008-02-08 02:11:27 +0000354 et->hash_shift = hash_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 et->hash_mask = size - 1;
356 et->table = dm_vcalloc(size, sizeof(struct list_head));
357 if (!et->table)
358 return -ENOMEM;
359
360 for (i = 0; i < size; i++)
361 INIT_LIST_HEAD(et->table + i);
362
363 return 0;
364}
365
Christoph Lametere18b8902006-12-06 20:33:20 -0800366static void exit_exception_table(struct exception_table *et, struct kmem_cache *mem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367{
368 struct list_head *slot;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100369 struct dm_snap_exception *ex, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 int i, size;
371
372 size = et->hash_mask + 1;
373 for (i = 0; i < size; i++) {
374 slot = et->table + i;
375
376 list_for_each_entry_safe (ex, next, slot, hash_list)
377 kmem_cache_free(mem, ex);
378 }
379
380 vfree(et->table);
381}
382
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100383static uint32_t exception_hash(struct exception_table *et, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384{
Milan Brozd74f81f2008-02-08 02:11:27 +0000385 return (chunk >> et->hash_shift) & et->hash_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386}
387
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100388static void insert_exception(struct exception_table *eh,
389 struct dm_snap_exception *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390{
391 struct list_head *l = &eh->table[exception_hash(eh, e->old_chunk)];
392 list_add(&e->hash_list, l);
393}
394
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100395static void remove_exception(struct dm_snap_exception *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396{
397 list_del(&e->hash_list);
398}
399
400/*
401 * Return the exception data for a sector, or NULL if not
402 * remapped.
403 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100404static struct dm_snap_exception *lookup_exception(struct exception_table *et,
405 chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406{
407 struct list_head *slot;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100408 struct dm_snap_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409
410 slot = &et->table[exception_hash(et, chunk)];
411 list_for_each_entry (e, slot, hash_list)
Milan Brozd74f81f2008-02-08 02:11:27 +0000412 if (chunk >= e->old_chunk &&
413 chunk <= e->old_chunk + dm_consecutive_chunk_count(e))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 return e;
415
416 return NULL;
417}
418
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100419static struct dm_snap_exception *alloc_exception(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100421 struct dm_snap_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422
423 e = kmem_cache_alloc(exception_cache, GFP_NOIO);
424 if (!e)
425 e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
426
427 return e;
428}
429
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100430static void free_exception(struct dm_snap_exception *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431{
432 kmem_cache_free(exception_cache, e);
433}
434
Mikulas Patocka92e86812008-07-21 12:00:35 +0100435static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436{
Mikulas Patocka92e86812008-07-21 12:00:35 +0100437 struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool,
438 GFP_NOIO);
439
Mikulas Patocka879129d22008-10-30 13:33:16 +0000440 atomic_inc(&s->pending_exceptions_count);
Mikulas Patocka92e86812008-07-21 12:00:35 +0100441 pe->snap = s;
442
443 return pe;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444}
445
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100446static void free_pending_exception(struct dm_snap_pending_exception *pe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447{
Mikulas Patocka879129d22008-10-30 13:33:16 +0000448 struct dm_snapshot *s = pe->snap;
449
450 mempool_free(pe, s->pending_pool);
451 smp_mb__before_atomic_dec();
452 atomic_dec(&s->pending_exceptions_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453}
454
Milan Brozd74f81f2008-02-08 02:11:27 +0000455static void insert_completed_exception(struct dm_snapshot *s,
456 struct dm_snap_exception *new_e)
457{
458 struct exception_table *eh = &s->complete;
459 struct list_head *l;
460 struct dm_snap_exception *e = NULL;
461
462 l = &eh->table[exception_hash(eh, new_e->old_chunk)];
463
464 /* Add immediately if this table doesn't support consecutive chunks */
465 if (!eh->hash_shift)
466 goto out;
467
468 /* List is ordered by old_chunk */
469 list_for_each_entry_reverse(e, l, hash_list) {
470 /* Insert after an existing chunk? */
471 if (new_e->old_chunk == (e->old_chunk +
472 dm_consecutive_chunk_count(e) + 1) &&
473 new_e->new_chunk == (dm_chunk_number(e->new_chunk) +
474 dm_consecutive_chunk_count(e) + 1)) {
475 dm_consecutive_chunk_count_inc(e);
476 free_exception(new_e);
477 return;
478 }
479
480 /* Insert before an existing chunk? */
481 if (new_e->old_chunk == (e->old_chunk - 1) &&
482 new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) {
483 dm_consecutive_chunk_count_inc(e);
484 e->old_chunk--;
485 e->new_chunk--;
486 free_exception(new_e);
487 return;
488 }
489
490 if (new_e->old_chunk > e->old_chunk)
491 break;
492 }
493
494out:
495 list_add(&new_e->hash_list, e ? &e->hash_list : l);
496}
497
Jonathan Brassowa159c1a2009-01-06 03:05:19 +0000498/*
499 * Callback used by the exception stores to load exceptions when
500 * initialising.
501 */
502static int dm_add_exception(void *context, chunk_t old, chunk_t new)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503{
Jonathan Brassowa159c1a2009-01-06 03:05:19 +0000504 struct dm_snapshot *s = context;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100505 struct dm_snap_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506
507 e = alloc_exception();
508 if (!e)
509 return -ENOMEM;
510
511 e->old_chunk = old;
Milan Brozd74f81f2008-02-08 02:11:27 +0000512
513 /* Consecutive_count is implicitly initialised to zero */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 e->new_chunk = new;
Milan Brozd74f81f2008-02-08 02:11:27 +0000515
516 insert_completed_exception(s, e);
517
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 return 0;
519}
520
521/*
522 * Hard coded magic.
523 */
524static int calc_max_buckets(void)
525{
526 /* use a fixed size of 2MB */
527 unsigned long mem = 2 * 1024 * 1024;
528 mem /= sizeof(struct list_head);
529
530 return mem;
531}
532
533/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 * Allocate room for a suitable hash table.
535 */
Jonathan Brassowfee19982009-04-02 19:55:34 +0100536static int init_hash_tables(struct dm_snapshot *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537{
538 sector_t hash_size, cow_dev_size, origin_dev_size, max_buckets;
539
540 /*
541 * Calculate based on the size of the original volume or
542 * the COW volume...
543 */
Jonathan Brassowfee19982009-04-02 19:55:34 +0100544 cow_dev_size = get_dev_size(s->store->cow->bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 origin_dev_size = get_dev_size(s->origin->bdev);
546 max_buckets = calc_max_buckets();
547
Jonathan Brassowfee19982009-04-02 19:55:34 +0100548 hash_size = min(origin_dev_size, cow_dev_size) >> s->store->chunk_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 hash_size = min(hash_size, max_buckets);
550
Robert P. J. Day8defd832008-02-08 02:10:06 +0000551 hash_size = rounddown_pow_of_two(hash_size);
Milan Brozd74f81f2008-02-08 02:11:27 +0000552 if (init_exception_table(&s->complete, hash_size,
553 DM_CHUNK_CONSECUTIVE_BITS))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 return -ENOMEM;
555
556 /*
557 * Allocate hash table for in-flight exceptions
558 * Make this smaller than the real hash table
559 */
560 hash_size >>= 3;
561 if (hash_size < 64)
562 hash_size = 64;
563
Milan Brozd74f81f2008-02-08 02:11:27 +0000564 if (init_exception_table(&s->pending, hash_size, 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 exit_exception_table(&s->complete, exception_cache);
566 return -ENOMEM;
567 }
568
569 return 0;
570}
571
572/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
574 */
575static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
576{
577 struct dm_snapshot *s;
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100578 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 char *origin_path;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100581 struct dm_exception_store *store;
582 unsigned args_used;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -0700584 if (argc != 4) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700585 ti->error = "requires exactly 4 arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 r = -EINVAL;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100587 goto bad_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 }
589
590 origin_path = argv[0];
Jonathan Brassowfee19982009-04-02 19:55:34 +0100591 argv++;
592 argc--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593
Jonathan Brassowfee19982009-04-02 19:55:34 +0100594 r = dm_exception_store_create(ti, argc, argv, &args_used, &store);
595 if (r) {
596 ti->error = "Couldn't create exception store";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 r = -EINVAL;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100598 goto bad_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599 }
600
Jonathan Brassowfee19982009-04-02 19:55:34 +0100601 argv += args_used;
602 argc -= args_used;
603
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 s = kmalloc(sizeof(*s), GFP_KERNEL);
Jonathan Brassowfee19982009-04-02 19:55:34 +0100605 if (!s) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 ti->error = "Cannot allocate snapshot context private "
607 "structure";
608 r = -ENOMEM;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100609 goto bad_snap;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 }
611
612 r = dm_get_device(ti, origin_path, 0, ti->len, FMODE_READ, &s->origin);
613 if (r) {
614 ti->error = "Cannot get origin device";
Jonathan Brassowfee19982009-04-02 19:55:34 +0100615 goto bad_origin;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 }
617
Jonathan Brassowfee19982009-04-02 19:55:34 +0100618 s->store = store;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 s->valid = 1;
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800620 s->active = 0;
Mikulas Patocka879129d22008-10-30 13:33:16 +0000621 atomic_set(&s->pending_exceptions_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 init_rwsem(&s->lock);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700623 spin_lock_init(&s->pe_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624
625 /* Allocate hash table for COW data */
Jonathan Brassowfee19982009-04-02 19:55:34 +0100626 if (init_hash_tables(s)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 ti->error = "Unable to allocate hash table space";
628 r = -ENOMEM;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100629 goto bad_hash_tables;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 }
631
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100632 r = dm_kcopyd_client_create(SNAPSHOT_PAGES, &s->kcopyd_client);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 if (r) {
634 ti->error = "Could not create kcopyd client";
Jonathan Brassowfee19982009-04-02 19:55:34 +0100635 goto bad_kcopyd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 }
637
Mikulas Patocka92e86812008-07-21 12:00:35 +0100638 s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache);
639 if (!s->pending_pool) {
640 ti->error = "Could not allocate mempool for pending exceptions";
Jonathan Brassowfee19982009-04-02 19:55:34 +0100641 goto bad_pending_pool;
Mikulas Patocka92e86812008-07-21 12:00:35 +0100642 }
643
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100644 s->tracked_chunk_pool = mempool_create_slab_pool(MIN_IOS,
645 tracked_chunk_cache);
646 if (!s->tracked_chunk_pool) {
647 ti->error = "Could not allocate tracked_chunk mempool for "
648 "tracking reads";
Mikulas Patocka92e86812008-07-21 12:00:35 +0100649 goto bad_tracked_chunk_pool;
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100650 }
651
652 for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
653 INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
654
655 spin_lock_init(&s->tracked_chunk_lock);
656
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800657 /* Metadata must only be loaded into one table at once */
Jonathan Brassow493df712009-04-02 19:55:31 +0100658 r = s->store->type->read_metadata(s->store, dm_add_exception,
659 (void *)s);
Milan Broz07641472007-07-12 17:28:13 +0100660 if (r < 0) {
Mark McLoughlinf9cea4f2006-10-03 01:15:25 -0700661 ti->error = "Failed to read snapshot metadata";
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100662 goto bad_load_and_register;
Milan Broz07641472007-07-12 17:28:13 +0100663 } else if (r > 0) {
664 s->valid = 0;
665 DMWARN("Snapshot is marked invalid.");
Mark McLoughlinf9cea4f2006-10-03 01:15:25 -0700666 }
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800667
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700668 bio_list_init(&s->queued_bios);
David Howellsc4028952006-11-22 14:57:56 +0000669 INIT_WORK(&s->queued_bios_work, flush_queued_bios);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700670
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 /* Add snapshot to the list of snapshots for this origin */
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800672 /* Exceptions aren't triggered till snapshot_resume() is called */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 if (register_snapshot(s)) {
674 r = -EINVAL;
675 ti->error = "Cannot register snapshot origin";
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100676 goto bad_load_and_register;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 }
678
679 ti->private = s;
Jonathan Brassowd0216842009-04-02 19:55:32 +0100680 ti->split_io = s->store->chunk_size;
Mikulas Patocka494b3ee2009-06-22 10:12:25 +0100681 ti->num_flush_requests = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682
683 return 0;
684
Jonathan Brassowfee19982009-04-02 19:55:34 +0100685bad_load_and_register:
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100686 mempool_destroy(s->tracked_chunk_pool);
687
Jonathan Brassowfee19982009-04-02 19:55:34 +0100688bad_tracked_chunk_pool:
Mikulas Patocka92e86812008-07-21 12:00:35 +0100689 mempool_destroy(s->pending_pool);
690
Jonathan Brassowfee19982009-04-02 19:55:34 +0100691bad_pending_pool:
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100692 dm_kcopyd_client_destroy(s->kcopyd_client);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693
Jonathan Brassowfee19982009-04-02 19:55:34 +0100694bad_kcopyd:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695 exit_exception_table(&s->pending, pending_cache);
696 exit_exception_table(&s->complete, exception_cache);
697
Jonathan Brassowfee19982009-04-02 19:55:34 +0100698bad_hash_tables:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699 dm_put_device(ti, s->origin);
700
Jonathan Brassowfee19982009-04-02 19:55:34 +0100701bad_origin:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702 kfree(s);
703
Jonathan Brassowfee19982009-04-02 19:55:34 +0100704bad_snap:
705 dm_exception_store_destroy(store);
706
707bad_args:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 return r;
709}
710
Milan Broz31c93a02006-12-08 02:41:11 -0800711static void __free_exceptions(struct dm_snapshot *s)
712{
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100713 dm_kcopyd_client_destroy(s->kcopyd_client);
Milan Broz31c93a02006-12-08 02:41:11 -0800714 s->kcopyd_client = NULL;
715
716 exit_exception_table(&s->pending, pending_cache);
717 exit_exception_table(&s->complete, exception_cache);
Milan Broz31c93a02006-12-08 02:41:11 -0800718}
719
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720static void snapshot_dtr(struct dm_target *ti)
721{
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100722#ifdef CONFIG_DM_DEBUG
723 int i;
724#endif
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100725 struct dm_snapshot *s = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700727 flush_workqueue(ksnapd);
728
Alasdair G Kergon138728dc2006-03-27 01:17:50 -0800729 /* Prevent further origin writes from using this snapshot. */
730 /* After this returns there can be no new kcopyd jobs. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 unregister_snapshot(s);
732
Mikulas Patocka879129d22008-10-30 13:33:16 +0000733 while (atomic_read(&s->pending_exceptions_count))
Mikulas Patocka90fa1522009-01-06 03:04:54 +0000734 msleep(1);
Mikulas Patocka879129d22008-10-30 13:33:16 +0000735 /*
736 * Ensure instructions in mempool_destroy aren't reordered
737 * before atomic_read.
738 */
739 smp_mb();
740
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100741#ifdef CONFIG_DM_DEBUG
742 for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
743 BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));
744#endif
745
746 mempool_destroy(s->tracked_chunk_pool);
747
Milan Broz31c93a02006-12-08 02:41:11 -0800748 __free_exceptions(s);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749
Mikulas Patocka92e86812008-07-21 12:00:35 +0100750 mempool_destroy(s->pending_pool);
751
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 dm_put_device(ti, s->origin);
Jonathan Brassowfee19982009-04-02 19:55:34 +0100753
754 dm_exception_store_destroy(s->store);
Alasdair G Kergon138728dc2006-03-27 01:17:50 -0800755
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 kfree(s);
757}
758
759/*
760 * Flush a list of buffers.
761 */
762static void flush_bios(struct bio *bio)
763{
764 struct bio *n;
765
766 while (bio) {
767 n = bio->bi_next;
768 bio->bi_next = NULL;
769 generic_make_request(bio);
770 bio = n;
771 }
772}
773
David Howellsc4028952006-11-22 14:57:56 +0000774static void flush_queued_bios(struct work_struct *work)
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700775{
David Howellsc4028952006-11-22 14:57:56 +0000776 struct dm_snapshot *s =
777 container_of(work, struct dm_snapshot, queued_bios_work);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700778 struct bio *queued_bios;
779 unsigned long flags;
780
781 spin_lock_irqsave(&s->pe_lock, flags);
782 queued_bios = bio_list_get(&s->queued_bios);
783 spin_unlock_irqrestore(&s->pe_lock, flags);
784
785 flush_bios(queued_bios);
786}
787
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788/*
789 * Error a list of buffers.
790 */
791static void error_bios(struct bio *bio)
792{
793 struct bio *n;
794
795 while (bio) {
796 n = bio->bi_next;
797 bio->bi_next = NULL;
NeilBrown6712ecf2007-09-27 12:47:43 +0200798 bio_io_error(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 bio = n;
800 }
801}
802
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700803static void __invalidate_snapshot(struct dm_snapshot *s, int err)
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800804{
805 if (!s->valid)
806 return;
807
808 if (err == -EIO)
809 DMERR("Invalidating snapshot: Error reading/writing.");
810 else if (err == -ENOMEM)
811 DMERR("Invalidating snapshot: Unable to allocate exception.");
812
Jonathan Brassow493df712009-04-02 19:55:31 +0100813 if (s->store->type->drop_snapshot)
814 s->store->type->drop_snapshot(s->store);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800815
816 s->valid = 0;
817
Jonathan Brassow0cea9c72009-04-02 19:55:32 +0100818 dm_table_event(s->store->ti->table);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800819}
820
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100821static void get_pending_exception(struct dm_snap_pending_exception *pe)
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700822{
823 atomic_inc(&pe->ref_count);
824}
825
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100826static struct bio *put_pending_exception(struct dm_snap_pending_exception *pe)
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700827{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100828 struct dm_snap_pending_exception *primary_pe;
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700829 struct bio *origin_bios = NULL;
830
831 primary_pe = pe->primary_pe;
832
833 /*
834 * If this pe is involved in a write to the origin and
835 * it is the last sibling to complete then release
836 * the bios for the original write to the origin.
837 */
838 if (primary_pe &&
Mikulas Patocka7c5f78b2008-10-21 17:44:51 +0100839 atomic_dec_and_test(&primary_pe->ref_count)) {
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700840 origin_bios = bio_list_get(&primary_pe->origin_bios);
Mikulas Patocka7c5f78b2008-10-21 17:44:51 +0100841 free_pending_exception(primary_pe);
842 }
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700843
844 /*
845 * Free the pe if it's not linked to an origin write or if
846 * it's not itself a primary pe.
847 */
848 if (!primary_pe || primary_pe != pe)
849 free_pending_exception(pe);
850
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700851 return origin_bios;
852}
853
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100854static void pending_complete(struct dm_snap_pending_exception *pe, int success)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100856 struct dm_snap_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 struct dm_snapshot *s = pe->snap;
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700858 struct bio *origin_bios = NULL;
859 struct bio *snapshot_bios = NULL;
860 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800862 if (!success) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863 /* Read/write error - snapshot is unusable */
864 down_write(&s->lock);
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700865 __invalidate_snapshot(s, -EIO);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700866 error = 1;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800867 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868 }
869
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800870 e = alloc_exception();
871 if (!e) {
872 down_write(&s->lock);
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700873 __invalidate_snapshot(s, -ENOMEM);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700874 error = 1;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800875 goto out;
876 }
877 *e = pe->e;
878
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700879 down_write(&s->lock);
880 if (!s->valid) {
881 free_exception(e);
882 error = 1;
883 goto out;
884 }
885
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800886 /*
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100887 * Check for conflicting reads. This is extremely improbable,
Mikulas Patocka90fa1522009-01-06 03:04:54 +0000888 * so msleep(1) is sufficient and there is no need for a wait queue.
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100889 */
890 while (__chunk_is_tracked(s, pe->e.old_chunk))
Mikulas Patocka90fa1522009-01-06 03:04:54 +0000891 msleep(1);
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100892
893 /*
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800894 * Add a proper exception, and remove the
895 * in-flight exception from the list.
896 */
Milan Brozd74f81f2008-02-08 02:11:27 +0000897 insert_completed_exception(s, e);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800898
Linus Torvalds1da177e2005-04-16 15:20:36 -0700899 out:
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700900 remove_exception(&pe->e);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700901 snapshot_bios = bio_list_get(&pe->snapshot_bios);
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700902 origin_bios = put_pending_exception(pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700904 up_write(&s->lock);
905
906 /* Submit any pending write bios */
907 if (error)
908 error_bios(snapshot_bios);
909 else
910 flush_bios(snapshot_bios);
911
912 flush_bios(origin_bios);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913}
914
915static void commit_callback(void *context, int success)
916{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100917 struct dm_snap_pending_exception *pe = context;
918
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919 pending_complete(pe, success);
920}
921
922/*
923 * Called when the copy I/O has finished. kcopyd actually runs
924 * this code so don't block.
925 */
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700926static void copy_callback(int read_err, unsigned long write_err, void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100928 struct dm_snap_pending_exception *pe = context;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 struct dm_snapshot *s = pe->snap;
930
931 if (read_err || write_err)
932 pending_complete(pe, 0);
933
934 else
935 /* Update the metadata if we are persistent */
Jonathan Brassow493df712009-04-02 19:55:31 +0100936 s->store->type->commit_exception(s->store, &pe->e,
937 commit_callback, pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938}
939
940/*
941 * Dispatches the copy operation to kcopyd.
942 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100943static void start_copy(struct dm_snap_pending_exception *pe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944{
945 struct dm_snapshot *s = pe->snap;
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +0100946 struct dm_io_region src, dest;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947 struct block_device *bdev = s->origin->bdev;
948 sector_t dev_size;
949
950 dev_size = get_dev_size(bdev);
951
952 src.bdev = bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +0100953 src.sector = chunk_to_sector(s->store, pe->e.old_chunk);
Jonathan Brassowd0216842009-04-02 19:55:32 +0100954 src.count = min(s->store->chunk_size, dev_size - src.sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955
Jonathan Brassow49beb2b2009-04-02 19:55:33 +0100956 dest.bdev = s->store->cow->bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +0100957 dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 dest.count = src.count;
959
960 /* Hand over to kcopyd */
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100961 dm_kcopyd_copy(s->kcopyd_client,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 &src, 1, &dest, 0, copy_callback, pe);
963}
964
Mikulas Patocka29138082009-04-02 19:55:25 +0100965static struct dm_snap_pending_exception *
966__lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)
967{
968 struct dm_snap_exception *e = lookup_exception(&s->pending, chunk);
969
970 if (!e)
971 return NULL;
972
973 return container_of(e, struct dm_snap_pending_exception, e);
974}
975
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976/*
977 * Looks to see if this snapshot already has a pending exception
978 * for this chunk, otherwise it allocates a new one and inserts
979 * it into the pending table.
980 *
981 * NOTE: a write lock must be held on snap->lock before calling
982 * this.
983 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100984static struct dm_snap_pending_exception *
Mikulas Patockac6621392009-04-02 19:55:25 +0100985__find_pending_exception(struct dm_snapshot *s,
986 struct dm_snap_pending_exception *pe, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987{
Mikulas Patockac6621392009-04-02 19:55:25 +0100988 struct dm_snap_pending_exception *pe2;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800989
Mikulas Patocka29138082009-04-02 19:55:25 +0100990 pe2 = __lookup_pending_exception(s, chunk);
991 if (pe2) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800992 free_pending_exception(pe);
Mikulas Patocka29138082009-04-02 19:55:25 +0100993 return pe2;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800994 }
995
996 pe->e.old_chunk = chunk;
997 bio_list_init(&pe->origin_bios);
998 bio_list_init(&pe->snapshot_bios);
999 pe->primary_pe = NULL;
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001000 atomic_set(&pe->ref_count, 0);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001001 pe->started = 0;
1002
Jonathan Brassow493df712009-04-02 19:55:31 +01001003 if (s->store->type->prepare_exception(s->store, &pe->e)) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001004 free_pending_exception(pe);
1005 return NULL;
1006 }
1007
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001008 get_pending_exception(pe);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001009 insert_exception(&s->pending, &pe->e);
1010
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 return pe;
1012}
1013
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001014static void remap_exception(struct dm_snapshot *s, struct dm_snap_exception *e,
Milan Brozd74f81f2008-02-08 02:11:27 +00001015 struct bio *bio, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016{
Jonathan Brassow49beb2b2009-04-02 19:55:33 +01001017 bio->bi_bdev = s->store->cow->bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +01001018 bio->bi_sector = chunk_to_sector(s->store,
1019 dm_chunk_number(e->new_chunk) +
1020 (chunk - e->old_chunk)) +
1021 (bio->bi_sector &
1022 s->store->chunk_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023}
1024
1025static int snapshot_map(struct dm_target *ti, struct bio *bio,
1026 union map_info *map_context)
1027{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001028 struct dm_snap_exception *e;
1029 struct dm_snapshot *s = ti->private;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001030 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 chunk_t chunk;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001032 struct dm_snap_pending_exception *pe = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033
Mikulas Patocka494b3ee2009-06-22 10:12:25 +01001034 if (unlikely(bio_empty_barrier(bio))) {
1035 bio->bi_bdev = s->store->cow->bdev;
1036 return DM_MAPIO_REMAPPED;
1037 }
1038
Jonathan Brassow71fab002009-04-02 19:55:33 +01001039 chunk = sector_to_chunk(s->store, bio->bi_sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040
1041 /* Full snapshots are not usable */
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001042 /* To get here the table must be live so s->active is always set. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043 if (!s->valid)
Alasdair G Kergonf6a80ea2005-07-12 15:53:01 -07001044 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001046 /* FIXME: should only take write lock if we need
1047 * to copy an exception */
1048 down_write(&s->lock);
1049
1050 if (!s->valid) {
1051 r = -EIO;
1052 goto out_unlock;
1053 }
1054
1055 /* If the block is already remapped - use that, else remap it */
1056 e = lookup_exception(&s->complete, chunk);
1057 if (e) {
Milan Brozd74f81f2008-02-08 02:11:27 +00001058 remap_exception(s, e, bio, chunk);
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001059 goto out_unlock;
1060 }
1061
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062 /*
1063 * Write to snapshot - higher level takes care of RW/RO
1064 * flags so we should only get this if we are
1065 * writeable.
1066 */
1067 if (bio_rw(bio) == WRITE) {
Mikulas Patocka29138082009-04-02 19:55:25 +01001068 pe = __lookup_pending_exception(s, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001069 if (!pe) {
Mikulas Patockac6621392009-04-02 19:55:25 +01001070 up_write(&s->lock);
1071 pe = alloc_pending_exception(s);
1072 down_write(&s->lock);
1073
1074 if (!s->valid) {
1075 free_pending_exception(pe);
1076 r = -EIO;
1077 goto out_unlock;
1078 }
1079
Mikulas Patocka35bf6592009-04-02 19:55:26 +01001080 e = lookup_exception(&s->complete, chunk);
1081 if (e) {
1082 free_pending_exception(pe);
1083 remap_exception(s, e, bio, chunk);
1084 goto out_unlock;
1085 }
1086
Mikulas Patockac6621392009-04-02 19:55:25 +01001087 pe = __find_pending_exception(s, pe, chunk);
Mikulas Patocka29138082009-04-02 19:55:25 +01001088 if (!pe) {
1089 __invalidate_snapshot(s, -ENOMEM);
1090 r = -EIO;
1091 goto out_unlock;
1092 }
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001093 }
1094
Milan Brozd74f81f2008-02-08 02:11:27 +00001095 remap_exception(s, &pe->e, bio, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001096 bio_list_add(&pe->snapshot_bios, bio);
1097
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001098 r = DM_MAPIO_SUBMITTED;
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001099
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001100 if (!pe->started) {
1101 /* this is protected by snap->lock */
1102 pe->started = 1;
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001103 up_write(&s->lock);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001104 start_copy(pe);
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001105 goto out;
1106 }
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001107 } else {
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001108 bio->bi_bdev = s->origin->bdev;
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001109 map_context->ptr = track_chunk(s, chunk);
1110 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001112 out_unlock:
1113 up_write(&s->lock);
1114 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 return r;
1116}
1117
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001118static int snapshot_end_io(struct dm_target *ti, struct bio *bio,
1119 int error, union map_info *map_context)
1120{
1121 struct dm_snapshot *s = ti->private;
1122 struct dm_snap_tracked_chunk *c = map_context->ptr;
1123
1124 if (c)
1125 stop_tracking_chunk(s, c);
1126
1127 return 0;
1128}
1129
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130static void snapshot_resume(struct dm_target *ti)
1131{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001132 struct dm_snapshot *s = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001133
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001134 down_write(&s->lock);
1135 s->active = 1;
1136 up_write(&s->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137}
1138
1139static int snapshot_status(struct dm_target *ti, status_type_t type,
1140 char *result, unsigned int maxlen)
1141{
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001142 unsigned sz = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001143 struct dm_snapshot *snap = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144
1145 switch (type) {
1146 case STATUSTYPE_INFO:
1147 if (!snap->valid)
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001148 DMEMIT("Invalid");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149 else {
Jonathan Brassow493df712009-04-02 19:55:31 +01001150 if (snap->store->type->fraction_full) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151 sector_t numerator, denominator;
Jonathan Brassow493df712009-04-02 19:55:31 +01001152 snap->store->type->fraction_full(snap->store,
1153 &numerator,
1154 &denominator);
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001155 DMEMIT("%llu/%llu",
1156 (unsigned long long)numerator,
1157 (unsigned long long)denominator);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001158 }
1159 else
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001160 DMEMIT("Unknown");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 }
1162 break;
1163
1164 case STATUSTYPE_TABLE:
1165 /*
1166 * kdevname returns a static pointer so we need
1167 * to make private copies if the output is to
1168 * make sense.
1169 */
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001170 DMEMIT("%s", snap->origin->name);
Jonathan Brassow1e302a92009-04-02 19:55:35 +01001171 snap->store->type->status(snap->store, type, result + sz,
1172 maxlen - sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173 break;
1174 }
1175
1176 return 0;
1177}
1178
Mike Snitzer8811f462009-09-04 20:40:19 +01001179static int snapshot_iterate_devices(struct dm_target *ti,
1180 iterate_devices_callout_fn fn, void *data)
1181{
1182 struct dm_snapshot *snap = ti->private;
1183
1184 return fn(ti, snap->origin, 0, ti->len, data);
1185}
1186
1187
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188/*-----------------------------------------------------------------
1189 * Origin methods
1190 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -07001191static int __origin_write(struct list_head *snapshots, struct bio *bio)
1192{
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001193 int r = DM_MAPIO_REMAPPED, first = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194 struct dm_snapshot *snap;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001195 struct dm_snap_exception *e;
1196 struct dm_snap_pending_exception *pe, *next_pe, *primary_pe = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197 chunk_t chunk;
Alasdair G Kergoneccf0812006-03-27 01:17:42 -08001198 LIST_HEAD(pe_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199
1200 /* Do all the snapshots on this origin */
1201 list_for_each_entry (snap, snapshots, list) {
1202
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001203 down_write(&snap->lock);
1204
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001205 /* Only deal with valid and active snapshots */
1206 if (!snap->valid || !snap->active)
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001207 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208
Alasdair G Kergond5e404c2005-07-12 15:53:05 -07001209 /* Nothing to do if writing beyond end of snapshot */
Jonathan Brassow0cea9c72009-04-02 19:55:32 +01001210 if (bio->bi_sector >= dm_table_get_size(snap->store->ti->table))
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001211 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212
1213 /*
1214 * Remember, different snapshots can have
1215 * different chunk sizes.
1216 */
Jonathan Brassow71fab002009-04-02 19:55:33 +01001217 chunk = sector_to_chunk(snap->store, bio->bi_sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218
1219 /*
1220 * Check exception table to see if block
1221 * is already remapped in this snapshot
1222 * and trigger an exception if not.
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001223 *
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001224 * ref_count is initialised to 1 so pending_complete()
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001225 * won't destroy the primary_pe while we're inside this loop.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001226 */
1227 e = lookup_exception(&snap->complete, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001228 if (e)
1229 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230
Mikulas Patocka29138082009-04-02 19:55:25 +01001231 pe = __lookup_pending_exception(snap, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001232 if (!pe) {
Mikulas Patockac6621392009-04-02 19:55:25 +01001233 up_write(&snap->lock);
1234 pe = alloc_pending_exception(snap);
1235 down_write(&snap->lock);
1236
1237 if (!snap->valid) {
1238 free_pending_exception(pe);
1239 goto next_snapshot;
1240 }
1241
Mikulas Patocka35bf6592009-04-02 19:55:26 +01001242 e = lookup_exception(&snap->complete, chunk);
1243 if (e) {
1244 free_pending_exception(pe);
1245 goto next_snapshot;
1246 }
1247
Mikulas Patockac6621392009-04-02 19:55:25 +01001248 pe = __find_pending_exception(snap, pe, chunk);
Mikulas Patocka29138082009-04-02 19:55:25 +01001249 if (!pe) {
1250 __invalidate_snapshot(snap, -ENOMEM);
1251 goto next_snapshot;
1252 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253 }
1254
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001255 if (!primary_pe) {
1256 /*
1257 * Either every pe here has same
1258 * primary_pe or none has one yet.
1259 */
1260 if (pe->primary_pe)
1261 primary_pe = pe->primary_pe;
1262 else {
1263 primary_pe = pe;
1264 first = 1;
1265 }
1266
1267 bio_list_add(&primary_pe->origin_bios, bio);
1268
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001269 r = DM_MAPIO_SUBMITTED;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001270 }
1271
1272 if (!pe->primary_pe) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001273 pe->primary_pe = primary_pe;
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001274 get_pending_exception(primary_pe);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001275 }
1276
1277 if (!pe->started) {
1278 pe->started = 1;
1279 list_add_tail(&pe->list, &pe_queue);
1280 }
1281
1282 next_snapshot:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283 up_write(&snap->lock);
1284 }
1285
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001286 if (!primary_pe)
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001287 return r;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001288
1289 /*
1290 * If this is the first time we're processing this chunk and
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001291 * ref_count is now 1 it means all the pending exceptions
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001292 * got completed while we were in the loop above, so it falls to
1293 * us here to remove the primary_pe and submit any origin_bios.
1294 */
1295
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001296 if (first && atomic_dec_and_test(&primary_pe->ref_count)) {
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001297 flush_bios(bio_list_get(&primary_pe->origin_bios));
1298 free_pending_exception(primary_pe);
1299 /* If we got here, pe_queue is necessarily empty. */
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001300 return r;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001301 }
1302
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303 /*
1304 * Now that we have a complete pe list we can start the copying.
1305 */
Alasdair G Kergoneccf0812006-03-27 01:17:42 -08001306 list_for_each_entry_safe(pe, next_pe, &pe_queue, list)
1307 start_copy(pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001308
1309 return r;
1310}
1311
1312/*
1313 * Called on a write from the origin driver.
1314 */
1315static int do_origin(struct dm_dev *origin, struct bio *bio)
1316{
1317 struct origin *o;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001318 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319
1320 down_read(&_origins_lock);
1321 o = __lookup_origin(origin->bdev);
1322 if (o)
1323 r = __origin_write(&o->snapshots, bio);
1324 up_read(&_origins_lock);
1325
1326 return r;
1327}
1328
1329/*
1330 * Origin: maps a linear range of a device, with hooks for snapshotting.
1331 */
1332
1333/*
1334 * Construct an origin mapping: <dev_path>
1335 * The context for an origin is merely a 'struct dm_dev *'
1336 * pointing to the real device.
1337 */
1338static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1339{
1340 int r;
1341 struct dm_dev *dev;
1342
1343 if (argc != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001344 ti->error = "origin: incorrect number of arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001345 return -EINVAL;
1346 }
1347
1348 r = dm_get_device(ti, argv[0], 0, ti->len,
1349 dm_table_get_mode(ti->table), &dev);
1350 if (r) {
1351 ti->error = "Cannot get target device";
1352 return r;
1353 }
1354
1355 ti->private = dev;
Mikulas Patocka494b3ee2009-06-22 10:12:25 +01001356 ti->num_flush_requests = 1;
1357
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358 return 0;
1359}
1360
1361static void origin_dtr(struct dm_target *ti)
1362{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001363 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364 dm_put_device(ti, dev);
1365}
1366
1367static int origin_map(struct dm_target *ti, struct bio *bio,
1368 union map_info *map_context)
1369{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001370 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371 bio->bi_bdev = dev->bdev;
1372
Mikulas Patocka494b3ee2009-06-22 10:12:25 +01001373 if (unlikely(bio_empty_barrier(bio)))
1374 return DM_MAPIO_REMAPPED;
1375
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376 /* Only tell snapshots if this is a write */
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001377 return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378}
1379
1380#define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))
1381
1382/*
1383 * Set the target "split_io" field to the minimum of all the snapshots'
1384 * chunk sizes.
1385 */
1386static void origin_resume(struct dm_target *ti)
1387{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001388 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389 struct dm_snapshot *snap;
1390 struct origin *o;
1391 chunk_t chunk_size = 0;
1392
1393 down_read(&_origins_lock);
1394 o = __lookup_origin(dev->bdev);
1395 if (o)
1396 list_for_each_entry (snap, &o->snapshots, list)
Jonathan Brassowd0216842009-04-02 19:55:32 +01001397 chunk_size = min_not_zero(chunk_size,
1398 snap->store->chunk_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399 up_read(&_origins_lock);
1400
1401 ti->split_io = chunk_size;
1402}
1403
1404static int origin_status(struct dm_target *ti, status_type_t type, char *result,
1405 unsigned int maxlen)
1406{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001407 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408
1409 switch (type) {
1410 case STATUSTYPE_INFO:
1411 result[0] = '\0';
1412 break;
1413
1414 case STATUSTYPE_TABLE:
1415 snprintf(result, maxlen, "%s", dev->name);
1416 break;
1417 }
1418
1419 return 0;
1420}
1421
Mike Snitzer8811f462009-09-04 20:40:19 +01001422static int origin_iterate_devices(struct dm_target *ti,
1423 iterate_devices_callout_fn fn, void *data)
1424{
1425 struct dm_dev *dev = ti->private;
1426
1427 return fn(ti, dev, 0, ti->len, data);
1428}
1429
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430static struct target_type origin_target = {
1431 .name = "snapshot-origin",
Mike Snitzer8811f462009-09-04 20:40:19 +01001432 .version = {1, 7, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433 .module = THIS_MODULE,
1434 .ctr = origin_ctr,
1435 .dtr = origin_dtr,
1436 .map = origin_map,
1437 .resume = origin_resume,
1438 .status = origin_status,
Mike Snitzer8811f462009-09-04 20:40:19 +01001439 .iterate_devices = origin_iterate_devices,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001440};
1441
1442static struct target_type snapshot_target = {
1443 .name = "snapshot",
Mike Snitzer8811f462009-09-04 20:40:19 +01001444 .version = {1, 7, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445 .module = THIS_MODULE,
1446 .ctr = snapshot_ctr,
1447 .dtr = snapshot_dtr,
1448 .map = snapshot_map,
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001449 .end_io = snapshot_end_io,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450 .resume = snapshot_resume,
1451 .status = snapshot_status,
Mike Snitzer8811f462009-09-04 20:40:19 +01001452 .iterate_devices = snapshot_iterate_devices,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453};
1454
1455static int __init dm_snapshot_init(void)
1456{
1457 int r;
1458
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001459 r = dm_exception_store_init();
1460 if (r) {
1461 DMERR("Failed to initialize exception stores");
1462 return r;
1463 }
1464
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465 r = dm_register_target(&snapshot_target);
1466 if (r) {
1467 DMERR("snapshot target register failed %d", r);
1468 return r;
1469 }
1470
1471 r = dm_register_target(&origin_target);
1472 if (r < 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001473 DMERR("Origin target register failed %d", r);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474 goto bad1;
1475 }
1476
1477 r = init_origin_hash();
1478 if (r) {
1479 DMERR("init_origin_hash failed.");
1480 goto bad2;
1481 }
1482
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001483 exception_cache = KMEM_CACHE(dm_snap_exception, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 if (!exception_cache) {
1485 DMERR("Couldn't create exception cache.");
1486 r = -ENOMEM;
1487 goto bad3;
1488 }
1489
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001490 pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491 if (!pending_cache) {
1492 DMERR("Couldn't create pending cache.");
1493 r = -ENOMEM;
1494 goto bad4;
1495 }
1496
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001497 tracked_chunk_cache = KMEM_CACHE(dm_snap_tracked_chunk, 0);
1498 if (!tracked_chunk_cache) {
1499 DMERR("Couldn't create cache to track chunks in use.");
1500 r = -ENOMEM;
1501 goto bad5;
1502 }
1503
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001504 ksnapd = create_singlethread_workqueue("ksnapd");
1505 if (!ksnapd) {
1506 DMERR("Failed to create ksnapd workqueue.");
1507 r = -ENOMEM;
Mikulas Patocka92e86812008-07-21 12:00:35 +01001508 goto bad_pending_pool;
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001509 }
1510
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 return 0;
1512
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001513bad_pending_pool:
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001514 kmem_cache_destroy(tracked_chunk_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001515bad5:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516 kmem_cache_destroy(pending_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001517bad4:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518 kmem_cache_destroy(exception_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001519bad3:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520 exit_origin_hash();
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001521bad2:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522 dm_unregister_target(&origin_target);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001523bad1:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524 dm_unregister_target(&snapshot_target);
1525 return r;
1526}
1527
1528static void __exit dm_snapshot_exit(void)
1529{
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001530 destroy_workqueue(ksnapd);
1531
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001532 dm_unregister_target(&snapshot_target);
1533 dm_unregister_target(&origin_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534
1535 exit_origin_hash();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001536 kmem_cache_destroy(pending_cache);
1537 kmem_cache_destroy(exception_cache);
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001538 kmem_cache_destroy(tracked_chunk_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001539
1540 dm_exception_store_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541}
1542
1543/* Module hooks */
1544module_init(dm_snapshot_init);
1545module_exit(dm_snapshot_exit);
1546
1547MODULE_DESCRIPTION(DM_NAME " snapshot target");
1548MODULE_AUTHOR("Joe Thornber");
1549MODULE_LICENSE("GPL");