blob: d73f17fc777823968e4ab93d5a65dc30255233fc [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * dm-snapshot.c
3 *
4 * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
5 *
6 * This file is released under the GPL.
7 */
8
9#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/device-mapper.h>
Mikulas Patocka90fa1522009-01-06 03:04:54 +000011#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/fs.h>
13#include <linux/init.h>
14#include <linux/kdev_t.h>
15#include <linux/list.h>
16#include <linux/mempool.h>
17#include <linux/module.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
vignesh babu6f3c3f02007-10-19 22:38:44 +010020#include <linux/log2.h>
Alasdair G Kergona765e202008-04-24 22:02:01 +010021#include <linux/dm-kcopyd.h>
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010022#include <linux/workqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
Jonathan Brassowaea53d92009-01-06 03:05:15 +000024#include "dm-exception-store.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Alasdair G Kergon72d94862006-06-26 00:27:35 -070026#define DM_MSG_PREFIX "snapshots"
27
Linus Torvalds1da177e2005-04-16 15:20:36 -070028/*
29 * The percentage increment we will wake up users at
30 */
31#define WAKE_UP_PERCENT 5
32
33/*
34 * kcopyd priority of snapshot operations
35 */
36#define SNAPSHOT_COPY_PRIORITY 2
37
38/*
Milan Broz8ee27672008-04-24 21:42:36 +010039 * Reserve 1MB for each snapshot initially (with minimum of 1 page).
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 */
Milan Broz8ee27672008-04-24 21:42:36 +010041#define SNAPSHOT_PAGES (((1UL << 20) >> PAGE_SHIFT) ? : 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
Mikulas Patockacd45daf2008-07-21 12:00:32 +010043/*
44 * The size of the mempool used to track chunks in use.
45 */
46#define MIN_IOS 256
47
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010048#define DM_TRACKED_CHUNK_HASH_SIZE 16
49#define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \
50 (DM_TRACKED_CHUNK_HASH_SIZE - 1))
51
52struct exception_table {
53 uint32_t hash_mask;
54 unsigned hash_shift;
55 struct list_head *table;
56};
57
58struct dm_snapshot {
59 struct rw_semaphore lock;
60
61 struct dm_dev *origin;
62
63 /* List of snapshots per Origin */
64 struct list_head list;
65
66 /* You can't use a snapshot if this is 0 (e.g. if full) */
67 int valid;
68
69 /* Origin writes don't trigger exceptions until this is set */
70 int active;
71
Jonathan Brassowccc45ea2009-04-02 19:55:34 +010072 mempool_t *pending_pool;
73
74 atomic_t pending_exceptions_count;
75
76 struct exception_table pending;
77 struct exception_table complete;
78
79 /*
80 * pe_lock protects all pending_exception operations and access
81 * as well as the snapshot_bios list.
82 */
83 spinlock_t pe_lock;
84
85 /* The on disk metadata handler */
86 struct dm_exception_store *store;
87
88 struct dm_kcopyd_client *kcopyd_client;
89
90 /* Queue of snapshot writes for ksnapd to flush */
91 struct bio_list queued_bios;
92 struct work_struct queued_bios_work;
93
94 /* Chunks with outstanding reads */
95 mempool_t *tracked_chunk_pool;
96 spinlock_t tracked_chunk_lock;
97 struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
98};
99
Adrian Bunkc642f9e2006-12-08 02:41:13 -0800100static struct workqueue_struct *ksnapd;
David Howellsc4028952006-11-22 14:57:56 +0000101static void flush_queued_bios(struct work_struct *work);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700102
Jonathan Brassowccc45ea2009-04-02 19:55:34 +0100103static sector_t chunk_to_sector(struct dm_exception_store *store,
104 chunk_t chunk)
105{
106 return chunk << store->chunk_shift;
107}
108
109static int bdev_equal(struct block_device *lhs, struct block_device *rhs)
110{
111 /*
112 * There is only ever one instance of a particular block
113 * device so we can compare pointers safely.
114 */
115 return lhs == rhs;
116}
117
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100118struct dm_snap_pending_exception {
119 struct dm_snap_exception e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120
121 /*
122 * Origin buffers waiting for this to complete are held
123 * in a bio list
124 */
125 struct bio_list origin_bios;
126 struct bio_list snapshot_bios;
127
128 /*
Alasdair G Kergoneccf0812006-03-27 01:17:42 -0800129 * Short-term queue of pending exceptions prior to submission.
130 */
131 struct list_head list;
132
133 /*
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -0800134 * The primary pending_exception is the one that holds
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700135 * the ref_count and the list of origin_bios for a
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -0800136 * group of pending_exceptions. It is always last to get freed.
137 * These fields get set up when writing to the origin.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100139 struct dm_snap_pending_exception *primary_pe;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -0800140
141 /*
142 * Number of pending_exceptions processing this chunk.
143 * When this drops to zero we must complete the origin bios.
144 * If incrementing or decrementing this, hold pe->snap->lock for
145 * the sibling concerned and not pe->primary_pe->snap->lock unless
146 * they are the same.
147 */
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700148 atomic_t ref_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
150 /* Pointer back to snapshot context */
151 struct dm_snapshot *snap;
152
153 /*
154 * 1 indicates the exception has already been sent to
155 * kcopyd.
156 */
157 int started;
158};
159
160/*
161 * Hash table mapping origin volumes to lists of snapshots and
162 * a lock to protect it
163 */
Christoph Lametere18b8902006-12-06 20:33:20 -0800164static struct kmem_cache *exception_cache;
165static struct kmem_cache *pending_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100167struct dm_snap_tracked_chunk {
168 struct hlist_node node;
169 chunk_t chunk;
170};
171
172static struct kmem_cache *tracked_chunk_cache;
173
174static struct dm_snap_tracked_chunk *track_chunk(struct dm_snapshot *s,
175 chunk_t chunk)
176{
177 struct dm_snap_tracked_chunk *c = mempool_alloc(s->tracked_chunk_pool,
178 GFP_NOIO);
179 unsigned long flags;
180
181 c->chunk = chunk;
182
183 spin_lock_irqsave(&s->tracked_chunk_lock, flags);
184 hlist_add_head(&c->node,
185 &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]);
186 spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
187
188 return c;
189}
190
191static void stop_tracking_chunk(struct dm_snapshot *s,
192 struct dm_snap_tracked_chunk *c)
193{
194 unsigned long flags;
195
196 spin_lock_irqsave(&s->tracked_chunk_lock, flags);
197 hlist_del(&c->node);
198 spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);
199
200 mempool_free(c, s->tracked_chunk_pool);
201}
202
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100203static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk)
204{
205 struct dm_snap_tracked_chunk *c;
206 struct hlist_node *hn;
207 int found = 0;
208
209 spin_lock_irq(&s->tracked_chunk_lock);
210
211 hlist_for_each_entry(c, hn,
212 &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) {
213 if (c->chunk == chunk) {
214 found = 1;
215 break;
216 }
217 }
218
219 spin_unlock_irq(&s->tracked_chunk_lock);
220
221 return found;
222}
223
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224/*
225 * One of these per registered origin, held in the snapshot_origins hash
226 */
227struct origin {
228 /* The origin device */
229 struct block_device *bdev;
230
231 struct list_head hash_list;
232
233 /* List of snapshots for this origin */
234 struct list_head snapshots;
235};
236
237/*
238 * Size of the hash table for origin volumes. If we make this
239 * the size of the minors list then it should be nearly perfect
240 */
241#define ORIGIN_HASH_SIZE 256
242#define ORIGIN_MASK 0xFF
243static struct list_head *_origins;
244static struct rw_semaphore _origins_lock;
245
246static int init_origin_hash(void)
247{
248 int i;
249
250 _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head),
251 GFP_KERNEL);
252 if (!_origins) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700253 DMERR("unable to allocate memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 return -ENOMEM;
255 }
256
257 for (i = 0; i < ORIGIN_HASH_SIZE; i++)
258 INIT_LIST_HEAD(_origins + i);
259 init_rwsem(&_origins_lock);
260
261 return 0;
262}
263
264static void exit_origin_hash(void)
265{
266 kfree(_origins);
267}
268
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100269static unsigned origin_hash(struct block_device *bdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270{
271 return bdev->bd_dev & ORIGIN_MASK;
272}
273
274static struct origin *__lookup_origin(struct block_device *origin)
275{
276 struct list_head *ol;
277 struct origin *o;
278
279 ol = &_origins[origin_hash(origin)];
280 list_for_each_entry (o, ol, hash_list)
281 if (bdev_equal(o->bdev, origin))
282 return o;
283
284 return NULL;
285}
286
287static void __insert_origin(struct origin *o)
288{
289 struct list_head *sl = &_origins[origin_hash(o->bdev)];
290 list_add_tail(&o->hash_list, sl);
291}
292
293/*
294 * Make a note of the snapshot and its origin so we can look it
295 * up when the origin has a write on it.
296 */
297static int register_snapshot(struct dm_snapshot *snap)
298{
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000299 struct origin *o, *new_o;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 struct block_device *bdev = snap->origin->bdev;
301
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000302 new_o = kmalloc(sizeof(*new_o), GFP_KERNEL);
303 if (!new_o)
304 return -ENOMEM;
305
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 down_write(&_origins_lock);
307 o = __lookup_origin(bdev);
308
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000309 if (o)
310 kfree(new_o);
311 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 /* New origin */
Mikulas Patocka60c856c82008-10-30 13:33:12 +0000313 o = new_o;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
315 /* Initialise the struct */
316 INIT_LIST_HEAD(&o->snapshots);
317 o->bdev = bdev;
318
319 __insert_origin(o);
320 }
321
322 list_add_tail(&snap->list, &o->snapshots);
323
324 up_write(&_origins_lock);
325 return 0;
326}
327
328static void unregister_snapshot(struct dm_snapshot *s)
329{
330 struct origin *o;
331
332 down_write(&_origins_lock);
333 o = __lookup_origin(s->origin->bdev);
334
335 list_del(&s->list);
336 if (list_empty(&o->snapshots)) {
337 list_del(&o->hash_list);
338 kfree(o);
339 }
340
341 up_write(&_origins_lock);
342}
343
344/*
345 * Implementation of the exception hash tables.
Milan Brozd74f81f2008-02-08 02:11:27 +0000346 * The lowest hash_shift bits of the chunk number are ignored, allowing
347 * some consecutive chunks to be grouped together.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 */
Milan Brozd74f81f2008-02-08 02:11:27 +0000349static int init_exception_table(struct exception_table *et, uint32_t size,
350 unsigned hash_shift)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351{
352 unsigned int i;
353
Milan Brozd74f81f2008-02-08 02:11:27 +0000354 et->hash_shift = hash_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 et->hash_mask = size - 1;
356 et->table = dm_vcalloc(size, sizeof(struct list_head));
357 if (!et->table)
358 return -ENOMEM;
359
360 for (i = 0; i < size; i++)
361 INIT_LIST_HEAD(et->table + i);
362
363 return 0;
364}
365
Christoph Lametere18b8902006-12-06 20:33:20 -0800366static void exit_exception_table(struct exception_table *et, struct kmem_cache *mem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367{
368 struct list_head *slot;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100369 struct dm_snap_exception *ex, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 int i, size;
371
372 size = et->hash_mask + 1;
373 for (i = 0; i < size; i++) {
374 slot = et->table + i;
375
376 list_for_each_entry_safe (ex, next, slot, hash_list)
377 kmem_cache_free(mem, ex);
378 }
379
380 vfree(et->table);
381}
382
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100383static uint32_t exception_hash(struct exception_table *et, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384{
Milan Brozd74f81f2008-02-08 02:11:27 +0000385 return (chunk >> et->hash_shift) & et->hash_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386}
387
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100388static void insert_exception(struct exception_table *eh,
389 struct dm_snap_exception *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390{
391 struct list_head *l = &eh->table[exception_hash(eh, e->old_chunk)];
392 list_add(&e->hash_list, l);
393}
394
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100395static void remove_exception(struct dm_snap_exception *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396{
397 list_del(&e->hash_list);
398}
399
400/*
401 * Return the exception data for a sector, or NULL if not
402 * remapped.
403 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100404static struct dm_snap_exception *lookup_exception(struct exception_table *et,
405 chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406{
407 struct list_head *slot;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100408 struct dm_snap_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409
410 slot = &et->table[exception_hash(et, chunk)];
411 list_for_each_entry (e, slot, hash_list)
Milan Brozd74f81f2008-02-08 02:11:27 +0000412 if (chunk >= e->old_chunk &&
413 chunk <= e->old_chunk + dm_consecutive_chunk_count(e))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 return e;
415
416 return NULL;
417}
418
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100419static struct dm_snap_exception *alloc_exception(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100421 struct dm_snap_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422
423 e = kmem_cache_alloc(exception_cache, GFP_NOIO);
424 if (!e)
425 e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
426
427 return e;
428}
429
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100430static void free_exception(struct dm_snap_exception *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431{
432 kmem_cache_free(exception_cache, e);
433}
434
Mikulas Patocka92e86812008-07-21 12:00:35 +0100435static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436{
Mikulas Patocka92e86812008-07-21 12:00:35 +0100437 struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool,
438 GFP_NOIO);
439
Mikulas Patocka879129d2008-10-30 13:33:16 +0000440 atomic_inc(&s->pending_exceptions_count);
Mikulas Patocka92e86812008-07-21 12:00:35 +0100441 pe->snap = s;
442
443 return pe;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444}
445
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100446static void free_pending_exception(struct dm_snap_pending_exception *pe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447{
Mikulas Patocka879129d2008-10-30 13:33:16 +0000448 struct dm_snapshot *s = pe->snap;
449
450 mempool_free(pe, s->pending_pool);
451 smp_mb__before_atomic_dec();
452 atomic_dec(&s->pending_exceptions_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453}
454
Milan Brozd74f81f2008-02-08 02:11:27 +0000455static void insert_completed_exception(struct dm_snapshot *s,
456 struct dm_snap_exception *new_e)
457{
458 struct exception_table *eh = &s->complete;
459 struct list_head *l;
460 struct dm_snap_exception *e = NULL;
461
462 l = &eh->table[exception_hash(eh, new_e->old_chunk)];
463
464 /* Add immediately if this table doesn't support consecutive chunks */
465 if (!eh->hash_shift)
466 goto out;
467
468 /* List is ordered by old_chunk */
469 list_for_each_entry_reverse(e, l, hash_list) {
470 /* Insert after an existing chunk? */
471 if (new_e->old_chunk == (e->old_chunk +
472 dm_consecutive_chunk_count(e) + 1) &&
473 new_e->new_chunk == (dm_chunk_number(e->new_chunk) +
474 dm_consecutive_chunk_count(e) + 1)) {
475 dm_consecutive_chunk_count_inc(e);
476 free_exception(new_e);
477 return;
478 }
479
480 /* Insert before an existing chunk? */
481 if (new_e->old_chunk == (e->old_chunk - 1) &&
482 new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) {
483 dm_consecutive_chunk_count_inc(e);
484 e->old_chunk--;
485 e->new_chunk--;
486 free_exception(new_e);
487 return;
488 }
489
490 if (new_e->old_chunk > e->old_chunk)
491 break;
492 }
493
494out:
495 list_add(&new_e->hash_list, e ? &e->hash_list : l);
496}
497
Jonathan Brassowa159c1a2009-01-06 03:05:19 +0000498/*
499 * Callback used by the exception stores to load exceptions when
500 * initialising.
501 */
502static int dm_add_exception(void *context, chunk_t old, chunk_t new)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503{
Jonathan Brassowa159c1a2009-01-06 03:05:19 +0000504 struct dm_snapshot *s = context;
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100505 struct dm_snap_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506
507 e = alloc_exception();
508 if (!e)
509 return -ENOMEM;
510
511 e->old_chunk = old;
Milan Brozd74f81f2008-02-08 02:11:27 +0000512
513 /* Consecutive_count is implicitly initialised to zero */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 e->new_chunk = new;
Milan Brozd74f81f2008-02-08 02:11:27 +0000515
516 insert_completed_exception(s, e);
517
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 return 0;
519}
520
521/*
522 * Hard coded magic.
523 */
524static int calc_max_buckets(void)
525{
526 /* use a fixed size of 2MB */
527 unsigned long mem = 2 * 1024 * 1024;
528 mem /= sizeof(struct list_head);
529
530 return mem;
531}
532
533/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 * Allocate room for a suitable hash table.
535 */
Jonathan Brassowfee19982009-04-02 19:55:34 +0100536static int init_hash_tables(struct dm_snapshot *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537{
538 sector_t hash_size, cow_dev_size, origin_dev_size, max_buckets;
539
540 /*
541 * Calculate based on the size of the original volume or
542 * the COW volume...
543 */
Jonathan Brassowfee19982009-04-02 19:55:34 +0100544 cow_dev_size = get_dev_size(s->store->cow->bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 origin_dev_size = get_dev_size(s->origin->bdev);
546 max_buckets = calc_max_buckets();
547
Jonathan Brassowfee19982009-04-02 19:55:34 +0100548 hash_size = min(origin_dev_size, cow_dev_size) >> s->store->chunk_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 hash_size = min(hash_size, max_buckets);
550
Robert P. J. Day8defd832008-02-08 02:10:06 +0000551 hash_size = rounddown_pow_of_two(hash_size);
Milan Brozd74f81f2008-02-08 02:11:27 +0000552 if (init_exception_table(&s->complete, hash_size,
553 DM_CHUNK_CONSECUTIVE_BITS))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 return -ENOMEM;
555
556 /*
557 * Allocate hash table for in-flight exceptions
558 * Make this smaller than the real hash table
559 */
560 hash_size >>= 3;
561 if (hash_size < 64)
562 hash_size = 64;
563
Milan Brozd74f81f2008-02-08 02:11:27 +0000564 if (init_exception_table(&s->pending, hash_size, 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 exit_exception_table(&s->complete, exception_cache);
566 return -ENOMEM;
567 }
568
569 return 0;
570}
571
572/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
574 */
575static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
576{
577 struct dm_snapshot *s;
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100578 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579 int r = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 char *origin_path;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100581 struct dm_exception_store *store;
582 unsigned args_used;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -0700584 if (argc != 4) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700585 ti->error = "requires exactly 4 arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 r = -EINVAL;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100587 goto bad_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 }
589
590 origin_path = argv[0];
Jonathan Brassowfee19982009-04-02 19:55:34 +0100591 argv++;
592 argc--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593
Jonathan Brassowfee19982009-04-02 19:55:34 +0100594 r = dm_exception_store_create(ti, argc, argv, &args_used, &store);
595 if (r) {
596 ti->error = "Couldn't create exception store";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 r = -EINVAL;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100598 goto bad_args;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599 }
600
Jonathan Brassowfee19982009-04-02 19:55:34 +0100601 argv += args_used;
602 argc -= args_used;
603
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 s = kmalloc(sizeof(*s), GFP_KERNEL);
Jonathan Brassowfee19982009-04-02 19:55:34 +0100605 if (!s) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 ti->error = "Cannot allocate snapshot context private "
607 "structure";
608 r = -ENOMEM;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100609 goto bad_snap;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 }
611
612 r = dm_get_device(ti, origin_path, 0, ti->len, FMODE_READ, &s->origin);
613 if (r) {
614 ti->error = "Cannot get origin device";
Jonathan Brassowfee19982009-04-02 19:55:34 +0100615 goto bad_origin;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 }
617
Jonathan Brassowfee19982009-04-02 19:55:34 +0100618 s->store = store;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 s->valid = 1;
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800620 s->active = 0;
Mikulas Patocka879129d2008-10-30 13:33:16 +0000621 atomic_set(&s->pending_exceptions_count, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 init_rwsem(&s->lock);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700623 spin_lock_init(&s->pe_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624
625 /* Allocate hash table for COW data */
Jonathan Brassowfee19982009-04-02 19:55:34 +0100626 if (init_hash_tables(s)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 ti->error = "Unable to allocate hash table space";
628 r = -ENOMEM;
Jonathan Brassowfee19982009-04-02 19:55:34 +0100629 goto bad_hash_tables;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 }
631
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100632 r = dm_kcopyd_client_create(SNAPSHOT_PAGES, &s->kcopyd_client);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 if (r) {
634 ti->error = "Could not create kcopyd client";
Jonathan Brassowfee19982009-04-02 19:55:34 +0100635 goto bad_kcopyd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 }
637
Mikulas Patocka92e86812008-07-21 12:00:35 +0100638 s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache);
639 if (!s->pending_pool) {
640 ti->error = "Could not allocate mempool for pending exceptions";
Jonathan Brassowfee19982009-04-02 19:55:34 +0100641 goto bad_pending_pool;
Mikulas Patocka92e86812008-07-21 12:00:35 +0100642 }
643
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100644 s->tracked_chunk_pool = mempool_create_slab_pool(MIN_IOS,
645 tracked_chunk_cache);
646 if (!s->tracked_chunk_pool) {
647 ti->error = "Could not allocate tracked_chunk mempool for "
648 "tracking reads";
Mikulas Patocka92e86812008-07-21 12:00:35 +0100649 goto bad_tracked_chunk_pool;
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100650 }
651
652 for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
653 INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
654
655 spin_lock_init(&s->tracked_chunk_lock);
656
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800657 /* Metadata must only be loaded into one table at once */
Jonathan Brassow493df712009-04-02 19:55:31 +0100658 r = s->store->type->read_metadata(s->store, dm_add_exception,
659 (void *)s);
Milan Broz07641472007-07-12 17:28:13 +0100660 if (r < 0) {
Mark McLoughlinf9cea4f2006-10-03 01:15:25 -0700661 ti->error = "Failed to read snapshot metadata";
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100662 goto bad_load_and_register;
Milan Broz07641472007-07-12 17:28:13 +0100663 } else if (r > 0) {
664 s->valid = 0;
665 DMWARN("Snapshot is marked invalid.");
Mark McLoughlinf9cea4f2006-10-03 01:15:25 -0700666 }
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800667
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700668 bio_list_init(&s->queued_bios);
David Howellsc4028952006-11-22 14:57:56 +0000669 INIT_WORK(&s->queued_bios_work, flush_queued_bios);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700670
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 /* Add snapshot to the list of snapshots for this origin */
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800672 /* Exceptions aren't triggered till snapshot_resume() is called */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 if (register_snapshot(s)) {
674 r = -EINVAL;
675 ti->error = "Cannot register snapshot origin";
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100676 goto bad_load_and_register;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 }
678
679 ti->private = s;
Jonathan Brassowd0216842009-04-02 19:55:32 +0100680 ti->split_io = s->store->chunk_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681
682 return 0;
683
Jonathan Brassowfee19982009-04-02 19:55:34 +0100684bad_load_and_register:
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100685 mempool_destroy(s->tracked_chunk_pool);
686
Jonathan Brassowfee19982009-04-02 19:55:34 +0100687bad_tracked_chunk_pool:
Mikulas Patocka92e86812008-07-21 12:00:35 +0100688 mempool_destroy(s->pending_pool);
689
Jonathan Brassowfee19982009-04-02 19:55:34 +0100690bad_pending_pool:
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100691 dm_kcopyd_client_destroy(s->kcopyd_client);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692
Jonathan Brassowfee19982009-04-02 19:55:34 +0100693bad_kcopyd:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 exit_exception_table(&s->pending, pending_cache);
695 exit_exception_table(&s->complete, exception_cache);
696
Jonathan Brassowfee19982009-04-02 19:55:34 +0100697bad_hash_tables:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 dm_put_device(ti, s->origin);
699
Jonathan Brassowfee19982009-04-02 19:55:34 +0100700bad_origin:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701 kfree(s);
702
Jonathan Brassowfee19982009-04-02 19:55:34 +0100703bad_snap:
704 dm_exception_store_destroy(store);
705
706bad_args:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 return r;
708}
709
Milan Broz31c93a02006-12-08 02:41:11 -0800710static void __free_exceptions(struct dm_snapshot *s)
711{
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100712 dm_kcopyd_client_destroy(s->kcopyd_client);
Milan Broz31c93a02006-12-08 02:41:11 -0800713 s->kcopyd_client = NULL;
714
715 exit_exception_table(&s->pending, pending_cache);
716 exit_exception_table(&s->complete, exception_cache);
Milan Broz31c93a02006-12-08 02:41:11 -0800717}
718
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719static void snapshot_dtr(struct dm_target *ti)
720{
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100721#ifdef CONFIG_DM_DEBUG
722 int i;
723#endif
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100724 struct dm_snapshot *s = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700725
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700726 flush_workqueue(ksnapd);
727
Alasdair G Kergon138728d2006-03-27 01:17:50 -0800728 /* Prevent further origin writes from using this snapshot. */
729 /* After this returns there can be no new kcopyd jobs. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 unregister_snapshot(s);
731
Mikulas Patocka879129d2008-10-30 13:33:16 +0000732 while (atomic_read(&s->pending_exceptions_count))
Mikulas Patocka90fa1522009-01-06 03:04:54 +0000733 msleep(1);
Mikulas Patocka879129d2008-10-30 13:33:16 +0000734 /*
735 * Ensure instructions in mempool_destroy aren't reordered
736 * before atomic_read.
737 */
738 smp_mb();
739
Mikulas Patockacd45daf2008-07-21 12:00:32 +0100740#ifdef CONFIG_DM_DEBUG
741 for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++)
742 BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));
743#endif
744
745 mempool_destroy(s->tracked_chunk_pool);
746
Milan Broz31c93a02006-12-08 02:41:11 -0800747 __free_exceptions(s);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748
Mikulas Patocka92e86812008-07-21 12:00:35 +0100749 mempool_destroy(s->pending_pool);
750
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 dm_put_device(ti, s->origin);
Jonathan Brassowfee19982009-04-02 19:55:34 +0100752
753 dm_exception_store_destroy(s->store);
Alasdair G Kergon138728d2006-03-27 01:17:50 -0800754
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755 kfree(s);
756}
757
758/*
759 * Flush a list of buffers.
760 */
761static void flush_bios(struct bio *bio)
762{
763 struct bio *n;
764
765 while (bio) {
766 n = bio->bi_next;
767 bio->bi_next = NULL;
768 generic_make_request(bio);
769 bio = n;
770 }
771}
772
David Howellsc4028952006-11-22 14:57:56 +0000773static void flush_queued_bios(struct work_struct *work)
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700774{
David Howellsc4028952006-11-22 14:57:56 +0000775 struct dm_snapshot *s =
776 container_of(work, struct dm_snapshot, queued_bios_work);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700777 struct bio *queued_bios;
778 unsigned long flags;
779
780 spin_lock_irqsave(&s->pe_lock, flags);
781 queued_bios = bio_list_get(&s->queued_bios);
782 spin_unlock_irqrestore(&s->pe_lock, flags);
783
784 flush_bios(queued_bios);
785}
786
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787/*
788 * Error a list of buffers.
789 */
790static void error_bios(struct bio *bio)
791{
792 struct bio *n;
793
794 while (bio) {
795 n = bio->bi_next;
796 bio->bi_next = NULL;
NeilBrown6712ecf2007-09-27 12:47:43 +0200797 bio_io_error(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798 bio = n;
799 }
800}
801
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700802static void __invalidate_snapshot(struct dm_snapshot *s, int err)
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800803{
804 if (!s->valid)
805 return;
806
807 if (err == -EIO)
808 DMERR("Invalidating snapshot: Error reading/writing.");
809 else if (err == -ENOMEM)
810 DMERR("Invalidating snapshot: Unable to allocate exception.");
811
Jonathan Brassow493df712009-04-02 19:55:31 +0100812 if (s->store->type->drop_snapshot)
813 s->store->type->drop_snapshot(s->store);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800814
815 s->valid = 0;
816
Jonathan Brassow0cea9c72009-04-02 19:55:32 +0100817 dm_table_event(s->store->ti->table);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800818}
819
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100820static void get_pending_exception(struct dm_snap_pending_exception *pe)
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700821{
822 atomic_inc(&pe->ref_count);
823}
824
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100825static struct bio *put_pending_exception(struct dm_snap_pending_exception *pe)
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700826{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100827 struct dm_snap_pending_exception *primary_pe;
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700828 struct bio *origin_bios = NULL;
829
830 primary_pe = pe->primary_pe;
831
832 /*
833 * If this pe is involved in a write to the origin and
834 * it is the last sibling to complete then release
835 * the bios for the original write to the origin.
836 */
837 if (primary_pe &&
Mikulas Patocka7c5f78b2008-10-21 17:44:51 +0100838 atomic_dec_and_test(&primary_pe->ref_count)) {
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700839 origin_bios = bio_list_get(&primary_pe->origin_bios);
Mikulas Patocka7c5f78b2008-10-21 17:44:51 +0100840 free_pending_exception(primary_pe);
841 }
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700842
843 /*
844 * Free the pe if it's not linked to an origin write or if
845 * it's not itself a primary pe.
846 */
847 if (!primary_pe || primary_pe != pe)
848 free_pending_exception(pe);
849
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700850 return origin_bios;
851}
852
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100853static void pending_complete(struct dm_snap_pending_exception *pe, int success)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100855 struct dm_snap_exception *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 struct dm_snapshot *s = pe->snap;
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700857 struct bio *origin_bios = NULL;
858 struct bio *snapshot_bios = NULL;
859 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800861 if (!success) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862 /* Read/write error - snapshot is unusable */
863 down_write(&s->lock);
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700864 __invalidate_snapshot(s, -EIO);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700865 error = 1;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800866 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867 }
868
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800869 e = alloc_exception();
870 if (!e) {
871 down_write(&s->lock);
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700872 __invalidate_snapshot(s, -ENOMEM);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700873 error = 1;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800874 goto out;
875 }
876 *e = pe->e;
877
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700878 down_write(&s->lock);
879 if (!s->valid) {
880 free_exception(e);
881 error = 1;
882 goto out;
883 }
884
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800885 /*
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100886 * Check for conflicting reads. This is extremely improbable,
Mikulas Patocka90fa1522009-01-06 03:04:54 +0000887 * so msleep(1) is sufficient and there is no need for a wait queue.
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100888 */
889 while (__chunk_is_tracked(s, pe->e.old_chunk))
Mikulas Patocka90fa1522009-01-06 03:04:54 +0000890 msleep(1);
Mikulas Patockaa8d41b52008-07-21 12:00:34 +0100891
892 /*
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800893 * Add a proper exception, and remove the
894 * in-flight exception from the list.
895 */
Milan Brozd74f81f2008-02-08 02:11:27 +0000896 insert_completed_exception(s, e);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800897
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898 out:
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700899 remove_exception(&pe->e);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700900 snapshot_bios = bio_list_get(&pe->snapshot_bios);
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700901 origin_bios = put_pending_exception(pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700903 up_write(&s->lock);
904
905 /* Submit any pending write bios */
906 if (error)
907 error_bios(snapshot_bios);
908 else
909 flush_bios(snapshot_bios);
910
911 flush_bios(origin_bios);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912}
913
914static void commit_callback(void *context, int success)
915{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100916 struct dm_snap_pending_exception *pe = context;
917
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918 pending_complete(pe, success);
919}
920
921/*
922 * Called when the copy I/O has finished. kcopyd actually runs
923 * this code so don't block.
924 */
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700925static void copy_callback(int read_err, unsigned long write_err, void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926{
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100927 struct dm_snap_pending_exception *pe = context;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 struct dm_snapshot *s = pe->snap;
929
930 if (read_err || write_err)
931 pending_complete(pe, 0);
932
933 else
934 /* Update the metadata if we are persistent */
Jonathan Brassow493df712009-04-02 19:55:31 +0100935 s->store->type->commit_exception(s->store, &pe->e,
936 commit_callback, pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937}
938
939/*
940 * Dispatches the copy operation to kcopyd.
941 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100942static void start_copy(struct dm_snap_pending_exception *pe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943{
944 struct dm_snapshot *s = pe->snap;
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +0100945 struct dm_io_region src, dest;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 struct block_device *bdev = s->origin->bdev;
947 sector_t dev_size;
948
949 dev_size = get_dev_size(bdev);
950
951 src.bdev = bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +0100952 src.sector = chunk_to_sector(s->store, pe->e.old_chunk);
Jonathan Brassowd0216842009-04-02 19:55:32 +0100953 src.count = min(s->store->chunk_size, dev_size - src.sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954
Jonathan Brassow49beb2b2009-04-02 19:55:33 +0100955 dest.bdev = s->store->cow->bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +0100956 dest.sector = chunk_to_sector(s->store, pe->e.new_chunk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957 dest.count = src.count;
958
959 /* Hand over to kcopyd */
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100960 dm_kcopyd_copy(s->kcopyd_client,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961 &src, 1, &dest, 0, copy_callback, pe);
962}
963
Mikulas Patocka29138082009-04-02 19:55:25 +0100964static struct dm_snap_pending_exception *
965__lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)
966{
967 struct dm_snap_exception *e = lookup_exception(&s->pending, chunk);
968
969 if (!e)
970 return NULL;
971
972 return container_of(e, struct dm_snap_pending_exception, e);
973}
974
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975/*
976 * Looks to see if this snapshot already has a pending exception
977 * for this chunk, otherwise it allocates a new one and inserts
978 * it into the pending table.
979 *
980 * NOTE: a write lock must be held on snap->lock before calling
981 * this.
982 */
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100983static struct dm_snap_pending_exception *
Mikulas Patockac6621392009-04-02 19:55:25 +0100984__find_pending_exception(struct dm_snapshot *s,
985 struct dm_snap_pending_exception *pe, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986{
Mikulas Patockac6621392009-04-02 19:55:25 +0100987 struct dm_snap_pending_exception *pe2;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800988
Mikulas Patocka29138082009-04-02 19:55:25 +0100989 pe2 = __lookup_pending_exception(s, chunk);
990 if (pe2) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800991 free_pending_exception(pe);
Mikulas Patocka29138082009-04-02 19:55:25 +0100992 return pe2;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800993 }
994
995 pe->e.old_chunk = chunk;
996 bio_list_init(&pe->origin_bios);
997 bio_list_init(&pe->snapshot_bios);
998 pe->primary_pe = NULL;
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700999 atomic_set(&pe->ref_count, 0);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001000 pe->started = 0;
1001
Jonathan Brassow493df712009-04-02 19:55:31 +01001002 if (s->store->type->prepare_exception(s->store, &pe->e)) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001003 free_pending_exception(pe);
1004 return NULL;
1005 }
1006
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001007 get_pending_exception(pe);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001008 insert_exception(&s->pending, &pe->e);
1009
Linus Torvalds1da177e2005-04-16 15:20:36 -07001010 return pe;
1011}
1012
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001013static void remap_exception(struct dm_snapshot *s, struct dm_snap_exception *e,
Milan Brozd74f81f2008-02-08 02:11:27 +00001014 struct bio *bio, chunk_t chunk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015{
Jonathan Brassow49beb2b2009-04-02 19:55:33 +01001016 bio->bi_bdev = s->store->cow->bdev;
Jonathan Brassow71fab002009-04-02 19:55:33 +01001017 bio->bi_sector = chunk_to_sector(s->store,
1018 dm_chunk_number(e->new_chunk) +
1019 (chunk - e->old_chunk)) +
1020 (bio->bi_sector &
1021 s->store->chunk_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001022}
1023
1024static int snapshot_map(struct dm_target *ti, struct bio *bio,
1025 union map_info *map_context)
1026{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001027 struct dm_snap_exception *e;
1028 struct dm_snapshot *s = ti->private;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001029 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030 chunk_t chunk;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001031 struct dm_snap_pending_exception *pe = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032
Jonathan Brassow71fab002009-04-02 19:55:33 +01001033 chunk = sector_to_chunk(s->store, bio->bi_sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034
1035 /* Full snapshots are not usable */
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001036 /* To get here the table must be live so s->active is always set. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 if (!s->valid)
Alasdair G Kergonf6a80ea2005-07-12 15:53:01 -07001038 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001040 /* FIXME: should only take write lock if we need
1041 * to copy an exception */
1042 down_write(&s->lock);
1043
1044 if (!s->valid) {
1045 r = -EIO;
1046 goto out_unlock;
1047 }
1048
1049 /* If the block is already remapped - use that, else remap it */
1050 e = lookup_exception(&s->complete, chunk);
1051 if (e) {
Milan Brozd74f81f2008-02-08 02:11:27 +00001052 remap_exception(s, e, bio, chunk);
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001053 goto out_unlock;
1054 }
1055
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 /*
1057 * Write to snapshot - higher level takes care of RW/RO
1058 * flags so we should only get this if we are
1059 * writeable.
1060 */
1061 if (bio_rw(bio) == WRITE) {
Mikulas Patocka29138082009-04-02 19:55:25 +01001062 pe = __lookup_pending_exception(s, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001063 if (!pe) {
Mikulas Patockac6621392009-04-02 19:55:25 +01001064 up_write(&s->lock);
1065 pe = alloc_pending_exception(s);
1066 down_write(&s->lock);
1067
1068 if (!s->valid) {
1069 free_pending_exception(pe);
1070 r = -EIO;
1071 goto out_unlock;
1072 }
1073
Mikulas Patocka35bf6592009-04-02 19:55:26 +01001074 e = lookup_exception(&s->complete, chunk);
1075 if (e) {
1076 free_pending_exception(pe);
1077 remap_exception(s, e, bio, chunk);
1078 goto out_unlock;
1079 }
1080
Mikulas Patockac6621392009-04-02 19:55:25 +01001081 pe = __find_pending_exception(s, pe, chunk);
Mikulas Patocka29138082009-04-02 19:55:25 +01001082 if (!pe) {
1083 __invalidate_snapshot(s, -ENOMEM);
1084 r = -EIO;
1085 goto out_unlock;
1086 }
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001087 }
1088
Milan Brozd74f81f2008-02-08 02:11:27 +00001089 remap_exception(s, &pe->e, bio, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001090 bio_list_add(&pe->snapshot_bios, bio);
1091
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001092 r = DM_MAPIO_SUBMITTED;
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001093
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001094 if (!pe->started) {
1095 /* this is protected by snap->lock */
1096 pe->started = 1;
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001097 up_write(&s->lock);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001098 start_copy(pe);
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001099 goto out;
1100 }
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001101 } else {
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001102 bio->bi_bdev = s->origin->bdev;
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001103 map_context->ptr = track_chunk(s, chunk);
1104 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -07001106 out_unlock:
1107 up_write(&s->lock);
1108 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 return r;
1110}
1111
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001112static int snapshot_end_io(struct dm_target *ti, struct bio *bio,
1113 int error, union map_info *map_context)
1114{
1115 struct dm_snapshot *s = ti->private;
1116 struct dm_snap_tracked_chunk *c = map_context->ptr;
1117
1118 if (c)
1119 stop_tracking_chunk(s, c);
1120
1121 return 0;
1122}
1123
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124static void snapshot_resume(struct dm_target *ti)
1125{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001126 struct dm_snapshot *s = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001128 down_write(&s->lock);
1129 s->active = 1;
1130 up_write(&s->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131}
1132
1133static int snapshot_status(struct dm_target *ti, status_type_t type,
1134 char *result, unsigned int maxlen)
1135{
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001136 unsigned sz = 0;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001137 struct dm_snapshot *snap = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138
1139 switch (type) {
1140 case STATUSTYPE_INFO:
1141 if (!snap->valid)
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001142 DMEMIT("Invalid");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143 else {
Jonathan Brassow493df712009-04-02 19:55:31 +01001144 if (snap->store->type->fraction_full) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145 sector_t numerator, denominator;
Jonathan Brassow493df712009-04-02 19:55:31 +01001146 snap->store->type->fraction_full(snap->store,
1147 &numerator,
1148 &denominator);
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001149 DMEMIT("%llu/%llu",
1150 (unsigned long long)numerator,
1151 (unsigned long long)denominator);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 }
1153 else
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001154 DMEMIT("Unknown");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 }
1156 break;
1157
1158 case STATUSTYPE_TABLE:
1159 /*
1160 * kdevname returns a static pointer so we need
1161 * to make private copies if the output is to
1162 * make sense.
1163 */
Jonathan Brassow2e4a31d2009-04-02 19:55:34 +01001164 DMEMIT("%s", snap->origin->name);
Jonathan Brassow1e302a92009-04-02 19:55:35 +01001165 snap->store->type->status(snap->store, type, result + sz,
1166 maxlen - sz);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001167 break;
1168 }
1169
1170 return 0;
1171}
1172
1173/*-----------------------------------------------------------------
1174 * Origin methods
1175 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176static int __origin_write(struct list_head *snapshots, struct bio *bio)
1177{
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001178 int r = DM_MAPIO_REMAPPED, first = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179 struct dm_snapshot *snap;
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001180 struct dm_snap_exception *e;
1181 struct dm_snap_pending_exception *pe, *next_pe, *primary_pe = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182 chunk_t chunk;
Alasdair G Kergoneccf0812006-03-27 01:17:42 -08001183 LIST_HEAD(pe_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184
1185 /* Do all the snapshots on this origin */
1186 list_for_each_entry (snap, snapshots, list) {
1187
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001188 down_write(&snap->lock);
1189
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001190 /* Only deal with valid and active snapshots */
1191 if (!snap->valid || !snap->active)
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001192 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001193
Alasdair G Kergond5e404c2005-07-12 15:53:05 -07001194 /* Nothing to do if writing beyond end of snapshot */
Jonathan Brassow0cea9c72009-04-02 19:55:32 +01001195 if (bio->bi_sector >= dm_table_get_size(snap->store->ti->table))
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001196 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001197
1198 /*
1199 * Remember, different snapshots can have
1200 * different chunk sizes.
1201 */
Jonathan Brassow71fab002009-04-02 19:55:33 +01001202 chunk = sector_to_chunk(snap->store, bio->bi_sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203
1204 /*
1205 * Check exception table to see if block
1206 * is already remapped in this snapshot
1207 * and trigger an exception if not.
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001208 *
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001209 * ref_count is initialised to 1 so pending_complete()
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001210 * won't destroy the primary_pe while we're inside this loop.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 */
1212 e = lookup_exception(&snap->complete, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001213 if (e)
1214 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215
Mikulas Patocka29138082009-04-02 19:55:25 +01001216 pe = __lookup_pending_exception(snap, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001217 if (!pe) {
Mikulas Patockac6621392009-04-02 19:55:25 +01001218 up_write(&snap->lock);
1219 pe = alloc_pending_exception(snap);
1220 down_write(&snap->lock);
1221
1222 if (!snap->valid) {
1223 free_pending_exception(pe);
1224 goto next_snapshot;
1225 }
1226
Mikulas Patocka35bf6592009-04-02 19:55:26 +01001227 e = lookup_exception(&snap->complete, chunk);
1228 if (e) {
1229 free_pending_exception(pe);
1230 goto next_snapshot;
1231 }
1232
Mikulas Patockac6621392009-04-02 19:55:25 +01001233 pe = __find_pending_exception(snap, pe, chunk);
Mikulas Patocka29138082009-04-02 19:55:25 +01001234 if (!pe) {
1235 __invalidate_snapshot(snap, -ENOMEM);
1236 goto next_snapshot;
1237 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238 }
1239
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001240 if (!primary_pe) {
1241 /*
1242 * Either every pe here has same
1243 * primary_pe or none has one yet.
1244 */
1245 if (pe->primary_pe)
1246 primary_pe = pe->primary_pe;
1247 else {
1248 primary_pe = pe;
1249 first = 1;
1250 }
1251
1252 bio_list_add(&primary_pe->origin_bios, bio);
1253
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001254 r = DM_MAPIO_SUBMITTED;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001255 }
1256
1257 if (!pe->primary_pe) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001258 pe->primary_pe = primary_pe;
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001259 get_pending_exception(primary_pe);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001260 }
1261
1262 if (!pe->started) {
1263 pe->started = 1;
1264 list_add_tail(&pe->list, &pe_queue);
1265 }
1266
1267 next_snapshot:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001268 up_write(&snap->lock);
1269 }
1270
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001271 if (!primary_pe)
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001272 return r;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001273
1274 /*
1275 * If this is the first time we're processing this chunk and
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001276 * ref_count is now 1 it means all the pending exceptions
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001277 * got completed while we were in the loop above, so it falls to
1278 * us here to remove the primary_pe and submit any origin_bios.
1279 */
1280
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001281 if (first && atomic_dec_and_test(&primary_pe->ref_count)) {
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001282 flush_bios(bio_list_get(&primary_pe->origin_bios));
1283 free_pending_exception(primary_pe);
1284 /* If we got here, pe_queue is necessarily empty. */
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001285 return r;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001286 }
1287
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288 /*
1289 * Now that we have a complete pe list we can start the copying.
1290 */
Alasdair G Kergoneccf0812006-03-27 01:17:42 -08001291 list_for_each_entry_safe(pe, next_pe, &pe_queue, list)
1292 start_copy(pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293
1294 return r;
1295}
1296
1297/*
1298 * Called on a write from the origin driver.
1299 */
1300static int do_origin(struct dm_dev *origin, struct bio *bio)
1301{
1302 struct origin *o;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001303 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001304
1305 down_read(&_origins_lock);
1306 o = __lookup_origin(origin->bdev);
1307 if (o)
1308 r = __origin_write(&o->snapshots, bio);
1309 up_read(&_origins_lock);
1310
1311 return r;
1312}
1313
1314/*
1315 * Origin: maps a linear range of a device, with hooks for snapshotting.
1316 */
1317
1318/*
1319 * Construct an origin mapping: <dev_path>
1320 * The context for an origin is merely a 'struct dm_dev *'
1321 * pointing to the real device.
1322 */
1323static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1324{
1325 int r;
1326 struct dm_dev *dev;
1327
1328 if (argc != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001329 ti->error = "origin: incorrect number of arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330 return -EINVAL;
1331 }
1332
1333 r = dm_get_device(ti, argv[0], 0, ti->len,
1334 dm_table_get_mode(ti->table), &dev);
1335 if (r) {
1336 ti->error = "Cannot get target device";
1337 return r;
1338 }
1339
1340 ti->private = dev;
1341 return 0;
1342}
1343
1344static void origin_dtr(struct dm_target *ti)
1345{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001346 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001347 dm_put_device(ti, dev);
1348}
1349
1350static int origin_map(struct dm_target *ti, struct bio *bio,
1351 union map_info *map_context)
1352{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001353 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354 bio->bi_bdev = dev->bdev;
1355
1356 /* Only tell snapshots if this is a write */
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001357 return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001358}
1359
1360#define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))
1361
1362/*
1363 * Set the target "split_io" field to the minimum of all the snapshots'
1364 * chunk sizes.
1365 */
1366static void origin_resume(struct dm_target *ti)
1367{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001368 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369 struct dm_snapshot *snap;
1370 struct origin *o;
1371 chunk_t chunk_size = 0;
1372
1373 down_read(&_origins_lock);
1374 o = __lookup_origin(dev->bdev);
1375 if (o)
1376 list_for_each_entry (snap, &o->snapshots, list)
Jonathan Brassowd0216842009-04-02 19:55:32 +01001377 chunk_size = min_not_zero(chunk_size,
1378 snap->store->chunk_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 up_read(&_origins_lock);
1380
1381 ti->split_io = chunk_size;
1382}
1383
1384static int origin_status(struct dm_target *ti, status_type_t type, char *result,
1385 unsigned int maxlen)
1386{
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001387 struct dm_dev *dev = ti->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388
1389 switch (type) {
1390 case STATUSTYPE_INFO:
1391 result[0] = '\0';
1392 break;
1393
1394 case STATUSTYPE_TABLE:
1395 snprintf(result, maxlen, "%s", dev->name);
1396 break;
1397 }
1398
1399 return 0;
1400}
1401
1402static struct target_type origin_target = {
1403 .name = "snapshot-origin",
Milan Brozd74f81f2008-02-08 02:11:27 +00001404 .version = {1, 6, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405 .module = THIS_MODULE,
1406 .ctr = origin_ctr,
1407 .dtr = origin_dtr,
1408 .map = origin_map,
1409 .resume = origin_resume,
1410 .status = origin_status,
1411};
1412
1413static struct target_type snapshot_target = {
1414 .name = "snapshot",
Milan Brozd74f81f2008-02-08 02:11:27 +00001415 .version = {1, 6, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001416 .module = THIS_MODULE,
1417 .ctr = snapshot_ctr,
1418 .dtr = snapshot_dtr,
1419 .map = snapshot_map,
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001420 .end_io = snapshot_end_io,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421 .resume = snapshot_resume,
1422 .status = snapshot_status,
1423};
1424
1425static int __init dm_snapshot_init(void)
1426{
1427 int r;
1428
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001429 r = dm_exception_store_init();
1430 if (r) {
1431 DMERR("Failed to initialize exception stores");
1432 return r;
1433 }
1434
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435 r = dm_register_target(&snapshot_target);
1436 if (r) {
1437 DMERR("snapshot target register failed %d", r);
1438 return r;
1439 }
1440
1441 r = dm_register_target(&origin_target);
1442 if (r < 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001443 DMERR("Origin target register failed %d", r);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001444 goto bad1;
1445 }
1446
1447 r = init_origin_hash();
1448 if (r) {
1449 DMERR("init_origin_hash failed.");
1450 goto bad2;
1451 }
1452
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001453 exception_cache = KMEM_CACHE(dm_snap_exception, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454 if (!exception_cache) {
1455 DMERR("Couldn't create exception cache.");
1456 r = -ENOMEM;
1457 goto bad3;
1458 }
1459
Alasdair G Kergon028867a2007-07-12 17:26:32 +01001460 pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001461 if (!pending_cache) {
1462 DMERR("Couldn't create pending cache.");
1463 r = -ENOMEM;
1464 goto bad4;
1465 }
1466
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001467 tracked_chunk_cache = KMEM_CACHE(dm_snap_tracked_chunk, 0);
1468 if (!tracked_chunk_cache) {
1469 DMERR("Couldn't create cache to track chunks in use.");
1470 r = -ENOMEM;
1471 goto bad5;
1472 }
1473
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001474 ksnapd = create_singlethread_workqueue("ksnapd");
1475 if (!ksnapd) {
1476 DMERR("Failed to create ksnapd workqueue.");
1477 r = -ENOMEM;
Mikulas Patocka92e86812008-07-21 12:00:35 +01001478 goto bad_pending_pool;
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001479 }
1480
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481 return 0;
1482
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001483bad_pending_pool:
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001484 kmem_cache_destroy(tracked_chunk_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001485bad5:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001486 kmem_cache_destroy(pending_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001487bad4:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488 kmem_cache_destroy(exception_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001489bad3:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490 exit_origin_hash();
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001491bad2:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492 dm_unregister_target(&origin_target);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001493bad1:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494 dm_unregister_target(&snapshot_target);
1495 return r;
1496}
1497
1498static void __exit dm_snapshot_exit(void)
1499{
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001500 destroy_workqueue(ksnapd);
1501
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001502 dm_unregister_target(&snapshot_target);
1503 dm_unregister_target(&origin_target);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001504
1505 exit_origin_hash();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506 kmem_cache_destroy(pending_cache);
1507 kmem_cache_destroy(exception_cache);
Mikulas Patockacd45daf2008-07-21 12:00:32 +01001508 kmem_cache_destroy(tracked_chunk_cache);
Alasdair G Kergon4db6bfe2009-01-06 03:05:17 +00001509
1510 dm_exception_store_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511}
1512
1513/* Module hooks */
1514module_init(dm_snapshot_init);
1515module_exit(dm_snapshot_exit);
1516
1517MODULE_DESCRIPTION(DM_NAME " snapshot target");
1518MODULE_AUTHOR("Joe Thornber");
1519MODULE_LICENSE("GPL");