blob: 0821a2b68a73a98719cb931c82196a74bfe471d4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * dm-snapshot.c
3 *
4 * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
5 *
6 * This file is released under the GPL.
7 */
8
9#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/ctype.h>
11#include <linux/device-mapper.h>
12#include <linux/fs.h>
13#include <linux/init.h>
14#include <linux/kdev_t.h>
15#include <linux/list.h>
16#include <linux/mempool.h>
17#include <linux/module.h>
18#include <linux/slab.h>
19#include <linux/vmalloc.h>
20
21#include "dm-snap.h"
22#include "dm-bio-list.h"
23#include "kcopyd.h"
24
Alasdair G Kergon72d94862006-06-26 00:27:35 -070025#define DM_MSG_PREFIX "snapshots"
26
Linus Torvalds1da177e2005-04-16 15:20:36 -070027/*
28 * The percentage increment we will wake up users at
29 */
30#define WAKE_UP_PERCENT 5
31
32/*
33 * kcopyd priority of snapshot operations
34 */
35#define SNAPSHOT_COPY_PRIORITY 2
36
37/*
38 * Each snapshot reserves this many pages for io
39 */
40#define SNAPSHOT_PAGES 256
41
Adrian Bunkc642f9e2006-12-08 02:41:13 -080042static struct workqueue_struct *ksnapd;
David Howellsc4028952006-11-22 14:57:56 +000043static void flush_queued_bios(struct work_struct *work);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -070044
Linus Torvalds1da177e2005-04-16 15:20:36 -070045struct pending_exception {
46 struct exception e;
47
48 /*
49 * Origin buffers waiting for this to complete are held
50 * in a bio list
51 */
52 struct bio_list origin_bios;
53 struct bio_list snapshot_bios;
54
55 /*
Alasdair G Kergoneccf0812006-03-27 01:17:42 -080056 * Short-term queue of pending exceptions prior to submission.
57 */
58 struct list_head list;
59
60 /*
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -080061 * The primary pending_exception is the one that holds
Alasdair G Kergon4b832e82006-10-03 01:15:30 -070062 * the ref_count and the list of origin_bios for a
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -080063 * group of pending_exceptions. It is always last to get freed.
64 * These fields get set up when writing to the origin.
Linus Torvalds1da177e2005-04-16 15:20:36 -070065 */
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -080066 struct pending_exception *primary_pe;
67
68 /*
69 * Number of pending_exceptions processing this chunk.
70 * When this drops to zero we must complete the origin bios.
71 * If incrementing or decrementing this, hold pe->snap->lock for
72 * the sibling concerned and not pe->primary_pe->snap->lock unless
73 * they are the same.
74 */
Alasdair G Kergon4b832e82006-10-03 01:15:30 -070075 atomic_t ref_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -070076
77 /* Pointer back to snapshot context */
78 struct dm_snapshot *snap;
79
80 /*
81 * 1 indicates the exception has already been sent to
82 * kcopyd.
83 */
84 int started;
85};
86
87/*
88 * Hash table mapping origin volumes to lists of snapshots and
89 * a lock to protect it
90 */
Christoph Lametere18b8902006-12-06 20:33:20 -080091static struct kmem_cache *exception_cache;
92static struct kmem_cache *pending_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -070093static mempool_t *pending_pool;
94
95/*
96 * One of these per registered origin, held in the snapshot_origins hash
97 */
98struct origin {
99 /* The origin device */
100 struct block_device *bdev;
101
102 struct list_head hash_list;
103
104 /* List of snapshots for this origin */
105 struct list_head snapshots;
106};
107
108/*
109 * Size of the hash table for origin volumes. If we make this
110 * the size of the minors list then it should be nearly perfect
111 */
112#define ORIGIN_HASH_SIZE 256
113#define ORIGIN_MASK 0xFF
114static struct list_head *_origins;
115static struct rw_semaphore _origins_lock;
116
117static int init_origin_hash(void)
118{
119 int i;
120
121 _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head),
122 GFP_KERNEL);
123 if (!_origins) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700124 DMERR("unable to allocate memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125 return -ENOMEM;
126 }
127
128 for (i = 0; i < ORIGIN_HASH_SIZE; i++)
129 INIT_LIST_HEAD(_origins + i);
130 init_rwsem(&_origins_lock);
131
132 return 0;
133}
134
135static void exit_origin_hash(void)
136{
137 kfree(_origins);
138}
139
140static inline unsigned int origin_hash(struct block_device *bdev)
141{
142 return bdev->bd_dev & ORIGIN_MASK;
143}
144
145static struct origin *__lookup_origin(struct block_device *origin)
146{
147 struct list_head *ol;
148 struct origin *o;
149
150 ol = &_origins[origin_hash(origin)];
151 list_for_each_entry (o, ol, hash_list)
152 if (bdev_equal(o->bdev, origin))
153 return o;
154
155 return NULL;
156}
157
158static void __insert_origin(struct origin *o)
159{
160 struct list_head *sl = &_origins[origin_hash(o->bdev)];
161 list_add_tail(&o->hash_list, sl);
162}
163
164/*
165 * Make a note of the snapshot and its origin so we can look it
166 * up when the origin has a write on it.
167 */
168static int register_snapshot(struct dm_snapshot *snap)
169{
170 struct origin *o;
171 struct block_device *bdev = snap->origin->bdev;
172
173 down_write(&_origins_lock);
174 o = __lookup_origin(bdev);
175
176 if (!o) {
177 /* New origin */
178 o = kmalloc(sizeof(*o), GFP_KERNEL);
179 if (!o) {
180 up_write(&_origins_lock);
181 return -ENOMEM;
182 }
183
184 /* Initialise the struct */
185 INIT_LIST_HEAD(&o->snapshots);
186 o->bdev = bdev;
187
188 __insert_origin(o);
189 }
190
191 list_add_tail(&snap->list, &o->snapshots);
192
193 up_write(&_origins_lock);
194 return 0;
195}
196
197static void unregister_snapshot(struct dm_snapshot *s)
198{
199 struct origin *o;
200
201 down_write(&_origins_lock);
202 o = __lookup_origin(s->origin->bdev);
203
204 list_del(&s->list);
205 if (list_empty(&o->snapshots)) {
206 list_del(&o->hash_list);
207 kfree(o);
208 }
209
210 up_write(&_origins_lock);
211}
212
213/*
214 * Implementation of the exception hash tables.
215 */
216static int init_exception_table(struct exception_table *et, uint32_t size)
217{
218 unsigned int i;
219
220 et->hash_mask = size - 1;
221 et->table = dm_vcalloc(size, sizeof(struct list_head));
222 if (!et->table)
223 return -ENOMEM;
224
225 for (i = 0; i < size; i++)
226 INIT_LIST_HEAD(et->table + i);
227
228 return 0;
229}
230
Christoph Lametere18b8902006-12-06 20:33:20 -0800231static void exit_exception_table(struct exception_table *et, struct kmem_cache *mem)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232{
233 struct list_head *slot;
234 struct exception *ex, *next;
235 int i, size;
236
237 size = et->hash_mask + 1;
238 for (i = 0; i < size; i++) {
239 slot = et->table + i;
240
241 list_for_each_entry_safe (ex, next, slot, hash_list)
242 kmem_cache_free(mem, ex);
243 }
244
245 vfree(et->table);
246}
247
248static inline uint32_t exception_hash(struct exception_table *et, chunk_t chunk)
249{
250 return chunk & et->hash_mask;
251}
252
253static void insert_exception(struct exception_table *eh, struct exception *e)
254{
255 struct list_head *l = &eh->table[exception_hash(eh, e->old_chunk)];
256 list_add(&e->hash_list, l);
257}
258
259static inline void remove_exception(struct exception *e)
260{
261 list_del(&e->hash_list);
262}
263
264/*
265 * Return the exception data for a sector, or NULL if not
266 * remapped.
267 */
268static struct exception *lookup_exception(struct exception_table *et,
269 chunk_t chunk)
270{
271 struct list_head *slot;
272 struct exception *e;
273
274 slot = &et->table[exception_hash(et, chunk)];
275 list_for_each_entry (e, slot, hash_list)
276 if (e->old_chunk == chunk)
277 return e;
278
279 return NULL;
280}
281
282static inline struct exception *alloc_exception(void)
283{
284 struct exception *e;
285
286 e = kmem_cache_alloc(exception_cache, GFP_NOIO);
287 if (!e)
288 e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
289
290 return e;
291}
292
293static inline void free_exception(struct exception *e)
294{
295 kmem_cache_free(exception_cache, e);
296}
297
298static inline struct pending_exception *alloc_pending_exception(void)
299{
300 return mempool_alloc(pending_pool, GFP_NOIO);
301}
302
303static inline void free_pending_exception(struct pending_exception *pe)
304{
305 mempool_free(pe, pending_pool);
306}
307
308int dm_add_exception(struct dm_snapshot *s, chunk_t old, chunk_t new)
309{
310 struct exception *e;
311
312 e = alloc_exception();
313 if (!e)
314 return -ENOMEM;
315
316 e->old_chunk = old;
317 e->new_chunk = new;
318 insert_exception(&s->complete, e);
319 return 0;
320}
321
322/*
323 * Hard coded magic.
324 */
325static int calc_max_buckets(void)
326{
327 /* use a fixed size of 2MB */
328 unsigned long mem = 2 * 1024 * 1024;
329 mem /= sizeof(struct list_head);
330
331 return mem;
332}
333
334/*
335 * Rounds a number down to a power of 2.
336 */
337static inline uint32_t round_down(uint32_t n)
338{
339 while (n & (n - 1))
340 n &= (n - 1);
341 return n;
342}
343
344/*
345 * Allocate room for a suitable hash table.
346 */
347static int init_hash_tables(struct dm_snapshot *s)
348{
349 sector_t hash_size, cow_dev_size, origin_dev_size, max_buckets;
350
351 /*
352 * Calculate based on the size of the original volume or
353 * the COW volume...
354 */
355 cow_dev_size = get_dev_size(s->cow->bdev);
356 origin_dev_size = get_dev_size(s->origin->bdev);
357 max_buckets = calc_max_buckets();
358
359 hash_size = min(origin_dev_size, cow_dev_size) >> s->chunk_shift;
360 hash_size = min(hash_size, max_buckets);
361
362 /* Round it down to a power of 2 */
363 hash_size = round_down(hash_size);
364 if (init_exception_table(&s->complete, hash_size))
365 return -ENOMEM;
366
367 /*
368 * Allocate hash table for in-flight exceptions
369 * Make this smaller than the real hash table
370 */
371 hash_size >>= 3;
372 if (hash_size < 64)
373 hash_size = 64;
374
375 if (init_exception_table(&s->pending, hash_size)) {
376 exit_exception_table(&s->complete, exception_cache);
377 return -ENOMEM;
378 }
379
380 return 0;
381}
382
383/*
384 * Round a number up to the nearest 'size' boundary. size must
385 * be a power of 2.
386 */
387static inline ulong round_up(ulong n, ulong size)
388{
389 size--;
390 return (n + size) & ~size;
391}
392
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -0700393static int set_chunk_size(struct dm_snapshot *s, const char *chunk_size_arg,
394 char **error)
395{
396 unsigned long chunk_size;
397 char *value;
398
399 chunk_size = simple_strtoul(chunk_size_arg, &value, 10);
400 if (*chunk_size_arg == '\0' || *value != '\0') {
401 *error = "Invalid chunk size";
402 return -EINVAL;
403 }
404
405 if (!chunk_size) {
406 s->chunk_size = s->chunk_mask = s->chunk_shift = 0;
407 return 0;
408 }
409
410 /*
411 * Chunk size must be multiple of page size. Silently
412 * round up if it's not.
413 */
414 chunk_size = round_up(chunk_size, PAGE_SIZE >> 9);
415
416 /* Check chunk_size is a power of 2 */
417 if (chunk_size & (chunk_size - 1)) {
418 *error = "Chunk size is not a power of 2";
419 return -EINVAL;
420 }
421
422 /* Validate the chunk size against the device block size */
423 if (chunk_size % (bdev_hardsect_size(s->cow->bdev) >> 9)) {
424 *error = "Chunk size is not a multiple of device blocksize";
425 return -EINVAL;
426 }
427
428 s->chunk_size = chunk_size;
429 s->chunk_mask = chunk_size - 1;
430 s->chunk_shift = ffs(chunk_size) - 1;
431
432 return 0;
433}
434
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435/*
436 * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
437 */
438static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
439{
440 struct dm_snapshot *s;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 int r = -EINVAL;
442 char persistent;
443 char *origin_path;
444 char *cow_path;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -0700446 if (argc != 4) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700447 ti->error = "requires exactly 4 arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 r = -EINVAL;
449 goto bad1;
450 }
451
452 origin_path = argv[0];
453 cow_path = argv[1];
454 persistent = toupper(*argv[2]);
455
456 if (persistent != 'P' && persistent != 'N') {
457 ti->error = "Persistent flag is not P or N";
458 r = -EINVAL;
459 goto bad1;
460 }
461
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462 s = kmalloc(sizeof(*s), GFP_KERNEL);
463 if (s == NULL) {
464 ti->error = "Cannot allocate snapshot context private "
465 "structure";
466 r = -ENOMEM;
467 goto bad1;
468 }
469
470 r = dm_get_device(ti, origin_path, 0, ti->len, FMODE_READ, &s->origin);
471 if (r) {
472 ti->error = "Cannot get origin device";
473 goto bad2;
474 }
475
476 r = dm_get_device(ti, cow_path, 0, 0,
477 FMODE_READ | FMODE_WRITE, &s->cow);
478 if (r) {
479 dm_put_device(ti, s->origin);
480 ti->error = "Cannot get COW device";
481 goto bad2;
482 }
483
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -0700484 r = set_chunk_size(s, argv[3], &ti->error);
485 if (r)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 goto bad3;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 s->type = persistent;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489
490 s->valid = 1;
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800491 s->active = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492 s->last_percent = 0;
493 init_rwsem(&s->lock);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700494 spin_lock_init(&s->pe_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 s->table = ti->table;
496
497 /* Allocate hash table for COW data */
498 if (init_hash_tables(s)) {
499 ti->error = "Unable to allocate hash table space";
500 r = -ENOMEM;
501 goto bad3;
502 }
503
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 s->store.snap = s;
505
506 if (persistent == 'P')
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -0700507 r = dm_create_persistent(&s->store);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 else
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -0700509 r = dm_create_transient(&s->store);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510
511 if (r) {
512 ti->error = "Couldn't create exception store";
513 r = -EINVAL;
514 goto bad4;
515 }
516
517 r = kcopyd_client_create(SNAPSHOT_PAGES, &s->kcopyd_client);
518 if (r) {
519 ti->error = "Could not create kcopyd client";
520 goto bad5;
521 }
522
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800523 /* Metadata must only be loaded into one table at once */
Mark McLoughlinf9cea4f2006-10-03 01:15:25 -0700524 r = s->store.read_metadata(&s->store);
525 if (r) {
526 ti->error = "Failed to read snapshot metadata";
527 goto bad6;
528 }
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800529
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700530 bio_list_init(&s->queued_bios);
David Howellsc4028952006-11-22 14:57:56 +0000531 INIT_WORK(&s->queued_bios_work, flush_queued_bios);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700532
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 /* Add snapshot to the list of snapshots for this origin */
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800534 /* Exceptions aren't triggered till snapshot_resume() is called */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 if (register_snapshot(s)) {
536 r = -EINVAL;
537 ti->error = "Cannot register snapshot origin";
538 goto bad6;
539 }
540
541 ti->private = s;
Alasdair G Kergonc51c2752006-06-26 00:27:18 -0700542 ti->split_io = s->chunk_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543
544 return 0;
545
546 bad6:
547 kcopyd_client_destroy(s->kcopyd_client);
548
549 bad5:
550 s->store.destroy(&s->store);
551
552 bad4:
553 exit_exception_table(&s->pending, pending_cache);
554 exit_exception_table(&s->complete, exception_cache);
555
556 bad3:
557 dm_put_device(ti, s->cow);
558 dm_put_device(ti, s->origin);
559
560 bad2:
561 kfree(s);
562
563 bad1:
564 return r;
565}
566
Milan Broz31c93a02006-12-08 02:41:11 -0800567static void __free_exceptions(struct dm_snapshot *s)
568{
569 kcopyd_client_destroy(s->kcopyd_client);
570 s->kcopyd_client = NULL;
571
572 exit_exception_table(&s->pending, pending_cache);
573 exit_exception_table(&s->complete, exception_cache);
574
575 s->store.destroy(&s->store);
576}
577
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578static void snapshot_dtr(struct dm_target *ti)
579{
580 struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
581
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700582 flush_workqueue(ksnapd);
583
Alasdair G Kergon138728d2006-03-27 01:17:50 -0800584 /* Prevent further origin writes from using this snapshot. */
585 /* After this returns there can be no new kcopyd jobs. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 unregister_snapshot(s);
587
Milan Broz31c93a02006-12-08 02:41:11 -0800588 __free_exceptions(s);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589
590 dm_put_device(ti, s->origin);
591 dm_put_device(ti, s->cow);
Alasdair G Kergon138728d2006-03-27 01:17:50 -0800592
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 kfree(s);
594}
595
596/*
597 * Flush a list of buffers.
598 */
599static void flush_bios(struct bio *bio)
600{
601 struct bio *n;
602
603 while (bio) {
604 n = bio->bi_next;
605 bio->bi_next = NULL;
606 generic_make_request(bio);
607 bio = n;
608 }
609}
610
David Howellsc4028952006-11-22 14:57:56 +0000611static void flush_queued_bios(struct work_struct *work)
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700612{
David Howellsc4028952006-11-22 14:57:56 +0000613 struct dm_snapshot *s =
614 container_of(work, struct dm_snapshot, queued_bios_work);
Alasdair G Kergonca3a9312006-10-03 01:15:30 -0700615 struct bio *queued_bios;
616 unsigned long flags;
617
618 spin_lock_irqsave(&s->pe_lock, flags);
619 queued_bios = bio_list_get(&s->queued_bios);
620 spin_unlock_irqrestore(&s->pe_lock, flags);
621
622 flush_bios(queued_bios);
623}
624
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625/*
626 * Error a list of buffers.
627 */
628static void error_bios(struct bio *bio)
629{
630 struct bio *n;
631
632 while (bio) {
633 n = bio->bi_next;
634 bio->bi_next = NULL;
635 bio_io_error(bio, bio->bi_size);
636 bio = n;
637 }
638}
639
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700640static void __invalidate_snapshot(struct dm_snapshot *s, int err)
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800641{
642 if (!s->valid)
643 return;
644
645 if (err == -EIO)
646 DMERR("Invalidating snapshot: Error reading/writing.");
647 else if (err == -ENOMEM)
648 DMERR("Invalidating snapshot: Unable to allocate exception.");
649
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800650 if (s->store.drop_snapshot)
651 s->store.drop_snapshot(&s->store);
652
653 s->valid = 0;
654
655 dm_table_event(s->table);
656}
657
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700658static void get_pending_exception(struct pending_exception *pe)
659{
660 atomic_inc(&pe->ref_count);
661}
662
663static struct bio *put_pending_exception(struct pending_exception *pe)
664{
665 struct pending_exception *primary_pe;
666 struct bio *origin_bios = NULL;
667
668 primary_pe = pe->primary_pe;
669
670 /*
671 * If this pe is involved in a write to the origin and
672 * it is the last sibling to complete then release
673 * the bios for the original write to the origin.
674 */
675 if (primary_pe &&
676 atomic_dec_and_test(&primary_pe->ref_count))
677 origin_bios = bio_list_get(&primary_pe->origin_bios);
678
679 /*
680 * Free the pe if it's not linked to an origin write or if
681 * it's not itself a primary pe.
682 */
683 if (!primary_pe || primary_pe != pe)
684 free_pending_exception(pe);
685
686 /*
687 * Free the primary pe if nothing references it.
688 */
689 if (primary_pe && !atomic_read(&primary_pe->ref_count))
690 free_pending_exception(primary_pe);
691
692 return origin_bios;
693}
694
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695static void pending_complete(struct pending_exception *pe, int success)
696{
697 struct exception *e;
698 struct dm_snapshot *s = pe->snap;
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700699 struct bio *origin_bios = NULL;
700 struct bio *snapshot_bios = NULL;
701 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800703 if (!success) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 /* Read/write error - snapshot is unusable */
705 down_write(&s->lock);
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700706 __invalidate_snapshot(s, -EIO);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700707 error = 1;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800708 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 }
710
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800711 e = alloc_exception();
712 if (!e) {
713 down_write(&s->lock);
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700714 __invalidate_snapshot(s, -ENOMEM);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700715 error = 1;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800716 goto out;
717 }
718 *e = pe->e;
719
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700720 down_write(&s->lock);
721 if (!s->valid) {
722 free_exception(e);
723 error = 1;
724 goto out;
725 }
726
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800727 /*
728 * Add a proper exception, and remove the
729 * in-flight exception from the list.
730 */
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800731 insert_exception(&s->complete, e);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800732
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 out:
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700734 remove_exception(&pe->e);
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700735 snapshot_bios = bio_list_get(&pe->snapshot_bios);
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700736 origin_bios = put_pending_exception(pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737
Alasdair G Kergon9d493fa2006-10-03 01:15:29 -0700738 up_write(&s->lock);
739
740 /* Submit any pending write bios */
741 if (error)
742 error_bios(snapshot_bios);
743 else
744 flush_bios(snapshot_bios);
745
746 flush_bios(origin_bios);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747}
748
749static void commit_callback(void *context, int success)
750{
751 struct pending_exception *pe = (struct pending_exception *) context;
752 pending_complete(pe, success);
753}
754
755/*
756 * Called when the copy I/O has finished. kcopyd actually runs
757 * this code so don't block.
758 */
759static void copy_callback(int read_err, unsigned int write_err, void *context)
760{
761 struct pending_exception *pe = (struct pending_exception *) context;
762 struct dm_snapshot *s = pe->snap;
763
764 if (read_err || write_err)
765 pending_complete(pe, 0);
766
767 else
768 /* Update the metadata if we are persistent */
769 s->store.commit_exception(&s->store, &pe->e, commit_callback,
770 pe);
771}
772
773/*
774 * Dispatches the copy operation to kcopyd.
775 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800776static void start_copy(struct pending_exception *pe)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777{
778 struct dm_snapshot *s = pe->snap;
779 struct io_region src, dest;
780 struct block_device *bdev = s->origin->bdev;
781 sector_t dev_size;
782
783 dev_size = get_dev_size(bdev);
784
785 src.bdev = bdev;
786 src.sector = chunk_to_sector(s, pe->e.old_chunk);
787 src.count = min(s->chunk_size, dev_size - src.sector);
788
789 dest.bdev = s->cow->bdev;
790 dest.sector = chunk_to_sector(s, pe->e.new_chunk);
791 dest.count = src.count;
792
793 /* Hand over to kcopyd */
794 kcopyd_copy(s->kcopyd_client,
795 &src, 1, &dest, 0, copy_callback, pe);
796}
797
798/*
799 * Looks to see if this snapshot already has a pending exception
800 * for this chunk, otherwise it allocates a new one and inserts
801 * it into the pending table.
802 *
803 * NOTE: a write lock must be held on snap->lock before calling
804 * this.
805 */
806static struct pending_exception *
807__find_pending_exception(struct dm_snapshot *s, struct bio *bio)
808{
809 struct exception *e;
810 struct pending_exception *pe;
811 chunk_t chunk = sector_to_chunk(s, bio->bi_sector);
812
813 /*
814 * Is there a pending exception for this already ?
815 */
816 e = lookup_exception(&s->pending, chunk);
817 if (e) {
818 /* cast the exception to a pending exception */
819 pe = container_of(e, struct pending_exception, e);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800820 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 }
822
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800823 /*
824 * Create a new pending exception, we don't want
825 * to hold the lock while we do this.
826 */
827 up_write(&s->lock);
828 pe = alloc_pending_exception();
829 down_write(&s->lock);
830
831 if (!s->valid) {
832 free_pending_exception(pe);
833 return NULL;
834 }
835
836 e = lookup_exception(&s->pending, chunk);
837 if (e) {
838 free_pending_exception(pe);
839 pe = container_of(e, struct pending_exception, e);
840 goto out;
841 }
842
843 pe->e.old_chunk = chunk;
844 bio_list_init(&pe->origin_bios);
845 bio_list_init(&pe->snapshot_bios);
846 pe->primary_pe = NULL;
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700847 atomic_set(&pe->ref_count, 0);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800848 pe->snap = s;
849 pe->started = 0;
850
851 if (s->store.prepare_exception(&s->store, &pe->e)) {
852 free_pending_exception(pe);
853 return NULL;
854 }
855
Alasdair G Kergon4b832e82006-10-03 01:15:30 -0700856 get_pending_exception(pe);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800857 insert_exception(&s->pending, &pe->e);
858
859 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860 return pe;
861}
862
863static inline void remap_exception(struct dm_snapshot *s, struct exception *e,
864 struct bio *bio)
865{
866 bio->bi_bdev = s->cow->bdev;
867 bio->bi_sector = chunk_to_sector(s, e->new_chunk) +
868 (bio->bi_sector & s->chunk_mask);
869}
870
871static int snapshot_map(struct dm_target *ti, struct bio *bio,
872 union map_info *map_context)
873{
874 struct exception *e;
875 struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -0800876 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 chunk_t chunk;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800878 struct pending_exception *pe = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879
880 chunk = sector_to_chunk(s, bio->bi_sector);
881
882 /* Full snapshots are not usable */
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800883 /* To get here the table must be live so s->active is always set. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 if (!s->valid)
Alasdair G Kergonf6a80ea2005-07-12 15:53:01 -0700885 return -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886
Alasdair G Kergon4aac0a62006-02-01 03:04:55 -0800887 if (unlikely(bio_barrier(bio)))
888 return -EOPNOTSUPP;
889
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -0700890 /* FIXME: should only take write lock if we need
891 * to copy an exception */
892 down_write(&s->lock);
893
894 if (!s->valid) {
895 r = -EIO;
896 goto out_unlock;
897 }
898
899 /* If the block is already remapped - use that, else remap it */
900 e = lookup_exception(&s->complete, chunk);
901 if (e) {
902 remap_exception(s, e, bio);
903 goto out_unlock;
904 }
905
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 /*
907 * Write to snapshot - higher level takes care of RW/RO
908 * flags so we should only get this if we are
909 * writeable.
910 */
911 if (bio_rw(bio) == WRITE) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800912 pe = __find_pending_exception(s, bio);
913 if (!pe) {
Alasdair G Kergon695368a2006-10-03 01:15:31 -0700914 __invalidate_snapshot(s, -ENOMEM);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800915 r = -EIO;
916 goto out_unlock;
917 }
918
919 remap_exception(s, &pe->e, bio);
920 bio_list_add(&pe->snapshot_bios, bio);
921
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -0800922 r = DM_MAPIO_SUBMITTED;
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -0700923
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800924 if (!pe->started) {
925 /* this is protected by snap->lock */
926 pe->started = 1;
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -0700927 up_write(&s->lock);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -0800928 start_copy(pe);
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -0700929 goto out;
930 }
931 } else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932 /*
933 * FIXME: this read path scares me because we
934 * always use the origin when we have a pending
935 * exception. However I can't think of a
936 * situation where this is wrong - ejt.
937 */
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -0700938 bio->bi_bdev = s->origin->bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939
Alasdair G Kergonba40a2a2006-10-03 01:15:28 -0700940 out_unlock:
941 up_write(&s->lock);
942 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 return r;
944}
945
946static void snapshot_resume(struct dm_target *ti)
947{
948 struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
949
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -0800950 down_write(&s->lock);
951 s->active = 1;
952 up_write(&s->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953}
954
955static int snapshot_status(struct dm_target *ti, status_type_t type,
956 char *result, unsigned int maxlen)
957{
958 struct dm_snapshot *snap = (struct dm_snapshot *) ti->private;
959
960 switch (type) {
961 case STATUSTYPE_INFO:
962 if (!snap->valid)
963 snprintf(result, maxlen, "Invalid");
964 else {
965 if (snap->store.fraction_full) {
966 sector_t numerator, denominator;
967 snap->store.fraction_full(&snap->store,
968 &numerator,
969 &denominator);
Andrew Morton4ee218c2006-03-27 01:17:48 -0800970 snprintf(result, maxlen, "%llu/%llu",
971 (unsigned long long)numerator,
972 (unsigned long long)denominator);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 }
974 else
975 snprintf(result, maxlen, "Unknown");
976 }
977 break;
978
979 case STATUSTYPE_TABLE:
980 /*
981 * kdevname returns a static pointer so we need
982 * to make private copies if the output is to
983 * make sense.
984 */
Andrew Morton4ee218c2006-03-27 01:17:48 -0800985 snprintf(result, maxlen, "%s %s %c %llu",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 snap->origin->name, snap->cow->name,
Andrew Morton4ee218c2006-03-27 01:17:48 -0800987 snap->type,
988 (unsigned long long)snap->chunk_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 break;
990 }
991
992 return 0;
993}
994
995/*-----------------------------------------------------------------
996 * Origin methods
997 *---------------------------------------------------------------*/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998static int __origin_write(struct list_head *snapshots, struct bio *bio)
999{
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001000 int r = DM_MAPIO_REMAPPED, first = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 struct dm_snapshot *snap;
1002 struct exception *e;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001003 struct pending_exception *pe, *next_pe, *primary_pe = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004 chunk_t chunk;
Alasdair G Kergoneccf0812006-03-27 01:17:42 -08001005 LIST_HEAD(pe_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006
1007 /* Do all the snapshots on this origin */
1008 list_for_each_entry (snap, snapshots, list) {
1009
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001010 down_write(&snap->lock);
1011
Alasdair G Kergonaa14ede2006-02-01 03:04:50 -08001012 /* Only deal with valid and active snapshots */
1013 if (!snap->valid || !snap->active)
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001014 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
Alasdair G Kergond5e404c2005-07-12 15:53:05 -07001016 /* Nothing to do if writing beyond end of snapshot */
1017 if (bio->bi_sector >= dm_table_get_size(snap->table))
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001018 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019
1020 /*
1021 * Remember, different snapshots can have
1022 * different chunk sizes.
1023 */
1024 chunk = sector_to_chunk(snap, bio->bi_sector);
1025
1026 /*
1027 * Check exception table to see if block
1028 * is already remapped in this snapshot
1029 * and trigger an exception if not.
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001030 *
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001031 * ref_count is initialised to 1 so pending_complete()
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001032 * won't destroy the primary_pe while we're inside this loop.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033 */
1034 e = lookup_exception(&snap->complete, chunk);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001035 if (e)
1036 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001038 pe = __find_pending_exception(snap, bio);
1039 if (!pe) {
Alasdair G Kergon695368a2006-10-03 01:15:31 -07001040 __invalidate_snapshot(snap, -ENOMEM);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001041 goto next_snapshot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042 }
1043
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001044 if (!primary_pe) {
1045 /*
1046 * Either every pe here has same
1047 * primary_pe or none has one yet.
1048 */
1049 if (pe->primary_pe)
1050 primary_pe = pe->primary_pe;
1051 else {
1052 primary_pe = pe;
1053 first = 1;
1054 }
1055
1056 bio_list_add(&primary_pe->origin_bios, bio);
1057
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001058 r = DM_MAPIO_SUBMITTED;
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001059 }
1060
1061 if (!pe->primary_pe) {
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001062 pe->primary_pe = primary_pe;
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001063 get_pending_exception(primary_pe);
Alasdair G Kergon76df1c62006-03-27 01:17:45 -08001064 }
1065
1066 if (!pe->started) {
1067 pe->started = 1;
1068 list_add_tail(&pe->list, &pe_queue);
1069 }
1070
1071 next_snapshot:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072 up_write(&snap->lock);
1073 }
1074
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001075 if (!primary_pe)
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001076 return r;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001077
1078 /*
1079 * If this is the first time we're processing this chunk and
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001080 * ref_count is now 1 it means all the pending exceptions
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001081 * got completed while we were in the loop above, so it falls to
1082 * us here to remove the primary_pe and submit any origin_bios.
1083 */
1084
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001085 if (first && atomic_dec_and_test(&primary_pe->ref_count)) {
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001086 flush_bios(bio_list_get(&primary_pe->origin_bios));
1087 free_pending_exception(primary_pe);
1088 /* If we got here, pe_queue is necessarily empty. */
Alasdair G Kergon4b832e82006-10-03 01:15:30 -07001089 return r;
Alasdair G Kergonb4b610f2006-03-27 01:17:44 -08001090 }
1091
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092 /*
1093 * Now that we have a complete pe list we can start the copying.
1094 */
Alasdair G Kergoneccf0812006-03-27 01:17:42 -08001095 list_for_each_entry_safe(pe, next_pe, &pe_queue, list)
1096 start_copy(pe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097
1098 return r;
1099}
1100
1101/*
1102 * Called on a write from the origin driver.
1103 */
1104static int do_origin(struct dm_dev *origin, struct bio *bio)
1105{
1106 struct origin *o;
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001107 int r = DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
1109 down_read(&_origins_lock);
1110 o = __lookup_origin(origin->bdev);
1111 if (o)
1112 r = __origin_write(&o->snapshots, bio);
1113 up_read(&_origins_lock);
1114
1115 return r;
1116}
1117
1118/*
1119 * Origin: maps a linear range of a device, with hooks for snapshotting.
1120 */
1121
1122/*
1123 * Construct an origin mapping: <dev_path>
1124 * The context for an origin is merely a 'struct dm_dev *'
1125 * pointing to the real device.
1126 */
1127static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1128{
1129 int r;
1130 struct dm_dev *dev;
1131
1132 if (argc != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001133 ti->error = "origin: incorrect number of arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 return -EINVAL;
1135 }
1136
1137 r = dm_get_device(ti, argv[0], 0, ti->len,
1138 dm_table_get_mode(ti->table), &dev);
1139 if (r) {
1140 ti->error = "Cannot get target device";
1141 return r;
1142 }
1143
1144 ti->private = dev;
1145 return 0;
1146}
1147
1148static void origin_dtr(struct dm_target *ti)
1149{
1150 struct dm_dev *dev = (struct dm_dev *) ti->private;
1151 dm_put_device(ti, dev);
1152}
1153
1154static int origin_map(struct dm_target *ti, struct bio *bio,
1155 union map_info *map_context)
1156{
1157 struct dm_dev *dev = (struct dm_dev *) ti->private;
1158 bio->bi_bdev = dev->bdev;
1159
Alasdair G Kergon4aac0a62006-02-01 03:04:55 -08001160 if (unlikely(bio_barrier(bio)))
1161 return -EOPNOTSUPP;
1162
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163 /* Only tell snapshots if this is a write */
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001164 return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165}
1166
1167#define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))
1168
1169/*
1170 * Set the target "split_io" field to the minimum of all the snapshots'
1171 * chunk sizes.
1172 */
1173static void origin_resume(struct dm_target *ti)
1174{
1175 struct dm_dev *dev = (struct dm_dev *) ti->private;
1176 struct dm_snapshot *snap;
1177 struct origin *o;
1178 chunk_t chunk_size = 0;
1179
1180 down_read(&_origins_lock);
1181 o = __lookup_origin(dev->bdev);
1182 if (o)
1183 list_for_each_entry (snap, &o->snapshots, list)
1184 chunk_size = min_not_zero(chunk_size, snap->chunk_size);
1185 up_read(&_origins_lock);
1186
1187 ti->split_io = chunk_size;
1188}
1189
1190static int origin_status(struct dm_target *ti, status_type_t type, char *result,
1191 unsigned int maxlen)
1192{
1193 struct dm_dev *dev = (struct dm_dev *) ti->private;
1194
1195 switch (type) {
1196 case STATUSTYPE_INFO:
1197 result[0] = '\0';
1198 break;
1199
1200 case STATUSTYPE_TABLE:
1201 snprintf(result, maxlen, "%s", dev->name);
1202 break;
1203 }
1204
1205 return 0;
1206}
1207
1208static struct target_type origin_target = {
1209 .name = "snapshot-origin",
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -07001210 .version = {1, 5, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 .module = THIS_MODULE,
1212 .ctr = origin_ctr,
1213 .dtr = origin_dtr,
1214 .map = origin_map,
1215 .resume = origin_resume,
1216 .status = origin_status,
1217};
1218
1219static struct target_type snapshot_target = {
1220 .name = "snapshot",
Mark McLoughlin4c7e3bf2006-10-03 01:15:25 -07001221 .version = {1, 5, 0},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222 .module = THIS_MODULE,
1223 .ctr = snapshot_ctr,
1224 .dtr = snapshot_dtr,
1225 .map = snapshot_map,
1226 .resume = snapshot_resume,
1227 .status = snapshot_status,
1228};
1229
1230static int __init dm_snapshot_init(void)
1231{
1232 int r;
1233
1234 r = dm_register_target(&snapshot_target);
1235 if (r) {
1236 DMERR("snapshot target register failed %d", r);
1237 return r;
1238 }
1239
1240 r = dm_register_target(&origin_target);
1241 if (r < 0) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -07001242 DMERR("Origin target register failed %d", r);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 goto bad1;
1244 }
1245
1246 r = init_origin_hash();
1247 if (r) {
1248 DMERR("init_origin_hash failed.");
1249 goto bad2;
1250 }
1251
1252 exception_cache = kmem_cache_create("dm-snapshot-ex",
1253 sizeof(struct exception),
1254 __alignof__(struct exception),
1255 0, NULL, NULL);
1256 if (!exception_cache) {
1257 DMERR("Couldn't create exception cache.");
1258 r = -ENOMEM;
1259 goto bad3;
1260 }
1261
1262 pending_cache =
1263 kmem_cache_create("dm-snapshot-in",
1264 sizeof(struct pending_exception),
1265 __alignof__(struct pending_exception),
1266 0, NULL, NULL);
1267 if (!pending_cache) {
1268 DMERR("Couldn't create pending cache.");
1269 r = -ENOMEM;
1270 goto bad4;
1271 }
1272
Matthew Dobson93d23412006-03-26 01:37:50 -08001273 pending_pool = mempool_create_slab_pool(128, pending_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274 if (!pending_pool) {
1275 DMERR("Couldn't create pending pool.");
1276 r = -ENOMEM;
1277 goto bad5;
1278 }
1279
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001280 ksnapd = create_singlethread_workqueue("ksnapd");
1281 if (!ksnapd) {
1282 DMERR("Failed to create ksnapd workqueue.");
1283 r = -ENOMEM;
1284 goto bad6;
1285 }
1286
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287 return 0;
1288
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001289 bad6:
1290 mempool_destroy(pending_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 bad5:
1292 kmem_cache_destroy(pending_cache);
1293 bad4:
1294 kmem_cache_destroy(exception_cache);
1295 bad3:
1296 exit_origin_hash();
1297 bad2:
1298 dm_unregister_target(&origin_target);
1299 bad1:
1300 dm_unregister_target(&snapshot_target);
1301 return r;
1302}
1303
1304static void __exit dm_snapshot_exit(void)
1305{
1306 int r;
1307
Alasdair G Kergonca3a9312006-10-03 01:15:30 -07001308 destroy_workqueue(ksnapd);
1309
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310 r = dm_unregister_target(&snapshot_target);
1311 if (r)
1312 DMERR("snapshot unregister failed %d", r);
1313
1314 r = dm_unregister_target(&origin_target);
1315 if (r)
1316 DMERR("origin unregister failed %d", r);
1317
1318 exit_origin_hash();
1319 mempool_destroy(pending_pool);
1320 kmem_cache_destroy(pending_cache);
1321 kmem_cache_destroy(exception_cache);
1322}
1323
1324/* Module hooks */
1325module_init(dm_snapshot_init);
1326module_exit(dm_snapshot_exit);
1327
1328MODULE_DESCRIPTION(DM_NAME " snapshot target");
1329MODULE_AUTHOR("Joe Thornber");
1330MODULE_LICENSE("GPL");