blob: 423df92f1c19800307786f716401197c8ba41814 [file] [log] [blame]
Joe Thornber991d9fa2011-10-31 20:21:18 +00001/*
2 * Copyright (C) 2011 Red Hat UK.
3 *
4 * This file is released under the GPL.
5 */
6
7#include "dm-thin-metadata.h"
8
9#include <linux/device-mapper.h>
10#include <linux/dm-io.h>
11#include <linux/dm-kcopyd.h>
12#include <linux/list.h>
13#include <linux/init.h>
14#include <linux/module.h>
15#include <linux/slab.h>
16
17#define DM_MSG_PREFIX "thin"
18
19/*
20 * Tunable constants
21 */
Alasdair G Kergon7768ed32012-07-27 15:07:57 +010022#define ENDIO_HOOK_POOL_SIZE 1024
Joe Thornber991d9fa2011-10-31 20:21:18 +000023#define DEFERRED_SET_SIZE 64
24#define MAPPING_POOL_SIZE 1024
25#define PRISON_CELLS 1024
Joe Thornber905e51b2012-03-28 18:41:27 +010026#define COMMIT_PERIOD HZ
Joe Thornber991d9fa2011-10-31 20:21:18 +000027
28/*
29 * The block size of the device holding pool data must be
30 * between 64KB and 1GB.
31 */
32#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
33#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
34
35/*
Joe Thornber991d9fa2011-10-31 20:21:18 +000036 * Device id is restricted to 24 bits.
37 */
38#define MAX_DEV_ID ((1 << 24) - 1)
39
40/*
41 * How do we handle breaking sharing of data blocks?
42 * =================================================
43 *
44 * We use a standard copy-on-write btree to store the mappings for the
45 * devices (note I'm talking about copy-on-write of the metadata here, not
46 * the data). When you take an internal snapshot you clone the root node
47 * of the origin btree. After this there is no concept of an origin or a
48 * snapshot. They are just two device trees that happen to point to the
49 * same data blocks.
50 *
51 * When we get a write in we decide if it's to a shared data block using
52 * some timestamp magic. If it is, we have to break sharing.
53 *
54 * Let's say we write to a shared block in what was the origin. The
55 * steps are:
56 *
57 * i) plug io further to this physical block. (see bio_prison code).
58 *
59 * ii) quiesce any read io to that shared data block. Obviously
60 * including all devices that share this block. (see deferred_set code)
61 *
62 * iii) copy the data block to a newly allocate block. This step can be
63 * missed out if the io covers the block. (schedule_copy).
64 *
65 * iv) insert the new mapping into the origin's btree
Joe Thornberfe878f32012-03-28 18:41:24 +010066 * (process_prepared_mapping). This act of inserting breaks some
Joe Thornber991d9fa2011-10-31 20:21:18 +000067 * sharing of btree nodes between the two devices. Breaking sharing only
68 * effects the btree of that specific device. Btrees for the other
69 * devices that share the block never change. The btree for the origin
70 * device as it was after the last commit is untouched, ie. we're using
71 * persistent data structures in the functional programming sense.
72 *
73 * v) unplug io to this physical block, including the io that triggered
74 * the breaking of sharing.
75 *
76 * Steps (ii) and (iii) occur in parallel.
77 *
78 * The metadata _doesn't_ need to be committed before the io continues. We
79 * get away with this because the io is always written to a _new_ block.
80 * If there's a crash, then:
81 *
82 * - The origin mapping will point to the old origin block (the shared
83 * one). This will contain the data as it was before the io that triggered
84 * the breaking of sharing came in.
85 *
86 * - The snap mapping still points to the old block. As it would after
87 * the commit.
88 *
89 * The downside of this scheme is the timestamp magic isn't perfect, and
90 * will continue to think that data block in the snapshot device is shared
91 * even after the write to the origin has broken sharing. I suspect data
92 * blocks will typically be shared by many different devices, so we're
93 * breaking sharing n + 1 times, rather than n, where n is the number of
94 * devices that reference this data block. At the moment I think the
95 * benefits far, far outweigh the disadvantages.
96 */
97
98/*----------------------------------------------------------------*/
99
100/*
101 * Sometimes we can't deal with a bio straight away. We put them in prison
102 * where they can't cause any mischief. Bios are put in a cell identified
103 * by a key, multiple bios can be in the same cell. When the cell is
104 * subsequently unlocked the bios become available.
105 */
106struct bio_prison;
107
108struct cell_key {
109 int virtual;
110 dm_thin_id dev;
111 dm_block_t block;
112};
113
Mike Snitzera24c2562012-06-03 00:30:00 +0100114struct dm_bio_prison_cell {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000115 struct hlist_node list;
116 struct bio_prison *prison;
117 struct cell_key key;
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100118 struct bio *holder;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000119 struct bio_list bios;
120};
121
122struct bio_prison {
123 spinlock_t lock;
124 mempool_t *cell_pool;
125
126 unsigned nr_buckets;
127 unsigned hash_mask;
128 struct hlist_head *cells;
129};
130
131static uint32_t calc_nr_buckets(unsigned nr_cells)
132{
133 uint32_t n = 128;
134
135 nr_cells /= 4;
136 nr_cells = min(nr_cells, 8192u);
137
138 while (n < nr_cells)
139 n <<= 1;
140
141 return n;
142}
143
Mike Snitzera24c2562012-06-03 00:30:00 +0100144static struct kmem_cache *_cell_cache;
145
Joe Thornber991d9fa2011-10-31 20:21:18 +0000146/*
147 * @nr_cells should be the number of cells you want in use _concurrently_.
148 * Don't confuse it with the number of distinct keys.
149 */
150static struct bio_prison *prison_create(unsigned nr_cells)
151{
152 unsigned i;
153 uint32_t nr_buckets = calc_nr_buckets(nr_cells);
154 size_t len = sizeof(struct bio_prison) +
155 (sizeof(struct hlist_head) * nr_buckets);
156 struct bio_prison *prison = kmalloc(len, GFP_KERNEL);
157
158 if (!prison)
159 return NULL;
160
161 spin_lock_init(&prison->lock);
Mike Snitzera24c2562012-06-03 00:30:00 +0100162 prison->cell_pool = mempool_create_slab_pool(nr_cells, _cell_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000163 if (!prison->cell_pool) {
164 kfree(prison);
165 return NULL;
166 }
167
168 prison->nr_buckets = nr_buckets;
169 prison->hash_mask = nr_buckets - 1;
170 prison->cells = (struct hlist_head *) (prison + 1);
171 for (i = 0; i < nr_buckets; i++)
172 INIT_HLIST_HEAD(prison->cells + i);
173
174 return prison;
175}
176
177static void prison_destroy(struct bio_prison *prison)
178{
179 mempool_destroy(prison->cell_pool);
180 kfree(prison);
181}
182
183static uint32_t hash_key(struct bio_prison *prison, struct cell_key *key)
184{
185 const unsigned long BIG_PRIME = 4294967291UL;
186 uint64_t hash = key->block * BIG_PRIME;
187
188 return (uint32_t) (hash & prison->hash_mask);
189}
190
191static int keys_equal(struct cell_key *lhs, struct cell_key *rhs)
192{
193 return (lhs->virtual == rhs->virtual) &&
194 (lhs->dev == rhs->dev) &&
195 (lhs->block == rhs->block);
196}
197
Mike Snitzera24c2562012-06-03 00:30:00 +0100198static struct dm_bio_prison_cell *__search_bucket(struct hlist_head *bucket,
199 struct cell_key *key)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000200{
Mike Snitzera24c2562012-06-03 00:30:00 +0100201 struct dm_bio_prison_cell *cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000202 struct hlist_node *tmp;
203
204 hlist_for_each_entry(cell, tmp, bucket, list)
205 if (keys_equal(&cell->key, key))
206 return cell;
207
208 return NULL;
209}
210
211/*
212 * This may block if a new cell needs allocating. You must ensure that
213 * cells will be unlocked even if the calling thread is blocked.
214 *
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100215 * Returns 1 if the cell was already held, 0 if @inmate is the new holder.
Joe Thornber991d9fa2011-10-31 20:21:18 +0000216 */
217static int bio_detain(struct bio_prison *prison, struct cell_key *key,
Mike Snitzera24c2562012-06-03 00:30:00 +0100218 struct bio *inmate, struct dm_bio_prison_cell **ref)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000219{
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100220 int r = 1;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000221 unsigned long flags;
222 uint32_t hash = hash_key(prison, key);
Mike Snitzera24c2562012-06-03 00:30:00 +0100223 struct dm_bio_prison_cell *cell, *cell2;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000224
225 BUG_ON(hash > prison->nr_buckets);
226
227 spin_lock_irqsave(&prison->lock, flags);
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100228
Joe Thornber991d9fa2011-10-31 20:21:18 +0000229 cell = __search_bucket(prison->cells + hash, key);
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100230 if (cell) {
231 bio_list_add(&cell->bios, inmate);
232 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000233 }
234
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100235 /*
236 * Allocate a new cell
237 */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000238 spin_unlock_irqrestore(&prison->lock, flags);
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100239 cell2 = mempool_alloc(prison->cell_pool, GFP_NOIO);
240 spin_lock_irqsave(&prison->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000241
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100242 /*
243 * We've been unlocked, so we have to double check that
244 * nobody else has inserted this cell in the meantime.
245 */
246 cell = __search_bucket(prison->cells + hash, key);
247 if (cell) {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000248 mempool_free(cell2, prison->cell_pool);
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100249 bio_list_add(&cell->bios, inmate);
250 goto out;
251 }
252
253 /*
254 * Use new cell.
255 */
256 cell = cell2;
257
258 cell->prison = prison;
259 memcpy(&cell->key, key, sizeof(cell->key));
260 cell->holder = inmate;
261 bio_list_init(&cell->bios);
262 hlist_add_head(&cell->list, prison->cells + hash);
263
264 r = 0;
265
266out:
267 spin_unlock_irqrestore(&prison->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000268
269 *ref = cell;
270
271 return r;
272}
273
274/*
275 * @inmates must have been initialised prior to this call
276 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100277static void __cell_release(struct dm_bio_prison_cell *cell, struct bio_list *inmates)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000278{
279 struct bio_prison *prison = cell->prison;
280
281 hlist_del(&cell->list);
282
Mike Snitzer03aaae72012-05-12 01:43:12 +0100283 if (inmates) {
284 bio_list_add(inmates, cell->holder);
285 bio_list_merge(inmates, &cell->bios);
286 }
Joe Thornber991d9fa2011-10-31 20:21:18 +0000287
288 mempool_free(cell, prison->cell_pool);
289}
290
Mike Snitzera24c2562012-06-03 00:30:00 +0100291static void cell_release(struct dm_bio_prison_cell *cell, struct bio_list *bios)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000292{
293 unsigned long flags;
294 struct bio_prison *prison = cell->prison;
295
296 spin_lock_irqsave(&prison->lock, flags);
297 __cell_release(cell, bios);
298 spin_unlock_irqrestore(&prison->lock, flags);
299}
300
301/*
302 * There are a couple of places where we put a bio into a cell briefly
303 * before taking it out again. In these situations we know that no other
304 * bio may be in the cell. This function releases the cell, and also does
305 * a sanity check.
306 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100307static void __cell_release_singleton(struct dm_bio_prison_cell *cell, struct bio *bio)
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100308{
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100309 BUG_ON(cell->holder != bio);
310 BUG_ON(!bio_list_empty(&cell->bios));
Mike Snitzer03aaae72012-05-12 01:43:12 +0100311
312 __cell_release(cell, NULL);
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100313}
314
Mike Snitzera24c2562012-06-03 00:30:00 +0100315static void cell_release_singleton(struct dm_bio_prison_cell *cell, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000316{
Joe Thornber991d9fa2011-10-31 20:21:18 +0000317 unsigned long flags;
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100318 struct bio_prison *prison = cell->prison;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000319
320 spin_lock_irqsave(&prison->lock, flags);
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100321 __cell_release_singleton(cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000322 spin_unlock_irqrestore(&prison->lock, flags);
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100323}
Joe Thornber991d9fa2011-10-31 20:21:18 +0000324
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100325/*
326 * Sometimes we don't want the holder, just the additional bios.
327 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100328static void __cell_release_no_holder(struct dm_bio_prison_cell *cell,
329 struct bio_list *inmates)
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100330{
331 struct bio_prison *prison = cell->prison;
332
333 hlist_del(&cell->list);
334 bio_list_merge(inmates, &cell->bios);
335
336 mempool_free(cell, prison->cell_pool);
337}
338
Mike Snitzera24c2562012-06-03 00:30:00 +0100339static void cell_release_no_holder(struct dm_bio_prison_cell *cell,
340 struct bio_list *inmates)
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100341{
342 unsigned long flags;
343 struct bio_prison *prison = cell->prison;
344
345 spin_lock_irqsave(&prison->lock, flags);
346 __cell_release_no_holder(cell, inmates);
347 spin_unlock_irqrestore(&prison->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000348}
349
Mike Snitzera24c2562012-06-03 00:30:00 +0100350static void cell_error(struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000351{
352 struct bio_prison *prison = cell->prison;
353 struct bio_list bios;
354 struct bio *bio;
355 unsigned long flags;
356
357 bio_list_init(&bios);
358
359 spin_lock_irqsave(&prison->lock, flags);
360 __cell_release(cell, &bios);
361 spin_unlock_irqrestore(&prison->lock, flags);
362
363 while ((bio = bio_list_pop(&bios)))
364 bio_io_error(bio);
365}
366
367/*----------------------------------------------------------------*/
368
369/*
370 * We use the deferred set to keep track of pending reads to shared blocks.
371 * We do this to ensure the new mapping caused by a write isn't performed
372 * until these prior reads have completed. Otherwise the insertion of the
373 * new mapping could free the old block that the read bios are mapped to.
374 */
375
376struct deferred_set;
377struct deferred_entry {
378 struct deferred_set *ds;
379 unsigned count;
380 struct list_head work_items;
381};
382
383struct deferred_set {
384 spinlock_t lock;
385 unsigned current_entry;
386 unsigned sweeper;
387 struct deferred_entry entries[DEFERRED_SET_SIZE];
388};
389
390static void ds_init(struct deferred_set *ds)
391{
392 int i;
393
394 spin_lock_init(&ds->lock);
395 ds->current_entry = 0;
396 ds->sweeper = 0;
397 for (i = 0; i < DEFERRED_SET_SIZE; i++) {
398 ds->entries[i].ds = ds;
399 ds->entries[i].count = 0;
400 INIT_LIST_HEAD(&ds->entries[i].work_items);
401 }
402}
403
404static struct deferred_entry *ds_inc(struct deferred_set *ds)
405{
406 unsigned long flags;
407 struct deferred_entry *entry;
408
409 spin_lock_irqsave(&ds->lock, flags);
410 entry = ds->entries + ds->current_entry;
411 entry->count++;
412 spin_unlock_irqrestore(&ds->lock, flags);
413
414 return entry;
415}
416
417static unsigned ds_next(unsigned index)
418{
419 return (index + 1) % DEFERRED_SET_SIZE;
420}
421
422static void __sweep(struct deferred_set *ds, struct list_head *head)
423{
424 while ((ds->sweeper != ds->current_entry) &&
425 !ds->entries[ds->sweeper].count) {
426 list_splice_init(&ds->entries[ds->sweeper].work_items, head);
427 ds->sweeper = ds_next(ds->sweeper);
428 }
429
430 if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
431 list_splice_init(&ds->entries[ds->sweeper].work_items, head);
432}
433
434static void ds_dec(struct deferred_entry *entry, struct list_head *head)
435{
436 unsigned long flags;
437
438 spin_lock_irqsave(&entry->ds->lock, flags);
439 BUG_ON(!entry->count);
440 --entry->count;
441 __sweep(entry->ds, head);
442 spin_unlock_irqrestore(&entry->ds->lock, flags);
443}
444
445/*
446 * Returns 1 if deferred or 0 if no pending items to delay job.
447 */
448static int ds_add_work(struct deferred_set *ds, struct list_head *work)
449{
450 int r = 1;
451 unsigned long flags;
452 unsigned next_entry;
453
454 spin_lock_irqsave(&ds->lock, flags);
455 if ((ds->sweeper == ds->current_entry) &&
456 !ds->entries[ds->current_entry].count)
457 r = 0;
458 else {
459 list_add(work, &ds->entries[ds->current_entry].work_items);
460 next_entry = ds_next(ds->current_entry);
461 if (!ds->entries[next_entry].count)
462 ds->current_entry = next_entry;
463 }
464 spin_unlock_irqrestore(&ds->lock, flags);
465
466 return r;
467}
468
469/*----------------------------------------------------------------*/
470
471/*
472 * Key building.
473 */
474static void build_data_key(struct dm_thin_device *td,
475 dm_block_t b, struct cell_key *key)
476{
477 key->virtual = 0;
478 key->dev = dm_thin_dev_id(td);
479 key->block = b;
480}
481
482static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
483 struct cell_key *key)
484{
485 key->virtual = 1;
486 key->dev = dm_thin_dev_id(td);
487 key->block = b;
488}
489
490/*----------------------------------------------------------------*/
491
492/*
493 * A pool device ties together a metadata device and a data device. It
494 * also provides the interface for creating and destroying internal
495 * devices.
496 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100497struct dm_thin_new_mapping;
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100498
499struct pool_features {
500 unsigned zero_new_blocks:1;
501 unsigned discard_enabled:1;
502 unsigned discard_passdown:1;
503};
504
Joe Thornber991d9fa2011-10-31 20:21:18 +0000505struct pool {
506 struct list_head list;
507 struct dm_target *ti; /* Only set if a pool target is bound */
508
509 struct mapped_device *pool_md;
510 struct block_device *md_dev;
511 struct dm_pool_metadata *pmd;
512
Joe Thornber991d9fa2011-10-31 20:21:18 +0000513 dm_block_t low_water_blocks;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100514 uint32_t sectors_per_block;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100515 int sectors_per_block_shift;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000516
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100517 struct pool_features pf;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000518 unsigned low_water_triggered:1; /* A dm event has been sent */
519 unsigned no_free_space:1; /* A -ENOSPC warning has been issued */
520
521 struct bio_prison *prison;
522 struct dm_kcopyd_client *copier;
523
524 struct workqueue_struct *wq;
525 struct work_struct worker;
Joe Thornber905e51b2012-03-28 18:41:27 +0100526 struct delayed_work waker;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000527
Joe Thornber905e51b2012-03-28 18:41:27 +0100528 unsigned long last_commit_jiffies;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100529 unsigned ref_count;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000530
531 spinlock_t lock;
532 struct bio_list deferred_bios;
533 struct bio_list deferred_flush_bios;
534 struct list_head prepared_mappings;
Joe Thornber104655f2012-03-28 18:41:28 +0100535 struct list_head prepared_discards;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000536
537 struct bio_list retry_on_resume_list;
538
Joe Thornbereb2aa482012-03-28 18:41:28 +0100539 struct deferred_set shared_read_ds;
Joe Thornber104655f2012-03-28 18:41:28 +0100540 struct deferred_set all_io_ds;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000541
Mike Snitzera24c2562012-06-03 00:30:00 +0100542 struct dm_thin_new_mapping *next_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000543 mempool_t *mapping_pool;
544 mempool_t *endio_hook_pool;
545};
546
547/*
548 * Target context for a pool.
549 */
550struct pool_c {
551 struct dm_target *ti;
552 struct pool *pool;
553 struct dm_dev *data_dev;
554 struct dm_dev *metadata_dev;
555 struct dm_target_callbacks callbacks;
556
557 dm_block_t low_water_blocks;
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100558 struct pool_features pf;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000559};
560
561/*
562 * Target context for a thin.
563 */
564struct thin_c {
565 struct dm_dev *pool_dev;
Joe Thornber2dd9c252012-03-28 18:41:28 +0100566 struct dm_dev *origin_dev;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000567 dm_thin_id dev_id;
568
569 struct pool *pool;
570 struct dm_thin_device *td;
571};
572
573/*----------------------------------------------------------------*/
574
575/*
576 * A global list of pools that uses a struct mapped_device as a key.
577 */
578static struct dm_thin_pool_table {
579 struct mutex mutex;
580 struct list_head pools;
581} dm_thin_pool_table;
582
583static void pool_table_init(void)
584{
585 mutex_init(&dm_thin_pool_table.mutex);
586 INIT_LIST_HEAD(&dm_thin_pool_table.pools);
587}
588
589static void __pool_table_insert(struct pool *pool)
590{
591 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
592 list_add(&pool->list, &dm_thin_pool_table.pools);
593}
594
595static void __pool_table_remove(struct pool *pool)
596{
597 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
598 list_del(&pool->list);
599}
600
601static struct pool *__pool_table_lookup(struct mapped_device *md)
602{
603 struct pool *pool = NULL, *tmp;
604
605 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
606
607 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
608 if (tmp->pool_md == md) {
609 pool = tmp;
610 break;
611 }
612 }
613
614 return pool;
615}
616
617static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
618{
619 struct pool *pool = NULL, *tmp;
620
621 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
622
623 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
624 if (tmp->md_dev == md_dev) {
625 pool = tmp;
626 break;
627 }
628 }
629
630 return pool;
631}
632
633/*----------------------------------------------------------------*/
634
Mike Snitzera24c2562012-06-03 00:30:00 +0100635struct dm_thin_endio_hook {
Joe Thornbereb2aa482012-03-28 18:41:28 +0100636 struct thin_c *tc;
637 struct deferred_entry *shared_read_entry;
Joe Thornber104655f2012-03-28 18:41:28 +0100638 struct deferred_entry *all_io_entry;
Mike Snitzera24c2562012-06-03 00:30:00 +0100639 struct dm_thin_new_mapping *overwrite_mapping;
Joe Thornbereb2aa482012-03-28 18:41:28 +0100640};
641
Joe Thornber991d9fa2011-10-31 20:21:18 +0000642static void __requeue_bio_list(struct thin_c *tc, struct bio_list *master)
643{
644 struct bio *bio;
645 struct bio_list bios;
646
647 bio_list_init(&bios);
648 bio_list_merge(&bios, master);
649 bio_list_init(master);
650
651 while ((bio = bio_list_pop(&bios))) {
Mike Snitzera24c2562012-06-03 00:30:00 +0100652 struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;
653
Joe Thornbereb2aa482012-03-28 18:41:28 +0100654 if (h->tc == tc)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000655 bio_endio(bio, DM_ENDIO_REQUEUE);
656 else
657 bio_list_add(master, bio);
658 }
659}
660
661static void requeue_io(struct thin_c *tc)
662{
663 struct pool *pool = tc->pool;
664 unsigned long flags;
665
666 spin_lock_irqsave(&pool->lock, flags);
667 __requeue_bio_list(tc, &pool->deferred_bios);
668 __requeue_bio_list(tc, &pool->retry_on_resume_list);
669 spin_unlock_irqrestore(&pool->lock, flags);
670}
671
672/*
673 * This section of code contains the logic for processing a thin device's IO.
674 * Much of the code depends on pool object resources (lists, workqueues, etc)
675 * but most is exclusively called from the thin target rather than the thin-pool
676 * target.
677 */
678
679static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
680{
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100681 sector_t block_nr = bio->bi_sector;
682
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100683 if (tc->pool->sectors_per_block_shift < 0)
684 (void) sector_div(block_nr, tc->pool->sectors_per_block);
685 else
686 block_nr >>= tc->pool->sectors_per_block_shift;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100687
688 return block_nr;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000689}
690
691static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
692{
693 struct pool *pool = tc->pool;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100694 sector_t bi_sector = bio->bi_sector;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000695
696 bio->bi_bdev = tc->pool_dev->bdev;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100697 if (tc->pool->sectors_per_block_shift < 0)
698 bio->bi_sector = (block * pool->sectors_per_block) +
699 sector_div(bi_sector, pool->sectors_per_block);
700 else
701 bio->bi_sector = (block << pool->sectors_per_block_shift) |
702 (bi_sector & (pool->sectors_per_block - 1));
Joe Thornber991d9fa2011-10-31 20:21:18 +0000703}
704
Joe Thornber2dd9c252012-03-28 18:41:28 +0100705static void remap_to_origin(struct thin_c *tc, struct bio *bio)
706{
707 bio->bi_bdev = tc->origin_dev->bdev;
708}
709
710static void issue(struct thin_c *tc, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000711{
712 struct pool *pool = tc->pool;
713 unsigned long flags;
714
Joe Thornber991d9fa2011-10-31 20:21:18 +0000715 /*
716 * Batch together any FUA/FLUSH bios we find and then issue
717 * a single commit for them in process_deferred_bios().
718 */
719 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
720 spin_lock_irqsave(&pool->lock, flags);
721 bio_list_add(&pool->deferred_flush_bios, bio);
722 spin_unlock_irqrestore(&pool->lock, flags);
723 } else
724 generic_make_request(bio);
725}
726
Joe Thornber2dd9c252012-03-28 18:41:28 +0100727static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
728{
729 remap_to_origin(tc, bio);
730 issue(tc, bio);
731}
732
733static void remap_and_issue(struct thin_c *tc, struct bio *bio,
734 dm_block_t block)
735{
736 remap(tc, bio, block);
737 issue(tc, bio);
738}
739
Joe Thornber991d9fa2011-10-31 20:21:18 +0000740/*
741 * wake_worker() is used when new work is queued and when pool_resume is
742 * ready to continue deferred IO processing.
743 */
744static void wake_worker(struct pool *pool)
745{
746 queue_work(pool->wq, &pool->worker);
747}
748
749/*----------------------------------------------------------------*/
750
751/*
752 * Bio endio functions.
753 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100754struct dm_thin_new_mapping {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000755 struct list_head list;
756
Joe Thornbereb2aa482012-03-28 18:41:28 +0100757 unsigned quiesced:1;
758 unsigned prepared:1;
Joe Thornber104655f2012-03-28 18:41:28 +0100759 unsigned pass_discard:1;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000760
761 struct thin_c *tc;
762 dm_block_t virt_block;
763 dm_block_t data_block;
Mike Snitzera24c2562012-06-03 00:30:00 +0100764 struct dm_bio_prison_cell *cell, *cell2;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000765 int err;
766
767 /*
768 * If the bio covers the whole area of a block then we can avoid
769 * zeroing or copying. Instead this bio is hooked. The bio will
770 * still be in the cell, so care has to be taken to avoid issuing
771 * the bio twice.
772 */
773 struct bio *bio;
774 bio_end_io_t *saved_bi_end_io;
775};
776
Mike Snitzera24c2562012-06-03 00:30:00 +0100777static void __maybe_add_mapping(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000778{
779 struct pool *pool = m->tc->pool;
780
Joe Thornbereb2aa482012-03-28 18:41:28 +0100781 if (m->quiesced && m->prepared) {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000782 list_add(&m->list, &pool->prepared_mappings);
783 wake_worker(pool);
784 }
785}
786
787static void copy_complete(int read_err, unsigned long write_err, void *context)
788{
789 unsigned long flags;
Mike Snitzera24c2562012-06-03 00:30:00 +0100790 struct dm_thin_new_mapping *m = context;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000791 struct pool *pool = m->tc->pool;
792
793 m->err = read_err || write_err ? -EIO : 0;
794
795 spin_lock_irqsave(&pool->lock, flags);
796 m->prepared = 1;
797 __maybe_add_mapping(m);
798 spin_unlock_irqrestore(&pool->lock, flags);
799}
800
801static void overwrite_endio(struct bio *bio, int err)
802{
803 unsigned long flags;
Mike Snitzera24c2562012-06-03 00:30:00 +0100804 struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;
805 struct dm_thin_new_mapping *m = h->overwrite_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000806 struct pool *pool = m->tc->pool;
807
808 m->err = err;
809
810 spin_lock_irqsave(&pool->lock, flags);
811 m->prepared = 1;
812 __maybe_add_mapping(m);
813 spin_unlock_irqrestore(&pool->lock, flags);
814}
815
Joe Thornber991d9fa2011-10-31 20:21:18 +0000816/*----------------------------------------------------------------*/
817
818/*
819 * Workqueue.
820 */
821
822/*
823 * Prepared mapping jobs.
824 */
825
826/*
827 * This sends the bios in the cell back to the deferred_bios list.
828 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100829static void cell_defer(struct thin_c *tc, struct dm_bio_prison_cell *cell,
Joe Thornber991d9fa2011-10-31 20:21:18 +0000830 dm_block_t data_block)
831{
832 struct pool *pool = tc->pool;
833 unsigned long flags;
834
835 spin_lock_irqsave(&pool->lock, flags);
836 cell_release(cell, &pool->deferred_bios);
837 spin_unlock_irqrestore(&tc->pool->lock, flags);
838
839 wake_worker(pool);
840}
841
842/*
843 * Same as cell_defer above, except it omits one particular detainee,
844 * a write bio that covers the block and has already been processed.
845 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100846static void cell_defer_except(struct thin_c *tc, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000847{
848 struct bio_list bios;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000849 struct pool *pool = tc->pool;
850 unsigned long flags;
851
852 bio_list_init(&bios);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000853
854 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100855 cell_release_no_holder(cell, &pool->deferred_bios);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000856 spin_unlock_irqrestore(&pool->lock, flags);
857
858 wake_worker(pool);
859}
860
Mike Snitzera24c2562012-06-03 00:30:00 +0100861static void process_prepared_mapping(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000862{
863 struct thin_c *tc = m->tc;
864 struct bio *bio;
865 int r;
866
867 bio = m->bio;
868 if (bio)
869 bio->bi_end_io = m->saved_bi_end_io;
870
871 if (m->err) {
872 cell_error(m->cell);
Joe Thornber905386f2012-07-27 15:08:05 +0100873 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000874 }
875
876 /*
877 * Commit the prepared block into the mapping btree.
878 * Any I/O for this block arriving after this point will get
879 * remapped to it directly.
880 */
881 r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block);
882 if (r) {
883 DMERR("dm_thin_insert_block() failed");
884 cell_error(m->cell);
Joe Thornber905386f2012-07-27 15:08:05 +0100885 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000886 }
887
888 /*
889 * Release any bios held while the block was being provisioned.
890 * If we are processing a write bio that completely covers the block,
891 * we already processed it so can ignore it now when processing
892 * the bios in the cell.
893 */
894 if (bio) {
Joe Thornber6f94a4c2012-03-28 18:41:23 +0100895 cell_defer_except(tc, m->cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000896 bio_endio(bio, 0);
897 } else
898 cell_defer(tc, m->cell, m->data_block);
899
Joe Thornber905386f2012-07-27 15:08:05 +0100900out:
Joe Thornber991d9fa2011-10-31 20:21:18 +0000901 list_del(&m->list);
902 mempool_free(m, tc->pool->mapping_pool);
903}
904
Mike Snitzera24c2562012-06-03 00:30:00 +0100905static void process_prepared_discard(struct dm_thin_new_mapping *m)
Joe Thornber104655f2012-03-28 18:41:28 +0100906{
907 int r;
908 struct thin_c *tc = m->tc;
909
910 r = dm_thin_remove_block(tc->td, m->virt_block);
911 if (r)
912 DMERR("dm_thin_remove_block() failed");
913
914 /*
915 * Pass the discard down to the underlying device?
916 */
917 if (m->pass_discard)
918 remap_and_issue(tc, m->bio, m->data_block);
919 else
920 bio_endio(m->bio, 0);
921
922 cell_defer_except(tc, m->cell);
923 cell_defer_except(tc, m->cell2);
924 mempool_free(m, tc->pool->mapping_pool);
925}
926
927static void process_prepared(struct pool *pool, struct list_head *head,
Mike Snitzera24c2562012-06-03 00:30:00 +0100928 void (*fn)(struct dm_thin_new_mapping *))
Joe Thornber991d9fa2011-10-31 20:21:18 +0000929{
930 unsigned long flags;
931 struct list_head maps;
Mike Snitzera24c2562012-06-03 00:30:00 +0100932 struct dm_thin_new_mapping *m, *tmp;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000933
934 INIT_LIST_HEAD(&maps);
935 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +0100936 list_splice_init(head, &maps);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000937 spin_unlock_irqrestore(&pool->lock, flags);
938
939 list_for_each_entry_safe(m, tmp, &maps, list)
Joe Thornber104655f2012-03-28 18:41:28 +0100940 fn(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000941}
942
943/*
944 * Deferred bio jobs.
945 */
Joe Thornber104655f2012-03-28 18:41:28 +0100946static int io_overlaps_block(struct pool *pool, struct bio *bio)
947{
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100948 return bio->bi_size == (pool->sectors_per_block << SECTOR_SHIFT);
Joe Thornber104655f2012-03-28 18:41:28 +0100949}
950
Joe Thornber991d9fa2011-10-31 20:21:18 +0000951static int io_overwrites_block(struct pool *pool, struct bio *bio)
952{
Joe Thornber104655f2012-03-28 18:41:28 +0100953 return (bio_data_dir(bio) == WRITE) &&
954 io_overlaps_block(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000955}
956
957static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
958 bio_end_io_t *fn)
959{
960 *save = bio->bi_end_io;
961 bio->bi_end_io = fn;
962}
963
964static int ensure_next_mapping(struct pool *pool)
965{
966 if (pool->next_mapping)
967 return 0;
968
969 pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
970
971 return pool->next_mapping ? 0 : -ENOMEM;
972}
973
Mike Snitzera24c2562012-06-03 00:30:00 +0100974static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000975{
Mike Snitzera24c2562012-06-03 00:30:00 +0100976 struct dm_thin_new_mapping *r = pool->next_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000977
978 BUG_ON(!pool->next_mapping);
979
980 pool->next_mapping = NULL;
981
982 return r;
983}
984
985static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
Joe Thornber2dd9c252012-03-28 18:41:28 +0100986 struct dm_dev *origin, dm_block_t data_origin,
987 dm_block_t data_dest,
Mike Snitzera24c2562012-06-03 00:30:00 +0100988 struct dm_bio_prison_cell *cell, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000989{
990 int r;
991 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +0100992 struct dm_thin_new_mapping *m = get_next_mapping(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000993
994 INIT_LIST_HEAD(&m->list);
Joe Thornbereb2aa482012-03-28 18:41:28 +0100995 m->quiesced = 0;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000996 m->prepared = 0;
997 m->tc = tc;
998 m->virt_block = virt_block;
999 m->data_block = data_dest;
1000 m->cell = cell;
1001 m->err = 0;
1002 m->bio = NULL;
1003
Joe Thornbereb2aa482012-03-28 18:41:28 +01001004 if (!ds_add_work(&pool->shared_read_ds, &m->list))
1005 m->quiesced = 1;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001006
1007 /*
1008 * IO to pool_dev remaps to the pool target's data_dev.
1009 *
1010 * If the whole block of data is being overwritten, we can issue the
1011 * bio immediately. Otherwise we use kcopyd to clone the data first.
1012 */
1013 if (io_overwrites_block(pool, bio)) {
Mike Snitzera24c2562012-06-03 00:30:00 +01001014 struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;
1015
Joe Thornbereb2aa482012-03-28 18:41:28 +01001016 h->overwrite_mapping = m;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001017 m->bio = bio;
1018 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001019 remap_and_issue(tc, bio, data_dest);
1020 } else {
1021 struct dm_io_region from, to;
1022
Joe Thornber2dd9c252012-03-28 18:41:28 +01001023 from.bdev = origin->bdev;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001024 from.sector = data_origin * pool->sectors_per_block;
1025 from.count = pool->sectors_per_block;
1026
1027 to.bdev = tc->pool_dev->bdev;
1028 to.sector = data_dest * pool->sectors_per_block;
1029 to.count = pool->sectors_per_block;
1030
1031 r = dm_kcopyd_copy(pool->copier, &from, 1, &to,
1032 0, copy_complete, m);
1033 if (r < 0) {
1034 mempool_free(m, pool->mapping_pool);
1035 DMERR("dm_kcopyd_copy() failed");
1036 cell_error(cell);
1037 }
1038 }
1039}
1040
Joe Thornber2dd9c252012-03-28 18:41:28 +01001041static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
1042 dm_block_t data_origin, dm_block_t data_dest,
Mike Snitzera24c2562012-06-03 00:30:00 +01001043 struct dm_bio_prison_cell *cell, struct bio *bio)
Joe Thornber2dd9c252012-03-28 18:41:28 +01001044{
1045 schedule_copy(tc, virt_block, tc->pool_dev,
1046 data_origin, data_dest, cell, bio);
1047}
1048
1049static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
1050 dm_block_t data_dest,
Mike Snitzera24c2562012-06-03 00:30:00 +01001051 struct dm_bio_prison_cell *cell, struct bio *bio)
Joe Thornber2dd9c252012-03-28 18:41:28 +01001052{
1053 schedule_copy(tc, virt_block, tc->origin_dev,
1054 virt_block, data_dest, cell, bio);
1055}
1056
Joe Thornber991d9fa2011-10-31 20:21:18 +00001057static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
Mike Snitzera24c2562012-06-03 00:30:00 +01001058 dm_block_t data_block, struct dm_bio_prison_cell *cell,
Joe Thornber991d9fa2011-10-31 20:21:18 +00001059 struct bio *bio)
1060{
1061 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +01001062 struct dm_thin_new_mapping *m = get_next_mapping(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001063
1064 INIT_LIST_HEAD(&m->list);
Joe Thornbereb2aa482012-03-28 18:41:28 +01001065 m->quiesced = 1;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001066 m->prepared = 0;
1067 m->tc = tc;
1068 m->virt_block = virt_block;
1069 m->data_block = data_block;
1070 m->cell = cell;
1071 m->err = 0;
1072 m->bio = NULL;
1073
1074 /*
1075 * If the whole block of data is being overwritten or we are not
1076 * zeroing pre-existing data, we can issue the bio immediately.
1077 * Otherwise we use kcopyd to zero the data first.
1078 */
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001079 if (!pool->pf.zero_new_blocks)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001080 process_prepared_mapping(m);
1081
1082 else if (io_overwrites_block(pool, bio)) {
Mike Snitzera24c2562012-06-03 00:30:00 +01001083 struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;
1084
Joe Thornbereb2aa482012-03-28 18:41:28 +01001085 h->overwrite_mapping = m;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001086 m->bio = bio;
1087 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001088 remap_and_issue(tc, bio, data_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001089 } else {
1090 int r;
1091 struct dm_io_region to;
1092
1093 to.bdev = tc->pool_dev->bdev;
1094 to.sector = data_block * pool->sectors_per_block;
1095 to.count = pool->sectors_per_block;
1096
1097 r = dm_kcopyd_zero(pool->copier, 1, &to, 0, copy_complete, m);
1098 if (r < 0) {
1099 mempool_free(m, pool->mapping_pool);
1100 DMERR("dm_kcopyd_zero() failed");
1101 cell_error(cell);
1102 }
1103 }
1104}
1105
1106static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1107{
1108 int r;
1109 dm_block_t free_blocks;
1110 unsigned long flags;
1111 struct pool *pool = tc->pool;
1112
1113 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1114 if (r)
1115 return r;
1116
1117 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1118 DMWARN("%s: reached low water mark, sending event.",
1119 dm_device_name(pool->pool_md));
1120 spin_lock_irqsave(&pool->lock, flags);
1121 pool->low_water_triggered = 1;
1122 spin_unlock_irqrestore(&pool->lock, flags);
1123 dm_table_event(pool->ti->table);
1124 }
1125
1126 if (!free_blocks) {
1127 if (pool->no_free_space)
1128 return -ENOSPC;
1129 else {
1130 /*
1131 * Try to commit to see if that will free up some
1132 * more space.
1133 */
1134 r = dm_pool_commit_metadata(pool->pmd);
1135 if (r) {
1136 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1137 __func__, r);
1138 return r;
1139 }
1140
1141 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1142 if (r)
1143 return r;
1144
1145 /*
1146 * If we still have no space we set a flag to avoid
1147 * doing all this checking and return -ENOSPC.
1148 */
1149 if (!free_blocks) {
1150 DMWARN("%s: no free space available.",
1151 dm_device_name(pool->pool_md));
1152 spin_lock_irqsave(&pool->lock, flags);
1153 pool->no_free_space = 1;
1154 spin_unlock_irqrestore(&pool->lock, flags);
1155 return -ENOSPC;
1156 }
1157 }
1158 }
1159
1160 r = dm_pool_alloc_data_block(pool->pmd, result);
1161 if (r)
1162 return r;
1163
1164 return 0;
1165}
1166
1167/*
1168 * If we have run out of space, queue bios until the device is
1169 * resumed, presumably after having been reloaded with more space.
1170 */
1171static void retry_on_resume(struct bio *bio)
1172{
Mike Snitzera24c2562012-06-03 00:30:00 +01001173 struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;
Joe Thornbereb2aa482012-03-28 18:41:28 +01001174 struct thin_c *tc = h->tc;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001175 struct pool *pool = tc->pool;
1176 unsigned long flags;
1177
1178 spin_lock_irqsave(&pool->lock, flags);
1179 bio_list_add(&pool->retry_on_resume_list, bio);
1180 spin_unlock_irqrestore(&pool->lock, flags);
1181}
1182
Mike Snitzera24c2562012-06-03 00:30:00 +01001183static void no_space(struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001184{
1185 struct bio *bio;
1186 struct bio_list bios;
1187
1188 bio_list_init(&bios);
1189 cell_release(cell, &bios);
1190
1191 while ((bio = bio_list_pop(&bios)))
1192 retry_on_resume(bio);
1193}
1194
Joe Thornber104655f2012-03-28 18:41:28 +01001195static void process_discard(struct thin_c *tc, struct bio *bio)
1196{
1197 int r;
Mike Snitzerc3a0ce22012-05-12 01:43:16 +01001198 unsigned long flags;
Joe Thornber104655f2012-03-28 18:41:28 +01001199 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +01001200 struct dm_bio_prison_cell *cell, *cell2;
Joe Thornber104655f2012-03-28 18:41:28 +01001201 struct cell_key key, key2;
1202 dm_block_t block = get_bio_block(tc, bio);
1203 struct dm_thin_lookup_result lookup_result;
Mike Snitzera24c2562012-06-03 00:30:00 +01001204 struct dm_thin_new_mapping *m;
Joe Thornber104655f2012-03-28 18:41:28 +01001205
1206 build_virtual_key(tc->td, block, &key);
1207 if (bio_detain(tc->pool->prison, &key, bio, &cell))
1208 return;
1209
1210 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1211 switch (r) {
1212 case 0:
1213 /*
1214 * Check nobody is fiddling with this pool block. This can
1215 * happen if someone's in the process of breaking sharing
1216 * on this block.
1217 */
1218 build_data_key(tc->td, lookup_result.block, &key2);
1219 if (bio_detain(tc->pool->prison, &key2, bio, &cell2)) {
1220 cell_release_singleton(cell, bio);
1221 break;
1222 }
1223
1224 if (io_overlaps_block(pool, bio)) {
1225 /*
1226 * IO may still be going to the destination block. We must
1227 * quiesce before we can do the removal.
1228 */
1229 m = get_next_mapping(pool);
1230 m->tc = tc;
Mike Snitzer17b7d632012-07-27 15:07:57 +01001231 m->pass_discard = (!lookup_result.shared) && pool->pf.discard_passdown;
Joe Thornber104655f2012-03-28 18:41:28 +01001232 m->virt_block = block;
1233 m->data_block = lookup_result.block;
1234 m->cell = cell;
1235 m->cell2 = cell2;
1236 m->err = 0;
1237 m->bio = bio;
1238
1239 if (!ds_add_work(&pool->all_io_ds, &m->list)) {
Mike Snitzerc3a0ce22012-05-12 01:43:16 +01001240 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +01001241 list_add(&m->list, &pool->prepared_discards);
Mike Snitzerc3a0ce22012-05-12 01:43:16 +01001242 spin_unlock_irqrestore(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +01001243 wake_worker(pool);
1244 }
1245 } else {
1246 /*
Mikulas Patocka49296302012-07-27 15:08:03 +01001247 * The DM core makes sure that the discard doesn't span
1248 * a block boundary. So we submit the discard of a
1249 * partial block appropriately.
Joe Thornber104655f2012-03-28 18:41:28 +01001250 */
Joe Thornber104655f2012-03-28 18:41:28 +01001251 cell_release_singleton(cell, bio);
1252 cell_release_singleton(cell2, bio);
Mikulas Patocka650d2a02012-07-20 14:25:05 +01001253 if ((!lookup_result.shared) && pool->pf.discard_passdown)
1254 remap_and_issue(tc, bio, lookup_result.block);
1255 else
1256 bio_endio(bio, 0);
Joe Thornber104655f2012-03-28 18:41:28 +01001257 }
1258 break;
1259
1260 case -ENODATA:
1261 /*
1262 * It isn't provisioned, just forget it.
1263 */
1264 cell_release_singleton(cell, bio);
1265 bio_endio(bio, 0);
1266 break;
1267
1268 default:
1269 DMERR("discard: find block unexpectedly returned %d", r);
1270 cell_release_singleton(cell, bio);
1271 bio_io_error(bio);
1272 break;
1273 }
1274}
1275
Joe Thornber991d9fa2011-10-31 20:21:18 +00001276static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
1277 struct cell_key *key,
1278 struct dm_thin_lookup_result *lookup_result,
Mike Snitzera24c2562012-06-03 00:30:00 +01001279 struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001280{
1281 int r;
1282 dm_block_t data_block;
1283
1284 r = alloc_data_block(tc, &data_block);
1285 switch (r) {
1286 case 0:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001287 schedule_internal_copy(tc, block, lookup_result->block,
1288 data_block, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001289 break;
1290
1291 case -ENOSPC:
1292 no_space(cell);
1293 break;
1294
1295 default:
1296 DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);
1297 cell_error(cell);
1298 break;
1299 }
1300}
1301
1302static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1303 dm_block_t block,
1304 struct dm_thin_lookup_result *lookup_result)
1305{
Mike Snitzera24c2562012-06-03 00:30:00 +01001306 struct dm_bio_prison_cell *cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001307 struct pool *pool = tc->pool;
1308 struct cell_key key;
1309
1310 /*
1311 * If cell is already occupied, then sharing is already in the process
1312 * of being broken so we have nothing further to do here.
1313 */
1314 build_data_key(tc->td, lookup_result->block, &key);
1315 if (bio_detain(pool->prison, &key, bio, &cell))
1316 return;
1317
1318 if (bio_data_dir(bio) == WRITE)
1319 break_sharing(tc, bio, block, &key, lookup_result, cell);
1320 else {
Mike Snitzera24c2562012-06-03 00:30:00 +01001321 struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001322
Joe Thornbereb2aa482012-03-28 18:41:28 +01001323 h->shared_read_entry = ds_inc(&pool->shared_read_ds);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001324
1325 cell_release_singleton(cell, bio);
1326 remap_and_issue(tc, bio, lookup_result->block);
1327 }
1328}
1329
1330static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
Mike Snitzera24c2562012-06-03 00:30:00 +01001331 struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001332{
1333 int r;
1334 dm_block_t data_block;
1335
1336 /*
1337 * Remap empty bios (flushes) immediately, without provisioning.
1338 */
1339 if (!bio->bi_size) {
1340 cell_release_singleton(cell, bio);
1341 remap_and_issue(tc, bio, 0);
1342 return;
1343 }
1344
1345 /*
1346 * Fill read bios with zeroes and complete them immediately.
1347 */
1348 if (bio_data_dir(bio) == READ) {
1349 zero_fill_bio(bio);
1350 cell_release_singleton(cell, bio);
1351 bio_endio(bio, 0);
1352 return;
1353 }
1354
1355 r = alloc_data_block(tc, &data_block);
1356 switch (r) {
1357 case 0:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001358 if (tc->origin_dev)
1359 schedule_external_copy(tc, block, data_block, cell, bio);
1360 else
1361 schedule_zero(tc, block, data_block, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001362 break;
1363
1364 case -ENOSPC:
1365 no_space(cell);
1366 break;
1367
1368 default:
1369 DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);
1370 cell_error(cell);
1371 break;
1372 }
1373}
1374
1375static void process_bio(struct thin_c *tc, struct bio *bio)
1376{
1377 int r;
1378 dm_block_t block = get_bio_block(tc, bio);
Mike Snitzera24c2562012-06-03 00:30:00 +01001379 struct dm_bio_prison_cell *cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001380 struct cell_key key;
1381 struct dm_thin_lookup_result lookup_result;
1382
1383 /*
1384 * If cell is already occupied, then the block is already
1385 * being provisioned so we have nothing further to do here.
1386 */
1387 build_virtual_key(tc->td, block, &key);
1388 if (bio_detain(tc->pool->prison, &key, bio, &cell))
1389 return;
1390
1391 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1392 switch (r) {
1393 case 0:
1394 /*
1395 * We can release this cell now. This thread is the only
1396 * one that puts bios into a cell, and we know there were
1397 * no preceding bios.
1398 */
1399 /*
1400 * TODO: this will probably have to change when discard goes
1401 * back in.
1402 */
1403 cell_release_singleton(cell, bio);
1404
1405 if (lookup_result.shared)
1406 process_shared_bio(tc, bio, block, &lookup_result);
1407 else
1408 remap_and_issue(tc, bio, lookup_result.block);
1409 break;
1410
1411 case -ENODATA:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001412 if (bio_data_dir(bio) == READ && tc->origin_dev) {
1413 cell_release_singleton(cell, bio);
1414 remap_to_origin_and_issue(tc, bio);
1415 } else
1416 provision_block(tc, bio, block, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001417 break;
1418
1419 default:
1420 DMERR("dm_thin_find_block() failed, error = %d", r);
Joe Thornber104655f2012-03-28 18:41:28 +01001421 cell_release_singleton(cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001422 bio_io_error(bio);
1423 break;
1424 }
1425}
1426
Joe Thornber905e51b2012-03-28 18:41:27 +01001427static int need_commit_due_to_time(struct pool *pool)
1428{
1429 return jiffies < pool->last_commit_jiffies ||
1430 jiffies > pool->last_commit_jiffies + COMMIT_PERIOD;
1431}
1432
Joe Thornber991d9fa2011-10-31 20:21:18 +00001433static void process_deferred_bios(struct pool *pool)
1434{
1435 unsigned long flags;
1436 struct bio *bio;
1437 struct bio_list bios;
1438 int r;
1439
1440 bio_list_init(&bios);
1441
1442 spin_lock_irqsave(&pool->lock, flags);
1443 bio_list_merge(&bios, &pool->deferred_bios);
1444 bio_list_init(&pool->deferred_bios);
1445 spin_unlock_irqrestore(&pool->lock, flags);
1446
1447 while ((bio = bio_list_pop(&bios))) {
Mike Snitzera24c2562012-06-03 00:30:00 +01001448 struct dm_thin_endio_hook *h = dm_get_mapinfo(bio)->ptr;
Joe Thornbereb2aa482012-03-28 18:41:28 +01001449 struct thin_c *tc = h->tc;
1450
Joe Thornber991d9fa2011-10-31 20:21:18 +00001451 /*
1452 * If we've got no free new_mapping structs, and processing
1453 * this bio might require one, we pause until there are some
1454 * prepared mappings to process.
1455 */
1456 if (ensure_next_mapping(pool)) {
1457 spin_lock_irqsave(&pool->lock, flags);
1458 bio_list_merge(&pool->deferred_bios, &bios);
1459 spin_unlock_irqrestore(&pool->lock, flags);
1460
1461 break;
1462 }
Joe Thornber104655f2012-03-28 18:41:28 +01001463
1464 if (bio->bi_rw & REQ_DISCARD)
1465 process_discard(tc, bio);
1466 else
1467 process_bio(tc, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001468 }
1469
1470 /*
1471 * If there are any deferred flush bios, we must commit
1472 * the metadata before issuing them.
1473 */
1474 bio_list_init(&bios);
1475 spin_lock_irqsave(&pool->lock, flags);
1476 bio_list_merge(&bios, &pool->deferred_flush_bios);
1477 bio_list_init(&pool->deferred_flush_bios);
1478 spin_unlock_irqrestore(&pool->lock, flags);
1479
Joe Thornber905e51b2012-03-28 18:41:27 +01001480 if (bio_list_empty(&bios) && !need_commit_due_to_time(pool))
Joe Thornber991d9fa2011-10-31 20:21:18 +00001481 return;
1482
1483 r = dm_pool_commit_metadata(pool->pmd);
1484 if (r) {
1485 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1486 __func__, r);
1487 while ((bio = bio_list_pop(&bios)))
1488 bio_io_error(bio);
1489 return;
1490 }
Joe Thornber905e51b2012-03-28 18:41:27 +01001491 pool->last_commit_jiffies = jiffies;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001492
1493 while ((bio = bio_list_pop(&bios)))
1494 generic_make_request(bio);
1495}
1496
1497static void do_worker(struct work_struct *ws)
1498{
1499 struct pool *pool = container_of(ws, struct pool, worker);
1500
Joe Thornber104655f2012-03-28 18:41:28 +01001501 process_prepared(pool, &pool->prepared_mappings, process_prepared_mapping);
1502 process_prepared(pool, &pool->prepared_discards, process_prepared_discard);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001503 process_deferred_bios(pool);
1504}
1505
Joe Thornber905e51b2012-03-28 18:41:27 +01001506/*
1507 * We want to commit periodically so that not too much
1508 * unwritten data builds up.
1509 */
1510static void do_waker(struct work_struct *ws)
1511{
1512 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
1513 wake_worker(pool);
1514 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
1515}
1516
Joe Thornber991d9fa2011-10-31 20:21:18 +00001517/*----------------------------------------------------------------*/
1518
1519/*
1520 * Mapping functions.
1521 */
1522
1523/*
1524 * Called only while mapping a thin bio to hand it over to the workqueue.
1525 */
1526static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
1527{
1528 unsigned long flags;
1529 struct pool *pool = tc->pool;
1530
1531 spin_lock_irqsave(&pool->lock, flags);
1532 bio_list_add(&pool->deferred_bios, bio);
1533 spin_unlock_irqrestore(&pool->lock, flags);
1534
1535 wake_worker(pool);
1536}
1537
Mike Snitzera24c2562012-06-03 00:30:00 +01001538static struct dm_thin_endio_hook *thin_hook_bio(struct thin_c *tc, struct bio *bio)
Joe Thornbereb2aa482012-03-28 18:41:28 +01001539{
1540 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +01001541 struct dm_thin_endio_hook *h = mempool_alloc(pool->endio_hook_pool, GFP_NOIO);
Joe Thornbereb2aa482012-03-28 18:41:28 +01001542
1543 h->tc = tc;
1544 h->shared_read_entry = NULL;
Joe Thornber104655f2012-03-28 18:41:28 +01001545 h->all_io_entry = bio->bi_rw & REQ_DISCARD ? NULL : ds_inc(&pool->all_io_ds);
Joe Thornbereb2aa482012-03-28 18:41:28 +01001546 h->overwrite_mapping = NULL;
1547
1548 return h;
1549}
1550
Joe Thornber991d9fa2011-10-31 20:21:18 +00001551/*
1552 * Non-blocking function called from the thin target's map function.
1553 */
1554static int thin_bio_map(struct dm_target *ti, struct bio *bio,
1555 union map_info *map_context)
1556{
1557 int r;
1558 struct thin_c *tc = ti->private;
1559 dm_block_t block = get_bio_block(tc, bio);
1560 struct dm_thin_device *td = tc->td;
1561 struct dm_thin_lookup_result result;
1562
Joe Thornbereb2aa482012-03-28 18:41:28 +01001563 map_context->ptr = thin_hook_bio(tc, bio);
Joe Thornber104655f2012-03-28 18:41:28 +01001564 if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001565 thin_defer_bio(tc, bio);
1566 return DM_MAPIO_SUBMITTED;
1567 }
1568
1569 r = dm_thin_find_block(td, block, 0, &result);
1570
1571 /*
1572 * Note that we defer readahead too.
1573 */
1574 switch (r) {
1575 case 0:
1576 if (unlikely(result.shared)) {
1577 /*
1578 * We have a race condition here between the
1579 * result.shared value returned by the lookup and
1580 * snapshot creation, which may cause new
1581 * sharing.
1582 *
1583 * To avoid this always quiesce the origin before
1584 * taking the snap. You want to do this anyway to
1585 * ensure a consistent application view
1586 * (i.e. lockfs).
1587 *
1588 * More distant ancestors are irrelevant. The
1589 * shared flag will be set in their case.
1590 */
1591 thin_defer_bio(tc, bio);
1592 r = DM_MAPIO_SUBMITTED;
1593 } else {
1594 remap(tc, bio, result.block);
1595 r = DM_MAPIO_REMAPPED;
1596 }
1597 break;
1598
1599 case -ENODATA:
1600 /*
1601 * In future, the failed dm_thin_find_block above could
1602 * provide the hint to load the metadata into cache.
1603 */
1604 case -EWOULDBLOCK:
1605 thin_defer_bio(tc, bio);
1606 r = DM_MAPIO_SUBMITTED;
1607 break;
1608 }
1609
1610 return r;
1611}
1612
1613static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
1614{
1615 int r;
1616 unsigned long flags;
1617 struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
1618
1619 spin_lock_irqsave(&pt->pool->lock, flags);
1620 r = !bio_list_empty(&pt->pool->retry_on_resume_list);
1621 spin_unlock_irqrestore(&pt->pool->lock, flags);
1622
1623 if (!r) {
1624 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
1625 r = bdi_congested(&q->backing_dev_info, bdi_bits);
1626 }
1627
1628 return r;
1629}
1630
1631static void __requeue_bios(struct pool *pool)
1632{
1633 bio_list_merge(&pool->deferred_bios, &pool->retry_on_resume_list);
1634 bio_list_init(&pool->retry_on_resume_list);
1635}
1636
1637/*----------------------------------------------------------------
1638 * Binding of control targets to a pool object
1639 *--------------------------------------------------------------*/
1640static int bind_control_target(struct pool *pool, struct dm_target *ti)
1641{
1642 struct pool_c *pt = ti->private;
1643
1644 pool->ti = ti;
1645 pool->low_water_blocks = pt->low_water_blocks;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001646 pool->pf = pt->pf;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001647
Mike Snitzerf4026932012-05-19 01:01:01 +01001648 /*
1649 * If discard_passdown was enabled verify that the data device
1650 * supports discards. Disable discard_passdown if not; otherwise
1651 * -EOPNOTSUPP will be returned.
1652 */
1653 if (pt->pf.discard_passdown) {
1654 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
1655 if (!q || !blk_queue_discard(q)) {
1656 char buf[BDEVNAME_SIZE];
1657 DMWARN("Discard unsupported by data device (%s): Disabling discard passdown.",
1658 bdevname(pt->data_dev->bdev, buf));
1659 pool->pf.discard_passdown = 0;
1660 }
1661 }
1662
Joe Thornber991d9fa2011-10-31 20:21:18 +00001663 return 0;
1664}
1665
1666static void unbind_control_target(struct pool *pool, struct dm_target *ti)
1667{
1668 if (pool->ti == ti)
1669 pool->ti = NULL;
1670}
1671
1672/*----------------------------------------------------------------
1673 * Pool creation
1674 *--------------------------------------------------------------*/
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001675/* Initialize pool features. */
1676static void pool_features_init(struct pool_features *pf)
1677{
1678 pf->zero_new_blocks = 1;
1679 pf->discard_enabled = 1;
1680 pf->discard_passdown = 1;
1681}
1682
Joe Thornber991d9fa2011-10-31 20:21:18 +00001683static void __pool_destroy(struct pool *pool)
1684{
1685 __pool_table_remove(pool);
1686
1687 if (dm_pool_metadata_close(pool->pmd) < 0)
1688 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
1689
1690 prison_destroy(pool->prison);
1691 dm_kcopyd_client_destroy(pool->copier);
1692
1693 if (pool->wq)
1694 destroy_workqueue(pool->wq);
1695
1696 if (pool->next_mapping)
1697 mempool_free(pool->next_mapping, pool->mapping_pool);
1698 mempool_destroy(pool->mapping_pool);
1699 mempool_destroy(pool->endio_hook_pool);
1700 kfree(pool);
1701}
1702
Mike Snitzera24c2562012-06-03 00:30:00 +01001703static struct kmem_cache *_new_mapping_cache;
1704static struct kmem_cache *_endio_hook_cache;
1705
Joe Thornber991d9fa2011-10-31 20:21:18 +00001706static struct pool *pool_create(struct mapped_device *pool_md,
1707 struct block_device *metadata_dev,
1708 unsigned long block_size, char **error)
1709{
1710 int r;
1711 void *err_p;
1712 struct pool *pool;
1713 struct dm_pool_metadata *pmd;
1714
1715 pmd = dm_pool_metadata_open(metadata_dev, block_size);
1716 if (IS_ERR(pmd)) {
1717 *error = "Error creating metadata object";
1718 return (struct pool *)pmd;
1719 }
1720
1721 pool = kmalloc(sizeof(*pool), GFP_KERNEL);
1722 if (!pool) {
1723 *error = "Error allocating memory for pool";
1724 err_p = ERR_PTR(-ENOMEM);
1725 goto bad_pool;
1726 }
1727
1728 pool->pmd = pmd;
1729 pool->sectors_per_block = block_size;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +01001730 if (block_size & (block_size - 1))
1731 pool->sectors_per_block_shift = -1;
1732 else
1733 pool->sectors_per_block_shift = __ffs(block_size);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001734 pool->low_water_blocks = 0;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001735 pool_features_init(&pool->pf);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001736 pool->prison = prison_create(PRISON_CELLS);
1737 if (!pool->prison) {
1738 *error = "Error creating pool's bio prison";
1739 err_p = ERR_PTR(-ENOMEM);
1740 goto bad_prison;
1741 }
1742
1743 pool->copier = dm_kcopyd_client_create();
1744 if (IS_ERR(pool->copier)) {
1745 r = PTR_ERR(pool->copier);
1746 *error = "Error creating pool's kcopyd client";
1747 err_p = ERR_PTR(r);
1748 goto bad_kcopyd_client;
1749 }
1750
1751 /*
1752 * Create singlethreaded workqueue that will service all devices
1753 * that use this metadata.
1754 */
1755 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
1756 if (!pool->wq) {
1757 *error = "Error creating pool's workqueue";
1758 err_p = ERR_PTR(-ENOMEM);
1759 goto bad_wq;
1760 }
1761
1762 INIT_WORK(&pool->worker, do_worker);
Joe Thornber905e51b2012-03-28 18:41:27 +01001763 INIT_DELAYED_WORK(&pool->waker, do_waker);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001764 spin_lock_init(&pool->lock);
1765 bio_list_init(&pool->deferred_bios);
1766 bio_list_init(&pool->deferred_flush_bios);
1767 INIT_LIST_HEAD(&pool->prepared_mappings);
Joe Thornber104655f2012-03-28 18:41:28 +01001768 INIT_LIST_HEAD(&pool->prepared_discards);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001769 pool->low_water_triggered = 0;
1770 pool->no_free_space = 0;
1771 bio_list_init(&pool->retry_on_resume_list);
Joe Thornbereb2aa482012-03-28 18:41:28 +01001772 ds_init(&pool->shared_read_ds);
Joe Thornber104655f2012-03-28 18:41:28 +01001773 ds_init(&pool->all_io_ds);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001774
1775 pool->next_mapping = NULL;
Mike Snitzera24c2562012-06-03 00:30:00 +01001776 pool->mapping_pool = mempool_create_slab_pool(MAPPING_POOL_SIZE,
1777 _new_mapping_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001778 if (!pool->mapping_pool) {
1779 *error = "Error creating pool's mapping mempool";
1780 err_p = ERR_PTR(-ENOMEM);
1781 goto bad_mapping_pool;
1782 }
1783
Mike Snitzera24c2562012-06-03 00:30:00 +01001784 pool->endio_hook_pool = mempool_create_slab_pool(ENDIO_HOOK_POOL_SIZE,
1785 _endio_hook_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001786 if (!pool->endio_hook_pool) {
1787 *error = "Error creating pool's endio_hook mempool";
1788 err_p = ERR_PTR(-ENOMEM);
1789 goto bad_endio_hook_pool;
1790 }
1791 pool->ref_count = 1;
Joe Thornber905e51b2012-03-28 18:41:27 +01001792 pool->last_commit_jiffies = jiffies;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001793 pool->pool_md = pool_md;
1794 pool->md_dev = metadata_dev;
1795 __pool_table_insert(pool);
1796
1797 return pool;
1798
1799bad_endio_hook_pool:
1800 mempool_destroy(pool->mapping_pool);
1801bad_mapping_pool:
1802 destroy_workqueue(pool->wq);
1803bad_wq:
1804 dm_kcopyd_client_destroy(pool->copier);
1805bad_kcopyd_client:
1806 prison_destroy(pool->prison);
1807bad_prison:
1808 kfree(pool);
1809bad_pool:
1810 if (dm_pool_metadata_close(pmd))
1811 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
1812
1813 return err_p;
1814}
1815
1816static void __pool_inc(struct pool *pool)
1817{
1818 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
1819 pool->ref_count++;
1820}
1821
1822static void __pool_dec(struct pool *pool)
1823{
1824 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
1825 BUG_ON(!pool->ref_count);
1826 if (!--pool->ref_count)
1827 __pool_destroy(pool);
1828}
1829
1830static struct pool *__pool_find(struct mapped_device *pool_md,
1831 struct block_device *metadata_dev,
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001832 unsigned long block_size, char **error,
1833 int *created)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001834{
1835 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
1836
1837 if (pool) {
Mike Snitzerf09996c2012-07-27 15:07:59 +01001838 if (pool->pool_md != pool_md) {
1839 *error = "metadata device already in use by a pool";
Joe Thornber991d9fa2011-10-31 20:21:18 +00001840 return ERR_PTR(-EBUSY);
Mike Snitzerf09996c2012-07-27 15:07:59 +01001841 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001842 __pool_inc(pool);
1843
1844 } else {
1845 pool = __pool_table_lookup(pool_md);
1846 if (pool) {
Mike Snitzerf09996c2012-07-27 15:07:59 +01001847 if (pool->md_dev != metadata_dev) {
1848 *error = "different pool cannot replace a pool";
Joe Thornber991d9fa2011-10-31 20:21:18 +00001849 return ERR_PTR(-EINVAL);
Mike Snitzerf09996c2012-07-27 15:07:59 +01001850 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001851 __pool_inc(pool);
1852
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001853 } else {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001854 pool = pool_create(pool_md, metadata_dev, block_size, error);
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001855 *created = 1;
1856 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001857 }
1858
1859 return pool;
1860}
1861
1862/*----------------------------------------------------------------
1863 * Pool target methods
1864 *--------------------------------------------------------------*/
1865static void pool_dtr(struct dm_target *ti)
1866{
1867 struct pool_c *pt = ti->private;
1868
1869 mutex_lock(&dm_thin_pool_table.mutex);
1870
1871 unbind_control_target(pt->pool, ti);
1872 __pool_dec(pt->pool);
1873 dm_put_device(ti, pt->metadata_dev);
1874 dm_put_device(ti, pt->data_dev);
1875 kfree(pt);
1876
1877 mutex_unlock(&dm_thin_pool_table.mutex);
1878}
1879
Joe Thornber991d9fa2011-10-31 20:21:18 +00001880static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
1881 struct dm_target *ti)
1882{
1883 int r;
1884 unsigned argc;
1885 const char *arg_name;
1886
1887 static struct dm_arg _args[] = {
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001888 {0, 3, "Invalid number of pool feature arguments"},
Joe Thornber991d9fa2011-10-31 20:21:18 +00001889 };
1890
1891 /*
1892 * No feature arguments supplied.
1893 */
1894 if (!as->argc)
1895 return 0;
1896
1897 r = dm_read_arg_group(_args, as, &argc, &ti->error);
1898 if (r)
1899 return -EINVAL;
1900
1901 while (argc && !r) {
1902 arg_name = dm_shift_arg(as);
1903 argc--;
1904
1905 if (!strcasecmp(arg_name, "skip_block_zeroing")) {
1906 pf->zero_new_blocks = 0;
1907 continue;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001908 } else if (!strcasecmp(arg_name, "ignore_discard")) {
1909 pf->discard_enabled = 0;
1910 continue;
1911 } else if (!strcasecmp(arg_name, "no_discard_passdown")) {
1912 pf->discard_passdown = 0;
1913 continue;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001914 }
1915
1916 ti->error = "Unrecognised pool feature requested";
1917 r = -EINVAL;
1918 }
1919
1920 return r;
1921}
1922
1923/*
1924 * thin-pool <metadata dev> <data dev>
1925 * <data block size (sectors)>
1926 * <low water mark (blocks)>
1927 * [<#feature args> [<arg>]*]
1928 *
1929 * Optional feature arguments are:
1930 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001931 * ignore_discard: disable discard
1932 * no_discard_passdown: don't pass discards down to the data device
Joe Thornber991d9fa2011-10-31 20:21:18 +00001933 */
1934static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
1935{
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001936 int r, pool_created = 0;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001937 struct pool_c *pt;
1938 struct pool *pool;
1939 struct pool_features pf;
1940 struct dm_arg_set as;
1941 struct dm_dev *data_dev;
1942 unsigned long block_size;
1943 dm_block_t low_water_blocks;
1944 struct dm_dev *metadata_dev;
1945 sector_t metadata_dev_size;
Mike Snitzerc4a69ec2012-03-28 18:41:28 +01001946 char b[BDEVNAME_SIZE];
Joe Thornber991d9fa2011-10-31 20:21:18 +00001947
1948 /*
1949 * FIXME Remove validation from scope of lock.
1950 */
1951 mutex_lock(&dm_thin_pool_table.mutex);
1952
1953 if (argc < 4) {
1954 ti->error = "Invalid argument count";
1955 r = -EINVAL;
1956 goto out_unlock;
1957 }
1958 as.argc = argc;
1959 as.argv = argv;
1960
1961 r = dm_get_device(ti, argv[0], FMODE_READ | FMODE_WRITE, &metadata_dev);
1962 if (r) {
1963 ti->error = "Error opening metadata block device";
1964 goto out_unlock;
1965 }
1966
1967 metadata_dev_size = i_size_read(metadata_dev->bdev->bd_inode) >> SECTOR_SHIFT;
Mike Snitzerc4a69ec2012-03-28 18:41:28 +01001968 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
1969 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
1970 bdevname(metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001971
1972 r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
1973 if (r) {
1974 ti->error = "Error getting data device";
1975 goto out_metadata;
1976 }
1977
1978 if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
1979 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
1980 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01001981 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001982 ti->error = "Invalid block size";
1983 r = -EINVAL;
1984 goto out;
1985 }
1986
1987 if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
1988 ti->error = "Invalid low water mark";
1989 r = -EINVAL;
1990 goto out;
1991 }
1992
1993 /*
1994 * Set default pool features.
1995 */
Joe Thornber67e2e2b2012-03-28 18:41:29 +01001996 pool_features_init(&pf);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001997
1998 dm_consume_args(&as, 4);
1999 r = parse_pool_features(&as, &pf, ti);
2000 if (r)
2001 goto out;
2002
2003 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
2004 if (!pt) {
2005 r = -ENOMEM;
2006 goto out;
2007 }
2008
2009 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev,
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002010 block_size, &ti->error, &pool_created);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002011 if (IS_ERR(pool)) {
2012 r = PTR_ERR(pool);
2013 goto out_free_pt;
2014 }
2015
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002016 /*
2017 * 'pool_created' reflects whether this is the first table load.
2018 * Top level discard support is not allowed to be changed after
2019 * initial load. This would require a pool reload to trigger thin
2020 * device changes.
2021 */
2022 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
2023 ti->error = "Discard support cannot be disabled once enabled";
2024 r = -EINVAL;
2025 goto out_flags_changed;
2026 }
2027
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002028 /*
2029 * The block layer requires discard_granularity to be a power of 2.
2030 */
2031 if (pf.discard_enabled && !is_power_of_2(block_size)) {
2032 ti->error = "Discard support must be disabled when the block size is not a power of 2";
2033 r = -EINVAL;
2034 goto out_flags_changed;
2035 }
2036
Joe Thornber991d9fa2011-10-31 20:21:18 +00002037 pt->pool = pool;
2038 pt->ti = ti;
2039 pt->metadata_dev = metadata_dev;
2040 pt->data_dev = data_dev;
2041 pt->low_water_blocks = low_water_blocks;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002042 pt->pf = pf;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002043 ti->num_flush_requests = 1;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002044 /*
2045 * Only need to enable discards if the pool should pass
2046 * them down to the data device. The thin device's discard
2047 * processing will cause mappings to be removed from the btree.
2048 */
2049 if (pf.discard_enabled && pf.discard_passdown) {
2050 ti->num_discard_requests = 1;
2051 /*
2052 * Setting 'discards_supported' circumvents the normal
2053 * stacking of discard limits (this keeps the pool and
2054 * thin devices' discard limits consistent).
2055 */
2056 ti->discards_supported = 1;
2057 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002058 ti->private = pt;
2059
2060 pt->callbacks.congested_fn = pool_is_congested;
2061 dm_table_add_target_callbacks(ti->table, &pt->callbacks);
2062
2063 mutex_unlock(&dm_thin_pool_table.mutex);
2064
2065 return 0;
2066
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002067out_flags_changed:
2068 __pool_dec(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002069out_free_pt:
2070 kfree(pt);
2071out:
2072 dm_put_device(ti, data_dev);
2073out_metadata:
2074 dm_put_device(ti, metadata_dev);
2075out_unlock:
2076 mutex_unlock(&dm_thin_pool_table.mutex);
2077
2078 return r;
2079}
2080
2081static int pool_map(struct dm_target *ti, struct bio *bio,
2082 union map_info *map_context)
2083{
2084 int r;
2085 struct pool_c *pt = ti->private;
2086 struct pool *pool = pt->pool;
2087 unsigned long flags;
2088
2089 /*
2090 * As this is a singleton target, ti->begin is always zero.
2091 */
2092 spin_lock_irqsave(&pool->lock, flags);
2093 bio->bi_bdev = pt->data_dev->bdev;
2094 r = DM_MAPIO_REMAPPED;
2095 spin_unlock_irqrestore(&pool->lock, flags);
2096
2097 return r;
2098}
2099
2100/*
2101 * Retrieves the number of blocks of the data device from
2102 * the superblock and compares it to the actual device size,
2103 * thus resizing the data device in case it has grown.
2104 *
2105 * This both copes with opening preallocated data devices in the ctr
2106 * being followed by a resume
2107 * -and-
2108 * calling the resume method individually after userspace has
2109 * grown the data device in reaction to a table event.
2110 */
2111static int pool_preresume(struct dm_target *ti)
2112{
2113 int r;
2114 struct pool_c *pt = ti->private;
2115 struct pool *pool = pt->pool;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002116 sector_t data_size = ti->len;
2117 dm_block_t sb_data_size;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002118
2119 /*
2120 * Take control of the pool object.
2121 */
2122 r = bind_control_target(pool, ti);
2123 if (r)
2124 return r;
2125
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002126 (void) sector_div(data_size, pool->sectors_per_block);
2127
Joe Thornber991d9fa2011-10-31 20:21:18 +00002128 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
2129 if (r) {
2130 DMERR("failed to retrieve data device size");
2131 return r;
2132 }
2133
2134 if (data_size < sb_data_size) {
2135 DMERR("pool target too small, is %llu blocks (expected %llu)",
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002136 (unsigned long long)data_size, sb_data_size);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002137 return -EINVAL;
2138
2139 } else if (data_size > sb_data_size) {
2140 r = dm_pool_resize_data_dev(pool->pmd, data_size);
2141 if (r) {
2142 DMERR("failed to resize data device");
2143 return r;
2144 }
2145
2146 r = dm_pool_commit_metadata(pool->pmd);
2147 if (r) {
2148 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
2149 __func__, r);
2150 return r;
2151 }
2152 }
2153
2154 return 0;
2155}
2156
2157static void pool_resume(struct dm_target *ti)
2158{
2159 struct pool_c *pt = ti->private;
2160 struct pool *pool = pt->pool;
2161 unsigned long flags;
2162
2163 spin_lock_irqsave(&pool->lock, flags);
2164 pool->low_water_triggered = 0;
2165 pool->no_free_space = 0;
2166 __requeue_bios(pool);
2167 spin_unlock_irqrestore(&pool->lock, flags);
2168
Joe Thornber905e51b2012-03-28 18:41:27 +01002169 do_waker(&pool->waker.work);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002170}
2171
2172static void pool_postsuspend(struct dm_target *ti)
2173{
2174 int r;
2175 struct pool_c *pt = ti->private;
2176 struct pool *pool = pt->pool;
2177
Joe Thornber905e51b2012-03-28 18:41:27 +01002178 cancel_delayed_work(&pool->waker);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002179 flush_workqueue(pool->wq);
2180
2181 r = dm_pool_commit_metadata(pool->pmd);
2182 if (r < 0) {
2183 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
2184 __func__, r);
2185 /* FIXME: invalidate device? error the next FUA or FLUSH bio ?*/
2186 }
2187}
2188
2189static int check_arg_count(unsigned argc, unsigned args_required)
2190{
2191 if (argc != args_required) {
2192 DMWARN("Message received with %u arguments instead of %u.",
2193 argc, args_required);
2194 return -EINVAL;
2195 }
2196
2197 return 0;
2198}
2199
2200static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
2201{
2202 if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
2203 *dev_id <= MAX_DEV_ID)
2204 return 0;
2205
2206 if (warning)
2207 DMWARN("Message received with invalid device id: %s", arg);
2208
2209 return -EINVAL;
2210}
2211
2212static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
2213{
2214 dm_thin_id dev_id;
2215 int r;
2216
2217 r = check_arg_count(argc, 2);
2218 if (r)
2219 return r;
2220
2221 r = read_dev_id(argv[1], &dev_id, 1);
2222 if (r)
2223 return r;
2224
2225 r = dm_pool_create_thin(pool->pmd, dev_id);
2226 if (r) {
2227 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
2228 argv[1]);
2229 return r;
2230 }
2231
2232 return 0;
2233}
2234
2235static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
2236{
2237 dm_thin_id dev_id;
2238 dm_thin_id origin_dev_id;
2239 int r;
2240
2241 r = check_arg_count(argc, 3);
2242 if (r)
2243 return r;
2244
2245 r = read_dev_id(argv[1], &dev_id, 1);
2246 if (r)
2247 return r;
2248
2249 r = read_dev_id(argv[2], &origin_dev_id, 1);
2250 if (r)
2251 return r;
2252
2253 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
2254 if (r) {
2255 DMWARN("Creation of new snapshot %s of device %s failed.",
2256 argv[1], argv[2]);
2257 return r;
2258 }
2259
2260 return 0;
2261}
2262
2263static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
2264{
2265 dm_thin_id dev_id;
2266 int r;
2267
2268 r = check_arg_count(argc, 2);
2269 if (r)
2270 return r;
2271
2272 r = read_dev_id(argv[1], &dev_id, 1);
2273 if (r)
2274 return r;
2275
2276 r = dm_pool_delete_thin_device(pool->pmd, dev_id);
2277 if (r)
2278 DMWARN("Deletion of thin device %s failed.", argv[1]);
2279
2280 return r;
2281}
2282
2283static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
2284{
2285 dm_thin_id old_id, new_id;
2286 int r;
2287
2288 r = check_arg_count(argc, 3);
2289 if (r)
2290 return r;
2291
2292 if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
2293 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
2294 return -EINVAL;
2295 }
2296
2297 if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
2298 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
2299 return -EINVAL;
2300 }
2301
2302 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
2303 if (r) {
2304 DMWARN("Failed to change transaction id from %s to %s.",
2305 argv[1], argv[2]);
2306 return r;
2307 }
2308
2309 return 0;
2310}
2311
Joe Thornbercc8394d2012-06-03 00:30:01 +01002312static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
2313{
2314 int r;
2315
2316 r = check_arg_count(argc, 1);
2317 if (r)
2318 return r;
2319
Joe Thornber0d200ae2012-07-03 12:55:31 +01002320 r = dm_pool_commit_metadata(pool->pmd);
2321 if (r) {
2322 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
2323 __func__, r);
2324 return r;
2325 }
2326
Joe Thornbercc8394d2012-06-03 00:30:01 +01002327 r = dm_pool_reserve_metadata_snap(pool->pmd);
2328 if (r)
2329 DMWARN("reserve_metadata_snap message failed.");
2330
2331 return r;
2332}
2333
2334static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
2335{
2336 int r;
2337
2338 r = check_arg_count(argc, 1);
2339 if (r)
2340 return r;
2341
2342 r = dm_pool_release_metadata_snap(pool->pmd);
2343 if (r)
2344 DMWARN("release_metadata_snap message failed.");
2345
2346 return r;
2347}
2348
Joe Thornber991d9fa2011-10-31 20:21:18 +00002349/*
2350 * Messages supported:
2351 * create_thin <dev_id>
2352 * create_snap <dev_id> <origin_id>
2353 * delete <dev_id>
2354 * trim <dev_id> <new_size_in_sectors>
2355 * set_transaction_id <current_trans_id> <new_trans_id>
Joe Thornbercc8394d2012-06-03 00:30:01 +01002356 * reserve_metadata_snap
2357 * release_metadata_snap
Joe Thornber991d9fa2011-10-31 20:21:18 +00002358 */
2359static int pool_message(struct dm_target *ti, unsigned argc, char **argv)
2360{
2361 int r = -EINVAL;
2362 struct pool_c *pt = ti->private;
2363 struct pool *pool = pt->pool;
2364
2365 if (!strcasecmp(argv[0], "create_thin"))
2366 r = process_create_thin_mesg(argc, argv, pool);
2367
2368 else if (!strcasecmp(argv[0], "create_snap"))
2369 r = process_create_snap_mesg(argc, argv, pool);
2370
2371 else if (!strcasecmp(argv[0], "delete"))
2372 r = process_delete_mesg(argc, argv, pool);
2373
2374 else if (!strcasecmp(argv[0], "set_transaction_id"))
2375 r = process_set_transaction_id_mesg(argc, argv, pool);
2376
Joe Thornbercc8394d2012-06-03 00:30:01 +01002377 else if (!strcasecmp(argv[0], "reserve_metadata_snap"))
2378 r = process_reserve_metadata_snap_mesg(argc, argv, pool);
2379
2380 else if (!strcasecmp(argv[0], "release_metadata_snap"))
2381 r = process_release_metadata_snap_mesg(argc, argv, pool);
2382
Joe Thornber991d9fa2011-10-31 20:21:18 +00002383 else
2384 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
2385
2386 if (!r) {
2387 r = dm_pool_commit_metadata(pool->pmd);
2388 if (r)
2389 DMERR("%s message: dm_pool_commit_metadata() failed, error = %d",
2390 argv[0], r);
2391 }
2392
2393 return r;
2394}
2395
2396/*
2397 * Status line is:
2398 * <transaction id> <used metadata sectors>/<total metadata sectors>
2399 * <used data sectors>/<total data sectors> <held metadata root>
2400 */
2401static int pool_status(struct dm_target *ti, status_type_t type,
2402 char *result, unsigned maxlen)
2403{
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002404 int r, count;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002405 unsigned sz = 0;
2406 uint64_t transaction_id;
2407 dm_block_t nr_free_blocks_data;
2408 dm_block_t nr_free_blocks_metadata;
2409 dm_block_t nr_blocks_data;
2410 dm_block_t nr_blocks_metadata;
2411 dm_block_t held_root;
2412 char buf[BDEVNAME_SIZE];
2413 char buf2[BDEVNAME_SIZE];
2414 struct pool_c *pt = ti->private;
2415 struct pool *pool = pt->pool;
2416
2417 switch (type) {
2418 case STATUSTYPE_INFO:
2419 r = dm_pool_get_metadata_transaction_id(pool->pmd,
2420 &transaction_id);
2421 if (r)
2422 return r;
2423
2424 r = dm_pool_get_free_metadata_block_count(pool->pmd,
2425 &nr_free_blocks_metadata);
2426 if (r)
2427 return r;
2428
2429 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
2430 if (r)
2431 return r;
2432
2433 r = dm_pool_get_free_block_count(pool->pmd,
2434 &nr_free_blocks_data);
2435 if (r)
2436 return r;
2437
2438 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
2439 if (r)
2440 return r;
2441
Joe Thornbercc8394d2012-06-03 00:30:01 +01002442 r = dm_pool_get_metadata_snap(pool->pmd, &held_root);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002443 if (r)
2444 return r;
2445
2446 DMEMIT("%llu %llu/%llu %llu/%llu ",
2447 (unsigned long long)transaction_id,
2448 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
2449 (unsigned long long)nr_blocks_metadata,
2450 (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
2451 (unsigned long long)nr_blocks_data);
2452
2453 if (held_root)
2454 DMEMIT("%llu", held_root);
2455 else
2456 DMEMIT("-");
2457
2458 break;
2459
2460 case STATUSTYPE_TABLE:
2461 DMEMIT("%s %s %lu %llu ",
2462 format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
2463 format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
2464 (unsigned long)pool->sectors_per_block,
2465 (unsigned long long)pt->low_water_blocks);
2466
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002467 count = !pool->pf.zero_new_blocks + !pool->pf.discard_enabled +
Mike Snitzerf4026932012-05-19 01:01:01 +01002468 !pt->pf.discard_passdown;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002469 DMEMIT("%u ", count);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002470
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002471 if (!pool->pf.zero_new_blocks)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002472 DMEMIT("skip_block_zeroing ");
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002473
2474 if (!pool->pf.discard_enabled)
2475 DMEMIT("ignore_discard ");
2476
Mike Snitzerf4026932012-05-19 01:01:01 +01002477 if (!pt->pf.discard_passdown)
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002478 DMEMIT("no_discard_passdown ");
2479
Joe Thornber991d9fa2011-10-31 20:21:18 +00002480 break;
2481 }
2482
2483 return 0;
2484}
2485
2486static int pool_iterate_devices(struct dm_target *ti,
2487 iterate_devices_callout_fn fn, void *data)
2488{
2489 struct pool_c *pt = ti->private;
2490
2491 return fn(ti, pt->data_dev, 0, ti->len, data);
2492}
2493
2494static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2495 struct bio_vec *biovec, int max_size)
2496{
2497 struct pool_c *pt = ti->private;
2498 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2499
2500 if (!q->merge_bvec_fn)
2501 return max_size;
2502
2503 bvm->bi_bdev = pt->data_dev->bdev;
2504
2505 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2506}
2507
Joe Thornber104655f2012-03-28 18:41:28 +01002508static void set_discard_limits(struct pool *pool, struct queue_limits *limits)
2509{
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002510 /*
2511 * FIXME: these limits may be incompatible with the pool's data device
2512 */
Joe Thornber104655f2012-03-28 18:41:28 +01002513 limits->max_discard_sectors = pool->sectors_per_block;
2514
2515 /*
2516 * This is just a hint, and not enforced. We have to cope with
Mikulas Patocka49296302012-07-27 15:08:03 +01002517 * bios that cover a block partially. A discard that spans a block
2518 * boundary is not sent to this target.
Joe Thornber104655f2012-03-28 18:41:28 +01002519 */
2520 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002521 limits->discard_zeroes_data = pool->pf.zero_new_blocks;
Joe Thornber104655f2012-03-28 18:41:28 +01002522}
2523
Joe Thornber991d9fa2011-10-31 20:21:18 +00002524static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
2525{
2526 struct pool_c *pt = ti->private;
2527 struct pool *pool = pt->pool;
2528
2529 blk_limits_io_min(limits, 0);
2530 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002531 if (pool->pf.discard_enabled)
2532 set_discard_limits(pool, limits);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002533}
2534
2535static struct target_type pool_target = {
2536 .name = "thin-pool",
2537 .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
2538 DM_TARGET_IMMUTABLE,
Joe Thornbercc8394d2012-06-03 00:30:01 +01002539 .version = {1, 2, 0},
Joe Thornber991d9fa2011-10-31 20:21:18 +00002540 .module = THIS_MODULE,
2541 .ctr = pool_ctr,
2542 .dtr = pool_dtr,
2543 .map = pool_map,
2544 .postsuspend = pool_postsuspend,
2545 .preresume = pool_preresume,
2546 .resume = pool_resume,
2547 .message = pool_message,
2548 .status = pool_status,
2549 .merge = pool_merge,
2550 .iterate_devices = pool_iterate_devices,
2551 .io_hints = pool_io_hints,
2552};
2553
2554/*----------------------------------------------------------------
2555 * Thin target methods
2556 *--------------------------------------------------------------*/
2557static void thin_dtr(struct dm_target *ti)
2558{
2559 struct thin_c *tc = ti->private;
2560
2561 mutex_lock(&dm_thin_pool_table.mutex);
2562
2563 __pool_dec(tc->pool);
2564 dm_pool_close_thin_device(tc->td);
2565 dm_put_device(ti, tc->pool_dev);
Joe Thornber2dd9c252012-03-28 18:41:28 +01002566 if (tc->origin_dev)
2567 dm_put_device(ti, tc->origin_dev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002568 kfree(tc);
2569
2570 mutex_unlock(&dm_thin_pool_table.mutex);
2571}
2572
2573/*
2574 * Thin target parameters:
2575 *
Joe Thornber2dd9c252012-03-28 18:41:28 +01002576 * <pool_dev> <dev_id> [origin_dev]
Joe Thornber991d9fa2011-10-31 20:21:18 +00002577 *
2578 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
2579 * dev_id: the internal device identifier
Joe Thornber2dd9c252012-03-28 18:41:28 +01002580 * origin_dev: a device external to the pool that should act as the origin
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002581 *
2582 * If the pool device has discards disabled, they get disabled for the thin
2583 * device as well.
Joe Thornber991d9fa2011-10-31 20:21:18 +00002584 */
2585static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
2586{
2587 int r;
2588 struct thin_c *tc;
Joe Thornber2dd9c252012-03-28 18:41:28 +01002589 struct dm_dev *pool_dev, *origin_dev;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002590 struct mapped_device *pool_md;
2591
2592 mutex_lock(&dm_thin_pool_table.mutex);
2593
Joe Thornber2dd9c252012-03-28 18:41:28 +01002594 if (argc != 2 && argc != 3) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00002595 ti->error = "Invalid argument count";
2596 r = -EINVAL;
2597 goto out_unlock;
2598 }
2599
2600 tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
2601 if (!tc) {
2602 ti->error = "Out of memory";
2603 r = -ENOMEM;
2604 goto out_unlock;
2605 }
2606
Joe Thornber2dd9c252012-03-28 18:41:28 +01002607 if (argc == 3) {
2608 r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev);
2609 if (r) {
2610 ti->error = "Error opening origin device";
2611 goto bad_origin_dev;
2612 }
2613 tc->origin_dev = origin_dev;
2614 }
2615
Joe Thornber991d9fa2011-10-31 20:21:18 +00002616 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
2617 if (r) {
2618 ti->error = "Error opening pool device";
2619 goto bad_pool_dev;
2620 }
2621 tc->pool_dev = pool_dev;
2622
2623 if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
2624 ti->error = "Invalid device id";
2625 r = -EINVAL;
2626 goto bad_common;
2627 }
2628
2629 pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
2630 if (!pool_md) {
2631 ti->error = "Couldn't get pool mapped device";
2632 r = -EINVAL;
2633 goto bad_common;
2634 }
2635
2636 tc->pool = __pool_table_lookup(pool_md);
2637 if (!tc->pool) {
2638 ti->error = "Couldn't find pool object";
2639 r = -EINVAL;
2640 goto bad_pool_lookup;
2641 }
2642 __pool_inc(tc->pool);
2643
2644 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
2645 if (r) {
2646 ti->error = "Couldn't open thin internal device";
2647 goto bad_thin_open;
2648 }
2649
Mike Snitzer542f9032012-07-27 15:08:00 +01002650 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
2651 if (r)
2652 goto bad_thin_open;
2653
Joe Thornber991d9fa2011-10-31 20:21:18 +00002654 ti->num_flush_requests = 1;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002655
2656 /* In case the pool supports discards, pass them on. */
2657 if (tc->pool->pf.discard_enabled) {
2658 ti->discards_supported = 1;
2659 ti->num_discard_requests = 1;
Mikulas Patocka650d2a02012-07-20 14:25:05 +01002660 ti->discard_zeroes_data_unsupported = 1;
Mikulas Patocka49296302012-07-27 15:08:03 +01002661 /* Discard requests must be split on a block boundary */
2662 ti->split_discard_requests = 1;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002663 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002664
2665 dm_put(pool_md);
2666
2667 mutex_unlock(&dm_thin_pool_table.mutex);
2668
2669 return 0;
2670
2671bad_thin_open:
2672 __pool_dec(tc->pool);
2673bad_pool_lookup:
2674 dm_put(pool_md);
2675bad_common:
2676 dm_put_device(ti, tc->pool_dev);
2677bad_pool_dev:
Joe Thornber2dd9c252012-03-28 18:41:28 +01002678 if (tc->origin_dev)
2679 dm_put_device(ti, tc->origin_dev);
2680bad_origin_dev:
Joe Thornber991d9fa2011-10-31 20:21:18 +00002681 kfree(tc);
2682out_unlock:
2683 mutex_unlock(&dm_thin_pool_table.mutex);
2684
2685 return r;
2686}
2687
2688static int thin_map(struct dm_target *ti, struct bio *bio,
2689 union map_info *map_context)
2690{
Alasdair G Kergon6efd6e82012-03-28 18:41:28 +01002691 bio->bi_sector = dm_target_offset(ti, bio->bi_sector);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002692
2693 return thin_bio_map(ti, bio, map_context);
2694}
2695
Joe Thornbereb2aa482012-03-28 18:41:28 +01002696static int thin_endio(struct dm_target *ti,
2697 struct bio *bio, int err,
2698 union map_info *map_context)
2699{
2700 unsigned long flags;
Mike Snitzera24c2562012-06-03 00:30:00 +01002701 struct dm_thin_endio_hook *h = map_context->ptr;
Joe Thornbereb2aa482012-03-28 18:41:28 +01002702 struct list_head work;
Mike Snitzera24c2562012-06-03 00:30:00 +01002703 struct dm_thin_new_mapping *m, *tmp;
Joe Thornbereb2aa482012-03-28 18:41:28 +01002704 struct pool *pool = h->tc->pool;
2705
2706 if (h->shared_read_entry) {
2707 INIT_LIST_HEAD(&work);
2708 ds_dec(h->shared_read_entry, &work);
2709
2710 spin_lock_irqsave(&pool->lock, flags);
2711 list_for_each_entry_safe(m, tmp, &work, list) {
2712 list_del(&m->list);
2713 m->quiesced = 1;
2714 __maybe_add_mapping(m);
2715 }
2716 spin_unlock_irqrestore(&pool->lock, flags);
2717 }
2718
Joe Thornber104655f2012-03-28 18:41:28 +01002719 if (h->all_io_entry) {
2720 INIT_LIST_HEAD(&work);
2721 ds_dec(h->all_io_entry, &work);
Mike Snitzerc3a0ce22012-05-12 01:43:16 +01002722 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +01002723 list_for_each_entry_safe(m, tmp, &work, list)
2724 list_add(&m->list, &pool->prepared_discards);
Mike Snitzerc3a0ce22012-05-12 01:43:16 +01002725 spin_unlock_irqrestore(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +01002726 }
2727
Joe Thornbereb2aa482012-03-28 18:41:28 +01002728 mempool_free(h, pool->endio_hook_pool);
2729
2730 return 0;
2731}
2732
Joe Thornber991d9fa2011-10-31 20:21:18 +00002733static void thin_postsuspend(struct dm_target *ti)
2734{
2735 if (dm_noflush_suspending(ti))
2736 requeue_io((struct thin_c *)ti->private);
2737}
2738
2739/*
2740 * <nr mapped sectors> <highest mapped sector>
2741 */
2742static int thin_status(struct dm_target *ti, status_type_t type,
2743 char *result, unsigned maxlen)
2744{
2745 int r;
2746 ssize_t sz = 0;
2747 dm_block_t mapped, highest;
2748 char buf[BDEVNAME_SIZE];
2749 struct thin_c *tc = ti->private;
2750
2751 if (!tc->td)
2752 DMEMIT("-");
2753 else {
2754 switch (type) {
2755 case STATUSTYPE_INFO:
2756 r = dm_thin_get_mapped_count(tc->td, &mapped);
2757 if (r)
2758 return r;
2759
2760 r = dm_thin_get_highest_mapped_block(tc->td, &highest);
2761 if (r < 0)
2762 return r;
2763
2764 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
2765 if (r)
2766 DMEMIT("%llu", ((highest + 1) *
2767 tc->pool->sectors_per_block) - 1);
2768 else
2769 DMEMIT("-");
2770 break;
2771
2772 case STATUSTYPE_TABLE:
2773 DMEMIT("%s %lu",
2774 format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
2775 (unsigned long) tc->dev_id);
Joe Thornber2dd9c252012-03-28 18:41:28 +01002776 if (tc->origin_dev)
2777 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
Joe Thornber991d9fa2011-10-31 20:21:18 +00002778 break;
2779 }
2780 }
2781
2782 return 0;
2783}
2784
2785static int thin_iterate_devices(struct dm_target *ti,
2786 iterate_devices_callout_fn fn, void *data)
2787{
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002788 sector_t blocks;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002789 struct thin_c *tc = ti->private;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002790 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002791
2792 /*
2793 * We can't call dm_pool_get_data_dev_size() since that blocks. So
2794 * we follow a more convoluted path through to the pool's target.
2795 */
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002796 if (!pool->ti)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002797 return 0; /* nothing is bound */
2798
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002799 blocks = pool->ti->len;
2800 (void) sector_div(blocks, pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002801 if (blocks)
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002802 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002803
2804 return 0;
2805}
2806
2807static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
2808{
2809 struct thin_c *tc = ti->private;
Joe Thornber104655f2012-03-28 18:41:28 +01002810 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002811
2812 blk_limits_io_min(limits, 0);
Joe Thornber104655f2012-03-28 18:41:28 +01002813 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
2814 set_discard_limits(pool, limits);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002815}
2816
2817static struct target_type thin_target = {
2818 .name = "thin",
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002819 .version = {1, 2, 0},
Joe Thornber991d9fa2011-10-31 20:21:18 +00002820 .module = THIS_MODULE,
2821 .ctr = thin_ctr,
2822 .dtr = thin_dtr,
2823 .map = thin_map,
Joe Thornbereb2aa482012-03-28 18:41:28 +01002824 .end_io = thin_endio,
Joe Thornber991d9fa2011-10-31 20:21:18 +00002825 .postsuspend = thin_postsuspend,
2826 .status = thin_status,
2827 .iterate_devices = thin_iterate_devices,
2828 .io_hints = thin_io_hints,
2829};
2830
2831/*----------------------------------------------------------------*/
2832
2833static int __init dm_thin_init(void)
2834{
2835 int r;
2836
2837 pool_table_init();
2838
2839 r = dm_register_target(&thin_target);
2840 if (r)
2841 return r;
2842
2843 r = dm_register_target(&pool_target);
2844 if (r)
Mike Snitzera24c2562012-06-03 00:30:00 +01002845 goto bad_pool_target;
2846
2847 r = -ENOMEM;
2848
2849 _cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
2850 if (!_cell_cache)
2851 goto bad_cell_cache;
2852
2853 _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
2854 if (!_new_mapping_cache)
2855 goto bad_new_mapping_cache;
2856
2857 _endio_hook_cache = KMEM_CACHE(dm_thin_endio_hook, 0);
2858 if (!_endio_hook_cache)
2859 goto bad_endio_hook_cache;
2860
2861 return 0;
2862
2863bad_endio_hook_cache:
2864 kmem_cache_destroy(_new_mapping_cache);
2865bad_new_mapping_cache:
2866 kmem_cache_destroy(_cell_cache);
2867bad_cell_cache:
2868 dm_unregister_target(&pool_target);
2869bad_pool_target:
2870 dm_unregister_target(&thin_target);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002871
2872 return r;
2873}
2874
2875static void dm_thin_exit(void)
2876{
2877 dm_unregister_target(&thin_target);
2878 dm_unregister_target(&pool_target);
Mike Snitzera24c2562012-06-03 00:30:00 +01002879
2880 kmem_cache_destroy(_cell_cache);
2881 kmem_cache_destroy(_new_mapping_cache);
2882 kmem_cache_destroy(_endio_hook_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002883}
2884
2885module_init(dm_thin_init);
2886module_exit(dm_thin_exit);
2887
Alasdair G Kergon7cab8bf2012-05-12 01:43:19 +01002888MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
Joe Thornber991d9fa2011-10-31 20:21:18 +00002889MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2890MODULE_LICENSE("GPL");