blob: 4843801173fe11a99519b59dd808e46e02425ee4 [file] [log] [blame]
Joe Thornber991d9fa2011-10-31 20:21:18 +00001/*
Joe Thornbere49e5822012-07-27 15:08:16 +01002 * Copyright (C) 2011-2012 Red Hat UK.
Joe Thornber991d9fa2011-10-31 20:21:18 +00003 *
4 * This file is released under the GPL.
5 */
6
7#include "dm-thin-metadata.h"
Mike Snitzer4f81a412012-10-12 21:02:13 +01008#include "dm-bio-prison.h"
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01009#include "dm.h"
Joe Thornber991d9fa2011-10-31 20:21:18 +000010
11#include <linux/device-mapper.h>
12#include <linux/dm-io.h>
13#include <linux/dm-kcopyd.h>
14#include <linux/list.h>
Mike Snitzerc140e1c2014-03-20 21:17:14 -040015#include <linux/rculist.h>
Joe Thornber991d9fa2011-10-31 20:21:18 +000016#include <linux/init.h>
17#include <linux/module.h>
18#include <linux/slab.h>
Mike Snitzer67324ea2014-03-21 18:33:41 -040019#include <linux/rbtree.h>
Joe Thornber991d9fa2011-10-31 20:21:18 +000020
21#define DM_MSG_PREFIX "thin"
22
23/*
24 * Tunable constants
25 */
Alasdair G Kergon7768ed32012-07-27 15:07:57 +010026#define ENDIO_HOOK_POOL_SIZE 1024
Joe Thornber991d9fa2011-10-31 20:21:18 +000027#define MAPPING_POOL_SIZE 1024
28#define PRISON_CELLS 1024
Joe Thornber905e51b2012-03-28 18:41:27 +010029#define COMMIT_PERIOD HZ
Mike Snitzer80c57892014-05-20 13:38:33 -040030#define NO_SPACE_TIMEOUT_SECS 60
31
32static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
Joe Thornber991d9fa2011-10-31 20:21:18 +000033
Mikulas Patockadf5d2e92013-03-01 22:45:49 +000034DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
35 "A percentage of time allocated for copy on write");
36
Joe Thornber991d9fa2011-10-31 20:21:18 +000037/*
38 * The block size of the device holding pool data must be
39 * between 64KB and 1GB.
40 */
41#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
42#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
43
44/*
Joe Thornber991d9fa2011-10-31 20:21:18 +000045 * Device id is restricted to 24 bits.
46 */
47#define MAX_DEV_ID ((1 << 24) - 1)
48
49/*
50 * How do we handle breaking sharing of data blocks?
51 * =================================================
52 *
53 * We use a standard copy-on-write btree to store the mappings for the
54 * devices (note I'm talking about copy-on-write of the metadata here, not
55 * the data). When you take an internal snapshot you clone the root node
56 * of the origin btree. After this there is no concept of an origin or a
57 * snapshot. They are just two device trees that happen to point to the
58 * same data blocks.
59 *
60 * When we get a write in we decide if it's to a shared data block using
61 * some timestamp magic. If it is, we have to break sharing.
62 *
63 * Let's say we write to a shared block in what was the origin. The
64 * steps are:
65 *
66 * i) plug io further to this physical block. (see bio_prison code).
67 *
68 * ii) quiesce any read io to that shared data block. Obviously
Mike Snitzer44feb382012-10-12 21:02:10 +010069 * including all devices that share this block. (see dm_deferred_set code)
Joe Thornber991d9fa2011-10-31 20:21:18 +000070 *
71 * iii) copy the data block to a newly allocate block. This step can be
72 * missed out if the io covers the block. (schedule_copy).
73 *
74 * iv) insert the new mapping into the origin's btree
Joe Thornberfe878f32012-03-28 18:41:24 +010075 * (process_prepared_mapping). This act of inserting breaks some
Joe Thornber991d9fa2011-10-31 20:21:18 +000076 * sharing of btree nodes between the two devices. Breaking sharing only
77 * effects the btree of that specific device. Btrees for the other
78 * devices that share the block never change. The btree for the origin
79 * device as it was after the last commit is untouched, ie. we're using
80 * persistent data structures in the functional programming sense.
81 *
82 * v) unplug io to this physical block, including the io that triggered
83 * the breaking of sharing.
84 *
85 * Steps (ii) and (iii) occur in parallel.
86 *
87 * The metadata _doesn't_ need to be committed before the io continues. We
88 * get away with this because the io is always written to a _new_ block.
89 * If there's a crash, then:
90 *
91 * - The origin mapping will point to the old origin block (the shared
92 * one). This will contain the data as it was before the io that triggered
93 * the breaking of sharing came in.
94 *
95 * - The snap mapping still points to the old block. As it would after
96 * the commit.
97 *
98 * The downside of this scheme is the timestamp magic isn't perfect, and
99 * will continue to think that data block in the snapshot device is shared
100 * even after the write to the origin has broken sharing. I suspect data
101 * blocks will typically be shared by many different devices, so we're
102 * breaking sharing n + 1 times, rather than n, where n is the number of
103 * devices that reference this data block. At the moment I think the
104 * benefits far, far outweigh the disadvantages.
105 */
106
107/*----------------------------------------------------------------*/
108
109/*
Joe Thornber991d9fa2011-10-31 20:21:18 +0000110 * Key building.
111 */
112static void build_data_key(struct dm_thin_device *td,
Mike Snitzer44feb382012-10-12 21:02:10 +0100113 dm_block_t b, struct dm_cell_key *key)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000114{
115 key->virtual = 0;
116 key->dev = dm_thin_dev_id(td);
117 key->block = b;
118}
119
120static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
Mike Snitzer44feb382012-10-12 21:02:10 +0100121 struct dm_cell_key *key)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000122{
123 key->virtual = 1;
124 key->dev = dm_thin_dev_id(td);
125 key->block = b;
126}
127
128/*----------------------------------------------------------------*/
129
130/*
131 * A pool device ties together a metadata device and a data device. It
132 * also provides the interface for creating and destroying internal
133 * devices.
134 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100135struct dm_thin_new_mapping;
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100136
Joe Thornbere49e5822012-07-27 15:08:16 +0100137/*
Joe Thornber3e1a0692014-03-03 16:03:26 +0000138 * The pool runs in 4 modes. Ordered in degraded order for comparisons.
Joe Thornbere49e5822012-07-27 15:08:16 +0100139 */
140enum pool_mode {
141 PM_WRITE, /* metadata may be changed */
Joe Thornber3e1a0692014-03-03 16:03:26 +0000142 PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */
Joe Thornbere49e5822012-07-27 15:08:16 +0100143 PM_READ_ONLY, /* metadata may not be changed */
144 PM_FAIL, /* all I/O fails */
145};
146
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100147struct pool_features {
Joe Thornbere49e5822012-07-27 15:08:16 +0100148 enum pool_mode mode;
149
Mike Snitzer9bc142d2012-09-26 23:45:46 +0100150 bool zero_new_blocks:1;
151 bool discard_enabled:1;
152 bool discard_passdown:1;
Mike Snitzer787a996c2013-12-06 16:21:43 -0500153 bool error_if_no_space:1;
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100154};
155
Joe Thornbere49e5822012-07-27 15:08:16 +0100156struct thin_c;
157typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
158typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
159
Joe Thornber991d9fa2011-10-31 20:21:18 +0000160struct pool {
161 struct list_head list;
162 struct dm_target *ti; /* Only set if a pool target is bound */
163
164 struct mapped_device *pool_md;
165 struct block_device *md_dev;
166 struct dm_pool_metadata *pmd;
167
Joe Thornber991d9fa2011-10-31 20:21:18 +0000168 dm_block_t low_water_blocks;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100169 uint32_t sectors_per_block;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100170 int sectors_per_block_shift;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000171
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100172 struct pool_features pf;
Joe Thornber88a66212013-12-04 20:16:12 -0500173 bool low_water_triggered:1; /* A dm event has been sent */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000174
Mike Snitzer44feb382012-10-12 21:02:10 +0100175 struct dm_bio_prison *prison;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000176 struct dm_kcopyd_client *copier;
177
178 struct workqueue_struct *wq;
179 struct work_struct worker;
Joe Thornber905e51b2012-03-28 18:41:27 +0100180 struct delayed_work waker;
Joe Thornber85ad643b2014-05-09 15:59:38 +0100181 struct delayed_work no_space_timeout;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000182
Joe Thornber905e51b2012-03-28 18:41:27 +0100183 unsigned long last_commit_jiffies;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100184 unsigned ref_count;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000185
186 spinlock_t lock;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000187 struct bio_list deferred_flush_bios;
188 struct list_head prepared_mappings;
Joe Thornber104655f2012-03-28 18:41:28 +0100189 struct list_head prepared_discards;
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400190 struct list_head active_thins;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000191
Mike Snitzer44feb382012-10-12 21:02:10 +0100192 struct dm_deferred_set *shared_read_ds;
193 struct dm_deferred_set *all_io_ds;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000194
Mike Snitzera24c2562012-06-03 00:30:00 +0100195 struct dm_thin_new_mapping *next_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000196 mempool_t *mapping_pool;
Joe Thornbere49e5822012-07-27 15:08:16 +0100197
198 process_bio_fn process_bio;
199 process_bio_fn process_discard;
200
201 process_mapping_fn process_prepared_mapping;
202 process_mapping_fn process_prepared_discard;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000203};
204
Joe Thornbere49e5822012-07-27 15:08:16 +0100205static enum pool_mode get_pool_mode(struct pool *pool);
Joe Thornberb5330652013-12-04 19:51:33 -0500206static void metadata_operation_failed(struct pool *pool, const char *op, int r);
Joe Thornbere49e5822012-07-27 15:08:16 +0100207
Joe Thornber991d9fa2011-10-31 20:21:18 +0000208/*
209 * Target context for a pool.
210 */
211struct pool_c {
212 struct dm_target *ti;
213 struct pool *pool;
214 struct dm_dev *data_dev;
215 struct dm_dev *metadata_dev;
216 struct dm_target_callbacks callbacks;
217
218 dm_block_t low_water_blocks;
Mike Snitzer0424caa2012-09-26 23:45:47 +0100219 struct pool_features requested_pf; /* Features requested during table load */
220 struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000221};
222
223/*
224 * Target context for a thin.
225 */
226struct thin_c {
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400227 struct list_head list;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000228 struct dm_dev *pool_dev;
Joe Thornber2dd9c252012-03-28 18:41:28 +0100229 struct dm_dev *origin_dev;
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100230 sector_t origin_size;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000231 dm_thin_id dev_id;
232
233 struct pool *pool;
234 struct dm_thin_device *td;
Joe Thornber738211f2014-03-03 15:52:28 +0000235 bool requeue_mode:1;
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400236 spinlock_t lock;
237 struct bio_list deferred_bio_list;
238 struct bio_list retry_on_resume_list;
Mike Snitzer67324ea2014-03-21 18:33:41 -0400239 struct rb_root sort_bio_list; /* sorted list of deferred bios */
Joe Thornberb10ebd32014-04-08 11:29:01 +0100240
241 /*
242 * Ensures the thin is not destroyed until the worker has finished
243 * iterating the active_thins list.
244 */
245 atomic_t refcount;
246 struct completion can_destroy;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000247};
248
249/*----------------------------------------------------------------*/
250
Joe Thornber025b9682013-03-01 22:45:50 +0000251/*
252 * wake_worker() is used when new work is queued and when pool_resume is
253 * ready to continue deferred IO processing.
254 */
255static void wake_worker(struct pool *pool)
256{
257 queue_work(pool->wq, &pool->worker);
258}
259
260/*----------------------------------------------------------------*/
261
Joe Thornber6beca5e2013-03-01 22:45:50 +0000262static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
263 struct dm_bio_prison_cell **cell_result)
264{
265 int r;
266 struct dm_bio_prison_cell *cell_prealloc;
267
268 /*
269 * Allocate a cell from the prison's mempool.
270 * This might block but it can't fail.
271 */
272 cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);
273
274 r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);
275 if (r)
276 /*
277 * We reused an old cell; we can get rid of
278 * the new one.
279 */
280 dm_bio_prison_free_cell(pool->prison, cell_prealloc);
281
282 return r;
283}
284
285static void cell_release(struct pool *pool,
286 struct dm_bio_prison_cell *cell,
287 struct bio_list *bios)
288{
289 dm_cell_release(pool->prison, cell, bios);
290 dm_bio_prison_free_cell(pool->prison, cell);
291}
292
293static void cell_release_no_holder(struct pool *pool,
294 struct dm_bio_prison_cell *cell,
295 struct bio_list *bios)
296{
297 dm_cell_release_no_holder(pool->prison, cell, bios);
298 dm_bio_prison_free_cell(pool->prison, cell);
299}
300
Joe Thornber025b9682013-03-01 22:45:50 +0000301static void cell_defer_no_holder_no_free(struct thin_c *tc,
302 struct dm_bio_prison_cell *cell)
303{
304 struct pool *pool = tc->pool;
305 unsigned long flags;
306
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400307 spin_lock_irqsave(&tc->lock, flags);
308 dm_cell_release_no_holder(pool->prison, cell, &tc->deferred_bio_list);
309 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber025b9682013-03-01 22:45:50 +0000310
311 wake_worker(pool);
312}
313
Mike Snitzeraf918052014-05-22 14:32:51 -0400314static void cell_error_with_code(struct pool *pool,
315 struct dm_bio_prison_cell *cell, int error_code)
Joe Thornber6beca5e2013-03-01 22:45:50 +0000316{
Mike Snitzeraf918052014-05-22 14:32:51 -0400317 dm_cell_error(pool->prison, cell, error_code);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000318 dm_bio_prison_free_cell(pool->prison, cell);
319}
320
Mike Snitzeraf918052014-05-22 14:32:51 -0400321static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
322{
323 cell_error_with_code(pool, cell, -EIO);
324}
325
Joe Thornber6beca5e2013-03-01 22:45:50 +0000326/*----------------------------------------------------------------*/
327
Joe Thornber991d9fa2011-10-31 20:21:18 +0000328/*
329 * A global list of pools that uses a struct mapped_device as a key.
330 */
331static struct dm_thin_pool_table {
332 struct mutex mutex;
333 struct list_head pools;
334} dm_thin_pool_table;
335
336static void pool_table_init(void)
337{
338 mutex_init(&dm_thin_pool_table.mutex);
339 INIT_LIST_HEAD(&dm_thin_pool_table.pools);
340}
341
342static void __pool_table_insert(struct pool *pool)
343{
344 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
345 list_add(&pool->list, &dm_thin_pool_table.pools);
346}
347
348static void __pool_table_remove(struct pool *pool)
349{
350 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
351 list_del(&pool->list);
352}
353
354static struct pool *__pool_table_lookup(struct mapped_device *md)
355{
356 struct pool *pool = NULL, *tmp;
357
358 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
359
360 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
361 if (tmp->pool_md == md) {
362 pool = tmp;
363 break;
364 }
365 }
366
367 return pool;
368}
369
370static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
371{
372 struct pool *pool = NULL, *tmp;
373
374 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
375
376 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
377 if (tmp->md_dev == md_dev) {
378 pool = tmp;
379 break;
380 }
381 }
382
383 return pool;
384}
385
386/*----------------------------------------------------------------*/
387
Mike Snitzera24c2562012-06-03 00:30:00 +0100388struct dm_thin_endio_hook {
Joe Thornbereb2aa482012-03-28 18:41:28 +0100389 struct thin_c *tc;
Mike Snitzer44feb382012-10-12 21:02:10 +0100390 struct dm_deferred_entry *shared_read_entry;
391 struct dm_deferred_entry *all_io_entry;
Mike Snitzera24c2562012-06-03 00:30:00 +0100392 struct dm_thin_new_mapping *overwrite_mapping;
Mike Snitzer67324ea2014-03-21 18:33:41 -0400393 struct rb_node rb_node;
Joe Thornbereb2aa482012-03-28 18:41:28 +0100394};
395
Joe Thornber18adc572014-03-03 15:46:42 +0000396static void requeue_bio_list(struct thin_c *tc, struct bio_list *master)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000397{
398 struct bio *bio;
399 struct bio_list bios;
Joe Thornber18adc572014-03-03 15:46:42 +0000400 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000401
402 bio_list_init(&bios);
Joe Thornber18adc572014-03-03 15:46:42 +0000403
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400404 spin_lock_irqsave(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000405 bio_list_merge(&bios, master);
406 bio_list_init(master);
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400407 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000408
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400409 while ((bio = bio_list_pop(&bios)))
410 bio_endio(bio, DM_ENDIO_REQUEUE);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000411}
412
413static void requeue_io(struct thin_c *tc)
414{
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400415 requeue_bio_list(tc, &tc->deferred_bio_list);
416 requeue_bio_list(tc, &tc->retry_on_resume_list);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000417}
418
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400419static void error_thin_retry_list(struct thin_c *tc)
Joe Thornber3e1a0692014-03-03 16:03:26 +0000420{
421 struct bio *bio;
422 unsigned long flags;
423 struct bio_list bios;
424
425 bio_list_init(&bios);
426
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400427 spin_lock_irqsave(&tc->lock, flags);
428 bio_list_merge(&bios, &tc->retry_on_resume_list);
429 bio_list_init(&tc->retry_on_resume_list);
430 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber3e1a0692014-03-03 16:03:26 +0000431
432 while ((bio = bio_list_pop(&bios)))
433 bio_io_error(bio);
434}
435
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400436static void error_retry_list(struct pool *pool)
437{
438 struct thin_c *tc;
439
440 rcu_read_lock();
441 list_for_each_entry_rcu(tc, &pool->active_thins, list)
442 error_thin_retry_list(tc);
443 rcu_read_unlock();
444}
445
Joe Thornber991d9fa2011-10-31 20:21:18 +0000446/*
447 * This section of code contains the logic for processing a thin device's IO.
448 * Much of the code depends on pool object resources (lists, workqueues, etc)
449 * but most is exclusively called from the thin target rather than the thin-pool
450 * target.
451 */
452
Mike Snitzer58f77a22013-03-01 22:45:45 +0000453static bool block_size_is_power_of_two(struct pool *pool)
454{
455 return pool->sectors_per_block_shift >= 0;
456}
457
Joe Thornber991d9fa2011-10-31 20:21:18 +0000458static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
459{
Mike Snitzer58f77a22013-03-01 22:45:45 +0000460 struct pool *pool = tc->pool;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700461 sector_t block_nr = bio->bi_iter.bi_sector;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100462
Mike Snitzer58f77a22013-03-01 22:45:45 +0000463 if (block_size_is_power_of_two(pool))
464 block_nr >>= pool->sectors_per_block_shift;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100465 else
Mike Snitzer58f77a22013-03-01 22:45:45 +0000466 (void) sector_div(block_nr, pool->sectors_per_block);
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100467
468 return block_nr;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000469}
470
471static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
472{
473 struct pool *pool = tc->pool;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700474 sector_t bi_sector = bio->bi_iter.bi_sector;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000475
476 bio->bi_bdev = tc->pool_dev->bdev;
Mike Snitzer58f77a22013-03-01 22:45:45 +0000477 if (block_size_is_power_of_two(pool))
Kent Overstreet4f024f32013-10-11 15:44:27 -0700478 bio->bi_iter.bi_sector =
479 (block << pool->sectors_per_block_shift) |
480 (bi_sector & (pool->sectors_per_block - 1));
Mike Snitzer58f77a22013-03-01 22:45:45 +0000481 else
Kent Overstreet4f024f32013-10-11 15:44:27 -0700482 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
Mike Snitzer58f77a22013-03-01 22:45:45 +0000483 sector_div(bi_sector, pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000484}
485
Joe Thornber2dd9c252012-03-28 18:41:28 +0100486static void remap_to_origin(struct thin_c *tc, struct bio *bio)
487{
488 bio->bi_bdev = tc->origin_dev->bdev;
489}
490
Joe Thornber4afdd682012-07-27 15:08:14 +0100491static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
492{
493 return (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) &&
494 dm_thin_changed_this_transaction(tc->td);
495}
496
Joe Thornbere8088072012-12-21 20:23:31 +0000497static void inc_all_io_entry(struct pool *pool, struct bio *bio)
498{
499 struct dm_thin_endio_hook *h;
500
501 if (bio->bi_rw & REQ_DISCARD)
502 return;
503
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +0000504 h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbere8088072012-12-21 20:23:31 +0000505 h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
506}
507
Joe Thornber2dd9c252012-03-28 18:41:28 +0100508static void issue(struct thin_c *tc, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000509{
510 struct pool *pool = tc->pool;
511 unsigned long flags;
512
Joe Thornbere49e5822012-07-27 15:08:16 +0100513 if (!bio_triggers_commit(tc, bio)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000514 generic_make_request(bio);
Joe Thornbere49e5822012-07-27 15:08:16 +0100515 return;
516 }
517
518 /*
519 * Complete bio with an error if earlier I/O caused changes to
520 * the metadata that can't be committed e.g, due to I/O errors
521 * on the metadata device.
522 */
523 if (dm_thin_aborted_changes(tc->td)) {
524 bio_io_error(bio);
525 return;
526 }
527
528 /*
529 * Batch together any bios that trigger commits and then issue a
530 * single commit for them in process_deferred_bios().
531 */
532 spin_lock_irqsave(&pool->lock, flags);
533 bio_list_add(&pool->deferred_flush_bios, bio);
534 spin_unlock_irqrestore(&pool->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000535}
536
Joe Thornber2dd9c252012-03-28 18:41:28 +0100537static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
538{
539 remap_to_origin(tc, bio);
540 issue(tc, bio);
541}
542
543static void remap_and_issue(struct thin_c *tc, struct bio *bio,
544 dm_block_t block)
545{
546 remap(tc, bio, block);
547 issue(tc, bio);
548}
549
Joe Thornber991d9fa2011-10-31 20:21:18 +0000550/*----------------------------------------------------------------*/
551
552/*
553 * Bio endio functions.
554 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100555struct dm_thin_new_mapping {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000556 struct list_head list;
557
Mike Snitzer7f214662013-12-17 13:43:31 -0500558 bool pass_discard:1;
559 bool definitely_not_shared:1;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000560
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100561 /*
562 * Track quiescing, copying and zeroing preparation actions. When this
563 * counter hits zero the block is prepared and can be inserted into the
564 * btree.
565 */
566 atomic_t prepare_actions;
567
Mike Snitzer7f214662013-12-17 13:43:31 -0500568 int err;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000569 struct thin_c *tc;
570 dm_block_t virt_block;
571 dm_block_t data_block;
Mike Snitzera24c2562012-06-03 00:30:00 +0100572 struct dm_bio_prison_cell *cell, *cell2;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000573
574 /*
575 * If the bio covers the whole area of a block then we can avoid
576 * zeroing or copying. Instead this bio is hooked. The bio will
577 * still be in the cell, so care has to be taken to avoid issuing
578 * the bio twice.
579 */
580 struct bio *bio;
581 bio_end_io_t *saved_bi_end_io;
582};
583
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100584static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000585{
586 struct pool *pool = m->tc->pool;
587
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100588 if (atomic_dec_and_test(&m->prepare_actions)) {
Mike Snitzerdaec3382013-12-11 14:01:20 -0500589 list_add_tail(&m->list, &pool->prepared_mappings);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000590 wake_worker(pool);
591 }
592}
593
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100594static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000595{
596 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000597 struct pool *pool = m->tc->pool;
598
Joe Thornber991d9fa2011-10-31 20:21:18 +0000599 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100600 __complete_mapping_preparation(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000601 spin_unlock_irqrestore(&pool->lock, flags);
602}
603
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100604static void copy_complete(int read_err, unsigned long write_err, void *context)
605{
606 struct dm_thin_new_mapping *m = context;
607
608 m->err = read_err || write_err ? -EIO : 0;
609 complete_mapping_preparation(m);
610}
611
Joe Thornber991d9fa2011-10-31 20:21:18 +0000612static void overwrite_endio(struct bio *bio, int err)
613{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +0000614 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Mike Snitzera24c2562012-06-03 00:30:00 +0100615 struct dm_thin_new_mapping *m = h->overwrite_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000616
617 m->err = err;
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100618 complete_mapping_preparation(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000619}
620
Joe Thornber991d9fa2011-10-31 20:21:18 +0000621/*----------------------------------------------------------------*/
622
623/*
624 * Workqueue.
625 */
626
627/*
628 * Prepared mapping jobs.
629 */
630
631/*
632 * This sends the bios in the cell back to the deferred_bios list.
633 */
Joe Thornber2aab3852012-12-21 20:23:33 +0000634static void cell_defer(struct thin_c *tc, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000635{
636 struct pool *pool = tc->pool;
637 unsigned long flags;
638
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400639 spin_lock_irqsave(&tc->lock, flags);
640 cell_release(pool, cell, &tc->deferred_bio_list);
641 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000642
643 wake_worker(pool);
644}
645
646/*
Joe Thornber6beca5e2013-03-01 22:45:50 +0000647 * Same as cell_defer above, except it omits the original holder of the cell.
Joe Thornber991d9fa2011-10-31 20:21:18 +0000648 */
Joe Thornberf286ba02012-12-21 20:23:33 +0000649static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000650{
Joe Thornber991d9fa2011-10-31 20:21:18 +0000651 struct pool *pool = tc->pool;
652 unsigned long flags;
653
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400654 spin_lock_irqsave(&tc->lock, flags);
655 cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
656 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000657
658 wake_worker(pool);
659}
660
Joe Thornbere49e5822012-07-27 15:08:16 +0100661static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
662{
Kent Overstreet196d38b2013-11-23 18:34:15 -0800663 if (m->bio) {
Joe Thornbere49e5822012-07-27 15:08:16 +0100664 m->bio->bi_end_io = m->saved_bi_end_io;
Kent Overstreet196d38b2013-11-23 18:34:15 -0800665 atomic_inc(&m->bio->bi_remaining);
666 }
Joe Thornber6beca5e2013-03-01 22:45:50 +0000667 cell_error(m->tc->pool, m->cell);
Joe Thornbere49e5822012-07-27 15:08:16 +0100668 list_del(&m->list);
669 mempool_free(m, m->tc->pool->mapping_pool);
670}
Joe Thornber025b9682013-03-01 22:45:50 +0000671
Mike Snitzera24c2562012-06-03 00:30:00 +0100672static void process_prepared_mapping(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000673{
674 struct thin_c *tc = m->tc;
Joe Thornber6beca5e2013-03-01 22:45:50 +0000675 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000676 struct bio *bio;
677 int r;
678
679 bio = m->bio;
Kent Overstreet196d38b2013-11-23 18:34:15 -0800680 if (bio) {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000681 bio->bi_end_io = m->saved_bi_end_io;
Kent Overstreet196d38b2013-11-23 18:34:15 -0800682 atomic_inc(&bio->bi_remaining);
683 }
Joe Thornber991d9fa2011-10-31 20:21:18 +0000684
685 if (m->err) {
Joe Thornber6beca5e2013-03-01 22:45:50 +0000686 cell_error(pool, m->cell);
Joe Thornber905386f2012-07-27 15:08:05 +0100687 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000688 }
689
690 /*
691 * Commit the prepared block into the mapping btree.
692 * Any I/O for this block arriving after this point will get
693 * remapped to it directly.
694 */
695 r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block);
696 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -0500697 metadata_operation_failed(pool, "dm_thin_insert_block", r);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000698 cell_error(pool, m->cell);
Joe Thornber905386f2012-07-27 15:08:05 +0100699 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000700 }
701
702 /*
703 * Release any bios held while the block was being provisioned.
704 * If we are processing a write bio that completely covers the block,
705 * we already processed it so can ignore it now when processing
706 * the bios in the cell.
707 */
708 if (bio) {
Joe Thornberf286ba02012-12-21 20:23:33 +0000709 cell_defer_no_holder(tc, m->cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000710 bio_endio(bio, 0);
711 } else
Joe Thornber2aab3852012-12-21 20:23:33 +0000712 cell_defer(tc, m->cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000713
Joe Thornber905386f2012-07-27 15:08:05 +0100714out:
Joe Thornber991d9fa2011-10-31 20:21:18 +0000715 list_del(&m->list);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000716 mempool_free(m, pool->mapping_pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000717}
718
Joe Thornbere49e5822012-07-27 15:08:16 +0100719static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
Joe Thornber104655f2012-03-28 18:41:28 +0100720{
Joe Thornber104655f2012-03-28 18:41:28 +0100721 struct thin_c *tc = m->tc;
722
Joe Thornbere49e5822012-07-27 15:08:16 +0100723 bio_io_error(m->bio);
Joe Thornberf286ba02012-12-21 20:23:33 +0000724 cell_defer_no_holder(tc, m->cell);
725 cell_defer_no_holder(tc, m->cell2);
Joe Thornbere49e5822012-07-27 15:08:16 +0100726 mempool_free(m, tc->pool->mapping_pool);
727}
Joe Thornber104655f2012-03-28 18:41:28 +0100728
Joe Thornbere49e5822012-07-27 15:08:16 +0100729static void process_prepared_discard_passdown(struct dm_thin_new_mapping *m)
730{
731 struct thin_c *tc = m->tc;
732
Joe Thornbere8088072012-12-21 20:23:31 +0000733 inc_all_io_entry(tc->pool, m->bio);
Joe Thornberf286ba02012-12-21 20:23:33 +0000734 cell_defer_no_holder(tc, m->cell);
735 cell_defer_no_holder(tc, m->cell2);
Joe Thornbere8088072012-12-21 20:23:31 +0000736
Joe Thornber104655f2012-03-28 18:41:28 +0100737 if (m->pass_discard)
Joe Thornber19fa1a62013-12-17 12:09:40 -0500738 if (m->definitely_not_shared)
739 remap_and_issue(tc, m->bio, m->data_block);
740 else {
741 bool used = false;
742 if (dm_pool_block_is_used(tc->pool->pmd, m->data_block, &used) || used)
743 bio_endio(m->bio, 0);
744 else
745 remap_and_issue(tc, m->bio, m->data_block);
746 }
Joe Thornber104655f2012-03-28 18:41:28 +0100747 else
748 bio_endio(m->bio, 0);
749
Joe Thornber104655f2012-03-28 18:41:28 +0100750 mempool_free(m, tc->pool->mapping_pool);
751}
752
Joe Thornbere49e5822012-07-27 15:08:16 +0100753static void process_prepared_discard(struct dm_thin_new_mapping *m)
754{
755 int r;
756 struct thin_c *tc = m->tc;
757
758 r = dm_thin_remove_block(tc->td, m->virt_block);
759 if (r)
Mike Snitzerc3977412012-12-21 20:23:34 +0000760 DMERR_LIMIT("dm_thin_remove_block() failed");
Joe Thornbere49e5822012-07-27 15:08:16 +0100761
762 process_prepared_discard_passdown(m);
763}
764
Joe Thornber104655f2012-03-28 18:41:28 +0100765static void process_prepared(struct pool *pool, struct list_head *head,
Joe Thornbere49e5822012-07-27 15:08:16 +0100766 process_mapping_fn *fn)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000767{
768 unsigned long flags;
769 struct list_head maps;
Mike Snitzera24c2562012-06-03 00:30:00 +0100770 struct dm_thin_new_mapping *m, *tmp;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000771
772 INIT_LIST_HEAD(&maps);
773 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +0100774 list_splice_init(head, &maps);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000775 spin_unlock_irqrestore(&pool->lock, flags);
776
777 list_for_each_entry_safe(m, tmp, &maps, list)
Joe Thornbere49e5822012-07-27 15:08:16 +0100778 (*fn)(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000779}
780
781/*
782 * Deferred bio jobs.
783 */
Joe Thornber104655f2012-03-28 18:41:28 +0100784static int io_overlaps_block(struct pool *pool, struct bio *bio)
785{
Kent Overstreet4f024f32013-10-11 15:44:27 -0700786 return bio->bi_iter.bi_size ==
787 (pool->sectors_per_block << SECTOR_SHIFT);
Joe Thornber104655f2012-03-28 18:41:28 +0100788}
789
Joe Thornber991d9fa2011-10-31 20:21:18 +0000790static int io_overwrites_block(struct pool *pool, struct bio *bio)
791{
Joe Thornber104655f2012-03-28 18:41:28 +0100792 return (bio_data_dir(bio) == WRITE) &&
793 io_overlaps_block(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000794}
795
796static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
797 bio_end_io_t *fn)
798{
799 *save = bio->bi_end_io;
800 bio->bi_end_io = fn;
801}
802
803static int ensure_next_mapping(struct pool *pool)
804{
805 if (pool->next_mapping)
806 return 0;
807
808 pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
809
810 return pool->next_mapping ? 0 : -ENOMEM;
811}
812
Mike Snitzera24c2562012-06-03 00:30:00 +0100813static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000814{
Mike Snitzer16961b02013-12-17 13:19:11 -0500815 struct dm_thin_new_mapping *m = pool->next_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000816
817 BUG_ON(!pool->next_mapping);
818
Mike Snitzer16961b02013-12-17 13:19:11 -0500819 memset(m, 0, sizeof(struct dm_thin_new_mapping));
820 INIT_LIST_HEAD(&m->list);
821 m->bio = NULL;
822
Joe Thornber991d9fa2011-10-31 20:21:18 +0000823 pool->next_mapping = NULL;
824
Mike Snitzer16961b02013-12-17 13:19:11 -0500825 return m;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000826}
827
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100828static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
829 sector_t begin, sector_t end)
830{
831 int r;
832 struct dm_io_region to;
833
834 to.bdev = tc->pool_dev->bdev;
835 to.sector = begin;
836 to.count = end - begin;
837
838 r = dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);
839 if (r < 0) {
840 DMERR_LIMIT("dm_kcopyd_zero() failed");
841 copy_complete(1, 1, m);
842 }
843}
844
845/*
846 * A partial copy also needs to zero the uncopied region.
847 */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000848static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
Joe Thornber2dd9c252012-03-28 18:41:28 +0100849 struct dm_dev *origin, dm_block_t data_origin,
850 dm_block_t data_dest,
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100851 struct dm_bio_prison_cell *cell, struct bio *bio,
852 sector_t len)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000853{
854 int r;
855 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +0100856 struct dm_thin_new_mapping *m = get_next_mapping(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000857
Joe Thornber991d9fa2011-10-31 20:21:18 +0000858 m->tc = tc;
859 m->virt_block = virt_block;
860 m->data_block = data_dest;
861 m->cell = cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000862
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100863 /*
864 * quiesce action + copy action + an extra reference held for the
865 * duration of this function (we may need to inc later for a
866 * partial zero).
867 */
868 atomic_set(&m->prepare_actions, 3);
869
Mike Snitzer44feb382012-10-12 21:02:10 +0100870 if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100871 complete_mapping_preparation(m); /* already quiesced */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000872
873 /*
874 * IO to pool_dev remaps to the pool target's data_dev.
875 *
876 * If the whole block of data is being overwritten, we can issue the
877 * bio immediately. Otherwise we use kcopyd to clone the data first.
878 */
879 if (io_overwrites_block(pool, bio)) {
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +0000880 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Mike Snitzera24c2562012-06-03 00:30:00 +0100881
Joe Thornbereb2aa482012-03-28 18:41:28 +0100882 h->overwrite_mapping = m;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000883 m->bio = bio;
884 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
Joe Thornbere8088072012-12-21 20:23:31 +0000885 inc_all_io_entry(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000886 remap_and_issue(tc, bio, data_dest);
887 } else {
888 struct dm_io_region from, to;
889
Joe Thornber2dd9c252012-03-28 18:41:28 +0100890 from.bdev = origin->bdev;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000891 from.sector = data_origin * pool->sectors_per_block;
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100892 from.count = len;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000893
894 to.bdev = tc->pool_dev->bdev;
895 to.sector = data_dest * pool->sectors_per_block;
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100896 to.count = len;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000897
898 r = dm_kcopyd_copy(pool->copier, &from, 1, &to,
899 0, copy_complete, m);
900 if (r < 0) {
Mike Snitzerc3977412012-12-21 20:23:34 +0000901 DMERR_LIMIT("dm_kcopyd_copy() failed");
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100902 copy_complete(1, 1, m);
903
904 /*
905 * We allow the zero to be issued, to simplify the
906 * error path. Otherwise we'd need to start
907 * worrying about decrementing the prepare_actions
908 * counter.
909 */
910 }
911
912 /*
913 * Do we need to zero a tail region?
914 */
915 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
916 atomic_inc(&m->prepare_actions);
917 ll_zero(tc, m,
918 data_dest * pool->sectors_per_block + len,
919 (data_dest + 1) * pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000920 }
921 }
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100922
923 complete_mapping_preparation(m); /* drop our ref */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000924}
925
Joe Thornber2dd9c252012-03-28 18:41:28 +0100926static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
927 dm_block_t data_origin, dm_block_t data_dest,
Mike Snitzera24c2562012-06-03 00:30:00 +0100928 struct dm_bio_prison_cell *cell, struct bio *bio)
Joe Thornber2dd9c252012-03-28 18:41:28 +0100929{
930 schedule_copy(tc, virt_block, tc->pool_dev,
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100931 data_origin, data_dest, cell, bio,
932 tc->pool->sectors_per_block);
Joe Thornber2dd9c252012-03-28 18:41:28 +0100933}
934
Joe Thornber991d9fa2011-10-31 20:21:18 +0000935static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
Mike Snitzera24c2562012-06-03 00:30:00 +0100936 dm_block_t data_block, struct dm_bio_prison_cell *cell,
Joe Thornber991d9fa2011-10-31 20:21:18 +0000937 struct bio *bio)
938{
939 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +0100940 struct dm_thin_new_mapping *m = get_next_mapping(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000941
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100942 atomic_set(&m->prepare_actions, 1); /* no need to quiesce */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000943 m->tc = tc;
944 m->virt_block = virt_block;
945 m->data_block = data_block;
946 m->cell = cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000947
948 /*
949 * If the whole block of data is being overwritten or we are not
950 * zeroing pre-existing data, we can issue the bio immediately.
951 * Otherwise we use kcopyd to zero the data first.
952 */
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100953 if (!pool->pf.zero_new_blocks)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000954 process_prepared_mapping(m);
955
956 else if (io_overwrites_block(pool, bio)) {
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +0000957 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Mike Snitzera24c2562012-06-03 00:30:00 +0100958
Joe Thornbereb2aa482012-03-28 18:41:28 +0100959 h->overwrite_mapping = m;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000960 m->bio = bio;
961 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
Joe Thornbere8088072012-12-21 20:23:31 +0000962 inc_all_io_entry(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000963 remap_and_issue(tc, bio, data_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000964
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100965 } else
966 ll_zero(tc, m,
967 data_block * pool->sectors_per_block,
968 (data_block + 1) * pool->sectors_per_block);
969}
Joe Thornber991d9fa2011-10-31 20:21:18 +0000970
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100971static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
972 dm_block_t data_dest,
973 struct dm_bio_prison_cell *cell, struct bio *bio)
974{
975 struct pool *pool = tc->pool;
976 sector_t virt_block_begin = virt_block * pool->sectors_per_block;
977 sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
978
979 if (virt_block_end <= tc->origin_size)
980 schedule_copy(tc, virt_block, tc->origin_dev,
981 virt_block, data_dest, cell, bio,
982 pool->sectors_per_block);
983
984 else if (virt_block_begin < tc->origin_size)
985 schedule_copy(tc, virt_block, tc->origin_dev,
986 virt_block, data_dest, cell, bio,
987 tc->origin_size - virt_block_begin);
988
989 else
990 schedule_zero(tc, virt_block, data_dest, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000991}
992
Joe Thornbere49e5822012-07-27 15:08:16 +0100993/*
994 * A non-zero return indicates read_only or fail_io mode.
995 * Many callers don't care about the return value.
996 */
Joe Thornber020cc3b2013-12-04 15:05:36 -0500997static int commit(struct pool *pool)
Joe Thornbere49e5822012-07-27 15:08:16 +0100998{
999 int r;
1000
Joe Thornber8d07e8a2014-05-06 16:28:14 +01001001 if (get_pool_mode(pool) >= PM_READ_ONLY)
Joe Thornbere49e5822012-07-27 15:08:16 +01001002 return -EINVAL;
1003
Joe Thornber020cc3b2013-12-04 15:05:36 -05001004 r = dm_pool_commit_metadata(pool->pmd);
Joe Thornberb5330652013-12-04 19:51:33 -05001005 if (r)
1006 metadata_operation_failed(pool, "dm_pool_commit_metadata", r);
Joe Thornbere49e5822012-07-27 15:08:16 +01001007
1008 return r;
1009}
1010
Joe Thornber88a66212013-12-04 20:16:12 -05001011static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
1012{
1013 unsigned long flags;
1014
1015 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1016 DMWARN("%s: reached low water mark for data device: sending event.",
1017 dm_device_name(pool->pool_md));
1018 spin_lock_irqsave(&pool->lock, flags);
1019 pool->low_water_triggered = true;
1020 spin_unlock_irqrestore(&pool->lock, flags);
1021 dm_table_event(pool->ti->table);
1022 }
1023}
1024
Joe Thornber3e1a0692014-03-03 16:03:26 +00001025static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
1026
Joe Thornber991d9fa2011-10-31 20:21:18 +00001027static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1028{
1029 int r;
1030 dm_block_t free_blocks;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001031 struct pool *pool = tc->pool;
1032
Joe Thornber3e1a0692014-03-03 16:03:26 +00001033 if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
Joe Thornber8d30abf2013-12-04 19:16:11 -05001034 return -EINVAL;
1035
Joe Thornber991d9fa2011-10-31 20:21:18 +00001036 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
Joe Thornberb5330652013-12-04 19:51:33 -05001037 if (r) {
1038 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001039 return r;
Joe Thornberb5330652013-12-04 19:51:33 -05001040 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001041
Joe Thornber88a66212013-12-04 20:16:12 -05001042 check_low_water_mark(pool, free_blocks);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001043
1044 if (!free_blocks) {
Mike Snitzer94563ba2013-08-22 09:56:18 -04001045 /*
1046 * Try to commit to see if that will free up some
1047 * more space.
1048 */
Joe Thornber020cc3b2013-12-04 15:05:36 -05001049 r = commit(pool);
1050 if (r)
1051 return r;
Mike Snitzer94563ba2013-08-22 09:56:18 -04001052
1053 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
Joe Thornberb5330652013-12-04 19:51:33 -05001054 if (r) {
1055 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
Mike Snitzer94563ba2013-08-22 09:56:18 -04001056 return r;
Joe Thornberb5330652013-12-04 19:51:33 -05001057 }
Mike Snitzer94563ba2013-08-22 09:56:18 -04001058
Mike Snitzer94563ba2013-08-22 09:56:18 -04001059 if (!free_blocks) {
Joe Thornber3e1a0692014-03-03 16:03:26 +00001060 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001061 return -ENOSPC;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001062 }
1063 }
1064
1065 r = dm_pool_alloc_data_block(pool->pmd, result);
Mike Snitzer4a02b342013-12-03 12:20:57 -05001066 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05001067 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001068 return r;
Mike Snitzer4a02b342013-12-03 12:20:57 -05001069 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001070
1071 return 0;
1072}
1073
1074/*
1075 * If we have run out of space, queue bios until the device is
1076 * resumed, presumably after having been reloaded with more space.
1077 */
1078static void retry_on_resume(struct bio *bio)
1079{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001080 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01001081 struct thin_c *tc = h->tc;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001082 unsigned long flags;
1083
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001084 spin_lock_irqsave(&tc->lock, flags);
1085 bio_list_add(&tc->retry_on_resume_list, bio);
1086 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001087}
1088
Mike Snitzeraf918052014-05-22 14:32:51 -04001089static int should_error_unserviceable_bio(struct pool *pool)
Joe Thornber3e1a0692014-03-03 16:03:26 +00001090{
1091 enum pool_mode m = get_pool_mode(pool);
1092
1093 switch (m) {
1094 case PM_WRITE:
1095 /* Shouldn't get here */
1096 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
Mike Snitzeraf918052014-05-22 14:32:51 -04001097 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001098
1099 case PM_OUT_OF_DATA_SPACE:
Mike Snitzeraf918052014-05-22 14:32:51 -04001100 return pool->pf.error_if_no_space ? -ENOSPC : 0;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001101
1102 case PM_READ_ONLY:
1103 case PM_FAIL:
Mike Snitzeraf918052014-05-22 14:32:51 -04001104 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001105 default:
1106 /* Shouldn't get here */
1107 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
Mike Snitzeraf918052014-05-22 14:32:51 -04001108 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001109 }
1110}
1111
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001112static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
1113{
Mike Snitzeraf918052014-05-22 14:32:51 -04001114 int error = should_error_unserviceable_bio(pool);
1115
1116 if (error)
1117 bio_endio(bio, error);
Mike Snitzer6d162022013-12-20 18:09:02 -05001118 else
1119 retry_on_resume(bio);
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001120}
1121
Mike Snitzer399cadd2013-12-05 16:03:33 -05001122static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001123{
1124 struct bio *bio;
1125 struct bio_list bios;
Mike Snitzeraf918052014-05-22 14:32:51 -04001126 int error;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001127
Mike Snitzeraf918052014-05-22 14:32:51 -04001128 error = should_error_unserviceable_bio(pool);
1129 if (error) {
1130 cell_error_with_code(pool, cell, error);
Joe Thornber3e1a0692014-03-03 16:03:26 +00001131 return;
1132 }
1133
Joe Thornber991d9fa2011-10-31 20:21:18 +00001134 bio_list_init(&bios);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001135 cell_release(pool, cell, &bios);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001136
Mike Snitzeraf918052014-05-22 14:32:51 -04001137 error = should_error_unserviceable_bio(pool);
1138 if (error)
Joe Thornber3e1a0692014-03-03 16:03:26 +00001139 while ((bio = bio_list_pop(&bios)))
Mike Snitzeraf918052014-05-22 14:32:51 -04001140 bio_endio(bio, error);
Joe Thornber3e1a0692014-03-03 16:03:26 +00001141 else
1142 while ((bio = bio_list_pop(&bios)))
1143 retry_on_resume(bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001144}
1145
Joe Thornber104655f2012-03-28 18:41:28 +01001146static void process_discard(struct thin_c *tc, struct bio *bio)
1147{
1148 int r;
Mike Snitzerc3a0ce22012-05-12 01:43:16 +01001149 unsigned long flags;
Joe Thornber104655f2012-03-28 18:41:28 +01001150 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +01001151 struct dm_bio_prison_cell *cell, *cell2;
Mike Snitzer44feb382012-10-12 21:02:10 +01001152 struct dm_cell_key key, key2;
Joe Thornber104655f2012-03-28 18:41:28 +01001153 dm_block_t block = get_bio_block(tc, bio);
1154 struct dm_thin_lookup_result lookup_result;
Mike Snitzera24c2562012-06-03 00:30:00 +01001155 struct dm_thin_new_mapping *m;
Joe Thornber104655f2012-03-28 18:41:28 +01001156
1157 build_virtual_key(tc->td, block, &key);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001158 if (bio_detain(tc->pool, &key, bio, &cell))
Joe Thornber104655f2012-03-28 18:41:28 +01001159 return;
1160
1161 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1162 switch (r) {
1163 case 0:
1164 /*
1165 * Check nobody is fiddling with this pool block. This can
1166 * happen if someone's in the process of breaking sharing
1167 * on this block.
1168 */
1169 build_data_key(tc->td, lookup_result.block, &key2);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001170 if (bio_detain(tc->pool, &key2, bio, &cell2)) {
Joe Thornberf286ba02012-12-21 20:23:33 +00001171 cell_defer_no_holder(tc, cell);
Joe Thornber104655f2012-03-28 18:41:28 +01001172 break;
1173 }
1174
1175 if (io_overlaps_block(pool, bio)) {
1176 /*
1177 * IO may still be going to the destination block. We must
1178 * quiesce before we can do the removal.
1179 */
1180 m = get_next_mapping(pool);
1181 m->tc = tc;
Joe Thornber19fa1a62013-12-17 12:09:40 -05001182 m->pass_discard = pool->pf.discard_passdown;
1183 m->definitely_not_shared = !lookup_result.shared;
Joe Thornber104655f2012-03-28 18:41:28 +01001184 m->virt_block = block;
1185 m->data_block = lookup_result.block;
1186 m->cell = cell;
1187 m->cell2 = cell2;
Joe Thornber104655f2012-03-28 18:41:28 +01001188 m->bio = bio;
1189
Mike Snitzer44feb382012-10-12 21:02:10 +01001190 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) {
Mike Snitzerc3a0ce22012-05-12 01:43:16 +01001191 spin_lock_irqsave(&pool->lock, flags);
Mike Snitzerdaec3382013-12-11 14:01:20 -05001192 list_add_tail(&m->list, &pool->prepared_discards);
Mike Snitzerc3a0ce22012-05-12 01:43:16 +01001193 spin_unlock_irqrestore(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +01001194 wake_worker(pool);
1195 }
1196 } else {
Joe Thornbere8088072012-12-21 20:23:31 +00001197 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001198 cell_defer_no_holder(tc, cell);
1199 cell_defer_no_holder(tc, cell2);
Joe Thornbere8088072012-12-21 20:23:31 +00001200
Joe Thornber104655f2012-03-28 18:41:28 +01001201 /*
Mikulas Patocka49296302012-07-27 15:08:03 +01001202 * The DM core makes sure that the discard doesn't span
1203 * a block boundary. So we submit the discard of a
1204 * partial block appropriately.
Joe Thornber104655f2012-03-28 18:41:28 +01001205 */
Mikulas Patocka650d2a02012-07-20 14:25:05 +01001206 if ((!lookup_result.shared) && pool->pf.discard_passdown)
1207 remap_and_issue(tc, bio, lookup_result.block);
1208 else
1209 bio_endio(bio, 0);
Joe Thornber104655f2012-03-28 18:41:28 +01001210 }
1211 break;
1212
1213 case -ENODATA:
1214 /*
1215 * It isn't provisioned, just forget it.
1216 */
Joe Thornberf286ba02012-12-21 20:23:33 +00001217 cell_defer_no_holder(tc, cell);
Joe Thornber104655f2012-03-28 18:41:28 +01001218 bio_endio(bio, 0);
1219 break;
1220
1221 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001222 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1223 __func__, r);
Joe Thornberf286ba02012-12-21 20:23:33 +00001224 cell_defer_no_holder(tc, cell);
Joe Thornber104655f2012-03-28 18:41:28 +01001225 bio_io_error(bio);
1226 break;
1227 }
1228}
1229
Joe Thornber991d9fa2011-10-31 20:21:18 +00001230static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
Mike Snitzer44feb382012-10-12 21:02:10 +01001231 struct dm_cell_key *key,
Joe Thornber991d9fa2011-10-31 20:21:18 +00001232 struct dm_thin_lookup_result *lookup_result,
Mike Snitzera24c2562012-06-03 00:30:00 +01001233 struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001234{
1235 int r;
1236 dm_block_t data_block;
Mike Snitzerd6fc2042013-08-21 17:40:11 -04001237 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001238
1239 r = alloc_data_block(tc, &data_block);
1240 switch (r) {
1241 case 0:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001242 schedule_internal_copy(tc, block, lookup_result->block,
1243 data_block, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001244 break;
1245
1246 case -ENOSPC:
Mike Snitzer399cadd2013-12-05 16:03:33 -05001247 retry_bios_on_resume(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001248 break;
1249
1250 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001251 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1252 __func__, r);
Mike Snitzerd6fc2042013-08-21 17:40:11 -04001253 cell_error(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001254 break;
1255 }
1256}
1257
1258static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1259 dm_block_t block,
1260 struct dm_thin_lookup_result *lookup_result)
1261{
Mike Snitzera24c2562012-06-03 00:30:00 +01001262 struct dm_bio_prison_cell *cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001263 struct pool *pool = tc->pool;
Mike Snitzer44feb382012-10-12 21:02:10 +01001264 struct dm_cell_key key;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001265
1266 /*
1267 * If cell is already occupied, then sharing is already in the process
1268 * of being broken so we have nothing further to do here.
1269 */
1270 build_data_key(tc->td, lookup_result->block, &key);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001271 if (bio_detain(pool, &key, bio, &cell))
Joe Thornber991d9fa2011-10-31 20:21:18 +00001272 return;
1273
Kent Overstreet4f024f32013-10-11 15:44:27 -07001274 if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001275 break_sharing(tc, bio, block, &key, lookup_result, cell);
1276 else {
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001277 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornber991d9fa2011-10-31 20:21:18 +00001278
Mike Snitzer44feb382012-10-12 21:02:10 +01001279 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
Joe Thornbere8088072012-12-21 20:23:31 +00001280 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001281 cell_defer_no_holder(tc, cell);
Joe Thornbere8088072012-12-21 20:23:31 +00001282
Joe Thornber991d9fa2011-10-31 20:21:18 +00001283 remap_and_issue(tc, bio, lookup_result->block);
1284 }
1285}
1286
1287static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
Mike Snitzera24c2562012-06-03 00:30:00 +01001288 struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001289{
1290 int r;
1291 dm_block_t data_block;
Joe Thornber6beca5e2013-03-01 22:45:50 +00001292 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001293
1294 /*
1295 * Remap empty bios (flushes) immediately, without provisioning.
1296 */
Kent Overstreet4f024f32013-10-11 15:44:27 -07001297 if (!bio->bi_iter.bi_size) {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001298 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001299 cell_defer_no_holder(tc, cell);
Joe Thornbere8088072012-12-21 20:23:31 +00001300
Joe Thornber991d9fa2011-10-31 20:21:18 +00001301 remap_and_issue(tc, bio, 0);
1302 return;
1303 }
1304
1305 /*
1306 * Fill read bios with zeroes and complete them immediately.
1307 */
1308 if (bio_data_dir(bio) == READ) {
1309 zero_fill_bio(bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001310 cell_defer_no_holder(tc, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001311 bio_endio(bio, 0);
1312 return;
1313 }
1314
1315 r = alloc_data_block(tc, &data_block);
1316 switch (r) {
1317 case 0:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001318 if (tc->origin_dev)
1319 schedule_external_copy(tc, block, data_block, cell, bio);
1320 else
1321 schedule_zero(tc, block, data_block, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001322 break;
1323
1324 case -ENOSPC:
Mike Snitzer399cadd2013-12-05 16:03:33 -05001325 retry_bios_on_resume(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001326 break;
1327
1328 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001329 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1330 __func__, r);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001331 cell_error(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001332 break;
1333 }
1334}
1335
1336static void process_bio(struct thin_c *tc, struct bio *bio)
1337{
1338 int r;
Joe Thornber6beca5e2013-03-01 22:45:50 +00001339 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001340 dm_block_t block = get_bio_block(tc, bio);
Mike Snitzera24c2562012-06-03 00:30:00 +01001341 struct dm_bio_prison_cell *cell;
Mike Snitzer44feb382012-10-12 21:02:10 +01001342 struct dm_cell_key key;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001343 struct dm_thin_lookup_result lookup_result;
1344
1345 /*
1346 * If cell is already occupied, then the block is already
1347 * being provisioned so we have nothing further to do here.
1348 */
1349 build_virtual_key(tc->td, block, &key);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001350 if (bio_detain(pool, &key, bio, &cell))
Joe Thornber991d9fa2011-10-31 20:21:18 +00001351 return;
1352
1353 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1354 switch (r) {
1355 case 0:
Joe Thornbere8088072012-12-21 20:23:31 +00001356 if (lookup_result.shared) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001357 process_shared_bio(tc, bio, block, &lookup_result);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001358 cell_defer_no_holder(tc, cell); /* FIXME: pass this cell into process_shared? */
Joe Thornbere8088072012-12-21 20:23:31 +00001359 } else {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001360 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001361 cell_defer_no_holder(tc, cell);
Joe Thornbere8088072012-12-21 20:23:31 +00001362
Joe Thornber991d9fa2011-10-31 20:21:18 +00001363 remap_and_issue(tc, bio, lookup_result.block);
Joe Thornbere8088072012-12-21 20:23:31 +00001364 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001365 break;
1366
1367 case -ENODATA:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001368 if (bio_data_dir(bio) == READ && tc->origin_dev) {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001369 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001370 cell_defer_no_holder(tc, cell);
Joe Thornbere8088072012-12-21 20:23:31 +00001371
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001372 if (bio_end_sector(bio) <= tc->origin_size)
1373 remap_to_origin_and_issue(tc, bio);
1374
1375 else if (bio->bi_iter.bi_sector < tc->origin_size) {
1376 zero_fill_bio(bio);
1377 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
1378 remap_to_origin_and_issue(tc, bio);
1379
1380 } else {
1381 zero_fill_bio(bio);
1382 bio_endio(bio, 0);
1383 }
Joe Thornber2dd9c252012-03-28 18:41:28 +01001384 } else
1385 provision_block(tc, bio, block, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001386 break;
1387
1388 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001389 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1390 __func__, r);
Joe Thornberf286ba02012-12-21 20:23:33 +00001391 cell_defer_no_holder(tc, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001392 bio_io_error(bio);
1393 break;
1394 }
1395}
1396
Joe Thornbere49e5822012-07-27 15:08:16 +01001397static void process_bio_read_only(struct thin_c *tc, struct bio *bio)
1398{
1399 int r;
1400 int rw = bio_data_dir(bio);
1401 dm_block_t block = get_bio_block(tc, bio);
1402 struct dm_thin_lookup_result lookup_result;
1403
1404 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1405 switch (r) {
1406 case 0:
Kent Overstreet4f024f32013-10-11 15:44:27 -07001407 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size)
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001408 handle_unserviceable_bio(tc->pool, bio);
Joe Thornbere8088072012-12-21 20:23:31 +00001409 else {
1410 inc_all_io_entry(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001411 remap_and_issue(tc, bio, lookup_result.block);
Joe Thornbere8088072012-12-21 20:23:31 +00001412 }
Joe Thornbere49e5822012-07-27 15:08:16 +01001413 break;
1414
1415 case -ENODATA:
1416 if (rw != READ) {
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001417 handle_unserviceable_bio(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001418 break;
1419 }
1420
1421 if (tc->origin_dev) {
Joe Thornbere8088072012-12-21 20:23:31 +00001422 inc_all_io_entry(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001423 remap_to_origin_and_issue(tc, bio);
1424 break;
1425 }
1426
1427 zero_fill_bio(bio);
1428 bio_endio(bio, 0);
1429 break;
1430
1431 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001432 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1433 __func__, r);
Joe Thornbere49e5822012-07-27 15:08:16 +01001434 bio_io_error(bio);
1435 break;
1436 }
1437}
1438
Joe Thornber3e1a0692014-03-03 16:03:26 +00001439static void process_bio_success(struct thin_c *tc, struct bio *bio)
1440{
1441 bio_endio(bio, 0);
1442}
1443
Joe Thornbere49e5822012-07-27 15:08:16 +01001444static void process_bio_fail(struct thin_c *tc, struct bio *bio)
1445{
1446 bio_io_error(bio);
1447}
1448
Joe Thornberac8c3f32013-05-10 14:37:21 +01001449/*
1450 * FIXME: should we also commit due to size of transaction, measured in
1451 * metadata blocks?
1452 */
Joe Thornber905e51b2012-03-28 18:41:27 +01001453static int need_commit_due_to_time(struct pool *pool)
1454{
1455 return jiffies < pool->last_commit_jiffies ||
1456 jiffies > pool->last_commit_jiffies + COMMIT_PERIOD;
1457}
1458
Mike Snitzer67324ea2014-03-21 18:33:41 -04001459#define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
1460#define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
1461
1462static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)
1463{
1464 struct rb_node **rbp, *parent;
1465 struct dm_thin_endio_hook *pbd;
1466 sector_t bi_sector = bio->bi_iter.bi_sector;
1467
1468 rbp = &tc->sort_bio_list.rb_node;
1469 parent = NULL;
1470 while (*rbp) {
1471 parent = *rbp;
1472 pbd = thin_pbd(parent);
1473
1474 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)
1475 rbp = &(*rbp)->rb_left;
1476 else
1477 rbp = &(*rbp)->rb_right;
1478 }
1479
1480 pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1481 rb_link_node(&pbd->rb_node, parent, rbp);
1482 rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);
1483}
1484
1485static void __extract_sorted_bios(struct thin_c *tc)
1486{
1487 struct rb_node *node;
1488 struct dm_thin_endio_hook *pbd;
1489 struct bio *bio;
1490
1491 for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) {
1492 pbd = thin_pbd(node);
1493 bio = thin_bio(pbd);
1494
1495 bio_list_add(&tc->deferred_bio_list, bio);
1496 rb_erase(&pbd->rb_node, &tc->sort_bio_list);
1497 }
1498
1499 WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));
1500}
1501
1502static void __sort_thin_deferred_bios(struct thin_c *tc)
1503{
1504 struct bio *bio;
1505 struct bio_list bios;
1506
1507 bio_list_init(&bios);
1508 bio_list_merge(&bios, &tc->deferred_bio_list);
1509 bio_list_init(&tc->deferred_bio_list);
1510
1511 /* Sort deferred_bio_list using rb-tree */
1512 while ((bio = bio_list_pop(&bios)))
1513 __thin_bio_rb_add(tc, bio);
1514
1515 /*
1516 * Transfer the sorted bios in sort_bio_list back to
1517 * deferred_bio_list to allow lockless submission of
1518 * all bios.
1519 */
1520 __extract_sorted_bios(tc);
1521}
1522
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001523static void process_thin_deferred_bios(struct thin_c *tc)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001524{
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001525 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001526 unsigned long flags;
1527 struct bio *bio;
1528 struct bio_list bios;
Mike Snitzer67324ea2014-03-21 18:33:41 -04001529 struct blk_plug plug;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001530
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001531 if (tc->requeue_mode) {
1532 requeue_bio_list(tc, &tc->deferred_bio_list);
1533 return;
1534 }
1535
Joe Thornber991d9fa2011-10-31 20:21:18 +00001536 bio_list_init(&bios);
1537
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001538 spin_lock_irqsave(&tc->lock, flags);
Mike Snitzer67324ea2014-03-21 18:33:41 -04001539
1540 if (bio_list_empty(&tc->deferred_bio_list)) {
1541 spin_unlock_irqrestore(&tc->lock, flags);
1542 return;
1543 }
1544
1545 __sort_thin_deferred_bios(tc);
1546
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001547 bio_list_merge(&bios, &tc->deferred_bio_list);
1548 bio_list_init(&tc->deferred_bio_list);
Mike Snitzer67324ea2014-03-21 18:33:41 -04001549
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001550 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001551
Mike Snitzer67324ea2014-03-21 18:33:41 -04001552 blk_start_plug(&plug);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001553 while ((bio = bio_list_pop(&bios))) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001554 /*
1555 * If we've got no free new_mapping structs, and processing
1556 * this bio might require one, we pause until there are some
1557 * prepared mappings to process.
1558 */
1559 if (ensure_next_mapping(pool)) {
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001560 spin_lock_irqsave(&tc->lock, flags);
1561 bio_list_add(&tc->deferred_bio_list, bio);
1562 bio_list_merge(&tc->deferred_bio_list, &bios);
1563 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001564 break;
1565 }
Joe Thornber104655f2012-03-28 18:41:28 +01001566
1567 if (bio->bi_rw & REQ_DISCARD)
Joe Thornbere49e5822012-07-27 15:08:16 +01001568 pool->process_discard(tc, bio);
Joe Thornber104655f2012-03-28 18:41:28 +01001569 else
Joe Thornbere49e5822012-07-27 15:08:16 +01001570 pool->process_bio(tc, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001571 }
Mike Snitzer67324ea2014-03-21 18:33:41 -04001572 blk_finish_plug(&plug);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001573}
1574
Joe Thornberb10ebd32014-04-08 11:29:01 +01001575static void thin_get(struct thin_c *tc);
1576static void thin_put(struct thin_c *tc);
1577
1578/*
1579 * We can't hold rcu_read_lock() around code that can block. So we
1580 * find a thin with the rcu lock held; bump a refcount; then drop
1581 * the lock.
1582 */
1583static struct thin_c *get_first_thin(struct pool *pool)
1584{
1585 struct thin_c *tc = NULL;
1586
1587 rcu_read_lock();
1588 if (!list_empty(&pool->active_thins)) {
1589 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list);
1590 thin_get(tc);
1591 }
1592 rcu_read_unlock();
1593
1594 return tc;
1595}
1596
1597static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
1598{
1599 struct thin_c *old_tc = tc;
1600
1601 rcu_read_lock();
1602 list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
1603 thin_get(tc);
1604 thin_put(old_tc);
1605 rcu_read_unlock();
1606 return tc;
1607 }
1608 thin_put(old_tc);
1609 rcu_read_unlock();
1610
1611 return NULL;
1612}
1613
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001614static void process_deferred_bios(struct pool *pool)
1615{
1616 unsigned long flags;
1617 struct bio *bio;
1618 struct bio_list bios;
1619 struct thin_c *tc;
1620
Joe Thornberb10ebd32014-04-08 11:29:01 +01001621 tc = get_first_thin(pool);
1622 while (tc) {
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001623 process_thin_deferred_bios(tc);
Joe Thornberb10ebd32014-04-08 11:29:01 +01001624 tc = get_next_thin(pool, tc);
1625 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001626
1627 /*
1628 * If there are any deferred flush bios, we must commit
1629 * the metadata before issuing them.
1630 */
1631 bio_list_init(&bios);
1632 spin_lock_irqsave(&pool->lock, flags);
1633 bio_list_merge(&bios, &pool->deferred_flush_bios);
1634 bio_list_init(&pool->deferred_flush_bios);
1635 spin_unlock_irqrestore(&pool->lock, flags);
1636
Mike Snitzer4d1662a2014-02-06 06:08:56 -05001637 if (bio_list_empty(&bios) &&
1638 !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool)))
Joe Thornber991d9fa2011-10-31 20:21:18 +00001639 return;
1640
Joe Thornber020cc3b2013-12-04 15:05:36 -05001641 if (commit(pool)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001642 while ((bio = bio_list_pop(&bios)))
1643 bio_io_error(bio);
1644 return;
1645 }
Joe Thornber905e51b2012-03-28 18:41:27 +01001646 pool->last_commit_jiffies = jiffies;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001647
1648 while ((bio = bio_list_pop(&bios)))
1649 generic_make_request(bio);
1650}
1651
1652static void do_worker(struct work_struct *ws)
1653{
1654 struct pool *pool = container_of(ws, struct pool, worker);
1655
Joe Thornbere49e5822012-07-27 15:08:16 +01001656 process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);
1657 process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001658 process_deferred_bios(pool);
1659}
1660
Joe Thornber905e51b2012-03-28 18:41:27 +01001661/*
1662 * We want to commit periodically so that not too much
1663 * unwritten data builds up.
1664 */
1665static void do_waker(struct work_struct *ws)
1666{
1667 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
1668 wake_worker(pool);
1669 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
1670}
1671
Joe Thornber85ad643b2014-05-09 15:59:38 +01001672/*
1673 * We're holding onto IO to allow userland time to react. After the
1674 * timeout either the pool will have been resized (and thus back in
1675 * PM_WRITE mode), or we degrade to PM_READ_ONLY and start erroring IO.
1676 */
1677static void do_no_space_timeout(struct work_struct *ws)
1678{
1679 struct pool *pool = container_of(to_delayed_work(ws), struct pool,
1680 no_space_timeout);
1681
1682 if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space)
1683 set_pool_mode(pool, PM_READ_ONLY);
1684}
1685
Joe Thornber991d9fa2011-10-31 20:21:18 +00001686/*----------------------------------------------------------------*/
1687
Joe Thornbere7a3e872014-05-13 16:14:14 -04001688struct pool_work {
Joe Thornber738211f2014-03-03 15:52:28 +00001689 struct work_struct worker;
Joe Thornbere7a3e872014-05-13 16:14:14 -04001690 struct completion complete;
Joe Thornber738211f2014-03-03 15:52:28 +00001691};
1692
Joe Thornbere7a3e872014-05-13 16:14:14 -04001693static struct pool_work *to_pool_work(struct work_struct *ws)
Joe Thornber738211f2014-03-03 15:52:28 +00001694{
Joe Thornbere7a3e872014-05-13 16:14:14 -04001695 return container_of(ws, struct pool_work, worker);
1696}
1697
1698static void pool_work_complete(struct pool_work *pw)
1699{
1700 complete(&pw->complete);
1701}
1702
1703static void pool_work_wait(struct pool_work *pw, struct pool *pool,
1704 void (*fn)(struct work_struct *))
1705{
1706 INIT_WORK_ONSTACK(&pw->worker, fn);
1707 init_completion(&pw->complete);
1708 queue_work(pool->wq, &pw->worker);
1709 wait_for_completion(&pw->complete);
1710}
1711
1712/*----------------------------------------------------------------*/
1713
1714struct noflush_work {
1715 struct pool_work pw;
1716 struct thin_c *tc;
1717};
1718
1719static struct noflush_work *to_noflush(struct work_struct *ws)
1720{
1721 return container_of(to_pool_work(ws), struct noflush_work, pw);
Joe Thornber738211f2014-03-03 15:52:28 +00001722}
1723
1724static void do_noflush_start(struct work_struct *ws)
1725{
Joe Thornbere7a3e872014-05-13 16:14:14 -04001726 struct noflush_work *w = to_noflush(ws);
Joe Thornber738211f2014-03-03 15:52:28 +00001727 w->tc->requeue_mode = true;
1728 requeue_io(w->tc);
Joe Thornbere7a3e872014-05-13 16:14:14 -04001729 pool_work_complete(&w->pw);
Joe Thornber738211f2014-03-03 15:52:28 +00001730}
1731
1732static void do_noflush_stop(struct work_struct *ws)
1733{
Joe Thornbere7a3e872014-05-13 16:14:14 -04001734 struct noflush_work *w = to_noflush(ws);
Joe Thornber738211f2014-03-03 15:52:28 +00001735 w->tc->requeue_mode = false;
Joe Thornbere7a3e872014-05-13 16:14:14 -04001736 pool_work_complete(&w->pw);
Joe Thornber738211f2014-03-03 15:52:28 +00001737}
1738
1739static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
1740{
1741 struct noflush_work w;
1742
Joe Thornber738211f2014-03-03 15:52:28 +00001743 w.tc = tc;
Joe Thornbere7a3e872014-05-13 16:14:14 -04001744 pool_work_wait(&w.pw, tc->pool, fn);
Joe Thornber738211f2014-03-03 15:52:28 +00001745}
1746
1747/*----------------------------------------------------------------*/
1748
Joe Thornbere49e5822012-07-27 15:08:16 +01001749static enum pool_mode get_pool_mode(struct pool *pool)
1750{
1751 return pool->pf.mode;
1752}
1753
Joe Thornber3e1a0692014-03-03 16:03:26 +00001754static void notify_of_pool_mode_change(struct pool *pool, const char *new_mode)
1755{
1756 dm_table_event(pool->ti->table);
1757 DMINFO("%s: switching pool to %s mode",
1758 dm_device_name(pool->pool_md), new_mode);
1759}
1760
Mike Snitzer8b64e882013-12-20 14:27:28 -05001761static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
Joe Thornbere49e5822012-07-27 15:08:16 +01001762{
Mike Snitzercdc2b412014-02-14 18:10:55 -05001763 struct pool_c *pt = pool->ti->private;
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05001764 bool needs_check = dm_pool_metadata_needs_check(pool->pmd);
1765 enum pool_mode old_mode = get_pool_mode(pool);
Mike Snitzer80c57892014-05-20 13:38:33 -04001766 unsigned long no_space_timeout = ACCESS_ONCE(no_space_timeout_secs) * HZ;
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05001767
1768 /*
1769 * Never allow the pool to transition to PM_WRITE mode if user
1770 * intervention is required to verify metadata and data consistency.
1771 */
1772 if (new_mode == PM_WRITE && needs_check) {
1773 DMERR("%s: unable to switch pool to write mode until repaired.",
1774 dm_device_name(pool->pool_md));
1775 if (old_mode != new_mode)
1776 new_mode = old_mode;
1777 else
1778 new_mode = PM_READ_ONLY;
1779 }
1780 /*
1781 * If we were in PM_FAIL mode, rollback of metadata failed. We're
1782 * not going to recover without a thin_repair. So we never let the
1783 * pool move out of the old mode.
1784 */
1785 if (old_mode == PM_FAIL)
1786 new_mode = old_mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01001787
Mike Snitzer8b64e882013-12-20 14:27:28 -05001788 switch (new_mode) {
Joe Thornbere49e5822012-07-27 15:08:16 +01001789 case PM_FAIL:
Mike Snitzer8b64e882013-12-20 14:27:28 -05001790 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00001791 notify_of_pool_mode_change(pool, "failure");
Joe Thornber5383ef32013-12-04 16:30:01 -05001792 dm_pool_metadata_read_only(pool->pmd);
Joe Thornbere49e5822012-07-27 15:08:16 +01001793 pool->process_bio = process_bio_fail;
1794 pool->process_discard = process_bio_fail;
1795 pool->process_prepared_mapping = process_prepared_mapping_fail;
1796 pool->process_prepared_discard = process_prepared_discard_fail;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001797
1798 error_retry_list(pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01001799 break;
1800
1801 case PM_READ_ONLY:
Mike Snitzer8b64e882013-12-20 14:27:28 -05001802 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00001803 notify_of_pool_mode_change(pool, "read-only");
1804 dm_pool_metadata_read_only(pool->pmd);
1805 pool->process_bio = process_bio_read_only;
1806 pool->process_discard = process_bio_success;
1807 pool->process_prepared_mapping = process_prepared_mapping_fail;
1808 pool->process_prepared_discard = process_prepared_discard_passdown;
1809
1810 error_retry_list(pool);
1811 break;
1812
1813 case PM_OUT_OF_DATA_SPACE:
1814 /*
1815 * Ideally we'd never hit this state; the low water mark
1816 * would trigger userland to extend the pool before we
1817 * completely run out of data space. However, many small
1818 * IOs to unprovisioned space can consume data space at an
1819 * alarming rate. Adjust your low water mark if you're
1820 * frequently seeing this mode.
1821 */
1822 if (old_mode != new_mode)
1823 notify_of_pool_mode_change(pool, "out-of-data-space");
1824 pool->process_bio = process_bio_read_only;
1825 pool->process_discard = process_discard;
1826 pool->process_prepared_mapping = process_prepared_mapping;
1827 pool->process_prepared_discard = process_prepared_discard_passdown;
Joe Thornber85ad643b2014-05-09 15:59:38 +01001828
Mike Snitzer80c57892014-05-20 13:38:33 -04001829 if (!pool->pf.error_if_no_space && no_space_timeout)
1830 queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);
Joe Thornbere49e5822012-07-27 15:08:16 +01001831 break;
1832
1833 case PM_WRITE:
Mike Snitzer8b64e882013-12-20 14:27:28 -05001834 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00001835 notify_of_pool_mode_change(pool, "write");
Joe Thornber9b7aaa62013-12-04 16:58:19 -05001836 dm_pool_metadata_read_write(pool->pmd);
Joe Thornbere49e5822012-07-27 15:08:16 +01001837 pool->process_bio = process_bio;
1838 pool->process_discard = process_discard;
1839 pool->process_prepared_mapping = process_prepared_mapping;
1840 pool->process_prepared_discard = process_prepared_discard;
1841 break;
1842 }
Mike Snitzer8b64e882013-12-20 14:27:28 -05001843
1844 pool->pf.mode = new_mode;
Mike Snitzercdc2b412014-02-14 18:10:55 -05001845 /*
1846 * The pool mode may have changed, sync it so bind_control_target()
1847 * doesn't cause an unexpected mode transition on resume.
1848 */
1849 pt->adjusted_pf.mode = new_mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01001850}
1851
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05001852static void abort_transaction(struct pool *pool)
1853{
1854 const char *dev_name = dm_device_name(pool->pool_md);
1855
1856 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
1857 if (dm_pool_abort_metadata(pool->pmd)) {
1858 DMERR("%s: failed to abort metadata transaction", dev_name);
1859 set_pool_mode(pool, PM_FAIL);
1860 }
1861
1862 if (dm_pool_metadata_set_needs_check(pool->pmd)) {
1863 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
1864 set_pool_mode(pool, PM_FAIL);
1865 }
1866}
1867
Joe Thornberb5330652013-12-04 19:51:33 -05001868static void metadata_operation_failed(struct pool *pool, const char *op, int r)
1869{
1870 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
1871 dm_device_name(pool->pool_md), op, r);
1872
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05001873 abort_transaction(pool);
Joe Thornberb5330652013-12-04 19:51:33 -05001874 set_pool_mode(pool, PM_READ_ONLY);
1875}
1876
Joe Thornbere49e5822012-07-27 15:08:16 +01001877/*----------------------------------------------------------------*/
1878
Joe Thornber991d9fa2011-10-31 20:21:18 +00001879/*
1880 * Mapping functions.
1881 */
1882
1883/*
1884 * Called only while mapping a thin bio to hand it over to the workqueue.
1885 */
1886static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
1887{
1888 unsigned long flags;
1889 struct pool *pool = tc->pool;
1890
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001891 spin_lock_irqsave(&tc->lock, flags);
1892 bio_list_add(&tc->deferred_bio_list, bio);
1893 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001894
1895 wake_worker(pool);
1896}
1897
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001898static void thin_hook_bio(struct thin_c *tc, struct bio *bio)
Joe Thornbereb2aa482012-03-28 18:41:28 +01001899{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001900 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01001901
1902 h->tc = tc;
1903 h->shared_read_entry = NULL;
Joe Thornbere8088072012-12-21 20:23:31 +00001904 h->all_io_entry = NULL;
Joe Thornbereb2aa482012-03-28 18:41:28 +01001905 h->overwrite_mapping = NULL;
Joe Thornbereb2aa482012-03-28 18:41:28 +01001906}
1907
Joe Thornber991d9fa2011-10-31 20:21:18 +00001908/*
1909 * Non-blocking function called from the thin target's map function.
1910 */
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00001911static int thin_bio_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001912{
1913 int r;
1914 struct thin_c *tc = ti->private;
1915 dm_block_t block = get_bio_block(tc, bio);
1916 struct dm_thin_device *td = tc->td;
1917 struct dm_thin_lookup_result result;
Joe Thornber025b9682013-03-01 22:45:50 +00001918 struct dm_bio_prison_cell cell1, cell2;
1919 struct dm_bio_prison_cell *cell_result;
Joe Thornbere8088072012-12-21 20:23:31 +00001920 struct dm_cell_key key;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001921
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001922 thin_hook_bio(tc, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001923
Joe Thornber738211f2014-03-03 15:52:28 +00001924 if (tc->requeue_mode) {
1925 bio_endio(bio, DM_ENDIO_REQUEUE);
1926 return DM_MAPIO_SUBMITTED;
1927 }
1928
Joe Thornbere49e5822012-07-27 15:08:16 +01001929 if (get_pool_mode(tc->pool) == PM_FAIL) {
1930 bio_io_error(bio);
1931 return DM_MAPIO_SUBMITTED;
1932 }
1933
Joe Thornber104655f2012-03-28 18:41:28 +01001934 if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001935 thin_defer_bio(tc, bio);
1936 return DM_MAPIO_SUBMITTED;
1937 }
1938
1939 r = dm_thin_find_block(td, block, 0, &result);
1940
1941 /*
1942 * Note that we defer readahead too.
1943 */
1944 switch (r) {
1945 case 0:
1946 if (unlikely(result.shared)) {
1947 /*
1948 * We have a race condition here between the
1949 * result.shared value returned by the lookup and
1950 * snapshot creation, which may cause new
1951 * sharing.
1952 *
1953 * To avoid this always quiesce the origin before
1954 * taking the snap. You want to do this anyway to
1955 * ensure a consistent application view
1956 * (i.e. lockfs).
1957 *
1958 * More distant ancestors are irrelevant. The
1959 * shared flag will be set in their case.
1960 */
1961 thin_defer_bio(tc, bio);
Joe Thornbere8088072012-12-21 20:23:31 +00001962 return DM_MAPIO_SUBMITTED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001963 }
Joe Thornbere8088072012-12-21 20:23:31 +00001964
1965 build_virtual_key(tc->td, block, &key);
Joe Thornber025b9682013-03-01 22:45:50 +00001966 if (dm_bio_detain(tc->pool->prison, &key, bio, &cell1, &cell_result))
Joe Thornbere8088072012-12-21 20:23:31 +00001967 return DM_MAPIO_SUBMITTED;
1968
1969 build_data_key(tc->td, result.block, &key);
Joe Thornber025b9682013-03-01 22:45:50 +00001970 if (dm_bio_detain(tc->pool->prison, &key, bio, &cell2, &cell_result)) {
1971 cell_defer_no_holder_no_free(tc, &cell1);
Joe Thornbere8088072012-12-21 20:23:31 +00001972 return DM_MAPIO_SUBMITTED;
1973 }
1974
1975 inc_all_io_entry(tc->pool, bio);
Joe Thornber025b9682013-03-01 22:45:50 +00001976 cell_defer_no_holder_no_free(tc, &cell2);
1977 cell_defer_no_holder_no_free(tc, &cell1);
Joe Thornbere8088072012-12-21 20:23:31 +00001978
1979 remap(tc, bio, result.block);
1980 return DM_MAPIO_REMAPPED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001981
1982 case -ENODATA:
Joe Thornbere49e5822012-07-27 15:08:16 +01001983 if (get_pool_mode(tc->pool) == PM_READ_ONLY) {
1984 /*
1985 * This block isn't provisioned, and we have no way
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001986 * of doing so.
Joe Thornbere49e5822012-07-27 15:08:16 +01001987 */
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001988 handle_unserviceable_bio(tc->pool, bio);
Joe Thornber2aab3852012-12-21 20:23:33 +00001989 return DM_MAPIO_SUBMITTED;
Joe Thornbere49e5822012-07-27 15:08:16 +01001990 }
1991 /* fall through */
1992
1993 case -EWOULDBLOCK:
Joe Thornber991d9fa2011-10-31 20:21:18 +00001994 /*
1995 * In future, the failed dm_thin_find_block above could
1996 * provide the hint to load the metadata into cache.
1997 */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001998 thin_defer_bio(tc, bio);
Joe Thornber2aab3852012-12-21 20:23:33 +00001999 return DM_MAPIO_SUBMITTED;
Joe Thornbere49e5822012-07-27 15:08:16 +01002000
2001 default:
2002 /*
2003 * Must always call bio_io_error on failure.
2004 * dm_thin_find_block can fail with -EINVAL if the
2005 * pool is switched to fail-io mode.
2006 */
2007 bio_io_error(bio);
Joe Thornber2aab3852012-12-21 20:23:33 +00002008 return DM_MAPIO_SUBMITTED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002009 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002010}
2011
2012static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
2013{
Joe Thornber991d9fa2011-10-31 20:21:18 +00002014 struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
Mike Snitzer760fe672014-03-20 08:36:47 -04002015 struct request_queue *q;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002016
Mike Snitzer760fe672014-03-20 08:36:47 -04002017 if (get_pool_mode(pt->pool) == PM_OUT_OF_DATA_SPACE)
2018 return 1;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002019
Mike Snitzer760fe672014-03-20 08:36:47 -04002020 q = bdev_get_queue(pt->data_dev->bdev);
2021 return bdi_congested(&q->backing_dev_info, bdi_bits);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002022}
2023
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002024static void requeue_bios(struct pool *pool)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002025{
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002026 unsigned long flags;
2027 struct thin_c *tc;
2028
2029 rcu_read_lock();
2030 list_for_each_entry_rcu(tc, &pool->active_thins, list) {
2031 spin_lock_irqsave(&tc->lock, flags);
2032 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list);
2033 bio_list_init(&tc->retry_on_resume_list);
2034 spin_unlock_irqrestore(&tc->lock, flags);
2035 }
2036 rcu_read_unlock();
Joe Thornber991d9fa2011-10-31 20:21:18 +00002037}
2038
2039/*----------------------------------------------------------------
2040 * Binding of control targets to a pool object
2041 *--------------------------------------------------------------*/
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002042static bool data_dev_supports_discard(struct pool_c *pt)
2043{
2044 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2045
2046 return q && blk_queue_discard(q);
2047}
2048
Joe Thornber58051b92013-03-20 17:21:25 +00002049static bool is_factor(sector_t block_size, uint32_t n)
2050{
2051 return !sector_div(block_size, n);
2052}
2053
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002054/*
2055 * If discard_passdown was enabled verify that the data device
Mike Snitzer0424caa2012-09-26 23:45:47 +01002056 * supports discards. Disable discard_passdown if not.
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002057 */
Mike Snitzer0424caa2012-09-26 23:45:47 +01002058static void disable_passdown_if_not_supported(struct pool_c *pt)
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002059{
Mike Snitzer0424caa2012-09-26 23:45:47 +01002060 struct pool *pool = pt->pool;
2061 struct block_device *data_bdev = pt->data_dev->bdev;
2062 struct queue_limits *data_limits = &bdev_get_queue(data_bdev)->limits;
2063 sector_t block_size = pool->sectors_per_block << SECTOR_SHIFT;
2064 const char *reason = NULL;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002065 char buf[BDEVNAME_SIZE];
2066
Mike Snitzer0424caa2012-09-26 23:45:47 +01002067 if (!pt->adjusted_pf.discard_passdown)
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002068 return;
2069
Mike Snitzer0424caa2012-09-26 23:45:47 +01002070 if (!data_dev_supports_discard(pt))
2071 reason = "discard unsupported";
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002072
Mike Snitzer0424caa2012-09-26 23:45:47 +01002073 else if (data_limits->max_discard_sectors < pool->sectors_per_block)
2074 reason = "max discard sectors smaller than a block";
2075
2076 else if (data_limits->discard_granularity > block_size)
2077 reason = "discard granularity larger than a block";
2078
Joe Thornber58051b92013-03-20 17:21:25 +00002079 else if (!is_factor(block_size, data_limits->discard_granularity))
Mike Snitzer0424caa2012-09-26 23:45:47 +01002080 reason = "discard granularity not a factor of block size";
2081
2082 if (reason) {
2083 DMWARN("Data device (%s) %s: Disabling discard passdown.", bdevname(data_bdev, buf), reason);
2084 pt->adjusted_pf.discard_passdown = false;
2085 }
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002086}
2087
Joe Thornber991d9fa2011-10-31 20:21:18 +00002088static int bind_control_target(struct pool *pool, struct dm_target *ti)
2089{
2090 struct pool_c *pt = ti->private;
2091
Joe Thornbere49e5822012-07-27 15:08:16 +01002092 /*
Joe Thornber9b7aaa62013-12-04 16:58:19 -05002093 * We want to make sure that a pool in PM_FAIL mode is never upgraded.
Joe Thornbere49e5822012-07-27 15:08:16 +01002094 */
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002095 enum pool_mode old_mode = get_pool_mode(pool);
Mike Snitzer0424caa2012-09-26 23:45:47 +01002096 enum pool_mode new_mode = pt->adjusted_pf.mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01002097
Joe Thornber9b7aaa62013-12-04 16:58:19 -05002098 /*
Mike Snitzer8b64e882013-12-20 14:27:28 -05002099 * Don't change the pool's mode until set_pool_mode() below.
2100 * Otherwise the pool's process_* function pointers may
2101 * not match the desired pool mode.
2102 */
2103 pt->adjusted_pf.mode = old_mode;
2104
2105 pool->ti = ti;
2106 pool->pf = pt->adjusted_pf;
2107 pool->low_water_blocks = pt->low_water_blocks;
2108
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002109 set_pool_mode(pool, new_mode);
Mike Snitzerf4026932012-05-19 01:01:01 +01002110
Joe Thornber991d9fa2011-10-31 20:21:18 +00002111 return 0;
2112}
2113
2114static void unbind_control_target(struct pool *pool, struct dm_target *ti)
2115{
2116 if (pool->ti == ti)
2117 pool->ti = NULL;
2118}
2119
2120/*----------------------------------------------------------------
2121 * Pool creation
2122 *--------------------------------------------------------------*/
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002123/* Initialize pool features. */
2124static void pool_features_init(struct pool_features *pf)
2125{
Joe Thornbere49e5822012-07-27 15:08:16 +01002126 pf->mode = PM_WRITE;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002127 pf->zero_new_blocks = true;
2128 pf->discard_enabled = true;
2129 pf->discard_passdown = true;
Mike Snitzer787a996c2013-12-06 16:21:43 -05002130 pf->error_if_no_space = false;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002131}
2132
Joe Thornber991d9fa2011-10-31 20:21:18 +00002133static void __pool_destroy(struct pool *pool)
2134{
2135 __pool_table_remove(pool);
2136
2137 if (dm_pool_metadata_close(pool->pmd) < 0)
2138 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2139
Mike Snitzer44feb382012-10-12 21:02:10 +01002140 dm_bio_prison_destroy(pool->prison);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002141 dm_kcopyd_client_destroy(pool->copier);
2142
2143 if (pool->wq)
2144 destroy_workqueue(pool->wq);
2145
2146 if (pool->next_mapping)
2147 mempool_free(pool->next_mapping, pool->mapping_pool);
2148 mempool_destroy(pool->mapping_pool);
Mike Snitzer44feb382012-10-12 21:02:10 +01002149 dm_deferred_set_destroy(pool->shared_read_ds);
2150 dm_deferred_set_destroy(pool->all_io_ds);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002151 kfree(pool);
2152}
2153
Mike Snitzera24c2562012-06-03 00:30:00 +01002154static struct kmem_cache *_new_mapping_cache;
Mike Snitzera24c2562012-06-03 00:30:00 +01002155
Joe Thornber991d9fa2011-10-31 20:21:18 +00002156static struct pool *pool_create(struct mapped_device *pool_md,
2157 struct block_device *metadata_dev,
Joe Thornbere49e5822012-07-27 15:08:16 +01002158 unsigned long block_size,
2159 int read_only, char **error)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002160{
2161 int r;
2162 void *err_p;
2163 struct pool *pool;
2164 struct dm_pool_metadata *pmd;
Joe Thornbere49e5822012-07-27 15:08:16 +01002165 bool format_device = read_only ? false : true;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002166
Joe Thornbere49e5822012-07-27 15:08:16 +01002167 pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002168 if (IS_ERR(pmd)) {
2169 *error = "Error creating metadata object";
2170 return (struct pool *)pmd;
2171 }
2172
2173 pool = kmalloc(sizeof(*pool), GFP_KERNEL);
2174 if (!pool) {
2175 *error = "Error allocating memory for pool";
2176 err_p = ERR_PTR(-ENOMEM);
2177 goto bad_pool;
2178 }
2179
2180 pool->pmd = pmd;
2181 pool->sectors_per_block = block_size;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +01002182 if (block_size & (block_size - 1))
2183 pool->sectors_per_block_shift = -1;
2184 else
2185 pool->sectors_per_block_shift = __ffs(block_size);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002186 pool->low_water_blocks = 0;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002187 pool_features_init(&pool->pf);
Mike Snitzer44feb382012-10-12 21:02:10 +01002188 pool->prison = dm_bio_prison_create(PRISON_CELLS);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002189 if (!pool->prison) {
2190 *error = "Error creating pool's bio prison";
2191 err_p = ERR_PTR(-ENOMEM);
2192 goto bad_prison;
2193 }
2194
Mikulas Patockadf5d2e92013-03-01 22:45:49 +00002195 pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002196 if (IS_ERR(pool->copier)) {
2197 r = PTR_ERR(pool->copier);
2198 *error = "Error creating pool's kcopyd client";
2199 err_p = ERR_PTR(r);
2200 goto bad_kcopyd_client;
2201 }
2202
2203 /*
2204 * Create singlethreaded workqueue that will service all devices
2205 * that use this metadata.
2206 */
2207 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
2208 if (!pool->wq) {
2209 *error = "Error creating pool's workqueue";
2210 err_p = ERR_PTR(-ENOMEM);
2211 goto bad_wq;
2212 }
2213
2214 INIT_WORK(&pool->worker, do_worker);
Joe Thornber905e51b2012-03-28 18:41:27 +01002215 INIT_DELAYED_WORK(&pool->waker, do_waker);
Joe Thornber85ad643b2014-05-09 15:59:38 +01002216 INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002217 spin_lock_init(&pool->lock);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002218 bio_list_init(&pool->deferred_flush_bios);
2219 INIT_LIST_HEAD(&pool->prepared_mappings);
Joe Thornber104655f2012-03-28 18:41:28 +01002220 INIT_LIST_HEAD(&pool->prepared_discards);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002221 INIT_LIST_HEAD(&pool->active_thins);
Joe Thornber88a66212013-12-04 20:16:12 -05002222 pool->low_water_triggered = false;
Mike Snitzer44feb382012-10-12 21:02:10 +01002223
2224 pool->shared_read_ds = dm_deferred_set_create();
2225 if (!pool->shared_read_ds) {
2226 *error = "Error creating pool's shared read deferred set";
2227 err_p = ERR_PTR(-ENOMEM);
2228 goto bad_shared_read_ds;
2229 }
2230
2231 pool->all_io_ds = dm_deferred_set_create();
2232 if (!pool->all_io_ds) {
2233 *error = "Error creating pool's all io deferred set";
2234 err_p = ERR_PTR(-ENOMEM);
2235 goto bad_all_io_ds;
2236 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002237
2238 pool->next_mapping = NULL;
Mike Snitzera24c2562012-06-03 00:30:00 +01002239 pool->mapping_pool = mempool_create_slab_pool(MAPPING_POOL_SIZE,
2240 _new_mapping_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002241 if (!pool->mapping_pool) {
2242 *error = "Error creating pool's mapping mempool";
2243 err_p = ERR_PTR(-ENOMEM);
2244 goto bad_mapping_pool;
2245 }
2246
Joe Thornber991d9fa2011-10-31 20:21:18 +00002247 pool->ref_count = 1;
Joe Thornber905e51b2012-03-28 18:41:27 +01002248 pool->last_commit_jiffies = jiffies;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002249 pool->pool_md = pool_md;
2250 pool->md_dev = metadata_dev;
2251 __pool_table_insert(pool);
2252
2253 return pool;
2254
Joe Thornber991d9fa2011-10-31 20:21:18 +00002255bad_mapping_pool:
Mike Snitzer44feb382012-10-12 21:02:10 +01002256 dm_deferred_set_destroy(pool->all_io_ds);
2257bad_all_io_ds:
2258 dm_deferred_set_destroy(pool->shared_read_ds);
2259bad_shared_read_ds:
Joe Thornber991d9fa2011-10-31 20:21:18 +00002260 destroy_workqueue(pool->wq);
2261bad_wq:
2262 dm_kcopyd_client_destroy(pool->copier);
2263bad_kcopyd_client:
Mike Snitzer44feb382012-10-12 21:02:10 +01002264 dm_bio_prison_destroy(pool->prison);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002265bad_prison:
2266 kfree(pool);
2267bad_pool:
2268 if (dm_pool_metadata_close(pmd))
2269 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2270
2271 return err_p;
2272}
2273
2274static void __pool_inc(struct pool *pool)
2275{
2276 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
2277 pool->ref_count++;
2278}
2279
2280static void __pool_dec(struct pool *pool)
2281{
2282 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
2283 BUG_ON(!pool->ref_count);
2284 if (!--pool->ref_count)
2285 __pool_destroy(pool);
2286}
2287
2288static struct pool *__pool_find(struct mapped_device *pool_md,
2289 struct block_device *metadata_dev,
Joe Thornbere49e5822012-07-27 15:08:16 +01002290 unsigned long block_size, int read_only,
2291 char **error, int *created)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002292{
2293 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
2294
2295 if (pool) {
Mike Snitzerf09996c2012-07-27 15:07:59 +01002296 if (pool->pool_md != pool_md) {
2297 *error = "metadata device already in use by a pool";
Joe Thornber991d9fa2011-10-31 20:21:18 +00002298 return ERR_PTR(-EBUSY);
Mike Snitzerf09996c2012-07-27 15:07:59 +01002299 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002300 __pool_inc(pool);
2301
2302 } else {
2303 pool = __pool_table_lookup(pool_md);
2304 if (pool) {
Mike Snitzerf09996c2012-07-27 15:07:59 +01002305 if (pool->md_dev != metadata_dev) {
2306 *error = "different pool cannot replace a pool";
Joe Thornber991d9fa2011-10-31 20:21:18 +00002307 return ERR_PTR(-EINVAL);
Mike Snitzerf09996c2012-07-27 15:07:59 +01002308 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002309 __pool_inc(pool);
2310
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002311 } else {
Joe Thornbere49e5822012-07-27 15:08:16 +01002312 pool = pool_create(pool_md, metadata_dev, block_size, read_only, error);
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002313 *created = 1;
2314 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002315 }
2316
2317 return pool;
2318}
2319
2320/*----------------------------------------------------------------
2321 * Pool target methods
2322 *--------------------------------------------------------------*/
2323static void pool_dtr(struct dm_target *ti)
2324{
2325 struct pool_c *pt = ti->private;
2326
2327 mutex_lock(&dm_thin_pool_table.mutex);
2328
2329 unbind_control_target(pt->pool, ti);
2330 __pool_dec(pt->pool);
2331 dm_put_device(ti, pt->metadata_dev);
2332 dm_put_device(ti, pt->data_dev);
2333 kfree(pt);
2334
2335 mutex_unlock(&dm_thin_pool_table.mutex);
2336}
2337
Joe Thornber991d9fa2011-10-31 20:21:18 +00002338static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
2339 struct dm_target *ti)
2340{
2341 int r;
2342 unsigned argc;
2343 const char *arg_name;
2344
2345 static struct dm_arg _args[] = {
Mike Snitzer74aa45c2014-01-15 19:07:58 -05002346 {0, 4, "Invalid number of pool feature arguments"},
Joe Thornber991d9fa2011-10-31 20:21:18 +00002347 };
2348
2349 /*
2350 * No feature arguments supplied.
2351 */
2352 if (!as->argc)
2353 return 0;
2354
2355 r = dm_read_arg_group(_args, as, &argc, &ti->error);
2356 if (r)
2357 return -EINVAL;
2358
2359 while (argc && !r) {
2360 arg_name = dm_shift_arg(as);
2361 argc--;
2362
Joe Thornbere49e5822012-07-27 15:08:16 +01002363 if (!strcasecmp(arg_name, "skip_block_zeroing"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002364 pf->zero_new_blocks = false;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002365
Joe Thornbere49e5822012-07-27 15:08:16 +01002366 else if (!strcasecmp(arg_name, "ignore_discard"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002367 pf->discard_enabled = false;
Joe Thornbere49e5822012-07-27 15:08:16 +01002368
2369 else if (!strcasecmp(arg_name, "no_discard_passdown"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002370 pf->discard_passdown = false;
Joe Thornbere49e5822012-07-27 15:08:16 +01002371
2372 else if (!strcasecmp(arg_name, "read_only"))
2373 pf->mode = PM_READ_ONLY;
2374
Mike Snitzer787a996c2013-12-06 16:21:43 -05002375 else if (!strcasecmp(arg_name, "error_if_no_space"))
2376 pf->error_if_no_space = true;
2377
Joe Thornbere49e5822012-07-27 15:08:16 +01002378 else {
2379 ti->error = "Unrecognised pool feature requested";
2380 r = -EINVAL;
2381 break;
2382 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002383 }
2384
2385 return r;
2386}
2387
Joe Thornberac8c3f32013-05-10 14:37:21 +01002388static void metadata_low_callback(void *context)
2389{
2390 struct pool *pool = context;
2391
2392 DMWARN("%s: reached low water mark for metadata device: sending event.",
2393 dm_device_name(pool->pool_md));
2394
2395 dm_table_event(pool->ti->table);
2396}
2397
Mike Snitzer7d489352014-02-12 23:58:15 -05002398static sector_t get_dev_size(struct block_device *bdev)
Joe Thornberb17446d2013-05-10 14:37:18 +01002399{
Mike Snitzer7d489352014-02-12 23:58:15 -05002400 return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
2401}
2402
2403static void warn_if_metadata_device_too_big(struct block_device *bdev)
2404{
2405 sector_t metadata_dev_size = get_dev_size(bdev);
Joe Thornberb17446d2013-05-10 14:37:18 +01002406 char buffer[BDEVNAME_SIZE];
2407
Mike Snitzer7d489352014-02-12 23:58:15 -05002408 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
Joe Thornberb17446d2013-05-10 14:37:18 +01002409 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
2410 bdevname(bdev, buffer), THIN_METADATA_MAX_SECTORS);
Mike Snitzer7d489352014-02-12 23:58:15 -05002411}
2412
2413static sector_t get_metadata_dev_size(struct block_device *bdev)
2414{
2415 sector_t metadata_dev_size = get_dev_size(bdev);
2416
2417 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)
2418 metadata_dev_size = THIN_METADATA_MAX_SECTORS;
Joe Thornberb17446d2013-05-10 14:37:18 +01002419
2420 return metadata_dev_size;
2421}
2422
Joe Thornber24347e92013-05-10 14:37:19 +01002423static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)
2424{
2425 sector_t metadata_dev_size = get_metadata_dev_size(bdev);
2426
Mike Snitzer7d489352014-02-12 23:58:15 -05002427 sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);
Joe Thornber24347e92013-05-10 14:37:19 +01002428
2429 return metadata_dev_size;
2430}
2431
Joe Thornber991d9fa2011-10-31 20:21:18 +00002432/*
Joe Thornberac8c3f32013-05-10 14:37:21 +01002433 * When a metadata threshold is crossed a dm event is triggered, and
2434 * userland should respond by growing the metadata device. We could let
2435 * userland set the threshold, like we do with the data threshold, but I'm
2436 * not sure they know enough to do this well.
2437 */
2438static dm_block_t calc_metadata_threshold(struct pool_c *pt)
2439{
2440 /*
2441 * 4M is ample for all ops with the possible exception of thin
2442 * device deletion which is harmless if it fails (just retry the
2443 * delete after you've grown the device).
2444 */
2445 dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4;
2446 return min((dm_block_t)1024ULL /* 4M */, quarter);
2447}
2448
2449/*
Joe Thornber991d9fa2011-10-31 20:21:18 +00002450 * thin-pool <metadata dev> <data dev>
2451 * <data block size (sectors)>
2452 * <low water mark (blocks)>
2453 * [<#feature args> [<arg>]*]
2454 *
2455 * Optional feature arguments are:
2456 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002457 * ignore_discard: disable discard
2458 * no_discard_passdown: don't pass discards down to the data device
Mike Snitzer787a996c2013-12-06 16:21:43 -05002459 * read_only: Don't allow any changes to be made to the pool metadata.
2460 * error_if_no_space: error IOs, instead of queueing, if no space.
Joe Thornber991d9fa2011-10-31 20:21:18 +00002461 */
2462static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
2463{
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002464 int r, pool_created = 0;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002465 struct pool_c *pt;
2466 struct pool *pool;
2467 struct pool_features pf;
2468 struct dm_arg_set as;
2469 struct dm_dev *data_dev;
2470 unsigned long block_size;
2471 dm_block_t low_water_blocks;
2472 struct dm_dev *metadata_dev;
Joe Thornber5d0db962013-05-10 14:37:19 +01002473 fmode_t metadata_mode;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002474
2475 /*
2476 * FIXME Remove validation from scope of lock.
2477 */
2478 mutex_lock(&dm_thin_pool_table.mutex);
2479
2480 if (argc < 4) {
2481 ti->error = "Invalid argument count";
2482 r = -EINVAL;
2483 goto out_unlock;
2484 }
Joe Thornber5d0db962013-05-10 14:37:19 +01002485
Joe Thornber991d9fa2011-10-31 20:21:18 +00002486 as.argc = argc;
2487 as.argv = argv;
2488
Joe Thornber5d0db962013-05-10 14:37:19 +01002489 /*
2490 * Set default pool features.
2491 */
2492 pool_features_init(&pf);
2493
2494 dm_consume_args(&as, 4);
2495 r = parse_pool_features(&as, &pf, ti);
2496 if (r)
2497 goto out_unlock;
2498
2499 metadata_mode = FMODE_READ | ((pf.mode == PM_READ_ONLY) ? 0 : FMODE_WRITE);
2500 r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002501 if (r) {
2502 ti->error = "Error opening metadata block device";
2503 goto out_unlock;
2504 }
Mike Snitzer7d489352014-02-12 23:58:15 -05002505 warn_if_metadata_device_too_big(metadata_dev->bdev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002506
2507 r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
2508 if (r) {
2509 ti->error = "Error getting data device";
2510 goto out_metadata;
2511 }
2512
2513 if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
2514 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
2515 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01002516 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00002517 ti->error = "Invalid block size";
2518 r = -EINVAL;
2519 goto out;
2520 }
2521
2522 if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
2523 ti->error = "Invalid low water mark";
2524 r = -EINVAL;
2525 goto out;
2526 }
2527
Joe Thornber991d9fa2011-10-31 20:21:18 +00002528 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
2529 if (!pt) {
2530 r = -ENOMEM;
2531 goto out;
2532 }
2533
2534 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev,
Joe Thornbere49e5822012-07-27 15:08:16 +01002535 block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002536 if (IS_ERR(pool)) {
2537 r = PTR_ERR(pool);
2538 goto out_free_pt;
2539 }
2540
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002541 /*
2542 * 'pool_created' reflects whether this is the first table load.
2543 * Top level discard support is not allowed to be changed after
2544 * initial load. This would require a pool reload to trigger thin
2545 * device changes.
2546 */
2547 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
2548 ti->error = "Discard support cannot be disabled once enabled";
2549 r = -EINVAL;
2550 goto out_flags_changed;
2551 }
2552
Joe Thornber991d9fa2011-10-31 20:21:18 +00002553 pt->pool = pool;
2554 pt->ti = ti;
2555 pt->metadata_dev = metadata_dev;
2556 pt->data_dev = data_dev;
2557 pt->low_water_blocks = low_water_blocks;
Mike Snitzer0424caa2012-09-26 23:45:47 +01002558 pt->adjusted_pf = pt->requested_pf = pf;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00002559 ti->num_flush_bios = 1;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002560
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002561 /*
2562 * Only need to enable discards if the pool should pass
2563 * them down to the data device. The thin device's discard
2564 * processing will cause mappings to be removed from the btree.
2565 */
Mike Snitzerb60ab992013-09-19 18:49:11 -04002566 ti->discard_zeroes_data_unsupported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002567 if (pf.discard_enabled && pf.discard_passdown) {
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00002568 ti->num_discard_bios = 1;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002569
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002570 /*
2571 * Setting 'discards_supported' circumvents the normal
2572 * stacking of discard limits (this keeps the pool and
2573 * thin devices' discard limits consistent).
2574 */
Alasdair G Kergon0ac55482012-07-27 15:08:08 +01002575 ti->discards_supported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002576 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002577 ti->private = pt;
2578
Joe Thornberac8c3f32013-05-10 14:37:21 +01002579 r = dm_pool_register_metadata_threshold(pt->pool->pmd,
2580 calc_metadata_threshold(pt),
2581 metadata_low_callback,
2582 pool);
2583 if (r)
2584 goto out_free_pt;
2585
Joe Thornber991d9fa2011-10-31 20:21:18 +00002586 pt->callbacks.congested_fn = pool_is_congested;
2587 dm_table_add_target_callbacks(ti->table, &pt->callbacks);
2588
2589 mutex_unlock(&dm_thin_pool_table.mutex);
2590
2591 return 0;
2592
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002593out_flags_changed:
2594 __pool_dec(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002595out_free_pt:
2596 kfree(pt);
2597out:
2598 dm_put_device(ti, data_dev);
2599out_metadata:
2600 dm_put_device(ti, metadata_dev);
2601out_unlock:
2602 mutex_unlock(&dm_thin_pool_table.mutex);
2603
2604 return r;
2605}
2606
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00002607static int pool_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002608{
2609 int r;
2610 struct pool_c *pt = ti->private;
2611 struct pool *pool = pt->pool;
2612 unsigned long flags;
2613
2614 /*
2615 * As this is a singleton target, ti->begin is always zero.
2616 */
2617 spin_lock_irqsave(&pool->lock, flags);
2618 bio->bi_bdev = pt->data_dev->bdev;
2619 r = DM_MAPIO_REMAPPED;
2620 spin_unlock_irqrestore(&pool->lock, flags);
2621
2622 return r;
2623}
2624
Joe Thornberb17446d2013-05-10 14:37:18 +01002625static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)
2626{
2627 int r;
2628 struct pool_c *pt = ti->private;
2629 struct pool *pool = pt->pool;
2630 sector_t data_size = ti->len;
2631 dm_block_t sb_data_size;
2632
2633 *need_commit = false;
2634
2635 (void) sector_div(data_size, pool->sectors_per_block);
2636
2637 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
2638 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04002639 DMERR("%s: failed to retrieve data device size",
2640 dm_device_name(pool->pool_md));
Joe Thornberb17446d2013-05-10 14:37:18 +01002641 return r;
2642 }
2643
2644 if (data_size < sb_data_size) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04002645 DMERR("%s: pool target (%llu blocks) too small: expected %llu",
2646 dm_device_name(pool->pool_md),
Joe Thornberb17446d2013-05-10 14:37:18 +01002647 (unsigned long long)data_size, sb_data_size);
2648 return -EINVAL;
2649
2650 } else if (data_size > sb_data_size) {
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002651 if (dm_pool_metadata_needs_check(pool->pmd)) {
2652 DMERR("%s: unable to grow the data device until repaired.",
2653 dm_device_name(pool->pool_md));
2654 return 0;
2655 }
2656
Mike Snitzer6f7f51d2013-12-04 10:25:53 -05002657 if (sb_data_size)
2658 DMINFO("%s: growing the data device from %llu to %llu blocks",
2659 dm_device_name(pool->pool_md),
2660 sb_data_size, (unsigned long long)data_size);
Joe Thornberb17446d2013-05-10 14:37:18 +01002661 r = dm_pool_resize_data_dev(pool->pmd, data_size);
2662 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05002663 metadata_operation_failed(pool, "dm_pool_resize_data_dev", r);
Joe Thornberb17446d2013-05-10 14:37:18 +01002664 return r;
2665 }
2666
2667 *need_commit = true;
2668 }
2669
2670 return 0;
2671}
2672
Joe Thornber24347e92013-05-10 14:37:19 +01002673static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)
2674{
2675 int r;
2676 struct pool_c *pt = ti->private;
2677 struct pool *pool = pt->pool;
2678 dm_block_t metadata_dev_size, sb_metadata_dev_size;
2679
2680 *need_commit = false;
2681
Alasdair G Kergon610bba82013-05-19 18:57:50 +01002682 metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev);
Joe Thornber24347e92013-05-10 14:37:19 +01002683
2684 r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size);
2685 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04002686 DMERR("%s: failed to retrieve metadata device size",
2687 dm_device_name(pool->pool_md));
Joe Thornber24347e92013-05-10 14:37:19 +01002688 return r;
2689 }
2690
2691 if (metadata_dev_size < sb_metadata_dev_size) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04002692 DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
2693 dm_device_name(pool->pool_md),
Joe Thornber24347e92013-05-10 14:37:19 +01002694 metadata_dev_size, sb_metadata_dev_size);
2695 return -EINVAL;
2696
2697 } else if (metadata_dev_size > sb_metadata_dev_size) {
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002698 if (dm_pool_metadata_needs_check(pool->pmd)) {
2699 DMERR("%s: unable to grow the metadata device until repaired.",
2700 dm_device_name(pool->pool_md));
2701 return 0;
2702 }
2703
Mike Snitzer7d489352014-02-12 23:58:15 -05002704 warn_if_metadata_device_too_big(pool->md_dev);
Mike Snitzer6f7f51d2013-12-04 10:25:53 -05002705 DMINFO("%s: growing the metadata device from %llu to %llu blocks",
2706 dm_device_name(pool->pool_md),
2707 sb_metadata_dev_size, metadata_dev_size);
Joe Thornber24347e92013-05-10 14:37:19 +01002708 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size);
2709 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05002710 metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r);
Joe Thornber24347e92013-05-10 14:37:19 +01002711 return r;
2712 }
2713
2714 *need_commit = true;
2715 }
2716
2717 return 0;
2718}
2719
Joe Thornber991d9fa2011-10-31 20:21:18 +00002720/*
2721 * Retrieves the number of blocks of the data device from
2722 * the superblock and compares it to the actual device size,
2723 * thus resizing the data device in case it has grown.
2724 *
2725 * This both copes with opening preallocated data devices in the ctr
2726 * being followed by a resume
2727 * -and-
2728 * calling the resume method individually after userspace has
2729 * grown the data device in reaction to a table event.
2730 */
2731static int pool_preresume(struct dm_target *ti)
2732{
2733 int r;
Joe Thornber24347e92013-05-10 14:37:19 +01002734 bool need_commit1, need_commit2;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002735 struct pool_c *pt = ti->private;
2736 struct pool *pool = pt->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002737
2738 /*
2739 * Take control of the pool object.
2740 */
2741 r = bind_control_target(pool, ti);
2742 if (r)
2743 return r;
2744
Joe Thornberb17446d2013-05-10 14:37:18 +01002745 r = maybe_resize_data_dev(ti, &need_commit1);
2746 if (r)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002747 return r;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002748
Joe Thornber24347e92013-05-10 14:37:19 +01002749 r = maybe_resize_metadata_dev(ti, &need_commit2);
2750 if (r)
2751 return r;
2752
2753 if (need_commit1 || need_commit2)
Joe Thornber020cc3b2013-12-04 15:05:36 -05002754 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002755
2756 return 0;
2757}
2758
2759static void pool_resume(struct dm_target *ti)
2760{
2761 struct pool_c *pt = ti->private;
2762 struct pool *pool = pt->pool;
2763 unsigned long flags;
2764
2765 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber88a66212013-12-04 20:16:12 -05002766 pool->low_water_triggered = false;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002767 spin_unlock_irqrestore(&pool->lock, flags);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002768 requeue_bios(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002769
Joe Thornber905e51b2012-03-28 18:41:27 +01002770 do_waker(&pool->waker.work);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002771}
2772
2773static void pool_postsuspend(struct dm_target *ti)
2774{
Joe Thornber991d9fa2011-10-31 20:21:18 +00002775 struct pool_c *pt = ti->private;
2776 struct pool *pool = pt->pool;
2777
Joe Thornber905e51b2012-03-28 18:41:27 +01002778 cancel_delayed_work(&pool->waker);
Joe Thornber85ad643b2014-05-09 15:59:38 +01002779 cancel_delayed_work(&pool->no_space_timeout);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002780 flush_workqueue(pool->wq);
Joe Thornber020cc3b2013-12-04 15:05:36 -05002781 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002782}
2783
2784static int check_arg_count(unsigned argc, unsigned args_required)
2785{
2786 if (argc != args_required) {
2787 DMWARN("Message received with %u arguments instead of %u.",
2788 argc, args_required);
2789 return -EINVAL;
2790 }
2791
2792 return 0;
2793}
2794
2795static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
2796{
2797 if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
2798 *dev_id <= MAX_DEV_ID)
2799 return 0;
2800
2801 if (warning)
2802 DMWARN("Message received with invalid device id: %s", arg);
2803
2804 return -EINVAL;
2805}
2806
2807static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
2808{
2809 dm_thin_id dev_id;
2810 int r;
2811
2812 r = check_arg_count(argc, 2);
2813 if (r)
2814 return r;
2815
2816 r = read_dev_id(argv[1], &dev_id, 1);
2817 if (r)
2818 return r;
2819
2820 r = dm_pool_create_thin(pool->pmd, dev_id);
2821 if (r) {
2822 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
2823 argv[1]);
2824 return r;
2825 }
2826
2827 return 0;
2828}
2829
2830static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
2831{
2832 dm_thin_id dev_id;
2833 dm_thin_id origin_dev_id;
2834 int r;
2835
2836 r = check_arg_count(argc, 3);
2837 if (r)
2838 return r;
2839
2840 r = read_dev_id(argv[1], &dev_id, 1);
2841 if (r)
2842 return r;
2843
2844 r = read_dev_id(argv[2], &origin_dev_id, 1);
2845 if (r)
2846 return r;
2847
2848 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
2849 if (r) {
2850 DMWARN("Creation of new snapshot %s of device %s failed.",
2851 argv[1], argv[2]);
2852 return r;
2853 }
2854
2855 return 0;
2856}
2857
2858static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
2859{
2860 dm_thin_id dev_id;
2861 int r;
2862
2863 r = check_arg_count(argc, 2);
2864 if (r)
2865 return r;
2866
2867 r = read_dev_id(argv[1], &dev_id, 1);
2868 if (r)
2869 return r;
2870
2871 r = dm_pool_delete_thin_device(pool->pmd, dev_id);
2872 if (r)
2873 DMWARN("Deletion of thin device %s failed.", argv[1]);
2874
2875 return r;
2876}
2877
2878static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
2879{
2880 dm_thin_id old_id, new_id;
2881 int r;
2882
2883 r = check_arg_count(argc, 3);
2884 if (r)
2885 return r;
2886
2887 if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
2888 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
2889 return -EINVAL;
2890 }
2891
2892 if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
2893 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
2894 return -EINVAL;
2895 }
2896
2897 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
2898 if (r) {
2899 DMWARN("Failed to change transaction id from %s to %s.",
2900 argv[1], argv[2]);
2901 return r;
2902 }
2903
2904 return 0;
2905}
2906
Joe Thornbercc8394d2012-06-03 00:30:01 +01002907static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
2908{
2909 int r;
2910
2911 r = check_arg_count(argc, 1);
2912 if (r)
2913 return r;
2914
Joe Thornber020cc3b2013-12-04 15:05:36 -05002915 (void) commit(pool);
Joe Thornber0d200ae2012-07-03 12:55:31 +01002916
Joe Thornbercc8394d2012-06-03 00:30:01 +01002917 r = dm_pool_reserve_metadata_snap(pool->pmd);
2918 if (r)
2919 DMWARN("reserve_metadata_snap message failed.");
2920
2921 return r;
2922}
2923
2924static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
2925{
2926 int r;
2927
2928 r = check_arg_count(argc, 1);
2929 if (r)
2930 return r;
2931
2932 r = dm_pool_release_metadata_snap(pool->pmd);
2933 if (r)
2934 DMWARN("release_metadata_snap message failed.");
2935
2936 return r;
2937}
2938
Joe Thornber991d9fa2011-10-31 20:21:18 +00002939/*
2940 * Messages supported:
2941 * create_thin <dev_id>
2942 * create_snap <dev_id> <origin_id>
2943 * delete <dev_id>
2944 * trim <dev_id> <new_size_in_sectors>
2945 * set_transaction_id <current_trans_id> <new_trans_id>
Joe Thornbercc8394d2012-06-03 00:30:01 +01002946 * reserve_metadata_snap
2947 * release_metadata_snap
Joe Thornber991d9fa2011-10-31 20:21:18 +00002948 */
2949static int pool_message(struct dm_target *ti, unsigned argc, char **argv)
2950{
2951 int r = -EINVAL;
2952 struct pool_c *pt = ti->private;
2953 struct pool *pool = pt->pool;
2954
2955 if (!strcasecmp(argv[0], "create_thin"))
2956 r = process_create_thin_mesg(argc, argv, pool);
2957
2958 else if (!strcasecmp(argv[0], "create_snap"))
2959 r = process_create_snap_mesg(argc, argv, pool);
2960
2961 else if (!strcasecmp(argv[0], "delete"))
2962 r = process_delete_mesg(argc, argv, pool);
2963
2964 else if (!strcasecmp(argv[0], "set_transaction_id"))
2965 r = process_set_transaction_id_mesg(argc, argv, pool);
2966
Joe Thornbercc8394d2012-06-03 00:30:01 +01002967 else if (!strcasecmp(argv[0], "reserve_metadata_snap"))
2968 r = process_reserve_metadata_snap_mesg(argc, argv, pool);
2969
2970 else if (!strcasecmp(argv[0], "release_metadata_snap"))
2971 r = process_release_metadata_snap_mesg(argc, argv, pool);
2972
Joe Thornber991d9fa2011-10-31 20:21:18 +00002973 else
2974 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
2975
Joe Thornbere49e5822012-07-27 15:08:16 +01002976 if (!r)
Joe Thornber020cc3b2013-12-04 15:05:36 -05002977 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002978
2979 return r;
2980}
2981
Joe Thornbere49e5822012-07-27 15:08:16 +01002982static void emit_flags(struct pool_features *pf, char *result,
2983 unsigned sz, unsigned maxlen)
2984{
2985 unsigned count = !pf->zero_new_blocks + !pf->discard_enabled +
Mike Snitzer787a996c2013-12-06 16:21:43 -05002986 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) +
2987 pf->error_if_no_space;
Joe Thornbere49e5822012-07-27 15:08:16 +01002988 DMEMIT("%u ", count);
2989
2990 if (!pf->zero_new_blocks)
2991 DMEMIT("skip_block_zeroing ");
2992
2993 if (!pf->discard_enabled)
2994 DMEMIT("ignore_discard ");
2995
2996 if (!pf->discard_passdown)
2997 DMEMIT("no_discard_passdown ");
2998
2999 if (pf->mode == PM_READ_ONLY)
3000 DMEMIT("read_only ");
Mike Snitzer787a996c2013-12-06 16:21:43 -05003001
3002 if (pf->error_if_no_space)
3003 DMEMIT("error_if_no_space ");
Joe Thornbere49e5822012-07-27 15:08:16 +01003004}
3005
Joe Thornber991d9fa2011-10-31 20:21:18 +00003006/*
3007 * Status line is:
3008 * <transaction id> <used metadata sectors>/<total metadata sectors>
3009 * <used data sectors>/<total data sectors> <held metadata root>
3010 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003011static void pool_status(struct dm_target *ti, status_type_t type,
3012 unsigned status_flags, char *result, unsigned maxlen)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003013{
Joe Thornbere49e5822012-07-27 15:08:16 +01003014 int r;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003015 unsigned sz = 0;
3016 uint64_t transaction_id;
3017 dm_block_t nr_free_blocks_data;
3018 dm_block_t nr_free_blocks_metadata;
3019 dm_block_t nr_blocks_data;
3020 dm_block_t nr_blocks_metadata;
3021 dm_block_t held_root;
3022 char buf[BDEVNAME_SIZE];
3023 char buf2[BDEVNAME_SIZE];
3024 struct pool_c *pt = ti->private;
3025 struct pool *pool = pt->pool;
3026
3027 switch (type) {
3028 case STATUSTYPE_INFO:
Joe Thornbere49e5822012-07-27 15:08:16 +01003029 if (get_pool_mode(pool) == PM_FAIL) {
3030 DMEMIT("Fail");
3031 break;
3032 }
3033
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01003034 /* Commit to ensure statistics aren't out-of-date */
3035 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
Joe Thornber020cc3b2013-12-04 15:05:36 -05003036 (void) commit(pool);
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01003037
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003038 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);
3039 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003040 DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
3041 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003042 goto err;
3043 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003044
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003045 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);
3046 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003047 DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
3048 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003049 goto err;
3050 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003051
3052 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003053 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003054 DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
3055 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003056 goto err;
3057 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003058
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003059 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);
3060 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003061 DMERR("%s: dm_pool_get_free_block_count returned %d",
3062 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003063 goto err;
3064 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003065
3066 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003067 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003068 DMERR("%s: dm_pool_get_data_dev_size returned %d",
3069 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003070 goto err;
3071 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003072
Joe Thornbercc8394d2012-06-03 00:30:01 +01003073 r = dm_pool_get_metadata_snap(pool->pmd, &held_root);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003074 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003075 DMERR("%s: dm_pool_get_metadata_snap returned %d",
3076 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003077 goto err;
3078 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003079
3080 DMEMIT("%llu %llu/%llu %llu/%llu ",
3081 (unsigned long long)transaction_id,
3082 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3083 (unsigned long long)nr_blocks_metadata,
3084 (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
3085 (unsigned long long)nr_blocks_data);
3086
3087 if (held_root)
Joe Thornbere49e5822012-07-27 15:08:16 +01003088 DMEMIT("%llu ", held_root);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003089 else
Joe Thornbere49e5822012-07-27 15:08:16 +01003090 DMEMIT("- ");
3091
Joe Thornber3e1a0692014-03-03 16:03:26 +00003092 if (pool->pf.mode == PM_OUT_OF_DATA_SPACE)
3093 DMEMIT("out_of_data_space ");
3094 else if (pool->pf.mode == PM_READ_ONLY)
Joe Thornbere49e5822012-07-27 15:08:16 +01003095 DMEMIT("ro ");
3096 else
3097 DMEMIT("rw ");
3098
Mike Snitzer018debe2012-12-21 20:23:32 +00003099 if (!pool->pf.discard_enabled)
Mike Snitzer787a996c2013-12-06 16:21:43 -05003100 DMEMIT("ignore_discard ");
Mike Snitzer018debe2012-12-21 20:23:32 +00003101 else if (pool->pf.discard_passdown)
Mike Snitzer787a996c2013-12-06 16:21:43 -05003102 DMEMIT("discard_passdown ");
Joe Thornbere49e5822012-07-27 15:08:16 +01003103 else
Mike Snitzer787a996c2013-12-06 16:21:43 -05003104 DMEMIT("no_discard_passdown ");
3105
3106 if (pool->pf.error_if_no_space)
3107 DMEMIT("error_if_no_space ");
3108 else
3109 DMEMIT("queue_if_no_space ");
Joe Thornber991d9fa2011-10-31 20:21:18 +00003110
3111 break;
3112
3113 case STATUSTYPE_TABLE:
3114 DMEMIT("%s %s %lu %llu ",
3115 format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
3116 format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
3117 (unsigned long)pool->sectors_per_block,
3118 (unsigned long long)pt->low_water_blocks);
Mike Snitzer0424caa2012-09-26 23:45:47 +01003119 emit_flags(&pt->requested_pf, result, sz, maxlen);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003120 break;
3121 }
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003122 return;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003123
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003124err:
3125 DMEMIT("Error");
Joe Thornber991d9fa2011-10-31 20:21:18 +00003126}
3127
3128static int pool_iterate_devices(struct dm_target *ti,
3129 iterate_devices_callout_fn fn, void *data)
3130{
3131 struct pool_c *pt = ti->private;
3132
3133 return fn(ti, pt->data_dev, 0, ti->len, data);
3134}
3135
3136static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
3137 struct bio_vec *biovec, int max_size)
3138{
3139 struct pool_c *pt = ti->private;
3140 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
3141
3142 if (!q->merge_bvec_fn)
3143 return max_size;
3144
3145 bvm->bi_bdev = pt->data_dev->bdev;
3146
3147 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
3148}
3149
Mike Snitzer0424caa2012-09-26 23:45:47 +01003150static void set_discard_limits(struct pool_c *pt, struct queue_limits *limits)
Joe Thornber104655f2012-03-28 18:41:28 +01003151{
Mike Snitzer0424caa2012-09-26 23:45:47 +01003152 struct pool *pool = pt->pool;
3153 struct queue_limits *data_limits;
3154
Joe Thornber104655f2012-03-28 18:41:28 +01003155 limits->max_discard_sectors = pool->sectors_per_block;
3156
3157 /*
Mike Snitzer0424caa2012-09-26 23:45:47 +01003158 * discard_granularity is just a hint, and not enforced.
Joe Thornber104655f2012-03-28 18:41:28 +01003159 */
Mike Snitzer0424caa2012-09-26 23:45:47 +01003160 if (pt->adjusted_pf.discard_passdown) {
3161 data_limits = &bdev_get_queue(pt->data_dev->bdev)->limits;
Lukas Czerner09869de2014-06-11 12:28:43 -04003162 limits->discard_granularity = max(data_limits->discard_granularity,
3163 pool->sectors_per_block << SECTOR_SHIFT);
Mike Snitzerf13945d2013-03-01 22:45:44 +00003164 } else
Mike Snitzer0424caa2012-09-26 23:45:47 +01003165 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
Joe Thornber104655f2012-03-28 18:41:28 +01003166}
3167
Joe Thornber991d9fa2011-10-31 20:21:18 +00003168static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
3169{
3170 struct pool_c *pt = ti->private;
3171 struct pool *pool = pt->pool;
Mike Snitzer0cc67cd2013-08-20 15:02:41 -04003172 uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003173
Mike Snitzer0cc67cd2013-08-20 15:02:41 -04003174 /*
3175 * If the system-determined stacked limits are compatible with the
3176 * pool's blocksize (io_opt is a factor) do not override them.
3177 */
3178 if (io_opt_sectors < pool->sectors_per_block ||
3179 do_div(io_opt_sectors, pool->sectors_per_block)) {
Mike Snitzerfdfb4c82014-07-18 17:59:43 -04003180 blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT);
Mike Snitzer0cc67cd2013-08-20 15:02:41 -04003181 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
3182 }
Mike Snitzer0424caa2012-09-26 23:45:47 +01003183
3184 /*
3185 * pt->adjusted_pf is a staging area for the actual features to use.
3186 * They get transferred to the live pool in bind_control_target()
3187 * called from pool_preresume().
3188 */
Mike Snitzerb60ab992013-09-19 18:49:11 -04003189 if (!pt->adjusted_pf.discard_enabled) {
3190 /*
3191 * Must explicitly disallow stacking discard limits otherwise the
3192 * block layer will stack them if pool's data device has support.
3193 * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the
3194 * user to see that, so make sure to set all discard limits to 0.
3195 */
3196 limits->discard_granularity = 0;
Mike Snitzer0424caa2012-09-26 23:45:47 +01003197 return;
Mike Snitzerb60ab992013-09-19 18:49:11 -04003198 }
Mike Snitzer0424caa2012-09-26 23:45:47 +01003199
3200 disable_passdown_if_not_supported(pt);
3201
3202 set_discard_limits(pt, limits);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003203}
3204
3205static struct target_type pool_target = {
3206 .name = "thin-pool",
3207 .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
3208 DM_TARGET_IMMUTABLE,
Joe Thornbere5aea7b2014-06-13 14:47:24 +01003209 .version = {1, 13, 0},
Joe Thornber991d9fa2011-10-31 20:21:18 +00003210 .module = THIS_MODULE,
3211 .ctr = pool_ctr,
3212 .dtr = pool_dtr,
3213 .map = pool_map,
3214 .postsuspend = pool_postsuspend,
3215 .preresume = pool_preresume,
3216 .resume = pool_resume,
3217 .message = pool_message,
3218 .status = pool_status,
3219 .merge = pool_merge,
3220 .iterate_devices = pool_iterate_devices,
3221 .io_hints = pool_io_hints,
3222};
3223
3224/*----------------------------------------------------------------
3225 * Thin target methods
3226 *--------------------------------------------------------------*/
Joe Thornberb10ebd32014-04-08 11:29:01 +01003227static void thin_get(struct thin_c *tc)
3228{
3229 atomic_inc(&tc->refcount);
3230}
3231
3232static void thin_put(struct thin_c *tc)
3233{
3234 if (atomic_dec_and_test(&tc->refcount))
3235 complete(&tc->can_destroy);
3236}
3237
Joe Thornber991d9fa2011-10-31 20:21:18 +00003238static void thin_dtr(struct dm_target *ti)
3239{
3240 struct thin_c *tc = ti->private;
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003241 unsigned long flags;
3242
Joe Thornberb10ebd32014-04-08 11:29:01 +01003243 thin_put(tc);
3244 wait_for_completion(&tc->can_destroy);
3245
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003246 spin_lock_irqsave(&tc->pool->lock, flags);
3247 list_del_rcu(&tc->list);
3248 spin_unlock_irqrestore(&tc->pool->lock, flags);
3249 synchronize_rcu();
Joe Thornber991d9fa2011-10-31 20:21:18 +00003250
3251 mutex_lock(&dm_thin_pool_table.mutex);
3252
3253 __pool_dec(tc->pool);
3254 dm_pool_close_thin_device(tc->td);
3255 dm_put_device(ti, tc->pool_dev);
Joe Thornber2dd9c252012-03-28 18:41:28 +01003256 if (tc->origin_dev)
3257 dm_put_device(ti, tc->origin_dev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003258 kfree(tc);
3259
3260 mutex_unlock(&dm_thin_pool_table.mutex);
3261}
3262
3263/*
3264 * Thin target parameters:
3265 *
Joe Thornber2dd9c252012-03-28 18:41:28 +01003266 * <pool_dev> <dev_id> [origin_dev]
Joe Thornber991d9fa2011-10-31 20:21:18 +00003267 *
3268 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
3269 * dev_id: the internal device identifier
Joe Thornber2dd9c252012-03-28 18:41:28 +01003270 * origin_dev: a device external to the pool that should act as the origin
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003271 *
3272 * If the pool device has discards disabled, they get disabled for the thin
3273 * device as well.
Joe Thornber991d9fa2011-10-31 20:21:18 +00003274 */
3275static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
3276{
3277 int r;
3278 struct thin_c *tc;
Joe Thornber2dd9c252012-03-28 18:41:28 +01003279 struct dm_dev *pool_dev, *origin_dev;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003280 struct mapped_device *pool_md;
Joe Thornber5e3283e2014-04-08 11:08:41 +01003281 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003282
3283 mutex_lock(&dm_thin_pool_table.mutex);
3284
Joe Thornber2dd9c252012-03-28 18:41:28 +01003285 if (argc != 2 && argc != 3) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00003286 ti->error = "Invalid argument count";
3287 r = -EINVAL;
3288 goto out_unlock;
3289 }
3290
3291 tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
3292 if (!tc) {
3293 ti->error = "Out of memory";
3294 r = -ENOMEM;
3295 goto out_unlock;
3296 }
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003297 spin_lock_init(&tc->lock);
3298 bio_list_init(&tc->deferred_bio_list);
3299 bio_list_init(&tc->retry_on_resume_list);
Mike Snitzer67324ea2014-03-21 18:33:41 -04003300 tc->sort_bio_list = RB_ROOT;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003301
Joe Thornber2dd9c252012-03-28 18:41:28 +01003302 if (argc == 3) {
3303 r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev);
3304 if (r) {
3305 ti->error = "Error opening origin device";
3306 goto bad_origin_dev;
3307 }
3308 tc->origin_dev = origin_dev;
3309 }
3310
Joe Thornber991d9fa2011-10-31 20:21:18 +00003311 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
3312 if (r) {
3313 ti->error = "Error opening pool device";
3314 goto bad_pool_dev;
3315 }
3316 tc->pool_dev = pool_dev;
3317
3318 if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
3319 ti->error = "Invalid device id";
3320 r = -EINVAL;
3321 goto bad_common;
3322 }
3323
3324 pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
3325 if (!pool_md) {
3326 ti->error = "Couldn't get pool mapped device";
3327 r = -EINVAL;
3328 goto bad_common;
3329 }
3330
3331 tc->pool = __pool_table_lookup(pool_md);
3332 if (!tc->pool) {
3333 ti->error = "Couldn't find pool object";
3334 r = -EINVAL;
3335 goto bad_pool_lookup;
3336 }
3337 __pool_inc(tc->pool);
3338
Joe Thornbere49e5822012-07-27 15:08:16 +01003339 if (get_pool_mode(tc->pool) == PM_FAIL) {
3340 ti->error = "Couldn't open thin device, Pool is in fail mode";
Mike Snitzer1acacc02014-02-19 20:32:33 -05003341 r = -EINVAL;
Joe Thornbere49e5822012-07-27 15:08:16 +01003342 goto bad_thin_open;
3343 }
3344
Joe Thornber991d9fa2011-10-31 20:21:18 +00003345 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
3346 if (r) {
3347 ti->error = "Couldn't open thin internal device";
3348 goto bad_thin_open;
3349 }
3350
Mike Snitzer542f9032012-07-27 15:08:00 +01003351 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
3352 if (r)
Mike Snitzer1acacc02014-02-19 20:32:33 -05003353 goto bad_target_max_io_len;
Mike Snitzer542f9032012-07-27 15:08:00 +01003354
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00003355 ti->num_flush_bios = 1;
Joe Thornber16ad3d12012-07-27 15:08:07 +01003356 ti->flush_supported = true;
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00003357 ti->per_bio_data_size = sizeof(struct dm_thin_endio_hook);
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003358
3359 /* In case the pool supports discards, pass them on. */
Mike Snitzerb60ab992013-09-19 18:49:11 -04003360 ti->discard_zeroes_data_unsupported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003361 if (tc->pool->pf.discard_enabled) {
Alasdair G Kergon0ac55482012-07-27 15:08:08 +01003362 ti->discards_supported = true;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00003363 ti->num_discard_bios = 1;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00003364 /* Discard bios must be split on a block boundary */
3365 ti->split_discard_bios = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003366 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003367
3368 dm_put(pool_md);
3369
3370 mutex_unlock(&dm_thin_pool_table.mutex);
3371
Joe Thornberb10ebd32014-04-08 11:29:01 +01003372 atomic_set(&tc->refcount, 1);
3373 init_completion(&tc->can_destroy);
3374
Joe Thornber5e3283e2014-04-08 11:08:41 +01003375 spin_lock_irqsave(&tc->pool->lock, flags);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003376 list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
Joe Thornber5e3283e2014-04-08 11:08:41 +01003377 spin_unlock_irqrestore(&tc->pool->lock, flags);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003378 /*
3379 * This synchronize_rcu() call is needed here otherwise we risk a
3380 * wake_worker() call finding no bios to process (because the newly
3381 * added tc isn't yet visible). So this reduces latency since we
3382 * aren't then dependent on the periodic commit to wake_worker().
3383 */
3384 synchronize_rcu();
3385
Joe Thornber991d9fa2011-10-31 20:21:18 +00003386 return 0;
3387
Mike Snitzer1acacc02014-02-19 20:32:33 -05003388bad_target_max_io_len:
3389 dm_pool_close_thin_device(tc->td);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003390bad_thin_open:
3391 __pool_dec(tc->pool);
3392bad_pool_lookup:
3393 dm_put(pool_md);
3394bad_common:
3395 dm_put_device(ti, tc->pool_dev);
3396bad_pool_dev:
Joe Thornber2dd9c252012-03-28 18:41:28 +01003397 if (tc->origin_dev)
3398 dm_put_device(ti, tc->origin_dev);
3399bad_origin_dev:
Joe Thornber991d9fa2011-10-31 20:21:18 +00003400 kfree(tc);
3401out_unlock:
3402 mutex_unlock(&dm_thin_pool_table.mutex);
3403
3404 return r;
3405}
3406
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00003407static int thin_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003408{
Kent Overstreet4f024f32013-10-11 15:44:27 -07003409 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003410
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00003411 return thin_bio_map(ti, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003412}
3413
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00003414static int thin_endio(struct dm_target *ti, struct bio *bio, int err)
Joe Thornbereb2aa482012-03-28 18:41:28 +01003415{
3416 unsigned long flags;
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00003417 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01003418 struct list_head work;
Mike Snitzera24c2562012-06-03 00:30:00 +01003419 struct dm_thin_new_mapping *m, *tmp;
Joe Thornbereb2aa482012-03-28 18:41:28 +01003420 struct pool *pool = h->tc->pool;
3421
3422 if (h->shared_read_entry) {
3423 INIT_LIST_HEAD(&work);
Mike Snitzer44feb382012-10-12 21:02:10 +01003424 dm_deferred_entry_dec(h->shared_read_entry, &work);
Joe Thornbereb2aa482012-03-28 18:41:28 +01003425
3426 spin_lock_irqsave(&pool->lock, flags);
3427 list_for_each_entry_safe(m, tmp, &work, list) {
3428 list_del(&m->list);
Joe Thornber50f3c3e2014-06-13 13:57:09 +01003429 __complete_mapping_preparation(m);
Joe Thornbereb2aa482012-03-28 18:41:28 +01003430 }
3431 spin_unlock_irqrestore(&pool->lock, flags);
3432 }
3433
Joe Thornber104655f2012-03-28 18:41:28 +01003434 if (h->all_io_entry) {
3435 INIT_LIST_HEAD(&work);
Mike Snitzer44feb382012-10-12 21:02:10 +01003436 dm_deferred_entry_dec(h->all_io_entry, &work);
Joe Thornber563af182012-12-21 20:23:31 +00003437 if (!list_empty(&work)) {
3438 spin_lock_irqsave(&pool->lock, flags);
3439 list_for_each_entry_safe(m, tmp, &work, list)
Mike Snitzerdaec3382013-12-11 14:01:20 -05003440 list_add_tail(&m->list, &pool->prepared_discards);
Joe Thornber563af182012-12-21 20:23:31 +00003441 spin_unlock_irqrestore(&pool->lock, flags);
3442 wake_worker(pool);
3443 }
Joe Thornber104655f2012-03-28 18:41:28 +01003444 }
3445
Joe Thornbereb2aa482012-03-28 18:41:28 +01003446 return 0;
3447}
3448
Joe Thornber738211f2014-03-03 15:52:28 +00003449static void thin_presuspend(struct dm_target *ti)
3450{
3451 struct thin_c *tc = ti->private;
3452
3453 if (dm_noflush_suspending(ti))
3454 noflush_work(tc, do_noflush_start);
3455}
3456
Joe Thornber991d9fa2011-10-31 20:21:18 +00003457static void thin_postsuspend(struct dm_target *ti)
3458{
Joe Thornber738211f2014-03-03 15:52:28 +00003459 struct thin_c *tc = ti->private;
3460
3461 /*
3462 * The dm_noflush_suspending flag has been cleared by now, so
3463 * unfortunately we must always run this.
3464 */
3465 noflush_work(tc, do_noflush_stop);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003466}
3467
Joe Thornbere5aea7b2014-06-13 14:47:24 +01003468static int thin_preresume(struct dm_target *ti)
3469{
3470 struct thin_c *tc = ti->private;
3471
3472 if (tc->origin_dev)
3473 tc->origin_size = get_dev_size(tc->origin_dev->bdev);
3474
3475 return 0;
3476}
3477
Joe Thornber991d9fa2011-10-31 20:21:18 +00003478/*
3479 * <nr mapped sectors> <highest mapped sector>
3480 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003481static void thin_status(struct dm_target *ti, status_type_t type,
3482 unsigned status_flags, char *result, unsigned maxlen)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003483{
3484 int r;
3485 ssize_t sz = 0;
3486 dm_block_t mapped, highest;
3487 char buf[BDEVNAME_SIZE];
3488 struct thin_c *tc = ti->private;
3489
Joe Thornbere49e5822012-07-27 15:08:16 +01003490 if (get_pool_mode(tc->pool) == PM_FAIL) {
3491 DMEMIT("Fail");
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003492 return;
Joe Thornbere49e5822012-07-27 15:08:16 +01003493 }
3494
Joe Thornber991d9fa2011-10-31 20:21:18 +00003495 if (!tc->td)
3496 DMEMIT("-");
3497 else {
3498 switch (type) {
3499 case STATUSTYPE_INFO:
3500 r = dm_thin_get_mapped_count(tc->td, &mapped);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003501 if (r) {
3502 DMERR("dm_thin_get_mapped_count returned %d", r);
3503 goto err;
3504 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003505
3506 r = dm_thin_get_highest_mapped_block(tc->td, &highest);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003507 if (r < 0) {
3508 DMERR("dm_thin_get_highest_mapped_block returned %d", r);
3509 goto err;
3510 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003511
3512 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
3513 if (r)
3514 DMEMIT("%llu", ((highest + 1) *
3515 tc->pool->sectors_per_block) - 1);
3516 else
3517 DMEMIT("-");
3518 break;
3519
3520 case STATUSTYPE_TABLE:
3521 DMEMIT("%s %lu",
3522 format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
3523 (unsigned long) tc->dev_id);
Joe Thornber2dd9c252012-03-28 18:41:28 +01003524 if (tc->origin_dev)
3525 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
Joe Thornber991d9fa2011-10-31 20:21:18 +00003526 break;
3527 }
3528 }
3529
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003530 return;
3531
3532err:
3533 DMEMIT("Error");
Joe Thornber991d9fa2011-10-31 20:21:18 +00003534}
3535
3536static int thin_iterate_devices(struct dm_target *ti,
3537 iterate_devices_callout_fn fn, void *data)
3538{
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01003539 sector_t blocks;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003540 struct thin_c *tc = ti->private;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01003541 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003542
3543 /*
3544 * We can't call dm_pool_get_data_dev_size() since that blocks. So
3545 * we follow a more convoluted path through to the pool's target.
3546 */
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01003547 if (!pool->ti)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003548 return 0; /* nothing is bound */
3549
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01003550 blocks = pool->ti->len;
3551 (void) sector_div(blocks, pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003552 if (blocks)
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01003553 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003554
3555 return 0;
3556}
3557
Joe Thornber991d9fa2011-10-31 20:21:18 +00003558static struct target_type thin_target = {
3559 .name = "thin",
Joe Thornbere5aea7b2014-06-13 14:47:24 +01003560 .version = {1, 13, 0},
Joe Thornber991d9fa2011-10-31 20:21:18 +00003561 .module = THIS_MODULE,
3562 .ctr = thin_ctr,
3563 .dtr = thin_dtr,
3564 .map = thin_map,
Joe Thornbereb2aa482012-03-28 18:41:28 +01003565 .end_io = thin_endio,
Joe Thornbere5aea7b2014-06-13 14:47:24 +01003566 .preresume = thin_preresume,
Joe Thornber738211f2014-03-03 15:52:28 +00003567 .presuspend = thin_presuspend,
Joe Thornber991d9fa2011-10-31 20:21:18 +00003568 .postsuspend = thin_postsuspend,
3569 .status = thin_status,
3570 .iterate_devices = thin_iterate_devices,
Joe Thornber991d9fa2011-10-31 20:21:18 +00003571};
3572
3573/*----------------------------------------------------------------*/
3574
3575static int __init dm_thin_init(void)
3576{
3577 int r;
3578
3579 pool_table_init();
3580
3581 r = dm_register_target(&thin_target);
3582 if (r)
3583 return r;
3584
3585 r = dm_register_target(&pool_target);
3586 if (r)
Mike Snitzera24c2562012-06-03 00:30:00 +01003587 goto bad_pool_target;
3588
3589 r = -ENOMEM;
3590
Mike Snitzera24c2562012-06-03 00:30:00 +01003591 _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
3592 if (!_new_mapping_cache)
3593 goto bad_new_mapping_cache;
3594
Mike Snitzera24c2562012-06-03 00:30:00 +01003595 return 0;
3596
Mike Snitzera24c2562012-06-03 00:30:00 +01003597bad_new_mapping_cache:
Mike Snitzera24c2562012-06-03 00:30:00 +01003598 dm_unregister_target(&pool_target);
3599bad_pool_target:
3600 dm_unregister_target(&thin_target);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003601
3602 return r;
3603}
3604
3605static void dm_thin_exit(void)
3606{
3607 dm_unregister_target(&thin_target);
3608 dm_unregister_target(&pool_target);
Mike Snitzera24c2562012-06-03 00:30:00 +01003609
Mike Snitzera24c2562012-06-03 00:30:00 +01003610 kmem_cache_destroy(_new_mapping_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003611}
3612
3613module_init(dm_thin_init);
3614module_exit(dm_thin_exit);
3615
Mike Snitzer80c57892014-05-20 13:38:33 -04003616module_param_named(no_space_timeout, no_space_timeout_secs, uint, S_IRUGO | S_IWUSR);
3617MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
3618
Alasdair G Kergon7cab8bf2012-05-12 01:43:19 +01003619MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
Joe Thornber991d9fa2011-10-31 20:21:18 +00003620MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3621MODULE_LICENSE("GPL");