blob: 6fcbfb0633665a7c7b91d036b771cd997560e3de [file] [log] [blame]
Joe Thornber991d9fa2011-10-31 20:21:18 +00001/*
Joe Thornbere49e5822012-07-27 15:08:16 +01002 * Copyright (C) 2011-2012 Red Hat UK.
Joe Thornber991d9fa2011-10-31 20:21:18 +00003 *
4 * This file is released under the GPL.
5 */
6
7#include "dm-thin-metadata.h"
Mike Snitzer4f81a412012-10-12 21:02:13 +01008#include "dm-bio-prison.h"
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01009#include "dm.h"
Joe Thornber991d9fa2011-10-31 20:21:18 +000010
11#include <linux/device-mapper.h>
12#include <linux/dm-io.h>
13#include <linux/dm-kcopyd.h>
Manuel Schölling0f30af92014-05-22 22:42:37 +020014#include <linux/jiffies.h>
Mike Snitzer604ea902014-10-09 18:43:25 -040015#include <linux/log2.h>
Joe Thornber991d9fa2011-10-31 20:21:18 +000016#include <linux/list.h>
Mike Snitzerc140e1c2014-03-20 21:17:14 -040017#include <linux/rculist.h>
Joe Thornber991d9fa2011-10-31 20:21:18 +000018#include <linux/init.h>
19#include <linux/module.h>
20#include <linux/slab.h>
Joe Thornbera822c832015-07-03 10:22:42 +010021#include <linux/vmalloc.h>
Joe Thornberac4c3f32014-10-10 16:42:10 +010022#include <linux/sort.h>
Mike Snitzer67324ea2014-03-21 18:33:41 -040023#include <linux/rbtree.h>
Joe Thornber991d9fa2011-10-31 20:21:18 +000024
25#define DM_MSG_PREFIX "thin"
26
27/*
28 * Tunable constants
29 */
Alasdair G Kergon7768ed32012-07-27 15:07:57 +010030#define ENDIO_HOOK_POOL_SIZE 1024
Joe Thornber991d9fa2011-10-31 20:21:18 +000031#define MAPPING_POOL_SIZE 1024
Joe Thornber905e51b2012-03-28 18:41:27 +010032#define COMMIT_PERIOD HZ
Mike Snitzer80c57892014-05-20 13:38:33 -040033#define NO_SPACE_TIMEOUT_SECS 60
34
35static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
Joe Thornber991d9fa2011-10-31 20:21:18 +000036
Mikulas Patockadf5d2e92013-03-01 22:45:49 +000037DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
38 "A percentage of time allocated for copy on write");
39
Joe Thornber991d9fa2011-10-31 20:21:18 +000040/*
41 * The block size of the device holding pool data must be
42 * between 64KB and 1GB.
43 */
44#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
45#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
46
47/*
Joe Thornber991d9fa2011-10-31 20:21:18 +000048 * Device id is restricted to 24 bits.
49 */
50#define MAX_DEV_ID ((1 << 24) - 1)
51
52/*
53 * How do we handle breaking sharing of data blocks?
54 * =================================================
55 *
56 * We use a standard copy-on-write btree to store the mappings for the
57 * devices (note I'm talking about copy-on-write of the metadata here, not
58 * the data). When you take an internal snapshot you clone the root node
59 * of the origin btree. After this there is no concept of an origin or a
60 * snapshot. They are just two device trees that happen to point to the
61 * same data blocks.
62 *
63 * When we get a write in we decide if it's to a shared data block using
64 * some timestamp magic. If it is, we have to break sharing.
65 *
66 * Let's say we write to a shared block in what was the origin. The
67 * steps are:
68 *
69 * i) plug io further to this physical block. (see bio_prison code).
70 *
71 * ii) quiesce any read io to that shared data block. Obviously
Mike Snitzer44feb382012-10-12 21:02:10 +010072 * including all devices that share this block. (see dm_deferred_set code)
Joe Thornber991d9fa2011-10-31 20:21:18 +000073 *
74 * iii) copy the data block to a newly allocate block. This step can be
75 * missed out if the io covers the block. (schedule_copy).
76 *
77 * iv) insert the new mapping into the origin's btree
Joe Thornberfe878f32012-03-28 18:41:24 +010078 * (process_prepared_mapping). This act of inserting breaks some
Joe Thornber991d9fa2011-10-31 20:21:18 +000079 * sharing of btree nodes between the two devices. Breaking sharing only
80 * effects the btree of that specific device. Btrees for the other
81 * devices that share the block never change. The btree for the origin
82 * device as it was after the last commit is untouched, ie. we're using
83 * persistent data structures in the functional programming sense.
84 *
85 * v) unplug io to this physical block, including the io that triggered
86 * the breaking of sharing.
87 *
88 * Steps (ii) and (iii) occur in parallel.
89 *
90 * The metadata _doesn't_ need to be committed before the io continues. We
91 * get away with this because the io is always written to a _new_ block.
92 * If there's a crash, then:
93 *
94 * - The origin mapping will point to the old origin block (the shared
95 * one). This will contain the data as it was before the io that triggered
96 * the breaking of sharing came in.
97 *
98 * - The snap mapping still points to the old block. As it would after
99 * the commit.
100 *
101 * The downside of this scheme is the timestamp magic isn't perfect, and
102 * will continue to think that data block in the snapshot device is shared
103 * even after the write to the origin has broken sharing. I suspect data
104 * blocks will typically be shared by many different devices, so we're
105 * breaking sharing n + 1 times, rather than n, where n is the number of
106 * devices that reference this data block. At the moment I think the
107 * benefits far, far outweigh the disadvantages.
108 */
109
110/*----------------------------------------------------------------*/
111
112/*
Joe Thornber991d9fa2011-10-31 20:21:18 +0000113 * Key building.
114 */
Joe Thornber34fbcf62015-04-16 12:58:35 +0100115enum lock_space {
116 VIRTUAL,
117 PHYSICAL
118};
119
120static void build_key(struct dm_thin_device *td, enum lock_space ls,
121 dm_block_t b, dm_block_t e, struct dm_cell_key *key)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000122{
Joe Thornber34fbcf62015-04-16 12:58:35 +0100123 key->virtual = (ls == VIRTUAL);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000124 key->dev = dm_thin_dev_id(td);
Joe Thornber5f274d82014-09-17 10:17:39 +0100125 key->block_begin = b;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100126 key->block_end = e;
127}
128
129static void build_data_key(struct dm_thin_device *td, dm_block_t b,
130 struct dm_cell_key *key)
131{
132 build_key(td, PHYSICAL, b, b + 1llu, key);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000133}
134
135static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
Mike Snitzer44feb382012-10-12 21:02:10 +0100136 struct dm_cell_key *key)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000137{
Joe Thornber34fbcf62015-04-16 12:58:35 +0100138 build_key(td, VIRTUAL, b, b + 1llu, key);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000139}
140
141/*----------------------------------------------------------------*/
142
Joe Thornber7d327fe2014-10-06 15:45:59 +0100143#define THROTTLE_THRESHOLD (1 * HZ)
144
145struct throttle {
146 struct rw_semaphore lock;
147 unsigned long threshold;
148 bool throttle_applied;
149};
150
151static void throttle_init(struct throttle *t)
152{
153 init_rwsem(&t->lock);
154 t->throttle_applied = false;
155}
156
157static void throttle_work_start(struct throttle *t)
158{
159 t->threshold = jiffies + THROTTLE_THRESHOLD;
160}
161
162static void throttle_work_update(struct throttle *t)
163{
164 if (!t->throttle_applied && jiffies > t->threshold) {
165 down_write(&t->lock);
166 t->throttle_applied = true;
167 }
168}
169
170static void throttle_work_complete(struct throttle *t)
171{
172 if (t->throttle_applied) {
173 t->throttle_applied = false;
174 up_write(&t->lock);
175 }
176}
177
178static void throttle_lock(struct throttle *t)
179{
180 down_read(&t->lock);
181}
182
183static void throttle_unlock(struct throttle *t)
184{
185 up_read(&t->lock);
186}
187
188/*----------------------------------------------------------------*/
189
Joe Thornber991d9fa2011-10-31 20:21:18 +0000190/*
191 * A pool device ties together a metadata device and a data device. It
192 * also provides the interface for creating and destroying internal
193 * devices.
194 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100195struct dm_thin_new_mapping;
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100196
Joe Thornbere49e5822012-07-27 15:08:16 +0100197/*
Joe Thornber3e1a0692014-03-03 16:03:26 +0000198 * The pool runs in 4 modes. Ordered in degraded order for comparisons.
Joe Thornbere49e5822012-07-27 15:08:16 +0100199 */
200enum pool_mode {
201 PM_WRITE, /* metadata may be changed */
Joe Thornber3e1a0692014-03-03 16:03:26 +0000202 PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */
Joe Thornbere49e5822012-07-27 15:08:16 +0100203 PM_READ_ONLY, /* metadata may not be changed */
204 PM_FAIL, /* all I/O fails */
205};
206
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100207struct pool_features {
Joe Thornbere49e5822012-07-27 15:08:16 +0100208 enum pool_mode mode;
209
Mike Snitzer9bc142d2012-09-26 23:45:46 +0100210 bool zero_new_blocks:1;
211 bool discard_enabled:1;
212 bool discard_passdown:1;
Mike Snitzer787a996c2013-12-06 16:21:43 -0500213 bool error_if_no_space:1;
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100214};
215
Joe Thornbere49e5822012-07-27 15:08:16 +0100216struct thin_c;
217typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
Joe Thornbera374bb22014-10-10 13:43:14 +0100218typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
Joe Thornbere49e5822012-07-27 15:08:16 +0100219typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
220
Joe Thornberac4c3f32014-10-10 16:42:10 +0100221#define CELL_SORT_ARRAY_SIZE 8192
222
Joe Thornber991d9fa2011-10-31 20:21:18 +0000223struct pool {
224 struct list_head list;
225 struct dm_target *ti; /* Only set if a pool target is bound */
226
227 struct mapped_device *pool_md;
228 struct block_device *md_dev;
229 struct dm_pool_metadata *pmd;
230
Joe Thornber991d9fa2011-10-31 20:21:18 +0000231 dm_block_t low_water_blocks;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100232 uint32_t sectors_per_block;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100233 int sectors_per_block_shift;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000234
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100235 struct pool_features pf;
Joe Thornber88a66212013-12-04 20:16:12 -0500236 bool low_water_triggered:1; /* A dm event has been sent */
Mike Snitzer80e96c52014-11-07 15:09:46 -0500237 bool suspended:1;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000238
Mike Snitzer44feb382012-10-12 21:02:10 +0100239 struct dm_bio_prison *prison;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000240 struct dm_kcopyd_client *copier;
241
242 struct workqueue_struct *wq;
Joe Thornber7d327fe2014-10-06 15:45:59 +0100243 struct throttle throttle;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000244 struct work_struct worker;
Joe Thornber905e51b2012-03-28 18:41:27 +0100245 struct delayed_work waker;
Joe Thornber85ad643b2014-05-09 15:59:38 +0100246 struct delayed_work no_space_timeout;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000247
Joe Thornber905e51b2012-03-28 18:41:27 +0100248 unsigned long last_commit_jiffies;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100249 unsigned ref_count;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000250
251 spinlock_t lock;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000252 struct bio_list deferred_flush_bios;
253 struct list_head prepared_mappings;
Joe Thornber104655f2012-03-28 18:41:28 +0100254 struct list_head prepared_discards;
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400255 struct list_head active_thins;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000256
Mike Snitzer44feb382012-10-12 21:02:10 +0100257 struct dm_deferred_set *shared_read_ds;
258 struct dm_deferred_set *all_io_ds;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000259
Mike Snitzera24c2562012-06-03 00:30:00 +0100260 struct dm_thin_new_mapping *next_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000261 mempool_t *mapping_pool;
Joe Thornbere49e5822012-07-27 15:08:16 +0100262
263 process_bio_fn process_bio;
264 process_bio_fn process_discard;
265
Joe Thornbera374bb22014-10-10 13:43:14 +0100266 process_cell_fn process_cell;
267 process_cell_fn process_discard_cell;
268
Joe Thornbere49e5822012-07-27 15:08:16 +0100269 process_mapping_fn process_prepared_mapping;
270 process_mapping_fn process_prepared_discard;
Joe Thornberac4c3f32014-10-10 16:42:10 +0100271
Joe Thornbera822c832015-07-03 10:22:42 +0100272 struct dm_bio_prison_cell **cell_sort_array;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000273};
274
Joe Thornbere49e5822012-07-27 15:08:16 +0100275static enum pool_mode get_pool_mode(struct pool *pool);
Joe Thornberb5330652013-12-04 19:51:33 -0500276static void metadata_operation_failed(struct pool *pool, const char *op, int r);
Joe Thornbere49e5822012-07-27 15:08:16 +0100277
Joe Thornber991d9fa2011-10-31 20:21:18 +0000278/*
279 * Target context for a pool.
280 */
281struct pool_c {
282 struct dm_target *ti;
283 struct pool *pool;
284 struct dm_dev *data_dev;
285 struct dm_dev *metadata_dev;
286 struct dm_target_callbacks callbacks;
287
288 dm_block_t low_water_blocks;
Mike Snitzer0424caa2012-09-26 23:45:47 +0100289 struct pool_features requested_pf; /* Features requested during table load */
290 struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000291};
292
293/*
294 * Target context for a thin.
295 */
296struct thin_c {
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400297 struct list_head list;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000298 struct dm_dev *pool_dev;
Joe Thornber2dd9c252012-03-28 18:41:28 +0100299 struct dm_dev *origin_dev;
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100300 sector_t origin_size;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000301 dm_thin_id dev_id;
302
303 struct pool *pool;
304 struct dm_thin_device *td;
Mike Snitzer583024d2014-10-28 20:58:45 -0400305 struct mapped_device *thin_md;
306
Joe Thornber738211f2014-03-03 15:52:28 +0000307 bool requeue_mode:1;
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400308 spinlock_t lock;
Joe Thornbera374bb22014-10-10 13:43:14 +0100309 struct list_head deferred_cells;
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400310 struct bio_list deferred_bio_list;
311 struct bio_list retry_on_resume_list;
Mike Snitzer67324ea2014-03-21 18:33:41 -0400312 struct rb_root sort_bio_list; /* sorted list of deferred bios */
Joe Thornberb10ebd32014-04-08 11:29:01 +0100313
314 /*
315 * Ensures the thin is not destroyed until the worker has finished
316 * iterating the active_thins list.
317 */
318 atomic_t refcount;
319 struct completion can_destroy;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000320};
321
322/*----------------------------------------------------------------*/
323
Joe Thornber34fbcf62015-04-16 12:58:35 +0100324/**
325 * __blkdev_issue_discard_async - queue a discard with async completion
326 * @bdev: blockdev to issue discard for
327 * @sector: start sector
328 * @nr_sects: number of sectors to discard
329 * @gfp_mask: memory allocation flags (for bio_alloc)
330 * @flags: BLKDEV_IFL_* flags to control behaviour
331 * @parent_bio: parent discard bio that all sub discards get chained to
332 *
333 * Description:
334 * Asynchronously issue a discard request for the sectors in question.
Joe Thornber34fbcf62015-04-16 12:58:35 +0100335 */
336static int __blkdev_issue_discard_async(struct block_device *bdev, sector_t sector,
337 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags,
338 struct bio *parent_bio)
339{
340 struct request_queue *q = bdev_get_queue(bdev);
341 int type = REQ_WRITE | REQ_DISCARD;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100342 struct bio *bio;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100343
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400344 if (!q || !nr_sects)
Joe Thornber34fbcf62015-04-16 12:58:35 +0100345 return -ENXIO;
346
347 if (!blk_queue_discard(q))
348 return -EOPNOTSUPP;
349
Joe Thornber34fbcf62015-04-16 12:58:35 +0100350 if (flags & BLKDEV_DISCARD_SECURE) {
351 if (!blk_queue_secdiscard(q))
352 return -EOPNOTSUPP;
353 type |= REQ_SECURE;
354 }
355
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400356 /*
357 * Required bio_put occurs in bio_endio thanks to bio_chain below
358 */
359 bio = bio_alloc(gfp_mask, 1);
360 if (!bio)
361 return -ENOMEM;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100362
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400363 bio_chain(bio, parent_bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +0100364
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400365 bio->bi_iter.bi_sector = sector;
366 bio->bi_bdev = bdev;
367 bio->bi_iter.bi_size = nr_sects << 9;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100368
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400369 submit_bio(type, bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +0100370
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400371 return 0;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100372}
373
374static bool block_size_is_power_of_two(struct pool *pool)
375{
376 return pool->sectors_per_block_shift >= 0;
377}
378
379static sector_t block_to_sectors(struct pool *pool, dm_block_t b)
380{
381 return block_size_is_power_of_two(pool) ?
382 (b << pool->sectors_per_block_shift) :
383 (b * pool->sectors_per_block);
384}
385
386static int issue_discard(struct thin_c *tc, dm_block_t data_b, dm_block_t data_e,
387 struct bio *parent_bio)
388{
389 sector_t s = block_to_sectors(tc->pool, data_b);
390 sector_t len = block_to_sectors(tc->pool, data_e - data_b);
391
392 return __blkdev_issue_discard_async(tc->pool_dev->bdev, s, len,
393 GFP_NOWAIT, 0, parent_bio);
394}
395
396/*----------------------------------------------------------------*/
397
Joe Thornber025b9682013-03-01 22:45:50 +0000398/*
399 * wake_worker() is used when new work is queued and when pool_resume is
400 * ready to continue deferred IO processing.
401 */
402static void wake_worker(struct pool *pool)
403{
404 queue_work(pool->wq, &pool->worker);
405}
406
407/*----------------------------------------------------------------*/
408
Joe Thornber6beca5e2013-03-01 22:45:50 +0000409static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
410 struct dm_bio_prison_cell **cell_result)
411{
412 int r;
413 struct dm_bio_prison_cell *cell_prealloc;
414
415 /*
416 * Allocate a cell from the prison's mempool.
417 * This might block but it can't fail.
418 */
419 cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);
420
421 r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);
422 if (r)
423 /*
424 * We reused an old cell; we can get rid of
425 * the new one.
426 */
427 dm_bio_prison_free_cell(pool->prison, cell_prealloc);
428
429 return r;
430}
431
432static void cell_release(struct pool *pool,
433 struct dm_bio_prison_cell *cell,
434 struct bio_list *bios)
435{
436 dm_cell_release(pool->prison, cell, bios);
437 dm_bio_prison_free_cell(pool->prison, cell);
438}
439
Joe Thornber2d759a42014-10-10 15:27:16 +0100440static void cell_visit_release(struct pool *pool,
441 void (*fn)(void *, struct dm_bio_prison_cell *),
442 void *context,
443 struct dm_bio_prison_cell *cell)
444{
445 dm_cell_visit_release(pool->prison, fn, context, cell);
446 dm_bio_prison_free_cell(pool->prison, cell);
447}
448
Joe Thornber6beca5e2013-03-01 22:45:50 +0000449static void cell_release_no_holder(struct pool *pool,
450 struct dm_bio_prison_cell *cell,
451 struct bio_list *bios)
452{
453 dm_cell_release_no_holder(pool->prison, cell, bios);
454 dm_bio_prison_free_cell(pool->prison, cell);
455}
456
Mike Snitzeraf918052014-05-22 14:32:51 -0400457static void cell_error_with_code(struct pool *pool,
458 struct dm_bio_prison_cell *cell, int error_code)
Joe Thornber6beca5e2013-03-01 22:45:50 +0000459{
Mike Snitzeraf918052014-05-22 14:32:51 -0400460 dm_cell_error(pool->prison, cell, error_code);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000461 dm_bio_prison_free_cell(pool->prison, cell);
462}
463
Mike Snitzeraf918052014-05-22 14:32:51 -0400464static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
465{
466 cell_error_with_code(pool, cell, -EIO);
467}
468
Joe Thornbera374bb22014-10-10 13:43:14 +0100469static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)
470{
471 cell_error_with_code(pool, cell, 0);
472}
473
474static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)
475{
476 cell_error_with_code(pool, cell, DM_ENDIO_REQUEUE);
477}
478
Joe Thornber6beca5e2013-03-01 22:45:50 +0000479/*----------------------------------------------------------------*/
480
Joe Thornber991d9fa2011-10-31 20:21:18 +0000481/*
482 * A global list of pools that uses a struct mapped_device as a key.
483 */
484static struct dm_thin_pool_table {
485 struct mutex mutex;
486 struct list_head pools;
487} dm_thin_pool_table;
488
489static void pool_table_init(void)
490{
491 mutex_init(&dm_thin_pool_table.mutex);
492 INIT_LIST_HEAD(&dm_thin_pool_table.pools);
493}
494
495static void __pool_table_insert(struct pool *pool)
496{
497 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
498 list_add(&pool->list, &dm_thin_pool_table.pools);
499}
500
501static void __pool_table_remove(struct pool *pool)
502{
503 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
504 list_del(&pool->list);
505}
506
507static struct pool *__pool_table_lookup(struct mapped_device *md)
508{
509 struct pool *pool = NULL, *tmp;
510
511 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
512
513 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
514 if (tmp->pool_md == md) {
515 pool = tmp;
516 break;
517 }
518 }
519
520 return pool;
521}
522
523static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
524{
525 struct pool *pool = NULL, *tmp;
526
527 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
528
529 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
530 if (tmp->md_dev == md_dev) {
531 pool = tmp;
532 break;
533 }
534 }
535
536 return pool;
537}
538
539/*----------------------------------------------------------------*/
540
Mike Snitzera24c2562012-06-03 00:30:00 +0100541struct dm_thin_endio_hook {
Joe Thornbereb2aa482012-03-28 18:41:28 +0100542 struct thin_c *tc;
Mike Snitzer44feb382012-10-12 21:02:10 +0100543 struct dm_deferred_entry *shared_read_entry;
544 struct dm_deferred_entry *all_io_entry;
Mike Snitzera24c2562012-06-03 00:30:00 +0100545 struct dm_thin_new_mapping *overwrite_mapping;
Mike Snitzer67324ea2014-03-21 18:33:41 -0400546 struct rb_node rb_node;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100547 struct dm_bio_prison_cell *cell;
Joe Thornbereb2aa482012-03-28 18:41:28 +0100548};
549
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400550static void __merge_bio_list(struct bio_list *bios, struct bio_list *master)
551{
552 bio_list_merge(bios, master);
553 bio_list_init(master);
554}
555
556static void error_bio_list(struct bio_list *bios, int error)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000557{
558 struct bio *bio;
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400559
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200560 while ((bio = bio_list_pop(bios))) {
561 bio->bi_error = error;
562 bio_endio(bio);
563 }
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400564}
565
566static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, int error)
567{
Joe Thornber991d9fa2011-10-31 20:21:18 +0000568 struct bio_list bios;
Joe Thornber18adc572014-03-03 15:46:42 +0000569 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000570
571 bio_list_init(&bios);
Joe Thornber18adc572014-03-03 15:46:42 +0000572
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400573 spin_lock_irqsave(&tc->lock, flags);
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400574 __merge_bio_list(&bios, master);
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400575 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000576
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400577 error_bio_list(&bios, error);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000578}
579
Joe Thornbera374bb22014-10-10 13:43:14 +0100580static void requeue_deferred_cells(struct thin_c *tc)
581{
582 struct pool *pool = tc->pool;
583 unsigned long flags;
584 struct list_head cells;
585 struct dm_bio_prison_cell *cell, *tmp;
586
587 INIT_LIST_HEAD(&cells);
588
589 spin_lock_irqsave(&tc->lock, flags);
590 list_splice_init(&tc->deferred_cells, &cells);
591 spin_unlock_irqrestore(&tc->lock, flags);
592
593 list_for_each_entry_safe(cell, tmp, &cells, user_list)
594 cell_requeue(pool, cell);
595}
596
Joe Thornber991d9fa2011-10-31 20:21:18 +0000597static void requeue_io(struct thin_c *tc)
598{
Joe Thornber3e1a0692014-03-03 16:03:26 +0000599 struct bio_list bios;
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400600 unsigned long flags;
Joe Thornber3e1a0692014-03-03 16:03:26 +0000601
602 bio_list_init(&bios);
603
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400604 spin_lock_irqsave(&tc->lock, flags);
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400605 __merge_bio_list(&bios, &tc->deferred_bio_list);
606 __merge_bio_list(&bios, &tc->retry_on_resume_list);
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400607 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber3e1a0692014-03-03 16:03:26 +0000608
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400609 error_bio_list(&bios, DM_ENDIO_REQUEUE);
610 requeue_deferred_cells(tc);
Joe Thornber3e1a0692014-03-03 16:03:26 +0000611}
612
Mike Snitzer0a927c22015-07-21 13:20:46 -0400613static void error_retry_list_with_code(struct pool *pool, int error)
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400614{
615 struct thin_c *tc;
616
617 rcu_read_lock();
618 list_for_each_entry_rcu(tc, &pool->active_thins, list)
Mike Snitzer0a927c22015-07-21 13:20:46 -0400619 error_thin_bio_list(tc, &tc->retry_on_resume_list, error);
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400620 rcu_read_unlock();
621}
622
Mike Snitzer0a927c22015-07-21 13:20:46 -0400623static void error_retry_list(struct pool *pool)
624{
625 return error_retry_list_with_code(pool, -EIO);
626}
627
Joe Thornber991d9fa2011-10-31 20:21:18 +0000628/*
629 * This section of code contains the logic for processing a thin device's IO.
630 * Much of the code depends on pool object resources (lists, workqueues, etc)
631 * but most is exclusively called from the thin target rather than the thin-pool
632 * target.
633 */
634
635static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
636{
Mike Snitzer58f77a22013-03-01 22:45:45 +0000637 struct pool *pool = tc->pool;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700638 sector_t block_nr = bio->bi_iter.bi_sector;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100639
Mike Snitzer58f77a22013-03-01 22:45:45 +0000640 if (block_size_is_power_of_two(pool))
641 block_nr >>= pool->sectors_per_block_shift;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100642 else
Mike Snitzer58f77a22013-03-01 22:45:45 +0000643 (void) sector_div(block_nr, pool->sectors_per_block);
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100644
645 return block_nr;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000646}
647
Joe Thornber34fbcf62015-04-16 12:58:35 +0100648/*
649 * Returns the _complete_ blocks that this bio covers.
650 */
651static void get_bio_block_range(struct thin_c *tc, struct bio *bio,
652 dm_block_t *begin, dm_block_t *end)
653{
654 struct pool *pool = tc->pool;
655 sector_t b = bio->bi_iter.bi_sector;
656 sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
657
658 b += pool->sectors_per_block - 1ull; /* so we round up */
659
660 if (block_size_is_power_of_two(pool)) {
661 b >>= pool->sectors_per_block_shift;
662 e >>= pool->sectors_per_block_shift;
663 } else {
664 (void) sector_div(b, pool->sectors_per_block);
665 (void) sector_div(e, pool->sectors_per_block);
666 }
667
668 if (e < b)
669 /* Can happen if the bio is within a single block. */
670 e = b;
671
672 *begin = b;
673 *end = e;
674}
675
Joe Thornber991d9fa2011-10-31 20:21:18 +0000676static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
677{
678 struct pool *pool = tc->pool;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700679 sector_t bi_sector = bio->bi_iter.bi_sector;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000680
681 bio->bi_bdev = tc->pool_dev->bdev;
Mike Snitzer58f77a22013-03-01 22:45:45 +0000682 if (block_size_is_power_of_two(pool))
Kent Overstreet4f024f32013-10-11 15:44:27 -0700683 bio->bi_iter.bi_sector =
684 (block << pool->sectors_per_block_shift) |
685 (bi_sector & (pool->sectors_per_block - 1));
Mike Snitzer58f77a22013-03-01 22:45:45 +0000686 else
Kent Overstreet4f024f32013-10-11 15:44:27 -0700687 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
Mike Snitzer58f77a22013-03-01 22:45:45 +0000688 sector_div(bi_sector, pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000689}
690
Joe Thornber2dd9c252012-03-28 18:41:28 +0100691static void remap_to_origin(struct thin_c *tc, struct bio *bio)
692{
693 bio->bi_bdev = tc->origin_dev->bdev;
694}
695
Joe Thornber4afdd682012-07-27 15:08:14 +0100696static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
697{
698 return (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) &&
699 dm_thin_changed_this_transaction(tc->td);
700}
701
Joe Thornbere8088072012-12-21 20:23:31 +0000702static void inc_all_io_entry(struct pool *pool, struct bio *bio)
703{
704 struct dm_thin_endio_hook *h;
705
706 if (bio->bi_rw & REQ_DISCARD)
707 return;
708
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +0000709 h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbere8088072012-12-21 20:23:31 +0000710 h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
711}
712
Joe Thornber2dd9c252012-03-28 18:41:28 +0100713static void issue(struct thin_c *tc, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000714{
715 struct pool *pool = tc->pool;
716 unsigned long flags;
717
Joe Thornbere49e5822012-07-27 15:08:16 +0100718 if (!bio_triggers_commit(tc, bio)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000719 generic_make_request(bio);
Joe Thornbere49e5822012-07-27 15:08:16 +0100720 return;
721 }
722
723 /*
724 * Complete bio with an error if earlier I/O caused changes to
725 * the metadata that can't be committed e.g, due to I/O errors
726 * on the metadata device.
727 */
728 if (dm_thin_aborted_changes(tc->td)) {
729 bio_io_error(bio);
730 return;
731 }
732
733 /*
734 * Batch together any bios that trigger commits and then issue a
735 * single commit for them in process_deferred_bios().
736 */
737 spin_lock_irqsave(&pool->lock, flags);
738 bio_list_add(&pool->deferred_flush_bios, bio);
739 spin_unlock_irqrestore(&pool->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000740}
741
Joe Thornber2dd9c252012-03-28 18:41:28 +0100742static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
743{
744 remap_to_origin(tc, bio);
745 issue(tc, bio);
746}
747
748static void remap_and_issue(struct thin_c *tc, struct bio *bio,
749 dm_block_t block)
750{
751 remap(tc, bio, block);
752 issue(tc, bio);
753}
754
Joe Thornber991d9fa2011-10-31 20:21:18 +0000755/*----------------------------------------------------------------*/
756
757/*
758 * Bio endio functions.
759 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100760struct dm_thin_new_mapping {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000761 struct list_head list;
762
Mike Snitzer7f214662013-12-17 13:43:31 -0500763 bool pass_discard:1;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100764 bool maybe_shared:1;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000765
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100766 /*
767 * Track quiescing, copying and zeroing preparation actions. When this
768 * counter hits zero the block is prepared and can be inserted into the
769 * btree.
770 */
771 atomic_t prepare_actions;
772
Mike Snitzer7f214662013-12-17 13:43:31 -0500773 int err;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000774 struct thin_c *tc;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100775 dm_block_t virt_begin, virt_end;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000776 dm_block_t data_block;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100777 struct dm_bio_prison_cell *cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000778
779 /*
780 * If the bio covers the whole area of a block then we can avoid
781 * zeroing or copying. Instead this bio is hooked. The bio will
782 * still be in the cell, so care has to be taken to avoid issuing
783 * the bio twice.
784 */
785 struct bio *bio;
786 bio_end_io_t *saved_bi_end_io;
787};
788
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100789static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000790{
791 struct pool *pool = m->tc->pool;
792
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100793 if (atomic_dec_and_test(&m->prepare_actions)) {
Mike Snitzerdaec3382013-12-11 14:01:20 -0500794 list_add_tail(&m->list, &pool->prepared_mappings);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000795 wake_worker(pool);
796 }
797}
798
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100799static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000800{
801 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000802 struct pool *pool = m->tc->pool;
803
Joe Thornber991d9fa2011-10-31 20:21:18 +0000804 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100805 __complete_mapping_preparation(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000806 spin_unlock_irqrestore(&pool->lock, flags);
807}
808
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100809static void copy_complete(int read_err, unsigned long write_err, void *context)
810{
811 struct dm_thin_new_mapping *m = context;
812
813 m->err = read_err || write_err ? -EIO : 0;
814 complete_mapping_preparation(m);
815}
816
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200817static void overwrite_endio(struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000818{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +0000819 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Mike Snitzera24c2562012-06-03 00:30:00 +0100820 struct dm_thin_new_mapping *m = h->overwrite_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000821
Mike Snitzer8b908f82015-05-13 17:53:13 -0400822 bio->bi_end_io = m->saved_bi_end_io;
823
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200824 m->err = bio->bi_error;
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100825 complete_mapping_preparation(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000826}
827
Joe Thornber991d9fa2011-10-31 20:21:18 +0000828/*----------------------------------------------------------------*/
829
830/*
831 * Workqueue.
832 */
833
834/*
835 * Prepared mapping jobs.
836 */
837
838/*
Joe Thornber2d759a42014-10-10 15:27:16 +0100839 * This sends the bios in the cell, except the original holder, back
840 * to the deferred_bios list.
Joe Thornber991d9fa2011-10-31 20:21:18 +0000841 */
Joe Thornberf286ba02012-12-21 20:23:33 +0000842static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000843{
Joe Thornber991d9fa2011-10-31 20:21:18 +0000844 struct pool *pool = tc->pool;
845 unsigned long flags;
846
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400847 spin_lock_irqsave(&tc->lock, flags);
848 cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
849 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000850
851 wake_worker(pool);
852}
853
Joe Thornbera374bb22014-10-10 13:43:14 +0100854static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
855
Joe Thornber2d759a42014-10-10 15:27:16 +0100856struct remap_info {
857 struct thin_c *tc;
858 struct bio_list defer_bios;
859 struct bio_list issue_bios;
860};
861
862static void __inc_remap_and_issue_cell(void *context,
863 struct dm_bio_prison_cell *cell)
864{
865 struct remap_info *info = context;
866 struct bio *bio;
867
868 while ((bio = bio_list_pop(&cell->bios))) {
869 if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA))
870 bio_list_add(&info->defer_bios, bio);
871 else {
872 inc_all_io_entry(info->tc->pool, bio);
873
874 /*
875 * We can't issue the bios with the bio prison lock
876 * held, so we add them to a list to issue on
877 * return from this function.
878 */
879 bio_list_add(&info->issue_bios, bio);
880 }
881 }
882}
883
Joe Thornbera374bb22014-10-10 13:43:14 +0100884static void inc_remap_and_issue_cell(struct thin_c *tc,
885 struct dm_bio_prison_cell *cell,
886 dm_block_t block)
887{
888 struct bio *bio;
Joe Thornber2d759a42014-10-10 15:27:16 +0100889 struct remap_info info;
Joe Thornbera374bb22014-10-10 13:43:14 +0100890
Joe Thornber2d759a42014-10-10 15:27:16 +0100891 info.tc = tc;
892 bio_list_init(&info.defer_bios);
893 bio_list_init(&info.issue_bios);
Joe Thornbera374bb22014-10-10 13:43:14 +0100894
Joe Thornber2d759a42014-10-10 15:27:16 +0100895 /*
896 * We have to be careful to inc any bios we're about to issue
897 * before the cell is released, and avoid a race with new bios
898 * being added to the cell.
899 */
900 cell_visit_release(tc->pool, __inc_remap_and_issue_cell,
901 &info, cell);
902
903 while ((bio = bio_list_pop(&info.defer_bios)))
904 thin_defer_bio(tc, bio);
905
906 while ((bio = bio_list_pop(&info.issue_bios)))
907 remap_and_issue(info.tc, bio, block);
Joe Thornbera374bb22014-10-10 13:43:14 +0100908}
909
Joe Thornbere49e5822012-07-27 15:08:16 +0100910static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
911{
Joe Thornber6beca5e2013-03-01 22:45:50 +0000912 cell_error(m->tc->pool, m->cell);
Joe Thornbere49e5822012-07-27 15:08:16 +0100913 list_del(&m->list);
914 mempool_free(m, m->tc->pool->mapping_pool);
915}
Joe Thornber025b9682013-03-01 22:45:50 +0000916
Mike Snitzera24c2562012-06-03 00:30:00 +0100917static void process_prepared_mapping(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000918{
919 struct thin_c *tc = m->tc;
Joe Thornber6beca5e2013-03-01 22:45:50 +0000920 struct pool *pool = tc->pool;
Mike Snitzer8b908f82015-05-13 17:53:13 -0400921 struct bio *bio = m->bio;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000922 int r;
923
Joe Thornber991d9fa2011-10-31 20:21:18 +0000924 if (m->err) {
Joe Thornber6beca5e2013-03-01 22:45:50 +0000925 cell_error(pool, m->cell);
Joe Thornber905386f2012-07-27 15:08:05 +0100926 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000927 }
928
929 /*
930 * Commit the prepared block into the mapping btree.
931 * Any I/O for this block arriving after this point will get
932 * remapped to it directly.
933 */
Joe Thornber34fbcf62015-04-16 12:58:35 +0100934 r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000935 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -0500936 metadata_operation_failed(pool, "dm_thin_insert_block", r);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000937 cell_error(pool, m->cell);
Joe Thornber905386f2012-07-27 15:08:05 +0100938 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000939 }
940
941 /*
942 * Release any bios held while the block was being provisioned.
943 * If we are processing a write bio that completely covers the block,
944 * we already processed it so can ignore it now when processing
945 * the bios in the cell.
946 */
947 if (bio) {
Joe Thornber2d759a42014-10-10 15:27:16 +0100948 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200949 bio_endio(bio);
Joe Thornber2d759a42014-10-10 15:27:16 +0100950 } else {
951 inc_all_io_entry(tc->pool, m->cell->holder);
952 remap_and_issue(tc, m->cell->holder, m->data_block);
953 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
954 }
Joe Thornber991d9fa2011-10-31 20:21:18 +0000955
Joe Thornber905386f2012-07-27 15:08:05 +0100956out:
Joe Thornber991d9fa2011-10-31 20:21:18 +0000957 list_del(&m->list);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000958 mempool_free(m, pool->mapping_pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000959}
960
Joe Thornber34fbcf62015-04-16 12:58:35 +0100961/*----------------------------------------------------------------*/
962
963static void free_discard_mapping(struct dm_thin_new_mapping *m)
964{
965 struct thin_c *tc = m->tc;
966 if (m->cell)
967 cell_defer_no_holder(tc, m->cell);
968 mempool_free(m, tc->pool->mapping_pool);
969}
970
Joe Thornbere49e5822012-07-27 15:08:16 +0100971static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
Joe Thornber104655f2012-03-28 18:41:28 +0100972{
Joe Thornbere49e5822012-07-27 15:08:16 +0100973 bio_io_error(m->bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +0100974 free_discard_mapping(m);
Joe Thornbere49e5822012-07-27 15:08:16 +0100975}
Joe Thornber104655f2012-03-28 18:41:28 +0100976
Joe Thornber34fbcf62015-04-16 12:58:35 +0100977static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
Joe Thornbere49e5822012-07-27 15:08:16 +0100978{
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200979 bio_endio(m->bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +0100980 free_discard_mapping(m);
Joe Thornber104655f2012-03-28 18:41:28 +0100981}
982
Joe Thornber34fbcf62015-04-16 12:58:35 +0100983static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
Joe Thornbere49e5822012-07-27 15:08:16 +0100984{
985 int r;
986 struct thin_c *tc = m->tc;
987
Joe Thornber34fbcf62015-04-16 12:58:35 +0100988 r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);
989 if (r) {
990 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);
991 bio_io_error(m->bio);
992 } else
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200993 bio_endio(m->bio);
Joe Thornbere49e5822012-07-27 15:08:16 +0100994
Joe Thornber34fbcf62015-04-16 12:58:35 +0100995 cell_defer_no_holder(tc, m->cell);
996 mempool_free(m, tc->pool->mapping_pool);
997}
998
999static int passdown_double_checking_shared_status(struct dm_thin_new_mapping *m)
1000{
1001 /*
1002 * We've already unmapped this range of blocks, but before we
1003 * passdown we have to check that these blocks are now unused.
1004 */
1005 int r;
1006 bool used = true;
1007 struct thin_c *tc = m->tc;
1008 struct pool *pool = tc->pool;
1009 dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;
1010
1011 while (b != end) {
1012 /* find start of unmapped run */
1013 for (; b < end; b++) {
1014 r = dm_pool_block_is_used(pool->pmd, b, &used);
1015 if (r)
1016 return r;
1017
1018 if (!used)
1019 break;
1020 }
1021
1022 if (b == end)
1023 break;
1024
1025 /* find end of run */
1026 for (e = b + 1; e != end; e++) {
1027 r = dm_pool_block_is_used(pool->pmd, e, &used);
1028 if (r)
1029 return r;
1030
1031 if (used)
1032 break;
1033 }
1034
1035 r = issue_discard(tc, b, e, m->bio);
1036 if (r)
1037 return r;
1038
1039 b = e;
1040 }
1041
1042 return 0;
1043}
1044
1045static void process_prepared_discard_passdown(struct dm_thin_new_mapping *m)
1046{
1047 int r;
1048 struct thin_c *tc = m->tc;
1049 struct pool *pool = tc->pool;
1050
1051 r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end);
1052 if (r)
1053 metadata_operation_failed(pool, "dm_thin_remove_range", r);
1054
1055 else if (m->maybe_shared)
1056 r = passdown_double_checking_shared_status(m);
1057 else
1058 r = issue_discard(tc, m->data_block, m->data_block + (m->virt_end - m->virt_begin), m->bio);
1059
1060 /*
1061 * Even if r is set, there could be sub discards in flight that we
1062 * need to wait for.
1063 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001064 m->bio->bi_error = r;
1065 bio_endio(m->bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +01001066 cell_defer_no_holder(tc, m->cell);
1067 mempool_free(m, pool->mapping_pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01001068}
1069
Joe Thornber104655f2012-03-28 18:41:28 +01001070static void process_prepared(struct pool *pool, struct list_head *head,
Joe Thornbere49e5822012-07-27 15:08:16 +01001071 process_mapping_fn *fn)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001072{
1073 unsigned long flags;
1074 struct list_head maps;
Mike Snitzera24c2562012-06-03 00:30:00 +01001075 struct dm_thin_new_mapping *m, *tmp;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001076
1077 INIT_LIST_HEAD(&maps);
1078 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +01001079 list_splice_init(head, &maps);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001080 spin_unlock_irqrestore(&pool->lock, flags);
1081
1082 list_for_each_entry_safe(m, tmp, &maps, list)
Joe Thornbere49e5822012-07-27 15:08:16 +01001083 (*fn)(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001084}
1085
1086/*
1087 * Deferred bio jobs.
1088 */
Joe Thornber104655f2012-03-28 18:41:28 +01001089static int io_overlaps_block(struct pool *pool, struct bio *bio)
1090{
Kent Overstreet4f024f32013-10-11 15:44:27 -07001091 return bio->bi_iter.bi_size ==
1092 (pool->sectors_per_block << SECTOR_SHIFT);
Joe Thornber104655f2012-03-28 18:41:28 +01001093}
1094
Joe Thornber991d9fa2011-10-31 20:21:18 +00001095static int io_overwrites_block(struct pool *pool, struct bio *bio)
1096{
Joe Thornber104655f2012-03-28 18:41:28 +01001097 return (bio_data_dir(bio) == WRITE) &&
1098 io_overlaps_block(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001099}
1100
1101static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
1102 bio_end_io_t *fn)
1103{
1104 *save = bio->bi_end_io;
1105 bio->bi_end_io = fn;
1106}
1107
1108static int ensure_next_mapping(struct pool *pool)
1109{
1110 if (pool->next_mapping)
1111 return 0;
1112
1113 pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
1114
1115 return pool->next_mapping ? 0 : -ENOMEM;
1116}
1117
Mike Snitzera24c2562012-06-03 00:30:00 +01001118static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001119{
Mike Snitzer16961b02013-12-17 13:19:11 -05001120 struct dm_thin_new_mapping *m = pool->next_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001121
1122 BUG_ON(!pool->next_mapping);
1123
Mike Snitzer16961b02013-12-17 13:19:11 -05001124 memset(m, 0, sizeof(struct dm_thin_new_mapping));
1125 INIT_LIST_HEAD(&m->list);
1126 m->bio = NULL;
1127
Joe Thornber991d9fa2011-10-31 20:21:18 +00001128 pool->next_mapping = NULL;
1129
Mike Snitzer16961b02013-12-17 13:19:11 -05001130 return m;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001131}
1132
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001133static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
1134 sector_t begin, sector_t end)
1135{
1136 int r;
1137 struct dm_io_region to;
1138
1139 to.bdev = tc->pool_dev->bdev;
1140 to.sector = begin;
1141 to.count = end - begin;
1142
1143 r = dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);
1144 if (r < 0) {
1145 DMERR_LIMIT("dm_kcopyd_zero() failed");
1146 copy_complete(1, 1, m);
1147 }
1148}
1149
Mike Snitzer452d7a62014-10-09 19:20:21 -04001150static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,
Joe Thornber34fbcf62015-04-16 12:58:35 +01001151 dm_block_t data_begin,
Mike Snitzer452d7a62014-10-09 19:20:21 -04001152 struct dm_thin_new_mapping *m)
1153{
1154 struct pool *pool = tc->pool;
1155 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1156
1157 h->overwrite_mapping = m;
1158 m->bio = bio;
1159 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
1160 inc_all_io_entry(pool, bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +01001161 remap_and_issue(tc, bio, data_begin);
Mike Snitzer452d7a62014-10-09 19:20:21 -04001162}
1163
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001164/*
1165 * A partial copy also needs to zero the uncopied region.
1166 */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001167static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
Joe Thornber2dd9c252012-03-28 18:41:28 +01001168 struct dm_dev *origin, dm_block_t data_origin,
1169 dm_block_t data_dest,
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001170 struct dm_bio_prison_cell *cell, struct bio *bio,
1171 sector_t len)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001172{
1173 int r;
1174 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +01001175 struct dm_thin_new_mapping *m = get_next_mapping(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001176
Joe Thornber991d9fa2011-10-31 20:21:18 +00001177 m->tc = tc;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001178 m->virt_begin = virt_block;
1179 m->virt_end = virt_block + 1u;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001180 m->data_block = data_dest;
1181 m->cell = cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001182
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001183 /*
1184 * quiesce action + copy action + an extra reference held for the
1185 * duration of this function (we may need to inc later for a
1186 * partial zero).
1187 */
1188 atomic_set(&m->prepare_actions, 3);
1189
Mike Snitzer44feb382012-10-12 21:02:10 +01001190 if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001191 complete_mapping_preparation(m); /* already quiesced */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001192
1193 /*
1194 * IO to pool_dev remaps to the pool target's data_dev.
1195 *
1196 * If the whole block of data is being overwritten, we can issue the
1197 * bio immediately. Otherwise we use kcopyd to clone the data first.
1198 */
Mike Snitzer452d7a62014-10-09 19:20:21 -04001199 if (io_overwrites_block(pool, bio))
1200 remap_and_issue_overwrite(tc, bio, data_dest, m);
1201 else {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001202 struct dm_io_region from, to;
1203
Joe Thornber2dd9c252012-03-28 18:41:28 +01001204 from.bdev = origin->bdev;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001205 from.sector = data_origin * pool->sectors_per_block;
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001206 from.count = len;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001207
1208 to.bdev = tc->pool_dev->bdev;
1209 to.sector = data_dest * pool->sectors_per_block;
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001210 to.count = len;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001211
1212 r = dm_kcopyd_copy(pool->copier, &from, 1, &to,
1213 0, copy_complete, m);
1214 if (r < 0) {
Mike Snitzerc3977412012-12-21 20:23:34 +00001215 DMERR_LIMIT("dm_kcopyd_copy() failed");
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001216 copy_complete(1, 1, m);
1217
1218 /*
1219 * We allow the zero to be issued, to simplify the
1220 * error path. Otherwise we'd need to start
1221 * worrying about decrementing the prepare_actions
1222 * counter.
1223 */
1224 }
1225
1226 /*
1227 * Do we need to zero a tail region?
1228 */
1229 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
1230 atomic_inc(&m->prepare_actions);
1231 ll_zero(tc, m,
1232 data_dest * pool->sectors_per_block + len,
1233 (data_dest + 1) * pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001234 }
1235 }
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001236
1237 complete_mapping_preparation(m); /* drop our ref */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001238}
1239
Joe Thornber2dd9c252012-03-28 18:41:28 +01001240static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
1241 dm_block_t data_origin, dm_block_t data_dest,
Mike Snitzera24c2562012-06-03 00:30:00 +01001242 struct dm_bio_prison_cell *cell, struct bio *bio)
Joe Thornber2dd9c252012-03-28 18:41:28 +01001243{
1244 schedule_copy(tc, virt_block, tc->pool_dev,
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001245 data_origin, data_dest, cell, bio,
1246 tc->pool->sectors_per_block);
Joe Thornber2dd9c252012-03-28 18:41:28 +01001247}
1248
Joe Thornber991d9fa2011-10-31 20:21:18 +00001249static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
Mike Snitzera24c2562012-06-03 00:30:00 +01001250 dm_block_t data_block, struct dm_bio_prison_cell *cell,
Joe Thornber991d9fa2011-10-31 20:21:18 +00001251 struct bio *bio)
1252{
1253 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +01001254 struct dm_thin_new_mapping *m = get_next_mapping(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001255
Joe Thornber50f3c3e2014-06-13 13:57:09 +01001256 atomic_set(&m->prepare_actions, 1); /* no need to quiesce */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001257 m->tc = tc;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001258 m->virt_begin = virt_block;
1259 m->virt_end = virt_block + 1u;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001260 m->data_block = data_block;
1261 m->cell = cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001262
1263 /*
1264 * If the whole block of data is being overwritten or we are not
1265 * zeroing pre-existing data, we can issue the bio immediately.
1266 * Otherwise we use kcopyd to zero the data first.
1267 */
Mike Snitzerf8ae7522015-05-14 11:28:37 -04001268 if (pool->pf.zero_new_blocks) {
1269 if (io_overwrites_block(pool, bio))
1270 remap_and_issue_overwrite(tc, bio, data_block, m);
1271 else
1272 ll_zero(tc, m, data_block * pool->sectors_per_block,
1273 (data_block + 1) * pool->sectors_per_block);
1274 } else
Joe Thornber991d9fa2011-10-31 20:21:18 +00001275 process_prepared_mapping(m);
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001276}
Joe Thornber991d9fa2011-10-31 20:21:18 +00001277
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001278static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
1279 dm_block_t data_dest,
1280 struct dm_bio_prison_cell *cell, struct bio *bio)
1281{
1282 struct pool *pool = tc->pool;
1283 sector_t virt_block_begin = virt_block * pool->sectors_per_block;
1284 sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
1285
1286 if (virt_block_end <= tc->origin_size)
1287 schedule_copy(tc, virt_block, tc->origin_dev,
1288 virt_block, data_dest, cell, bio,
1289 pool->sectors_per_block);
1290
1291 else if (virt_block_begin < tc->origin_size)
1292 schedule_copy(tc, virt_block, tc->origin_dev,
1293 virt_block, data_dest, cell, bio,
1294 tc->origin_size - virt_block_begin);
1295
1296 else
1297 schedule_zero(tc, virt_block, data_dest, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001298}
1299
Joe Thornber2c43fd22014-12-11 11:12:19 +00001300static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
1301
1302static void check_for_space(struct pool *pool)
1303{
1304 int r;
1305 dm_block_t nr_free;
1306
1307 if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)
1308 return;
1309
1310 r = dm_pool_get_free_block_count(pool->pmd, &nr_free);
1311 if (r)
1312 return;
1313
1314 if (nr_free)
1315 set_pool_mode(pool, PM_WRITE);
1316}
1317
Joe Thornbere49e5822012-07-27 15:08:16 +01001318/*
1319 * A non-zero return indicates read_only or fail_io mode.
1320 * Many callers don't care about the return value.
1321 */
Joe Thornber020cc3b2013-12-04 15:05:36 -05001322static int commit(struct pool *pool)
Joe Thornbere49e5822012-07-27 15:08:16 +01001323{
1324 int r;
1325
Joe Thornber8d07e8a2014-05-06 16:28:14 +01001326 if (get_pool_mode(pool) >= PM_READ_ONLY)
Joe Thornbere49e5822012-07-27 15:08:16 +01001327 return -EINVAL;
1328
Joe Thornber020cc3b2013-12-04 15:05:36 -05001329 r = dm_pool_commit_metadata(pool->pmd);
Joe Thornberb5330652013-12-04 19:51:33 -05001330 if (r)
1331 metadata_operation_failed(pool, "dm_pool_commit_metadata", r);
Joe Thornber2c43fd22014-12-11 11:12:19 +00001332 else
1333 check_for_space(pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01001334
1335 return r;
1336}
1337
Joe Thornber88a66212013-12-04 20:16:12 -05001338static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
1339{
1340 unsigned long flags;
1341
1342 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1343 DMWARN("%s: reached low water mark for data device: sending event.",
1344 dm_device_name(pool->pool_md));
1345 spin_lock_irqsave(&pool->lock, flags);
1346 pool->low_water_triggered = true;
1347 spin_unlock_irqrestore(&pool->lock, flags);
1348 dm_table_event(pool->ti->table);
1349 }
1350}
1351
Joe Thornber991d9fa2011-10-31 20:21:18 +00001352static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1353{
1354 int r;
1355 dm_block_t free_blocks;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001356 struct pool *pool = tc->pool;
1357
Joe Thornber3e1a0692014-03-03 16:03:26 +00001358 if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
Joe Thornber8d30abf2013-12-04 19:16:11 -05001359 return -EINVAL;
1360
Joe Thornber991d9fa2011-10-31 20:21:18 +00001361 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
Joe Thornberb5330652013-12-04 19:51:33 -05001362 if (r) {
1363 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001364 return r;
Joe Thornberb5330652013-12-04 19:51:33 -05001365 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001366
Joe Thornber88a66212013-12-04 20:16:12 -05001367 check_low_water_mark(pool, free_blocks);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001368
1369 if (!free_blocks) {
Mike Snitzer94563ba2013-08-22 09:56:18 -04001370 /*
1371 * Try to commit to see if that will free up some
1372 * more space.
1373 */
Joe Thornber020cc3b2013-12-04 15:05:36 -05001374 r = commit(pool);
1375 if (r)
1376 return r;
Mike Snitzer94563ba2013-08-22 09:56:18 -04001377
1378 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
Joe Thornberb5330652013-12-04 19:51:33 -05001379 if (r) {
1380 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
Mike Snitzer94563ba2013-08-22 09:56:18 -04001381 return r;
Joe Thornberb5330652013-12-04 19:51:33 -05001382 }
Mike Snitzer94563ba2013-08-22 09:56:18 -04001383
Mike Snitzer94563ba2013-08-22 09:56:18 -04001384 if (!free_blocks) {
Joe Thornber3e1a0692014-03-03 16:03:26 +00001385 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001386 return -ENOSPC;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001387 }
1388 }
1389
1390 r = dm_pool_alloc_data_block(pool->pmd, result);
Mike Snitzer4a02b342013-12-03 12:20:57 -05001391 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05001392 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001393 return r;
Mike Snitzer4a02b342013-12-03 12:20:57 -05001394 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001395
1396 return 0;
1397}
1398
1399/*
1400 * If we have run out of space, queue bios until the device is
1401 * resumed, presumably after having been reloaded with more space.
1402 */
1403static void retry_on_resume(struct bio *bio)
1404{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001405 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01001406 struct thin_c *tc = h->tc;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001407 unsigned long flags;
1408
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001409 spin_lock_irqsave(&tc->lock, flags);
1410 bio_list_add(&tc->retry_on_resume_list, bio);
1411 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001412}
1413
Mike Snitzeraf918052014-05-22 14:32:51 -04001414static int should_error_unserviceable_bio(struct pool *pool)
Joe Thornber3e1a0692014-03-03 16:03:26 +00001415{
1416 enum pool_mode m = get_pool_mode(pool);
1417
1418 switch (m) {
1419 case PM_WRITE:
1420 /* Shouldn't get here */
1421 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
Mike Snitzeraf918052014-05-22 14:32:51 -04001422 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001423
1424 case PM_OUT_OF_DATA_SPACE:
Mike Snitzeraf918052014-05-22 14:32:51 -04001425 return pool->pf.error_if_no_space ? -ENOSPC : 0;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001426
1427 case PM_READ_ONLY:
1428 case PM_FAIL:
Mike Snitzeraf918052014-05-22 14:32:51 -04001429 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001430 default:
1431 /* Shouldn't get here */
1432 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
Mike Snitzeraf918052014-05-22 14:32:51 -04001433 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001434 }
1435}
1436
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001437static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
1438{
Mike Snitzeraf918052014-05-22 14:32:51 -04001439 int error = should_error_unserviceable_bio(pool);
1440
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001441 if (error) {
1442 bio->bi_error = error;
1443 bio_endio(bio);
1444 } else
Mike Snitzer6d162022013-12-20 18:09:02 -05001445 retry_on_resume(bio);
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001446}
1447
Mike Snitzer399cadd2013-12-05 16:03:33 -05001448static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001449{
1450 struct bio *bio;
1451 struct bio_list bios;
Mike Snitzeraf918052014-05-22 14:32:51 -04001452 int error;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001453
Mike Snitzeraf918052014-05-22 14:32:51 -04001454 error = should_error_unserviceable_bio(pool);
1455 if (error) {
1456 cell_error_with_code(pool, cell, error);
Joe Thornber3e1a0692014-03-03 16:03:26 +00001457 return;
1458 }
1459
Joe Thornber991d9fa2011-10-31 20:21:18 +00001460 bio_list_init(&bios);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001461 cell_release(pool, cell, &bios);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001462
Mike Snitzer9d094ee2014-10-19 08:23:09 -04001463 while ((bio = bio_list_pop(&bios)))
1464 retry_on_resume(bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001465}
1466
Joe Thornber34fbcf62015-04-16 12:58:35 +01001467static void process_discard_cell_no_passdown(struct thin_c *tc,
1468 struct dm_bio_prison_cell *virt_cell)
Joe Thornber104655f2012-03-28 18:41:28 +01001469{
Joe Thornber104655f2012-03-28 18:41:28 +01001470 struct pool *pool = tc->pool;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001471 struct dm_thin_new_mapping *m = get_next_mapping(pool);
1472
1473 /*
1474 * We don't need to lock the data blocks, since there's no
1475 * passdown. We only lock data blocks for allocation and breaking sharing.
1476 */
1477 m->tc = tc;
1478 m->virt_begin = virt_cell->key.block_begin;
1479 m->virt_end = virt_cell->key.block_end;
1480 m->cell = virt_cell;
1481 m->bio = virt_cell->holder;
1482
1483 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1484 pool->process_prepared_discard(m);
1485}
1486
1487/*
Mike Snitzer84f8bd82015-08-18 10:31:09 -04001488 * __bio_inc_remaining() is used to defer parent bios's end_io until
1489 * we _know_ all chained sub range discard bios have completed.
Joe Thornber34fbcf62015-04-16 12:58:35 +01001490 */
1491static inline void __bio_inc_remaining(struct bio *bio)
1492{
1493 bio->bi_flags |= (1 << BIO_CHAIN);
1494 smp_mb__before_atomic();
1495 atomic_inc(&bio->__bi_remaining);
1496}
1497
1498static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,
1499 struct bio *bio)
1500{
1501 struct pool *pool = tc->pool;
1502
1503 int r;
1504 bool maybe_shared;
1505 struct dm_cell_key data_key;
1506 struct dm_bio_prison_cell *data_cell;
Mike Snitzera24c2562012-06-03 00:30:00 +01001507 struct dm_thin_new_mapping *m;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001508 dm_block_t virt_begin, virt_end, data_begin;
Joe Thornber104655f2012-03-28 18:41:28 +01001509
Joe Thornber34fbcf62015-04-16 12:58:35 +01001510 while (begin != end) {
1511 r = ensure_next_mapping(pool);
1512 if (r)
1513 /* we did our best */
1514 return;
Joe Thornber104655f2012-03-28 18:41:28 +01001515
Joe Thornber34fbcf62015-04-16 12:58:35 +01001516 r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,
1517 &data_begin, &maybe_shared);
1518 if (r)
1519 /*
1520 * Silently fail, letting any mappings we've
1521 * created complete.
1522 */
Joe Thornber104655f2012-03-28 18:41:28 +01001523 break;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001524
1525 build_key(tc->td, PHYSICAL, data_begin, data_begin + (virt_end - virt_begin), &data_key);
1526 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
1527 /* contention, we'll give up with this range */
1528 begin = virt_end;
1529 continue;
Joe Thornber104655f2012-03-28 18:41:28 +01001530 }
1531
Joe Thornber104655f2012-03-28 18:41:28 +01001532 /*
Joe Thornber34fbcf62015-04-16 12:58:35 +01001533 * IO may still be going to the destination block. We must
1534 * quiesce before we can do the removal.
Joe Thornber104655f2012-03-28 18:41:28 +01001535 */
Joe Thornber34fbcf62015-04-16 12:58:35 +01001536 m = get_next_mapping(pool);
1537 m->tc = tc;
1538 m->maybe_shared = maybe_shared;
1539 m->virt_begin = virt_begin;
1540 m->virt_end = virt_end;
1541 m->data_block = data_begin;
1542 m->cell = data_cell;
1543 m->bio = bio;
Joe Thornber104655f2012-03-28 18:41:28 +01001544
Joe Thornber34fbcf62015-04-16 12:58:35 +01001545 /*
1546 * The parent bio must not complete before sub discard bios are
1547 * chained to it (see __blkdev_issue_discard_async's bio_chain)!
1548 *
1549 * This per-mapping bi_remaining increment is paired with
1550 * the implicit decrement that occurs via bio_endio() in
1551 * process_prepared_discard_{passdown,no_passdown}.
1552 */
1553 __bio_inc_remaining(bio);
1554 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1555 pool->process_prepared_discard(m);
1556
1557 begin = virt_end;
Joe Thornber104655f2012-03-28 18:41:28 +01001558 }
1559}
1560
Joe Thornber34fbcf62015-04-16 12:58:35 +01001561static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)
1562{
1563 struct bio *bio = virt_cell->holder;
1564 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1565
1566 /*
1567 * The virt_cell will only get freed once the origin bio completes.
1568 * This means it will remain locked while all the individual
1569 * passdown bios are in flight.
1570 */
1571 h->cell = virt_cell;
1572 break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio);
1573
1574 /*
1575 * We complete the bio now, knowing that the bi_remaining field
1576 * will prevent completion until the sub range discards have
1577 * completed.
1578 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001579 bio_endio(bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +01001580}
1581
Joe Thornbera374bb22014-10-10 13:43:14 +01001582static void process_discard_bio(struct thin_c *tc, struct bio *bio)
1583{
Joe Thornber34fbcf62015-04-16 12:58:35 +01001584 dm_block_t begin, end;
1585 struct dm_cell_key virt_key;
1586 struct dm_bio_prison_cell *virt_cell;
Joe Thornbera374bb22014-10-10 13:43:14 +01001587
Joe Thornber34fbcf62015-04-16 12:58:35 +01001588 get_bio_block_range(tc, bio, &begin, &end);
1589 if (begin == end) {
1590 /*
1591 * The discard covers less than a block.
1592 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001593 bio_endio(bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +01001594 return;
1595 }
1596
1597 build_key(tc->td, VIRTUAL, begin, end, &virt_key);
1598 if (bio_detain(tc->pool, &virt_key, bio, &virt_cell))
1599 /*
1600 * Potential starvation issue: We're relying on the
1601 * fs/application being well behaved, and not trying to
1602 * send IO to a region at the same time as discarding it.
1603 * If they do this persistently then it's possible this
1604 * cell will never be granted.
1605 */
Joe Thornbera374bb22014-10-10 13:43:14 +01001606 return;
1607
Joe Thornber34fbcf62015-04-16 12:58:35 +01001608 tc->pool->process_discard_cell(tc, virt_cell);
Joe Thornbera374bb22014-10-10 13:43:14 +01001609}
1610
Joe Thornber991d9fa2011-10-31 20:21:18 +00001611static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
Mike Snitzer44feb382012-10-12 21:02:10 +01001612 struct dm_cell_key *key,
Joe Thornber991d9fa2011-10-31 20:21:18 +00001613 struct dm_thin_lookup_result *lookup_result,
Mike Snitzera24c2562012-06-03 00:30:00 +01001614 struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001615{
1616 int r;
1617 dm_block_t data_block;
Mike Snitzerd6fc2042013-08-21 17:40:11 -04001618 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001619
1620 r = alloc_data_block(tc, &data_block);
1621 switch (r) {
1622 case 0:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001623 schedule_internal_copy(tc, block, lookup_result->block,
1624 data_block, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001625 break;
1626
1627 case -ENOSPC:
Mike Snitzer399cadd2013-12-05 16:03:33 -05001628 retry_bios_on_resume(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001629 break;
1630
1631 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001632 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1633 __func__, r);
Mike Snitzerd6fc2042013-08-21 17:40:11 -04001634 cell_error(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001635 break;
1636 }
1637}
1638
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001639static void __remap_and_issue_shared_cell(void *context,
1640 struct dm_bio_prison_cell *cell)
1641{
1642 struct remap_info *info = context;
1643 struct bio *bio;
1644
1645 while ((bio = bio_list_pop(&cell->bios))) {
1646 if ((bio_data_dir(bio) == WRITE) ||
1647 (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)))
1648 bio_list_add(&info->defer_bios, bio);
1649 else {
1650 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));;
1651
1652 h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
1653 inc_all_io_entry(info->tc->pool, bio);
1654 bio_list_add(&info->issue_bios, bio);
1655 }
1656 }
1657}
1658
1659static void remap_and_issue_shared_cell(struct thin_c *tc,
1660 struct dm_bio_prison_cell *cell,
1661 dm_block_t block)
1662{
1663 struct bio *bio;
1664 struct remap_info info;
1665
1666 info.tc = tc;
1667 bio_list_init(&info.defer_bios);
1668 bio_list_init(&info.issue_bios);
1669
1670 cell_visit_release(tc->pool, __remap_and_issue_shared_cell,
1671 &info, cell);
1672
1673 while ((bio = bio_list_pop(&info.defer_bios)))
1674 thin_defer_bio(tc, bio);
1675
1676 while ((bio = bio_list_pop(&info.issue_bios)))
1677 remap_and_issue(tc, bio, block);
1678}
1679
Joe Thornber991d9fa2011-10-31 20:21:18 +00001680static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1681 dm_block_t block,
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001682 struct dm_thin_lookup_result *lookup_result,
1683 struct dm_bio_prison_cell *virt_cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001684{
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001685 struct dm_bio_prison_cell *data_cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001686 struct pool *pool = tc->pool;
Mike Snitzer44feb382012-10-12 21:02:10 +01001687 struct dm_cell_key key;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001688
1689 /*
1690 * If cell is already occupied, then sharing is already in the process
1691 * of being broken so we have nothing further to do here.
1692 */
1693 build_data_key(tc->td, lookup_result->block, &key);
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001694 if (bio_detain(pool, &key, bio, &data_cell)) {
1695 cell_defer_no_holder(tc, virt_cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001696 return;
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001697 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001698
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001699 if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
1700 break_sharing(tc, bio, block, &key, lookup_result, data_cell);
1701 cell_defer_no_holder(tc, virt_cell);
1702 } else {
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001703 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornber991d9fa2011-10-31 20:21:18 +00001704
Mike Snitzer44feb382012-10-12 21:02:10 +01001705 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
Joe Thornbere8088072012-12-21 20:23:31 +00001706 inc_all_io_entry(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001707 remap_and_issue(tc, bio, lookup_result->block);
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001708
1709 remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);
1710 remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001711 }
1712}
1713
1714static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
Mike Snitzera24c2562012-06-03 00:30:00 +01001715 struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001716{
1717 int r;
1718 dm_block_t data_block;
Joe Thornber6beca5e2013-03-01 22:45:50 +00001719 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001720
1721 /*
1722 * Remap empty bios (flushes) immediately, without provisioning.
1723 */
Kent Overstreet4f024f32013-10-11 15:44:27 -07001724 if (!bio->bi_iter.bi_size) {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001725 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001726 cell_defer_no_holder(tc, cell);
Joe Thornbere8088072012-12-21 20:23:31 +00001727
Joe Thornber991d9fa2011-10-31 20:21:18 +00001728 remap_and_issue(tc, bio, 0);
1729 return;
1730 }
1731
1732 /*
1733 * Fill read bios with zeroes and complete them immediately.
1734 */
1735 if (bio_data_dir(bio) == READ) {
1736 zero_fill_bio(bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001737 cell_defer_no_holder(tc, cell);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001738 bio_endio(bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001739 return;
1740 }
1741
1742 r = alloc_data_block(tc, &data_block);
1743 switch (r) {
1744 case 0:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001745 if (tc->origin_dev)
1746 schedule_external_copy(tc, block, data_block, cell, bio);
1747 else
1748 schedule_zero(tc, block, data_block, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001749 break;
1750
1751 case -ENOSPC:
Mike Snitzer399cadd2013-12-05 16:03:33 -05001752 retry_bios_on_resume(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001753 break;
1754
1755 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001756 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1757 __func__, r);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001758 cell_error(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001759 break;
1760 }
1761}
1762
Joe Thornbera374bb22014-10-10 13:43:14 +01001763static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001764{
1765 int r;
Joe Thornber6beca5e2013-03-01 22:45:50 +00001766 struct pool *pool = tc->pool;
Joe Thornbera374bb22014-10-10 13:43:14 +01001767 struct bio *bio = cell->holder;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001768 dm_block_t block = get_bio_block(tc, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001769 struct dm_thin_lookup_result lookup_result;
1770
Joe Thornbera374bb22014-10-10 13:43:14 +01001771 if (tc->requeue_mode) {
1772 cell_requeue(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001773 return;
Joe Thornbera374bb22014-10-10 13:43:14 +01001774 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001775
1776 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1777 switch (r) {
1778 case 0:
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001779 if (lookup_result.shared)
1780 process_shared_bio(tc, bio, block, &lookup_result, cell);
1781 else {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001782 inc_all_io_entry(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001783 remap_and_issue(tc, bio, lookup_result.block);
Joe Thornbera374bb22014-10-10 13:43:14 +01001784 inc_remap_and_issue_cell(tc, cell, lookup_result.block);
Joe Thornbere8088072012-12-21 20:23:31 +00001785 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001786 break;
1787
1788 case -ENODATA:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001789 if (bio_data_dir(bio) == READ && tc->origin_dev) {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001790 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001791 cell_defer_no_holder(tc, cell);
Joe Thornbere8088072012-12-21 20:23:31 +00001792
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001793 if (bio_end_sector(bio) <= tc->origin_size)
1794 remap_to_origin_and_issue(tc, bio);
1795
1796 else if (bio->bi_iter.bi_sector < tc->origin_size) {
1797 zero_fill_bio(bio);
1798 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
1799 remap_to_origin_and_issue(tc, bio);
1800
1801 } else {
1802 zero_fill_bio(bio);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001803 bio_endio(bio);
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001804 }
Joe Thornber2dd9c252012-03-28 18:41:28 +01001805 } else
1806 provision_block(tc, bio, block, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001807 break;
1808
1809 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001810 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1811 __func__, r);
Joe Thornberf286ba02012-12-21 20:23:33 +00001812 cell_defer_no_holder(tc, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001813 bio_io_error(bio);
1814 break;
1815 }
1816}
1817
Joe Thornbera374bb22014-10-10 13:43:14 +01001818static void process_bio(struct thin_c *tc, struct bio *bio)
1819{
1820 struct pool *pool = tc->pool;
1821 dm_block_t block = get_bio_block(tc, bio);
1822 struct dm_bio_prison_cell *cell;
1823 struct dm_cell_key key;
1824
1825 /*
1826 * If cell is already occupied, then the block is already
1827 * being provisioned so we have nothing further to do here.
1828 */
1829 build_virtual_key(tc->td, block, &key);
1830 if (bio_detain(pool, &key, bio, &cell))
1831 return;
1832
1833 process_cell(tc, cell);
1834}
1835
1836static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,
1837 struct dm_bio_prison_cell *cell)
Joe Thornbere49e5822012-07-27 15:08:16 +01001838{
1839 int r;
1840 int rw = bio_data_dir(bio);
1841 dm_block_t block = get_bio_block(tc, bio);
1842 struct dm_thin_lookup_result lookup_result;
1843
1844 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1845 switch (r) {
1846 case 0:
Joe Thornbera374bb22014-10-10 13:43:14 +01001847 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001848 handle_unserviceable_bio(tc->pool, bio);
Joe Thornbera374bb22014-10-10 13:43:14 +01001849 if (cell)
1850 cell_defer_no_holder(tc, cell);
1851 } else {
Joe Thornbere8088072012-12-21 20:23:31 +00001852 inc_all_io_entry(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001853 remap_and_issue(tc, bio, lookup_result.block);
Joe Thornbera374bb22014-10-10 13:43:14 +01001854 if (cell)
1855 inc_remap_and_issue_cell(tc, cell, lookup_result.block);
Joe Thornbere8088072012-12-21 20:23:31 +00001856 }
Joe Thornbere49e5822012-07-27 15:08:16 +01001857 break;
1858
1859 case -ENODATA:
Joe Thornbera374bb22014-10-10 13:43:14 +01001860 if (cell)
1861 cell_defer_no_holder(tc, cell);
Joe Thornbere49e5822012-07-27 15:08:16 +01001862 if (rw != READ) {
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001863 handle_unserviceable_bio(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001864 break;
1865 }
1866
1867 if (tc->origin_dev) {
Joe Thornbere8088072012-12-21 20:23:31 +00001868 inc_all_io_entry(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001869 remap_to_origin_and_issue(tc, bio);
1870 break;
1871 }
1872
1873 zero_fill_bio(bio);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001874 bio_endio(bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001875 break;
1876
1877 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001878 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1879 __func__, r);
Joe Thornbera374bb22014-10-10 13:43:14 +01001880 if (cell)
1881 cell_defer_no_holder(tc, cell);
Joe Thornbere49e5822012-07-27 15:08:16 +01001882 bio_io_error(bio);
1883 break;
1884 }
1885}
1886
Joe Thornbera374bb22014-10-10 13:43:14 +01001887static void process_bio_read_only(struct thin_c *tc, struct bio *bio)
1888{
1889 __process_bio_read_only(tc, bio, NULL);
1890}
1891
1892static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1893{
1894 __process_bio_read_only(tc, cell->holder, cell);
1895}
1896
Joe Thornber3e1a0692014-03-03 16:03:26 +00001897static void process_bio_success(struct thin_c *tc, struct bio *bio)
1898{
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001899 bio_endio(bio);
Joe Thornber3e1a0692014-03-03 16:03:26 +00001900}
1901
Joe Thornbere49e5822012-07-27 15:08:16 +01001902static void process_bio_fail(struct thin_c *tc, struct bio *bio)
1903{
1904 bio_io_error(bio);
1905}
1906
Joe Thornbera374bb22014-10-10 13:43:14 +01001907static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1908{
1909 cell_success(tc->pool, cell);
1910}
1911
1912static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1913{
1914 cell_error(tc->pool, cell);
1915}
1916
Joe Thornberac8c3f32013-05-10 14:37:21 +01001917/*
1918 * FIXME: should we also commit due to size of transaction, measured in
1919 * metadata blocks?
1920 */
Joe Thornber905e51b2012-03-28 18:41:27 +01001921static int need_commit_due_to_time(struct pool *pool)
1922{
Manuel Schölling0f30af92014-05-22 22:42:37 +02001923 return !time_in_range(jiffies, pool->last_commit_jiffies,
1924 pool->last_commit_jiffies + COMMIT_PERIOD);
Joe Thornber905e51b2012-03-28 18:41:27 +01001925}
1926
Mike Snitzer67324ea2014-03-21 18:33:41 -04001927#define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
1928#define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
1929
1930static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)
1931{
1932 struct rb_node **rbp, *parent;
1933 struct dm_thin_endio_hook *pbd;
1934 sector_t bi_sector = bio->bi_iter.bi_sector;
1935
1936 rbp = &tc->sort_bio_list.rb_node;
1937 parent = NULL;
1938 while (*rbp) {
1939 parent = *rbp;
1940 pbd = thin_pbd(parent);
1941
1942 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)
1943 rbp = &(*rbp)->rb_left;
1944 else
1945 rbp = &(*rbp)->rb_right;
1946 }
1947
1948 pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1949 rb_link_node(&pbd->rb_node, parent, rbp);
1950 rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);
1951}
1952
1953static void __extract_sorted_bios(struct thin_c *tc)
1954{
1955 struct rb_node *node;
1956 struct dm_thin_endio_hook *pbd;
1957 struct bio *bio;
1958
1959 for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) {
1960 pbd = thin_pbd(node);
1961 bio = thin_bio(pbd);
1962
1963 bio_list_add(&tc->deferred_bio_list, bio);
1964 rb_erase(&pbd->rb_node, &tc->sort_bio_list);
1965 }
1966
1967 WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));
1968}
1969
1970static void __sort_thin_deferred_bios(struct thin_c *tc)
1971{
1972 struct bio *bio;
1973 struct bio_list bios;
1974
1975 bio_list_init(&bios);
1976 bio_list_merge(&bios, &tc->deferred_bio_list);
1977 bio_list_init(&tc->deferred_bio_list);
1978
1979 /* Sort deferred_bio_list using rb-tree */
1980 while ((bio = bio_list_pop(&bios)))
1981 __thin_bio_rb_add(tc, bio);
1982
1983 /*
1984 * Transfer the sorted bios in sort_bio_list back to
1985 * deferred_bio_list to allow lockless submission of
1986 * all bios.
1987 */
1988 __extract_sorted_bios(tc);
1989}
1990
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001991static void process_thin_deferred_bios(struct thin_c *tc)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001992{
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001993 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001994 unsigned long flags;
1995 struct bio *bio;
1996 struct bio_list bios;
Mike Snitzer67324ea2014-03-21 18:33:41 -04001997 struct blk_plug plug;
Joe Thornber8a01a6a2014-10-06 15:28:30 +01001998 unsigned count = 0;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001999
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002000 if (tc->requeue_mode) {
Mike Snitzer42d6a8c2014-10-19 07:52:44 -04002001 error_thin_bio_list(tc, &tc->deferred_bio_list, DM_ENDIO_REQUEUE);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002002 return;
2003 }
2004
Joe Thornber991d9fa2011-10-31 20:21:18 +00002005 bio_list_init(&bios);
2006
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002007 spin_lock_irqsave(&tc->lock, flags);
Mike Snitzer67324ea2014-03-21 18:33:41 -04002008
2009 if (bio_list_empty(&tc->deferred_bio_list)) {
2010 spin_unlock_irqrestore(&tc->lock, flags);
2011 return;
2012 }
2013
2014 __sort_thin_deferred_bios(tc);
2015
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002016 bio_list_merge(&bios, &tc->deferred_bio_list);
2017 bio_list_init(&tc->deferred_bio_list);
Mike Snitzer67324ea2014-03-21 18:33:41 -04002018
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002019 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002020
Mike Snitzer67324ea2014-03-21 18:33:41 -04002021 blk_start_plug(&plug);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002022 while ((bio = bio_list_pop(&bios))) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00002023 /*
2024 * If we've got no free new_mapping structs, and processing
2025 * this bio might require one, we pause until there are some
2026 * prepared mappings to process.
2027 */
2028 if (ensure_next_mapping(pool)) {
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002029 spin_lock_irqsave(&tc->lock, flags);
2030 bio_list_add(&tc->deferred_bio_list, bio);
2031 bio_list_merge(&tc->deferred_bio_list, &bios);
2032 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002033 break;
2034 }
Joe Thornber104655f2012-03-28 18:41:28 +01002035
2036 if (bio->bi_rw & REQ_DISCARD)
Joe Thornbere49e5822012-07-27 15:08:16 +01002037 pool->process_discard(tc, bio);
Joe Thornber104655f2012-03-28 18:41:28 +01002038 else
Joe Thornbere49e5822012-07-27 15:08:16 +01002039 pool->process_bio(tc, bio);
Joe Thornber8a01a6a2014-10-06 15:28:30 +01002040
2041 if ((count++ & 127) == 0) {
Joe Thornber7d327fe2014-10-06 15:45:59 +01002042 throttle_work_update(&pool->throttle);
Joe Thornber8a01a6a2014-10-06 15:28:30 +01002043 dm_pool_issue_prefetches(pool->pmd);
2044 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002045 }
Mike Snitzer67324ea2014-03-21 18:33:41 -04002046 blk_finish_plug(&plug);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002047}
2048
Joe Thornberac4c3f32014-10-10 16:42:10 +01002049static int cmp_cells(const void *lhs, const void *rhs)
2050{
2051 struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs);
2052 struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs);
2053
2054 BUG_ON(!lhs_cell->holder);
2055 BUG_ON(!rhs_cell->holder);
2056
2057 if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)
2058 return -1;
2059
2060 if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)
2061 return 1;
2062
2063 return 0;
2064}
2065
2066static unsigned sort_cells(struct pool *pool, struct list_head *cells)
2067{
2068 unsigned count = 0;
2069 struct dm_bio_prison_cell *cell, *tmp;
2070
2071 list_for_each_entry_safe(cell, tmp, cells, user_list) {
2072 if (count >= CELL_SORT_ARRAY_SIZE)
2073 break;
2074
2075 pool->cell_sort_array[count++] = cell;
2076 list_del(&cell->user_list);
2077 }
2078
2079 sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);
2080
2081 return count;
2082}
2083
Joe Thornbera374bb22014-10-10 13:43:14 +01002084static void process_thin_deferred_cells(struct thin_c *tc)
2085{
2086 struct pool *pool = tc->pool;
2087 unsigned long flags;
2088 struct list_head cells;
Joe Thornberac4c3f32014-10-10 16:42:10 +01002089 struct dm_bio_prison_cell *cell;
2090 unsigned i, j, count;
Joe Thornbera374bb22014-10-10 13:43:14 +01002091
2092 INIT_LIST_HEAD(&cells);
2093
2094 spin_lock_irqsave(&tc->lock, flags);
2095 list_splice_init(&tc->deferred_cells, &cells);
2096 spin_unlock_irqrestore(&tc->lock, flags);
2097
2098 if (list_empty(&cells))
2099 return;
2100
Joe Thornberac4c3f32014-10-10 16:42:10 +01002101 do {
2102 count = sort_cells(tc->pool, &cells);
Joe Thornbera374bb22014-10-10 13:43:14 +01002103
Joe Thornberac4c3f32014-10-10 16:42:10 +01002104 for (i = 0; i < count; i++) {
2105 cell = pool->cell_sort_array[i];
2106 BUG_ON(!cell->holder);
2107
2108 /*
2109 * If we've got no free new_mapping structs, and processing
2110 * this bio might require one, we pause until there are some
2111 * prepared mappings to process.
2112 */
2113 if (ensure_next_mapping(pool)) {
2114 for (j = i; j < count; j++)
2115 list_add(&pool->cell_sort_array[j]->user_list, &cells);
2116
2117 spin_lock_irqsave(&tc->lock, flags);
2118 list_splice(&cells, &tc->deferred_cells);
2119 spin_unlock_irqrestore(&tc->lock, flags);
2120 return;
2121 }
2122
2123 if (cell->holder->bi_rw & REQ_DISCARD)
2124 pool->process_discard_cell(tc, cell);
2125 else
2126 pool->process_cell(tc, cell);
Joe Thornbera374bb22014-10-10 13:43:14 +01002127 }
Joe Thornberac4c3f32014-10-10 16:42:10 +01002128 } while (!list_empty(&cells));
Joe Thornbera374bb22014-10-10 13:43:14 +01002129}
2130
Joe Thornberb10ebd32014-04-08 11:29:01 +01002131static void thin_get(struct thin_c *tc);
2132static void thin_put(struct thin_c *tc);
2133
2134/*
2135 * We can't hold rcu_read_lock() around code that can block. So we
2136 * find a thin with the rcu lock held; bump a refcount; then drop
2137 * the lock.
2138 */
2139static struct thin_c *get_first_thin(struct pool *pool)
2140{
2141 struct thin_c *tc = NULL;
2142
2143 rcu_read_lock();
2144 if (!list_empty(&pool->active_thins)) {
2145 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list);
2146 thin_get(tc);
2147 }
2148 rcu_read_unlock();
2149
2150 return tc;
2151}
2152
2153static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
2154{
2155 struct thin_c *old_tc = tc;
2156
2157 rcu_read_lock();
2158 list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
2159 thin_get(tc);
2160 thin_put(old_tc);
2161 rcu_read_unlock();
2162 return tc;
2163 }
2164 thin_put(old_tc);
2165 rcu_read_unlock();
2166
2167 return NULL;
2168}
2169
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002170static void process_deferred_bios(struct pool *pool)
2171{
2172 unsigned long flags;
2173 struct bio *bio;
2174 struct bio_list bios;
2175 struct thin_c *tc;
2176
Joe Thornberb10ebd32014-04-08 11:29:01 +01002177 tc = get_first_thin(pool);
2178 while (tc) {
Joe Thornbera374bb22014-10-10 13:43:14 +01002179 process_thin_deferred_cells(tc);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002180 process_thin_deferred_bios(tc);
Joe Thornberb10ebd32014-04-08 11:29:01 +01002181 tc = get_next_thin(pool, tc);
2182 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002183
2184 /*
2185 * If there are any deferred flush bios, we must commit
2186 * the metadata before issuing them.
2187 */
2188 bio_list_init(&bios);
2189 spin_lock_irqsave(&pool->lock, flags);
2190 bio_list_merge(&bios, &pool->deferred_flush_bios);
2191 bio_list_init(&pool->deferred_flush_bios);
2192 spin_unlock_irqrestore(&pool->lock, flags);
2193
Mike Snitzer4d1662a2014-02-06 06:08:56 -05002194 if (bio_list_empty(&bios) &&
2195 !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool)))
Joe Thornber991d9fa2011-10-31 20:21:18 +00002196 return;
2197
Joe Thornber020cc3b2013-12-04 15:05:36 -05002198 if (commit(pool)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00002199 while ((bio = bio_list_pop(&bios)))
2200 bio_io_error(bio);
2201 return;
2202 }
Joe Thornber905e51b2012-03-28 18:41:27 +01002203 pool->last_commit_jiffies = jiffies;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002204
2205 while ((bio = bio_list_pop(&bios)))
2206 generic_make_request(bio);
2207}
2208
2209static void do_worker(struct work_struct *ws)
2210{
2211 struct pool *pool = container_of(ws, struct pool, worker);
2212
Joe Thornber7d327fe2014-10-06 15:45:59 +01002213 throttle_work_start(&pool->throttle);
Joe Thornber8a01a6a2014-10-06 15:28:30 +01002214 dm_pool_issue_prefetches(pool->pmd);
Joe Thornber7d327fe2014-10-06 15:45:59 +01002215 throttle_work_update(&pool->throttle);
Joe Thornbere49e5822012-07-27 15:08:16 +01002216 process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);
Joe Thornber7d327fe2014-10-06 15:45:59 +01002217 throttle_work_update(&pool->throttle);
Joe Thornbere49e5822012-07-27 15:08:16 +01002218 process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);
Joe Thornber7d327fe2014-10-06 15:45:59 +01002219 throttle_work_update(&pool->throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002220 process_deferred_bios(pool);
Joe Thornber7d327fe2014-10-06 15:45:59 +01002221 throttle_work_complete(&pool->throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002222}
2223
Joe Thornber905e51b2012-03-28 18:41:27 +01002224/*
2225 * We want to commit periodically so that not too much
2226 * unwritten data builds up.
2227 */
2228static void do_waker(struct work_struct *ws)
2229{
2230 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
2231 wake_worker(pool);
2232 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
2233}
2234
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002235static void notify_of_pool_mode_change_to_oods(struct pool *pool);
2236
Joe Thornber85ad643b2014-05-09 15:59:38 +01002237/*
2238 * We're holding onto IO to allow userland time to react. After the
2239 * timeout either the pool will have been resized (and thus back in
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002240 * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.
Joe Thornber85ad643b2014-05-09 15:59:38 +01002241 */
2242static void do_no_space_timeout(struct work_struct *ws)
2243{
2244 struct pool *pool = container_of(to_delayed_work(ws), struct pool,
2245 no_space_timeout);
2246
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002247 if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) {
2248 pool->pf.error_if_no_space = true;
2249 notify_of_pool_mode_change_to_oods(pool);
Mike Snitzer0a927c22015-07-21 13:20:46 -04002250 error_retry_list_with_code(pool, -ENOSPC);
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002251 }
Joe Thornber85ad643b2014-05-09 15:59:38 +01002252}
2253
Joe Thornber991d9fa2011-10-31 20:21:18 +00002254/*----------------------------------------------------------------*/
2255
Joe Thornbere7a3e872014-05-13 16:14:14 -04002256struct pool_work {
Joe Thornber738211f2014-03-03 15:52:28 +00002257 struct work_struct worker;
Joe Thornbere7a3e872014-05-13 16:14:14 -04002258 struct completion complete;
Joe Thornber738211f2014-03-03 15:52:28 +00002259};
2260
Joe Thornbere7a3e872014-05-13 16:14:14 -04002261static struct pool_work *to_pool_work(struct work_struct *ws)
Joe Thornber738211f2014-03-03 15:52:28 +00002262{
Joe Thornbere7a3e872014-05-13 16:14:14 -04002263 return container_of(ws, struct pool_work, worker);
2264}
2265
2266static void pool_work_complete(struct pool_work *pw)
2267{
2268 complete(&pw->complete);
2269}
2270
2271static void pool_work_wait(struct pool_work *pw, struct pool *pool,
2272 void (*fn)(struct work_struct *))
2273{
2274 INIT_WORK_ONSTACK(&pw->worker, fn);
2275 init_completion(&pw->complete);
2276 queue_work(pool->wq, &pw->worker);
2277 wait_for_completion(&pw->complete);
2278}
2279
2280/*----------------------------------------------------------------*/
2281
2282struct noflush_work {
2283 struct pool_work pw;
2284 struct thin_c *tc;
2285};
2286
2287static struct noflush_work *to_noflush(struct work_struct *ws)
2288{
2289 return container_of(to_pool_work(ws), struct noflush_work, pw);
Joe Thornber738211f2014-03-03 15:52:28 +00002290}
2291
2292static void do_noflush_start(struct work_struct *ws)
2293{
Joe Thornbere7a3e872014-05-13 16:14:14 -04002294 struct noflush_work *w = to_noflush(ws);
Joe Thornber738211f2014-03-03 15:52:28 +00002295 w->tc->requeue_mode = true;
2296 requeue_io(w->tc);
Joe Thornbere7a3e872014-05-13 16:14:14 -04002297 pool_work_complete(&w->pw);
Joe Thornber738211f2014-03-03 15:52:28 +00002298}
2299
2300static void do_noflush_stop(struct work_struct *ws)
2301{
Joe Thornbere7a3e872014-05-13 16:14:14 -04002302 struct noflush_work *w = to_noflush(ws);
Joe Thornber738211f2014-03-03 15:52:28 +00002303 w->tc->requeue_mode = false;
Joe Thornbere7a3e872014-05-13 16:14:14 -04002304 pool_work_complete(&w->pw);
Joe Thornber738211f2014-03-03 15:52:28 +00002305}
2306
2307static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
2308{
2309 struct noflush_work w;
2310
Joe Thornber738211f2014-03-03 15:52:28 +00002311 w.tc = tc;
Joe Thornbere7a3e872014-05-13 16:14:14 -04002312 pool_work_wait(&w.pw, tc->pool, fn);
Joe Thornber738211f2014-03-03 15:52:28 +00002313}
2314
2315/*----------------------------------------------------------------*/
2316
Joe Thornbere49e5822012-07-27 15:08:16 +01002317static enum pool_mode get_pool_mode(struct pool *pool)
2318{
2319 return pool->pf.mode;
2320}
2321
Joe Thornber3e1a0692014-03-03 16:03:26 +00002322static void notify_of_pool_mode_change(struct pool *pool, const char *new_mode)
2323{
2324 dm_table_event(pool->ti->table);
2325 DMINFO("%s: switching pool to %s mode",
2326 dm_device_name(pool->pool_md), new_mode);
2327}
2328
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002329static void notify_of_pool_mode_change_to_oods(struct pool *pool)
2330{
2331 if (!pool->pf.error_if_no_space)
2332 notify_of_pool_mode_change(pool, "out-of-data-space (queue IO)");
2333 else
2334 notify_of_pool_mode_change(pool, "out-of-data-space (error IO)");
2335}
2336
Joe Thornber34fbcf62015-04-16 12:58:35 +01002337static bool passdown_enabled(struct pool_c *pt)
2338{
2339 return pt->adjusted_pf.discard_passdown;
2340}
2341
2342static void set_discard_callbacks(struct pool *pool)
2343{
2344 struct pool_c *pt = pool->ti->private;
2345
2346 if (passdown_enabled(pt)) {
2347 pool->process_discard_cell = process_discard_cell_passdown;
2348 pool->process_prepared_discard = process_prepared_discard_passdown;
2349 } else {
2350 pool->process_discard_cell = process_discard_cell_no_passdown;
2351 pool->process_prepared_discard = process_prepared_discard_no_passdown;
2352 }
2353}
2354
Mike Snitzer8b64e882013-12-20 14:27:28 -05002355static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
Joe Thornbere49e5822012-07-27 15:08:16 +01002356{
Mike Snitzercdc2b412014-02-14 18:10:55 -05002357 struct pool_c *pt = pool->ti->private;
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002358 bool needs_check = dm_pool_metadata_needs_check(pool->pmd);
2359 enum pool_mode old_mode = get_pool_mode(pool);
Mike Snitzer80c57892014-05-20 13:38:33 -04002360 unsigned long no_space_timeout = ACCESS_ONCE(no_space_timeout_secs) * HZ;
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002361
2362 /*
2363 * Never allow the pool to transition to PM_WRITE mode if user
2364 * intervention is required to verify metadata and data consistency.
2365 */
2366 if (new_mode == PM_WRITE && needs_check) {
2367 DMERR("%s: unable to switch pool to write mode until repaired.",
2368 dm_device_name(pool->pool_md));
2369 if (old_mode != new_mode)
2370 new_mode = old_mode;
2371 else
2372 new_mode = PM_READ_ONLY;
2373 }
2374 /*
2375 * If we were in PM_FAIL mode, rollback of metadata failed. We're
2376 * not going to recover without a thin_repair. So we never let the
2377 * pool move out of the old mode.
2378 */
2379 if (old_mode == PM_FAIL)
2380 new_mode = old_mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01002381
Mike Snitzer8b64e882013-12-20 14:27:28 -05002382 switch (new_mode) {
Joe Thornbere49e5822012-07-27 15:08:16 +01002383 case PM_FAIL:
Mike Snitzer8b64e882013-12-20 14:27:28 -05002384 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00002385 notify_of_pool_mode_change(pool, "failure");
Joe Thornber5383ef32013-12-04 16:30:01 -05002386 dm_pool_metadata_read_only(pool->pmd);
Joe Thornbere49e5822012-07-27 15:08:16 +01002387 pool->process_bio = process_bio_fail;
2388 pool->process_discard = process_bio_fail;
Joe Thornbera374bb22014-10-10 13:43:14 +01002389 pool->process_cell = process_cell_fail;
2390 pool->process_discard_cell = process_cell_fail;
Joe Thornbere49e5822012-07-27 15:08:16 +01002391 pool->process_prepared_mapping = process_prepared_mapping_fail;
2392 pool->process_prepared_discard = process_prepared_discard_fail;
Joe Thornber3e1a0692014-03-03 16:03:26 +00002393
2394 error_retry_list(pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01002395 break;
2396
2397 case PM_READ_ONLY:
Mike Snitzer8b64e882013-12-20 14:27:28 -05002398 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00002399 notify_of_pool_mode_change(pool, "read-only");
2400 dm_pool_metadata_read_only(pool->pmd);
2401 pool->process_bio = process_bio_read_only;
2402 pool->process_discard = process_bio_success;
Joe Thornbera374bb22014-10-10 13:43:14 +01002403 pool->process_cell = process_cell_read_only;
2404 pool->process_discard_cell = process_cell_success;
Joe Thornber3e1a0692014-03-03 16:03:26 +00002405 pool->process_prepared_mapping = process_prepared_mapping_fail;
Joe Thornber34fbcf62015-04-16 12:58:35 +01002406 pool->process_prepared_discard = process_prepared_discard_success;
Joe Thornber3e1a0692014-03-03 16:03:26 +00002407
2408 error_retry_list(pool);
2409 break;
2410
2411 case PM_OUT_OF_DATA_SPACE:
2412 /*
2413 * Ideally we'd never hit this state; the low water mark
2414 * would trigger userland to extend the pool before we
2415 * completely run out of data space. However, many small
2416 * IOs to unprovisioned space can consume data space at an
2417 * alarming rate. Adjust your low water mark if you're
2418 * frequently seeing this mode.
2419 */
2420 if (old_mode != new_mode)
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002421 notify_of_pool_mode_change_to_oods(pool);
Joe Thornber3e1a0692014-03-03 16:03:26 +00002422 pool->process_bio = process_bio_read_only;
Joe Thornbera374bb22014-10-10 13:43:14 +01002423 pool->process_discard = process_discard_bio;
2424 pool->process_cell = process_cell_read_only;
Joe Thornber3e1a0692014-03-03 16:03:26 +00002425 pool->process_prepared_mapping = process_prepared_mapping;
Joe Thornber34fbcf62015-04-16 12:58:35 +01002426 set_discard_callbacks(pool);
Joe Thornber85ad643b2014-05-09 15:59:38 +01002427
Mike Snitzer80c57892014-05-20 13:38:33 -04002428 if (!pool->pf.error_if_no_space && no_space_timeout)
2429 queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);
Joe Thornbere49e5822012-07-27 15:08:16 +01002430 break;
2431
2432 case PM_WRITE:
Mike Snitzer8b64e882013-12-20 14:27:28 -05002433 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00002434 notify_of_pool_mode_change(pool, "write");
Joe Thornber9b7aaa62013-12-04 16:58:19 -05002435 dm_pool_metadata_read_write(pool->pmd);
Joe Thornbere49e5822012-07-27 15:08:16 +01002436 pool->process_bio = process_bio;
Joe Thornbera374bb22014-10-10 13:43:14 +01002437 pool->process_discard = process_discard_bio;
2438 pool->process_cell = process_cell;
Joe Thornbere49e5822012-07-27 15:08:16 +01002439 pool->process_prepared_mapping = process_prepared_mapping;
Joe Thornber34fbcf62015-04-16 12:58:35 +01002440 set_discard_callbacks(pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01002441 break;
2442 }
Mike Snitzer8b64e882013-12-20 14:27:28 -05002443
2444 pool->pf.mode = new_mode;
Mike Snitzercdc2b412014-02-14 18:10:55 -05002445 /*
2446 * The pool mode may have changed, sync it so bind_control_target()
2447 * doesn't cause an unexpected mode transition on resume.
2448 */
2449 pt->adjusted_pf.mode = new_mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01002450}
2451
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002452static void abort_transaction(struct pool *pool)
2453{
2454 const char *dev_name = dm_device_name(pool->pool_md);
2455
2456 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
2457 if (dm_pool_abort_metadata(pool->pmd)) {
2458 DMERR("%s: failed to abort metadata transaction", dev_name);
2459 set_pool_mode(pool, PM_FAIL);
2460 }
2461
2462 if (dm_pool_metadata_set_needs_check(pool->pmd)) {
2463 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
2464 set_pool_mode(pool, PM_FAIL);
2465 }
2466}
2467
Joe Thornberb5330652013-12-04 19:51:33 -05002468static void metadata_operation_failed(struct pool *pool, const char *op, int r)
2469{
2470 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
2471 dm_device_name(pool->pool_md), op, r);
2472
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002473 abort_transaction(pool);
Joe Thornberb5330652013-12-04 19:51:33 -05002474 set_pool_mode(pool, PM_READ_ONLY);
2475}
2476
Joe Thornbere49e5822012-07-27 15:08:16 +01002477/*----------------------------------------------------------------*/
2478
Joe Thornber991d9fa2011-10-31 20:21:18 +00002479/*
2480 * Mapping functions.
2481 */
2482
2483/*
2484 * Called only while mapping a thin bio to hand it over to the workqueue.
2485 */
2486static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
2487{
2488 unsigned long flags;
2489 struct pool *pool = tc->pool;
2490
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002491 spin_lock_irqsave(&tc->lock, flags);
2492 bio_list_add(&tc->deferred_bio_list, bio);
2493 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002494
2495 wake_worker(pool);
2496}
2497
Joe Thornber7d327fe2014-10-06 15:45:59 +01002498static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio)
2499{
2500 struct pool *pool = tc->pool;
2501
2502 throttle_lock(&pool->throttle);
2503 thin_defer_bio(tc, bio);
2504 throttle_unlock(&pool->throttle);
2505}
2506
Joe Thornbera374bb22014-10-10 13:43:14 +01002507static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2508{
2509 unsigned long flags;
2510 struct pool *pool = tc->pool;
2511
2512 throttle_lock(&pool->throttle);
2513 spin_lock_irqsave(&tc->lock, flags);
2514 list_add_tail(&cell->user_list, &tc->deferred_cells);
2515 spin_unlock_irqrestore(&tc->lock, flags);
2516 throttle_unlock(&pool->throttle);
2517
2518 wake_worker(pool);
2519}
2520
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00002521static void thin_hook_bio(struct thin_c *tc, struct bio *bio)
Joe Thornbereb2aa482012-03-28 18:41:28 +01002522{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00002523 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01002524
2525 h->tc = tc;
2526 h->shared_read_entry = NULL;
Joe Thornbere8088072012-12-21 20:23:31 +00002527 h->all_io_entry = NULL;
Joe Thornbereb2aa482012-03-28 18:41:28 +01002528 h->overwrite_mapping = NULL;
Joe Thornber34fbcf62015-04-16 12:58:35 +01002529 h->cell = NULL;
Joe Thornbereb2aa482012-03-28 18:41:28 +01002530}
2531
Joe Thornber991d9fa2011-10-31 20:21:18 +00002532/*
2533 * Non-blocking function called from the thin target's map function.
2534 */
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00002535static int thin_bio_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002536{
2537 int r;
2538 struct thin_c *tc = ti->private;
2539 dm_block_t block = get_bio_block(tc, bio);
2540 struct dm_thin_device *td = tc->td;
2541 struct dm_thin_lookup_result result;
Joe Thornbera374bb22014-10-10 13:43:14 +01002542 struct dm_bio_prison_cell *virt_cell, *data_cell;
Joe Thornbere8088072012-12-21 20:23:31 +00002543 struct dm_cell_key key;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002544
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00002545 thin_hook_bio(tc, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01002546
Joe Thornber738211f2014-03-03 15:52:28 +00002547 if (tc->requeue_mode) {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002548 bio->bi_error = DM_ENDIO_REQUEUE;
2549 bio_endio(bio);
Joe Thornber738211f2014-03-03 15:52:28 +00002550 return DM_MAPIO_SUBMITTED;
2551 }
2552
Joe Thornbere49e5822012-07-27 15:08:16 +01002553 if (get_pool_mode(tc->pool) == PM_FAIL) {
2554 bio_io_error(bio);
2555 return DM_MAPIO_SUBMITTED;
2556 }
2557
Joe Thornber104655f2012-03-28 18:41:28 +01002558 if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) {
Joe Thornber7d327fe2014-10-06 15:45:59 +01002559 thin_defer_bio_with_throttle(tc, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002560 return DM_MAPIO_SUBMITTED;
2561 }
2562
Joe Thornberc822ed92014-10-10 09:41:09 +01002563 /*
2564 * We must hold the virtual cell before doing the lookup, otherwise
2565 * there's a race with discard.
2566 */
2567 build_virtual_key(tc->td, block, &key);
Joe Thornbera374bb22014-10-10 13:43:14 +01002568 if (bio_detain(tc->pool, &key, bio, &virt_cell))
Joe Thornberc822ed92014-10-10 09:41:09 +01002569 return DM_MAPIO_SUBMITTED;
2570
Joe Thornber991d9fa2011-10-31 20:21:18 +00002571 r = dm_thin_find_block(td, block, 0, &result);
2572
2573 /*
2574 * Note that we defer readahead too.
2575 */
2576 switch (r) {
2577 case 0:
2578 if (unlikely(result.shared)) {
2579 /*
2580 * We have a race condition here between the
2581 * result.shared value returned by the lookup and
2582 * snapshot creation, which may cause new
2583 * sharing.
2584 *
2585 * To avoid this always quiesce the origin before
2586 * taking the snap. You want to do this anyway to
2587 * ensure a consistent application view
2588 * (i.e. lockfs).
2589 *
2590 * More distant ancestors are irrelevant. The
2591 * shared flag will be set in their case.
2592 */
Joe Thornbera374bb22014-10-10 13:43:14 +01002593 thin_defer_cell(tc, virt_cell);
Joe Thornbere8088072012-12-21 20:23:31 +00002594 return DM_MAPIO_SUBMITTED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002595 }
Joe Thornbere8088072012-12-21 20:23:31 +00002596
Joe Thornbere8088072012-12-21 20:23:31 +00002597 build_data_key(tc->td, result.block, &key);
Joe Thornbera374bb22014-10-10 13:43:14 +01002598 if (bio_detain(tc->pool, &key, bio, &data_cell)) {
2599 cell_defer_no_holder(tc, virt_cell);
Joe Thornbere8088072012-12-21 20:23:31 +00002600 return DM_MAPIO_SUBMITTED;
2601 }
2602
2603 inc_all_io_entry(tc->pool, bio);
Joe Thornbera374bb22014-10-10 13:43:14 +01002604 cell_defer_no_holder(tc, data_cell);
2605 cell_defer_no_holder(tc, virt_cell);
Joe Thornbere8088072012-12-21 20:23:31 +00002606
2607 remap(tc, bio, result.block);
2608 return DM_MAPIO_REMAPPED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002609
2610 case -ENODATA:
Joe Thornbere49e5822012-07-27 15:08:16 +01002611 case -EWOULDBLOCK:
Joe Thornbera374bb22014-10-10 13:43:14 +01002612 thin_defer_cell(tc, virt_cell);
Joe Thornber2aab3852012-12-21 20:23:33 +00002613 return DM_MAPIO_SUBMITTED;
Joe Thornbere49e5822012-07-27 15:08:16 +01002614
2615 default:
2616 /*
2617 * Must always call bio_io_error on failure.
2618 * dm_thin_find_block can fail with -EINVAL if the
2619 * pool is switched to fail-io mode.
2620 */
2621 bio_io_error(bio);
Joe Thornbera374bb22014-10-10 13:43:14 +01002622 cell_defer_no_holder(tc, virt_cell);
Joe Thornber2aab3852012-12-21 20:23:33 +00002623 return DM_MAPIO_SUBMITTED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002624 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002625}
2626
2627static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
2628{
Joe Thornber991d9fa2011-10-31 20:21:18 +00002629 struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
Mike Snitzer760fe672014-03-20 08:36:47 -04002630 struct request_queue *q;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002631
Mike Snitzer760fe672014-03-20 08:36:47 -04002632 if (get_pool_mode(pt->pool) == PM_OUT_OF_DATA_SPACE)
2633 return 1;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002634
Mike Snitzer760fe672014-03-20 08:36:47 -04002635 q = bdev_get_queue(pt->data_dev->bdev);
2636 return bdi_congested(&q->backing_dev_info, bdi_bits);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002637}
2638
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002639static void requeue_bios(struct pool *pool)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002640{
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002641 unsigned long flags;
2642 struct thin_c *tc;
2643
2644 rcu_read_lock();
2645 list_for_each_entry_rcu(tc, &pool->active_thins, list) {
2646 spin_lock_irqsave(&tc->lock, flags);
2647 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list);
2648 bio_list_init(&tc->retry_on_resume_list);
2649 spin_unlock_irqrestore(&tc->lock, flags);
2650 }
2651 rcu_read_unlock();
Joe Thornber991d9fa2011-10-31 20:21:18 +00002652}
2653
2654/*----------------------------------------------------------------
2655 * Binding of control targets to a pool object
2656 *--------------------------------------------------------------*/
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002657static bool data_dev_supports_discard(struct pool_c *pt)
2658{
2659 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2660
2661 return q && blk_queue_discard(q);
2662}
2663
Joe Thornber58051b92013-03-20 17:21:25 +00002664static bool is_factor(sector_t block_size, uint32_t n)
2665{
2666 return !sector_div(block_size, n);
2667}
2668
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002669/*
2670 * If discard_passdown was enabled verify that the data device
Mike Snitzer0424caa2012-09-26 23:45:47 +01002671 * supports discards. Disable discard_passdown if not.
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002672 */
Mike Snitzer0424caa2012-09-26 23:45:47 +01002673static void disable_passdown_if_not_supported(struct pool_c *pt)
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002674{
Mike Snitzer0424caa2012-09-26 23:45:47 +01002675 struct pool *pool = pt->pool;
2676 struct block_device *data_bdev = pt->data_dev->bdev;
2677 struct queue_limits *data_limits = &bdev_get_queue(data_bdev)->limits;
Mike Snitzer0424caa2012-09-26 23:45:47 +01002678 const char *reason = NULL;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002679 char buf[BDEVNAME_SIZE];
2680
Mike Snitzer0424caa2012-09-26 23:45:47 +01002681 if (!pt->adjusted_pf.discard_passdown)
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002682 return;
2683
Mike Snitzer0424caa2012-09-26 23:45:47 +01002684 if (!data_dev_supports_discard(pt))
2685 reason = "discard unsupported";
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002686
Mike Snitzer0424caa2012-09-26 23:45:47 +01002687 else if (data_limits->max_discard_sectors < pool->sectors_per_block)
2688 reason = "max discard sectors smaller than a block";
2689
Mike Snitzer0424caa2012-09-26 23:45:47 +01002690 if (reason) {
2691 DMWARN("Data device (%s) %s: Disabling discard passdown.", bdevname(data_bdev, buf), reason);
2692 pt->adjusted_pf.discard_passdown = false;
2693 }
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002694}
2695
Joe Thornber991d9fa2011-10-31 20:21:18 +00002696static int bind_control_target(struct pool *pool, struct dm_target *ti)
2697{
2698 struct pool_c *pt = ti->private;
2699
Joe Thornbere49e5822012-07-27 15:08:16 +01002700 /*
Joe Thornber9b7aaa62013-12-04 16:58:19 -05002701 * We want to make sure that a pool in PM_FAIL mode is never upgraded.
Joe Thornbere49e5822012-07-27 15:08:16 +01002702 */
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002703 enum pool_mode old_mode = get_pool_mode(pool);
Mike Snitzer0424caa2012-09-26 23:45:47 +01002704 enum pool_mode new_mode = pt->adjusted_pf.mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01002705
Joe Thornber9b7aaa62013-12-04 16:58:19 -05002706 /*
Mike Snitzer8b64e882013-12-20 14:27:28 -05002707 * Don't change the pool's mode until set_pool_mode() below.
2708 * Otherwise the pool's process_* function pointers may
2709 * not match the desired pool mode.
2710 */
2711 pt->adjusted_pf.mode = old_mode;
2712
2713 pool->ti = ti;
2714 pool->pf = pt->adjusted_pf;
2715 pool->low_water_blocks = pt->low_water_blocks;
2716
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002717 set_pool_mode(pool, new_mode);
Mike Snitzerf4026932012-05-19 01:01:01 +01002718
Joe Thornber991d9fa2011-10-31 20:21:18 +00002719 return 0;
2720}
2721
2722static void unbind_control_target(struct pool *pool, struct dm_target *ti)
2723{
2724 if (pool->ti == ti)
2725 pool->ti = NULL;
2726}
2727
2728/*----------------------------------------------------------------
2729 * Pool creation
2730 *--------------------------------------------------------------*/
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002731/* Initialize pool features. */
2732static void pool_features_init(struct pool_features *pf)
2733{
Joe Thornbere49e5822012-07-27 15:08:16 +01002734 pf->mode = PM_WRITE;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002735 pf->zero_new_blocks = true;
2736 pf->discard_enabled = true;
2737 pf->discard_passdown = true;
Mike Snitzer787a996c2013-12-06 16:21:43 -05002738 pf->error_if_no_space = false;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002739}
2740
Joe Thornber991d9fa2011-10-31 20:21:18 +00002741static void __pool_destroy(struct pool *pool)
2742{
2743 __pool_table_remove(pool);
2744
Joe Thornbera822c832015-07-03 10:22:42 +01002745 vfree(pool->cell_sort_array);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002746 if (dm_pool_metadata_close(pool->pmd) < 0)
2747 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2748
Mike Snitzer44feb382012-10-12 21:02:10 +01002749 dm_bio_prison_destroy(pool->prison);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002750 dm_kcopyd_client_destroy(pool->copier);
2751
2752 if (pool->wq)
2753 destroy_workqueue(pool->wq);
2754
2755 if (pool->next_mapping)
2756 mempool_free(pool->next_mapping, pool->mapping_pool);
2757 mempool_destroy(pool->mapping_pool);
Mike Snitzer44feb382012-10-12 21:02:10 +01002758 dm_deferred_set_destroy(pool->shared_read_ds);
2759 dm_deferred_set_destroy(pool->all_io_ds);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002760 kfree(pool);
2761}
2762
Mike Snitzera24c2562012-06-03 00:30:00 +01002763static struct kmem_cache *_new_mapping_cache;
Mike Snitzera24c2562012-06-03 00:30:00 +01002764
Joe Thornber991d9fa2011-10-31 20:21:18 +00002765static struct pool *pool_create(struct mapped_device *pool_md,
2766 struct block_device *metadata_dev,
Joe Thornbere49e5822012-07-27 15:08:16 +01002767 unsigned long block_size,
2768 int read_only, char **error)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002769{
2770 int r;
2771 void *err_p;
2772 struct pool *pool;
2773 struct dm_pool_metadata *pmd;
Joe Thornbere49e5822012-07-27 15:08:16 +01002774 bool format_device = read_only ? false : true;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002775
Joe Thornbere49e5822012-07-27 15:08:16 +01002776 pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002777 if (IS_ERR(pmd)) {
2778 *error = "Error creating metadata object";
2779 return (struct pool *)pmd;
2780 }
2781
2782 pool = kmalloc(sizeof(*pool), GFP_KERNEL);
2783 if (!pool) {
2784 *error = "Error allocating memory for pool";
2785 err_p = ERR_PTR(-ENOMEM);
2786 goto bad_pool;
2787 }
2788
2789 pool->pmd = pmd;
2790 pool->sectors_per_block = block_size;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +01002791 if (block_size & (block_size - 1))
2792 pool->sectors_per_block_shift = -1;
2793 else
2794 pool->sectors_per_block_shift = __ffs(block_size);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002795 pool->low_water_blocks = 0;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002796 pool_features_init(&pool->pf);
Joe Thornbera195db22014-10-06 16:30:06 -04002797 pool->prison = dm_bio_prison_create();
Joe Thornber991d9fa2011-10-31 20:21:18 +00002798 if (!pool->prison) {
2799 *error = "Error creating pool's bio prison";
2800 err_p = ERR_PTR(-ENOMEM);
2801 goto bad_prison;
2802 }
2803
Mikulas Patockadf5d2e92013-03-01 22:45:49 +00002804 pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002805 if (IS_ERR(pool->copier)) {
2806 r = PTR_ERR(pool->copier);
2807 *error = "Error creating pool's kcopyd client";
2808 err_p = ERR_PTR(r);
2809 goto bad_kcopyd_client;
2810 }
2811
2812 /*
2813 * Create singlethreaded workqueue that will service all devices
2814 * that use this metadata.
2815 */
2816 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
2817 if (!pool->wq) {
2818 *error = "Error creating pool's workqueue";
2819 err_p = ERR_PTR(-ENOMEM);
2820 goto bad_wq;
2821 }
2822
Joe Thornber7d327fe2014-10-06 15:45:59 +01002823 throttle_init(&pool->throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002824 INIT_WORK(&pool->worker, do_worker);
Joe Thornber905e51b2012-03-28 18:41:27 +01002825 INIT_DELAYED_WORK(&pool->waker, do_waker);
Joe Thornber85ad643b2014-05-09 15:59:38 +01002826 INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002827 spin_lock_init(&pool->lock);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002828 bio_list_init(&pool->deferred_flush_bios);
2829 INIT_LIST_HEAD(&pool->prepared_mappings);
Joe Thornber104655f2012-03-28 18:41:28 +01002830 INIT_LIST_HEAD(&pool->prepared_discards);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002831 INIT_LIST_HEAD(&pool->active_thins);
Joe Thornber88a66212013-12-04 20:16:12 -05002832 pool->low_water_triggered = false;
Mike Snitzer80e96c52014-11-07 15:09:46 -05002833 pool->suspended = true;
Mike Snitzer44feb382012-10-12 21:02:10 +01002834
2835 pool->shared_read_ds = dm_deferred_set_create();
2836 if (!pool->shared_read_ds) {
2837 *error = "Error creating pool's shared read deferred set";
2838 err_p = ERR_PTR(-ENOMEM);
2839 goto bad_shared_read_ds;
2840 }
2841
2842 pool->all_io_ds = dm_deferred_set_create();
2843 if (!pool->all_io_ds) {
2844 *error = "Error creating pool's all io deferred set";
2845 err_p = ERR_PTR(-ENOMEM);
2846 goto bad_all_io_ds;
2847 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002848
2849 pool->next_mapping = NULL;
Mike Snitzera24c2562012-06-03 00:30:00 +01002850 pool->mapping_pool = mempool_create_slab_pool(MAPPING_POOL_SIZE,
2851 _new_mapping_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002852 if (!pool->mapping_pool) {
2853 *error = "Error creating pool's mapping mempool";
2854 err_p = ERR_PTR(-ENOMEM);
2855 goto bad_mapping_pool;
2856 }
2857
Joe Thornbera822c832015-07-03 10:22:42 +01002858 pool->cell_sort_array = vmalloc(sizeof(*pool->cell_sort_array) * CELL_SORT_ARRAY_SIZE);
2859 if (!pool->cell_sort_array) {
2860 *error = "Error allocating cell sort array";
2861 err_p = ERR_PTR(-ENOMEM);
2862 goto bad_sort_array;
2863 }
2864
Joe Thornber991d9fa2011-10-31 20:21:18 +00002865 pool->ref_count = 1;
Joe Thornber905e51b2012-03-28 18:41:27 +01002866 pool->last_commit_jiffies = jiffies;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002867 pool->pool_md = pool_md;
2868 pool->md_dev = metadata_dev;
2869 __pool_table_insert(pool);
2870
2871 return pool;
2872
Joe Thornbera822c832015-07-03 10:22:42 +01002873bad_sort_array:
2874 mempool_destroy(pool->mapping_pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002875bad_mapping_pool:
Mike Snitzer44feb382012-10-12 21:02:10 +01002876 dm_deferred_set_destroy(pool->all_io_ds);
2877bad_all_io_ds:
2878 dm_deferred_set_destroy(pool->shared_read_ds);
2879bad_shared_read_ds:
Joe Thornber991d9fa2011-10-31 20:21:18 +00002880 destroy_workqueue(pool->wq);
2881bad_wq:
2882 dm_kcopyd_client_destroy(pool->copier);
2883bad_kcopyd_client:
Mike Snitzer44feb382012-10-12 21:02:10 +01002884 dm_bio_prison_destroy(pool->prison);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002885bad_prison:
2886 kfree(pool);
2887bad_pool:
2888 if (dm_pool_metadata_close(pmd))
2889 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2890
2891 return err_p;
2892}
2893
2894static void __pool_inc(struct pool *pool)
2895{
2896 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
2897 pool->ref_count++;
2898}
2899
2900static void __pool_dec(struct pool *pool)
2901{
2902 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
2903 BUG_ON(!pool->ref_count);
2904 if (!--pool->ref_count)
2905 __pool_destroy(pool);
2906}
2907
2908static struct pool *__pool_find(struct mapped_device *pool_md,
2909 struct block_device *metadata_dev,
Joe Thornbere49e5822012-07-27 15:08:16 +01002910 unsigned long block_size, int read_only,
2911 char **error, int *created)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002912{
2913 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
2914
2915 if (pool) {
Mike Snitzerf09996c2012-07-27 15:07:59 +01002916 if (pool->pool_md != pool_md) {
2917 *error = "metadata device already in use by a pool";
Joe Thornber991d9fa2011-10-31 20:21:18 +00002918 return ERR_PTR(-EBUSY);
Mike Snitzerf09996c2012-07-27 15:07:59 +01002919 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002920 __pool_inc(pool);
2921
2922 } else {
2923 pool = __pool_table_lookup(pool_md);
2924 if (pool) {
Mike Snitzerf09996c2012-07-27 15:07:59 +01002925 if (pool->md_dev != metadata_dev) {
2926 *error = "different pool cannot replace a pool";
Joe Thornber991d9fa2011-10-31 20:21:18 +00002927 return ERR_PTR(-EINVAL);
Mike Snitzerf09996c2012-07-27 15:07:59 +01002928 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002929 __pool_inc(pool);
2930
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002931 } else {
Joe Thornbere49e5822012-07-27 15:08:16 +01002932 pool = pool_create(pool_md, metadata_dev, block_size, read_only, error);
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002933 *created = 1;
2934 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002935 }
2936
2937 return pool;
2938}
2939
2940/*----------------------------------------------------------------
2941 * Pool target methods
2942 *--------------------------------------------------------------*/
2943static void pool_dtr(struct dm_target *ti)
2944{
2945 struct pool_c *pt = ti->private;
2946
2947 mutex_lock(&dm_thin_pool_table.mutex);
2948
2949 unbind_control_target(pt->pool, ti);
2950 __pool_dec(pt->pool);
2951 dm_put_device(ti, pt->metadata_dev);
2952 dm_put_device(ti, pt->data_dev);
2953 kfree(pt);
2954
2955 mutex_unlock(&dm_thin_pool_table.mutex);
2956}
2957
Joe Thornber991d9fa2011-10-31 20:21:18 +00002958static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
2959 struct dm_target *ti)
2960{
2961 int r;
2962 unsigned argc;
2963 const char *arg_name;
2964
2965 static struct dm_arg _args[] = {
Mike Snitzer74aa45c2014-01-15 19:07:58 -05002966 {0, 4, "Invalid number of pool feature arguments"},
Joe Thornber991d9fa2011-10-31 20:21:18 +00002967 };
2968
2969 /*
2970 * No feature arguments supplied.
2971 */
2972 if (!as->argc)
2973 return 0;
2974
2975 r = dm_read_arg_group(_args, as, &argc, &ti->error);
2976 if (r)
2977 return -EINVAL;
2978
2979 while (argc && !r) {
2980 arg_name = dm_shift_arg(as);
2981 argc--;
2982
Joe Thornbere49e5822012-07-27 15:08:16 +01002983 if (!strcasecmp(arg_name, "skip_block_zeroing"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002984 pf->zero_new_blocks = false;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002985
Joe Thornbere49e5822012-07-27 15:08:16 +01002986 else if (!strcasecmp(arg_name, "ignore_discard"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002987 pf->discard_enabled = false;
Joe Thornbere49e5822012-07-27 15:08:16 +01002988
2989 else if (!strcasecmp(arg_name, "no_discard_passdown"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002990 pf->discard_passdown = false;
Joe Thornbere49e5822012-07-27 15:08:16 +01002991
2992 else if (!strcasecmp(arg_name, "read_only"))
2993 pf->mode = PM_READ_ONLY;
2994
Mike Snitzer787a996c2013-12-06 16:21:43 -05002995 else if (!strcasecmp(arg_name, "error_if_no_space"))
2996 pf->error_if_no_space = true;
2997
Joe Thornbere49e5822012-07-27 15:08:16 +01002998 else {
2999 ti->error = "Unrecognised pool feature requested";
3000 r = -EINVAL;
3001 break;
3002 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003003 }
3004
3005 return r;
3006}
3007
Joe Thornberac8c3f32013-05-10 14:37:21 +01003008static void metadata_low_callback(void *context)
3009{
3010 struct pool *pool = context;
3011
3012 DMWARN("%s: reached low water mark for metadata device: sending event.",
3013 dm_device_name(pool->pool_md));
3014
3015 dm_table_event(pool->ti->table);
3016}
3017
Mike Snitzer7d489352014-02-12 23:58:15 -05003018static sector_t get_dev_size(struct block_device *bdev)
Joe Thornberb17446d2013-05-10 14:37:18 +01003019{
Mike Snitzer7d489352014-02-12 23:58:15 -05003020 return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
3021}
3022
3023static void warn_if_metadata_device_too_big(struct block_device *bdev)
3024{
3025 sector_t metadata_dev_size = get_dev_size(bdev);
Joe Thornberb17446d2013-05-10 14:37:18 +01003026 char buffer[BDEVNAME_SIZE];
3027
Mike Snitzer7d489352014-02-12 23:58:15 -05003028 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
Joe Thornberb17446d2013-05-10 14:37:18 +01003029 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
3030 bdevname(bdev, buffer), THIN_METADATA_MAX_SECTORS);
Mike Snitzer7d489352014-02-12 23:58:15 -05003031}
3032
3033static sector_t get_metadata_dev_size(struct block_device *bdev)
3034{
3035 sector_t metadata_dev_size = get_dev_size(bdev);
3036
3037 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)
3038 metadata_dev_size = THIN_METADATA_MAX_SECTORS;
Joe Thornberb17446d2013-05-10 14:37:18 +01003039
3040 return metadata_dev_size;
3041}
3042
Joe Thornber24347e92013-05-10 14:37:19 +01003043static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)
3044{
3045 sector_t metadata_dev_size = get_metadata_dev_size(bdev);
3046
Mike Snitzer7d489352014-02-12 23:58:15 -05003047 sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);
Joe Thornber24347e92013-05-10 14:37:19 +01003048
3049 return metadata_dev_size;
3050}
3051
Joe Thornber991d9fa2011-10-31 20:21:18 +00003052/*
Joe Thornberac8c3f32013-05-10 14:37:21 +01003053 * When a metadata threshold is crossed a dm event is triggered, and
3054 * userland should respond by growing the metadata device. We could let
3055 * userland set the threshold, like we do with the data threshold, but I'm
3056 * not sure they know enough to do this well.
3057 */
3058static dm_block_t calc_metadata_threshold(struct pool_c *pt)
3059{
3060 /*
3061 * 4M is ample for all ops with the possible exception of thin
3062 * device deletion which is harmless if it fails (just retry the
3063 * delete after you've grown the device).
3064 */
3065 dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4;
3066 return min((dm_block_t)1024ULL /* 4M */, quarter);
3067}
3068
3069/*
Joe Thornber991d9fa2011-10-31 20:21:18 +00003070 * thin-pool <metadata dev> <data dev>
3071 * <data block size (sectors)>
3072 * <low water mark (blocks)>
3073 * [<#feature args> [<arg>]*]
3074 *
3075 * Optional feature arguments are:
3076 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003077 * ignore_discard: disable discard
3078 * no_discard_passdown: don't pass discards down to the data device
Mike Snitzer787a996c2013-12-06 16:21:43 -05003079 * read_only: Don't allow any changes to be made to the pool metadata.
3080 * error_if_no_space: error IOs, instead of queueing, if no space.
Joe Thornber991d9fa2011-10-31 20:21:18 +00003081 */
3082static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
3083{
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003084 int r, pool_created = 0;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003085 struct pool_c *pt;
3086 struct pool *pool;
3087 struct pool_features pf;
3088 struct dm_arg_set as;
3089 struct dm_dev *data_dev;
3090 unsigned long block_size;
3091 dm_block_t low_water_blocks;
3092 struct dm_dev *metadata_dev;
Joe Thornber5d0db962013-05-10 14:37:19 +01003093 fmode_t metadata_mode;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003094
3095 /*
3096 * FIXME Remove validation from scope of lock.
3097 */
3098 mutex_lock(&dm_thin_pool_table.mutex);
3099
3100 if (argc < 4) {
3101 ti->error = "Invalid argument count";
3102 r = -EINVAL;
3103 goto out_unlock;
3104 }
Joe Thornber5d0db962013-05-10 14:37:19 +01003105
Joe Thornber991d9fa2011-10-31 20:21:18 +00003106 as.argc = argc;
3107 as.argv = argv;
3108
Joe Thornber5d0db962013-05-10 14:37:19 +01003109 /*
3110 * Set default pool features.
3111 */
3112 pool_features_init(&pf);
3113
3114 dm_consume_args(&as, 4);
3115 r = parse_pool_features(&as, &pf, ti);
3116 if (r)
3117 goto out_unlock;
3118
3119 metadata_mode = FMODE_READ | ((pf.mode == PM_READ_ONLY) ? 0 : FMODE_WRITE);
3120 r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003121 if (r) {
3122 ti->error = "Error opening metadata block device";
3123 goto out_unlock;
3124 }
Mike Snitzer7d489352014-02-12 23:58:15 -05003125 warn_if_metadata_device_too_big(metadata_dev->bdev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003126
3127 r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
3128 if (r) {
3129 ti->error = "Error getting data device";
3130 goto out_metadata;
3131 }
3132
3133 if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
3134 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
3135 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01003136 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00003137 ti->error = "Invalid block size";
3138 r = -EINVAL;
3139 goto out;
3140 }
3141
3142 if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
3143 ti->error = "Invalid low water mark";
3144 r = -EINVAL;
3145 goto out;
3146 }
3147
Joe Thornber991d9fa2011-10-31 20:21:18 +00003148 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
3149 if (!pt) {
3150 r = -ENOMEM;
3151 goto out;
3152 }
3153
3154 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev,
Joe Thornbere49e5822012-07-27 15:08:16 +01003155 block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003156 if (IS_ERR(pool)) {
3157 r = PTR_ERR(pool);
3158 goto out_free_pt;
3159 }
3160
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003161 /*
3162 * 'pool_created' reflects whether this is the first table load.
3163 * Top level discard support is not allowed to be changed after
3164 * initial load. This would require a pool reload to trigger thin
3165 * device changes.
3166 */
3167 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
3168 ti->error = "Discard support cannot be disabled once enabled";
3169 r = -EINVAL;
3170 goto out_flags_changed;
3171 }
3172
Joe Thornber991d9fa2011-10-31 20:21:18 +00003173 pt->pool = pool;
3174 pt->ti = ti;
3175 pt->metadata_dev = metadata_dev;
3176 pt->data_dev = data_dev;
3177 pt->low_water_blocks = low_water_blocks;
Mike Snitzer0424caa2012-09-26 23:45:47 +01003178 pt->adjusted_pf = pt->requested_pf = pf;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00003179 ti->num_flush_bios = 1;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01003180
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003181 /*
3182 * Only need to enable discards if the pool should pass
3183 * them down to the data device. The thin device's discard
3184 * processing will cause mappings to be removed from the btree.
3185 */
Mike Snitzerb60ab992013-09-19 18:49:11 -04003186 ti->discard_zeroes_data_unsupported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003187 if (pf.discard_enabled && pf.discard_passdown) {
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00003188 ti->num_discard_bios = 1;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01003189
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003190 /*
3191 * Setting 'discards_supported' circumvents the normal
3192 * stacking of discard limits (this keeps the pool and
3193 * thin devices' discard limits consistent).
3194 */
Alasdair G Kergon0ac55482012-07-27 15:08:08 +01003195 ti->discards_supported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003196 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003197 ti->private = pt;
3198
Joe Thornberac8c3f32013-05-10 14:37:21 +01003199 r = dm_pool_register_metadata_threshold(pt->pool->pmd,
3200 calc_metadata_threshold(pt),
3201 metadata_low_callback,
3202 pool);
3203 if (r)
3204 goto out_free_pt;
3205
Joe Thornber991d9fa2011-10-31 20:21:18 +00003206 pt->callbacks.congested_fn = pool_is_congested;
3207 dm_table_add_target_callbacks(ti->table, &pt->callbacks);
3208
3209 mutex_unlock(&dm_thin_pool_table.mutex);
3210
3211 return 0;
3212
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003213out_flags_changed:
3214 __pool_dec(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003215out_free_pt:
3216 kfree(pt);
3217out:
3218 dm_put_device(ti, data_dev);
3219out_metadata:
3220 dm_put_device(ti, metadata_dev);
3221out_unlock:
3222 mutex_unlock(&dm_thin_pool_table.mutex);
3223
3224 return r;
3225}
3226
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00003227static int pool_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003228{
3229 int r;
3230 struct pool_c *pt = ti->private;
3231 struct pool *pool = pt->pool;
3232 unsigned long flags;
3233
3234 /*
3235 * As this is a singleton target, ti->begin is always zero.
3236 */
3237 spin_lock_irqsave(&pool->lock, flags);
3238 bio->bi_bdev = pt->data_dev->bdev;
3239 r = DM_MAPIO_REMAPPED;
3240 spin_unlock_irqrestore(&pool->lock, flags);
3241
3242 return r;
3243}
3244
Joe Thornberb17446d2013-05-10 14:37:18 +01003245static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)
3246{
3247 int r;
3248 struct pool_c *pt = ti->private;
3249 struct pool *pool = pt->pool;
3250 sector_t data_size = ti->len;
3251 dm_block_t sb_data_size;
3252
3253 *need_commit = false;
3254
3255 (void) sector_div(data_size, pool->sectors_per_block);
3256
3257 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
3258 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003259 DMERR("%s: failed to retrieve data device size",
3260 dm_device_name(pool->pool_md));
Joe Thornberb17446d2013-05-10 14:37:18 +01003261 return r;
3262 }
3263
3264 if (data_size < sb_data_size) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003265 DMERR("%s: pool target (%llu blocks) too small: expected %llu",
3266 dm_device_name(pool->pool_md),
Joe Thornberb17446d2013-05-10 14:37:18 +01003267 (unsigned long long)data_size, sb_data_size);
3268 return -EINVAL;
3269
3270 } else if (data_size > sb_data_size) {
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05003271 if (dm_pool_metadata_needs_check(pool->pmd)) {
3272 DMERR("%s: unable to grow the data device until repaired.",
3273 dm_device_name(pool->pool_md));
3274 return 0;
3275 }
3276
Mike Snitzer6f7f51d2013-12-04 10:25:53 -05003277 if (sb_data_size)
3278 DMINFO("%s: growing the data device from %llu to %llu blocks",
3279 dm_device_name(pool->pool_md),
3280 sb_data_size, (unsigned long long)data_size);
Joe Thornberb17446d2013-05-10 14:37:18 +01003281 r = dm_pool_resize_data_dev(pool->pmd, data_size);
3282 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05003283 metadata_operation_failed(pool, "dm_pool_resize_data_dev", r);
Joe Thornberb17446d2013-05-10 14:37:18 +01003284 return r;
3285 }
3286
3287 *need_commit = true;
3288 }
3289
3290 return 0;
3291}
3292
Joe Thornber24347e92013-05-10 14:37:19 +01003293static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)
3294{
3295 int r;
3296 struct pool_c *pt = ti->private;
3297 struct pool *pool = pt->pool;
3298 dm_block_t metadata_dev_size, sb_metadata_dev_size;
3299
3300 *need_commit = false;
3301
Alasdair G Kergon610bba82013-05-19 18:57:50 +01003302 metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev);
Joe Thornber24347e92013-05-10 14:37:19 +01003303
3304 r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size);
3305 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003306 DMERR("%s: failed to retrieve metadata device size",
3307 dm_device_name(pool->pool_md));
Joe Thornber24347e92013-05-10 14:37:19 +01003308 return r;
3309 }
3310
3311 if (metadata_dev_size < sb_metadata_dev_size) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003312 DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
3313 dm_device_name(pool->pool_md),
Joe Thornber24347e92013-05-10 14:37:19 +01003314 metadata_dev_size, sb_metadata_dev_size);
3315 return -EINVAL;
3316
3317 } else if (metadata_dev_size > sb_metadata_dev_size) {
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05003318 if (dm_pool_metadata_needs_check(pool->pmd)) {
3319 DMERR("%s: unable to grow the metadata device until repaired.",
3320 dm_device_name(pool->pool_md));
3321 return 0;
3322 }
3323
Mike Snitzer7d489352014-02-12 23:58:15 -05003324 warn_if_metadata_device_too_big(pool->md_dev);
Mike Snitzer6f7f51d2013-12-04 10:25:53 -05003325 DMINFO("%s: growing the metadata device from %llu to %llu blocks",
3326 dm_device_name(pool->pool_md),
3327 sb_metadata_dev_size, metadata_dev_size);
Joe Thornber24347e92013-05-10 14:37:19 +01003328 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size);
3329 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05003330 metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r);
Joe Thornber24347e92013-05-10 14:37:19 +01003331 return r;
3332 }
3333
3334 *need_commit = true;
3335 }
3336
3337 return 0;
3338}
3339
Joe Thornber991d9fa2011-10-31 20:21:18 +00003340/*
3341 * Retrieves the number of blocks of the data device from
3342 * the superblock and compares it to the actual device size,
3343 * thus resizing the data device in case it has grown.
3344 *
3345 * This both copes with opening preallocated data devices in the ctr
3346 * being followed by a resume
3347 * -and-
3348 * calling the resume method individually after userspace has
3349 * grown the data device in reaction to a table event.
3350 */
3351static int pool_preresume(struct dm_target *ti)
3352{
3353 int r;
Joe Thornber24347e92013-05-10 14:37:19 +01003354 bool need_commit1, need_commit2;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003355 struct pool_c *pt = ti->private;
3356 struct pool *pool = pt->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003357
3358 /*
3359 * Take control of the pool object.
3360 */
3361 r = bind_control_target(pool, ti);
3362 if (r)
3363 return r;
3364
Joe Thornberb17446d2013-05-10 14:37:18 +01003365 r = maybe_resize_data_dev(ti, &need_commit1);
3366 if (r)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003367 return r;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003368
Joe Thornber24347e92013-05-10 14:37:19 +01003369 r = maybe_resize_metadata_dev(ti, &need_commit2);
3370 if (r)
3371 return r;
3372
3373 if (need_commit1 || need_commit2)
Joe Thornber020cc3b2013-12-04 15:05:36 -05003374 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003375
3376 return 0;
3377}
3378
Mike Snitzer583024d2014-10-28 20:58:45 -04003379static void pool_suspend_active_thins(struct pool *pool)
3380{
3381 struct thin_c *tc;
3382
3383 /* Suspend all active thin devices */
3384 tc = get_first_thin(pool);
3385 while (tc) {
3386 dm_internal_suspend_noflush(tc->thin_md);
3387 tc = get_next_thin(pool, tc);
3388 }
3389}
3390
3391static void pool_resume_active_thins(struct pool *pool)
3392{
3393 struct thin_c *tc;
3394
3395 /* Resume all active thin devices */
3396 tc = get_first_thin(pool);
3397 while (tc) {
3398 dm_internal_resume(tc->thin_md);
3399 tc = get_next_thin(pool, tc);
3400 }
3401}
3402
Joe Thornber991d9fa2011-10-31 20:21:18 +00003403static void pool_resume(struct dm_target *ti)
3404{
3405 struct pool_c *pt = ti->private;
3406 struct pool *pool = pt->pool;
3407 unsigned long flags;
3408
Mike Snitzer583024d2014-10-28 20:58:45 -04003409 /*
3410 * Must requeue active_thins' bios and then resume
3411 * active_thins _before_ clearing 'suspend' flag.
3412 */
3413 requeue_bios(pool);
3414 pool_resume_active_thins(pool);
3415
Joe Thornber991d9fa2011-10-31 20:21:18 +00003416 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber88a66212013-12-04 20:16:12 -05003417 pool->low_water_triggered = false;
Mike Snitzer80e96c52014-11-07 15:09:46 -05003418 pool->suspended = false;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003419 spin_unlock_irqrestore(&pool->lock, flags);
Mike Snitzer80e96c52014-11-07 15:09:46 -05003420
Joe Thornber905e51b2012-03-28 18:41:27 +01003421 do_waker(&pool->waker.work);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003422}
3423
Mike Snitzer80e96c52014-11-07 15:09:46 -05003424static void pool_presuspend(struct dm_target *ti)
3425{
3426 struct pool_c *pt = ti->private;
3427 struct pool *pool = pt->pool;
3428 unsigned long flags;
3429
3430 spin_lock_irqsave(&pool->lock, flags);
3431 pool->suspended = true;
3432 spin_unlock_irqrestore(&pool->lock, flags);
Mike Snitzer583024d2014-10-28 20:58:45 -04003433
3434 pool_suspend_active_thins(pool);
Mike Snitzer80e96c52014-11-07 15:09:46 -05003435}
3436
3437static void pool_presuspend_undo(struct dm_target *ti)
3438{
3439 struct pool_c *pt = ti->private;
3440 struct pool *pool = pt->pool;
3441 unsigned long flags;
3442
Mike Snitzer583024d2014-10-28 20:58:45 -04003443 pool_resume_active_thins(pool);
3444
Mike Snitzer80e96c52014-11-07 15:09:46 -05003445 spin_lock_irqsave(&pool->lock, flags);
3446 pool->suspended = false;
3447 spin_unlock_irqrestore(&pool->lock, flags);
3448}
3449
Joe Thornber991d9fa2011-10-31 20:21:18 +00003450static void pool_postsuspend(struct dm_target *ti)
3451{
Joe Thornber991d9fa2011-10-31 20:21:18 +00003452 struct pool_c *pt = ti->private;
3453 struct pool *pool = pt->pool;
3454
Joe Thornber905e51b2012-03-28 18:41:27 +01003455 cancel_delayed_work(&pool->waker);
Joe Thornber85ad643b2014-05-09 15:59:38 +01003456 cancel_delayed_work(&pool->no_space_timeout);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003457 flush_workqueue(pool->wq);
Joe Thornber020cc3b2013-12-04 15:05:36 -05003458 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003459}
3460
3461static int check_arg_count(unsigned argc, unsigned args_required)
3462{
3463 if (argc != args_required) {
3464 DMWARN("Message received with %u arguments instead of %u.",
3465 argc, args_required);
3466 return -EINVAL;
3467 }
3468
3469 return 0;
3470}
3471
3472static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
3473{
3474 if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
3475 *dev_id <= MAX_DEV_ID)
3476 return 0;
3477
3478 if (warning)
3479 DMWARN("Message received with invalid device id: %s", arg);
3480
3481 return -EINVAL;
3482}
3483
3484static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
3485{
3486 dm_thin_id dev_id;
3487 int r;
3488
3489 r = check_arg_count(argc, 2);
3490 if (r)
3491 return r;
3492
3493 r = read_dev_id(argv[1], &dev_id, 1);
3494 if (r)
3495 return r;
3496
3497 r = dm_pool_create_thin(pool->pmd, dev_id);
3498 if (r) {
3499 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
3500 argv[1]);
3501 return r;
3502 }
3503
3504 return 0;
3505}
3506
3507static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3508{
3509 dm_thin_id dev_id;
3510 dm_thin_id origin_dev_id;
3511 int r;
3512
3513 r = check_arg_count(argc, 3);
3514 if (r)
3515 return r;
3516
3517 r = read_dev_id(argv[1], &dev_id, 1);
3518 if (r)
3519 return r;
3520
3521 r = read_dev_id(argv[2], &origin_dev_id, 1);
3522 if (r)
3523 return r;
3524
3525 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
3526 if (r) {
3527 DMWARN("Creation of new snapshot %s of device %s failed.",
3528 argv[1], argv[2]);
3529 return r;
3530 }
3531
3532 return 0;
3533}
3534
3535static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
3536{
3537 dm_thin_id dev_id;
3538 int r;
3539
3540 r = check_arg_count(argc, 2);
3541 if (r)
3542 return r;
3543
3544 r = read_dev_id(argv[1], &dev_id, 1);
3545 if (r)
3546 return r;
3547
3548 r = dm_pool_delete_thin_device(pool->pmd, dev_id);
3549 if (r)
3550 DMWARN("Deletion of thin device %s failed.", argv[1]);
3551
3552 return r;
3553}
3554
3555static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
3556{
3557 dm_thin_id old_id, new_id;
3558 int r;
3559
3560 r = check_arg_count(argc, 3);
3561 if (r)
3562 return r;
3563
3564 if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
3565 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
3566 return -EINVAL;
3567 }
3568
3569 if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
3570 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
3571 return -EINVAL;
3572 }
3573
3574 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
3575 if (r) {
3576 DMWARN("Failed to change transaction id from %s to %s.",
3577 argv[1], argv[2]);
3578 return r;
3579 }
3580
3581 return 0;
3582}
3583
Joe Thornbercc8394d2012-06-03 00:30:01 +01003584static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3585{
3586 int r;
3587
3588 r = check_arg_count(argc, 1);
3589 if (r)
3590 return r;
3591
Joe Thornber020cc3b2013-12-04 15:05:36 -05003592 (void) commit(pool);
Joe Thornber0d200ae2012-07-03 12:55:31 +01003593
Joe Thornbercc8394d2012-06-03 00:30:01 +01003594 r = dm_pool_reserve_metadata_snap(pool->pmd);
3595 if (r)
3596 DMWARN("reserve_metadata_snap message failed.");
3597
3598 return r;
3599}
3600
3601static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3602{
3603 int r;
3604
3605 r = check_arg_count(argc, 1);
3606 if (r)
3607 return r;
3608
3609 r = dm_pool_release_metadata_snap(pool->pmd);
3610 if (r)
3611 DMWARN("release_metadata_snap message failed.");
3612
3613 return r;
3614}
3615
Joe Thornber991d9fa2011-10-31 20:21:18 +00003616/*
3617 * Messages supported:
3618 * create_thin <dev_id>
3619 * create_snap <dev_id> <origin_id>
3620 * delete <dev_id>
Joe Thornber991d9fa2011-10-31 20:21:18 +00003621 * set_transaction_id <current_trans_id> <new_trans_id>
Joe Thornbercc8394d2012-06-03 00:30:01 +01003622 * reserve_metadata_snap
3623 * release_metadata_snap
Joe Thornber991d9fa2011-10-31 20:21:18 +00003624 */
3625static int pool_message(struct dm_target *ti, unsigned argc, char **argv)
3626{
3627 int r = -EINVAL;
3628 struct pool_c *pt = ti->private;
3629 struct pool *pool = pt->pool;
3630
Joe Thornber2a7eaea2015-01-26 11:38:21 +00003631 if (get_pool_mode(pool) >= PM_READ_ONLY) {
3632 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",
3633 dm_device_name(pool->pool_md));
Mike Snitzerfd467692015-06-09 12:31:26 -04003634 return -EOPNOTSUPP;
Joe Thornber2a7eaea2015-01-26 11:38:21 +00003635 }
3636
Joe Thornber991d9fa2011-10-31 20:21:18 +00003637 if (!strcasecmp(argv[0], "create_thin"))
3638 r = process_create_thin_mesg(argc, argv, pool);
3639
3640 else if (!strcasecmp(argv[0], "create_snap"))
3641 r = process_create_snap_mesg(argc, argv, pool);
3642
3643 else if (!strcasecmp(argv[0], "delete"))
3644 r = process_delete_mesg(argc, argv, pool);
3645
3646 else if (!strcasecmp(argv[0], "set_transaction_id"))
3647 r = process_set_transaction_id_mesg(argc, argv, pool);
3648
Joe Thornbercc8394d2012-06-03 00:30:01 +01003649 else if (!strcasecmp(argv[0], "reserve_metadata_snap"))
3650 r = process_reserve_metadata_snap_mesg(argc, argv, pool);
3651
3652 else if (!strcasecmp(argv[0], "release_metadata_snap"))
3653 r = process_release_metadata_snap_mesg(argc, argv, pool);
3654
Joe Thornber991d9fa2011-10-31 20:21:18 +00003655 else
3656 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
3657
Joe Thornbere49e5822012-07-27 15:08:16 +01003658 if (!r)
Joe Thornber020cc3b2013-12-04 15:05:36 -05003659 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003660
3661 return r;
3662}
3663
Joe Thornbere49e5822012-07-27 15:08:16 +01003664static void emit_flags(struct pool_features *pf, char *result,
3665 unsigned sz, unsigned maxlen)
3666{
3667 unsigned count = !pf->zero_new_blocks + !pf->discard_enabled +
Mike Snitzer787a996c2013-12-06 16:21:43 -05003668 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) +
3669 pf->error_if_no_space;
Joe Thornbere49e5822012-07-27 15:08:16 +01003670 DMEMIT("%u ", count);
3671
3672 if (!pf->zero_new_blocks)
3673 DMEMIT("skip_block_zeroing ");
3674
3675 if (!pf->discard_enabled)
3676 DMEMIT("ignore_discard ");
3677
3678 if (!pf->discard_passdown)
3679 DMEMIT("no_discard_passdown ");
3680
3681 if (pf->mode == PM_READ_ONLY)
3682 DMEMIT("read_only ");
Mike Snitzer787a996c2013-12-06 16:21:43 -05003683
3684 if (pf->error_if_no_space)
3685 DMEMIT("error_if_no_space ");
Joe Thornbere49e5822012-07-27 15:08:16 +01003686}
3687
Joe Thornber991d9fa2011-10-31 20:21:18 +00003688/*
3689 * Status line is:
3690 * <transaction id> <used metadata sectors>/<total metadata sectors>
3691 * <used data sectors>/<total data sectors> <held metadata root>
Mike Snitzere4c78e22015-07-15 11:40:24 -04003692 * <pool mode> <discard config> <no space config> <needs_check>
Joe Thornber991d9fa2011-10-31 20:21:18 +00003693 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003694static void pool_status(struct dm_target *ti, status_type_t type,
3695 unsigned status_flags, char *result, unsigned maxlen)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003696{
Joe Thornbere49e5822012-07-27 15:08:16 +01003697 int r;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003698 unsigned sz = 0;
3699 uint64_t transaction_id;
3700 dm_block_t nr_free_blocks_data;
3701 dm_block_t nr_free_blocks_metadata;
3702 dm_block_t nr_blocks_data;
3703 dm_block_t nr_blocks_metadata;
3704 dm_block_t held_root;
3705 char buf[BDEVNAME_SIZE];
3706 char buf2[BDEVNAME_SIZE];
3707 struct pool_c *pt = ti->private;
3708 struct pool *pool = pt->pool;
3709
3710 switch (type) {
3711 case STATUSTYPE_INFO:
Joe Thornbere49e5822012-07-27 15:08:16 +01003712 if (get_pool_mode(pool) == PM_FAIL) {
3713 DMEMIT("Fail");
3714 break;
3715 }
3716
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01003717 /* Commit to ensure statistics aren't out-of-date */
3718 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
Joe Thornber020cc3b2013-12-04 15:05:36 -05003719 (void) commit(pool);
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01003720
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003721 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);
3722 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003723 DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
3724 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003725 goto err;
3726 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003727
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003728 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);
3729 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003730 DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
3731 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003732 goto err;
3733 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003734
3735 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003736 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003737 DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
3738 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003739 goto err;
3740 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003741
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003742 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);
3743 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003744 DMERR("%s: dm_pool_get_free_block_count returned %d",
3745 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003746 goto err;
3747 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003748
3749 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003750 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003751 DMERR("%s: dm_pool_get_data_dev_size returned %d",
3752 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003753 goto err;
3754 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003755
Joe Thornbercc8394d2012-06-03 00:30:01 +01003756 r = dm_pool_get_metadata_snap(pool->pmd, &held_root);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003757 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003758 DMERR("%s: dm_pool_get_metadata_snap returned %d",
3759 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003760 goto err;
3761 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003762
3763 DMEMIT("%llu %llu/%llu %llu/%llu ",
3764 (unsigned long long)transaction_id,
3765 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3766 (unsigned long long)nr_blocks_metadata,
3767 (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
3768 (unsigned long long)nr_blocks_data);
3769
3770 if (held_root)
Joe Thornbere49e5822012-07-27 15:08:16 +01003771 DMEMIT("%llu ", held_root);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003772 else
Joe Thornbere49e5822012-07-27 15:08:16 +01003773 DMEMIT("- ");
3774
Joe Thornber3e1a0692014-03-03 16:03:26 +00003775 if (pool->pf.mode == PM_OUT_OF_DATA_SPACE)
3776 DMEMIT("out_of_data_space ");
3777 else if (pool->pf.mode == PM_READ_ONLY)
Joe Thornbere49e5822012-07-27 15:08:16 +01003778 DMEMIT("ro ");
3779 else
3780 DMEMIT("rw ");
3781
Mike Snitzer018debe2012-12-21 20:23:32 +00003782 if (!pool->pf.discard_enabled)
Mike Snitzer787a996c2013-12-06 16:21:43 -05003783 DMEMIT("ignore_discard ");
Mike Snitzer018debe2012-12-21 20:23:32 +00003784 else if (pool->pf.discard_passdown)
Mike Snitzer787a996c2013-12-06 16:21:43 -05003785 DMEMIT("discard_passdown ");
Joe Thornbere49e5822012-07-27 15:08:16 +01003786 else
Mike Snitzer787a996c2013-12-06 16:21:43 -05003787 DMEMIT("no_discard_passdown ");
3788
3789 if (pool->pf.error_if_no_space)
3790 DMEMIT("error_if_no_space ");
3791 else
3792 DMEMIT("queue_if_no_space ");
Joe Thornber991d9fa2011-10-31 20:21:18 +00003793
Mike Snitzere4c78e22015-07-15 11:40:24 -04003794 if (dm_pool_metadata_needs_check(pool->pmd))
3795 DMEMIT("needs_check ");
3796 else
3797 DMEMIT("- ");
3798
Joe Thornber991d9fa2011-10-31 20:21:18 +00003799 break;
3800
3801 case STATUSTYPE_TABLE:
3802 DMEMIT("%s %s %lu %llu ",
3803 format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
3804 format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
3805 (unsigned long)pool->sectors_per_block,
3806 (unsigned long long)pt->low_water_blocks);
Mike Snitzer0424caa2012-09-26 23:45:47 +01003807 emit_flags(&pt->requested_pf, result, sz, maxlen);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003808 break;
3809 }
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003810 return;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003811
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003812err:
3813 DMEMIT("Error");
Joe Thornber991d9fa2011-10-31 20:21:18 +00003814}
3815
3816static int pool_iterate_devices(struct dm_target *ti,
3817 iterate_devices_callout_fn fn, void *data)
3818{
3819 struct pool_c *pt = ti->private;
3820
3821 return fn(ti, pt->data_dev, 0, ti->len, data);
3822}
3823
Joe Thornber991d9fa2011-10-31 20:21:18 +00003824static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
3825{
3826 struct pool_c *pt = ti->private;
3827 struct pool *pool = pt->pool;
Mike Snitzer604ea902014-10-09 18:43:25 -04003828 sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3829
3830 /*
Mike Snitzerd200c302014-11-20 18:07:43 -05003831 * If max_sectors is smaller than pool->sectors_per_block adjust it
3832 * to the highest possible power-of-2 factor of pool->sectors_per_block.
3833 * This is especially beneficial when the pool's data device is a RAID
3834 * device that has a full stripe width that matches pool->sectors_per_block
3835 * -- because even though partial RAID stripe-sized IOs will be issued to a
3836 * single RAID stripe; when aggregated they will end on a full RAID stripe
3837 * boundary.. which avoids additional partial RAID stripe writes cascading
Mike Snitzer604ea902014-10-09 18:43:25 -04003838 */
Mike Snitzer604ea902014-10-09 18:43:25 -04003839 if (limits->max_sectors < pool->sectors_per_block) {
3840 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) {
3841 if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)
3842 limits->max_sectors--;
3843 limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);
3844 }
Mike Snitzer604ea902014-10-09 18:43:25 -04003845 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003846
Mike Snitzer0cc67cd2013-08-20 15:02:41 -04003847 /*
3848 * If the system-determined stacked limits are compatible with the
3849 * pool's blocksize (io_opt is a factor) do not override them.
3850 */
3851 if (io_opt_sectors < pool->sectors_per_block ||
Mike Snitzer604ea902014-10-09 18:43:25 -04003852 !is_factor(io_opt_sectors, pool->sectors_per_block)) {
3853 if (is_factor(pool->sectors_per_block, limits->max_sectors))
3854 blk_limits_io_min(limits, limits->max_sectors << SECTOR_SHIFT);
3855 else
3856 blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT);
Mike Snitzer0cc67cd2013-08-20 15:02:41 -04003857 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
3858 }
Mike Snitzer0424caa2012-09-26 23:45:47 +01003859
3860 /*
3861 * pt->adjusted_pf is a staging area for the actual features to use.
3862 * They get transferred to the live pool in bind_control_target()
3863 * called from pool_preresume().
3864 */
Mike Snitzerb60ab992013-09-19 18:49:11 -04003865 if (!pt->adjusted_pf.discard_enabled) {
3866 /*
3867 * Must explicitly disallow stacking discard limits otherwise the
3868 * block layer will stack them if pool's data device has support.
3869 * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the
3870 * user to see that, so make sure to set all discard limits to 0.
3871 */
3872 limits->discard_granularity = 0;
Mike Snitzer0424caa2012-09-26 23:45:47 +01003873 return;
Mike Snitzerb60ab992013-09-19 18:49:11 -04003874 }
Mike Snitzer0424caa2012-09-26 23:45:47 +01003875
3876 disable_passdown_if_not_supported(pt);
3877
Joe Thornber34fbcf62015-04-16 12:58:35 +01003878 /*
3879 * The pool uses the same discard limits as the underlying data
3880 * device. DM core has already set this up.
3881 */
Joe Thornber991d9fa2011-10-31 20:21:18 +00003882}
3883
3884static struct target_type pool_target = {
3885 .name = "thin-pool",
3886 .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
3887 DM_TARGET_IMMUTABLE,
Mike Snitzere4c78e22015-07-15 11:40:24 -04003888 .version = {1, 16, 0},
Joe Thornber991d9fa2011-10-31 20:21:18 +00003889 .module = THIS_MODULE,
3890 .ctr = pool_ctr,
3891 .dtr = pool_dtr,
3892 .map = pool_map,
Mike Snitzer80e96c52014-11-07 15:09:46 -05003893 .presuspend = pool_presuspend,
3894 .presuspend_undo = pool_presuspend_undo,
Joe Thornber991d9fa2011-10-31 20:21:18 +00003895 .postsuspend = pool_postsuspend,
3896 .preresume = pool_preresume,
3897 .resume = pool_resume,
3898 .message = pool_message,
3899 .status = pool_status,
Joe Thornber991d9fa2011-10-31 20:21:18 +00003900 .iterate_devices = pool_iterate_devices,
3901 .io_hints = pool_io_hints,
3902};
3903
3904/*----------------------------------------------------------------
3905 * Thin target methods
3906 *--------------------------------------------------------------*/
Joe Thornberb10ebd32014-04-08 11:29:01 +01003907static void thin_get(struct thin_c *tc)
3908{
3909 atomic_inc(&tc->refcount);
3910}
3911
3912static void thin_put(struct thin_c *tc)
3913{
3914 if (atomic_dec_and_test(&tc->refcount))
3915 complete(&tc->can_destroy);
3916}
3917
Joe Thornber991d9fa2011-10-31 20:21:18 +00003918static void thin_dtr(struct dm_target *ti)
3919{
3920 struct thin_c *tc = ti->private;
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003921 unsigned long flags;
3922
3923 spin_lock_irqsave(&tc->pool->lock, flags);
3924 list_del_rcu(&tc->list);
3925 spin_unlock_irqrestore(&tc->pool->lock, flags);
3926 synchronize_rcu();
Joe Thornber991d9fa2011-10-31 20:21:18 +00003927
Mikulas Patocka17181fb2014-11-05 17:00:13 -05003928 thin_put(tc);
3929 wait_for_completion(&tc->can_destroy);
3930
Joe Thornber991d9fa2011-10-31 20:21:18 +00003931 mutex_lock(&dm_thin_pool_table.mutex);
3932
3933 __pool_dec(tc->pool);
3934 dm_pool_close_thin_device(tc->td);
3935 dm_put_device(ti, tc->pool_dev);
Joe Thornber2dd9c252012-03-28 18:41:28 +01003936 if (tc->origin_dev)
3937 dm_put_device(ti, tc->origin_dev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003938 kfree(tc);
3939
3940 mutex_unlock(&dm_thin_pool_table.mutex);
3941}
3942
3943/*
3944 * Thin target parameters:
3945 *
Joe Thornber2dd9c252012-03-28 18:41:28 +01003946 * <pool_dev> <dev_id> [origin_dev]
Joe Thornber991d9fa2011-10-31 20:21:18 +00003947 *
3948 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
3949 * dev_id: the internal device identifier
Joe Thornber2dd9c252012-03-28 18:41:28 +01003950 * origin_dev: a device external to the pool that should act as the origin
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003951 *
3952 * If the pool device has discards disabled, they get disabled for the thin
3953 * device as well.
Joe Thornber991d9fa2011-10-31 20:21:18 +00003954 */
3955static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
3956{
3957 int r;
3958 struct thin_c *tc;
Joe Thornber2dd9c252012-03-28 18:41:28 +01003959 struct dm_dev *pool_dev, *origin_dev;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003960 struct mapped_device *pool_md;
Joe Thornber5e3283e2014-04-08 11:08:41 +01003961 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003962
3963 mutex_lock(&dm_thin_pool_table.mutex);
3964
Joe Thornber2dd9c252012-03-28 18:41:28 +01003965 if (argc != 2 && argc != 3) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00003966 ti->error = "Invalid argument count";
3967 r = -EINVAL;
3968 goto out_unlock;
3969 }
3970
3971 tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
3972 if (!tc) {
3973 ti->error = "Out of memory";
3974 r = -ENOMEM;
3975 goto out_unlock;
3976 }
Mike Snitzer583024d2014-10-28 20:58:45 -04003977 tc->thin_md = dm_table_get_md(ti->table);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003978 spin_lock_init(&tc->lock);
Joe Thornbera374bb22014-10-10 13:43:14 +01003979 INIT_LIST_HEAD(&tc->deferred_cells);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003980 bio_list_init(&tc->deferred_bio_list);
3981 bio_list_init(&tc->retry_on_resume_list);
Mike Snitzer67324ea2014-03-21 18:33:41 -04003982 tc->sort_bio_list = RB_ROOT;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003983
Joe Thornber2dd9c252012-03-28 18:41:28 +01003984 if (argc == 3) {
3985 r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev);
3986 if (r) {
3987 ti->error = "Error opening origin device";
3988 goto bad_origin_dev;
3989 }
3990 tc->origin_dev = origin_dev;
3991 }
3992
Joe Thornber991d9fa2011-10-31 20:21:18 +00003993 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
3994 if (r) {
3995 ti->error = "Error opening pool device";
3996 goto bad_pool_dev;
3997 }
3998 tc->pool_dev = pool_dev;
3999
4000 if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
4001 ti->error = "Invalid device id";
4002 r = -EINVAL;
4003 goto bad_common;
4004 }
4005
4006 pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
4007 if (!pool_md) {
4008 ti->error = "Couldn't get pool mapped device";
4009 r = -EINVAL;
4010 goto bad_common;
4011 }
4012
4013 tc->pool = __pool_table_lookup(pool_md);
4014 if (!tc->pool) {
4015 ti->error = "Couldn't find pool object";
4016 r = -EINVAL;
4017 goto bad_pool_lookup;
4018 }
4019 __pool_inc(tc->pool);
4020
Joe Thornbere49e5822012-07-27 15:08:16 +01004021 if (get_pool_mode(tc->pool) == PM_FAIL) {
4022 ti->error = "Couldn't open thin device, Pool is in fail mode";
Mike Snitzer1acacc02014-02-19 20:32:33 -05004023 r = -EINVAL;
Mike Snitzer80e96c52014-11-07 15:09:46 -05004024 goto bad_pool;
Joe Thornbere49e5822012-07-27 15:08:16 +01004025 }
4026
Joe Thornber991d9fa2011-10-31 20:21:18 +00004027 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
4028 if (r) {
4029 ti->error = "Couldn't open thin internal device";
Mike Snitzer80e96c52014-11-07 15:09:46 -05004030 goto bad_pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00004031 }
4032
Mike Snitzer542f9032012-07-27 15:08:00 +01004033 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
4034 if (r)
Mike Snitzer80e96c52014-11-07 15:09:46 -05004035 goto bad;
Mike Snitzer542f9032012-07-27 15:08:00 +01004036
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00004037 ti->num_flush_bios = 1;
Joe Thornber16ad3d12012-07-27 15:08:07 +01004038 ti->flush_supported = true;
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00004039 ti->per_bio_data_size = sizeof(struct dm_thin_endio_hook);
Joe Thornber67e2e2b2012-03-28 18:41:29 +01004040
4041 /* In case the pool supports discards, pass them on. */
Mike Snitzerb60ab992013-09-19 18:49:11 -04004042 ti->discard_zeroes_data_unsupported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01004043 if (tc->pool->pf.discard_enabled) {
Alasdair G Kergon0ac55482012-07-27 15:08:08 +01004044 ti->discards_supported = true;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00004045 ti->num_discard_bios = 1;
Joe Thornber34fbcf62015-04-16 12:58:35 +01004046 ti->split_discard_bios = false;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01004047 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00004048
Joe Thornber991d9fa2011-10-31 20:21:18 +00004049 mutex_unlock(&dm_thin_pool_table.mutex);
4050
Joe Thornber5e3283e2014-04-08 11:08:41 +01004051 spin_lock_irqsave(&tc->pool->lock, flags);
Mike Snitzer80e96c52014-11-07 15:09:46 -05004052 if (tc->pool->suspended) {
4053 spin_unlock_irqrestore(&tc->pool->lock, flags);
4054 mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */
4055 ti->error = "Unable to activate thin device while pool is suspended";
4056 r = -EINVAL;
4057 goto bad;
4058 }
Marc Dionne2b94e892014-12-17 07:59:59 -05004059 atomic_set(&tc->refcount, 1);
4060 init_completion(&tc->can_destroy);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04004061 list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
Joe Thornber5e3283e2014-04-08 11:08:41 +01004062 spin_unlock_irqrestore(&tc->pool->lock, flags);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04004063 /*
4064 * This synchronize_rcu() call is needed here otherwise we risk a
4065 * wake_worker() call finding no bios to process (because the newly
4066 * added tc isn't yet visible). So this reduces latency since we
4067 * aren't then dependent on the periodic commit to wake_worker().
4068 */
4069 synchronize_rcu();
4070
Mike Snitzer80e96c52014-11-07 15:09:46 -05004071 dm_put(pool_md);
4072
Joe Thornber991d9fa2011-10-31 20:21:18 +00004073 return 0;
4074
Mike Snitzer80e96c52014-11-07 15:09:46 -05004075bad:
Mike Snitzer1acacc02014-02-19 20:32:33 -05004076 dm_pool_close_thin_device(tc->td);
Mike Snitzer80e96c52014-11-07 15:09:46 -05004077bad_pool:
Joe Thornber991d9fa2011-10-31 20:21:18 +00004078 __pool_dec(tc->pool);
4079bad_pool_lookup:
4080 dm_put(pool_md);
4081bad_common:
4082 dm_put_device(ti, tc->pool_dev);
4083bad_pool_dev:
Joe Thornber2dd9c252012-03-28 18:41:28 +01004084 if (tc->origin_dev)
4085 dm_put_device(ti, tc->origin_dev);
4086bad_origin_dev:
Joe Thornber991d9fa2011-10-31 20:21:18 +00004087 kfree(tc);
4088out_unlock:
4089 mutex_unlock(&dm_thin_pool_table.mutex);
4090
4091 return r;
4092}
4093
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00004094static int thin_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00004095{
Kent Overstreet4f024f32013-10-11 15:44:27 -07004096 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004097
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00004098 return thin_bio_map(ti, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004099}
4100
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00004101static int thin_endio(struct dm_target *ti, struct bio *bio, int err)
Joe Thornbereb2aa482012-03-28 18:41:28 +01004102{
4103 unsigned long flags;
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00004104 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01004105 struct list_head work;
Mike Snitzera24c2562012-06-03 00:30:00 +01004106 struct dm_thin_new_mapping *m, *tmp;
Joe Thornbereb2aa482012-03-28 18:41:28 +01004107 struct pool *pool = h->tc->pool;
4108
4109 if (h->shared_read_entry) {
4110 INIT_LIST_HEAD(&work);
Mike Snitzer44feb382012-10-12 21:02:10 +01004111 dm_deferred_entry_dec(h->shared_read_entry, &work);
Joe Thornbereb2aa482012-03-28 18:41:28 +01004112
4113 spin_lock_irqsave(&pool->lock, flags);
4114 list_for_each_entry_safe(m, tmp, &work, list) {
4115 list_del(&m->list);
Joe Thornber50f3c3e2014-06-13 13:57:09 +01004116 __complete_mapping_preparation(m);
Joe Thornbereb2aa482012-03-28 18:41:28 +01004117 }
4118 spin_unlock_irqrestore(&pool->lock, flags);
4119 }
4120
Joe Thornber104655f2012-03-28 18:41:28 +01004121 if (h->all_io_entry) {
4122 INIT_LIST_HEAD(&work);
Mike Snitzer44feb382012-10-12 21:02:10 +01004123 dm_deferred_entry_dec(h->all_io_entry, &work);
Joe Thornber563af182012-12-21 20:23:31 +00004124 if (!list_empty(&work)) {
4125 spin_lock_irqsave(&pool->lock, flags);
4126 list_for_each_entry_safe(m, tmp, &work, list)
Mike Snitzerdaec3382013-12-11 14:01:20 -05004127 list_add_tail(&m->list, &pool->prepared_discards);
Joe Thornber563af182012-12-21 20:23:31 +00004128 spin_unlock_irqrestore(&pool->lock, flags);
4129 wake_worker(pool);
4130 }
Joe Thornber104655f2012-03-28 18:41:28 +01004131 }
4132
Joe Thornber34fbcf62015-04-16 12:58:35 +01004133 if (h->cell)
4134 cell_defer_no_holder(h->tc, h->cell);
4135
Joe Thornbereb2aa482012-03-28 18:41:28 +01004136 return 0;
4137}
4138
Joe Thornber738211f2014-03-03 15:52:28 +00004139static void thin_presuspend(struct dm_target *ti)
4140{
4141 struct thin_c *tc = ti->private;
4142
4143 if (dm_noflush_suspending(ti))
4144 noflush_work(tc, do_noflush_start);
4145}
4146
Joe Thornber991d9fa2011-10-31 20:21:18 +00004147static void thin_postsuspend(struct dm_target *ti)
4148{
Joe Thornber738211f2014-03-03 15:52:28 +00004149 struct thin_c *tc = ti->private;
4150
4151 /*
4152 * The dm_noflush_suspending flag has been cleared by now, so
4153 * unfortunately we must always run this.
4154 */
4155 noflush_work(tc, do_noflush_stop);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004156}
4157
Joe Thornbere5aea7b2014-06-13 14:47:24 +01004158static int thin_preresume(struct dm_target *ti)
4159{
4160 struct thin_c *tc = ti->private;
4161
4162 if (tc->origin_dev)
4163 tc->origin_size = get_dev_size(tc->origin_dev->bdev);
4164
4165 return 0;
4166}
4167
Joe Thornber991d9fa2011-10-31 20:21:18 +00004168/*
4169 * <nr mapped sectors> <highest mapped sector>
4170 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004171static void thin_status(struct dm_target *ti, status_type_t type,
4172 unsigned status_flags, char *result, unsigned maxlen)
Joe Thornber991d9fa2011-10-31 20:21:18 +00004173{
4174 int r;
4175 ssize_t sz = 0;
4176 dm_block_t mapped, highest;
4177 char buf[BDEVNAME_SIZE];
4178 struct thin_c *tc = ti->private;
4179
Joe Thornbere49e5822012-07-27 15:08:16 +01004180 if (get_pool_mode(tc->pool) == PM_FAIL) {
4181 DMEMIT("Fail");
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004182 return;
Joe Thornbere49e5822012-07-27 15:08:16 +01004183 }
4184
Joe Thornber991d9fa2011-10-31 20:21:18 +00004185 if (!tc->td)
4186 DMEMIT("-");
4187 else {
4188 switch (type) {
4189 case STATUSTYPE_INFO:
4190 r = dm_thin_get_mapped_count(tc->td, &mapped);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004191 if (r) {
4192 DMERR("dm_thin_get_mapped_count returned %d", r);
4193 goto err;
4194 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00004195
4196 r = dm_thin_get_highest_mapped_block(tc->td, &highest);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004197 if (r < 0) {
4198 DMERR("dm_thin_get_highest_mapped_block returned %d", r);
4199 goto err;
4200 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00004201
4202 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
4203 if (r)
4204 DMEMIT("%llu", ((highest + 1) *
4205 tc->pool->sectors_per_block) - 1);
4206 else
4207 DMEMIT("-");
4208 break;
4209
4210 case STATUSTYPE_TABLE:
4211 DMEMIT("%s %lu",
4212 format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
4213 (unsigned long) tc->dev_id);
Joe Thornber2dd9c252012-03-28 18:41:28 +01004214 if (tc->origin_dev)
4215 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
Joe Thornber991d9fa2011-10-31 20:21:18 +00004216 break;
4217 }
4218 }
4219
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004220 return;
4221
4222err:
4223 DMEMIT("Error");
Joe Thornber991d9fa2011-10-31 20:21:18 +00004224}
4225
4226static int thin_iterate_devices(struct dm_target *ti,
4227 iterate_devices_callout_fn fn, void *data)
4228{
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004229 sector_t blocks;
Joe Thornber991d9fa2011-10-31 20:21:18 +00004230 struct thin_c *tc = ti->private;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004231 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00004232
4233 /*
4234 * We can't call dm_pool_get_data_dev_size() since that blocks. So
4235 * we follow a more convoluted path through to the pool's target.
4236 */
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004237 if (!pool->ti)
Joe Thornber991d9fa2011-10-31 20:21:18 +00004238 return 0; /* nothing is bound */
4239
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004240 blocks = pool->ti->len;
4241 (void) sector_div(blocks, pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004242 if (blocks)
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004243 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004244
4245 return 0;
4246}
4247
Joe Thornber34fbcf62015-04-16 12:58:35 +01004248static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
4249{
4250 struct thin_c *tc = ti->private;
4251 struct pool *pool = tc->pool;
Mike Snitzer21607672015-09-08 08:56:13 -04004252 struct queue_limits *pool_limits = dm_get_queue_limits(pool->pool_md);
4253
4254 if (!pool_limits->discard_granularity)
4255 return; /* pool's discard support is disabled */
Joe Thornber34fbcf62015-04-16 12:58:35 +01004256
4257 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
4258 limits->max_discard_sectors = 2048 * 1024 * 16; /* 16G */
4259}
4260
Joe Thornber991d9fa2011-10-31 20:21:18 +00004261static struct target_type thin_target = {
4262 .name = "thin",
Mike Snitzere4c78e22015-07-15 11:40:24 -04004263 .version = {1, 16, 0},
Joe Thornber991d9fa2011-10-31 20:21:18 +00004264 .module = THIS_MODULE,
4265 .ctr = thin_ctr,
4266 .dtr = thin_dtr,
4267 .map = thin_map,
Joe Thornbereb2aa482012-03-28 18:41:28 +01004268 .end_io = thin_endio,
Joe Thornbere5aea7b2014-06-13 14:47:24 +01004269 .preresume = thin_preresume,
Joe Thornber738211f2014-03-03 15:52:28 +00004270 .presuspend = thin_presuspend,
Joe Thornber991d9fa2011-10-31 20:21:18 +00004271 .postsuspend = thin_postsuspend,
4272 .status = thin_status,
4273 .iterate_devices = thin_iterate_devices,
Joe Thornber34fbcf62015-04-16 12:58:35 +01004274 .io_hints = thin_io_hints,
Joe Thornber991d9fa2011-10-31 20:21:18 +00004275};
4276
4277/*----------------------------------------------------------------*/
4278
4279static int __init dm_thin_init(void)
4280{
4281 int r;
4282
4283 pool_table_init();
4284
4285 r = dm_register_target(&thin_target);
4286 if (r)
4287 return r;
4288
4289 r = dm_register_target(&pool_target);
4290 if (r)
Mike Snitzera24c2562012-06-03 00:30:00 +01004291 goto bad_pool_target;
4292
4293 r = -ENOMEM;
4294
Mike Snitzera24c2562012-06-03 00:30:00 +01004295 _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
4296 if (!_new_mapping_cache)
4297 goto bad_new_mapping_cache;
4298
Mike Snitzera24c2562012-06-03 00:30:00 +01004299 return 0;
4300
Mike Snitzera24c2562012-06-03 00:30:00 +01004301bad_new_mapping_cache:
Mike Snitzera24c2562012-06-03 00:30:00 +01004302 dm_unregister_target(&pool_target);
4303bad_pool_target:
4304 dm_unregister_target(&thin_target);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004305
4306 return r;
4307}
4308
4309static void dm_thin_exit(void)
4310{
4311 dm_unregister_target(&thin_target);
4312 dm_unregister_target(&pool_target);
Mike Snitzera24c2562012-06-03 00:30:00 +01004313
Mike Snitzera24c2562012-06-03 00:30:00 +01004314 kmem_cache_destroy(_new_mapping_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004315}
4316
4317module_init(dm_thin_init);
4318module_exit(dm_thin_exit);
4319
Mike Snitzer80c57892014-05-20 13:38:33 -04004320module_param_named(no_space_timeout, no_space_timeout_secs, uint, S_IRUGO | S_IWUSR);
4321MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
4322
Alasdair G Kergon7cab8bf2012-05-12 01:43:19 +01004323MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
Joe Thornber991d9fa2011-10-31 20:21:18 +00004324MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
4325MODULE_LICENSE("GPL");