blob: 63903a5a5d9ee3b580d552673b42b9716b99322c [file] [log] [blame]
Joe Thornber991d9fa2011-10-31 20:21:18 +00001/*
Joe Thornbere49e5822012-07-27 15:08:16 +01002 * Copyright (C) 2011-2012 Red Hat UK.
Joe Thornber991d9fa2011-10-31 20:21:18 +00003 *
4 * This file is released under the GPL.
5 */
6
7#include "dm-thin-metadata.h"
Mike Snitzer4f81a412012-10-12 21:02:13 +01008#include "dm-bio-prison.h"
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01009#include "dm.h"
Joe Thornber991d9fa2011-10-31 20:21:18 +000010
11#include <linux/device-mapper.h>
12#include <linux/dm-io.h>
13#include <linux/dm-kcopyd.h>
Manuel Schölling0f30af92014-05-22 22:42:37 +020014#include <linux/jiffies.h>
Mike Snitzer604ea902014-10-09 18:43:25 -040015#include <linux/log2.h>
Joe Thornber991d9fa2011-10-31 20:21:18 +000016#include <linux/list.h>
Mike Snitzerc140e1c2014-03-20 21:17:14 -040017#include <linux/rculist.h>
Joe Thornber991d9fa2011-10-31 20:21:18 +000018#include <linux/init.h>
19#include <linux/module.h>
20#include <linux/slab.h>
Joe Thornbera822c832015-07-03 10:22:42 +010021#include <linux/vmalloc.h>
Joe Thornberac4c3f32014-10-10 16:42:10 +010022#include <linux/sort.h>
Mike Snitzer67324ea2014-03-21 18:33:41 -040023#include <linux/rbtree.h>
Joe Thornber991d9fa2011-10-31 20:21:18 +000024
25#define DM_MSG_PREFIX "thin"
26
27/*
28 * Tunable constants
29 */
Alasdair G Kergon7768ed32012-07-27 15:07:57 +010030#define ENDIO_HOOK_POOL_SIZE 1024
Joe Thornber991d9fa2011-10-31 20:21:18 +000031#define MAPPING_POOL_SIZE 1024
Joe Thornber905e51b2012-03-28 18:41:27 +010032#define COMMIT_PERIOD HZ
Mike Snitzer80c57892014-05-20 13:38:33 -040033#define NO_SPACE_TIMEOUT_SECS 60
34
35static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
Joe Thornber991d9fa2011-10-31 20:21:18 +000036
Mikulas Patockadf5d2e92013-03-01 22:45:49 +000037DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
38 "A percentage of time allocated for copy on write");
39
Joe Thornber991d9fa2011-10-31 20:21:18 +000040/*
41 * The block size of the device holding pool data must be
42 * between 64KB and 1GB.
43 */
44#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
45#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
46
47/*
Joe Thornber991d9fa2011-10-31 20:21:18 +000048 * Device id is restricted to 24 bits.
49 */
50#define MAX_DEV_ID ((1 << 24) - 1)
51
52/*
53 * How do we handle breaking sharing of data blocks?
54 * =================================================
55 *
56 * We use a standard copy-on-write btree to store the mappings for the
57 * devices (note I'm talking about copy-on-write of the metadata here, not
58 * the data). When you take an internal snapshot you clone the root node
59 * of the origin btree. After this there is no concept of an origin or a
60 * snapshot. They are just two device trees that happen to point to the
61 * same data blocks.
62 *
63 * When we get a write in we decide if it's to a shared data block using
64 * some timestamp magic. If it is, we have to break sharing.
65 *
66 * Let's say we write to a shared block in what was the origin. The
67 * steps are:
68 *
69 * i) plug io further to this physical block. (see bio_prison code).
70 *
71 * ii) quiesce any read io to that shared data block. Obviously
Mike Snitzer44feb382012-10-12 21:02:10 +010072 * including all devices that share this block. (see dm_deferred_set code)
Joe Thornber991d9fa2011-10-31 20:21:18 +000073 *
74 * iii) copy the data block to a newly allocate block. This step can be
75 * missed out if the io covers the block. (schedule_copy).
76 *
77 * iv) insert the new mapping into the origin's btree
Joe Thornberfe878f32012-03-28 18:41:24 +010078 * (process_prepared_mapping). This act of inserting breaks some
Joe Thornber991d9fa2011-10-31 20:21:18 +000079 * sharing of btree nodes between the two devices. Breaking sharing only
80 * effects the btree of that specific device. Btrees for the other
81 * devices that share the block never change. The btree for the origin
82 * device as it was after the last commit is untouched, ie. we're using
83 * persistent data structures in the functional programming sense.
84 *
85 * v) unplug io to this physical block, including the io that triggered
86 * the breaking of sharing.
87 *
88 * Steps (ii) and (iii) occur in parallel.
89 *
90 * The metadata _doesn't_ need to be committed before the io continues. We
91 * get away with this because the io is always written to a _new_ block.
92 * If there's a crash, then:
93 *
94 * - The origin mapping will point to the old origin block (the shared
95 * one). This will contain the data as it was before the io that triggered
96 * the breaking of sharing came in.
97 *
98 * - The snap mapping still points to the old block. As it would after
99 * the commit.
100 *
101 * The downside of this scheme is the timestamp magic isn't perfect, and
102 * will continue to think that data block in the snapshot device is shared
103 * even after the write to the origin has broken sharing. I suspect data
104 * blocks will typically be shared by many different devices, so we're
105 * breaking sharing n + 1 times, rather than n, where n is the number of
106 * devices that reference this data block. At the moment I think the
107 * benefits far, far outweigh the disadvantages.
108 */
109
110/*----------------------------------------------------------------*/
111
112/*
Joe Thornber991d9fa2011-10-31 20:21:18 +0000113 * Key building.
114 */
Joe Thornber34fbcf62015-04-16 12:58:35 +0100115enum lock_space {
116 VIRTUAL,
117 PHYSICAL
118};
119
120static void build_key(struct dm_thin_device *td, enum lock_space ls,
121 dm_block_t b, dm_block_t e, struct dm_cell_key *key)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000122{
Joe Thornber34fbcf62015-04-16 12:58:35 +0100123 key->virtual = (ls == VIRTUAL);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000124 key->dev = dm_thin_dev_id(td);
Joe Thornber5f274d82014-09-17 10:17:39 +0100125 key->block_begin = b;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100126 key->block_end = e;
127}
128
129static void build_data_key(struct dm_thin_device *td, dm_block_t b,
130 struct dm_cell_key *key)
131{
132 build_key(td, PHYSICAL, b, b + 1llu, key);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000133}
134
135static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
Mike Snitzer44feb382012-10-12 21:02:10 +0100136 struct dm_cell_key *key)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000137{
Joe Thornber34fbcf62015-04-16 12:58:35 +0100138 build_key(td, VIRTUAL, b, b + 1llu, key);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000139}
140
141/*----------------------------------------------------------------*/
142
Joe Thornber7d327fe2014-10-06 15:45:59 +0100143#define THROTTLE_THRESHOLD (1 * HZ)
144
145struct throttle {
146 struct rw_semaphore lock;
147 unsigned long threshold;
148 bool throttle_applied;
149};
150
151static void throttle_init(struct throttle *t)
152{
153 init_rwsem(&t->lock);
154 t->throttle_applied = false;
155}
156
157static void throttle_work_start(struct throttle *t)
158{
159 t->threshold = jiffies + THROTTLE_THRESHOLD;
160}
161
162static void throttle_work_update(struct throttle *t)
163{
164 if (!t->throttle_applied && jiffies > t->threshold) {
165 down_write(&t->lock);
166 t->throttle_applied = true;
167 }
168}
169
170static void throttle_work_complete(struct throttle *t)
171{
172 if (t->throttle_applied) {
173 t->throttle_applied = false;
174 up_write(&t->lock);
175 }
176}
177
178static void throttle_lock(struct throttle *t)
179{
180 down_read(&t->lock);
181}
182
183static void throttle_unlock(struct throttle *t)
184{
185 up_read(&t->lock);
186}
187
188/*----------------------------------------------------------------*/
189
Joe Thornber991d9fa2011-10-31 20:21:18 +0000190/*
191 * A pool device ties together a metadata device and a data device. It
192 * also provides the interface for creating and destroying internal
193 * devices.
194 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100195struct dm_thin_new_mapping;
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100196
Joe Thornbere49e5822012-07-27 15:08:16 +0100197/*
Joe Thornber3e1a0692014-03-03 16:03:26 +0000198 * The pool runs in 4 modes. Ordered in degraded order for comparisons.
Joe Thornbere49e5822012-07-27 15:08:16 +0100199 */
200enum pool_mode {
201 PM_WRITE, /* metadata may be changed */
Joe Thornber3e1a0692014-03-03 16:03:26 +0000202 PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */
Joe Thornbere49e5822012-07-27 15:08:16 +0100203 PM_READ_ONLY, /* metadata may not be changed */
204 PM_FAIL, /* all I/O fails */
205};
206
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100207struct pool_features {
Joe Thornbere49e5822012-07-27 15:08:16 +0100208 enum pool_mode mode;
209
Mike Snitzer9bc142d2012-09-26 23:45:46 +0100210 bool zero_new_blocks:1;
211 bool discard_enabled:1;
212 bool discard_passdown:1;
Mike Snitzer787a996c2013-12-06 16:21:43 -0500213 bool error_if_no_space:1;
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100214};
215
Joe Thornbere49e5822012-07-27 15:08:16 +0100216struct thin_c;
217typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
Joe Thornbera374bb22014-10-10 13:43:14 +0100218typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
Joe Thornbere49e5822012-07-27 15:08:16 +0100219typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
220
Joe Thornberac4c3f32014-10-10 16:42:10 +0100221#define CELL_SORT_ARRAY_SIZE 8192
222
Joe Thornber991d9fa2011-10-31 20:21:18 +0000223struct pool {
224 struct list_head list;
225 struct dm_target *ti; /* Only set if a pool target is bound */
226
227 struct mapped_device *pool_md;
228 struct block_device *md_dev;
229 struct dm_pool_metadata *pmd;
230
Joe Thornber991d9fa2011-10-31 20:21:18 +0000231 dm_block_t low_water_blocks;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100232 uint32_t sectors_per_block;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100233 int sectors_per_block_shift;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000234
Joe Thornber67e2e2b2012-03-28 18:41:29 +0100235 struct pool_features pf;
Joe Thornber88a66212013-12-04 20:16:12 -0500236 bool low_water_triggered:1; /* A dm event has been sent */
Mike Snitzer80e96c52014-11-07 15:09:46 -0500237 bool suspended:1;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000238
Mike Snitzer44feb382012-10-12 21:02:10 +0100239 struct dm_bio_prison *prison;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000240 struct dm_kcopyd_client *copier;
241
242 struct workqueue_struct *wq;
Joe Thornber7d327fe2014-10-06 15:45:59 +0100243 struct throttle throttle;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000244 struct work_struct worker;
Joe Thornber905e51b2012-03-28 18:41:27 +0100245 struct delayed_work waker;
Joe Thornber85ad643b2014-05-09 15:59:38 +0100246 struct delayed_work no_space_timeout;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000247
Joe Thornber905e51b2012-03-28 18:41:27 +0100248 unsigned long last_commit_jiffies;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100249 unsigned ref_count;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000250
251 spinlock_t lock;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000252 struct bio_list deferred_flush_bios;
253 struct list_head prepared_mappings;
Joe Thornber104655f2012-03-28 18:41:28 +0100254 struct list_head prepared_discards;
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400255 struct list_head active_thins;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000256
Mike Snitzer44feb382012-10-12 21:02:10 +0100257 struct dm_deferred_set *shared_read_ds;
258 struct dm_deferred_set *all_io_ds;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000259
Mike Snitzera24c2562012-06-03 00:30:00 +0100260 struct dm_thin_new_mapping *next_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000261 mempool_t *mapping_pool;
Joe Thornbere49e5822012-07-27 15:08:16 +0100262
263 process_bio_fn process_bio;
264 process_bio_fn process_discard;
265
Joe Thornbera374bb22014-10-10 13:43:14 +0100266 process_cell_fn process_cell;
267 process_cell_fn process_discard_cell;
268
Joe Thornbere49e5822012-07-27 15:08:16 +0100269 process_mapping_fn process_prepared_mapping;
270 process_mapping_fn process_prepared_discard;
Joe Thornberac4c3f32014-10-10 16:42:10 +0100271
Joe Thornbera822c832015-07-03 10:22:42 +0100272 struct dm_bio_prison_cell **cell_sort_array;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000273};
274
Joe Thornbere49e5822012-07-27 15:08:16 +0100275static enum pool_mode get_pool_mode(struct pool *pool);
Joe Thornberb5330652013-12-04 19:51:33 -0500276static void metadata_operation_failed(struct pool *pool, const char *op, int r);
Joe Thornbere49e5822012-07-27 15:08:16 +0100277
Joe Thornber991d9fa2011-10-31 20:21:18 +0000278/*
279 * Target context for a pool.
280 */
281struct pool_c {
282 struct dm_target *ti;
283 struct pool *pool;
284 struct dm_dev *data_dev;
285 struct dm_dev *metadata_dev;
286 struct dm_target_callbacks callbacks;
287
288 dm_block_t low_water_blocks;
Mike Snitzer0424caa2012-09-26 23:45:47 +0100289 struct pool_features requested_pf; /* Features requested during table load */
290 struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */
Joe Thornber991d9fa2011-10-31 20:21:18 +0000291};
292
293/*
294 * Target context for a thin.
295 */
296struct thin_c {
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400297 struct list_head list;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000298 struct dm_dev *pool_dev;
Joe Thornber2dd9c252012-03-28 18:41:28 +0100299 struct dm_dev *origin_dev;
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100300 sector_t origin_size;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000301 dm_thin_id dev_id;
302
303 struct pool *pool;
304 struct dm_thin_device *td;
Mike Snitzer583024d2014-10-28 20:58:45 -0400305 struct mapped_device *thin_md;
306
Joe Thornber738211f2014-03-03 15:52:28 +0000307 bool requeue_mode:1;
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400308 spinlock_t lock;
Joe Thornbera374bb22014-10-10 13:43:14 +0100309 struct list_head deferred_cells;
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400310 struct bio_list deferred_bio_list;
311 struct bio_list retry_on_resume_list;
Mike Snitzer67324ea2014-03-21 18:33:41 -0400312 struct rb_root sort_bio_list; /* sorted list of deferred bios */
Joe Thornberb10ebd32014-04-08 11:29:01 +0100313
314 /*
315 * Ensures the thin is not destroyed until the worker has finished
316 * iterating the active_thins list.
317 */
318 atomic_t refcount;
319 struct completion can_destroy;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000320};
321
322/*----------------------------------------------------------------*/
323
Joe Thornber34fbcf62015-04-16 12:58:35 +0100324/**
325 * __blkdev_issue_discard_async - queue a discard with async completion
326 * @bdev: blockdev to issue discard for
327 * @sector: start sector
328 * @nr_sects: number of sectors to discard
329 * @gfp_mask: memory allocation flags (for bio_alloc)
330 * @flags: BLKDEV_IFL_* flags to control behaviour
331 * @parent_bio: parent discard bio that all sub discards get chained to
332 *
333 * Description:
334 * Asynchronously issue a discard request for the sectors in question.
Joe Thornber34fbcf62015-04-16 12:58:35 +0100335 */
336static int __blkdev_issue_discard_async(struct block_device *bdev, sector_t sector,
337 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags,
338 struct bio *parent_bio)
339{
340 struct request_queue *q = bdev_get_queue(bdev);
341 int type = REQ_WRITE | REQ_DISCARD;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100342 struct bio *bio;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100343
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400344 if (!q || !nr_sects)
Joe Thornber34fbcf62015-04-16 12:58:35 +0100345 return -ENXIO;
346
347 if (!blk_queue_discard(q))
348 return -EOPNOTSUPP;
349
Joe Thornber34fbcf62015-04-16 12:58:35 +0100350 if (flags & BLKDEV_DISCARD_SECURE) {
351 if (!blk_queue_secdiscard(q))
352 return -EOPNOTSUPP;
353 type |= REQ_SECURE;
354 }
355
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400356 /*
357 * Required bio_put occurs in bio_endio thanks to bio_chain below
358 */
359 bio = bio_alloc(gfp_mask, 1);
360 if (!bio)
361 return -ENOMEM;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100362
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400363 bio_chain(bio, parent_bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +0100364
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400365 bio->bi_iter.bi_sector = sector;
366 bio->bi_bdev = bdev;
367 bio->bi_iter.bi_size = nr_sects << 9;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100368
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400369 submit_bio(type, bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +0100370
Mike Snitzer84f8bd82015-08-18 10:31:09 -0400371 return 0;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100372}
373
374static bool block_size_is_power_of_two(struct pool *pool)
375{
376 return pool->sectors_per_block_shift >= 0;
377}
378
379static sector_t block_to_sectors(struct pool *pool, dm_block_t b)
380{
381 return block_size_is_power_of_two(pool) ?
382 (b << pool->sectors_per_block_shift) :
383 (b * pool->sectors_per_block);
384}
385
386static int issue_discard(struct thin_c *tc, dm_block_t data_b, dm_block_t data_e,
387 struct bio *parent_bio)
388{
389 sector_t s = block_to_sectors(tc->pool, data_b);
390 sector_t len = block_to_sectors(tc->pool, data_e - data_b);
391
392 return __blkdev_issue_discard_async(tc->pool_dev->bdev, s, len,
393 GFP_NOWAIT, 0, parent_bio);
394}
395
396/*----------------------------------------------------------------*/
397
Joe Thornber025b9682013-03-01 22:45:50 +0000398/*
399 * wake_worker() is used when new work is queued and when pool_resume is
400 * ready to continue deferred IO processing.
401 */
402static void wake_worker(struct pool *pool)
403{
404 queue_work(pool->wq, &pool->worker);
405}
406
407/*----------------------------------------------------------------*/
408
Joe Thornber6beca5e2013-03-01 22:45:50 +0000409static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
410 struct dm_bio_prison_cell **cell_result)
411{
412 int r;
413 struct dm_bio_prison_cell *cell_prealloc;
414
415 /*
416 * Allocate a cell from the prison's mempool.
417 * This might block but it can't fail.
418 */
419 cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);
420
421 r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);
422 if (r)
423 /*
424 * We reused an old cell; we can get rid of
425 * the new one.
426 */
427 dm_bio_prison_free_cell(pool->prison, cell_prealloc);
428
429 return r;
430}
431
432static void cell_release(struct pool *pool,
433 struct dm_bio_prison_cell *cell,
434 struct bio_list *bios)
435{
436 dm_cell_release(pool->prison, cell, bios);
437 dm_bio_prison_free_cell(pool->prison, cell);
438}
439
Joe Thornber2d759a42014-10-10 15:27:16 +0100440static void cell_visit_release(struct pool *pool,
441 void (*fn)(void *, struct dm_bio_prison_cell *),
442 void *context,
443 struct dm_bio_prison_cell *cell)
444{
445 dm_cell_visit_release(pool->prison, fn, context, cell);
446 dm_bio_prison_free_cell(pool->prison, cell);
447}
448
Joe Thornber6beca5e2013-03-01 22:45:50 +0000449static void cell_release_no_holder(struct pool *pool,
450 struct dm_bio_prison_cell *cell,
451 struct bio_list *bios)
452{
453 dm_cell_release_no_holder(pool->prison, cell, bios);
454 dm_bio_prison_free_cell(pool->prison, cell);
455}
456
Mike Snitzeraf918052014-05-22 14:32:51 -0400457static void cell_error_with_code(struct pool *pool,
458 struct dm_bio_prison_cell *cell, int error_code)
Joe Thornber6beca5e2013-03-01 22:45:50 +0000459{
Mike Snitzeraf918052014-05-22 14:32:51 -0400460 dm_cell_error(pool->prison, cell, error_code);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000461 dm_bio_prison_free_cell(pool->prison, cell);
462}
463
Mike Snitzeraf918052014-05-22 14:32:51 -0400464static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
465{
466 cell_error_with_code(pool, cell, -EIO);
467}
468
Joe Thornbera374bb22014-10-10 13:43:14 +0100469static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)
470{
471 cell_error_with_code(pool, cell, 0);
472}
473
474static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)
475{
476 cell_error_with_code(pool, cell, DM_ENDIO_REQUEUE);
477}
478
Joe Thornber6beca5e2013-03-01 22:45:50 +0000479/*----------------------------------------------------------------*/
480
Joe Thornber991d9fa2011-10-31 20:21:18 +0000481/*
482 * A global list of pools that uses a struct mapped_device as a key.
483 */
484static struct dm_thin_pool_table {
485 struct mutex mutex;
486 struct list_head pools;
487} dm_thin_pool_table;
488
489static void pool_table_init(void)
490{
491 mutex_init(&dm_thin_pool_table.mutex);
492 INIT_LIST_HEAD(&dm_thin_pool_table.pools);
493}
494
495static void __pool_table_insert(struct pool *pool)
496{
497 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
498 list_add(&pool->list, &dm_thin_pool_table.pools);
499}
500
501static void __pool_table_remove(struct pool *pool)
502{
503 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
504 list_del(&pool->list);
505}
506
507static struct pool *__pool_table_lookup(struct mapped_device *md)
508{
509 struct pool *pool = NULL, *tmp;
510
511 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
512
513 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
514 if (tmp->pool_md == md) {
515 pool = tmp;
516 break;
517 }
518 }
519
520 return pool;
521}
522
523static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
524{
525 struct pool *pool = NULL, *tmp;
526
527 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
528
529 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
530 if (tmp->md_dev == md_dev) {
531 pool = tmp;
532 break;
533 }
534 }
535
536 return pool;
537}
538
539/*----------------------------------------------------------------*/
540
Mike Snitzera24c2562012-06-03 00:30:00 +0100541struct dm_thin_endio_hook {
Joe Thornbereb2aa482012-03-28 18:41:28 +0100542 struct thin_c *tc;
Mike Snitzer44feb382012-10-12 21:02:10 +0100543 struct dm_deferred_entry *shared_read_entry;
544 struct dm_deferred_entry *all_io_entry;
Mike Snitzera24c2562012-06-03 00:30:00 +0100545 struct dm_thin_new_mapping *overwrite_mapping;
Mike Snitzer67324ea2014-03-21 18:33:41 -0400546 struct rb_node rb_node;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100547 struct dm_bio_prison_cell *cell;
Joe Thornbereb2aa482012-03-28 18:41:28 +0100548};
549
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400550static void __merge_bio_list(struct bio_list *bios, struct bio_list *master)
551{
552 bio_list_merge(bios, master);
553 bio_list_init(master);
554}
555
556static void error_bio_list(struct bio_list *bios, int error)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000557{
558 struct bio *bio;
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400559
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200560 while ((bio = bio_list_pop(bios))) {
561 bio->bi_error = error;
562 bio_endio(bio);
563 }
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400564}
565
566static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, int error)
567{
Joe Thornber991d9fa2011-10-31 20:21:18 +0000568 struct bio_list bios;
Joe Thornber18adc572014-03-03 15:46:42 +0000569 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000570
571 bio_list_init(&bios);
Joe Thornber18adc572014-03-03 15:46:42 +0000572
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400573 spin_lock_irqsave(&tc->lock, flags);
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400574 __merge_bio_list(&bios, master);
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400575 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000576
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400577 error_bio_list(&bios, error);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000578}
579
Joe Thornbera374bb22014-10-10 13:43:14 +0100580static void requeue_deferred_cells(struct thin_c *tc)
581{
582 struct pool *pool = tc->pool;
583 unsigned long flags;
584 struct list_head cells;
585 struct dm_bio_prison_cell *cell, *tmp;
586
587 INIT_LIST_HEAD(&cells);
588
589 spin_lock_irqsave(&tc->lock, flags);
590 list_splice_init(&tc->deferred_cells, &cells);
591 spin_unlock_irqrestore(&tc->lock, flags);
592
593 list_for_each_entry_safe(cell, tmp, &cells, user_list)
594 cell_requeue(pool, cell);
595}
596
Joe Thornber991d9fa2011-10-31 20:21:18 +0000597static void requeue_io(struct thin_c *tc)
598{
Joe Thornber3e1a0692014-03-03 16:03:26 +0000599 struct bio_list bios;
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400600 unsigned long flags;
Joe Thornber3e1a0692014-03-03 16:03:26 +0000601
602 bio_list_init(&bios);
603
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400604 spin_lock_irqsave(&tc->lock, flags);
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400605 __merge_bio_list(&bios, &tc->deferred_bio_list);
606 __merge_bio_list(&bios, &tc->retry_on_resume_list);
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400607 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber3e1a0692014-03-03 16:03:26 +0000608
Mike Snitzer42d6a8c2014-10-19 07:52:44 -0400609 error_bio_list(&bios, DM_ENDIO_REQUEUE);
610 requeue_deferred_cells(tc);
Joe Thornber3e1a0692014-03-03 16:03:26 +0000611}
612
Mike Snitzer0a927c22015-07-21 13:20:46 -0400613static void error_retry_list_with_code(struct pool *pool, int error)
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400614{
615 struct thin_c *tc;
616
617 rcu_read_lock();
618 list_for_each_entry_rcu(tc, &pool->active_thins, list)
Mike Snitzer0a927c22015-07-21 13:20:46 -0400619 error_thin_bio_list(tc, &tc->retry_on_resume_list, error);
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400620 rcu_read_unlock();
621}
622
Mike Snitzer0a927c22015-07-21 13:20:46 -0400623static void error_retry_list(struct pool *pool)
624{
625 return error_retry_list_with_code(pool, -EIO);
626}
627
Joe Thornber991d9fa2011-10-31 20:21:18 +0000628/*
629 * This section of code contains the logic for processing a thin device's IO.
630 * Much of the code depends on pool object resources (lists, workqueues, etc)
631 * but most is exclusively called from the thin target rather than the thin-pool
632 * target.
633 */
634
635static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
636{
Mike Snitzer58f77a22013-03-01 22:45:45 +0000637 struct pool *pool = tc->pool;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700638 sector_t block_nr = bio->bi_iter.bi_sector;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100639
Mike Snitzer58f77a22013-03-01 22:45:45 +0000640 if (block_size_is_power_of_two(pool))
641 block_nr >>= pool->sectors_per_block_shift;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +0100642 else
Mike Snitzer58f77a22013-03-01 22:45:45 +0000643 (void) sector_div(block_nr, pool->sectors_per_block);
Mike Snitzer55f2b8b2012-07-27 15:08:02 +0100644
645 return block_nr;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000646}
647
Joe Thornber34fbcf62015-04-16 12:58:35 +0100648/*
649 * Returns the _complete_ blocks that this bio covers.
650 */
651static void get_bio_block_range(struct thin_c *tc, struct bio *bio,
652 dm_block_t *begin, dm_block_t *end)
653{
654 struct pool *pool = tc->pool;
655 sector_t b = bio->bi_iter.bi_sector;
656 sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
657
658 b += pool->sectors_per_block - 1ull; /* so we round up */
659
660 if (block_size_is_power_of_two(pool)) {
661 b >>= pool->sectors_per_block_shift;
662 e >>= pool->sectors_per_block_shift;
663 } else {
664 (void) sector_div(b, pool->sectors_per_block);
665 (void) sector_div(e, pool->sectors_per_block);
666 }
667
668 if (e < b)
669 /* Can happen if the bio is within a single block. */
670 e = b;
671
672 *begin = b;
673 *end = e;
674}
675
Joe Thornber991d9fa2011-10-31 20:21:18 +0000676static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
677{
678 struct pool *pool = tc->pool;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700679 sector_t bi_sector = bio->bi_iter.bi_sector;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000680
681 bio->bi_bdev = tc->pool_dev->bdev;
Mike Snitzer58f77a22013-03-01 22:45:45 +0000682 if (block_size_is_power_of_two(pool))
Kent Overstreet4f024f32013-10-11 15:44:27 -0700683 bio->bi_iter.bi_sector =
684 (block << pool->sectors_per_block_shift) |
685 (bi_sector & (pool->sectors_per_block - 1));
Mike Snitzer58f77a22013-03-01 22:45:45 +0000686 else
Kent Overstreet4f024f32013-10-11 15:44:27 -0700687 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
Mike Snitzer58f77a22013-03-01 22:45:45 +0000688 sector_div(bi_sector, pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000689}
690
Joe Thornber2dd9c252012-03-28 18:41:28 +0100691static void remap_to_origin(struct thin_c *tc, struct bio *bio)
692{
693 bio->bi_bdev = tc->origin_dev->bdev;
694}
695
Joe Thornber4afdd682012-07-27 15:08:14 +0100696static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
697{
698 return (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) &&
699 dm_thin_changed_this_transaction(tc->td);
700}
701
Joe Thornbere8088072012-12-21 20:23:31 +0000702static void inc_all_io_entry(struct pool *pool, struct bio *bio)
703{
704 struct dm_thin_endio_hook *h;
705
706 if (bio->bi_rw & REQ_DISCARD)
707 return;
708
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +0000709 h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbere8088072012-12-21 20:23:31 +0000710 h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
711}
712
Joe Thornber2dd9c252012-03-28 18:41:28 +0100713static void issue(struct thin_c *tc, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000714{
715 struct pool *pool = tc->pool;
716 unsigned long flags;
717
Joe Thornbere49e5822012-07-27 15:08:16 +0100718 if (!bio_triggers_commit(tc, bio)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000719 generic_make_request(bio);
Joe Thornbere49e5822012-07-27 15:08:16 +0100720 return;
721 }
722
723 /*
724 * Complete bio with an error if earlier I/O caused changes to
725 * the metadata that can't be committed e.g, due to I/O errors
726 * on the metadata device.
727 */
728 if (dm_thin_aborted_changes(tc->td)) {
729 bio_io_error(bio);
730 return;
731 }
732
733 /*
734 * Batch together any bios that trigger commits and then issue a
735 * single commit for them in process_deferred_bios().
736 */
737 spin_lock_irqsave(&pool->lock, flags);
738 bio_list_add(&pool->deferred_flush_bios, bio);
739 spin_unlock_irqrestore(&pool->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000740}
741
Joe Thornber2dd9c252012-03-28 18:41:28 +0100742static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
743{
744 remap_to_origin(tc, bio);
745 issue(tc, bio);
746}
747
748static void remap_and_issue(struct thin_c *tc, struct bio *bio,
749 dm_block_t block)
750{
751 remap(tc, bio, block);
752 issue(tc, bio);
753}
754
Joe Thornber991d9fa2011-10-31 20:21:18 +0000755/*----------------------------------------------------------------*/
756
757/*
758 * Bio endio functions.
759 */
Mike Snitzera24c2562012-06-03 00:30:00 +0100760struct dm_thin_new_mapping {
Joe Thornber991d9fa2011-10-31 20:21:18 +0000761 struct list_head list;
762
Mike Snitzer7f214662013-12-17 13:43:31 -0500763 bool pass_discard:1;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100764 bool maybe_shared:1;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000765
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100766 /*
767 * Track quiescing, copying and zeroing preparation actions. When this
768 * counter hits zero the block is prepared and can be inserted into the
769 * btree.
770 */
771 atomic_t prepare_actions;
772
Mike Snitzer7f214662013-12-17 13:43:31 -0500773 int err;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000774 struct thin_c *tc;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100775 dm_block_t virt_begin, virt_end;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000776 dm_block_t data_block;
Joe Thornber34fbcf62015-04-16 12:58:35 +0100777 struct dm_bio_prison_cell *cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000778
779 /*
780 * If the bio covers the whole area of a block then we can avoid
781 * zeroing or copying. Instead this bio is hooked. The bio will
782 * still be in the cell, so care has to be taken to avoid issuing
783 * the bio twice.
784 */
785 struct bio *bio;
786 bio_end_io_t *saved_bi_end_io;
787};
788
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100789static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000790{
791 struct pool *pool = m->tc->pool;
792
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100793 if (atomic_dec_and_test(&m->prepare_actions)) {
Mike Snitzerdaec3382013-12-11 14:01:20 -0500794 list_add_tail(&m->list, &pool->prepared_mappings);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000795 wake_worker(pool);
796 }
797}
798
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100799static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000800{
801 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000802 struct pool *pool = m->tc->pool;
803
Joe Thornber991d9fa2011-10-31 20:21:18 +0000804 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber50f3c3e2014-06-13 13:57:09 +0100805 __complete_mapping_preparation(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000806 spin_unlock_irqrestore(&pool->lock, flags);
807}
808
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100809static void copy_complete(int read_err, unsigned long write_err, void *context)
810{
811 struct dm_thin_new_mapping *m = context;
812
813 m->err = read_err || write_err ? -EIO : 0;
814 complete_mapping_preparation(m);
815}
816
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200817static void overwrite_endio(struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000818{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +0000819 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Mike Snitzera24c2562012-06-03 00:30:00 +0100820 struct dm_thin_new_mapping *m = h->overwrite_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000821
Mike Snitzer8b908f82015-05-13 17:53:13 -0400822 bio->bi_end_io = m->saved_bi_end_io;
823
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200824 m->err = bio->bi_error;
Joe Thornbere5aea7b2014-06-13 14:47:24 +0100825 complete_mapping_preparation(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000826}
827
Joe Thornber991d9fa2011-10-31 20:21:18 +0000828/*----------------------------------------------------------------*/
829
830/*
831 * Workqueue.
832 */
833
834/*
835 * Prepared mapping jobs.
836 */
837
838/*
Joe Thornber2d759a42014-10-10 15:27:16 +0100839 * This sends the bios in the cell, except the original holder, back
840 * to the deferred_bios list.
Joe Thornber991d9fa2011-10-31 20:21:18 +0000841 */
Joe Thornberf286ba02012-12-21 20:23:33 +0000842static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000843{
Joe Thornber991d9fa2011-10-31 20:21:18 +0000844 struct pool *pool = tc->pool;
845 unsigned long flags;
846
Mike Snitzerc140e1c2014-03-20 21:17:14 -0400847 spin_lock_irqsave(&tc->lock, flags);
848 cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
849 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000850
851 wake_worker(pool);
852}
853
Joe Thornbera374bb22014-10-10 13:43:14 +0100854static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
855
Joe Thornber2d759a42014-10-10 15:27:16 +0100856struct remap_info {
857 struct thin_c *tc;
858 struct bio_list defer_bios;
859 struct bio_list issue_bios;
860};
861
862static void __inc_remap_and_issue_cell(void *context,
863 struct dm_bio_prison_cell *cell)
864{
865 struct remap_info *info = context;
866 struct bio *bio;
867
868 while ((bio = bio_list_pop(&cell->bios))) {
869 if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA))
870 bio_list_add(&info->defer_bios, bio);
871 else {
872 inc_all_io_entry(info->tc->pool, bio);
873
874 /*
875 * We can't issue the bios with the bio prison lock
876 * held, so we add them to a list to issue on
877 * return from this function.
878 */
879 bio_list_add(&info->issue_bios, bio);
880 }
881 }
882}
883
Joe Thornbera374bb22014-10-10 13:43:14 +0100884static void inc_remap_and_issue_cell(struct thin_c *tc,
885 struct dm_bio_prison_cell *cell,
886 dm_block_t block)
887{
888 struct bio *bio;
Joe Thornber2d759a42014-10-10 15:27:16 +0100889 struct remap_info info;
Joe Thornbera374bb22014-10-10 13:43:14 +0100890
Joe Thornber2d759a42014-10-10 15:27:16 +0100891 info.tc = tc;
892 bio_list_init(&info.defer_bios);
893 bio_list_init(&info.issue_bios);
Joe Thornbera374bb22014-10-10 13:43:14 +0100894
Joe Thornber2d759a42014-10-10 15:27:16 +0100895 /*
896 * We have to be careful to inc any bios we're about to issue
897 * before the cell is released, and avoid a race with new bios
898 * being added to the cell.
899 */
900 cell_visit_release(tc->pool, __inc_remap_and_issue_cell,
901 &info, cell);
902
903 while ((bio = bio_list_pop(&info.defer_bios)))
904 thin_defer_bio(tc, bio);
905
906 while ((bio = bio_list_pop(&info.issue_bios)))
907 remap_and_issue(info.tc, bio, block);
Joe Thornbera374bb22014-10-10 13:43:14 +0100908}
909
Joe Thornbere49e5822012-07-27 15:08:16 +0100910static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
911{
Joe Thornber6beca5e2013-03-01 22:45:50 +0000912 cell_error(m->tc->pool, m->cell);
Joe Thornbere49e5822012-07-27 15:08:16 +0100913 list_del(&m->list);
914 mempool_free(m, m->tc->pool->mapping_pool);
915}
Joe Thornber025b9682013-03-01 22:45:50 +0000916
Mike Snitzera24c2562012-06-03 00:30:00 +0100917static void process_prepared_mapping(struct dm_thin_new_mapping *m)
Joe Thornber991d9fa2011-10-31 20:21:18 +0000918{
919 struct thin_c *tc = m->tc;
Joe Thornber6beca5e2013-03-01 22:45:50 +0000920 struct pool *pool = tc->pool;
Mike Snitzer8b908f82015-05-13 17:53:13 -0400921 struct bio *bio = m->bio;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000922 int r;
923
Joe Thornber991d9fa2011-10-31 20:21:18 +0000924 if (m->err) {
Joe Thornber6beca5e2013-03-01 22:45:50 +0000925 cell_error(pool, m->cell);
Joe Thornber905386f2012-07-27 15:08:05 +0100926 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000927 }
928
929 /*
930 * Commit the prepared block into the mapping btree.
931 * Any I/O for this block arriving after this point will get
932 * remapped to it directly.
933 */
Joe Thornber34fbcf62015-04-16 12:58:35 +0100934 r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000935 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -0500936 metadata_operation_failed(pool, "dm_thin_insert_block", r);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000937 cell_error(pool, m->cell);
Joe Thornber905386f2012-07-27 15:08:05 +0100938 goto out;
Joe Thornber991d9fa2011-10-31 20:21:18 +0000939 }
940
941 /*
942 * Release any bios held while the block was being provisioned.
943 * If we are processing a write bio that completely covers the block,
944 * we already processed it so can ignore it now when processing
945 * the bios in the cell.
946 */
947 if (bio) {
Joe Thornber2d759a42014-10-10 15:27:16 +0100948 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200949 bio_endio(bio);
Joe Thornber2d759a42014-10-10 15:27:16 +0100950 } else {
951 inc_all_io_entry(tc->pool, m->cell->holder);
952 remap_and_issue(tc, m->cell->holder, m->data_block);
953 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
954 }
Joe Thornber991d9fa2011-10-31 20:21:18 +0000955
Joe Thornber905386f2012-07-27 15:08:05 +0100956out:
Joe Thornber991d9fa2011-10-31 20:21:18 +0000957 list_del(&m->list);
Joe Thornber6beca5e2013-03-01 22:45:50 +0000958 mempool_free(m, pool->mapping_pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +0000959}
960
Joe Thornber34fbcf62015-04-16 12:58:35 +0100961/*----------------------------------------------------------------*/
962
963static void free_discard_mapping(struct dm_thin_new_mapping *m)
964{
965 struct thin_c *tc = m->tc;
966 if (m->cell)
967 cell_defer_no_holder(tc, m->cell);
968 mempool_free(m, tc->pool->mapping_pool);
969}
970
Joe Thornbere49e5822012-07-27 15:08:16 +0100971static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
Joe Thornber104655f2012-03-28 18:41:28 +0100972{
Joe Thornbere49e5822012-07-27 15:08:16 +0100973 bio_io_error(m->bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +0100974 free_discard_mapping(m);
Joe Thornbere49e5822012-07-27 15:08:16 +0100975}
Joe Thornber104655f2012-03-28 18:41:28 +0100976
Joe Thornber34fbcf62015-04-16 12:58:35 +0100977static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
Joe Thornbere49e5822012-07-27 15:08:16 +0100978{
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200979 bio_endio(m->bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +0100980 free_discard_mapping(m);
Joe Thornber104655f2012-03-28 18:41:28 +0100981}
982
Joe Thornber34fbcf62015-04-16 12:58:35 +0100983static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
Joe Thornbere49e5822012-07-27 15:08:16 +0100984{
985 int r;
986 struct thin_c *tc = m->tc;
987
Joe Thornber34fbcf62015-04-16 12:58:35 +0100988 r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);
989 if (r) {
990 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);
991 bio_io_error(m->bio);
992 } else
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200993 bio_endio(m->bio);
Joe Thornbere49e5822012-07-27 15:08:16 +0100994
Joe Thornber34fbcf62015-04-16 12:58:35 +0100995 cell_defer_no_holder(tc, m->cell);
996 mempool_free(m, tc->pool->mapping_pool);
997}
998
999static int passdown_double_checking_shared_status(struct dm_thin_new_mapping *m)
1000{
1001 /*
1002 * We've already unmapped this range of blocks, but before we
1003 * passdown we have to check that these blocks are now unused.
1004 */
1005 int r;
1006 bool used = true;
1007 struct thin_c *tc = m->tc;
1008 struct pool *pool = tc->pool;
1009 dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;
1010
1011 while (b != end) {
1012 /* find start of unmapped run */
1013 for (; b < end; b++) {
1014 r = dm_pool_block_is_used(pool->pmd, b, &used);
1015 if (r)
1016 return r;
1017
1018 if (!used)
1019 break;
1020 }
1021
1022 if (b == end)
1023 break;
1024
1025 /* find end of run */
1026 for (e = b + 1; e != end; e++) {
1027 r = dm_pool_block_is_used(pool->pmd, e, &used);
1028 if (r)
1029 return r;
1030
1031 if (used)
1032 break;
1033 }
1034
1035 r = issue_discard(tc, b, e, m->bio);
1036 if (r)
1037 return r;
1038
1039 b = e;
1040 }
1041
1042 return 0;
1043}
1044
1045static void process_prepared_discard_passdown(struct dm_thin_new_mapping *m)
1046{
1047 int r;
1048 struct thin_c *tc = m->tc;
1049 struct pool *pool = tc->pool;
1050
1051 r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end);
1052 if (r)
1053 metadata_operation_failed(pool, "dm_thin_remove_range", r);
1054
1055 else if (m->maybe_shared)
1056 r = passdown_double_checking_shared_status(m);
1057 else
1058 r = issue_discard(tc, m->data_block, m->data_block + (m->virt_end - m->virt_begin), m->bio);
1059
1060 /*
1061 * Even if r is set, there could be sub discards in flight that we
1062 * need to wait for.
1063 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001064 m->bio->bi_error = r;
1065 bio_endio(m->bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +01001066 cell_defer_no_holder(tc, m->cell);
1067 mempool_free(m, pool->mapping_pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01001068}
1069
Joe Thornber104655f2012-03-28 18:41:28 +01001070static void process_prepared(struct pool *pool, struct list_head *head,
Joe Thornbere49e5822012-07-27 15:08:16 +01001071 process_mapping_fn *fn)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001072{
1073 unsigned long flags;
1074 struct list_head maps;
Mike Snitzera24c2562012-06-03 00:30:00 +01001075 struct dm_thin_new_mapping *m, *tmp;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001076
1077 INIT_LIST_HEAD(&maps);
1078 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber104655f2012-03-28 18:41:28 +01001079 list_splice_init(head, &maps);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001080 spin_unlock_irqrestore(&pool->lock, flags);
1081
1082 list_for_each_entry_safe(m, tmp, &maps, list)
Joe Thornbere49e5822012-07-27 15:08:16 +01001083 (*fn)(m);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001084}
1085
1086/*
1087 * Deferred bio jobs.
1088 */
Joe Thornber104655f2012-03-28 18:41:28 +01001089static int io_overlaps_block(struct pool *pool, struct bio *bio)
1090{
Kent Overstreet4f024f32013-10-11 15:44:27 -07001091 return bio->bi_iter.bi_size ==
1092 (pool->sectors_per_block << SECTOR_SHIFT);
Joe Thornber104655f2012-03-28 18:41:28 +01001093}
1094
Joe Thornber991d9fa2011-10-31 20:21:18 +00001095static int io_overwrites_block(struct pool *pool, struct bio *bio)
1096{
Joe Thornber104655f2012-03-28 18:41:28 +01001097 return (bio_data_dir(bio) == WRITE) &&
1098 io_overlaps_block(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001099}
1100
1101static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
1102 bio_end_io_t *fn)
1103{
1104 *save = bio->bi_end_io;
1105 bio->bi_end_io = fn;
1106}
1107
1108static int ensure_next_mapping(struct pool *pool)
1109{
1110 if (pool->next_mapping)
1111 return 0;
1112
1113 pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
1114
1115 return pool->next_mapping ? 0 : -ENOMEM;
1116}
1117
Mike Snitzera24c2562012-06-03 00:30:00 +01001118static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001119{
Mike Snitzer16961b02013-12-17 13:19:11 -05001120 struct dm_thin_new_mapping *m = pool->next_mapping;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001121
1122 BUG_ON(!pool->next_mapping);
1123
Mike Snitzer16961b02013-12-17 13:19:11 -05001124 memset(m, 0, sizeof(struct dm_thin_new_mapping));
1125 INIT_LIST_HEAD(&m->list);
1126 m->bio = NULL;
1127
Joe Thornber991d9fa2011-10-31 20:21:18 +00001128 pool->next_mapping = NULL;
1129
Mike Snitzer16961b02013-12-17 13:19:11 -05001130 return m;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001131}
1132
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001133static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
1134 sector_t begin, sector_t end)
1135{
1136 int r;
1137 struct dm_io_region to;
1138
1139 to.bdev = tc->pool_dev->bdev;
1140 to.sector = begin;
1141 to.count = end - begin;
1142
1143 r = dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);
1144 if (r < 0) {
1145 DMERR_LIMIT("dm_kcopyd_zero() failed");
1146 copy_complete(1, 1, m);
1147 }
1148}
1149
Mike Snitzer452d7a62014-10-09 19:20:21 -04001150static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,
Joe Thornber34fbcf62015-04-16 12:58:35 +01001151 dm_block_t data_begin,
Mike Snitzer452d7a62014-10-09 19:20:21 -04001152 struct dm_thin_new_mapping *m)
1153{
1154 struct pool *pool = tc->pool;
1155 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1156
1157 h->overwrite_mapping = m;
1158 m->bio = bio;
1159 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
1160 inc_all_io_entry(pool, bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +01001161 remap_and_issue(tc, bio, data_begin);
Mike Snitzer452d7a62014-10-09 19:20:21 -04001162}
1163
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001164/*
1165 * A partial copy also needs to zero the uncopied region.
1166 */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001167static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
Joe Thornber2dd9c252012-03-28 18:41:28 +01001168 struct dm_dev *origin, dm_block_t data_origin,
1169 dm_block_t data_dest,
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001170 struct dm_bio_prison_cell *cell, struct bio *bio,
1171 sector_t len)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001172{
1173 int r;
1174 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +01001175 struct dm_thin_new_mapping *m = get_next_mapping(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001176
Joe Thornber991d9fa2011-10-31 20:21:18 +00001177 m->tc = tc;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001178 m->virt_begin = virt_block;
1179 m->virt_end = virt_block + 1u;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001180 m->data_block = data_dest;
1181 m->cell = cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001182
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001183 /*
1184 * quiesce action + copy action + an extra reference held for the
1185 * duration of this function (we may need to inc later for a
1186 * partial zero).
1187 */
1188 atomic_set(&m->prepare_actions, 3);
1189
Mike Snitzer44feb382012-10-12 21:02:10 +01001190 if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001191 complete_mapping_preparation(m); /* already quiesced */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001192
1193 /*
1194 * IO to pool_dev remaps to the pool target's data_dev.
1195 *
1196 * If the whole block of data is being overwritten, we can issue the
1197 * bio immediately. Otherwise we use kcopyd to clone the data first.
1198 */
Mike Snitzer452d7a62014-10-09 19:20:21 -04001199 if (io_overwrites_block(pool, bio))
1200 remap_and_issue_overwrite(tc, bio, data_dest, m);
1201 else {
Joe Thornber991d9fa2011-10-31 20:21:18 +00001202 struct dm_io_region from, to;
1203
Joe Thornber2dd9c252012-03-28 18:41:28 +01001204 from.bdev = origin->bdev;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001205 from.sector = data_origin * pool->sectors_per_block;
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001206 from.count = len;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001207
1208 to.bdev = tc->pool_dev->bdev;
1209 to.sector = data_dest * pool->sectors_per_block;
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001210 to.count = len;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001211
1212 r = dm_kcopyd_copy(pool->copier, &from, 1, &to,
1213 0, copy_complete, m);
1214 if (r < 0) {
Mike Snitzerc3977412012-12-21 20:23:34 +00001215 DMERR_LIMIT("dm_kcopyd_copy() failed");
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001216 copy_complete(1, 1, m);
1217
1218 /*
1219 * We allow the zero to be issued, to simplify the
1220 * error path. Otherwise we'd need to start
1221 * worrying about decrementing the prepare_actions
1222 * counter.
1223 */
1224 }
1225
1226 /*
1227 * Do we need to zero a tail region?
1228 */
1229 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
1230 atomic_inc(&m->prepare_actions);
1231 ll_zero(tc, m,
1232 data_dest * pool->sectors_per_block + len,
1233 (data_dest + 1) * pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001234 }
1235 }
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001236
1237 complete_mapping_preparation(m); /* drop our ref */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001238}
1239
Joe Thornber2dd9c252012-03-28 18:41:28 +01001240static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
1241 dm_block_t data_origin, dm_block_t data_dest,
Mike Snitzera24c2562012-06-03 00:30:00 +01001242 struct dm_bio_prison_cell *cell, struct bio *bio)
Joe Thornber2dd9c252012-03-28 18:41:28 +01001243{
1244 schedule_copy(tc, virt_block, tc->pool_dev,
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001245 data_origin, data_dest, cell, bio,
1246 tc->pool->sectors_per_block);
Joe Thornber2dd9c252012-03-28 18:41:28 +01001247}
1248
Joe Thornber991d9fa2011-10-31 20:21:18 +00001249static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
Mike Snitzera24c2562012-06-03 00:30:00 +01001250 dm_block_t data_block, struct dm_bio_prison_cell *cell,
Joe Thornber991d9fa2011-10-31 20:21:18 +00001251 struct bio *bio)
1252{
1253 struct pool *pool = tc->pool;
Mike Snitzera24c2562012-06-03 00:30:00 +01001254 struct dm_thin_new_mapping *m = get_next_mapping(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001255
Joe Thornber50f3c3e2014-06-13 13:57:09 +01001256 atomic_set(&m->prepare_actions, 1); /* no need to quiesce */
Joe Thornber991d9fa2011-10-31 20:21:18 +00001257 m->tc = tc;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001258 m->virt_begin = virt_block;
1259 m->virt_end = virt_block + 1u;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001260 m->data_block = data_block;
1261 m->cell = cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001262
1263 /*
1264 * If the whole block of data is being overwritten or we are not
1265 * zeroing pre-existing data, we can issue the bio immediately.
1266 * Otherwise we use kcopyd to zero the data first.
1267 */
Mike Snitzerf8ae7522015-05-14 11:28:37 -04001268 if (pool->pf.zero_new_blocks) {
1269 if (io_overwrites_block(pool, bio))
1270 remap_and_issue_overwrite(tc, bio, data_block, m);
1271 else
1272 ll_zero(tc, m, data_block * pool->sectors_per_block,
1273 (data_block + 1) * pool->sectors_per_block);
1274 } else
Joe Thornber991d9fa2011-10-31 20:21:18 +00001275 process_prepared_mapping(m);
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001276}
Joe Thornber991d9fa2011-10-31 20:21:18 +00001277
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001278static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
1279 dm_block_t data_dest,
1280 struct dm_bio_prison_cell *cell, struct bio *bio)
1281{
1282 struct pool *pool = tc->pool;
1283 sector_t virt_block_begin = virt_block * pool->sectors_per_block;
1284 sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
1285
1286 if (virt_block_end <= tc->origin_size)
1287 schedule_copy(tc, virt_block, tc->origin_dev,
1288 virt_block, data_dest, cell, bio,
1289 pool->sectors_per_block);
1290
1291 else if (virt_block_begin < tc->origin_size)
1292 schedule_copy(tc, virt_block, tc->origin_dev,
1293 virt_block, data_dest, cell, bio,
1294 tc->origin_size - virt_block_begin);
1295
1296 else
1297 schedule_zero(tc, virt_block, data_dest, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001298}
1299
Joe Thornber2c43fd22014-12-11 11:12:19 +00001300static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
1301
1302static void check_for_space(struct pool *pool)
1303{
1304 int r;
1305 dm_block_t nr_free;
1306
1307 if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)
1308 return;
1309
1310 r = dm_pool_get_free_block_count(pool->pmd, &nr_free);
1311 if (r)
1312 return;
1313
1314 if (nr_free)
1315 set_pool_mode(pool, PM_WRITE);
1316}
1317
Joe Thornbere49e5822012-07-27 15:08:16 +01001318/*
1319 * A non-zero return indicates read_only or fail_io mode.
1320 * Many callers don't care about the return value.
1321 */
Joe Thornber020cc3b2013-12-04 15:05:36 -05001322static int commit(struct pool *pool)
Joe Thornbere49e5822012-07-27 15:08:16 +01001323{
1324 int r;
1325
Joe Thornber8d07e8a2014-05-06 16:28:14 +01001326 if (get_pool_mode(pool) >= PM_READ_ONLY)
Joe Thornbere49e5822012-07-27 15:08:16 +01001327 return -EINVAL;
1328
Joe Thornber020cc3b2013-12-04 15:05:36 -05001329 r = dm_pool_commit_metadata(pool->pmd);
Joe Thornberb5330652013-12-04 19:51:33 -05001330 if (r)
1331 metadata_operation_failed(pool, "dm_pool_commit_metadata", r);
Joe Thornber2c43fd22014-12-11 11:12:19 +00001332 else
1333 check_for_space(pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01001334
1335 return r;
1336}
1337
Joe Thornber88a66212013-12-04 20:16:12 -05001338static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
1339{
1340 unsigned long flags;
1341
1342 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1343 DMWARN("%s: reached low water mark for data device: sending event.",
1344 dm_device_name(pool->pool_md));
1345 spin_lock_irqsave(&pool->lock, flags);
1346 pool->low_water_triggered = true;
1347 spin_unlock_irqrestore(&pool->lock, flags);
1348 dm_table_event(pool->ti->table);
1349 }
1350}
1351
Joe Thornber991d9fa2011-10-31 20:21:18 +00001352static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1353{
1354 int r;
1355 dm_block_t free_blocks;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001356 struct pool *pool = tc->pool;
1357
Joe Thornber3e1a0692014-03-03 16:03:26 +00001358 if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
Joe Thornber8d30abf2013-12-04 19:16:11 -05001359 return -EINVAL;
1360
Joe Thornber991d9fa2011-10-31 20:21:18 +00001361 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
Joe Thornberb5330652013-12-04 19:51:33 -05001362 if (r) {
1363 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001364 return r;
Joe Thornberb5330652013-12-04 19:51:33 -05001365 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001366
Joe Thornber88a66212013-12-04 20:16:12 -05001367 check_low_water_mark(pool, free_blocks);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001368
1369 if (!free_blocks) {
Mike Snitzer94563ba2013-08-22 09:56:18 -04001370 /*
1371 * Try to commit to see if that will free up some
1372 * more space.
1373 */
Joe Thornber020cc3b2013-12-04 15:05:36 -05001374 r = commit(pool);
1375 if (r)
1376 return r;
Mike Snitzer94563ba2013-08-22 09:56:18 -04001377
1378 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
Joe Thornberb5330652013-12-04 19:51:33 -05001379 if (r) {
1380 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
Mike Snitzer94563ba2013-08-22 09:56:18 -04001381 return r;
Joe Thornberb5330652013-12-04 19:51:33 -05001382 }
Mike Snitzer94563ba2013-08-22 09:56:18 -04001383
Mike Snitzer94563ba2013-08-22 09:56:18 -04001384 if (!free_blocks) {
Joe Thornber3e1a0692014-03-03 16:03:26 +00001385 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001386 return -ENOSPC;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001387 }
1388 }
1389
1390 r = dm_pool_alloc_data_block(pool->pmd, result);
Mike Snitzer4a02b342013-12-03 12:20:57 -05001391 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05001392 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001393 return r;
Mike Snitzer4a02b342013-12-03 12:20:57 -05001394 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001395
1396 return 0;
1397}
1398
1399/*
1400 * If we have run out of space, queue bios until the device is
1401 * resumed, presumably after having been reloaded with more space.
1402 */
1403static void retry_on_resume(struct bio *bio)
1404{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001405 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01001406 struct thin_c *tc = h->tc;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001407 unsigned long flags;
1408
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001409 spin_lock_irqsave(&tc->lock, flags);
1410 bio_list_add(&tc->retry_on_resume_list, bio);
1411 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001412}
1413
Mike Snitzeraf918052014-05-22 14:32:51 -04001414static int should_error_unserviceable_bio(struct pool *pool)
Joe Thornber3e1a0692014-03-03 16:03:26 +00001415{
1416 enum pool_mode m = get_pool_mode(pool);
1417
1418 switch (m) {
1419 case PM_WRITE:
1420 /* Shouldn't get here */
1421 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
Mike Snitzeraf918052014-05-22 14:32:51 -04001422 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001423
1424 case PM_OUT_OF_DATA_SPACE:
Mike Snitzeraf918052014-05-22 14:32:51 -04001425 return pool->pf.error_if_no_space ? -ENOSPC : 0;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001426
1427 case PM_READ_ONLY:
1428 case PM_FAIL:
Mike Snitzeraf918052014-05-22 14:32:51 -04001429 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001430 default:
1431 /* Shouldn't get here */
1432 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
Mike Snitzeraf918052014-05-22 14:32:51 -04001433 return -EIO;
Joe Thornber3e1a0692014-03-03 16:03:26 +00001434 }
1435}
1436
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001437static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
1438{
Mike Snitzeraf918052014-05-22 14:32:51 -04001439 int error = should_error_unserviceable_bio(pool);
1440
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001441 if (error) {
1442 bio->bi_error = error;
1443 bio_endio(bio);
1444 } else
Mike Snitzer6d162022013-12-20 18:09:02 -05001445 retry_on_resume(bio);
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001446}
1447
Mike Snitzer399cadd2013-12-05 16:03:33 -05001448static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001449{
1450 struct bio *bio;
1451 struct bio_list bios;
Mike Snitzeraf918052014-05-22 14:32:51 -04001452 int error;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001453
Mike Snitzeraf918052014-05-22 14:32:51 -04001454 error = should_error_unserviceable_bio(pool);
1455 if (error) {
1456 cell_error_with_code(pool, cell, error);
Joe Thornber3e1a0692014-03-03 16:03:26 +00001457 return;
1458 }
1459
Joe Thornber991d9fa2011-10-31 20:21:18 +00001460 bio_list_init(&bios);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001461 cell_release(pool, cell, &bios);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001462
Mike Snitzer9d094ee2014-10-19 08:23:09 -04001463 while ((bio = bio_list_pop(&bios)))
1464 retry_on_resume(bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001465}
1466
Joe Thornber34fbcf62015-04-16 12:58:35 +01001467static void process_discard_cell_no_passdown(struct thin_c *tc,
1468 struct dm_bio_prison_cell *virt_cell)
Joe Thornber104655f2012-03-28 18:41:28 +01001469{
Joe Thornber104655f2012-03-28 18:41:28 +01001470 struct pool *pool = tc->pool;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001471 struct dm_thin_new_mapping *m = get_next_mapping(pool);
1472
1473 /*
1474 * We don't need to lock the data blocks, since there's no
1475 * passdown. We only lock data blocks for allocation and breaking sharing.
1476 */
1477 m->tc = tc;
1478 m->virt_begin = virt_cell->key.block_begin;
1479 m->virt_end = virt_cell->key.block_end;
1480 m->cell = virt_cell;
1481 m->bio = virt_cell->holder;
1482
1483 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1484 pool->process_prepared_discard(m);
1485}
1486
1487/*
Mike Snitzer84f8bd82015-08-18 10:31:09 -04001488 * __bio_inc_remaining() is used to defer parent bios's end_io until
1489 * we _know_ all chained sub range discard bios have completed.
Joe Thornber34fbcf62015-04-16 12:58:35 +01001490 */
1491static inline void __bio_inc_remaining(struct bio *bio)
1492{
1493 bio->bi_flags |= (1 << BIO_CHAIN);
1494 smp_mb__before_atomic();
1495 atomic_inc(&bio->__bi_remaining);
1496}
1497
1498static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,
1499 struct bio *bio)
1500{
1501 struct pool *pool = tc->pool;
1502
1503 int r;
1504 bool maybe_shared;
1505 struct dm_cell_key data_key;
1506 struct dm_bio_prison_cell *data_cell;
Mike Snitzera24c2562012-06-03 00:30:00 +01001507 struct dm_thin_new_mapping *m;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001508 dm_block_t virt_begin, virt_end, data_begin;
Joe Thornber104655f2012-03-28 18:41:28 +01001509
Joe Thornber34fbcf62015-04-16 12:58:35 +01001510 while (begin != end) {
1511 r = ensure_next_mapping(pool);
1512 if (r)
1513 /* we did our best */
1514 return;
Joe Thornber104655f2012-03-28 18:41:28 +01001515
Joe Thornber34fbcf62015-04-16 12:58:35 +01001516 r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,
1517 &data_begin, &maybe_shared);
1518 if (r)
1519 /*
1520 * Silently fail, letting any mappings we've
1521 * created complete.
1522 */
Joe Thornber104655f2012-03-28 18:41:28 +01001523 break;
Joe Thornber34fbcf62015-04-16 12:58:35 +01001524
1525 build_key(tc->td, PHYSICAL, data_begin, data_begin + (virt_end - virt_begin), &data_key);
1526 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
1527 /* contention, we'll give up with this range */
1528 begin = virt_end;
1529 continue;
Joe Thornber104655f2012-03-28 18:41:28 +01001530 }
1531
Joe Thornber104655f2012-03-28 18:41:28 +01001532 /*
Joe Thornber34fbcf62015-04-16 12:58:35 +01001533 * IO may still be going to the destination block. We must
1534 * quiesce before we can do the removal.
Joe Thornber104655f2012-03-28 18:41:28 +01001535 */
Joe Thornber34fbcf62015-04-16 12:58:35 +01001536 m = get_next_mapping(pool);
1537 m->tc = tc;
1538 m->maybe_shared = maybe_shared;
1539 m->virt_begin = virt_begin;
1540 m->virt_end = virt_end;
1541 m->data_block = data_begin;
1542 m->cell = data_cell;
1543 m->bio = bio;
Joe Thornber104655f2012-03-28 18:41:28 +01001544
Joe Thornber34fbcf62015-04-16 12:58:35 +01001545 /*
1546 * The parent bio must not complete before sub discard bios are
1547 * chained to it (see __blkdev_issue_discard_async's bio_chain)!
1548 *
1549 * This per-mapping bi_remaining increment is paired with
1550 * the implicit decrement that occurs via bio_endio() in
1551 * process_prepared_discard_{passdown,no_passdown}.
1552 */
1553 __bio_inc_remaining(bio);
1554 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1555 pool->process_prepared_discard(m);
1556
1557 begin = virt_end;
Joe Thornber104655f2012-03-28 18:41:28 +01001558 }
1559}
1560
Joe Thornber34fbcf62015-04-16 12:58:35 +01001561static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)
1562{
1563 struct bio *bio = virt_cell->holder;
1564 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1565
1566 /*
1567 * The virt_cell will only get freed once the origin bio completes.
1568 * This means it will remain locked while all the individual
1569 * passdown bios are in flight.
1570 */
1571 h->cell = virt_cell;
1572 break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio);
1573
1574 /*
1575 * We complete the bio now, knowing that the bi_remaining field
1576 * will prevent completion until the sub range discards have
1577 * completed.
1578 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001579 bio_endio(bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +01001580}
1581
Joe Thornbera374bb22014-10-10 13:43:14 +01001582static void process_discard_bio(struct thin_c *tc, struct bio *bio)
1583{
Joe Thornber34fbcf62015-04-16 12:58:35 +01001584 dm_block_t begin, end;
1585 struct dm_cell_key virt_key;
1586 struct dm_bio_prison_cell *virt_cell;
Joe Thornbera374bb22014-10-10 13:43:14 +01001587
Joe Thornber34fbcf62015-04-16 12:58:35 +01001588 get_bio_block_range(tc, bio, &begin, &end);
1589 if (begin == end) {
1590 /*
1591 * The discard covers less than a block.
1592 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001593 bio_endio(bio);
Joe Thornber34fbcf62015-04-16 12:58:35 +01001594 return;
1595 }
1596
1597 build_key(tc->td, VIRTUAL, begin, end, &virt_key);
1598 if (bio_detain(tc->pool, &virt_key, bio, &virt_cell))
1599 /*
1600 * Potential starvation issue: We're relying on the
1601 * fs/application being well behaved, and not trying to
1602 * send IO to a region at the same time as discarding it.
1603 * If they do this persistently then it's possible this
1604 * cell will never be granted.
1605 */
Joe Thornbera374bb22014-10-10 13:43:14 +01001606 return;
1607
Joe Thornber34fbcf62015-04-16 12:58:35 +01001608 tc->pool->process_discard_cell(tc, virt_cell);
Joe Thornbera374bb22014-10-10 13:43:14 +01001609}
1610
Joe Thornber991d9fa2011-10-31 20:21:18 +00001611static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
Mike Snitzer44feb382012-10-12 21:02:10 +01001612 struct dm_cell_key *key,
Joe Thornber991d9fa2011-10-31 20:21:18 +00001613 struct dm_thin_lookup_result *lookup_result,
Mike Snitzera24c2562012-06-03 00:30:00 +01001614 struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001615{
1616 int r;
1617 dm_block_t data_block;
Mike Snitzerd6fc2042013-08-21 17:40:11 -04001618 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001619
1620 r = alloc_data_block(tc, &data_block);
1621 switch (r) {
1622 case 0:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001623 schedule_internal_copy(tc, block, lookup_result->block,
1624 data_block, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001625 break;
1626
1627 case -ENOSPC:
Mike Snitzer399cadd2013-12-05 16:03:33 -05001628 retry_bios_on_resume(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001629 break;
1630
1631 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001632 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1633 __func__, r);
Mike Snitzerd6fc2042013-08-21 17:40:11 -04001634 cell_error(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001635 break;
1636 }
1637}
1638
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001639static void __remap_and_issue_shared_cell(void *context,
1640 struct dm_bio_prison_cell *cell)
1641{
1642 struct remap_info *info = context;
1643 struct bio *bio;
1644
1645 while ((bio = bio_list_pop(&cell->bios))) {
1646 if ((bio_data_dir(bio) == WRITE) ||
1647 (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)))
1648 bio_list_add(&info->defer_bios, bio);
1649 else {
1650 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));;
1651
1652 h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
1653 inc_all_io_entry(info->tc->pool, bio);
1654 bio_list_add(&info->issue_bios, bio);
1655 }
1656 }
1657}
1658
1659static void remap_and_issue_shared_cell(struct thin_c *tc,
1660 struct dm_bio_prison_cell *cell,
1661 dm_block_t block)
1662{
1663 struct bio *bio;
1664 struct remap_info info;
1665
1666 info.tc = tc;
1667 bio_list_init(&info.defer_bios);
1668 bio_list_init(&info.issue_bios);
1669
1670 cell_visit_release(tc->pool, __remap_and_issue_shared_cell,
1671 &info, cell);
1672
1673 while ((bio = bio_list_pop(&info.defer_bios)))
1674 thin_defer_bio(tc, bio);
1675
1676 while ((bio = bio_list_pop(&info.issue_bios)))
1677 remap_and_issue(tc, bio, block);
1678}
1679
Joe Thornber991d9fa2011-10-31 20:21:18 +00001680static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1681 dm_block_t block,
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001682 struct dm_thin_lookup_result *lookup_result,
1683 struct dm_bio_prison_cell *virt_cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001684{
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001685 struct dm_bio_prison_cell *data_cell;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001686 struct pool *pool = tc->pool;
Mike Snitzer44feb382012-10-12 21:02:10 +01001687 struct dm_cell_key key;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001688
1689 /*
1690 * If cell is already occupied, then sharing is already in the process
1691 * of being broken so we have nothing further to do here.
1692 */
1693 build_data_key(tc->td, lookup_result->block, &key);
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001694 if (bio_detain(pool, &key, bio, &data_cell)) {
1695 cell_defer_no_holder(tc, virt_cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001696 return;
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001697 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001698
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001699 if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
1700 break_sharing(tc, bio, block, &key, lookup_result, data_cell);
1701 cell_defer_no_holder(tc, virt_cell);
1702 } else {
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00001703 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornber991d9fa2011-10-31 20:21:18 +00001704
Mike Snitzer44feb382012-10-12 21:02:10 +01001705 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
Joe Thornbere8088072012-12-21 20:23:31 +00001706 inc_all_io_entry(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001707 remap_and_issue(tc, bio, lookup_result->block);
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001708
1709 remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);
1710 remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001711 }
1712}
1713
1714static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
Mike Snitzera24c2562012-06-03 00:30:00 +01001715 struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001716{
1717 int r;
1718 dm_block_t data_block;
Joe Thornber6beca5e2013-03-01 22:45:50 +00001719 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001720
1721 /*
1722 * Remap empty bios (flushes) immediately, without provisioning.
1723 */
Kent Overstreet4f024f32013-10-11 15:44:27 -07001724 if (!bio->bi_iter.bi_size) {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001725 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001726 cell_defer_no_holder(tc, cell);
Joe Thornbere8088072012-12-21 20:23:31 +00001727
Joe Thornber991d9fa2011-10-31 20:21:18 +00001728 remap_and_issue(tc, bio, 0);
1729 return;
1730 }
1731
1732 /*
1733 * Fill read bios with zeroes and complete them immediately.
1734 */
1735 if (bio_data_dir(bio) == READ) {
1736 zero_fill_bio(bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001737 cell_defer_no_holder(tc, cell);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001738 bio_endio(bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001739 return;
1740 }
1741
1742 r = alloc_data_block(tc, &data_block);
1743 switch (r) {
1744 case 0:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001745 if (tc->origin_dev)
1746 schedule_external_copy(tc, block, data_block, cell, bio);
1747 else
1748 schedule_zero(tc, block, data_block, cell, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001749 break;
1750
1751 case -ENOSPC:
Mike Snitzer399cadd2013-12-05 16:03:33 -05001752 retry_bios_on_resume(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001753 break;
1754
1755 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001756 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1757 __func__, r);
Joe Thornber6beca5e2013-03-01 22:45:50 +00001758 cell_error(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001759 break;
1760 }
1761}
1762
Joe Thornbera374bb22014-10-10 13:43:14 +01001763static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001764{
1765 int r;
Joe Thornber6beca5e2013-03-01 22:45:50 +00001766 struct pool *pool = tc->pool;
Joe Thornbera374bb22014-10-10 13:43:14 +01001767 struct bio *bio = cell->holder;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001768 dm_block_t block = get_bio_block(tc, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001769 struct dm_thin_lookup_result lookup_result;
1770
Joe Thornbera374bb22014-10-10 13:43:14 +01001771 if (tc->requeue_mode) {
1772 cell_requeue(pool, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001773 return;
Joe Thornbera374bb22014-10-10 13:43:14 +01001774 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001775
1776 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1777 switch (r) {
1778 case 0:
Joe Thornber23ca2bb2014-10-15 14:46:58 +01001779 if (lookup_result.shared)
1780 process_shared_bio(tc, bio, block, &lookup_result, cell);
1781 else {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001782 inc_all_io_entry(pool, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001783 remap_and_issue(tc, bio, lookup_result.block);
Joe Thornbera374bb22014-10-10 13:43:14 +01001784 inc_remap_and_issue_cell(tc, cell, lookup_result.block);
Joe Thornbere8088072012-12-21 20:23:31 +00001785 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00001786 break;
1787
1788 case -ENODATA:
Joe Thornber2dd9c252012-03-28 18:41:28 +01001789 if (bio_data_dir(bio) == READ && tc->origin_dev) {
Joe Thornber6beca5e2013-03-01 22:45:50 +00001790 inc_all_io_entry(pool, bio);
Joe Thornberf286ba02012-12-21 20:23:33 +00001791 cell_defer_no_holder(tc, cell);
Joe Thornbere8088072012-12-21 20:23:31 +00001792
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001793 if (bio_end_sector(bio) <= tc->origin_size)
1794 remap_to_origin_and_issue(tc, bio);
1795
1796 else if (bio->bi_iter.bi_sector < tc->origin_size) {
1797 zero_fill_bio(bio);
1798 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
1799 remap_to_origin_and_issue(tc, bio);
1800
1801 } else {
1802 zero_fill_bio(bio);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001803 bio_endio(bio);
Joe Thornbere5aea7b2014-06-13 14:47:24 +01001804 }
Joe Thornber2dd9c252012-03-28 18:41:28 +01001805 } else
1806 provision_block(tc, bio, block, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001807 break;
1808
1809 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001810 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1811 __func__, r);
Joe Thornberf286ba02012-12-21 20:23:33 +00001812 cell_defer_no_holder(tc, cell);
Joe Thornber991d9fa2011-10-31 20:21:18 +00001813 bio_io_error(bio);
1814 break;
1815 }
1816}
1817
Joe Thornbera374bb22014-10-10 13:43:14 +01001818static void process_bio(struct thin_c *tc, struct bio *bio)
1819{
1820 struct pool *pool = tc->pool;
1821 dm_block_t block = get_bio_block(tc, bio);
1822 struct dm_bio_prison_cell *cell;
1823 struct dm_cell_key key;
1824
1825 /*
1826 * If cell is already occupied, then the block is already
1827 * being provisioned so we have nothing further to do here.
1828 */
1829 build_virtual_key(tc->td, block, &key);
1830 if (bio_detain(pool, &key, bio, &cell))
1831 return;
1832
1833 process_cell(tc, cell);
1834}
1835
1836static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,
1837 struct dm_bio_prison_cell *cell)
Joe Thornbere49e5822012-07-27 15:08:16 +01001838{
1839 int r;
1840 int rw = bio_data_dir(bio);
1841 dm_block_t block = get_bio_block(tc, bio);
1842 struct dm_thin_lookup_result lookup_result;
1843
1844 r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1845 switch (r) {
1846 case 0:
Joe Thornbera374bb22014-10-10 13:43:14 +01001847 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001848 handle_unserviceable_bio(tc->pool, bio);
Joe Thornbera374bb22014-10-10 13:43:14 +01001849 if (cell)
1850 cell_defer_no_holder(tc, cell);
1851 } else {
Joe Thornbere8088072012-12-21 20:23:31 +00001852 inc_all_io_entry(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001853 remap_and_issue(tc, bio, lookup_result.block);
Joe Thornbera374bb22014-10-10 13:43:14 +01001854 if (cell)
1855 inc_remap_and_issue_cell(tc, cell, lookup_result.block);
Joe Thornbere8088072012-12-21 20:23:31 +00001856 }
Joe Thornbere49e5822012-07-27 15:08:16 +01001857 break;
1858
1859 case -ENODATA:
Joe Thornbera374bb22014-10-10 13:43:14 +01001860 if (cell)
1861 cell_defer_no_holder(tc, cell);
Joe Thornbere49e5822012-07-27 15:08:16 +01001862 if (rw != READ) {
Mike Snitzer8c0f0e82013-12-05 15:47:24 -05001863 handle_unserviceable_bio(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001864 break;
1865 }
1866
1867 if (tc->origin_dev) {
Joe Thornbere8088072012-12-21 20:23:31 +00001868 inc_all_io_entry(tc->pool, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001869 remap_to_origin_and_issue(tc, bio);
1870 break;
1871 }
1872
1873 zero_fill_bio(bio);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001874 bio_endio(bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01001875 break;
1876
1877 default:
Mike Snitzerc3977412012-12-21 20:23:34 +00001878 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1879 __func__, r);
Joe Thornbera374bb22014-10-10 13:43:14 +01001880 if (cell)
1881 cell_defer_no_holder(tc, cell);
Joe Thornbere49e5822012-07-27 15:08:16 +01001882 bio_io_error(bio);
1883 break;
1884 }
1885}
1886
Joe Thornbera374bb22014-10-10 13:43:14 +01001887static void process_bio_read_only(struct thin_c *tc, struct bio *bio)
1888{
1889 __process_bio_read_only(tc, bio, NULL);
1890}
1891
1892static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1893{
1894 __process_bio_read_only(tc, cell->holder, cell);
1895}
1896
Joe Thornber3e1a0692014-03-03 16:03:26 +00001897static void process_bio_success(struct thin_c *tc, struct bio *bio)
1898{
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02001899 bio_endio(bio);
Joe Thornber3e1a0692014-03-03 16:03:26 +00001900}
1901
Joe Thornbere49e5822012-07-27 15:08:16 +01001902static void process_bio_fail(struct thin_c *tc, struct bio *bio)
1903{
1904 bio_io_error(bio);
1905}
1906
Joe Thornbera374bb22014-10-10 13:43:14 +01001907static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1908{
1909 cell_success(tc->pool, cell);
1910}
1911
1912static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1913{
1914 cell_error(tc->pool, cell);
1915}
1916
Joe Thornberac8c3f32013-05-10 14:37:21 +01001917/*
1918 * FIXME: should we also commit due to size of transaction, measured in
1919 * metadata blocks?
1920 */
Joe Thornber905e51b2012-03-28 18:41:27 +01001921static int need_commit_due_to_time(struct pool *pool)
1922{
Manuel Schölling0f30af92014-05-22 22:42:37 +02001923 return !time_in_range(jiffies, pool->last_commit_jiffies,
1924 pool->last_commit_jiffies + COMMIT_PERIOD);
Joe Thornber905e51b2012-03-28 18:41:27 +01001925}
1926
Mike Snitzer67324ea2014-03-21 18:33:41 -04001927#define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
1928#define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
1929
1930static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)
1931{
1932 struct rb_node **rbp, *parent;
1933 struct dm_thin_endio_hook *pbd;
1934 sector_t bi_sector = bio->bi_iter.bi_sector;
1935
1936 rbp = &tc->sort_bio_list.rb_node;
1937 parent = NULL;
1938 while (*rbp) {
1939 parent = *rbp;
1940 pbd = thin_pbd(parent);
1941
1942 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)
1943 rbp = &(*rbp)->rb_left;
1944 else
1945 rbp = &(*rbp)->rb_right;
1946 }
1947
1948 pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1949 rb_link_node(&pbd->rb_node, parent, rbp);
1950 rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);
1951}
1952
1953static void __extract_sorted_bios(struct thin_c *tc)
1954{
1955 struct rb_node *node;
1956 struct dm_thin_endio_hook *pbd;
1957 struct bio *bio;
1958
1959 for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) {
1960 pbd = thin_pbd(node);
1961 bio = thin_bio(pbd);
1962
1963 bio_list_add(&tc->deferred_bio_list, bio);
1964 rb_erase(&pbd->rb_node, &tc->sort_bio_list);
1965 }
1966
1967 WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));
1968}
1969
1970static void __sort_thin_deferred_bios(struct thin_c *tc)
1971{
1972 struct bio *bio;
1973 struct bio_list bios;
1974
1975 bio_list_init(&bios);
1976 bio_list_merge(&bios, &tc->deferred_bio_list);
1977 bio_list_init(&tc->deferred_bio_list);
1978
1979 /* Sort deferred_bio_list using rb-tree */
1980 while ((bio = bio_list_pop(&bios)))
1981 __thin_bio_rb_add(tc, bio);
1982
1983 /*
1984 * Transfer the sorted bios in sort_bio_list back to
1985 * deferred_bio_list to allow lockless submission of
1986 * all bios.
1987 */
1988 __extract_sorted_bios(tc);
1989}
1990
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001991static void process_thin_deferred_bios(struct thin_c *tc)
Joe Thornber991d9fa2011-10-31 20:21:18 +00001992{
Mike Snitzerc140e1c2014-03-20 21:17:14 -04001993 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001994 unsigned long flags;
1995 struct bio *bio;
1996 struct bio_list bios;
Mike Snitzer67324ea2014-03-21 18:33:41 -04001997 struct blk_plug plug;
Joe Thornber8a01a6a2014-10-06 15:28:30 +01001998 unsigned count = 0;
Joe Thornber991d9fa2011-10-31 20:21:18 +00001999
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002000 if (tc->requeue_mode) {
Mike Snitzer42d6a8c2014-10-19 07:52:44 -04002001 error_thin_bio_list(tc, &tc->deferred_bio_list, DM_ENDIO_REQUEUE);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002002 return;
2003 }
2004
Joe Thornber991d9fa2011-10-31 20:21:18 +00002005 bio_list_init(&bios);
2006
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002007 spin_lock_irqsave(&tc->lock, flags);
Mike Snitzer67324ea2014-03-21 18:33:41 -04002008
2009 if (bio_list_empty(&tc->deferred_bio_list)) {
2010 spin_unlock_irqrestore(&tc->lock, flags);
2011 return;
2012 }
2013
2014 __sort_thin_deferred_bios(tc);
2015
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002016 bio_list_merge(&bios, &tc->deferred_bio_list);
2017 bio_list_init(&tc->deferred_bio_list);
Mike Snitzer67324ea2014-03-21 18:33:41 -04002018
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002019 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002020
Mike Snitzer67324ea2014-03-21 18:33:41 -04002021 blk_start_plug(&plug);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002022 while ((bio = bio_list_pop(&bios))) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00002023 /*
2024 * If we've got no free new_mapping structs, and processing
2025 * this bio might require one, we pause until there are some
2026 * prepared mappings to process.
2027 */
2028 if (ensure_next_mapping(pool)) {
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002029 spin_lock_irqsave(&tc->lock, flags);
2030 bio_list_add(&tc->deferred_bio_list, bio);
2031 bio_list_merge(&tc->deferred_bio_list, &bios);
2032 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002033 break;
2034 }
Joe Thornber104655f2012-03-28 18:41:28 +01002035
2036 if (bio->bi_rw & REQ_DISCARD)
Joe Thornbere49e5822012-07-27 15:08:16 +01002037 pool->process_discard(tc, bio);
Joe Thornber104655f2012-03-28 18:41:28 +01002038 else
Joe Thornbere49e5822012-07-27 15:08:16 +01002039 pool->process_bio(tc, bio);
Joe Thornber8a01a6a2014-10-06 15:28:30 +01002040
2041 if ((count++ & 127) == 0) {
Joe Thornber7d327fe2014-10-06 15:45:59 +01002042 throttle_work_update(&pool->throttle);
Joe Thornber8a01a6a2014-10-06 15:28:30 +01002043 dm_pool_issue_prefetches(pool->pmd);
2044 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002045 }
Mike Snitzer67324ea2014-03-21 18:33:41 -04002046 blk_finish_plug(&plug);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002047}
2048
Joe Thornberac4c3f32014-10-10 16:42:10 +01002049static int cmp_cells(const void *lhs, const void *rhs)
2050{
2051 struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs);
2052 struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs);
2053
2054 BUG_ON(!lhs_cell->holder);
2055 BUG_ON(!rhs_cell->holder);
2056
2057 if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)
2058 return -1;
2059
2060 if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)
2061 return 1;
2062
2063 return 0;
2064}
2065
2066static unsigned sort_cells(struct pool *pool, struct list_head *cells)
2067{
2068 unsigned count = 0;
2069 struct dm_bio_prison_cell *cell, *tmp;
2070
2071 list_for_each_entry_safe(cell, tmp, cells, user_list) {
2072 if (count >= CELL_SORT_ARRAY_SIZE)
2073 break;
2074
2075 pool->cell_sort_array[count++] = cell;
2076 list_del(&cell->user_list);
2077 }
2078
2079 sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);
2080
2081 return count;
2082}
2083
Joe Thornbera374bb22014-10-10 13:43:14 +01002084static void process_thin_deferred_cells(struct thin_c *tc)
2085{
2086 struct pool *pool = tc->pool;
2087 unsigned long flags;
2088 struct list_head cells;
Joe Thornberac4c3f32014-10-10 16:42:10 +01002089 struct dm_bio_prison_cell *cell;
2090 unsigned i, j, count;
Joe Thornbera374bb22014-10-10 13:43:14 +01002091
2092 INIT_LIST_HEAD(&cells);
2093
2094 spin_lock_irqsave(&tc->lock, flags);
2095 list_splice_init(&tc->deferred_cells, &cells);
2096 spin_unlock_irqrestore(&tc->lock, flags);
2097
2098 if (list_empty(&cells))
2099 return;
2100
Joe Thornberac4c3f32014-10-10 16:42:10 +01002101 do {
2102 count = sort_cells(tc->pool, &cells);
Joe Thornbera374bb22014-10-10 13:43:14 +01002103
Joe Thornberac4c3f32014-10-10 16:42:10 +01002104 for (i = 0; i < count; i++) {
2105 cell = pool->cell_sort_array[i];
2106 BUG_ON(!cell->holder);
2107
2108 /*
2109 * If we've got no free new_mapping structs, and processing
2110 * this bio might require one, we pause until there are some
2111 * prepared mappings to process.
2112 */
2113 if (ensure_next_mapping(pool)) {
2114 for (j = i; j < count; j++)
2115 list_add(&pool->cell_sort_array[j]->user_list, &cells);
2116
2117 spin_lock_irqsave(&tc->lock, flags);
2118 list_splice(&cells, &tc->deferred_cells);
2119 spin_unlock_irqrestore(&tc->lock, flags);
2120 return;
2121 }
2122
2123 if (cell->holder->bi_rw & REQ_DISCARD)
2124 pool->process_discard_cell(tc, cell);
2125 else
2126 pool->process_cell(tc, cell);
Joe Thornbera374bb22014-10-10 13:43:14 +01002127 }
Joe Thornberac4c3f32014-10-10 16:42:10 +01002128 } while (!list_empty(&cells));
Joe Thornbera374bb22014-10-10 13:43:14 +01002129}
2130
Joe Thornberb10ebd32014-04-08 11:29:01 +01002131static void thin_get(struct thin_c *tc);
2132static void thin_put(struct thin_c *tc);
2133
2134/*
2135 * We can't hold rcu_read_lock() around code that can block. So we
2136 * find a thin with the rcu lock held; bump a refcount; then drop
2137 * the lock.
2138 */
2139static struct thin_c *get_first_thin(struct pool *pool)
2140{
2141 struct thin_c *tc = NULL;
2142
2143 rcu_read_lock();
2144 if (!list_empty(&pool->active_thins)) {
2145 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list);
2146 thin_get(tc);
2147 }
2148 rcu_read_unlock();
2149
2150 return tc;
2151}
2152
2153static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
2154{
2155 struct thin_c *old_tc = tc;
2156
2157 rcu_read_lock();
2158 list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
2159 thin_get(tc);
2160 thin_put(old_tc);
2161 rcu_read_unlock();
2162 return tc;
2163 }
2164 thin_put(old_tc);
2165 rcu_read_unlock();
2166
2167 return NULL;
2168}
2169
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002170static void process_deferred_bios(struct pool *pool)
2171{
2172 unsigned long flags;
2173 struct bio *bio;
2174 struct bio_list bios;
2175 struct thin_c *tc;
2176
Joe Thornberb10ebd32014-04-08 11:29:01 +01002177 tc = get_first_thin(pool);
2178 while (tc) {
Joe Thornbera374bb22014-10-10 13:43:14 +01002179 process_thin_deferred_cells(tc);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002180 process_thin_deferred_bios(tc);
Joe Thornberb10ebd32014-04-08 11:29:01 +01002181 tc = get_next_thin(pool, tc);
2182 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002183
2184 /*
2185 * If there are any deferred flush bios, we must commit
2186 * the metadata before issuing them.
2187 */
2188 bio_list_init(&bios);
2189 spin_lock_irqsave(&pool->lock, flags);
2190 bio_list_merge(&bios, &pool->deferred_flush_bios);
2191 bio_list_init(&pool->deferred_flush_bios);
2192 spin_unlock_irqrestore(&pool->lock, flags);
2193
Mike Snitzer4d1662a2014-02-06 06:08:56 -05002194 if (bio_list_empty(&bios) &&
2195 !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool)))
Joe Thornber991d9fa2011-10-31 20:21:18 +00002196 return;
2197
Joe Thornber020cc3b2013-12-04 15:05:36 -05002198 if (commit(pool)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00002199 while ((bio = bio_list_pop(&bios)))
2200 bio_io_error(bio);
2201 return;
2202 }
Joe Thornber905e51b2012-03-28 18:41:27 +01002203 pool->last_commit_jiffies = jiffies;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002204
2205 while ((bio = bio_list_pop(&bios)))
2206 generic_make_request(bio);
2207}
2208
2209static void do_worker(struct work_struct *ws)
2210{
2211 struct pool *pool = container_of(ws, struct pool, worker);
2212
Joe Thornber7d327fe2014-10-06 15:45:59 +01002213 throttle_work_start(&pool->throttle);
Joe Thornber8a01a6a2014-10-06 15:28:30 +01002214 dm_pool_issue_prefetches(pool->pmd);
Joe Thornber7d327fe2014-10-06 15:45:59 +01002215 throttle_work_update(&pool->throttle);
Joe Thornbere49e5822012-07-27 15:08:16 +01002216 process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);
Joe Thornber7d327fe2014-10-06 15:45:59 +01002217 throttle_work_update(&pool->throttle);
Joe Thornbere49e5822012-07-27 15:08:16 +01002218 process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);
Joe Thornber7d327fe2014-10-06 15:45:59 +01002219 throttle_work_update(&pool->throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002220 process_deferred_bios(pool);
Joe Thornber7d327fe2014-10-06 15:45:59 +01002221 throttle_work_complete(&pool->throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002222}
2223
Joe Thornber905e51b2012-03-28 18:41:27 +01002224/*
2225 * We want to commit periodically so that not too much
2226 * unwritten data builds up.
2227 */
2228static void do_waker(struct work_struct *ws)
2229{
2230 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
2231 wake_worker(pool);
2232 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
2233}
2234
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002235static void notify_of_pool_mode_change_to_oods(struct pool *pool);
2236
Joe Thornber85ad643b2014-05-09 15:59:38 +01002237/*
2238 * We're holding onto IO to allow userland time to react. After the
2239 * timeout either the pool will have been resized (and thus back in
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002240 * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.
Joe Thornber85ad643b2014-05-09 15:59:38 +01002241 */
2242static void do_no_space_timeout(struct work_struct *ws)
2243{
2244 struct pool *pool = container_of(to_delayed_work(ws), struct pool,
2245 no_space_timeout);
2246
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002247 if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) {
2248 pool->pf.error_if_no_space = true;
2249 notify_of_pool_mode_change_to_oods(pool);
Mike Snitzer0a927c22015-07-21 13:20:46 -04002250 error_retry_list_with_code(pool, -ENOSPC);
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002251 }
Joe Thornber85ad643b2014-05-09 15:59:38 +01002252}
2253
Joe Thornber991d9fa2011-10-31 20:21:18 +00002254/*----------------------------------------------------------------*/
2255
Joe Thornbere7a3e872014-05-13 16:14:14 -04002256struct pool_work {
Joe Thornber738211f2014-03-03 15:52:28 +00002257 struct work_struct worker;
Joe Thornbere7a3e872014-05-13 16:14:14 -04002258 struct completion complete;
Joe Thornber738211f2014-03-03 15:52:28 +00002259};
2260
Joe Thornbere7a3e872014-05-13 16:14:14 -04002261static struct pool_work *to_pool_work(struct work_struct *ws)
Joe Thornber738211f2014-03-03 15:52:28 +00002262{
Joe Thornbere7a3e872014-05-13 16:14:14 -04002263 return container_of(ws, struct pool_work, worker);
2264}
2265
2266static void pool_work_complete(struct pool_work *pw)
2267{
2268 complete(&pw->complete);
2269}
2270
2271static void pool_work_wait(struct pool_work *pw, struct pool *pool,
2272 void (*fn)(struct work_struct *))
2273{
2274 INIT_WORK_ONSTACK(&pw->worker, fn);
2275 init_completion(&pw->complete);
2276 queue_work(pool->wq, &pw->worker);
2277 wait_for_completion(&pw->complete);
2278}
2279
2280/*----------------------------------------------------------------*/
2281
2282struct noflush_work {
2283 struct pool_work pw;
2284 struct thin_c *tc;
2285};
2286
2287static struct noflush_work *to_noflush(struct work_struct *ws)
2288{
2289 return container_of(to_pool_work(ws), struct noflush_work, pw);
Joe Thornber738211f2014-03-03 15:52:28 +00002290}
2291
2292static void do_noflush_start(struct work_struct *ws)
2293{
Joe Thornbere7a3e872014-05-13 16:14:14 -04002294 struct noflush_work *w = to_noflush(ws);
Joe Thornber738211f2014-03-03 15:52:28 +00002295 w->tc->requeue_mode = true;
2296 requeue_io(w->tc);
Joe Thornbere7a3e872014-05-13 16:14:14 -04002297 pool_work_complete(&w->pw);
Joe Thornber738211f2014-03-03 15:52:28 +00002298}
2299
2300static void do_noflush_stop(struct work_struct *ws)
2301{
Joe Thornbere7a3e872014-05-13 16:14:14 -04002302 struct noflush_work *w = to_noflush(ws);
Joe Thornber738211f2014-03-03 15:52:28 +00002303 w->tc->requeue_mode = false;
Joe Thornbere7a3e872014-05-13 16:14:14 -04002304 pool_work_complete(&w->pw);
Joe Thornber738211f2014-03-03 15:52:28 +00002305}
2306
2307static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
2308{
2309 struct noflush_work w;
2310
Joe Thornber738211f2014-03-03 15:52:28 +00002311 w.tc = tc;
Joe Thornbere7a3e872014-05-13 16:14:14 -04002312 pool_work_wait(&w.pw, tc->pool, fn);
Joe Thornber738211f2014-03-03 15:52:28 +00002313}
2314
2315/*----------------------------------------------------------------*/
2316
Joe Thornbere49e5822012-07-27 15:08:16 +01002317static enum pool_mode get_pool_mode(struct pool *pool)
2318{
2319 return pool->pf.mode;
2320}
2321
Joe Thornber3e1a0692014-03-03 16:03:26 +00002322static void notify_of_pool_mode_change(struct pool *pool, const char *new_mode)
2323{
2324 dm_table_event(pool->ti->table);
2325 DMINFO("%s: switching pool to %s mode",
2326 dm_device_name(pool->pool_md), new_mode);
2327}
2328
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002329static void notify_of_pool_mode_change_to_oods(struct pool *pool)
2330{
2331 if (!pool->pf.error_if_no_space)
2332 notify_of_pool_mode_change(pool, "out-of-data-space (queue IO)");
2333 else
2334 notify_of_pool_mode_change(pool, "out-of-data-space (error IO)");
2335}
2336
Joe Thornber34fbcf62015-04-16 12:58:35 +01002337static bool passdown_enabled(struct pool_c *pt)
2338{
2339 return pt->adjusted_pf.discard_passdown;
2340}
2341
2342static void set_discard_callbacks(struct pool *pool)
2343{
2344 struct pool_c *pt = pool->ti->private;
2345
2346 if (passdown_enabled(pt)) {
2347 pool->process_discard_cell = process_discard_cell_passdown;
2348 pool->process_prepared_discard = process_prepared_discard_passdown;
2349 } else {
2350 pool->process_discard_cell = process_discard_cell_no_passdown;
2351 pool->process_prepared_discard = process_prepared_discard_no_passdown;
2352 }
2353}
2354
Mike Snitzer8b64e882013-12-20 14:27:28 -05002355static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
Joe Thornbere49e5822012-07-27 15:08:16 +01002356{
Mike Snitzercdc2b412014-02-14 18:10:55 -05002357 struct pool_c *pt = pool->ti->private;
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002358 bool needs_check = dm_pool_metadata_needs_check(pool->pmd);
2359 enum pool_mode old_mode = get_pool_mode(pool);
Mike Snitzer80c57892014-05-20 13:38:33 -04002360 unsigned long no_space_timeout = ACCESS_ONCE(no_space_timeout_secs) * HZ;
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002361
2362 /*
2363 * Never allow the pool to transition to PM_WRITE mode if user
2364 * intervention is required to verify metadata and data consistency.
2365 */
2366 if (new_mode == PM_WRITE && needs_check) {
2367 DMERR("%s: unable to switch pool to write mode until repaired.",
2368 dm_device_name(pool->pool_md));
2369 if (old_mode != new_mode)
2370 new_mode = old_mode;
2371 else
2372 new_mode = PM_READ_ONLY;
2373 }
2374 /*
2375 * If we were in PM_FAIL mode, rollback of metadata failed. We're
2376 * not going to recover without a thin_repair. So we never let the
2377 * pool move out of the old mode.
2378 */
2379 if (old_mode == PM_FAIL)
2380 new_mode = old_mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01002381
Mike Snitzer8b64e882013-12-20 14:27:28 -05002382 switch (new_mode) {
Joe Thornbere49e5822012-07-27 15:08:16 +01002383 case PM_FAIL:
Mike Snitzer8b64e882013-12-20 14:27:28 -05002384 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00002385 notify_of_pool_mode_change(pool, "failure");
Joe Thornber5383ef32013-12-04 16:30:01 -05002386 dm_pool_metadata_read_only(pool->pmd);
Joe Thornbere49e5822012-07-27 15:08:16 +01002387 pool->process_bio = process_bio_fail;
2388 pool->process_discard = process_bio_fail;
Joe Thornbera374bb22014-10-10 13:43:14 +01002389 pool->process_cell = process_cell_fail;
2390 pool->process_discard_cell = process_cell_fail;
Joe Thornbere49e5822012-07-27 15:08:16 +01002391 pool->process_prepared_mapping = process_prepared_mapping_fail;
2392 pool->process_prepared_discard = process_prepared_discard_fail;
Joe Thornber3e1a0692014-03-03 16:03:26 +00002393
2394 error_retry_list(pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01002395 break;
2396
2397 case PM_READ_ONLY:
Mike Snitzer8b64e882013-12-20 14:27:28 -05002398 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00002399 notify_of_pool_mode_change(pool, "read-only");
2400 dm_pool_metadata_read_only(pool->pmd);
2401 pool->process_bio = process_bio_read_only;
2402 pool->process_discard = process_bio_success;
Joe Thornbera374bb22014-10-10 13:43:14 +01002403 pool->process_cell = process_cell_read_only;
2404 pool->process_discard_cell = process_cell_success;
Joe Thornber3e1a0692014-03-03 16:03:26 +00002405 pool->process_prepared_mapping = process_prepared_mapping_fail;
Joe Thornber34fbcf62015-04-16 12:58:35 +01002406 pool->process_prepared_discard = process_prepared_discard_success;
Joe Thornber3e1a0692014-03-03 16:03:26 +00002407
2408 error_retry_list(pool);
2409 break;
2410
2411 case PM_OUT_OF_DATA_SPACE:
2412 /*
2413 * Ideally we'd never hit this state; the low water mark
2414 * would trigger userland to extend the pool before we
2415 * completely run out of data space. However, many small
2416 * IOs to unprovisioned space can consume data space at an
2417 * alarming rate. Adjust your low water mark if you're
2418 * frequently seeing this mode.
2419 */
2420 if (old_mode != new_mode)
Mike Snitzerbcc696f2015-07-15 16:52:04 -04002421 notify_of_pool_mode_change_to_oods(pool);
Joe Thornber3e1a0692014-03-03 16:03:26 +00002422 pool->process_bio = process_bio_read_only;
Joe Thornbera374bb22014-10-10 13:43:14 +01002423 pool->process_discard = process_discard_bio;
2424 pool->process_cell = process_cell_read_only;
Joe Thornber3e1a0692014-03-03 16:03:26 +00002425 pool->process_prepared_mapping = process_prepared_mapping;
Joe Thornber34fbcf62015-04-16 12:58:35 +01002426 set_discard_callbacks(pool);
Joe Thornber85ad643b2014-05-09 15:59:38 +01002427
Mike Snitzer80c57892014-05-20 13:38:33 -04002428 if (!pool->pf.error_if_no_space && no_space_timeout)
2429 queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);
Joe Thornbere49e5822012-07-27 15:08:16 +01002430 break;
2431
2432 case PM_WRITE:
Mike Snitzer8b64e882013-12-20 14:27:28 -05002433 if (old_mode != new_mode)
Joe Thornber3e1a0692014-03-03 16:03:26 +00002434 notify_of_pool_mode_change(pool, "write");
Mike Snitzer172c2382015-11-06 10:53:01 -05002435 pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space;
Joe Thornber9b7aaa62013-12-04 16:58:19 -05002436 dm_pool_metadata_read_write(pool->pmd);
Joe Thornbere49e5822012-07-27 15:08:16 +01002437 pool->process_bio = process_bio;
Joe Thornbera374bb22014-10-10 13:43:14 +01002438 pool->process_discard = process_discard_bio;
2439 pool->process_cell = process_cell;
Joe Thornbere49e5822012-07-27 15:08:16 +01002440 pool->process_prepared_mapping = process_prepared_mapping;
Joe Thornber34fbcf62015-04-16 12:58:35 +01002441 set_discard_callbacks(pool);
Joe Thornbere49e5822012-07-27 15:08:16 +01002442 break;
2443 }
Mike Snitzer8b64e882013-12-20 14:27:28 -05002444
2445 pool->pf.mode = new_mode;
Mike Snitzercdc2b412014-02-14 18:10:55 -05002446 /*
2447 * The pool mode may have changed, sync it so bind_control_target()
2448 * doesn't cause an unexpected mode transition on resume.
2449 */
2450 pt->adjusted_pf.mode = new_mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01002451}
2452
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002453static void abort_transaction(struct pool *pool)
2454{
2455 const char *dev_name = dm_device_name(pool->pool_md);
2456
2457 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
2458 if (dm_pool_abort_metadata(pool->pmd)) {
2459 DMERR("%s: failed to abort metadata transaction", dev_name);
2460 set_pool_mode(pool, PM_FAIL);
2461 }
2462
2463 if (dm_pool_metadata_set_needs_check(pool->pmd)) {
2464 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
2465 set_pool_mode(pool, PM_FAIL);
2466 }
2467}
2468
Joe Thornberb5330652013-12-04 19:51:33 -05002469static void metadata_operation_failed(struct pool *pool, const char *op, int r)
2470{
2471 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
2472 dm_device_name(pool->pool_md), op, r);
2473
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002474 abort_transaction(pool);
Joe Thornberb5330652013-12-04 19:51:33 -05002475 set_pool_mode(pool, PM_READ_ONLY);
2476}
2477
Joe Thornbere49e5822012-07-27 15:08:16 +01002478/*----------------------------------------------------------------*/
2479
Joe Thornber991d9fa2011-10-31 20:21:18 +00002480/*
2481 * Mapping functions.
2482 */
2483
2484/*
2485 * Called only while mapping a thin bio to hand it over to the workqueue.
2486 */
2487static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
2488{
2489 unsigned long flags;
2490 struct pool *pool = tc->pool;
2491
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002492 spin_lock_irqsave(&tc->lock, flags);
2493 bio_list_add(&tc->deferred_bio_list, bio);
2494 spin_unlock_irqrestore(&tc->lock, flags);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002495
2496 wake_worker(pool);
2497}
2498
Joe Thornber7d327fe2014-10-06 15:45:59 +01002499static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio)
2500{
2501 struct pool *pool = tc->pool;
2502
2503 throttle_lock(&pool->throttle);
2504 thin_defer_bio(tc, bio);
2505 throttle_unlock(&pool->throttle);
2506}
2507
Joe Thornbera374bb22014-10-10 13:43:14 +01002508static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2509{
2510 unsigned long flags;
2511 struct pool *pool = tc->pool;
2512
2513 throttle_lock(&pool->throttle);
2514 spin_lock_irqsave(&tc->lock, flags);
2515 list_add_tail(&cell->user_list, &tc->deferred_cells);
2516 spin_unlock_irqrestore(&tc->lock, flags);
2517 throttle_unlock(&pool->throttle);
2518
2519 wake_worker(pool);
2520}
2521
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00002522static void thin_hook_bio(struct thin_c *tc, struct bio *bio)
Joe Thornbereb2aa482012-03-28 18:41:28 +01002523{
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00002524 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01002525
2526 h->tc = tc;
2527 h->shared_read_entry = NULL;
Joe Thornbere8088072012-12-21 20:23:31 +00002528 h->all_io_entry = NULL;
Joe Thornbereb2aa482012-03-28 18:41:28 +01002529 h->overwrite_mapping = NULL;
Joe Thornber34fbcf62015-04-16 12:58:35 +01002530 h->cell = NULL;
Joe Thornbereb2aa482012-03-28 18:41:28 +01002531}
2532
Joe Thornber991d9fa2011-10-31 20:21:18 +00002533/*
2534 * Non-blocking function called from the thin target's map function.
2535 */
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00002536static int thin_bio_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002537{
2538 int r;
2539 struct thin_c *tc = ti->private;
2540 dm_block_t block = get_bio_block(tc, bio);
2541 struct dm_thin_device *td = tc->td;
2542 struct dm_thin_lookup_result result;
Joe Thornbera374bb22014-10-10 13:43:14 +01002543 struct dm_bio_prison_cell *virt_cell, *data_cell;
Joe Thornbere8088072012-12-21 20:23:31 +00002544 struct dm_cell_key key;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002545
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00002546 thin_hook_bio(tc, bio);
Joe Thornbere49e5822012-07-27 15:08:16 +01002547
Joe Thornber738211f2014-03-03 15:52:28 +00002548 if (tc->requeue_mode) {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002549 bio->bi_error = DM_ENDIO_REQUEUE;
2550 bio_endio(bio);
Joe Thornber738211f2014-03-03 15:52:28 +00002551 return DM_MAPIO_SUBMITTED;
2552 }
2553
Joe Thornbere49e5822012-07-27 15:08:16 +01002554 if (get_pool_mode(tc->pool) == PM_FAIL) {
2555 bio_io_error(bio);
2556 return DM_MAPIO_SUBMITTED;
2557 }
2558
Joe Thornber104655f2012-03-28 18:41:28 +01002559 if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) {
Joe Thornber7d327fe2014-10-06 15:45:59 +01002560 thin_defer_bio_with_throttle(tc, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002561 return DM_MAPIO_SUBMITTED;
2562 }
2563
Joe Thornberc822ed92014-10-10 09:41:09 +01002564 /*
2565 * We must hold the virtual cell before doing the lookup, otherwise
2566 * there's a race with discard.
2567 */
2568 build_virtual_key(tc->td, block, &key);
Joe Thornbera374bb22014-10-10 13:43:14 +01002569 if (bio_detain(tc->pool, &key, bio, &virt_cell))
Joe Thornberc822ed92014-10-10 09:41:09 +01002570 return DM_MAPIO_SUBMITTED;
2571
Joe Thornber991d9fa2011-10-31 20:21:18 +00002572 r = dm_thin_find_block(td, block, 0, &result);
2573
2574 /*
2575 * Note that we defer readahead too.
2576 */
2577 switch (r) {
2578 case 0:
2579 if (unlikely(result.shared)) {
2580 /*
2581 * We have a race condition here between the
2582 * result.shared value returned by the lookup and
2583 * snapshot creation, which may cause new
2584 * sharing.
2585 *
2586 * To avoid this always quiesce the origin before
2587 * taking the snap. You want to do this anyway to
2588 * ensure a consistent application view
2589 * (i.e. lockfs).
2590 *
2591 * More distant ancestors are irrelevant. The
2592 * shared flag will be set in their case.
2593 */
Joe Thornbera374bb22014-10-10 13:43:14 +01002594 thin_defer_cell(tc, virt_cell);
Joe Thornbere8088072012-12-21 20:23:31 +00002595 return DM_MAPIO_SUBMITTED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002596 }
Joe Thornbere8088072012-12-21 20:23:31 +00002597
Joe Thornbere8088072012-12-21 20:23:31 +00002598 build_data_key(tc->td, result.block, &key);
Joe Thornbera374bb22014-10-10 13:43:14 +01002599 if (bio_detain(tc->pool, &key, bio, &data_cell)) {
2600 cell_defer_no_holder(tc, virt_cell);
Joe Thornbere8088072012-12-21 20:23:31 +00002601 return DM_MAPIO_SUBMITTED;
2602 }
2603
2604 inc_all_io_entry(tc->pool, bio);
Joe Thornbera374bb22014-10-10 13:43:14 +01002605 cell_defer_no_holder(tc, data_cell);
2606 cell_defer_no_holder(tc, virt_cell);
Joe Thornbere8088072012-12-21 20:23:31 +00002607
2608 remap(tc, bio, result.block);
2609 return DM_MAPIO_REMAPPED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002610
2611 case -ENODATA:
Joe Thornbere49e5822012-07-27 15:08:16 +01002612 case -EWOULDBLOCK:
Joe Thornbera374bb22014-10-10 13:43:14 +01002613 thin_defer_cell(tc, virt_cell);
Joe Thornber2aab3852012-12-21 20:23:33 +00002614 return DM_MAPIO_SUBMITTED;
Joe Thornbere49e5822012-07-27 15:08:16 +01002615
2616 default:
2617 /*
2618 * Must always call bio_io_error on failure.
2619 * dm_thin_find_block can fail with -EINVAL if the
2620 * pool is switched to fail-io mode.
2621 */
2622 bio_io_error(bio);
Joe Thornbera374bb22014-10-10 13:43:14 +01002623 cell_defer_no_holder(tc, virt_cell);
Joe Thornber2aab3852012-12-21 20:23:33 +00002624 return DM_MAPIO_SUBMITTED;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002625 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002626}
2627
2628static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
2629{
Joe Thornber991d9fa2011-10-31 20:21:18 +00002630 struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
Mike Snitzer760fe672014-03-20 08:36:47 -04002631 struct request_queue *q;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002632
Mike Snitzer760fe672014-03-20 08:36:47 -04002633 if (get_pool_mode(pt->pool) == PM_OUT_OF_DATA_SPACE)
2634 return 1;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002635
Mike Snitzer760fe672014-03-20 08:36:47 -04002636 q = bdev_get_queue(pt->data_dev->bdev);
2637 return bdi_congested(&q->backing_dev_info, bdi_bits);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002638}
2639
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002640static void requeue_bios(struct pool *pool)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002641{
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002642 unsigned long flags;
2643 struct thin_c *tc;
2644
2645 rcu_read_lock();
2646 list_for_each_entry_rcu(tc, &pool->active_thins, list) {
2647 spin_lock_irqsave(&tc->lock, flags);
2648 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list);
2649 bio_list_init(&tc->retry_on_resume_list);
2650 spin_unlock_irqrestore(&tc->lock, flags);
2651 }
2652 rcu_read_unlock();
Joe Thornber991d9fa2011-10-31 20:21:18 +00002653}
2654
2655/*----------------------------------------------------------------
2656 * Binding of control targets to a pool object
2657 *--------------------------------------------------------------*/
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002658static bool data_dev_supports_discard(struct pool_c *pt)
2659{
2660 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2661
2662 return q && blk_queue_discard(q);
2663}
2664
Joe Thornber58051b92013-03-20 17:21:25 +00002665static bool is_factor(sector_t block_size, uint32_t n)
2666{
2667 return !sector_div(block_size, n);
2668}
2669
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002670/*
2671 * If discard_passdown was enabled verify that the data device
Mike Snitzer0424caa2012-09-26 23:45:47 +01002672 * supports discards. Disable discard_passdown if not.
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002673 */
Mike Snitzer0424caa2012-09-26 23:45:47 +01002674static void disable_passdown_if_not_supported(struct pool_c *pt)
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002675{
Mike Snitzer0424caa2012-09-26 23:45:47 +01002676 struct pool *pool = pt->pool;
2677 struct block_device *data_bdev = pt->data_dev->bdev;
2678 struct queue_limits *data_limits = &bdev_get_queue(data_bdev)->limits;
Mike Snitzer0424caa2012-09-26 23:45:47 +01002679 const char *reason = NULL;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002680 char buf[BDEVNAME_SIZE];
2681
Mike Snitzer0424caa2012-09-26 23:45:47 +01002682 if (!pt->adjusted_pf.discard_passdown)
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002683 return;
2684
Mike Snitzer0424caa2012-09-26 23:45:47 +01002685 if (!data_dev_supports_discard(pt))
2686 reason = "discard unsupported";
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002687
Mike Snitzer0424caa2012-09-26 23:45:47 +01002688 else if (data_limits->max_discard_sectors < pool->sectors_per_block)
2689 reason = "max discard sectors smaller than a block";
2690
Mike Snitzer0424caa2012-09-26 23:45:47 +01002691 if (reason) {
2692 DMWARN("Data device (%s) %s: Disabling discard passdown.", bdevname(data_bdev, buf), reason);
2693 pt->adjusted_pf.discard_passdown = false;
2694 }
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002695}
2696
Joe Thornber991d9fa2011-10-31 20:21:18 +00002697static int bind_control_target(struct pool *pool, struct dm_target *ti)
2698{
2699 struct pool_c *pt = ti->private;
2700
Joe Thornbere49e5822012-07-27 15:08:16 +01002701 /*
Joe Thornber9b7aaa62013-12-04 16:58:19 -05002702 * We want to make sure that a pool in PM_FAIL mode is never upgraded.
Joe Thornbere49e5822012-07-27 15:08:16 +01002703 */
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05002704 enum pool_mode old_mode = get_pool_mode(pool);
Mike Snitzer0424caa2012-09-26 23:45:47 +01002705 enum pool_mode new_mode = pt->adjusted_pf.mode;
Joe Thornbere49e5822012-07-27 15:08:16 +01002706
Joe Thornber9b7aaa62013-12-04 16:58:19 -05002707 /*
Mike Snitzer8b64e882013-12-20 14:27:28 -05002708 * Don't change the pool's mode until set_pool_mode() below.
2709 * Otherwise the pool's process_* function pointers may
2710 * not match the desired pool mode.
2711 */
2712 pt->adjusted_pf.mode = old_mode;
2713
2714 pool->ti = ti;
2715 pool->pf = pt->adjusted_pf;
2716 pool->low_water_blocks = pt->low_water_blocks;
2717
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002718 set_pool_mode(pool, new_mode);
Mike Snitzerf4026932012-05-19 01:01:01 +01002719
Joe Thornber991d9fa2011-10-31 20:21:18 +00002720 return 0;
2721}
2722
2723static void unbind_control_target(struct pool *pool, struct dm_target *ti)
2724{
2725 if (pool->ti == ti)
2726 pool->ti = NULL;
2727}
2728
2729/*----------------------------------------------------------------
2730 * Pool creation
2731 *--------------------------------------------------------------*/
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002732/* Initialize pool features. */
2733static void pool_features_init(struct pool_features *pf)
2734{
Joe Thornbere49e5822012-07-27 15:08:16 +01002735 pf->mode = PM_WRITE;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002736 pf->zero_new_blocks = true;
2737 pf->discard_enabled = true;
2738 pf->discard_passdown = true;
Mike Snitzer787a996c2013-12-06 16:21:43 -05002739 pf->error_if_no_space = false;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002740}
2741
Joe Thornber991d9fa2011-10-31 20:21:18 +00002742static void __pool_destroy(struct pool *pool)
2743{
2744 __pool_table_remove(pool);
2745
Joe Thornbera822c832015-07-03 10:22:42 +01002746 vfree(pool->cell_sort_array);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002747 if (dm_pool_metadata_close(pool->pmd) < 0)
2748 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2749
Mike Snitzer44feb382012-10-12 21:02:10 +01002750 dm_bio_prison_destroy(pool->prison);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002751 dm_kcopyd_client_destroy(pool->copier);
2752
2753 if (pool->wq)
2754 destroy_workqueue(pool->wq);
2755
2756 if (pool->next_mapping)
2757 mempool_free(pool->next_mapping, pool->mapping_pool);
2758 mempool_destroy(pool->mapping_pool);
Mike Snitzer44feb382012-10-12 21:02:10 +01002759 dm_deferred_set_destroy(pool->shared_read_ds);
2760 dm_deferred_set_destroy(pool->all_io_ds);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002761 kfree(pool);
2762}
2763
Mike Snitzera24c2562012-06-03 00:30:00 +01002764static struct kmem_cache *_new_mapping_cache;
Mike Snitzera24c2562012-06-03 00:30:00 +01002765
Joe Thornber991d9fa2011-10-31 20:21:18 +00002766static struct pool *pool_create(struct mapped_device *pool_md,
2767 struct block_device *metadata_dev,
Joe Thornbere49e5822012-07-27 15:08:16 +01002768 unsigned long block_size,
2769 int read_only, char **error)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002770{
2771 int r;
2772 void *err_p;
2773 struct pool *pool;
2774 struct dm_pool_metadata *pmd;
Joe Thornbere49e5822012-07-27 15:08:16 +01002775 bool format_device = read_only ? false : true;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002776
Joe Thornbere49e5822012-07-27 15:08:16 +01002777 pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002778 if (IS_ERR(pmd)) {
2779 *error = "Error creating metadata object";
2780 return (struct pool *)pmd;
2781 }
2782
2783 pool = kmalloc(sizeof(*pool), GFP_KERNEL);
2784 if (!pool) {
2785 *error = "Error allocating memory for pool";
2786 err_p = ERR_PTR(-ENOMEM);
2787 goto bad_pool;
2788 }
2789
2790 pool->pmd = pmd;
2791 pool->sectors_per_block = block_size;
Mikulas Patockaf9a8e0c2012-07-27 15:08:03 +01002792 if (block_size & (block_size - 1))
2793 pool->sectors_per_block_shift = -1;
2794 else
2795 pool->sectors_per_block_shift = __ffs(block_size);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002796 pool->low_water_blocks = 0;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002797 pool_features_init(&pool->pf);
Joe Thornbera195db22014-10-06 16:30:06 -04002798 pool->prison = dm_bio_prison_create();
Joe Thornber991d9fa2011-10-31 20:21:18 +00002799 if (!pool->prison) {
2800 *error = "Error creating pool's bio prison";
2801 err_p = ERR_PTR(-ENOMEM);
2802 goto bad_prison;
2803 }
2804
Mikulas Patockadf5d2e92013-03-01 22:45:49 +00002805 pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002806 if (IS_ERR(pool->copier)) {
2807 r = PTR_ERR(pool->copier);
2808 *error = "Error creating pool's kcopyd client";
2809 err_p = ERR_PTR(r);
2810 goto bad_kcopyd_client;
2811 }
2812
2813 /*
2814 * Create singlethreaded workqueue that will service all devices
2815 * that use this metadata.
2816 */
2817 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
2818 if (!pool->wq) {
2819 *error = "Error creating pool's workqueue";
2820 err_p = ERR_PTR(-ENOMEM);
2821 goto bad_wq;
2822 }
2823
Joe Thornber7d327fe2014-10-06 15:45:59 +01002824 throttle_init(&pool->throttle);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002825 INIT_WORK(&pool->worker, do_worker);
Joe Thornber905e51b2012-03-28 18:41:27 +01002826 INIT_DELAYED_WORK(&pool->waker, do_waker);
Joe Thornber85ad643b2014-05-09 15:59:38 +01002827 INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002828 spin_lock_init(&pool->lock);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002829 bio_list_init(&pool->deferred_flush_bios);
2830 INIT_LIST_HEAD(&pool->prepared_mappings);
Joe Thornber104655f2012-03-28 18:41:28 +01002831 INIT_LIST_HEAD(&pool->prepared_discards);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04002832 INIT_LIST_HEAD(&pool->active_thins);
Joe Thornber88a66212013-12-04 20:16:12 -05002833 pool->low_water_triggered = false;
Mike Snitzer80e96c52014-11-07 15:09:46 -05002834 pool->suspended = true;
Mike Snitzer44feb382012-10-12 21:02:10 +01002835
2836 pool->shared_read_ds = dm_deferred_set_create();
2837 if (!pool->shared_read_ds) {
2838 *error = "Error creating pool's shared read deferred set";
2839 err_p = ERR_PTR(-ENOMEM);
2840 goto bad_shared_read_ds;
2841 }
2842
2843 pool->all_io_ds = dm_deferred_set_create();
2844 if (!pool->all_io_ds) {
2845 *error = "Error creating pool's all io deferred set";
2846 err_p = ERR_PTR(-ENOMEM);
2847 goto bad_all_io_ds;
2848 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002849
2850 pool->next_mapping = NULL;
Mike Snitzera24c2562012-06-03 00:30:00 +01002851 pool->mapping_pool = mempool_create_slab_pool(MAPPING_POOL_SIZE,
2852 _new_mapping_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002853 if (!pool->mapping_pool) {
2854 *error = "Error creating pool's mapping mempool";
2855 err_p = ERR_PTR(-ENOMEM);
2856 goto bad_mapping_pool;
2857 }
2858
Joe Thornbera822c832015-07-03 10:22:42 +01002859 pool->cell_sort_array = vmalloc(sizeof(*pool->cell_sort_array) * CELL_SORT_ARRAY_SIZE);
2860 if (!pool->cell_sort_array) {
2861 *error = "Error allocating cell sort array";
2862 err_p = ERR_PTR(-ENOMEM);
2863 goto bad_sort_array;
2864 }
2865
Joe Thornber991d9fa2011-10-31 20:21:18 +00002866 pool->ref_count = 1;
Joe Thornber905e51b2012-03-28 18:41:27 +01002867 pool->last_commit_jiffies = jiffies;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002868 pool->pool_md = pool_md;
2869 pool->md_dev = metadata_dev;
2870 __pool_table_insert(pool);
2871
2872 return pool;
2873
Joe Thornbera822c832015-07-03 10:22:42 +01002874bad_sort_array:
2875 mempool_destroy(pool->mapping_pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002876bad_mapping_pool:
Mike Snitzer44feb382012-10-12 21:02:10 +01002877 dm_deferred_set_destroy(pool->all_io_ds);
2878bad_all_io_ds:
2879 dm_deferred_set_destroy(pool->shared_read_ds);
2880bad_shared_read_ds:
Joe Thornber991d9fa2011-10-31 20:21:18 +00002881 destroy_workqueue(pool->wq);
2882bad_wq:
2883 dm_kcopyd_client_destroy(pool->copier);
2884bad_kcopyd_client:
Mike Snitzer44feb382012-10-12 21:02:10 +01002885 dm_bio_prison_destroy(pool->prison);
Joe Thornber991d9fa2011-10-31 20:21:18 +00002886bad_prison:
2887 kfree(pool);
2888bad_pool:
2889 if (dm_pool_metadata_close(pmd))
2890 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2891
2892 return err_p;
2893}
2894
2895static void __pool_inc(struct pool *pool)
2896{
2897 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
2898 pool->ref_count++;
2899}
2900
2901static void __pool_dec(struct pool *pool)
2902{
2903 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
2904 BUG_ON(!pool->ref_count);
2905 if (!--pool->ref_count)
2906 __pool_destroy(pool);
2907}
2908
2909static struct pool *__pool_find(struct mapped_device *pool_md,
2910 struct block_device *metadata_dev,
Joe Thornbere49e5822012-07-27 15:08:16 +01002911 unsigned long block_size, int read_only,
2912 char **error, int *created)
Joe Thornber991d9fa2011-10-31 20:21:18 +00002913{
2914 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
2915
2916 if (pool) {
Mike Snitzerf09996c2012-07-27 15:07:59 +01002917 if (pool->pool_md != pool_md) {
2918 *error = "metadata device already in use by a pool";
Joe Thornber991d9fa2011-10-31 20:21:18 +00002919 return ERR_PTR(-EBUSY);
Mike Snitzerf09996c2012-07-27 15:07:59 +01002920 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002921 __pool_inc(pool);
2922
2923 } else {
2924 pool = __pool_table_lookup(pool_md);
2925 if (pool) {
Mike Snitzerf09996c2012-07-27 15:07:59 +01002926 if (pool->md_dev != metadata_dev) {
2927 *error = "different pool cannot replace a pool";
Joe Thornber991d9fa2011-10-31 20:21:18 +00002928 return ERR_PTR(-EINVAL);
Mike Snitzerf09996c2012-07-27 15:07:59 +01002929 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002930 __pool_inc(pool);
2931
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002932 } else {
Joe Thornbere49e5822012-07-27 15:08:16 +01002933 pool = pool_create(pool_md, metadata_dev, block_size, read_only, error);
Joe Thornber67e2e2b2012-03-28 18:41:29 +01002934 *created = 1;
2935 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00002936 }
2937
2938 return pool;
2939}
2940
2941/*----------------------------------------------------------------
2942 * Pool target methods
2943 *--------------------------------------------------------------*/
2944static void pool_dtr(struct dm_target *ti)
2945{
2946 struct pool_c *pt = ti->private;
2947
2948 mutex_lock(&dm_thin_pool_table.mutex);
2949
2950 unbind_control_target(pt->pool, ti);
2951 __pool_dec(pt->pool);
2952 dm_put_device(ti, pt->metadata_dev);
2953 dm_put_device(ti, pt->data_dev);
2954 kfree(pt);
2955
2956 mutex_unlock(&dm_thin_pool_table.mutex);
2957}
2958
Joe Thornber991d9fa2011-10-31 20:21:18 +00002959static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
2960 struct dm_target *ti)
2961{
2962 int r;
2963 unsigned argc;
2964 const char *arg_name;
2965
2966 static struct dm_arg _args[] = {
Mike Snitzer74aa45c2014-01-15 19:07:58 -05002967 {0, 4, "Invalid number of pool feature arguments"},
Joe Thornber991d9fa2011-10-31 20:21:18 +00002968 };
2969
2970 /*
2971 * No feature arguments supplied.
2972 */
2973 if (!as->argc)
2974 return 0;
2975
2976 r = dm_read_arg_group(_args, as, &argc, &ti->error);
2977 if (r)
2978 return -EINVAL;
2979
2980 while (argc && !r) {
2981 arg_name = dm_shift_arg(as);
2982 argc--;
2983
Joe Thornbere49e5822012-07-27 15:08:16 +01002984 if (!strcasecmp(arg_name, "skip_block_zeroing"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002985 pf->zero_new_blocks = false;
Joe Thornber991d9fa2011-10-31 20:21:18 +00002986
Joe Thornbere49e5822012-07-27 15:08:16 +01002987 else if (!strcasecmp(arg_name, "ignore_discard"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002988 pf->discard_enabled = false;
Joe Thornbere49e5822012-07-27 15:08:16 +01002989
2990 else if (!strcasecmp(arg_name, "no_discard_passdown"))
Mike Snitzer9bc142d2012-09-26 23:45:46 +01002991 pf->discard_passdown = false;
Joe Thornbere49e5822012-07-27 15:08:16 +01002992
2993 else if (!strcasecmp(arg_name, "read_only"))
2994 pf->mode = PM_READ_ONLY;
2995
Mike Snitzer787a996c2013-12-06 16:21:43 -05002996 else if (!strcasecmp(arg_name, "error_if_no_space"))
2997 pf->error_if_no_space = true;
2998
Joe Thornbere49e5822012-07-27 15:08:16 +01002999 else {
3000 ti->error = "Unrecognised pool feature requested";
3001 r = -EINVAL;
3002 break;
3003 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003004 }
3005
3006 return r;
3007}
3008
Joe Thornberac8c3f32013-05-10 14:37:21 +01003009static void metadata_low_callback(void *context)
3010{
3011 struct pool *pool = context;
3012
3013 DMWARN("%s: reached low water mark for metadata device: sending event.",
3014 dm_device_name(pool->pool_md));
3015
3016 dm_table_event(pool->ti->table);
3017}
3018
Mike Snitzer7d489352014-02-12 23:58:15 -05003019static sector_t get_dev_size(struct block_device *bdev)
Joe Thornberb17446d2013-05-10 14:37:18 +01003020{
Mike Snitzer7d489352014-02-12 23:58:15 -05003021 return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
3022}
3023
3024static void warn_if_metadata_device_too_big(struct block_device *bdev)
3025{
3026 sector_t metadata_dev_size = get_dev_size(bdev);
Joe Thornberb17446d2013-05-10 14:37:18 +01003027 char buffer[BDEVNAME_SIZE];
3028
Mike Snitzer7d489352014-02-12 23:58:15 -05003029 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
Joe Thornberb17446d2013-05-10 14:37:18 +01003030 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
3031 bdevname(bdev, buffer), THIN_METADATA_MAX_SECTORS);
Mike Snitzer7d489352014-02-12 23:58:15 -05003032}
3033
3034static sector_t get_metadata_dev_size(struct block_device *bdev)
3035{
3036 sector_t metadata_dev_size = get_dev_size(bdev);
3037
3038 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)
3039 metadata_dev_size = THIN_METADATA_MAX_SECTORS;
Joe Thornberb17446d2013-05-10 14:37:18 +01003040
3041 return metadata_dev_size;
3042}
3043
Joe Thornber24347e92013-05-10 14:37:19 +01003044static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)
3045{
3046 sector_t metadata_dev_size = get_metadata_dev_size(bdev);
3047
Mike Snitzer7d489352014-02-12 23:58:15 -05003048 sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);
Joe Thornber24347e92013-05-10 14:37:19 +01003049
3050 return metadata_dev_size;
3051}
3052
Joe Thornber991d9fa2011-10-31 20:21:18 +00003053/*
Joe Thornberac8c3f32013-05-10 14:37:21 +01003054 * When a metadata threshold is crossed a dm event is triggered, and
3055 * userland should respond by growing the metadata device. We could let
3056 * userland set the threshold, like we do with the data threshold, but I'm
3057 * not sure they know enough to do this well.
3058 */
3059static dm_block_t calc_metadata_threshold(struct pool_c *pt)
3060{
3061 /*
3062 * 4M is ample for all ops with the possible exception of thin
3063 * device deletion which is harmless if it fails (just retry the
3064 * delete after you've grown the device).
3065 */
3066 dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4;
3067 return min((dm_block_t)1024ULL /* 4M */, quarter);
3068}
3069
3070/*
Joe Thornber991d9fa2011-10-31 20:21:18 +00003071 * thin-pool <metadata dev> <data dev>
3072 * <data block size (sectors)>
3073 * <low water mark (blocks)>
3074 * [<#feature args> [<arg>]*]
3075 *
3076 * Optional feature arguments are:
3077 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003078 * ignore_discard: disable discard
3079 * no_discard_passdown: don't pass discards down to the data device
Mike Snitzer787a996c2013-12-06 16:21:43 -05003080 * read_only: Don't allow any changes to be made to the pool metadata.
3081 * error_if_no_space: error IOs, instead of queueing, if no space.
Joe Thornber991d9fa2011-10-31 20:21:18 +00003082 */
3083static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
3084{
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003085 int r, pool_created = 0;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003086 struct pool_c *pt;
3087 struct pool *pool;
3088 struct pool_features pf;
3089 struct dm_arg_set as;
3090 struct dm_dev *data_dev;
3091 unsigned long block_size;
3092 dm_block_t low_water_blocks;
3093 struct dm_dev *metadata_dev;
Joe Thornber5d0db962013-05-10 14:37:19 +01003094 fmode_t metadata_mode;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003095
3096 /*
3097 * FIXME Remove validation from scope of lock.
3098 */
3099 mutex_lock(&dm_thin_pool_table.mutex);
3100
3101 if (argc < 4) {
3102 ti->error = "Invalid argument count";
3103 r = -EINVAL;
3104 goto out_unlock;
3105 }
Joe Thornber5d0db962013-05-10 14:37:19 +01003106
Joe Thornber991d9fa2011-10-31 20:21:18 +00003107 as.argc = argc;
3108 as.argv = argv;
3109
Joe Thornber5d0db962013-05-10 14:37:19 +01003110 /*
3111 * Set default pool features.
3112 */
3113 pool_features_init(&pf);
3114
3115 dm_consume_args(&as, 4);
3116 r = parse_pool_features(&as, &pf, ti);
3117 if (r)
3118 goto out_unlock;
3119
3120 metadata_mode = FMODE_READ | ((pf.mode == PM_READ_ONLY) ? 0 : FMODE_WRITE);
3121 r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003122 if (r) {
3123 ti->error = "Error opening metadata block device";
3124 goto out_unlock;
3125 }
Mike Snitzer7d489352014-02-12 23:58:15 -05003126 warn_if_metadata_device_too_big(metadata_dev->bdev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003127
3128 r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
3129 if (r) {
3130 ti->error = "Error getting data device";
3131 goto out_metadata;
3132 }
3133
3134 if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
3135 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
3136 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01003137 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00003138 ti->error = "Invalid block size";
3139 r = -EINVAL;
3140 goto out;
3141 }
3142
3143 if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
3144 ti->error = "Invalid low water mark";
3145 r = -EINVAL;
3146 goto out;
3147 }
3148
Joe Thornber991d9fa2011-10-31 20:21:18 +00003149 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
3150 if (!pt) {
3151 r = -ENOMEM;
3152 goto out;
3153 }
3154
3155 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev,
Joe Thornbere49e5822012-07-27 15:08:16 +01003156 block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003157 if (IS_ERR(pool)) {
3158 r = PTR_ERR(pool);
3159 goto out_free_pt;
3160 }
3161
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003162 /*
3163 * 'pool_created' reflects whether this is the first table load.
3164 * Top level discard support is not allowed to be changed after
3165 * initial load. This would require a pool reload to trigger thin
3166 * device changes.
3167 */
3168 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
3169 ti->error = "Discard support cannot be disabled once enabled";
3170 r = -EINVAL;
3171 goto out_flags_changed;
3172 }
3173
Joe Thornber991d9fa2011-10-31 20:21:18 +00003174 pt->pool = pool;
3175 pt->ti = ti;
3176 pt->metadata_dev = metadata_dev;
3177 pt->data_dev = data_dev;
3178 pt->low_water_blocks = low_water_blocks;
Mike Snitzer0424caa2012-09-26 23:45:47 +01003179 pt->adjusted_pf = pt->requested_pf = pf;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00003180 ti->num_flush_bios = 1;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01003181
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003182 /*
3183 * Only need to enable discards if the pool should pass
3184 * them down to the data device. The thin device's discard
3185 * processing will cause mappings to be removed from the btree.
3186 */
Mike Snitzerb60ab992013-09-19 18:49:11 -04003187 ti->discard_zeroes_data_unsupported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003188 if (pf.discard_enabled && pf.discard_passdown) {
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00003189 ti->num_discard_bios = 1;
Mike Snitzer9bc142d2012-09-26 23:45:46 +01003190
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003191 /*
3192 * Setting 'discards_supported' circumvents the normal
3193 * stacking of discard limits (this keeps the pool and
3194 * thin devices' discard limits consistent).
3195 */
Alasdair G Kergon0ac55482012-07-27 15:08:08 +01003196 ti->discards_supported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003197 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003198 ti->private = pt;
3199
Joe Thornberac8c3f32013-05-10 14:37:21 +01003200 r = dm_pool_register_metadata_threshold(pt->pool->pmd,
3201 calc_metadata_threshold(pt),
3202 metadata_low_callback,
3203 pool);
3204 if (r)
Mike Snitzerba306702015-10-13 12:04:28 -04003205 goto out_flags_changed;
Joe Thornberac8c3f32013-05-10 14:37:21 +01003206
Joe Thornber991d9fa2011-10-31 20:21:18 +00003207 pt->callbacks.congested_fn = pool_is_congested;
3208 dm_table_add_target_callbacks(ti->table, &pt->callbacks);
3209
3210 mutex_unlock(&dm_thin_pool_table.mutex);
3211
3212 return 0;
3213
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003214out_flags_changed:
3215 __pool_dec(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003216out_free_pt:
3217 kfree(pt);
3218out:
3219 dm_put_device(ti, data_dev);
3220out_metadata:
3221 dm_put_device(ti, metadata_dev);
3222out_unlock:
3223 mutex_unlock(&dm_thin_pool_table.mutex);
3224
3225 return r;
3226}
3227
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00003228static int pool_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003229{
3230 int r;
3231 struct pool_c *pt = ti->private;
3232 struct pool *pool = pt->pool;
3233 unsigned long flags;
3234
3235 /*
3236 * As this is a singleton target, ti->begin is always zero.
3237 */
3238 spin_lock_irqsave(&pool->lock, flags);
3239 bio->bi_bdev = pt->data_dev->bdev;
3240 r = DM_MAPIO_REMAPPED;
3241 spin_unlock_irqrestore(&pool->lock, flags);
3242
3243 return r;
3244}
3245
Joe Thornberb17446d2013-05-10 14:37:18 +01003246static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)
3247{
3248 int r;
3249 struct pool_c *pt = ti->private;
3250 struct pool *pool = pt->pool;
3251 sector_t data_size = ti->len;
3252 dm_block_t sb_data_size;
3253
3254 *need_commit = false;
3255
3256 (void) sector_div(data_size, pool->sectors_per_block);
3257
3258 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
3259 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003260 DMERR("%s: failed to retrieve data device size",
3261 dm_device_name(pool->pool_md));
Joe Thornberb17446d2013-05-10 14:37:18 +01003262 return r;
3263 }
3264
3265 if (data_size < sb_data_size) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003266 DMERR("%s: pool target (%llu blocks) too small: expected %llu",
3267 dm_device_name(pool->pool_md),
Joe Thornberb17446d2013-05-10 14:37:18 +01003268 (unsigned long long)data_size, sb_data_size);
3269 return -EINVAL;
3270
3271 } else if (data_size > sb_data_size) {
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05003272 if (dm_pool_metadata_needs_check(pool->pmd)) {
3273 DMERR("%s: unable to grow the data device until repaired.",
3274 dm_device_name(pool->pool_md));
3275 return 0;
3276 }
3277
Mike Snitzer6f7f51d2013-12-04 10:25:53 -05003278 if (sb_data_size)
3279 DMINFO("%s: growing the data device from %llu to %llu blocks",
3280 dm_device_name(pool->pool_md),
3281 sb_data_size, (unsigned long long)data_size);
Joe Thornberb17446d2013-05-10 14:37:18 +01003282 r = dm_pool_resize_data_dev(pool->pmd, data_size);
3283 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05003284 metadata_operation_failed(pool, "dm_pool_resize_data_dev", r);
Joe Thornberb17446d2013-05-10 14:37:18 +01003285 return r;
3286 }
3287
3288 *need_commit = true;
3289 }
3290
3291 return 0;
3292}
3293
Joe Thornber24347e92013-05-10 14:37:19 +01003294static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)
3295{
3296 int r;
3297 struct pool_c *pt = ti->private;
3298 struct pool *pool = pt->pool;
3299 dm_block_t metadata_dev_size, sb_metadata_dev_size;
3300
3301 *need_commit = false;
3302
Alasdair G Kergon610bba82013-05-19 18:57:50 +01003303 metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev);
Joe Thornber24347e92013-05-10 14:37:19 +01003304
3305 r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size);
3306 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003307 DMERR("%s: failed to retrieve metadata device size",
3308 dm_device_name(pool->pool_md));
Joe Thornber24347e92013-05-10 14:37:19 +01003309 return r;
3310 }
3311
3312 if (metadata_dev_size < sb_metadata_dev_size) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003313 DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
3314 dm_device_name(pool->pool_md),
Joe Thornber24347e92013-05-10 14:37:19 +01003315 metadata_dev_size, sb_metadata_dev_size);
3316 return -EINVAL;
3317
3318 } else if (metadata_dev_size > sb_metadata_dev_size) {
Mike Snitzer07f2b6e2014-02-14 11:58:41 -05003319 if (dm_pool_metadata_needs_check(pool->pmd)) {
3320 DMERR("%s: unable to grow the metadata device until repaired.",
3321 dm_device_name(pool->pool_md));
3322 return 0;
3323 }
3324
Mike Snitzer7d489352014-02-12 23:58:15 -05003325 warn_if_metadata_device_too_big(pool->md_dev);
Mike Snitzer6f7f51d2013-12-04 10:25:53 -05003326 DMINFO("%s: growing the metadata device from %llu to %llu blocks",
3327 dm_device_name(pool->pool_md),
3328 sb_metadata_dev_size, metadata_dev_size);
Joe Thornber24347e92013-05-10 14:37:19 +01003329 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size);
3330 if (r) {
Joe Thornberb5330652013-12-04 19:51:33 -05003331 metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r);
Joe Thornber24347e92013-05-10 14:37:19 +01003332 return r;
3333 }
3334
3335 *need_commit = true;
3336 }
3337
3338 return 0;
3339}
3340
Joe Thornber991d9fa2011-10-31 20:21:18 +00003341/*
3342 * Retrieves the number of blocks of the data device from
3343 * the superblock and compares it to the actual device size,
3344 * thus resizing the data device in case it has grown.
3345 *
3346 * This both copes with opening preallocated data devices in the ctr
3347 * being followed by a resume
3348 * -and-
3349 * calling the resume method individually after userspace has
3350 * grown the data device in reaction to a table event.
3351 */
3352static int pool_preresume(struct dm_target *ti)
3353{
3354 int r;
Joe Thornber24347e92013-05-10 14:37:19 +01003355 bool need_commit1, need_commit2;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003356 struct pool_c *pt = ti->private;
3357 struct pool *pool = pt->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003358
3359 /*
3360 * Take control of the pool object.
3361 */
3362 r = bind_control_target(pool, ti);
3363 if (r)
3364 return r;
3365
Joe Thornberb17446d2013-05-10 14:37:18 +01003366 r = maybe_resize_data_dev(ti, &need_commit1);
3367 if (r)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003368 return r;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003369
Joe Thornber24347e92013-05-10 14:37:19 +01003370 r = maybe_resize_metadata_dev(ti, &need_commit2);
3371 if (r)
3372 return r;
3373
3374 if (need_commit1 || need_commit2)
Joe Thornber020cc3b2013-12-04 15:05:36 -05003375 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003376
3377 return 0;
3378}
3379
Mike Snitzer583024d2014-10-28 20:58:45 -04003380static void pool_suspend_active_thins(struct pool *pool)
3381{
3382 struct thin_c *tc;
3383
3384 /* Suspend all active thin devices */
3385 tc = get_first_thin(pool);
3386 while (tc) {
3387 dm_internal_suspend_noflush(tc->thin_md);
3388 tc = get_next_thin(pool, tc);
3389 }
3390}
3391
3392static void pool_resume_active_thins(struct pool *pool)
3393{
3394 struct thin_c *tc;
3395
3396 /* Resume all active thin devices */
3397 tc = get_first_thin(pool);
3398 while (tc) {
3399 dm_internal_resume(tc->thin_md);
3400 tc = get_next_thin(pool, tc);
3401 }
3402}
3403
Joe Thornber991d9fa2011-10-31 20:21:18 +00003404static void pool_resume(struct dm_target *ti)
3405{
3406 struct pool_c *pt = ti->private;
3407 struct pool *pool = pt->pool;
3408 unsigned long flags;
3409
Mike Snitzer583024d2014-10-28 20:58:45 -04003410 /*
3411 * Must requeue active_thins' bios and then resume
3412 * active_thins _before_ clearing 'suspend' flag.
3413 */
3414 requeue_bios(pool);
3415 pool_resume_active_thins(pool);
3416
Joe Thornber991d9fa2011-10-31 20:21:18 +00003417 spin_lock_irqsave(&pool->lock, flags);
Joe Thornber88a66212013-12-04 20:16:12 -05003418 pool->low_water_triggered = false;
Mike Snitzer80e96c52014-11-07 15:09:46 -05003419 pool->suspended = false;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003420 spin_unlock_irqrestore(&pool->lock, flags);
Mike Snitzer80e96c52014-11-07 15:09:46 -05003421
Joe Thornber905e51b2012-03-28 18:41:27 +01003422 do_waker(&pool->waker.work);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003423}
3424
Mike Snitzer80e96c52014-11-07 15:09:46 -05003425static void pool_presuspend(struct dm_target *ti)
3426{
3427 struct pool_c *pt = ti->private;
3428 struct pool *pool = pt->pool;
3429 unsigned long flags;
3430
3431 spin_lock_irqsave(&pool->lock, flags);
3432 pool->suspended = true;
3433 spin_unlock_irqrestore(&pool->lock, flags);
Mike Snitzer583024d2014-10-28 20:58:45 -04003434
3435 pool_suspend_active_thins(pool);
Mike Snitzer80e96c52014-11-07 15:09:46 -05003436}
3437
3438static void pool_presuspend_undo(struct dm_target *ti)
3439{
3440 struct pool_c *pt = ti->private;
3441 struct pool *pool = pt->pool;
3442 unsigned long flags;
3443
Mike Snitzer583024d2014-10-28 20:58:45 -04003444 pool_resume_active_thins(pool);
3445
Mike Snitzer80e96c52014-11-07 15:09:46 -05003446 spin_lock_irqsave(&pool->lock, flags);
3447 pool->suspended = false;
3448 spin_unlock_irqrestore(&pool->lock, flags);
3449}
3450
Joe Thornber991d9fa2011-10-31 20:21:18 +00003451static void pool_postsuspend(struct dm_target *ti)
3452{
Joe Thornber991d9fa2011-10-31 20:21:18 +00003453 struct pool_c *pt = ti->private;
3454 struct pool *pool = pt->pool;
3455
Joe Thornber905e51b2012-03-28 18:41:27 +01003456 cancel_delayed_work(&pool->waker);
Joe Thornber85ad643b2014-05-09 15:59:38 +01003457 cancel_delayed_work(&pool->no_space_timeout);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003458 flush_workqueue(pool->wq);
Joe Thornber020cc3b2013-12-04 15:05:36 -05003459 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003460}
3461
3462static int check_arg_count(unsigned argc, unsigned args_required)
3463{
3464 if (argc != args_required) {
3465 DMWARN("Message received with %u arguments instead of %u.",
3466 argc, args_required);
3467 return -EINVAL;
3468 }
3469
3470 return 0;
3471}
3472
3473static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
3474{
3475 if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
3476 *dev_id <= MAX_DEV_ID)
3477 return 0;
3478
3479 if (warning)
3480 DMWARN("Message received with invalid device id: %s", arg);
3481
3482 return -EINVAL;
3483}
3484
3485static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
3486{
3487 dm_thin_id dev_id;
3488 int r;
3489
3490 r = check_arg_count(argc, 2);
3491 if (r)
3492 return r;
3493
3494 r = read_dev_id(argv[1], &dev_id, 1);
3495 if (r)
3496 return r;
3497
3498 r = dm_pool_create_thin(pool->pmd, dev_id);
3499 if (r) {
3500 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
3501 argv[1]);
3502 return r;
3503 }
3504
3505 return 0;
3506}
3507
3508static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3509{
3510 dm_thin_id dev_id;
3511 dm_thin_id origin_dev_id;
3512 int r;
3513
3514 r = check_arg_count(argc, 3);
3515 if (r)
3516 return r;
3517
3518 r = read_dev_id(argv[1], &dev_id, 1);
3519 if (r)
3520 return r;
3521
3522 r = read_dev_id(argv[2], &origin_dev_id, 1);
3523 if (r)
3524 return r;
3525
3526 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
3527 if (r) {
3528 DMWARN("Creation of new snapshot %s of device %s failed.",
3529 argv[1], argv[2]);
3530 return r;
3531 }
3532
3533 return 0;
3534}
3535
3536static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
3537{
3538 dm_thin_id dev_id;
3539 int r;
3540
3541 r = check_arg_count(argc, 2);
3542 if (r)
3543 return r;
3544
3545 r = read_dev_id(argv[1], &dev_id, 1);
3546 if (r)
3547 return r;
3548
3549 r = dm_pool_delete_thin_device(pool->pmd, dev_id);
3550 if (r)
3551 DMWARN("Deletion of thin device %s failed.", argv[1]);
3552
3553 return r;
3554}
3555
3556static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
3557{
3558 dm_thin_id old_id, new_id;
3559 int r;
3560
3561 r = check_arg_count(argc, 3);
3562 if (r)
3563 return r;
3564
3565 if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
3566 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
3567 return -EINVAL;
3568 }
3569
3570 if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
3571 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
3572 return -EINVAL;
3573 }
3574
3575 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
3576 if (r) {
3577 DMWARN("Failed to change transaction id from %s to %s.",
3578 argv[1], argv[2]);
3579 return r;
3580 }
3581
3582 return 0;
3583}
3584
Joe Thornbercc8394d2012-06-03 00:30:01 +01003585static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3586{
3587 int r;
3588
3589 r = check_arg_count(argc, 1);
3590 if (r)
3591 return r;
3592
Joe Thornber020cc3b2013-12-04 15:05:36 -05003593 (void) commit(pool);
Joe Thornber0d200ae2012-07-03 12:55:31 +01003594
Joe Thornbercc8394d2012-06-03 00:30:01 +01003595 r = dm_pool_reserve_metadata_snap(pool->pmd);
3596 if (r)
3597 DMWARN("reserve_metadata_snap message failed.");
3598
3599 return r;
3600}
3601
3602static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
3603{
3604 int r;
3605
3606 r = check_arg_count(argc, 1);
3607 if (r)
3608 return r;
3609
3610 r = dm_pool_release_metadata_snap(pool->pmd);
3611 if (r)
3612 DMWARN("release_metadata_snap message failed.");
3613
3614 return r;
3615}
3616
Joe Thornber991d9fa2011-10-31 20:21:18 +00003617/*
3618 * Messages supported:
3619 * create_thin <dev_id>
3620 * create_snap <dev_id> <origin_id>
3621 * delete <dev_id>
Joe Thornber991d9fa2011-10-31 20:21:18 +00003622 * set_transaction_id <current_trans_id> <new_trans_id>
Joe Thornbercc8394d2012-06-03 00:30:01 +01003623 * reserve_metadata_snap
3624 * release_metadata_snap
Joe Thornber991d9fa2011-10-31 20:21:18 +00003625 */
3626static int pool_message(struct dm_target *ti, unsigned argc, char **argv)
3627{
3628 int r = -EINVAL;
3629 struct pool_c *pt = ti->private;
3630 struct pool *pool = pt->pool;
3631
Joe Thornber2a7eaea2015-01-26 11:38:21 +00003632 if (get_pool_mode(pool) >= PM_READ_ONLY) {
3633 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",
3634 dm_device_name(pool->pool_md));
Mike Snitzerfd467692015-06-09 12:31:26 -04003635 return -EOPNOTSUPP;
Joe Thornber2a7eaea2015-01-26 11:38:21 +00003636 }
3637
Joe Thornber991d9fa2011-10-31 20:21:18 +00003638 if (!strcasecmp(argv[0], "create_thin"))
3639 r = process_create_thin_mesg(argc, argv, pool);
3640
3641 else if (!strcasecmp(argv[0], "create_snap"))
3642 r = process_create_snap_mesg(argc, argv, pool);
3643
3644 else if (!strcasecmp(argv[0], "delete"))
3645 r = process_delete_mesg(argc, argv, pool);
3646
3647 else if (!strcasecmp(argv[0], "set_transaction_id"))
3648 r = process_set_transaction_id_mesg(argc, argv, pool);
3649
Joe Thornbercc8394d2012-06-03 00:30:01 +01003650 else if (!strcasecmp(argv[0], "reserve_metadata_snap"))
3651 r = process_reserve_metadata_snap_mesg(argc, argv, pool);
3652
3653 else if (!strcasecmp(argv[0], "release_metadata_snap"))
3654 r = process_release_metadata_snap_mesg(argc, argv, pool);
3655
Joe Thornber991d9fa2011-10-31 20:21:18 +00003656 else
3657 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
3658
Joe Thornbere49e5822012-07-27 15:08:16 +01003659 if (!r)
Joe Thornber020cc3b2013-12-04 15:05:36 -05003660 (void) commit(pool);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003661
3662 return r;
3663}
3664
Joe Thornbere49e5822012-07-27 15:08:16 +01003665static void emit_flags(struct pool_features *pf, char *result,
3666 unsigned sz, unsigned maxlen)
3667{
3668 unsigned count = !pf->zero_new_blocks + !pf->discard_enabled +
Mike Snitzer787a996c2013-12-06 16:21:43 -05003669 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) +
3670 pf->error_if_no_space;
Joe Thornbere49e5822012-07-27 15:08:16 +01003671 DMEMIT("%u ", count);
3672
3673 if (!pf->zero_new_blocks)
3674 DMEMIT("skip_block_zeroing ");
3675
3676 if (!pf->discard_enabled)
3677 DMEMIT("ignore_discard ");
3678
3679 if (!pf->discard_passdown)
3680 DMEMIT("no_discard_passdown ");
3681
3682 if (pf->mode == PM_READ_ONLY)
3683 DMEMIT("read_only ");
Mike Snitzer787a996c2013-12-06 16:21:43 -05003684
3685 if (pf->error_if_no_space)
3686 DMEMIT("error_if_no_space ");
Joe Thornbere49e5822012-07-27 15:08:16 +01003687}
3688
Joe Thornber991d9fa2011-10-31 20:21:18 +00003689/*
3690 * Status line is:
3691 * <transaction id> <used metadata sectors>/<total metadata sectors>
3692 * <used data sectors>/<total data sectors> <held metadata root>
Mike Snitzere4c78e22015-07-15 11:40:24 -04003693 * <pool mode> <discard config> <no space config> <needs_check>
Joe Thornber991d9fa2011-10-31 20:21:18 +00003694 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003695static void pool_status(struct dm_target *ti, status_type_t type,
3696 unsigned status_flags, char *result, unsigned maxlen)
Joe Thornber991d9fa2011-10-31 20:21:18 +00003697{
Joe Thornbere49e5822012-07-27 15:08:16 +01003698 int r;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003699 unsigned sz = 0;
3700 uint64_t transaction_id;
3701 dm_block_t nr_free_blocks_data;
3702 dm_block_t nr_free_blocks_metadata;
3703 dm_block_t nr_blocks_data;
3704 dm_block_t nr_blocks_metadata;
3705 dm_block_t held_root;
3706 char buf[BDEVNAME_SIZE];
3707 char buf2[BDEVNAME_SIZE];
3708 struct pool_c *pt = ti->private;
3709 struct pool *pool = pt->pool;
3710
3711 switch (type) {
3712 case STATUSTYPE_INFO:
Joe Thornbere49e5822012-07-27 15:08:16 +01003713 if (get_pool_mode(pool) == PM_FAIL) {
3714 DMEMIT("Fail");
3715 break;
3716 }
3717
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01003718 /* Commit to ensure statistics aren't out-of-date */
3719 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
Joe Thornber020cc3b2013-12-04 15:05:36 -05003720 (void) commit(pool);
Alasdair G Kergon1f4e0ff2012-07-27 15:08:16 +01003721
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003722 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);
3723 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003724 DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
3725 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003726 goto err;
3727 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003728
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003729 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);
3730 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003731 DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
3732 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003733 goto err;
3734 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003735
3736 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003737 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003738 DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
3739 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003740 goto err;
3741 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003742
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003743 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);
3744 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003745 DMERR("%s: dm_pool_get_free_block_count returned %d",
3746 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003747 goto err;
3748 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003749
3750 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003751 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003752 DMERR("%s: dm_pool_get_data_dev_size returned %d",
3753 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003754 goto err;
3755 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003756
Joe Thornbercc8394d2012-06-03 00:30:01 +01003757 r = dm_pool_get_metadata_snap(pool->pmd, &held_root);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003758 if (r) {
Mike Snitzer4fa59712013-08-21 17:30:40 -04003759 DMERR("%s: dm_pool_get_metadata_snap returned %d",
3760 dm_device_name(pool->pool_md), r);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003761 goto err;
3762 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003763
3764 DMEMIT("%llu %llu/%llu %llu/%llu ",
3765 (unsigned long long)transaction_id,
3766 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3767 (unsigned long long)nr_blocks_metadata,
3768 (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
3769 (unsigned long long)nr_blocks_data);
3770
3771 if (held_root)
Joe Thornbere49e5822012-07-27 15:08:16 +01003772 DMEMIT("%llu ", held_root);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003773 else
Joe Thornbere49e5822012-07-27 15:08:16 +01003774 DMEMIT("- ");
3775
Joe Thornber3e1a0692014-03-03 16:03:26 +00003776 if (pool->pf.mode == PM_OUT_OF_DATA_SPACE)
3777 DMEMIT("out_of_data_space ");
3778 else if (pool->pf.mode == PM_READ_ONLY)
Joe Thornbere49e5822012-07-27 15:08:16 +01003779 DMEMIT("ro ");
3780 else
3781 DMEMIT("rw ");
3782
Mike Snitzer018debe2012-12-21 20:23:32 +00003783 if (!pool->pf.discard_enabled)
Mike Snitzer787a996c2013-12-06 16:21:43 -05003784 DMEMIT("ignore_discard ");
Mike Snitzer018debe2012-12-21 20:23:32 +00003785 else if (pool->pf.discard_passdown)
Mike Snitzer787a996c2013-12-06 16:21:43 -05003786 DMEMIT("discard_passdown ");
Joe Thornbere49e5822012-07-27 15:08:16 +01003787 else
Mike Snitzer787a996c2013-12-06 16:21:43 -05003788 DMEMIT("no_discard_passdown ");
3789
3790 if (pool->pf.error_if_no_space)
3791 DMEMIT("error_if_no_space ");
3792 else
3793 DMEMIT("queue_if_no_space ");
Joe Thornber991d9fa2011-10-31 20:21:18 +00003794
Mike Snitzere4c78e22015-07-15 11:40:24 -04003795 if (dm_pool_metadata_needs_check(pool->pmd))
3796 DMEMIT("needs_check ");
3797 else
3798 DMEMIT("- ");
3799
Joe Thornber991d9fa2011-10-31 20:21:18 +00003800 break;
3801
3802 case STATUSTYPE_TABLE:
3803 DMEMIT("%s %s %lu %llu ",
3804 format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
3805 format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
3806 (unsigned long)pool->sectors_per_block,
3807 (unsigned long long)pt->low_water_blocks);
Mike Snitzer0424caa2012-09-26 23:45:47 +01003808 emit_flags(&pt->requested_pf, result, sz, maxlen);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003809 break;
3810 }
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003811 return;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003812
Mikulas Patockafd7c0922013-03-01 22:45:44 +00003813err:
3814 DMEMIT("Error");
Joe Thornber991d9fa2011-10-31 20:21:18 +00003815}
3816
3817static int pool_iterate_devices(struct dm_target *ti,
3818 iterate_devices_callout_fn fn, void *data)
3819{
3820 struct pool_c *pt = ti->private;
3821
3822 return fn(ti, pt->data_dev, 0, ti->len, data);
3823}
3824
Joe Thornber991d9fa2011-10-31 20:21:18 +00003825static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
3826{
3827 struct pool_c *pt = ti->private;
3828 struct pool *pool = pt->pool;
Mike Snitzer604ea902014-10-09 18:43:25 -04003829 sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3830
3831 /*
Mike Snitzerd200c302014-11-20 18:07:43 -05003832 * If max_sectors is smaller than pool->sectors_per_block adjust it
3833 * to the highest possible power-of-2 factor of pool->sectors_per_block.
3834 * This is especially beneficial when the pool's data device is a RAID
3835 * device that has a full stripe width that matches pool->sectors_per_block
3836 * -- because even though partial RAID stripe-sized IOs will be issued to a
3837 * single RAID stripe; when aggregated they will end on a full RAID stripe
3838 * boundary.. which avoids additional partial RAID stripe writes cascading
Mike Snitzer604ea902014-10-09 18:43:25 -04003839 */
Mike Snitzer604ea902014-10-09 18:43:25 -04003840 if (limits->max_sectors < pool->sectors_per_block) {
3841 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) {
3842 if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)
3843 limits->max_sectors--;
3844 limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);
3845 }
Mike Snitzer604ea902014-10-09 18:43:25 -04003846 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00003847
Mike Snitzer0cc67cd2013-08-20 15:02:41 -04003848 /*
3849 * If the system-determined stacked limits are compatible with the
3850 * pool's blocksize (io_opt is a factor) do not override them.
3851 */
3852 if (io_opt_sectors < pool->sectors_per_block ||
Mike Snitzer604ea902014-10-09 18:43:25 -04003853 !is_factor(io_opt_sectors, pool->sectors_per_block)) {
3854 if (is_factor(pool->sectors_per_block, limits->max_sectors))
3855 blk_limits_io_min(limits, limits->max_sectors << SECTOR_SHIFT);
3856 else
3857 blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT);
Mike Snitzer0cc67cd2013-08-20 15:02:41 -04003858 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
3859 }
Mike Snitzer0424caa2012-09-26 23:45:47 +01003860
3861 /*
3862 * pt->adjusted_pf is a staging area for the actual features to use.
3863 * They get transferred to the live pool in bind_control_target()
3864 * called from pool_preresume().
3865 */
Mike Snitzerb60ab992013-09-19 18:49:11 -04003866 if (!pt->adjusted_pf.discard_enabled) {
3867 /*
3868 * Must explicitly disallow stacking discard limits otherwise the
3869 * block layer will stack them if pool's data device has support.
3870 * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the
3871 * user to see that, so make sure to set all discard limits to 0.
3872 */
3873 limits->discard_granularity = 0;
Mike Snitzer0424caa2012-09-26 23:45:47 +01003874 return;
Mike Snitzerb60ab992013-09-19 18:49:11 -04003875 }
Mike Snitzer0424caa2012-09-26 23:45:47 +01003876
3877 disable_passdown_if_not_supported(pt);
3878
Joe Thornber34fbcf62015-04-16 12:58:35 +01003879 /*
3880 * The pool uses the same discard limits as the underlying data
3881 * device. DM core has already set this up.
3882 */
Joe Thornber991d9fa2011-10-31 20:21:18 +00003883}
3884
3885static struct target_type pool_target = {
3886 .name = "thin-pool",
3887 .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
3888 DM_TARGET_IMMUTABLE,
Mike Snitzere4c78e22015-07-15 11:40:24 -04003889 .version = {1, 16, 0},
Joe Thornber991d9fa2011-10-31 20:21:18 +00003890 .module = THIS_MODULE,
3891 .ctr = pool_ctr,
3892 .dtr = pool_dtr,
3893 .map = pool_map,
Mike Snitzer80e96c52014-11-07 15:09:46 -05003894 .presuspend = pool_presuspend,
3895 .presuspend_undo = pool_presuspend_undo,
Joe Thornber991d9fa2011-10-31 20:21:18 +00003896 .postsuspend = pool_postsuspend,
3897 .preresume = pool_preresume,
3898 .resume = pool_resume,
3899 .message = pool_message,
3900 .status = pool_status,
Joe Thornber991d9fa2011-10-31 20:21:18 +00003901 .iterate_devices = pool_iterate_devices,
3902 .io_hints = pool_io_hints,
3903};
3904
3905/*----------------------------------------------------------------
3906 * Thin target methods
3907 *--------------------------------------------------------------*/
Joe Thornberb10ebd32014-04-08 11:29:01 +01003908static void thin_get(struct thin_c *tc)
3909{
3910 atomic_inc(&tc->refcount);
3911}
3912
3913static void thin_put(struct thin_c *tc)
3914{
3915 if (atomic_dec_and_test(&tc->refcount))
3916 complete(&tc->can_destroy);
3917}
3918
Joe Thornber991d9fa2011-10-31 20:21:18 +00003919static void thin_dtr(struct dm_target *ti)
3920{
3921 struct thin_c *tc = ti->private;
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003922 unsigned long flags;
3923
3924 spin_lock_irqsave(&tc->pool->lock, flags);
3925 list_del_rcu(&tc->list);
3926 spin_unlock_irqrestore(&tc->pool->lock, flags);
3927 synchronize_rcu();
Joe Thornber991d9fa2011-10-31 20:21:18 +00003928
Mikulas Patocka17181fb2014-11-05 17:00:13 -05003929 thin_put(tc);
3930 wait_for_completion(&tc->can_destroy);
3931
Joe Thornber991d9fa2011-10-31 20:21:18 +00003932 mutex_lock(&dm_thin_pool_table.mutex);
3933
3934 __pool_dec(tc->pool);
3935 dm_pool_close_thin_device(tc->td);
3936 dm_put_device(ti, tc->pool_dev);
Joe Thornber2dd9c252012-03-28 18:41:28 +01003937 if (tc->origin_dev)
3938 dm_put_device(ti, tc->origin_dev);
Joe Thornber991d9fa2011-10-31 20:21:18 +00003939 kfree(tc);
3940
3941 mutex_unlock(&dm_thin_pool_table.mutex);
3942}
3943
3944/*
3945 * Thin target parameters:
3946 *
Joe Thornber2dd9c252012-03-28 18:41:28 +01003947 * <pool_dev> <dev_id> [origin_dev]
Joe Thornber991d9fa2011-10-31 20:21:18 +00003948 *
3949 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
3950 * dev_id: the internal device identifier
Joe Thornber2dd9c252012-03-28 18:41:28 +01003951 * origin_dev: a device external to the pool that should act as the origin
Joe Thornber67e2e2b2012-03-28 18:41:29 +01003952 *
3953 * If the pool device has discards disabled, they get disabled for the thin
3954 * device as well.
Joe Thornber991d9fa2011-10-31 20:21:18 +00003955 */
3956static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
3957{
3958 int r;
3959 struct thin_c *tc;
Joe Thornber2dd9c252012-03-28 18:41:28 +01003960 struct dm_dev *pool_dev, *origin_dev;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003961 struct mapped_device *pool_md;
Joe Thornber5e3283e2014-04-08 11:08:41 +01003962 unsigned long flags;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003963
3964 mutex_lock(&dm_thin_pool_table.mutex);
3965
Joe Thornber2dd9c252012-03-28 18:41:28 +01003966 if (argc != 2 && argc != 3) {
Joe Thornber991d9fa2011-10-31 20:21:18 +00003967 ti->error = "Invalid argument count";
3968 r = -EINVAL;
3969 goto out_unlock;
3970 }
3971
3972 tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
3973 if (!tc) {
3974 ti->error = "Out of memory";
3975 r = -ENOMEM;
3976 goto out_unlock;
3977 }
Mike Snitzer583024d2014-10-28 20:58:45 -04003978 tc->thin_md = dm_table_get_md(ti->table);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003979 spin_lock_init(&tc->lock);
Joe Thornbera374bb22014-10-10 13:43:14 +01003980 INIT_LIST_HEAD(&tc->deferred_cells);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04003981 bio_list_init(&tc->deferred_bio_list);
3982 bio_list_init(&tc->retry_on_resume_list);
Mike Snitzer67324ea2014-03-21 18:33:41 -04003983 tc->sort_bio_list = RB_ROOT;
Joe Thornber991d9fa2011-10-31 20:21:18 +00003984
Joe Thornber2dd9c252012-03-28 18:41:28 +01003985 if (argc == 3) {
3986 r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev);
3987 if (r) {
3988 ti->error = "Error opening origin device";
3989 goto bad_origin_dev;
3990 }
3991 tc->origin_dev = origin_dev;
3992 }
3993
Joe Thornber991d9fa2011-10-31 20:21:18 +00003994 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
3995 if (r) {
3996 ti->error = "Error opening pool device";
3997 goto bad_pool_dev;
3998 }
3999 tc->pool_dev = pool_dev;
4000
4001 if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
4002 ti->error = "Invalid device id";
4003 r = -EINVAL;
4004 goto bad_common;
4005 }
4006
4007 pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
4008 if (!pool_md) {
4009 ti->error = "Couldn't get pool mapped device";
4010 r = -EINVAL;
4011 goto bad_common;
4012 }
4013
4014 tc->pool = __pool_table_lookup(pool_md);
4015 if (!tc->pool) {
4016 ti->error = "Couldn't find pool object";
4017 r = -EINVAL;
4018 goto bad_pool_lookup;
4019 }
4020 __pool_inc(tc->pool);
4021
Joe Thornbere49e5822012-07-27 15:08:16 +01004022 if (get_pool_mode(tc->pool) == PM_FAIL) {
4023 ti->error = "Couldn't open thin device, Pool is in fail mode";
Mike Snitzer1acacc02014-02-19 20:32:33 -05004024 r = -EINVAL;
Mike Snitzer80e96c52014-11-07 15:09:46 -05004025 goto bad_pool;
Joe Thornbere49e5822012-07-27 15:08:16 +01004026 }
4027
Joe Thornber991d9fa2011-10-31 20:21:18 +00004028 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
4029 if (r) {
4030 ti->error = "Couldn't open thin internal device";
Mike Snitzer80e96c52014-11-07 15:09:46 -05004031 goto bad_pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00004032 }
4033
Mike Snitzer542f9032012-07-27 15:08:00 +01004034 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
4035 if (r)
Mike Snitzer80e96c52014-11-07 15:09:46 -05004036 goto bad;
Mike Snitzer542f9032012-07-27 15:08:00 +01004037
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00004038 ti->num_flush_bios = 1;
Joe Thornber16ad3d12012-07-27 15:08:07 +01004039 ti->flush_supported = true;
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00004040 ti->per_bio_data_size = sizeof(struct dm_thin_endio_hook);
Joe Thornber67e2e2b2012-03-28 18:41:29 +01004041
4042 /* In case the pool supports discards, pass them on. */
Mike Snitzerb60ab992013-09-19 18:49:11 -04004043 ti->discard_zeroes_data_unsupported = true;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01004044 if (tc->pool->pf.discard_enabled) {
Alasdair G Kergon0ac55482012-07-27 15:08:08 +01004045 ti->discards_supported = true;
Alasdair G Kergon55a62ee2013-03-01 22:45:47 +00004046 ti->num_discard_bios = 1;
Joe Thornber34fbcf62015-04-16 12:58:35 +01004047 ti->split_discard_bios = false;
Joe Thornber67e2e2b2012-03-28 18:41:29 +01004048 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00004049
Joe Thornber991d9fa2011-10-31 20:21:18 +00004050 mutex_unlock(&dm_thin_pool_table.mutex);
4051
Joe Thornber5e3283e2014-04-08 11:08:41 +01004052 spin_lock_irqsave(&tc->pool->lock, flags);
Mike Snitzer80e96c52014-11-07 15:09:46 -05004053 if (tc->pool->suspended) {
4054 spin_unlock_irqrestore(&tc->pool->lock, flags);
4055 mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */
4056 ti->error = "Unable to activate thin device while pool is suspended";
4057 r = -EINVAL;
4058 goto bad;
4059 }
Marc Dionne2b94e892014-12-17 07:59:59 -05004060 atomic_set(&tc->refcount, 1);
4061 init_completion(&tc->can_destroy);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04004062 list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
Joe Thornber5e3283e2014-04-08 11:08:41 +01004063 spin_unlock_irqrestore(&tc->pool->lock, flags);
Mike Snitzerc140e1c2014-03-20 21:17:14 -04004064 /*
4065 * This synchronize_rcu() call is needed here otherwise we risk a
4066 * wake_worker() call finding no bios to process (because the newly
4067 * added tc isn't yet visible). So this reduces latency since we
4068 * aren't then dependent on the periodic commit to wake_worker().
4069 */
4070 synchronize_rcu();
4071
Mike Snitzer80e96c52014-11-07 15:09:46 -05004072 dm_put(pool_md);
4073
Joe Thornber991d9fa2011-10-31 20:21:18 +00004074 return 0;
4075
Mike Snitzer80e96c52014-11-07 15:09:46 -05004076bad:
Mike Snitzer1acacc02014-02-19 20:32:33 -05004077 dm_pool_close_thin_device(tc->td);
Mike Snitzer80e96c52014-11-07 15:09:46 -05004078bad_pool:
Joe Thornber991d9fa2011-10-31 20:21:18 +00004079 __pool_dec(tc->pool);
4080bad_pool_lookup:
4081 dm_put(pool_md);
4082bad_common:
4083 dm_put_device(ti, tc->pool_dev);
4084bad_pool_dev:
Joe Thornber2dd9c252012-03-28 18:41:28 +01004085 if (tc->origin_dev)
4086 dm_put_device(ti, tc->origin_dev);
4087bad_origin_dev:
Joe Thornber991d9fa2011-10-31 20:21:18 +00004088 kfree(tc);
4089out_unlock:
4090 mutex_unlock(&dm_thin_pool_table.mutex);
4091
4092 return r;
4093}
4094
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00004095static int thin_map(struct dm_target *ti, struct bio *bio)
Joe Thornber991d9fa2011-10-31 20:21:18 +00004096{
Kent Overstreet4f024f32013-10-11 15:44:27 -07004097 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004098
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00004099 return thin_bio_map(ti, bio);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004100}
4101
Mikulas Patocka7de3ee52012-12-21 20:23:41 +00004102static int thin_endio(struct dm_target *ti, struct bio *bio, int err)
Joe Thornbereb2aa482012-03-28 18:41:28 +01004103{
4104 unsigned long flags;
Mikulas Patocka59c3d2c2012-12-21 20:23:40 +00004105 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
Joe Thornbereb2aa482012-03-28 18:41:28 +01004106 struct list_head work;
Mike Snitzera24c2562012-06-03 00:30:00 +01004107 struct dm_thin_new_mapping *m, *tmp;
Joe Thornbereb2aa482012-03-28 18:41:28 +01004108 struct pool *pool = h->tc->pool;
4109
4110 if (h->shared_read_entry) {
4111 INIT_LIST_HEAD(&work);
Mike Snitzer44feb382012-10-12 21:02:10 +01004112 dm_deferred_entry_dec(h->shared_read_entry, &work);
Joe Thornbereb2aa482012-03-28 18:41:28 +01004113
4114 spin_lock_irqsave(&pool->lock, flags);
4115 list_for_each_entry_safe(m, tmp, &work, list) {
4116 list_del(&m->list);
Joe Thornber50f3c3e2014-06-13 13:57:09 +01004117 __complete_mapping_preparation(m);
Joe Thornbereb2aa482012-03-28 18:41:28 +01004118 }
4119 spin_unlock_irqrestore(&pool->lock, flags);
4120 }
4121
Joe Thornber104655f2012-03-28 18:41:28 +01004122 if (h->all_io_entry) {
4123 INIT_LIST_HEAD(&work);
Mike Snitzer44feb382012-10-12 21:02:10 +01004124 dm_deferred_entry_dec(h->all_io_entry, &work);
Joe Thornber563af182012-12-21 20:23:31 +00004125 if (!list_empty(&work)) {
4126 spin_lock_irqsave(&pool->lock, flags);
4127 list_for_each_entry_safe(m, tmp, &work, list)
Mike Snitzerdaec3382013-12-11 14:01:20 -05004128 list_add_tail(&m->list, &pool->prepared_discards);
Joe Thornber563af182012-12-21 20:23:31 +00004129 spin_unlock_irqrestore(&pool->lock, flags);
4130 wake_worker(pool);
4131 }
Joe Thornber104655f2012-03-28 18:41:28 +01004132 }
4133
Joe Thornber34fbcf62015-04-16 12:58:35 +01004134 if (h->cell)
4135 cell_defer_no_holder(h->tc, h->cell);
4136
Joe Thornbereb2aa482012-03-28 18:41:28 +01004137 return 0;
4138}
4139
Joe Thornber738211f2014-03-03 15:52:28 +00004140static void thin_presuspend(struct dm_target *ti)
4141{
4142 struct thin_c *tc = ti->private;
4143
4144 if (dm_noflush_suspending(ti))
4145 noflush_work(tc, do_noflush_start);
4146}
4147
Joe Thornber991d9fa2011-10-31 20:21:18 +00004148static void thin_postsuspend(struct dm_target *ti)
4149{
Joe Thornber738211f2014-03-03 15:52:28 +00004150 struct thin_c *tc = ti->private;
4151
4152 /*
4153 * The dm_noflush_suspending flag has been cleared by now, so
4154 * unfortunately we must always run this.
4155 */
4156 noflush_work(tc, do_noflush_stop);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004157}
4158
Joe Thornbere5aea7b2014-06-13 14:47:24 +01004159static int thin_preresume(struct dm_target *ti)
4160{
4161 struct thin_c *tc = ti->private;
4162
4163 if (tc->origin_dev)
4164 tc->origin_size = get_dev_size(tc->origin_dev->bdev);
4165
4166 return 0;
4167}
4168
Joe Thornber991d9fa2011-10-31 20:21:18 +00004169/*
4170 * <nr mapped sectors> <highest mapped sector>
4171 */
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004172static void thin_status(struct dm_target *ti, status_type_t type,
4173 unsigned status_flags, char *result, unsigned maxlen)
Joe Thornber991d9fa2011-10-31 20:21:18 +00004174{
4175 int r;
4176 ssize_t sz = 0;
4177 dm_block_t mapped, highest;
4178 char buf[BDEVNAME_SIZE];
4179 struct thin_c *tc = ti->private;
4180
Joe Thornbere49e5822012-07-27 15:08:16 +01004181 if (get_pool_mode(tc->pool) == PM_FAIL) {
4182 DMEMIT("Fail");
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004183 return;
Joe Thornbere49e5822012-07-27 15:08:16 +01004184 }
4185
Joe Thornber991d9fa2011-10-31 20:21:18 +00004186 if (!tc->td)
4187 DMEMIT("-");
4188 else {
4189 switch (type) {
4190 case STATUSTYPE_INFO:
4191 r = dm_thin_get_mapped_count(tc->td, &mapped);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004192 if (r) {
4193 DMERR("dm_thin_get_mapped_count returned %d", r);
4194 goto err;
4195 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00004196
4197 r = dm_thin_get_highest_mapped_block(tc->td, &highest);
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004198 if (r < 0) {
4199 DMERR("dm_thin_get_highest_mapped_block returned %d", r);
4200 goto err;
4201 }
Joe Thornber991d9fa2011-10-31 20:21:18 +00004202
4203 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
4204 if (r)
4205 DMEMIT("%llu", ((highest + 1) *
4206 tc->pool->sectors_per_block) - 1);
4207 else
4208 DMEMIT("-");
4209 break;
4210
4211 case STATUSTYPE_TABLE:
4212 DMEMIT("%s %lu",
4213 format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
4214 (unsigned long) tc->dev_id);
Joe Thornber2dd9c252012-03-28 18:41:28 +01004215 if (tc->origin_dev)
4216 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
Joe Thornber991d9fa2011-10-31 20:21:18 +00004217 break;
4218 }
4219 }
4220
Mikulas Patockafd7c0922013-03-01 22:45:44 +00004221 return;
4222
4223err:
4224 DMEMIT("Error");
Joe Thornber991d9fa2011-10-31 20:21:18 +00004225}
4226
4227static int thin_iterate_devices(struct dm_target *ti,
4228 iterate_devices_callout_fn fn, void *data)
4229{
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004230 sector_t blocks;
Joe Thornber991d9fa2011-10-31 20:21:18 +00004231 struct thin_c *tc = ti->private;
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004232 struct pool *pool = tc->pool;
Joe Thornber991d9fa2011-10-31 20:21:18 +00004233
4234 /*
4235 * We can't call dm_pool_get_data_dev_size() since that blocks. So
4236 * we follow a more convoluted path through to the pool's target.
4237 */
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004238 if (!pool->ti)
Joe Thornber991d9fa2011-10-31 20:21:18 +00004239 return 0; /* nothing is bound */
4240
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004241 blocks = pool->ti->len;
4242 (void) sector_div(blocks, pool->sectors_per_block);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004243 if (blocks)
Mike Snitzer55f2b8b2012-07-27 15:08:02 +01004244 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004245
4246 return 0;
4247}
4248
Joe Thornber34fbcf62015-04-16 12:58:35 +01004249static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
4250{
4251 struct thin_c *tc = ti->private;
4252 struct pool *pool = tc->pool;
Mike Snitzer21607672015-09-08 08:56:13 -04004253
Mike Snitzer0fcb04d2015-11-23 13:44:38 -05004254 if (!pool->pf.discard_enabled)
4255 return;
Joe Thornber34fbcf62015-04-16 12:58:35 +01004256
4257 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
4258 limits->max_discard_sectors = 2048 * 1024 * 16; /* 16G */
4259}
4260
Joe Thornber991d9fa2011-10-31 20:21:18 +00004261static struct target_type thin_target = {
4262 .name = "thin",
Mike Snitzere4c78e22015-07-15 11:40:24 -04004263 .version = {1, 16, 0},
Joe Thornber991d9fa2011-10-31 20:21:18 +00004264 .module = THIS_MODULE,
4265 .ctr = thin_ctr,
4266 .dtr = thin_dtr,
4267 .map = thin_map,
Joe Thornbereb2aa482012-03-28 18:41:28 +01004268 .end_io = thin_endio,
Joe Thornbere5aea7b2014-06-13 14:47:24 +01004269 .preresume = thin_preresume,
Joe Thornber738211f2014-03-03 15:52:28 +00004270 .presuspend = thin_presuspend,
Joe Thornber991d9fa2011-10-31 20:21:18 +00004271 .postsuspend = thin_postsuspend,
4272 .status = thin_status,
4273 .iterate_devices = thin_iterate_devices,
Joe Thornber34fbcf62015-04-16 12:58:35 +01004274 .io_hints = thin_io_hints,
Joe Thornber991d9fa2011-10-31 20:21:18 +00004275};
4276
4277/*----------------------------------------------------------------*/
4278
4279static int __init dm_thin_init(void)
4280{
4281 int r;
4282
4283 pool_table_init();
4284
4285 r = dm_register_target(&thin_target);
4286 if (r)
4287 return r;
4288
4289 r = dm_register_target(&pool_target);
4290 if (r)
Mike Snitzera24c2562012-06-03 00:30:00 +01004291 goto bad_pool_target;
4292
4293 r = -ENOMEM;
4294
Mike Snitzera24c2562012-06-03 00:30:00 +01004295 _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
4296 if (!_new_mapping_cache)
4297 goto bad_new_mapping_cache;
4298
Mike Snitzera24c2562012-06-03 00:30:00 +01004299 return 0;
4300
Mike Snitzera24c2562012-06-03 00:30:00 +01004301bad_new_mapping_cache:
Mike Snitzera24c2562012-06-03 00:30:00 +01004302 dm_unregister_target(&pool_target);
4303bad_pool_target:
4304 dm_unregister_target(&thin_target);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004305
4306 return r;
4307}
4308
4309static void dm_thin_exit(void)
4310{
4311 dm_unregister_target(&thin_target);
4312 dm_unregister_target(&pool_target);
Mike Snitzera24c2562012-06-03 00:30:00 +01004313
Mike Snitzera24c2562012-06-03 00:30:00 +01004314 kmem_cache_destroy(_new_mapping_cache);
Joe Thornber991d9fa2011-10-31 20:21:18 +00004315}
4316
4317module_init(dm_thin_init);
4318module_exit(dm_thin_exit);
4319
Mike Snitzer80c57892014-05-20 13:38:33 -04004320module_param_named(no_space_timeout, no_space_timeout_secs, uint, S_IRUGO | S_IWUSR);
4321MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
4322
Alasdair G Kergon7cab8bf2012-05-12 01:43:19 +01004323MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
Joe Thornber991d9fa2011-10-31 20:21:18 +00004324MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
4325MODULE_LICENSE("GPL");