blob: 9e9d04cb7d51f53e8292674abd9198911f8a3dc8 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2002 Sistina Software (UK) Limited.
Milan Broz373a3922007-05-09 02:33:02 -07003 * Copyright (C) 2006 Red Hat GmbH
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 *
7 * Kcopyd provides a simple interface for copying an area of one
8 * block-device to one or more other block-devices, with an asynchronous
9 * completion notification.
10 */
11
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +010012#include <linux/types.h>
Arun Sharma600634972011-07-26 16:09:06 -070013#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/fs.h>
16#include <linux/init.h>
17#include <linux/list.h>
18#include <linux/mempool.h>
19#include <linux/module.h>
20#include <linux/pagemap.h>
21#include <linux/slab.h>
22#include <linux/vmalloc.h>
23#include <linux/workqueue.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080024#include <linux/mutex.h>
Mikulas Patockadf5d2e92013-03-01 22:45:49 +000025#include <linux/delay.h>
Mikulas Patocka586e80e2008-10-21 17:44:59 +010026#include <linux/device-mapper.h>
Alasdair G Kergona765e202008-04-24 22:02:01 +010027#include <linux/dm-kcopyd.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Mike Snitzer4cc96132016-05-12 16:28:10 -040029#include "dm-core.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Mikulas Patockac6ea41f2011-05-29 13:03:00 +010031#define SUB_JOB_SIZE 128
32#define SPLIT_COUNT 8
33#define MIN_JOBS 8
Mikulas Patocka5f43ba22011-05-29 13:03:11 +010034#define RESERVE_PAGES (DIV_ROUND_UP(SUB_JOB_SIZE << SECTOR_SHIFT, PAGE_SIZE))
Mikulas Patockac6ea41f2011-05-29 13:03:00 +010035
Linus Torvalds1da177e2005-04-16 15:20:36 -070036/*-----------------------------------------------------------------
37 * Each kcopyd client has its own little pool of preallocated
38 * pages for kcopyd io.
39 *---------------------------------------------------------------*/
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +010040struct dm_kcopyd_client {
Linus Torvalds1da177e2005-04-16 15:20:36 -070041 struct page_list *pages;
Mikulas Patockad0471452011-05-29 13:03:07 +010042 unsigned nr_reserved_pages;
43 unsigned nr_free_pages;
Alasdair G Kergon138728dc2006-03-27 01:17:50 -080044
Milan Broz373a3922007-05-09 02:33:02 -070045 struct dm_io_client *io_client;
46
Alasdair G Kergon138728dc2006-03-27 01:17:50 -080047 wait_queue_head_t destroyq;
48 atomic_t nr_jobs;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +010049
Mikulas Patocka08d87572008-04-24 21:43:46 +010050 mempool_t *job_pool;
51
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +010052 struct workqueue_struct *kcopyd_wq;
53 struct work_struct kcopyd_work;
54
Mikulas Patockadf5d2e92013-03-01 22:45:49 +000055 struct dm_kcopyd_throttle *throttle;
56
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +010057/*
58 * We maintain three lists of jobs:
59 *
60 * i) jobs waiting for pages
61 * ii) jobs that have pages, and are waiting for the io to be issued.
62 * iii) jobs that have completed.
63 *
64 * All three of these are protected by job_lock.
65 */
66 spinlock_t job_lock;
67 struct list_head complete_jobs;
68 struct list_head io_jobs;
69 struct list_head pages_jobs;
Linus Torvalds1da177e2005-04-16 15:20:36 -070070};
71
Mikulas Patocka7f069652011-10-31 20:18:58 +000072static struct page_list zero_page_list;
73
Mikulas Patockadf5d2e92013-03-01 22:45:49 +000074static DEFINE_SPINLOCK(throttle_spinlock);
75
76/*
77 * IO/IDLE accounting slowly decays after (1 << ACCOUNT_INTERVAL_SHIFT) period.
78 * When total_period >= (1 << ACCOUNT_INTERVAL_SHIFT) the counters are divided
79 * by 2.
80 */
81#define ACCOUNT_INTERVAL_SHIFT SHIFT_HZ
82
83/*
84 * Sleep this number of milliseconds.
85 *
86 * The value was decided experimentally.
87 * Smaller values seem to cause an increased copy rate above the limit.
88 * The reason for this is unknown but possibly due to jiffies rounding errors
89 * or read/write cache inside the disk.
90 */
91#define SLEEP_MSEC 100
92
93/*
94 * Maximum number of sleep events. There is a theoretical livelock if more
95 * kcopyd clients do work simultaneously which this limit avoids.
96 */
97#define MAX_SLEEPS 10
98
99static void io_job_start(struct dm_kcopyd_throttle *t)
100{
101 unsigned throttle, now, difference;
102 int slept = 0, skew;
103
104 if (unlikely(!t))
105 return;
106
107try_again:
108 spin_lock_irq(&throttle_spinlock);
109
110 throttle = ACCESS_ONCE(t->throttle);
111
112 if (likely(throttle >= 100))
113 goto skip_limit;
114
115 now = jiffies;
116 difference = now - t->last_jiffies;
117 t->last_jiffies = now;
118 if (t->num_io_jobs)
119 t->io_period += difference;
120 t->total_period += difference;
121
122 /*
123 * Maintain sane values if we got a temporary overflow.
124 */
125 if (unlikely(t->io_period > t->total_period))
126 t->io_period = t->total_period;
127
128 if (unlikely(t->total_period >= (1 << ACCOUNT_INTERVAL_SHIFT))) {
129 int shift = fls(t->total_period >> ACCOUNT_INTERVAL_SHIFT);
130 t->total_period >>= shift;
131 t->io_period >>= shift;
132 }
133
134 skew = t->io_period - throttle * t->total_period / 100;
135
136 if (unlikely(skew > 0) && slept < MAX_SLEEPS) {
137 slept++;
138 spin_unlock_irq(&throttle_spinlock);
139 msleep(SLEEP_MSEC);
140 goto try_again;
141 }
142
143skip_limit:
144 t->num_io_jobs++;
145
146 spin_unlock_irq(&throttle_spinlock);
147}
148
149static void io_job_finish(struct dm_kcopyd_throttle *t)
150{
151 unsigned long flags;
152
153 if (unlikely(!t))
154 return;
155
156 spin_lock_irqsave(&throttle_spinlock, flags);
157
158 t->num_io_jobs--;
159
160 if (likely(ACCESS_ONCE(t->throttle) >= 100))
161 goto skip_limit;
162
163 if (!t->num_io_jobs) {
164 unsigned now, difference;
165
166 now = jiffies;
167 difference = now - t->last_jiffies;
168 t->last_jiffies = now;
169
170 t->io_period += difference;
171 t->total_period += difference;
172
173 /*
174 * Maintain sane values if we got a temporary overflow.
175 */
176 if (unlikely(t->io_period > t->total_period))
177 t->io_period = t->total_period;
178 }
179
180skip_limit:
181 spin_unlock_irqrestore(&throttle_spinlock, flags);
182}
183
184
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100185static void wake(struct dm_kcopyd_client *kc)
186{
187 queue_work(kc->kcopyd_wq, &kc->kcopyd_work);
188}
189
Mikulas Patockad0471452011-05-29 13:03:07 +0100190/*
191 * Obtain one page for the use of kcopyd.
192 */
Mikulas Patockaf99b55e2011-05-29 13:03:04 +0100193static struct page_list *alloc_pl(gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194{
195 struct page_list *pl;
196
Mikulas Patockaf99b55e2011-05-29 13:03:04 +0100197 pl = kmalloc(sizeof(*pl), gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 if (!pl)
199 return NULL;
200
Mikulas Patockaf99b55e2011-05-29 13:03:04 +0100201 pl->page = alloc_page(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 if (!pl->page) {
203 kfree(pl);
204 return NULL;
205 }
206
207 return pl;
208}
209
210static void free_pl(struct page_list *pl)
211{
212 __free_page(pl->page);
213 kfree(pl);
214}
215
Mikulas Patockad0471452011-05-29 13:03:07 +0100216/*
217 * Add the provided pages to a client's free page list, releasing
218 * back to the system any beyond the reserved_pages limit.
219 */
220static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl)
221{
222 struct page_list *next;
223
224 do {
225 next = pl->next;
226
227 if (kc->nr_free_pages >= kc->nr_reserved_pages)
228 free_pl(pl);
229 else {
230 pl->next = kc->pages;
231 kc->pages = pl;
232 kc->nr_free_pages++;
233 }
234
235 pl = next;
236 } while (pl);
237}
238
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100239static int kcopyd_get_pages(struct dm_kcopyd_client *kc,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 unsigned int nr, struct page_list **pages)
241{
242 struct page_list *pl;
243
Mikulas Patockad0471452011-05-29 13:03:07 +0100244 *pages = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245
Mikulas Patockad0471452011-05-29 13:03:07 +0100246 do {
Mel Gormand0164ad2015-11-06 16:28:21 -0800247 pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY | __GFP_KSWAPD_RECLAIM);
Mikulas Patockad0471452011-05-29 13:03:07 +0100248 if (unlikely(!pl)) {
249 /* Use reserved pages */
250 pl = kc->pages;
251 if (unlikely(!pl))
252 goto out_of_memory;
253 kc->pages = pl->next;
254 kc->nr_free_pages--;
255 }
256 pl->next = *pages;
257 *pages = pl;
258 } while (--nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261
Mikulas Patockad0471452011-05-29 13:03:07 +0100262out_of_memory:
263 if (*pages)
264 kcopyd_put_pages(kc, *pages);
265 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266}
267
268/*
269 * These three functions resize the page pool.
270 */
271static void drop_pages(struct page_list *pl)
272{
273 struct page_list *next;
274
275 while (pl) {
276 next = pl->next;
277 free_pl(pl);
278 pl = next;
279 }
280}
281
Mikulas Patockad0471452011-05-29 13:03:07 +0100282/*
283 * Allocate and reserve nr_pages for the use of a specific client.
284 */
285static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286{
Mikulas Patockad0471452011-05-29 13:03:07 +0100287 unsigned i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 struct page_list *pl = NULL, *next;
289
Mikulas Patockad0471452011-05-29 13:03:07 +0100290 for (i = 0; i < nr_pages; i++) {
Mikulas Patockaf99b55e2011-05-29 13:03:04 +0100291 next = alloc_pl(GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 if (!next) {
293 if (pl)
294 drop_pages(pl);
295 return -ENOMEM;
296 }
297 next->next = pl;
298 pl = next;
299 }
300
Mikulas Patockad0471452011-05-29 13:03:07 +0100301 kc->nr_reserved_pages += nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 kcopyd_put_pages(kc, pl);
Mikulas Patockad0471452011-05-29 13:03:07 +0100303
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 return 0;
305}
306
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100307static void client_free_pages(struct dm_kcopyd_client *kc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308{
Mikulas Patockad0471452011-05-29 13:03:07 +0100309 BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 drop_pages(kc->pages);
311 kc->pages = NULL;
Mikulas Patockad0471452011-05-29 13:03:07 +0100312 kc->nr_free_pages = kc->nr_reserved_pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313}
314
315/*-----------------------------------------------------------------
316 * kcopyd_jobs need to be allocated by the *clients* of kcopyd,
317 * for this reason we use a mempool to prevent the client from
318 * ever having to do io (which could cause a deadlock).
319 *---------------------------------------------------------------*/
320struct kcopyd_job {
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100321 struct dm_kcopyd_client *kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 struct list_head list;
323 unsigned long flags;
324
325 /*
326 * Error state of the job.
327 */
328 int read_err;
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700329 unsigned long write_err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330
331 /*
332 * Either READ or WRITE
333 */
334 int rw;
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +0100335 struct dm_io_region source;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336
337 /*
338 * The destinations for the transfer.
339 */
340 unsigned int num_dests;
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100341 struct dm_io_region dests[DM_KCOPYD_MAX_REGIONS];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 struct page_list *pages;
344
345 /*
346 * Set this to ensure you are notified when the job has
347 * completed. 'context' is for callback to use.
348 */
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100349 dm_kcopyd_notify_fn fn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 void *context;
351
352 /*
353 * These fields are only used if the job has been split
354 * into more manageable parts.
355 */
Matthias Kaehlckedef5b5b2007-10-19 22:38:52 +0100356 struct mutex lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 atomic_t sub_jobs;
358 sector_t progress;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100360 struct kcopyd_job *master_job;
361};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362
Christoph Lametere18b8902006-12-06 20:33:20 -0800363static struct kmem_cache *_job_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100365int __init dm_kcopyd_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366{
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100367 _job_cache = kmem_cache_create("kcopyd_job",
368 sizeof(struct kcopyd_job) * (SPLIT_COUNT + 1),
369 __alignof__(struct kcopyd_job), 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 if (!_job_cache)
371 return -ENOMEM;
372
Mikulas Patocka7f069652011-10-31 20:18:58 +0000373 zero_page_list.next = &zero_page_list;
374 zero_page_list.page = ZERO_PAGE(0);
375
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 return 0;
377}
378
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100379void dm_kcopyd_exit(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 kmem_cache_destroy(_job_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 _job_cache = NULL;
383}
384
385/*
386 * Functions to push and pop a job onto the head of a given job
387 * list.
388 */
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100389static struct kcopyd_job *pop(struct list_head *jobs,
390 struct dm_kcopyd_client *kc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391{
392 struct kcopyd_job *job = NULL;
393 unsigned long flags;
394
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100395 spin_lock_irqsave(&kc->job_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
397 if (!list_empty(jobs)) {
398 job = list_entry(jobs->next, struct kcopyd_job, list);
399 list_del(&job->list);
400 }
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100401 spin_unlock_irqrestore(&kc->job_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402
403 return job;
404}
405
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100406static void push(struct list_head *jobs, struct kcopyd_job *job)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407{
408 unsigned long flags;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100409 struct dm_kcopyd_client *kc = job->kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100411 spin_lock_irqsave(&kc->job_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412 list_add_tail(&job->list, jobs);
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100413 spin_unlock_irqrestore(&kc->job_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414}
415
Kazuo Itob673c3a2008-10-21 17:44:50 +0100416
417static void push_head(struct list_head *jobs, struct kcopyd_job *job)
418{
419 unsigned long flags;
420 struct dm_kcopyd_client *kc = job->kc;
421
422 spin_lock_irqsave(&kc->job_lock, flags);
423 list_add(&job->list, jobs);
424 spin_unlock_irqrestore(&kc->job_lock, flags);
425}
426
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427/*
428 * These three functions process 1 item from the corresponding
429 * job list.
430 *
431 * They return:
432 * < 0: error
433 * 0: success
434 * > 0: can't process yet.
435 */
436static int run_complete_job(struct kcopyd_job *job)
437{
438 void *context = job->context;
439 int read_err = job->read_err;
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700440 unsigned long write_err = job->write_err;
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100441 dm_kcopyd_notify_fn fn = job->fn;
442 struct dm_kcopyd_client *kc = job->kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443
Mikulas Patocka7f069652011-10-31 20:18:58 +0000444 if (job->pages && job->pages != &zero_page_list)
Mikulas Patocka73830852009-04-09 00:27:16 +0100445 kcopyd_put_pages(kc, job->pages);
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100446 /*
447 * If this is the master job, the sub jobs have already
448 * completed so we can free everything.
449 */
450 if (job->master_job == job)
451 mempool_free(job, kc->job_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 fn(read_err, write_err, context);
Alasdair G Kergon138728dc2006-03-27 01:17:50 -0800453
454 if (atomic_dec_and_test(&kc->nr_jobs))
455 wake_up(&kc->destroyq);
456
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457 return 0;
458}
459
460static void complete_io(unsigned long error, void *context)
461{
462 struct kcopyd_job *job = (struct kcopyd_job *) context;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100463 struct dm_kcopyd_client *kc = job->kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464
Mikulas Patockadf5d2e92013-03-01 22:45:49 +0000465 io_job_finish(kc->throttle);
466
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 if (error) {
Mike Christie51111662016-06-05 14:31:46 -0500468 if (op_is_write(job->rw))
Jonathan Brassowce503f52006-06-26 00:27:30 -0700469 job->write_err |= error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 else
471 job->read_err = 1;
472
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100473 if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100474 push(&kc->complete_jobs, job);
475 wake(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 return;
477 }
478 }
479
Mike Christie51111662016-06-05 14:31:46 -0500480 if (op_is_write(job->rw))
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100481 push(&kc->complete_jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482
483 else {
484 job->rw = WRITE;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100485 push(&kc->io_jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 }
487
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100488 wake(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489}
490
491/*
492 * Request io on as many buffer heads as we can currently get for
493 * a particular job.
494 */
495static int run_io_job(struct kcopyd_job *job)
496{
497 int r;
Milan Broz373a3922007-05-09 02:33:02 -0700498 struct dm_io_request io_req = {
Mike Christiee6047142016-06-05 14:32:04 -0500499 .bi_op = job->rw,
500 .bi_op_flags = 0,
Milan Broz373a3922007-05-09 02:33:02 -0700501 .mem.type = DM_IO_PAGE_LIST,
502 .mem.ptr.pl = job->pages,
Mikulas Patocka4622afb2011-08-02 12:32:02 +0100503 .mem.offset = 0,
Milan Broz373a3922007-05-09 02:33:02 -0700504 .notify.fn = complete_io,
505 .notify.context = job,
506 .client = job->kc->io_client,
507 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508
Mikulas Patockadf5d2e92013-03-01 22:45:49 +0000509 io_job_start(job->kc->throttle);
510
Jens Axboe7eaceac2011-03-10 08:52:07 +0100511 if (job->rw == READ)
Milan Broz373a3922007-05-09 02:33:02 -0700512 r = dm_io(&io_req, 1, &job->source, NULL);
Jens Axboe721a9602011-03-09 11:56:30 +0100513 else
Milan Broz373a3922007-05-09 02:33:02 -0700514 r = dm_io(&io_req, job->num_dests, job->dests, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515
516 return r;
517}
518
519static int run_pages_job(struct kcopyd_job *job)
520{
521 int r;
Mikulas Patocka5bf45a32011-08-02 12:32:02 +0100522 unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
Mikulas Patocka5bf45a32011-08-02 12:32:02 +0100524 r = kcopyd_get_pages(job->kc, nr_pages, &job->pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 if (!r) {
526 /* this job is ready for io */
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100527 push(&job->kc->io_jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 return 0;
529 }
530
531 if (r == -ENOMEM)
532 /* can't complete now */
533 return 1;
534
535 return r;
536}
537
538/*
539 * Run through a list for as long as possible. Returns the count
540 * of successful jobs.
541 */
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100542static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc,
543 int (*fn) (struct kcopyd_job *))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544{
545 struct kcopyd_job *job;
546 int r, count = 0;
547
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100548 while ((job = pop(jobs, kc))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549
550 r = fn(job);
551
552 if (r < 0) {
553 /* error this rogue job */
Mike Christie51111662016-06-05 14:31:46 -0500554 if (op_is_write(job->rw))
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700555 job->write_err = (unsigned long) -1L;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 else
557 job->read_err = 1;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100558 push(&kc->complete_jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 break;
560 }
561
562 if (r > 0) {
563 /*
564 * We couldn't service this job ATM, so
565 * push this job back onto the list.
566 */
Kazuo Itob673c3a2008-10-21 17:44:50 +0100567 push_head(jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568 break;
569 }
570
571 count++;
572 }
573
574 return count;
575}
576
577/*
578 * kcopyd does this every time it's woken up.
579 */
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100580static void do_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581{
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100582 struct dm_kcopyd_client *kc = container_of(work,
583 struct dm_kcopyd_client, kcopyd_work);
Jens Axboe7eaceac2011-03-10 08:52:07 +0100584 struct blk_plug plug;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100585
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 /*
587 * The order that these are called is *very* important.
588 * complete jobs can free some pages for pages jobs.
589 * Pages jobs when successful will jump onto the io jobs
590 * list. io jobs call wake when they complete and it all
591 * starts again.
592 */
Jens Axboe7eaceac2011-03-10 08:52:07 +0100593 blk_start_plug(&plug);
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100594 process_jobs(&kc->complete_jobs, kc, run_complete_job);
595 process_jobs(&kc->pages_jobs, kc, run_pages_job);
596 process_jobs(&kc->io_jobs, kc, run_io_job);
Jens Axboe7eaceac2011-03-10 08:52:07 +0100597 blk_finish_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598}
599
600/*
601 * If we are copying a small region we just dispatch a single job
602 * to do the copy, otherwise the io has to be split up into many
603 * jobs.
604 */
605static void dispatch_job(struct kcopyd_job *job)
606{
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100607 struct dm_kcopyd_client *kc = job->kc;
608 atomic_inc(&kc->nr_jobs);
Mikulas Patocka9ca170a2009-12-10 23:52:13 +0000609 if (unlikely(!job->source.count))
610 push(&kc->complete_jobs, job);
Mikulas Patocka7f069652011-10-31 20:18:58 +0000611 else if (job->pages == &zero_page_list)
612 push(&kc->io_jobs, job);
Mikulas Patocka9ca170a2009-12-10 23:52:13 +0000613 else
614 push(&kc->pages_jobs, job);
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100615 wake(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616}
617
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700618static void segment_complete(int read_err, unsigned long write_err,
619 void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620{
621 /* FIXME: tidy this function */
622 sector_t progress = 0;
623 sector_t count = 0;
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100624 struct kcopyd_job *sub_job = (struct kcopyd_job *) context;
625 struct kcopyd_job *job = sub_job->master_job;
Mikulas Patocka73830852009-04-09 00:27:16 +0100626 struct dm_kcopyd_client *kc = job->kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627
Matthias Kaehlckedef5b5b2007-10-19 22:38:52 +0100628 mutex_lock(&job->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629
630 /* update the error */
631 if (read_err)
632 job->read_err = 1;
633
634 if (write_err)
Jonathan Brassowce503f52006-06-26 00:27:30 -0700635 job->write_err |= write_err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636
637 /*
638 * Only dispatch more work if there hasn't been an error.
639 */
640 if ((!job->read_err && !job->write_err) ||
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100641 test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 /* get the next chunk of work */
643 progress = job->progress;
644 count = job->source.count - progress;
645 if (count) {
646 if (count > SUB_JOB_SIZE)
647 count = SUB_JOB_SIZE;
648
649 job->progress += count;
650 }
651 }
Matthias Kaehlckedef5b5b2007-10-19 22:38:52 +0100652 mutex_unlock(&job->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653
654 if (count) {
655 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656
657 *sub_job = *job;
658 sub_job->source.sector += progress;
659 sub_job->source.count = count;
660
661 for (i = 0; i < job->num_dests; i++) {
662 sub_job->dests[i].sector += progress;
663 sub_job->dests[i].count = count;
664 }
665
666 sub_job->fn = segment_complete;
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100667 sub_job->context = sub_job;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668 dispatch_job(sub_job);
669
670 } else if (atomic_dec_and_test(&job->sub_jobs)) {
671
672 /*
Mikulas Patocka340cd442009-04-09 00:27:17 +0100673 * Queue the completion callback to the kcopyd thread.
674 *
675 * Some callers assume that all the completions are called
676 * from a single thread and don't race with each other.
677 *
678 * We must not call the callback directly here because this
679 * code may not be executing in the thread.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 */
Mikulas Patocka340cd442009-04-09 00:27:17 +0100681 push(&kc->complete_jobs, job);
682 wake(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 }
684}
685
686/*
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100687 * Create some sub jobs to share the work between them.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688 */
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100689static void split_job(struct kcopyd_job *master_job)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690{
691 int i;
692
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100693 atomic_inc(&master_job->kc->nr_jobs);
Mikulas Patocka340cd442009-04-09 00:27:17 +0100694
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100695 atomic_set(&master_job->sub_jobs, SPLIT_COUNT);
696 for (i = 0; i < SPLIT_COUNT; i++) {
697 master_job[i + 1].master_job = master_job;
698 segment_complete(0, 0u, &master_job[i + 1]);
699 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700}
701
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100702int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
703 unsigned int num_dests, struct dm_io_region *dests,
704 unsigned int flags, dm_kcopyd_notify_fn fn, void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705{
706 struct kcopyd_job *job;
Mike Snitzer70d6c402012-12-21 20:23:37 +0000707 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708
709 /*
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100710 * Allocate an array of jobs consisting of one master job
711 * followed by SPLIT_COUNT sub jobs.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 */
Mikulas Patocka08d87572008-04-24 21:43:46 +0100713 job = mempool_alloc(kc->job_pool, GFP_NOIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714
715 /*
716 * set up for the read.
717 */
718 job->kc = kc;
719 job->flags = flags;
720 job->read_err = 0;
721 job->write_err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722
723 job->num_dests = num_dests;
724 memcpy(&job->dests, dests, sizeof(*dests) * num_dests);
725
Mikulas Patocka7f069652011-10-31 20:18:58 +0000726 if (from) {
727 job->source = *from;
728 job->pages = NULL;
729 job->rw = READ;
730 } else {
731 memset(&job->source, 0, sizeof job->source);
732 job->source.count = job->dests[0].count;
733 job->pages = &zero_page_list;
Mike Snitzer70d6c402012-12-21 20:23:37 +0000734
735 /*
736 * Use WRITE SAME to optimize zeroing if all dests support it.
737 */
Mike Christiee6047142016-06-05 14:32:04 -0500738 job->rw = REQ_OP_WRITE_SAME;
Mike Snitzer70d6c402012-12-21 20:23:37 +0000739 for (i = 0; i < job->num_dests; i++)
740 if (!bdev_write_same(job->dests[i].bdev)) {
741 job->rw = WRITE;
742 break;
743 }
Mikulas Patocka7f069652011-10-31 20:18:58 +0000744 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745
746 job->fn = fn;
747 job->context = context;
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100748 job->master_job = job;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749
Mikulas Patockaa705a342011-05-29 13:02:58 +0100750 if (job->source.count <= SUB_JOB_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 dispatch_job(job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 else {
Matthias Kaehlckedef5b5b2007-10-19 22:38:52 +0100753 mutex_init(&job->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 job->progress = 0;
755 split_job(job);
756 }
757
758 return 0;
759}
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100760EXPORT_SYMBOL(dm_kcopyd_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
Mikulas Patocka7f069652011-10-31 20:18:58 +0000762int dm_kcopyd_zero(struct dm_kcopyd_client *kc,
763 unsigned num_dests, struct dm_io_region *dests,
764 unsigned flags, dm_kcopyd_notify_fn fn, void *context)
765{
766 return dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context);
767}
768EXPORT_SYMBOL(dm_kcopyd_zero);
769
Mikulas Patockaa6e50b42011-08-02 12:32:04 +0100770void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,
771 dm_kcopyd_notify_fn fn, void *context)
772{
773 struct kcopyd_job *job;
774
775 job = mempool_alloc(kc->job_pool, GFP_NOIO);
776
777 memset(job, 0, sizeof(struct kcopyd_job));
778 job->kc = kc;
779 job->fn = fn;
780 job->context = context;
Alasdair G Kergond136f2e2011-10-23 20:55:17 +0100781 job->master_job = job;
Mikulas Patockaa6e50b42011-08-02 12:32:04 +0100782
783 atomic_inc(&kc->nr_jobs);
784
785 return job;
786}
787EXPORT_SYMBOL(dm_kcopyd_prepare_callback);
788
789void dm_kcopyd_do_callback(void *j, int read_err, unsigned long write_err)
790{
791 struct kcopyd_job *job = j;
792 struct dm_kcopyd_client *kc = job->kc;
793
794 job->read_err = read_err;
795 job->write_err = write_err;
796
797 push(&kc->complete_jobs, job);
798 wake(kc);
799}
800EXPORT_SYMBOL(dm_kcopyd_do_callback);
801
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802/*
803 * Cancels a kcopyd job, eg. someone might be deactivating a
804 * mirror.
805 */
Adrian Bunk0b563062006-01-06 00:20:08 -0800806#if 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807int kcopyd_cancel(struct kcopyd_job *job, int block)
808{
809 /* FIXME: finish */
810 return -1;
811}
Adrian Bunk0b563062006-01-06 00:20:08 -0800812#endif /* 0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813
814/*-----------------------------------------------------------------
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100815 * Client setup
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 *---------------------------------------------------------------*/
Mikulas Patockadf5d2e92013-03-01 22:45:49 +0000817struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *throttle)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818{
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100819 int r = -ENOMEM;
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100820 struct dm_kcopyd_client *kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 kc = kmalloc(sizeof(*kc), GFP_KERNEL);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100823 if (!kc)
Mikulas Patockafa34ce72011-05-29 13:03:13 +0100824 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100826 spin_lock_init(&kc->job_lock);
827 INIT_LIST_HEAD(&kc->complete_jobs);
828 INIT_LIST_HEAD(&kc->io_jobs);
829 INIT_LIST_HEAD(&kc->pages_jobs);
Mikulas Patockadf5d2e92013-03-01 22:45:49 +0000830 kc->throttle = throttle;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100831
Mikulas Patocka08d87572008-04-24 21:43:46 +0100832 kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100833 if (!kc->job_pool)
834 goto bad_slab;
Mikulas Patocka08d87572008-04-24 21:43:46 +0100835
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100836 INIT_WORK(&kc->kcopyd_work, do_work);
Tejun Heo670368a2013-07-30 08:40:21 -0400837 kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100838 if (!kc->kcopyd_wq)
839 goto bad_workqueue;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100840
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841 kc->pages = NULL;
Mikulas Patockad0471452011-05-29 13:03:07 +0100842 kc->nr_reserved_pages = kc->nr_free_pages = 0;
Mikulas Patocka5f43ba22011-05-29 13:03:11 +0100843 r = client_reserve_pages(kc, RESERVE_PAGES);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100844 if (r)
845 goto bad_client_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846
Mikulas Patockabda8efe2011-05-29 13:03:09 +0100847 kc->io_client = dm_io_client_create();
Milan Broz373a3922007-05-09 02:33:02 -0700848 if (IS_ERR(kc->io_client)) {
849 r = PTR_ERR(kc->io_client);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100850 goto bad_io_client;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 }
852
Alasdair G Kergon138728dc2006-03-27 01:17:50 -0800853 init_waitqueue_head(&kc->destroyq);
854 atomic_set(&kc->nr_jobs, 0);
855
Mikulas Patockafa34ce72011-05-29 13:03:13 +0100856 return kc;
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100857
858bad_io_client:
859 client_free_pages(kc);
860bad_client_pages:
861 destroy_workqueue(kc->kcopyd_wq);
862bad_workqueue:
863 mempool_destroy(kc->job_pool);
864bad_slab:
865 kfree(kc);
866
Mikulas Patockafa34ce72011-05-29 13:03:13 +0100867 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868}
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100869EXPORT_SYMBOL(dm_kcopyd_client_create);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100871void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872{
Alasdair G Kergon138728dc2006-03-27 01:17:50 -0800873 /* Wait for completion of all jobs submitted by this client. */
874 wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs));
875
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100876 BUG_ON(!list_empty(&kc->complete_jobs));
877 BUG_ON(!list_empty(&kc->io_jobs));
878 BUG_ON(!list_empty(&kc->pages_jobs));
879 destroy_workqueue(kc->kcopyd_wq);
Milan Broz373a3922007-05-09 02:33:02 -0700880 dm_io_client_destroy(kc->io_client);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881 client_free_pages(kc);
Mikulas Patocka08d87572008-04-24 21:43:46 +0100882 mempool_destroy(kc->job_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 kfree(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884}
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100885EXPORT_SYMBOL(dm_kcopyd_client_destroy);