blob: 68c02673263ba87b9749cda3e119838e75edf22a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2002 Sistina Software (UK) Limited.
Milan Broz373a3922007-05-09 02:33:02 -07003 * Copyright (C) 2006 Red Hat GmbH
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 *
7 * Kcopyd provides a simple interface for copying an area of one
8 * block-device to one or more other block-devices, with an asynchronous
9 * completion notification.
10 */
11
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +010012#include <linux/types.h>
Arun Sharma600634972011-07-26 16:09:06 -070013#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/blkdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/fs.h>
16#include <linux/init.h>
17#include <linux/list.h>
18#include <linux/mempool.h>
19#include <linux/module.h>
20#include <linux/pagemap.h>
21#include <linux/slab.h>
22#include <linux/vmalloc.h>
23#include <linux/workqueue.h>
Arjan van de Ven48c9c272006-03-27 01:18:20 -080024#include <linux/mutex.h>
Mikulas Patocka586e80e2008-10-21 17:44:59 +010025#include <linux/device-mapper.h>
Alasdair G Kergona765e202008-04-24 22:02:01 +010026#include <linux/dm-kcopyd.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +010028#include "dm.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
Mikulas Patockac6ea41f2011-05-29 13:03:00 +010030#define SUB_JOB_SIZE 128
31#define SPLIT_COUNT 8
32#define MIN_JOBS 8
Mikulas Patocka5f43ba22011-05-29 13:03:11 +010033#define RESERVE_PAGES (DIV_ROUND_UP(SUB_JOB_SIZE << SECTOR_SHIFT, PAGE_SIZE))
Mikulas Patockac6ea41f2011-05-29 13:03:00 +010034
Linus Torvalds1da177e2005-04-16 15:20:36 -070035/*-----------------------------------------------------------------
36 * Each kcopyd client has its own little pool of preallocated
37 * pages for kcopyd io.
38 *---------------------------------------------------------------*/
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +010039struct dm_kcopyd_client {
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 struct page_list *pages;
Mikulas Patockad0471452011-05-29 13:03:07 +010041 unsigned nr_reserved_pages;
42 unsigned nr_free_pages;
Alasdair G Kergon138728dc2006-03-27 01:17:50 -080043
Milan Broz373a3922007-05-09 02:33:02 -070044 struct dm_io_client *io_client;
45
Alasdair G Kergon138728dc2006-03-27 01:17:50 -080046 wait_queue_head_t destroyq;
47 atomic_t nr_jobs;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +010048
Mikulas Patocka08d87572008-04-24 21:43:46 +010049 mempool_t *job_pool;
50
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +010051 struct workqueue_struct *kcopyd_wq;
52 struct work_struct kcopyd_work;
53
54/*
55 * We maintain three lists of jobs:
56 *
57 * i) jobs waiting for pages
58 * ii) jobs that have pages, and are waiting for the io to be issued.
59 * iii) jobs that have completed.
60 *
61 * All three of these are protected by job_lock.
62 */
63 spinlock_t job_lock;
64 struct list_head complete_jobs;
65 struct list_head io_jobs;
66 struct list_head pages_jobs;
Linus Torvalds1da177e2005-04-16 15:20:36 -070067};
68
Mikulas Patocka7f069652011-10-31 20:18:58 +000069static struct page_list zero_page_list;
70
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +010071static void wake(struct dm_kcopyd_client *kc)
72{
73 queue_work(kc->kcopyd_wq, &kc->kcopyd_work);
74}
75
Mikulas Patockad0471452011-05-29 13:03:07 +010076/*
77 * Obtain one page for the use of kcopyd.
78 */
Mikulas Patockaf99b55e2011-05-29 13:03:04 +010079static struct page_list *alloc_pl(gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -070080{
81 struct page_list *pl;
82
Mikulas Patockaf99b55e2011-05-29 13:03:04 +010083 pl = kmalloc(sizeof(*pl), gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 if (!pl)
85 return NULL;
86
Mikulas Patockaf99b55e2011-05-29 13:03:04 +010087 pl->page = alloc_page(gfp);
Linus Torvalds1da177e2005-04-16 15:20:36 -070088 if (!pl->page) {
89 kfree(pl);
90 return NULL;
91 }
92
93 return pl;
94}
95
96static void free_pl(struct page_list *pl)
97{
98 __free_page(pl->page);
99 kfree(pl);
100}
101
Mikulas Patockad0471452011-05-29 13:03:07 +0100102/*
103 * Add the provided pages to a client's free page list, releasing
104 * back to the system any beyond the reserved_pages limit.
105 */
106static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl)
107{
108 struct page_list *next;
109
110 do {
111 next = pl->next;
112
113 if (kc->nr_free_pages >= kc->nr_reserved_pages)
114 free_pl(pl);
115 else {
116 pl->next = kc->pages;
117 kc->pages = pl;
118 kc->nr_free_pages++;
119 }
120
121 pl = next;
122 } while (pl);
123}
124
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100125static int kcopyd_get_pages(struct dm_kcopyd_client *kc,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 unsigned int nr, struct page_list **pages)
127{
128 struct page_list *pl;
129
Mikulas Patockad0471452011-05-29 13:03:07 +0100130 *pages = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131
Mikulas Patockad0471452011-05-29 13:03:07 +0100132 do {
133 pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY);
134 if (unlikely(!pl)) {
135 /* Use reserved pages */
136 pl = kc->pages;
137 if (unlikely(!pl))
138 goto out_of_memory;
139 kc->pages = pl->next;
140 kc->nr_free_pages--;
141 }
142 pl->next = *pages;
143 *pages = pl;
144 } while (--nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147
Mikulas Patockad0471452011-05-29 13:03:07 +0100148out_of_memory:
149 if (*pages)
150 kcopyd_put_pages(kc, *pages);
151 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152}
153
154/*
155 * These three functions resize the page pool.
156 */
157static void drop_pages(struct page_list *pl)
158{
159 struct page_list *next;
160
161 while (pl) {
162 next = pl->next;
163 free_pl(pl);
164 pl = next;
165 }
166}
167
Mikulas Patockad0471452011-05-29 13:03:07 +0100168/*
169 * Allocate and reserve nr_pages for the use of a specific client.
170 */
171static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172{
Mikulas Patockad0471452011-05-29 13:03:07 +0100173 unsigned i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 struct page_list *pl = NULL, *next;
175
Mikulas Patockad0471452011-05-29 13:03:07 +0100176 for (i = 0; i < nr_pages; i++) {
Mikulas Patockaf99b55e2011-05-29 13:03:04 +0100177 next = alloc_pl(GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 if (!next) {
179 if (pl)
180 drop_pages(pl);
181 return -ENOMEM;
182 }
183 next->next = pl;
184 pl = next;
185 }
186
Mikulas Patockad0471452011-05-29 13:03:07 +0100187 kc->nr_reserved_pages += nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 kcopyd_put_pages(kc, pl);
Mikulas Patockad0471452011-05-29 13:03:07 +0100189
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 return 0;
191}
192
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100193static void client_free_pages(struct dm_kcopyd_client *kc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194{
Mikulas Patockad0471452011-05-29 13:03:07 +0100195 BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 drop_pages(kc->pages);
197 kc->pages = NULL;
Mikulas Patockad0471452011-05-29 13:03:07 +0100198 kc->nr_free_pages = kc->nr_reserved_pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199}
200
201/*-----------------------------------------------------------------
202 * kcopyd_jobs need to be allocated by the *clients* of kcopyd,
203 * for this reason we use a mempool to prevent the client from
204 * ever having to do io (which could cause a deadlock).
205 *---------------------------------------------------------------*/
206struct kcopyd_job {
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100207 struct dm_kcopyd_client *kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 struct list_head list;
209 unsigned long flags;
210
211 /*
212 * Error state of the job.
213 */
214 int read_err;
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700215 unsigned long write_err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217 /*
218 * Either READ or WRITE
219 */
220 int rw;
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +0100221 struct dm_io_region source;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
223 /*
224 * The destinations for the transfer.
225 */
226 unsigned int num_dests;
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100227 struct dm_io_region dests[DM_KCOPYD_MAX_REGIONS];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 struct page_list *pages;
230
231 /*
232 * Set this to ensure you are notified when the job has
233 * completed. 'context' is for callback to use.
234 */
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100235 dm_kcopyd_notify_fn fn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 void *context;
237
238 /*
239 * These fields are only used if the job has been split
240 * into more manageable parts.
241 */
Matthias Kaehlckedef5b5b2007-10-19 22:38:52 +0100242 struct mutex lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 atomic_t sub_jobs;
244 sector_t progress;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100246 struct kcopyd_job *master_job;
247};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248
Christoph Lametere18b8902006-12-06 20:33:20 -0800249static struct kmem_cache *_job_cache;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100251int __init dm_kcopyd_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252{
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100253 _job_cache = kmem_cache_create("kcopyd_job",
254 sizeof(struct kcopyd_job) * (SPLIT_COUNT + 1),
255 __alignof__(struct kcopyd_job), 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 if (!_job_cache)
257 return -ENOMEM;
258
Mikulas Patocka7f069652011-10-31 20:18:58 +0000259 zero_page_list.next = &zero_page_list;
260 zero_page_list.page = ZERO_PAGE(0);
261
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 return 0;
263}
264
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100265void dm_kcopyd_exit(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 kmem_cache_destroy(_job_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 _job_cache = NULL;
269}
270
271/*
272 * Functions to push and pop a job onto the head of a given job
273 * list.
274 */
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100275static struct kcopyd_job *pop(struct list_head *jobs,
276 struct dm_kcopyd_client *kc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277{
278 struct kcopyd_job *job = NULL;
279 unsigned long flags;
280
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100281 spin_lock_irqsave(&kc->job_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282
283 if (!list_empty(jobs)) {
284 job = list_entry(jobs->next, struct kcopyd_job, list);
285 list_del(&job->list);
286 }
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100287 spin_unlock_irqrestore(&kc->job_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
289 return job;
290}
291
Alasdair G Kergon028867a2007-07-12 17:26:32 +0100292static void push(struct list_head *jobs, struct kcopyd_job *job)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293{
294 unsigned long flags;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100295 struct dm_kcopyd_client *kc = job->kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100297 spin_lock_irqsave(&kc->job_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 list_add_tail(&job->list, jobs);
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100299 spin_unlock_irqrestore(&kc->job_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300}
301
Kazuo Itob673c3a2008-10-21 17:44:50 +0100302
303static void push_head(struct list_head *jobs, struct kcopyd_job *job)
304{
305 unsigned long flags;
306 struct dm_kcopyd_client *kc = job->kc;
307
308 spin_lock_irqsave(&kc->job_lock, flags);
309 list_add(&job->list, jobs);
310 spin_unlock_irqrestore(&kc->job_lock, flags);
311}
312
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313/*
314 * These three functions process 1 item from the corresponding
315 * job list.
316 *
317 * They return:
318 * < 0: error
319 * 0: success
320 * > 0: can't process yet.
321 */
322static int run_complete_job(struct kcopyd_job *job)
323{
324 void *context = job->context;
325 int read_err = job->read_err;
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700326 unsigned long write_err = job->write_err;
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100327 dm_kcopyd_notify_fn fn = job->fn;
328 struct dm_kcopyd_client *kc = job->kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329
Mikulas Patocka7f069652011-10-31 20:18:58 +0000330 if (job->pages && job->pages != &zero_page_list)
Mikulas Patocka73830852009-04-09 00:27:16 +0100331 kcopyd_put_pages(kc, job->pages);
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100332 /*
333 * If this is the master job, the sub jobs have already
334 * completed so we can free everything.
335 */
336 if (job->master_job == job)
337 mempool_free(job, kc->job_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 fn(read_err, write_err, context);
Alasdair G Kergon138728dc2006-03-27 01:17:50 -0800339
340 if (atomic_dec_and_test(&kc->nr_jobs))
341 wake_up(&kc->destroyq);
342
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 return 0;
344}
345
346static void complete_io(unsigned long error, void *context)
347{
348 struct kcopyd_job *job = (struct kcopyd_job *) context;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100349 struct dm_kcopyd_client *kc = job->kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
351 if (error) {
Mike Snitzer70d6c402012-12-21 20:23:37 +0000352 if (job->rw & WRITE)
Jonathan Brassowce503f52006-06-26 00:27:30 -0700353 job->write_err |= error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 else
355 job->read_err = 1;
356
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100357 if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100358 push(&kc->complete_jobs, job);
359 wake(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 return;
361 }
362 }
363
Mike Snitzer70d6c402012-12-21 20:23:37 +0000364 if (job->rw & WRITE)
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100365 push(&kc->complete_jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366
367 else {
368 job->rw = WRITE;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100369 push(&kc->io_jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 }
371
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100372 wake(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373}
374
375/*
376 * Request io on as many buffer heads as we can currently get for
377 * a particular job.
378 */
379static int run_io_job(struct kcopyd_job *job)
380{
381 int r;
Milan Broz373a3922007-05-09 02:33:02 -0700382 struct dm_io_request io_req = {
Mikulas Patocka8d35d3e2011-01-13 19:59:50 +0000383 .bi_rw = job->rw,
Milan Broz373a3922007-05-09 02:33:02 -0700384 .mem.type = DM_IO_PAGE_LIST,
385 .mem.ptr.pl = job->pages,
Mikulas Patocka4622afb2011-08-02 12:32:02 +0100386 .mem.offset = 0,
Milan Broz373a3922007-05-09 02:33:02 -0700387 .notify.fn = complete_io,
388 .notify.context = job,
389 .client = job->kc->io_client,
390 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391
Jens Axboe7eaceac2011-03-10 08:52:07 +0100392 if (job->rw == READ)
Milan Broz373a3922007-05-09 02:33:02 -0700393 r = dm_io(&io_req, 1, &job->source, NULL);
Jens Axboe721a9602011-03-09 11:56:30 +0100394 else
Milan Broz373a3922007-05-09 02:33:02 -0700395 r = dm_io(&io_req, job->num_dests, job->dests, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
397 return r;
398}
399
400static int run_pages_job(struct kcopyd_job *job)
401{
402 int r;
Mikulas Patocka5bf45a32011-08-02 12:32:02 +0100403 unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404
Mikulas Patocka5bf45a32011-08-02 12:32:02 +0100405 r = kcopyd_get_pages(job->kc, nr_pages, &job->pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 if (!r) {
407 /* this job is ready for io */
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100408 push(&job->kc->io_jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 return 0;
410 }
411
412 if (r == -ENOMEM)
413 /* can't complete now */
414 return 1;
415
416 return r;
417}
418
419/*
420 * Run through a list for as long as possible. Returns the count
421 * of successful jobs.
422 */
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100423static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc,
424 int (*fn) (struct kcopyd_job *))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425{
426 struct kcopyd_job *job;
427 int r, count = 0;
428
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100429 while ((job = pop(jobs, kc))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430
431 r = fn(job);
432
433 if (r < 0) {
434 /* error this rogue job */
Mike Snitzer70d6c402012-12-21 20:23:37 +0000435 if (job->rw & WRITE)
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700436 job->write_err = (unsigned long) -1L;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 else
438 job->read_err = 1;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100439 push(&kc->complete_jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 break;
441 }
442
443 if (r > 0) {
444 /*
445 * We couldn't service this job ATM, so
446 * push this job back onto the list.
447 */
Kazuo Itob673c3a2008-10-21 17:44:50 +0100448 push_head(jobs, job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 break;
450 }
451
452 count++;
453 }
454
455 return count;
456}
457
458/*
459 * kcopyd does this every time it's woken up.
460 */
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100461static void do_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462{
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100463 struct dm_kcopyd_client *kc = container_of(work,
464 struct dm_kcopyd_client, kcopyd_work);
Jens Axboe7eaceac2011-03-10 08:52:07 +0100465 struct blk_plug plug;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100466
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 /*
468 * The order that these are called is *very* important.
469 * complete jobs can free some pages for pages jobs.
470 * Pages jobs when successful will jump onto the io jobs
471 * list. io jobs call wake when they complete and it all
472 * starts again.
473 */
Jens Axboe7eaceac2011-03-10 08:52:07 +0100474 blk_start_plug(&plug);
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100475 process_jobs(&kc->complete_jobs, kc, run_complete_job);
476 process_jobs(&kc->pages_jobs, kc, run_pages_job);
477 process_jobs(&kc->io_jobs, kc, run_io_job);
Jens Axboe7eaceac2011-03-10 08:52:07 +0100478 blk_finish_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479}
480
481/*
482 * If we are copying a small region we just dispatch a single job
483 * to do the copy, otherwise the io has to be split up into many
484 * jobs.
485 */
486static void dispatch_job(struct kcopyd_job *job)
487{
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100488 struct dm_kcopyd_client *kc = job->kc;
489 atomic_inc(&kc->nr_jobs);
Mikulas Patocka9ca170a2009-12-10 23:52:13 +0000490 if (unlikely(!job->source.count))
491 push(&kc->complete_jobs, job);
Mikulas Patocka7f069652011-10-31 20:18:58 +0000492 else if (job->pages == &zero_page_list)
493 push(&kc->io_jobs, job);
Mikulas Patocka9ca170a2009-12-10 23:52:13 +0000494 else
495 push(&kc->pages_jobs, job);
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100496 wake(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497}
498
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700499static void segment_complete(int read_err, unsigned long write_err,
500 void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700501{
502 /* FIXME: tidy this function */
503 sector_t progress = 0;
504 sector_t count = 0;
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100505 struct kcopyd_job *sub_job = (struct kcopyd_job *) context;
506 struct kcopyd_job *job = sub_job->master_job;
Mikulas Patocka73830852009-04-09 00:27:16 +0100507 struct dm_kcopyd_client *kc = job->kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508
Matthias Kaehlckedef5b5b2007-10-19 22:38:52 +0100509 mutex_lock(&job->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510
511 /* update the error */
512 if (read_err)
513 job->read_err = 1;
514
515 if (write_err)
Jonathan Brassowce503f52006-06-26 00:27:30 -0700516 job->write_err |= write_err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517
518 /*
519 * Only dispatch more work if there hasn't been an error.
520 */
521 if ((!job->read_err && !job->write_err) ||
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100522 test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 /* get the next chunk of work */
524 progress = job->progress;
525 count = job->source.count - progress;
526 if (count) {
527 if (count > SUB_JOB_SIZE)
528 count = SUB_JOB_SIZE;
529
530 job->progress += count;
531 }
532 }
Matthias Kaehlckedef5b5b2007-10-19 22:38:52 +0100533 mutex_unlock(&job->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534
535 if (count) {
536 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537
538 *sub_job = *job;
539 sub_job->source.sector += progress;
540 sub_job->source.count = count;
541
542 for (i = 0; i < job->num_dests; i++) {
543 sub_job->dests[i].sector += progress;
544 sub_job->dests[i].count = count;
545 }
546
547 sub_job->fn = segment_complete;
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100548 sub_job->context = sub_job;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 dispatch_job(sub_job);
550
551 } else if (atomic_dec_and_test(&job->sub_jobs)) {
552
553 /*
Mikulas Patocka340cd442009-04-09 00:27:17 +0100554 * Queue the completion callback to the kcopyd thread.
555 *
556 * Some callers assume that all the completions are called
557 * from a single thread and don't race with each other.
558 *
559 * We must not call the callback directly here because this
560 * code may not be executing in the thread.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 */
Mikulas Patocka340cd442009-04-09 00:27:17 +0100562 push(&kc->complete_jobs, job);
563 wake(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 }
565}
566
567/*
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100568 * Create some sub jobs to share the work between them.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 */
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100570static void split_job(struct kcopyd_job *master_job)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571{
572 int i;
573
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100574 atomic_inc(&master_job->kc->nr_jobs);
Mikulas Patocka340cd442009-04-09 00:27:17 +0100575
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100576 atomic_set(&master_job->sub_jobs, SPLIT_COUNT);
577 for (i = 0; i < SPLIT_COUNT; i++) {
578 master_job[i + 1].master_job = master_job;
579 segment_complete(0, 0u, &master_job[i + 1]);
580 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581}
582
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100583int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
584 unsigned int num_dests, struct dm_io_region *dests,
585 unsigned int flags, dm_kcopyd_notify_fn fn, void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586{
587 struct kcopyd_job *job;
Mike Snitzer70d6c402012-12-21 20:23:37 +0000588 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589
590 /*
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100591 * Allocate an array of jobs consisting of one master job
592 * followed by SPLIT_COUNT sub jobs.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593 */
Mikulas Patocka08d87572008-04-24 21:43:46 +0100594 job = mempool_alloc(kc->job_pool, GFP_NOIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
596 /*
597 * set up for the read.
598 */
599 job->kc = kc;
600 job->flags = flags;
601 job->read_err = 0;
602 job->write_err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603
604 job->num_dests = num_dests;
605 memcpy(&job->dests, dests, sizeof(*dests) * num_dests);
606
Mikulas Patocka7f069652011-10-31 20:18:58 +0000607 if (from) {
608 job->source = *from;
609 job->pages = NULL;
610 job->rw = READ;
611 } else {
612 memset(&job->source, 0, sizeof job->source);
613 job->source.count = job->dests[0].count;
614 job->pages = &zero_page_list;
Mike Snitzer70d6c402012-12-21 20:23:37 +0000615
616 /*
617 * Use WRITE SAME to optimize zeroing if all dests support it.
618 */
619 job->rw = WRITE | REQ_WRITE_SAME;
620 for (i = 0; i < job->num_dests; i++)
621 if (!bdev_write_same(job->dests[i].bdev)) {
622 job->rw = WRITE;
623 break;
624 }
Mikulas Patocka7f069652011-10-31 20:18:58 +0000625 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626
627 job->fn = fn;
628 job->context = context;
Mikulas Patockac6ea41f2011-05-29 13:03:00 +0100629 job->master_job = job;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630
Mikulas Patockaa705a342011-05-29 13:02:58 +0100631 if (job->source.count <= SUB_JOB_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 dispatch_job(job);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 else {
Matthias Kaehlckedef5b5b2007-10-19 22:38:52 +0100634 mutex_init(&job->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 job->progress = 0;
636 split_job(job);
637 }
638
639 return 0;
640}
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100641EXPORT_SYMBOL(dm_kcopyd_copy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642
Mikulas Patocka7f069652011-10-31 20:18:58 +0000643int dm_kcopyd_zero(struct dm_kcopyd_client *kc,
644 unsigned num_dests, struct dm_io_region *dests,
645 unsigned flags, dm_kcopyd_notify_fn fn, void *context)
646{
647 return dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context);
648}
649EXPORT_SYMBOL(dm_kcopyd_zero);
650
Mikulas Patockaa6e50b42011-08-02 12:32:04 +0100651void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,
652 dm_kcopyd_notify_fn fn, void *context)
653{
654 struct kcopyd_job *job;
655
656 job = mempool_alloc(kc->job_pool, GFP_NOIO);
657
658 memset(job, 0, sizeof(struct kcopyd_job));
659 job->kc = kc;
660 job->fn = fn;
661 job->context = context;
Alasdair G Kergond136f2e2011-10-23 20:55:17 +0100662 job->master_job = job;
Mikulas Patockaa6e50b42011-08-02 12:32:04 +0100663
664 atomic_inc(&kc->nr_jobs);
665
666 return job;
667}
668EXPORT_SYMBOL(dm_kcopyd_prepare_callback);
669
670void dm_kcopyd_do_callback(void *j, int read_err, unsigned long write_err)
671{
672 struct kcopyd_job *job = j;
673 struct dm_kcopyd_client *kc = job->kc;
674
675 job->read_err = read_err;
676 job->write_err = write_err;
677
678 push(&kc->complete_jobs, job);
679 wake(kc);
680}
681EXPORT_SYMBOL(dm_kcopyd_do_callback);
682
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683/*
684 * Cancels a kcopyd job, eg. someone might be deactivating a
685 * mirror.
686 */
Adrian Bunk0b563062006-01-06 00:20:08 -0800687#if 0
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688int kcopyd_cancel(struct kcopyd_job *job, int block)
689{
690 /* FIXME: finish */
691 return -1;
692}
Adrian Bunk0b563062006-01-06 00:20:08 -0800693#endif /* 0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694
695/*-----------------------------------------------------------------
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100696 * Client setup
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697 *---------------------------------------------------------------*/
Mikulas Patockafa34ce72011-05-29 13:03:13 +0100698struct dm_kcopyd_client *dm_kcopyd_client_create(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699{
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100700 int r = -ENOMEM;
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100701 struct dm_kcopyd_client *kc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 kc = kmalloc(sizeof(*kc), GFP_KERNEL);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100704 if (!kc)
Mikulas Patockafa34ce72011-05-29 13:03:13 +0100705 return ERR_PTR(-ENOMEM);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100707 spin_lock_init(&kc->job_lock);
708 INIT_LIST_HEAD(&kc->complete_jobs);
709 INIT_LIST_HEAD(&kc->io_jobs);
710 INIT_LIST_HEAD(&kc->pages_jobs);
711
Mikulas Patocka08d87572008-04-24 21:43:46 +0100712 kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100713 if (!kc->job_pool)
714 goto bad_slab;
Mikulas Patocka08d87572008-04-24 21:43:46 +0100715
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100716 INIT_WORK(&kc->kcopyd_work, do_work);
Tejun Heo9c4376d2011-01-13 19:59:58 +0000717 kc->kcopyd_wq = alloc_workqueue("kcopyd",
718 WQ_NON_REENTRANT | WQ_MEM_RECLAIM, 0);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100719 if (!kc->kcopyd_wq)
720 goto bad_workqueue;
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100721
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722 kc->pages = NULL;
Mikulas Patockad0471452011-05-29 13:03:07 +0100723 kc->nr_reserved_pages = kc->nr_free_pages = 0;
Mikulas Patocka5f43ba22011-05-29 13:03:11 +0100724 r = client_reserve_pages(kc, RESERVE_PAGES);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100725 if (r)
726 goto bad_client_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727
Mikulas Patockabda8efe2011-05-29 13:03:09 +0100728 kc->io_client = dm_io_client_create();
Milan Broz373a3922007-05-09 02:33:02 -0700729 if (IS_ERR(kc->io_client)) {
730 r = PTR_ERR(kc->io_client);
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100731 goto bad_io_client;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732 }
733
Alasdair G Kergon138728dc2006-03-27 01:17:50 -0800734 init_waitqueue_head(&kc->destroyq);
735 atomic_set(&kc->nr_jobs, 0);
736
Mikulas Patockafa34ce72011-05-29 13:03:13 +0100737 return kc;
Mikulas Patocka945fa4d2008-04-24 21:43:49 +0100738
739bad_io_client:
740 client_free_pages(kc);
741bad_client_pages:
742 destroy_workqueue(kc->kcopyd_wq);
743bad_workqueue:
744 mempool_destroy(kc->job_pool);
745bad_slab:
746 kfree(kc);
747
Mikulas Patockafa34ce72011-05-29 13:03:13 +0100748 return ERR_PTR(r);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749}
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100750EXPORT_SYMBOL(dm_kcopyd_client_create);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100752void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753{
Alasdair G Kergon138728dc2006-03-27 01:17:50 -0800754 /* Wait for completion of all jobs submitted by this client. */
755 wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs));
756
Mikulas Patocka8c0cbc22008-04-24 21:43:44 +0100757 BUG_ON(!list_empty(&kc->complete_jobs));
758 BUG_ON(!list_empty(&kc->io_jobs));
759 BUG_ON(!list_empty(&kc->pages_jobs));
760 destroy_workqueue(kc->kcopyd_wq);
Milan Broz373a3922007-05-09 02:33:02 -0700761 dm_io_client_destroy(kc->io_client);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762 client_free_pages(kc);
Mikulas Patocka08d87572008-04-24 21:43:46 +0100763 mempool_destroy(kc->job_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 kfree(kc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765}
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100766EXPORT_SYMBOL(dm_kcopyd_client_destroy);