blob: a086014dcd49915d5dad8b2c57b9bbfa30b39e5c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * raid5.c : Multiple Devices driver for Linux
3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
4 * Copyright (C) 1999, 2000 Ingo Molnar
NeilBrown16a53ec2006-06-26 00:27:38 -07005 * Copyright (C) 2002, 2003 H. Peter Anvin
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 *
NeilBrown16a53ec2006-06-26 00:27:38 -07007 * RAID-4/5/6 management functions.
8 * Thanks to Penguin Computing for making the RAID-6 development possible
9 * by donating a test server!
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
14 * any later version.
15 *
16 * You should have received a copy of the GNU General Public License
17 * (for example /usr/src/linux/COPYING); if not, write to the Free
18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
NeilBrownae3c20c2006-07-10 04:44:17 -070021/*
22 * BITMAP UNPLUGGING:
23 *
24 * The sequencing for updating the bitmap reliably is a little
25 * subtle (and I got it wrong the first time) so it deserves some
26 * explanation.
27 *
28 * We group bitmap updates into batches. Each batch has a number.
29 * We may write out several batches at once, but that isn't very important.
NeilBrown7c13edc2011-04-18 18:25:43 +100030 * conf->seq_write is the number of the last batch successfully written.
31 * conf->seq_flush is the number of the last batch that was closed to
NeilBrownae3c20c2006-07-10 04:44:17 -070032 * new additions.
33 * When we discover that we will need to write to any block in a stripe
34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
NeilBrown7c13edc2011-04-18 18:25:43 +100035 * the number of the batch it will be in. This is seq_flush+1.
NeilBrownae3c20c2006-07-10 04:44:17 -070036 * When we are ready to do a write, if that batch hasn't been written yet,
37 * we plug the array and queue the stripe for later.
38 * When an unplug happens, we increment bm_flush, thus closing the current
39 * batch.
40 * When we notice that bm_flush > bm_write, we write out all pending updates
41 * to the bitmap, and advance bm_write to where bm_flush was.
42 * This may occasionally write a bit out twice, but is sure never to
43 * miss any bits.
44 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070045
NeilBrownbff61972009-03-31 14:33:13 +110046#include <linux/blkdev.h>
NeilBrownf6705572006-03-27 01:18:11 -080047#include <linux/kthread.h>
Dan Williamsf701d582009-03-31 15:09:39 +110048#include <linux/raid/pq.h>
Dan Williams91c00922007-01-02 13:52:30 -070049#include <linux/async_tx.h>
Paul Gortmaker056075c2011-07-03 13:58:33 -040050#include <linux/module.h>
Dan Williams07a3b412009-08-29 19:13:13 -070051#include <linux/async.h>
NeilBrownbff61972009-03-31 14:33:13 +110052#include <linux/seq_file.h>
Dan Williams36d1c642009-07-14 11:48:22 -070053#include <linux/cpu.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090054#include <linux/slab.h>
Christian Dietrich8bda4702011-07-27 11:00:36 +100055#include <linux/ratelimit.h>
Shaohua Li851c30c2013-08-28 14:30:16 +080056#include <linux/nodemask.h>
shli@kernel.org46d5b782014-12-15 12:57:02 +110057#include <linux/flex_array.h>
NeilBrowna9add5d2012-10-31 11:59:09 +110058#include <trace/events/block.h>
59
NeilBrown43b2e5d2009-03-31 14:33:13 +110060#include "md.h"
NeilBrownbff61972009-03-31 14:33:13 +110061#include "raid5.h"
Trela Maciej54071b32010-03-08 16:02:42 +110062#include "raid0.h"
Christoph Hellwigef740c32009-03-31 14:27:03 +110063#include "bitmap.h"
NeilBrown72626682005-09-09 16:23:54 -070064
Shaohua Li851c30c2013-08-28 14:30:16 +080065#define cpu_to_group(cpu) cpu_to_node(cpu)
66#define ANY_GROUP NUMA_NO_NODE
67
NeilBrown8e0e99b2014-10-02 13:45:00 +100068static bool devices_handle_discard_safely = false;
69module_param(devices_handle_discard_safely, bool, 0644);
70MODULE_PARM_DESC(devices_handle_discard_safely,
71 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
Shaohua Li851c30c2013-08-28 14:30:16 +080072static struct workqueue_struct *raid5_wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -070073/*
74 * Stripe cache
75 */
76
77#define NR_STRIPES 256
78#define STRIPE_SIZE PAGE_SIZE
79#define STRIPE_SHIFT (PAGE_SHIFT - 9)
80#define STRIPE_SECTORS (STRIPE_SIZE>>9)
81#define IO_THRESHOLD 1
Dan Williams8b3e6cd2008-04-28 02:15:53 -070082#define BYPASS_THRESHOLD 1
NeilBrownfccddba2006-01-06 00:20:33 -080083#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
Linus Torvalds1da177e2005-04-16 15:20:36 -070084#define HASH_MASK (NR_HASH - 1)
Shaohua Libfc90cb2013-08-29 15:40:32 +080085#define MAX_STRIPE_BATCH 8
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
NeilBrownd1688a62011-10-11 16:49:52 +110087static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
NeilBrowndb298e12011-10-07 14:23:00 +110088{
89 int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
90 return &conf->stripe_hashtbl[hash];
91}
Linus Torvalds1da177e2005-04-16 15:20:36 -070092
Shaohua Li566c09c2013-11-14 15:16:17 +110093static inline int stripe_hash_locks_hash(sector_t sect)
94{
95 return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
96}
97
98static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
99{
100 spin_lock_irq(conf->hash_locks + hash);
101 spin_lock(&conf->device_lock);
102}
103
104static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
105{
106 spin_unlock(&conf->device_lock);
107 spin_unlock_irq(conf->hash_locks + hash);
108}
109
110static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
111{
112 int i;
113 local_irq_disable();
114 spin_lock(conf->hash_locks);
115 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
116 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
117 spin_lock(&conf->device_lock);
118}
119
120static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
121{
122 int i;
123 spin_unlock(&conf->device_lock);
124 for (i = NR_STRIPE_HASH_LOCKS; i; i--)
125 spin_unlock(conf->hash_locks + i - 1);
126 local_irq_enable();
127}
128
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129/* bio's attached to a stripe+device for I/O are linked together in bi_sector
130 * order without overlap. There may be several bio's per stripe+device, and
131 * a bio could span several devices.
132 * When walking this list for a particular stripe+device, we must never proceed
133 * beyond a bio that extends past this device, as the next bio might no longer
134 * be valid.
NeilBrowndb298e12011-10-07 14:23:00 +1100135 * This function is used to determine the 'next' bio in the list, given the sector
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136 * of the current stripe+device
137 */
NeilBrowndb298e12011-10-07 14:23:00 +1100138static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
139{
Kent Overstreetaa8b57a2013-02-05 15:19:29 -0800140 int sectors = bio_sectors(bio);
Kent Overstreet4f024f32013-10-11 15:44:27 -0700141 if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
NeilBrowndb298e12011-10-07 14:23:00 +1100142 return bio->bi_next;
143 else
144 return NULL;
145}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146
Jens Axboe960e7392008-08-15 10:41:18 +0200147/*
Jens Axboe5b99c2f2008-08-15 10:56:11 +0200148 * We maintain a biased count of active stripes in the bottom 16 bits of
149 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
Jens Axboe960e7392008-08-15 10:41:18 +0200150 */
Shaohua Lie7836bd62012-07-19 16:01:31 +1000151static inline int raid5_bi_processed_stripes(struct bio *bio)
Jens Axboe960e7392008-08-15 10:41:18 +0200152{
Shaohua Lie7836bd62012-07-19 16:01:31 +1000153 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
154 return (atomic_read(segments) >> 16) & 0xffff;
Jens Axboe960e7392008-08-15 10:41:18 +0200155}
156
Shaohua Lie7836bd62012-07-19 16:01:31 +1000157static inline int raid5_dec_bi_active_stripes(struct bio *bio)
Jens Axboe960e7392008-08-15 10:41:18 +0200158{
Shaohua Lie7836bd62012-07-19 16:01:31 +1000159 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
160 return atomic_sub_return(1, segments) & 0xffff;
Jens Axboe960e7392008-08-15 10:41:18 +0200161}
162
Shaohua Lie7836bd62012-07-19 16:01:31 +1000163static inline void raid5_inc_bi_active_stripes(struct bio *bio)
Jens Axboe960e7392008-08-15 10:41:18 +0200164{
Shaohua Lie7836bd62012-07-19 16:01:31 +1000165 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
166 atomic_inc(segments);
Jens Axboe960e7392008-08-15 10:41:18 +0200167}
168
Shaohua Lie7836bd62012-07-19 16:01:31 +1000169static inline void raid5_set_bi_processed_stripes(struct bio *bio,
170 unsigned int cnt)
Jens Axboe960e7392008-08-15 10:41:18 +0200171{
Shaohua Lie7836bd62012-07-19 16:01:31 +1000172 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
173 int old, new;
Jens Axboe960e7392008-08-15 10:41:18 +0200174
Shaohua Lie7836bd62012-07-19 16:01:31 +1000175 do {
176 old = atomic_read(segments);
177 new = (old & 0xffff) | (cnt << 16);
178 } while (atomic_cmpxchg(segments, old, new) != old);
Jens Axboe960e7392008-08-15 10:41:18 +0200179}
180
Shaohua Lie7836bd62012-07-19 16:01:31 +1000181static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
Jens Axboe960e7392008-08-15 10:41:18 +0200182{
Shaohua Lie7836bd62012-07-19 16:01:31 +1000183 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
184 atomic_set(segments, cnt);
Jens Axboe960e7392008-08-15 10:41:18 +0200185}
186
NeilBrownd0dabf72009-03-31 14:39:38 +1100187/* Find first data disk in a raid6 stripe */
188static inline int raid6_d0(struct stripe_head *sh)
189{
NeilBrown67cc2b82009-03-31 14:39:38 +1100190 if (sh->ddf_layout)
191 /* ddf always start from first device */
192 return 0;
193 /* md starts just after Q block */
NeilBrownd0dabf72009-03-31 14:39:38 +1100194 if (sh->qd_idx == sh->disks - 1)
195 return 0;
196 else
197 return sh->qd_idx + 1;
198}
NeilBrown16a53ec2006-06-26 00:27:38 -0700199static inline int raid6_next_disk(int disk, int raid_disks)
200{
201 disk++;
202 return (disk < raid_disks) ? disk : 0;
203}
Dan Williamsa4456852007-07-09 11:56:43 -0700204
NeilBrownd0dabf72009-03-31 14:39:38 +1100205/* When walking through the disks in a raid5, starting at raid6_d0,
206 * We need to map each disk to a 'slot', where the data disks are slot
207 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
208 * is raid_disks-1. This help does that mapping.
209 */
NeilBrown67cc2b82009-03-31 14:39:38 +1100210static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
211 int *count, int syndrome_disks)
NeilBrownd0dabf72009-03-31 14:39:38 +1100212{
Dan Williams66295422009-10-19 18:09:32 -0700213 int slot = *count;
NeilBrown67cc2b82009-03-31 14:39:38 +1100214
NeilBrowne4424fe2009-10-16 16:27:34 +1100215 if (sh->ddf_layout)
Dan Williams66295422009-10-19 18:09:32 -0700216 (*count)++;
NeilBrownd0dabf72009-03-31 14:39:38 +1100217 if (idx == sh->pd_idx)
NeilBrown67cc2b82009-03-31 14:39:38 +1100218 return syndrome_disks;
NeilBrownd0dabf72009-03-31 14:39:38 +1100219 if (idx == sh->qd_idx)
NeilBrown67cc2b82009-03-31 14:39:38 +1100220 return syndrome_disks + 1;
NeilBrowne4424fe2009-10-16 16:27:34 +1100221 if (!sh->ddf_layout)
Dan Williams66295422009-10-19 18:09:32 -0700222 (*count)++;
NeilBrownd0dabf72009-03-31 14:39:38 +1100223 return slot;
224}
225
NeilBrown34a6f802015-08-14 12:07:57 +1000226static void return_io(struct bio_list *return_bi)
Dan Williamsa4456852007-07-09 11:56:43 -0700227{
NeilBrown34a6f802015-08-14 12:07:57 +1000228 struct bio *bi;
229 while ((bi = bio_list_pop(return_bi)) != NULL) {
Kent Overstreet4f024f32013-10-11 15:44:27 -0700230 bi->bi_iter.bi_size = 0;
Linus Torvalds0a82a8d2013-04-18 09:00:26 -0700231 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
232 bi, 0);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200233 bio_endio(bi);
Dan Williamsa4456852007-07-09 11:56:43 -0700234 }
235}
236
NeilBrownd1688a62011-10-11 16:49:52 +1100237static void print_raid5_conf (struct r5conf *conf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
Dan Williams600aa102008-06-28 08:32:05 +1000239static int stripe_operations_active(struct stripe_head *sh)
240{
241 return sh->check_state || sh->reconstruct_state ||
242 test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
243 test_bit(STRIPE_COMPUTE_RUN, &sh->state);
244}
245
Shaohua Li851c30c2013-08-28 14:30:16 +0800246static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
247{
248 struct r5conf *conf = sh->raid_conf;
249 struct r5worker_group *group;
Shaohua Libfc90cb2013-08-29 15:40:32 +0800250 int thread_cnt;
Shaohua Li851c30c2013-08-28 14:30:16 +0800251 int i, cpu = sh->cpu;
252
253 if (!cpu_online(cpu)) {
254 cpu = cpumask_any(cpu_online_mask);
255 sh->cpu = cpu;
256 }
257
258 if (list_empty(&sh->lru)) {
259 struct r5worker_group *group;
260 group = conf->worker_groups + cpu_to_group(cpu);
261 list_add_tail(&sh->lru, &group->handle_list);
Shaohua Libfc90cb2013-08-29 15:40:32 +0800262 group->stripes_cnt++;
263 sh->group = group;
Shaohua Li851c30c2013-08-28 14:30:16 +0800264 }
265
266 if (conf->worker_cnt_per_group == 0) {
267 md_wakeup_thread(conf->mddev->thread);
268 return;
269 }
270
271 group = conf->worker_groups + cpu_to_group(sh->cpu);
272
Shaohua Libfc90cb2013-08-29 15:40:32 +0800273 group->workers[0].working = true;
274 /* at least one worker should run to avoid race */
275 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
276
277 thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
278 /* wakeup more workers */
279 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
280 if (group->workers[i].working == false) {
281 group->workers[i].working = true;
282 queue_work_on(sh->cpu, raid5_wq,
283 &group->workers[i].work);
284 thread_cnt--;
285 }
286 }
Shaohua Li851c30c2013-08-28 14:30:16 +0800287}
288
Shaohua Li566c09c2013-11-14 15:16:17 +1100289static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
290 struct list_head *temp_inactive_list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291{
Shaohua Li4eb788d2012-07-19 16:01:31 +1000292 BUG_ON(!list_empty(&sh->lru));
293 BUG_ON(atomic_read(&conf->active_stripes)==0);
294 if (test_bit(STRIPE_HANDLE, &sh->state)) {
295 if (test_bit(STRIPE_DELAYED, &sh->state) &&
Jes Sorensenad3ab8b2015-01-29 12:38:29 -0500296 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
Shaohua Li4eb788d2012-07-19 16:01:31 +1000297 list_add_tail(&sh->lru, &conf->delayed_list);
Jes Sorensenad3ab8b2015-01-29 12:38:29 -0500298 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
Shaohua Li4eb788d2012-07-19 16:01:31 +1000299 sh->bm_seq - conf->seq_write > 0)
300 list_add_tail(&sh->lru, &conf->bitmap_list);
301 else {
302 clear_bit(STRIPE_DELAYED, &sh->state);
303 clear_bit(STRIPE_BIT_DELAY, &sh->state);
Shaohua Li851c30c2013-08-28 14:30:16 +0800304 if (conf->worker_cnt_per_group == 0) {
305 list_add_tail(&sh->lru, &conf->handle_list);
306 } else {
307 raid5_wakeup_stripe_thread(sh);
308 return;
309 }
Shaohua Li4eb788d2012-07-19 16:01:31 +1000310 }
311 md_wakeup_thread(conf->mddev->thread);
312 } else {
313 BUG_ON(stripe_operations_active(sh));
314 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
315 if (atomic_dec_return(&conf->preread_active_stripes)
316 < IO_THRESHOLD)
317 md_wakeup_thread(conf->mddev->thread);
318 atomic_dec(&conf->active_stripes);
Shaohua Li566c09c2013-11-14 15:16:17 +1100319 if (!test_bit(STRIPE_EXPANDING, &sh->state))
320 list_add_tail(&sh->lru, temp_inactive_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321 }
322}
NeilBrownd0dabf72009-03-31 14:39:38 +1100323
Shaohua Li566c09c2013-11-14 15:16:17 +1100324static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
325 struct list_head *temp_inactive_list)
Shaohua Li4eb788d2012-07-19 16:01:31 +1000326{
327 if (atomic_dec_and_test(&sh->count))
Shaohua Li566c09c2013-11-14 15:16:17 +1100328 do_release_stripe(conf, sh, temp_inactive_list);
329}
330
331/*
332 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
333 *
334 * Be careful: Only one task can add/delete stripes from temp_inactive_list at
335 * given time. Adding stripes only takes device lock, while deleting stripes
336 * only takes hash lock.
337 */
338static void release_inactive_stripe_list(struct r5conf *conf,
339 struct list_head *temp_inactive_list,
340 int hash)
341{
342 int size;
Yuanhan Liue9e4c372015-05-08 18:19:07 +1000343 unsigned long do_wakeup = 0;
344 int i = 0;
Shaohua Li566c09c2013-11-14 15:16:17 +1100345 unsigned long flags;
346
347 if (hash == NR_STRIPE_HASH_LOCKS) {
348 size = NR_STRIPE_HASH_LOCKS;
349 hash = NR_STRIPE_HASH_LOCKS - 1;
350 } else
351 size = 1;
352 while (size) {
353 struct list_head *list = &temp_inactive_list[size - 1];
354
355 /*
Shaohua Li6d036f72015-08-13 14:31:57 -0700356 * We don't hold any lock here yet, raid5_get_active_stripe() might
Shaohua Li566c09c2013-11-14 15:16:17 +1100357 * remove stripes from the list
358 */
359 if (!list_empty_careful(list)) {
360 spin_lock_irqsave(conf->hash_locks + hash, flags);
Shaohua Li4bda5562013-11-14 15:16:17 +1100361 if (list_empty(conf->inactive_list + hash) &&
362 !list_empty(list))
363 atomic_dec(&conf->empty_inactive_list_nr);
Shaohua Li566c09c2013-11-14 15:16:17 +1100364 list_splice_tail_init(list, conf->inactive_list + hash);
Yuanhan Liue9e4c372015-05-08 18:19:07 +1000365 do_wakeup |= 1 << hash;
Shaohua Li566c09c2013-11-14 15:16:17 +1100366 spin_unlock_irqrestore(conf->hash_locks + hash, flags);
367 }
368 size--;
369 hash--;
370 }
371
Yuanhan Liue9e4c372015-05-08 18:19:07 +1000372 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) {
373 if (do_wakeup & (1 << i))
374 wake_up(&conf->wait_for_stripe[i]);
375 }
376
Shaohua Li566c09c2013-11-14 15:16:17 +1100377 if (do_wakeup) {
Yuanhan Liub1b46482015-05-08 18:19:06 +1000378 if (atomic_read(&conf->active_stripes) == 0)
379 wake_up(&conf->wait_for_quiescent);
Shaohua Li566c09c2013-11-14 15:16:17 +1100380 if (conf->retry_read_aligned)
381 md_wakeup_thread(conf->mddev->thread);
382 }
Shaohua Li4eb788d2012-07-19 16:01:31 +1000383}
384
Shaohua Li773ca822013-08-27 17:50:39 +0800385/* should hold conf->device_lock already */
Shaohua Li566c09c2013-11-14 15:16:17 +1100386static int release_stripe_list(struct r5conf *conf,
387 struct list_head *temp_inactive_list)
Shaohua Li773ca822013-08-27 17:50:39 +0800388{
389 struct stripe_head *sh;
390 int count = 0;
391 struct llist_node *head;
392
393 head = llist_del_all(&conf->released_stripes);
Shaohua Lid265d9d2013-08-28 14:29:05 +0800394 head = llist_reverse_order(head);
Shaohua Li773ca822013-08-27 17:50:39 +0800395 while (head) {
Shaohua Li566c09c2013-11-14 15:16:17 +1100396 int hash;
397
Shaohua Li773ca822013-08-27 17:50:39 +0800398 sh = llist_entry(head, struct stripe_head, release_list);
399 head = llist_next(head);
400 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
401 smp_mb();
402 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
403 /*
404 * Don't worry the bit is set here, because if the bit is set
405 * again, the count is always > 1. This is true for
406 * STRIPE_ON_UNPLUG_LIST bit too.
407 */
Shaohua Li566c09c2013-11-14 15:16:17 +1100408 hash = sh->hash_lock_index;
409 __release_stripe(conf, sh, &temp_inactive_list[hash]);
Shaohua Li773ca822013-08-27 17:50:39 +0800410 count++;
411 }
412
413 return count;
414}
415
Shaohua Li6d036f72015-08-13 14:31:57 -0700416void raid5_release_stripe(struct stripe_head *sh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417{
NeilBrownd1688a62011-10-11 16:49:52 +1100418 struct r5conf *conf = sh->raid_conf;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419 unsigned long flags;
Shaohua Li566c09c2013-11-14 15:16:17 +1100420 struct list_head list;
421 int hash;
Shaohua Li773ca822013-08-27 17:50:39 +0800422 bool wakeup;
NeilBrown16a53ec2006-06-26 00:27:38 -0700423
Eivind Sartocf170f32014-05-28 13:39:23 +1000424 /* Avoid release_list until the last reference.
425 */
426 if (atomic_add_unless(&sh->count, -1, 1))
427 return;
428
majianpengad4068d2013-11-14 15:16:15 +1100429 if (unlikely(!conf->mddev->thread) ||
430 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
Shaohua Li773ca822013-08-27 17:50:39 +0800431 goto slow_path;
432 wakeup = llist_add(&sh->release_list, &conf->released_stripes);
433 if (wakeup)
434 md_wakeup_thread(conf->mddev->thread);
435 return;
436slow_path:
Shaohua Li4eb788d2012-07-19 16:01:31 +1000437 local_irq_save(flags);
Shaohua Li773ca822013-08-27 17:50:39 +0800438 /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
Shaohua Li4eb788d2012-07-19 16:01:31 +1000439 if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
Shaohua Li566c09c2013-11-14 15:16:17 +1100440 INIT_LIST_HEAD(&list);
441 hash = sh->hash_lock_index;
442 do_release_stripe(conf, sh, &list);
Shaohua Li4eb788d2012-07-19 16:01:31 +1000443 spin_unlock(&conf->device_lock);
Shaohua Li566c09c2013-11-14 15:16:17 +1100444 release_inactive_stripe_list(conf, &list, hash);
Shaohua Li4eb788d2012-07-19 16:01:31 +1000445 }
446 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447}
448
NeilBrownfccddba2006-01-06 00:20:33 -0800449static inline void remove_hash(struct stripe_head *sh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450{
Dan Williams45b42332007-07-09 11:56:43 -0700451 pr_debug("remove_hash(), stripe %llu\n",
452 (unsigned long long)sh->sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453
NeilBrownfccddba2006-01-06 00:20:33 -0800454 hlist_del_init(&sh->hash);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455}
456
NeilBrownd1688a62011-10-11 16:49:52 +1100457static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458{
NeilBrownfccddba2006-01-06 00:20:33 -0800459 struct hlist_head *hp = stripe_hash(conf, sh->sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460
Dan Williams45b42332007-07-09 11:56:43 -0700461 pr_debug("insert_hash(), stripe %llu\n",
462 (unsigned long long)sh->sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463
NeilBrownfccddba2006-01-06 00:20:33 -0800464 hlist_add_head(&sh->hash, hp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465}
466
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467/* find an idle stripe, make sure it is unhashed, and return it. */
Shaohua Li566c09c2013-11-14 15:16:17 +1100468static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469{
470 struct stripe_head *sh = NULL;
471 struct list_head *first;
472
Shaohua Li566c09c2013-11-14 15:16:17 +1100473 if (list_empty(conf->inactive_list + hash))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 goto out;
Shaohua Li566c09c2013-11-14 15:16:17 +1100475 first = (conf->inactive_list + hash)->next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 sh = list_entry(first, struct stripe_head, lru);
477 list_del_init(first);
478 remove_hash(sh);
479 atomic_inc(&conf->active_stripes);
Shaohua Li566c09c2013-11-14 15:16:17 +1100480 BUG_ON(hash != sh->hash_lock_index);
Shaohua Li4bda5562013-11-14 15:16:17 +1100481 if (list_empty(conf->inactive_list + hash))
482 atomic_inc(&conf->empty_inactive_list_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483out:
484 return sh;
485}
486
NeilBrowne4e11e32010-06-16 16:45:16 +1000487static void shrink_buffers(struct stripe_head *sh)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488{
489 struct page *p;
490 int i;
NeilBrowne4e11e32010-06-16 16:45:16 +1000491 int num = sh->raid_conf->pool_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492
NeilBrowne4e11e32010-06-16 16:45:16 +1000493 for (i = 0; i < num ; i++) {
Shaohua Lid592a992014-05-21 17:57:44 +0800494 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 p = sh->dev[i].page;
496 if (!p)
497 continue;
498 sh->dev[i].page = NULL;
NeilBrown2d1f3b52006-01-06 00:20:31 -0800499 put_page(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500 }
501}
502
NeilBrowna9683a72015-02-25 12:02:51 +1100503static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504{
505 int i;
NeilBrowne4e11e32010-06-16 16:45:16 +1000506 int num = sh->raid_conf->pool_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507
NeilBrowne4e11e32010-06-16 16:45:16 +1000508 for (i = 0; i < num; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 struct page *page;
510
NeilBrowna9683a72015-02-25 12:02:51 +1100511 if (!(page = alloc_page(gfp))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512 return 1;
513 }
514 sh->dev[i].page = page;
Shaohua Lid592a992014-05-21 17:57:44 +0800515 sh->dev[i].orig_page = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 }
517 return 0;
518}
519
NeilBrown784052e2009-03-31 15:19:07 +1100520static void raid5_build_block(struct stripe_head *sh, int i, int previous);
NeilBrownd1688a62011-10-11 16:49:52 +1100521static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
NeilBrown911d4ee2009-03-31 14:39:38 +1100522 struct stripe_head *sh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
NeilBrownb5663ba2009-03-31 14:39:38 +1100524static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525{
NeilBrownd1688a62011-10-11 16:49:52 +1100526 struct r5conf *conf = sh->raid_conf;
Shaohua Li566c09c2013-11-14 15:16:17 +1100527 int i, seq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528
Eric Sesterhenn78bafeb2006-04-02 13:31:42 +0200529 BUG_ON(atomic_read(&sh->count) != 0);
530 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
Dan Williams600aa102008-06-28 08:32:05 +1000531 BUG_ON(stripe_operations_active(sh));
shli@kernel.org59fc6302014-12-15 12:57:03 +1100532 BUG_ON(sh->batch_head);
Dan Williamsd84e0f12007-01-02 13:52:30 -0700533
Dan Williams45b42332007-07-09 11:56:43 -0700534 pr_debug("init_stripe called, stripe %llu\n",
Markus Stockhausenb8e6a152014-08-23 20:19:27 +1000535 (unsigned long long)sector);
Shaohua Li566c09c2013-11-14 15:16:17 +1100536retry:
537 seq = read_seqcount_begin(&conf->gen_lock);
NeilBrown86b42c72009-03-31 15:19:03 +1100538 sh->generation = conf->generation - previous;
NeilBrownb5663ba2009-03-31 14:39:38 +1100539 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540 sh->sector = sector;
NeilBrown911d4ee2009-03-31 14:39:38 +1100541 stripe_set_idx(sector, conf, previous, sh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 sh->state = 0;
543
NeilBrown7ecaa1e2006-03-27 01:18:08 -0800544 for (i = sh->disks; i--; ) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 struct r5dev *dev = &sh->dev[i];
546
Dan Williamsd84e0f12007-01-02 13:52:30 -0700547 if (dev->toread || dev->read || dev->towrite || dev->written ||
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 test_bit(R5_LOCKED, &dev->flags)) {
Dan Williamsd84e0f12007-01-02 13:52:30 -0700549 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 (unsigned long long)sh->sector, i, dev->toread,
Dan Williamsd84e0f12007-01-02 13:52:30 -0700551 dev->read, dev->towrite, dev->written,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 test_bit(R5_LOCKED, &dev->flags));
NeilBrown8cfa7b02011-07-27 11:00:36 +1000553 WARN_ON(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 }
555 dev->flags = 0;
NeilBrown784052e2009-03-31 15:19:07 +1100556 raid5_build_block(sh, i, previous);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 }
Shaohua Li566c09c2013-11-14 15:16:17 +1100558 if (read_seqcount_retry(&conf->gen_lock, seq))
559 goto retry;
shli@kernel.org7a87f432014-12-15 12:57:03 +1100560 sh->overwrite_disks = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700561 insert_hash(conf, sh);
Shaohua Li851c30c2013-08-28 14:30:16 +0800562 sh->cpu = smp_processor_id();
shli@kernel.orgda41ba62014-12-15 12:57:03 +1100563 set_bit(STRIPE_BATCH_READY, &sh->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564}
565
NeilBrownd1688a62011-10-11 16:49:52 +1100566static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
NeilBrown86b42c72009-03-31 15:19:03 +1100567 short generation)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568{
569 struct stripe_head *sh;
570
Dan Williams45b42332007-07-09 11:56:43 -0700571 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
Sasha Levinb67bfe02013-02-27 17:06:00 -0800572 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
NeilBrown86b42c72009-03-31 15:19:03 +1100573 if (sh->sector == sector && sh->generation == generation)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 return sh;
Dan Williams45b42332007-07-09 11:56:43 -0700575 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 return NULL;
577}
578
NeilBrown674806d2010-06-16 17:17:53 +1000579/*
580 * Need to check if array has failed when deciding whether to:
581 * - start an array
582 * - remove non-faulty devices
583 * - add a spare
584 * - allow a reshape
585 * This determination is simple when no reshape is happening.
586 * However if there is a reshape, we need to carefully check
587 * both the before and after sections.
588 * This is because some failed devices may only affect one
589 * of the two sections, and some non-in_sync devices may
590 * be insync in the section most affected by failed devices.
591 */
NeilBrown908f4fb2011-12-23 10:17:50 +1100592static int calc_degraded(struct r5conf *conf)
NeilBrown674806d2010-06-16 17:17:53 +1000593{
NeilBrown908f4fb2011-12-23 10:17:50 +1100594 int degraded, degraded2;
NeilBrown674806d2010-06-16 17:17:53 +1000595 int i;
NeilBrown674806d2010-06-16 17:17:53 +1000596
597 rcu_read_lock();
598 degraded = 0;
599 for (i = 0; i < conf->previous_raid_disks; i++) {
NeilBrown3cb03002011-10-11 16:45:26 +1100600 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
NeilBrowne5c86472012-09-19 12:52:30 +1000601 if (rdev && test_bit(Faulty, &rdev->flags))
602 rdev = rcu_dereference(conf->disks[i].replacement);
NeilBrown674806d2010-06-16 17:17:53 +1000603 if (!rdev || test_bit(Faulty, &rdev->flags))
604 degraded++;
605 else if (test_bit(In_sync, &rdev->flags))
606 ;
607 else
608 /* not in-sync or faulty.
609 * If the reshape increases the number of devices,
610 * this is being recovered by the reshape, so
611 * this 'previous' section is not in_sync.
612 * If the number of devices is being reduced however,
613 * the device can only be part of the array if
614 * we are reverting a reshape, so this section will
615 * be in-sync.
616 */
617 if (conf->raid_disks >= conf->previous_raid_disks)
618 degraded++;
619 }
620 rcu_read_unlock();
NeilBrown908f4fb2011-12-23 10:17:50 +1100621 if (conf->raid_disks == conf->previous_raid_disks)
622 return degraded;
NeilBrown674806d2010-06-16 17:17:53 +1000623 rcu_read_lock();
NeilBrown908f4fb2011-12-23 10:17:50 +1100624 degraded2 = 0;
NeilBrown674806d2010-06-16 17:17:53 +1000625 for (i = 0; i < conf->raid_disks; i++) {
NeilBrown3cb03002011-10-11 16:45:26 +1100626 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
NeilBrowne5c86472012-09-19 12:52:30 +1000627 if (rdev && test_bit(Faulty, &rdev->flags))
628 rdev = rcu_dereference(conf->disks[i].replacement);
NeilBrown674806d2010-06-16 17:17:53 +1000629 if (!rdev || test_bit(Faulty, &rdev->flags))
NeilBrown908f4fb2011-12-23 10:17:50 +1100630 degraded2++;
NeilBrown674806d2010-06-16 17:17:53 +1000631 else if (test_bit(In_sync, &rdev->flags))
632 ;
633 else
634 /* not in-sync or faulty.
635 * If reshape increases the number of devices, this
636 * section has already been recovered, else it
637 * almost certainly hasn't.
638 */
639 if (conf->raid_disks <= conf->previous_raid_disks)
NeilBrown908f4fb2011-12-23 10:17:50 +1100640 degraded2++;
NeilBrown674806d2010-06-16 17:17:53 +1000641 }
642 rcu_read_unlock();
NeilBrown908f4fb2011-12-23 10:17:50 +1100643 if (degraded2 > degraded)
644 return degraded2;
645 return degraded;
646}
647
648static int has_failed(struct r5conf *conf)
649{
650 int degraded;
651
652 if (conf->mddev->reshape_position == MaxSector)
653 return conf->mddev->degraded > conf->max_degraded;
654
655 degraded = calc_degraded(conf);
NeilBrown674806d2010-06-16 17:17:53 +1000656 if (degraded > conf->max_degraded)
657 return 1;
658 return 0;
659}
660
Shaohua Li6d036f72015-08-13 14:31:57 -0700661struct stripe_head *
662raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
663 int previous, int noblock, int noquiesce)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664{
665 struct stripe_head *sh;
Shaohua Li566c09c2013-11-14 15:16:17 +1100666 int hash = stripe_hash_locks_hash(sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667
Dan Williams45b42332007-07-09 11:56:43 -0700668 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669
Shaohua Li566c09c2013-11-14 15:16:17 +1100670 spin_lock_irq(conf->hash_locks + hash);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671
672 do {
Yuanhan Liub1b46482015-05-08 18:19:06 +1000673 wait_event_lock_irq(conf->wait_for_quiescent,
NeilBrowna8c906c2009-06-09 14:39:59 +1000674 conf->quiesce == 0 || noquiesce,
Shaohua Li566c09c2013-11-14 15:16:17 +1100675 *(conf->hash_locks + hash));
NeilBrown86b42c72009-03-31 15:19:03 +1100676 sh = __find_stripe(conf, sector, conf->generation - previous);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 if (!sh) {
NeilBrownedbe83a2015-02-26 12:47:56 +1100678 if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
Shaohua Li566c09c2013-11-14 15:16:17 +1100679 sh = get_free_stripe(conf, hash);
Shaohua Li713bc5c2015-05-28 17:33:47 -0700680 if (!sh && !test_bit(R5_DID_ALLOC,
681 &conf->cache_state))
NeilBrownedbe83a2015-02-26 12:47:56 +1100682 set_bit(R5_ALLOC_MORE,
683 &conf->cache_state);
684 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 if (noblock && sh == NULL)
686 break;
687 if (!sh) {
NeilBrown54233992015-02-26 12:21:04 +1100688 set_bit(R5_INACTIVE_BLOCKED,
689 &conf->cache_state);
Yuanhan Liue9e4c372015-05-08 18:19:07 +1000690 wait_event_exclusive_cmd(
691 conf->wait_for_stripe[hash],
Shaohua Li566c09c2013-11-14 15:16:17 +1100692 !list_empty(conf->inactive_list + hash) &&
693 (atomic_read(&conf->active_stripes)
694 < (conf->max_nr_stripes * 3 / 4)
NeilBrown54233992015-02-26 12:21:04 +1100695 || !test_bit(R5_INACTIVE_BLOCKED,
696 &conf->cache_state)),
Yuanhan Liue9e4c372015-05-08 18:19:07 +1000697 spin_unlock_irq(conf->hash_locks + hash),
698 spin_lock_irq(conf->hash_locks + hash));
NeilBrown54233992015-02-26 12:21:04 +1100699 clear_bit(R5_INACTIVE_BLOCKED,
700 &conf->cache_state);
NeilBrown7da9d452014-01-22 11:45:03 +1100701 } else {
NeilBrownb5663ba2009-03-31 14:39:38 +1100702 init_stripe(sh, sector, previous);
NeilBrown7da9d452014-01-22 11:45:03 +1100703 atomic_inc(&sh->count);
704 }
Shaohua Lie240c182014-04-09 11:27:42 +0800705 } else if (!atomic_inc_not_zero(&sh->count)) {
NeilBrown6d183de2013-11-28 10:55:27 +1100706 spin_lock(&conf->device_lock);
Shaohua Lie240c182014-04-09 11:27:42 +0800707 if (!atomic_read(&sh->count)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 if (!test_bit(STRIPE_HANDLE, &sh->state))
709 atomic_inc(&conf->active_stripes);
NeilBrown5af9bef2014-01-14 15:16:10 +1100710 BUG_ON(list_empty(&sh->lru) &&
711 !test_bit(STRIPE_EXPANDING, &sh->state));
NeilBrown16a53ec2006-06-26 00:27:38 -0700712 list_del_init(&sh->lru);
Shaohua Libfc90cb2013-08-29 15:40:32 +0800713 if (sh->group) {
714 sh->group->stripes_cnt--;
715 sh->group = NULL;
716 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717 }
NeilBrown7da9d452014-01-22 11:45:03 +1100718 atomic_inc(&sh->count);
NeilBrown6d183de2013-11-28 10:55:27 +1100719 spin_unlock(&conf->device_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 }
721 } while (sh == NULL);
722
Yuanhan Liue9e4c372015-05-08 18:19:07 +1000723 if (!list_empty(conf->inactive_list + hash))
724 wake_up(&conf->wait_for_stripe[hash]);
725
Shaohua Li566c09c2013-11-14 15:16:17 +1100726 spin_unlock_irq(conf->hash_locks + hash);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727 return sh;
728}
729
shli@kernel.org7a87f432014-12-15 12:57:03 +1100730static bool is_full_stripe_write(struct stripe_head *sh)
731{
732 BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
733 return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
734}
735
shli@kernel.org59fc6302014-12-15 12:57:03 +1100736static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
737{
738 local_irq_disable();
739 if (sh1 > sh2) {
740 spin_lock(&sh2->stripe_lock);
741 spin_lock_nested(&sh1->stripe_lock, 1);
742 } else {
743 spin_lock(&sh1->stripe_lock);
744 spin_lock_nested(&sh2->stripe_lock, 1);
745 }
746}
747
748static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
749{
750 spin_unlock(&sh1->stripe_lock);
751 spin_unlock(&sh2->stripe_lock);
752 local_irq_enable();
753}
754
755/* Only freshly new full stripe normal write stripe can be added to a batch list */
756static bool stripe_can_batch(struct stripe_head *sh)
757{
Shaohua Li9c3e3332015-08-13 14:32:02 -0700758 struct r5conf *conf = sh->raid_conf;
759
760 if (conf->log)
761 return false;
shli@kernel.org59fc6302014-12-15 12:57:03 +1100762 return test_bit(STRIPE_BATCH_READY, &sh->state) &&
NeilBrownd0852df52015-05-27 08:43:45 +1000763 !test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&
shli@kernel.org59fc6302014-12-15 12:57:03 +1100764 is_full_stripe_write(sh);
765}
766
767/* we only do back search */
768static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
769{
770 struct stripe_head *head;
771 sector_t head_sector, tmp_sec;
772 int hash;
773 int dd_idx;
774
shli@kernel.org59fc6302014-12-15 12:57:03 +1100775 /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
776 tmp_sec = sh->sector;
777 if (!sector_div(tmp_sec, conf->chunk_sectors))
778 return;
779 head_sector = sh->sector - STRIPE_SECTORS;
780
781 hash = stripe_hash_locks_hash(head_sector);
782 spin_lock_irq(conf->hash_locks + hash);
783 head = __find_stripe(conf, head_sector, conf->generation);
784 if (head && !atomic_inc_not_zero(&head->count)) {
785 spin_lock(&conf->device_lock);
786 if (!atomic_read(&head->count)) {
787 if (!test_bit(STRIPE_HANDLE, &head->state))
788 atomic_inc(&conf->active_stripes);
789 BUG_ON(list_empty(&head->lru) &&
790 !test_bit(STRIPE_EXPANDING, &head->state));
791 list_del_init(&head->lru);
792 if (head->group) {
793 head->group->stripes_cnt--;
794 head->group = NULL;
795 }
796 }
797 atomic_inc(&head->count);
798 spin_unlock(&conf->device_lock);
799 }
800 spin_unlock_irq(conf->hash_locks + hash);
801
802 if (!head)
803 return;
804 if (!stripe_can_batch(head))
805 goto out;
806
807 lock_two_stripes(head, sh);
808 /* clear_batch_ready clear the flag */
809 if (!stripe_can_batch(head) || !stripe_can_batch(sh))
810 goto unlock_out;
811
812 if (sh->batch_head)
813 goto unlock_out;
814
815 dd_idx = 0;
816 while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
817 dd_idx++;
818 if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw)
819 goto unlock_out;
820
821 if (head->batch_head) {
822 spin_lock(&head->batch_head->batch_lock);
823 /* This batch list is already running */
824 if (!stripe_can_batch(head)) {
825 spin_unlock(&head->batch_head->batch_lock);
826 goto unlock_out;
827 }
828
829 /*
830 * at this point, head's BATCH_READY could be cleared, but we
831 * can still add the stripe to batch list
832 */
833 list_add(&sh->batch_list, &head->batch_list);
834 spin_unlock(&head->batch_head->batch_lock);
835
836 sh->batch_head = head->batch_head;
837 } else {
838 head->batch_head = head;
839 sh->batch_head = head->batch_head;
840 spin_lock(&head->batch_lock);
841 list_add_tail(&sh->batch_list, &head->batch_list);
842 spin_unlock(&head->batch_lock);
843 }
844
845 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
846 if (atomic_dec_return(&conf->preread_active_stripes)
847 < IO_THRESHOLD)
848 md_wakeup_thread(conf->mddev->thread);
849
NeilBrown2b6b2452015-05-21 15:10:01 +1000850 if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {
851 int seq = sh->bm_seq;
852 if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&
853 sh->batch_head->bm_seq > seq)
854 seq = sh->batch_head->bm_seq;
855 set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);
856 sh->batch_head->bm_seq = seq;
857 }
858
shli@kernel.org59fc6302014-12-15 12:57:03 +1100859 atomic_inc(&sh->count);
860unlock_out:
861 unlock_two_stripes(head, sh);
862out:
Shaohua Li6d036f72015-08-13 14:31:57 -0700863 raid5_release_stripe(head);
shli@kernel.org59fc6302014-12-15 12:57:03 +1100864}
865
NeilBrown05616be2012-05-21 09:27:00 +1000866/* Determine if 'data_offset' or 'new_data_offset' should be used
867 * in this stripe_head.
868 */
869static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
870{
871 sector_t progress = conf->reshape_progress;
872 /* Need a memory barrier to make sure we see the value
873 * of conf->generation, or ->data_offset that was set before
874 * reshape_progress was updated.
875 */
876 smp_rmb();
877 if (progress == MaxSector)
878 return 0;
879 if (sh->generation == conf->generation - 1)
880 return 0;
881 /* We are in a reshape, and this is a new-generation stripe,
882 * so use new_data_offset.
883 */
884 return 1;
885}
886
NeilBrown6712ecf2007-09-27 12:47:43 +0200887static void
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200888raid5_end_read_request(struct bio *bi);
NeilBrown6712ecf2007-09-27 12:47:43 +0200889static void
Christoph Hellwig4246a0b2015-07-20 15:29:37 +0200890raid5_end_write_request(struct bio *bi);
Dan Williams91c00922007-01-02 13:52:30 -0700891
Dan Williamsc4e5ac02008-06-28 08:31:53 +1000892static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
Dan Williams91c00922007-01-02 13:52:30 -0700893{
NeilBrownd1688a62011-10-11 16:49:52 +1100894 struct r5conf *conf = sh->raid_conf;
Dan Williams91c00922007-01-02 13:52:30 -0700895 int i, disks = sh->disks;
shli@kernel.org59fc6302014-12-15 12:57:03 +1100896 struct stripe_head *head_sh = sh;
Dan Williams91c00922007-01-02 13:52:30 -0700897
898 might_sleep();
899
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700900 if (r5l_write_stripe(conf->log, sh) == 0)
901 return;
Dan Williams91c00922007-01-02 13:52:30 -0700902 for (i = disks; i--; ) {
903 int rw;
NeilBrown9a3e1102011-12-23 10:17:53 +1100904 int replace_only = 0;
NeilBrown977df362011-12-23 10:17:53 +1100905 struct bio *bi, *rbi;
906 struct md_rdev *rdev, *rrdev = NULL;
shli@kernel.org59fc6302014-12-15 12:57:03 +1100907
908 sh = head_sh;
Tejun Heoe9c74692010-09-03 11:56:18 +0200909 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
910 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
911 rw = WRITE_FUA;
912 else
913 rw = WRITE;
Shaohua Li9e4447682012-10-11 13:49:49 +1100914 if (test_bit(R5_Discard, &sh->dev[i].flags))
Shaohua Li620125f2012-10-11 13:49:05 +1100915 rw |= REQ_DISCARD;
Tejun Heoe9c74692010-09-03 11:56:18 +0200916 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
Dan Williams91c00922007-01-02 13:52:30 -0700917 rw = READ;
NeilBrown9a3e1102011-12-23 10:17:53 +1100918 else if (test_and_clear_bit(R5_WantReplace,
919 &sh->dev[i].flags)) {
920 rw = WRITE;
921 replace_only = 1;
922 } else
Dan Williams91c00922007-01-02 13:52:30 -0700923 continue;
Shaohua Libc0934f2012-05-22 13:55:05 +1000924 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
925 rw |= REQ_SYNC;
Dan Williams91c00922007-01-02 13:52:30 -0700926
shli@kernel.org59fc6302014-12-15 12:57:03 +1100927again:
Dan Williams91c00922007-01-02 13:52:30 -0700928 bi = &sh->dev[i].req;
NeilBrown977df362011-12-23 10:17:53 +1100929 rbi = &sh->dev[i].rreq; /* For writing to replacement */
Dan Williams91c00922007-01-02 13:52:30 -0700930
Dan Williams91c00922007-01-02 13:52:30 -0700931 rcu_read_lock();
NeilBrown9a3e1102011-12-23 10:17:53 +1100932 rrdev = rcu_dereference(conf->disks[i].replacement);
NeilBrowndd054fc2011-12-23 10:17:53 +1100933 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
934 rdev = rcu_dereference(conf->disks[i].rdev);
935 if (!rdev) {
936 rdev = rrdev;
937 rrdev = NULL;
938 }
NeilBrown9a3e1102011-12-23 10:17:53 +1100939 if (rw & WRITE) {
940 if (replace_only)
941 rdev = NULL;
NeilBrowndd054fc2011-12-23 10:17:53 +1100942 if (rdev == rrdev)
943 /* We raced and saw duplicates */
944 rrdev = NULL;
NeilBrown9a3e1102011-12-23 10:17:53 +1100945 } else {
shli@kernel.org59fc6302014-12-15 12:57:03 +1100946 if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
NeilBrown9a3e1102011-12-23 10:17:53 +1100947 rdev = rrdev;
948 rrdev = NULL;
949 }
NeilBrown977df362011-12-23 10:17:53 +1100950
Dan Williams91c00922007-01-02 13:52:30 -0700951 if (rdev && test_bit(Faulty, &rdev->flags))
952 rdev = NULL;
953 if (rdev)
954 atomic_inc(&rdev->nr_pending);
NeilBrown977df362011-12-23 10:17:53 +1100955 if (rrdev && test_bit(Faulty, &rrdev->flags))
956 rrdev = NULL;
957 if (rrdev)
958 atomic_inc(&rrdev->nr_pending);
Dan Williams91c00922007-01-02 13:52:30 -0700959 rcu_read_unlock();
960
NeilBrown73e92e52011-07-28 11:39:22 +1000961 /* We have already checked bad blocks for reads. Now
NeilBrown977df362011-12-23 10:17:53 +1100962 * need to check for writes. We never accept write errors
963 * on the replacement, so we don't to check rrdev.
NeilBrown73e92e52011-07-28 11:39:22 +1000964 */
965 while ((rw & WRITE) && rdev &&
966 test_bit(WriteErrorSeen, &rdev->flags)) {
967 sector_t first_bad;
968 int bad_sectors;
969 int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
970 &first_bad, &bad_sectors);
971 if (!bad)
972 break;
973
974 if (bad < 0) {
975 set_bit(BlockedBadBlocks, &rdev->flags);
976 if (!conf->mddev->external &&
977 conf->mddev->flags) {
978 /* It is very unlikely, but we might
979 * still need to write out the
980 * bad block log - better give it
981 * a chance*/
982 md_check_recovery(conf->mddev);
983 }
majianpeng18507532012-07-03 12:11:54 +1000984 /*
985 * Because md_wait_for_blocked_rdev
986 * will dec nr_pending, we must
987 * increment it first.
988 */
989 atomic_inc(&rdev->nr_pending);
NeilBrown73e92e52011-07-28 11:39:22 +1000990 md_wait_for_blocked_rdev(rdev, conf->mddev);
991 } else {
992 /* Acknowledged bad block - skip the write */
993 rdev_dec_pending(rdev, conf->mddev);
994 rdev = NULL;
995 }
996 }
997
Dan Williams91c00922007-01-02 13:52:30 -0700998 if (rdev) {
NeilBrown9a3e1102011-12-23 10:17:53 +1100999 if (s->syncing || s->expanding || s->expanded
1000 || s->replacing)
Dan Williams91c00922007-01-02 13:52:30 -07001001 md_sync_acct(rdev->bdev, STRIPE_SECTORS);
1002
Dan Williams2b7497f2008-06-28 08:31:52 +10001003 set_bit(STRIPE_IO_STARTED, &sh->state);
1004
Kent Overstreet2f6db2a2012-09-11 12:26:38 -07001005 bio_reset(bi);
Dan Williams91c00922007-01-02 13:52:30 -07001006 bi->bi_bdev = rdev->bdev;
Kent Overstreet2f6db2a2012-09-11 12:26:38 -07001007 bi->bi_rw = rw;
1008 bi->bi_end_io = (rw & WRITE)
1009 ? raid5_end_write_request
1010 : raid5_end_read_request;
1011 bi->bi_private = sh;
1012
Dan Williams91c00922007-01-02 13:52:30 -07001013 pr_debug("%s: for %llu schedule op %ld on disc %d\n",
Harvey Harrisone46b2722008-04-28 02:15:50 -07001014 __func__, (unsigned long long)sh->sector,
Dan Williams91c00922007-01-02 13:52:30 -07001015 bi->bi_rw, i);
1016 atomic_inc(&sh->count);
shli@kernel.org59fc6302014-12-15 12:57:03 +11001017 if (sh != head_sh)
1018 atomic_inc(&head_sh->count);
NeilBrown05616be2012-05-21 09:27:00 +10001019 if (use_new_offset(conf, sh))
Kent Overstreet4f024f32013-10-11 15:44:27 -07001020 bi->bi_iter.bi_sector = (sh->sector
NeilBrown05616be2012-05-21 09:27:00 +10001021 + rdev->new_data_offset);
1022 else
Kent Overstreet4f024f32013-10-11 15:44:27 -07001023 bi->bi_iter.bi_sector = (sh->sector
NeilBrown05616be2012-05-21 09:27:00 +10001024 + rdev->data_offset);
shli@kernel.org59fc6302014-12-15 12:57:03 +11001025 if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
majianpenge59aa232013-11-14 15:16:19 +11001026 bi->bi_rw |= REQ_NOMERGE;
majianpeng3f9e7c12012-07-31 10:04:21 +10001027
Shaohua Lid592a992014-05-21 17:57:44 +08001028 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1029 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1030 sh->dev[i].vec.bv_page = sh->dev[i].page;
Kent Overstreet4997b722013-05-30 08:44:39 +02001031 bi->bi_vcnt = 1;
Dan Williams91c00922007-01-02 13:52:30 -07001032 bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1033 bi->bi_io_vec[0].bv_offset = 0;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001034 bi->bi_iter.bi_size = STRIPE_SIZE;
Shaohua Li37c61ff2013-10-19 14:50:28 +08001035 /*
1036 * If this is discard request, set bi_vcnt 0. We don't
1037 * want to confuse SCSI because SCSI will replace payload
1038 */
1039 if (rw & REQ_DISCARD)
1040 bi->bi_vcnt = 0;
NeilBrown977df362011-12-23 10:17:53 +11001041 if (rrdev)
1042 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
Jonathan Brassowe3620a32013-03-07 16:22:01 -06001043
1044 if (conf->mddev->gendisk)
1045 trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
1046 bi, disk_devt(conf->mddev->gendisk),
1047 sh->dev[i].sector);
Dan Williams91c00922007-01-02 13:52:30 -07001048 generic_make_request(bi);
NeilBrown977df362011-12-23 10:17:53 +11001049 }
1050 if (rrdev) {
NeilBrown9a3e1102011-12-23 10:17:53 +11001051 if (s->syncing || s->expanding || s->expanded
1052 || s->replacing)
NeilBrown977df362011-12-23 10:17:53 +11001053 md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
1054
1055 set_bit(STRIPE_IO_STARTED, &sh->state);
1056
Kent Overstreet2f6db2a2012-09-11 12:26:38 -07001057 bio_reset(rbi);
NeilBrown977df362011-12-23 10:17:53 +11001058 rbi->bi_bdev = rrdev->bdev;
Kent Overstreet2f6db2a2012-09-11 12:26:38 -07001059 rbi->bi_rw = rw;
1060 BUG_ON(!(rw & WRITE));
1061 rbi->bi_end_io = raid5_end_write_request;
1062 rbi->bi_private = sh;
1063
NeilBrown977df362011-12-23 10:17:53 +11001064 pr_debug("%s: for %llu schedule op %ld on "
1065 "replacement disc %d\n",
1066 __func__, (unsigned long long)sh->sector,
1067 rbi->bi_rw, i);
1068 atomic_inc(&sh->count);
shli@kernel.org59fc6302014-12-15 12:57:03 +11001069 if (sh != head_sh)
1070 atomic_inc(&head_sh->count);
NeilBrown05616be2012-05-21 09:27:00 +10001071 if (use_new_offset(conf, sh))
Kent Overstreet4f024f32013-10-11 15:44:27 -07001072 rbi->bi_iter.bi_sector = (sh->sector
NeilBrown05616be2012-05-21 09:27:00 +10001073 + rrdev->new_data_offset);
1074 else
Kent Overstreet4f024f32013-10-11 15:44:27 -07001075 rbi->bi_iter.bi_sector = (sh->sector
NeilBrown05616be2012-05-21 09:27:00 +10001076 + rrdev->data_offset);
Shaohua Lid592a992014-05-21 17:57:44 +08001077 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1078 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1079 sh->dev[i].rvec.bv_page = sh->dev[i].page;
Kent Overstreet4997b722013-05-30 08:44:39 +02001080 rbi->bi_vcnt = 1;
NeilBrown977df362011-12-23 10:17:53 +11001081 rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1082 rbi->bi_io_vec[0].bv_offset = 0;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001083 rbi->bi_iter.bi_size = STRIPE_SIZE;
Shaohua Li37c61ff2013-10-19 14:50:28 +08001084 /*
1085 * If this is discard request, set bi_vcnt 0. We don't
1086 * want to confuse SCSI because SCSI will replace payload
1087 */
1088 if (rw & REQ_DISCARD)
1089 rbi->bi_vcnt = 0;
Jonathan Brassowe3620a32013-03-07 16:22:01 -06001090 if (conf->mddev->gendisk)
1091 trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
1092 rbi, disk_devt(conf->mddev->gendisk),
1093 sh->dev[i].sector);
NeilBrown977df362011-12-23 10:17:53 +11001094 generic_make_request(rbi);
1095 }
1096 if (!rdev && !rrdev) {
Namhyung Kimb0629622011-06-14 14:20:19 +10001097 if (rw & WRITE)
Dan Williams91c00922007-01-02 13:52:30 -07001098 set_bit(STRIPE_DEGRADED, &sh->state);
1099 pr_debug("skip op %ld on disc %d for sector %llu\n",
1100 bi->bi_rw, i, (unsigned long long)sh->sector);
1101 clear_bit(R5_LOCKED, &sh->dev[i].flags);
1102 set_bit(STRIPE_HANDLE, &sh->state);
1103 }
shli@kernel.org59fc6302014-12-15 12:57:03 +11001104
1105 if (!head_sh->batch_head)
1106 continue;
1107 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1108 batch_list);
1109 if (sh != head_sh)
1110 goto again;
Dan Williams91c00922007-01-02 13:52:30 -07001111 }
1112}
1113
1114static struct dma_async_tx_descriptor *
Shaohua Lid592a992014-05-21 17:57:44 +08001115async_copy_data(int frombio, struct bio *bio, struct page **page,
1116 sector_t sector, struct dma_async_tx_descriptor *tx,
1117 struct stripe_head *sh)
Dan Williams91c00922007-01-02 13:52:30 -07001118{
Kent Overstreet79886132013-11-23 17:19:00 -08001119 struct bio_vec bvl;
1120 struct bvec_iter iter;
Dan Williams91c00922007-01-02 13:52:30 -07001121 struct page *bio_page;
Dan Williams91c00922007-01-02 13:52:30 -07001122 int page_offset;
Dan Williamsa08abd82009-06-03 11:43:59 -07001123 struct async_submit_ctl submit;
Dan Williams0403e382009-09-08 17:42:50 -07001124 enum async_tx_flags flags = 0;
Dan Williams91c00922007-01-02 13:52:30 -07001125
Kent Overstreet4f024f32013-10-11 15:44:27 -07001126 if (bio->bi_iter.bi_sector >= sector)
1127 page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
Dan Williams91c00922007-01-02 13:52:30 -07001128 else
Kent Overstreet4f024f32013-10-11 15:44:27 -07001129 page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
Dan Williamsa08abd82009-06-03 11:43:59 -07001130
Dan Williams0403e382009-09-08 17:42:50 -07001131 if (frombio)
1132 flags |= ASYNC_TX_FENCE;
1133 init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
1134
Kent Overstreet79886132013-11-23 17:19:00 -08001135 bio_for_each_segment(bvl, bio, iter) {
1136 int len = bvl.bv_len;
Dan Williams91c00922007-01-02 13:52:30 -07001137 int clen;
1138 int b_offset = 0;
1139
1140 if (page_offset < 0) {
1141 b_offset = -page_offset;
1142 page_offset += b_offset;
1143 len -= b_offset;
1144 }
1145
1146 if (len > 0 && page_offset + len > STRIPE_SIZE)
1147 clen = STRIPE_SIZE - page_offset;
1148 else
1149 clen = len;
1150
1151 if (clen > 0) {
Kent Overstreet79886132013-11-23 17:19:00 -08001152 b_offset += bvl.bv_offset;
1153 bio_page = bvl.bv_page;
Shaohua Lid592a992014-05-21 17:57:44 +08001154 if (frombio) {
1155 if (sh->raid_conf->skip_copy &&
1156 b_offset == 0 && page_offset == 0 &&
1157 clen == STRIPE_SIZE)
1158 *page = bio_page;
1159 else
1160 tx = async_memcpy(*page, bio_page, page_offset,
Dan Williamsa08abd82009-06-03 11:43:59 -07001161 b_offset, clen, &submit);
Shaohua Lid592a992014-05-21 17:57:44 +08001162 } else
1163 tx = async_memcpy(bio_page, *page, b_offset,
Dan Williamsa08abd82009-06-03 11:43:59 -07001164 page_offset, clen, &submit);
Dan Williams91c00922007-01-02 13:52:30 -07001165 }
Dan Williamsa08abd82009-06-03 11:43:59 -07001166 /* chain the operations */
1167 submit.depend_tx = tx;
1168
Dan Williams91c00922007-01-02 13:52:30 -07001169 if (clen < len) /* hit end of page */
1170 break;
1171 page_offset += len;
1172 }
1173
1174 return tx;
1175}
1176
1177static void ops_complete_biofill(void *stripe_head_ref)
1178{
1179 struct stripe_head *sh = stripe_head_ref;
NeilBrown34a6f802015-08-14 12:07:57 +10001180 struct bio_list return_bi = BIO_EMPTY_LIST;
Dan Williamse4d84902007-09-24 10:06:13 -07001181 int i;
Dan Williams91c00922007-01-02 13:52:30 -07001182
Harvey Harrisone46b2722008-04-28 02:15:50 -07001183 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001184 (unsigned long long)sh->sector);
1185
1186 /* clear completed biofills */
1187 for (i = sh->disks; i--; ) {
1188 struct r5dev *dev = &sh->dev[i];
Dan Williams91c00922007-01-02 13:52:30 -07001189
1190 /* acknowledge completion of a biofill operation */
Dan Williamse4d84902007-09-24 10:06:13 -07001191 /* and check if we need to reply to a read request,
1192 * new R5_Wantfill requests are held off until
Dan Williams83de75c2008-06-28 08:31:58 +10001193 * !STRIPE_BIOFILL_RUN
Dan Williamse4d84902007-09-24 10:06:13 -07001194 */
1195 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
Dan Williams91c00922007-01-02 13:52:30 -07001196 struct bio *rbi, *rbi2;
Dan Williams91c00922007-01-02 13:52:30 -07001197
Dan Williams91c00922007-01-02 13:52:30 -07001198 BUG_ON(!dev->read);
1199 rbi = dev->read;
1200 dev->read = NULL;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001201 while (rbi && rbi->bi_iter.bi_sector <
Dan Williams91c00922007-01-02 13:52:30 -07001202 dev->sector + STRIPE_SECTORS) {
1203 rbi2 = r5_next_bio(rbi, dev->sector);
NeilBrown34a6f802015-08-14 12:07:57 +10001204 if (!raid5_dec_bi_active_stripes(rbi))
1205 bio_list_add(&return_bi, rbi);
Dan Williams91c00922007-01-02 13:52:30 -07001206 rbi = rbi2;
1207 }
1208 }
1209 }
Dan Williams83de75c2008-06-28 08:31:58 +10001210 clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
Dan Williams91c00922007-01-02 13:52:30 -07001211
NeilBrown34a6f802015-08-14 12:07:57 +10001212 return_io(&return_bi);
Dan Williams91c00922007-01-02 13:52:30 -07001213
Dan Williamse4d84902007-09-24 10:06:13 -07001214 set_bit(STRIPE_HANDLE, &sh->state);
Shaohua Li6d036f72015-08-13 14:31:57 -07001215 raid5_release_stripe(sh);
Dan Williams91c00922007-01-02 13:52:30 -07001216}
1217
1218static void ops_run_biofill(struct stripe_head *sh)
1219{
1220 struct dma_async_tx_descriptor *tx = NULL;
Dan Williamsa08abd82009-06-03 11:43:59 -07001221 struct async_submit_ctl submit;
Dan Williams91c00922007-01-02 13:52:30 -07001222 int i;
1223
shli@kernel.org59fc6302014-12-15 12:57:03 +11001224 BUG_ON(sh->batch_head);
Harvey Harrisone46b2722008-04-28 02:15:50 -07001225 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001226 (unsigned long long)sh->sector);
1227
1228 for (i = sh->disks; i--; ) {
1229 struct r5dev *dev = &sh->dev[i];
1230 if (test_bit(R5_Wantfill, &dev->flags)) {
1231 struct bio *rbi;
Shaohua Lib17459c2012-07-19 16:01:31 +10001232 spin_lock_irq(&sh->stripe_lock);
Dan Williams91c00922007-01-02 13:52:30 -07001233 dev->read = rbi = dev->toread;
1234 dev->toread = NULL;
Shaohua Lib17459c2012-07-19 16:01:31 +10001235 spin_unlock_irq(&sh->stripe_lock);
Kent Overstreet4f024f32013-10-11 15:44:27 -07001236 while (rbi && rbi->bi_iter.bi_sector <
Dan Williams91c00922007-01-02 13:52:30 -07001237 dev->sector + STRIPE_SECTORS) {
Shaohua Lid592a992014-05-21 17:57:44 +08001238 tx = async_copy_data(0, rbi, &dev->page,
1239 dev->sector, tx, sh);
Dan Williams91c00922007-01-02 13:52:30 -07001240 rbi = r5_next_bio(rbi, dev->sector);
1241 }
1242 }
1243 }
1244
1245 atomic_inc(&sh->count);
Dan Williamsa08abd82009-06-03 11:43:59 -07001246 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
1247 async_trigger_callback(&submit);
Dan Williams91c00922007-01-02 13:52:30 -07001248}
1249
Dan Williams4e7d2c02009-08-29 19:13:11 -07001250static void mark_target_uptodate(struct stripe_head *sh, int target)
1251{
1252 struct r5dev *tgt;
1253
1254 if (target < 0)
1255 return;
1256
1257 tgt = &sh->dev[target];
1258 set_bit(R5_UPTODATE, &tgt->flags);
1259 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1260 clear_bit(R5_Wantcompute, &tgt->flags);
1261}
1262
Dan Williamsac6b53b2009-07-14 13:40:19 -07001263static void ops_complete_compute(void *stripe_head_ref)
Dan Williams91c00922007-01-02 13:52:30 -07001264{
1265 struct stripe_head *sh = stripe_head_ref;
Dan Williams91c00922007-01-02 13:52:30 -07001266
Harvey Harrisone46b2722008-04-28 02:15:50 -07001267 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001268 (unsigned long long)sh->sector);
1269
Dan Williamsac6b53b2009-07-14 13:40:19 -07001270 /* mark the computed target(s) as uptodate */
Dan Williams4e7d2c02009-08-29 19:13:11 -07001271 mark_target_uptodate(sh, sh->ops.target);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001272 mark_target_uptodate(sh, sh->ops.target2);
Dan Williams4e7d2c02009-08-29 19:13:11 -07001273
Dan Williamsecc65c92008-06-28 08:31:57 +10001274 clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
1275 if (sh->check_state == check_state_compute_run)
1276 sh->check_state = check_state_compute_result;
Dan Williams91c00922007-01-02 13:52:30 -07001277 set_bit(STRIPE_HANDLE, &sh->state);
Shaohua Li6d036f72015-08-13 14:31:57 -07001278 raid5_release_stripe(sh);
Dan Williams91c00922007-01-02 13:52:30 -07001279}
1280
Dan Williamsd6f38f32009-07-14 11:50:52 -07001281/* return a pointer to the address conversion region of the scribble buffer */
1282static addr_conv_t *to_addr_conv(struct stripe_head *sh,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001283 struct raid5_percpu *percpu, int i)
Dan Williams91c00922007-01-02 13:52:30 -07001284{
shli@kernel.org46d5b782014-12-15 12:57:02 +11001285 void *addr;
1286
1287 addr = flex_array_get(percpu->scribble, i);
1288 return addr + sizeof(struct page *) * (sh->disks + 2);
1289}
1290
1291/* return a pointer to the address conversion region of the scribble buffer */
1292static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
1293{
1294 void *addr;
1295
1296 addr = flex_array_get(percpu->scribble, i);
1297 return addr;
Dan Williamsd6f38f32009-07-14 11:50:52 -07001298}
1299
1300static struct dma_async_tx_descriptor *
1301ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1302{
Dan Williams91c00922007-01-02 13:52:30 -07001303 int disks = sh->disks;
shli@kernel.org46d5b782014-12-15 12:57:02 +11001304 struct page **xor_srcs = to_addr_page(percpu, 0);
Dan Williams91c00922007-01-02 13:52:30 -07001305 int target = sh->ops.target;
1306 struct r5dev *tgt = &sh->dev[target];
1307 struct page *xor_dest = tgt->page;
1308 int count = 0;
1309 struct dma_async_tx_descriptor *tx;
Dan Williamsa08abd82009-06-03 11:43:59 -07001310 struct async_submit_ctl submit;
Dan Williams91c00922007-01-02 13:52:30 -07001311 int i;
1312
shli@kernel.org59fc6302014-12-15 12:57:03 +11001313 BUG_ON(sh->batch_head);
1314
Dan Williams91c00922007-01-02 13:52:30 -07001315 pr_debug("%s: stripe %llu block: %d\n",
Harvey Harrisone46b2722008-04-28 02:15:50 -07001316 __func__, (unsigned long long)sh->sector, target);
Dan Williams91c00922007-01-02 13:52:30 -07001317 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1318
1319 for (i = disks; i--; )
1320 if (i != target)
1321 xor_srcs[count++] = sh->dev[i].page;
1322
1323 atomic_inc(&sh->count);
1324
Dan Williams0403e382009-09-08 17:42:50 -07001325 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001326 ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
Dan Williams91c00922007-01-02 13:52:30 -07001327 if (unlikely(count == 1))
Dan Williamsa08abd82009-06-03 11:43:59 -07001328 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
Dan Williams91c00922007-01-02 13:52:30 -07001329 else
Dan Williamsa08abd82009-06-03 11:43:59 -07001330 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
Dan Williams91c00922007-01-02 13:52:30 -07001331
Dan Williams91c00922007-01-02 13:52:30 -07001332 return tx;
1333}
1334
Dan Williamsac6b53b2009-07-14 13:40:19 -07001335/* set_syndrome_sources - populate source buffers for gen_syndrome
1336 * @srcs - (struct page *) array of size sh->disks
1337 * @sh - stripe_head to parse
1338 *
1339 * Populates srcs in proper layout order for the stripe and returns the
1340 * 'count' of sources to be used in a call to async_gen_syndrome. The P
1341 * destination buffer is recorded in srcs[count] and the Q destination
1342 * is recorded in srcs[count+1]].
1343 */
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001344static int set_syndrome_sources(struct page **srcs,
1345 struct stripe_head *sh,
1346 int srctype)
Dan Williamsac6b53b2009-07-14 13:40:19 -07001347{
1348 int disks = sh->disks;
1349 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
1350 int d0_idx = raid6_d0(sh);
1351 int count;
1352 int i;
1353
1354 for (i = 0; i < disks; i++)
NeilBrown5dd33c92009-10-16 16:40:25 +11001355 srcs[i] = NULL;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001356
1357 count = 0;
1358 i = d0_idx;
1359 do {
1360 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001361 struct r5dev *dev = &sh->dev[i];
Dan Williamsac6b53b2009-07-14 13:40:19 -07001362
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001363 if (i == sh->qd_idx || i == sh->pd_idx ||
1364 (srctype == SYNDROME_SRC_ALL) ||
1365 (srctype == SYNDROME_SRC_WANT_DRAIN &&
1366 test_bit(R5_Wantdrain, &dev->flags)) ||
1367 (srctype == SYNDROME_SRC_WRITTEN &&
1368 dev->written))
1369 srcs[slot] = sh->dev[i].page;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001370 i = raid6_next_disk(i, disks);
1371 } while (i != d0_idx);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001372
NeilBrowne4424fe2009-10-16 16:27:34 +11001373 return syndrome_disks;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001374}
1375
1376static struct dma_async_tx_descriptor *
1377ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
1378{
1379 int disks = sh->disks;
shli@kernel.org46d5b782014-12-15 12:57:02 +11001380 struct page **blocks = to_addr_page(percpu, 0);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001381 int target;
1382 int qd_idx = sh->qd_idx;
1383 struct dma_async_tx_descriptor *tx;
1384 struct async_submit_ctl submit;
1385 struct r5dev *tgt;
1386 struct page *dest;
1387 int i;
1388 int count;
1389
shli@kernel.org59fc6302014-12-15 12:57:03 +11001390 BUG_ON(sh->batch_head);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001391 if (sh->ops.target < 0)
1392 target = sh->ops.target2;
1393 else if (sh->ops.target2 < 0)
1394 target = sh->ops.target;
1395 else
1396 /* we should only have one valid target */
1397 BUG();
1398 BUG_ON(target < 0);
1399 pr_debug("%s: stripe %llu block: %d\n",
1400 __func__, (unsigned long long)sh->sector, target);
1401
1402 tgt = &sh->dev[target];
1403 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1404 dest = tgt->page;
1405
1406 atomic_inc(&sh->count);
1407
1408 if (target == qd_idx) {
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001409 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001410 blocks[count] = NULL; /* regenerating p is not necessary */
1411 BUG_ON(blocks[count+1] != dest); /* q should already be set */
Dan Williams0403e382009-09-08 17:42:50 -07001412 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1413 ops_complete_compute, sh,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001414 to_addr_conv(sh, percpu, 0));
Dan Williamsac6b53b2009-07-14 13:40:19 -07001415 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
1416 } else {
1417 /* Compute any data- or p-drive using XOR */
1418 count = 0;
1419 for (i = disks; i-- ; ) {
1420 if (i == target || i == qd_idx)
1421 continue;
1422 blocks[count++] = sh->dev[i].page;
1423 }
1424
Dan Williams0403e382009-09-08 17:42:50 -07001425 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1426 NULL, ops_complete_compute, sh,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001427 to_addr_conv(sh, percpu, 0));
Dan Williamsac6b53b2009-07-14 13:40:19 -07001428 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
1429 }
1430
1431 return tx;
1432}
1433
1434static struct dma_async_tx_descriptor *
1435ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
1436{
1437 int i, count, disks = sh->disks;
1438 int syndrome_disks = sh->ddf_layout ? disks : disks-2;
1439 int d0_idx = raid6_d0(sh);
1440 int faila = -1, failb = -1;
1441 int target = sh->ops.target;
1442 int target2 = sh->ops.target2;
1443 struct r5dev *tgt = &sh->dev[target];
1444 struct r5dev *tgt2 = &sh->dev[target2];
1445 struct dma_async_tx_descriptor *tx;
shli@kernel.org46d5b782014-12-15 12:57:02 +11001446 struct page **blocks = to_addr_page(percpu, 0);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001447 struct async_submit_ctl submit;
1448
shli@kernel.org59fc6302014-12-15 12:57:03 +11001449 BUG_ON(sh->batch_head);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001450 pr_debug("%s: stripe %llu block1: %d block2: %d\n",
1451 __func__, (unsigned long long)sh->sector, target, target2);
1452 BUG_ON(target < 0 || target2 < 0);
1453 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1454 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
1455
Dan Williams6c910a72009-09-16 12:24:54 -07001456 /* we need to open-code set_syndrome_sources to handle the
Dan Williamsac6b53b2009-07-14 13:40:19 -07001457 * slot number conversion for 'faila' and 'failb'
1458 */
1459 for (i = 0; i < disks ; i++)
NeilBrown5dd33c92009-10-16 16:40:25 +11001460 blocks[i] = NULL;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001461 count = 0;
1462 i = d0_idx;
1463 do {
1464 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1465
1466 blocks[slot] = sh->dev[i].page;
1467
1468 if (i == target)
1469 faila = slot;
1470 if (i == target2)
1471 failb = slot;
1472 i = raid6_next_disk(i, disks);
1473 } while (i != d0_idx);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001474
1475 BUG_ON(faila == failb);
1476 if (failb < faila)
1477 swap(faila, failb);
1478 pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
1479 __func__, (unsigned long long)sh->sector, faila, failb);
1480
1481 atomic_inc(&sh->count);
1482
1483 if (failb == syndrome_disks+1) {
1484 /* Q disk is one of the missing disks */
1485 if (faila == syndrome_disks) {
1486 /* Missing P+Q, just recompute */
Dan Williams0403e382009-09-08 17:42:50 -07001487 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1488 ops_complete_compute, sh,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001489 to_addr_conv(sh, percpu, 0));
NeilBrowne4424fe2009-10-16 16:27:34 +11001490 return async_gen_syndrome(blocks, 0, syndrome_disks+2,
Dan Williamsac6b53b2009-07-14 13:40:19 -07001491 STRIPE_SIZE, &submit);
1492 } else {
1493 struct page *dest;
1494 int data_target;
1495 int qd_idx = sh->qd_idx;
1496
1497 /* Missing D+Q: recompute D from P, then recompute Q */
1498 if (target == qd_idx)
1499 data_target = target2;
1500 else
1501 data_target = target;
1502
1503 count = 0;
1504 for (i = disks; i-- ; ) {
1505 if (i == data_target || i == qd_idx)
1506 continue;
1507 blocks[count++] = sh->dev[i].page;
1508 }
1509 dest = sh->dev[data_target].page;
Dan Williams0403e382009-09-08 17:42:50 -07001510 init_async_submit(&submit,
1511 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1512 NULL, NULL, NULL,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001513 to_addr_conv(sh, percpu, 0));
Dan Williamsac6b53b2009-07-14 13:40:19 -07001514 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
1515 &submit);
1516
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001517 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
Dan Williams0403e382009-09-08 17:42:50 -07001518 init_async_submit(&submit, ASYNC_TX_FENCE, tx,
1519 ops_complete_compute, sh,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001520 to_addr_conv(sh, percpu, 0));
Dan Williamsac6b53b2009-07-14 13:40:19 -07001521 return async_gen_syndrome(blocks, 0, count+2,
1522 STRIPE_SIZE, &submit);
1523 }
Dan Williamsac6b53b2009-07-14 13:40:19 -07001524 } else {
Dan Williams6c910a72009-09-16 12:24:54 -07001525 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1526 ops_complete_compute, sh,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001527 to_addr_conv(sh, percpu, 0));
Dan Williams6c910a72009-09-16 12:24:54 -07001528 if (failb == syndrome_disks) {
1529 /* We're missing D+P. */
1530 return async_raid6_datap_recov(syndrome_disks+2,
1531 STRIPE_SIZE, faila,
1532 blocks, &submit);
1533 } else {
1534 /* We're missing D+D. */
1535 return async_raid6_2data_recov(syndrome_disks+2,
1536 STRIPE_SIZE, faila, failb,
1537 blocks, &submit);
1538 }
Dan Williamsac6b53b2009-07-14 13:40:19 -07001539 }
1540}
1541
Dan Williams91c00922007-01-02 13:52:30 -07001542static void ops_complete_prexor(void *stripe_head_ref)
1543{
1544 struct stripe_head *sh = stripe_head_ref;
1545
Harvey Harrisone46b2722008-04-28 02:15:50 -07001546 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001547 (unsigned long long)sh->sector);
Dan Williams91c00922007-01-02 13:52:30 -07001548}
1549
1550static struct dma_async_tx_descriptor *
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001551ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
1552 struct dma_async_tx_descriptor *tx)
Dan Williams91c00922007-01-02 13:52:30 -07001553{
Dan Williams91c00922007-01-02 13:52:30 -07001554 int disks = sh->disks;
shli@kernel.org46d5b782014-12-15 12:57:02 +11001555 struct page **xor_srcs = to_addr_page(percpu, 0);
Dan Williams91c00922007-01-02 13:52:30 -07001556 int count = 0, pd_idx = sh->pd_idx, i;
Dan Williamsa08abd82009-06-03 11:43:59 -07001557 struct async_submit_ctl submit;
Dan Williams91c00922007-01-02 13:52:30 -07001558
1559 /* existing parity data subtracted */
1560 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1561
shli@kernel.org59fc6302014-12-15 12:57:03 +11001562 BUG_ON(sh->batch_head);
Harvey Harrisone46b2722008-04-28 02:15:50 -07001563 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001564 (unsigned long long)sh->sector);
1565
1566 for (i = disks; i--; ) {
1567 struct r5dev *dev = &sh->dev[i];
1568 /* Only process blocks that are known to be uptodate */
Dan Williamsd8ee0722008-06-28 08:32:06 +10001569 if (test_bit(R5_Wantdrain, &dev->flags))
Dan Williams91c00922007-01-02 13:52:30 -07001570 xor_srcs[count++] = dev->page;
1571 }
1572
Dan Williams0403e382009-09-08 17:42:50 -07001573 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001574 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
Dan Williamsa08abd82009-06-03 11:43:59 -07001575 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
Dan Williams91c00922007-01-02 13:52:30 -07001576
1577 return tx;
1578}
1579
1580static struct dma_async_tx_descriptor *
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001581ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
1582 struct dma_async_tx_descriptor *tx)
1583{
1584 struct page **blocks = to_addr_page(percpu, 0);
1585 int count;
1586 struct async_submit_ctl submit;
1587
1588 pr_debug("%s: stripe %llu\n", __func__,
1589 (unsigned long long)sh->sector);
1590
1591 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);
1592
1593 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
1594 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1595 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
1596
1597 return tx;
1598}
1599
1600static struct dma_async_tx_descriptor *
Dan Williamsd8ee0722008-06-28 08:32:06 +10001601ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
Dan Williams91c00922007-01-02 13:52:30 -07001602{
1603 int disks = sh->disks;
Dan Williamsd8ee0722008-06-28 08:32:06 +10001604 int i;
shli@kernel.org59fc6302014-12-15 12:57:03 +11001605 struct stripe_head *head_sh = sh;
Dan Williams91c00922007-01-02 13:52:30 -07001606
Harvey Harrisone46b2722008-04-28 02:15:50 -07001607 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001608 (unsigned long long)sh->sector);
1609
1610 for (i = disks; i--; ) {
shli@kernel.org59fc6302014-12-15 12:57:03 +11001611 struct r5dev *dev;
Dan Williams91c00922007-01-02 13:52:30 -07001612 struct bio *chosen;
Dan Williams91c00922007-01-02 13:52:30 -07001613
shli@kernel.org59fc6302014-12-15 12:57:03 +11001614 sh = head_sh;
1615 if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
Dan Williams91c00922007-01-02 13:52:30 -07001616 struct bio *wbi;
1617
shli@kernel.org59fc6302014-12-15 12:57:03 +11001618again:
1619 dev = &sh->dev[i];
Shaohua Lib17459c2012-07-19 16:01:31 +10001620 spin_lock_irq(&sh->stripe_lock);
Dan Williams91c00922007-01-02 13:52:30 -07001621 chosen = dev->towrite;
1622 dev->towrite = NULL;
shli@kernel.org7a87f432014-12-15 12:57:03 +11001623 sh->overwrite_disks = 0;
Dan Williams91c00922007-01-02 13:52:30 -07001624 BUG_ON(dev->written);
1625 wbi = dev->written = chosen;
Shaohua Lib17459c2012-07-19 16:01:31 +10001626 spin_unlock_irq(&sh->stripe_lock);
Shaohua Lid592a992014-05-21 17:57:44 +08001627 WARN_ON(dev->page != dev->orig_page);
Dan Williams91c00922007-01-02 13:52:30 -07001628
Kent Overstreet4f024f32013-10-11 15:44:27 -07001629 while (wbi && wbi->bi_iter.bi_sector <
Dan Williams91c00922007-01-02 13:52:30 -07001630 dev->sector + STRIPE_SECTORS) {
Tejun Heoe9c74692010-09-03 11:56:18 +02001631 if (wbi->bi_rw & REQ_FUA)
1632 set_bit(R5_WantFUA, &dev->flags);
Shaohua Libc0934f2012-05-22 13:55:05 +10001633 if (wbi->bi_rw & REQ_SYNC)
1634 set_bit(R5_SyncIO, &dev->flags);
Shaohua Li9e4447682012-10-11 13:49:49 +11001635 if (wbi->bi_rw & REQ_DISCARD)
Shaohua Li620125f2012-10-11 13:49:05 +11001636 set_bit(R5_Discard, &dev->flags);
Shaohua Lid592a992014-05-21 17:57:44 +08001637 else {
1638 tx = async_copy_data(1, wbi, &dev->page,
1639 dev->sector, tx, sh);
1640 if (dev->page != dev->orig_page) {
1641 set_bit(R5_SkipCopy, &dev->flags);
1642 clear_bit(R5_UPTODATE, &dev->flags);
1643 clear_bit(R5_OVERWRITE, &dev->flags);
1644 }
1645 }
Dan Williams91c00922007-01-02 13:52:30 -07001646 wbi = r5_next_bio(wbi, dev->sector);
1647 }
shli@kernel.org59fc6302014-12-15 12:57:03 +11001648
1649 if (head_sh->batch_head) {
1650 sh = list_first_entry(&sh->batch_list,
1651 struct stripe_head,
1652 batch_list);
1653 if (sh == head_sh)
1654 continue;
1655 goto again;
1656 }
Dan Williams91c00922007-01-02 13:52:30 -07001657 }
1658 }
1659
1660 return tx;
1661}
1662
Dan Williamsac6b53b2009-07-14 13:40:19 -07001663static void ops_complete_reconstruct(void *stripe_head_ref)
Dan Williams91c00922007-01-02 13:52:30 -07001664{
1665 struct stripe_head *sh = stripe_head_ref;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001666 int disks = sh->disks;
1667 int pd_idx = sh->pd_idx;
1668 int qd_idx = sh->qd_idx;
1669 int i;
Shaohua Li9e4447682012-10-11 13:49:49 +11001670 bool fua = false, sync = false, discard = false;
Dan Williams91c00922007-01-02 13:52:30 -07001671
Harvey Harrisone46b2722008-04-28 02:15:50 -07001672 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001673 (unsigned long long)sh->sector);
1674
Shaohua Libc0934f2012-05-22 13:55:05 +10001675 for (i = disks; i--; ) {
Tejun Heoe9c74692010-09-03 11:56:18 +02001676 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
Shaohua Libc0934f2012-05-22 13:55:05 +10001677 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
Shaohua Li9e4447682012-10-11 13:49:49 +11001678 discard |= test_bit(R5_Discard, &sh->dev[i].flags);
Shaohua Libc0934f2012-05-22 13:55:05 +10001679 }
Tejun Heoe9c74692010-09-03 11:56:18 +02001680
Dan Williams91c00922007-01-02 13:52:30 -07001681 for (i = disks; i--; ) {
1682 struct r5dev *dev = &sh->dev[i];
Dan Williamsac6b53b2009-07-14 13:40:19 -07001683
Tejun Heoe9c74692010-09-03 11:56:18 +02001684 if (dev->written || i == pd_idx || i == qd_idx) {
Shaohua Lid592a992014-05-21 17:57:44 +08001685 if (!discard && !test_bit(R5_SkipCopy, &dev->flags))
Shaohua Li9e4447682012-10-11 13:49:49 +11001686 set_bit(R5_UPTODATE, &dev->flags);
Tejun Heoe9c74692010-09-03 11:56:18 +02001687 if (fua)
1688 set_bit(R5_WantFUA, &dev->flags);
Shaohua Libc0934f2012-05-22 13:55:05 +10001689 if (sync)
1690 set_bit(R5_SyncIO, &dev->flags);
Tejun Heoe9c74692010-09-03 11:56:18 +02001691 }
Dan Williams91c00922007-01-02 13:52:30 -07001692 }
1693
Dan Williamsd8ee0722008-06-28 08:32:06 +10001694 if (sh->reconstruct_state == reconstruct_state_drain_run)
1695 sh->reconstruct_state = reconstruct_state_drain_result;
1696 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
1697 sh->reconstruct_state = reconstruct_state_prexor_drain_result;
1698 else {
1699 BUG_ON(sh->reconstruct_state != reconstruct_state_run);
1700 sh->reconstruct_state = reconstruct_state_result;
1701 }
Dan Williams91c00922007-01-02 13:52:30 -07001702
1703 set_bit(STRIPE_HANDLE, &sh->state);
Shaohua Li6d036f72015-08-13 14:31:57 -07001704 raid5_release_stripe(sh);
Dan Williams91c00922007-01-02 13:52:30 -07001705}
1706
1707static void
Dan Williamsac6b53b2009-07-14 13:40:19 -07001708ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1709 struct dma_async_tx_descriptor *tx)
Dan Williams91c00922007-01-02 13:52:30 -07001710{
Dan Williams91c00922007-01-02 13:52:30 -07001711 int disks = sh->disks;
shli@kernel.org59fc6302014-12-15 12:57:03 +11001712 struct page **xor_srcs;
Dan Williamsa08abd82009-06-03 11:43:59 -07001713 struct async_submit_ctl submit;
shli@kernel.org59fc6302014-12-15 12:57:03 +11001714 int count, pd_idx = sh->pd_idx, i;
Dan Williams91c00922007-01-02 13:52:30 -07001715 struct page *xor_dest;
Dan Williamsd8ee0722008-06-28 08:32:06 +10001716 int prexor = 0;
Dan Williams91c00922007-01-02 13:52:30 -07001717 unsigned long flags;
shli@kernel.org59fc6302014-12-15 12:57:03 +11001718 int j = 0;
1719 struct stripe_head *head_sh = sh;
1720 int last_stripe;
Dan Williams91c00922007-01-02 13:52:30 -07001721
Harvey Harrisone46b2722008-04-28 02:15:50 -07001722 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001723 (unsigned long long)sh->sector);
1724
Shaohua Li620125f2012-10-11 13:49:05 +11001725 for (i = 0; i < sh->disks; i++) {
1726 if (pd_idx == i)
1727 continue;
1728 if (!test_bit(R5_Discard, &sh->dev[i].flags))
1729 break;
1730 }
1731 if (i >= sh->disks) {
1732 atomic_inc(&sh->count);
Shaohua Li620125f2012-10-11 13:49:05 +11001733 set_bit(R5_Discard, &sh->dev[pd_idx].flags);
1734 ops_complete_reconstruct(sh);
1735 return;
1736 }
shli@kernel.org59fc6302014-12-15 12:57:03 +11001737again:
1738 count = 0;
1739 xor_srcs = to_addr_page(percpu, j);
Dan Williams91c00922007-01-02 13:52:30 -07001740 /* check if prexor is active which means only process blocks
1741 * that are part of a read-modify-write (written)
1742 */
shli@kernel.org59fc6302014-12-15 12:57:03 +11001743 if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
Dan Williamsd8ee0722008-06-28 08:32:06 +10001744 prexor = 1;
Dan Williams91c00922007-01-02 13:52:30 -07001745 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1746 for (i = disks; i--; ) {
1747 struct r5dev *dev = &sh->dev[i];
shli@kernel.org59fc6302014-12-15 12:57:03 +11001748 if (head_sh->dev[i].written)
Dan Williams91c00922007-01-02 13:52:30 -07001749 xor_srcs[count++] = dev->page;
1750 }
1751 } else {
1752 xor_dest = sh->dev[pd_idx].page;
1753 for (i = disks; i--; ) {
1754 struct r5dev *dev = &sh->dev[i];
1755 if (i != pd_idx)
1756 xor_srcs[count++] = dev->page;
1757 }
1758 }
1759
Dan Williams91c00922007-01-02 13:52:30 -07001760 /* 1/ if we prexor'd then the dest is reused as a source
1761 * 2/ if we did not prexor then we are redoing the parity
1762 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
1763 * for the synchronous xor case
1764 */
shli@kernel.org59fc6302014-12-15 12:57:03 +11001765 last_stripe = !head_sh->batch_head ||
1766 list_first_entry(&sh->batch_list,
1767 struct stripe_head, batch_list) == head_sh;
1768 if (last_stripe) {
1769 flags = ASYNC_TX_ACK |
1770 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
Dan Williams91c00922007-01-02 13:52:30 -07001771
shli@kernel.org59fc6302014-12-15 12:57:03 +11001772 atomic_inc(&head_sh->count);
1773 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
1774 to_addr_conv(sh, percpu, j));
1775 } else {
1776 flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
1777 init_async_submit(&submit, flags, tx, NULL, NULL,
1778 to_addr_conv(sh, percpu, j));
1779 }
Dan Williams91c00922007-01-02 13:52:30 -07001780
Dan Williamsa08abd82009-06-03 11:43:59 -07001781 if (unlikely(count == 1))
1782 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1783 else
1784 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
shli@kernel.org59fc6302014-12-15 12:57:03 +11001785 if (!last_stripe) {
1786 j++;
1787 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1788 batch_list);
1789 goto again;
1790 }
Dan Williams91c00922007-01-02 13:52:30 -07001791}
1792
Dan Williamsac6b53b2009-07-14 13:40:19 -07001793static void
1794ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
1795 struct dma_async_tx_descriptor *tx)
1796{
1797 struct async_submit_ctl submit;
shli@kernel.org59fc6302014-12-15 12:57:03 +11001798 struct page **blocks;
1799 int count, i, j = 0;
1800 struct stripe_head *head_sh = sh;
1801 int last_stripe;
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001802 int synflags;
1803 unsigned long txflags;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001804
1805 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
1806
Shaohua Li620125f2012-10-11 13:49:05 +11001807 for (i = 0; i < sh->disks; i++) {
1808 if (sh->pd_idx == i || sh->qd_idx == i)
1809 continue;
1810 if (!test_bit(R5_Discard, &sh->dev[i].flags))
1811 break;
1812 }
1813 if (i >= sh->disks) {
1814 atomic_inc(&sh->count);
Shaohua Li620125f2012-10-11 13:49:05 +11001815 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
1816 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
1817 ops_complete_reconstruct(sh);
1818 return;
1819 }
1820
shli@kernel.org59fc6302014-12-15 12:57:03 +11001821again:
1822 blocks = to_addr_page(percpu, j);
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001823
1824 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1825 synflags = SYNDROME_SRC_WRITTEN;
1826 txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
1827 } else {
1828 synflags = SYNDROME_SRC_ALL;
1829 txflags = ASYNC_TX_ACK;
1830 }
1831
1832 count = set_syndrome_sources(blocks, sh, synflags);
shli@kernel.org59fc6302014-12-15 12:57:03 +11001833 last_stripe = !head_sh->batch_head ||
1834 list_first_entry(&sh->batch_list,
1835 struct stripe_head, batch_list) == head_sh;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001836
shli@kernel.org59fc6302014-12-15 12:57:03 +11001837 if (last_stripe) {
1838 atomic_inc(&head_sh->count);
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001839 init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
shli@kernel.org59fc6302014-12-15 12:57:03 +11001840 head_sh, to_addr_conv(sh, percpu, j));
1841 } else
1842 init_async_submit(&submit, 0, tx, NULL, NULL,
1843 to_addr_conv(sh, percpu, j));
Shaohua Li48769692015-05-13 09:30:08 -07001844 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
shli@kernel.org59fc6302014-12-15 12:57:03 +11001845 if (!last_stripe) {
1846 j++;
1847 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1848 batch_list);
1849 goto again;
1850 }
Dan Williams91c00922007-01-02 13:52:30 -07001851}
1852
1853static void ops_complete_check(void *stripe_head_ref)
1854{
1855 struct stripe_head *sh = stripe_head_ref;
Dan Williams91c00922007-01-02 13:52:30 -07001856
Harvey Harrisone46b2722008-04-28 02:15:50 -07001857 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001858 (unsigned long long)sh->sector);
1859
Dan Williamsecc65c92008-06-28 08:31:57 +10001860 sh->check_state = check_state_check_result;
Dan Williams91c00922007-01-02 13:52:30 -07001861 set_bit(STRIPE_HANDLE, &sh->state);
Shaohua Li6d036f72015-08-13 14:31:57 -07001862 raid5_release_stripe(sh);
Dan Williams91c00922007-01-02 13:52:30 -07001863}
1864
Dan Williamsac6b53b2009-07-14 13:40:19 -07001865static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
Dan Williams91c00922007-01-02 13:52:30 -07001866{
Dan Williams91c00922007-01-02 13:52:30 -07001867 int disks = sh->disks;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001868 int pd_idx = sh->pd_idx;
1869 int qd_idx = sh->qd_idx;
1870 struct page *xor_dest;
shli@kernel.org46d5b782014-12-15 12:57:02 +11001871 struct page **xor_srcs = to_addr_page(percpu, 0);
Dan Williams91c00922007-01-02 13:52:30 -07001872 struct dma_async_tx_descriptor *tx;
Dan Williamsa08abd82009-06-03 11:43:59 -07001873 struct async_submit_ctl submit;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001874 int count;
1875 int i;
Dan Williams91c00922007-01-02 13:52:30 -07001876
Harvey Harrisone46b2722008-04-28 02:15:50 -07001877 pr_debug("%s: stripe %llu\n", __func__,
Dan Williams91c00922007-01-02 13:52:30 -07001878 (unsigned long long)sh->sector);
1879
shli@kernel.org59fc6302014-12-15 12:57:03 +11001880 BUG_ON(sh->batch_head);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001881 count = 0;
1882 xor_dest = sh->dev[pd_idx].page;
1883 xor_srcs[count++] = xor_dest;
Dan Williams91c00922007-01-02 13:52:30 -07001884 for (i = disks; i--; ) {
Dan Williamsac6b53b2009-07-14 13:40:19 -07001885 if (i == pd_idx || i == qd_idx)
1886 continue;
1887 xor_srcs[count++] = sh->dev[i].page;
Dan Williams91c00922007-01-02 13:52:30 -07001888 }
1889
Dan Williamsd6f38f32009-07-14 11:50:52 -07001890 init_async_submit(&submit, 0, NULL, NULL, NULL,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001891 to_addr_conv(sh, percpu, 0));
Dan Williams099f53c2009-04-08 14:28:37 -07001892 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
Dan Williamsa08abd82009-06-03 11:43:59 -07001893 &sh->ops.zero_sum_result, &submit);
Dan Williams91c00922007-01-02 13:52:30 -07001894
Dan Williams91c00922007-01-02 13:52:30 -07001895 atomic_inc(&sh->count);
Dan Williamsa08abd82009-06-03 11:43:59 -07001896 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
1897 tx = async_trigger_callback(&submit);
Dan Williams91c00922007-01-02 13:52:30 -07001898}
1899
Dan Williamsac6b53b2009-07-14 13:40:19 -07001900static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
1901{
shli@kernel.org46d5b782014-12-15 12:57:02 +11001902 struct page **srcs = to_addr_page(percpu, 0);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001903 struct async_submit_ctl submit;
1904 int count;
1905
1906 pr_debug("%s: stripe %llu checkp: %d\n", __func__,
1907 (unsigned long long)sh->sector, checkp);
1908
shli@kernel.org59fc6302014-12-15 12:57:03 +11001909 BUG_ON(sh->batch_head);
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001910 count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL);
Dan Williamsac6b53b2009-07-14 13:40:19 -07001911 if (!checkp)
1912 srcs[count] = NULL;
1913
1914 atomic_inc(&sh->count);
1915 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
shli@kernel.org46d5b782014-12-15 12:57:02 +11001916 sh, to_addr_conv(sh, percpu, 0));
Dan Williamsac6b53b2009-07-14 13:40:19 -07001917 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
1918 &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1919}
1920
NeilBrown51acbce2013-02-28 09:08:34 +11001921static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
Dan Williams91c00922007-01-02 13:52:30 -07001922{
1923 int overlap_clear = 0, i, disks = sh->disks;
1924 struct dma_async_tx_descriptor *tx = NULL;
NeilBrownd1688a62011-10-11 16:49:52 +11001925 struct r5conf *conf = sh->raid_conf;
Dan Williamsac6b53b2009-07-14 13:40:19 -07001926 int level = conf->level;
Dan Williamsd6f38f32009-07-14 11:50:52 -07001927 struct raid5_percpu *percpu;
1928 unsigned long cpu;
Dan Williams91c00922007-01-02 13:52:30 -07001929
Dan Williamsd6f38f32009-07-14 11:50:52 -07001930 cpu = get_cpu();
1931 percpu = per_cpu_ptr(conf->percpu, cpu);
Dan Williams83de75c2008-06-28 08:31:58 +10001932 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
Dan Williams91c00922007-01-02 13:52:30 -07001933 ops_run_biofill(sh);
1934 overlap_clear++;
1935 }
1936
Dan Williams7b3a8712008-06-28 08:32:09 +10001937 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
Dan Williamsac6b53b2009-07-14 13:40:19 -07001938 if (level < 6)
1939 tx = ops_run_compute5(sh, percpu);
1940 else {
1941 if (sh->ops.target2 < 0 || sh->ops.target < 0)
1942 tx = ops_run_compute6_1(sh, percpu);
1943 else
1944 tx = ops_run_compute6_2(sh, percpu);
1945 }
1946 /* terminate the chain if reconstruct is not set to be run */
1947 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
Dan Williams7b3a8712008-06-28 08:32:09 +10001948 async_tx_ack(tx);
1949 }
Dan Williams91c00922007-01-02 13:52:30 -07001950
Markus Stockhausen584acdd2014-12-15 12:57:05 +11001951 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
1952 if (level < 6)
1953 tx = ops_run_prexor5(sh, percpu, tx);
1954 else
1955 tx = ops_run_prexor6(sh, percpu, tx);
1956 }
Dan Williams91c00922007-01-02 13:52:30 -07001957
Dan Williams600aa102008-06-28 08:32:05 +10001958 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
Dan Williamsd8ee0722008-06-28 08:32:06 +10001959 tx = ops_run_biodrain(sh, tx);
Dan Williams91c00922007-01-02 13:52:30 -07001960 overlap_clear++;
1961 }
1962
Dan Williamsac6b53b2009-07-14 13:40:19 -07001963 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
1964 if (level < 6)
1965 ops_run_reconstruct5(sh, percpu, tx);
1966 else
1967 ops_run_reconstruct6(sh, percpu, tx);
1968 }
Dan Williams91c00922007-01-02 13:52:30 -07001969
Dan Williamsac6b53b2009-07-14 13:40:19 -07001970 if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
1971 if (sh->check_state == check_state_run)
1972 ops_run_check_p(sh, percpu);
1973 else if (sh->check_state == check_state_run_q)
1974 ops_run_check_pq(sh, percpu, 0);
1975 else if (sh->check_state == check_state_run_pq)
1976 ops_run_check_pq(sh, percpu, 1);
1977 else
1978 BUG();
1979 }
Dan Williams91c00922007-01-02 13:52:30 -07001980
shli@kernel.org59fc6302014-12-15 12:57:03 +11001981 if (overlap_clear && !sh->batch_head)
Dan Williams91c00922007-01-02 13:52:30 -07001982 for (i = disks; i--; ) {
1983 struct r5dev *dev = &sh->dev[i];
1984 if (test_and_clear_bit(R5_Overlap, &dev->flags))
1985 wake_up(&sh->raid_conf->wait_for_overlap);
1986 }
Dan Williamsd6f38f32009-07-14 11:50:52 -07001987 put_cpu();
Dan Williams91c00922007-01-02 13:52:30 -07001988}
1989
NeilBrownf18c1a32015-05-08 18:19:04 +10001990static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp)
1991{
1992 struct stripe_head *sh;
1993
1994 sh = kmem_cache_zalloc(sc, gfp);
1995 if (sh) {
1996 spin_lock_init(&sh->stripe_lock);
1997 spin_lock_init(&sh->batch_lock);
1998 INIT_LIST_HEAD(&sh->batch_list);
1999 INIT_LIST_HEAD(&sh->lru);
2000 atomic_set(&sh->count, 1);
2001 }
2002 return sh;
2003}
NeilBrown486f0642015-02-25 12:10:35 +11002004static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002005{
2006 struct stripe_head *sh;
NeilBrownf18c1a32015-05-08 18:19:04 +10002007
2008 sh = alloc_stripe(conf->slab_cache, gfp);
NeilBrown3f294f42005-11-08 21:39:25 -08002009 if (!sh)
2010 return 0;
Namhyung Kim6ce32842011-07-18 17:38:50 +10002011
NeilBrown3f294f42005-11-08 21:39:25 -08002012 sh->raid_conf = conf;
NeilBrown3f294f42005-11-08 21:39:25 -08002013
NeilBrowna9683a72015-02-25 12:02:51 +11002014 if (grow_buffers(sh, gfp)) {
NeilBrowne4e11e32010-06-16 16:45:16 +10002015 shrink_buffers(sh);
NeilBrown3f294f42005-11-08 21:39:25 -08002016 kmem_cache_free(conf->slab_cache, sh);
2017 return 0;
2018 }
NeilBrown486f0642015-02-25 12:10:35 +11002019 sh->hash_lock_index =
2020 conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
NeilBrown3f294f42005-11-08 21:39:25 -08002021 /* we just created an active stripe so... */
NeilBrown3f294f42005-11-08 21:39:25 -08002022 atomic_inc(&conf->active_stripes);
shli@kernel.org59fc6302014-12-15 12:57:03 +11002023
Shaohua Li6d036f72015-08-13 14:31:57 -07002024 raid5_release_stripe(sh);
NeilBrown486f0642015-02-25 12:10:35 +11002025 conf->max_nr_stripes++;
NeilBrown3f294f42005-11-08 21:39:25 -08002026 return 1;
2027}
2028
NeilBrownd1688a62011-10-11 16:49:52 +11002029static int grow_stripes(struct r5conf *conf, int num)
NeilBrown3f294f42005-11-08 21:39:25 -08002030{
Christoph Lametere18b8902006-12-06 20:33:20 -08002031 struct kmem_cache *sc;
NeilBrown5e5e3e72009-10-16 16:35:30 +11002032 int devs = max(conf->raid_disks, conf->previous_raid_disks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002033
NeilBrownf4be6b42010-06-01 19:37:25 +10002034 if (conf->mddev->gendisk)
2035 sprintf(conf->cache_name[0],
2036 "raid%d-%s", conf->level, mdname(conf->mddev));
2037 else
2038 sprintf(conf->cache_name[0],
2039 "raid%d-%p", conf->level, conf->mddev);
2040 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
2041
NeilBrownad01c9e2006-03-27 01:18:07 -08002042 conf->active_name = 0;
2043 sc = kmem_cache_create(conf->cache_name[conf->active_name],
Linus Torvalds1da177e2005-04-16 15:20:36 -07002044 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
Paul Mundt20c2df82007-07-20 10:11:58 +09002045 0, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002046 if (!sc)
2047 return 1;
2048 conf->slab_cache = sc;
NeilBrownad01c9e2006-03-27 01:18:07 -08002049 conf->pool_size = devs;
NeilBrown486f0642015-02-25 12:10:35 +11002050 while (num--)
2051 if (!grow_one_stripe(conf, GFP_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002052 return 1;
NeilBrown486f0642015-02-25 12:10:35 +11002053
Linus Torvalds1da177e2005-04-16 15:20:36 -07002054 return 0;
2055}
NeilBrown29269552006-03-27 01:18:10 -08002056
Dan Williamsd6f38f32009-07-14 11:50:52 -07002057/**
2058 * scribble_len - return the required size of the scribble region
2059 * @num - total number of disks in the array
2060 *
2061 * The size must be enough to contain:
2062 * 1/ a struct page pointer for each device in the array +2
2063 * 2/ room to convert each entry in (1) to its corresponding dma
2064 * (dma_map_page()) or page (page_address()) address.
2065 *
2066 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
2067 * calculate over all devices (not just the data blocks), using zeros in place
2068 * of the P and Q blocks.
2069 */
shli@kernel.org46d5b782014-12-15 12:57:02 +11002070static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
Dan Williamsd6f38f32009-07-14 11:50:52 -07002071{
shli@kernel.org46d5b782014-12-15 12:57:02 +11002072 struct flex_array *ret;
Dan Williamsd6f38f32009-07-14 11:50:52 -07002073 size_t len;
2074
2075 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
shli@kernel.org46d5b782014-12-15 12:57:02 +11002076 ret = flex_array_alloc(len, cnt, flags);
2077 if (!ret)
2078 return NULL;
2079 /* always prealloc all elements, so no locking is required */
2080 if (flex_array_prealloc(ret, 0, cnt, flags)) {
2081 flex_array_free(ret);
2082 return NULL;
2083 }
2084 return ret;
Dan Williamsd6f38f32009-07-14 11:50:52 -07002085}
2086
NeilBrown738a2732015-05-08 18:19:39 +10002087static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
2088{
2089 unsigned long cpu;
2090 int err = 0;
2091
2092 mddev_suspend(conf->mddev);
2093 get_online_cpus();
2094 for_each_present_cpu(cpu) {
2095 struct raid5_percpu *percpu;
2096 struct flex_array *scribble;
2097
2098 percpu = per_cpu_ptr(conf->percpu, cpu);
2099 scribble = scribble_alloc(new_disks,
2100 new_sectors / STRIPE_SECTORS,
2101 GFP_NOIO);
2102
2103 if (scribble) {
2104 flex_array_free(percpu->scribble);
2105 percpu->scribble = scribble;
2106 } else {
2107 err = -ENOMEM;
2108 break;
2109 }
2110 }
2111 put_online_cpus();
2112 mddev_resume(conf->mddev);
2113 return err;
2114}
2115
NeilBrownd1688a62011-10-11 16:49:52 +11002116static int resize_stripes(struct r5conf *conf, int newsize)
NeilBrownad01c9e2006-03-27 01:18:07 -08002117{
2118 /* Make all the stripes able to hold 'newsize' devices.
2119 * New slots in each stripe get 'page' set to a new page.
2120 *
2121 * This happens in stages:
2122 * 1/ create a new kmem_cache and allocate the required number of
2123 * stripe_heads.
Masanari Iida83f0d772012-10-30 00:18:08 +09002124 * 2/ gather all the old stripe_heads and transfer the pages across
NeilBrownad01c9e2006-03-27 01:18:07 -08002125 * to the new stripe_heads. This will have the side effect of
2126 * freezing the array as once all stripe_heads have been collected,
2127 * no IO will be possible. Old stripe heads are freed once their
2128 * pages have been transferred over, and the old kmem_cache is
2129 * freed when all stripes are done.
2130 * 3/ reallocate conf->disks to be suitable bigger. If this fails,
2131 * we simple return a failre status - no need to clean anything up.
2132 * 4/ allocate new pages for the new slots in the new stripe_heads.
2133 * If this fails, we don't bother trying the shrink the
2134 * stripe_heads down again, we just leave them as they are.
2135 * As each stripe_head is processed the new one is released into
2136 * active service.
2137 *
2138 * Once step2 is started, we cannot afford to wait for a write,
2139 * so we use GFP_NOIO allocations.
2140 */
2141 struct stripe_head *osh, *nsh;
2142 LIST_HEAD(newstripes);
2143 struct disk_info *ndisks;
Dan Williamsb5470dc2008-06-27 21:44:04 -07002144 int err;
Christoph Lametere18b8902006-12-06 20:33:20 -08002145 struct kmem_cache *sc;
NeilBrownad01c9e2006-03-27 01:18:07 -08002146 int i;
Shaohua Li566c09c2013-11-14 15:16:17 +11002147 int hash, cnt;
NeilBrownad01c9e2006-03-27 01:18:07 -08002148
2149 if (newsize <= conf->pool_size)
2150 return 0; /* never bother to shrink */
2151
Dan Williamsb5470dc2008-06-27 21:44:04 -07002152 err = md_allow_write(conf->mddev);
2153 if (err)
2154 return err;
NeilBrown2a2275d2007-01-26 00:57:11 -08002155
NeilBrownad01c9e2006-03-27 01:18:07 -08002156 /* Step 1 */
2157 sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
2158 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
Paul Mundt20c2df82007-07-20 10:11:58 +09002159 0, 0, NULL);
NeilBrownad01c9e2006-03-27 01:18:07 -08002160 if (!sc)
2161 return -ENOMEM;
2162
NeilBrown2d5b5692015-07-06 12:49:23 +10002163 /* Need to ensure auto-resizing doesn't interfere */
2164 mutex_lock(&conf->cache_size_mutex);
2165
NeilBrownad01c9e2006-03-27 01:18:07 -08002166 for (i = conf->max_nr_stripes; i; i--) {
NeilBrownf18c1a32015-05-08 18:19:04 +10002167 nsh = alloc_stripe(sc, GFP_KERNEL);
NeilBrownad01c9e2006-03-27 01:18:07 -08002168 if (!nsh)
2169 break;
2170
NeilBrownad01c9e2006-03-27 01:18:07 -08002171 nsh->raid_conf = conf;
NeilBrownad01c9e2006-03-27 01:18:07 -08002172 list_add(&nsh->lru, &newstripes);
2173 }
2174 if (i) {
2175 /* didn't get enough, give up */
2176 while (!list_empty(&newstripes)) {
2177 nsh = list_entry(newstripes.next, struct stripe_head, lru);
2178 list_del(&nsh->lru);
2179 kmem_cache_free(sc, nsh);
2180 }
2181 kmem_cache_destroy(sc);
NeilBrown2d5b5692015-07-06 12:49:23 +10002182 mutex_unlock(&conf->cache_size_mutex);
NeilBrownad01c9e2006-03-27 01:18:07 -08002183 return -ENOMEM;
2184 }
2185 /* Step 2 - Must use GFP_NOIO now.
2186 * OK, we have enough stripes, start collecting inactive
2187 * stripes and copying them over
2188 */
Shaohua Li566c09c2013-11-14 15:16:17 +11002189 hash = 0;
2190 cnt = 0;
NeilBrownad01c9e2006-03-27 01:18:07 -08002191 list_for_each_entry(nsh, &newstripes, lru) {
Shaohua Li566c09c2013-11-14 15:16:17 +11002192 lock_device_hash_lock(conf, hash);
Yuanhan Liue9e4c372015-05-08 18:19:07 +10002193 wait_event_exclusive_cmd(conf->wait_for_stripe[hash],
Shaohua Li566c09c2013-11-14 15:16:17 +11002194 !list_empty(conf->inactive_list + hash),
2195 unlock_device_hash_lock(conf, hash),
2196 lock_device_hash_lock(conf, hash));
2197 osh = get_free_stripe(conf, hash);
2198 unlock_device_hash_lock(conf, hash);
NeilBrownf18c1a32015-05-08 18:19:04 +10002199
Shaohua Lid592a992014-05-21 17:57:44 +08002200 for(i=0; i<conf->pool_size; i++) {
NeilBrownad01c9e2006-03-27 01:18:07 -08002201 nsh->dev[i].page = osh->dev[i].page;
Shaohua Lid592a992014-05-21 17:57:44 +08002202 nsh->dev[i].orig_page = osh->dev[i].page;
2203 }
Shaohua Li566c09c2013-11-14 15:16:17 +11002204 nsh->hash_lock_index = hash;
NeilBrownad01c9e2006-03-27 01:18:07 -08002205 kmem_cache_free(conf->slab_cache, osh);
Shaohua Li566c09c2013-11-14 15:16:17 +11002206 cnt++;
2207 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
2208 !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
2209 hash++;
2210 cnt = 0;
2211 }
NeilBrownad01c9e2006-03-27 01:18:07 -08002212 }
2213 kmem_cache_destroy(conf->slab_cache);
2214
2215 /* Step 3.
2216 * At this point, we are holding all the stripes so the array
2217 * is completely stalled, so now is a good time to resize
Dan Williamsd6f38f32009-07-14 11:50:52 -07002218 * conf->disks and the scribble region
NeilBrownad01c9e2006-03-27 01:18:07 -08002219 */
2220 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
2221 if (ndisks) {
2222 for (i=0; i<conf->raid_disks; i++)
2223 ndisks[i] = conf->disks[i];
2224 kfree(conf->disks);
2225 conf->disks = ndisks;
2226 } else
2227 err = -ENOMEM;
2228
NeilBrown2d5b5692015-07-06 12:49:23 +10002229 mutex_unlock(&conf->cache_size_mutex);
NeilBrownad01c9e2006-03-27 01:18:07 -08002230 /* Step 4, return new stripes to service */
2231 while(!list_empty(&newstripes)) {
2232 nsh = list_entry(newstripes.next, struct stripe_head, lru);
2233 list_del_init(&nsh->lru);
Dan Williamsd6f38f32009-07-14 11:50:52 -07002234
NeilBrownad01c9e2006-03-27 01:18:07 -08002235 for (i=conf->raid_disks; i < newsize; i++)
2236 if (nsh->dev[i].page == NULL) {
2237 struct page *p = alloc_page(GFP_NOIO);
2238 nsh->dev[i].page = p;
Shaohua Lid592a992014-05-21 17:57:44 +08002239 nsh->dev[i].orig_page = p;
NeilBrownad01c9e2006-03-27 01:18:07 -08002240 if (!p)
2241 err = -ENOMEM;
2242 }
Shaohua Li6d036f72015-08-13 14:31:57 -07002243 raid5_release_stripe(nsh);
NeilBrownad01c9e2006-03-27 01:18:07 -08002244 }
2245 /* critical section pass, GFP_NOIO no longer needed */
2246
2247 conf->slab_cache = sc;
2248 conf->active_name = 1-conf->active_name;
NeilBrown6e9eac22015-05-08 18:19:34 +10002249 if (!err)
2250 conf->pool_size = newsize;
NeilBrownad01c9e2006-03-27 01:18:07 -08002251 return err;
2252}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002253
NeilBrown486f0642015-02-25 12:10:35 +11002254static int drop_one_stripe(struct r5conf *conf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255{
2256 struct stripe_head *sh;
NeilBrown49895bc2015-08-03 17:09:57 +10002257 int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002258
Shaohua Li566c09c2013-11-14 15:16:17 +11002259 spin_lock_irq(conf->hash_locks + hash);
2260 sh = get_free_stripe(conf, hash);
2261 spin_unlock_irq(conf->hash_locks + hash);
NeilBrown3f294f42005-11-08 21:39:25 -08002262 if (!sh)
2263 return 0;
Eric Sesterhenn78bafeb2006-04-02 13:31:42 +02002264 BUG_ON(atomic_read(&sh->count));
NeilBrowne4e11e32010-06-16 16:45:16 +10002265 shrink_buffers(sh);
NeilBrown3f294f42005-11-08 21:39:25 -08002266 kmem_cache_free(conf->slab_cache, sh);
2267 atomic_dec(&conf->active_stripes);
NeilBrown486f0642015-02-25 12:10:35 +11002268 conf->max_nr_stripes--;
NeilBrown3f294f42005-11-08 21:39:25 -08002269 return 1;
2270}
2271
NeilBrownd1688a62011-10-11 16:49:52 +11002272static void shrink_stripes(struct r5conf *conf)
NeilBrown3f294f42005-11-08 21:39:25 -08002273{
NeilBrown486f0642015-02-25 12:10:35 +11002274 while (conf->max_nr_stripes &&
2275 drop_one_stripe(conf))
2276 ;
NeilBrown3f294f42005-11-08 21:39:25 -08002277
Julia Lawall644df1a2015-09-13 14:15:10 +02002278 kmem_cache_destroy(conf->slab_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002279 conf->slab_cache = NULL;
2280}
2281
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002282static void raid5_end_read_request(struct bio * bi)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002283{
NeilBrown99c0fb52009-03-31 14:39:38 +11002284 struct stripe_head *sh = bi->bi_private;
NeilBrownd1688a62011-10-11 16:49:52 +11002285 struct r5conf *conf = sh->raid_conf;
NeilBrown7ecaa1e2006-03-27 01:18:08 -08002286 int disks = sh->disks, i;
NeilBrownd6950432006-07-10 04:44:20 -07002287 char b[BDEVNAME_SIZE];
NeilBrowndd054fc2011-12-23 10:17:53 +11002288 struct md_rdev *rdev = NULL;
NeilBrown05616be2012-05-21 09:27:00 +10002289 sector_t s;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002290
2291 for (i=0 ; i<disks; i++)
2292 if (bi == &sh->dev[i].req)
2293 break;
2294
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002295 pr_debug("end_read_request %llu/%d, count: %d, error %d.\n",
Dan Williams45b42332007-07-09 11:56:43 -07002296 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002297 bi->bi_error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002298 if (i == disks) {
2299 BUG();
NeilBrown6712ecf2007-09-27 12:47:43 +02002300 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002301 }
NeilBrown14a75d32011-12-23 10:17:52 +11002302 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
NeilBrowndd054fc2011-12-23 10:17:53 +11002303 /* If replacement finished while this request was outstanding,
2304 * 'replacement' might be NULL already.
2305 * In that case it moved down to 'rdev'.
2306 * rdev is not removed until all requests are finished.
2307 */
NeilBrown14a75d32011-12-23 10:17:52 +11002308 rdev = conf->disks[i].replacement;
NeilBrowndd054fc2011-12-23 10:17:53 +11002309 if (!rdev)
NeilBrown14a75d32011-12-23 10:17:52 +11002310 rdev = conf->disks[i].rdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002311
NeilBrown05616be2012-05-21 09:27:00 +10002312 if (use_new_offset(conf, sh))
2313 s = sh->sector + rdev->new_data_offset;
2314 else
2315 s = sh->sector + rdev->data_offset;
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002316 if (!bi->bi_error) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002317 set_bit(R5_UPTODATE, &sh->dev[i].flags);
NeilBrown4e5314b2005-11-08 21:39:22 -08002318 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
NeilBrown14a75d32011-12-23 10:17:52 +11002319 /* Note that this cannot happen on a
2320 * replacement device. We just fail those on
2321 * any error
2322 */
Christian Dietrich8bda4702011-07-27 11:00:36 +10002323 printk_ratelimited(
2324 KERN_INFO
2325 "md/raid:%s: read error corrected"
2326 " (%lu sectors at %llu on %s)\n",
2327 mdname(conf->mddev), STRIPE_SECTORS,
NeilBrown05616be2012-05-21 09:27:00 +10002328 (unsigned long long)s,
Christian Dietrich8bda4702011-07-27 11:00:36 +10002329 bdevname(rdev->bdev, b));
Namhyung Kimddd51152011-07-27 11:00:36 +10002330 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
NeilBrown4e5314b2005-11-08 21:39:22 -08002331 clear_bit(R5_ReadError, &sh->dev[i].flags);
2332 clear_bit(R5_ReWrite, &sh->dev[i].flags);
majianpeng3f9e7c12012-07-31 10:04:21 +10002333 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2334 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2335
NeilBrown14a75d32011-12-23 10:17:52 +11002336 if (atomic_read(&rdev->read_errors))
2337 atomic_set(&rdev->read_errors, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338 } else {
NeilBrown14a75d32011-12-23 10:17:52 +11002339 const char *bdn = bdevname(rdev->bdev, b);
NeilBrownba22dcb2005-11-08 21:39:31 -08002340 int retry = 0;
majianpeng2e8ac3032012-07-03 15:57:02 +10002341 int set_bad = 0;
NeilBrownd6950432006-07-10 04:44:20 -07002342
Linus Torvalds1da177e2005-04-16 15:20:36 -07002343 clear_bit(R5_UPTODATE, &sh->dev[i].flags);
NeilBrownd6950432006-07-10 04:44:20 -07002344 atomic_inc(&rdev->read_errors);
NeilBrown14a75d32011-12-23 10:17:52 +11002345 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2346 printk_ratelimited(
2347 KERN_WARNING
2348 "md/raid:%s: read error on replacement device "
2349 "(sector %llu on %s).\n",
2350 mdname(conf->mddev),
NeilBrown05616be2012-05-21 09:27:00 +10002351 (unsigned long long)s,
NeilBrown14a75d32011-12-23 10:17:52 +11002352 bdn);
majianpeng2e8ac3032012-07-03 15:57:02 +10002353 else if (conf->mddev->degraded >= conf->max_degraded) {
2354 set_bad = 1;
Christian Dietrich8bda4702011-07-27 11:00:36 +10002355 printk_ratelimited(
2356 KERN_WARNING
2357 "md/raid:%s: read error not correctable "
2358 "(sector %llu on %s).\n",
2359 mdname(conf->mddev),
NeilBrown05616be2012-05-21 09:27:00 +10002360 (unsigned long long)s,
Christian Dietrich8bda4702011-07-27 11:00:36 +10002361 bdn);
majianpeng2e8ac3032012-07-03 15:57:02 +10002362 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
NeilBrown4e5314b2005-11-08 21:39:22 -08002363 /* Oh, no!!! */
majianpeng2e8ac3032012-07-03 15:57:02 +10002364 set_bad = 1;
Christian Dietrich8bda4702011-07-27 11:00:36 +10002365 printk_ratelimited(
2366 KERN_WARNING
2367 "md/raid:%s: read error NOT corrected!! "
2368 "(sector %llu on %s).\n",
2369 mdname(conf->mddev),
NeilBrown05616be2012-05-21 09:27:00 +10002370 (unsigned long long)s,
Christian Dietrich8bda4702011-07-27 11:00:36 +10002371 bdn);
majianpeng2e8ac3032012-07-03 15:57:02 +10002372 } else if (atomic_read(&rdev->read_errors)
NeilBrownba22dcb2005-11-08 21:39:31 -08002373 > conf->max_nr_stripes)
NeilBrown14f8d262006-01-06 00:20:14 -08002374 printk(KERN_WARNING
NeilBrown0c55e022010-05-03 14:09:02 +10002375 "md/raid:%s: Too many read errors, failing device %s.\n",
NeilBrownd6950432006-07-10 04:44:20 -07002376 mdname(conf->mddev), bdn);
NeilBrownba22dcb2005-11-08 21:39:31 -08002377 else
2378 retry = 1;
Bian Yuedfa1f62013-11-14 15:16:17 +11002379 if (set_bad && test_bit(In_sync, &rdev->flags)
2380 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2381 retry = 1;
NeilBrownba22dcb2005-11-08 21:39:31 -08002382 if (retry)
majianpeng3f9e7c12012-07-31 10:04:21 +10002383 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
2384 set_bit(R5_ReadError, &sh->dev[i].flags);
2385 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2386 } else
2387 set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
NeilBrownba22dcb2005-11-08 21:39:31 -08002388 else {
NeilBrown4e5314b2005-11-08 21:39:22 -08002389 clear_bit(R5_ReadError, &sh->dev[i].flags);
2390 clear_bit(R5_ReWrite, &sh->dev[i].flags);
majianpeng2e8ac3032012-07-03 15:57:02 +10002391 if (!(set_bad
2392 && test_bit(In_sync, &rdev->flags)
2393 && rdev_set_badblocks(
2394 rdev, sh->sector, STRIPE_SECTORS, 0)))
2395 md_error(conf->mddev, rdev);
NeilBrownba22dcb2005-11-08 21:39:31 -08002396 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002397 }
NeilBrown14a75d32011-12-23 10:17:52 +11002398 rdev_dec_pending(rdev, conf->mddev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002399 clear_bit(R5_LOCKED, &sh->dev[i].flags);
2400 set_bit(STRIPE_HANDLE, &sh->state);
Shaohua Li6d036f72015-08-13 14:31:57 -07002401 raid5_release_stripe(sh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002402}
2403
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002404static void raid5_end_write_request(struct bio *bi)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002405{
NeilBrown99c0fb52009-03-31 14:39:38 +11002406 struct stripe_head *sh = bi->bi_private;
NeilBrownd1688a62011-10-11 16:49:52 +11002407 struct r5conf *conf = sh->raid_conf;
NeilBrown7ecaa1e2006-03-27 01:18:08 -08002408 int disks = sh->disks, i;
NeilBrown977df362011-12-23 10:17:53 +11002409 struct md_rdev *uninitialized_var(rdev);
NeilBrownb84db562011-07-28 11:39:23 +10002410 sector_t first_bad;
2411 int bad_sectors;
NeilBrown977df362011-12-23 10:17:53 +11002412 int replacement = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002413
NeilBrown977df362011-12-23 10:17:53 +11002414 for (i = 0 ; i < disks; i++) {
2415 if (bi == &sh->dev[i].req) {
2416 rdev = conf->disks[i].rdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002417 break;
NeilBrown977df362011-12-23 10:17:53 +11002418 }
2419 if (bi == &sh->dev[i].rreq) {
2420 rdev = conf->disks[i].replacement;
NeilBrowndd054fc2011-12-23 10:17:53 +11002421 if (rdev)
2422 replacement = 1;
2423 else
2424 /* rdev was removed and 'replacement'
2425 * replaced it. rdev is not removed
2426 * until all requests are finished.
2427 */
2428 rdev = conf->disks[i].rdev;
NeilBrown977df362011-12-23 10:17:53 +11002429 break;
2430 }
2431 }
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002432 pr_debug("end_write_request %llu/%d, count %d, error: %d.\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002433 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002434 bi->bi_error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002435 if (i == disks) {
2436 BUG();
NeilBrown6712ecf2007-09-27 12:47:43 +02002437 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002438 }
2439
NeilBrown977df362011-12-23 10:17:53 +11002440 if (replacement) {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002441 if (bi->bi_error)
NeilBrown977df362011-12-23 10:17:53 +11002442 md_error(conf->mddev, rdev);
2443 else if (is_badblock(rdev, sh->sector,
2444 STRIPE_SECTORS,
2445 &first_bad, &bad_sectors))
2446 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2447 } else {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002448 if (bi->bi_error) {
NeilBrown9f97e4b2014-01-16 09:35:38 +11002449 set_bit(STRIPE_DEGRADED, &sh->state);
NeilBrown977df362011-12-23 10:17:53 +11002450 set_bit(WriteErrorSeen, &rdev->flags);
2451 set_bit(R5_WriteError, &sh->dev[i].flags);
NeilBrown3a6de292011-12-23 10:17:54 +11002452 if (!test_and_set_bit(WantReplacement, &rdev->flags))
2453 set_bit(MD_RECOVERY_NEEDED,
2454 &rdev->mddev->recovery);
NeilBrown977df362011-12-23 10:17:53 +11002455 } else if (is_badblock(rdev, sh->sector,
2456 STRIPE_SECTORS,
NeilBrownc0b32972013-04-24 11:42:42 +10002457 &first_bad, &bad_sectors)) {
NeilBrown977df362011-12-23 10:17:53 +11002458 set_bit(R5_MadeGood, &sh->dev[i].flags);
NeilBrownc0b32972013-04-24 11:42:42 +10002459 if (test_bit(R5_ReadError, &sh->dev[i].flags))
2460 /* That was a successful write so make
2461 * sure it looks like we already did
2462 * a re-write.
2463 */
2464 set_bit(R5_ReWrite, &sh->dev[i].flags);
2465 }
NeilBrown977df362011-12-23 10:17:53 +11002466 }
2467 rdev_dec_pending(rdev, conf->mddev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002468
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002469 if (sh->batch_head && bi->bi_error && !replacement)
shli@kernel.org72ac7332014-12-15 12:57:03 +11002470 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
2471
NeilBrown977df362011-12-23 10:17:53 +11002472 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2473 clear_bit(R5_LOCKED, &sh->dev[i].flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002474 set_bit(STRIPE_HANDLE, &sh->state);
Shaohua Li6d036f72015-08-13 14:31:57 -07002475 raid5_release_stripe(sh);
shli@kernel.org59fc6302014-12-15 12:57:03 +11002476
2477 if (sh->batch_head && sh != sh->batch_head)
Shaohua Li6d036f72015-08-13 14:31:57 -07002478 raid5_release_stripe(sh->batch_head);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002479}
2480
NeilBrown784052e2009-03-31 15:19:07 +11002481static void raid5_build_block(struct stripe_head *sh, int i, int previous)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002482{
2483 struct r5dev *dev = &sh->dev[i];
2484
2485 bio_init(&dev->req);
2486 dev->req.bi_io_vec = &dev->vec;
Shaohua Lid592a992014-05-21 17:57:44 +08002487 dev->req.bi_max_vecs = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002488 dev->req.bi_private = sh;
2489
NeilBrown977df362011-12-23 10:17:53 +11002490 bio_init(&dev->rreq);
2491 dev->rreq.bi_io_vec = &dev->rvec;
Shaohua Lid592a992014-05-21 17:57:44 +08002492 dev->rreq.bi_max_vecs = 1;
NeilBrown977df362011-12-23 10:17:53 +11002493 dev->rreq.bi_private = sh;
NeilBrown977df362011-12-23 10:17:53 +11002494
Linus Torvalds1da177e2005-04-16 15:20:36 -07002495 dev->flags = 0;
Shaohua Li6d036f72015-08-13 14:31:57 -07002496 dev->sector = raid5_compute_blocknr(sh, i, previous);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002497}
2498
NeilBrownfd01b882011-10-11 16:47:53 +11002499static void error(struct mddev *mddev, struct md_rdev *rdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002500{
2501 char b[BDEVNAME_SIZE];
NeilBrownd1688a62011-10-11 16:49:52 +11002502 struct r5conf *conf = mddev->private;
NeilBrown908f4fb2011-12-23 10:17:50 +11002503 unsigned long flags;
NeilBrown0c55e022010-05-03 14:09:02 +10002504 pr_debug("raid456: error called\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002505
NeilBrown908f4fb2011-12-23 10:17:50 +11002506 spin_lock_irqsave(&conf->device_lock, flags);
2507 clear_bit(In_sync, &rdev->flags);
2508 mddev->degraded = calc_degraded(conf);
2509 spin_unlock_irqrestore(&conf->device_lock, flags);
2510 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2511
NeilBrownde393cd2011-07-28 11:31:48 +10002512 set_bit(Blocked, &rdev->flags);
NeilBrown6f8d0c72011-05-11 14:38:44 +10002513 set_bit(Faulty, &rdev->flags);
2514 set_bit(MD_CHANGE_DEVS, &mddev->flags);
NeilBrownc3cce6c2015-08-14 12:47:33 +10002515 set_bit(MD_CHANGE_PENDING, &mddev->flags);
NeilBrown6f8d0c72011-05-11 14:38:44 +10002516 printk(KERN_ALERT
2517 "md/raid:%s: Disk failure on %s, disabling device.\n"
2518 "md/raid:%s: Operation continuing on %d devices.\n",
2519 mdname(mddev),
2520 bdevname(rdev->bdev, b),
2521 mdname(mddev),
2522 conf->raid_disks - mddev->degraded);
NeilBrown16a53ec2006-06-26 00:27:38 -07002523}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002524
2525/*
2526 * Input: a 'big' sector number,
2527 * Output: index of the data and parity disk, and the sector # in them.
2528 */
Shaohua Li6d036f72015-08-13 14:31:57 -07002529sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
2530 int previous, int *dd_idx,
2531 struct stripe_head *sh)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002532{
NeilBrown6e3b96e2010-04-23 07:08:28 +10002533 sector_t stripe, stripe2;
NeilBrown35f2a592010-04-20 14:13:34 +10002534 sector_t chunk_number;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002535 unsigned int chunk_offset;
NeilBrown911d4ee2009-03-31 14:39:38 +11002536 int pd_idx, qd_idx;
NeilBrown67cc2b82009-03-31 14:39:38 +11002537 int ddf_layout = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002538 sector_t new_sector;
NeilBrowne183eae2009-03-31 15:20:22 +11002539 int algorithm = previous ? conf->prev_algo
2540 : conf->algorithm;
Andre Noll09c9e5f2009-06-18 08:45:55 +10002541 int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2542 : conf->chunk_sectors;
NeilBrown112bf892009-03-31 14:39:38 +11002543 int raid_disks = previous ? conf->previous_raid_disks
2544 : conf->raid_disks;
2545 int data_disks = raid_disks - conf->max_degraded;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002546
2547 /* First compute the information on this sector */
2548
2549 /*
2550 * Compute the chunk number and the sector offset inside the chunk
2551 */
2552 chunk_offset = sector_div(r_sector, sectors_per_chunk);
2553 chunk_number = r_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002554
2555 /*
2556 * Compute the stripe number
2557 */
NeilBrown35f2a592010-04-20 14:13:34 +10002558 stripe = chunk_number;
2559 *dd_idx = sector_div(stripe, data_disks);
NeilBrown6e3b96e2010-04-23 07:08:28 +10002560 stripe2 = stripe;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002561 /*
2562 * Select the parity disk based on the user selected algorithm.
2563 */
NeilBrown84789552011-07-27 11:00:36 +10002564 pd_idx = qd_idx = -1;
NeilBrown16a53ec2006-06-26 00:27:38 -07002565 switch(conf->level) {
2566 case 4:
NeilBrown911d4ee2009-03-31 14:39:38 +11002567 pd_idx = data_disks;
NeilBrown16a53ec2006-06-26 00:27:38 -07002568 break;
2569 case 5:
NeilBrowne183eae2009-03-31 15:20:22 +11002570 switch (algorithm) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002571 case ALGORITHM_LEFT_ASYMMETRIC:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002572 pd_idx = data_disks - sector_div(stripe2, raid_disks);
NeilBrown911d4ee2009-03-31 14:39:38 +11002573 if (*dd_idx >= pd_idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002574 (*dd_idx)++;
2575 break;
2576 case ALGORITHM_RIGHT_ASYMMETRIC:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002577 pd_idx = sector_div(stripe2, raid_disks);
NeilBrown911d4ee2009-03-31 14:39:38 +11002578 if (*dd_idx >= pd_idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002579 (*dd_idx)++;
2580 break;
2581 case ALGORITHM_LEFT_SYMMETRIC:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002582 pd_idx = data_disks - sector_div(stripe2, raid_disks);
NeilBrown911d4ee2009-03-31 14:39:38 +11002583 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002584 break;
2585 case ALGORITHM_RIGHT_SYMMETRIC:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002586 pd_idx = sector_div(stripe2, raid_disks);
NeilBrown911d4ee2009-03-31 14:39:38 +11002587 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002588 break;
NeilBrown99c0fb52009-03-31 14:39:38 +11002589 case ALGORITHM_PARITY_0:
2590 pd_idx = 0;
2591 (*dd_idx)++;
2592 break;
2593 case ALGORITHM_PARITY_N:
2594 pd_idx = data_disks;
2595 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002596 default:
NeilBrown99c0fb52009-03-31 14:39:38 +11002597 BUG();
NeilBrown16a53ec2006-06-26 00:27:38 -07002598 }
2599 break;
2600 case 6:
2601
NeilBrowne183eae2009-03-31 15:20:22 +11002602 switch (algorithm) {
NeilBrown16a53ec2006-06-26 00:27:38 -07002603 case ALGORITHM_LEFT_ASYMMETRIC:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002604 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
NeilBrown911d4ee2009-03-31 14:39:38 +11002605 qd_idx = pd_idx + 1;
2606 if (pd_idx == raid_disks-1) {
NeilBrown99c0fb52009-03-31 14:39:38 +11002607 (*dd_idx)++; /* Q D D D P */
NeilBrown911d4ee2009-03-31 14:39:38 +11002608 qd_idx = 0;
2609 } else if (*dd_idx >= pd_idx)
NeilBrown16a53ec2006-06-26 00:27:38 -07002610 (*dd_idx) += 2; /* D D P Q D */
2611 break;
2612 case ALGORITHM_RIGHT_ASYMMETRIC:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002613 pd_idx = sector_div(stripe2, raid_disks);
NeilBrown911d4ee2009-03-31 14:39:38 +11002614 qd_idx = pd_idx + 1;
2615 if (pd_idx == raid_disks-1) {
NeilBrown99c0fb52009-03-31 14:39:38 +11002616 (*dd_idx)++; /* Q D D D P */
NeilBrown911d4ee2009-03-31 14:39:38 +11002617 qd_idx = 0;
2618 } else if (*dd_idx >= pd_idx)
NeilBrown16a53ec2006-06-26 00:27:38 -07002619 (*dd_idx) += 2; /* D D P Q D */
2620 break;
2621 case ALGORITHM_LEFT_SYMMETRIC:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002622 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
NeilBrown911d4ee2009-03-31 14:39:38 +11002623 qd_idx = (pd_idx + 1) % raid_disks;
2624 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
NeilBrown16a53ec2006-06-26 00:27:38 -07002625 break;
2626 case ALGORITHM_RIGHT_SYMMETRIC:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002627 pd_idx = sector_div(stripe2, raid_disks);
NeilBrown911d4ee2009-03-31 14:39:38 +11002628 qd_idx = (pd_idx + 1) % raid_disks;
2629 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
NeilBrown16a53ec2006-06-26 00:27:38 -07002630 break;
NeilBrown99c0fb52009-03-31 14:39:38 +11002631
2632 case ALGORITHM_PARITY_0:
2633 pd_idx = 0;
2634 qd_idx = 1;
2635 (*dd_idx) += 2;
2636 break;
2637 case ALGORITHM_PARITY_N:
2638 pd_idx = data_disks;
2639 qd_idx = data_disks + 1;
2640 break;
2641
2642 case ALGORITHM_ROTATING_ZERO_RESTART:
2643 /* Exactly the same as RIGHT_ASYMMETRIC, but or
2644 * of blocks for computing Q is different.
2645 */
NeilBrown6e3b96e2010-04-23 07:08:28 +10002646 pd_idx = sector_div(stripe2, raid_disks);
NeilBrown99c0fb52009-03-31 14:39:38 +11002647 qd_idx = pd_idx + 1;
2648 if (pd_idx == raid_disks-1) {
2649 (*dd_idx)++; /* Q D D D P */
2650 qd_idx = 0;
2651 } else if (*dd_idx >= pd_idx)
2652 (*dd_idx) += 2; /* D D P Q D */
NeilBrown67cc2b82009-03-31 14:39:38 +11002653 ddf_layout = 1;
NeilBrown99c0fb52009-03-31 14:39:38 +11002654 break;
2655
2656 case ALGORITHM_ROTATING_N_RESTART:
2657 /* Same a left_asymmetric, by first stripe is
2658 * D D D P Q rather than
2659 * Q D D D P
2660 */
NeilBrown6e3b96e2010-04-23 07:08:28 +10002661 stripe2 += 1;
2662 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
NeilBrown99c0fb52009-03-31 14:39:38 +11002663 qd_idx = pd_idx + 1;
2664 if (pd_idx == raid_disks-1) {
2665 (*dd_idx)++; /* Q D D D P */
2666 qd_idx = 0;
2667 } else if (*dd_idx >= pd_idx)
2668 (*dd_idx) += 2; /* D D P Q D */
NeilBrown67cc2b82009-03-31 14:39:38 +11002669 ddf_layout = 1;
NeilBrown99c0fb52009-03-31 14:39:38 +11002670 break;
2671
2672 case ALGORITHM_ROTATING_N_CONTINUE:
2673 /* Same as left_symmetric but Q is before P */
NeilBrown6e3b96e2010-04-23 07:08:28 +10002674 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
NeilBrown99c0fb52009-03-31 14:39:38 +11002675 qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
2676 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
NeilBrown67cc2b82009-03-31 14:39:38 +11002677 ddf_layout = 1;
NeilBrown99c0fb52009-03-31 14:39:38 +11002678 break;
2679
2680 case ALGORITHM_LEFT_ASYMMETRIC_6:
2681 /* RAID5 left_asymmetric, with Q on last device */
NeilBrown6e3b96e2010-04-23 07:08:28 +10002682 pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
NeilBrown99c0fb52009-03-31 14:39:38 +11002683 if (*dd_idx >= pd_idx)
2684 (*dd_idx)++;
2685 qd_idx = raid_disks - 1;
2686 break;
2687
2688 case ALGORITHM_RIGHT_ASYMMETRIC_6:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002689 pd_idx = sector_div(stripe2, raid_disks-1);
NeilBrown99c0fb52009-03-31 14:39:38 +11002690 if (*dd_idx >= pd_idx)
2691 (*dd_idx)++;
2692 qd_idx = raid_disks - 1;
2693 break;
2694
2695 case ALGORITHM_LEFT_SYMMETRIC_6:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002696 pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
NeilBrown99c0fb52009-03-31 14:39:38 +11002697 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2698 qd_idx = raid_disks - 1;
2699 break;
2700
2701 case ALGORITHM_RIGHT_SYMMETRIC_6:
NeilBrown6e3b96e2010-04-23 07:08:28 +10002702 pd_idx = sector_div(stripe2, raid_disks-1);
NeilBrown99c0fb52009-03-31 14:39:38 +11002703 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2704 qd_idx = raid_disks - 1;
2705 break;
2706
2707 case ALGORITHM_PARITY_0_6:
2708 pd_idx = 0;
2709 (*dd_idx)++;
2710 qd_idx = raid_disks - 1;
2711 break;
2712
NeilBrown16a53ec2006-06-26 00:27:38 -07002713 default:
NeilBrown99c0fb52009-03-31 14:39:38 +11002714 BUG();
NeilBrown16a53ec2006-06-26 00:27:38 -07002715 }
2716 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002717 }
2718
NeilBrown911d4ee2009-03-31 14:39:38 +11002719 if (sh) {
2720 sh->pd_idx = pd_idx;
2721 sh->qd_idx = qd_idx;
NeilBrown67cc2b82009-03-31 14:39:38 +11002722 sh->ddf_layout = ddf_layout;
NeilBrown911d4ee2009-03-31 14:39:38 +11002723 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002724 /*
2725 * Finally, compute the new sector number
2726 */
2727 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
2728 return new_sector;
2729}
2730
Shaohua Li6d036f72015-08-13 14:31:57 -07002731sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002732{
NeilBrownd1688a62011-10-11 16:49:52 +11002733 struct r5conf *conf = sh->raid_conf;
NeilBrownb875e532006-12-10 02:20:49 -08002734 int raid_disks = sh->disks;
2735 int data_disks = raid_disks - conf->max_degraded;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002736 sector_t new_sector = sh->sector, check;
Andre Noll09c9e5f2009-06-18 08:45:55 +10002737 int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2738 : conf->chunk_sectors;
NeilBrowne183eae2009-03-31 15:20:22 +11002739 int algorithm = previous ? conf->prev_algo
2740 : conf->algorithm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002741 sector_t stripe;
2742 int chunk_offset;
NeilBrown35f2a592010-04-20 14:13:34 +10002743 sector_t chunk_number;
2744 int dummy1, dd_idx = i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002745 sector_t r_sector;
NeilBrown911d4ee2009-03-31 14:39:38 +11002746 struct stripe_head sh2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002747
2748 chunk_offset = sector_div(new_sector, sectors_per_chunk);
2749 stripe = new_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002750
NeilBrown16a53ec2006-06-26 00:27:38 -07002751 if (i == sh->pd_idx)
2752 return 0;
2753 switch(conf->level) {
2754 case 4: break;
2755 case 5:
NeilBrowne183eae2009-03-31 15:20:22 +11002756 switch (algorithm) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002757 case ALGORITHM_LEFT_ASYMMETRIC:
2758 case ALGORITHM_RIGHT_ASYMMETRIC:
2759 if (i > sh->pd_idx)
2760 i--;
2761 break;
2762 case ALGORITHM_LEFT_SYMMETRIC:
2763 case ALGORITHM_RIGHT_SYMMETRIC:
2764 if (i < sh->pd_idx)
2765 i += raid_disks;
2766 i -= (sh->pd_idx + 1);
2767 break;
NeilBrown99c0fb52009-03-31 14:39:38 +11002768 case ALGORITHM_PARITY_0:
2769 i -= 1;
2770 break;
2771 case ALGORITHM_PARITY_N:
2772 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002773 default:
NeilBrown99c0fb52009-03-31 14:39:38 +11002774 BUG();
NeilBrown16a53ec2006-06-26 00:27:38 -07002775 }
2776 break;
2777 case 6:
NeilBrownd0dabf72009-03-31 14:39:38 +11002778 if (i == sh->qd_idx)
NeilBrown16a53ec2006-06-26 00:27:38 -07002779 return 0; /* It is the Q disk */
NeilBrowne183eae2009-03-31 15:20:22 +11002780 switch (algorithm) {
NeilBrown16a53ec2006-06-26 00:27:38 -07002781 case ALGORITHM_LEFT_ASYMMETRIC:
2782 case ALGORITHM_RIGHT_ASYMMETRIC:
NeilBrown99c0fb52009-03-31 14:39:38 +11002783 case ALGORITHM_ROTATING_ZERO_RESTART:
2784 case ALGORITHM_ROTATING_N_RESTART:
2785 if (sh->pd_idx == raid_disks-1)
2786 i--; /* Q D D D P */
NeilBrown16a53ec2006-06-26 00:27:38 -07002787 else if (i > sh->pd_idx)
2788 i -= 2; /* D D P Q D */
2789 break;
2790 case ALGORITHM_LEFT_SYMMETRIC:
2791 case ALGORITHM_RIGHT_SYMMETRIC:
2792 if (sh->pd_idx == raid_disks-1)
2793 i--; /* Q D D D P */
2794 else {
2795 /* D D P Q D */
2796 if (i < sh->pd_idx)
2797 i += raid_disks;
2798 i -= (sh->pd_idx + 2);
2799 }
2800 break;
NeilBrown99c0fb52009-03-31 14:39:38 +11002801 case ALGORITHM_PARITY_0:
2802 i -= 2;
2803 break;
2804 case ALGORITHM_PARITY_N:
2805 break;
2806 case ALGORITHM_ROTATING_N_CONTINUE:
NeilBrowne4424fe2009-10-16 16:27:34 +11002807 /* Like left_symmetric, but P is before Q */
NeilBrown99c0fb52009-03-31 14:39:38 +11002808 if (sh->pd_idx == 0)
2809 i--; /* P D D D Q */
NeilBrowne4424fe2009-10-16 16:27:34 +11002810 else {
2811 /* D D Q P D */
2812 if (i < sh->pd_idx)
2813 i += raid_disks;
2814 i -= (sh->pd_idx + 1);
2815 }
NeilBrown99c0fb52009-03-31 14:39:38 +11002816 break;
2817 case ALGORITHM_LEFT_ASYMMETRIC_6:
2818 case ALGORITHM_RIGHT_ASYMMETRIC_6:
2819 if (i > sh->pd_idx)
2820 i--;
2821 break;
2822 case ALGORITHM_LEFT_SYMMETRIC_6:
2823 case ALGORITHM_RIGHT_SYMMETRIC_6:
2824 if (i < sh->pd_idx)
2825 i += data_disks + 1;
2826 i -= (sh->pd_idx + 1);
2827 break;
2828 case ALGORITHM_PARITY_0_6:
2829 i -= 1;
2830 break;
NeilBrown16a53ec2006-06-26 00:27:38 -07002831 default:
NeilBrown99c0fb52009-03-31 14:39:38 +11002832 BUG();
NeilBrown16a53ec2006-06-26 00:27:38 -07002833 }
2834 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002835 }
2836
2837 chunk_number = stripe * data_disks + i;
NeilBrown35f2a592010-04-20 14:13:34 +10002838 r_sector = chunk_number * sectors_per_chunk + chunk_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002839
NeilBrown112bf892009-03-31 14:39:38 +11002840 check = raid5_compute_sector(conf, r_sector,
NeilBrown784052e2009-03-31 15:19:07 +11002841 previous, &dummy1, &sh2);
NeilBrown911d4ee2009-03-31 14:39:38 +11002842 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
2843 || sh2.qd_idx != sh->qd_idx) {
NeilBrown0c55e022010-05-03 14:09:02 +10002844 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
2845 mdname(conf->mddev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002846 return 0;
2847 }
2848 return r_sector;
2849}
2850
Dan Williams600aa102008-06-28 08:32:05 +10002851static void
Yuri Tikhonovc0f7bdd2009-08-29 19:13:12 -07002852schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
Dan Williams600aa102008-06-28 08:32:05 +10002853 int rcw, int expand)
Dan Williamse33129d2007-01-02 13:52:30 -07002854{
Markus Stockhausen584acdd2014-12-15 12:57:05 +11002855 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
NeilBrownd1688a62011-10-11 16:49:52 +11002856 struct r5conf *conf = sh->raid_conf;
Yuri Tikhonovc0f7bdd2009-08-29 19:13:12 -07002857 int level = conf->level;
NeilBrown16a53ec2006-06-26 00:27:38 -07002858
Dan Williamse33129d2007-01-02 13:52:30 -07002859 if (rcw) {
Dan Williamse33129d2007-01-02 13:52:30 -07002860
2861 for (i = disks; i--; ) {
2862 struct r5dev *dev = &sh->dev[i];
2863
2864 if (dev->towrite) {
2865 set_bit(R5_LOCKED, &dev->flags);
Dan Williamsd8ee0722008-06-28 08:32:06 +10002866 set_bit(R5_Wantdrain, &dev->flags);
Dan Williamse33129d2007-01-02 13:52:30 -07002867 if (!expand)
2868 clear_bit(R5_UPTODATE, &dev->flags);
Dan Williams600aa102008-06-28 08:32:05 +10002869 s->locked++;
Dan Williamse33129d2007-01-02 13:52:30 -07002870 }
2871 }
NeilBrownce7d3632013-03-04 12:37:14 +11002872 /* if we are not expanding this is a proper write request, and
2873 * there will be bios with new data to be drained into the
2874 * stripe cache
2875 */
2876 if (!expand) {
2877 if (!s->locked)
2878 /* False alarm, nothing to do */
2879 return;
2880 sh->reconstruct_state = reconstruct_state_drain_run;
2881 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2882 } else
2883 sh->reconstruct_state = reconstruct_state_run;
2884
2885 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2886
Yuri Tikhonovc0f7bdd2009-08-29 19:13:12 -07002887 if (s->locked + conf->max_degraded == disks)
Dan Williams8b3e6cd2008-04-28 02:15:53 -07002888 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
Yuri Tikhonovc0f7bdd2009-08-29 19:13:12 -07002889 atomic_inc(&conf->pending_full_writes);
Dan Williamse33129d2007-01-02 13:52:30 -07002890 } else {
2891 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
2892 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
Markus Stockhausen584acdd2014-12-15 12:57:05 +11002893 BUG_ON(level == 6 &&
2894 (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
2895 test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
Dan Williamse33129d2007-01-02 13:52:30 -07002896
Dan Williamse33129d2007-01-02 13:52:30 -07002897 for (i = disks; i--; ) {
2898 struct r5dev *dev = &sh->dev[i];
Markus Stockhausen584acdd2014-12-15 12:57:05 +11002899 if (i == pd_idx || i == qd_idx)
Dan Williamse33129d2007-01-02 13:52:30 -07002900 continue;
2901
Dan Williamse33129d2007-01-02 13:52:30 -07002902 if (dev->towrite &&
2903 (test_bit(R5_UPTODATE, &dev->flags) ||
Dan Williamsd8ee0722008-06-28 08:32:06 +10002904 test_bit(R5_Wantcompute, &dev->flags))) {
2905 set_bit(R5_Wantdrain, &dev->flags);
Dan Williamse33129d2007-01-02 13:52:30 -07002906 set_bit(R5_LOCKED, &dev->flags);
2907 clear_bit(R5_UPTODATE, &dev->flags);
Dan Williams600aa102008-06-28 08:32:05 +10002908 s->locked++;
Dan Williamse33129d2007-01-02 13:52:30 -07002909 }
2910 }
NeilBrownce7d3632013-03-04 12:37:14 +11002911 if (!s->locked)
2912 /* False alarm - nothing to do */
2913 return;
2914 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
2915 set_bit(STRIPE_OP_PREXOR, &s->ops_request);
2916 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2917 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
Dan Williamse33129d2007-01-02 13:52:30 -07002918 }
2919
Yuri Tikhonovc0f7bdd2009-08-29 19:13:12 -07002920 /* keep the parity disk(s) locked while asynchronous operations
Dan Williamse33129d2007-01-02 13:52:30 -07002921 * are in flight
2922 */
2923 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
2924 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
Dan Williams600aa102008-06-28 08:32:05 +10002925 s->locked++;
Dan Williamse33129d2007-01-02 13:52:30 -07002926
Yuri Tikhonovc0f7bdd2009-08-29 19:13:12 -07002927 if (level == 6) {
2928 int qd_idx = sh->qd_idx;
2929 struct r5dev *dev = &sh->dev[qd_idx];
2930
2931 set_bit(R5_LOCKED, &dev->flags);
2932 clear_bit(R5_UPTODATE, &dev->flags);
2933 s->locked++;
2934 }
2935
Dan Williams600aa102008-06-28 08:32:05 +10002936 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
Harvey Harrisone46b2722008-04-28 02:15:50 -07002937 __func__, (unsigned long long)sh->sector,
Dan Williams600aa102008-06-28 08:32:05 +10002938 s->locked, s->ops_request);
Dan Williamse33129d2007-01-02 13:52:30 -07002939}
NeilBrown16a53ec2006-06-26 00:27:38 -07002940
Linus Torvalds1da177e2005-04-16 15:20:36 -07002941/*
2942 * Each stripe/dev can have one or more bion attached.
NeilBrown16a53ec2006-06-26 00:27:38 -07002943 * toread/towrite point to the first in a chain.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002944 * The bi_next chain must be in order.
2945 */
shli@kernel.orgda41ba62014-12-15 12:57:03 +11002946static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
2947 int forwrite, int previous)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002948{
2949 struct bio **bip;
NeilBrownd1688a62011-10-11 16:49:52 +11002950 struct r5conf *conf = sh->raid_conf;
NeilBrown72626682005-09-09 16:23:54 -07002951 int firstwrite=0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002952
NeilBrowncbe47ec2011-07-26 11:20:35 +10002953 pr_debug("adding bi b#%llu to stripe s#%llu\n",
Kent Overstreet4f024f32013-10-11 15:44:27 -07002954 (unsigned long long)bi->bi_iter.bi_sector,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002955 (unsigned long long)sh->sector);
2956
Shaohua Lib17459c2012-07-19 16:01:31 +10002957 /*
2958 * If several bio share a stripe. The bio bi_phys_segments acts as a
2959 * reference count to avoid race. The reference count should already be
2960 * increased before this function is called (for example, in
2961 * make_request()), so other bio sharing this stripe will not free the
2962 * stripe. If a stripe is owned by one stripe, the stripe lock will
2963 * protect it.
2964 */
2965 spin_lock_irq(&sh->stripe_lock);
shli@kernel.org59fc6302014-12-15 12:57:03 +11002966 /* Don't allow new IO added to stripes in batch list */
2967 if (sh->batch_head)
2968 goto overlap;
NeilBrown72626682005-09-09 16:23:54 -07002969 if (forwrite) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002970 bip = &sh->dev[dd_idx].towrite;
Shaohua Li7eaf7e82012-07-19 16:01:31 +10002971 if (*bip == NULL)
NeilBrown72626682005-09-09 16:23:54 -07002972 firstwrite = 1;
2973 } else
Linus Torvalds1da177e2005-04-16 15:20:36 -07002974 bip = &sh->dev[dd_idx].toread;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002975 while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
2976 if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002977 goto overlap;
2978 bip = & (*bip)->bi_next;
2979 }
Kent Overstreet4f024f32013-10-11 15:44:27 -07002980 if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002981 goto overlap;
2982
shli@kernel.orgda41ba62014-12-15 12:57:03 +11002983 if (!forwrite || previous)
2984 clear_bit(STRIPE_BATCH_READY, &sh->state);
2985
Eric Sesterhenn78bafeb2006-04-02 13:31:42 +02002986 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002987 if (*bip)
2988 bi->bi_next = *bip;
2989 *bip = bi;
Shaohua Lie7836bd62012-07-19 16:01:31 +10002990 raid5_inc_bi_active_stripes(bi);
NeilBrown72626682005-09-09 16:23:54 -07002991
Linus Torvalds1da177e2005-04-16 15:20:36 -07002992 if (forwrite) {
2993 /* check if page is covered */
2994 sector_t sector = sh->dev[dd_idx].sector;
2995 for (bi=sh->dev[dd_idx].towrite;
2996 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
Kent Overstreet4f024f32013-10-11 15:44:27 -07002997 bi && bi->bi_iter.bi_sector <= sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002998 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002999 if (bio_end_sector(bi) >= sector)
3000 sector = bio_end_sector(bi);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001 }
3002 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
shli@kernel.org7a87f432014-12-15 12:57:03 +11003003 if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
3004 sh->overwrite_disks++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003005 }
NeilBrowncbe47ec2011-07-26 11:20:35 +10003006
3007 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
Kent Overstreet4f024f32013-10-11 15:44:27 -07003008 (unsigned long long)(*bip)->bi_iter.bi_sector,
NeilBrowncbe47ec2011-07-26 11:20:35 +10003009 (unsigned long long)sh->sector, dd_idx);
3010
3011 if (conf->mddev->bitmap && firstwrite) {
NeilBrownd0852df52015-05-27 08:43:45 +10003012 /* Cannot hold spinlock over bitmap_startwrite,
3013 * but must ensure this isn't added to a batch until
3014 * we have added to the bitmap and set bm_seq.
3015 * So set STRIPE_BITMAP_PENDING to prevent
3016 * batching.
3017 * If multiple add_stripe_bio() calls race here they
3018 * much all set STRIPE_BITMAP_PENDING. So only the first one
3019 * to complete "bitmap_startwrite" gets to set
3020 * STRIPE_BIT_DELAY. This is important as once a stripe
3021 * is added to a batch, STRIPE_BIT_DELAY cannot be changed
3022 * any more.
3023 */
3024 set_bit(STRIPE_BITMAP_PENDING, &sh->state);
3025 spin_unlock_irq(&sh->stripe_lock);
NeilBrowncbe47ec2011-07-26 11:20:35 +10003026 bitmap_startwrite(conf->mddev->bitmap, sh->sector,
3027 STRIPE_SECTORS, 0);
NeilBrownd0852df52015-05-27 08:43:45 +10003028 spin_lock_irq(&sh->stripe_lock);
3029 clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
3030 if (!sh->batch_head) {
3031 sh->bm_seq = conf->seq_flush+1;
3032 set_bit(STRIPE_BIT_DELAY, &sh->state);
3033 }
NeilBrowncbe47ec2011-07-26 11:20:35 +10003034 }
NeilBrownd0852df52015-05-27 08:43:45 +10003035 spin_unlock_irq(&sh->stripe_lock);
shli@kernel.org59fc6302014-12-15 12:57:03 +11003036
3037 if (stripe_can_batch(sh))
3038 stripe_add_to_batch_list(conf, sh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003039 return 1;
3040
3041 overlap:
3042 set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
Shaohua Lib17459c2012-07-19 16:01:31 +10003043 spin_unlock_irq(&sh->stripe_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003044 return 0;
3045}
3046
NeilBrownd1688a62011-10-11 16:49:52 +11003047static void end_reshape(struct r5conf *conf);
NeilBrown29269552006-03-27 01:18:10 -08003048
NeilBrownd1688a62011-10-11 16:49:52 +11003049static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
NeilBrown911d4ee2009-03-31 14:39:38 +11003050 struct stripe_head *sh)
NeilBrownccfcc3c2006-03-27 01:18:09 -08003051{
NeilBrown784052e2009-03-31 15:19:07 +11003052 int sectors_per_chunk =
Andre Noll09c9e5f2009-06-18 08:45:55 +10003053 previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
NeilBrown911d4ee2009-03-31 14:39:38 +11003054 int dd_idx;
Coywolf Qi Hunt2d2063c2006-10-03 01:15:50 -07003055 int chunk_offset = sector_div(stripe, sectors_per_chunk);
NeilBrown112bf892009-03-31 14:39:38 +11003056 int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
Coywolf Qi Hunt2d2063c2006-10-03 01:15:50 -07003057
NeilBrown112bf892009-03-31 14:39:38 +11003058 raid5_compute_sector(conf,
3059 stripe * (disks - conf->max_degraded)
NeilBrownb875e532006-12-10 02:20:49 -08003060 *sectors_per_chunk + chunk_offset,
NeilBrown112bf892009-03-31 14:39:38 +11003061 previous,
NeilBrown911d4ee2009-03-31 14:39:38 +11003062 &dd_idx, sh);
NeilBrownccfcc3c2006-03-27 01:18:09 -08003063}
3064
Dan Williamsa4456852007-07-09 11:56:43 -07003065static void
NeilBrownd1688a62011-10-11 16:49:52 +11003066handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
Dan Williamsa4456852007-07-09 11:56:43 -07003067 struct stripe_head_state *s, int disks,
NeilBrown34a6f802015-08-14 12:07:57 +10003068 struct bio_list *return_bi)
Dan Williamsa4456852007-07-09 11:56:43 -07003069{
3070 int i;
shli@kernel.org59fc6302014-12-15 12:57:03 +11003071 BUG_ON(sh->batch_head);
Dan Williamsa4456852007-07-09 11:56:43 -07003072 for (i = disks; i--; ) {
3073 struct bio *bi;
3074 int bitmap_end = 0;
3075
3076 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
NeilBrown3cb03002011-10-11 16:45:26 +11003077 struct md_rdev *rdev;
Dan Williamsa4456852007-07-09 11:56:43 -07003078 rcu_read_lock();
3079 rdev = rcu_dereference(conf->disks[i].rdev);
3080 if (rdev && test_bit(In_sync, &rdev->flags))
NeilBrown7f0da592011-07-28 11:39:22 +10003081 atomic_inc(&rdev->nr_pending);
3082 else
3083 rdev = NULL;
Dan Williamsa4456852007-07-09 11:56:43 -07003084 rcu_read_unlock();
NeilBrown7f0da592011-07-28 11:39:22 +10003085 if (rdev) {
3086 if (!rdev_set_badblocks(
3087 rdev,
3088 sh->sector,
3089 STRIPE_SECTORS, 0))
3090 md_error(conf->mddev, rdev);
3091 rdev_dec_pending(rdev, conf->mddev);
3092 }
Dan Williamsa4456852007-07-09 11:56:43 -07003093 }
Shaohua Lib17459c2012-07-19 16:01:31 +10003094 spin_lock_irq(&sh->stripe_lock);
Dan Williamsa4456852007-07-09 11:56:43 -07003095 /* fail all writes first */
3096 bi = sh->dev[i].towrite;
3097 sh->dev[i].towrite = NULL;
shli@kernel.org7a87f432014-12-15 12:57:03 +11003098 sh->overwrite_disks = 0;
Shaohua Lib17459c2012-07-19 16:01:31 +10003099 spin_unlock_irq(&sh->stripe_lock);
NeilBrown1ed850f2012-10-11 13:50:13 +11003100 if (bi)
Dan Williamsa4456852007-07-09 11:56:43 -07003101 bitmap_end = 1;
Dan Williamsa4456852007-07-09 11:56:43 -07003102
Shaohua Li0576b1c2015-08-13 14:32:00 -07003103 r5l_stripe_write_finished(sh);
3104
Dan Williamsa4456852007-07-09 11:56:43 -07003105 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3106 wake_up(&conf->wait_for_overlap);
3107
Kent Overstreet4f024f32013-10-11 15:44:27 -07003108 while (bi && bi->bi_iter.bi_sector <
Dan Williamsa4456852007-07-09 11:56:43 -07003109 sh->dev[i].sector + STRIPE_SECTORS) {
3110 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003111
3112 bi->bi_error = -EIO;
Shaohua Lie7836bd62012-07-19 16:01:31 +10003113 if (!raid5_dec_bi_active_stripes(bi)) {
Dan Williamsa4456852007-07-09 11:56:43 -07003114 md_write_end(conf->mddev);
NeilBrown34a6f802015-08-14 12:07:57 +10003115 bio_list_add(return_bi, bi);
Dan Williamsa4456852007-07-09 11:56:43 -07003116 }
3117 bi = nextbi;
3118 }
Shaohua Li7eaf7e82012-07-19 16:01:31 +10003119 if (bitmap_end)
3120 bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3121 STRIPE_SECTORS, 0, 0);
3122 bitmap_end = 0;
Dan Williamsa4456852007-07-09 11:56:43 -07003123 /* and fail all 'written' */
3124 bi = sh->dev[i].written;
3125 sh->dev[i].written = NULL;
Shaohua Lid592a992014-05-21 17:57:44 +08003126 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
3127 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
3128 sh->dev[i].page = sh->dev[i].orig_page;
3129 }
3130
Dan Williamsa4456852007-07-09 11:56:43 -07003131 if (bi) bitmap_end = 1;
Kent Overstreet4f024f32013-10-11 15:44:27 -07003132 while (bi && bi->bi_iter.bi_sector <
Dan Williamsa4456852007-07-09 11:56:43 -07003133 sh->dev[i].sector + STRIPE_SECTORS) {
3134 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003135
3136 bi->bi_error = -EIO;
Shaohua Lie7836bd62012-07-19 16:01:31 +10003137 if (!raid5_dec_bi_active_stripes(bi)) {
Dan Williamsa4456852007-07-09 11:56:43 -07003138 md_write_end(conf->mddev);
NeilBrown34a6f802015-08-14 12:07:57 +10003139 bio_list_add(return_bi, bi);
Dan Williamsa4456852007-07-09 11:56:43 -07003140 }
3141 bi = bi2;
3142 }
3143
Dan Williamsb5e98d62007-01-02 13:52:31 -07003144 /* fail any reads if this device is non-operational and
3145 * the data has not reached the cache yet.
3146 */
3147 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
Shaohua Li6e74a9c2015-10-08 21:54:08 -07003148 s->failed > conf->max_degraded &&
Dan Williamsb5e98d62007-01-02 13:52:31 -07003149 (!test_bit(R5_Insync, &sh->dev[i].flags) ||
3150 test_bit(R5_ReadError, &sh->dev[i].flags))) {
NeilBrown143c4d02012-10-11 13:50:12 +11003151 spin_lock_irq(&sh->stripe_lock);
Dan Williamsa4456852007-07-09 11:56:43 -07003152 bi = sh->dev[i].toread;
3153 sh->dev[i].toread = NULL;
NeilBrown143c4d02012-10-11 13:50:12 +11003154 spin_unlock_irq(&sh->stripe_lock);
Dan Williamsa4456852007-07-09 11:56:43 -07003155 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3156 wake_up(&conf->wait_for_overlap);
Shaohua Liebda7802015-09-18 10:20:13 -07003157 if (bi)
3158 s->to_read--;
Kent Overstreet4f024f32013-10-11 15:44:27 -07003159 while (bi && bi->bi_iter.bi_sector <
Dan Williamsa4456852007-07-09 11:56:43 -07003160 sh->dev[i].sector + STRIPE_SECTORS) {
3161 struct bio *nextbi =
3162 r5_next_bio(bi, sh->dev[i].sector);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003163
3164 bi->bi_error = -EIO;
NeilBrown34a6f802015-08-14 12:07:57 +10003165 if (!raid5_dec_bi_active_stripes(bi))
3166 bio_list_add(return_bi, bi);
Dan Williamsa4456852007-07-09 11:56:43 -07003167 bi = nextbi;
3168 }
3169 }
Dan Williamsa4456852007-07-09 11:56:43 -07003170 if (bitmap_end)
3171 bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3172 STRIPE_SECTORS, 0, 0);
NeilBrown8cfa7b02011-07-27 11:00:36 +10003173 /* If we were in the middle of a write the parity block might
3174 * still be locked - so just clear all R5_LOCKED flags
3175 */
3176 clear_bit(R5_LOCKED, &sh->dev[i].flags);
Dan Williamsa4456852007-07-09 11:56:43 -07003177 }
Shaohua Liebda7802015-09-18 10:20:13 -07003178 s->to_write = 0;
3179 s->written = 0;
Dan Williamsa4456852007-07-09 11:56:43 -07003180
Dan Williams8b3e6cd2008-04-28 02:15:53 -07003181 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
3182 if (atomic_dec_and_test(&conf->pending_full_writes))
3183 md_wakeup_thread(conf->mddev->thread);
Dan Williamsa4456852007-07-09 11:56:43 -07003184}
3185
NeilBrown7f0da592011-07-28 11:39:22 +10003186static void
NeilBrownd1688a62011-10-11 16:49:52 +11003187handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
NeilBrown7f0da592011-07-28 11:39:22 +10003188 struct stripe_head_state *s)
3189{
3190 int abort = 0;
3191 int i;
3192
shli@kernel.org59fc6302014-12-15 12:57:03 +11003193 BUG_ON(sh->batch_head);
NeilBrown7f0da592011-07-28 11:39:22 +10003194 clear_bit(STRIPE_SYNCING, &sh->state);
NeilBrownf8dfcff2013-03-12 12:18:06 +11003195 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
3196 wake_up(&conf->wait_for_overlap);
NeilBrown7f0da592011-07-28 11:39:22 +10003197 s->syncing = 0;
NeilBrown9a3e1102011-12-23 10:17:53 +11003198 s->replacing = 0;
NeilBrown7f0da592011-07-28 11:39:22 +10003199 /* There is nothing more to do for sync/check/repair.
NeilBrown18b98372012-04-01 23:48:38 +10003200 * Don't even need to abort as that is handled elsewhere
3201 * if needed, and not always wanted e.g. if there is a known
3202 * bad block here.
NeilBrown9a3e1102011-12-23 10:17:53 +11003203 * For recover/replace we need to record a bad block on all
NeilBrown7f0da592011-07-28 11:39:22 +10003204 * non-sync devices, or abort the recovery
3205 */
NeilBrown18b98372012-04-01 23:48:38 +10003206 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
3207 /* During recovery devices cannot be removed, so
3208 * locking and refcounting of rdevs is not needed
3209 */
3210 for (i = 0; i < conf->raid_disks; i++) {
3211 struct md_rdev *rdev = conf->disks[i].rdev;
3212 if (rdev
3213 && !test_bit(Faulty, &rdev->flags)
3214 && !test_bit(In_sync, &rdev->flags)
3215 && !rdev_set_badblocks(rdev, sh->sector,
3216 STRIPE_SECTORS, 0))
3217 abort = 1;
3218 rdev = conf->disks[i].replacement;
3219 if (rdev
3220 && !test_bit(Faulty, &rdev->flags)
3221 && !test_bit(In_sync, &rdev->flags)
3222 && !rdev_set_badblocks(rdev, sh->sector,
3223 STRIPE_SECTORS, 0))
3224 abort = 1;
3225 }
3226 if (abort)
3227 conf->recovery_disabled =
3228 conf->mddev->recovery_disabled;
NeilBrown7f0da592011-07-28 11:39:22 +10003229 }
NeilBrown18b98372012-04-01 23:48:38 +10003230 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
NeilBrown7f0da592011-07-28 11:39:22 +10003231}
3232
NeilBrown9a3e1102011-12-23 10:17:53 +11003233static int want_replace(struct stripe_head *sh, int disk_idx)
3234{
3235 struct md_rdev *rdev;
3236 int rv = 0;
3237 /* Doing recovery so rcu locking not required */
3238 rdev = sh->raid_conf->disks[disk_idx].replacement;
3239 if (rdev
3240 && !test_bit(Faulty, &rdev->flags)
3241 && !test_bit(In_sync, &rdev->flags)
3242 && (rdev->recovery_offset <= sh->sector
3243 || rdev->mddev->recovery_cp <= sh->sector))
3244 rv = 1;
3245
3246 return rv;
3247}
3248
NeilBrown93b3dbc2011-07-27 11:00:36 +10003249/* fetch_block - checks the given member device to see if its data needs
Dan Williams1fe797e2008-06-28 09:16:30 +10003250 * to be read or computed to satisfy a request.
3251 *
3252 * Returns 1 when no more member devices need to be checked, otherwise returns
NeilBrown93b3dbc2011-07-27 11:00:36 +10003253 * 0 to tell the loop in handle_stripe_fill to continue
Dan Williamsf38e1212007-01-02 13:52:30 -07003254 */
NeilBrown2c58f062015-02-02 11:32:23 +11003255
3256static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
3257 int disk_idx, int disks)
Dan Williamsf38e1212007-01-02 13:52:30 -07003258{
3259 struct r5dev *dev = &sh->dev[disk_idx];
NeilBrown93b3dbc2011-07-27 11:00:36 +10003260 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
3261 &sh->dev[s->failed_num[1]] };
NeilBrownea664c82015-02-02 14:03:28 +11003262 int i;
Dan Williamsf38e1212007-01-02 13:52:30 -07003263
NeilBrowna79cfe12015-02-02 11:37:59 +11003264
3265 if (test_bit(R5_LOCKED, &dev->flags) ||
3266 test_bit(R5_UPTODATE, &dev->flags))
3267 /* No point reading this as we already have it or have
3268 * decided to get it.
3269 */
3270 return 0;
3271
3272 if (dev->toread ||
3273 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))
3274 /* We need this block to directly satisfy a request */
3275 return 1;
3276
3277 if (s->syncing || s->expanding ||
3278 (s->replacing && want_replace(sh, disk_idx)))
3279 /* When syncing, or expanding we read everything.
3280 * When replacing, we need the replaced block.
3281 */
3282 return 1;
3283
3284 if ((s->failed >= 1 && fdev[0]->toread) ||
3285 (s->failed >= 2 && fdev[1]->toread))
3286 /* If we want to read from a failed device, then
3287 * we need to actually read every other device.
3288 */
3289 return 1;
3290
NeilBrowna9d56952015-02-02 11:49:10 +11003291 /* Sometimes neither read-modify-write nor reconstruct-write
3292 * cycles can work. In those cases we read every block we
3293 * can. Then the parity-update is certain to have enough to
3294 * work with.
3295 * This can only be a problem when we need to write something,
3296 * and some device has failed. If either of those tests
3297 * fail we need look no further.
3298 */
3299 if (!s->failed || !s->to_write)
3300 return 0;
3301
3302 if (test_bit(R5_Insync, &dev->flags) &&
3303 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3304 /* Pre-reads at not permitted until after short delay
3305 * to gather multiple requests. However if this
3306 * device is no Insync, the block could only be be computed
3307 * and there is no need to delay that.
3308 */
3309 return 0;
NeilBrownea664c82015-02-02 14:03:28 +11003310
NeilBrown36707bb2015-09-24 15:25:36 +10003311 for (i = 0; i < s->failed && i < 2; i++) {
NeilBrownea664c82015-02-02 14:03:28 +11003312 if (fdev[i]->towrite &&
3313 !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
3314 !test_bit(R5_OVERWRITE, &fdev[i]->flags))
3315 /* If we have a partial write to a failed
3316 * device, then we will need to reconstruct
3317 * the content of that device, so all other
3318 * devices must be read.
3319 */
3320 return 1;
3321 }
3322
3323 /* If we are forced to do a reconstruct-write, either because
3324 * the current RAID6 implementation only supports that, or
3325 * or because parity cannot be trusted and we are currently
3326 * recovering it, there is extra need to be careful.
3327 * If one of the devices that we would need to read, because
3328 * it is not being overwritten (and maybe not written at all)
3329 * is missing/faulty, then we need to read everything we can.
3330 */
3331 if (sh->raid_conf->level != 6 &&
3332 sh->sector < sh->raid_conf->mddev->recovery_cp)
3333 /* reconstruct-write isn't being forced */
3334 return 0;
NeilBrown36707bb2015-09-24 15:25:36 +10003335 for (i = 0; i < s->failed && i < 2; i++) {
NeilBrown10d82c52015-05-08 18:19:33 +10003336 if (s->failed_num[i] != sh->pd_idx &&
3337 s->failed_num[i] != sh->qd_idx &&
3338 !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
NeilBrownea664c82015-02-02 14:03:28 +11003339 !test_bit(R5_OVERWRITE, &fdev[i]->flags))
3340 return 1;
3341 }
3342
NeilBrown2c58f062015-02-02 11:32:23 +11003343 return 0;
3344}
3345
3346static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
3347 int disk_idx, int disks)
3348{
3349 struct r5dev *dev = &sh->dev[disk_idx];
3350
3351 /* is the data in this block needed, and can we get it? */
3352 if (need_this_block(sh, s, disk_idx, disks)) {
Yuri Tikhonov5599bec2009-08-29 19:13:12 -07003353 /* we would like to get this block, possibly by computing it,
3354 * otherwise read it if the backing disk is insync
3355 */
3356 BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
3357 BUG_ON(test_bit(R5_Wantread, &dev->flags));
NeilBrownb0c783b2015-05-08 18:19:32 +10003358 BUG_ON(sh->batch_head);
Yuri Tikhonov5599bec2009-08-29 19:13:12 -07003359 if ((s->uptodate == disks - 1) &&
NeilBrownf2b3b442011-07-26 11:35:19 +10003360 (s->failed && (disk_idx == s->failed_num[0] ||
3361 disk_idx == s->failed_num[1]))) {
Yuri Tikhonov5599bec2009-08-29 19:13:12 -07003362 /* have disk failed, and we're requested to fetch it;
3363 * do compute it
3364 */
3365 pr_debug("Computing stripe %llu block %d\n",
3366 (unsigned long long)sh->sector, disk_idx);
3367 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3368 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3369 set_bit(R5_Wantcompute, &dev->flags);
3370 sh->ops.target = disk_idx;
3371 sh->ops.target2 = -1; /* no 2nd target */
3372 s->req_compute = 1;
NeilBrown93b3dbc2011-07-27 11:00:36 +10003373 /* Careful: from this point on 'uptodate' is in the eye
3374 * of raid_run_ops which services 'compute' operations
3375 * before writes. R5_Wantcompute flags a block that will
3376 * be R5_UPTODATE by the time it is needed for a
3377 * subsequent operation.
3378 */
Yuri Tikhonov5599bec2009-08-29 19:13:12 -07003379 s->uptodate++;
3380 return 1;
3381 } else if (s->uptodate == disks-2 && s->failed >= 2) {
3382 /* Computing 2-failure is *very* expensive; only
3383 * do it if failed >= 2
3384 */
3385 int other;
3386 for (other = disks; other--; ) {
3387 if (other == disk_idx)
3388 continue;
3389 if (!test_bit(R5_UPTODATE,
3390 &sh->dev[other].flags))
3391 break;
3392 }
3393 BUG_ON(other < 0);
3394 pr_debug("Computing stripe %llu blocks %d,%d\n",
3395 (unsigned long long)sh->sector,
3396 disk_idx, other);
3397 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3398 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3399 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
3400 set_bit(R5_Wantcompute, &sh->dev[other].flags);
3401 sh->ops.target = disk_idx;
3402 sh->ops.target2 = other;
3403 s->uptodate += 2;
3404 s->req_compute = 1;
3405 return 1;
3406 } else if (test_bit(R5_Insync, &dev->flags)) {
3407 set_bit(R5_LOCKED, &dev->flags);
3408 set_bit(R5_Wantread, &dev->flags);
3409 s->locked++;
3410 pr_debug("Reading block %d (sync=%d)\n",
3411 disk_idx, s->syncing);
3412 }
3413 }
3414
3415 return 0;
3416}
3417
3418/**
NeilBrown93b3dbc2011-07-27 11:00:36 +10003419 * handle_stripe_fill - read or compute data to satisfy pending requests.
Yuri Tikhonov5599bec2009-08-29 19:13:12 -07003420 */
NeilBrown93b3dbc2011-07-27 11:00:36 +10003421static void handle_stripe_fill(struct stripe_head *sh,
3422 struct stripe_head_state *s,
3423 int disks)
Dan Williamsa4456852007-07-09 11:56:43 -07003424{
3425 int i;
Yuri Tikhonov5599bec2009-08-29 19:13:12 -07003426
3427 /* look for blocks to read/compute, skip this if a compute
3428 * is already in flight, or if the stripe contents are in the
3429 * midst of changing due to a write
3430 */
3431 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
3432 !sh->reconstruct_state)
3433 for (i = disks; i--; )
NeilBrown93b3dbc2011-07-27 11:00:36 +10003434 if (fetch_block(sh, s, i, disks))
Yuri Tikhonov5599bec2009-08-29 19:13:12 -07003435 break;
Dan Williamsa4456852007-07-09 11:56:43 -07003436 set_bit(STRIPE_HANDLE, &sh->state);
3437}
3438
NeilBrown787b76f2015-05-21 12:56:41 +10003439static void break_stripe_batch_list(struct stripe_head *head_sh,
3440 unsigned long handle_flags);
Dan Williams1fe797e2008-06-28 09:16:30 +10003441/* handle_stripe_clean_event
Dan Williamsa4456852007-07-09 11:56:43 -07003442 * any written block on an uptodate or failed drive can be returned.
3443 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
3444 * never LOCKED, so we don't need to test 'failed' directly.
3445 */
NeilBrownd1688a62011-10-11 16:49:52 +11003446static void handle_stripe_clean_event(struct r5conf *conf,
NeilBrown34a6f802015-08-14 12:07:57 +10003447 struct stripe_head *sh, int disks, struct bio_list *return_bi)
Dan Williamsa4456852007-07-09 11:56:43 -07003448{
3449 int i;
3450 struct r5dev *dev;
NeilBrownf8dfcff2013-03-12 12:18:06 +11003451 int discard_pending = 0;
shli@kernel.org59fc6302014-12-15 12:57:03 +11003452 struct stripe_head *head_sh = sh;
3453 bool do_endio = false;
Dan Williamsa4456852007-07-09 11:56:43 -07003454
3455 for (i = disks; i--; )
3456 if (sh->dev[i].written) {
3457 dev = &sh->dev[i];
3458 if (!test_bit(R5_LOCKED, &dev->flags) &&
Shaohua Li9e4447682012-10-11 13:49:49 +11003459 (test_bit(R5_UPTODATE, &dev->flags) ||
Shaohua Lid592a992014-05-21 17:57:44 +08003460 test_bit(R5_Discard, &dev->flags) ||
3461 test_bit(R5_SkipCopy, &dev->flags))) {
Dan Williamsa4456852007-07-09 11:56:43 -07003462 /* We can return any write requests */
3463 struct bio *wbi, *wbi2;
Dan Williams45b42332007-07-09 11:56:43 -07003464 pr_debug("Return write for disc %d\n", i);
NeilBrownca64cae2012-11-21 16:33:40 +11003465 if (test_and_clear_bit(R5_Discard, &dev->flags))
3466 clear_bit(R5_UPTODATE, &dev->flags);
Shaohua Lid592a992014-05-21 17:57:44 +08003467 if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
3468 WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
Shaohua Lid592a992014-05-21 17:57:44 +08003469 }
shli@kernel.org59fc6302014-12-15 12:57:03 +11003470 do_endio = true;
3471
3472returnbi:
3473 dev->page = dev->orig_page;
Dan Williamsa4456852007-07-09 11:56:43 -07003474 wbi = dev->written;
3475 dev->written = NULL;
Kent Overstreet4f024f32013-10-11 15:44:27 -07003476 while (wbi && wbi->bi_iter.bi_sector <
Dan Williamsa4456852007-07-09 11:56:43 -07003477 dev->sector + STRIPE_SECTORS) {
3478 wbi2 = r5_next_bio(wbi, dev->sector);
Shaohua Lie7836bd62012-07-19 16:01:31 +10003479 if (!raid5_dec_bi_active_stripes(wbi)) {
Dan Williamsa4456852007-07-09 11:56:43 -07003480 md_write_end(conf->mddev);
NeilBrown34a6f802015-08-14 12:07:57 +10003481 bio_list_add(return_bi, wbi);
Dan Williamsa4456852007-07-09 11:56:43 -07003482 }
3483 wbi = wbi2;
3484 }
Shaohua Li7eaf7e82012-07-19 16:01:31 +10003485 bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3486 STRIPE_SECTORS,
Dan Williamsa4456852007-07-09 11:56:43 -07003487 !test_bit(STRIPE_DEGRADED, &sh->state),
Shaohua Li7eaf7e82012-07-19 16:01:31 +10003488 0);
shli@kernel.org59fc6302014-12-15 12:57:03 +11003489 if (head_sh->batch_head) {
3490 sh = list_first_entry(&sh->batch_list,
3491 struct stripe_head,
3492 batch_list);
3493 if (sh != head_sh) {
3494 dev = &sh->dev[i];
3495 goto returnbi;
3496 }
3497 }
3498 sh = head_sh;
3499 dev = &sh->dev[i];
NeilBrownf8dfcff2013-03-12 12:18:06 +11003500 } else if (test_bit(R5_Discard, &dev->flags))
3501 discard_pending = 1;
Shaohua Lid592a992014-05-21 17:57:44 +08003502 WARN_ON(test_bit(R5_SkipCopy, &dev->flags));
3503 WARN_ON(dev->page != dev->orig_page);
NeilBrownf8dfcff2013-03-12 12:18:06 +11003504 }
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003505
Shaohua Li0576b1c2015-08-13 14:32:00 -07003506 r5l_stripe_write_finished(sh);
3507
NeilBrownf8dfcff2013-03-12 12:18:06 +11003508 if (!discard_pending &&
3509 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
Roman Gushchinb8a9d662015-10-31 10:53:50 +11003510 int hash;
NeilBrownf8dfcff2013-03-12 12:18:06 +11003511 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
3512 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
3513 if (sh->qd_idx >= 0) {
3514 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
3515 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
3516 }
3517 /* now that discard is done we can proceed with any sync */
3518 clear_bit(STRIPE_DISCARD, &sh->state);
Shaohua Lid47648f2013-10-19 14:51:42 +08003519 /*
3520 * SCSI discard will change some bio fields and the stripe has
3521 * no updated data, so remove it from hash list and the stripe
3522 * will be reinitialized
3523 */
shli@kernel.org59fc6302014-12-15 12:57:03 +11003524unhash:
Roman Gushchinb8a9d662015-10-31 10:53:50 +11003525 hash = sh->hash_lock_index;
3526 spin_lock_irq(conf->hash_locks + hash);
Shaohua Lid47648f2013-10-19 14:51:42 +08003527 remove_hash(sh);
Roman Gushchinb8a9d662015-10-31 10:53:50 +11003528 spin_unlock_irq(conf->hash_locks + hash);
shli@kernel.org59fc6302014-12-15 12:57:03 +11003529 if (head_sh->batch_head) {
3530 sh = list_first_entry(&sh->batch_list,
3531 struct stripe_head, batch_list);
3532 if (sh != head_sh)
3533 goto unhash;
3534 }
shli@kernel.org59fc6302014-12-15 12:57:03 +11003535 sh = head_sh;
3536
NeilBrownf8dfcff2013-03-12 12:18:06 +11003537 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
3538 set_bit(STRIPE_HANDLE, &sh->state);
3539
3540 }
Dan Williams8b3e6cd2008-04-28 02:15:53 -07003541
3542 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
3543 if (atomic_dec_and_test(&conf->pending_full_writes))
3544 md_wakeup_thread(conf->mddev->thread);
shli@kernel.org59fc6302014-12-15 12:57:03 +11003545
NeilBrown787b76f2015-05-21 12:56:41 +10003546 if (head_sh->batch_head && do_endio)
3547 break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS);
Dan Williamsa4456852007-07-09 11:56:43 -07003548}
3549
NeilBrownd1688a62011-10-11 16:49:52 +11003550static void handle_stripe_dirtying(struct r5conf *conf,
NeilBrownc8ac1802011-07-27 11:00:36 +10003551 struct stripe_head *sh,
3552 struct stripe_head_state *s,
3553 int disks)
Dan Williamsa4456852007-07-09 11:56:43 -07003554{
3555 int rmw = 0, rcw = 0, i;
Alexander Lyakasa7854482012-10-11 13:50:12 +11003556 sector_t recovery_cp = conf->mddev->recovery_cp;
3557
Markus Stockhausen584acdd2014-12-15 12:57:05 +11003558 /* Check whether resync is now happening or should start.
Alexander Lyakasa7854482012-10-11 13:50:12 +11003559 * If yes, then the array is dirty (after unclean shutdown or
3560 * initial creation), so parity in some stripes might be inconsistent.
3561 * In this case, we need to always do reconstruct-write, to ensure
3562 * that in case of drive failure or read-error correction, we
3563 * generate correct data from the parity.
3564 */
Markus Stockhausen584acdd2014-12-15 12:57:05 +11003565 if (conf->rmw_level == PARITY_DISABLE_RMW ||
NeilBrown26ac1072015-02-18 11:35:14 +11003566 (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
3567 s->failed == 0)) {
Alexander Lyakasa7854482012-10-11 13:50:12 +11003568 /* Calculate the real rcw later - for now make it
NeilBrownc8ac1802011-07-27 11:00:36 +10003569 * look like rcw is cheaper
3570 */
3571 rcw = 1; rmw = 2;
Markus Stockhausen584acdd2014-12-15 12:57:05 +11003572 pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
3573 conf->rmw_level, (unsigned long long)recovery_cp,
Alexander Lyakasa7854482012-10-11 13:50:12 +11003574 (unsigned long long)sh->sector);
NeilBrownc8ac1802011-07-27 11:00:36 +10003575 } else for (i = disks; i--; ) {
Dan Williamsa4456852007-07-09 11:56:43 -07003576 /* would I have to read this buffer for read_modify_write */
3577 struct r5dev *dev = &sh->dev[i];
Markus Stockhausen584acdd2014-12-15 12:57:05 +11003578 if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
Dan Williamsa4456852007-07-09 11:56:43 -07003579 !test_bit(R5_LOCKED, &dev->flags) &&
Dan Williamsf38e1212007-01-02 13:52:30 -07003580 !(test_bit(R5_UPTODATE, &dev->flags) ||
3581 test_bit(R5_Wantcompute, &dev->flags))) {
Dan Williamsa4456852007-07-09 11:56:43 -07003582 if (test_bit(R5_Insync, &dev->flags))
3583 rmw++;
3584 else
3585 rmw += 2*disks; /* cannot read it */
3586 }
3587 /* Would I have to read this buffer for reconstruct_write */
Markus Stockhausen584acdd2014-12-15 12:57:05 +11003588 if (!test_bit(R5_OVERWRITE, &dev->flags) &&
3589 i != sh->pd_idx && i != sh->qd_idx &&
Dan Williamsa4456852007-07-09 11:56:43 -07003590 !test_bit(R5_LOCKED, &dev->flags) &&
Dan Williamsf38e1212007-01-02 13:52:30 -07003591 !(test_bit(R5_UPTODATE, &dev->flags) ||
3592 test_bit(R5_Wantcompute, &dev->flags))) {
NeilBrown67f45542014-05-28 13:39:22 +10003593 if (test_bit(R5_Insync, &dev->flags))
3594 rcw++;
Dan Williamsa4456852007-07-09 11:56:43 -07003595 else
3596 rcw += 2*disks;
3597 }
3598 }
Dan Williams45b42332007-07-09 11:56:43 -07003599 pr_debug("for sector %llu, rmw=%d rcw=%d\n",
Dan Williamsa4456852007-07-09 11:56:43 -07003600 (unsigned long long)sh->sector, rmw, rcw);
3601 set_bit(STRIPE_HANDLE, &sh->state);
Markus Stockhausen584acdd2014-12-15 12:57:05 +11003602 if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) {
Dan Williamsa4456852007-07-09 11:56:43 -07003603 /* prefer read-modify-write, but need to get some data */
Jonathan Brassowe3620a32013-03-07 16:22:01 -06003604 if (conf->mddev->queue)
3605 blk_add_trace_msg(conf->mddev->queue,
3606 "raid5 rmw %llu %d",
3607 (unsigned long long)sh->sector, rmw);
Dan Williamsa4456852007-07-09 11:56:43 -07003608 for (i = disks; i--; ) {
3609 struct r5dev *dev = &sh->dev[i];
Markus Stockhausen584acdd2014-12-15 12:57:05 +11003610 if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
Dan Williamsa4456852007-07-09 11:56:43 -07003611 !test_bit(R5_LOCKED, &dev->flags) &&
Dan Williamsf38e1212007-01-02 13:52:30 -07003612 !(test_bit(R5_UPTODATE, &dev->flags) ||
3613 test_bit(R5_Wantcompute, &dev->flags)) &&
Dan Williamsa4456852007-07-09 11:56:43 -07003614 test_bit(R5_Insync, &dev->flags)) {
NeilBrown67f45542014-05-28 13:39:22 +10003615 if (test_bit(STRIPE_PREREAD_ACTIVE,
3616 &sh->state)) {
3617 pr_debug("Read_old block %d for r-m-w\n",
3618 i);
Dan Williamsa4456852007-07-09 11:56:43 -07003619 set_bit(R5_LOCKED, &dev->flags);
3620 set_bit(R5_Wantread, &dev->flags);
3621 s->locked++;
3622 } else {
3623 set_bit(STRIPE_DELAYED, &sh->state);
3624 set_bit(STRIPE_HANDLE, &sh->state);
3625 }
3626 }
3627 }
NeilBrowna9add5d2012-10-31 11:59:09 +11003628 }
Markus Stockhausen584acdd2014-12-15 12:57:05 +11003629 if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) {
Dan Williamsa4456852007-07-09 11:56:43 -07003630 /* want reconstruct write, but need to get some data */
NeilBrowna9add5d2012-10-31 11:59:09 +11003631 int qread =0;
NeilBrownc8ac1802011-07-27 11:00:36 +10003632 rcw = 0;
Dan Williamsa4456852007-07-09 11:56:43 -07003633 for (i = disks; i--; ) {
3634 struct r5dev *dev = &sh->dev[i];
3635 if (!test_bit(R5_OVERWRITE, &dev->flags) &&
NeilBrownc8ac1802011-07-27 11:00:36 +10003636 i != sh->pd_idx && i != sh->qd_idx &&
Dan Williamsa4456852007-07-09 11:56:43 -07003637 !test_bit(R5_LOCKED, &dev->flags) &&
Dan Williamsf38e1212007-01-02 13:52:30 -07003638 !(test_bit(R5_UPTODATE, &dev->flags) ||
NeilBrownc8ac1802011-07-27 11:00:36 +10003639 test_bit(R5_Wantcompute, &dev->flags))) {
3640 rcw++;
NeilBrown67f45542014-05-28 13:39:22 +10003641 if (test_bit(R5_Insync, &dev->flags) &&
3642 test_bit(STRIPE_PREREAD_ACTIVE,
3643 &sh->state)) {
Dan Williams45b42332007-07-09 11:56:43 -07003644 pr_debug("Read_old block "
Dan Williamsa4456852007-07-09 11:56:43 -07003645 "%d for Reconstruct\n", i);
3646 set_bit(R5_LOCKED, &dev->flags);
3647 set_bit(R5_Wantread, &dev->flags);
3648 s->locked++;
NeilBrowna9add5d2012-10-31 11:59:09 +11003649 qread++;
Dan Williamsa4456852007-07-09 11:56:43 -07003650 } else {
3651 set_bit(STRIPE_DELAYED, &sh->state);
3652 set_bit(STRIPE_HANDLE, &sh->state);
3653 }
3654 }
3655 }
Jonathan Brassowe3620a32013-03-07 16:22:01 -06003656 if (rcw && conf->mddev->queue)
NeilBrowna9add5d2012-10-31 11:59:09 +11003657 blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
3658 (unsigned long long)sh->sector,
3659 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
NeilBrownc8ac1802011-07-27 11:00:36 +10003660 }
NeilBrownb1b02fe2015-02-02 10:44:29 +11003661
3662 if (rcw > disks && rmw > disks &&
3663 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3664 set_bit(STRIPE_DELAYED, &sh->state);
3665
Dan Williamsa4456852007-07-09 11:56:43 -07003666 /* now if nothing is locked, and if we have enough data,
3667 * we can start a write request
3668 */
Dan Williamsf38e1212007-01-02 13:52:30 -07003669 /* since handle_stripe can be called at any time we need to handle the
3670 * case where a compute block operation has been submitted and then a
Dan Williamsac6b53b2009-07-14 13:40:19 -07003671 * subsequent call wants to start a write request. raid_run_ops only
3672 * handles the case where compute block and reconstruct are requested
Dan Williamsf38e1212007-01-02 13:52:30 -07003673 * simultaneously. If this is not the case then new writes need to be
3674 * held off until the compute completes.
3675 */
Dan Williams976ea8d2008-06-28 08:32:03 +10003676 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
3677 (s->locked == 0 && (rcw == 0 || rmw == 0) &&
3678 !test_bit(STRIPE_BIT_DELAY, &sh->state)))
Yuri Tikhonovc0f7bdd2009-08-29 19:13:12 -07003679 schedule_reconstruction(sh, s, rcw == 0, 0);
Dan Williamsa4456852007-07-09 11:56:43 -07003680}
3681
NeilBrownd1688a62011-10-11 16:49:52 +11003682static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
Dan Williamsa4456852007-07-09 11:56:43 -07003683 struct stripe_head_state *s, int disks)
3684{
Dan Williamsecc65c92008-06-28 08:31:57 +10003685 struct r5dev *dev = NULL;
Dan Williamse89f8962007-01-02 13:52:31 -07003686
shli@kernel.org59fc6302014-12-15 12:57:03 +11003687 BUG_ON(sh->batch_head);
Dan Williamsbd2ab672008-04-10 21:29:27 -07003688 set_bit(STRIPE_HANDLE, &sh->state);
3689
Dan Williamsecc65c92008-06-28 08:31:57 +10003690 switch (sh->check_state) {
3691 case check_state_idle:
3692 /* start a new check operation if there are no failures */
Dan Williamsbd2ab672008-04-10 21:29:27 -07003693 if (s->failed == 0) {
Dan Williamsbd2ab672008-04-10 21:29:27 -07003694 BUG_ON(s->uptodate != disks);
Dan Williamsecc65c92008-06-28 08:31:57 +10003695 sh->check_state = check_state_run;
3696 set_bit(STRIPE_OP_CHECK, &s->ops_request);
Dan Williamsbd2ab672008-04-10 21:29:27 -07003697 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
Dan Williamsbd2ab672008-04-10 21:29:27 -07003698 s->uptodate--;
Dan Williamsecc65c92008-06-28 08:31:57 +10003699 break;
Dan Williamsbd2ab672008-04-10 21:29:27 -07003700 }
NeilBrownf2b3b442011-07-26 11:35:19 +10003701 dev = &sh->dev[s->failed_num[0]];
Dan Williamsecc65c92008-06-28 08:31:57 +10003702 /* fall through */
3703 case check_state_compute_result:
3704 sh->check_state = check_state_idle;
3705 if (!dev)
3706 dev = &sh->dev[sh->pd_idx];
3707
3708 /* check that a write has not made the stripe insync */
3709 if (test_bit(STRIPE_INSYNC, &sh->state))
3710 break;
3711
3712 /* either failed parity check, or recovery is happening */
Dan Williamsa4456852007-07-09 11:56:43 -07003713 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
3714 BUG_ON(s->uptodate != disks);
3715
3716 set_bit(R5_LOCKED, &dev->flags);
Dan Williamsecc65c92008-06-28 08:31:57 +10003717 s->locked++;
Dan Williamsa4456852007-07-09 11:56:43 -07003718 set_bit(R5_Wantwrite, &dev->flags);
Dan Williams830ea012007-01-02 13:52:31 -07003719
Dan Williamsa4456852007-07-09 11:56:43 -07003720 clear_bit(STRIPE_DEGRADED, &sh->state);
Dan Williamsa4456852007-07-09 11:56:43 -07003721 set_bit(STRIPE_INSYNC, &sh->state);
Dan Williamsecc65c92008-06-28 08:31:57 +10003722 break;
3723 case check_state_run:
3724 break; /* we will be called again upon completion */
3725 case check_state_check_result:
3726 sh->check_state = check_state_idle;
3727
3728 /* if a failure occurred during the check operation, leave
3729 * STRIPE_INSYNC not set and let the stripe be handled again
3730 */
3731 if (s->failed)
3732 break;
3733
3734 /* handle a successful check operation, if parity is correct
3735 * we are done. Otherwise update the mismatch count and repair
3736 * parity if !MD_RECOVERY_CHECK
3737 */
Dan Williamsad283ea2009-08-29 19:09:26 -07003738 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
Dan Williamsecc65c92008-06-28 08:31:57 +10003739 /* parity is correct (on disc,
3740 * not in buffer any more)
3741 */
3742 set_bit(STRIPE_INSYNC, &sh->state);
3743 else {
Jianpeng Ma7f7583d2012-10-11 14:17:59 +11003744 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
Dan Williamsecc65c92008-06-28 08:31:57 +10003745 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
3746 /* don't try to repair!! */
3747 set_bit(STRIPE_INSYNC, &sh->state);
3748 else {
3749 sh->check_state = check_state_compute_run;
Dan Williams976ea8d2008-06-28 08:32:03 +10003750 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
Dan Williamsecc65c92008-06-28 08:31:57 +10003751 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3752 set_bit(R5_Wantcompute,
3753 &sh->dev[sh->pd_idx].flags);
3754 sh->ops.target = sh->pd_idx;
Dan Williamsac6b53b2009-07-14 13:40:19 -07003755 sh->ops.target2 = -1;
Dan Williamsecc65c92008-06-28 08:31:57 +10003756 s->uptodate++;
3757 }
3758 }
3759 break;
3760 case check_state_compute_run:
3761 break;
3762 default:
3763 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3764 __func__, sh->check_state,
3765 (unsigned long long) sh->sector);
3766 BUG();
Dan Williamsa4456852007-07-09 11:56:43 -07003767 }
3768}
3769
NeilBrownd1688a62011-10-11 16:49:52 +11003770static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
Dan Williams36d1c642009-07-14 11:48:22 -07003771 struct stripe_head_state *s,
NeilBrownf2b3b442011-07-26 11:35:19 +10003772 int disks)
Dan Williamsa4456852007-07-09 11:56:43 -07003773{
Dan Williamsa4456852007-07-09 11:56:43 -07003774 int pd_idx = sh->pd_idx;
NeilBrown34e04e82009-03-31 15:10:16 +11003775 int qd_idx = sh->qd_idx;
Dan Williamsd82dfee2009-07-14 13:40:57 -07003776 struct r5dev *dev;
Dan Williamsa4456852007-07-09 11:56:43 -07003777
shli@kernel.org59fc6302014-12-15 12:57:03 +11003778 BUG_ON(sh->batch_head);
Dan Williamsa4456852007-07-09 11:56:43 -07003779 set_bit(STRIPE_HANDLE, &sh->state);
3780
3781 BUG_ON(s->failed > 2);
Dan Williamsd82dfee2009-07-14 13:40:57 -07003782
Dan Williamsa4456852007-07-09 11:56:43 -07003783 /* Want to check and possibly repair P and Q.
3784 * However there could be one 'failed' device, in which
3785 * case we can only check one of them, possibly using the
3786 * other to generate missing data
3787 */
3788
Dan Williamsd82dfee2009-07-14 13:40:57 -07003789 switch (sh->check_state) {
3790 case check_state_idle:
3791 /* start a new check operation if there are < 2 failures */
NeilBrownf2b3b442011-07-26 11:35:19 +10003792 if (s->failed == s->q_failed) {
Dan Williamsd82dfee2009-07-14 13:40:57 -07003793 /* The only possible failed device holds Q, so it
Dan Williamsa4456852007-07-09 11:56:43 -07003794 * makes sense to check P (If anything else were failed,
3795 * we would have used P to recreate it).
3796 */
Dan Williamsd82dfee2009-07-14 13:40:57 -07003797 sh->check_state = check_state_run;
Dan Williamsa4456852007-07-09 11:56:43 -07003798 }
NeilBrownf2b3b442011-07-26 11:35:19 +10003799 if (!s->q_failed && s->failed < 2) {
Dan Williamsd82dfee2009-07-14 13:40:57 -07003800 /* Q is not failed, and we didn't use it to generate
Dan Williamsa4456852007-07-09 11:56:43 -07003801 * anything, so it makes sense to check it
3802 */
Dan Williamsd82dfee2009-07-14 13:40:57 -07003803 if (sh->check_state == check_state_run)
3804 sh->check_state = check_state_run_pq;
3805 else
3806 sh->check_state = check_state_run_q;
Dan Williamsa4456852007-07-09 11:56:43 -07003807 }
Dan Williams36d1c642009-07-14 11:48:22 -07003808
Dan Williamsd82dfee2009-07-14 13:40:57 -07003809 /* discard potentially stale zero_sum_result */
3810 sh->ops.zero_sum_result = 0;
Dan Williams36d1c642009-07-14 11:48:22 -07003811
Dan Williamsd82dfee2009-07-14 13:40:57 -07003812 if (sh->check_state == check_state_run) {
3813 /* async_xor_zero_sum destroys the contents of P */
3814 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
3815 s->uptodate--;
Dan Williamsa4456852007-07-09 11:56:43 -07003816 }
Dan Williamsd82dfee2009-07-14 13:40:57 -07003817 if (sh->check_state >= check_state_run &&
3818 sh->check_state <= check_state_run_pq) {
3819 /* async_syndrome_zero_sum preserves P and Q, so
3820 * no need to mark them !uptodate here
3821 */
3822 set_bit(STRIPE_OP_CHECK, &s->ops_request);
3823 break;
3824 }
Dan Williams36d1c642009-07-14 11:48:22 -07003825
Dan Williamsd82dfee2009-07-14 13:40:57 -07003826 /* we have 2-disk failure */
3827 BUG_ON(s->failed != 2);
3828 /* fall through */
3829 case check_state_compute_result:
3830 sh->check_state = check_state_idle;
Dan Williams36d1c642009-07-14 11:48:22 -07003831
Dan Williamsd82dfee2009-07-14 13:40:57 -07003832 /* check that a write has not made the stripe insync */
3833 if (test_bit(STRIPE_INSYNC, &sh->state))
3834 break;
Dan Williamsa4456852007-07-09 11:56:43 -07003835
3836 /* now write out any block on a failed drive,
Dan Williamsd82dfee2009-07-14 13:40:57 -07003837 * or P or Q if they were recomputed
Dan Williamsa4456852007-07-09 11:56:43 -07003838 */
Dan Williamsd82dfee2009-07-14 13:40:57 -07003839 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
Dan Williamsa4456852007-07-09 11:56:43 -07003840 if (s->failed == 2) {
NeilBrownf2b3b442011-07-26 11:35:19 +10003841 dev = &sh->dev[s->failed_num[1]];
Dan Williamsa4456852007-07-09 11:56:43 -07003842 s->locked++;
3843 set_bit(R5_LOCKED, &dev->flags);
3844 set_bit(R5_Wantwrite, &dev->flags);
3845 }
3846 if (s->failed >= 1) {
NeilBrownf2b3b442011-07-26 11:35:19 +10003847 dev = &sh->dev[s->failed_num[0]];
Dan Williamsa4456852007-07-09 11:56:43 -07003848 s->locked++;
3849 set_bit(R5_LOCKED, &dev->flags);
3850 set_bit(R5_Wantwrite, &dev->flags);
3851 }
Dan Williamsd82dfee2009-07-14 13:40:57 -07003852 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
Dan Williamsa4456852007-07-09 11:56:43 -07003853 dev = &sh->dev[pd_idx];
3854 s->locked++;
3855 set_bit(R5_LOCKED, &dev->flags);
3856 set_bit(R5_Wantwrite, &dev->flags);
3857 }
Dan Williamsd82dfee2009-07-14 13:40:57 -07003858 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
Dan Williamsa4456852007-07-09 11:56:43 -07003859 dev = &sh->dev[qd_idx];
3860 s->locked++;
3861 set_bit(R5_LOCKED, &dev->flags);
3862 set_bit(R5_Wantwrite, &dev->flags);
3863 }
3864 clear_bit(STRIPE_DEGRADED, &sh->state);
3865
3866 set_bit(STRIPE_INSYNC, &sh->state);
Dan Williamsd82dfee2009-07-14 13:40:57 -07003867 break;
3868 case check_state_run:
3869 case check_state_run_q:
3870 case check_state_run_pq:
3871 break; /* we will be called again upon completion */
3872 case check_state_check_result:
3873 sh->check_state = check_state_idle;
3874
3875 /* handle a successful check operation, if parity is correct
3876 * we are done. Otherwise update the mismatch count and repair
3877 * parity if !MD_RECOVERY_CHECK
3878 */
3879 if (sh->ops.zero_sum_result == 0) {
3880 /* both parities are correct */
3881 if (!s->failed)
3882 set_bit(STRIPE_INSYNC, &sh->state);
3883 else {
3884 /* in contrast to the raid5 case we can validate
3885 * parity, but still have a failure to write
3886 * back
3887 */
3888 sh->check_state = check_state_compute_result;
3889 /* Returning at this point means that we may go
3890 * off and bring p and/or q uptodate again so
3891 * we make sure to check zero_sum_result again
3892 * to verify if p or q need writeback
3893 */
3894 }
3895 } else {
Jianpeng Ma7f7583d2012-10-11 14:17:59 +11003896 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
Dan Williamsd82dfee2009-07-14 13:40:57 -07003897 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
3898 /* don't try to repair!! */
3899 set_bit(STRIPE_INSYNC, &sh->state);
3900 else {
3901 int *target = &sh->ops.target;
3902
3903 sh->ops.target = -1;
3904 sh->ops.target2 = -1;
3905 sh->check_state = check_state_compute_run;
3906 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3907 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3908 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3909 set_bit(R5_Wantcompute,
3910 &sh->dev[pd_idx].flags);
3911 *target = pd_idx;
3912 target = &sh->ops.target2;
3913 s->uptodate++;
3914 }
3915 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3916 set_bit(R5_Wantcompute,
3917 &sh->dev[qd_idx].flags);
3918 *target = qd_idx;
3919 s->uptodate++;
3920 }
3921 }
3922 }
3923 break;
3924 case check_state_compute_run:
3925 break;
3926 default:
3927 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3928 __func__, sh->check_state,
3929 (unsigned long long) sh->sector);
3930 BUG();
Dan Williamsa4456852007-07-09 11:56:43 -07003931 }
3932}
3933
NeilBrownd1688a62011-10-11 16:49:52 +11003934static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
Dan Williamsa4456852007-07-09 11:56:43 -07003935{
3936 int i;
3937
3938 /* We have read all the blocks in this stripe and now we need to
3939 * copy some of them into a target stripe for expand.
3940 */
Dan Williamsf0a50d32007-01-02 13:52:31 -07003941 struct dma_async_tx_descriptor *tx = NULL;
shli@kernel.org59fc6302014-12-15 12:57:03 +11003942 BUG_ON(sh->batch_head);
Dan Williamsa4456852007-07-09 11:56:43 -07003943 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3944 for (i = 0; i < sh->disks; i++)
NeilBrown34e04e82009-03-31 15:10:16 +11003945 if (i != sh->pd_idx && i != sh->qd_idx) {
NeilBrown911d4ee2009-03-31 14:39:38 +11003946 int dd_idx, j;
Dan Williamsa4456852007-07-09 11:56:43 -07003947 struct stripe_head *sh2;
Dan Williamsa08abd82009-06-03 11:43:59 -07003948 struct async_submit_ctl submit;
Dan Williamsa4456852007-07-09 11:56:43 -07003949
Shaohua Li6d036f72015-08-13 14:31:57 -07003950 sector_t bn = raid5_compute_blocknr(sh, i, 1);
NeilBrown911d4ee2009-03-31 14:39:38 +11003951 sector_t s = raid5_compute_sector(conf, bn, 0,
3952 &dd_idx, NULL);
Shaohua Li6d036f72015-08-13 14:31:57 -07003953 sh2 = raid5_get_active_stripe(conf, s, 0, 1, 1);
Dan Williamsa4456852007-07-09 11:56:43 -07003954 if (sh2 == NULL)
3955 /* so far only the early blocks of this stripe
3956 * have been requested. When later blocks
3957 * get requested, we will try again
3958 */
3959 continue;
3960 if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
3961 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
3962 /* must have already done this block */
Shaohua Li6d036f72015-08-13 14:31:57 -07003963 raid5_release_stripe(sh2);
Dan Williamsa4456852007-07-09 11:56:43 -07003964 continue;
3965 }
Dan Williamsf0a50d32007-01-02 13:52:31 -07003966
3967 /* place all the copies on one channel */
Dan Williamsa08abd82009-06-03 11:43:59 -07003968 init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
Dan Williamsf0a50d32007-01-02 13:52:31 -07003969 tx = async_memcpy(sh2->dev[dd_idx].page,
Dan Williams88ba2aa2009-04-09 16:16:18 -07003970 sh->dev[i].page, 0, 0, STRIPE_SIZE,
Dan Williamsa08abd82009-06-03 11:43:59 -07003971 &submit);
Dan Williamsf0a50d32007-01-02 13:52:31 -07003972
Dan Williamsa4456852007-07-09 11:56:43 -07003973 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
3974 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
3975 for (j = 0; j < conf->raid_disks; j++)
3976 if (j != sh2->pd_idx &&
NeilBrown86c374b2011-07-27 11:00:36 +10003977 j != sh2->qd_idx &&
Dan Williamsa4456852007-07-09 11:56:43 -07003978 !test_bit(R5_Expanded, &sh2->dev[j].flags))
3979 break;
3980 if (j == conf->raid_disks) {
3981 set_bit(STRIPE_EXPAND_READY, &sh2->state);
3982 set_bit(STRIPE_HANDLE, &sh2->state);
3983 }
Shaohua Li6d036f72015-08-13 14:31:57 -07003984 raid5_release_stripe(sh2);
Dan Williamsf0a50d32007-01-02 13:52:31 -07003985
Dan Williamsa4456852007-07-09 11:56:43 -07003986 }
NeilBrowna2e08552007-09-11 15:23:36 -07003987 /* done submitting copies, wait for them to complete */
NeilBrown749586b2012-11-20 14:11:15 +11003988 async_tx_quiesce(&tx);
Dan Williamsa4456852007-07-09 11:56:43 -07003989}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003990
3991/*
3992 * handle_stripe - do things to a stripe.
3993 *
NeilBrown9a3e1102011-12-23 10:17:53 +11003994 * We lock the stripe by setting STRIPE_ACTIVE and then examine the
3995 * state of various bits to see what needs to be done.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003996 * Possible results:
NeilBrown9a3e1102011-12-23 10:17:53 +11003997 * return some read requests which now have data
3998 * return some write requests which are safely on storage
Linus Torvalds1da177e2005-04-16 15:20:36 -07003999 * schedule a read on some buffers
4000 * schedule a write of some buffers
4001 * return confirmation of parity correctness
4002 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07004003 */
Dan Williamsa4456852007-07-09 11:56:43 -07004004
NeilBrownacfe7262011-07-27 11:00:36 +10004005static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
NeilBrown16a53ec2006-06-26 00:27:38 -07004006{
NeilBrownd1688a62011-10-11 16:49:52 +11004007 struct r5conf *conf = sh->raid_conf;
NeilBrownf4168852007-02-28 20:11:53 -08004008 int disks = sh->disks;
NeilBrown474af965fe2011-07-27 11:00:36 +10004009 struct r5dev *dev;
4010 int i;
NeilBrown9a3e1102011-12-23 10:17:53 +11004011 int do_recovery = 0;
NeilBrown16a53ec2006-06-26 00:27:38 -07004012
NeilBrownacfe7262011-07-27 11:00:36 +10004013 memset(s, 0, sizeof(*s));
NeilBrown16a53ec2006-06-26 00:27:38 -07004014
shli@kernel.orgdabc4ec2014-12-15 12:57:04 +11004015 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
4016 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
NeilBrownacfe7262011-07-27 11:00:36 +10004017 s->failed_num[0] = -1;
4018 s->failed_num[1] = -1;
Shaohua Li6e74a9c2015-10-08 21:54:08 -07004019 s->log_failed = r5l_log_disk_error(conf);
NeilBrownacfe7262011-07-27 11:00:36 +10004020
4021 /* Now to look around and see what can be done */
NeilBrown16a53ec2006-06-26 00:27:38 -07004022 rcu_read_lock();
4023 for (i=disks; i--; ) {
NeilBrown3cb03002011-10-11 16:45:26 +11004024 struct md_rdev *rdev;
NeilBrown31c176e2011-07-28 11:39:22 +10004025 sector_t first_bad;
4026 int bad_sectors;
4027 int is_bad = 0;
NeilBrownacfe7262011-07-27 11:00:36 +10004028
NeilBrown16a53ec2006-06-26 00:27:38 -07004029 dev = &sh->dev[i];
NeilBrown16a53ec2006-06-26 00:27:38 -07004030
Dan Williams45b42332007-07-09 11:56:43 -07004031 pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
NeilBrown9a3e1102011-12-23 10:17:53 +11004032 i, dev->flags,
4033 dev->toread, dev->towrite, dev->written);
Yuri Tikhonov6c0069c2009-08-29 19:13:13 -07004034 /* maybe we can reply to a read
4035 *
4036 * new wantfill requests are only permitted while
4037 * ops_complete_biofill is guaranteed to be inactive
4038 */
4039 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
4040 !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
4041 set_bit(R5_Wantfill, &dev->flags);
NeilBrown16a53ec2006-06-26 00:27:38 -07004042
4043 /* now count some things */
NeilBrowncc940152011-07-26 11:35:35 +10004044 if (test_bit(R5_LOCKED, &dev->flags))
4045 s->locked++;
4046 if (test_bit(R5_UPTODATE, &dev->flags))
4047 s->uptodate++;
Dan Williams2d6e4ec2009-09-16 12:11:54 -07004048 if (test_bit(R5_Wantcompute, &dev->flags)) {
NeilBrowncc940152011-07-26 11:35:35 +10004049 s->compute++;
4050 BUG_ON(s->compute > 2);
Dan Williams2d6e4ec2009-09-16 12:11:54 -07004051 }
NeilBrown16a53ec2006-06-26 00:27:38 -07004052
NeilBrownacfe7262011-07-27 11:00:36 +10004053 if (test_bit(R5_Wantfill, &dev->flags))
NeilBrowncc940152011-07-26 11:35:35 +10004054 s->to_fill++;
NeilBrownacfe7262011-07-27 11:00:36 +10004055 else if (dev->toread)
NeilBrowncc940152011-07-26 11:35:35 +10004056 s->to_read++;
NeilBrown16a53ec2006-06-26 00:27:38 -07004057 if (dev->towrite) {
NeilBrowncc940152011-07-26 11:35:35 +10004058 s->to_write++;
NeilBrown16a53ec2006-06-26 00:27:38 -07004059 if (!test_bit(R5_OVERWRITE, &dev->flags))
NeilBrowncc940152011-07-26 11:35:35 +10004060 s->non_overwrite++;
NeilBrown16a53ec2006-06-26 00:27:38 -07004061 }
Dan Williamsa4456852007-07-09 11:56:43 -07004062 if (dev->written)
NeilBrowncc940152011-07-26 11:35:35 +10004063 s->written++;
NeilBrown14a75d32011-12-23 10:17:52 +11004064 /* Prefer to use the replacement for reads, but only
4065 * if it is recovered enough and has no bad blocks.
4066 */
4067 rdev = rcu_dereference(conf->disks[i].replacement);
4068 if (rdev && !test_bit(Faulty, &rdev->flags) &&
4069 rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
4070 !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
4071 &first_bad, &bad_sectors))
4072 set_bit(R5_ReadRepl, &dev->flags);
4073 else {
NeilBrowne6030cb2015-07-17 13:26:23 +10004074 if (rdev && !test_bit(Faulty, &rdev->flags))
NeilBrown9a3e1102011-12-23 10:17:53 +11004075 set_bit(R5_NeedReplace, &dev->flags);
NeilBrowne6030cb2015-07-17 13:26:23 +10004076 else
4077 clear_bit(R5_NeedReplace, &dev->flags);
NeilBrown14a75d32011-12-23 10:17:52 +11004078 rdev = rcu_dereference(conf->disks[i].rdev);
4079 clear_bit(R5_ReadRepl, &dev->flags);
4080 }
NeilBrown9283d8c2011-12-08 16:27:57 +11004081 if (rdev && test_bit(Faulty, &rdev->flags))
4082 rdev = NULL;
NeilBrown31c176e2011-07-28 11:39:22 +10004083 if (rdev) {
4084 is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
4085 &first_bad, &bad_sectors);
4086 if (s->blocked_rdev == NULL
4087 && (test_bit(Blocked, &rdev->flags)
4088 || is_bad < 0)) {
4089 if (is_bad < 0)
4090 set_bit(BlockedBadBlocks,
4091 &rdev->flags);
4092 s->blocked_rdev = rdev;
4093 atomic_inc(&rdev->nr_pending);
4094 }
Dan Williams6bfe0b42008-04-30 00:52:32 -07004095 }
NeilBrown415e72d2010-06-17 17:25:21 +10004096 clear_bit(R5_Insync, &dev->flags);
4097 if (!rdev)
4098 /* Not in-sync */;
NeilBrown31c176e2011-07-28 11:39:22 +10004099 else if (is_bad) {
4100 /* also not in-sync */
NeilBrown18b98372012-04-01 23:48:38 +10004101 if (!test_bit(WriteErrorSeen, &rdev->flags) &&
4102 test_bit(R5_UPTODATE, &dev->flags)) {
NeilBrown31c176e2011-07-28 11:39:22 +10004103 /* treat as in-sync, but with a read error
4104 * which we can now try to correct
4105 */
4106 set_bit(R5_Insync, &dev->flags);
4107 set_bit(R5_ReadError, &dev->flags);
4108 }
4109 } else if (test_bit(In_sync, &rdev->flags))
NeilBrown415e72d2010-06-17 17:25:21 +10004110 set_bit(R5_Insync, &dev->flags);
NeilBrown30d7a482011-12-23 09:57:00 +11004111 else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
NeilBrown415e72d2010-06-17 17:25:21 +10004112 /* in sync if before recovery_offset */
NeilBrown30d7a482011-12-23 09:57:00 +11004113 set_bit(R5_Insync, &dev->flags);
4114 else if (test_bit(R5_UPTODATE, &dev->flags) &&
4115 test_bit(R5_Expanded, &dev->flags))
4116 /* If we've reshaped into here, we assume it is Insync.
4117 * We will shortly update recovery_offset to make
4118 * it official.
4119 */
4120 set_bit(R5_Insync, &dev->flags);
4121
NeilBrown1cc03eb2014-01-06 13:19:42 +11004122 if (test_bit(R5_WriteError, &dev->flags)) {
NeilBrown14a75d32011-12-23 10:17:52 +11004123 /* This flag does not apply to '.replacement'
4124 * only to .rdev, so make sure to check that*/
4125 struct md_rdev *rdev2 = rcu_dereference(
4126 conf->disks[i].rdev);
4127 if (rdev2 == rdev)
4128 clear_bit(R5_Insync, &dev->flags);
4129 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
NeilBrownbc2607f2011-07-28 11:39:22 +10004130 s->handle_bad_blocks = 1;
NeilBrown14a75d32011-12-23 10:17:52 +11004131 atomic_inc(&rdev2->nr_pending);
NeilBrownbc2607f2011-07-28 11:39:22 +10004132 } else
4133 clear_bit(R5_WriteError, &dev->flags);
4134 }
NeilBrown1cc03eb2014-01-06 13:19:42 +11004135 if (test_bit(R5_MadeGood, &dev->flags)) {
NeilBrown14a75d32011-12-23 10:17:52 +11004136 /* This flag does not apply to '.replacement'
4137 * only to .rdev, so make sure to check that*/
4138 struct md_rdev *rdev2 = rcu_dereference(
4139 conf->disks[i].rdev);
4140 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
NeilBrownb84db562011-07-28 11:39:23 +10004141 s->handle_bad_blocks = 1;
NeilBrown14a75d32011-12-23 10:17:52 +11004142 atomic_inc(&rdev2->nr_pending);
NeilBrownb84db562011-07-28 11:39:23 +10004143 } else
4144 clear_bit(R5_MadeGood, &dev->flags);
4145 }
NeilBrown977df362011-12-23 10:17:53 +11004146 if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
4147 struct md_rdev *rdev2 = rcu_dereference(
4148 conf->disks[i].replacement);
4149 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
4150 s->handle_bad_blocks = 1;
4151 atomic_inc(&rdev2->nr_pending);
4152 } else
4153 clear_bit(R5_MadeGoodRepl, &dev->flags);
4154 }
NeilBrown415e72d2010-06-17 17:25:21 +10004155 if (!test_bit(R5_Insync, &dev->flags)) {
NeilBrown16a53ec2006-06-26 00:27:38 -07004156 /* The ReadError flag will just be confusing now */
4157 clear_bit(R5_ReadError, &dev->flags);
4158 clear_bit(R5_ReWrite, &dev->flags);
4159 }
NeilBrown415e72d2010-06-17 17:25:21 +10004160 if (test_bit(R5_ReadError, &dev->flags))
4161 clear_bit(R5_Insync, &dev->flags);
4162 if (!test_bit(R5_Insync, &dev->flags)) {
NeilBrowncc940152011-07-26 11:35:35 +10004163 if (s->failed < 2)
4164 s->failed_num[s->failed] = i;
4165 s->failed++;
NeilBrown9a3e1102011-12-23 10:17:53 +11004166 if (rdev && !test_bit(Faulty, &rdev->flags))
4167 do_recovery = 1;
NeilBrown415e72d2010-06-17 17:25:21 +10004168 }
NeilBrown16a53ec2006-06-26 00:27:38 -07004169 }
NeilBrown9a3e1102011-12-23 10:17:53 +11004170 if (test_bit(STRIPE_SYNCING, &sh->state)) {
4171 /* If there is a failed device being replaced,
4172 * we must be recovering.
4173 * else if we are after recovery_cp, we must be syncing
majianpengc6d2e082012-04-02 01:16:59 +10004174 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
NeilBrown9a3e1102011-12-23 10:17:53 +11004175 * else we can only be replacing
4176 * sync and recovery both need to read all devices, and so
4177 * use the same flag.
4178 */
4179 if (do_recovery ||
majianpengc6d2e082012-04-02 01:16:59 +10004180 sh->sector >= conf->mddev->recovery_cp ||
4181 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
NeilBrown9a3e1102011-12-23 10:17:53 +11004182 s->syncing = 1;
4183 else
4184 s->replacing = 1;
4185 }
NeilBrown16a53ec2006-06-26 00:27:38 -07004186 rcu_read_unlock();
NeilBrowncc940152011-07-26 11:35:35 +10004187}
NeilBrownf4168852007-02-28 20:11:53 -08004188
shli@kernel.org59fc6302014-12-15 12:57:03 +11004189static int clear_batch_ready(struct stripe_head *sh)
4190{
NeilBrownb15a9db2015-05-22 15:20:04 +10004191 /* Return '1' if this is a member of batch, or
4192 * '0' if it is a lone stripe or a head which can now be
4193 * handled.
4194 */
shli@kernel.org59fc6302014-12-15 12:57:03 +11004195 struct stripe_head *tmp;
4196 if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
NeilBrownb15a9db2015-05-22 15:20:04 +10004197 return (sh->batch_head && sh->batch_head != sh);
shli@kernel.org59fc6302014-12-15 12:57:03 +11004198 spin_lock(&sh->stripe_lock);
4199 if (!sh->batch_head) {
4200 spin_unlock(&sh->stripe_lock);
4201 return 0;
4202 }
4203
4204 /*
4205 * this stripe could be added to a batch list before we check
4206 * BATCH_READY, skips it
4207 */
4208 if (sh->batch_head != sh) {
4209 spin_unlock(&sh->stripe_lock);
4210 return 1;
4211 }
4212 spin_lock(&sh->batch_lock);
4213 list_for_each_entry(tmp, &sh->batch_list, batch_list)
4214 clear_bit(STRIPE_BATCH_READY, &tmp->state);
4215 spin_unlock(&sh->batch_lock);
4216 spin_unlock(&sh->stripe_lock);
4217
4218 /*
4219 * BATCH_READY is cleared, no new stripes can be added.
4220 * batch_list can be accessed without lock
4221 */
4222 return 0;
4223}
4224
NeilBrown3960ce72015-05-21 12:20:36 +10004225static void break_stripe_batch_list(struct stripe_head *head_sh,
4226 unsigned long handle_flags)
shli@kernel.org72ac7332014-12-15 12:57:03 +11004227{
NeilBrown4e3d62f2015-05-21 11:50:16 +10004228 struct stripe_head *sh, *next;
shli@kernel.org72ac7332014-12-15 12:57:03 +11004229 int i;
NeilBrownfb642b92015-05-21 12:00:47 +10004230 int do_wakeup = 0;
shli@kernel.org72ac7332014-12-15 12:57:03 +11004231
NeilBrownbb270512015-05-08 18:19:40 +10004232 list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {
4233
shli@kernel.org72ac7332014-12-15 12:57:03 +11004234 list_del_init(&sh->batch_list);
4235
NeilBrown1b956f72015-05-21 12:40:26 +10004236 WARN_ON_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |
4237 (1 << STRIPE_SYNCING) |
4238 (1 << STRIPE_REPLACED) |
4239 (1 << STRIPE_PREREAD_ACTIVE) |
4240 (1 << STRIPE_DELAYED) |
4241 (1 << STRIPE_BIT_DELAY) |
4242 (1 << STRIPE_FULL_WRITE) |
4243 (1 << STRIPE_BIOFILL_RUN) |
4244 (1 << STRIPE_COMPUTE_RUN) |
4245 (1 << STRIPE_OPS_REQ_PENDING) |
4246 (1 << STRIPE_DISCARD) |
4247 (1 << STRIPE_BATCH_READY) |
4248 (1 << STRIPE_BATCH_ERR) |
4249 (1 << STRIPE_BITMAP_PENDING)));
4250 WARN_ON_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) |
4251 (1 << STRIPE_REPLACED)));
4252
4253 set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |
4254 (1 << STRIPE_DEGRADED)),
4255 head_sh->state & (1 << STRIPE_INSYNC));
4256
shli@kernel.org72ac7332014-12-15 12:57:03 +11004257 sh->check_state = head_sh->check_state;
4258 sh->reconstruct_state = head_sh->reconstruct_state;
NeilBrownfb642b92015-05-21 12:00:47 +10004259 for (i = 0; i < sh->disks; i++) {
4260 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
4261 do_wakeup = 1;
shli@kernel.org72ac7332014-12-15 12:57:03 +11004262 sh->dev[i].flags = head_sh->dev[i].flags &
4263 (~((1 << R5_WriteError) | (1 << R5_Overlap)));
NeilBrownfb642b92015-05-21 12:00:47 +10004264 }
shli@kernel.org72ac7332014-12-15 12:57:03 +11004265 spin_lock_irq(&sh->stripe_lock);
4266 sh->batch_head = NULL;
4267 spin_unlock_irq(&sh->stripe_lock);
NeilBrown3960ce72015-05-21 12:20:36 +10004268 if (handle_flags == 0 ||
4269 sh->state & handle_flags)
4270 set_bit(STRIPE_HANDLE, &sh->state);
Shaohua Li6d036f72015-08-13 14:31:57 -07004271 raid5_release_stripe(sh);
shli@kernel.org72ac7332014-12-15 12:57:03 +11004272 }
NeilBrownfb642b92015-05-21 12:00:47 +10004273 spin_lock_irq(&head_sh->stripe_lock);
4274 head_sh->batch_head = NULL;
4275 spin_unlock_irq(&head_sh->stripe_lock);
4276 for (i = 0; i < head_sh->disks; i++)
4277 if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))
4278 do_wakeup = 1;
NeilBrown3960ce72015-05-21 12:20:36 +10004279 if (head_sh->state & handle_flags)
4280 set_bit(STRIPE_HANDLE, &head_sh->state);
NeilBrownfb642b92015-05-21 12:00:47 +10004281
4282 if (do_wakeup)
4283 wake_up(&head_sh->raid_conf->wait_for_overlap);
shli@kernel.org72ac7332014-12-15 12:57:03 +11004284}
4285
NeilBrowncc940152011-07-26 11:35:35 +10004286static void handle_stripe(struct stripe_head *sh)
4287{
4288 struct stripe_head_state s;
NeilBrownd1688a62011-10-11 16:49:52 +11004289 struct r5conf *conf = sh->raid_conf;
NeilBrown3687c062011-07-27 11:00:36 +10004290 int i;
NeilBrown84789552011-07-27 11:00:36 +10004291 int prexor;
4292 int disks = sh->disks;
NeilBrown474af965fe2011-07-27 11:00:36 +10004293 struct r5dev *pdev, *qdev;
NeilBrowncc940152011-07-26 11:35:35 +10004294
4295 clear_bit(STRIPE_HANDLE, &sh->state);
Dan Williams257a4b42011-11-08 16:22:06 +11004296 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
NeilBrowncc940152011-07-26 11:35:35 +10004297 /* already being handled, ensure it gets handled
4298 * again when current action finishes */
4299 set_bit(STRIPE_HANDLE, &sh->state);
4300 return;
4301 }
4302
shli@kernel.org59fc6302014-12-15 12:57:03 +11004303 if (clear_batch_ready(sh) ) {
4304 clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4305 return;
4306 }
4307
NeilBrown4e3d62f2015-05-21 11:50:16 +10004308 if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
NeilBrown3960ce72015-05-21 12:20:36 +10004309 break_stripe_batch_list(sh, 0);
shli@kernel.org72ac7332014-12-15 12:57:03 +11004310
shli@kernel.orgdabc4ec2014-12-15 12:57:04 +11004311 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
NeilBrownf8dfcff2013-03-12 12:18:06 +11004312 spin_lock(&sh->stripe_lock);
4313 /* Cannot process 'sync' concurrently with 'discard' */
4314 if (!test_bit(STRIPE_DISCARD, &sh->state) &&
4315 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
4316 set_bit(STRIPE_SYNCING, &sh->state);
4317 clear_bit(STRIPE_INSYNC, &sh->state);
NeilBrownf94c0b62013-07-22 12:57:21 +10004318 clear_bit(STRIPE_REPLACED, &sh->state);
NeilBrownf8dfcff2013-03-12 12:18:06 +11004319 }
4320 spin_unlock(&sh->stripe_lock);
NeilBrowncc940152011-07-26 11:35:35 +10004321 }
4322 clear_bit(STRIPE_DELAYED, &sh->state);
4323
4324 pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
4325 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
4326 (unsigned long long)sh->sector, sh->state,
4327 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
4328 sh->check_state, sh->reconstruct_state);
NeilBrowncc940152011-07-26 11:35:35 +10004329
NeilBrownacfe7262011-07-27 11:00:36 +10004330 analyse_stripe(sh, &s);
NeilBrownc5a31002011-07-27 11:00:36 +10004331
Shaohua Lib70abcb22015-08-13 14:31:58 -07004332 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
4333 goto finish;
4334
NeilBrownbc2607f2011-07-28 11:39:22 +10004335 if (s.handle_bad_blocks) {
4336 set_bit(STRIPE_HANDLE, &sh->state);
4337 goto finish;
4338 }
4339
NeilBrown474af965fe2011-07-27 11:00:36 +10004340 if (unlikely(s.blocked_rdev)) {
4341 if (s.syncing || s.expanding || s.expanded ||
NeilBrown9a3e1102011-12-23 10:17:53 +11004342 s.replacing || s.to_write || s.written) {
NeilBrown474af965fe2011-07-27 11:00:36 +10004343 set_bit(STRIPE_HANDLE, &sh->state);
4344 goto finish;
4345 }
4346 /* There is nothing for the blocked_rdev to block */
4347 rdev_dec_pending(s.blocked_rdev, conf->mddev);
4348 s.blocked_rdev = NULL;
4349 }
4350
4351 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
4352 set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
4353 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
4354 }
4355
4356 pr_debug("locked=%d uptodate=%d to_read=%d"
4357 " to_write=%d failed=%d failed_num=%d,%d\n",
4358 s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
4359 s.failed_num[0], s.failed_num[1]);
4360 /* check if the array has lost more than max_degraded devices and,
4361 * if so, some requests might need to be failed.
4362 */
Shaohua Li6e74a9c2015-10-08 21:54:08 -07004363 if (s.failed > conf->max_degraded || s.log_failed) {
NeilBrown9a3f5302011-11-08 16:22:01 +11004364 sh->check_state = 0;
4365 sh->reconstruct_state = 0;
NeilBrown626f2092015-05-22 14:03:10 +10004366 break_stripe_batch_list(sh, 0);
NeilBrown9a3f5302011-11-08 16:22:01 +11004367 if (s.to_read+s.to_write+s.written)
4368 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
NeilBrown9a3e1102011-12-23 10:17:53 +11004369 if (s.syncing + s.replacing)
NeilBrown9a3f5302011-11-08 16:22:01 +11004370 handle_failed_sync(conf, sh, &s);
4371 }
NeilBrown474af965fe2011-07-27 11:00:36 +10004372
NeilBrown84789552011-07-27 11:00:36 +10004373 /* Now we check to see if any write operations have recently
4374 * completed
4375 */
4376 prexor = 0;
4377 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
4378 prexor = 1;
4379 if (sh->reconstruct_state == reconstruct_state_drain_result ||
4380 sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
4381 sh->reconstruct_state = reconstruct_state_idle;
4382
4383 /* All the 'written' buffers and the parity block are ready to
4384 * be written back to disk
4385 */
Shaohua Li9e4447682012-10-11 13:49:49 +11004386 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
4387 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
NeilBrown84789552011-07-27 11:00:36 +10004388 BUG_ON(sh->qd_idx >= 0 &&
Shaohua Li9e4447682012-10-11 13:49:49 +11004389 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
4390 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
NeilBrown84789552011-07-27 11:00:36 +10004391 for (i = disks; i--; ) {
4392 struct r5dev *dev = &sh->dev[i];
4393 if (test_bit(R5_LOCKED, &dev->flags) &&
4394 (i == sh->pd_idx || i == sh->qd_idx ||
4395 dev->written)) {
4396 pr_debug("Writing block %d\n", i);
4397 set_bit(R5_Wantwrite, &dev->flags);
4398 if (prexor)
4399 continue;
NeilBrown9c4bdf62014-08-13 09:57:07 +10004400 if (s.failed > 1)
4401 continue;
NeilBrown84789552011-07-27 11:00:36 +10004402 if (!test_bit(R5_Insync, &dev->flags) ||
4403 ((i == sh->pd_idx || i == sh->qd_idx) &&
4404 s.failed == 0))
4405 set_bit(STRIPE_INSYNC, &sh->state);
4406 }
4407 }
4408 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4409 s.dec_preread_active = 1;
4410 }
4411
NeilBrownef5b7c62012-11-22 09:13:36 +11004412 /*
4413 * might be able to return some write requests if the parity blocks
4414 * are safe, or on a failed drive
4415 */
4416 pdev = &sh->dev[sh->pd_idx];
4417 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
4418 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
4419 qdev = &sh->dev[sh->qd_idx];
4420 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
4421 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
4422 || conf->level < 6;
4423
4424 if (s.written &&
4425 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
4426 && !test_bit(R5_LOCKED, &pdev->flags)
4427 && (test_bit(R5_UPTODATE, &pdev->flags) ||
4428 test_bit(R5_Discard, &pdev->flags))))) &&
4429 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
4430 && !test_bit(R5_LOCKED, &qdev->flags)
4431 && (test_bit(R5_UPTODATE, &qdev->flags) ||
4432 test_bit(R5_Discard, &qdev->flags))))))
4433 handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
4434
4435 /* Now we might consider reading some blocks, either to check/generate
4436 * parity, or to satisfy requests
4437 * or to load a block that is being partially written.
4438 */
4439 if (s.to_read || s.non_overwrite
4440 || (conf->level == 6 && s.to_write && s.failed)
4441 || (s.syncing && (s.uptodate + s.compute < disks))
4442 || s.replacing
4443 || s.expanding)
4444 handle_stripe_fill(sh, &s, disks);
4445
NeilBrown84789552011-07-27 11:00:36 +10004446 /* Now to consider new write requests and what else, if anything
4447 * should be read. We do not handle new writes when:
4448 * 1/ A 'write' operation (copy+xor) is already in flight.
4449 * 2/ A 'check' operation is in flight, as it may clobber the parity
4450 * block.
4451 */
4452 if (s.to_write && !sh->reconstruct_state && !sh->check_state)
4453 handle_stripe_dirtying(conf, sh, &s, disks);
4454
4455 /* maybe we need to check and possibly fix the parity for this stripe
4456 * Any reads will already have been scheduled, so we just see if enough
4457 * data is available. The parity check is held off while parity
4458 * dependent operations are in flight.
4459 */
4460 if (sh->check_state ||
4461 (s.syncing && s.locked == 0 &&
4462 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
4463 !test_bit(STRIPE_INSYNC, &sh->state))) {
4464 if (conf->level == 6)
4465 handle_parity_checks6(conf, sh, &s, disks);
4466 else
4467 handle_parity_checks5(conf, sh, &s, disks);
4468 }
NeilBrownc5a31002011-07-27 11:00:36 +10004469
NeilBrownf94c0b62013-07-22 12:57:21 +10004470 if ((s.replacing || s.syncing) && s.locked == 0
4471 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
4472 && !test_bit(STRIPE_REPLACED, &sh->state)) {
NeilBrown9a3e1102011-12-23 10:17:53 +11004473 /* Write out to replacement devices where possible */
4474 for (i = 0; i < conf->raid_disks; i++)
NeilBrownf94c0b62013-07-22 12:57:21 +10004475 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
4476 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
NeilBrown9a3e1102011-12-23 10:17:53 +11004477 set_bit(R5_WantReplace, &sh->dev[i].flags);
4478 set_bit(R5_LOCKED, &sh->dev[i].flags);
4479 s.locked++;
4480 }
NeilBrownf94c0b62013-07-22 12:57:21 +10004481 if (s.replacing)
4482 set_bit(STRIPE_INSYNC, &sh->state);
4483 set_bit(STRIPE_REPLACED, &sh->state);
NeilBrown9a3e1102011-12-23 10:17:53 +11004484 }
4485 if ((s.syncing || s.replacing) && s.locked == 0 &&
NeilBrownf94c0b62013-07-22 12:57:21 +10004486 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
NeilBrown9a3e1102011-12-23 10:17:53 +11004487 test_bit(STRIPE_INSYNC, &sh->state)) {
NeilBrownc5a31002011-07-27 11:00:36 +10004488 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
4489 clear_bit(STRIPE_SYNCING, &sh->state);
NeilBrownf8dfcff2013-03-12 12:18:06 +11004490 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
4491 wake_up(&conf->wait_for_overlap);
NeilBrownc5a31002011-07-27 11:00:36 +10004492 }
4493
4494 /* If the failed drives are just a ReadError, then we might need
4495 * to progress the repair/check process
4496 */
4497 if (s.failed <= conf->max_degraded && !conf->mddev->ro)
4498 for (i = 0; i < s.failed; i++) {
4499 struct r5dev *dev = &sh->dev[s.failed_num[i]];
4500 if (test_bit(R5_ReadError, &dev->flags)
4501 && !test_bit(R5_LOCKED, &dev->flags)
4502 && test_bit(R5_UPTODATE, &dev->flags)
4503 ) {
4504 if (!test_bit(R5_ReWrite, &dev->flags)) {
4505 set_bit(R5_Wantwrite, &dev->flags);
4506 set_bit(R5_ReWrite, &dev->flags);
4507 set_bit(R5_LOCKED, &dev->flags);
4508 s.locked++;
4509 } else {
4510 /* let's read it back */
4511 set_bit(R5_Wantread, &dev->flags);
4512 set_bit(R5_LOCKED, &dev->flags);
4513 s.locked++;
4514 }
4515 }
4516 }
4517
NeilBrown3687c062011-07-27 11:00:36 +10004518 /* Finish reconstruct operations initiated by the expansion process */
4519 if (sh->reconstruct_state == reconstruct_state_result) {
4520 struct stripe_head *sh_src
Shaohua Li6d036f72015-08-13 14:31:57 -07004521 = raid5_get_active_stripe(conf, sh->sector, 1, 1, 1);
NeilBrown3687c062011-07-27 11:00:36 +10004522 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
4523 /* sh cannot be written until sh_src has been read.
4524 * so arrange for sh to be delayed a little
4525 */
4526 set_bit(STRIPE_DELAYED, &sh->state);
4527 set_bit(STRIPE_HANDLE, &sh->state);
4528 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
4529 &sh_src->state))
4530 atomic_inc(&conf->preread_active_stripes);
Shaohua Li6d036f72015-08-13 14:31:57 -07004531 raid5_release_stripe(sh_src);
NeilBrown3687c062011-07-27 11:00:36 +10004532 goto finish;
4533 }
4534 if (sh_src)
Shaohua Li6d036f72015-08-13 14:31:57 -07004535 raid5_release_stripe(sh_src);
NeilBrown16a53ec2006-06-26 00:27:38 -07004536
NeilBrown3687c062011-07-27 11:00:36 +10004537 sh->reconstruct_state = reconstruct_state_idle;
4538 clear_bit(STRIPE_EXPANDING, &sh->state);
4539 for (i = conf->raid_disks; i--; ) {
4540 set_bit(R5_Wantwrite, &sh->dev[i].flags);
4541 set_bit(R5_LOCKED, &sh->dev[i].flags);
4542 s.locked++;
4543 }
4544 }
4545
4546 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
4547 !sh->reconstruct_state) {
4548 /* Need to write out all blocks after computing parity */
4549 sh->disks = conf->raid_disks;
4550 stripe_set_idx(sh->sector, conf, 0, sh);
4551 schedule_reconstruction(sh, &s, 1, 1);
4552 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
4553 clear_bit(STRIPE_EXPAND_READY, &sh->state);
4554 atomic_dec(&conf->reshape_stripes);
4555 wake_up(&conf->wait_for_overlap);
4556 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
4557 }
4558
4559 if (s.expanding && s.locked == 0 &&
4560 !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
4561 handle_stripe_expansion(conf, sh);
4562
4563finish:
Dan Williams6bfe0b42008-04-30 00:52:32 -07004564 /* wait for this device to become unblocked */
NeilBrown5f066c62012-07-03 12:13:29 +10004565 if (unlikely(s.blocked_rdev)) {
4566 if (conf->mddev->external)
4567 md_wait_for_blocked_rdev(s.blocked_rdev,
4568 conf->mddev);
4569 else
4570 /* Internal metadata will immediately
4571 * be written by raid5d, so we don't
4572 * need to wait here.
4573 */
4574 rdev_dec_pending(s.blocked_rdev,
4575 conf->mddev);
4576 }
Dan Williams6bfe0b42008-04-30 00:52:32 -07004577
NeilBrownbc2607f2011-07-28 11:39:22 +10004578 if (s.handle_bad_blocks)
4579 for (i = disks; i--; ) {
NeilBrown3cb03002011-10-11 16:45:26 +11004580 struct md_rdev *rdev;
NeilBrownbc2607f2011-07-28 11:39:22 +10004581 struct r5dev *dev = &sh->dev[i];
4582 if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
4583 /* We own a safe reference to the rdev */
4584 rdev = conf->disks[i].rdev;
4585 if (!rdev_set_badblocks(rdev, sh->sector,
4586 STRIPE_SECTORS, 0))
4587 md_error(conf->mddev, rdev);
4588 rdev_dec_pending(rdev, conf->mddev);
4589 }
NeilBrownb84db562011-07-28 11:39:23 +10004590 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
4591 rdev = conf->disks[i].rdev;
4592 rdev_clear_badblocks(rdev, sh->sector,
NeilBrownc6563a82012-05-21 09:27:00 +10004593 STRIPE_SECTORS, 0);
NeilBrownb84db562011-07-28 11:39:23 +10004594 rdev_dec_pending(rdev, conf->mddev);
4595 }
NeilBrown977df362011-12-23 10:17:53 +11004596 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
4597 rdev = conf->disks[i].replacement;
NeilBrowndd054fc2011-12-23 10:17:53 +11004598 if (!rdev)
4599 /* rdev have been moved down */
4600 rdev = conf->disks[i].rdev;
NeilBrown977df362011-12-23 10:17:53 +11004601 rdev_clear_badblocks(rdev, sh->sector,
NeilBrownc6563a82012-05-21 09:27:00 +10004602 STRIPE_SECTORS, 0);
NeilBrown977df362011-12-23 10:17:53 +11004603 rdev_dec_pending(rdev, conf->mddev);
4604 }
NeilBrownbc2607f2011-07-28 11:39:22 +10004605 }
4606
Yuri Tikhonov6c0069c2009-08-29 19:13:13 -07004607 if (s.ops_request)
4608 raid_run_ops(sh, s.ops_request);
4609
Dan Williamsf0e43bc2008-06-28 08:31:55 +10004610 ops_run_io(sh, &s);
4611
NeilBrownc5709ef2011-07-26 11:35:20 +10004612 if (s.dec_preread_active) {
NeilBrown729a1862009-12-14 12:49:50 +11004613 /* We delay this until after ops_run_io so that if make_request
Tejun Heoe9c74692010-09-03 11:56:18 +02004614 * is waiting on a flush, it won't continue until the writes
NeilBrown729a1862009-12-14 12:49:50 +11004615 * have actually been submitted.
4616 */
4617 atomic_dec(&conf->preread_active_stripes);
4618 if (atomic_read(&conf->preread_active_stripes) <
4619 IO_THRESHOLD)
4620 md_wakeup_thread(conf->mddev->thread);
4621 }
4622
NeilBrownc3cce6c2015-08-14 12:47:33 +10004623 if (!bio_list_empty(&s.return_bi)) {
4624 if (test_bit(MD_CHANGE_PENDING, &conf->mddev->flags)) {
4625 spin_lock_irq(&conf->device_lock);
4626 bio_list_merge(&conf->return_bi, &s.return_bi);
4627 spin_unlock_irq(&conf->device_lock);
4628 md_wakeup_thread(conf->mddev->thread);
4629 } else
4630 return_io(&s.return_bi);
4631 }
NeilBrown16a53ec2006-06-26 00:27:38 -07004632
Dan Williams257a4b42011-11-08 16:22:06 +11004633 clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
NeilBrown16a53ec2006-06-26 00:27:38 -07004634}
4635
NeilBrownd1688a62011-10-11 16:49:52 +11004636static void raid5_activate_delayed(struct r5conf *conf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004637{
4638 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
4639 while (!list_empty(&conf->delayed_list)) {
4640 struct list_head *l = conf->delayed_list.next;
4641 struct stripe_head *sh;
4642 sh = list_entry(l, struct stripe_head, lru);
4643 list_del_init(l);
4644 clear_bit(STRIPE_DELAYED, &sh->state);
4645 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4646 atomic_inc(&conf->preread_active_stripes);
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004647 list_add_tail(&sh->lru, &conf->hold_list);
Shaohua Li851c30c2013-08-28 14:30:16 +08004648 raid5_wakeup_stripe_thread(sh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004649 }
NeilBrown482c0832011-04-18 18:25:42 +10004650 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004651}
4652
Shaohua Li566c09c2013-11-14 15:16:17 +11004653static void activate_bit_delay(struct r5conf *conf,
4654 struct list_head *temp_inactive_list)
NeilBrown72626682005-09-09 16:23:54 -07004655{
4656 /* device_lock is held */
4657 struct list_head head;
4658 list_add(&head, &conf->bitmap_list);
4659 list_del_init(&conf->bitmap_list);
4660 while (!list_empty(&head)) {
4661 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
Shaohua Li566c09c2013-11-14 15:16:17 +11004662 int hash;
NeilBrown72626682005-09-09 16:23:54 -07004663 list_del_init(&sh->lru);
4664 atomic_inc(&sh->count);
Shaohua Li566c09c2013-11-14 15:16:17 +11004665 hash = sh->hash_lock_index;
4666 __release_stripe(conf, sh, &temp_inactive_list[hash]);
NeilBrown72626682005-09-09 16:23:54 -07004667 }
4668}
4669
NeilBrown5c675f82014-12-15 12:56:56 +11004670static int raid5_congested(struct mddev *mddev, int bits)
NeilBrownf022b2f2006-10-03 01:15:56 -07004671{
NeilBrownd1688a62011-10-11 16:49:52 +11004672 struct r5conf *conf = mddev->private;
NeilBrownf022b2f2006-10-03 01:15:56 -07004673
4674 /* No difference between reads and writes. Just check
4675 * how busy the stripe_cache is
4676 */
NeilBrown3fa841d2009-09-23 18:10:29 +10004677
NeilBrown54233992015-02-26 12:21:04 +11004678 if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
NeilBrownf022b2f2006-10-03 01:15:56 -07004679 return 1;
4680 if (conf->quiesce)
4681 return 1;
Shaohua Li4bda5562013-11-14 15:16:17 +11004682 if (atomic_read(&conf->empty_inactive_list_nr))
NeilBrownf022b2f2006-10-03 01:15:56 -07004683 return 1;
4684
4685 return 0;
4686}
4687
NeilBrownfd01b882011-10-11 16:47:53 +11004688static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004689{
NeilBrown3cb5edf2015-07-15 17:24:17 +10004690 struct r5conf *conf = mddev->private;
Kent Overstreet4f024f32013-10-11 15:44:27 -07004691 sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
NeilBrown3cb5edf2015-07-15 17:24:17 +10004692 unsigned int chunk_sectors;
Kent Overstreetaa8b57a2013-02-05 15:19:29 -08004693 unsigned int bio_sectors = bio_sectors(bio);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004694
NeilBrown3cb5edf2015-07-15 17:24:17 +10004695 chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004696 return chunk_sectors >=
4697 ((sector & (chunk_sectors - 1)) + bio_sectors);
4698}
4699
4700/*
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004701 * add bio to the retry LIFO ( in O(1) ... we are in interrupt )
4702 * later sampled by raid5d.
4703 */
NeilBrownd1688a62011-10-11 16:49:52 +11004704static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004705{
4706 unsigned long flags;
4707
4708 spin_lock_irqsave(&conf->device_lock, flags);
4709
4710 bi->bi_next = conf->retry_read_aligned_list;
4711 conf->retry_read_aligned_list = bi;
4712
4713 spin_unlock_irqrestore(&conf->device_lock, flags);
4714 md_wakeup_thread(conf->mddev->thread);
4715}
4716
NeilBrownd1688a62011-10-11 16:49:52 +11004717static struct bio *remove_bio_from_retry(struct r5conf *conf)
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004718{
4719 struct bio *bi;
4720
4721 bi = conf->retry_read_aligned;
4722 if (bi) {
4723 conf->retry_read_aligned = NULL;
4724 return bi;
4725 }
4726 bi = conf->retry_read_aligned_list;
4727 if(bi) {
Neil Brown387bb172007-02-08 14:20:29 -08004728 conf->retry_read_aligned_list = bi->bi_next;
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004729 bi->bi_next = NULL;
Jens Axboe960e7392008-08-15 10:41:18 +02004730 /*
4731 * this sets the active strip count to 1 and the processed
4732 * strip count to zero (upper 8 bits)
4733 */
Shaohua Lie7836bd62012-07-19 16:01:31 +10004734 raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004735 }
4736
4737 return bi;
4738}
4739
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004740/*
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004741 * The "raid5_align_endio" should check if the read succeeded and if it
4742 * did, call bio_endio on the original bio (having bio_put the new bio
4743 * first).
4744 * If the read failed..
4745 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02004746static void raid5_align_endio(struct bio *bi)
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004747{
4748 struct bio* raid_bi = bi->bi_private;
NeilBrownfd01b882011-10-11 16:47:53 +11004749 struct mddev *mddev;
NeilBrownd1688a62011-10-11 16:49:52 +11004750 struct r5conf *conf;
NeilBrown3cb03002011-10-11 16:45:26 +11004751 struct md_rdev *rdev;
Sasha Levin9b81c842015-08-10 19:05:18 -04004752 int error = bi->bi_error;
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004753
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004754 bio_put(bi);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004755
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004756 rdev = (void*)raid_bi->bi_next;
4757 raid_bi->bi_next = NULL;
NeilBrown2b7f2222010-03-25 16:06:03 +11004758 mddev = rdev->mddev;
4759 conf = mddev->private;
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004760
4761 rdev_dec_pending(rdev, conf->mddev);
4762
Sasha Levin9b81c842015-08-10 19:05:18 -04004763 if (!error) {
Linus Torvalds0a82a8d2013-04-18 09:00:26 -07004764 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
4765 raid_bi, 0);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02004766 bio_endio(raid_bi);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004767 if (atomic_dec_and_test(&conf->active_aligned_reads))
Yuanhan Liub1b46482015-05-08 18:19:06 +10004768 wake_up(&conf->wait_for_quiescent);
NeilBrown6712ecf2007-09-27 12:47:43 +02004769 return;
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004770 }
4771
Dan Williams45b42332007-07-09 11:56:43 -07004772 pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004773
4774 add_bio_to_retry(raid_bi, conf);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004775}
4776
Ming Lin7ef6b122015-05-06 22:51:24 -07004777static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004778{
NeilBrownd1688a62011-10-11 16:49:52 +11004779 struct r5conf *conf = mddev->private;
NeilBrown8553fe7ec2009-12-14 12:49:47 +11004780 int dd_idx;
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004781 struct bio* align_bi;
NeilBrown3cb03002011-10-11 16:45:26 +11004782 struct md_rdev *rdev;
NeilBrown671488c2011-12-23 10:17:52 +11004783 sector_t end_sector;
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004784
4785 if (!in_chunk_boundary(mddev, raid_bio)) {
Ming Lin7ef6b122015-05-06 22:51:24 -07004786 pr_debug("%s: non aligned\n", __func__);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004787 return 0;
4788 }
4789 /*
NeilBrowna167f662010-10-26 18:31:13 +11004790 * use bio_clone_mddev to make a copy of the bio
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004791 */
NeilBrowna167f662010-10-26 18:31:13 +11004792 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004793 if (!align_bi)
4794 return 0;
4795 /*
4796 * set bi_end_io to a new function, and set bi_private to the
4797 * original bio.
4798 */
4799 align_bi->bi_end_io = raid5_align_endio;
4800 align_bi->bi_private = raid_bio;
4801 /*
4802 * compute position
4803 */
Kent Overstreet4f024f32013-10-11 15:44:27 -07004804 align_bi->bi_iter.bi_sector =
4805 raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
4806 0, &dd_idx, NULL);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004807
Kent Overstreetf73a1c72012-09-25 15:05:12 -07004808 end_sector = bio_end_sector(align_bi);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004809 rcu_read_lock();
NeilBrown671488c2011-12-23 10:17:52 +11004810 rdev = rcu_dereference(conf->disks[dd_idx].replacement);
4811 if (!rdev || test_bit(Faulty, &rdev->flags) ||
4812 rdev->recovery_offset < end_sector) {
4813 rdev = rcu_dereference(conf->disks[dd_idx].rdev);
4814 if (rdev &&
4815 (test_bit(Faulty, &rdev->flags) ||
4816 !(test_bit(In_sync, &rdev->flags) ||
4817 rdev->recovery_offset >= end_sector)))
4818 rdev = NULL;
4819 }
4820 if (rdev) {
NeilBrown31c176e2011-07-28 11:39:22 +10004821 sector_t first_bad;
4822 int bad_sectors;
4823
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004824 atomic_inc(&rdev->nr_pending);
4825 rcu_read_unlock();
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004826 raid_bio->bi_next = (void*)rdev;
4827 align_bi->bi_bdev = rdev->bdev;
Jens Axboeb7c44ed2015-07-24 12:37:59 -06004828 bio_clear_flag(align_bi, BIO_SEG_VALID);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004829
Kent Overstreet7140aaf2013-09-25 13:37:01 -07004830 if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
Kent Overstreet4f024f32013-10-11 15:44:27 -07004831 bio_sectors(align_bi),
NeilBrown31c176e2011-07-28 11:39:22 +10004832 &first_bad, &bad_sectors)) {
Neil Brown387bb172007-02-08 14:20:29 -08004833 bio_put(align_bi);
4834 rdev_dec_pending(rdev, mddev);
4835 return 0;
4836 }
4837
majianpeng6c0544e2012-06-12 08:31:10 +08004838 /* No reshape active, so we can trust rdev->data_offset */
Kent Overstreet4f024f32013-10-11 15:44:27 -07004839 align_bi->bi_iter.bi_sector += rdev->data_offset;
majianpeng6c0544e2012-06-12 08:31:10 +08004840
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004841 spin_lock_irq(&conf->device_lock);
Yuanhan Liub1b46482015-05-08 18:19:06 +10004842 wait_event_lock_irq(conf->wait_for_quiescent,
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004843 conf->quiesce == 0,
Lukas Czernereed8c022012-11-30 11:42:40 +01004844 conf->device_lock);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004845 atomic_inc(&conf->active_aligned_reads);
4846 spin_unlock_irq(&conf->device_lock);
4847
Jonathan Brassowe3620a32013-03-07 16:22:01 -06004848 if (mddev->gendisk)
4849 trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
4850 align_bi, disk_devt(mddev->gendisk),
Kent Overstreet4f024f32013-10-11 15:44:27 -07004851 raid_bio->bi_iter.bi_sector);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004852 generic_make_request(align_bi);
4853 return 1;
4854 } else {
4855 rcu_read_unlock();
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08004856 bio_put(align_bi);
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004857 return 0;
4858 }
4859}
4860
Ming Lin7ef6b122015-05-06 22:51:24 -07004861static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio)
4862{
4863 struct bio *split;
4864
4865 do {
4866 sector_t sector = raid_bio->bi_iter.bi_sector;
4867 unsigned chunk_sects = mddev->chunk_sectors;
4868 unsigned sectors = chunk_sects - (sector & (chunk_sects-1));
4869
4870 if (sectors < bio_sectors(raid_bio)) {
4871 split = bio_split(raid_bio, sectors, GFP_NOIO, fs_bio_set);
4872 bio_chain(split, raid_bio);
4873 } else
4874 split = raid_bio;
4875
4876 if (!raid5_read_one_chunk(mddev, split)) {
4877 if (split != raid_bio)
4878 generic_make_request(raid_bio);
4879 return split;
4880 }
4881 } while (split != raid_bio);
4882
4883 return NULL;
4884}
4885
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004886/* __get_priority_stripe - get the next stripe to process
4887 *
4888 * Full stripe writes are allowed to pass preread active stripes up until
4889 * the bypass_threshold is exceeded. In general the bypass_count
4890 * increments when the handle_list is handled before the hold_list; however, it
4891 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
4892 * stripe with in flight i/o. The bypass_count will be reset when the
4893 * head of the hold_list has changed, i.e. the head was promoted to the
4894 * handle_list.
4895 */
Shaohua Li851c30c2013-08-28 14:30:16 +08004896static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004897{
Shaohua Li851c30c2013-08-28 14:30:16 +08004898 struct stripe_head *sh = NULL, *tmp;
4899 struct list_head *handle_list = NULL;
Shaohua Libfc90cb2013-08-29 15:40:32 +08004900 struct r5worker_group *wg = NULL;
Shaohua Li851c30c2013-08-28 14:30:16 +08004901
4902 if (conf->worker_cnt_per_group == 0) {
4903 handle_list = &conf->handle_list;
4904 } else if (group != ANY_GROUP) {
4905 handle_list = &conf->worker_groups[group].handle_list;
Shaohua Libfc90cb2013-08-29 15:40:32 +08004906 wg = &conf->worker_groups[group];
Shaohua Li851c30c2013-08-28 14:30:16 +08004907 } else {
4908 int i;
4909 for (i = 0; i < conf->group_cnt; i++) {
4910 handle_list = &conf->worker_groups[i].handle_list;
Shaohua Libfc90cb2013-08-29 15:40:32 +08004911 wg = &conf->worker_groups[i];
Shaohua Li851c30c2013-08-28 14:30:16 +08004912 if (!list_empty(handle_list))
4913 break;
4914 }
4915 }
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004916
4917 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
4918 __func__,
Shaohua Li851c30c2013-08-28 14:30:16 +08004919 list_empty(handle_list) ? "empty" : "busy",
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004920 list_empty(&conf->hold_list) ? "empty" : "busy",
4921 atomic_read(&conf->pending_full_writes), conf->bypass_count);
4922
Shaohua Li851c30c2013-08-28 14:30:16 +08004923 if (!list_empty(handle_list)) {
4924 sh = list_entry(handle_list->next, typeof(*sh), lru);
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004925
4926 if (list_empty(&conf->hold_list))
4927 conf->bypass_count = 0;
4928 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
4929 if (conf->hold_list.next == conf->last_hold)
4930 conf->bypass_count++;
4931 else {
4932 conf->last_hold = conf->hold_list.next;
4933 conf->bypass_count -= conf->bypass_threshold;
4934 if (conf->bypass_count < 0)
4935 conf->bypass_count = 0;
4936 }
4937 }
4938 } else if (!list_empty(&conf->hold_list) &&
4939 ((conf->bypass_threshold &&
4940 conf->bypass_count > conf->bypass_threshold) ||
4941 atomic_read(&conf->pending_full_writes) == 0)) {
Shaohua Li851c30c2013-08-28 14:30:16 +08004942
4943 list_for_each_entry(tmp, &conf->hold_list, lru) {
4944 if (conf->worker_cnt_per_group == 0 ||
4945 group == ANY_GROUP ||
4946 !cpu_online(tmp->cpu) ||
4947 cpu_to_group(tmp->cpu) == group) {
4948 sh = tmp;
4949 break;
4950 }
4951 }
4952
4953 if (sh) {
4954 conf->bypass_count -= conf->bypass_threshold;
4955 if (conf->bypass_count < 0)
4956 conf->bypass_count = 0;
4957 }
Shaohua Libfc90cb2013-08-29 15:40:32 +08004958 wg = NULL;
Shaohua Li851c30c2013-08-28 14:30:16 +08004959 }
4960
4961 if (!sh)
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004962 return NULL;
4963
Shaohua Libfc90cb2013-08-29 15:40:32 +08004964 if (wg) {
4965 wg->stripes_cnt--;
4966 sh->group = NULL;
4967 }
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004968 list_del_init(&sh->lru);
Shaohua Lic7a6d352014-04-15 09:12:54 +08004969 BUG_ON(atomic_inc_return(&sh->count) != 1);
Dan Williams8b3e6cd2008-04-28 02:15:53 -07004970 return sh;
4971}
Raz Ben-Jehuda(caro)f6796232006-12-10 02:20:46 -08004972
Shaohua Li8811b592012-08-02 08:33:00 +10004973struct raid5_plug_cb {
4974 struct blk_plug_cb cb;
4975 struct list_head list;
Shaohua Li566c09c2013-11-14 15:16:17 +11004976 struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
Shaohua Li8811b592012-08-02 08:33:00 +10004977};
4978
4979static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
4980{
4981 struct raid5_plug_cb *cb = container_of(
4982 blk_cb, struct raid5_plug_cb, cb);
4983 struct stripe_head *sh;
4984 struct mddev *mddev = cb->cb.data;
4985 struct r5conf *conf = mddev->private;
NeilBrowna9add5d2012-10-31 11:59:09 +11004986 int cnt = 0;
Shaohua Li566c09c2013-11-14 15:16:17 +11004987 int hash;
Shaohua Li8811b592012-08-02 08:33:00 +10004988
4989 if (cb->list.next && !list_empty(&cb->list)) {
4990 spin_lock_irq(&conf->device_lock);
4991 while (!list_empty(&cb->list)) {
4992 sh = list_first_entry(&cb->list, struct stripe_head, lru);
4993 list_del_init(&sh->lru);
4994 /*
4995 * avoid race release_stripe_plug() sees
4996 * STRIPE_ON_UNPLUG_LIST clear but the stripe
4997 * is still in our list
4998 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01004999 smp_mb__before_atomic();
Shaohua Li8811b592012-08-02 08:33:00 +10005000 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
Shaohua Li773ca822013-08-27 17:50:39 +08005001 /*
5002 * STRIPE_ON_RELEASE_LIST could be set here. In that
5003 * case, the count is always > 1 here
5004 */
Shaohua Li566c09c2013-11-14 15:16:17 +11005005 hash = sh->hash_lock_index;
5006 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
NeilBrowna9add5d2012-10-31 11:59:09 +11005007 cnt++;
Shaohua Li8811b592012-08-02 08:33:00 +10005008 }
5009 spin_unlock_irq(&conf->device_lock);
5010 }
Shaohua Li566c09c2013-11-14 15:16:17 +11005011 release_inactive_stripe_list(conf, cb->temp_inactive_list,
5012 NR_STRIPE_HASH_LOCKS);
Jonathan Brassowe3620a32013-03-07 16:22:01 -06005013 if (mddev->queue)
5014 trace_block_unplug(mddev->queue, cnt, !from_schedule);
Shaohua Li8811b592012-08-02 08:33:00 +10005015 kfree(cb);
5016}
5017
5018static void release_stripe_plug(struct mddev *mddev,
5019 struct stripe_head *sh)
5020{
5021 struct blk_plug_cb *blk_cb = blk_check_plugged(
5022 raid5_unplug, mddev,
5023 sizeof(struct raid5_plug_cb));
5024 struct raid5_plug_cb *cb;
5025
5026 if (!blk_cb) {
Shaohua Li6d036f72015-08-13 14:31:57 -07005027 raid5_release_stripe(sh);
Shaohua Li8811b592012-08-02 08:33:00 +10005028 return;
5029 }
5030
5031 cb = container_of(blk_cb, struct raid5_plug_cb, cb);
5032
Shaohua Li566c09c2013-11-14 15:16:17 +11005033 if (cb->list.next == NULL) {
5034 int i;
Shaohua Li8811b592012-08-02 08:33:00 +10005035 INIT_LIST_HEAD(&cb->list);
Shaohua Li566c09c2013-11-14 15:16:17 +11005036 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
5037 INIT_LIST_HEAD(cb->temp_inactive_list + i);
5038 }
Shaohua Li8811b592012-08-02 08:33:00 +10005039
5040 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
5041 list_add_tail(&sh->lru, &cb->list);
5042 else
Shaohua Li6d036f72015-08-13 14:31:57 -07005043 raid5_release_stripe(sh);
Shaohua Li8811b592012-08-02 08:33:00 +10005044}
5045
Shaohua Li620125f2012-10-11 13:49:05 +11005046static void make_discard_request(struct mddev *mddev, struct bio *bi)
5047{
5048 struct r5conf *conf = mddev->private;
5049 sector_t logical_sector, last_sector;
5050 struct stripe_head *sh;
5051 int remaining;
5052 int stripe_sectors;
5053
5054 if (mddev->reshape_position != MaxSector)
5055 /* Skip discard while reshape is happening */
5056 return;
5057
Kent Overstreet4f024f32013-10-11 15:44:27 -07005058 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
5059 last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
Shaohua Li620125f2012-10-11 13:49:05 +11005060
5061 bi->bi_next = NULL;
5062 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
5063
5064 stripe_sectors = conf->chunk_sectors *
5065 (conf->raid_disks - conf->max_degraded);
5066 logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
5067 stripe_sectors);
5068 sector_div(last_sector, stripe_sectors);
5069
5070 logical_sector *= conf->chunk_sectors;
5071 last_sector *= conf->chunk_sectors;
5072
5073 for (; logical_sector < last_sector;
5074 logical_sector += STRIPE_SECTORS) {
5075 DEFINE_WAIT(w);
5076 int d;
5077 again:
Shaohua Li6d036f72015-08-13 14:31:57 -07005078 sh = raid5_get_active_stripe(conf, logical_sector, 0, 0, 0);
Shaohua Li620125f2012-10-11 13:49:05 +11005079 prepare_to_wait(&conf->wait_for_overlap, &w,
5080 TASK_UNINTERRUPTIBLE);
NeilBrownf8dfcff2013-03-12 12:18:06 +11005081 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
5082 if (test_bit(STRIPE_SYNCING, &sh->state)) {
Shaohua Li6d036f72015-08-13 14:31:57 -07005083 raid5_release_stripe(sh);
NeilBrownf8dfcff2013-03-12 12:18:06 +11005084 schedule();
5085 goto again;
5086 }
5087 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
Shaohua Li620125f2012-10-11 13:49:05 +11005088 spin_lock_irq(&sh->stripe_lock);
5089 for (d = 0; d < conf->raid_disks; d++) {
5090 if (d == sh->pd_idx || d == sh->qd_idx)
5091 continue;
5092 if (sh->dev[d].towrite || sh->dev[d].toread) {
5093 set_bit(R5_Overlap, &sh->dev[d].flags);
5094 spin_unlock_irq(&sh->stripe_lock);
Shaohua Li6d036f72015-08-13 14:31:57 -07005095 raid5_release_stripe(sh);
Shaohua Li620125f2012-10-11 13:49:05 +11005096 schedule();
5097 goto again;
5098 }
5099 }
NeilBrownf8dfcff2013-03-12 12:18:06 +11005100 set_bit(STRIPE_DISCARD, &sh->state);
Shaohua Li620125f2012-10-11 13:49:05 +11005101 finish_wait(&conf->wait_for_overlap, &w);
shli@kernel.org7a87f432014-12-15 12:57:03 +11005102 sh->overwrite_disks = 0;
Shaohua Li620125f2012-10-11 13:49:05 +11005103 for (d = 0; d < conf->raid_disks; d++) {
5104 if (d == sh->pd_idx || d == sh->qd_idx)
5105 continue;
5106 sh->dev[d].towrite = bi;
5107 set_bit(R5_OVERWRITE, &sh->dev[d].flags);
5108 raid5_inc_bi_active_stripes(bi);
shli@kernel.org7a87f432014-12-15 12:57:03 +11005109 sh->overwrite_disks++;
Shaohua Li620125f2012-10-11 13:49:05 +11005110 }
5111 spin_unlock_irq(&sh->stripe_lock);
5112 if (conf->mddev->bitmap) {
5113 for (d = 0;
5114 d < conf->raid_disks - conf->max_degraded;
5115 d++)
5116 bitmap_startwrite(mddev->bitmap,
5117 sh->sector,
5118 STRIPE_SECTORS,
5119 0);
5120 sh->bm_seq = conf->seq_flush + 1;
5121 set_bit(STRIPE_BIT_DELAY, &sh->state);
5122 }
5123
5124 set_bit(STRIPE_HANDLE, &sh->state);
5125 clear_bit(STRIPE_DELAYED, &sh->state);
5126 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5127 atomic_inc(&conf->preread_active_stripes);
5128 release_stripe_plug(mddev, sh);
5129 }
5130
5131 remaining = raid5_dec_bi_active_stripes(bi);
5132 if (remaining == 0) {
5133 md_write_end(mddev);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02005134 bio_endio(bi);
Shaohua Li620125f2012-10-11 13:49:05 +11005135 }
5136}
5137
Linus Torvaldsb4fdcb02011-11-04 17:06:58 -07005138static void make_request(struct mddev *mddev, struct bio * bi)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005139{
NeilBrownd1688a62011-10-11 16:49:52 +11005140 struct r5conf *conf = mddev->private;
NeilBrown911d4ee2009-03-31 14:39:38 +11005141 int dd_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005142 sector_t new_sector;
5143 sector_t logical_sector, last_sector;
5144 struct stripe_head *sh;
Jens Axboea3623572005-11-01 09:26:16 +01005145 const int rw = bio_data_dir(bi);
NeilBrown49077322010-03-25 16:20:56 +11005146 int remaining;
Shaohua Li27c0f682014-04-09 11:25:47 +08005147 DEFINE_WAIT(w);
5148 bool do_prepare;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005149
Tejun Heoe9c74692010-09-03 11:56:18 +02005150 if (unlikely(bi->bi_rw & REQ_FLUSH)) {
Shaohua Li828cbe92015-09-02 13:49:49 -07005151 int ret = r5l_handle_flush_request(conf->log, bi);
5152
5153 if (ret == 0)
5154 return;
5155 if (ret == -ENODEV) {
5156 md_flush_request(mddev, bi);
5157 return;
5158 }
5159 /* ret == -EAGAIN, fallback */
NeilBrowne5dcdd82005-09-09 16:23:41 -07005160 }
5161
NeilBrown3d310eb2005-06-21 17:17:26 -07005162 md_write_start(mddev, bi);
NeilBrown06d91a52005-06-21 17:17:12 -07005163
Eric Mei9ffc8f72015-03-18 23:39:11 -06005164 /*
5165 * If array is degraded, better not do chunk aligned read because
5166 * later we might have to read it again in order to reconstruct
5167 * data on failed drives.
5168 */
5169 if (rw == READ && mddev->degraded == 0 &&
Ming Lin7ef6b122015-05-06 22:51:24 -07005170 mddev->reshape_position == MaxSector) {
5171 bi = chunk_aligned_read(mddev, bi);
5172 if (!bi)
5173 return;
5174 }
Raz Ben-Jehuda(caro)52488612006-12-10 02:20:48 -08005175
Shaohua Li620125f2012-10-11 13:49:05 +11005176 if (unlikely(bi->bi_rw & REQ_DISCARD)) {
5177 make_discard_request(mddev, bi);
5178 return;
5179 }
5180
Kent Overstreet4f024f32013-10-11 15:44:27 -07005181 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
Kent Overstreetf73a1c72012-09-25 15:05:12 -07005182 last_sector = bio_end_sector(bi);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005183 bi->bi_next = NULL;
5184 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
NeilBrown06d91a52005-06-21 17:17:12 -07005185
Shaohua Li27c0f682014-04-09 11:25:47 +08005186 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005187 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
NeilBrownb5663ba2009-03-31 14:39:38 +11005188 int previous;
NeilBrownc46501b2013-08-27 15:52:13 +10005189 int seq;
NeilBrownb578d552006-03-27 01:18:12 -08005190
Shaohua Li27c0f682014-04-09 11:25:47 +08005191 do_prepare = false;
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005192 retry:
NeilBrownc46501b2013-08-27 15:52:13 +10005193 seq = read_seqcount_begin(&conf->gen_lock);
NeilBrownb5663ba2009-03-31 14:39:38 +11005194 previous = 0;
Shaohua Li27c0f682014-04-09 11:25:47 +08005195 if (do_prepare)
5196 prepare_to_wait(&conf->wait_for_overlap, &w,
5197 TASK_UNINTERRUPTIBLE);
NeilBrownb0f9ec02009-03-31 15:27:18 +11005198 if (unlikely(conf->reshape_progress != MaxSector)) {
NeilBrownfef9c612009-03-31 15:16:46 +11005199 /* spinlock is needed as reshape_progress may be
NeilBrowndf8e7f72006-03-27 01:18:15 -08005200 * 64bit on a 32bit platform, and so it might be
5201 * possible to see a half-updated value
Jesper Juhlaeb878b2011-04-10 18:06:17 +02005202 * Of course reshape_progress could change after
NeilBrowndf8e7f72006-03-27 01:18:15 -08005203 * the lock is dropped, so once we get a reference
5204 * to the stripe that we think it is, we will have
5205 * to check again.
5206 */
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005207 spin_lock_irq(&conf->device_lock);
NeilBrown2c810cd2012-05-21 09:27:00 +10005208 if (mddev->reshape_backwards
NeilBrownfef9c612009-03-31 15:16:46 +11005209 ? logical_sector < conf->reshape_progress
5210 : logical_sector >= conf->reshape_progress) {
NeilBrownb5663ba2009-03-31 14:39:38 +11005211 previous = 1;
5212 } else {
NeilBrown2c810cd2012-05-21 09:27:00 +10005213 if (mddev->reshape_backwards
NeilBrownfef9c612009-03-31 15:16:46 +11005214 ? logical_sector < conf->reshape_safe
5215 : logical_sector >= conf->reshape_safe) {
NeilBrownb578d552006-03-27 01:18:12 -08005216 spin_unlock_irq(&conf->device_lock);
5217 schedule();
Shaohua Li27c0f682014-04-09 11:25:47 +08005218 do_prepare = true;
NeilBrownb578d552006-03-27 01:18:12 -08005219 goto retry;
5220 }
5221 }
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005222 spin_unlock_irq(&conf->device_lock);
5223 }
NeilBrown16a53ec2006-06-26 00:27:38 -07005224
NeilBrown112bf892009-03-31 14:39:38 +11005225 new_sector = raid5_compute_sector(conf, logical_sector,
5226 previous,
NeilBrown911d4ee2009-03-31 14:39:38 +11005227 &dd_idx, NULL);
NeilBrown0c55e022010-05-03 14:09:02 +10005228 pr_debug("raid456: make_request, sector %llu logical %llu\n",
NeilBrownc46501b2013-08-27 15:52:13 +10005229 (unsigned long long)new_sector,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005230 (unsigned long long)logical_sector);
5231
Shaohua Li6d036f72015-08-13 14:31:57 -07005232 sh = raid5_get_active_stripe(conf, new_sector, previous,
NeilBrowna8c906c2009-06-09 14:39:59 +10005233 (bi->bi_rw&RWA_MASK), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005234 if (sh) {
NeilBrownb0f9ec02009-03-31 15:27:18 +11005235 if (unlikely(previous)) {
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005236 /* expansion might have moved on while waiting for a
NeilBrowndf8e7f72006-03-27 01:18:15 -08005237 * stripe, so we must do the range check again.
5238 * Expansion could still move past after this
5239 * test, but as we are holding a reference to
5240 * 'sh', we know that if that happens,
5241 * STRIPE_EXPANDING will get set and the expansion
5242 * won't proceed until we finish with the stripe.
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005243 */
5244 int must_retry = 0;
5245 spin_lock_irq(&conf->device_lock);
NeilBrown2c810cd2012-05-21 09:27:00 +10005246 if (mddev->reshape_backwards
NeilBrownb0f9ec02009-03-31 15:27:18 +11005247 ? logical_sector >= conf->reshape_progress
5248 : logical_sector < conf->reshape_progress)
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005249 /* mismatch, need to try again */
5250 must_retry = 1;
5251 spin_unlock_irq(&conf->device_lock);
5252 if (must_retry) {
Shaohua Li6d036f72015-08-13 14:31:57 -07005253 raid5_release_stripe(sh);
Dan Williams7a3ab902009-06-16 16:00:33 -07005254 schedule();
Shaohua Li27c0f682014-04-09 11:25:47 +08005255 do_prepare = true;
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005256 goto retry;
5257 }
5258 }
NeilBrownc46501b2013-08-27 15:52:13 +10005259 if (read_seqcount_retry(&conf->gen_lock, seq)) {
5260 /* Might have got the wrong stripe_head
5261 * by accident
5262 */
Shaohua Li6d036f72015-08-13 14:31:57 -07005263 raid5_release_stripe(sh);
NeilBrownc46501b2013-08-27 15:52:13 +10005264 goto retry;
5265 }
NeilBrowne62e58a2009-07-01 13:15:35 +10005266
Namhyung Kimffd96e32011-07-18 17:38:51 +10005267 if (rw == WRITE &&
NeilBrowna5c308d2009-07-01 13:15:35 +10005268 logical_sector >= mddev->suspend_lo &&
NeilBrowne464eaf2006-03-27 01:18:14 -08005269 logical_sector < mddev->suspend_hi) {
Shaohua Li6d036f72015-08-13 14:31:57 -07005270 raid5_release_stripe(sh);
NeilBrowne62e58a2009-07-01 13:15:35 +10005271 /* As the suspend_* range is controlled by
5272 * userspace, we want an interruptible
5273 * wait.
5274 */
5275 flush_signals(current);
5276 prepare_to_wait(&conf->wait_for_overlap,
5277 &w, TASK_INTERRUPTIBLE);
5278 if (logical_sector >= mddev->suspend_lo &&
Shaohua Li27c0f682014-04-09 11:25:47 +08005279 logical_sector < mddev->suspend_hi) {
NeilBrowne62e58a2009-07-01 13:15:35 +10005280 schedule();
Shaohua Li27c0f682014-04-09 11:25:47 +08005281 do_prepare = true;
5282 }
NeilBrowne464eaf2006-03-27 01:18:14 -08005283 goto retry;
5284 }
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005285
5286 if (test_bit(STRIPE_EXPANDING, &sh->state) ||
shli@kernel.orgda41ba62014-12-15 12:57:03 +11005287 !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {
NeilBrown7ecaa1e2006-03-27 01:18:08 -08005288 /* Stripe is busy expanding or
5289 * add failed due to overlap. Flush everything
Linus Torvalds1da177e2005-04-16 15:20:36 -07005290 * and wait a while
5291 */
NeilBrown482c0832011-04-18 18:25:42 +10005292 md_wakeup_thread(mddev->thread);
Shaohua Li6d036f72015-08-13 14:31:57 -07005293 raid5_release_stripe(sh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005294 schedule();
Shaohua Li27c0f682014-04-09 11:25:47 +08005295 do_prepare = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005296 goto retry;
5297 }
NeilBrown6ed30032008-02-06 01:40:00 -08005298 set_bit(STRIPE_HANDLE, &sh->state);
5299 clear_bit(STRIPE_DELAYED, &sh->state);
shli@kernel.org59fc6302014-12-15 12:57:03 +11005300 if ((!sh->batch_head || sh == sh->batch_head) &&
5301 (bi->bi_rw & REQ_SYNC) &&
NeilBrown729a1862009-12-14 12:49:50 +11005302 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5303 atomic_inc(&conf->preread_active_stripes);
Shaohua Li8811b592012-08-02 08:33:00 +10005304 release_stripe_plug(mddev, sh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005305 } else {
5306 /* cannot get stripe for read-ahead, just give-up */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02005307 bi->bi_error = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005308 break;
5309 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005310 }
Shaohua Li27c0f682014-04-09 11:25:47 +08005311 finish_wait(&conf->wait_for_overlap, &w);
NeilBrown7c13edc2011-04-18 18:25:43 +10005312
Shaohua Lie7836bd62012-07-19 16:01:31 +10005313 remaining = raid5_dec_bi_active_stripes(bi);
NeilBrownf6344752006-03-27 01:18:17 -08005314 if (remaining == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005315
NeilBrown16a53ec2006-06-26 00:27:38 -07005316 if ( rw == WRITE )
Linus Torvalds1da177e2005-04-16 15:20:36 -07005317 md_write_end(mddev);
NeilBrown6712ecf2007-09-27 12:47:43 +02005318
Linus Torvalds0a82a8d2013-04-18 09:00:26 -07005319 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
5320 bi, 0);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02005321 bio_endio(bi);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005322 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005323}
5324
NeilBrownfd01b882011-10-11 16:47:53 +11005325static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
Dan Williamsb522adc2009-03-31 15:00:31 +11005326
NeilBrownfd01b882011-10-11 16:47:53 +11005327static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005328{
NeilBrown52c03292006-06-26 00:27:43 -07005329 /* reshaping is quite different to recovery/resync so it is
5330 * handled quite separately ... here.
5331 *
5332 * On each call to sync_request, we gather one chunk worth of
5333 * destination stripes and flag them as expanding.
5334 * Then we find all the source stripes and request reads.
5335 * As the reads complete, handle_stripe will copy the data
5336 * into the destination stripe and release that stripe.
5337 */
NeilBrownd1688a62011-10-11 16:49:52 +11005338 struct r5conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005339 struct stripe_head *sh;
NeilBrownccfcc3c2006-03-27 01:18:09 -08005340 sector_t first_sector, last_sector;
NeilBrownf4168852007-02-28 20:11:53 -08005341 int raid_disks = conf->previous_raid_disks;
5342 int data_disks = raid_disks - conf->max_degraded;
5343 int new_data_disks = conf->raid_disks - conf->max_degraded;
NeilBrown52c03292006-06-26 00:27:43 -07005344 int i;
5345 int dd_idx;
NeilBrownc8f517c2009-03-31 15:28:40 +11005346 sector_t writepos, readpos, safepos;
NeilBrownec32a2b2009-03-31 15:17:38 +11005347 sector_t stripe_addr;
NeilBrown7a661382009-03-31 15:21:40 +11005348 int reshape_sectors;
NeilBrownab69ae12009-03-31 15:26:47 +11005349 struct list_head stripes;
NeilBrown92140482015-07-06 12:28:45 +10005350 sector_t retn;
NeilBrown52c03292006-06-26 00:27:43 -07005351
NeilBrownfef9c612009-03-31 15:16:46 +11005352 if (sector_nr == 0) {
5353 /* If restarting in the middle, skip the initial sectors */
NeilBrown2c810cd2012-05-21 09:27:00 +10005354 if (mddev->reshape_backwards &&
NeilBrownfef9c612009-03-31 15:16:46 +11005355 conf->reshape_progress < raid5_size(mddev, 0, 0)) {
5356 sector_nr = raid5_size(mddev, 0, 0)
5357 - conf->reshape_progress;
NeilBrown6cbd8142015-07-24 13:30:32 +10005358 } else if (mddev->reshape_backwards &&
5359 conf->reshape_progress == MaxSector) {
5360 /* shouldn't happen, but just in case, finish up.*/
5361 sector_nr = MaxSector;
NeilBrown2c810cd2012-05-21 09:27:00 +10005362 } else if (!mddev->reshape_backwards &&
NeilBrownfef9c612009-03-31 15:16:46 +11005363 conf->reshape_progress > 0)
5364 sector_nr = conf->reshape_progress;
NeilBrownf4168852007-02-28 20:11:53 -08005365 sector_div(sector_nr, new_data_disks);
NeilBrownfef9c612009-03-31 15:16:46 +11005366 if (sector_nr) {
NeilBrown8dee7212009-11-06 14:59:29 +11005367 mddev->curr_resync_completed = sector_nr;
5368 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
NeilBrownfef9c612009-03-31 15:16:46 +11005369 *skipped = 1;
NeilBrown92140482015-07-06 12:28:45 +10005370 retn = sector_nr;
5371 goto finish;
NeilBrownfef9c612009-03-31 15:16:46 +11005372 }
NeilBrown52c03292006-06-26 00:27:43 -07005373 }
5374
NeilBrown7a661382009-03-31 15:21:40 +11005375 /* We need to process a full chunk at a time.
5376 * If old and new chunk sizes differ, we need to process the
5377 * largest of these
5378 */
NeilBrown3cb5edf2015-07-15 17:24:17 +10005379
5380 reshape_sectors = max(conf->chunk_sectors, conf->prev_chunk_sectors);
NeilBrown7a661382009-03-31 15:21:40 +11005381
NeilBrownb5254dd2012-05-21 09:27:01 +10005382 /* We update the metadata at least every 10 seconds, or when
5383 * the data about to be copied would over-write the source of
5384 * the data at the front of the range. i.e. one new_stripe
5385 * along from reshape_progress new_maps to after where
5386 * reshape_safe old_maps to
NeilBrown52c03292006-06-26 00:27:43 -07005387 */
NeilBrownfef9c612009-03-31 15:16:46 +11005388 writepos = conf->reshape_progress;
NeilBrownf4168852007-02-28 20:11:53 -08005389 sector_div(writepos, new_data_disks);
NeilBrownc8f517c2009-03-31 15:28:40 +11005390 readpos = conf->reshape_progress;
5391 sector_div(readpos, data_disks);
NeilBrownfef9c612009-03-31 15:16:46 +11005392 safepos = conf->reshape_safe;
NeilBrownf4168852007-02-28 20:11:53 -08005393 sector_div(safepos, data_disks);
NeilBrown2c810cd2012-05-21 09:27:00 +10005394 if (mddev->reshape_backwards) {
NeilBrownc74c0d72015-07-15 17:54:15 +10005395 BUG_ON(writepos < reshape_sectors);
5396 writepos -= reshape_sectors;
NeilBrownc8f517c2009-03-31 15:28:40 +11005397 readpos += reshape_sectors;
NeilBrown7a661382009-03-31 15:21:40 +11005398 safepos += reshape_sectors;
NeilBrownfef9c612009-03-31 15:16:46 +11005399 } else {
NeilBrown7a661382009-03-31 15:21:40 +11005400 writepos += reshape_sectors;
NeilBrownc74c0d72015-07-15 17:54:15 +10005401 /* readpos and safepos are worst-case calculations.
5402 * A negative number is overly pessimistic, and causes
5403 * obvious problems for unsigned storage. So clip to 0.
5404 */
NeilBrowned37d832009-05-27 21:39:05 +10005405 readpos -= min_t(sector_t, reshape_sectors, readpos);
5406 safepos -= min_t(sector_t, reshape_sectors, safepos);
NeilBrownfef9c612009-03-31 15:16:46 +11005407 }
NeilBrown52c03292006-06-26 00:27:43 -07005408
NeilBrownb5254dd2012-05-21 09:27:01 +10005409 /* Having calculated the 'writepos' possibly use it
5410 * to set 'stripe_addr' which is where we will write to.
5411 */
5412 if (mddev->reshape_backwards) {
5413 BUG_ON(conf->reshape_progress == 0);
5414 stripe_addr = writepos;
5415 BUG_ON((mddev->dev_sectors &
5416 ~((sector_t)reshape_sectors - 1))
5417 - reshape_sectors - stripe_addr
5418 != sector_nr);
5419 } else {
5420 BUG_ON(writepos != sector_nr + reshape_sectors);
5421 stripe_addr = sector_nr;
5422 }
5423
NeilBrownc8f517c2009-03-31 15:28:40 +11005424 /* 'writepos' is the most advanced device address we might write.
5425 * 'readpos' is the least advanced device address we might read.
5426 * 'safepos' is the least address recorded in the metadata as having
5427 * been reshaped.
NeilBrownb5254dd2012-05-21 09:27:01 +10005428 * If there is a min_offset_diff, these are adjusted either by
5429 * increasing the safepos/readpos if diff is negative, or
5430 * increasing writepos if diff is positive.
5431 * If 'readpos' is then behind 'writepos', there is no way that we can
NeilBrownc8f517c2009-03-31 15:28:40 +11005432 * ensure safety in the face of a crash - that must be done by userspace
5433 * making a backup of the data. So in that case there is no particular
5434 * rush to update metadata.
5435 * Otherwise if 'safepos' is behind 'writepos', then we really need to
5436 * update the metadata to advance 'safepos' to match 'readpos' so that
5437 * we can be safe in the event of a crash.
5438 * So we insist on updating metadata if safepos is behind writepos and
5439 * readpos is beyond writepos.
5440 * In any case, update the metadata every 10 seconds.
5441 * Maybe that number should be configurable, but I'm not sure it is
5442 * worth it.... maybe it could be a multiple of safemode_delay???
5443 */
NeilBrownb5254dd2012-05-21 09:27:01 +10005444 if (conf->min_offset_diff < 0) {
5445 safepos += -conf->min_offset_diff;
5446 readpos += -conf->min_offset_diff;
5447 } else
5448 writepos += conf->min_offset_diff;
5449
NeilBrown2c810cd2012-05-21 09:27:00 +10005450 if ((mddev->reshape_backwards
NeilBrownc8f517c2009-03-31 15:28:40 +11005451 ? (safepos > writepos && readpos < writepos)
5452 : (safepos < writepos && readpos > writepos)) ||
5453 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
NeilBrown52c03292006-06-26 00:27:43 -07005454 /* Cannot proceed until we've updated the superblock... */
5455 wait_event(conf->wait_for_overlap,
NeilBrownc91abf52013-11-19 12:02:01 +11005456 atomic_read(&conf->reshape_stripes)==0
5457 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5458 if (atomic_read(&conf->reshape_stripes) != 0)
5459 return 0;
NeilBrownfef9c612009-03-31 15:16:46 +11005460 mddev->reshape_position = conf->reshape_progress;
NeilBrown75d3da42011-01-14 09:14:34 +11005461 mddev->curr_resync_completed = sector_nr;
NeilBrownc8f517c2009-03-31 15:28:40 +11005462 conf->reshape_checkpoint = jiffies;
NeilBrown850b2b42006-10-03 01:15:46 -07005463 set_bit(MD_CHANGE_DEVS, &mddev->flags);
NeilBrown52c03292006-06-26 00:27:43 -07005464 md_wakeup_thread(mddev->thread);
NeilBrown850b2b42006-10-03 01:15:46 -07005465 wait_event(mddev->sb_wait, mddev->flags == 0 ||
NeilBrownc91abf52013-11-19 12:02:01 +11005466 test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5467 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5468 return 0;
NeilBrown52c03292006-06-26 00:27:43 -07005469 spin_lock_irq(&conf->device_lock);
NeilBrownfef9c612009-03-31 15:16:46 +11005470 conf->reshape_safe = mddev->reshape_position;
NeilBrown52c03292006-06-26 00:27:43 -07005471 spin_unlock_irq(&conf->device_lock);
5472 wake_up(&conf->wait_for_overlap);
NeilBrownacb180b2009-04-14 16:28:34 +10005473 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
NeilBrown52c03292006-06-26 00:27:43 -07005474 }
5475
NeilBrownab69ae12009-03-31 15:26:47 +11005476 INIT_LIST_HEAD(&stripes);
NeilBrown7a661382009-03-31 15:21:40 +11005477 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
NeilBrown52c03292006-06-26 00:27:43 -07005478 int j;
NeilBrowna9f326e2009-09-23 18:06:41 +10005479 int skipped_disk = 0;
Shaohua Li6d036f72015-08-13 14:31:57 -07005480 sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
NeilBrown52c03292006-06-26 00:27:43 -07005481 set_bit(STRIPE_EXPANDING, &sh->state);
5482 atomic_inc(&conf->reshape_stripes);
5483 /* If any of this stripe is beyond the end of the old
5484 * array, then we need to zero those blocks
5485 */
5486 for (j=sh->disks; j--;) {
5487 sector_t s;
5488 if (j == sh->pd_idx)
5489 continue;
NeilBrownf4168852007-02-28 20:11:53 -08005490 if (conf->level == 6 &&
NeilBrownd0dabf72009-03-31 14:39:38 +11005491 j == sh->qd_idx)
NeilBrownf4168852007-02-28 20:11:53 -08005492 continue;
Shaohua Li6d036f72015-08-13 14:31:57 -07005493 s = raid5_compute_blocknr(sh, j, 0);
Dan Williamsb522adc2009-03-31 15:00:31 +11005494 if (s < raid5_size(mddev, 0, 0)) {
NeilBrowna9f326e2009-09-23 18:06:41 +10005495 skipped_disk = 1;
NeilBrown52c03292006-06-26 00:27:43 -07005496 continue;
5497 }
5498 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
5499 set_bit(R5_Expanded, &sh->dev[j].flags);
5500 set_bit(R5_UPTODATE, &sh->dev[j].flags);
5501 }
NeilBrowna9f326e2009-09-23 18:06:41 +10005502 if (!skipped_disk) {
NeilBrown52c03292006-06-26 00:27:43 -07005503 set_bit(STRIPE_EXPAND_READY, &sh->state);
5504 set_bit(STRIPE_HANDLE, &sh->state);
5505 }
NeilBrownab69ae12009-03-31 15:26:47 +11005506 list_add(&sh->lru, &stripes);
NeilBrown52c03292006-06-26 00:27:43 -07005507 }
5508 spin_lock_irq(&conf->device_lock);
NeilBrown2c810cd2012-05-21 09:27:00 +10005509 if (mddev->reshape_backwards)
NeilBrown7a661382009-03-31 15:21:40 +11005510 conf->reshape_progress -= reshape_sectors * new_data_disks;
NeilBrownfef9c612009-03-31 15:16:46 +11005511 else
NeilBrown7a661382009-03-31 15:21:40 +11005512 conf->reshape_progress += reshape_sectors * new_data_disks;
NeilBrown52c03292006-06-26 00:27:43 -07005513 spin_unlock_irq(&conf->device_lock);
5514 /* Ok, those stripe are ready. We can start scheduling
5515 * reads on the source stripes.
5516 * The source stripes are determined by mapping the first and last
5517 * block on the destination stripes.
5518 */
NeilBrown52c03292006-06-26 00:27:43 -07005519 first_sector =
NeilBrownec32a2b2009-03-31 15:17:38 +11005520 raid5_compute_sector(conf, stripe_addr*(new_data_disks),
NeilBrown911d4ee2009-03-31 14:39:38 +11005521 1, &dd_idx, NULL);
NeilBrown52c03292006-06-26 00:27:43 -07005522 last_sector =
NeilBrown0e6e0272009-06-09 16:32:22 +10005523 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
Andre Noll09c9e5f2009-06-18 08:45:55 +10005524 * new_data_disks - 1),
NeilBrown911d4ee2009-03-31 14:39:38 +11005525 1, &dd_idx, NULL);
Andre Noll58c0fed2009-03-31 14:33:13 +11005526 if (last_sector >= mddev->dev_sectors)
5527 last_sector = mddev->dev_sectors - 1;
NeilBrown52c03292006-06-26 00:27:43 -07005528 while (first_sector <= last_sector) {
Shaohua Li6d036f72015-08-13 14:31:57 -07005529 sh = raid5_get_active_stripe(conf, first_sector, 1, 0, 1);
NeilBrown52c03292006-06-26 00:27:43 -07005530 set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
5531 set_bit(STRIPE_HANDLE, &sh->state);
Shaohua Li6d036f72015-08-13 14:31:57 -07005532 raid5_release_stripe(sh);
NeilBrown52c03292006-06-26 00:27:43 -07005533 first_sector += STRIPE_SECTORS;
5534 }
NeilBrownab69ae12009-03-31 15:26:47 +11005535 /* Now that the sources are clearly marked, we can release
5536 * the destination stripes
5537 */
5538 while (!list_empty(&stripes)) {
5539 sh = list_entry(stripes.next, struct stripe_head, lru);
5540 list_del_init(&sh->lru);
Shaohua Li6d036f72015-08-13 14:31:57 -07005541 raid5_release_stripe(sh);
NeilBrownab69ae12009-03-31 15:26:47 +11005542 }
NeilBrownc6207272008-02-06 01:39:52 -08005543 /* If this takes us to the resync_max point where we have to pause,
5544 * then we need to write out the superblock.
5545 */
NeilBrown7a661382009-03-31 15:21:40 +11005546 sector_nr += reshape_sectors;
NeilBrown92140482015-07-06 12:28:45 +10005547 retn = reshape_sectors;
5548finish:
NeilBrownc5e19d92015-07-17 12:06:02 +10005549 if (mddev->curr_resync_completed > mddev->resync_max ||
5550 (sector_nr - mddev->curr_resync_completed) * 2
NeilBrownc03f6a12009-04-17 11:06:30 +10005551 >= mddev->resync_max - mddev->curr_resync_completed) {
NeilBrownc6207272008-02-06 01:39:52 -08005552 /* Cannot proceed until we've updated the superblock... */
5553 wait_event(conf->wait_for_overlap,
NeilBrownc91abf52013-11-19 12:02:01 +11005554 atomic_read(&conf->reshape_stripes) == 0
5555 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5556 if (atomic_read(&conf->reshape_stripes) != 0)
5557 goto ret;
NeilBrownfef9c612009-03-31 15:16:46 +11005558 mddev->reshape_position = conf->reshape_progress;
NeilBrown75d3da42011-01-14 09:14:34 +11005559 mddev->curr_resync_completed = sector_nr;
NeilBrownc8f517c2009-03-31 15:28:40 +11005560 conf->reshape_checkpoint = jiffies;
NeilBrownc6207272008-02-06 01:39:52 -08005561 set_bit(MD_CHANGE_DEVS, &mddev->flags);
5562 md_wakeup_thread(mddev->thread);
5563 wait_event(mddev->sb_wait,
5564 !test_bit(MD_CHANGE_DEVS, &mddev->flags)
NeilBrownc91abf52013-11-19 12:02:01 +11005565 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5566 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5567 goto ret;
NeilBrownc6207272008-02-06 01:39:52 -08005568 spin_lock_irq(&conf->device_lock);
NeilBrownfef9c612009-03-31 15:16:46 +11005569 conf->reshape_safe = mddev->reshape_position;
NeilBrownc6207272008-02-06 01:39:52 -08005570 spin_unlock_irq(&conf->device_lock);
5571 wake_up(&conf->wait_for_overlap);
NeilBrownacb180b2009-04-14 16:28:34 +10005572 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
NeilBrownc6207272008-02-06 01:39:52 -08005573 }
NeilBrownc91abf52013-11-19 12:02:01 +11005574ret:
NeilBrown92140482015-07-06 12:28:45 +10005575 return retn;
NeilBrown52c03292006-06-26 00:27:43 -07005576}
5577
NeilBrown09314792015-02-19 16:04:40 +11005578static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
NeilBrown52c03292006-06-26 00:27:43 -07005579{
NeilBrownd1688a62011-10-11 16:49:52 +11005580 struct r5conf *conf = mddev->private;
NeilBrown52c03292006-06-26 00:27:43 -07005581 struct stripe_head *sh;
Andre Noll58c0fed2009-03-31 14:33:13 +11005582 sector_t max_sector = mddev->dev_sectors;
NeilBrown57dab0b2010-10-19 10:03:39 +11005583 sector_t sync_blocks;
NeilBrown16a53ec2006-06-26 00:27:38 -07005584 int still_degraded = 0;
5585 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005586
NeilBrown72626682005-09-09 16:23:54 -07005587 if (sector_nr >= max_sector) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005588 /* just being told to finish up .. nothing much to do */
NeilBrowncea9c222009-03-31 15:15:05 +11005589
NeilBrown29269552006-03-27 01:18:10 -08005590 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
5591 end_reshape(conf);
5592 return 0;
5593 }
NeilBrown72626682005-09-09 16:23:54 -07005594
5595 if (mddev->curr_resync < max_sector) /* aborted */
5596 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
5597 &sync_blocks, 1);
NeilBrown16a53ec2006-06-26 00:27:38 -07005598 else /* completed sync */
NeilBrown72626682005-09-09 16:23:54 -07005599 conf->fullsync = 0;
5600 bitmap_close_sync(mddev->bitmap);
5601
Linus Torvalds1da177e2005-04-16 15:20:36 -07005602 return 0;
5603 }
NeilBrownccfcc3c2006-03-27 01:18:09 -08005604
NeilBrown64bd6602009-08-03 10:59:58 +10005605 /* Allow raid5_quiesce to complete */
5606 wait_event(conf->wait_for_overlap, conf->quiesce != 2);
5607
NeilBrown52c03292006-06-26 00:27:43 -07005608 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
5609 return reshape_request(mddev, sector_nr, skipped);
NeilBrownf6705572006-03-27 01:18:11 -08005610
NeilBrownc6207272008-02-06 01:39:52 -08005611 /* No need to check resync_max as we never do more than one
5612 * stripe, and as resync_max will always be on a chunk boundary,
5613 * if the check in md_do_sync didn't fire, there is no chance
5614 * of overstepping resync_max here
5615 */
5616
NeilBrown16a53ec2006-06-26 00:27:38 -07005617 /* if there is too many failed drives and we are trying
Linus Torvalds1da177e2005-04-16 15:20:36 -07005618 * to resync, then assert that we are finished, because there is
5619 * nothing we can do.
5620 */
NeilBrown3285edf2006-06-26 00:27:55 -07005621 if (mddev->degraded >= conf->max_degraded &&
NeilBrown16a53ec2006-06-26 00:27:38 -07005622 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
Andre Noll58c0fed2009-03-31 14:33:13 +11005623 sector_t rv = mddev->dev_sectors - sector_nr;
NeilBrown57afd892005-06-21 17:17:13 -07005624 *skipped = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005625 return rv;
5626 }
majianpeng6f608042013-04-24 11:42:41 +10005627 if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
5628 !conf->fullsync &&
5629 !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
5630 sync_blocks >= STRIPE_SECTORS) {
NeilBrown72626682005-09-09 16:23:54 -07005631 /* we can skip this block, and probably more */
5632 sync_blocks /= STRIPE_SECTORS;
5633 *skipped = 1;
5634 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
5635 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005636
Goldwyn Rodriguesc40f3412015-08-19 08:14:42 +10005637 bitmap_cond_end_sync(mddev->bitmap, sector_nr, false);
NeilBrownb47490c2008-02-06 01:39:50 -08005638
Shaohua Li6d036f72015-08-13 14:31:57 -07005639 sh = raid5_get_active_stripe(conf, sector_nr, 0, 1, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005640 if (sh == NULL) {
Shaohua Li6d036f72015-08-13 14:31:57 -07005641 sh = raid5_get_active_stripe(conf, sector_nr, 0, 0, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005642 /* make sure we don't swamp the stripe cache if someone else
NeilBrown16a53ec2006-06-26 00:27:38 -07005643 * is trying to get access
Linus Torvalds1da177e2005-04-16 15:20:36 -07005644 */
Nishanth Aravamudan66c006a2005-11-07 01:01:17 -08005645 schedule_timeout_uninterruptible(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005646 }
NeilBrown16a53ec2006-06-26 00:27:38 -07005647 /* Need to check if array will still be degraded after recovery/resync
Eric Mei16d9cfa2015-01-06 09:35:02 -08005648 * Note in case of > 1 drive failures it's possible we're rebuilding
5649 * one drive while leaving another faulty drive in array.
NeilBrown16a53ec2006-06-26 00:27:38 -07005650 */
Eric Mei16d9cfa2015-01-06 09:35:02 -08005651 rcu_read_lock();
5652 for (i = 0; i < conf->raid_disks; i++) {
5653 struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev);
5654
5655 if (rdev == NULL || test_bit(Faulty, &rdev->flags))
NeilBrown16a53ec2006-06-26 00:27:38 -07005656 still_degraded = 1;
Eric Mei16d9cfa2015-01-06 09:35:02 -08005657 }
5658 rcu_read_unlock();
NeilBrown16a53ec2006-06-26 00:27:38 -07005659
5660 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
5661
NeilBrown83206d62011-07-26 11:19:49 +10005662 set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
Eivind Sarto053f5b62014-06-09 17:06:19 -07005663 set_bit(STRIPE_HANDLE, &sh->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005664
Shaohua Li6d036f72015-08-13 14:31:57 -07005665 raid5_release_stripe(sh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005666
5667 return STRIPE_SECTORS;
5668}
5669
NeilBrownd1688a62011-10-11 16:49:52 +11005670static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005671{
5672 /* We may not be able to submit a whole bio at once as there
5673 * may not be enough stripe_heads available.
5674 * We cannot pre-allocate enough stripe_heads as we may need
5675 * more than exist in the cache (if we allow ever large chunks).
5676 * So we do one stripe head at a time and record in
5677 * ->bi_hw_segments how many have been done.
5678 *
5679 * We *know* that this entire raid_bio is in one chunk, so
5680 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
5681 */
5682 struct stripe_head *sh;
NeilBrown911d4ee2009-03-31 14:39:38 +11005683 int dd_idx;
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005684 sector_t sector, logical_sector, last_sector;
5685 int scnt = 0;
5686 int remaining;
5687 int handled = 0;
5688
Kent Overstreet4f024f32013-10-11 15:44:27 -07005689 logical_sector = raid_bio->bi_iter.bi_sector &
5690 ~((sector_t)STRIPE_SECTORS-1);
NeilBrown112bf892009-03-31 14:39:38 +11005691 sector = raid5_compute_sector(conf, logical_sector,
NeilBrown911d4ee2009-03-31 14:39:38 +11005692 0, &dd_idx, NULL);
Kent Overstreetf73a1c72012-09-25 15:05:12 -07005693 last_sector = bio_end_sector(raid_bio);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005694
5695 for (; logical_sector < last_sector;
Neil Brown387bb172007-02-08 14:20:29 -08005696 logical_sector += STRIPE_SECTORS,
5697 sector += STRIPE_SECTORS,
5698 scnt++) {
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005699
Shaohua Lie7836bd62012-07-19 16:01:31 +10005700 if (scnt < raid5_bi_processed_stripes(raid_bio))
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005701 /* already done this stripe */
5702 continue;
5703
Shaohua Li6d036f72015-08-13 14:31:57 -07005704 sh = raid5_get_active_stripe(conf, sector, 0, 1, 1);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005705
5706 if (!sh) {
5707 /* failed to get a stripe - must wait */
Shaohua Lie7836bd62012-07-19 16:01:31 +10005708 raid5_set_bi_processed_stripes(raid_bio, scnt);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005709 conf->retry_read_aligned = raid_bio;
5710 return handled;
5711 }
5712
shli@kernel.orgda41ba62014-12-15 12:57:03 +11005713 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
Shaohua Li6d036f72015-08-13 14:31:57 -07005714 raid5_release_stripe(sh);
Shaohua Lie7836bd62012-07-19 16:01:31 +10005715 raid5_set_bi_processed_stripes(raid_bio, scnt);
Neil Brown387bb172007-02-08 14:20:29 -08005716 conf->retry_read_aligned = raid_bio;
5717 return handled;
5718 }
5719
majianpeng3f9e7c12012-07-31 10:04:21 +10005720 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
Dan Williams36d1c642009-07-14 11:48:22 -07005721 handle_stripe(sh);
Shaohua Li6d036f72015-08-13 14:31:57 -07005722 raid5_release_stripe(sh);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005723 handled++;
5724 }
Shaohua Lie7836bd62012-07-19 16:01:31 +10005725 remaining = raid5_dec_bi_active_stripes(raid_bio);
Linus Torvalds0a82a8d2013-04-18 09:00:26 -07005726 if (remaining == 0) {
5727 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
5728 raid_bio, 0);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02005729 bio_endio(raid_bio);
Linus Torvalds0a82a8d2013-04-18 09:00:26 -07005730 }
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005731 if (atomic_dec_and_test(&conf->active_aligned_reads))
Yuanhan Liub1b46482015-05-08 18:19:06 +10005732 wake_up(&conf->wait_for_quiescent);
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005733 return handled;
5734}
5735
Shaohua Libfc90cb2013-08-29 15:40:32 +08005736static int handle_active_stripes(struct r5conf *conf, int group,
Shaohua Li566c09c2013-11-14 15:16:17 +11005737 struct r5worker *worker,
5738 struct list_head *temp_inactive_list)
Shaohua Li46a06402012-08-02 08:33:15 +10005739{
5740 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
Shaohua Li566c09c2013-11-14 15:16:17 +11005741 int i, batch_size = 0, hash;
5742 bool release_inactive = false;
Shaohua Li46a06402012-08-02 08:33:15 +10005743
5744 while (batch_size < MAX_STRIPE_BATCH &&
Shaohua Li851c30c2013-08-28 14:30:16 +08005745 (sh = __get_priority_stripe(conf, group)) != NULL)
Shaohua Li46a06402012-08-02 08:33:15 +10005746 batch[batch_size++] = sh;
5747
Shaohua Li566c09c2013-11-14 15:16:17 +11005748 if (batch_size == 0) {
5749 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
5750 if (!list_empty(temp_inactive_list + i))
5751 break;
Shaohua Lia8c34f92015-09-02 13:49:46 -07005752 if (i == NR_STRIPE_HASH_LOCKS) {
5753 spin_unlock_irq(&conf->device_lock);
5754 r5l_flush_stripe_to_raid(conf->log);
5755 spin_lock_irq(&conf->device_lock);
Shaohua Li566c09c2013-11-14 15:16:17 +11005756 return batch_size;
Shaohua Lia8c34f92015-09-02 13:49:46 -07005757 }
Shaohua Li566c09c2013-11-14 15:16:17 +11005758 release_inactive = true;
5759 }
Shaohua Li46a06402012-08-02 08:33:15 +10005760 spin_unlock_irq(&conf->device_lock);
5761
Shaohua Li566c09c2013-11-14 15:16:17 +11005762 release_inactive_stripe_list(conf, temp_inactive_list,
5763 NR_STRIPE_HASH_LOCKS);
5764
Shaohua Lia8c34f92015-09-02 13:49:46 -07005765 r5l_flush_stripe_to_raid(conf->log);
Shaohua Li566c09c2013-11-14 15:16:17 +11005766 if (release_inactive) {
5767 spin_lock_irq(&conf->device_lock);
5768 return 0;
5769 }
5770
Shaohua Li46a06402012-08-02 08:33:15 +10005771 for (i = 0; i < batch_size; i++)
5772 handle_stripe(batch[i]);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07005773 r5l_write_stripe_run(conf->log);
Shaohua Li46a06402012-08-02 08:33:15 +10005774
5775 cond_resched();
5776
5777 spin_lock_irq(&conf->device_lock);
Shaohua Li566c09c2013-11-14 15:16:17 +11005778 for (i = 0; i < batch_size; i++) {
5779 hash = batch[i]->hash_lock_index;
5780 __release_stripe(conf, batch[i], &temp_inactive_list[hash]);
5781 }
Shaohua Li46a06402012-08-02 08:33:15 +10005782 return batch_size;
5783}
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005784
Shaohua Li851c30c2013-08-28 14:30:16 +08005785static void raid5_do_work(struct work_struct *work)
5786{
5787 struct r5worker *worker = container_of(work, struct r5worker, work);
5788 struct r5worker_group *group = worker->group;
5789 struct r5conf *conf = group->conf;
5790 int group_id = group - conf->worker_groups;
5791 int handled;
5792 struct blk_plug plug;
5793
5794 pr_debug("+++ raid5worker active\n");
5795
5796 blk_start_plug(&plug);
5797 handled = 0;
5798 spin_lock_irq(&conf->device_lock);
5799 while (1) {
5800 int batch_size, released;
5801
Shaohua Li566c09c2013-11-14 15:16:17 +11005802 released = release_stripe_list(conf, worker->temp_inactive_list);
Shaohua Li851c30c2013-08-28 14:30:16 +08005803
Shaohua Li566c09c2013-11-14 15:16:17 +11005804 batch_size = handle_active_stripes(conf, group_id, worker,
5805 worker->temp_inactive_list);
Shaohua Libfc90cb2013-08-29 15:40:32 +08005806 worker->working = false;
Shaohua Li851c30c2013-08-28 14:30:16 +08005807 if (!batch_size && !released)
5808 break;
5809 handled += batch_size;
5810 }
5811 pr_debug("%d stripes handled\n", handled);
5812
5813 spin_unlock_irq(&conf->device_lock);
5814 blk_finish_plug(&plug);
5815
5816 pr_debug("--- raid5worker inactive\n");
5817}
5818
Linus Torvalds1da177e2005-04-16 15:20:36 -07005819/*
5820 * This is our raid5 kernel thread.
5821 *
5822 * We scan the hash table for stripes which can be handled now.
5823 * During the scan, completed stripes are saved for us by the interrupt
5824 * handler, so that they will not have to wait for our next wakeup.
5825 */
Shaohua Li4ed87312012-10-11 13:34:00 +11005826static void raid5d(struct md_thread *thread)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005827{
Shaohua Li4ed87312012-10-11 13:34:00 +11005828 struct mddev *mddev = thread->mddev;
NeilBrownd1688a62011-10-11 16:49:52 +11005829 struct r5conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005830 int handled;
NeilBrowne1dfa0a2011-04-18 18:25:41 +10005831 struct blk_plug plug;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005832
Dan Williams45b42332007-07-09 11:56:43 -07005833 pr_debug("+++ raid5d active\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005834
5835 md_check_recovery(mddev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005836
NeilBrownc3cce6c2015-08-14 12:47:33 +10005837 if (!bio_list_empty(&conf->return_bi) &&
5838 !test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
5839 struct bio_list tmp = BIO_EMPTY_LIST;
5840 spin_lock_irq(&conf->device_lock);
5841 if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
5842 bio_list_merge(&tmp, &conf->return_bi);
5843 bio_list_init(&conf->return_bi);
5844 }
5845 spin_unlock_irq(&conf->device_lock);
5846 return_io(&tmp);
5847 }
5848
NeilBrowne1dfa0a2011-04-18 18:25:41 +10005849 blk_start_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005850 handled = 0;
5851 spin_lock_irq(&conf->device_lock);
5852 while (1) {
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005853 struct bio *bio;
Shaohua Li773ca822013-08-27 17:50:39 +08005854 int batch_size, released;
5855
Shaohua Li566c09c2013-11-14 15:16:17 +11005856 released = release_stripe_list(conf, conf->temp_inactive_list);
NeilBrownedbe83a2015-02-26 12:47:56 +11005857 if (released)
5858 clear_bit(R5_DID_ALLOC, &conf->cache_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005859
NeilBrown0021b7b2012-07-31 09:08:14 +02005860 if (
NeilBrown7c13edc2011-04-18 18:25:43 +10005861 !list_empty(&conf->bitmap_list)) {
5862 /* Now is a good time to flush some bitmap updates */
5863 conf->seq_flush++;
NeilBrown700e4322005-11-28 13:44:10 -08005864 spin_unlock_irq(&conf->device_lock);
NeilBrown72626682005-09-09 16:23:54 -07005865 bitmap_unplug(mddev->bitmap);
NeilBrown700e4322005-11-28 13:44:10 -08005866 spin_lock_irq(&conf->device_lock);
NeilBrown7c13edc2011-04-18 18:25:43 +10005867 conf->seq_write = conf->seq_flush;
Shaohua Li566c09c2013-11-14 15:16:17 +11005868 activate_bit_delay(conf, conf->temp_inactive_list);
NeilBrown72626682005-09-09 16:23:54 -07005869 }
NeilBrown0021b7b2012-07-31 09:08:14 +02005870 raid5_activate_delayed(conf);
NeilBrown72626682005-09-09 16:23:54 -07005871
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08005872 while ((bio = remove_bio_from_retry(conf))) {
5873 int ok;
5874 spin_unlock_irq(&conf->device_lock);
5875 ok = retry_aligned_read(conf, bio);
5876 spin_lock_irq(&conf->device_lock);
5877 if (!ok)
5878 break;
5879 handled++;
5880 }
5881
Shaohua Li566c09c2013-11-14 15:16:17 +11005882 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
5883 conf->temp_inactive_list);
Shaohua Li773ca822013-08-27 17:50:39 +08005884 if (!batch_size && !released)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005885 break;
Shaohua Li46a06402012-08-02 08:33:15 +10005886 handled += batch_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005887
Shaohua Li46a06402012-08-02 08:33:15 +10005888 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
5889 spin_unlock_irq(&conf->device_lock);
NeilBrownde393cd2011-07-28 11:31:48 +10005890 md_check_recovery(mddev);
Shaohua Li46a06402012-08-02 08:33:15 +10005891 spin_lock_irq(&conf->device_lock);
5892 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005893 }
Dan Williams45b42332007-07-09 11:56:43 -07005894 pr_debug("%d stripes handled\n", handled);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005895
5896 spin_unlock_irq(&conf->device_lock);
NeilBrown2d5b5692015-07-06 12:49:23 +10005897 if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) &&
5898 mutex_trylock(&conf->cache_size_mutex)) {
NeilBrownedbe83a2015-02-26 12:47:56 +11005899 grow_one_stripe(conf, __GFP_NOWARN);
5900 /* Set flag even if allocation failed. This helps
5901 * slow down allocation requests when mem is short
5902 */
5903 set_bit(R5_DID_ALLOC, &conf->cache_state);
NeilBrown2d5b5692015-07-06 12:49:23 +10005904 mutex_unlock(&conf->cache_size_mutex);
NeilBrownedbe83a2015-02-26 12:47:56 +11005905 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005906
Shaohua Li0576b1c2015-08-13 14:32:00 -07005907 r5l_flush_stripe_to_raid(conf->log);
5908
Dan Williamsc9f21aa2008-07-23 12:05:51 -07005909 async_tx_issue_pending_all();
NeilBrowne1dfa0a2011-04-18 18:25:41 +10005910 blk_finish_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005911
Dan Williams45b42332007-07-09 11:56:43 -07005912 pr_debug("--- raid5d inactive\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005913}
5914
NeilBrown3f294f42005-11-08 21:39:25 -08005915static ssize_t
NeilBrownfd01b882011-10-11 16:47:53 +11005916raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
NeilBrown3f294f42005-11-08 21:39:25 -08005917{
NeilBrown7b1485b2014-12-15 12:56:59 +11005918 struct r5conf *conf;
5919 int ret = 0;
5920 spin_lock(&mddev->lock);
5921 conf = mddev->private;
NeilBrown96de1e62005-11-08 21:39:39 -08005922 if (conf)
NeilBrownedbe83a2015-02-26 12:47:56 +11005923 ret = sprintf(page, "%d\n", conf->min_nr_stripes);
NeilBrown7b1485b2014-12-15 12:56:59 +11005924 spin_unlock(&mddev->lock);
5925 return ret;
NeilBrown3f294f42005-11-08 21:39:25 -08005926}
5927
NeilBrownc41d4ac2010-06-01 19:37:24 +10005928int
NeilBrownfd01b882011-10-11 16:47:53 +11005929raid5_set_cache_size(struct mddev *mddev, int size)
NeilBrownc41d4ac2010-06-01 19:37:24 +10005930{
NeilBrownd1688a62011-10-11 16:49:52 +11005931 struct r5conf *conf = mddev->private;
NeilBrownc41d4ac2010-06-01 19:37:24 +10005932 int err;
5933
5934 if (size <= 16 || size > 32768)
5935 return -EINVAL;
NeilBrown486f0642015-02-25 12:10:35 +11005936
NeilBrownedbe83a2015-02-26 12:47:56 +11005937 conf->min_nr_stripes = size;
NeilBrown2d5b5692015-07-06 12:49:23 +10005938 mutex_lock(&conf->cache_size_mutex);
NeilBrown486f0642015-02-25 12:10:35 +11005939 while (size < conf->max_nr_stripes &&
5940 drop_one_stripe(conf))
5941 ;
NeilBrown2d5b5692015-07-06 12:49:23 +10005942 mutex_unlock(&conf->cache_size_mutex);
NeilBrown486f0642015-02-25 12:10:35 +11005943
NeilBrownedbe83a2015-02-26 12:47:56 +11005944
NeilBrownc41d4ac2010-06-01 19:37:24 +10005945 err = md_allow_write(mddev);
5946 if (err)
5947 return err;
NeilBrown486f0642015-02-25 12:10:35 +11005948
NeilBrown2d5b5692015-07-06 12:49:23 +10005949 mutex_lock(&conf->cache_size_mutex);
NeilBrown486f0642015-02-25 12:10:35 +11005950 while (size > conf->max_nr_stripes)
5951 if (!grow_one_stripe(conf, GFP_KERNEL))
5952 break;
NeilBrown2d5b5692015-07-06 12:49:23 +10005953 mutex_unlock(&conf->cache_size_mutex);
NeilBrown486f0642015-02-25 12:10:35 +11005954
NeilBrownc41d4ac2010-06-01 19:37:24 +10005955 return 0;
5956}
5957EXPORT_SYMBOL(raid5_set_cache_size);
5958
NeilBrown3f294f42005-11-08 21:39:25 -08005959static ssize_t
NeilBrownfd01b882011-10-11 16:47:53 +11005960raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
NeilBrown3f294f42005-11-08 21:39:25 -08005961{
NeilBrown67918752014-12-15 12:57:01 +11005962 struct r5conf *conf;
Dan Williams4ef197d82008-04-28 02:15:54 -07005963 unsigned long new;
Dan Williamsb5470dc2008-06-27 21:44:04 -07005964 int err;
5965
NeilBrown3f294f42005-11-08 21:39:25 -08005966 if (len >= PAGE_SIZE)
5967 return -EINVAL;
Jingoo Hanb29bebd2013-06-01 16:15:16 +09005968 if (kstrtoul(page, 10, &new))
NeilBrown3f294f42005-11-08 21:39:25 -08005969 return -EINVAL;
NeilBrown67918752014-12-15 12:57:01 +11005970 err = mddev_lock(mddev);
Dan Williamsb5470dc2008-06-27 21:44:04 -07005971 if (err)
5972 return err;
NeilBrown67918752014-12-15 12:57:01 +11005973 conf = mddev->private;
5974 if (!conf)
5975 err = -ENODEV;
5976 else
5977 err = raid5_set_cache_size(mddev, new);
5978 mddev_unlock(mddev);
5979
5980 return err ?: len;
NeilBrown3f294f42005-11-08 21:39:25 -08005981}
NeilBrown007583c2005-11-08 21:39:30 -08005982
NeilBrown96de1e62005-11-08 21:39:39 -08005983static struct md_sysfs_entry
5984raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
5985 raid5_show_stripe_cache_size,
5986 raid5_store_stripe_cache_size);
NeilBrown3f294f42005-11-08 21:39:25 -08005987
5988static ssize_t
Markus Stockhausend06f1912014-12-15 12:57:05 +11005989raid5_show_rmw_level(struct mddev *mddev, char *page)
5990{
5991 struct r5conf *conf = mddev->private;
5992 if (conf)
5993 return sprintf(page, "%d\n", conf->rmw_level);
5994 else
5995 return 0;
5996}
5997
5998static ssize_t
5999raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len)
6000{
6001 struct r5conf *conf = mddev->private;
6002 unsigned long new;
6003
6004 if (!conf)
6005 return -ENODEV;
6006
6007 if (len >= PAGE_SIZE)
6008 return -EINVAL;
6009
6010 if (kstrtoul(page, 10, &new))
6011 return -EINVAL;
6012
6013 if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
6014 return -EINVAL;
6015
6016 if (new != PARITY_DISABLE_RMW &&
6017 new != PARITY_ENABLE_RMW &&
6018 new != PARITY_PREFER_RMW)
6019 return -EINVAL;
6020
6021 conf->rmw_level = new;
6022 return len;
6023}
6024
6025static struct md_sysfs_entry
6026raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR,
6027 raid5_show_rmw_level,
6028 raid5_store_rmw_level);
6029
6030
6031static ssize_t
NeilBrownfd01b882011-10-11 16:47:53 +11006032raid5_show_preread_threshold(struct mddev *mddev, char *page)
Dan Williams8b3e6cd2008-04-28 02:15:53 -07006033{
NeilBrown7b1485b2014-12-15 12:56:59 +11006034 struct r5conf *conf;
6035 int ret = 0;
6036 spin_lock(&mddev->lock);
6037 conf = mddev->private;
Dan Williams8b3e6cd2008-04-28 02:15:53 -07006038 if (conf)
NeilBrown7b1485b2014-12-15 12:56:59 +11006039 ret = sprintf(page, "%d\n", conf->bypass_threshold);
6040 spin_unlock(&mddev->lock);
6041 return ret;
Dan Williams8b3e6cd2008-04-28 02:15:53 -07006042}
6043
6044static ssize_t
NeilBrownfd01b882011-10-11 16:47:53 +11006045raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
Dan Williams8b3e6cd2008-04-28 02:15:53 -07006046{
NeilBrown67918752014-12-15 12:57:01 +11006047 struct r5conf *conf;
Dan Williams4ef197d82008-04-28 02:15:54 -07006048 unsigned long new;
NeilBrown67918752014-12-15 12:57:01 +11006049 int err;
6050
Dan Williams8b3e6cd2008-04-28 02:15:53 -07006051 if (len >= PAGE_SIZE)
6052 return -EINVAL;
Jingoo Hanb29bebd2013-06-01 16:15:16 +09006053 if (kstrtoul(page, 10, &new))
Dan Williams8b3e6cd2008-04-28 02:15:53 -07006054 return -EINVAL;
NeilBrown67918752014-12-15 12:57:01 +11006055
6056 err = mddev_lock(mddev);
6057 if (err)
6058 return err;
6059 conf = mddev->private;
6060 if (!conf)
6061 err = -ENODEV;
NeilBrownedbe83a2015-02-26 12:47:56 +11006062 else if (new > conf->min_nr_stripes)
NeilBrown67918752014-12-15 12:57:01 +11006063 err = -EINVAL;
6064 else
6065 conf->bypass_threshold = new;
6066 mddev_unlock(mddev);
6067 return err ?: len;
Dan Williams8b3e6cd2008-04-28 02:15:53 -07006068}
6069
6070static struct md_sysfs_entry
6071raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
6072 S_IRUGO | S_IWUSR,
6073 raid5_show_preread_threshold,
6074 raid5_store_preread_threshold);
6075
6076static ssize_t
Shaohua Lid592a992014-05-21 17:57:44 +08006077raid5_show_skip_copy(struct mddev *mddev, char *page)
6078{
NeilBrown7b1485b2014-12-15 12:56:59 +11006079 struct r5conf *conf;
6080 int ret = 0;
6081 spin_lock(&mddev->lock);
6082 conf = mddev->private;
Shaohua Lid592a992014-05-21 17:57:44 +08006083 if (conf)
NeilBrown7b1485b2014-12-15 12:56:59 +11006084 ret = sprintf(page, "%d\n", conf->skip_copy);
6085 spin_unlock(&mddev->lock);
6086 return ret;
Shaohua Lid592a992014-05-21 17:57:44 +08006087}
6088
6089static ssize_t
6090raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
6091{
NeilBrown67918752014-12-15 12:57:01 +11006092 struct r5conf *conf;
Shaohua Lid592a992014-05-21 17:57:44 +08006093 unsigned long new;
NeilBrown67918752014-12-15 12:57:01 +11006094 int err;
6095
Shaohua Lid592a992014-05-21 17:57:44 +08006096 if (len >= PAGE_SIZE)
6097 return -EINVAL;
Shaohua Lid592a992014-05-21 17:57:44 +08006098 if (kstrtoul(page, 10, &new))
6099 return -EINVAL;
6100 new = !!new;
Shaohua Lid592a992014-05-21 17:57:44 +08006101
NeilBrown67918752014-12-15 12:57:01 +11006102 err = mddev_lock(mddev);
6103 if (err)
6104 return err;
6105 conf = mddev->private;
6106 if (!conf)
6107 err = -ENODEV;
6108 else if (new != conf->skip_copy) {
6109 mddev_suspend(mddev);
6110 conf->skip_copy = new;
6111 if (new)
6112 mddev->queue->backing_dev_info.capabilities |=
6113 BDI_CAP_STABLE_WRITES;
6114 else
6115 mddev->queue->backing_dev_info.capabilities &=
6116 ~BDI_CAP_STABLE_WRITES;
6117 mddev_resume(mddev);
6118 }
6119 mddev_unlock(mddev);
6120 return err ?: len;
Shaohua Lid592a992014-05-21 17:57:44 +08006121}
6122
6123static struct md_sysfs_entry
6124raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
6125 raid5_show_skip_copy,
6126 raid5_store_skip_copy);
6127
Shaohua Lid592a992014-05-21 17:57:44 +08006128static ssize_t
NeilBrownfd01b882011-10-11 16:47:53 +11006129stripe_cache_active_show(struct mddev *mddev, char *page)
NeilBrown3f294f42005-11-08 21:39:25 -08006130{
NeilBrownd1688a62011-10-11 16:49:52 +11006131 struct r5conf *conf = mddev->private;
NeilBrown96de1e62005-11-08 21:39:39 -08006132 if (conf)
6133 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
6134 else
6135 return 0;
NeilBrown3f294f42005-11-08 21:39:25 -08006136}
6137
NeilBrown96de1e62005-11-08 21:39:39 -08006138static struct md_sysfs_entry
6139raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
NeilBrown3f294f42005-11-08 21:39:25 -08006140
Shaohua Lib7214202013-08-27 17:50:42 +08006141static ssize_t
6142raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
6143{
NeilBrown7b1485b2014-12-15 12:56:59 +11006144 struct r5conf *conf;
6145 int ret = 0;
6146 spin_lock(&mddev->lock);
6147 conf = mddev->private;
Shaohua Lib7214202013-08-27 17:50:42 +08006148 if (conf)
NeilBrown7b1485b2014-12-15 12:56:59 +11006149 ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);
6150 spin_unlock(&mddev->lock);
6151 return ret;
Shaohua Lib7214202013-08-27 17:50:42 +08006152}
6153
majianpeng60aaf932013-11-14 15:16:20 +11006154static int alloc_thread_groups(struct r5conf *conf, int cnt,
6155 int *group_cnt,
6156 int *worker_cnt_per_group,
6157 struct r5worker_group **worker_groups);
Shaohua Lib7214202013-08-27 17:50:42 +08006158static ssize_t
6159raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
6160{
NeilBrown67918752014-12-15 12:57:01 +11006161 struct r5conf *conf;
Shaohua Lib7214202013-08-27 17:50:42 +08006162 unsigned long new;
6163 int err;
majianpeng60aaf932013-11-14 15:16:20 +11006164 struct r5worker_group *new_groups, *old_groups;
6165 int group_cnt, worker_cnt_per_group;
Shaohua Lib7214202013-08-27 17:50:42 +08006166
6167 if (len >= PAGE_SIZE)
6168 return -EINVAL;
Shaohua Lib7214202013-08-27 17:50:42 +08006169 if (kstrtoul(page, 10, &new))
6170 return -EINVAL;
6171
NeilBrown67918752014-12-15 12:57:01 +11006172 err = mddev_lock(mddev);
Shaohua Lib7214202013-08-27 17:50:42 +08006173 if (err)
6174 return err;
NeilBrown67918752014-12-15 12:57:01 +11006175 conf = mddev->private;
6176 if (!conf)
6177 err = -ENODEV;
6178 else if (new != conf->worker_cnt_per_group) {
6179 mddev_suspend(mddev);
6180
6181 old_groups = conf->worker_groups;
6182 if (old_groups)
6183 flush_workqueue(raid5_wq);
6184
6185 err = alloc_thread_groups(conf, new,
6186 &group_cnt, &worker_cnt_per_group,
6187 &new_groups);
6188 if (!err) {
6189 spin_lock_irq(&conf->device_lock);
6190 conf->group_cnt = group_cnt;
6191 conf->worker_cnt_per_group = worker_cnt_per_group;
6192 conf->worker_groups = new_groups;
6193 spin_unlock_irq(&conf->device_lock);
6194
6195 if (old_groups)
6196 kfree(old_groups[0].workers);
6197 kfree(old_groups);
6198 }
6199 mddev_resume(mddev);
6200 }
6201 mddev_unlock(mddev);
6202
6203 return err ?: len;
Shaohua Lib7214202013-08-27 17:50:42 +08006204}
6205
6206static struct md_sysfs_entry
6207raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
6208 raid5_show_group_thread_cnt,
6209 raid5_store_group_thread_cnt);
6210
NeilBrown007583c2005-11-08 21:39:30 -08006211static struct attribute *raid5_attrs[] = {
NeilBrown3f294f42005-11-08 21:39:25 -08006212 &raid5_stripecache_size.attr,
6213 &raid5_stripecache_active.attr,
Dan Williams8b3e6cd2008-04-28 02:15:53 -07006214 &raid5_preread_bypass_threshold.attr,
Shaohua Lib7214202013-08-27 17:50:42 +08006215 &raid5_group_thread_cnt.attr,
Shaohua Lid592a992014-05-21 17:57:44 +08006216 &raid5_skip_copy.attr,
Markus Stockhausend06f1912014-12-15 12:57:05 +11006217 &raid5_rmw_level.attr,
NeilBrown3f294f42005-11-08 21:39:25 -08006218 NULL,
6219};
NeilBrown007583c2005-11-08 21:39:30 -08006220static struct attribute_group raid5_attrs_group = {
6221 .name = NULL,
6222 .attrs = raid5_attrs,
NeilBrown3f294f42005-11-08 21:39:25 -08006223};
6224
majianpeng60aaf932013-11-14 15:16:20 +11006225static int alloc_thread_groups(struct r5conf *conf, int cnt,
6226 int *group_cnt,
6227 int *worker_cnt_per_group,
6228 struct r5worker_group **worker_groups)
Shaohua Li851c30c2013-08-28 14:30:16 +08006229{
Shaohua Li566c09c2013-11-14 15:16:17 +11006230 int i, j, k;
Shaohua Li851c30c2013-08-28 14:30:16 +08006231 ssize_t size;
6232 struct r5worker *workers;
6233
majianpeng60aaf932013-11-14 15:16:20 +11006234 *worker_cnt_per_group = cnt;
Shaohua Li851c30c2013-08-28 14:30:16 +08006235 if (cnt == 0) {
majianpeng60aaf932013-11-14 15:16:20 +11006236 *group_cnt = 0;
6237 *worker_groups = NULL;
Shaohua Li851c30c2013-08-28 14:30:16 +08006238 return 0;
6239 }
majianpeng60aaf932013-11-14 15:16:20 +11006240 *group_cnt = num_possible_nodes();
Shaohua Li851c30c2013-08-28 14:30:16 +08006241 size = sizeof(struct r5worker) * cnt;
majianpeng60aaf932013-11-14 15:16:20 +11006242 workers = kzalloc(size * *group_cnt, GFP_NOIO);
6243 *worker_groups = kzalloc(sizeof(struct r5worker_group) *
6244 *group_cnt, GFP_NOIO);
6245 if (!*worker_groups || !workers) {
Shaohua Li851c30c2013-08-28 14:30:16 +08006246 kfree(workers);
majianpeng60aaf932013-11-14 15:16:20 +11006247 kfree(*worker_groups);
Shaohua Li851c30c2013-08-28 14:30:16 +08006248 return -ENOMEM;
6249 }
6250
majianpeng60aaf932013-11-14 15:16:20 +11006251 for (i = 0; i < *group_cnt; i++) {
Shaohua Li851c30c2013-08-28 14:30:16 +08006252 struct r5worker_group *group;
6253
NeilBrown0c775d52013-11-25 11:12:43 +11006254 group = &(*worker_groups)[i];
Shaohua Li851c30c2013-08-28 14:30:16 +08006255 INIT_LIST_HEAD(&group->handle_list);
6256 group->conf = conf;
6257 group->workers = workers + i * cnt;
6258
6259 for (j = 0; j < cnt; j++) {
Shaohua Li566c09c2013-11-14 15:16:17 +11006260 struct r5worker *worker = group->workers + j;
6261 worker->group = group;
6262 INIT_WORK(&worker->work, raid5_do_work);
6263
6264 for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
6265 INIT_LIST_HEAD(worker->temp_inactive_list + k);
Shaohua Li851c30c2013-08-28 14:30:16 +08006266 }
6267 }
6268
6269 return 0;
6270}
6271
6272static void free_thread_groups(struct r5conf *conf)
6273{
6274 if (conf->worker_groups)
6275 kfree(conf->worker_groups[0].workers);
6276 kfree(conf->worker_groups);
6277 conf->worker_groups = NULL;
6278}
6279
Dan Williams80c3a6c2009-03-17 18:10:40 -07006280static sector_t
NeilBrownfd01b882011-10-11 16:47:53 +11006281raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
Dan Williams80c3a6c2009-03-17 18:10:40 -07006282{
NeilBrownd1688a62011-10-11 16:49:52 +11006283 struct r5conf *conf = mddev->private;
Dan Williams80c3a6c2009-03-17 18:10:40 -07006284
6285 if (!sectors)
6286 sectors = mddev->dev_sectors;
NeilBrown5e5e3e72009-10-16 16:35:30 +11006287 if (!raid_disks)
NeilBrown7ec05472009-03-31 15:10:36 +11006288 /* size is defined by the smallest of previous and new size */
NeilBrown5e5e3e72009-10-16 16:35:30 +11006289 raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
Dan Williams80c3a6c2009-03-17 18:10:40 -07006290
NeilBrown3cb5edf2015-07-15 17:24:17 +10006291 sectors &= ~((sector_t)conf->chunk_sectors - 1);
6292 sectors &= ~((sector_t)conf->prev_chunk_sectors - 1);
Dan Williams80c3a6c2009-03-17 18:10:40 -07006293 return sectors * (raid_disks - conf->max_degraded);
6294}
6295
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306296static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
6297{
6298 safe_put_page(percpu->spare_page);
shli@kernel.org46d5b782014-12-15 12:57:02 +11006299 if (percpu->scribble)
6300 flex_array_free(percpu->scribble);
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306301 percpu->spare_page = NULL;
6302 percpu->scribble = NULL;
6303}
6304
6305static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
6306{
6307 if (conf->level == 6 && !percpu->spare_page)
6308 percpu->spare_page = alloc_page(GFP_KERNEL);
6309 if (!percpu->scribble)
shli@kernel.org46d5b782014-12-15 12:57:02 +11006310 percpu->scribble = scribble_alloc(max(conf->raid_disks,
NeilBrown738a2732015-05-08 18:19:39 +10006311 conf->previous_raid_disks),
6312 max(conf->chunk_sectors,
6313 conf->prev_chunk_sectors)
6314 / STRIPE_SECTORS,
6315 GFP_KERNEL);
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306316
6317 if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) {
6318 free_scratch_buffer(conf, percpu);
6319 return -ENOMEM;
6320 }
6321
6322 return 0;
6323}
6324
NeilBrownd1688a62011-10-11 16:49:52 +11006325static void raid5_free_percpu(struct r5conf *conf)
Dan Williams36d1c642009-07-14 11:48:22 -07006326{
Dan Williams36d1c642009-07-14 11:48:22 -07006327 unsigned long cpu;
6328
6329 if (!conf->percpu)
6330 return;
6331
Dan Williams36d1c642009-07-14 11:48:22 -07006332#ifdef CONFIG_HOTPLUG_CPU
6333 unregister_cpu_notifier(&conf->cpu_notify);
6334#endif
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306335
6336 get_online_cpus();
6337 for_each_possible_cpu(cpu)
6338 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
Dan Williams36d1c642009-07-14 11:48:22 -07006339 put_online_cpus();
6340
6341 free_percpu(conf->percpu);
6342}
6343
NeilBrownd1688a62011-10-11 16:49:52 +11006344static void free_conf(struct r5conf *conf)
Dan Williams95fc17a2009-07-31 12:39:15 +10006345{
Shaohua Li5c7e81c2015-08-13 14:32:04 -07006346 if (conf->log)
6347 r5l_exit_log(conf->log);
NeilBrownedbe83a2015-02-26 12:47:56 +11006348 if (conf->shrinker.seeks)
6349 unregister_shrinker(&conf->shrinker);
Shaohua Li5c7e81c2015-08-13 14:32:04 -07006350
Shaohua Li851c30c2013-08-28 14:30:16 +08006351 free_thread_groups(conf);
Dan Williams95fc17a2009-07-31 12:39:15 +10006352 shrink_stripes(conf);
Dan Williams36d1c642009-07-14 11:48:22 -07006353 raid5_free_percpu(conf);
Dan Williams95fc17a2009-07-31 12:39:15 +10006354 kfree(conf->disks);
6355 kfree(conf->stripe_hashtbl);
6356 kfree(conf);
6357}
6358
Dan Williams36d1c642009-07-14 11:48:22 -07006359#ifdef CONFIG_HOTPLUG_CPU
6360static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
6361 void *hcpu)
6362{
NeilBrownd1688a62011-10-11 16:49:52 +11006363 struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
Dan Williams36d1c642009-07-14 11:48:22 -07006364 long cpu = (long)hcpu;
6365 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
6366
6367 switch (action) {
6368 case CPU_UP_PREPARE:
6369 case CPU_UP_PREPARE_FROZEN:
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306370 if (alloc_scratch_buffer(conf, percpu)) {
Dan Williams36d1c642009-07-14 11:48:22 -07006371 pr_err("%s: failed memory allocation for cpu%ld\n",
6372 __func__, cpu);
Akinobu Mita55af6bb2010-05-26 14:43:35 -07006373 return notifier_from_errno(-ENOMEM);
Dan Williams36d1c642009-07-14 11:48:22 -07006374 }
6375 break;
6376 case CPU_DEAD:
6377 case CPU_DEAD_FROZEN:
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306378 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
Dan Williams36d1c642009-07-14 11:48:22 -07006379 break;
6380 default:
6381 break;
6382 }
6383 return NOTIFY_OK;
6384}
6385#endif
6386
NeilBrownd1688a62011-10-11 16:49:52 +11006387static int raid5_alloc_percpu(struct r5conf *conf)
Dan Williams36d1c642009-07-14 11:48:22 -07006388{
6389 unsigned long cpu;
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306390 int err = 0;
Dan Williams36d1c642009-07-14 11:48:22 -07006391
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306392 conf->percpu = alloc_percpu(struct raid5_percpu);
6393 if (!conf->percpu)
Dan Williams36d1c642009-07-14 11:48:22 -07006394 return -ENOMEM;
Dan Williams36d1c642009-07-14 11:48:22 -07006395
Dan Williams36d1c642009-07-14 11:48:22 -07006396#ifdef CONFIG_HOTPLUG_CPU
6397 conf->cpu_notify.notifier_call = raid456_cpu_notify;
6398 conf->cpu_notify.priority = 0;
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306399 err = register_cpu_notifier(&conf->cpu_notify);
6400 if (err)
6401 return err;
Dan Williams36d1c642009-07-14 11:48:22 -07006402#endif
Oleg Nesterov789b5e02014-02-06 03:42:45 +05306403
6404 get_online_cpus();
6405 for_each_present_cpu(cpu) {
6406 err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
6407 if (err) {
6408 pr_err("%s: failed memory allocation for cpu%ld\n",
6409 __func__, cpu);
6410 break;
6411 }
6412 }
Dan Williams36d1c642009-07-14 11:48:22 -07006413 put_online_cpus();
6414
6415 return err;
6416}
6417
NeilBrownedbe83a2015-02-26 12:47:56 +11006418static unsigned long raid5_cache_scan(struct shrinker *shrink,
6419 struct shrink_control *sc)
6420{
6421 struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
NeilBrown2d5b5692015-07-06 12:49:23 +10006422 unsigned long ret = SHRINK_STOP;
6423
6424 if (mutex_trylock(&conf->cache_size_mutex)) {
6425 ret= 0;
NeilBrown49895bc2015-08-03 17:09:57 +10006426 while (ret < sc->nr_to_scan &&
6427 conf->max_nr_stripes > conf->min_nr_stripes) {
NeilBrown2d5b5692015-07-06 12:49:23 +10006428 if (drop_one_stripe(conf) == 0) {
6429 ret = SHRINK_STOP;
6430 break;
6431 }
6432 ret++;
6433 }
6434 mutex_unlock(&conf->cache_size_mutex);
NeilBrownedbe83a2015-02-26 12:47:56 +11006435 }
6436 return ret;
6437}
6438
6439static unsigned long raid5_cache_count(struct shrinker *shrink,
6440 struct shrink_control *sc)
6441{
6442 struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
6443
6444 if (conf->max_nr_stripes < conf->min_nr_stripes)
6445 /* unlikely, but not impossible */
6446 return 0;
6447 return conf->max_nr_stripes - conf->min_nr_stripes;
6448}
6449
NeilBrownd1688a62011-10-11 16:49:52 +11006450static struct r5conf *setup_conf(struct mddev *mddev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006451{
NeilBrownd1688a62011-10-11 16:49:52 +11006452 struct r5conf *conf;
NeilBrown5e5e3e72009-10-16 16:35:30 +11006453 int raid_disk, memory, max_disks;
NeilBrown3cb03002011-10-11 16:45:26 +11006454 struct md_rdev *rdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006455 struct disk_info *disk;
NeilBrown02326052012-07-03 15:56:52 +10006456 char pers_name[6];
Shaohua Li566c09c2013-11-14 15:16:17 +11006457 int i;
majianpeng60aaf932013-11-14 15:16:20 +11006458 int group_cnt, worker_cnt_per_group;
6459 struct r5worker_group *new_group;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006460
NeilBrown91adb562009-03-31 14:39:39 +11006461 if (mddev->new_level != 5
6462 && mddev->new_level != 4
6463 && mddev->new_level != 6) {
NeilBrown0c55e022010-05-03 14:09:02 +10006464 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
NeilBrown91adb562009-03-31 14:39:39 +11006465 mdname(mddev), mddev->new_level);
6466 return ERR_PTR(-EIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006467 }
NeilBrown91adb562009-03-31 14:39:39 +11006468 if ((mddev->new_level == 5
6469 && !algorithm_valid_raid5(mddev->new_layout)) ||
6470 (mddev->new_level == 6
6471 && !algorithm_valid_raid6(mddev->new_layout))) {
NeilBrown0c55e022010-05-03 14:09:02 +10006472 printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
NeilBrown91adb562009-03-31 14:39:39 +11006473 mdname(mddev), mddev->new_layout);
6474 return ERR_PTR(-EIO);
6475 }
6476 if (mddev->new_level == 6 && mddev->raid_disks < 4) {
NeilBrown0c55e022010-05-03 14:09:02 +10006477 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
NeilBrown91adb562009-03-31 14:39:39 +11006478 mdname(mddev), mddev->raid_disks);
6479 return ERR_PTR(-EINVAL);
NeilBrown99c0fb52009-03-31 14:39:38 +11006480 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006481
Andre Noll664e7c42009-06-18 08:45:27 +10006482 if (!mddev->new_chunk_sectors ||
6483 (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
6484 !is_power_of_2(mddev->new_chunk_sectors)) {
NeilBrown0c55e022010-05-03 14:09:02 +10006485 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
6486 mdname(mddev), mddev->new_chunk_sectors << 9);
NeilBrown91adb562009-03-31 14:39:39 +11006487 return ERR_PTR(-EINVAL);
NeilBrown4bbf3772008-10-13 11:55:12 +11006488 }
6489
NeilBrownd1688a62011-10-11 16:49:52 +11006490 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
NeilBrown91adb562009-03-31 14:39:39 +11006491 if (conf == NULL)
6492 goto abort;
Shaohua Li851c30c2013-08-28 14:30:16 +08006493 /* Don't enable multi-threading by default*/
majianpeng60aaf932013-11-14 15:16:20 +11006494 if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
6495 &new_group)) {
6496 conf->group_cnt = group_cnt;
6497 conf->worker_cnt_per_group = worker_cnt_per_group;
6498 conf->worker_groups = new_group;
6499 } else
Shaohua Li851c30c2013-08-28 14:30:16 +08006500 goto abort;
Dan Williamsf5efd452009-10-16 15:55:38 +11006501 spin_lock_init(&conf->device_lock);
NeilBrownc46501b2013-08-27 15:52:13 +10006502 seqcount_init(&conf->gen_lock);
NeilBrown2d5b5692015-07-06 12:49:23 +10006503 mutex_init(&conf->cache_size_mutex);
Yuanhan Liub1b46482015-05-08 18:19:06 +10006504 init_waitqueue_head(&conf->wait_for_quiescent);
Yuanhan Liue9e4c372015-05-08 18:19:07 +10006505 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) {
6506 init_waitqueue_head(&conf->wait_for_stripe[i]);
6507 }
Dan Williamsf5efd452009-10-16 15:55:38 +11006508 init_waitqueue_head(&conf->wait_for_overlap);
6509 INIT_LIST_HEAD(&conf->handle_list);
6510 INIT_LIST_HEAD(&conf->hold_list);
6511 INIT_LIST_HEAD(&conf->delayed_list);
6512 INIT_LIST_HEAD(&conf->bitmap_list);
NeilBrownc3cce6c2015-08-14 12:47:33 +10006513 bio_list_init(&conf->return_bi);
Shaohua Li773ca822013-08-27 17:50:39 +08006514 init_llist_head(&conf->released_stripes);
Dan Williamsf5efd452009-10-16 15:55:38 +11006515 atomic_set(&conf->active_stripes, 0);
6516 atomic_set(&conf->preread_active_stripes, 0);
6517 atomic_set(&conf->active_aligned_reads, 0);
6518 conf->bypass_threshold = BYPASS_THRESHOLD;
NeilBrownd890fa22011-10-26 11:54:39 +11006519 conf->recovery_disabled = mddev->recovery_disabled - 1;
NeilBrown91adb562009-03-31 14:39:39 +11006520
6521 conf->raid_disks = mddev->raid_disks;
6522 if (mddev->reshape_position == MaxSector)
6523 conf->previous_raid_disks = mddev->raid_disks;
6524 else
6525 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
NeilBrown5e5e3e72009-10-16 16:35:30 +11006526 max_disks = max(conf->raid_disks, conf->previous_raid_disks);
NeilBrown91adb562009-03-31 14:39:39 +11006527
NeilBrown5e5e3e72009-10-16 16:35:30 +11006528 conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
NeilBrown91adb562009-03-31 14:39:39 +11006529 GFP_KERNEL);
6530 if (!conf->disks)
6531 goto abort;
6532
6533 conf->mddev = mddev;
6534
6535 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
6536 goto abort;
6537
Shaohua Li566c09c2013-11-14 15:16:17 +11006538 /* We init hash_locks[0] separately to that it can be used
6539 * as the reference lock in the spin_lock_nest_lock() call
6540 * in lock_all_device_hash_locks_irq in order to convince
6541 * lockdep that we know what we are doing.
6542 */
6543 spin_lock_init(conf->hash_locks);
6544 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
6545 spin_lock_init(conf->hash_locks + i);
6546
6547 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
6548 INIT_LIST_HEAD(conf->inactive_list + i);
6549
6550 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
6551 INIT_LIST_HEAD(conf->temp_inactive_list + i);
6552
Dan Williams36d1c642009-07-14 11:48:22 -07006553 conf->level = mddev->new_level;
shli@kernel.org46d5b782014-12-15 12:57:02 +11006554 conf->chunk_sectors = mddev->new_chunk_sectors;
Dan Williams36d1c642009-07-14 11:48:22 -07006555 if (raid5_alloc_percpu(conf) != 0)
6556 goto abort;
6557
NeilBrown0c55e022010-05-03 14:09:02 +10006558 pr_debug("raid456: run(%s) called.\n", mdname(mddev));
NeilBrown91adb562009-03-31 14:39:39 +11006559
NeilBrowndafb20f2012-03-19 12:46:39 +11006560 rdev_for_each(rdev, mddev) {
NeilBrown91adb562009-03-31 14:39:39 +11006561 raid_disk = rdev->raid_disk;
NeilBrown5e5e3e72009-10-16 16:35:30 +11006562 if (raid_disk >= max_disks
Shaohua Lif2076e72015-10-08 21:54:12 -07006563 || raid_disk < 0 || test_bit(Journal, &rdev->flags))
NeilBrown91adb562009-03-31 14:39:39 +11006564 continue;
6565 disk = conf->disks + raid_disk;
6566
NeilBrown17045f52011-12-23 10:17:53 +11006567 if (test_bit(Replacement, &rdev->flags)) {
6568 if (disk->replacement)
6569 goto abort;
6570 disk->replacement = rdev;
6571 } else {
6572 if (disk->rdev)
6573 goto abort;
6574 disk->rdev = rdev;
6575 }
NeilBrown91adb562009-03-31 14:39:39 +11006576
6577 if (test_bit(In_sync, &rdev->flags)) {
6578 char b[BDEVNAME_SIZE];
NeilBrown0c55e022010-05-03 14:09:02 +10006579 printk(KERN_INFO "md/raid:%s: device %s operational as raid"
6580 " disk %d\n",
6581 mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
Jonathan Brassowd6b212f2011-06-08 18:00:28 -05006582 } else if (rdev->saved_raid_disk != raid_disk)
NeilBrown91adb562009-03-31 14:39:39 +11006583 /* Cannot rely on bitmap to complete recovery */
6584 conf->fullsync = 1;
6585 }
6586
NeilBrown91adb562009-03-31 14:39:39 +11006587 conf->level = mddev->new_level;
Markus Stockhausen584acdd2014-12-15 12:57:05 +11006588 if (conf->level == 6) {
NeilBrown91adb562009-03-31 14:39:39 +11006589 conf->max_degraded = 2;
Markus Stockhausen584acdd2014-12-15 12:57:05 +11006590 if (raid6_call.xor_syndrome)
6591 conf->rmw_level = PARITY_ENABLE_RMW;
6592 else
6593 conf->rmw_level = PARITY_DISABLE_RMW;
6594 } else {
NeilBrown91adb562009-03-31 14:39:39 +11006595 conf->max_degraded = 1;
Markus Stockhausen584acdd2014-12-15 12:57:05 +11006596 conf->rmw_level = PARITY_ENABLE_RMW;
6597 }
NeilBrown91adb562009-03-31 14:39:39 +11006598 conf->algorithm = mddev->new_layout;
NeilBrownfef9c612009-03-31 15:16:46 +11006599 conf->reshape_progress = mddev->reshape_position;
NeilBrowne183eae2009-03-31 15:20:22 +11006600 if (conf->reshape_progress != MaxSector) {
Andre Noll09c9e5f2009-06-18 08:45:55 +10006601 conf->prev_chunk_sectors = mddev->chunk_sectors;
NeilBrowne183eae2009-03-31 15:20:22 +11006602 conf->prev_algo = mddev->layout;
NeilBrown5cac6bc2015-07-17 12:17:50 +10006603 } else {
6604 conf->prev_chunk_sectors = conf->chunk_sectors;
6605 conf->prev_algo = conf->algorithm;
NeilBrowne183eae2009-03-31 15:20:22 +11006606 }
NeilBrown91adb562009-03-31 14:39:39 +11006607
NeilBrownedbe83a2015-02-26 12:47:56 +11006608 conf->min_nr_stripes = NR_STRIPES;
6609 memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
NeilBrown5e5e3e72009-10-16 16:35:30 +11006610 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
Shaohua Li4bda5562013-11-14 15:16:17 +11006611 atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
NeilBrownedbe83a2015-02-26 12:47:56 +11006612 if (grow_stripes(conf, conf->min_nr_stripes)) {
NeilBrown91adb562009-03-31 14:39:39 +11006613 printk(KERN_ERR
NeilBrown0c55e022010-05-03 14:09:02 +10006614 "md/raid:%s: couldn't allocate %dkB for buffers\n",
6615 mdname(mddev), memory);
NeilBrown91adb562009-03-31 14:39:39 +11006616 goto abort;
6617 } else
NeilBrown0c55e022010-05-03 14:09:02 +10006618 printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
6619 mdname(mddev), memory);
NeilBrownedbe83a2015-02-26 12:47:56 +11006620 /*
6621 * Losing a stripe head costs more than the time to refill it,
6622 * it reduces the queue depth and so can hurt throughput.
6623 * So set it rather large, scaled by number of devices.
6624 */
6625 conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4;
6626 conf->shrinker.scan_objects = raid5_cache_scan;
6627 conf->shrinker.count_objects = raid5_cache_count;
6628 conf->shrinker.batch = 128;
6629 conf->shrinker.flags = 0;
6630 register_shrinker(&conf->shrinker);
NeilBrown91adb562009-03-31 14:39:39 +11006631
NeilBrown02326052012-07-03 15:56:52 +10006632 sprintf(pers_name, "raid%d", mddev->new_level);
6633 conf->thread = md_register_thread(raid5d, mddev, pers_name);
NeilBrown91adb562009-03-31 14:39:39 +11006634 if (!conf->thread) {
6635 printk(KERN_ERR
NeilBrown0c55e022010-05-03 14:09:02 +10006636 "md/raid:%s: couldn't allocate thread.\n",
NeilBrown91adb562009-03-31 14:39:39 +11006637 mdname(mddev));
6638 goto abort;
6639 }
6640
6641 return conf;
6642
6643 abort:
6644 if (conf) {
Dan Williams95fc17a2009-07-31 12:39:15 +10006645 free_conf(conf);
NeilBrown91adb562009-03-31 14:39:39 +11006646 return ERR_PTR(-EIO);
6647 } else
6648 return ERR_PTR(-ENOMEM);
6649}
6650
NeilBrownc148ffd2009-11-13 17:47:00 +11006651static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
6652{
6653 switch (algo) {
6654 case ALGORITHM_PARITY_0:
6655 if (raid_disk < max_degraded)
6656 return 1;
6657 break;
6658 case ALGORITHM_PARITY_N:
6659 if (raid_disk >= raid_disks - max_degraded)
6660 return 1;
6661 break;
6662 case ALGORITHM_PARITY_0_6:
NeilBrownf72ffdd2014-09-30 14:23:59 +10006663 if (raid_disk == 0 ||
NeilBrownc148ffd2009-11-13 17:47:00 +11006664 raid_disk == raid_disks - 1)
6665 return 1;
6666 break;
6667 case ALGORITHM_LEFT_ASYMMETRIC_6:
6668 case ALGORITHM_RIGHT_ASYMMETRIC_6:
6669 case ALGORITHM_LEFT_SYMMETRIC_6:
6670 case ALGORITHM_RIGHT_SYMMETRIC_6:
6671 if (raid_disk == raid_disks - 1)
6672 return 1;
6673 }
6674 return 0;
6675}
6676
NeilBrownfd01b882011-10-11 16:47:53 +11006677static int run(struct mddev *mddev)
NeilBrown91adb562009-03-31 14:39:39 +11006678{
NeilBrownd1688a62011-10-11 16:49:52 +11006679 struct r5conf *conf;
NeilBrown9f7c2222010-07-26 12:04:13 +10006680 int working_disks = 0;
NeilBrownc148ffd2009-11-13 17:47:00 +11006681 int dirty_parity_disks = 0;
NeilBrown3cb03002011-10-11 16:45:26 +11006682 struct md_rdev *rdev;
Shaohua Li713cf5a2015-08-13 14:32:03 -07006683 struct md_rdev *journal_dev = NULL;
NeilBrownc148ffd2009-11-13 17:47:00 +11006684 sector_t reshape_offset = 0;
NeilBrown17045f52011-12-23 10:17:53 +11006685 int i;
NeilBrownb5254dd2012-05-21 09:27:01 +10006686 long long min_offset_diff = 0;
6687 int first = 1;
NeilBrown91adb562009-03-31 14:39:39 +11006688
Andre Noll8c6ac862009-06-18 08:48:06 +10006689 if (mddev->recovery_cp != MaxSector)
NeilBrown0c55e022010-05-03 14:09:02 +10006690 printk(KERN_NOTICE "md/raid:%s: not clean"
Andre Noll8c6ac862009-06-18 08:48:06 +10006691 " -- starting background reconstruction\n",
6692 mdname(mddev));
NeilBrownb5254dd2012-05-21 09:27:01 +10006693
6694 rdev_for_each(rdev, mddev) {
6695 long long diff;
Shaohua Li713cf5a2015-08-13 14:32:03 -07006696
Shaohua Lif2076e72015-10-08 21:54:12 -07006697 if (test_bit(Journal, &rdev->flags)) {
Shaohua Li713cf5a2015-08-13 14:32:03 -07006698 journal_dev = rdev;
Shaohua Lif2076e72015-10-08 21:54:12 -07006699 continue;
6700 }
NeilBrownb5254dd2012-05-21 09:27:01 +10006701 if (rdev->raid_disk < 0)
6702 continue;
6703 diff = (rdev->new_data_offset - rdev->data_offset);
6704 if (first) {
6705 min_offset_diff = diff;
6706 first = 0;
6707 } else if (mddev->reshape_backwards &&
6708 diff < min_offset_diff)
6709 min_offset_diff = diff;
6710 else if (!mddev->reshape_backwards &&
6711 diff > min_offset_diff)
6712 min_offset_diff = diff;
6713 }
6714
NeilBrownf6705572006-03-27 01:18:11 -08006715 if (mddev->reshape_position != MaxSector) {
6716 /* Check that we can continue the reshape.
NeilBrownb5254dd2012-05-21 09:27:01 +10006717 * Difficulties arise if the stripe we would write to
6718 * next is at or after the stripe we would read from next.
6719 * For a reshape that changes the number of devices, this
6720 * is only possible for a very short time, and mdadm makes
6721 * sure that time appears to have past before assembling
6722 * the array. So we fail if that time hasn't passed.
6723 * For a reshape that keeps the number of devices the same
6724 * mdadm must be monitoring the reshape can keeping the
6725 * critical areas read-only and backed up. It will start
6726 * the array in read-only mode, so we check for that.
NeilBrownf6705572006-03-27 01:18:11 -08006727 */
6728 sector_t here_new, here_old;
6729 int old_disks;
Andre Noll18b00332009-03-31 15:00:56 +11006730 int max_degraded = (mddev->level == 6 ? 2 : 1);
NeilBrown05256d92015-07-15 17:36:21 +10006731 int chunk_sectors;
6732 int new_data_disks;
NeilBrownf6705572006-03-27 01:18:11 -08006733
Shaohua Li713cf5a2015-08-13 14:32:03 -07006734 if (journal_dev) {
6735 printk(KERN_ERR "md/raid:%s: don't support reshape with journal - aborting.\n",
6736 mdname(mddev));
6737 return -EINVAL;
6738 }
6739
NeilBrown88ce4932009-03-31 15:24:23 +11006740 if (mddev->new_level != mddev->level) {
NeilBrown0c55e022010-05-03 14:09:02 +10006741 printk(KERN_ERR "md/raid:%s: unsupported reshape "
NeilBrownf4168852007-02-28 20:11:53 -08006742 "required - aborting.\n",
NeilBrownf6705572006-03-27 01:18:11 -08006743 mdname(mddev));
6744 return -EINVAL;
6745 }
NeilBrownf6705572006-03-27 01:18:11 -08006746 old_disks = mddev->raid_disks - mddev->delta_disks;
6747 /* reshape_position must be on a new-stripe boundary, and one
NeilBrownf4168852007-02-28 20:11:53 -08006748 * further up in new geometry must map after here in old
6749 * geometry.
NeilBrown05256d92015-07-15 17:36:21 +10006750 * If the chunk sizes are different, then as we perform reshape
6751 * in units of the largest of the two, reshape_position needs
6752 * be a multiple of the largest chunk size times new data disks.
NeilBrownf6705572006-03-27 01:18:11 -08006753 */
6754 here_new = mddev->reshape_position;
NeilBrown05256d92015-07-15 17:36:21 +10006755 chunk_sectors = max(mddev->chunk_sectors, mddev->new_chunk_sectors);
6756 new_data_disks = mddev->raid_disks - max_degraded;
6757 if (sector_div(here_new, chunk_sectors * new_data_disks)) {
NeilBrown0c55e022010-05-03 14:09:02 +10006758 printk(KERN_ERR "md/raid:%s: reshape_position not "
6759 "on a stripe boundary\n", mdname(mddev));
NeilBrownf6705572006-03-27 01:18:11 -08006760 return -EINVAL;
6761 }
NeilBrown05256d92015-07-15 17:36:21 +10006762 reshape_offset = here_new * chunk_sectors;
NeilBrownf6705572006-03-27 01:18:11 -08006763 /* here_new is the stripe we will write to */
6764 here_old = mddev->reshape_position;
NeilBrown05256d92015-07-15 17:36:21 +10006765 sector_div(here_old, chunk_sectors * (old_disks-max_degraded));
NeilBrownf4168852007-02-28 20:11:53 -08006766 /* here_old is the first stripe that we might need to read
6767 * from */
NeilBrown67ac6012009-08-13 10:06:24 +10006768 if (mddev->delta_disks == 0) {
6769 /* We cannot be sure it is safe to start an in-place
NeilBrownb5254dd2012-05-21 09:27:01 +10006770 * reshape. It is only safe if user-space is monitoring
NeilBrown67ac6012009-08-13 10:06:24 +10006771 * and taking constant backups.
6772 * mdadm always starts a situation like this in
6773 * readonly mode so it can take control before
6774 * allowing any writes. So just check for that.
6775 */
NeilBrownb5254dd2012-05-21 09:27:01 +10006776 if (abs(min_offset_diff) >= mddev->chunk_sectors &&
6777 abs(min_offset_diff) >= mddev->new_chunk_sectors)
6778 /* not really in-place - so OK */;
6779 else if (mddev->ro == 0) {
6780 printk(KERN_ERR "md/raid:%s: in-place reshape "
6781 "must be started in read-only mode "
6782 "- aborting\n",
NeilBrown0c55e022010-05-03 14:09:02 +10006783 mdname(mddev));
NeilBrown67ac6012009-08-13 10:06:24 +10006784 return -EINVAL;
6785 }
NeilBrown2c810cd2012-05-21 09:27:00 +10006786 } else if (mddev->reshape_backwards
NeilBrown05256d92015-07-15 17:36:21 +10006787 ? (here_new * chunk_sectors + min_offset_diff <=
6788 here_old * chunk_sectors)
6789 : (here_new * chunk_sectors >=
6790 here_old * chunk_sectors + (-min_offset_diff))) {
NeilBrownf6705572006-03-27 01:18:11 -08006791 /* Reading from the same stripe as writing to - bad */
NeilBrown0c55e022010-05-03 14:09:02 +10006792 printk(KERN_ERR "md/raid:%s: reshape_position too early for "
6793 "auto-recovery - aborting.\n",
6794 mdname(mddev));
NeilBrownf6705572006-03-27 01:18:11 -08006795 return -EINVAL;
6796 }
NeilBrown0c55e022010-05-03 14:09:02 +10006797 printk(KERN_INFO "md/raid:%s: reshape will continue\n",
6798 mdname(mddev));
NeilBrownf6705572006-03-27 01:18:11 -08006799 /* OK, we should be able to continue; */
NeilBrownf6705572006-03-27 01:18:11 -08006800 } else {
NeilBrown91adb562009-03-31 14:39:39 +11006801 BUG_ON(mddev->level != mddev->new_level);
6802 BUG_ON(mddev->layout != mddev->new_layout);
Andre Noll664e7c42009-06-18 08:45:27 +10006803 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
NeilBrown91adb562009-03-31 14:39:39 +11006804 BUG_ON(mddev->delta_disks != 0);
NeilBrownf6705572006-03-27 01:18:11 -08006805 }
6806
NeilBrown245f46c2009-03-31 14:39:39 +11006807 if (mddev->private == NULL)
6808 conf = setup_conf(mddev);
6809 else
6810 conf = mddev->private;
6811
NeilBrown91adb562009-03-31 14:39:39 +11006812 if (IS_ERR(conf))
6813 return PTR_ERR(conf);
NeilBrown9ffae0c2006-01-06 00:20:32 -08006814
Shaohua Li7dde2ad2015-10-08 21:54:10 -07006815 if (test_bit(MD_HAS_JOURNAL, &mddev->flags) && !journal_dev) {
6816 printk(KERN_ERR "md/raid:%s: journal disk is missing, force array readonly\n",
6817 mdname(mddev));
6818 mddev->ro = 1;
6819 set_disk_ro(mddev->gendisk, 1);
6820 }
6821
NeilBrownb5254dd2012-05-21 09:27:01 +10006822 conf->min_offset_diff = min_offset_diff;
NeilBrown91adb562009-03-31 14:39:39 +11006823 mddev->thread = conf->thread;
6824 conf->thread = NULL;
6825 mddev->private = conf;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006826
NeilBrown17045f52011-12-23 10:17:53 +11006827 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
6828 i++) {
6829 rdev = conf->disks[i].rdev;
6830 if (!rdev && conf->disks[i].replacement) {
6831 /* The replacement is all we have yet */
6832 rdev = conf->disks[i].replacement;
6833 conf->disks[i].replacement = NULL;
6834 clear_bit(Replacement, &rdev->flags);
6835 conf->disks[i].rdev = rdev;
6836 }
6837 if (!rdev)
NeilBrownc148ffd2009-11-13 17:47:00 +11006838 continue;
NeilBrown17045f52011-12-23 10:17:53 +11006839 if (conf->disks[i].replacement &&
6840 conf->reshape_progress != MaxSector) {
6841 /* replacements and reshape simply do not mix. */
6842 printk(KERN_ERR "md: cannot handle concurrent "
6843 "replacement and reshape.\n");
6844 goto abort;
6845 }
NeilBrown2f115882010-06-17 17:41:03 +10006846 if (test_bit(In_sync, &rdev->flags)) {
NeilBrown91adb562009-03-31 14:39:39 +11006847 working_disks++;
NeilBrown2f115882010-06-17 17:41:03 +10006848 continue;
6849 }
NeilBrownc148ffd2009-11-13 17:47:00 +11006850 /* This disc is not fully in-sync. However if it
6851 * just stored parity (beyond the recovery_offset),
6852 * when we don't need to be concerned about the
6853 * array being dirty.
6854 * When reshape goes 'backwards', we never have
6855 * partially completed devices, so we only need
6856 * to worry about reshape going forwards.
6857 */
6858 /* Hack because v0.91 doesn't store recovery_offset properly. */
6859 if (mddev->major_version == 0 &&
6860 mddev->minor_version > 90)
6861 rdev->recovery_offset = reshape_offset;
H. Peter Anvin5026d7a2013-06-12 07:37:43 -07006862
NeilBrownc148ffd2009-11-13 17:47:00 +11006863 if (rdev->recovery_offset < reshape_offset) {
6864 /* We need to check old and new layout */
6865 if (!only_parity(rdev->raid_disk,
6866 conf->algorithm,
6867 conf->raid_disks,
6868 conf->max_degraded))
6869 continue;
6870 }
6871 if (!only_parity(rdev->raid_disk,
6872 conf->prev_algo,
6873 conf->previous_raid_disks,
6874 conf->max_degraded))
6875 continue;
6876 dirty_parity_disks++;
6877 }
NeilBrown91adb562009-03-31 14:39:39 +11006878
NeilBrown17045f52011-12-23 10:17:53 +11006879 /*
6880 * 0 for a fully functional array, 1 or 2 for a degraded array.
6881 */
NeilBrown908f4fb2011-12-23 10:17:50 +11006882 mddev->degraded = calc_degraded(conf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006883
NeilBrown674806d2010-06-16 17:17:53 +10006884 if (has_failed(conf)) {
NeilBrown0c55e022010-05-03 14:09:02 +10006885 printk(KERN_ERR "md/raid:%s: not enough operational devices"
Linus Torvalds1da177e2005-04-16 15:20:36 -07006886 " (%d/%d failed)\n",
NeilBrown02c2de82006-10-03 01:15:47 -07006887 mdname(mddev), mddev->degraded, conf->raid_disks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006888 goto abort;
6889 }
6890
NeilBrown91adb562009-03-31 14:39:39 +11006891 /* device size must be a multiple of chunk size */
Andre Noll9d8f0362009-06-18 08:45:01 +10006892 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
NeilBrown91adb562009-03-31 14:39:39 +11006893 mddev->resync_max_sectors = mddev->dev_sectors;
6894
NeilBrownc148ffd2009-11-13 17:47:00 +11006895 if (mddev->degraded > dirty_parity_disks &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07006896 mddev->recovery_cp != MaxSector) {
NeilBrown6ff8d8ec2006-01-06 00:20:15 -08006897 if (mddev->ok_start_degraded)
6898 printk(KERN_WARNING
NeilBrown0c55e022010-05-03 14:09:02 +10006899 "md/raid:%s: starting dirty degraded array"
6900 " - data corruption possible.\n",
NeilBrown6ff8d8ec2006-01-06 00:20:15 -08006901 mdname(mddev));
6902 else {
6903 printk(KERN_ERR
NeilBrown0c55e022010-05-03 14:09:02 +10006904 "md/raid:%s: cannot start dirty degraded array.\n",
NeilBrown6ff8d8ec2006-01-06 00:20:15 -08006905 mdname(mddev));
6906 goto abort;
6907 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006908 }
6909
Linus Torvalds1da177e2005-04-16 15:20:36 -07006910 if (mddev->degraded == 0)
NeilBrown0c55e022010-05-03 14:09:02 +10006911 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
6912 " devices, algorithm %d\n", mdname(mddev), conf->level,
NeilBrowne183eae2009-03-31 15:20:22 +11006913 mddev->raid_disks-mddev->degraded, mddev->raid_disks,
6914 mddev->new_layout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006915 else
NeilBrown0c55e022010-05-03 14:09:02 +10006916 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
6917 " out of %d devices, algorithm %d\n",
6918 mdname(mddev), conf->level,
6919 mddev->raid_disks - mddev->degraded,
6920 mddev->raid_disks, mddev->new_layout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006921
6922 print_raid5_conf(conf);
6923
NeilBrownfef9c612009-03-31 15:16:46 +11006924 if (conf->reshape_progress != MaxSector) {
NeilBrownfef9c612009-03-31 15:16:46 +11006925 conf->reshape_safe = conf->reshape_progress;
NeilBrownf6705572006-03-27 01:18:11 -08006926 atomic_set(&conf->reshape_stripes, 0);
6927 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
6928 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
6929 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
6930 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
6931 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
NeilBrown0da3c612009-09-23 18:09:45 +10006932 "reshape");
NeilBrownf6705572006-03-27 01:18:11 -08006933 }
6934
Linus Torvalds1da177e2005-04-16 15:20:36 -07006935 /* Ok, everything is just fine now */
NeilBrowna64c8762010-04-14 17:15:37 +10006936 if (mddev->to_remove == &raid5_attrs_group)
6937 mddev->to_remove = NULL;
NeilBrown00bcb4a2010-06-01 19:37:23 +10006938 else if (mddev->kobj.sd &&
6939 sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
NeilBrown5e55e2f2007-03-26 21:32:14 -08006940 printk(KERN_WARNING
NeilBrown4a5add42010-06-01 19:37:28 +10006941 "raid5: failed to create sysfs attributes for %s\n",
NeilBrown5e55e2f2007-03-26 21:32:14 -08006942 mdname(mddev));
NeilBrown4a5add42010-06-01 19:37:28 +10006943 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
6944
6945 if (mddev->queue) {
NeilBrown9f7c2222010-07-26 12:04:13 +10006946 int chunk_size;
Shaohua Li620125f2012-10-11 13:49:05 +11006947 bool discard_supported = true;
NeilBrown4a5add42010-06-01 19:37:28 +10006948 /* read-ahead size must cover two whole stripes, which
6949 * is 2 * (datadisks) * chunksize where 'n' is the
6950 * number of raid devices
6951 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006952 int data_disks = conf->previous_raid_disks - conf->max_degraded;
6953 int stripe = data_disks *
6954 ((mddev->chunk_sectors << 9) / PAGE_SIZE);
6955 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
6956 mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
NeilBrown4a5add42010-06-01 19:37:28 +10006957
NeilBrown9f7c2222010-07-26 12:04:13 +10006958 chunk_size = mddev->chunk_sectors << 9;
6959 blk_queue_io_min(mddev->queue, chunk_size);
6960 blk_queue_io_opt(mddev->queue, chunk_size *
6961 (conf->raid_disks - conf->max_degraded));
Kent Overstreetc78afc62013-07-11 22:39:53 -07006962 mddev->queue->limits.raid_partial_stripes_expensive = 1;
Shaohua Li620125f2012-10-11 13:49:05 +11006963 /*
6964 * We can only discard a whole stripe. It doesn't make sense to
6965 * discard data disk but write parity disk
6966 */
6967 stripe = stripe * PAGE_SIZE;
NeilBrown4ac68752012-11-19 13:11:26 +11006968 /* Round up to power of 2, as discard handling
6969 * currently assumes that */
6970 while ((stripe-1) & stripe)
6971 stripe = (stripe | (stripe-1)) + 1;
Shaohua Li620125f2012-10-11 13:49:05 +11006972 mddev->queue->limits.discard_alignment = stripe;
6973 mddev->queue->limits.discard_granularity = stripe;
6974 /*
6975 * unaligned part of discard request will be ignored, so can't
NeilBrown8e0e99b2014-10-02 13:45:00 +10006976 * guarantee discard_zeroes_data
Shaohua Li620125f2012-10-11 13:49:05 +11006977 */
6978 mddev->queue->limits.discard_zeroes_data = 0;
NeilBrown9f7c2222010-07-26 12:04:13 +10006979
H. Peter Anvin5026d7a2013-06-12 07:37:43 -07006980 blk_queue_max_write_same_sectors(mddev->queue, 0);
6981
NeilBrown05616be2012-05-21 09:27:00 +10006982 rdev_for_each(rdev, mddev) {
NeilBrown9f7c2222010-07-26 12:04:13 +10006983 disk_stack_limits(mddev->gendisk, rdev->bdev,
6984 rdev->data_offset << 9);
NeilBrown05616be2012-05-21 09:27:00 +10006985 disk_stack_limits(mddev->gendisk, rdev->bdev,
6986 rdev->new_data_offset << 9);
Shaohua Li620125f2012-10-11 13:49:05 +11006987 /*
6988 * discard_zeroes_data is required, otherwise data
6989 * could be lost. Consider a scenario: discard a stripe
6990 * (the stripe could be inconsistent if
6991 * discard_zeroes_data is 0); write one disk of the
6992 * stripe (the stripe could be inconsistent again
6993 * depending on which disks are used to calculate
6994 * parity); the disk is broken; The stripe data of this
6995 * disk is lost.
6996 */
6997 if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) ||
6998 !bdev_get_queue(rdev->bdev)->
6999 limits.discard_zeroes_data)
7000 discard_supported = false;
NeilBrown8e0e99b2014-10-02 13:45:00 +10007001 /* Unfortunately, discard_zeroes_data is not currently
7002 * a guarantee - just a hint. So we only allow DISCARD
7003 * if the sysadmin has confirmed that only safe devices
7004 * are in use by setting a module parameter.
7005 */
7006 if (!devices_handle_discard_safely) {
7007 if (discard_supported) {
7008 pr_info("md/raid456: discard support disabled due to uncertainty.\n");
7009 pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n");
7010 }
7011 discard_supported = false;
7012 }
NeilBrown05616be2012-05-21 09:27:00 +10007013 }
Shaohua Li620125f2012-10-11 13:49:05 +11007014
7015 if (discard_supported &&
7016 mddev->queue->limits.max_discard_sectors >= stripe &&
7017 mddev->queue->limits.discard_granularity >= stripe)
7018 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
7019 mddev->queue);
7020 else
7021 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
7022 mddev->queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007023 }
7024
Shaohua Li5c7e81c2015-08-13 14:32:04 -07007025 if (journal_dev) {
7026 char b[BDEVNAME_SIZE];
7027
7028 printk(KERN_INFO"md/raid:%s: using device %s as journal\n",
7029 mdname(mddev), bdevname(journal_dev->bdev, b));
7030 r5l_init_log(conf, journal_dev);
7031 }
7032
Linus Torvalds1da177e2005-04-16 15:20:36 -07007033 return 0;
7034abort:
NeilBrown01f96c02011-09-21 15:30:20 +10007035 md_unregister_thread(&mddev->thread);
NeilBrowne4f869d2011-10-07 14:22:49 +11007036 print_raid5_conf(conf);
7037 free_conf(conf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007038 mddev->private = NULL;
NeilBrown0c55e022010-05-03 14:09:02 +10007039 printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07007040 return -EIO;
7041}
7042
NeilBrownafa0f552014-12-15 12:56:58 +11007043static void raid5_free(struct mddev *mddev, void *priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007044{
NeilBrownafa0f552014-12-15 12:56:58 +11007045 struct r5conf *conf = priv;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007046
Dan Williams95fc17a2009-07-31 12:39:15 +10007047 free_conf(conf);
NeilBrowna64c8762010-04-14 17:15:37 +10007048 mddev->to_remove = &raid5_attrs_group;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007049}
7050
NeilBrownfd01b882011-10-11 16:47:53 +11007051static void status(struct seq_file *seq, struct mddev *mddev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007052{
NeilBrownd1688a62011-10-11 16:49:52 +11007053 struct r5conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007054 int i;
7055
Andre Noll9d8f0362009-06-18 08:45:01 +10007056 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
NeilBrown3cb5edf2015-07-15 17:24:17 +10007057 conf->chunk_sectors / 2, mddev->layout);
NeilBrown02c2de82006-10-03 01:15:47 -07007058 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007059 for (i = 0; i < conf->raid_disks; i++)
7060 seq_printf (seq, "%s",
7061 conf->disks[i].rdev &&
NeilBrownb2d444d2005-11-08 21:39:31 -08007062 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
Linus Torvalds1da177e2005-04-16 15:20:36 -07007063 seq_printf (seq, "]");
Linus Torvalds1da177e2005-04-16 15:20:36 -07007064}
7065
NeilBrownd1688a62011-10-11 16:49:52 +11007066static void print_raid5_conf (struct r5conf *conf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007067{
7068 int i;
7069 struct disk_info *tmp;
7070
NeilBrown0c55e022010-05-03 14:09:02 +10007071 printk(KERN_DEBUG "RAID conf printout:\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07007072 if (!conf) {
7073 printk("(conf==NULL)\n");
7074 return;
7075 }
NeilBrown0c55e022010-05-03 14:09:02 +10007076 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
7077 conf->raid_disks,
7078 conf->raid_disks - conf->mddev->degraded);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007079
7080 for (i = 0; i < conf->raid_disks; i++) {
7081 char b[BDEVNAME_SIZE];
7082 tmp = conf->disks + i;
7083 if (tmp->rdev)
NeilBrown0c55e022010-05-03 14:09:02 +10007084 printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
7085 i, !test_bit(Faulty, &tmp->rdev->flags),
7086 bdevname(tmp->rdev->bdev, b));
Linus Torvalds1da177e2005-04-16 15:20:36 -07007087 }
7088}
7089
NeilBrownfd01b882011-10-11 16:47:53 +11007090static int raid5_spare_active(struct mddev *mddev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007091{
7092 int i;
NeilBrownd1688a62011-10-11 16:49:52 +11007093 struct r5conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007094 struct disk_info *tmp;
NeilBrown6b965622010-08-18 11:56:59 +10007095 int count = 0;
7096 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007097
7098 for (i = 0; i < conf->raid_disks; i++) {
7099 tmp = conf->disks + i;
NeilBrowndd054fc2011-12-23 10:17:53 +11007100 if (tmp->replacement
7101 && tmp->replacement->recovery_offset == MaxSector
7102 && !test_bit(Faulty, &tmp->replacement->flags)
7103 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
7104 /* Replacement has just become active. */
7105 if (!tmp->rdev
7106 || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
7107 count++;
7108 if (tmp->rdev) {
7109 /* Replaced device not technically faulty,
7110 * but we need to be sure it gets removed
7111 * and never re-added.
7112 */
7113 set_bit(Faulty, &tmp->rdev->flags);
7114 sysfs_notify_dirent_safe(
7115 tmp->rdev->sysfs_state);
7116 }
7117 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
7118 } else if (tmp->rdev
NeilBrown70fffd02010-06-16 17:01:25 +10007119 && tmp->rdev->recovery_offset == MaxSector
NeilBrownb2d444d2005-11-08 21:39:31 -08007120 && !test_bit(Faulty, &tmp->rdev->flags)
NeilBrownc04be0a2006-10-03 01:15:53 -07007121 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
NeilBrown6b965622010-08-18 11:56:59 +10007122 count++;
Jonathan Brassow43c73ca2011-01-14 09:14:33 +11007123 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007124 }
7125 }
NeilBrown6b965622010-08-18 11:56:59 +10007126 spin_lock_irqsave(&conf->device_lock, flags);
NeilBrown908f4fb2011-12-23 10:17:50 +11007127 mddev->degraded = calc_degraded(conf);
NeilBrown6b965622010-08-18 11:56:59 +10007128 spin_unlock_irqrestore(&conf->device_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007129 print_raid5_conf(conf);
NeilBrown6b965622010-08-18 11:56:59 +10007130 return count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007131}
7132
NeilBrownb8321b62011-12-23 10:17:51 +11007133static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007134{
NeilBrownd1688a62011-10-11 16:49:52 +11007135 struct r5conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007136 int err = 0;
NeilBrownb8321b62011-12-23 10:17:51 +11007137 int number = rdev->raid_disk;
NeilBrown657e3e42011-12-23 10:17:52 +11007138 struct md_rdev **rdevp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007139 struct disk_info *p = conf->disks + number;
7140
7141 print_raid5_conf(conf);
Shaohua Lif6b6ec52015-12-21 10:51:02 +11007142 if (test_bit(Journal, &rdev->flags) && conf->log) {
7143 struct r5l_log *log;
Shaohua Lic2bb6242015-10-08 21:54:07 -07007144 /*
Shaohua Lif6b6ec52015-12-21 10:51:02 +11007145 * we can't wait pending write here, as this is called in
7146 * raid5d, wait will deadlock.
Shaohua Lic2bb6242015-10-08 21:54:07 -07007147 */
Shaohua Lif6b6ec52015-12-21 10:51:02 +11007148 if (atomic_read(&mddev->writes_pending))
7149 return -EBUSY;
7150 log = conf->log;
7151 conf->log = NULL;
7152 synchronize_rcu();
7153 r5l_exit_log(log);
7154 return 0;
Shaohua Lic2bb6242015-10-08 21:54:07 -07007155 }
NeilBrown657e3e42011-12-23 10:17:52 +11007156 if (rdev == p->rdev)
7157 rdevp = &p->rdev;
7158 else if (rdev == p->replacement)
7159 rdevp = &p->replacement;
7160 else
7161 return 0;
NeilBrownec32a2b2009-03-31 15:17:38 +11007162
NeilBrown657e3e42011-12-23 10:17:52 +11007163 if (number >= conf->raid_disks &&
7164 conf->reshape_progress == MaxSector)
7165 clear_bit(In_sync, &rdev->flags);
7166
7167 if (test_bit(In_sync, &rdev->flags) ||
7168 atomic_read(&rdev->nr_pending)) {
7169 err = -EBUSY;
7170 goto abort;
7171 }
7172 /* Only remove non-faulty devices if recovery
7173 * isn't possible.
7174 */
7175 if (!test_bit(Faulty, &rdev->flags) &&
7176 mddev->recovery_disabled != conf->recovery_disabled &&
7177 !has_failed(conf) &&
NeilBrowndd054fc2011-12-23 10:17:53 +11007178 (!p->replacement || p->replacement == rdev) &&
NeilBrown657e3e42011-12-23 10:17:52 +11007179 number < conf->raid_disks) {
7180 err = -EBUSY;
7181 goto abort;
7182 }
7183 *rdevp = NULL;
7184 synchronize_rcu();
7185 if (atomic_read(&rdev->nr_pending)) {
7186 /* lost the race, try later */
7187 err = -EBUSY;
7188 *rdevp = rdev;
NeilBrowndd054fc2011-12-23 10:17:53 +11007189 } else if (p->replacement) {
7190 /* We must have just cleared 'rdev' */
7191 p->rdev = p->replacement;
7192 clear_bit(Replacement, &p->replacement->flags);
7193 smp_mb(); /* Make sure other CPUs may see both as identical
7194 * but will never see neither - if they are careful
7195 */
7196 p->replacement = NULL;
7197 clear_bit(WantReplacement, &rdev->flags);
7198 } else
7199 /* We might have just removed the Replacement as faulty-
7200 * clear the bit just in case
7201 */
7202 clear_bit(WantReplacement, &rdev->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007203abort:
7204
7205 print_raid5_conf(conf);
7206 return err;
7207}
7208
NeilBrownfd01b882011-10-11 16:47:53 +11007209static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007210{
NeilBrownd1688a62011-10-11 16:49:52 +11007211 struct r5conf *conf = mddev->private;
Neil Brown199050e2008-06-28 08:31:33 +10007212 int err = -EEXIST;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007213 int disk;
7214 struct disk_info *p;
Neil Brown6c2fce22008-06-28 08:31:31 +10007215 int first = 0;
7216 int last = conf->raid_disks - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007217
Shaohua Lif6b6ec52015-12-21 10:51:02 +11007218 if (test_bit(Journal, &rdev->flags)) {
7219 char b[BDEVNAME_SIZE];
7220 if (conf->log)
7221 return -EBUSY;
7222
7223 rdev->raid_disk = 0;
7224 /*
7225 * The array is in readonly mode if journal is missing, so no
7226 * write requests running. We should be safe
7227 */
7228 r5l_init_log(conf, rdev);
7229 printk(KERN_INFO"md/raid:%s: using device %s as journal\n",
7230 mdname(mddev), bdevname(rdev->bdev, b));
7231 return 0;
7232 }
NeilBrown7f0da592011-07-28 11:39:22 +10007233 if (mddev->recovery_disabled == conf->recovery_disabled)
7234 return -EBUSY;
7235
NeilBrowndc10c642012-03-19 12:46:37 +11007236 if (rdev->saved_raid_disk < 0 && has_failed(conf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07007237 /* no point adding a device */
Neil Brown199050e2008-06-28 08:31:33 +10007238 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007239
Neil Brown6c2fce22008-06-28 08:31:31 +10007240 if (rdev->raid_disk >= 0)
7241 first = last = rdev->raid_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007242
7243 /*
NeilBrown16a53ec2006-06-26 00:27:38 -07007244 * find the disk ... but prefer rdev->saved_raid_disk
7245 * if possible.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007246 */
NeilBrown16a53ec2006-06-26 00:27:38 -07007247 if (rdev->saved_raid_disk >= 0 &&
Neil Brown6c2fce22008-06-28 08:31:31 +10007248 rdev->saved_raid_disk >= first &&
NeilBrown16a53ec2006-06-26 00:27:38 -07007249 conf->disks[rdev->saved_raid_disk].rdev == NULL)
NeilBrown5cfb22a2012-07-03 11:46:53 +10007250 first = rdev->saved_raid_disk;
7251
7252 for (disk = first; disk <= last; disk++) {
NeilBrown7bfec5f2011-12-23 10:17:53 +11007253 p = conf->disks + disk;
7254 if (p->rdev == NULL) {
NeilBrownb2d444d2005-11-08 21:39:31 -08007255 clear_bit(In_sync, &rdev->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007256 rdev->raid_disk = disk;
Neil Brown199050e2008-06-28 08:31:33 +10007257 err = 0;
NeilBrown72626682005-09-09 16:23:54 -07007258 if (rdev->saved_raid_disk != disk)
7259 conf->fullsync = 1;
Suzanne Woodd6065f72005-11-08 21:39:27 -08007260 rcu_assign_pointer(p->rdev, rdev);
NeilBrown5cfb22a2012-07-03 11:46:53 +10007261 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007262 }
NeilBrown5cfb22a2012-07-03 11:46:53 +10007263 }
7264 for (disk = first; disk <= last; disk++) {
7265 p = conf->disks + disk;
NeilBrown7bfec5f2011-12-23 10:17:53 +11007266 if (test_bit(WantReplacement, &p->rdev->flags) &&
7267 p->replacement == NULL) {
7268 clear_bit(In_sync, &rdev->flags);
7269 set_bit(Replacement, &rdev->flags);
7270 rdev->raid_disk = disk;
7271 err = 0;
7272 conf->fullsync = 1;
7273 rcu_assign_pointer(p->replacement, rdev);
7274 break;
7275 }
7276 }
NeilBrown5cfb22a2012-07-03 11:46:53 +10007277out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07007278 print_raid5_conf(conf);
Neil Brown199050e2008-06-28 08:31:33 +10007279 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007280}
7281
NeilBrownfd01b882011-10-11 16:47:53 +11007282static int raid5_resize(struct mddev *mddev, sector_t sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007283{
7284 /* no resync is happening, and there is enough space
7285 * on all devices, so we can resize.
7286 * We need to make sure resync covers any new space.
7287 * If the array is shrinking we should possibly wait until
7288 * any io in the removed space completes, but it hardly seems
7289 * worth it.
7290 */
NeilBrowna4a61252012-05-22 13:55:27 +10007291 sector_t newsize;
NeilBrown3cb5edf2015-07-15 17:24:17 +10007292 struct r5conf *conf = mddev->private;
7293
Shaohua Li713cf5a2015-08-13 14:32:03 -07007294 if (conf->log)
7295 return -EINVAL;
NeilBrown3cb5edf2015-07-15 17:24:17 +10007296 sectors &= ~((sector_t)conf->chunk_sectors - 1);
NeilBrowna4a61252012-05-22 13:55:27 +10007297 newsize = raid5_size(mddev, sectors, mddev->raid_disks);
7298 if (mddev->external_size &&
7299 mddev->array_sectors > newsize)
Dan Williamsb522adc2009-03-31 15:00:31 +11007300 return -EINVAL;
NeilBrowna4a61252012-05-22 13:55:27 +10007301 if (mddev->bitmap) {
7302 int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
7303 if (ret)
7304 return ret;
7305 }
7306 md_set_array_sectors(mddev, newsize);
Andre Nollf233ea52008-07-21 17:05:22 +10007307 set_capacity(mddev->gendisk, mddev->array_sectors);
NeilBrown449aad32009-08-03 10:59:58 +10007308 revalidate_disk(mddev->gendisk);
NeilBrownb0986362011-05-11 15:52:21 +10007309 if (sectors > mddev->dev_sectors &&
7310 mddev->recovery_cp > mddev->dev_sectors) {
Andre Noll58c0fed2009-03-31 14:33:13 +11007311 mddev->recovery_cp = mddev->dev_sectors;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007312 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
7313 }
Andre Noll58c0fed2009-03-31 14:33:13 +11007314 mddev->dev_sectors = sectors;
NeilBrown4b5c7ae2005-07-27 11:43:28 -07007315 mddev->resync_max_sectors = sectors;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007316 return 0;
7317}
7318
NeilBrownfd01b882011-10-11 16:47:53 +11007319static int check_stripe_cache(struct mddev *mddev)
NeilBrown01ee22b2009-06-18 08:47:20 +10007320{
7321 /* Can only proceed if there are plenty of stripe_heads.
7322 * We need a minimum of one full stripe,, and for sensible progress
7323 * it is best to have about 4 times that.
7324 * If we require 4 times, then the default 256 4K stripe_heads will
7325 * allow for chunk sizes up to 256K, which is probably OK.
7326 * If the chunk size is greater, user-space should request more
7327 * stripe_heads first.
7328 */
NeilBrownd1688a62011-10-11 16:49:52 +11007329 struct r5conf *conf = mddev->private;
NeilBrown01ee22b2009-06-18 08:47:20 +10007330 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
NeilBrownedbe83a2015-02-26 12:47:56 +11007331 > conf->min_nr_stripes ||
NeilBrown01ee22b2009-06-18 08:47:20 +10007332 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
NeilBrownedbe83a2015-02-26 12:47:56 +11007333 > conf->min_nr_stripes) {
NeilBrown0c55e022010-05-03 14:09:02 +10007334 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n",
7335 mdname(mddev),
NeilBrown01ee22b2009-06-18 08:47:20 +10007336 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
7337 / STRIPE_SIZE)*4);
7338 return 0;
7339 }
7340 return 1;
7341}
7342
NeilBrownfd01b882011-10-11 16:47:53 +11007343static int check_reshape(struct mddev *mddev)
NeilBrown29269552006-03-27 01:18:10 -08007344{
NeilBrownd1688a62011-10-11 16:49:52 +11007345 struct r5conf *conf = mddev->private;
NeilBrown29269552006-03-27 01:18:10 -08007346
Shaohua Li713cf5a2015-08-13 14:32:03 -07007347 if (conf->log)
7348 return -EINVAL;
NeilBrown88ce4932009-03-31 15:24:23 +11007349 if (mddev->delta_disks == 0 &&
7350 mddev->new_layout == mddev->layout &&
Andre Noll664e7c42009-06-18 08:45:27 +10007351 mddev->new_chunk_sectors == mddev->chunk_sectors)
NeilBrown50ac1682009-06-18 08:47:55 +10007352 return 0; /* nothing to do */
NeilBrown674806d2010-06-16 17:17:53 +10007353 if (has_failed(conf))
NeilBrownec32a2b2009-03-31 15:17:38 +11007354 return -EINVAL;
NeilBrownfdcfbbb2013-07-04 16:38:16 +10007355 if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
NeilBrownec32a2b2009-03-31 15:17:38 +11007356 /* We might be able to shrink, but the devices must
7357 * be made bigger first.
7358 * For raid6, 4 is the minimum size.
7359 * Otherwise 2 is the minimum
7360 */
7361 int min = 2;
7362 if (mddev->level == 6)
7363 min = 4;
7364 if (mddev->raid_disks + mddev->delta_disks < min)
7365 return -EINVAL;
7366 }
NeilBrown29269552006-03-27 01:18:10 -08007367
NeilBrown01ee22b2009-06-18 08:47:20 +10007368 if (!check_stripe_cache(mddev))
NeilBrown29269552006-03-27 01:18:10 -08007369 return -ENOSPC;
NeilBrown29269552006-03-27 01:18:10 -08007370
NeilBrown738a2732015-05-08 18:19:39 +10007371 if (mddev->new_chunk_sectors > mddev->chunk_sectors ||
7372 mddev->delta_disks > 0)
7373 if (resize_chunks(conf,
7374 conf->previous_raid_disks
7375 + max(0, mddev->delta_disks),
7376 max(mddev->new_chunk_sectors,
7377 mddev->chunk_sectors)
7378 ) < 0)
7379 return -ENOMEM;
NeilBrowne56108d62012-10-11 14:24:13 +11007380 return resize_stripes(conf, (conf->previous_raid_disks
7381 + mddev->delta_disks));
NeilBrown63c70c42006-03-27 01:18:13 -08007382}
7383
NeilBrownfd01b882011-10-11 16:47:53 +11007384static int raid5_start_reshape(struct mddev *mddev)
NeilBrown63c70c42006-03-27 01:18:13 -08007385{
NeilBrownd1688a62011-10-11 16:49:52 +11007386 struct r5conf *conf = mddev->private;
NeilBrown3cb03002011-10-11 16:45:26 +11007387 struct md_rdev *rdev;
NeilBrown63c70c42006-03-27 01:18:13 -08007388 int spares = 0;
NeilBrownc04be0a2006-10-03 01:15:53 -07007389 unsigned long flags;
NeilBrown63c70c42006-03-27 01:18:13 -08007390
NeilBrownf4168852007-02-28 20:11:53 -08007391 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
NeilBrown63c70c42006-03-27 01:18:13 -08007392 return -EBUSY;
7393
NeilBrown01ee22b2009-06-18 08:47:20 +10007394 if (!check_stripe_cache(mddev))
7395 return -ENOSPC;
7396
NeilBrown30b67642012-05-22 13:55:28 +10007397 if (has_failed(conf))
7398 return -EINVAL;
7399
NeilBrownc6563a82012-05-21 09:27:00 +10007400 rdev_for_each(rdev, mddev) {
NeilBrown469518a2011-01-31 11:57:43 +11007401 if (!test_bit(In_sync, &rdev->flags)
7402 && !test_bit(Faulty, &rdev->flags))
NeilBrown29269552006-03-27 01:18:10 -08007403 spares++;
NeilBrownc6563a82012-05-21 09:27:00 +10007404 }
NeilBrown63c70c42006-03-27 01:18:13 -08007405
NeilBrownf4168852007-02-28 20:11:53 -08007406 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
NeilBrown29269552006-03-27 01:18:10 -08007407 /* Not enough devices even to make a degraded array
7408 * of that size
7409 */
7410 return -EINVAL;
7411
NeilBrownec32a2b2009-03-31 15:17:38 +11007412 /* Refuse to reduce size of the array. Any reductions in
7413 * array size must be through explicit setting of array_size
7414 * attribute.
7415 */
7416 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
7417 < mddev->array_sectors) {
NeilBrown0c55e022010-05-03 14:09:02 +10007418 printk(KERN_ERR "md/raid:%s: array size must be reduced "
NeilBrownec32a2b2009-03-31 15:17:38 +11007419 "before number of disks\n", mdname(mddev));
7420 return -EINVAL;
7421 }
7422
NeilBrownf6705572006-03-27 01:18:11 -08007423 atomic_set(&conf->reshape_stripes, 0);
NeilBrown29269552006-03-27 01:18:10 -08007424 spin_lock_irq(&conf->device_lock);
NeilBrownc46501b2013-08-27 15:52:13 +10007425 write_seqcount_begin(&conf->gen_lock);
NeilBrown29269552006-03-27 01:18:10 -08007426 conf->previous_raid_disks = conf->raid_disks;
NeilBrown63c70c42006-03-27 01:18:13 -08007427 conf->raid_disks += mddev->delta_disks;
Andre Noll09c9e5f2009-06-18 08:45:55 +10007428 conf->prev_chunk_sectors = conf->chunk_sectors;
7429 conf->chunk_sectors = mddev->new_chunk_sectors;
NeilBrown88ce4932009-03-31 15:24:23 +11007430 conf->prev_algo = conf->algorithm;
7431 conf->algorithm = mddev->new_layout;
NeilBrown05616be2012-05-21 09:27:00 +10007432 conf->generation++;
7433 /* Code that selects data_offset needs to see the generation update
7434 * if reshape_progress has been set - so a memory barrier needed.
7435 */
7436 smp_mb();
NeilBrown2c810cd2012-05-21 09:27:00 +10007437 if (mddev->reshape_backwards)
NeilBrownfef9c612009-03-31 15:16:46 +11007438 conf->reshape_progress = raid5_size(mddev, 0, 0);
7439 else
7440 conf->reshape_progress = 0;
7441 conf->reshape_safe = conf->reshape_progress;
NeilBrownc46501b2013-08-27 15:52:13 +10007442 write_seqcount_end(&conf->gen_lock);
NeilBrown29269552006-03-27 01:18:10 -08007443 spin_unlock_irq(&conf->device_lock);
7444
NeilBrown4d77e3b2013-08-27 15:57:47 +10007445 /* Now make sure any requests that proceeded on the assumption
7446 * the reshape wasn't running - like Discard or Read - have
7447 * completed.
7448 */
7449 mddev_suspend(mddev);
7450 mddev_resume(mddev);
7451
NeilBrown29269552006-03-27 01:18:10 -08007452 /* Add some new drives, as many as will fit.
7453 * We know there are enough to make the newly sized array work.
NeilBrown3424bf62010-06-17 17:48:26 +10007454 * Don't add devices if we are reducing the number of
7455 * devices in the array. This is because it is not possible
7456 * to correctly record the "partially reconstructed" state of
7457 * such devices during the reshape and confusion could result.
NeilBrown29269552006-03-27 01:18:10 -08007458 */
NeilBrown87a8dec2011-01-31 11:57:43 +11007459 if (mddev->delta_disks >= 0) {
NeilBrowndafb20f2012-03-19 12:46:39 +11007460 rdev_for_each(rdev, mddev)
NeilBrown87a8dec2011-01-31 11:57:43 +11007461 if (rdev->raid_disk < 0 &&
7462 !test_bit(Faulty, &rdev->flags)) {
7463 if (raid5_add_disk(mddev, rdev) == 0) {
NeilBrown87a8dec2011-01-31 11:57:43 +11007464 if (rdev->raid_disk
NeilBrown9d4c7d82012-03-13 11:21:21 +11007465 >= conf->previous_raid_disks)
NeilBrown87a8dec2011-01-31 11:57:43 +11007466 set_bit(In_sync, &rdev->flags);
NeilBrown9d4c7d82012-03-13 11:21:21 +11007467 else
NeilBrown87a8dec2011-01-31 11:57:43 +11007468 rdev->recovery_offset = 0;
Namhyung Kim36fad852011-07-27 11:00:36 +10007469
7470 if (sysfs_link_rdev(mddev, rdev))
NeilBrown87a8dec2011-01-31 11:57:43 +11007471 /* Failure here is OK */;
NeilBrown50da0842011-01-31 11:57:43 +11007472 }
NeilBrown87a8dec2011-01-31 11:57:43 +11007473 } else if (rdev->raid_disk >= conf->previous_raid_disks
7474 && !test_bit(Faulty, &rdev->flags)) {
7475 /* This is a spare that was manually added */
7476 set_bit(In_sync, &rdev->flags);
NeilBrown87a8dec2011-01-31 11:57:43 +11007477 }
NeilBrown29269552006-03-27 01:18:10 -08007478
NeilBrown87a8dec2011-01-31 11:57:43 +11007479 /* When a reshape changes the number of devices,
7480 * ->degraded is measured against the larger of the
7481 * pre and post number of devices.
7482 */
NeilBrownec32a2b2009-03-31 15:17:38 +11007483 spin_lock_irqsave(&conf->device_lock, flags);
NeilBrown908f4fb2011-12-23 10:17:50 +11007484 mddev->degraded = calc_degraded(conf);
NeilBrownec32a2b2009-03-31 15:17:38 +11007485 spin_unlock_irqrestore(&conf->device_lock, flags);
7486 }
NeilBrown63c70c42006-03-27 01:18:13 -08007487 mddev->raid_disks = conf->raid_disks;
NeilBrowne5164022009-08-03 10:59:57 +10007488 mddev->reshape_position = conf->reshape_progress;
NeilBrown850b2b42006-10-03 01:15:46 -07007489 set_bit(MD_CHANGE_DEVS, &mddev->flags);
NeilBrownf6705572006-03-27 01:18:11 -08007490
NeilBrown29269552006-03-27 01:18:10 -08007491 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
7492 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
NeilBrownea358cd2015-06-12 20:05:04 +10007493 clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
NeilBrown29269552006-03-27 01:18:10 -08007494 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
7495 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
7496 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
NeilBrown0da3c612009-09-23 18:09:45 +10007497 "reshape");
NeilBrown29269552006-03-27 01:18:10 -08007498 if (!mddev->sync_thread) {
7499 mddev->recovery = 0;
7500 spin_lock_irq(&conf->device_lock);
NeilBrownba8805b2013-11-14 15:16:15 +11007501 write_seqcount_begin(&conf->gen_lock);
NeilBrown29269552006-03-27 01:18:10 -08007502 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
NeilBrownba8805b2013-11-14 15:16:15 +11007503 mddev->new_chunk_sectors =
7504 conf->chunk_sectors = conf->prev_chunk_sectors;
7505 mddev->new_layout = conf->algorithm = conf->prev_algo;
NeilBrown05616be2012-05-21 09:27:00 +10007506 rdev_for_each(rdev, mddev)
7507 rdev->new_data_offset = rdev->data_offset;
7508 smp_wmb();
NeilBrownba8805b2013-11-14 15:16:15 +11007509 conf->generation --;
NeilBrownfef9c612009-03-31 15:16:46 +11007510 conf->reshape_progress = MaxSector;
NeilBrown1e3fa9b2012-03-13 11:21:18 +11007511 mddev->reshape_position = MaxSector;
NeilBrownba8805b2013-11-14 15:16:15 +11007512 write_seqcount_end(&conf->gen_lock);
NeilBrown29269552006-03-27 01:18:10 -08007513 spin_unlock_irq(&conf->device_lock);
7514 return -EAGAIN;
7515 }
NeilBrownc8f517c2009-03-31 15:28:40 +11007516 conf->reshape_checkpoint = jiffies;
NeilBrown29269552006-03-27 01:18:10 -08007517 md_wakeup_thread(mddev->sync_thread);
7518 md_new_event(mddev);
7519 return 0;
7520}
NeilBrown29269552006-03-27 01:18:10 -08007521
NeilBrownec32a2b2009-03-31 15:17:38 +11007522/* This is called from the reshape thread and should make any
7523 * changes needed in 'conf'
7524 */
NeilBrownd1688a62011-10-11 16:49:52 +11007525static void end_reshape(struct r5conf *conf)
NeilBrown29269552006-03-27 01:18:10 -08007526{
NeilBrown29269552006-03-27 01:18:10 -08007527
NeilBrownf6705572006-03-27 01:18:11 -08007528 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
NeilBrown05616be2012-05-21 09:27:00 +10007529 struct md_rdev *rdev;
Dan Williams80c3a6c2009-03-17 18:10:40 -07007530
NeilBrownf6705572006-03-27 01:18:11 -08007531 spin_lock_irq(&conf->device_lock);
NeilBrowncea9c222009-03-31 15:15:05 +11007532 conf->previous_raid_disks = conf->raid_disks;
NeilBrown05616be2012-05-21 09:27:00 +10007533 rdev_for_each(rdev, conf->mddev)
7534 rdev->data_offset = rdev->new_data_offset;
7535 smp_wmb();
NeilBrownfef9c612009-03-31 15:16:46 +11007536 conf->reshape_progress = MaxSector;
NeilBrown6cbd8142015-07-24 13:30:32 +10007537 conf->mddev->reshape_position = MaxSector;
NeilBrownf6705572006-03-27 01:18:11 -08007538 spin_unlock_irq(&conf->device_lock);
NeilBrownb0f9ec02009-03-31 15:27:18 +11007539 wake_up(&conf->wait_for_overlap);
NeilBrown16a53ec2006-06-26 00:27:38 -07007540
7541 /* read-ahead size must cover two whole stripes, which is
7542 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
7543 */
NeilBrown4a5add42010-06-01 19:37:28 +10007544 if (conf->mddev->queue) {
NeilBrowncea9c222009-03-31 15:15:05 +11007545 int data_disks = conf->raid_disks - conf->max_degraded;
Andre Noll09c9e5f2009-06-18 08:45:55 +10007546 int stripe = data_disks * ((conf->chunk_sectors << 9)
NeilBrowncea9c222009-03-31 15:15:05 +11007547 / PAGE_SIZE);
NeilBrown16a53ec2006-06-26 00:27:38 -07007548 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
7549 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
7550 }
NeilBrown29269552006-03-27 01:18:10 -08007551 }
NeilBrown29269552006-03-27 01:18:10 -08007552}
7553
NeilBrownec32a2b2009-03-31 15:17:38 +11007554/* This is called from the raid5d thread with mddev_lock held.
7555 * It makes config changes to the device.
7556 */
NeilBrownfd01b882011-10-11 16:47:53 +11007557static void raid5_finish_reshape(struct mddev *mddev)
NeilBrowncea9c222009-03-31 15:15:05 +11007558{
NeilBrownd1688a62011-10-11 16:49:52 +11007559 struct r5conf *conf = mddev->private;
NeilBrowncea9c222009-03-31 15:15:05 +11007560
7561 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
7562
NeilBrownec32a2b2009-03-31 15:17:38 +11007563 if (mddev->delta_disks > 0) {
7564 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
7565 set_capacity(mddev->gendisk, mddev->array_sectors);
NeilBrown449aad32009-08-03 10:59:58 +10007566 revalidate_disk(mddev->gendisk);
NeilBrownec32a2b2009-03-31 15:17:38 +11007567 } else {
7568 int d;
NeilBrown908f4fb2011-12-23 10:17:50 +11007569 spin_lock_irq(&conf->device_lock);
7570 mddev->degraded = calc_degraded(conf);
7571 spin_unlock_irq(&conf->device_lock);
NeilBrownec32a2b2009-03-31 15:17:38 +11007572 for (d = conf->raid_disks ;
7573 d < conf->raid_disks - mddev->delta_disks;
NeilBrown1a67dde2009-08-13 10:41:49 +10007574 d++) {
NeilBrown3cb03002011-10-11 16:45:26 +11007575 struct md_rdev *rdev = conf->disks[d].rdev;
NeilBrownda7613b2012-05-22 13:55:33 +10007576 if (rdev)
7577 clear_bit(In_sync, &rdev->flags);
7578 rdev = conf->disks[d].replacement;
7579 if (rdev)
7580 clear_bit(In_sync, &rdev->flags);
NeilBrown1a67dde2009-08-13 10:41:49 +10007581 }
NeilBrowncea9c222009-03-31 15:15:05 +11007582 }
NeilBrown88ce4932009-03-31 15:24:23 +11007583 mddev->layout = conf->algorithm;
Andre Noll09c9e5f2009-06-18 08:45:55 +10007584 mddev->chunk_sectors = conf->chunk_sectors;
NeilBrownec32a2b2009-03-31 15:17:38 +11007585 mddev->reshape_position = MaxSector;
7586 mddev->delta_disks = 0;
NeilBrown2c810cd2012-05-21 09:27:00 +10007587 mddev->reshape_backwards = 0;
NeilBrowncea9c222009-03-31 15:15:05 +11007588 }
7589}
7590
NeilBrownfd01b882011-10-11 16:47:53 +11007591static void raid5_quiesce(struct mddev *mddev, int state)
NeilBrown72626682005-09-09 16:23:54 -07007592{
NeilBrownd1688a62011-10-11 16:49:52 +11007593 struct r5conf *conf = mddev->private;
NeilBrown72626682005-09-09 16:23:54 -07007594
7595 switch(state) {
NeilBrowne464eaf2006-03-27 01:18:14 -08007596 case 2: /* resume for a suspend */
7597 wake_up(&conf->wait_for_overlap);
7598 break;
7599
NeilBrown72626682005-09-09 16:23:54 -07007600 case 1: /* stop all writes */
Shaohua Li566c09c2013-11-14 15:16:17 +11007601 lock_all_device_hash_locks_irq(conf);
NeilBrown64bd6602009-08-03 10:59:58 +10007602 /* '2' tells resync/reshape to pause so that all
7603 * active stripes can drain
7604 */
7605 conf->quiesce = 2;
Yuanhan Liub1b46482015-05-08 18:19:06 +10007606 wait_event_cmd(conf->wait_for_quiescent,
Raz Ben-Jehuda(caro)46031f92006-12-10 02:20:47 -08007607 atomic_read(&conf->active_stripes) == 0 &&
7608 atomic_read(&conf->active_aligned_reads) == 0,
Shaohua Li566c09c2013-11-14 15:16:17 +11007609 unlock_all_device_hash_locks_irq(conf),
7610 lock_all_device_hash_locks_irq(conf));
NeilBrown64bd6602009-08-03 10:59:58 +10007611 conf->quiesce = 1;
Shaohua Li566c09c2013-11-14 15:16:17 +11007612 unlock_all_device_hash_locks_irq(conf);
NeilBrown64bd6602009-08-03 10:59:58 +10007613 /* allow reshape to continue */
7614 wake_up(&conf->wait_for_overlap);
NeilBrown72626682005-09-09 16:23:54 -07007615 break;
7616
7617 case 0: /* re-enable writes */
Shaohua Li566c09c2013-11-14 15:16:17 +11007618 lock_all_device_hash_locks_irq(conf);
NeilBrown72626682005-09-09 16:23:54 -07007619 conf->quiesce = 0;
Yuanhan Liub1b46482015-05-08 18:19:06 +10007620 wake_up(&conf->wait_for_quiescent);
NeilBrowne464eaf2006-03-27 01:18:14 -08007621 wake_up(&conf->wait_for_overlap);
Shaohua Li566c09c2013-11-14 15:16:17 +11007622 unlock_all_device_hash_locks_irq(conf);
NeilBrown72626682005-09-09 16:23:54 -07007623 break;
7624 }
Shaohua Lie6c033f2015-10-04 09:20:12 -07007625 r5l_quiesce(conf->log, state);
NeilBrown72626682005-09-09 16:23:54 -07007626}
NeilBrownb15c2e52006-01-06 00:20:16 -08007627
NeilBrownfd01b882011-10-11 16:47:53 +11007628static void *raid45_takeover_raid0(struct mddev *mddev, int level)
Trela Maciej54071b32010-03-08 16:02:42 +11007629{
NeilBrowne373ab12011-10-11 16:48:59 +11007630 struct r0conf *raid0_conf = mddev->private;
Randy Dunlapd76c8422011-04-21 09:07:26 -07007631 sector_t sectors;
Trela Maciej54071b32010-03-08 16:02:42 +11007632
Dan Williamsf1b29bc2010-05-01 18:09:05 -07007633 /* for raid0 takeover only one zone is supported */
NeilBrowne373ab12011-10-11 16:48:59 +11007634 if (raid0_conf->nr_strip_zones > 1) {
NeilBrown0c55e022010-05-03 14:09:02 +10007635 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
7636 mdname(mddev));
Dan Williamsf1b29bc2010-05-01 18:09:05 -07007637 return ERR_PTR(-EINVAL);
7638 }
7639
NeilBrowne373ab12011-10-11 16:48:59 +11007640 sectors = raid0_conf->strip_zone[0].zone_end;
7641 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
NeilBrown3b71bd92011-04-20 15:38:18 +10007642 mddev->dev_sectors = sectors;
Dan Williamsf1b29bc2010-05-01 18:09:05 -07007643 mddev->new_level = level;
Trela Maciej54071b32010-03-08 16:02:42 +11007644 mddev->new_layout = ALGORITHM_PARITY_N;
7645 mddev->new_chunk_sectors = mddev->chunk_sectors;
7646 mddev->raid_disks += 1;
7647 mddev->delta_disks = 1;
7648 /* make sure it will be not marked as dirty */
7649 mddev->recovery_cp = MaxSector;
7650
7651 return setup_conf(mddev);
7652}
7653
NeilBrownfd01b882011-10-11 16:47:53 +11007654static void *raid5_takeover_raid1(struct mddev *mddev)
NeilBrownd562b0c2009-03-31 14:39:39 +11007655{
7656 int chunksect;
7657
7658 if (mddev->raid_disks != 2 ||
7659 mddev->degraded > 1)
7660 return ERR_PTR(-EINVAL);
7661
7662 /* Should check if there are write-behind devices? */
7663
7664 chunksect = 64*2; /* 64K by default */
7665
7666 /* The array must be an exact multiple of chunksize */
7667 while (chunksect && (mddev->array_sectors & (chunksect-1)))
7668 chunksect >>= 1;
7669
7670 if ((chunksect<<9) < STRIPE_SIZE)
7671 /* array size does not allow a suitable chunk size */
7672 return ERR_PTR(-EINVAL);
7673
7674 mddev->new_level = 5;
7675 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
Andre Noll664e7c42009-06-18 08:45:27 +10007676 mddev->new_chunk_sectors = chunksect;
NeilBrownd562b0c2009-03-31 14:39:39 +11007677
7678 return setup_conf(mddev);
7679}
7680
NeilBrownfd01b882011-10-11 16:47:53 +11007681static void *raid5_takeover_raid6(struct mddev *mddev)
NeilBrownfc9739c2009-03-31 14:57:20 +11007682{
7683 int new_layout;
7684
7685 switch (mddev->layout) {
7686 case ALGORITHM_LEFT_ASYMMETRIC_6:
7687 new_layout = ALGORITHM_LEFT_ASYMMETRIC;
7688 break;
7689 case ALGORITHM_RIGHT_ASYMMETRIC_6:
7690 new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
7691 break;
7692 case ALGORITHM_LEFT_SYMMETRIC_6:
7693 new_layout = ALGORITHM_LEFT_SYMMETRIC;
7694 break;
7695 case ALGORITHM_RIGHT_SYMMETRIC_6:
7696 new_layout = ALGORITHM_RIGHT_SYMMETRIC;
7697 break;
7698 case ALGORITHM_PARITY_0_6:
7699 new_layout = ALGORITHM_PARITY_0;
7700 break;
7701 case ALGORITHM_PARITY_N:
7702 new_layout = ALGORITHM_PARITY_N;
7703 break;
7704 default:
7705 return ERR_PTR(-EINVAL);
7706 }
7707 mddev->new_level = 5;
7708 mddev->new_layout = new_layout;
7709 mddev->delta_disks = -1;
7710 mddev->raid_disks -= 1;
7711 return setup_conf(mddev);
7712}
7713
NeilBrownfd01b882011-10-11 16:47:53 +11007714static int raid5_check_reshape(struct mddev *mddev)
NeilBrownb3546032009-03-31 14:56:41 +11007715{
NeilBrown88ce4932009-03-31 15:24:23 +11007716 /* For a 2-drive array, the layout and chunk size can be changed
7717 * immediately as not restriping is needed.
7718 * For larger arrays we record the new value - after validation
7719 * to be used by a reshape pass.
NeilBrownb3546032009-03-31 14:56:41 +11007720 */
NeilBrownd1688a62011-10-11 16:49:52 +11007721 struct r5conf *conf = mddev->private;
NeilBrown597a7112009-06-18 08:47:42 +10007722 int new_chunk = mddev->new_chunk_sectors;
NeilBrownb3546032009-03-31 14:56:41 +11007723
NeilBrown597a7112009-06-18 08:47:42 +10007724 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
NeilBrownb3546032009-03-31 14:56:41 +11007725 return -EINVAL;
7726 if (new_chunk > 0) {
Andre Noll0ba459d2009-06-18 08:46:10 +10007727 if (!is_power_of_2(new_chunk))
NeilBrownb3546032009-03-31 14:56:41 +11007728 return -EINVAL;
NeilBrown597a7112009-06-18 08:47:42 +10007729 if (new_chunk < (PAGE_SIZE>>9))
NeilBrownb3546032009-03-31 14:56:41 +11007730 return -EINVAL;
NeilBrown597a7112009-06-18 08:47:42 +10007731 if (mddev->array_sectors & (new_chunk-1))
NeilBrownb3546032009-03-31 14:56:41 +11007732 /* not factor of array size */
7733 return -EINVAL;
7734 }
7735
7736 /* They look valid */
7737
NeilBrown88ce4932009-03-31 15:24:23 +11007738 if (mddev->raid_disks == 2) {
NeilBrown597a7112009-06-18 08:47:42 +10007739 /* can make the change immediately */
7740 if (mddev->new_layout >= 0) {
7741 conf->algorithm = mddev->new_layout;
7742 mddev->layout = mddev->new_layout;
NeilBrown88ce4932009-03-31 15:24:23 +11007743 }
7744 if (new_chunk > 0) {
NeilBrown597a7112009-06-18 08:47:42 +10007745 conf->chunk_sectors = new_chunk ;
7746 mddev->chunk_sectors = new_chunk;
NeilBrown88ce4932009-03-31 15:24:23 +11007747 }
7748 set_bit(MD_CHANGE_DEVS, &mddev->flags);
7749 md_wakeup_thread(mddev->thread);
NeilBrownb3546032009-03-31 14:56:41 +11007750 }
NeilBrown50ac1682009-06-18 08:47:55 +10007751 return check_reshape(mddev);
NeilBrown88ce4932009-03-31 15:24:23 +11007752}
7753
NeilBrownfd01b882011-10-11 16:47:53 +11007754static int raid6_check_reshape(struct mddev *mddev)
NeilBrown88ce4932009-03-31 15:24:23 +11007755{
NeilBrown597a7112009-06-18 08:47:42 +10007756 int new_chunk = mddev->new_chunk_sectors;
NeilBrown50ac1682009-06-18 08:47:55 +10007757
NeilBrown597a7112009-06-18 08:47:42 +10007758 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
NeilBrown88ce4932009-03-31 15:24:23 +11007759 return -EINVAL;
NeilBrownb3546032009-03-31 14:56:41 +11007760 if (new_chunk > 0) {
Andre Noll0ba459d2009-06-18 08:46:10 +10007761 if (!is_power_of_2(new_chunk))
NeilBrown88ce4932009-03-31 15:24:23 +11007762 return -EINVAL;
NeilBrown597a7112009-06-18 08:47:42 +10007763 if (new_chunk < (PAGE_SIZE >> 9))
NeilBrown88ce4932009-03-31 15:24:23 +11007764 return -EINVAL;
NeilBrown597a7112009-06-18 08:47:42 +10007765 if (mddev->array_sectors & (new_chunk-1))
NeilBrown88ce4932009-03-31 15:24:23 +11007766 /* not factor of array size */
7767 return -EINVAL;
NeilBrownb3546032009-03-31 14:56:41 +11007768 }
NeilBrown88ce4932009-03-31 15:24:23 +11007769
7770 /* They look valid */
NeilBrown50ac1682009-06-18 08:47:55 +10007771 return check_reshape(mddev);
NeilBrownb3546032009-03-31 14:56:41 +11007772}
7773
NeilBrownfd01b882011-10-11 16:47:53 +11007774static void *raid5_takeover(struct mddev *mddev)
NeilBrownd562b0c2009-03-31 14:39:39 +11007775{
7776 /* raid5 can take over:
Dan Williamsf1b29bc2010-05-01 18:09:05 -07007777 * raid0 - if there is only one strip zone - make it a raid4 layout
NeilBrownd562b0c2009-03-31 14:39:39 +11007778 * raid1 - if there are two drives. We need to know the chunk size
7779 * raid4 - trivial - just use a raid4 layout.
7780 * raid6 - Providing it is a *_6 layout
NeilBrownd562b0c2009-03-31 14:39:39 +11007781 */
Dan Williamsf1b29bc2010-05-01 18:09:05 -07007782 if (mddev->level == 0)
7783 return raid45_takeover_raid0(mddev, 5);
NeilBrownd562b0c2009-03-31 14:39:39 +11007784 if (mddev->level == 1)
7785 return raid5_takeover_raid1(mddev);
NeilBrowne9d47582009-03-31 14:57:09 +11007786 if (mddev->level == 4) {
7787 mddev->new_layout = ALGORITHM_PARITY_N;
7788 mddev->new_level = 5;
7789 return setup_conf(mddev);
7790 }
NeilBrownfc9739c2009-03-31 14:57:20 +11007791 if (mddev->level == 6)
7792 return raid5_takeover_raid6(mddev);
NeilBrownd562b0c2009-03-31 14:39:39 +11007793
7794 return ERR_PTR(-EINVAL);
7795}
7796
NeilBrownfd01b882011-10-11 16:47:53 +11007797static void *raid4_takeover(struct mddev *mddev)
NeilBrowna78d38a2010-03-22 16:53:49 +11007798{
Dan Williamsf1b29bc2010-05-01 18:09:05 -07007799 /* raid4 can take over:
7800 * raid0 - if there is only one strip zone
7801 * raid5 - if layout is right
NeilBrowna78d38a2010-03-22 16:53:49 +11007802 */
Dan Williamsf1b29bc2010-05-01 18:09:05 -07007803 if (mddev->level == 0)
7804 return raid45_takeover_raid0(mddev, 4);
NeilBrowna78d38a2010-03-22 16:53:49 +11007805 if (mddev->level == 5 &&
7806 mddev->layout == ALGORITHM_PARITY_N) {
7807 mddev->new_layout = 0;
7808 mddev->new_level = 4;
7809 return setup_conf(mddev);
7810 }
7811 return ERR_PTR(-EINVAL);
7812}
NeilBrownd562b0c2009-03-31 14:39:39 +11007813
NeilBrown84fc4b52011-10-11 16:49:58 +11007814static struct md_personality raid5_personality;
NeilBrown245f46c2009-03-31 14:39:39 +11007815
NeilBrownfd01b882011-10-11 16:47:53 +11007816static void *raid6_takeover(struct mddev *mddev)
NeilBrown245f46c2009-03-31 14:39:39 +11007817{
7818 /* Currently can only take over a raid5. We map the
7819 * personality to an equivalent raid6 personality
7820 * with the Q block at the end.
7821 */
7822 int new_layout;
7823
7824 if (mddev->pers != &raid5_personality)
7825 return ERR_PTR(-EINVAL);
7826 if (mddev->degraded > 1)
7827 return ERR_PTR(-EINVAL);
7828 if (mddev->raid_disks > 253)
7829 return ERR_PTR(-EINVAL);
7830 if (mddev->raid_disks < 3)
7831 return ERR_PTR(-EINVAL);
7832
7833 switch (mddev->layout) {
7834 case ALGORITHM_LEFT_ASYMMETRIC:
7835 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
7836 break;
7837 case ALGORITHM_RIGHT_ASYMMETRIC:
7838 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
7839 break;
7840 case ALGORITHM_LEFT_SYMMETRIC:
7841 new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
7842 break;
7843 case ALGORITHM_RIGHT_SYMMETRIC:
7844 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
7845 break;
7846 case ALGORITHM_PARITY_0:
7847 new_layout = ALGORITHM_PARITY_0_6;
7848 break;
7849 case ALGORITHM_PARITY_N:
7850 new_layout = ALGORITHM_PARITY_N;
7851 break;
7852 default:
7853 return ERR_PTR(-EINVAL);
7854 }
7855 mddev->new_level = 6;
7856 mddev->new_layout = new_layout;
7857 mddev->delta_disks = 1;
7858 mddev->raid_disks += 1;
7859 return setup_conf(mddev);
7860}
7861
NeilBrown84fc4b52011-10-11 16:49:58 +11007862static struct md_personality raid6_personality =
NeilBrown16a53ec2006-06-26 00:27:38 -07007863{
7864 .name = "raid6",
7865 .level = 6,
7866 .owner = THIS_MODULE,
7867 .make_request = make_request,
7868 .run = run,
NeilBrownafa0f552014-12-15 12:56:58 +11007869 .free = raid5_free,
NeilBrown16a53ec2006-06-26 00:27:38 -07007870 .status = status,
7871 .error_handler = error,
7872 .hot_add_disk = raid5_add_disk,
7873 .hot_remove_disk= raid5_remove_disk,
7874 .spare_active = raid5_spare_active,
7875 .sync_request = sync_request,
7876 .resize = raid5_resize,
Dan Williams80c3a6c2009-03-17 18:10:40 -07007877 .size = raid5_size,
NeilBrown50ac1682009-06-18 08:47:55 +10007878 .check_reshape = raid6_check_reshape,
NeilBrownf4168852007-02-28 20:11:53 -08007879 .start_reshape = raid5_start_reshape,
NeilBrowncea9c222009-03-31 15:15:05 +11007880 .finish_reshape = raid5_finish_reshape,
NeilBrown16a53ec2006-06-26 00:27:38 -07007881 .quiesce = raid5_quiesce,
NeilBrown245f46c2009-03-31 14:39:39 +11007882 .takeover = raid6_takeover,
NeilBrown5c675f82014-12-15 12:56:56 +11007883 .congested = raid5_congested,
NeilBrown16a53ec2006-06-26 00:27:38 -07007884};
NeilBrown84fc4b52011-10-11 16:49:58 +11007885static struct md_personality raid5_personality =
Linus Torvalds1da177e2005-04-16 15:20:36 -07007886{
7887 .name = "raid5",
NeilBrown2604b702006-01-06 00:20:36 -08007888 .level = 5,
Linus Torvalds1da177e2005-04-16 15:20:36 -07007889 .owner = THIS_MODULE,
7890 .make_request = make_request,
7891 .run = run,
NeilBrownafa0f552014-12-15 12:56:58 +11007892 .free = raid5_free,
Linus Torvalds1da177e2005-04-16 15:20:36 -07007893 .status = status,
7894 .error_handler = error,
7895 .hot_add_disk = raid5_add_disk,
7896 .hot_remove_disk= raid5_remove_disk,
7897 .spare_active = raid5_spare_active,
7898 .sync_request = sync_request,
7899 .resize = raid5_resize,
Dan Williams80c3a6c2009-03-17 18:10:40 -07007900 .size = raid5_size,
NeilBrown63c70c42006-03-27 01:18:13 -08007901 .check_reshape = raid5_check_reshape,
7902 .start_reshape = raid5_start_reshape,
NeilBrowncea9c222009-03-31 15:15:05 +11007903 .finish_reshape = raid5_finish_reshape,
NeilBrown72626682005-09-09 16:23:54 -07007904 .quiesce = raid5_quiesce,
NeilBrownd562b0c2009-03-31 14:39:39 +11007905 .takeover = raid5_takeover,
NeilBrown5c675f82014-12-15 12:56:56 +11007906 .congested = raid5_congested,
Linus Torvalds1da177e2005-04-16 15:20:36 -07007907};
7908
NeilBrown84fc4b52011-10-11 16:49:58 +11007909static struct md_personality raid4_personality =
Linus Torvalds1da177e2005-04-16 15:20:36 -07007910{
NeilBrown2604b702006-01-06 00:20:36 -08007911 .name = "raid4",
7912 .level = 4,
7913 .owner = THIS_MODULE,
7914 .make_request = make_request,
7915 .run = run,
NeilBrownafa0f552014-12-15 12:56:58 +11007916 .free = raid5_free,
NeilBrown2604b702006-01-06 00:20:36 -08007917 .status = status,
7918 .error_handler = error,
7919 .hot_add_disk = raid5_add_disk,
7920 .hot_remove_disk= raid5_remove_disk,
7921 .spare_active = raid5_spare_active,
7922 .sync_request = sync_request,
7923 .resize = raid5_resize,
Dan Williams80c3a6c2009-03-17 18:10:40 -07007924 .size = raid5_size,
NeilBrown3d378902007-03-26 21:32:13 -08007925 .check_reshape = raid5_check_reshape,
7926 .start_reshape = raid5_start_reshape,
NeilBrowncea9c222009-03-31 15:15:05 +11007927 .finish_reshape = raid5_finish_reshape,
NeilBrown2604b702006-01-06 00:20:36 -08007928 .quiesce = raid5_quiesce,
NeilBrowna78d38a2010-03-22 16:53:49 +11007929 .takeover = raid4_takeover,
NeilBrown5c675f82014-12-15 12:56:56 +11007930 .congested = raid5_congested,
NeilBrown2604b702006-01-06 00:20:36 -08007931};
7932
7933static int __init raid5_init(void)
7934{
Shaohua Li851c30c2013-08-28 14:30:16 +08007935 raid5_wq = alloc_workqueue("raid5wq",
7936 WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
7937 if (!raid5_wq)
7938 return -ENOMEM;
NeilBrown16a53ec2006-06-26 00:27:38 -07007939 register_md_personality(&raid6_personality);
NeilBrown2604b702006-01-06 00:20:36 -08007940 register_md_personality(&raid5_personality);
7941 register_md_personality(&raid4_personality);
7942 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007943}
7944
NeilBrown2604b702006-01-06 00:20:36 -08007945static void raid5_exit(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007946{
NeilBrown16a53ec2006-06-26 00:27:38 -07007947 unregister_md_personality(&raid6_personality);
NeilBrown2604b702006-01-06 00:20:36 -08007948 unregister_md_personality(&raid5_personality);
7949 unregister_md_personality(&raid4_personality);
Shaohua Li851c30c2013-08-28 14:30:16 +08007950 destroy_workqueue(raid5_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007951}
7952
7953module_init(raid5_init);
7954module_exit(raid5_exit);
7955MODULE_LICENSE("GPL");
NeilBrown0efb9e62009-12-14 12:49:58 +11007956MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
Linus Torvalds1da177e2005-04-16 15:20:36 -07007957MODULE_ALIAS("md-personality-4"); /* RAID5 */
NeilBrownd9d166c2006-01-06 00:20:51 -08007958MODULE_ALIAS("md-raid5");
7959MODULE_ALIAS("md-raid4");
NeilBrown2604b702006-01-06 00:20:36 -08007960MODULE_ALIAS("md-level-5");
7961MODULE_ALIAS("md-level-4");
NeilBrown16a53ec2006-06-26 00:27:38 -07007962MODULE_ALIAS("md-personality-8"); /* RAID6 */
7963MODULE_ALIAS("md-raid6");
7964MODULE_ALIAS("md-level-6");
7965
7966/* This used to be two separate modules, they were: */
7967MODULE_ALIAS("raid5");
7968MODULE_ALIAS("raid6");