blob: f1c86d938502e61c93d8951680767f1af5de98e5 [file] [log] [blame]
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001/*
2 * Copyright (C) 2015 Shaohua Li <shli@fb.com>
Song Liub4c625c2016-11-17 15:24:43 -08003 * Copyright (C) 2016 Song Liu <songliubraving@fb.com>
Shaohua Lif6bed0e2015-08-13 14:31:59 -07004 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15#include <linux/kernel.h>
16#include <linux/wait.h>
17#include <linux/blkdev.h>
18#include <linux/slab.h>
19#include <linux/raid/md_p.h>
Shaohua Li5cb2fbd2015-10-28 08:41:25 -070020#include <linux/crc32c.h>
Shaohua Lif6bed0e2015-08-13 14:31:59 -070021#include <linux/random.h>
Shaohua Lice1ccd02016-11-21 10:29:18 -080022#include <linux/kthread.h>
Song Liu03b047f2017-01-11 13:39:14 -080023#include <linux/types.h>
Shaohua Lif6bed0e2015-08-13 14:31:59 -070024#include "md.h"
25#include "raid5.h"
Mike Snitzer935fe092017-10-10 17:02:41 -040026#include "md-bitmap.h"
Song Liu70d466f2017-05-11 15:28:28 -070027#include "raid5-log.h"
Shaohua Lif6bed0e2015-08-13 14:31:59 -070028
29/*
30 * metadata/data stored in disk with 4k size unit (a block) regardless
31 * underneath hardware sector size. only works with PAGE_SIZE == 4096
32 */
33#define BLOCK_SECTORS (8)
Song Liueffe6ee2017-03-07 16:49:17 -080034#define BLOCK_SECTOR_SHIFT (3)
Shaohua Lif6bed0e2015-08-13 14:31:59 -070035
Shaohua Li0576b1c2015-08-13 14:32:00 -070036/*
Song Liua39f7af2016-11-17 15:24:40 -080037 * log->max_free_space is min(1/4 disk size, 10G reclaimable space).
38 *
39 * In write through mode, the reclaim runs every log->max_free_space.
40 * This can prevent the recovery scans for too long
Shaohua Li0576b1c2015-08-13 14:32:00 -070041 */
42#define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
43#define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
44
Song Liua39f7af2016-11-17 15:24:40 -080045/* wake up reclaim thread periodically */
46#define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ)
47/* start flush with these full stripes */
Shaohua Li84890c02017-02-15 19:58:05 -080048#define R5C_FULL_STRIPE_FLUSH_BATCH(conf) (conf->max_nr_stripes / 4)
Song Liua39f7af2016-11-17 15:24:40 -080049/* reclaim stripes in groups */
50#define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2)
51
Christoph Hellwigc38d29b2015-12-21 10:51:02 +110052/*
53 * We only need 2 bios per I/O unit to make progress, but ensure we
54 * have a few more available to not get too tight.
55 */
56#define R5L_POOL_SIZE 4
57
Song Liu2c7da142016-11-17 15:24:41 -080058static char *r5c_journal_mode_str[] = {"write-through",
59 "write-back"};
Song Liu2ded3702016-11-17 15:24:38 -080060/*
61 * raid5 cache state machine
62 *
JackieLiu9b691732016-11-28 16:19:18 +080063 * With the RAID cache, each stripe works in two phases:
Song Liu2ded3702016-11-17 15:24:38 -080064 * - caching phase
65 * - writing-out phase
66 *
67 * These two phases are controlled by bit STRIPE_R5C_CACHING:
68 * if STRIPE_R5C_CACHING == 0, the stripe is in writing-out phase
69 * if STRIPE_R5C_CACHING == 1, the stripe is in caching phase
70 *
71 * When there is no journal, or the journal is in write-through mode,
72 * the stripe is always in writing-out phase.
73 *
74 * For write-back journal, the stripe is sent to caching phase on write
75 * (r5c_try_caching_write). r5c_make_stripe_write_out() kicks off
76 * the write-out phase by clearing STRIPE_R5C_CACHING.
77 *
78 * Stripes in caching phase do not write the raid disks. Instead, all
79 * writes are committed from the log device. Therefore, a stripe in
80 * caching phase handles writes as:
81 * - write to log device
82 * - return IO
83 *
84 * Stripes in writing-out phase handle writes as:
85 * - calculate parity
86 * - write pending data and parity to journal
87 * - write data and parity to raid disks
88 * - return IO for pending writes
89 */
90
Shaohua Lif6bed0e2015-08-13 14:31:59 -070091struct r5l_log {
92 struct md_rdev *rdev;
93
94 u32 uuid_checksum;
95
96 sector_t device_size; /* log device size, round to
97 * BLOCK_SECTORS */
Shaohua Li0576b1c2015-08-13 14:32:00 -070098 sector_t max_free_space; /* reclaim run if free space is at
99 * this size */
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700100
101 sector_t last_checkpoint; /* log tail. where recovery scan
102 * starts from */
103 u64 last_cp_seq; /* log tail sequence */
104
105 sector_t log_start; /* log head. where new data appends */
106 u64 seq; /* log head sequence */
107
Christoph Hellwig17036462015-10-05 09:31:06 +0200108 sector_t next_checkpoint;
Christoph Hellwig17036462015-10-05 09:31:06 +0200109
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700110 struct mutex io_mutex;
111 struct r5l_io_unit *current_io; /* current io_unit accepting new data */
112
113 spinlock_t io_list_lock;
114 struct list_head running_ios; /* io_units which are still running,
115 * and have not yet been completely
116 * written to the log */
117 struct list_head io_end_ios; /* io_units which have been completely
118 * written to the log but not yet written
119 * to the RAID */
Shaohua Lia8c34f92015-09-02 13:49:46 -0700120 struct list_head flushing_ios; /* io_units which are waiting for log
121 * cache flush */
Christoph Hellwig04732f72015-10-05 09:31:07 +0200122 struct list_head finished_ios; /* io_units which settle down in log disk */
Shaohua Lia8c34f92015-09-02 13:49:46 -0700123 struct bio flush_bio;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700124
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100125 struct list_head no_mem_stripes; /* pending stripes, -ENOMEM */
126
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700127 struct kmem_cache *io_kc;
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100128 mempool_t *io_pool;
Christoph Hellwigc38d29b2015-12-21 10:51:02 +1100129 struct bio_set *bs;
Christoph Hellwige8deb632015-12-21 10:51:02 +1100130 mempool_t *meta_pool;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700131
Shaohua Li0576b1c2015-08-13 14:32:00 -0700132 struct md_thread *reclaim_thread;
133 unsigned long reclaim_target; /* number of space that need to be
134 * reclaimed. if it's 0, reclaim spaces
135 * used by io_units which are in
136 * IO_UNIT_STRIPE_END state (eg, reclaim
137 * dones't wait for specific io_unit
138 * switching to IO_UNIT_STRIPE_END
139 * state) */
Shaohua Li0fd22b42015-09-02 13:49:47 -0700140 wait_queue_head_t iounit_wait;
Shaohua Li0576b1c2015-08-13 14:32:00 -0700141
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700142 struct list_head no_space_stripes; /* pending stripes, log has no space */
143 spinlock_t no_space_stripes_lock;
Christoph Hellwig56fef7c2015-10-05 09:31:09 +0200144
145 bool need_cache_flush;
Song Liu2ded3702016-11-17 15:24:38 -0800146
147 /* for r5c_cache */
148 enum r5c_journal_mode r5c_journal_mode;
Song Liua39f7af2016-11-17 15:24:40 -0800149
150 /* all stripes in r5cache, in the order of seq at sh->log_start */
151 struct list_head stripe_in_journal_list;
152
153 spinlock_t stripe_in_journal_lock;
154 atomic_t stripe_in_journal_count;
Song Liu3bddb7f2016-11-18 16:46:50 -0800155
156 /* to submit async io_units, to fulfill ordering of flush */
157 struct work_struct deferred_io_work;
Song Liu2e38a372017-01-24 10:45:30 -0800158 /* to disable write back during in degraded mode */
159 struct work_struct disable_writeback_work;
Song Liu03b047f2017-01-11 13:39:14 -0800160
161 /* to for chunk_aligned_read in writeback mode, details below */
162 spinlock_t tree_lock;
163 struct radix_tree_root big_stripe_tree;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700164};
165
166/*
Song Liu03b047f2017-01-11 13:39:14 -0800167 * Enable chunk_aligned_read() with write back cache.
168 *
169 * Each chunk may contain more than one stripe (for example, a 256kB
170 * chunk contains 64 4kB-page, so this chunk contain 64 stripes). For
171 * chunk_aligned_read, these stripes are grouped into one "big_stripe".
172 * For each big_stripe, we count how many stripes of this big_stripe
173 * are in the write back cache. These data are tracked in a radix tree
174 * (big_stripe_tree). We use radix_tree item pointer as the counter.
175 * r5c_tree_index() is used to calculate keys for the radix tree.
176 *
177 * chunk_aligned_read() calls r5c_big_stripe_cached() to look up
178 * big_stripe of each chunk in the tree. If this big_stripe is in the
179 * tree, chunk_aligned_read() aborts. This look up is protected by
180 * rcu_read_lock().
181 *
182 * It is necessary to remember whether a stripe is counted in
183 * big_stripe_tree. Instead of adding new flag, we reuses existing flags:
184 * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE. If either of these
185 * two flags are set, the stripe is counted in big_stripe_tree. This
186 * requires moving set_bit(STRIPE_R5C_PARTIAL_STRIPE) to
187 * r5c_try_caching_write(); and moving clear_bit of
188 * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE to
189 * r5c_finish_stripe_write_out().
190 */
191
192/*
193 * radix tree requests lowest 2 bits of data pointer to be 2b'00.
194 * So it is necessary to left shift the counter by 2 bits before using it
195 * as data pointer of the tree.
196 */
197#define R5C_RADIX_COUNT_SHIFT 2
198
199/*
200 * calculate key for big_stripe_tree
201 *
202 * sect: align_bi->bi_iter.bi_sector or sh->sector
203 */
204static inline sector_t r5c_tree_index(struct r5conf *conf,
205 sector_t sect)
206{
207 sector_t offset;
208
209 offset = sector_div(sect, conf->chunk_sectors);
210 return sect;
211}
212
213/*
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700214 * an IO range starts from a meta data block and end at the next meta data
215 * block. The io unit's the meta data block tracks data/parity followed it. io
216 * unit is written to log disk with normal write, as we always flush log disk
217 * first and then start move data to raid disks, there is no requirement to
218 * write io unit with FLUSH/FUA
219 */
220struct r5l_io_unit {
221 struct r5l_log *log;
222
223 struct page *meta_page; /* store meta block */
224 int meta_offset; /* current offset in meta_page */
225
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700226 struct bio *current_bio;/* current_bio accepting new data */
227
228 atomic_t pending_stripe;/* how many stripes not flushed to raid */
229 u64 seq; /* seq number of the metablock */
230 sector_t log_start; /* where the io_unit starts */
231 sector_t log_end; /* where the io_unit ends */
232 struct list_head log_sibling; /* log->running_ios */
233 struct list_head stripe_list; /* stripes added to the io_unit */
234
235 int state;
Christoph Hellwig6143e2c2015-10-05 09:31:16 +0200236 bool need_split_bio;
Song Liu3bddb7f2016-11-18 16:46:50 -0800237 struct bio *split_bio;
238
Song Liua9501d72017-08-03 10:03:17 -0700239 unsigned int has_flush:1; /* include flush request */
240 unsigned int has_fua:1; /* include fua request */
241 unsigned int has_null_flush:1; /* include null flush request */
242 unsigned int has_flush_payload:1; /* include flush payload */
Song Liu3bddb7f2016-11-18 16:46:50 -0800243 /*
244 * io isn't sent yet, flush/fua request can only be submitted till it's
245 * the first IO in running_ios list
246 */
247 unsigned int io_deferred:1;
248
249 struct bio_list flush_barriers; /* size == 0 flush bios */
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700250};
251
252/* r5l_io_unit state */
253enum r5l_io_unit_state {
254 IO_UNIT_RUNNING = 0, /* accepting new IO */
255 IO_UNIT_IO_START = 1, /* io_unit bio start writing to log,
256 * don't accepting new bio */
257 IO_UNIT_IO_END = 2, /* io_unit bio finish writing to log */
Shaohua Lia8c34f92015-09-02 13:49:46 -0700258 IO_UNIT_STRIPE_END = 3, /* stripes data finished writing to raid */
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700259};
260
Song Liu2ded3702016-11-17 15:24:38 -0800261bool r5c_is_writeback(struct r5l_log *log)
262{
263 return (log != NULL &&
264 log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK);
265}
266
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700267static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc)
268{
269 start += inc;
270 if (start >= log->device_size)
271 start = start - log->device_size;
272 return start;
273}
274
275static sector_t r5l_ring_distance(struct r5l_log *log, sector_t start,
276 sector_t end)
277{
278 if (end >= start)
279 return end - start;
280 else
281 return end + log->device_size - start;
282}
283
284static bool r5l_has_free_space(struct r5l_log *log, sector_t size)
285{
286 sector_t used_size;
287
288 used_size = r5l_ring_distance(log, log->last_checkpoint,
289 log->log_start);
290
291 return log->device_size > used_size + size;
292}
293
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700294static void __r5l_set_io_unit_state(struct r5l_io_unit *io,
295 enum r5l_io_unit_state state)
296{
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700297 if (WARN_ON(io->state >= state))
298 return;
299 io->state = state;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700300}
301
Song Liu1e6d6902016-11-17 15:24:39 -0800302static void
NeilBrownbd83d0a2017-03-15 14:05:12 +1100303r5c_return_dev_pending_writes(struct r5conf *conf, struct r5dev *dev)
Song Liu1e6d6902016-11-17 15:24:39 -0800304{
305 struct bio *wbi, *wbi2;
306
307 wbi = dev->written;
308 dev->written = NULL;
309 while (wbi && wbi->bi_iter.bi_sector <
310 dev->sector + STRIPE_SECTORS) {
311 wbi2 = r5_next_bio(wbi, dev->sector);
NeilBrown49728052017-03-15 14:05:12 +1100312 md_write_end(conf->mddev);
NeilBrown016c76a2017-03-15 14:05:13 +1100313 bio_endio(wbi);
Song Liu1e6d6902016-11-17 15:24:39 -0800314 wbi = wbi2;
315 }
316}
317
318void r5c_handle_cached_data_endio(struct r5conf *conf,
NeilBrownbd83d0a2017-03-15 14:05:12 +1100319 struct stripe_head *sh, int disks)
Song Liu1e6d6902016-11-17 15:24:39 -0800320{
321 int i;
322
323 for (i = sh->disks; i--; ) {
324 if (sh->dev[i].written) {
325 set_bit(R5_UPTODATE, &sh->dev[i].flags);
NeilBrownbd83d0a2017-03-15 14:05:12 +1100326 r5c_return_dev_pending_writes(conf, &sh->dev[i]);
Song Liu1e6d6902016-11-17 15:24:39 -0800327 bitmap_endwrite(conf->mddev->bitmap, sh->sector,
328 STRIPE_SECTORS,
329 !test_bit(STRIPE_DEGRADED, &sh->state),
330 0);
331 }
332 }
333}
334
Artur Paszkiewiczff875732017-03-09 09:59:58 +0100335void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
336
Song Liua39f7af2016-11-17 15:24:40 -0800337/* Check whether we should flush some stripes to free up stripe cache */
338void r5c_check_stripe_cache_usage(struct r5conf *conf)
339{
340 int total_cached;
341
342 if (!r5c_is_writeback(conf->log))
343 return;
344
345 total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
346 atomic_read(&conf->r5c_cached_full_stripes);
347
348 /*
349 * The following condition is true for either of the following:
350 * - stripe cache pressure high:
351 * total_cached > 3/4 min_nr_stripes ||
352 * empty_inactive_list_nr > 0
353 * - stripe cache pressure moderate:
354 * total_cached > 1/2 min_nr_stripes
355 */
356 if (total_cached > conf->min_nr_stripes * 1 / 2 ||
357 atomic_read(&conf->empty_inactive_list_nr) > 0)
358 r5l_wake_reclaim(conf->log, 0);
359}
360
361/*
362 * flush cache when there are R5C_FULL_STRIPE_FLUSH_BATCH or more full
363 * stripes in the cache
364 */
365void r5c_check_cached_full_stripe(struct r5conf *conf)
366{
367 if (!r5c_is_writeback(conf->log))
368 return;
369
370 /*
371 * wake up reclaim for R5C_FULL_STRIPE_FLUSH_BATCH cached stripes
372 * or a full stripe (chunk size / 4k stripes).
373 */
374 if (atomic_read(&conf->r5c_cached_full_stripes) >=
Shaohua Li84890c02017-02-15 19:58:05 -0800375 min(R5C_FULL_STRIPE_FLUSH_BATCH(conf),
Song Liua39f7af2016-11-17 15:24:40 -0800376 conf->chunk_sectors >> STRIPE_SHIFT))
377 r5l_wake_reclaim(conf->log, 0);
378}
379
380/*
381 * Total log space (in sectors) needed to flush all data in cache
382 *
Song Liu39b99582017-01-24 14:08:23 -0800383 * To avoid deadlock due to log space, it is necessary to reserve log
384 * space to flush critical stripes (stripes that occupying log space near
385 * last_checkpoint). This function helps check how much log space is
386 * required to flush all cached stripes.
Song Liua39f7af2016-11-17 15:24:40 -0800387 *
Song Liu39b99582017-01-24 14:08:23 -0800388 * To reduce log space requirements, two mechanisms are used to give cache
389 * flush higher priorities:
390 * 1. In handle_stripe_dirtying() and schedule_reconstruction(),
391 * stripes ALREADY in journal can be flushed w/o pending writes;
392 * 2. In r5l_write_stripe() and r5c_cache_data(), stripes NOT in journal
393 * can be delayed (r5l_add_no_space_stripe).
Song Liua39f7af2016-11-17 15:24:40 -0800394 *
Song Liu39b99582017-01-24 14:08:23 -0800395 * In cache flush, the stripe goes through 1 and then 2. For a stripe that
396 * already passed 1, flushing it requires at most (conf->max_degraded + 1)
397 * pages of journal space. For stripes that has not passed 1, flushing it
398 * requires (conf->raid_disks + 1) pages of journal space. There are at
399 * most (conf->group_cnt + 1) stripe that passed 1. So total journal space
400 * required to flush all cached stripes (in pages) is:
401 *
402 * (stripe_in_journal_count - group_cnt - 1) * (max_degraded + 1) +
403 * (group_cnt + 1) * (raid_disks + 1)
404 * or
405 * (stripe_in_journal_count) * (max_degraded + 1) +
406 * (group_cnt + 1) * (raid_disks - max_degraded)
Song Liua39f7af2016-11-17 15:24:40 -0800407 */
408static sector_t r5c_log_required_to_flush_cache(struct r5conf *conf)
409{
410 struct r5l_log *log = conf->log;
411
412 if (!r5c_is_writeback(log))
413 return 0;
414
Song Liu39b99582017-01-24 14:08:23 -0800415 return BLOCK_SECTORS *
416 ((conf->max_degraded + 1) * atomic_read(&log->stripe_in_journal_count) +
417 (conf->raid_disks - conf->max_degraded) * (conf->group_cnt + 1));
Song Liua39f7af2016-11-17 15:24:40 -0800418}
419
420/*
421 * evaluate log space usage and update R5C_LOG_TIGHT and R5C_LOG_CRITICAL
422 *
423 * R5C_LOG_TIGHT is set when free space on the log device is less than 3x of
424 * reclaim_required_space. R5C_LOG_CRITICAL is set when free space on the log
425 * device is less than 2x of reclaim_required_space.
426 */
427static inline void r5c_update_log_state(struct r5l_log *log)
428{
429 struct r5conf *conf = log->rdev->mddev->private;
430 sector_t free_space;
431 sector_t reclaim_space;
Song Liuf687a332016-11-30 16:57:54 -0800432 bool wake_reclaim = false;
Song Liua39f7af2016-11-17 15:24:40 -0800433
434 if (!r5c_is_writeback(log))
435 return;
436
437 free_space = r5l_ring_distance(log, log->log_start,
438 log->last_checkpoint);
439 reclaim_space = r5c_log_required_to_flush_cache(conf);
440 if (free_space < 2 * reclaim_space)
441 set_bit(R5C_LOG_CRITICAL, &conf->cache_state);
Song Liuf687a332016-11-30 16:57:54 -0800442 else {
443 if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
444 wake_reclaim = true;
Song Liua39f7af2016-11-17 15:24:40 -0800445 clear_bit(R5C_LOG_CRITICAL, &conf->cache_state);
Song Liuf687a332016-11-30 16:57:54 -0800446 }
Song Liua39f7af2016-11-17 15:24:40 -0800447 if (free_space < 3 * reclaim_space)
448 set_bit(R5C_LOG_TIGHT, &conf->cache_state);
449 else
450 clear_bit(R5C_LOG_TIGHT, &conf->cache_state);
Song Liuf687a332016-11-30 16:57:54 -0800451
452 if (wake_reclaim)
453 r5l_wake_reclaim(log, 0);
Song Liua39f7af2016-11-17 15:24:40 -0800454}
455
Song Liu2ded3702016-11-17 15:24:38 -0800456/*
457 * Put the stripe into writing-out phase by clearing STRIPE_R5C_CACHING.
458 * This function should only be called in write-back mode.
459 */
Song Liua39f7af2016-11-17 15:24:40 -0800460void r5c_make_stripe_write_out(struct stripe_head *sh)
Song Liu2ded3702016-11-17 15:24:38 -0800461{
462 struct r5conf *conf = sh->raid_conf;
463 struct r5l_log *log = conf->log;
464
465 BUG_ON(!r5c_is_writeback(log));
466
467 WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
468 clear_bit(STRIPE_R5C_CACHING, &sh->state);
Song Liu1e6d6902016-11-17 15:24:39 -0800469
470 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
471 atomic_inc(&conf->preread_active_stripes);
Song Liu1e6d6902016-11-17 15:24:39 -0800472}
473
474static void r5c_handle_data_cached(struct stripe_head *sh)
475{
476 int i;
477
478 for (i = sh->disks; i--; )
479 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
480 set_bit(R5_InJournal, &sh->dev[i].flags);
481 clear_bit(R5_LOCKED, &sh->dev[i].flags);
482 }
483 clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
484}
485
486/*
487 * this journal write must contain full parity,
488 * it may also contain some data pages
489 */
490static void r5c_handle_parity_cached(struct stripe_head *sh)
491{
492 int i;
493
494 for (i = sh->disks; i--; )
495 if (test_bit(R5_InJournal, &sh->dev[i].flags))
496 set_bit(R5_Wantwrite, &sh->dev[i].flags);
Song Liu2ded3702016-11-17 15:24:38 -0800497}
498
499/*
500 * Setting proper flags after writing (or flushing) data and/or parity to the
501 * log device. This is called from r5l_log_endio() or r5l_log_flush_endio().
502 */
503static void r5c_finish_cache_stripe(struct stripe_head *sh)
504{
505 struct r5l_log *log = sh->raid_conf->log;
506
507 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
508 BUG_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
509 /*
510 * Set R5_InJournal for parity dev[pd_idx]. This means
511 * all data AND parity in the journal. For RAID 6, it is
512 * NOT necessary to set the flag for dev[qd_idx], as the
513 * two parities are written out together.
514 */
515 set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
Song Liu1e6d6902016-11-17 15:24:39 -0800516 } else if (test_bit(STRIPE_R5C_CACHING, &sh->state)) {
517 r5c_handle_data_cached(sh);
518 } else {
519 r5c_handle_parity_cached(sh);
520 set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
521 }
Song Liu2ded3702016-11-17 15:24:38 -0800522}
523
Christoph Hellwigd8858f42015-10-05 09:31:08 +0200524static void r5l_io_run_stripes(struct r5l_io_unit *io)
525{
526 struct stripe_head *sh, *next;
527
528 list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {
529 list_del_init(&sh->log_list);
Song Liu2ded3702016-11-17 15:24:38 -0800530
531 r5c_finish_cache_stripe(sh);
532
Christoph Hellwigd8858f42015-10-05 09:31:08 +0200533 set_bit(STRIPE_HANDLE, &sh->state);
534 raid5_release_stripe(sh);
535 }
536}
537
Christoph Hellwig56fef7c2015-10-05 09:31:09 +0200538static void r5l_log_run_stripes(struct r5l_log *log)
539{
540 struct r5l_io_unit *io, *next;
541
Shaohua Liefa4b772017-10-18 22:08:13 -0700542 lockdep_assert_held(&log->io_list_lock);
Christoph Hellwig56fef7c2015-10-05 09:31:09 +0200543
544 list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
545 /* don't change list order */
546 if (io->state < IO_UNIT_IO_END)
547 break;
548
549 list_move_tail(&io->log_sibling, &log->finished_ios);
550 r5l_io_run_stripes(io);
551 }
552}
553
Christoph Hellwig3848c0b2015-12-21 10:51:01 +1100554static void r5l_move_to_end_ios(struct r5l_log *log)
555{
556 struct r5l_io_unit *io, *next;
557
Shaohua Liefa4b772017-10-18 22:08:13 -0700558 lockdep_assert_held(&log->io_list_lock);
Christoph Hellwig3848c0b2015-12-21 10:51:01 +1100559
560 list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
561 /* don't change list order */
562 if (io->state < IO_UNIT_IO_END)
563 break;
564 list_move_tail(&io->log_sibling, &log->io_end_ios);
565 }
566}
567
Song Liu3bddb7f2016-11-18 16:46:50 -0800568static void __r5l_stripe_write_finished(struct r5l_io_unit *io);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700569static void r5l_log_endio(struct bio *bio)
570{
571 struct r5l_io_unit *io = bio->bi_private;
Song Liu3bddb7f2016-11-18 16:46:50 -0800572 struct r5l_io_unit *io_deferred;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700573 struct r5l_log *log = io->log;
Christoph Hellwig509ffec2015-09-02 13:49:48 -0700574 unsigned long flags;
Song Liua9501d72017-08-03 10:03:17 -0700575 bool has_null_flush;
576 bool has_flush_payload;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700577
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200578 if (bio->bi_status)
Shaohua Li6e74a9c2015-10-08 21:54:08 -0700579 md_error(log->rdev->mddev, log->rdev);
580
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700581 bio_put(bio);
Christoph Hellwige8deb632015-12-21 10:51:02 +1100582 mempool_free(io->meta_page, log->meta_pool);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700583
Christoph Hellwig509ffec2015-09-02 13:49:48 -0700584 spin_lock_irqsave(&log->io_list_lock, flags);
585 __r5l_set_io_unit_state(io, IO_UNIT_IO_END);
Song Liua9501d72017-08-03 10:03:17 -0700586
587 /*
588 * if the io doesn't not have null_flush or flush payload,
589 * it is not safe to access it after releasing io_list_lock.
590 * Therefore, it is necessary to check the condition with
591 * the lock held.
592 */
593 has_null_flush = io->has_null_flush;
594 has_flush_payload = io->has_flush_payload;
595
Song Liuea174812017-03-09 21:23:39 -0800596 if (log->need_cache_flush && !list_empty(&io->stripe_list))
Christoph Hellwig3848c0b2015-12-21 10:51:01 +1100597 r5l_move_to_end_ios(log);
Christoph Hellwig56fef7c2015-10-05 09:31:09 +0200598 else
599 r5l_log_run_stripes(log);
Song Liu3bddb7f2016-11-18 16:46:50 -0800600 if (!list_empty(&log->running_ios)) {
601 /*
602 * FLUSH/FUA io_unit is deferred because of ordering, now we
603 * can dispatch it
604 */
605 io_deferred = list_first_entry(&log->running_ios,
606 struct r5l_io_unit, log_sibling);
607 if (io_deferred->io_deferred)
608 schedule_work(&log->deferred_io_work);
609 }
610
Christoph Hellwig509ffec2015-09-02 13:49:48 -0700611 spin_unlock_irqrestore(&log->io_list_lock, flags);
612
Christoph Hellwig56fef7c2015-10-05 09:31:09 +0200613 if (log->need_cache_flush)
614 md_wakeup_thread(log->rdev->mddev->thread);
Song Liu3bddb7f2016-11-18 16:46:50 -0800615
Song Liua9501d72017-08-03 10:03:17 -0700616 /* finish flush only io_unit and PAYLOAD_FLUSH only io_unit */
617 if (has_null_flush) {
Song Liu3bddb7f2016-11-18 16:46:50 -0800618 struct bio *bi;
619
620 WARN_ON(bio_list_empty(&io->flush_barriers));
621 while ((bi = bio_list_pop(&io->flush_barriers)) != NULL) {
622 bio_endio(bi);
Song Liua9501d72017-08-03 10:03:17 -0700623 if (atomic_dec_and_test(&io->pending_stripe)) {
624 __r5l_stripe_write_finished(io);
625 return;
626 }
Song Liu3bddb7f2016-11-18 16:46:50 -0800627 }
Song Liu3bddb7f2016-11-18 16:46:50 -0800628 }
Song Liua9501d72017-08-03 10:03:17 -0700629 /* decrease pending_stripe for flush payload */
630 if (has_flush_payload)
631 if (atomic_dec_and_test(&io->pending_stripe))
632 __r5l_stripe_write_finished(io);
Song Liu3bddb7f2016-11-18 16:46:50 -0800633}
634
635static void r5l_do_submit_io(struct r5l_log *log, struct r5l_io_unit *io)
636{
637 unsigned long flags;
638
639 spin_lock_irqsave(&log->io_list_lock, flags);
640 __r5l_set_io_unit_state(io, IO_UNIT_IO_START);
641 spin_unlock_irqrestore(&log->io_list_lock, flags);
642
Song Liubb3338d2017-05-08 17:39:24 -0700643 /*
644 * In case of journal device failures, submit_bio will get error
645 * and calls endio, then active stripes will continue write
646 * process. Therefore, it is not necessary to check Faulty bit
647 * of journal device here.
648 *
649 * We can't check split_bio after current_bio is submitted. If
650 * io->split_bio is null, after current_bio is submitted, current_bio
651 * might already be completed and the io_unit is freed. We submit
652 * split_bio first to avoid the issue.
653 */
654 if (io->split_bio) {
655 if (io->has_flush)
656 io->split_bio->bi_opf |= REQ_PREFLUSH;
657 if (io->has_fua)
658 io->split_bio->bi_opf |= REQ_FUA;
659 submit_bio(io->split_bio);
660 }
661
Song Liu3bddb7f2016-11-18 16:46:50 -0800662 if (io->has_flush)
Shaohua Li20737732016-12-13 12:40:15 -0800663 io->current_bio->bi_opf |= REQ_PREFLUSH;
Song Liu3bddb7f2016-11-18 16:46:50 -0800664 if (io->has_fua)
Shaohua Li20737732016-12-13 12:40:15 -0800665 io->current_bio->bi_opf |= REQ_FUA;
Song Liu3bddb7f2016-11-18 16:46:50 -0800666 submit_bio(io->current_bio);
Song Liu3bddb7f2016-11-18 16:46:50 -0800667}
668
669/* deferred io_unit will be dispatched here */
670static void r5l_submit_io_async(struct work_struct *work)
671{
672 struct r5l_log *log = container_of(work, struct r5l_log,
673 deferred_io_work);
674 struct r5l_io_unit *io = NULL;
675 unsigned long flags;
676
677 spin_lock_irqsave(&log->io_list_lock, flags);
678 if (!list_empty(&log->running_ios)) {
679 io = list_first_entry(&log->running_ios, struct r5l_io_unit,
680 log_sibling);
681 if (!io->io_deferred)
682 io = NULL;
683 else
684 io->io_deferred = 0;
685 }
686 spin_unlock_irqrestore(&log->io_list_lock, flags);
687 if (io)
688 r5l_do_submit_io(log, io);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700689}
690
Song Liu2e38a372017-01-24 10:45:30 -0800691static void r5c_disable_writeback_async(struct work_struct *work)
692{
693 struct r5l_log *log = container_of(work, struct r5l_log,
694 disable_writeback_work);
695 struct mddev *mddev = log->rdev->mddev;
NeilBrown4d5324f2017-10-19 12:17:16 +1100696 struct r5conf *conf = mddev->private;
697 int locked = 0;
Song Liu2e38a372017-01-24 10:45:30 -0800698
699 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
700 return;
701 pr_info("md/raid:%s: Disabling writeback cache for degraded array.\n",
702 mdname(mddev));
Song Liu70d466f2017-05-11 15:28:28 -0700703
704 /* wait superblock change before suspend */
705 wait_event(mddev->sb_wait,
NeilBrown4d5324f2017-10-19 12:17:16 +1100706 conf->log == NULL ||
707 (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags) &&
708 (locked = mddev_trylock(mddev))));
709 if (locked) {
710 mddev_suspend(mddev);
711 log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
712 mddev_resume(mddev);
713 mddev_unlock(mddev);
714 }
Song Liu2e38a372017-01-24 10:45:30 -0800715}
716
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700717static void r5l_submit_current_io(struct r5l_log *log)
718{
719 struct r5l_io_unit *io = log->current_io;
Song Liu3bddb7f2016-11-18 16:46:50 -0800720 struct bio *bio;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700721 struct r5l_meta_block *block;
Christoph Hellwig509ffec2015-09-02 13:49:48 -0700722 unsigned long flags;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700723 u32 crc;
Song Liu3bddb7f2016-11-18 16:46:50 -0800724 bool do_submit = true;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700725
726 if (!io)
727 return;
728
729 block = page_address(io->meta_page);
730 block->meta_size = cpu_to_le32(io->meta_offset);
Shaohua Li5cb2fbd2015-10-28 08:41:25 -0700731 crc = crc32c_le(log->uuid_checksum, block, PAGE_SIZE);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700732 block->checksum = cpu_to_le32(crc);
Song Liu3bddb7f2016-11-18 16:46:50 -0800733 bio = io->current_bio;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700734
735 log->current_io = NULL;
Christoph Hellwig509ffec2015-09-02 13:49:48 -0700736 spin_lock_irqsave(&log->io_list_lock, flags);
Song Liu3bddb7f2016-11-18 16:46:50 -0800737 if (io->has_flush || io->has_fua) {
738 if (io != list_first_entry(&log->running_ios,
739 struct r5l_io_unit, log_sibling)) {
740 io->io_deferred = 1;
741 do_submit = false;
742 }
743 }
Christoph Hellwig509ffec2015-09-02 13:49:48 -0700744 spin_unlock_irqrestore(&log->io_list_lock, flags);
Song Liu3bddb7f2016-11-18 16:46:50 -0800745 if (do_submit)
746 r5l_do_submit_io(log, io);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700747}
748
Christoph Hellwig6143e2c2015-10-05 09:31:16 +0200749static struct bio *r5l_bio_alloc(struct r5l_log *log)
Christoph Hellwigb349feb2015-10-05 09:31:11 +0200750{
Christoph Hellwigc38d29b2015-12-21 10:51:02 +1100751 struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, log->bs);
Christoph Hellwigb349feb2015-10-05 09:31:11 +0200752
Mike Christie796a5cf2016-06-05 14:32:07 -0500753 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
Christoph Hellwig74d46992017-08-23 19:10:32 +0200754 bio_set_dev(bio, log->rdev->bdev);
Christoph Hellwig1e932a32015-10-05 09:31:12 +0200755 bio->bi_iter.bi_sector = log->rdev->data_offset + log->log_start;
Christoph Hellwigb349feb2015-10-05 09:31:11 +0200756
Christoph Hellwigb349feb2015-10-05 09:31:11 +0200757 return bio;
758}
759
Christoph Hellwigc1b99192015-10-05 09:31:14 +0200760static void r5_reserve_log_entry(struct r5l_log *log, struct r5l_io_unit *io)
761{
762 log->log_start = r5l_ring_add(log, log->log_start, BLOCK_SECTORS);
763
Song Liua39f7af2016-11-17 15:24:40 -0800764 r5c_update_log_state(log);
Christoph Hellwigc1b99192015-10-05 09:31:14 +0200765 /*
766 * If we filled up the log device start from the beginning again,
767 * which will require a new bio.
768 *
769 * Note: for this to work properly the log size needs to me a multiple
770 * of BLOCK_SECTORS.
771 */
772 if (log->log_start == 0)
Christoph Hellwig6143e2c2015-10-05 09:31:16 +0200773 io->need_split_bio = true;
Christoph Hellwigc1b99192015-10-05 09:31:14 +0200774
775 io->log_end = log->log_start;
776}
777
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700778static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
779{
780 struct r5l_io_unit *io;
781 struct r5l_meta_block *block;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700782
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100783 io = mempool_alloc(log->io_pool, GFP_ATOMIC);
784 if (!io)
785 return NULL;
786 memset(io, 0, sizeof(*io));
787
Christoph Hellwig51039cd2015-10-05 09:31:13 +0200788 io->log = log;
Christoph Hellwig51039cd2015-10-05 09:31:13 +0200789 INIT_LIST_HEAD(&io->log_sibling);
790 INIT_LIST_HEAD(&io->stripe_list);
Song Liu3bddb7f2016-11-18 16:46:50 -0800791 bio_list_init(&io->flush_barriers);
Christoph Hellwig51039cd2015-10-05 09:31:13 +0200792 io->state = IO_UNIT_RUNNING;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700793
Christoph Hellwige8deb632015-12-21 10:51:02 +1100794 io->meta_page = mempool_alloc(log->meta_pool, GFP_NOIO);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700795 block = page_address(io->meta_page);
Christoph Hellwige8deb632015-12-21 10:51:02 +1100796 clear_page(block);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700797 block->magic = cpu_to_le32(R5LOG_MAGIC);
798 block->version = R5LOG_VERSION;
799 block->seq = cpu_to_le64(log->seq);
800 block->position = cpu_to_le64(log->log_start);
801
802 io->log_start = log->log_start;
803 io->meta_offset = sizeof(struct r5l_meta_block);
Christoph Hellwig2b8ef162015-10-05 09:31:15 +0200804 io->seq = log->seq++;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700805
Christoph Hellwig6143e2c2015-10-05 09:31:16 +0200806 io->current_bio = r5l_bio_alloc(log);
807 io->current_bio->bi_end_io = r5l_log_endio;
808 io->current_bio->bi_private = io;
Christoph Hellwigb349feb2015-10-05 09:31:11 +0200809 bio_add_page(io->current_bio, io->meta_page, PAGE_SIZE, 0);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700810
Christoph Hellwigc1b99192015-10-05 09:31:14 +0200811 r5_reserve_log_entry(log, io);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700812
813 spin_lock_irq(&log->io_list_lock);
814 list_add_tail(&io->log_sibling, &log->running_ios);
815 spin_unlock_irq(&log->io_list_lock);
816
817 return io;
818}
819
820static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
821{
Christoph Hellwig22581f52015-10-05 09:31:10 +0200822 if (log->current_io &&
823 log->current_io->meta_offset + payload_size > PAGE_SIZE)
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700824 r5l_submit_current_io(log);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700825
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100826 if (!log->current_io) {
Christoph Hellwig22581f52015-10-05 09:31:10 +0200827 log->current_io = r5l_new_meta(log);
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100828 if (!log->current_io)
829 return -ENOMEM;
830 }
831
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700832 return 0;
833}
834
835static void r5l_append_payload_meta(struct r5l_log *log, u16 type,
836 sector_t location,
837 u32 checksum1, u32 checksum2,
838 bool checksum2_valid)
839{
840 struct r5l_io_unit *io = log->current_io;
841 struct r5l_payload_data_parity *payload;
842
843 payload = page_address(io->meta_page) + io->meta_offset;
844 payload->header.type = cpu_to_le16(type);
845 payload->header.flags = cpu_to_le16(0);
846 payload->size = cpu_to_le32((1 + !!checksum2_valid) <<
847 (PAGE_SHIFT - 9));
848 payload->location = cpu_to_le64(location);
849 payload->checksum[0] = cpu_to_le32(checksum1);
850 if (checksum2_valid)
851 payload->checksum[1] = cpu_to_le32(checksum2);
852
853 io->meta_offset += sizeof(struct r5l_payload_data_parity) +
854 sizeof(__le32) * (1 + !!checksum2_valid);
855}
856
857static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
858{
859 struct r5l_io_unit *io = log->current_io;
860
Christoph Hellwig6143e2c2015-10-05 09:31:16 +0200861 if (io->need_split_bio) {
Song Liu3bddb7f2016-11-18 16:46:50 -0800862 BUG_ON(io->split_bio);
863 io->split_bio = io->current_bio;
Christoph Hellwig6143e2c2015-10-05 09:31:16 +0200864 io->current_bio = r5l_bio_alloc(log);
Song Liu3bddb7f2016-11-18 16:46:50 -0800865 bio_chain(io->current_bio, io->split_bio);
866 io->need_split_bio = false;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700867 }
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700868
Christoph Hellwig6143e2c2015-10-05 09:31:16 +0200869 if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0))
870 BUG();
871
Christoph Hellwigc1b99192015-10-05 09:31:14 +0200872 r5_reserve_log_entry(log, io);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700873}
874
Song Liuea174812017-03-09 21:23:39 -0800875static void r5l_append_flush_payload(struct r5l_log *log, sector_t sect)
876{
877 struct mddev *mddev = log->rdev->mddev;
878 struct r5conf *conf = mddev->private;
879 struct r5l_io_unit *io;
880 struct r5l_payload_flush *payload;
881 int meta_size;
882
883 /*
884 * payload_flush requires extra writes to the journal.
885 * To avoid handling the extra IO in quiesce, just skip
886 * flush_payload
887 */
888 if (conf->quiesce)
889 return;
890
891 mutex_lock(&log->io_mutex);
892 meta_size = sizeof(struct r5l_payload_flush) + sizeof(__le64);
893
894 if (r5l_get_meta(log, meta_size)) {
895 mutex_unlock(&log->io_mutex);
896 return;
897 }
898
899 /* current implementation is one stripe per flush payload */
900 io = log->current_io;
901 payload = page_address(io->meta_page) + io->meta_offset;
902 payload->header.type = cpu_to_le16(R5LOG_PAYLOAD_FLUSH);
903 payload->header.flags = cpu_to_le16(0);
904 payload->size = cpu_to_le32(sizeof(__le64));
905 payload->flush_stripes[0] = cpu_to_le64(sect);
906 io->meta_offset += meta_size;
Song Liua9501d72017-08-03 10:03:17 -0700907 /* multiple flush payloads count as one pending_stripe */
908 if (!io->has_flush_payload) {
909 io->has_flush_payload = 1;
910 atomic_inc(&io->pending_stripe);
911 }
Song Liuea174812017-03-09 21:23:39 -0800912 mutex_unlock(&log->io_mutex);
913}
914
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100915static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700916 int data_pages, int parity_pages)
917{
918 int i;
919 int meta_size;
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100920 int ret;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700921 struct r5l_io_unit *io;
922
923 meta_size =
924 ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32))
925 * data_pages) +
926 sizeof(struct r5l_payload_data_parity) +
927 sizeof(__le32) * parity_pages;
928
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100929 ret = r5l_get_meta(log, meta_size);
930 if (ret)
931 return ret;
932
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700933 io = log->current_io;
934
Song Liu3bddb7f2016-11-18 16:46:50 -0800935 if (test_and_clear_bit(STRIPE_R5C_PREFLUSH, &sh->state))
936 io->has_flush = 1;
937
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700938 for (i = 0; i < sh->disks; i++) {
Song Liu1e6d6902016-11-17 15:24:39 -0800939 if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
940 test_bit(R5_InJournal, &sh->dev[i].flags))
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700941 continue;
942 if (i == sh->pd_idx || i == sh->qd_idx)
943 continue;
Song Liu3bddb7f2016-11-18 16:46:50 -0800944 if (test_bit(R5_WantFUA, &sh->dev[i].flags) &&
945 log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK) {
946 io->has_fua = 1;
947 /*
948 * we need to flush journal to make sure recovery can
949 * reach the data with fua flag
950 */
951 io->has_flush = 1;
952 }
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700953 r5l_append_payload_meta(log, R5LOG_PAYLOAD_DATA,
954 raid5_compute_blocknr(sh, i, 0),
955 sh->dev[i].log_checksum, 0, false);
956 r5l_append_payload_page(log, sh->dev[i].page);
957 }
958
Song Liu2ded3702016-11-17 15:24:38 -0800959 if (parity_pages == 2) {
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700960 r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
961 sh->sector, sh->dev[sh->pd_idx].log_checksum,
962 sh->dev[sh->qd_idx].log_checksum, true);
963 r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
964 r5l_append_payload_page(log, sh->dev[sh->qd_idx].page);
Song Liu2ded3702016-11-17 15:24:38 -0800965 } else if (parity_pages == 1) {
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700966 r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
967 sh->sector, sh->dev[sh->pd_idx].log_checksum,
968 0, false);
969 r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
Song Liu2ded3702016-11-17 15:24:38 -0800970 } else /* Just writing data, not parity, in caching phase */
971 BUG_ON(parity_pages != 0);
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700972
973 list_add_tail(&sh->log_list, &io->stripe_list);
974 atomic_inc(&io->pending_stripe);
975 sh->log_io = io;
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100976
Song Liua39f7af2016-11-17 15:24:40 -0800977 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
978 return 0;
979
980 if (sh->log_start == MaxSector) {
981 BUG_ON(!list_empty(&sh->r5c));
982 sh->log_start = io->log_start;
983 spin_lock_irq(&log->stripe_in_journal_lock);
984 list_add_tail(&sh->r5c,
985 &log->stripe_in_journal_list);
986 spin_unlock_irq(&log->stripe_in_journal_lock);
987 atomic_inc(&log->stripe_in_journal_count);
988 }
Christoph Hellwig5036c3902015-12-21 10:51:02 +1100989 return 0;
Shaohua Lif6bed0e2015-08-13 14:31:59 -0700990}
991
Song Liua39f7af2016-11-17 15:24:40 -0800992/* add stripe to no_space_stripes, and then wake up reclaim */
993static inline void r5l_add_no_space_stripe(struct r5l_log *log,
994 struct stripe_head *sh)
995{
996 spin_lock(&log->no_space_stripes_lock);
997 list_add_tail(&sh->log_list, &log->no_space_stripes);
998 spin_unlock(&log->no_space_stripes_lock);
999}
1000
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001001/*
1002 * running in raid5d, where reclaim could wait for raid5d too (when it flushes
1003 * data from log to raid disks), so we shouldn't wait for reclaim here
1004 */
1005int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
1006{
Song Liua39f7af2016-11-17 15:24:40 -08001007 struct r5conf *conf = sh->raid_conf;
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001008 int write_disks = 0;
1009 int data_pages, parity_pages;
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001010 int reserve;
1011 int i;
Christoph Hellwig5036c3902015-12-21 10:51:02 +11001012 int ret = 0;
Song Liua39f7af2016-11-17 15:24:40 -08001013 bool wake_reclaim = false;
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001014
1015 if (!log)
1016 return -EAGAIN;
1017 /* Don't support stripe batch */
1018 if (sh->log_io || !test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags) ||
1019 test_bit(STRIPE_SYNCING, &sh->state)) {
1020 /* the stripe is written to log, we start writing it to raid */
1021 clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
1022 return -EAGAIN;
1023 }
1024
Song Liu2ded3702016-11-17 15:24:38 -08001025 WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
1026
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001027 for (i = 0; i < sh->disks; i++) {
1028 void *addr;
1029
Song Liu1e6d6902016-11-17 15:24:39 -08001030 if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
1031 test_bit(R5_InJournal, &sh->dev[i].flags))
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001032 continue;
Song Liu1e6d6902016-11-17 15:24:39 -08001033
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001034 write_disks++;
1035 /* checksum is already calculated in last run */
1036 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
1037 continue;
1038 addr = kmap_atomic(sh->dev[i].page);
Shaohua Li5cb2fbd2015-10-28 08:41:25 -07001039 sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
1040 addr, PAGE_SIZE);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001041 kunmap_atomic(addr);
1042 }
1043 parity_pages = 1 + !!(sh->qd_idx >= 0);
1044 data_pages = write_disks - parity_pages;
1045
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001046 set_bit(STRIPE_LOG_TRAPPED, &sh->state);
Shaohua Li253f9fd42015-09-04 14:14:16 -07001047 /*
1048 * The stripe must enter state machine again to finish the write, so
1049 * don't delay.
1050 */
1051 clear_bit(STRIPE_DELAYED, &sh->state);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001052 atomic_inc(&sh->count);
1053
1054 mutex_lock(&log->io_mutex);
1055 /* meta + data */
1056 reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001057
Song Liua39f7af2016-11-17 15:24:40 -08001058 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
1059 if (!r5l_has_free_space(log, reserve)) {
1060 r5l_add_no_space_stripe(log, sh);
1061 wake_reclaim = true;
1062 } else {
1063 ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
1064 if (ret) {
1065 spin_lock_irq(&log->io_list_lock);
1066 list_add_tail(&sh->log_list,
1067 &log->no_mem_stripes);
1068 spin_unlock_irq(&log->io_list_lock);
1069 }
1070 }
1071 } else { /* R5C_JOURNAL_MODE_WRITE_BACK */
1072 /*
1073 * log space critical, do not process stripes that are
1074 * not in cache yet (sh->log_start == MaxSector).
1075 */
1076 if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
1077 sh->log_start == MaxSector) {
1078 r5l_add_no_space_stripe(log, sh);
1079 wake_reclaim = true;
1080 reserve = 0;
1081 } else if (!r5l_has_free_space(log, reserve)) {
1082 if (sh->log_start == log->last_checkpoint)
1083 BUG();
1084 else
1085 r5l_add_no_space_stripe(log, sh);
1086 } else {
1087 ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
1088 if (ret) {
1089 spin_lock_irq(&log->io_list_lock);
1090 list_add_tail(&sh->log_list,
1091 &log->no_mem_stripes);
1092 spin_unlock_irq(&log->io_list_lock);
1093 }
Christoph Hellwig5036c3902015-12-21 10:51:02 +11001094 }
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001095 }
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001096
Christoph Hellwig5036c3902015-12-21 10:51:02 +11001097 mutex_unlock(&log->io_mutex);
Song Liua39f7af2016-11-17 15:24:40 -08001098 if (wake_reclaim)
1099 r5l_wake_reclaim(log, reserve);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001100 return 0;
1101}
1102
1103void r5l_write_stripe_run(struct r5l_log *log)
1104{
1105 if (!log)
1106 return;
1107 mutex_lock(&log->io_mutex);
1108 r5l_submit_current_io(log);
1109 mutex_unlock(&log->io_mutex);
1110}
1111
Shaohua Li828cbe92015-09-02 13:49:49 -07001112int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio)
1113{
1114 if (!log)
1115 return -ENODEV;
Song Liu3bddb7f2016-11-18 16:46:50 -08001116
1117 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
1118 /*
1119 * in write through (journal only)
1120 * we flush log disk cache first, then write stripe data to
1121 * raid disks. So if bio is finished, the log disk cache is
1122 * flushed already. The recovery guarantees we can recovery
1123 * the bio from log disk, so we don't need to flush again
1124 */
1125 if (bio->bi_iter.bi_size == 0) {
1126 bio_endio(bio);
1127 return 0;
1128 }
1129 bio->bi_opf &= ~REQ_PREFLUSH;
1130 } else {
1131 /* write back (with cache) */
1132 if (bio->bi_iter.bi_size == 0) {
1133 mutex_lock(&log->io_mutex);
1134 r5l_get_meta(log, 0);
1135 bio_list_add(&log->current_io->flush_barriers, bio);
1136 log->current_io->has_flush = 1;
1137 log->current_io->has_null_flush = 1;
1138 atomic_inc(&log->current_io->pending_stripe);
1139 r5l_submit_current_io(log);
1140 mutex_unlock(&log->io_mutex);
1141 return 0;
1142 }
Shaohua Li828cbe92015-09-02 13:49:49 -07001143 }
Shaohua Li828cbe92015-09-02 13:49:49 -07001144 return -EAGAIN;
1145}
1146
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001147/* This will run after log space is reclaimed */
1148static void r5l_run_no_space_stripes(struct r5l_log *log)
1149{
1150 struct stripe_head *sh;
1151
1152 spin_lock(&log->no_space_stripes_lock);
1153 while (!list_empty(&log->no_space_stripes)) {
1154 sh = list_first_entry(&log->no_space_stripes,
1155 struct stripe_head, log_list);
1156 list_del_init(&sh->log_list);
1157 set_bit(STRIPE_HANDLE, &sh->state);
1158 raid5_release_stripe(sh);
1159 }
1160 spin_unlock(&log->no_space_stripes_lock);
1161}
1162
Song Liua39f7af2016-11-17 15:24:40 -08001163/*
1164 * calculate new last_checkpoint
1165 * for write through mode, returns log->next_checkpoint
1166 * for write back, returns log_start of first sh in stripe_in_journal_list
1167 */
1168static sector_t r5c_calculate_new_cp(struct r5conf *conf)
1169{
1170 struct stripe_head *sh;
1171 struct r5l_log *log = conf->log;
1172 sector_t new_cp;
1173 unsigned long flags;
1174
1175 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
1176 return log->next_checkpoint;
1177
1178 spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
1179 if (list_empty(&conf->log->stripe_in_journal_list)) {
1180 /* all stripes flushed */
Dan Carpenterd3014e22016-11-24 14:13:04 +03001181 spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
Song Liua39f7af2016-11-17 15:24:40 -08001182 return log->next_checkpoint;
1183 }
1184 sh = list_first_entry(&conf->log->stripe_in_journal_list,
1185 struct stripe_head, r5c);
1186 new_cp = sh->log_start;
1187 spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
1188 return new_cp;
1189}
1190
Christoph Hellwig17036462015-10-05 09:31:06 +02001191static sector_t r5l_reclaimable_space(struct r5l_log *log)
1192{
Song Liua39f7af2016-11-17 15:24:40 -08001193 struct r5conf *conf = log->rdev->mddev->private;
1194
Christoph Hellwig17036462015-10-05 09:31:06 +02001195 return r5l_ring_distance(log, log->last_checkpoint,
Song Liua39f7af2016-11-17 15:24:40 -08001196 r5c_calculate_new_cp(conf));
Christoph Hellwig17036462015-10-05 09:31:06 +02001197}
1198
Christoph Hellwig5036c3902015-12-21 10:51:02 +11001199static void r5l_run_no_mem_stripe(struct r5l_log *log)
1200{
1201 struct stripe_head *sh;
1202
Shaohua Liefa4b772017-10-18 22:08:13 -07001203 lockdep_assert_held(&log->io_list_lock);
Christoph Hellwig5036c3902015-12-21 10:51:02 +11001204
1205 if (!list_empty(&log->no_mem_stripes)) {
1206 sh = list_first_entry(&log->no_mem_stripes,
1207 struct stripe_head, log_list);
1208 list_del_init(&sh->log_list);
1209 set_bit(STRIPE_HANDLE, &sh->state);
1210 raid5_release_stripe(sh);
1211 }
1212}
1213
Christoph Hellwig04732f72015-10-05 09:31:07 +02001214static bool r5l_complete_finished_ios(struct r5l_log *log)
Christoph Hellwig17036462015-10-05 09:31:06 +02001215{
1216 struct r5l_io_unit *io, *next;
1217 bool found = false;
1218
Shaohua Liefa4b772017-10-18 22:08:13 -07001219 lockdep_assert_held(&log->io_list_lock);
Christoph Hellwig17036462015-10-05 09:31:06 +02001220
Christoph Hellwig04732f72015-10-05 09:31:07 +02001221 list_for_each_entry_safe(io, next, &log->finished_ios, log_sibling) {
Christoph Hellwig17036462015-10-05 09:31:06 +02001222 /* don't change list order */
1223 if (io->state < IO_UNIT_STRIPE_END)
1224 break;
1225
1226 log->next_checkpoint = io->log_start;
Christoph Hellwig17036462015-10-05 09:31:06 +02001227
1228 list_del(&io->log_sibling);
Christoph Hellwig5036c3902015-12-21 10:51:02 +11001229 mempool_free(io, log->io_pool);
1230 r5l_run_no_mem_stripe(log);
Christoph Hellwig17036462015-10-05 09:31:06 +02001231
1232 found = true;
1233 }
1234
1235 return found;
1236}
1237
Christoph Hellwig509ffec2015-09-02 13:49:48 -07001238static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
1239{
1240 struct r5l_log *log = io->log;
Song Liua39f7af2016-11-17 15:24:40 -08001241 struct r5conf *conf = log->rdev->mddev->private;
Christoph Hellwig509ffec2015-09-02 13:49:48 -07001242 unsigned long flags;
1243
1244 spin_lock_irqsave(&log->io_list_lock, flags);
1245 __r5l_set_io_unit_state(io, IO_UNIT_STRIPE_END);
Christoph Hellwig17036462015-10-05 09:31:06 +02001246
Christoph Hellwig04732f72015-10-05 09:31:07 +02001247 if (!r5l_complete_finished_ios(log)) {
Shaohua Li85f2f9a2015-09-04 14:14:05 -07001248 spin_unlock_irqrestore(&log->io_list_lock, flags);
1249 return;
1250 }
Christoph Hellwig509ffec2015-09-02 13:49:48 -07001251
Song Liua39f7af2016-11-17 15:24:40 -08001252 if (r5l_reclaimable_space(log) > log->max_free_space ||
1253 test_bit(R5C_LOG_TIGHT, &conf->cache_state))
Christoph Hellwig509ffec2015-09-02 13:49:48 -07001254 r5l_wake_reclaim(log, 0);
1255
Christoph Hellwig509ffec2015-09-02 13:49:48 -07001256 spin_unlock_irqrestore(&log->io_list_lock, flags);
1257 wake_up(&log->iounit_wait);
1258}
1259
Shaohua Li0576b1c2015-08-13 14:32:00 -07001260void r5l_stripe_write_finished(struct stripe_head *sh)
1261{
1262 struct r5l_io_unit *io;
1263
Shaohua Li0576b1c2015-08-13 14:32:00 -07001264 io = sh->log_io;
Shaohua Li0576b1c2015-08-13 14:32:00 -07001265 sh->log_io = NULL;
1266
Christoph Hellwig509ffec2015-09-02 13:49:48 -07001267 if (io && atomic_dec_and_test(&io->pending_stripe))
1268 __r5l_stripe_write_finished(io);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001269}
1270
Shaohua Lia8c34f92015-09-02 13:49:46 -07001271static void r5l_log_flush_endio(struct bio *bio)
1272{
1273 struct r5l_log *log = container_of(bio, struct r5l_log,
1274 flush_bio);
1275 unsigned long flags;
1276 struct r5l_io_unit *io;
Shaohua Lia8c34f92015-09-02 13:49:46 -07001277
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +02001278 if (bio->bi_status)
Shaohua Li6e74a9c2015-10-08 21:54:08 -07001279 md_error(log->rdev->mddev, log->rdev);
1280
Shaohua Lia8c34f92015-09-02 13:49:46 -07001281 spin_lock_irqsave(&log->io_list_lock, flags);
Christoph Hellwigd8858f42015-10-05 09:31:08 +02001282 list_for_each_entry(io, &log->flushing_ios, log_sibling)
1283 r5l_io_run_stripes(io);
Christoph Hellwig04732f72015-10-05 09:31:07 +02001284 list_splice_tail_init(&log->flushing_ios, &log->finished_ios);
Shaohua Lia8c34f92015-09-02 13:49:46 -07001285 spin_unlock_irqrestore(&log->io_list_lock, flags);
1286}
1287
Shaohua Li0576b1c2015-08-13 14:32:00 -07001288/*
1289 * Starting dispatch IO to raid.
1290 * io_unit(meta) consists of a log. There is one situation we want to avoid. A
1291 * broken meta in the middle of a log causes recovery can't find meta at the
1292 * head of log. If operations require meta at the head persistent in log, we
1293 * must make sure meta before it persistent in log too. A case is:
1294 *
1295 * stripe data/parity is in log, we start write stripe to raid disks. stripe
1296 * data/parity must be persistent in log before we do the write to raid disks.
1297 *
1298 * The solution is we restrictly maintain io_unit list order. In this case, we
1299 * only write stripes of an io_unit to raid disks till the io_unit is the first
1300 * one whose data/parity is in log.
1301 */
1302void r5l_flush_stripe_to_raid(struct r5l_log *log)
1303{
Shaohua Lia8c34f92015-09-02 13:49:46 -07001304 bool do_flush;
Christoph Hellwig56fef7c2015-10-05 09:31:09 +02001305
1306 if (!log || !log->need_cache_flush)
Shaohua Li0576b1c2015-08-13 14:32:00 -07001307 return;
Shaohua Li0576b1c2015-08-13 14:32:00 -07001308
Shaohua Lia8c34f92015-09-02 13:49:46 -07001309 spin_lock_irq(&log->io_list_lock);
1310 /* flush bio is running */
1311 if (!list_empty(&log->flushing_ios)) {
1312 spin_unlock_irq(&log->io_list_lock);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001313 return;
Shaohua Li0576b1c2015-08-13 14:32:00 -07001314 }
Shaohua Lia8c34f92015-09-02 13:49:46 -07001315 list_splice_tail_init(&log->io_end_ios, &log->flushing_ios);
1316 do_flush = !list_empty(&log->flushing_ios);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001317 spin_unlock_irq(&log->io_list_lock);
Shaohua Lia8c34f92015-09-02 13:49:46 -07001318
1319 if (!do_flush)
1320 return;
1321 bio_reset(&log->flush_bio);
Christoph Hellwig74d46992017-08-23 19:10:32 +02001322 bio_set_dev(&log->flush_bio, log->rdev->bdev);
Shaohua Lia8c34f92015-09-02 13:49:46 -07001323 log->flush_bio.bi_end_io = r5l_log_flush_endio;
Christoph Hellwig70fd7612016-11-01 07:40:10 -06001324 log->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
Mike Christie4e49ea42016-06-05 14:31:41 -05001325 submit_bio(&log->flush_bio);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001326}
1327
Shaohua Li0576b1c2015-08-13 14:32:00 -07001328static void r5l_write_super(struct r5l_log *log, sector_t cp);
Shaohua Li4b482042015-10-08 21:54:06 -07001329static void r5l_write_super_and_discard_space(struct r5l_log *log,
1330 sector_t end)
1331{
1332 struct block_device *bdev = log->rdev->bdev;
1333 struct mddev *mddev;
1334
1335 r5l_write_super(log, end);
1336
1337 if (!blk_queue_discard(bdev_get_queue(bdev)))
1338 return;
1339
1340 mddev = log->rdev->mddev;
1341 /*
Shaohua Li8e018c22016-08-25 10:09:39 -07001342 * Discard could zero data, so before discard we must make sure
1343 * superblock is updated to new log tail. Updating superblock (either
1344 * directly call md_update_sb() or depend on md thread) must hold
1345 * reconfig mutex. On the other hand, raid5_quiesce is called with
1346 * reconfig_mutex hold. The first step of raid5_quiesce() is waitting
1347 * for all IO finish, hence waitting for reclaim thread, while reclaim
1348 * thread is calling this function and waitting for reconfig mutex. So
1349 * there is a deadlock. We workaround this issue with a trylock.
1350 * FIXME: we could miss discard if we can't take reconfig mutex
Shaohua Li4b482042015-10-08 21:54:06 -07001351 */
Shaohua Li29530792016-12-08 15:48:19 -08001352 set_mask_bits(&mddev->sb_flags, 0,
1353 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
Shaohua Li8e018c22016-08-25 10:09:39 -07001354 if (!mddev_trylock(mddev))
1355 return;
1356 md_update_sb(mddev, 1);
1357 mddev_unlock(mddev);
Shaohua Li4b482042015-10-08 21:54:06 -07001358
Shaohua Li6e74a9c2015-10-08 21:54:08 -07001359 /* discard IO error really doesn't matter, ignore it */
Shaohua Li4b482042015-10-08 21:54:06 -07001360 if (log->last_checkpoint < end) {
1361 blkdev_issue_discard(bdev,
1362 log->last_checkpoint + log->rdev->data_offset,
1363 end - log->last_checkpoint, GFP_NOIO, 0);
1364 } else {
1365 blkdev_issue_discard(bdev,
1366 log->last_checkpoint + log->rdev->data_offset,
1367 log->device_size - log->last_checkpoint,
1368 GFP_NOIO, 0);
1369 blkdev_issue_discard(bdev, log->rdev->data_offset, end,
1370 GFP_NOIO, 0);
1371 }
1372}
1373
Song Liua39f7af2016-11-17 15:24:40 -08001374/*
1375 * r5c_flush_stripe moves stripe from cached list to handle_list. When called,
1376 * the stripe must be on r5c_cached_full_stripes or r5c_cached_partial_stripes.
1377 *
1378 * must hold conf->device_lock
1379 */
1380static void r5c_flush_stripe(struct r5conf *conf, struct stripe_head *sh)
1381{
1382 BUG_ON(list_empty(&sh->lru));
1383 BUG_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
1384 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
1385
1386 /*
1387 * The stripe is not ON_RELEASE_LIST, so it is safe to call
1388 * raid5_release_stripe() while holding conf->device_lock
1389 */
1390 BUG_ON(test_bit(STRIPE_ON_RELEASE_LIST, &sh->state));
Shaohua Liefa4b772017-10-18 22:08:13 -07001391 lockdep_assert_held(&conf->device_lock);
Song Liua39f7af2016-11-17 15:24:40 -08001392
1393 list_del_init(&sh->lru);
1394 atomic_inc(&sh->count);
1395
1396 set_bit(STRIPE_HANDLE, &sh->state);
1397 atomic_inc(&conf->active_stripes);
1398 r5c_make_stripe_write_out(sh);
1399
Shaohua Lie33fbb92017-02-10 16:18:09 -08001400 if (test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
1401 atomic_inc(&conf->r5c_flushing_partial_stripes);
1402 else
1403 atomic_inc(&conf->r5c_flushing_full_stripes);
Song Liua39f7af2016-11-17 15:24:40 -08001404 raid5_release_stripe(sh);
1405}
1406
1407/*
1408 * if num == 0, flush all full stripes
1409 * if num > 0, flush all full stripes. If less than num full stripes are
1410 * flushed, flush some partial stripes until totally num stripes are
1411 * flushed or there is no more cached stripes.
1412 */
1413void r5c_flush_cache(struct r5conf *conf, int num)
1414{
1415 int count;
1416 struct stripe_head *sh, *next;
1417
Shaohua Liefa4b772017-10-18 22:08:13 -07001418 lockdep_assert_held(&conf->device_lock);
Song Liua39f7af2016-11-17 15:24:40 -08001419 if (!conf->log)
1420 return;
1421
1422 count = 0;
1423 list_for_each_entry_safe(sh, next, &conf->r5c_full_stripe_list, lru) {
1424 r5c_flush_stripe(conf, sh);
1425 count++;
1426 }
1427
1428 if (count >= num)
1429 return;
1430 list_for_each_entry_safe(sh, next,
1431 &conf->r5c_partial_stripe_list, lru) {
1432 r5c_flush_stripe(conf, sh);
1433 if (++count >= num)
1434 break;
1435 }
1436}
1437
1438static void r5c_do_reclaim(struct r5conf *conf)
1439{
1440 struct r5l_log *log = conf->log;
1441 struct stripe_head *sh;
1442 int count = 0;
1443 unsigned long flags;
1444 int total_cached;
1445 int stripes_to_flush;
Shaohua Lie33fbb92017-02-10 16:18:09 -08001446 int flushing_partial, flushing_full;
Song Liua39f7af2016-11-17 15:24:40 -08001447
1448 if (!r5c_is_writeback(log))
1449 return;
1450
Shaohua Lie33fbb92017-02-10 16:18:09 -08001451 flushing_partial = atomic_read(&conf->r5c_flushing_partial_stripes);
1452 flushing_full = atomic_read(&conf->r5c_flushing_full_stripes);
Song Liua39f7af2016-11-17 15:24:40 -08001453 total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
Shaohua Lie33fbb92017-02-10 16:18:09 -08001454 atomic_read(&conf->r5c_cached_full_stripes) -
1455 flushing_full - flushing_partial;
Song Liua39f7af2016-11-17 15:24:40 -08001456
1457 if (total_cached > conf->min_nr_stripes * 3 / 4 ||
1458 atomic_read(&conf->empty_inactive_list_nr) > 0)
1459 /*
1460 * if stripe cache pressure high, flush all full stripes and
1461 * some partial stripes
1462 */
1463 stripes_to_flush = R5C_RECLAIM_STRIPE_GROUP;
1464 else if (total_cached > conf->min_nr_stripes * 1 / 2 ||
Shaohua Lie33fbb92017-02-10 16:18:09 -08001465 atomic_read(&conf->r5c_cached_full_stripes) - flushing_full >
Shaohua Li84890c02017-02-15 19:58:05 -08001466 R5C_FULL_STRIPE_FLUSH_BATCH(conf))
Song Liua39f7af2016-11-17 15:24:40 -08001467 /*
1468 * if stripe cache pressure moderate, or if there is many full
1469 * stripes,flush all full stripes
1470 */
1471 stripes_to_flush = 0;
1472 else
1473 /* no need to flush */
1474 stripes_to_flush = -1;
1475
1476 if (stripes_to_flush >= 0) {
1477 spin_lock_irqsave(&conf->device_lock, flags);
1478 r5c_flush_cache(conf, stripes_to_flush);
1479 spin_unlock_irqrestore(&conf->device_lock, flags);
1480 }
1481
1482 /* if log space is tight, flush stripes on stripe_in_journal_list */
1483 if (test_bit(R5C_LOG_TIGHT, &conf->cache_state)) {
1484 spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
1485 spin_lock(&conf->device_lock);
1486 list_for_each_entry(sh, &log->stripe_in_journal_list, r5c) {
1487 /*
1488 * stripes on stripe_in_journal_list could be in any
1489 * state of the stripe_cache state machine. In this
1490 * case, we only want to flush stripe on
1491 * r5c_cached_full/partial_stripes. The following
1492 * condition makes sure the stripe is on one of the
1493 * two lists.
1494 */
1495 if (!list_empty(&sh->lru) &&
1496 !test_bit(STRIPE_HANDLE, &sh->state) &&
1497 atomic_read(&sh->count) == 0) {
1498 r5c_flush_stripe(conf, sh);
Shaohua Lie8fd52e2017-02-10 16:18:08 -08001499 if (count++ >= R5C_RECLAIM_STRIPE_GROUP)
1500 break;
Song Liua39f7af2016-11-17 15:24:40 -08001501 }
Song Liua39f7af2016-11-17 15:24:40 -08001502 }
1503 spin_unlock(&conf->device_lock);
1504 spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
1505 }
Song Liuf687a332016-11-30 16:57:54 -08001506
1507 if (!test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
1508 r5l_run_no_space_stripes(log);
1509
Song Liua39f7af2016-11-17 15:24:40 -08001510 md_wakeup_thread(conf->mddev->thread);
1511}
Shaohua Li4b482042015-10-08 21:54:06 -07001512
Shaohua Li0576b1c2015-08-13 14:32:00 -07001513static void r5l_do_reclaim(struct r5l_log *log)
1514{
Song Liua39f7af2016-11-17 15:24:40 -08001515 struct r5conf *conf = log->rdev->mddev->private;
Shaohua Li0576b1c2015-08-13 14:32:00 -07001516 sector_t reclaim_target = xchg(&log->reclaim_target, 0);
Christoph Hellwig17036462015-10-05 09:31:06 +02001517 sector_t reclaimable;
1518 sector_t next_checkpoint;
Song Liua39f7af2016-11-17 15:24:40 -08001519 bool write_super;
Shaohua Li0576b1c2015-08-13 14:32:00 -07001520
1521 spin_lock_irq(&log->io_list_lock);
Song Liua39f7af2016-11-17 15:24:40 -08001522 write_super = r5l_reclaimable_space(log) > log->max_free_space ||
1523 reclaim_target != 0 || !list_empty(&log->no_space_stripes);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001524 /*
1525 * move proper io_unit to reclaim list. We should not change the order.
1526 * reclaimable/unreclaimable io_unit can be mixed in the list, we
1527 * shouldn't reuse space of an unreclaimable io_unit
1528 */
1529 while (1) {
Christoph Hellwig17036462015-10-05 09:31:06 +02001530 reclaimable = r5l_reclaimable_space(log);
1531 if (reclaimable >= reclaim_target ||
Shaohua Li0576b1c2015-08-13 14:32:00 -07001532 (list_empty(&log->running_ios) &&
1533 list_empty(&log->io_end_ios) &&
Shaohua Lia8c34f92015-09-02 13:49:46 -07001534 list_empty(&log->flushing_ios) &&
Christoph Hellwig04732f72015-10-05 09:31:07 +02001535 list_empty(&log->finished_ios)))
Shaohua Li0576b1c2015-08-13 14:32:00 -07001536 break;
1537
Christoph Hellwig17036462015-10-05 09:31:06 +02001538 md_wakeup_thread(log->rdev->mddev->thread);
1539 wait_event_lock_irq(log->iounit_wait,
1540 r5l_reclaimable_space(log) > reclaimable,
1541 log->io_list_lock);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001542 }
Christoph Hellwig17036462015-10-05 09:31:06 +02001543
Song Liua39f7af2016-11-17 15:24:40 -08001544 next_checkpoint = r5c_calculate_new_cp(conf);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001545 spin_unlock_irq(&log->io_list_lock);
1546
Song Liua39f7af2016-11-17 15:24:40 -08001547 if (reclaimable == 0 || !write_super)
Shaohua Li0576b1c2015-08-13 14:32:00 -07001548 return;
1549
Shaohua Li0576b1c2015-08-13 14:32:00 -07001550 /*
1551 * write_super will flush cache of each raid disk. We must write super
1552 * here, because the log area might be reused soon and we don't want to
1553 * confuse recovery
1554 */
Shaohua Li4b482042015-10-08 21:54:06 -07001555 r5l_write_super_and_discard_space(log, next_checkpoint);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001556
1557 mutex_lock(&log->io_mutex);
Christoph Hellwig17036462015-10-05 09:31:06 +02001558 log->last_checkpoint = next_checkpoint;
Song Liua39f7af2016-11-17 15:24:40 -08001559 r5c_update_log_state(log);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001560 mutex_unlock(&log->io_mutex);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001561
Christoph Hellwig17036462015-10-05 09:31:06 +02001562 r5l_run_no_space_stripes(log);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001563}
1564
1565static void r5l_reclaim_thread(struct md_thread *thread)
1566{
1567 struct mddev *mddev = thread->mddev;
1568 struct r5conf *conf = mddev->private;
1569 struct r5l_log *log = conf->log;
1570
1571 if (!log)
1572 return;
Song Liua39f7af2016-11-17 15:24:40 -08001573 r5c_do_reclaim(conf);
Shaohua Li0576b1c2015-08-13 14:32:00 -07001574 r5l_do_reclaim(log);
1575}
1576
Song Liua39f7af2016-11-17 15:24:40 -08001577void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001578{
Shaohua Li0576b1c2015-08-13 14:32:00 -07001579 unsigned long target;
1580 unsigned long new = (unsigned long)space; /* overflow in theory */
1581
Song Liua39f7af2016-11-17 15:24:40 -08001582 if (!log)
1583 return;
Shaohua Li0576b1c2015-08-13 14:32:00 -07001584 do {
1585 target = log->reclaim_target;
1586 if (new < target)
1587 return;
1588 } while (cmpxchg(&log->reclaim_target, target, new) != target);
1589 md_wakeup_thread(log->reclaim_thread);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07001590}
1591
NeilBrownb03e0cc2017-10-19 12:49:15 +11001592void r5l_quiesce(struct r5l_log *log, int quiesce)
Shaohua Lie6c033f2015-10-04 09:20:12 -07001593{
Shaohua Li4b482042015-10-08 21:54:06 -07001594 struct mddev *mddev;
NeilBrownb03e0cc2017-10-19 12:49:15 +11001595 if (!log)
Shaohua Lie6c033f2015-10-04 09:20:12 -07001596 return;
NeilBrownb03e0cc2017-10-19 12:49:15 +11001597
1598 if (quiesce) {
Shaohua Li4b482042015-10-08 21:54:06 -07001599 /* make sure r5l_write_super_and_discard_space exits */
1600 mddev = log->rdev->mddev;
1601 wake_up(&mddev->sb_wait);
Shaohua Lice1ccd02016-11-21 10:29:18 -08001602 kthread_park(log->reclaim_thread->tsk);
Song Liua39f7af2016-11-17 15:24:40 -08001603 r5l_wake_reclaim(log, MaxSector);
Shaohua Lie6c033f2015-10-04 09:20:12 -07001604 r5l_do_reclaim(log);
NeilBrownb03e0cc2017-10-19 12:49:15 +11001605 } else
1606 kthread_unpark(log->reclaim_thread->tsk);
Shaohua Lie6c033f2015-10-04 09:20:12 -07001607}
1608
Shaohua Li6e74a9c2015-10-08 21:54:08 -07001609bool r5l_log_disk_error(struct r5conf *conf)
1610{
Shaohua Lif6b6ec52015-12-21 10:51:02 +11001611 struct r5l_log *log;
1612 bool ret;
Shaohua Li7dde2ad2015-10-08 21:54:10 -07001613 /* don't allow write if journal disk is missing */
Shaohua Lif6b6ec52015-12-21 10:51:02 +11001614 rcu_read_lock();
1615 log = rcu_dereference(conf->log);
1616
1617 if (!log)
1618 ret = test_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
1619 else
1620 ret = test_bit(Faulty, &log->rdev->flags);
1621 rcu_read_unlock();
1622 return ret;
Shaohua Li6e74a9c2015-10-08 21:54:08 -07001623}
1624
Song Liueffe6ee2017-03-07 16:49:17 -08001625#define R5L_RECOVERY_PAGE_POOL_SIZE 256
1626
Shaohua Li355810d2015-08-13 14:32:01 -07001627struct r5l_recovery_ctx {
1628 struct page *meta_page; /* current meta */
1629 sector_t meta_total_blocks; /* total size of current meta and data */
1630 sector_t pos; /* recovery position */
1631 u64 seq; /* recovery position seq */
Song Liub4c625c2016-11-17 15:24:43 -08001632 int data_parity_stripes; /* number of data_parity stripes */
1633 int data_only_stripes; /* number of data_only stripes */
1634 struct list_head cached_list;
Song Liueffe6ee2017-03-07 16:49:17 -08001635
1636 /*
1637 * read ahead page pool (ra_pool)
1638 * in recovery, log is read sequentially. It is not efficient to
1639 * read every page with sync_page_io(). The read ahead page pool
1640 * reads multiple pages with one IO, so further log read can
1641 * just copy data from the pool.
1642 */
1643 struct page *ra_pool[R5L_RECOVERY_PAGE_POOL_SIZE];
1644 sector_t pool_offset; /* offset of first page in the pool */
1645 int total_pages; /* total allocated pages */
1646 int valid_pages; /* pages with valid data */
1647 struct bio *ra_bio; /* bio to do the read ahead */
Shaohua Li355810d2015-08-13 14:32:01 -07001648};
1649
Song Liueffe6ee2017-03-07 16:49:17 -08001650static int r5l_recovery_allocate_ra_pool(struct r5l_log *log,
1651 struct r5l_recovery_ctx *ctx)
1652{
1653 struct page *page;
1654
1655 ctx->ra_bio = bio_alloc_bioset(GFP_KERNEL, BIO_MAX_PAGES, log->bs);
1656 if (!ctx->ra_bio)
1657 return -ENOMEM;
1658
1659 ctx->valid_pages = 0;
1660 ctx->total_pages = 0;
1661 while (ctx->total_pages < R5L_RECOVERY_PAGE_POOL_SIZE) {
1662 page = alloc_page(GFP_KERNEL);
1663
1664 if (!page)
1665 break;
1666 ctx->ra_pool[ctx->total_pages] = page;
1667 ctx->total_pages += 1;
1668 }
1669
1670 if (ctx->total_pages == 0) {
1671 bio_put(ctx->ra_bio);
1672 return -ENOMEM;
1673 }
1674
1675 ctx->pool_offset = 0;
1676 return 0;
1677}
1678
1679static void r5l_recovery_free_ra_pool(struct r5l_log *log,
1680 struct r5l_recovery_ctx *ctx)
1681{
1682 int i;
1683
1684 for (i = 0; i < ctx->total_pages; ++i)
1685 put_page(ctx->ra_pool[i]);
1686 bio_put(ctx->ra_bio);
1687}
1688
1689/*
1690 * fetch ctx->valid_pages pages from offset
1691 * In normal cases, ctx->valid_pages == ctx->total_pages after the call.
1692 * However, if the offset is close to the end of the journal device,
1693 * ctx->valid_pages could be smaller than ctx->total_pages
1694 */
1695static int r5l_recovery_fetch_ra_pool(struct r5l_log *log,
1696 struct r5l_recovery_ctx *ctx,
1697 sector_t offset)
1698{
1699 bio_reset(ctx->ra_bio);
Christoph Hellwig74d46992017-08-23 19:10:32 +02001700 bio_set_dev(ctx->ra_bio, log->rdev->bdev);
Song Liueffe6ee2017-03-07 16:49:17 -08001701 bio_set_op_attrs(ctx->ra_bio, REQ_OP_READ, 0);
1702 ctx->ra_bio->bi_iter.bi_sector = log->rdev->data_offset + offset;
1703
1704 ctx->valid_pages = 0;
1705 ctx->pool_offset = offset;
1706
1707 while (ctx->valid_pages < ctx->total_pages) {
1708 bio_add_page(ctx->ra_bio,
1709 ctx->ra_pool[ctx->valid_pages], PAGE_SIZE, 0);
1710 ctx->valid_pages += 1;
1711
1712 offset = r5l_ring_add(log, offset, BLOCK_SECTORS);
1713
1714 if (offset == 0) /* reached end of the device */
1715 break;
1716 }
1717
1718 return submit_bio_wait(ctx->ra_bio);
1719}
1720
1721/*
1722 * try read a page from the read ahead page pool, if the page is not in the
1723 * pool, call r5l_recovery_fetch_ra_pool
1724 */
1725static int r5l_recovery_read_page(struct r5l_log *log,
1726 struct r5l_recovery_ctx *ctx,
1727 struct page *page,
1728 sector_t offset)
1729{
1730 int ret;
1731
1732 if (offset < ctx->pool_offset ||
1733 offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS) {
1734 ret = r5l_recovery_fetch_ra_pool(log, ctx, offset);
1735 if (ret)
1736 return ret;
1737 }
1738
1739 BUG_ON(offset < ctx->pool_offset ||
1740 offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS);
1741
1742 memcpy(page_address(page),
1743 page_address(ctx->ra_pool[(offset - ctx->pool_offset) >>
1744 BLOCK_SECTOR_SHIFT]),
1745 PAGE_SIZE);
1746 return 0;
1747}
1748
Song Liu9ed988f52016-11-17 15:24:42 -08001749static int r5l_recovery_read_meta_block(struct r5l_log *log,
1750 struct r5l_recovery_ctx *ctx)
Shaohua Li355810d2015-08-13 14:32:01 -07001751{
1752 struct page *page = ctx->meta_page;
1753 struct r5l_meta_block *mb;
1754 u32 crc, stored_crc;
Song Liueffe6ee2017-03-07 16:49:17 -08001755 int ret;
Shaohua Li355810d2015-08-13 14:32:01 -07001756
Song Liueffe6ee2017-03-07 16:49:17 -08001757 ret = r5l_recovery_read_page(log, ctx, page, ctx->pos);
1758 if (ret != 0)
1759 return ret;
Shaohua Li355810d2015-08-13 14:32:01 -07001760
1761 mb = page_address(page);
1762 stored_crc = le32_to_cpu(mb->checksum);
1763 mb->checksum = 0;
1764
1765 if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
1766 le64_to_cpu(mb->seq) != ctx->seq ||
1767 mb->version != R5LOG_VERSION ||
1768 le64_to_cpu(mb->position) != ctx->pos)
1769 return -EINVAL;
1770
Shaohua Li5cb2fbd2015-10-28 08:41:25 -07001771 crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
Shaohua Li355810d2015-08-13 14:32:01 -07001772 if (stored_crc != crc)
1773 return -EINVAL;
1774
1775 if (le32_to_cpu(mb->meta_size) > PAGE_SIZE)
1776 return -EINVAL;
1777
1778 ctx->meta_total_blocks = BLOCK_SECTORS;
1779
1780 return 0;
1781}
1782
Song Liu9ed988f52016-11-17 15:24:42 -08001783static void
1784r5l_recovery_create_empty_meta_block(struct r5l_log *log,
1785 struct page *page,
1786 sector_t pos, u64 seq)
Shaohua Li355810d2015-08-13 14:32:01 -07001787{
Shaohua Li355810d2015-08-13 14:32:01 -07001788 struct r5l_meta_block *mb;
Shaohua Li355810d2015-08-13 14:32:01 -07001789
Shaohua Li355810d2015-08-13 14:32:01 -07001790 mb = page_address(page);
Song Liu9ed988f52016-11-17 15:24:42 -08001791 clear_page(mb);
Shaohua Li355810d2015-08-13 14:32:01 -07001792 mb->magic = cpu_to_le32(R5LOG_MAGIC);
1793 mb->version = R5LOG_VERSION;
1794 mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
1795 mb->seq = cpu_to_le64(seq);
1796 mb->position = cpu_to_le64(pos);
Shaohua Li355810d2015-08-13 14:32:01 -07001797}
1798
1799static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
1800 u64 seq)
1801{
1802 struct page *page;
1803 struct r5l_meta_block *mb;
Shaohua Li355810d2015-08-13 14:32:01 -07001804
Song Liu9ed988f52016-11-17 15:24:42 -08001805 page = alloc_page(GFP_KERNEL);
Shaohua Li355810d2015-08-13 14:32:01 -07001806 if (!page)
1807 return -ENOMEM;
Song Liu9ed988f52016-11-17 15:24:42 -08001808 r5l_recovery_create_empty_meta_block(log, page, pos, seq);
Shaohua Li355810d2015-08-13 14:32:01 -07001809 mb = page_address(page);
Song Liu5c88f402016-12-07 09:42:05 -08001810 mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
1811 mb, PAGE_SIZE));
Mike Christie796a5cf2016-06-05 14:32:07 -05001812 if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE,
Jan Kara5a8948f2017-05-31 09:44:33 +02001813 REQ_SYNC | REQ_FUA, false)) {
Shaohua Li355810d2015-08-13 14:32:01 -07001814 __free_page(page);
1815 return -EIO;
1816 }
1817 __free_page(page);
1818 return 0;
1819}
1820
Song Liub4c625c2016-11-17 15:24:43 -08001821/*
1822 * r5l_recovery_load_data and r5l_recovery_load_parity uses flag R5_Wantwrite
1823 * to mark valid (potentially not flushed) data in the journal.
1824 *
1825 * We already verified checksum in r5l_recovery_verify_data_checksum_for_mb,
1826 * so there should not be any mismatch here.
1827 */
1828static void r5l_recovery_load_data(struct r5l_log *log,
1829 struct stripe_head *sh,
1830 struct r5l_recovery_ctx *ctx,
1831 struct r5l_payload_data_parity *payload,
1832 sector_t log_offset)
1833{
1834 struct mddev *mddev = log->rdev->mddev;
1835 struct r5conf *conf = mddev->private;
1836 int dd_idx;
1837
1838 raid5_compute_sector(conf,
1839 le64_to_cpu(payload->location), 0,
1840 &dd_idx, sh);
Song Liueffe6ee2017-03-07 16:49:17 -08001841 r5l_recovery_read_page(log, ctx, sh->dev[dd_idx].page, log_offset);
Song Liub4c625c2016-11-17 15:24:43 -08001842 sh->dev[dd_idx].log_checksum =
1843 le32_to_cpu(payload->checksum[0]);
1844 ctx->meta_total_blocks += BLOCK_SECTORS;
1845
1846 set_bit(R5_Wantwrite, &sh->dev[dd_idx].flags);
1847 set_bit(STRIPE_R5C_CACHING, &sh->state);
1848}
1849
1850static void r5l_recovery_load_parity(struct r5l_log *log,
1851 struct stripe_head *sh,
1852 struct r5l_recovery_ctx *ctx,
1853 struct r5l_payload_data_parity *payload,
1854 sector_t log_offset)
1855{
1856 struct mddev *mddev = log->rdev->mddev;
1857 struct r5conf *conf = mddev->private;
1858
1859 ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
Song Liueffe6ee2017-03-07 16:49:17 -08001860 r5l_recovery_read_page(log, ctx, sh->dev[sh->pd_idx].page, log_offset);
Song Liub4c625c2016-11-17 15:24:43 -08001861 sh->dev[sh->pd_idx].log_checksum =
1862 le32_to_cpu(payload->checksum[0]);
1863 set_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags);
1864
1865 if (sh->qd_idx >= 0) {
Song Liueffe6ee2017-03-07 16:49:17 -08001866 r5l_recovery_read_page(
1867 log, ctx, sh->dev[sh->qd_idx].page,
1868 r5l_ring_add(log, log_offset, BLOCK_SECTORS));
Song Liub4c625c2016-11-17 15:24:43 -08001869 sh->dev[sh->qd_idx].log_checksum =
1870 le32_to_cpu(payload->checksum[1]);
1871 set_bit(R5_Wantwrite, &sh->dev[sh->qd_idx].flags);
1872 }
1873 clear_bit(STRIPE_R5C_CACHING, &sh->state);
1874}
1875
1876static void r5l_recovery_reset_stripe(struct stripe_head *sh)
1877{
1878 int i;
1879
1880 sh->state = 0;
1881 sh->log_start = MaxSector;
1882 for (i = sh->disks; i--; )
1883 sh->dev[i].flags = 0;
1884}
1885
1886static void
1887r5l_recovery_replay_one_stripe(struct r5conf *conf,
1888 struct stripe_head *sh,
1889 struct r5l_recovery_ctx *ctx)
1890{
1891 struct md_rdev *rdev, *rrdev;
1892 int disk_index;
1893 int data_count = 0;
1894
1895 for (disk_index = 0; disk_index < sh->disks; disk_index++) {
1896 if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
1897 continue;
1898 if (disk_index == sh->qd_idx || disk_index == sh->pd_idx)
1899 continue;
1900 data_count++;
1901 }
1902
1903 /*
1904 * stripes that only have parity must have been flushed
1905 * before the crash that we are now recovering from, so
1906 * there is nothing more to recovery.
1907 */
1908 if (data_count == 0)
1909 goto out;
1910
1911 for (disk_index = 0; disk_index < sh->disks; disk_index++) {
1912 if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
1913 continue;
1914
1915 /* in case device is broken */
1916 rcu_read_lock();
1917 rdev = rcu_dereference(conf->disks[disk_index].rdev);
1918 if (rdev) {
1919 atomic_inc(&rdev->nr_pending);
1920 rcu_read_unlock();
1921 sync_page_io(rdev, sh->sector, PAGE_SIZE,
1922 sh->dev[disk_index].page, REQ_OP_WRITE, 0,
1923 false);
1924 rdev_dec_pending(rdev, rdev->mddev);
1925 rcu_read_lock();
1926 }
1927 rrdev = rcu_dereference(conf->disks[disk_index].replacement);
1928 if (rrdev) {
1929 atomic_inc(&rrdev->nr_pending);
1930 rcu_read_unlock();
1931 sync_page_io(rrdev, sh->sector, PAGE_SIZE,
1932 sh->dev[disk_index].page, REQ_OP_WRITE, 0,
1933 false);
1934 rdev_dec_pending(rrdev, rrdev->mddev);
1935 rcu_read_lock();
1936 }
1937 rcu_read_unlock();
1938 }
1939 ctx->data_parity_stripes++;
1940out:
1941 r5l_recovery_reset_stripe(sh);
1942}
1943
1944static struct stripe_head *
1945r5c_recovery_alloc_stripe(struct r5conf *conf,
Song Liu3c66abb2016-12-14 15:38:01 -08001946 sector_t stripe_sect)
Song Liub4c625c2016-11-17 15:24:43 -08001947{
1948 struct stripe_head *sh;
1949
1950 sh = raid5_get_active_stripe(conf, stripe_sect, 0, 1, 0);
1951 if (!sh)
1952 return NULL; /* no more stripe available */
1953
1954 r5l_recovery_reset_stripe(sh);
Song Liub4c625c2016-11-17 15:24:43 -08001955
1956 return sh;
1957}
1958
1959static struct stripe_head *
1960r5c_recovery_lookup_stripe(struct list_head *list, sector_t sect)
1961{
1962 struct stripe_head *sh;
1963
1964 list_for_each_entry(sh, list, lru)
1965 if (sh->sector == sect)
1966 return sh;
1967 return NULL;
1968}
1969
1970static void
1971r5c_recovery_drop_stripes(struct list_head *cached_stripe_list,
1972 struct r5l_recovery_ctx *ctx)
1973{
1974 struct stripe_head *sh, *next;
1975
1976 list_for_each_entry_safe(sh, next, cached_stripe_list, lru) {
1977 r5l_recovery_reset_stripe(sh);
1978 list_del_init(&sh->lru);
1979 raid5_release_stripe(sh);
1980 }
1981}
1982
1983static void
1984r5c_recovery_replay_stripes(struct list_head *cached_stripe_list,
1985 struct r5l_recovery_ctx *ctx)
1986{
1987 struct stripe_head *sh, *next;
1988
1989 list_for_each_entry_safe(sh, next, cached_stripe_list, lru)
1990 if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
1991 r5l_recovery_replay_one_stripe(sh->raid_conf, sh, ctx);
1992 list_del_init(&sh->lru);
1993 raid5_release_stripe(sh);
1994 }
1995}
1996
1997/* if matches return 0; otherwise return -EINVAL */
1998static int
Song Liueffe6ee2017-03-07 16:49:17 -08001999r5l_recovery_verify_data_checksum(struct r5l_log *log,
2000 struct r5l_recovery_ctx *ctx,
2001 struct page *page,
Song Liub4c625c2016-11-17 15:24:43 -08002002 sector_t log_offset, __le32 log_checksum)
2003{
2004 void *addr;
2005 u32 checksum;
2006
Song Liueffe6ee2017-03-07 16:49:17 -08002007 r5l_recovery_read_page(log, ctx, page, log_offset);
Song Liub4c625c2016-11-17 15:24:43 -08002008 addr = kmap_atomic(page);
2009 checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
2010 kunmap_atomic(addr);
2011 return (le32_to_cpu(log_checksum) == checksum) ? 0 : -EINVAL;
2012}
2013
2014/*
2015 * before loading data to stripe cache, we need verify checksum for all data,
2016 * if there is mismatch for any data page, we drop all data in the mata block
2017 */
2018static int
2019r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log,
2020 struct r5l_recovery_ctx *ctx)
2021{
2022 struct mddev *mddev = log->rdev->mddev;
2023 struct r5conf *conf = mddev->private;
2024 struct r5l_meta_block *mb = page_address(ctx->meta_page);
2025 sector_t mb_offset = sizeof(struct r5l_meta_block);
2026 sector_t log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2027 struct page *page;
2028 struct r5l_payload_data_parity *payload;
Song Liu2d4f4682017-03-07 17:44:21 -08002029 struct r5l_payload_flush *payload_flush;
Song Liub4c625c2016-11-17 15:24:43 -08002030
2031 page = alloc_page(GFP_KERNEL);
2032 if (!page)
2033 return -ENOMEM;
2034
2035 while (mb_offset < le32_to_cpu(mb->meta_size)) {
2036 payload = (void *)mb + mb_offset;
Song Liu2d4f4682017-03-07 17:44:21 -08002037 payload_flush = (void *)mb + mb_offset;
Song Liub4c625c2016-11-17 15:24:43 -08002038
Jason Yan1ad45a92017-03-25 09:44:39 +08002039 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
Song Liub4c625c2016-11-17 15:24:43 -08002040 if (r5l_recovery_verify_data_checksum(
Song Liueffe6ee2017-03-07 16:49:17 -08002041 log, ctx, page, log_offset,
Song Liub4c625c2016-11-17 15:24:43 -08002042 payload->checksum[0]) < 0)
2043 goto mismatch;
Jason Yan1ad45a92017-03-25 09:44:39 +08002044 } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY) {
Song Liub4c625c2016-11-17 15:24:43 -08002045 if (r5l_recovery_verify_data_checksum(
Song Liueffe6ee2017-03-07 16:49:17 -08002046 log, ctx, page, log_offset,
Song Liub4c625c2016-11-17 15:24:43 -08002047 payload->checksum[0]) < 0)
2048 goto mismatch;
2049 if (conf->max_degraded == 2 && /* q for RAID 6 */
2050 r5l_recovery_verify_data_checksum(
Song Liueffe6ee2017-03-07 16:49:17 -08002051 log, ctx, page,
Song Liub4c625c2016-11-17 15:24:43 -08002052 r5l_ring_add(log, log_offset,
2053 BLOCK_SECTORS),
2054 payload->checksum[1]) < 0)
2055 goto mismatch;
Jason Yan1ad45a92017-03-25 09:44:39 +08002056 } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
Song Liu2d4f4682017-03-07 17:44:21 -08002057 /* nothing to do for R5LOG_PAYLOAD_FLUSH here */
2058 } else /* not R5LOG_PAYLOAD_DATA/PARITY/FLUSH */
Song Liub4c625c2016-11-17 15:24:43 -08002059 goto mismatch;
2060
Jason Yan1ad45a92017-03-25 09:44:39 +08002061 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
Song Liu2d4f4682017-03-07 17:44:21 -08002062 mb_offset += sizeof(struct r5l_payload_flush) +
2063 le32_to_cpu(payload_flush->size);
2064 } else {
2065 /* DATA or PARITY payload */
2066 log_offset = r5l_ring_add(log, log_offset,
2067 le32_to_cpu(payload->size));
2068 mb_offset += sizeof(struct r5l_payload_data_parity) +
2069 sizeof(__le32) *
2070 (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
2071 }
Song Liub4c625c2016-11-17 15:24:43 -08002072
Song Liub4c625c2016-11-17 15:24:43 -08002073 }
2074
2075 put_page(page);
2076 return 0;
2077
2078mismatch:
2079 put_page(page);
2080 return -EINVAL;
2081}
2082
2083/*
2084 * Analyze all data/parity pages in one meta block
2085 * Returns:
2086 * 0 for success
2087 * -EINVAL for unknown playload type
2088 * -EAGAIN for checksum mismatch of data page
2089 * -ENOMEM for run out of memory (alloc_page failed or run out of stripes)
2090 */
2091static int
2092r5c_recovery_analyze_meta_block(struct r5l_log *log,
2093 struct r5l_recovery_ctx *ctx,
2094 struct list_head *cached_stripe_list)
2095{
2096 struct mddev *mddev = log->rdev->mddev;
2097 struct r5conf *conf = mddev->private;
2098 struct r5l_meta_block *mb;
2099 struct r5l_payload_data_parity *payload;
Song Liu2d4f4682017-03-07 17:44:21 -08002100 struct r5l_payload_flush *payload_flush;
Song Liub4c625c2016-11-17 15:24:43 -08002101 int mb_offset;
2102 sector_t log_offset;
2103 sector_t stripe_sect;
2104 struct stripe_head *sh;
2105 int ret;
2106
2107 /*
2108 * for mismatch in data blocks, we will drop all data in this mb, but
2109 * we will still read next mb for other data with FLUSH flag, as
2110 * io_unit could finish out of order.
2111 */
2112 ret = r5l_recovery_verify_data_checksum_for_mb(log, ctx);
2113 if (ret == -EINVAL)
2114 return -EAGAIN;
2115 else if (ret)
2116 return ret; /* -ENOMEM duo to alloc_page() failed */
2117
2118 mb = page_address(ctx->meta_page);
2119 mb_offset = sizeof(struct r5l_meta_block);
2120 log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2121
2122 while (mb_offset < le32_to_cpu(mb->meta_size)) {
2123 int dd;
2124
2125 payload = (void *)mb + mb_offset;
Song Liu2d4f4682017-03-07 17:44:21 -08002126 payload_flush = (void *)mb + mb_offset;
2127
Jason Yan1ad45a92017-03-25 09:44:39 +08002128 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
Song Liu2d4f4682017-03-07 17:44:21 -08002129 int i, count;
2130
2131 count = le32_to_cpu(payload_flush->size) / sizeof(__le64);
2132 for (i = 0; i < count; ++i) {
2133 stripe_sect = le64_to_cpu(payload_flush->flush_stripes[i]);
2134 sh = r5c_recovery_lookup_stripe(cached_stripe_list,
2135 stripe_sect);
2136 if (sh) {
2137 WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
2138 r5l_recovery_reset_stripe(sh);
2139 list_del_init(&sh->lru);
2140 raid5_release_stripe(sh);
2141 }
2142 }
2143
2144 mb_offset += sizeof(struct r5l_payload_flush) +
2145 le32_to_cpu(payload_flush->size);
2146 continue;
2147 }
2148
2149 /* DATA or PARITY payload */
Jason Yan1ad45a92017-03-25 09:44:39 +08002150 stripe_sect = (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) ?
Song Liub4c625c2016-11-17 15:24:43 -08002151 raid5_compute_sector(
2152 conf, le64_to_cpu(payload->location), 0, &dd,
2153 NULL)
2154 : le64_to_cpu(payload->location);
2155
2156 sh = r5c_recovery_lookup_stripe(cached_stripe_list,
2157 stripe_sect);
2158
2159 if (!sh) {
Song Liu3c66abb2016-12-14 15:38:01 -08002160 sh = r5c_recovery_alloc_stripe(conf, stripe_sect);
Song Liub4c625c2016-11-17 15:24:43 -08002161 /*
2162 * cannot get stripe from raid5_get_active_stripe
2163 * try replay some stripes
2164 */
2165 if (!sh) {
2166 r5c_recovery_replay_stripes(
2167 cached_stripe_list, ctx);
2168 sh = r5c_recovery_alloc_stripe(
Song Liu3c66abb2016-12-14 15:38:01 -08002169 conf, stripe_sect);
Song Liub4c625c2016-11-17 15:24:43 -08002170 }
2171 if (!sh) {
2172 pr_debug("md/raid:%s: Increasing stripe cache size to %d to recovery data on journal.\n",
2173 mdname(mddev),
2174 conf->min_nr_stripes * 2);
2175 raid5_set_cache_size(mddev,
2176 conf->min_nr_stripes * 2);
Song Liu3c66abb2016-12-14 15:38:01 -08002177 sh = r5c_recovery_alloc_stripe(conf,
2178 stripe_sect);
Song Liub4c625c2016-11-17 15:24:43 -08002179 }
2180 if (!sh) {
2181 pr_err("md/raid:%s: Cannot get enough stripes due to memory pressure. Recovery failed.\n",
2182 mdname(mddev));
2183 return -ENOMEM;
2184 }
2185 list_add_tail(&sh->lru, cached_stripe_list);
2186 }
2187
Jason Yan1ad45a92017-03-25 09:44:39 +08002188 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
Zhengyuan Liuf7b7bee2016-11-26 10:57:13 +08002189 if (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
2190 test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags)) {
Song Liub4c625c2016-11-17 15:24:43 -08002191 r5l_recovery_replay_one_stripe(conf, sh, ctx);
Song Liub4c625c2016-11-17 15:24:43 -08002192 list_move_tail(&sh->lru, cached_stripe_list);
2193 }
2194 r5l_recovery_load_data(log, sh, ctx, payload,
2195 log_offset);
Jason Yan1ad45a92017-03-25 09:44:39 +08002196 } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY)
Song Liub4c625c2016-11-17 15:24:43 -08002197 r5l_recovery_load_parity(log, sh, ctx, payload,
2198 log_offset);
2199 else
2200 return -EINVAL;
2201
2202 log_offset = r5l_ring_add(log, log_offset,
2203 le32_to_cpu(payload->size));
2204
2205 mb_offset += sizeof(struct r5l_payload_data_parity) +
2206 sizeof(__le32) *
2207 (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
2208 }
2209
2210 return 0;
2211}
2212
2213/*
2214 * Load the stripe into cache. The stripe will be written out later by
2215 * the stripe cache state machine.
2216 */
2217static void r5c_recovery_load_one_stripe(struct r5l_log *log,
2218 struct stripe_head *sh)
2219{
Song Liub4c625c2016-11-17 15:24:43 -08002220 struct r5dev *dev;
2221 int i;
2222
2223 for (i = sh->disks; i--; ) {
2224 dev = sh->dev + i;
2225 if (test_and_clear_bit(R5_Wantwrite, &dev->flags)) {
2226 set_bit(R5_InJournal, &dev->flags);
2227 set_bit(R5_UPTODATE, &dev->flags);
2228 }
2229 }
Song Liub4c625c2016-11-17 15:24:43 -08002230}
2231
2232/*
2233 * Scan through the log for all to-be-flushed data
2234 *
2235 * For stripes with data and parity, namely Data-Parity stripe
2236 * (STRIPE_R5C_CACHING == 0), we simply replay all the writes.
2237 *
2238 * For stripes with only data, namely Data-Only stripe
2239 * (STRIPE_R5C_CACHING == 1), we load them to stripe cache state machine.
2240 *
2241 * For a stripe, if we see data after parity, we should discard all previous
2242 * data and parity for this stripe, as these data are already flushed to
2243 * the array.
2244 *
2245 * At the end of the scan, we return the new journal_tail, which points to
2246 * first data-only stripe on the journal device, or next invalid meta block.
2247 */
2248static int r5c_recovery_flush_log(struct r5l_log *log,
2249 struct r5l_recovery_ctx *ctx)
2250{
JackieLiubc8f1672016-11-28 16:19:20 +08002251 struct stripe_head *sh;
Song Liub4c625c2016-11-17 15:24:43 -08002252 int ret = 0;
2253
2254 /* scan through the log */
2255 while (1) {
2256 if (r5l_recovery_read_meta_block(log, ctx))
2257 break;
2258
2259 ret = r5c_recovery_analyze_meta_block(log, ctx,
2260 &ctx->cached_list);
2261 /*
2262 * -EAGAIN means mismatch in data block, in this case, we still
2263 * try scan the next metablock
2264 */
2265 if (ret && ret != -EAGAIN)
2266 break; /* ret == -EINVAL or -ENOMEM */
2267 ctx->seq++;
2268 ctx->pos = r5l_ring_add(log, ctx->pos, ctx->meta_total_blocks);
2269 }
2270
2271 if (ret == -ENOMEM) {
2272 r5c_recovery_drop_stripes(&ctx->cached_list, ctx);
2273 return ret;
2274 }
2275
2276 /* replay data-parity stripes */
2277 r5c_recovery_replay_stripes(&ctx->cached_list, ctx);
2278
2279 /* load data-only stripes to stripe cache */
JackieLiubc8f1672016-11-28 16:19:20 +08002280 list_for_each_entry(sh, &ctx->cached_list, lru) {
Song Liub4c625c2016-11-17 15:24:43 -08002281 WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
2282 r5c_recovery_load_one_stripe(log, sh);
Song Liub4c625c2016-11-17 15:24:43 -08002283 ctx->data_only_stripes++;
2284 }
2285
2286 return 0;
2287}
2288
2289/*
2290 * we did a recovery. Now ctx.pos points to an invalid meta block. New
2291 * log will start here. but we can't let superblock point to last valid
2292 * meta block. The log might looks like:
2293 * | meta 1| meta 2| meta 3|
2294 * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
2295 * superblock points to meta 1, we write a new valid meta 2n. if crash
2296 * happens again, new recovery will start from meta 1. Since meta 2n is
2297 * valid now, recovery will think meta 3 is valid, which is wrong.
2298 * The solution is we create a new meta in meta2 with its seq == meta
Song Liu3c6edc62016-12-07 09:42:06 -08002299 * 1's seq + 10000 and let superblock points to meta2. The same recovery
2300 * will not think meta 3 is a valid meta, because its seq doesn't match
Song Liub4c625c2016-11-17 15:24:43 -08002301 */
2302
2303/*
2304 * Before recovery, the log looks like the following
2305 *
2306 * ---------------------------------------------
2307 * | valid log | invalid log |
2308 * ---------------------------------------------
2309 * ^
2310 * |- log->last_checkpoint
2311 * |- log->last_cp_seq
2312 *
2313 * Now we scan through the log until we see invalid entry
2314 *
2315 * ---------------------------------------------
2316 * | valid log | invalid log |
2317 * ---------------------------------------------
2318 * ^ ^
2319 * |- log->last_checkpoint |- ctx->pos
2320 * |- log->last_cp_seq |- ctx->seq
2321 *
2322 * From this point, we need to increase seq number by 10 to avoid
2323 * confusing next recovery.
2324 *
2325 * ---------------------------------------------
2326 * | valid log | invalid log |
2327 * ---------------------------------------------
2328 * ^ ^
2329 * |- log->last_checkpoint |- ctx->pos+1
Song Liu3c6edc62016-12-07 09:42:06 -08002330 * |- log->last_cp_seq |- ctx->seq+10001
Song Liub4c625c2016-11-17 15:24:43 -08002331 *
2332 * However, it is not safe to start the state machine yet, because data only
2333 * parities are not yet secured in RAID. To save these data only parities, we
2334 * rewrite them from seq+11.
2335 *
2336 * -----------------------------------------------------------------
2337 * | valid log | data only stripes | invalid log |
2338 * -----------------------------------------------------------------
2339 * ^ ^
2340 * |- log->last_checkpoint |- ctx->pos+n
Song Liu3c6edc62016-12-07 09:42:06 -08002341 * |- log->last_cp_seq |- ctx->seq+10000+n
Song Liub4c625c2016-11-17 15:24:43 -08002342 *
2343 * If failure happens again during this process, the recovery can safe start
2344 * again from log->last_checkpoint.
2345 *
2346 * Once data only stripes are rewritten to journal, we move log_tail
2347 *
2348 * -----------------------------------------------------------------
2349 * | old log | data only stripes | invalid log |
2350 * -----------------------------------------------------------------
2351 * ^ ^
2352 * |- log->last_checkpoint |- ctx->pos+n
Song Liu3c6edc62016-12-07 09:42:06 -08002353 * |- log->last_cp_seq |- ctx->seq+10000+n
Song Liub4c625c2016-11-17 15:24:43 -08002354 *
2355 * Then we can safely start the state machine. If failure happens from this
2356 * point on, the recovery will start from new log->last_checkpoint.
2357 */
2358static int
2359r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log,
2360 struct r5l_recovery_ctx *ctx)
2361{
Song Liua85dd7b2017-01-23 17:12:57 -08002362 struct stripe_head *sh;
Song Liub4c625c2016-11-17 15:24:43 -08002363 struct mddev *mddev = log->rdev->mddev;
2364 struct page *page;
Song Liu3c66abb2016-12-14 15:38:01 -08002365 sector_t next_checkpoint = MaxSector;
Song Liub4c625c2016-11-17 15:24:43 -08002366
2367 page = alloc_page(GFP_KERNEL);
2368 if (!page) {
2369 pr_err("md/raid:%s: cannot allocate memory to rewrite data only stripes\n",
2370 mdname(mddev));
2371 return -ENOMEM;
2372 }
2373
Song Liu3c66abb2016-12-14 15:38:01 -08002374 WARN_ON(list_empty(&ctx->cached_list));
2375
Song Liua85dd7b2017-01-23 17:12:57 -08002376 list_for_each_entry(sh, &ctx->cached_list, lru) {
Song Liub4c625c2016-11-17 15:24:43 -08002377 struct r5l_meta_block *mb;
2378 int i;
2379 int offset;
2380 sector_t write_pos;
2381
2382 WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
2383 r5l_recovery_create_empty_meta_block(log, page,
2384 ctx->pos, ctx->seq);
2385 mb = page_address(page);
2386 offset = le32_to_cpu(mb->meta_size);
JackieLiufc833c22016-11-28 16:19:19 +08002387 write_pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
Song Liub4c625c2016-11-17 15:24:43 -08002388
2389 for (i = sh->disks; i--; ) {
2390 struct r5dev *dev = &sh->dev[i];
2391 struct r5l_payload_data_parity *payload;
2392 void *addr;
2393
2394 if (test_bit(R5_InJournal, &dev->flags)) {
2395 payload = (void *)mb + offset;
2396 payload->header.type = cpu_to_le16(
2397 R5LOG_PAYLOAD_DATA);
Jason Yan1ad45a92017-03-25 09:44:39 +08002398 payload->size = cpu_to_le32(BLOCK_SECTORS);
Song Liub4c625c2016-11-17 15:24:43 -08002399 payload->location = cpu_to_le64(
2400 raid5_compute_blocknr(sh, i, 0));
2401 addr = kmap_atomic(dev->page);
2402 payload->checksum[0] = cpu_to_le32(
2403 crc32c_le(log->uuid_checksum, addr,
2404 PAGE_SIZE));
2405 kunmap_atomic(addr);
2406 sync_page_io(log->rdev, write_pos, PAGE_SIZE,
2407 dev->page, REQ_OP_WRITE, 0, false);
2408 write_pos = r5l_ring_add(log, write_pos,
2409 BLOCK_SECTORS);
2410 offset += sizeof(__le32) +
2411 sizeof(struct r5l_payload_data_parity);
2412
2413 }
2414 }
2415 mb->meta_size = cpu_to_le32(offset);
Song Liu5c88f402016-12-07 09:42:05 -08002416 mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
2417 mb, PAGE_SIZE));
Song Liub4c625c2016-11-17 15:24:43 -08002418 sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page,
Jan Kara5a8948f2017-05-31 09:44:33 +02002419 REQ_OP_WRITE, REQ_SYNC | REQ_FUA, false);
Song Liub4c625c2016-11-17 15:24:43 -08002420 sh->log_start = ctx->pos;
Song Liu3c66abb2016-12-14 15:38:01 -08002421 list_add_tail(&sh->r5c, &log->stripe_in_journal_list);
2422 atomic_inc(&log->stripe_in_journal_count);
Song Liub4c625c2016-11-17 15:24:43 -08002423 ctx->pos = write_pos;
2424 ctx->seq += 1;
Song Liu3c66abb2016-12-14 15:38:01 -08002425 next_checkpoint = sh->log_start;
Song Liub4c625c2016-11-17 15:24:43 -08002426 }
Song Liu3c66abb2016-12-14 15:38:01 -08002427 log->next_checkpoint = next_checkpoint;
Song Liub4c625c2016-11-17 15:24:43 -08002428 __free_page(page);
2429 return 0;
2430}
2431
Song Liua85dd7b2017-01-23 17:12:57 -08002432static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log,
2433 struct r5l_recovery_ctx *ctx)
2434{
2435 struct mddev *mddev = log->rdev->mddev;
2436 struct r5conf *conf = mddev->private;
2437 struct stripe_head *sh, *next;
2438
2439 if (ctx->data_only_stripes == 0)
2440 return;
2441
2442 log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_BACK;
2443
2444 list_for_each_entry_safe(sh, next, &ctx->cached_list, lru) {
2445 r5c_make_stripe_write_out(sh);
2446 set_bit(STRIPE_HANDLE, &sh->state);
2447 list_del_init(&sh->lru);
2448 raid5_release_stripe(sh);
2449 }
2450
2451 md_wakeup_thread(conf->mddev->thread);
2452 /* reuse conf->wait_for_quiescent in recovery */
2453 wait_event(conf->wait_for_quiescent,
2454 atomic_read(&conf->active_stripes) == 0);
2455
2456 log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
2457}
2458
Shaohua Lif6bed0e2015-08-13 14:31:59 -07002459static int r5l_recovery_log(struct r5l_log *log)
2460{
Song Liu5aabf7c2016-11-17 15:24:44 -08002461 struct mddev *mddev = log->rdev->mddev;
Song Liueffe6ee2017-03-07 16:49:17 -08002462 struct r5l_recovery_ctx *ctx;
Song Liu5aabf7c2016-11-17 15:24:44 -08002463 int ret;
JackieLiu43b96742016-12-05 11:58:53 +08002464 sector_t pos;
Shaohua Li355810d2015-08-13 14:32:01 -07002465
Song Liueffe6ee2017-03-07 16:49:17 -08002466 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2467 if (!ctx)
Shaohua Li355810d2015-08-13 14:32:01 -07002468 return -ENOMEM;
2469
Song Liueffe6ee2017-03-07 16:49:17 -08002470 ctx->pos = log->last_checkpoint;
2471 ctx->seq = log->last_cp_seq;
2472 INIT_LIST_HEAD(&ctx->cached_list);
2473 ctx->meta_page = alloc_page(GFP_KERNEL);
2474
2475 if (!ctx->meta_page) {
2476 ret = -ENOMEM;
2477 goto meta_page;
2478 }
2479
2480 if (r5l_recovery_allocate_ra_pool(log, ctx) != 0) {
2481 ret = -ENOMEM;
2482 goto ra_pool;
2483 }
2484
2485 ret = r5c_recovery_flush_log(log, ctx);
Shaohua Li355810d2015-08-13 14:32:01 -07002486
Song Liu5aabf7c2016-11-17 15:24:44 -08002487 if (ret)
Song Liueffe6ee2017-03-07 16:49:17 -08002488 goto error;
Shaohua Li355810d2015-08-13 14:32:01 -07002489
Song Liueffe6ee2017-03-07 16:49:17 -08002490 pos = ctx->pos;
2491 ctx->seq += 10000;
JackieLiu43b96742016-12-05 11:58:53 +08002492
Song Liueffe6ee2017-03-07 16:49:17 -08002493 if ((ctx->data_only_stripes == 0) && (ctx->data_parity_stripes == 0))
Song Liu5aabf7c2016-11-17 15:24:44 -08002494 pr_debug("md/raid:%s: starting from clean shutdown\n",
2495 mdname(mddev));
Song Liua85dd7b2017-01-23 17:12:57 -08002496 else
Colin Ian King99f17892016-12-23 00:52:30 +00002497 pr_debug("md/raid:%s: recovering %d data-only stripes and %d data-parity stripes\n",
Song Liueffe6ee2017-03-07 16:49:17 -08002498 mdname(mddev), ctx->data_only_stripes,
2499 ctx->data_parity_stripes);
Song Liu5aabf7c2016-11-17 15:24:44 -08002500
Song Liueffe6ee2017-03-07 16:49:17 -08002501 if (ctx->data_only_stripes == 0) {
2502 log->next_checkpoint = ctx->pos;
2503 r5l_log_write_empty_meta_block(log, ctx->pos, ctx->seq++);
2504 ctx->pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2505 } else if (r5c_recovery_rewrite_data_only_stripes(log, ctx)) {
Song Liua85dd7b2017-01-23 17:12:57 -08002506 pr_err("md/raid:%s: failed to rewrite stripes to journal\n",
2507 mdname(mddev));
Song Liueffe6ee2017-03-07 16:49:17 -08002508 ret = -EIO;
2509 goto error;
Shaohua Li355810d2015-08-13 14:32:01 -07002510 }
Song Liub4c625c2016-11-17 15:24:43 -08002511
Song Liueffe6ee2017-03-07 16:49:17 -08002512 log->log_start = ctx->pos;
2513 log->seq = ctx->seq;
JackieLiu43b96742016-12-05 11:58:53 +08002514 log->last_checkpoint = pos;
2515 r5l_write_super(log, pos);
Song Liua85dd7b2017-01-23 17:12:57 -08002516
Song Liueffe6ee2017-03-07 16:49:17 -08002517 r5c_recovery_flush_data_only_stripes(log, ctx);
2518 ret = 0;
2519error:
2520 r5l_recovery_free_ra_pool(log, ctx);
2521ra_pool:
2522 __free_page(ctx->meta_page);
2523meta_page:
2524 kfree(ctx);
2525 return ret;
Shaohua Lif6bed0e2015-08-13 14:31:59 -07002526}
2527
2528static void r5l_write_super(struct r5l_log *log, sector_t cp)
2529{
2530 struct mddev *mddev = log->rdev->mddev;
2531
2532 log->rdev->journal_tail = cp;
Shaohua Li29530792016-12-08 15:48:19 -08002533 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07002534}
2535
Song Liu2c7da142016-11-17 15:24:41 -08002536static ssize_t r5c_journal_mode_show(struct mddev *mddev, char *page)
2537{
Song Liua72cbf82017-08-08 22:56:52 -07002538 struct r5conf *conf;
Song Liu2c7da142016-11-17 15:24:41 -08002539 int ret;
2540
Song Liua72cbf82017-08-08 22:56:52 -07002541 ret = mddev_lock(mddev);
2542 if (ret)
2543 return ret;
2544
2545 conf = mddev->private;
2546 if (!conf || !conf->log) {
2547 mddev_unlock(mddev);
Song Liu2c7da142016-11-17 15:24:41 -08002548 return 0;
Song Liua72cbf82017-08-08 22:56:52 -07002549 }
Song Liu2c7da142016-11-17 15:24:41 -08002550
2551 switch (conf->log->r5c_journal_mode) {
2552 case R5C_JOURNAL_MODE_WRITE_THROUGH:
2553 ret = snprintf(
2554 page, PAGE_SIZE, "[%s] %s\n",
2555 r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
2556 r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
2557 break;
2558 case R5C_JOURNAL_MODE_WRITE_BACK:
2559 ret = snprintf(
2560 page, PAGE_SIZE, "%s [%s]\n",
2561 r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
2562 r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
2563 break;
2564 default:
2565 ret = 0;
2566 }
Song Liua72cbf82017-08-08 22:56:52 -07002567 mddev_unlock(mddev);
Song Liu2c7da142016-11-17 15:24:41 -08002568 return ret;
2569}
2570
Heinz Mauelshagen78e470c2017-03-22 17:44:37 +01002571/*
2572 * Set journal cache mode on @mddev (external API initially needed by dm-raid).
2573 *
2574 * @mode as defined in 'enum r5c_journal_mode'.
2575 *
2576 */
2577int r5c_journal_mode_set(struct mddev *mddev, int mode)
Song Liu2c7da142016-11-17 15:24:41 -08002578{
Song Liub44886c2017-07-31 14:52:26 -07002579 struct r5conf *conf;
2580 int err;
Song Liu2c7da142016-11-17 15:24:41 -08002581
Heinz Mauelshagen78e470c2017-03-22 17:44:37 +01002582 if (mode < R5C_JOURNAL_MODE_WRITE_THROUGH ||
2583 mode > R5C_JOURNAL_MODE_WRITE_BACK)
Song Liu2c7da142016-11-17 15:24:41 -08002584 return -EINVAL;
2585
Song Liub44886c2017-07-31 14:52:26 -07002586 err = mddev_lock(mddev);
2587 if (err)
2588 return err;
2589 conf = mddev->private;
2590 if (!conf || !conf->log) {
2591 mddev_unlock(mddev);
2592 return -ENODEV;
2593 }
2594
Song Liu2e38a372017-01-24 10:45:30 -08002595 if (raid5_calc_degraded(conf) > 0 &&
Song Liub44886c2017-07-31 14:52:26 -07002596 mode == R5C_JOURNAL_MODE_WRITE_BACK) {
2597 mddev_unlock(mddev);
Song Liu2e38a372017-01-24 10:45:30 -08002598 return -EINVAL;
Song Liub44886c2017-07-31 14:52:26 -07002599 }
Song Liu2e38a372017-01-24 10:45:30 -08002600
Song Liu2c7da142016-11-17 15:24:41 -08002601 mddev_suspend(mddev);
Heinz Mauelshagen78e470c2017-03-22 17:44:37 +01002602 conf->log->r5c_journal_mode = mode;
Song Liu2c7da142016-11-17 15:24:41 -08002603 mddev_resume(mddev);
Song Liub44886c2017-07-31 14:52:26 -07002604 mddev_unlock(mddev);
Song Liu2c7da142016-11-17 15:24:41 -08002605
2606 pr_debug("md/raid:%s: setting r5c cache mode to %d: %s\n",
Heinz Mauelshagen78e470c2017-03-22 17:44:37 +01002607 mdname(mddev), mode, r5c_journal_mode_str[mode]);
2608 return 0;
2609}
2610EXPORT_SYMBOL(r5c_journal_mode_set);
2611
2612static ssize_t r5c_journal_mode_store(struct mddev *mddev,
2613 const char *page, size_t length)
2614{
2615 int mode = ARRAY_SIZE(r5c_journal_mode_str);
2616 size_t len = length;
2617
2618 if (len < 2)
2619 return -EINVAL;
2620
2621 if (page[len - 1] == '\n')
2622 len--;
2623
2624 while (mode--)
2625 if (strlen(r5c_journal_mode_str[mode]) == len &&
2626 !strncmp(page, r5c_journal_mode_str[mode], len))
2627 break;
2628
2629 return r5c_journal_mode_set(mddev, mode) ?: length;
Song Liu2c7da142016-11-17 15:24:41 -08002630}
2631
2632struct md_sysfs_entry
2633r5c_journal_mode = __ATTR(journal_mode, 0644,
2634 r5c_journal_mode_show, r5c_journal_mode_store);
2635
Song Liu2ded3702016-11-17 15:24:38 -08002636/*
2637 * Try handle write operation in caching phase. This function should only
2638 * be called in write-back mode.
2639 *
2640 * If all outstanding writes can be handled in caching phase, returns 0
2641 * If writes requires write-out phase, call r5c_make_stripe_write_out()
2642 * and returns -EAGAIN
2643 */
2644int r5c_try_caching_write(struct r5conf *conf,
2645 struct stripe_head *sh,
2646 struct stripe_head_state *s,
2647 int disks)
2648{
2649 struct r5l_log *log = conf->log;
Song Liu1e6d6902016-11-17 15:24:39 -08002650 int i;
2651 struct r5dev *dev;
2652 int to_cache = 0;
Song Liu03b047f2017-01-11 13:39:14 -08002653 void **pslot;
2654 sector_t tree_index;
2655 int ret;
2656 uintptr_t refcount;
Song Liu2ded3702016-11-17 15:24:38 -08002657
2658 BUG_ON(!r5c_is_writeback(log));
2659
Song Liu1e6d6902016-11-17 15:24:39 -08002660 if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
2661 /*
2662 * There are two different scenarios here:
2663 * 1. The stripe has some data cached, and it is sent to
2664 * write-out phase for reclaim
2665 * 2. The stripe is clean, and this is the first write
2666 *
2667 * For 1, return -EAGAIN, so we continue with
2668 * handle_stripe_dirtying().
2669 *
2670 * For 2, set STRIPE_R5C_CACHING and continue with caching
2671 * write.
2672 */
2673
2674 /* case 1: anything injournal or anything in written */
2675 if (s->injournal > 0 || s->written > 0)
2676 return -EAGAIN;
2677 /* case 2 */
2678 set_bit(STRIPE_R5C_CACHING, &sh->state);
2679 }
2680
Song Liu2e38a372017-01-24 10:45:30 -08002681 /*
2682 * When run in degraded mode, array is set to write-through mode.
2683 * This check helps drain pending write safely in the transition to
2684 * write-through mode.
Song Liu5ddf0442017-05-11 17:03:44 -07002685 *
2686 * When a stripe is syncing, the write is also handled in write
2687 * through mode.
Song Liu2e38a372017-01-24 10:45:30 -08002688 */
Song Liu5ddf0442017-05-11 17:03:44 -07002689 if (s->failed || test_bit(STRIPE_SYNCING, &sh->state)) {
Song Liu2e38a372017-01-24 10:45:30 -08002690 r5c_make_stripe_write_out(sh);
2691 return -EAGAIN;
2692 }
2693
Song Liu1e6d6902016-11-17 15:24:39 -08002694 for (i = disks; i--; ) {
2695 dev = &sh->dev[i];
2696 /* if non-overwrite, use writing-out phase */
2697 if (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags) &&
2698 !test_bit(R5_InJournal, &dev->flags)) {
2699 r5c_make_stripe_write_out(sh);
2700 return -EAGAIN;
2701 }
2702 }
2703
Song Liu03b047f2017-01-11 13:39:14 -08002704 /* if the stripe is not counted in big_stripe_tree, add it now */
2705 if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
2706 !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2707 tree_index = r5c_tree_index(conf, sh->sector);
2708 spin_lock(&log->tree_lock);
2709 pslot = radix_tree_lookup_slot(&log->big_stripe_tree,
2710 tree_index);
2711 if (pslot) {
2712 refcount = (uintptr_t)radix_tree_deref_slot_protected(
2713 pslot, &log->tree_lock) >>
2714 R5C_RADIX_COUNT_SHIFT;
2715 radix_tree_replace_slot(
2716 &log->big_stripe_tree, pslot,
2717 (void *)((refcount + 1) << R5C_RADIX_COUNT_SHIFT));
2718 } else {
2719 /*
2720 * this radix_tree_insert can fail safely, so no
2721 * need to call radix_tree_preload()
2722 */
2723 ret = radix_tree_insert(
2724 &log->big_stripe_tree, tree_index,
2725 (void *)(1 << R5C_RADIX_COUNT_SHIFT));
2726 if (ret) {
2727 spin_unlock(&log->tree_lock);
2728 r5c_make_stripe_write_out(sh);
2729 return -EAGAIN;
2730 }
2731 }
2732 spin_unlock(&log->tree_lock);
2733
2734 /*
2735 * set STRIPE_R5C_PARTIAL_STRIPE, this shows the stripe is
2736 * counted in the radix tree
2737 */
2738 set_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state);
2739 atomic_inc(&conf->r5c_cached_partial_stripes);
2740 }
2741
Song Liu1e6d6902016-11-17 15:24:39 -08002742 for (i = disks; i--; ) {
2743 dev = &sh->dev[i];
2744 if (dev->towrite) {
2745 set_bit(R5_Wantwrite, &dev->flags);
2746 set_bit(R5_Wantdrain, &dev->flags);
2747 set_bit(R5_LOCKED, &dev->flags);
2748 to_cache++;
2749 }
2750 }
2751
2752 if (to_cache) {
2753 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2754 /*
2755 * set STRIPE_LOG_TRAPPED, which triggers r5c_cache_data()
2756 * in ops_run_io(). STRIPE_LOG_TRAPPED will be cleared in
2757 * r5c_handle_data_cached()
2758 */
2759 set_bit(STRIPE_LOG_TRAPPED, &sh->state);
2760 }
2761
2762 return 0;
2763}
2764
2765/*
2766 * free extra pages (orig_page) we allocated for prexor
2767 */
2768void r5c_release_extra_page(struct stripe_head *sh)
2769{
Song Liud7bd3982016-11-23 22:50:39 -08002770 struct r5conf *conf = sh->raid_conf;
Song Liu1e6d6902016-11-17 15:24:39 -08002771 int i;
Song Liud7bd3982016-11-23 22:50:39 -08002772 bool using_disk_info_extra_page;
2773
2774 using_disk_info_extra_page =
2775 sh->dev[0].orig_page == conf->disks[0].extra_page;
Song Liu1e6d6902016-11-17 15:24:39 -08002776
2777 for (i = sh->disks; i--; )
2778 if (sh->dev[i].page != sh->dev[i].orig_page) {
2779 struct page *p = sh->dev[i].orig_page;
2780
2781 sh->dev[i].orig_page = sh->dev[i].page;
Song Liu86aa1392017-01-12 17:22:41 -08002782 clear_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
2783
Song Liud7bd3982016-11-23 22:50:39 -08002784 if (!using_disk_info_extra_page)
2785 put_page(p);
Song Liu1e6d6902016-11-17 15:24:39 -08002786 }
Song Liud7bd3982016-11-23 22:50:39 -08002787
2788 if (using_disk_info_extra_page) {
2789 clear_bit(R5C_EXTRA_PAGE_IN_USE, &conf->cache_state);
2790 md_wakeup_thread(conf->mddev->thread);
2791 }
2792}
2793
2794void r5c_use_extra_page(struct stripe_head *sh)
2795{
2796 struct r5conf *conf = sh->raid_conf;
2797 int i;
2798 struct r5dev *dev;
2799
2800 for (i = sh->disks; i--; ) {
2801 dev = &sh->dev[i];
2802 if (dev->orig_page != dev->page)
2803 put_page(dev->orig_page);
2804 dev->orig_page = conf->disks[i].extra_page;
2805 }
Song Liu2ded3702016-11-17 15:24:38 -08002806}
2807
2808/*
2809 * clean up the stripe (clear R5_InJournal for dev[pd_idx] etc.) after the
2810 * stripe is committed to RAID disks.
2811 */
2812void r5c_finish_stripe_write_out(struct r5conf *conf,
2813 struct stripe_head *sh,
2814 struct stripe_head_state *s)
2815{
Song Liu03b047f2017-01-11 13:39:14 -08002816 struct r5l_log *log = conf->log;
Song Liu1e6d6902016-11-17 15:24:39 -08002817 int i;
2818 int do_wakeup = 0;
Song Liu03b047f2017-01-11 13:39:14 -08002819 sector_t tree_index;
2820 void **pslot;
2821 uintptr_t refcount;
Song Liu1e6d6902016-11-17 15:24:39 -08002822
Song Liu03b047f2017-01-11 13:39:14 -08002823 if (!log || !test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags))
Song Liu2ded3702016-11-17 15:24:38 -08002824 return;
2825
2826 WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
2827 clear_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
2828
Song Liu03b047f2017-01-11 13:39:14 -08002829 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
Song Liu2ded3702016-11-17 15:24:38 -08002830 return;
Song Liu1e6d6902016-11-17 15:24:39 -08002831
2832 for (i = sh->disks; i--; ) {
2833 clear_bit(R5_InJournal, &sh->dev[i].flags);
2834 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2835 do_wakeup = 1;
2836 }
2837
2838 /*
2839 * analyse_stripe() runs before r5c_finish_stripe_write_out(),
2840 * We updated R5_InJournal, so we also update s->injournal.
2841 */
2842 s->injournal = 0;
2843
2844 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2845 if (atomic_dec_and_test(&conf->pending_full_writes))
2846 md_wakeup_thread(conf->mddev->thread);
2847
2848 if (do_wakeup)
2849 wake_up(&conf->wait_for_overlap);
Song Liua39f7af2016-11-17 15:24:40 -08002850
Song Liu03b047f2017-01-11 13:39:14 -08002851 spin_lock_irq(&log->stripe_in_journal_lock);
Song Liua39f7af2016-11-17 15:24:40 -08002852 list_del_init(&sh->r5c);
Song Liu03b047f2017-01-11 13:39:14 -08002853 spin_unlock_irq(&log->stripe_in_journal_lock);
Song Liua39f7af2016-11-17 15:24:40 -08002854 sh->log_start = MaxSector;
Song Liu03b047f2017-01-11 13:39:14 -08002855
2856 atomic_dec(&log->stripe_in_journal_count);
2857 r5c_update_log_state(log);
2858
2859 /* stop counting this stripe in big_stripe_tree */
2860 if (test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) ||
2861 test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2862 tree_index = r5c_tree_index(conf, sh->sector);
2863 spin_lock(&log->tree_lock);
2864 pslot = radix_tree_lookup_slot(&log->big_stripe_tree,
2865 tree_index);
2866 BUG_ON(pslot == NULL);
2867 refcount = (uintptr_t)radix_tree_deref_slot_protected(
2868 pslot, &log->tree_lock) >>
2869 R5C_RADIX_COUNT_SHIFT;
2870 if (refcount == 1)
2871 radix_tree_delete(&log->big_stripe_tree, tree_index);
2872 else
2873 radix_tree_replace_slot(
2874 &log->big_stripe_tree, pslot,
2875 (void *)((refcount - 1) << R5C_RADIX_COUNT_SHIFT));
2876 spin_unlock(&log->tree_lock);
2877 }
2878
2879 if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) {
2880 BUG_ON(atomic_read(&conf->r5c_cached_partial_stripes) == 0);
Shaohua Lie33fbb92017-02-10 16:18:09 -08002881 atomic_dec(&conf->r5c_flushing_partial_stripes);
Song Liu03b047f2017-01-11 13:39:14 -08002882 atomic_dec(&conf->r5c_cached_partial_stripes);
2883 }
2884
2885 if (test_and_clear_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2886 BUG_ON(atomic_read(&conf->r5c_cached_full_stripes) == 0);
Shaohua Lie33fbb92017-02-10 16:18:09 -08002887 atomic_dec(&conf->r5c_flushing_full_stripes);
Song Liu03b047f2017-01-11 13:39:14 -08002888 atomic_dec(&conf->r5c_cached_full_stripes);
2889 }
Song Liuea174812017-03-09 21:23:39 -08002890
2891 r5l_append_flush_payload(log, sh->sector);
Song Liu5ddf0442017-05-11 17:03:44 -07002892 /* stripe is flused to raid disks, we can do resync now */
2893 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
2894 set_bit(STRIPE_HANDLE, &sh->state);
Song Liu1e6d6902016-11-17 15:24:39 -08002895}
2896
Artur Paszkiewiczff875732017-03-09 09:59:58 +01002897int r5c_cache_data(struct r5l_log *log, struct stripe_head *sh)
Song Liu1e6d6902016-11-17 15:24:39 -08002898{
Song Liua39f7af2016-11-17 15:24:40 -08002899 struct r5conf *conf = sh->raid_conf;
Song Liu1e6d6902016-11-17 15:24:39 -08002900 int pages = 0;
2901 int reserve;
2902 int i;
2903 int ret = 0;
2904
2905 BUG_ON(!log);
2906
2907 for (i = 0; i < sh->disks; i++) {
2908 void *addr;
2909
2910 if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
2911 continue;
2912 addr = kmap_atomic(sh->dev[i].page);
2913 sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
2914 addr, PAGE_SIZE);
2915 kunmap_atomic(addr);
2916 pages++;
2917 }
2918 WARN_ON(pages == 0);
2919
2920 /*
2921 * The stripe must enter state machine again to call endio, so
2922 * don't delay.
2923 */
2924 clear_bit(STRIPE_DELAYED, &sh->state);
2925 atomic_inc(&sh->count);
2926
2927 mutex_lock(&log->io_mutex);
2928 /* meta + data */
2929 reserve = (1 + pages) << (PAGE_SHIFT - 9);
Song Liu1e6d6902016-11-17 15:24:39 -08002930
Song Liua39f7af2016-11-17 15:24:40 -08002931 if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
2932 sh->log_start == MaxSector)
2933 r5l_add_no_space_stripe(log, sh);
2934 else if (!r5l_has_free_space(log, reserve)) {
2935 if (sh->log_start == log->last_checkpoint)
2936 BUG();
2937 else
2938 r5l_add_no_space_stripe(log, sh);
Song Liu1e6d6902016-11-17 15:24:39 -08002939 } else {
2940 ret = r5l_log_stripe(log, sh, pages, 0);
2941 if (ret) {
2942 spin_lock_irq(&log->io_list_lock);
2943 list_add_tail(&sh->log_list, &log->no_mem_stripes);
2944 spin_unlock_irq(&log->io_list_lock);
2945 }
2946 }
2947
2948 mutex_unlock(&log->io_mutex);
2949 return 0;
Shaohua Lif6bed0e2015-08-13 14:31:59 -07002950}
2951
Song Liu03b047f2017-01-11 13:39:14 -08002952/* check whether this big stripe is in write back cache. */
2953bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect)
2954{
2955 struct r5l_log *log = conf->log;
2956 sector_t tree_index;
2957 void *slot;
2958
2959 if (!log)
2960 return false;
2961
2962 WARN_ON_ONCE(!rcu_read_lock_held());
2963 tree_index = r5c_tree_index(conf, sect);
2964 slot = radix_tree_lookup(&log->big_stripe_tree, tree_index);
2965 return slot != NULL;
2966}
2967
Shaohua Lif6bed0e2015-08-13 14:31:59 -07002968static int r5l_load_log(struct r5l_log *log)
2969{
2970 struct md_rdev *rdev = log->rdev;
2971 struct page *page;
2972 struct r5l_meta_block *mb;
2973 sector_t cp = log->rdev->journal_tail;
2974 u32 stored_crc, expected_crc;
2975 bool create_super = false;
JackieLiud30dfeb2016-12-08 08:47:39 +08002976 int ret = 0;
Shaohua Lif6bed0e2015-08-13 14:31:59 -07002977
2978 /* Make sure it's valid */
2979 if (cp >= rdev->sectors || round_down(cp, BLOCK_SECTORS) != cp)
2980 cp = 0;
2981 page = alloc_page(GFP_KERNEL);
2982 if (!page)
2983 return -ENOMEM;
2984
Mike Christie796a5cf2016-06-05 14:32:07 -05002985 if (!sync_page_io(rdev, cp, PAGE_SIZE, page, REQ_OP_READ, 0, false)) {
Shaohua Lif6bed0e2015-08-13 14:31:59 -07002986 ret = -EIO;
2987 goto ioerr;
2988 }
2989 mb = page_address(page);
2990
2991 if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
2992 mb->version != R5LOG_VERSION) {
2993 create_super = true;
2994 goto create;
2995 }
2996 stored_crc = le32_to_cpu(mb->checksum);
2997 mb->checksum = 0;
Shaohua Li5cb2fbd2015-10-28 08:41:25 -07002998 expected_crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07002999 if (stored_crc != expected_crc) {
3000 create_super = true;
3001 goto create;
3002 }
3003 if (le64_to_cpu(mb->position) != cp) {
3004 create_super = true;
3005 goto create;
3006 }
3007create:
3008 if (create_super) {
3009 log->last_cp_seq = prandom_u32();
3010 cp = 0;
Zhengyuan Liu56056c22016-10-24 16:15:59 +08003011 r5l_log_write_empty_meta_block(log, cp, log->last_cp_seq);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003012 /*
3013 * Make sure super points to correct address. Log might have
3014 * data very soon. If super hasn't correct log tail address,
3015 * recovery can't find the log
3016 */
3017 r5l_write_super(log, cp);
3018 } else
3019 log->last_cp_seq = le64_to_cpu(mb->seq);
3020
3021 log->device_size = round_down(rdev->sectors, BLOCK_SECTORS);
Shaohua Li0576b1c2015-08-13 14:32:00 -07003022 log->max_free_space = log->device_size >> RECLAIM_MAX_FREE_SPACE_SHIFT;
3023 if (log->max_free_space > RECLAIM_MAX_FREE_SPACE)
3024 log->max_free_space = RECLAIM_MAX_FREE_SPACE;
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003025 log->last_checkpoint = cp;
3026
3027 __free_page(page);
3028
JackieLiud30dfeb2016-12-08 08:47:39 +08003029 if (create_super) {
3030 log->log_start = r5l_ring_add(log, cp, BLOCK_SECTORS);
3031 log->seq = log->last_cp_seq + 1;
3032 log->next_checkpoint = cp;
3033 } else
3034 ret = r5l_recovery_log(log);
3035
Zhengyuan Liu3d7e7e12016-12-04 16:49:44 +08003036 r5c_update_log_state(log);
3037 return ret;
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003038ioerr:
3039 __free_page(page);
3040 return ret;
3041}
3042
Song Liu70d466f2017-05-11 15:28:28 -07003043void r5c_update_on_rdev_error(struct mddev *mddev, struct md_rdev *rdev)
Song Liu2e38a372017-01-24 10:45:30 -08003044{
3045 struct r5conf *conf = mddev->private;
3046 struct r5l_log *log = conf->log;
3047
3048 if (!log)
3049 return;
3050
Song Liu70d466f2017-05-11 15:28:28 -07003051 if ((raid5_calc_degraded(conf) > 0 ||
3052 test_bit(Journal, &rdev->flags)) &&
Song Liu2e38a372017-01-24 10:45:30 -08003053 conf->log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK)
3054 schedule_work(&log->disable_writeback_work);
3055}
3056
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003057int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
3058{
Jens Axboec888a8f2016-04-13 13:33:19 -06003059 struct request_queue *q = bdev_get_queue(rdev->bdev);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003060 struct r5l_log *log;
Artur Paszkiewiczff875732017-03-09 09:59:58 +01003061 char b[BDEVNAME_SIZE];
3062
3063 pr_debug("md/raid:%s: using device %s as journal\n",
3064 mdname(conf->mddev), bdevname(rdev->bdev, b));
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003065
3066 if (PAGE_SIZE != 4096)
3067 return -EINVAL;
Song Liuc757ec92016-11-17 15:24:36 -08003068
3069 /*
3070 * The PAGE_SIZE must be big enough to hold 1 r5l_meta_block and
3071 * raid_disks r5l_payload_data_parity.
3072 *
3073 * Write journal and cache does not work for very big array
3074 * (raid_disks > 203)
3075 */
3076 if (sizeof(struct r5l_meta_block) +
3077 ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32)) *
3078 conf->raid_disks) > PAGE_SIZE) {
3079 pr_err("md/raid:%s: write journal/cache doesn't work for array with %d disks\n",
3080 mdname(conf->mddev), conf->raid_disks);
3081 return -EINVAL;
3082 }
3083
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003084 log = kzalloc(sizeof(*log), GFP_KERNEL);
3085 if (!log)
3086 return -ENOMEM;
3087 log->rdev = rdev;
3088
Jens Axboec888a8f2016-04-13 13:33:19 -06003089 log->need_cache_flush = test_bit(QUEUE_FLAG_WC, &q->queue_flags) != 0;
Christoph Hellwig56fef7c2015-10-05 09:31:09 +02003090
Shaohua Li5cb2fbd2015-10-28 08:41:25 -07003091 log->uuid_checksum = crc32c_le(~0, rdev->mddev->uuid,
3092 sizeof(rdev->mddev->uuid));
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003093
3094 mutex_init(&log->io_mutex);
3095
3096 spin_lock_init(&log->io_list_lock);
3097 INIT_LIST_HEAD(&log->running_ios);
Shaohua Li0576b1c2015-08-13 14:32:00 -07003098 INIT_LIST_HEAD(&log->io_end_ios);
Shaohua Lia8c34f92015-09-02 13:49:46 -07003099 INIT_LIST_HEAD(&log->flushing_ios);
Christoph Hellwig04732f72015-10-05 09:31:07 +02003100 INIT_LIST_HEAD(&log->finished_ios);
Ming Lei3a83f462016-11-22 08:57:21 -07003101 bio_init(&log->flush_bio, NULL, 0);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003102
3103 log->io_kc = KMEM_CACHE(r5l_io_unit, 0);
3104 if (!log->io_kc)
3105 goto io_kc;
3106
Christoph Hellwig5036c3902015-12-21 10:51:02 +11003107 log->io_pool = mempool_create_slab_pool(R5L_POOL_SIZE, log->io_kc);
3108 if (!log->io_pool)
3109 goto io_pool;
3110
NeilBrown011067b2017-06-18 14:38:57 +10003111 log->bs = bioset_create(R5L_POOL_SIZE, 0, BIOSET_NEED_BVECS);
Christoph Hellwigc38d29b2015-12-21 10:51:02 +11003112 if (!log->bs)
3113 goto io_bs;
3114
Christoph Hellwige8deb632015-12-21 10:51:02 +11003115 log->meta_pool = mempool_create_page_pool(R5L_POOL_SIZE, 0);
3116 if (!log->meta_pool)
3117 goto out_mempool;
3118
Song Liu03b047f2017-01-11 13:39:14 -08003119 spin_lock_init(&log->tree_lock);
3120 INIT_RADIX_TREE(&log->big_stripe_tree, GFP_NOWAIT | __GFP_NOWARN);
3121
Shaohua Li0576b1c2015-08-13 14:32:00 -07003122 log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
3123 log->rdev->mddev, "reclaim");
3124 if (!log->reclaim_thread)
3125 goto reclaim_thread;
Song Liua39f7af2016-11-17 15:24:40 -08003126 log->reclaim_thread->timeout = R5C_RECLAIM_WAKEUP_INTERVAL;
3127
Shaohua Li0fd22b42015-09-02 13:49:47 -07003128 init_waitqueue_head(&log->iounit_wait);
Shaohua Li0576b1c2015-08-13 14:32:00 -07003129
Christoph Hellwig5036c3902015-12-21 10:51:02 +11003130 INIT_LIST_HEAD(&log->no_mem_stripes);
3131
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003132 INIT_LIST_HEAD(&log->no_space_stripes);
3133 spin_lock_init(&log->no_space_stripes_lock);
3134
Song Liu3bddb7f2016-11-18 16:46:50 -08003135 INIT_WORK(&log->deferred_io_work, r5l_submit_io_async);
Song Liu2e38a372017-01-24 10:45:30 -08003136 INIT_WORK(&log->disable_writeback_work, r5c_disable_writeback_async);
Song Liu3bddb7f2016-11-18 16:46:50 -08003137
Song Liu2ded3702016-11-17 15:24:38 -08003138 log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
Song Liua39f7af2016-11-17 15:24:40 -08003139 INIT_LIST_HEAD(&log->stripe_in_journal_list);
3140 spin_lock_init(&log->stripe_in_journal_lock);
3141 atomic_set(&log->stripe_in_journal_count, 0);
Song Liu2ded3702016-11-17 15:24:38 -08003142
Song Liud2250f12016-12-14 15:38:02 -08003143 rcu_assign_pointer(conf->log, log);
3144
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003145 if (r5l_load_log(log))
3146 goto error;
3147
Shaohua Lia62ab492016-01-06 14:37:13 -08003148 set_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003149 return 0;
Christoph Hellwige8deb632015-12-21 10:51:02 +11003150
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003151error:
Song Liud2250f12016-12-14 15:38:02 -08003152 rcu_assign_pointer(conf->log, NULL);
Shaohua Li0576b1c2015-08-13 14:32:00 -07003153 md_unregister_thread(&log->reclaim_thread);
3154reclaim_thread:
Christoph Hellwige8deb632015-12-21 10:51:02 +11003155 mempool_destroy(log->meta_pool);
3156out_mempool:
Christoph Hellwigc38d29b2015-12-21 10:51:02 +11003157 bioset_free(log->bs);
3158io_bs:
Christoph Hellwig5036c3902015-12-21 10:51:02 +11003159 mempool_destroy(log->io_pool);
3160io_pool:
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003161 kmem_cache_destroy(log->io_kc);
3162io_kc:
3163 kfree(log);
3164 return -EINVAL;
3165}
3166
Artur Paszkiewiczff875732017-03-09 09:59:58 +01003167void r5l_exit_log(struct r5conf *conf)
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003168{
Artur Paszkiewiczff875732017-03-09 09:59:58 +01003169 struct r5l_log *log = conf->log;
3170
3171 conf->log = NULL;
3172 synchronize_rcu();
3173
NeilBrown4d5324f2017-10-19 12:17:16 +11003174 /* Ensure disable_writeback_work wakes up and exits */
3175 wake_up(&conf->mddev->sb_wait);
Song Liu2e38a372017-01-24 10:45:30 -08003176 flush_work(&log->disable_writeback_work);
Shaohua Li0576b1c2015-08-13 14:32:00 -07003177 md_unregister_thread(&log->reclaim_thread);
Christoph Hellwige8deb632015-12-21 10:51:02 +11003178 mempool_destroy(log->meta_pool);
Christoph Hellwigc38d29b2015-12-21 10:51:02 +11003179 bioset_free(log->bs);
Christoph Hellwig5036c3902015-12-21 10:51:02 +11003180 mempool_destroy(log->io_pool);
Shaohua Lif6bed0e2015-08-13 14:31:59 -07003181 kmem_cache_destroy(log->io_kc);
3182 kfree(log);
3183}