Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2015 Shaohua Li <shli@fb.com> |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify it |
| 5 | * under the terms and conditions of the GNU General Public License, |
| 6 | * version 2, as published by the Free Software Foundation. |
| 7 | * |
| 8 | * This program is distributed in the hope it will be useful, but WITHOUT |
| 9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 11 | * more details. |
| 12 | * |
| 13 | */ |
| 14 | #include <linux/kernel.h> |
| 15 | #include <linux/wait.h> |
| 16 | #include <linux/blkdev.h> |
| 17 | #include <linux/slab.h> |
| 18 | #include <linux/raid/md_p.h> |
Shaohua Li | 5cb2fbd | 2015-10-28 08:41:25 -0700 | [diff] [blame] | 19 | #include <linux/crc32c.h> |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 20 | #include <linux/random.h> |
| 21 | #include "md.h" |
| 22 | #include "raid5.h" |
| 23 | |
| 24 | /* |
| 25 | * metadata/data stored in disk with 4k size unit (a block) regardless |
| 26 | * underneath hardware sector size. only works with PAGE_SIZE == 4096 |
| 27 | */ |
| 28 | #define BLOCK_SECTORS (8) |
| 29 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 30 | /* |
| 31 | * reclaim runs every 1/4 disk size or 10G reclaimable space. This can prevent |
| 32 | * recovery scans a very long log |
| 33 | */ |
| 34 | #define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */ |
| 35 | #define RECLAIM_MAX_FREE_SPACE_SHIFT (2) |
| 36 | |
Christoph Hellwig | c38d29b | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 37 | /* |
| 38 | * We only need 2 bios per I/O unit to make progress, but ensure we |
| 39 | * have a few more available to not get too tight. |
| 40 | */ |
| 41 | #define R5L_POOL_SIZE 4 |
| 42 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 43 | struct r5l_log { |
| 44 | struct md_rdev *rdev; |
| 45 | |
| 46 | u32 uuid_checksum; |
| 47 | |
| 48 | sector_t device_size; /* log device size, round to |
| 49 | * BLOCK_SECTORS */ |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 50 | sector_t max_free_space; /* reclaim run if free space is at |
| 51 | * this size */ |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 52 | |
| 53 | sector_t last_checkpoint; /* log tail. where recovery scan |
| 54 | * starts from */ |
| 55 | u64 last_cp_seq; /* log tail sequence */ |
| 56 | |
| 57 | sector_t log_start; /* log head. where new data appends */ |
| 58 | u64 seq; /* log head sequence */ |
| 59 | |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 60 | sector_t next_checkpoint; |
| 61 | u64 next_cp_seq; |
| 62 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 63 | struct mutex io_mutex; |
| 64 | struct r5l_io_unit *current_io; /* current io_unit accepting new data */ |
| 65 | |
| 66 | spinlock_t io_list_lock; |
| 67 | struct list_head running_ios; /* io_units which are still running, |
| 68 | * and have not yet been completely |
| 69 | * written to the log */ |
| 70 | struct list_head io_end_ios; /* io_units which have been completely |
| 71 | * written to the log but not yet written |
| 72 | * to the RAID */ |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 73 | struct list_head flushing_ios; /* io_units which are waiting for log |
| 74 | * cache flush */ |
Christoph Hellwig | 04732f7 | 2015-10-05 09:31:07 +0200 | [diff] [blame] | 75 | struct list_head finished_ios; /* io_units which settle down in log disk */ |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 76 | struct bio flush_bio; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 77 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 78 | struct list_head no_mem_stripes; /* pending stripes, -ENOMEM */ |
| 79 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 80 | struct kmem_cache *io_kc; |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 81 | mempool_t *io_pool; |
Christoph Hellwig | c38d29b | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 82 | struct bio_set *bs; |
Christoph Hellwig | e8deb63 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 83 | mempool_t *meta_pool; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 84 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 85 | struct md_thread *reclaim_thread; |
| 86 | unsigned long reclaim_target; /* number of space that need to be |
| 87 | * reclaimed. if it's 0, reclaim spaces |
| 88 | * used by io_units which are in |
| 89 | * IO_UNIT_STRIPE_END state (eg, reclaim |
| 90 | * dones't wait for specific io_unit |
| 91 | * switching to IO_UNIT_STRIPE_END |
| 92 | * state) */ |
Shaohua Li | 0fd22b4 | 2015-09-02 13:49:47 -0700 | [diff] [blame] | 93 | wait_queue_head_t iounit_wait; |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 94 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 95 | struct list_head no_space_stripes; /* pending stripes, log has no space */ |
| 96 | spinlock_t no_space_stripes_lock; |
Christoph Hellwig | 56fef7c | 2015-10-05 09:31:09 +0200 | [diff] [blame] | 97 | |
| 98 | bool need_cache_flush; |
Shaohua Li | 4b48204 | 2015-10-08 21:54:06 -0700 | [diff] [blame] | 99 | bool in_teardown; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 100 | }; |
| 101 | |
| 102 | /* |
| 103 | * an IO range starts from a meta data block and end at the next meta data |
| 104 | * block. The io unit's the meta data block tracks data/parity followed it. io |
| 105 | * unit is written to log disk with normal write, as we always flush log disk |
| 106 | * first and then start move data to raid disks, there is no requirement to |
| 107 | * write io unit with FLUSH/FUA |
| 108 | */ |
| 109 | struct r5l_io_unit { |
| 110 | struct r5l_log *log; |
| 111 | |
| 112 | struct page *meta_page; /* store meta block */ |
| 113 | int meta_offset; /* current offset in meta_page */ |
| 114 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 115 | struct bio *current_bio;/* current_bio accepting new data */ |
| 116 | |
| 117 | atomic_t pending_stripe;/* how many stripes not flushed to raid */ |
| 118 | u64 seq; /* seq number of the metablock */ |
| 119 | sector_t log_start; /* where the io_unit starts */ |
| 120 | sector_t log_end; /* where the io_unit ends */ |
| 121 | struct list_head log_sibling; /* log->running_ios */ |
| 122 | struct list_head stripe_list; /* stripes added to the io_unit */ |
| 123 | |
| 124 | int state; |
Christoph Hellwig | 6143e2c | 2015-10-05 09:31:16 +0200 | [diff] [blame] | 125 | bool need_split_bio; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 126 | }; |
| 127 | |
| 128 | /* r5l_io_unit state */ |
| 129 | enum r5l_io_unit_state { |
| 130 | IO_UNIT_RUNNING = 0, /* accepting new IO */ |
| 131 | IO_UNIT_IO_START = 1, /* io_unit bio start writing to log, |
| 132 | * don't accepting new bio */ |
| 133 | IO_UNIT_IO_END = 2, /* io_unit bio finish writing to log */ |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 134 | IO_UNIT_STRIPE_END = 3, /* stripes data finished writing to raid */ |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 135 | }; |
| 136 | |
| 137 | static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc) |
| 138 | { |
| 139 | start += inc; |
| 140 | if (start >= log->device_size) |
| 141 | start = start - log->device_size; |
| 142 | return start; |
| 143 | } |
| 144 | |
| 145 | static sector_t r5l_ring_distance(struct r5l_log *log, sector_t start, |
| 146 | sector_t end) |
| 147 | { |
| 148 | if (end >= start) |
| 149 | return end - start; |
| 150 | else |
| 151 | return end + log->device_size - start; |
| 152 | } |
| 153 | |
| 154 | static bool r5l_has_free_space(struct r5l_log *log, sector_t size) |
| 155 | { |
| 156 | sector_t used_size; |
| 157 | |
| 158 | used_size = r5l_ring_distance(log, log->last_checkpoint, |
| 159 | log->log_start); |
| 160 | |
| 161 | return log->device_size > used_size + size; |
| 162 | } |
| 163 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 164 | static void __r5l_set_io_unit_state(struct r5l_io_unit *io, |
| 165 | enum r5l_io_unit_state state) |
| 166 | { |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 167 | if (WARN_ON(io->state >= state)) |
| 168 | return; |
| 169 | io->state = state; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 170 | } |
| 171 | |
Christoph Hellwig | d8858f4 | 2015-10-05 09:31:08 +0200 | [diff] [blame] | 172 | static void r5l_io_run_stripes(struct r5l_io_unit *io) |
| 173 | { |
| 174 | struct stripe_head *sh, *next; |
| 175 | |
| 176 | list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) { |
| 177 | list_del_init(&sh->log_list); |
| 178 | set_bit(STRIPE_HANDLE, &sh->state); |
| 179 | raid5_release_stripe(sh); |
| 180 | } |
| 181 | } |
| 182 | |
Christoph Hellwig | 56fef7c | 2015-10-05 09:31:09 +0200 | [diff] [blame] | 183 | static void r5l_log_run_stripes(struct r5l_log *log) |
| 184 | { |
| 185 | struct r5l_io_unit *io, *next; |
| 186 | |
| 187 | assert_spin_locked(&log->io_list_lock); |
| 188 | |
| 189 | list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) { |
| 190 | /* don't change list order */ |
| 191 | if (io->state < IO_UNIT_IO_END) |
| 192 | break; |
| 193 | |
| 194 | list_move_tail(&io->log_sibling, &log->finished_ios); |
| 195 | r5l_io_run_stripes(io); |
| 196 | } |
| 197 | } |
| 198 | |
Christoph Hellwig | 3848c0b | 2015-12-21 10:51:01 +1100 | [diff] [blame] | 199 | static void r5l_move_to_end_ios(struct r5l_log *log) |
| 200 | { |
| 201 | struct r5l_io_unit *io, *next; |
| 202 | |
| 203 | assert_spin_locked(&log->io_list_lock); |
| 204 | |
| 205 | list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) { |
| 206 | /* don't change list order */ |
| 207 | if (io->state < IO_UNIT_IO_END) |
| 208 | break; |
| 209 | list_move_tail(&io->log_sibling, &log->io_end_ios); |
| 210 | } |
| 211 | } |
| 212 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 213 | static void r5l_log_endio(struct bio *bio) |
| 214 | { |
| 215 | struct r5l_io_unit *io = bio->bi_private; |
| 216 | struct r5l_log *log = io->log; |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 217 | unsigned long flags; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 218 | |
Shaohua Li | 6e74a9c | 2015-10-08 21:54:08 -0700 | [diff] [blame] | 219 | if (bio->bi_error) |
| 220 | md_error(log->rdev->mddev, log->rdev); |
| 221 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 222 | bio_put(bio); |
Christoph Hellwig | e8deb63 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 223 | mempool_free(io->meta_page, log->meta_pool); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 224 | |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 225 | spin_lock_irqsave(&log->io_list_lock, flags); |
| 226 | __r5l_set_io_unit_state(io, IO_UNIT_IO_END); |
Christoph Hellwig | 56fef7c | 2015-10-05 09:31:09 +0200 | [diff] [blame] | 227 | if (log->need_cache_flush) |
Christoph Hellwig | 3848c0b | 2015-12-21 10:51:01 +1100 | [diff] [blame] | 228 | r5l_move_to_end_ios(log); |
Christoph Hellwig | 56fef7c | 2015-10-05 09:31:09 +0200 | [diff] [blame] | 229 | else |
| 230 | r5l_log_run_stripes(log); |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 231 | spin_unlock_irqrestore(&log->io_list_lock, flags); |
| 232 | |
Christoph Hellwig | 56fef7c | 2015-10-05 09:31:09 +0200 | [diff] [blame] | 233 | if (log->need_cache_flush) |
| 234 | md_wakeup_thread(log->rdev->mddev->thread); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 235 | } |
| 236 | |
| 237 | static void r5l_submit_current_io(struct r5l_log *log) |
| 238 | { |
| 239 | struct r5l_io_unit *io = log->current_io; |
| 240 | struct r5l_meta_block *block; |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 241 | unsigned long flags; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 242 | u32 crc; |
| 243 | |
| 244 | if (!io) |
| 245 | return; |
| 246 | |
| 247 | block = page_address(io->meta_page); |
| 248 | block->meta_size = cpu_to_le32(io->meta_offset); |
Shaohua Li | 5cb2fbd | 2015-10-28 08:41:25 -0700 | [diff] [blame] | 249 | crc = crc32c_le(log->uuid_checksum, block, PAGE_SIZE); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 250 | block->checksum = cpu_to_le32(crc); |
| 251 | |
| 252 | log->current_io = NULL; |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 253 | spin_lock_irqsave(&log->io_list_lock, flags); |
| 254 | __r5l_set_io_unit_state(io, IO_UNIT_IO_START); |
| 255 | spin_unlock_irqrestore(&log->io_list_lock, flags); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 256 | |
Christoph Hellwig | 6143e2c | 2015-10-05 09:31:16 +0200 | [diff] [blame] | 257 | submit_bio(WRITE, io->current_bio); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 258 | } |
| 259 | |
Christoph Hellwig | 6143e2c | 2015-10-05 09:31:16 +0200 | [diff] [blame] | 260 | static struct bio *r5l_bio_alloc(struct r5l_log *log) |
Christoph Hellwig | b349feb | 2015-10-05 09:31:11 +0200 | [diff] [blame] | 261 | { |
Christoph Hellwig | c38d29b | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 262 | struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, log->bs); |
Christoph Hellwig | b349feb | 2015-10-05 09:31:11 +0200 | [diff] [blame] | 263 | |
| 264 | bio->bi_rw = WRITE; |
| 265 | bio->bi_bdev = log->rdev->bdev; |
Christoph Hellwig | 1e932a3 | 2015-10-05 09:31:12 +0200 | [diff] [blame] | 266 | bio->bi_iter.bi_sector = log->rdev->data_offset + log->log_start; |
Christoph Hellwig | b349feb | 2015-10-05 09:31:11 +0200 | [diff] [blame] | 267 | |
Christoph Hellwig | b349feb | 2015-10-05 09:31:11 +0200 | [diff] [blame] | 268 | return bio; |
| 269 | } |
| 270 | |
Christoph Hellwig | c1b9919 | 2015-10-05 09:31:14 +0200 | [diff] [blame] | 271 | static void r5_reserve_log_entry(struct r5l_log *log, struct r5l_io_unit *io) |
| 272 | { |
| 273 | log->log_start = r5l_ring_add(log, log->log_start, BLOCK_SECTORS); |
| 274 | |
| 275 | /* |
| 276 | * If we filled up the log device start from the beginning again, |
| 277 | * which will require a new bio. |
| 278 | * |
| 279 | * Note: for this to work properly the log size needs to me a multiple |
| 280 | * of BLOCK_SECTORS. |
| 281 | */ |
| 282 | if (log->log_start == 0) |
Christoph Hellwig | 6143e2c | 2015-10-05 09:31:16 +0200 | [diff] [blame] | 283 | io->need_split_bio = true; |
Christoph Hellwig | c1b9919 | 2015-10-05 09:31:14 +0200 | [diff] [blame] | 284 | |
| 285 | io->log_end = log->log_start; |
| 286 | } |
| 287 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 288 | static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log) |
| 289 | { |
| 290 | struct r5l_io_unit *io; |
| 291 | struct r5l_meta_block *block; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 292 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 293 | io = mempool_alloc(log->io_pool, GFP_ATOMIC); |
| 294 | if (!io) |
| 295 | return NULL; |
| 296 | memset(io, 0, sizeof(*io)); |
| 297 | |
Christoph Hellwig | 51039cd | 2015-10-05 09:31:13 +0200 | [diff] [blame] | 298 | io->log = log; |
Christoph Hellwig | 51039cd | 2015-10-05 09:31:13 +0200 | [diff] [blame] | 299 | INIT_LIST_HEAD(&io->log_sibling); |
| 300 | INIT_LIST_HEAD(&io->stripe_list); |
| 301 | io->state = IO_UNIT_RUNNING; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 302 | |
Christoph Hellwig | e8deb63 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 303 | io->meta_page = mempool_alloc(log->meta_pool, GFP_NOIO); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 304 | block = page_address(io->meta_page); |
Christoph Hellwig | e8deb63 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 305 | clear_page(block); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 306 | block->magic = cpu_to_le32(R5LOG_MAGIC); |
| 307 | block->version = R5LOG_VERSION; |
| 308 | block->seq = cpu_to_le64(log->seq); |
| 309 | block->position = cpu_to_le64(log->log_start); |
| 310 | |
| 311 | io->log_start = log->log_start; |
| 312 | io->meta_offset = sizeof(struct r5l_meta_block); |
Christoph Hellwig | 2b8ef16 | 2015-10-05 09:31:15 +0200 | [diff] [blame] | 313 | io->seq = log->seq++; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 314 | |
Christoph Hellwig | 6143e2c | 2015-10-05 09:31:16 +0200 | [diff] [blame] | 315 | io->current_bio = r5l_bio_alloc(log); |
| 316 | io->current_bio->bi_end_io = r5l_log_endio; |
| 317 | io->current_bio->bi_private = io; |
Christoph Hellwig | b349feb | 2015-10-05 09:31:11 +0200 | [diff] [blame] | 318 | bio_add_page(io->current_bio, io->meta_page, PAGE_SIZE, 0); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 319 | |
Christoph Hellwig | c1b9919 | 2015-10-05 09:31:14 +0200 | [diff] [blame] | 320 | r5_reserve_log_entry(log, io); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 321 | |
| 322 | spin_lock_irq(&log->io_list_lock); |
| 323 | list_add_tail(&io->log_sibling, &log->running_ios); |
| 324 | spin_unlock_irq(&log->io_list_lock); |
| 325 | |
| 326 | return io; |
| 327 | } |
| 328 | |
| 329 | static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size) |
| 330 | { |
Christoph Hellwig | 22581f5 | 2015-10-05 09:31:10 +0200 | [diff] [blame] | 331 | if (log->current_io && |
| 332 | log->current_io->meta_offset + payload_size > PAGE_SIZE) |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 333 | r5l_submit_current_io(log); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 334 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 335 | if (!log->current_io) { |
Christoph Hellwig | 22581f5 | 2015-10-05 09:31:10 +0200 | [diff] [blame] | 336 | log->current_io = r5l_new_meta(log); |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 337 | if (!log->current_io) |
| 338 | return -ENOMEM; |
| 339 | } |
| 340 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 341 | return 0; |
| 342 | } |
| 343 | |
| 344 | static void r5l_append_payload_meta(struct r5l_log *log, u16 type, |
| 345 | sector_t location, |
| 346 | u32 checksum1, u32 checksum2, |
| 347 | bool checksum2_valid) |
| 348 | { |
| 349 | struct r5l_io_unit *io = log->current_io; |
| 350 | struct r5l_payload_data_parity *payload; |
| 351 | |
| 352 | payload = page_address(io->meta_page) + io->meta_offset; |
| 353 | payload->header.type = cpu_to_le16(type); |
| 354 | payload->header.flags = cpu_to_le16(0); |
| 355 | payload->size = cpu_to_le32((1 + !!checksum2_valid) << |
| 356 | (PAGE_SHIFT - 9)); |
| 357 | payload->location = cpu_to_le64(location); |
| 358 | payload->checksum[0] = cpu_to_le32(checksum1); |
| 359 | if (checksum2_valid) |
| 360 | payload->checksum[1] = cpu_to_le32(checksum2); |
| 361 | |
| 362 | io->meta_offset += sizeof(struct r5l_payload_data_parity) + |
| 363 | sizeof(__le32) * (1 + !!checksum2_valid); |
| 364 | } |
| 365 | |
| 366 | static void r5l_append_payload_page(struct r5l_log *log, struct page *page) |
| 367 | { |
| 368 | struct r5l_io_unit *io = log->current_io; |
| 369 | |
Christoph Hellwig | 6143e2c | 2015-10-05 09:31:16 +0200 | [diff] [blame] | 370 | if (io->need_split_bio) { |
| 371 | struct bio *prev = io->current_bio; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 372 | |
Christoph Hellwig | 6143e2c | 2015-10-05 09:31:16 +0200 | [diff] [blame] | 373 | io->current_bio = r5l_bio_alloc(log); |
| 374 | bio_chain(io->current_bio, prev); |
| 375 | |
| 376 | submit_bio(WRITE, prev); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 377 | } |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 378 | |
Christoph Hellwig | 6143e2c | 2015-10-05 09:31:16 +0200 | [diff] [blame] | 379 | if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0)) |
| 380 | BUG(); |
| 381 | |
Christoph Hellwig | c1b9919 | 2015-10-05 09:31:14 +0200 | [diff] [blame] | 382 | r5_reserve_log_entry(log, io); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 383 | } |
| 384 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 385 | static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh, |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 386 | int data_pages, int parity_pages) |
| 387 | { |
| 388 | int i; |
| 389 | int meta_size; |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 390 | int ret; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 391 | struct r5l_io_unit *io; |
| 392 | |
| 393 | meta_size = |
| 394 | ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32)) |
| 395 | * data_pages) + |
| 396 | sizeof(struct r5l_payload_data_parity) + |
| 397 | sizeof(__le32) * parity_pages; |
| 398 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 399 | ret = r5l_get_meta(log, meta_size); |
| 400 | if (ret) |
| 401 | return ret; |
| 402 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 403 | io = log->current_io; |
| 404 | |
| 405 | for (i = 0; i < sh->disks; i++) { |
| 406 | if (!test_bit(R5_Wantwrite, &sh->dev[i].flags)) |
| 407 | continue; |
| 408 | if (i == sh->pd_idx || i == sh->qd_idx) |
| 409 | continue; |
| 410 | r5l_append_payload_meta(log, R5LOG_PAYLOAD_DATA, |
| 411 | raid5_compute_blocknr(sh, i, 0), |
| 412 | sh->dev[i].log_checksum, 0, false); |
| 413 | r5l_append_payload_page(log, sh->dev[i].page); |
| 414 | } |
| 415 | |
| 416 | if (sh->qd_idx >= 0) { |
| 417 | r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY, |
| 418 | sh->sector, sh->dev[sh->pd_idx].log_checksum, |
| 419 | sh->dev[sh->qd_idx].log_checksum, true); |
| 420 | r5l_append_payload_page(log, sh->dev[sh->pd_idx].page); |
| 421 | r5l_append_payload_page(log, sh->dev[sh->qd_idx].page); |
| 422 | } else { |
| 423 | r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY, |
| 424 | sh->sector, sh->dev[sh->pd_idx].log_checksum, |
| 425 | 0, false); |
| 426 | r5l_append_payload_page(log, sh->dev[sh->pd_idx].page); |
| 427 | } |
| 428 | |
| 429 | list_add_tail(&sh->log_list, &io->stripe_list); |
| 430 | atomic_inc(&io->pending_stripe); |
| 431 | sh->log_io = io; |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 432 | |
| 433 | return 0; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 434 | } |
| 435 | |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 436 | static void r5l_wake_reclaim(struct r5l_log *log, sector_t space); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 437 | /* |
| 438 | * running in raid5d, where reclaim could wait for raid5d too (when it flushes |
| 439 | * data from log to raid disks), so we shouldn't wait for reclaim here |
| 440 | */ |
| 441 | int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh) |
| 442 | { |
| 443 | int write_disks = 0; |
| 444 | int data_pages, parity_pages; |
| 445 | int meta_size; |
| 446 | int reserve; |
| 447 | int i; |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 448 | int ret = 0; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 449 | |
| 450 | if (!log) |
| 451 | return -EAGAIN; |
| 452 | /* Don't support stripe batch */ |
| 453 | if (sh->log_io || !test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags) || |
| 454 | test_bit(STRIPE_SYNCING, &sh->state)) { |
| 455 | /* the stripe is written to log, we start writing it to raid */ |
| 456 | clear_bit(STRIPE_LOG_TRAPPED, &sh->state); |
| 457 | return -EAGAIN; |
| 458 | } |
| 459 | |
| 460 | for (i = 0; i < sh->disks; i++) { |
| 461 | void *addr; |
| 462 | |
| 463 | if (!test_bit(R5_Wantwrite, &sh->dev[i].flags)) |
| 464 | continue; |
| 465 | write_disks++; |
| 466 | /* checksum is already calculated in last run */ |
| 467 | if (test_bit(STRIPE_LOG_TRAPPED, &sh->state)) |
| 468 | continue; |
| 469 | addr = kmap_atomic(sh->dev[i].page); |
Shaohua Li | 5cb2fbd | 2015-10-28 08:41:25 -0700 | [diff] [blame] | 470 | sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum, |
| 471 | addr, PAGE_SIZE); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 472 | kunmap_atomic(addr); |
| 473 | } |
| 474 | parity_pages = 1 + !!(sh->qd_idx >= 0); |
| 475 | data_pages = write_disks - parity_pages; |
| 476 | |
| 477 | meta_size = |
| 478 | ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32)) |
| 479 | * data_pages) + |
| 480 | sizeof(struct r5l_payload_data_parity) + |
| 481 | sizeof(__le32) * parity_pages; |
| 482 | /* Doesn't work with very big raid array */ |
| 483 | if (meta_size + sizeof(struct r5l_meta_block) > PAGE_SIZE) |
| 484 | return -EINVAL; |
| 485 | |
| 486 | set_bit(STRIPE_LOG_TRAPPED, &sh->state); |
Shaohua Li | 253f9fd | 2015-09-04 14:14:16 -0700 | [diff] [blame] | 487 | /* |
| 488 | * The stripe must enter state machine again to finish the write, so |
| 489 | * don't delay. |
| 490 | */ |
| 491 | clear_bit(STRIPE_DELAYED, &sh->state); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 492 | atomic_inc(&sh->count); |
| 493 | |
| 494 | mutex_lock(&log->io_mutex); |
| 495 | /* meta + data */ |
| 496 | reserve = (1 + write_disks) << (PAGE_SHIFT - 9); |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 497 | if (!r5l_has_free_space(log, reserve)) { |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 498 | spin_lock(&log->no_space_stripes_lock); |
| 499 | list_add_tail(&sh->log_list, &log->no_space_stripes); |
| 500 | spin_unlock(&log->no_space_stripes_lock); |
| 501 | |
| 502 | r5l_wake_reclaim(log, reserve); |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 503 | } else { |
| 504 | ret = r5l_log_stripe(log, sh, data_pages, parity_pages); |
| 505 | if (ret) { |
| 506 | spin_lock_irq(&log->io_list_lock); |
| 507 | list_add_tail(&sh->log_list, &log->no_mem_stripes); |
| 508 | spin_unlock_irq(&log->io_list_lock); |
| 509 | } |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 510 | } |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 511 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 512 | mutex_unlock(&log->io_mutex); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 513 | return 0; |
| 514 | } |
| 515 | |
| 516 | void r5l_write_stripe_run(struct r5l_log *log) |
| 517 | { |
| 518 | if (!log) |
| 519 | return; |
| 520 | mutex_lock(&log->io_mutex); |
| 521 | r5l_submit_current_io(log); |
| 522 | mutex_unlock(&log->io_mutex); |
| 523 | } |
| 524 | |
Shaohua Li | 828cbe9 | 2015-09-02 13:49:49 -0700 | [diff] [blame] | 525 | int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio) |
| 526 | { |
| 527 | if (!log) |
| 528 | return -ENODEV; |
| 529 | /* |
| 530 | * we flush log disk cache first, then write stripe data to raid disks. |
| 531 | * So if bio is finished, the log disk cache is flushed already. The |
| 532 | * recovery guarantees we can recovery the bio from log disk, so we |
| 533 | * don't need to flush again |
| 534 | */ |
| 535 | if (bio->bi_iter.bi_size == 0) { |
| 536 | bio_endio(bio); |
| 537 | return 0; |
| 538 | } |
| 539 | bio->bi_rw &= ~REQ_FLUSH; |
| 540 | return -EAGAIN; |
| 541 | } |
| 542 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 543 | /* This will run after log space is reclaimed */ |
| 544 | static void r5l_run_no_space_stripes(struct r5l_log *log) |
| 545 | { |
| 546 | struct stripe_head *sh; |
| 547 | |
| 548 | spin_lock(&log->no_space_stripes_lock); |
| 549 | while (!list_empty(&log->no_space_stripes)) { |
| 550 | sh = list_first_entry(&log->no_space_stripes, |
| 551 | struct stripe_head, log_list); |
| 552 | list_del_init(&sh->log_list); |
| 553 | set_bit(STRIPE_HANDLE, &sh->state); |
| 554 | raid5_release_stripe(sh); |
| 555 | } |
| 556 | spin_unlock(&log->no_space_stripes_lock); |
| 557 | } |
| 558 | |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 559 | static sector_t r5l_reclaimable_space(struct r5l_log *log) |
| 560 | { |
| 561 | return r5l_ring_distance(log, log->last_checkpoint, |
| 562 | log->next_checkpoint); |
| 563 | } |
| 564 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 565 | static void r5l_run_no_mem_stripe(struct r5l_log *log) |
| 566 | { |
| 567 | struct stripe_head *sh; |
| 568 | |
| 569 | assert_spin_locked(&log->io_list_lock); |
| 570 | |
| 571 | if (!list_empty(&log->no_mem_stripes)) { |
| 572 | sh = list_first_entry(&log->no_mem_stripes, |
| 573 | struct stripe_head, log_list); |
| 574 | list_del_init(&sh->log_list); |
| 575 | set_bit(STRIPE_HANDLE, &sh->state); |
| 576 | raid5_release_stripe(sh); |
| 577 | } |
| 578 | } |
| 579 | |
Christoph Hellwig | 04732f7 | 2015-10-05 09:31:07 +0200 | [diff] [blame] | 580 | static bool r5l_complete_finished_ios(struct r5l_log *log) |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 581 | { |
| 582 | struct r5l_io_unit *io, *next; |
| 583 | bool found = false; |
| 584 | |
| 585 | assert_spin_locked(&log->io_list_lock); |
| 586 | |
Christoph Hellwig | 04732f7 | 2015-10-05 09:31:07 +0200 | [diff] [blame] | 587 | list_for_each_entry_safe(io, next, &log->finished_ios, log_sibling) { |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 588 | /* don't change list order */ |
| 589 | if (io->state < IO_UNIT_STRIPE_END) |
| 590 | break; |
| 591 | |
| 592 | log->next_checkpoint = io->log_start; |
| 593 | log->next_cp_seq = io->seq; |
| 594 | |
| 595 | list_del(&io->log_sibling); |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 596 | mempool_free(io, log->io_pool); |
| 597 | r5l_run_no_mem_stripe(log); |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 598 | |
| 599 | found = true; |
| 600 | } |
| 601 | |
| 602 | return found; |
| 603 | } |
| 604 | |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 605 | static void __r5l_stripe_write_finished(struct r5l_io_unit *io) |
| 606 | { |
| 607 | struct r5l_log *log = io->log; |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 608 | unsigned long flags; |
| 609 | |
| 610 | spin_lock_irqsave(&log->io_list_lock, flags); |
| 611 | __r5l_set_io_unit_state(io, IO_UNIT_STRIPE_END); |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 612 | |
Christoph Hellwig | 04732f7 | 2015-10-05 09:31:07 +0200 | [diff] [blame] | 613 | if (!r5l_complete_finished_ios(log)) { |
Shaohua Li | 85f2f9a | 2015-09-04 14:14:05 -0700 | [diff] [blame] | 614 | spin_unlock_irqrestore(&log->io_list_lock, flags); |
| 615 | return; |
| 616 | } |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 617 | |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 618 | if (r5l_reclaimable_space(log) > log->max_free_space) |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 619 | r5l_wake_reclaim(log, 0); |
| 620 | |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 621 | spin_unlock_irqrestore(&log->io_list_lock, flags); |
| 622 | wake_up(&log->iounit_wait); |
| 623 | } |
| 624 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 625 | void r5l_stripe_write_finished(struct stripe_head *sh) |
| 626 | { |
| 627 | struct r5l_io_unit *io; |
| 628 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 629 | io = sh->log_io; |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 630 | sh->log_io = NULL; |
| 631 | |
Christoph Hellwig | 509ffec | 2015-09-02 13:49:48 -0700 | [diff] [blame] | 632 | if (io && atomic_dec_and_test(&io->pending_stripe)) |
| 633 | __r5l_stripe_write_finished(io); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 634 | } |
| 635 | |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 636 | static void r5l_log_flush_endio(struct bio *bio) |
| 637 | { |
| 638 | struct r5l_log *log = container_of(bio, struct r5l_log, |
| 639 | flush_bio); |
| 640 | unsigned long flags; |
| 641 | struct r5l_io_unit *io; |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 642 | |
Shaohua Li | 6e74a9c | 2015-10-08 21:54:08 -0700 | [diff] [blame] | 643 | if (bio->bi_error) |
| 644 | md_error(log->rdev->mddev, log->rdev); |
| 645 | |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 646 | spin_lock_irqsave(&log->io_list_lock, flags); |
Christoph Hellwig | d8858f4 | 2015-10-05 09:31:08 +0200 | [diff] [blame] | 647 | list_for_each_entry(io, &log->flushing_ios, log_sibling) |
| 648 | r5l_io_run_stripes(io); |
Christoph Hellwig | 04732f7 | 2015-10-05 09:31:07 +0200 | [diff] [blame] | 649 | list_splice_tail_init(&log->flushing_ios, &log->finished_ios); |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 650 | spin_unlock_irqrestore(&log->io_list_lock, flags); |
| 651 | } |
| 652 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 653 | /* |
| 654 | * Starting dispatch IO to raid. |
| 655 | * io_unit(meta) consists of a log. There is one situation we want to avoid. A |
| 656 | * broken meta in the middle of a log causes recovery can't find meta at the |
| 657 | * head of log. If operations require meta at the head persistent in log, we |
| 658 | * must make sure meta before it persistent in log too. A case is: |
| 659 | * |
| 660 | * stripe data/parity is in log, we start write stripe to raid disks. stripe |
| 661 | * data/parity must be persistent in log before we do the write to raid disks. |
| 662 | * |
| 663 | * The solution is we restrictly maintain io_unit list order. In this case, we |
| 664 | * only write stripes of an io_unit to raid disks till the io_unit is the first |
| 665 | * one whose data/parity is in log. |
| 666 | */ |
| 667 | void r5l_flush_stripe_to_raid(struct r5l_log *log) |
| 668 | { |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 669 | bool do_flush; |
Christoph Hellwig | 56fef7c | 2015-10-05 09:31:09 +0200 | [diff] [blame] | 670 | |
| 671 | if (!log || !log->need_cache_flush) |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 672 | return; |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 673 | |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 674 | spin_lock_irq(&log->io_list_lock); |
| 675 | /* flush bio is running */ |
| 676 | if (!list_empty(&log->flushing_ios)) { |
| 677 | spin_unlock_irq(&log->io_list_lock); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 678 | return; |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 679 | } |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 680 | list_splice_tail_init(&log->io_end_ios, &log->flushing_ios); |
| 681 | do_flush = !list_empty(&log->flushing_ios); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 682 | spin_unlock_irq(&log->io_list_lock); |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 683 | |
| 684 | if (!do_flush) |
| 685 | return; |
| 686 | bio_reset(&log->flush_bio); |
| 687 | log->flush_bio.bi_bdev = log->rdev->bdev; |
| 688 | log->flush_bio.bi_end_io = r5l_log_flush_endio; |
| 689 | submit_bio(WRITE_FLUSH, &log->flush_bio); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 690 | } |
| 691 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 692 | static void r5l_write_super(struct r5l_log *log, sector_t cp); |
Shaohua Li | 4b48204 | 2015-10-08 21:54:06 -0700 | [diff] [blame] | 693 | static void r5l_write_super_and_discard_space(struct r5l_log *log, |
| 694 | sector_t end) |
| 695 | { |
| 696 | struct block_device *bdev = log->rdev->bdev; |
| 697 | struct mddev *mddev; |
| 698 | |
| 699 | r5l_write_super(log, end); |
| 700 | |
| 701 | if (!blk_queue_discard(bdev_get_queue(bdev))) |
| 702 | return; |
| 703 | |
| 704 | mddev = log->rdev->mddev; |
| 705 | /* |
| 706 | * This is to avoid a deadlock. r5l_quiesce holds reconfig_mutex and |
| 707 | * wait for this thread to finish. This thread waits for |
| 708 | * MD_CHANGE_PENDING clear, which is supposed to be done in |
| 709 | * md_check_recovery(). md_check_recovery() tries to get |
| 710 | * reconfig_mutex. Since r5l_quiesce already holds the mutex, |
| 711 | * md_check_recovery() fails, so the PENDING never get cleared. The |
| 712 | * in_teardown check workaround this issue. |
| 713 | */ |
| 714 | if (!log->in_teardown) { |
| 715 | set_bit(MD_CHANGE_DEVS, &mddev->flags); |
| 716 | set_bit(MD_CHANGE_PENDING, &mddev->flags); |
| 717 | md_wakeup_thread(mddev->thread); |
| 718 | wait_event(mddev->sb_wait, |
| 719 | !test_bit(MD_CHANGE_PENDING, &mddev->flags) || |
| 720 | log->in_teardown); |
| 721 | /* |
| 722 | * r5l_quiesce could run after in_teardown check and hold |
| 723 | * mutex first. Superblock might get updated twice. |
| 724 | */ |
| 725 | if (log->in_teardown) |
| 726 | md_update_sb(mddev, 1); |
| 727 | } else { |
| 728 | WARN_ON(!mddev_is_locked(mddev)); |
| 729 | md_update_sb(mddev, 1); |
| 730 | } |
| 731 | |
Shaohua Li | 6e74a9c | 2015-10-08 21:54:08 -0700 | [diff] [blame] | 732 | /* discard IO error really doesn't matter, ignore it */ |
Shaohua Li | 4b48204 | 2015-10-08 21:54:06 -0700 | [diff] [blame] | 733 | if (log->last_checkpoint < end) { |
| 734 | blkdev_issue_discard(bdev, |
| 735 | log->last_checkpoint + log->rdev->data_offset, |
| 736 | end - log->last_checkpoint, GFP_NOIO, 0); |
| 737 | } else { |
| 738 | blkdev_issue_discard(bdev, |
| 739 | log->last_checkpoint + log->rdev->data_offset, |
| 740 | log->device_size - log->last_checkpoint, |
| 741 | GFP_NOIO, 0); |
| 742 | blkdev_issue_discard(bdev, log->rdev->data_offset, end, |
| 743 | GFP_NOIO, 0); |
| 744 | } |
| 745 | } |
| 746 | |
| 747 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 748 | static void r5l_do_reclaim(struct r5l_log *log) |
| 749 | { |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 750 | sector_t reclaim_target = xchg(&log->reclaim_target, 0); |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 751 | sector_t reclaimable; |
| 752 | sector_t next_checkpoint; |
| 753 | u64 next_cp_seq; |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 754 | |
| 755 | spin_lock_irq(&log->io_list_lock); |
| 756 | /* |
| 757 | * move proper io_unit to reclaim list. We should not change the order. |
| 758 | * reclaimable/unreclaimable io_unit can be mixed in the list, we |
| 759 | * shouldn't reuse space of an unreclaimable io_unit |
| 760 | */ |
| 761 | while (1) { |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 762 | reclaimable = r5l_reclaimable_space(log); |
| 763 | if (reclaimable >= reclaim_target || |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 764 | (list_empty(&log->running_ios) && |
| 765 | list_empty(&log->io_end_ios) && |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 766 | list_empty(&log->flushing_ios) && |
Christoph Hellwig | 04732f7 | 2015-10-05 09:31:07 +0200 | [diff] [blame] | 767 | list_empty(&log->finished_ios))) |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 768 | break; |
| 769 | |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 770 | md_wakeup_thread(log->rdev->mddev->thread); |
| 771 | wait_event_lock_irq(log->iounit_wait, |
| 772 | r5l_reclaimable_space(log) > reclaimable, |
| 773 | log->io_list_lock); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 774 | } |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 775 | |
| 776 | next_checkpoint = log->next_checkpoint; |
| 777 | next_cp_seq = log->next_cp_seq; |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 778 | spin_unlock_irq(&log->io_list_lock); |
| 779 | |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 780 | BUG_ON(reclaimable < 0); |
| 781 | if (reclaimable == 0) |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 782 | return; |
| 783 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 784 | /* |
| 785 | * write_super will flush cache of each raid disk. We must write super |
| 786 | * here, because the log area might be reused soon and we don't want to |
| 787 | * confuse recovery |
| 788 | */ |
Shaohua Li | 4b48204 | 2015-10-08 21:54:06 -0700 | [diff] [blame] | 789 | r5l_write_super_and_discard_space(log, next_checkpoint); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 790 | |
| 791 | mutex_lock(&log->io_mutex); |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 792 | log->last_checkpoint = next_checkpoint; |
| 793 | log->last_cp_seq = next_cp_seq; |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 794 | mutex_unlock(&log->io_mutex); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 795 | |
Christoph Hellwig | 1703646 | 2015-10-05 09:31:06 +0200 | [diff] [blame] | 796 | r5l_run_no_space_stripes(log); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 797 | } |
| 798 | |
| 799 | static void r5l_reclaim_thread(struct md_thread *thread) |
| 800 | { |
| 801 | struct mddev *mddev = thread->mddev; |
| 802 | struct r5conf *conf = mddev->private; |
| 803 | struct r5l_log *log = conf->log; |
| 804 | |
| 805 | if (!log) |
| 806 | return; |
| 807 | r5l_do_reclaim(log); |
| 808 | } |
| 809 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 810 | static void r5l_wake_reclaim(struct r5l_log *log, sector_t space) |
| 811 | { |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 812 | unsigned long target; |
| 813 | unsigned long new = (unsigned long)space; /* overflow in theory */ |
| 814 | |
| 815 | do { |
| 816 | target = log->reclaim_target; |
| 817 | if (new < target) |
| 818 | return; |
| 819 | } while (cmpxchg(&log->reclaim_target, target, new) != target); |
| 820 | md_wakeup_thread(log->reclaim_thread); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 821 | } |
| 822 | |
Shaohua Li | e6c033f | 2015-10-04 09:20:12 -0700 | [diff] [blame] | 823 | void r5l_quiesce(struct r5l_log *log, int state) |
| 824 | { |
Shaohua Li | 4b48204 | 2015-10-08 21:54:06 -0700 | [diff] [blame] | 825 | struct mddev *mddev; |
Shaohua Li | e6c033f | 2015-10-04 09:20:12 -0700 | [diff] [blame] | 826 | if (!log || state == 2) |
| 827 | return; |
| 828 | if (state == 0) { |
Shaohua Li | 4b48204 | 2015-10-08 21:54:06 -0700 | [diff] [blame] | 829 | log->in_teardown = 0; |
Shaohua Li | 16a43f6 | 2016-01-06 14:37:15 -0800 | [diff] [blame] | 830 | /* |
| 831 | * This is a special case for hotadd. In suspend, the array has |
| 832 | * no journal. In resume, journal is initialized as well as the |
| 833 | * reclaim thread. |
| 834 | */ |
| 835 | if (log->reclaim_thread) |
| 836 | return; |
Shaohua Li | e6c033f | 2015-10-04 09:20:12 -0700 | [diff] [blame] | 837 | log->reclaim_thread = md_register_thread(r5l_reclaim_thread, |
| 838 | log->rdev->mddev, "reclaim"); |
| 839 | } else if (state == 1) { |
| 840 | /* |
| 841 | * at this point all stripes are finished, so io_unit is at |
| 842 | * least in STRIPE_END state |
| 843 | */ |
Shaohua Li | 4b48204 | 2015-10-08 21:54:06 -0700 | [diff] [blame] | 844 | log->in_teardown = 1; |
| 845 | /* make sure r5l_write_super_and_discard_space exits */ |
| 846 | mddev = log->rdev->mddev; |
| 847 | wake_up(&mddev->sb_wait); |
Shaohua Li | e6c033f | 2015-10-04 09:20:12 -0700 | [diff] [blame] | 848 | r5l_wake_reclaim(log, -1L); |
| 849 | md_unregister_thread(&log->reclaim_thread); |
| 850 | r5l_do_reclaim(log); |
| 851 | } |
| 852 | } |
| 853 | |
Shaohua Li | 6e74a9c | 2015-10-08 21:54:08 -0700 | [diff] [blame] | 854 | bool r5l_log_disk_error(struct r5conf *conf) |
| 855 | { |
Shaohua Li | f6b6ec5 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 856 | struct r5l_log *log; |
| 857 | bool ret; |
Shaohua Li | 7dde2ad | 2015-10-08 21:54:10 -0700 | [diff] [blame] | 858 | /* don't allow write if journal disk is missing */ |
Shaohua Li | f6b6ec5 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 859 | rcu_read_lock(); |
| 860 | log = rcu_dereference(conf->log); |
| 861 | |
| 862 | if (!log) |
| 863 | ret = test_bit(MD_HAS_JOURNAL, &conf->mddev->flags); |
| 864 | else |
| 865 | ret = test_bit(Faulty, &log->rdev->flags); |
| 866 | rcu_read_unlock(); |
| 867 | return ret; |
Shaohua Li | 6e74a9c | 2015-10-08 21:54:08 -0700 | [diff] [blame] | 868 | } |
| 869 | |
Shaohua Li | 355810d | 2015-08-13 14:32:01 -0700 | [diff] [blame] | 870 | struct r5l_recovery_ctx { |
| 871 | struct page *meta_page; /* current meta */ |
| 872 | sector_t meta_total_blocks; /* total size of current meta and data */ |
| 873 | sector_t pos; /* recovery position */ |
| 874 | u64 seq; /* recovery position seq */ |
| 875 | }; |
| 876 | |
| 877 | static int r5l_read_meta_block(struct r5l_log *log, |
| 878 | struct r5l_recovery_ctx *ctx) |
| 879 | { |
| 880 | struct page *page = ctx->meta_page; |
| 881 | struct r5l_meta_block *mb; |
| 882 | u32 crc, stored_crc; |
| 883 | |
| 884 | if (!sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page, READ, false)) |
| 885 | return -EIO; |
| 886 | |
| 887 | mb = page_address(page); |
| 888 | stored_crc = le32_to_cpu(mb->checksum); |
| 889 | mb->checksum = 0; |
| 890 | |
| 891 | if (le32_to_cpu(mb->magic) != R5LOG_MAGIC || |
| 892 | le64_to_cpu(mb->seq) != ctx->seq || |
| 893 | mb->version != R5LOG_VERSION || |
| 894 | le64_to_cpu(mb->position) != ctx->pos) |
| 895 | return -EINVAL; |
| 896 | |
Shaohua Li | 5cb2fbd | 2015-10-28 08:41:25 -0700 | [diff] [blame] | 897 | crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE); |
Shaohua Li | 355810d | 2015-08-13 14:32:01 -0700 | [diff] [blame] | 898 | if (stored_crc != crc) |
| 899 | return -EINVAL; |
| 900 | |
| 901 | if (le32_to_cpu(mb->meta_size) > PAGE_SIZE) |
| 902 | return -EINVAL; |
| 903 | |
| 904 | ctx->meta_total_blocks = BLOCK_SECTORS; |
| 905 | |
| 906 | return 0; |
| 907 | } |
| 908 | |
| 909 | static int r5l_recovery_flush_one_stripe(struct r5l_log *log, |
| 910 | struct r5l_recovery_ctx *ctx, |
| 911 | sector_t stripe_sect, |
| 912 | int *offset, sector_t *log_offset) |
| 913 | { |
| 914 | struct r5conf *conf = log->rdev->mddev->private; |
| 915 | struct stripe_head *sh; |
| 916 | struct r5l_payload_data_parity *payload; |
| 917 | int disk_index; |
| 918 | |
| 919 | sh = raid5_get_active_stripe(conf, stripe_sect, 0, 0, 0); |
| 920 | while (1) { |
| 921 | payload = page_address(ctx->meta_page) + *offset; |
| 922 | |
| 923 | if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) { |
| 924 | raid5_compute_sector(conf, |
| 925 | le64_to_cpu(payload->location), 0, |
| 926 | &disk_index, sh); |
| 927 | |
| 928 | sync_page_io(log->rdev, *log_offset, PAGE_SIZE, |
| 929 | sh->dev[disk_index].page, READ, false); |
| 930 | sh->dev[disk_index].log_checksum = |
| 931 | le32_to_cpu(payload->checksum[0]); |
| 932 | set_bit(R5_Wantwrite, &sh->dev[disk_index].flags); |
| 933 | ctx->meta_total_blocks += BLOCK_SECTORS; |
| 934 | } else { |
| 935 | disk_index = sh->pd_idx; |
| 936 | sync_page_io(log->rdev, *log_offset, PAGE_SIZE, |
| 937 | sh->dev[disk_index].page, READ, false); |
| 938 | sh->dev[disk_index].log_checksum = |
| 939 | le32_to_cpu(payload->checksum[0]); |
| 940 | set_bit(R5_Wantwrite, &sh->dev[disk_index].flags); |
| 941 | |
| 942 | if (sh->qd_idx >= 0) { |
| 943 | disk_index = sh->qd_idx; |
| 944 | sync_page_io(log->rdev, |
| 945 | r5l_ring_add(log, *log_offset, BLOCK_SECTORS), |
| 946 | PAGE_SIZE, sh->dev[disk_index].page, |
| 947 | READ, false); |
| 948 | sh->dev[disk_index].log_checksum = |
| 949 | le32_to_cpu(payload->checksum[1]); |
| 950 | set_bit(R5_Wantwrite, |
| 951 | &sh->dev[disk_index].flags); |
| 952 | } |
| 953 | ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded; |
| 954 | } |
| 955 | |
| 956 | *log_offset = r5l_ring_add(log, *log_offset, |
| 957 | le32_to_cpu(payload->size)); |
| 958 | *offset += sizeof(struct r5l_payload_data_parity) + |
| 959 | sizeof(__le32) * |
| 960 | (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9)); |
| 961 | if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY) |
| 962 | break; |
| 963 | } |
| 964 | |
| 965 | for (disk_index = 0; disk_index < sh->disks; disk_index++) { |
| 966 | void *addr; |
| 967 | u32 checksum; |
| 968 | |
| 969 | if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags)) |
| 970 | continue; |
| 971 | addr = kmap_atomic(sh->dev[disk_index].page); |
Shaohua Li | 5cb2fbd | 2015-10-28 08:41:25 -0700 | [diff] [blame] | 972 | checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE); |
Shaohua Li | 355810d | 2015-08-13 14:32:01 -0700 | [diff] [blame] | 973 | kunmap_atomic(addr); |
| 974 | if (checksum != sh->dev[disk_index].log_checksum) |
| 975 | goto error; |
| 976 | } |
| 977 | |
| 978 | for (disk_index = 0; disk_index < sh->disks; disk_index++) { |
| 979 | struct md_rdev *rdev, *rrdev; |
| 980 | |
| 981 | if (!test_and_clear_bit(R5_Wantwrite, |
| 982 | &sh->dev[disk_index].flags)) |
| 983 | continue; |
| 984 | |
| 985 | /* in case device is broken */ |
| 986 | rdev = rcu_dereference(conf->disks[disk_index].rdev); |
| 987 | if (rdev) |
| 988 | sync_page_io(rdev, stripe_sect, PAGE_SIZE, |
| 989 | sh->dev[disk_index].page, WRITE, false); |
| 990 | rrdev = rcu_dereference(conf->disks[disk_index].replacement); |
| 991 | if (rrdev) |
| 992 | sync_page_io(rrdev, stripe_sect, PAGE_SIZE, |
| 993 | sh->dev[disk_index].page, WRITE, false); |
| 994 | } |
| 995 | raid5_release_stripe(sh); |
| 996 | return 0; |
| 997 | |
| 998 | error: |
| 999 | for (disk_index = 0; disk_index < sh->disks; disk_index++) |
| 1000 | sh->dev[disk_index].flags = 0; |
| 1001 | raid5_release_stripe(sh); |
| 1002 | return -EINVAL; |
| 1003 | } |
| 1004 | |
| 1005 | static int r5l_recovery_flush_one_meta(struct r5l_log *log, |
| 1006 | struct r5l_recovery_ctx *ctx) |
| 1007 | { |
| 1008 | struct r5conf *conf = log->rdev->mddev->private; |
| 1009 | struct r5l_payload_data_parity *payload; |
| 1010 | struct r5l_meta_block *mb; |
| 1011 | int offset; |
| 1012 | sector_t log_offset; |
| 1013 | sector_t stripe_sector; |
| 1014 | |
| 1015 | mb = page_address(ctx->meta_page); |
| 1016 | offset = sizeof(struct r5l_meta_block); |
| 1017 | log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS); |
| 1018 | |
| 1019 | while (offset < le32_to_cpu(mb->meta_size)) { |
| 1020 | int dd; |
| 1021 | |
| 1022 | payload = (void *)mb + offset; |
| 1023 | stripe_sector = raid5_compute_sector(conf, |
| 1024 | le64_to_cpu(payload->location), 0, &dd, NULL); |
| 1025 | if (r5l_recovery_flush_one_stripe(log, ctx, stripe_sector, |
| 1026 | &offset, &log_offset)) |
| 1027 | return -EINVAL; |
| 1028 | } |
| 1029 | return 0; |
| 1030 | } |
| 1031 | |
| 1032 | /* copy data/parity from log to raid disks */ |
| 1033 | static void r5l_recovery_flush_log(struct r5l_log *log, |
| 1034 | struct r5l_recovery_ctx *ctx) |
| 1035 | { |
| 1036 | while (1) { |
| 1037 | if (r5l_read_meta_block(log, ctx)) |
| 1038 | return; |
| 1039 | if (r5l_recovery_flush_one_meta(log, ctx)) |
| 1040 | return; |
| 1041 | ctx->seq++; |
| 1042 | ctx->pos = r5l_ring_add(log, ctx->pos, ctx->meta_total_blocks); |
| 1043 | } |
| 1044 | } |
| 1045 | |
| 1046 | static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos, |
| 1047 | u64 seq) |
| 1048 | { |
| 1049 | struct page *page; |
| 1050 | struct r5l_meta_block *mb; |
| 1051 | u32 crc; |
| 1052 | |
| 1053 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); |
| 1054 | if (!page) |
| 1055 | return -ENOMEM; |
| 1056 | mb = page_address(page); |
| 1057 | mb->magic = cpu_to_le32(R5LOG_MAGIC); |
| 1058 | mb->version = R5LOG_VERSION; |
| 1059 | mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block)); |
| 1060 | mb->seq = cpu_to_le64(seq); |
| 1061 | mb->position = cpu_to_le64(pos); |
Shaohua Li | 5cb2fbd | 2015-10-28 08:41:25 -0700 | [diff] [blame] | 1062 | crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE); |
Shaohua Li | 355810d | 2015-08-13 14:32:01 -0700 | [diff] [blame] | 1063 | mb->checksum = cpu_to_le32(crc); |
| 1064 | |
| 1065 | if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, WRITE_FUA, false)) { |
| 1066 | __free_page(page); |
| 1067 | return -EIO; |
| 1068 | } |
| 1069 | __free_page(page); |
| 1070 | return 0; |
| 1071 | } |
| 1072 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1073 | static int r5l_recovery_log(struct r5l_log *log) |
| 1074 | { |
Shaohua Li | 355810d | 2015-08-13 14:32:01 -0700 | [diff] [blame] | 1075 | struct r5l_recovery_ctx ctx; |
| 1076 | |
| 1077 | ctx.pos = log->last_checkpoint; |
| 1078 | ctx.seq = log->last_cp_seq; |
| 1079 | ctx.meta_page = alloc_page(GFP_KERNEL); |
| 1080 | if (!ctx.meta_page) |
| 1081 | return -ENOMEM; |
| 1082 | |
| 1083 | r5l_recovery_flush_log(log, &ctx); |
| 1084 | __free_page(ctx.meta_page); |
| 1085 | |
| 1086 | /* |
| 1087 | * we did a recovery. Now ctx.pos points to an invalid meta block. New |
| 1088 | * log will start here. but we can't let superblock point to last valid |
| 1089 | * meta block. The log might looks like: |
| 1090 | * | meta 1| meta 2| meta 3| |
| 1091 | * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If |
| 1092 | * superblock points to meta 1, we write a new valid meta 2n. if crash |
| 1093 | * happens again, new recovery will start from meta 1. Since meta 2n is |
| 1094 | * valid now, recovery will think meta 3 is valid, which is wrong. |
| 1095 | * The solution is we create a new meta in meta2 with its seq == meta |
| 1096 | * 1's seq + 10 and let superblock points to meta2. The same recovery will |
| 1097 | * not think meta 3 is a valid meta, because its seq doesn't match |
| 1098 | */ |
| 1099 | if (ctx.seq > log->last_cp_seq + 1) { |
| 1100 | int ret; |
| 1101 | |
| 1102 | ret = r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq + 10); |
| 1103 | if (ret) |
| 1104 | return ret; |
| 1105 | log->seq = ctx.seq + 11; |
| 1106 | log->log_start = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS); |
| 1107 | r5l_write_super(log, ctx.pos); |
| 1108 | } else { |
| 1109 | log->log_start = ctx.pos; |
| 1110 | log->seq = ctx.seq; |
| 1111 | } |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1112 | return 0; |
| 1113 | } |
| 1114 | |
| 1115 | static void r5l_write_super(struct r5l_log *log, sector_t cp) |
| 1116 | { |
| 1117 | struct mddev *mddev = log->rdev->mddev; |
| 1118 | |
| 1119 | log->rdev->journal_tail = cp; |
| 1120 | set_bit(MD_CHANGE_DEVS, &mddev->flags); |
| 1121 | } |
| 1122 | |
| 1123 | static int r5l_load_log(struct r5l_log *log) |
| 1124 | { |
| 1125 | struct md_rdev *rdev = log->rdev; |
| 1126 | struct page *page; |
| 1127 | struct r5l_meta_block *mb; |
| 1128 | sector_t cp = log->rdev->journal_tail; |
| 1129 | u32 stored_crc, expected_crc; |
| 1130 | bool create_super = false; |
| 1131 | int ret; |
| 1132 | |
| 1133 | /* Make sure it's valid */ |
| 1134 | if (cp >= rdev->sectors || round_down(cp, BLOCK_SECTORS) != cp) |
| 1135 | cp = 0; |
| 1136 | page = alloc_page(GFP_KERNEL); |
| 1137 | if (!page) |
| 1138 | return -ENOMEM; |
| 1139 | |
| 1140 | if (!sync_page_io(rdev, cp, PAGE_SIZE, page, READ, false)) { |
| 1141 | ret = -EIO; |
| 1142 | goto ioerr; |
| 1143 | } |
| 1144 | mb = page_address(page); |
| 1145 | |
| 1146 | if (le32_to_cpu(mb->magic) != R5LOG_MAGIC || |
| 1147 | mb->version != R5LOG_VERSION) { |
| 1148 | create_super = true; |
| 1149 | goto create; |
| 1150 | } |
| 1151 | stored_crc = le32_to_cpu(mb->checksum); |
| 1152 | mb->checksum = 0; |
Shaohua Li | 5cb2fbd | 2015-10-28 08:41:25 -0700 | [diff] [blame] | 1153 | expected_crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1154 | if (stored_crc != expected_crc) { |
| 1155 | create_super = true; |
| 1156 | goto create; |
| 1157 | } |
| 1158 | if (le64_to_cpu(mb->position) != cp) { |
| 1159 | create_super = true; |
| 1160 | goto create; |
| 1161 | } |
| 1162 | create: |
| 1163 | if (create_super) { |
| 1164 | log->last_cp_seq = prandom_u32(); |
| 1165 | cp = 0; |
| 1166 | /* |
| 1167 | * Make sure super points to correct address. Log might have |
| 1168 | * data very soon. If super hasn't correct log tail address, |
| 1169 | * recovery can't find the log |
| 1170 | */ |
| 1171 | r5l_write_super(log, cp); |
| 1172 | } else |
| 1173 | log->last_cp_seq = le64_to_cpu(mb->seq); |
| 1174 | |
| 1175 | log->device_size = round_down(rdev->sectors, BLOCK_SECTORS); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 1176 | log->max_free_space = log->device_size >> RECLAIM_MAX_FREE_SPACE_SHIFT; |
| 1177 | if (log->max_free_space > RECLAIM_MAX_FREE_SPACE) |
| 1178 | log->max_free_space = RECLAIM_MAX_FREE_SPACE; |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1179 | log->last_checkpoint = cp; |
| 1180 | |
| 1181 | __free_page(page); |
| 1182 | |
| 1183 | return r5l_recovery_log(log); |
| 1184 | ioerr: |
| 1185 | __free_page(page); |
| 1186 | return ret; |
| 1187 | } |
| 1188 | |
| 1189 | int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev) |
| 1190 | { |
| 1191 | struct r5l_log *log; |
| 1192 | |
| 1193 | if (PAGE_SIZE != 4096) |
| 1194 | return -EINVAL; |
| 1195 | log = kzalloc(sizeof(*log), GFP_KERNEL); |
| 1196 | if (!log) |
| 1197 | return -ENOMEM; |
| 1198 | log->rdev = rdev; |
| 1199 | |
Christoph Hellwig | 56fef7c | 2015-10-05 09:31:09 +0200 | [diff] [blame] | 1200 | log->need_cache_flush = (rdev->bdev->bd_disk->queue->flush_flags != 0); |
| 1201 | |
Shaohua Li | 5cb2fbd | 2015-10-28 08:41:25 -0700 | [diff] [blame] | 1202 | log->uuid_checksum = crc32c_le(~0, rdev->mddev->uuid, |
| 1203 | sizeof(rdev->mddev->uuid)); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1204 | |
| 1205 | mutex_init(&log->io_mutex); |
| 1206 | |
| 1207 | spin_lock_init(&log->io_list_lock); |
| 1208 | INIT_LIST_HEAD(&log->running_ios); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 1209 | INIT_LIST_HEAD(&log->io_end_ios); |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 1210 | INIT_LIST_HEAD(&log->flushing_ios); |
Christoph Hellwig | 04732f7 | 2015-10-05 09:31:07 +0200 | [diff] [blame] | 1211 | INIT_LIST_HEAD(&log->finished_ios); |
Shaohua Li | a8c34f9 | 2015-09-02 13:49:46 -0700 | [diff] [blame] | 1212 | bio_init(&log->flush_bio); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1213 | |
| 1214 | log->io_kc = KMEM_CACHE(r5l_io_unit, 0); |
| 1215 | if (!log->io_kc) |
| 1216 | goto io_kc; |
| 1217 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1218 | log->io_pool = mempool_create_slab_pool(R5L_POOL_SIZE, log->io_kc); |
| 1219 | if (!log->io_pool) |
| 1220 | goto io_pool; |
| 1221 | |
Christoph Hellwig | c38d29b | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1222 | log->bs = bioset_create(R5L_POOL_SIZE, 0); |
| 1223 | if (!log->bs) |
| 1224 | goto io_bs; |
| 1225 | |
Christoph Hellwig | e8deb63 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1226 | log->meta_pool = mempool_create_page_pool(R5L_POOL_SIZE, 0); |
| 1227 | if (!log->meta_pool) |
| 1228 | goto out_mempool; |
| 1229 | |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 1230 | log->reclaim_thread = md_register_thread(r5l_reclaim_thread, |
| 1231 | log->rdev->mddev, "reclaim"); |
| 1232 | if (!log->reclaim_thread) |
| 1233 | goto reclaim_thread; |
Shaohua Li | 0fd22b4 | 2015-09-02 13:49:47 -0700 | [diff] [blame] | 1234 | init_waitqueue_head(&log->iounit_wait); |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 1235 | |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1236 | INIT_LIST_HEAD(&log->no_mem_stripes); |
| 1237 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1238 | INIT_LIST_HEAD(&log->no_space_stripes); |
| 1239 | spin_lock_init(&log->no_space_stripes_lock); |
| 1240 | |
| 1241 | if (r5l_load_log(log)) |
| 1242 | goto error; |
| 1243 | |
Shaohua Li | f6b6ec5 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1244 | rcu_assign_pointer(conf->log, log); |
Shaohua Li | a62ab49 | 2016-01-06 14:37:13 -0800 | [diff] [blame] | 1245 | set_bit(MD_HAS_JOURNAL, &conf->mddev->flags); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1246 | return 0; |
Christoph Hellwig | e8deb63 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1247 | |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1248 | error: |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 1249 | md_unregister_thread(&log->reclaim_thread); |
| 1250 | reclaim_thread: |
Christoph Hellwig | e8deb63 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1251 | mempool_destroy(log->meta_pool); |
| 1252 | out_mempool: |
Christoph Hellwig | c38d29b | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1253 | bioset_free(log->bs); |
| 1254 | io_bs: |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1255 | mempool_destroy(log->io_pool); |
| 1256 | io_pool: |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1257 | kmem_cache_destroy(log->io_kc); |
| 1258 | io_kc: |
| 1259 | kfree(log); |
| 1260 | return -EINVAL; |
| 1261 | } |
| 1262 | |
| 1263 | void r5l_exit_log(struct r5l_log *log) |
| 1264 | { |
Shaohua Li | 0576b1c | 2015-08-13 14:32:00 -0700 | [diff] [blame] | 1265 | md_unregister_thread(&log->reclaim_thread); |
Christoph Hellwig | e8deb63 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1266 | mempool_destroy(log->meta_pool); |
Christoph Hellwig | c38d29b | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1267 | bioset_free(log->bs); |
Christoph Hellwig | 5036c39 | 2015-12-21 10:51:02 +1100 | [diff] [blame] | 1268 | mempool_destroy(log->io_pool); |
Shaohua Li | f6bed0e | 2015-08-13 14:31:59 -0700 | [diff] [blame] | 1269 | kmem_cache_destroy(log->io_kc); |
| 1270 | kfree(log); |
| 1271 | } |