blob: 9e1569ffbf6ea66f1324022db9f9aba559339c73 [file] [log] [blame]
Arne Jansena2de7332011-03-08 14:14:00 +01001/*
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01002 * Copyright (C) 2011, 2012 STRATO. All rights reserved.
Arne Jansena2de7332011-03-08 14:14:00 +01003 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
Arne Jansena2de7332011-03-08 14:14:00 +010019#include <linux/blkdev.h>
Jan Schmidt558540c2011-06-13 19:59:12 +020020#include <linux/ratelimit.h>
Arne Jansena2de7332011-03-08 14:14:00 +010021#include "ctree.h"
22#include "volumes.h"
23#include "disk-io.h"
24#include "ordered-data.h"
Jan Schmidt0ef8e452011-06-13 20:04:15 +020025#include "transaction.h"
Jan Schmidt558540c2011-06-13 19:59:12 +020026#include "backref.h"
Jan Schmidt5da6fcb2011-08-04 18:11:04 +020027#include "extent_io.h"
Stefan Behrensff023aa2012-11-06 11:43:11 +010028#include "dev-replace.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010029#include "check-integrity.h"
Josef Bacik606686e2012-06-04 14:03:51 -040030#include "rcu-string.h"
David Woodhouse53b381b2013-01-29 18:40:14 -050031#include "raid56.h"
Arne Jansena2de7332011-03-08 14:14:00 +010032
33/*
34 * This is only the first step towards a full-features scrub. It reads all
35 * extent and super block and verifies the checksums. In case a bad checksum
36 * is found or the extent cannot be read, good data will be written back if
37 * any can be found.
38 *
39 * Future enhancements:
Arne Jansena2de7332011-03-08 14:14:00 +010040 * - In case an unrepairable extent is encountered, track which files are
41 * affected and report them
Arne Jansena2de7332011-03-08 14:14:00 +010042 * - track and record media errors, throw out bad devices
Arne Jansena2de7332011-03-08 14:14:00 +010043 * - add a mode to also read unallocated space
Arne Jansena2de7332011-03-08 14:14:00 +010044 */
45
Stefan Behrensb5d67f62012-03-27 14:21:27 -040046struct scrub_block;
Stefan Behrensd9d181c2012-11-02 09:58:09 +010047struct scrub_ctx;
Arne Jansena2de7332011-03-08 14:14:00 +010048
Stefan Behrensff023aa2012-11-06 11:43:11 +010049/*
50 * the following three values only influence the performance.
51 * The last one configures the number of parallel and outstanding I/O
52 * operations. The first two values configure an upper limit for the number
53 * of (dynamically allocated) pages that are added to a bio.
54 */
55#define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */
56#define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */
57#define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */
Stefan Behrens7a9e9982012-11-02 14:58:04 +010058
59/*
60 * the following value times PAGE_SIZE needs to be large enough to match the
61 * largest node/leaf/sector size that shall be supported.
62 * Values larger than BTRFS_STRIPE_LEN are not supported.
63 */
Stefan Behrensb5d67f62012-03-27 14:21:27 -040064#define SCRUB_MAX_PAGES_PER_BLOCK 16 /* 64k per node/leaf/sector */
Arne Jansena2de7332011-03-08 14:14:00 +010065
Miao Xieaf8e2d12014-10-23 14:42:50 +080066struct scrub_recover {
67 atomic_t refs;
68 struct btrfs_bio *bbio;
69 u64 *raid_map;
70 u64 map_length;
71};
72
Arne Jansena2de7332011-03-08 14:14:00 +010073struct scrub_page {
Stefan Behrensb5d67f62012-03-27 14:21:27 -040074 struct scrub_block *sblock;
75 struct page *page;
Stefan Behrens442a4f62012-05-25 16:06:08 +020076 struct btrfs_device *dev;
Miao Xie5a6ac9e2014-11-06 17:20:58 +080077 struct list_head list;
Arne Jansena2de7332011-03-08 14:14:00 +010078 u64 flags; /* extent flags */
79 u64 generation;
Stefan Behrensb5d67f62012-03-27 14:21:27 -040080 u64 logical;
81 u64 physical;
Stefan Behrensff023aa2012-11-06 11:43:11 +010082 u64 physical_for_dev_replace;
Stefan Behrens7a9e9982012-11-02 14:58:04 +010083 atomic_t ref_count;
Stefan Behrensb5d67f62012-03-27 14:21:27 -040084 struct {
85 unsigned int mirror_num:8;
86 unsigned int have_csum:1;
87 unsigned int io_error:1;
88 };
Arne Jansena2de7332011-03-08 14:14:00 +010089 u8 csum[BTRFS_CSUM_SIZE];
Miao Xieaf8e2d12014-10-23 14:42:50 +080090
91 struct scrub_recover *recover;
Arne Jansena2de7332011-03-08 14:14:00 +010092};
93
94struct scrub_bio {
95 int index;
Stefan Behrensd9d181c2012-11-02 09:58:09 +010096 struct scrub_ctx *sctx;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +010097 struct btrfs_device *dev;
Arne Jansena2de7332011-03-08 14:14:00 +010098 struct bio *bio;
99 int err;
100 u64 logical;
101 u64 physical;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100102#if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
103 struct scrub_page *pagev[SCRUB_PAGES_PER_WR_BIO];
104#else
105 struct scrub_page *pagev[SCRUB_PAGES_PER_RD_BIO];
106#endif
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400107 int page_count;
Arne Jansena2de7332011-03-08 14:14:00 +0100108 int next_free;
109 struct btrfs_work work;
110};
111
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400112struct scrub_block {
Stefan Behrens7a9e9982012-11-02 14:58:04 +0100113 struct scrub_page *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400114 int page_count;
115 atomic_t outstanding_pages;
116 atomic_t ref_count; /* free mem on transition to zero */
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100117 struct scrub_ctx *sctx;
Miao Xie5a6ac9e2014-11-06 17:20:58 +0800118 struct scrub_parity *sparity;
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400119 struct {
120 unsigned int header_error:1;
121 unsigned int checksum_error:1;
122 unsigned int no_io_error_seen:1;
Stefan Behrens442a4f62012-05-25 16:06:08 +0200123 unsigned int generation_error:1; /* also sets header_error */
Miao Xie5a6ac9e2014-11-06 17:20:58 +0800124
125 /* The following is for the data used to check parity */
126 /* It is for the data with checksum */
127 unsigned int data_corrected:1;
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400128 };
129};
130
Miao Xie5a6ac9e2014-11-06 17:20:58 +0800131/* Used for the chunks with parity stripe such RAID5/6 */
132struct scrub_parity {
133 struct scrub_ctx *sctx;
134
135 struct btrfs_device *scrub_dev;
136
137 u64 logic_start;
138
139 u64 logic_end;
140
141 int nsectors;
142
143 int stripe_len;
144
145 atomic_t ref_count;
146
147 struct list_head spages;
148
149 /* Work of parity check and repair */
150 struct btrfs_work work;
151
152 /* Mark the parity blocks which have data */
153 unsigned long *dbitmap;
154
155 /*
156 * Mark the parity blocks which have data, but errors happen when
157 * read data or check data
158 */
159 unsigned long *ebitmap;
160
161 unsigned long bitmap[0];
162};
163
Stefan Behrensff023aa2012-11-06 11:43:11 +0100164struct scrub_wr_ctx {
165 struct scrub_bio *wr_curr_bio;
166 struct btrfs_device *tgtdev;
167 int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
168 atomic_t flush_all_writes;
169 struct mutex wr_lock;
170};
171
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100172struct scrub_ctx {
Stefan Behrensff023aa2012-11-06 11:43:11 +0100173 struct scrub_bio *bios[SCRUB_BIOS_PER_SCTX];
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100174 struct btrfs_root *dev_root;
Arne Jansena2de7332011-03-08 14:14:00 +0100175 int first_free;
176 int curr;
Stefan Behrensb6bfebc2012-11-02 16:44:58 +0100177 atomic_t bios_in_flight;
178 atomic_t workers_pending;
Arne Jansena2de7332011-03-08 14:14:00 +0100179 spinlock_t list_lock;
180 wait_queue_head_t list_wait;
181 u16 csum_size;
182 struct list_head csum_list;
183 atomic_t cancel_req;
Arne Jansen86287642011-03-23 16:34:19 +0100184 int readonly;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100185 int pages_per_rd_bio;
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400186 u32 sectorsize;
187 u32 nodesize;
Stefan Behrens63a212a2012-11-05 18:29:28 +0100188
189 int is_dev_replace;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100190 struct scrub_wr_ctx wr_ctx;
Stefan Behrens63a212a2012-11-05 18:29:28 +0100191
Arne Jansena2de7332011-03-08 14:14:00 +0100192 /*
193 * statistics
194 */
195 struct btrfs_scrub_progress stat;
196 spinlock_t stat_lock;
197};
198
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200199struct scrub_fixup_nodatasum {
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100200 struct scrub_ctx *sctx;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100201 struct btrfs_device *dev;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200202 u64 logical;
203 struct btrfs_root *root;
204 struct btrfs_work work;
205 int mirror_num;
206};
207
Josef Bacik652f25a2013-09-12 16:58:28 -0400208struct scrub_nocow_inode {
209 u64 inum;
210 u64 offset;
211 u64 root;
212 struct list_head list;
213};
214
Stefan Behrensff023aa2012-11-06 11:43:11 +0100215struct scrub_copy_nocow_ctx {
216 struct scrub_ctx *sctx;
217 u64 logical;
218 u64 len;
219 int mirror_num;
220 u64 physical_for_dev_replace;
Josef Bacik652f25a2013-09-12 16:58:28 -0400221 struct list_head inodes;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100222 struct btrfs_work work;
223};
224
Jan Schmidt558540c2011-06-13 19:59:12 +0200225struct scrub_warning {
226 struct btrfs_path *path;
227 u64 extent_item_size;
Jan Schmidt558540c2011-06-13 19:59:12 +0200228 const char *errstr;
229 sector_t sector;
230 u64 logical;
231 struct btrfs_device *dev;
Jan Schmidt558540c2011-06-13 19:59:12 +0200232};
233
Stefan Behrensb6bfebc2012-11-02 16:44:58 +0100234static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
235static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
236static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
237static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400238static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100239static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
Stefan Behrens3ec706c2012-11-05 15:46:42 +0100240 struct btrfs_fs_info *fs_info,
Stefan Behrensff023aa2012-11-06 11:43:11 +0100241 struct scrub_block *original_sblock,
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400242 u64 length, u64 logical,
Stefan Behrensff023aa2012-11-06 11:43:11 +0100243 struct scrub_block *sblocks_for_recheck);
Stefan Behrens34f5c8e2012-11-02 16:16:26 +0100244static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
245 struct scrub_block *sblock, int is_metadata,
246 int have_csum, u8 *csum, u64 generation,
Miao Xieaf8e2d12014-10-23 14:42:50 +0800247 u16 csum_size, int retry_failed_mirror);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400248static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
249 struct scrub_block *sblock,
250 int is_metadata, int have_csum,
251 const u8 *csum, u64 generation,
252 u16 csum_size);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400253static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
254 struct scrub_block *sblock_good,
255 int force_write);
256static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
257 struct scrub_block *sblock_good,
258 int page_num, int force_write);
Stefan Behrensff023aa2012-11-06 11:43:11 +0100259static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
260static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
261 int page_num);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400262static int scrub_checksum_data(struct scrub_block *sblock);
263static int scrub_checksum_tree_block(struct scrub_block *sblock);
264static int scrub_checksum_super(struct scrub_block *sblock);
265static void scrub_block_get(struct scrub_block *sblock);
266static void scrub_block_put(struct scrub_block *sblock);
Stefan Behrens7a9e9982012-11-02 14:58:04 +0100267static void scrub_page_get(struct scrub_page *spage);
268static void scrub_page_put(struct scrub_page *spage);
Miao Xie5a6ac9e2014-11-06 17:20:58 +0800269static void scrub_parity_get(struct scrub_parity *sparity);
270static void scrub_parity_put(struct scrub_parity *sparity);
Stefan Behrensff023aa2012-11-06 11:43:11 +0100271static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
272 struct scrub_page *spage);
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100273static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100274 u64 physical, struct btrfs_device *dev, u64 flags,
Stefan Behrensff023aa2012-11-06 11:43:11 +0100275 u64 gen, int mirror_num, u8 *csum, int force,
276 u64 physical_for_dev_replace);
Stefan Behrens1623ede2012-03-27 14:21:26 -0400277static void scrub_bio_end_io(struct bio *bio, int err);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400278static void scrub_bio_end_io_worker(struct btrfs_work *work);
279static void scrub_block_complete(struct scrub_block *sblock);
Stefan Behrensff023aa2012-11-06 11:43:11 +0100280static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
281 u64 extent_logical, u64 extent_len,
282 u64 *extent_physical,
283 struct btrfs_device **extent_dev,
284 int *extent_mirror_num);
285static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
286 struct scrub_wr_ctx *wr_ctx,
287 struct btrfs_fs_info *fs_info,
288 struct btrfs_device *dev,
289 int is_dev_replace);
290static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
291static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
292 struct scrub_page *spage);
293static void scrub_wr_submit(struct scrub_ctx *sctx);
294static void scrub_wr_bio_end_io(struct bio *bio, int err);
295static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
296static int write_page_nocow(struct scrub_ctx *sctx,
297 u64 physical_for_dev_replace, struct page *page);
298static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
Josef Bacik652f25a2013-09-12 16:58:28 -0400299 struct scrub_copy_nocow_ctx *ctx);
Stefan Behrensff023aa2012-11-06 11:43:11 +0100300static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
301 int mirror_num, u64 physical_for_dev_replace);
302static void copy_nocow_pages_worker(struct btrfs_work *work);
Wang Shilongcb7ab022013-12-04 21:16:53 +0800303static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
Wang Shilong3cb09292013-12-04 21:15:19 +0800304static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
Stefan Behrens1623ede2012-03-27 14:21:26 -0400305
306
Stefan Behrensb6bfebc2012-11-02 16:44:58 +0100307static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
308{
309 atomic_inc(&sctx->bios_in_flight);
310}
311
312static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
313{
314 atomic_dec(&sctx->bios_in_flight);
315 wake_up(&sctx->list_wait);
316}
317
Wang Shilongcb7ab022013-12-04 21:16:53 +0800318static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
Wang Shilong3cb09292013-12-04 21:15:19 +0800319{
320 while (atomic_read(&fs_info->scrub_pause_req)) {
321 mutex_unlock(&fs_info->scrub_lock);
322 wait_event(fs_info->scrub_pause_wait,
323 atomic_read(&fs_info->scrub_pause_req) == 0);
324 mutex_lock(&fs_info->scrub_lock);
325 }
326}
327
Wang Shilongcb7ab022013-12-04 21:16:53 +0800328static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
329{
330 atomic_inc(&fs_info->scrubs_paused);
331 wake_up(&fs_info->scrub_pause_wait);
332
333 mutex_lock(&fs_info->scrub_lock);
334 __scrub_blocked_if_needed(fs_info);
335 atomic_dec(&fs_info->scrubs_paused);
336 mutex_unlock(&fs_info->scrub_lock);
337
338 wake_up(&fs_info->scrub_pause_wait);
339}
340
Stefan Behrensb6bfebc2012-11-02 16:44:58 +0100341/*
342 * used for workers that require transaction commits (i.e., for the
343 * NOCOW case)
344 */
345static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
346{
347 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
348
349 /*
350 * increment scrubs_running to prevent cancel requests from
351 * completing as long as a worker is running. we must also
352 * increment scrubs_paused to prevent deadlocking on pause
353 * requests used for transactions commits (as the worker uses a
354 * transaction context). it is safe to regard the worker
355 * as paused for all matters practical. effectively, we only
356 * avoid cancellation requests from completing.
357 */
358 mutex_lock(&fs_info->scrub_lock);
359 atomic_inc(&fs_info->scrubs_running);
360 atomic_inc(&fs_info->scrubs_paused);
361 mutex_unlock(&fs_info->scrub_lock);
Wang Shilong32a44782014-02-19 19:24:19 +0800362
363 /*
364 * check if @scrubs_running=@scrubs_paused condition
365 * inside wait_event() is not an atomic operation.
366 * which means we may inc/dec @scrub_running/paused
367 * at any time. Let's wake up @scrub_pause_wait as
368 * much as we can to let commit transaction blocked less.
369 */
370 wake_up(&fs_info->scrub_pause_wait);
371
Stefan Behrensb6bfebc2012-11-02 16:44:58 +0100372 atomic_inc(&sctx->workers_pending);
373}
374
375/* used for workers that require transaction commits */
376static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
377{
378 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
379
380 /*
381 * see scrub_pending_trans_workers_inc() why we're pretending
382 * to be paused in the scrub counters
383 */
384 mutex_lock(&fs_info->scrub_lock);
385 atomic_dec(&fs_info->scrubs_running);
386 atomic_dec(&fs_info->scrubs_paused);
387 mutex_unlock(&fs_info->scrub_lock);
388 atomic_dec(&sctx->workers_pending);
389 wake_up(&fs_info->scrub_pause_wait);
390 wake_up(&sctx->list_wait);
391}
392
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100393static void scrub_free_csums(struct scrub_ctx *sctx)
Arne Jansena2de7332011-03-08 14:14:00 +0100394{
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100395 while (!list_empty(&sctx->csum_list)) {
Arne Jansena2de7332011-03-08 14:14:00 +0100396 struct btrfs_ordered_sum *sum;
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100397 sum = list_first_entry(&sctx->csum_list,
Arne Jansena2de7332011-03-08 14:14:00 +0100398 struct btrfs_ordered_sum, list);
399 list_del(&sum->list);
400 kfree(sum);
401 }
402}
403
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100404static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
Arne Jansena2de7332011-03-08 14:14:00 +0100405{
406 int i;
Arne Jansena2de7332011-03-08 14:14:00 +0100407
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100408 if (!sctx)
Arne Jansena2de7332011-03-08 14:14:00 +0100409 return;
410
Stefan Behrensff023aa2012-11-06 11:43:11 +0100411 scrub_free_wr_ctx(&sctx->wr_ctx);
412
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400413 /* this can happen when scrub is cancelled */
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100414 if (sctx->curr != -1) {
415 struct scrub_bio *sbio = sctx->bios[sctx->curr];
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400416
417 for (i = 0; i < sbio->page_count; i++) {
Stefan Behrensff023aa2012-11-06 11:43:11 +0100418 WARN_ON(!sbio->pagev[i]->page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400419 scrub_block_put(sbio->pagev[i]->sblock);
420 }
421 bio_put(sbio->bio);
422 }
423
Stefan Behrensff023aa2012-11-06 11:43:11 +0100424 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100425 struct scrub_bio *sbio = sctx->bios[i];
Arne Jansena2de7332011-03-08 14:14:00 +0100426
427 if (!sbio)
428 break;
Arne Jansena2de7332011-03-08 14:14:00 +0100429 kfree(sbio);
430 }
431
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100432 scrub_free_csums(sctx);
433 kfree(sctx);
Arne Jansena2de7332011-03-08 14:14:00 +0100434}
435
436static noinline_for_stack
Stefan Behrens63a212a2012-11-05 18:29:28 +0100437struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
Arne Jansena2de7332011-03-08 14:14:00 +0100438{
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100439 struct scrub_ctx *sctx;
Arne Jansena2de7332011-03-08 14:14:00 +0100440 int i;
Arne Jansena2de7332011-03-08 14:14:00 +0100441 struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100442 int pages_per_rd_bio;
443 int ret;
Arne Jansena2de7332011-03-08 14:14:00 +0100444
Stefan Behrensff023aa2012-11-06 11:43:11 +0100445 /*
446 * the setting of pages_per_rd_bio is correct for scrub but might
447 * be wrong for the dev_replace code where we might read from
448 * different devices in the initial huge bios. However, that
449 * code is able to correctly handle the case when adding a page
450 * to a bio fails.
451 */
452 if (dev->bdev)
453 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
454 bio_get_nr_vecs(dev->bdev));
455 else
456 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100457 sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
458 if (!sctx)
Arne Jansena2de7332011-03-08 14:14:00 +0100459 goto nomem;
Stefan Behrens63a212a2012-11-05 18:29:28 +0100460 sctx->is_dev_replace = is_dev_replace;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100461 sctx->pages_per_rd_bio = pages_per_rd_bio;
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100462 sctx->curr = -1;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100463 sctx->dev_root = dev->dev_root;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100464 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
Arne Jansena2de7332011-03-08 14:14:00 +0100465 struct scrub_bio *sbio;
466
467 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
468 if (!sbio)
469 goto nomem;
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100470 sctx->bios[i] = sbio;
Arne Jansena2de7332011-03-08 14:14:00 +0100471
Arne Jansena2de7332011-03-08 14:14:00 +0100472 sbio->index = i;
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100473 sbio->sctx = sctx;
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400474 sbio->page_count = 0;
Liu Bo9e0af232014-08-15 23:36:53 +0800475 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
476 scrub_bio_end_io_worker, NULL, NULL);
Arne Jansena2de7332011-03-08 14:14:00 +0100477
Stefan Behrensff023aa2012-11-06 11:43:11 +0100478 if (i != SCRUB_BIOS_PER_SCTX - 1)
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100479 sctx->bios[i]->next_free = i + 1;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200480 else
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100481 sctx->bios[i]->next_free = -1;
Arne Jansena2de7332011-03-08 14:14:00 +0100482 }
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100483 sctx->first_free = 0;
484 sctx->nodesize = dev->dev_root->nodesize;
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100485 sctx->sectorsize = dev->dev_root->sectorsize;
Stefan Behrensb6bfebc2012-11-02 16:44:58 +0100486 atomic_set(&sctx->bios_in_flight, 0);
487 atomic_set(&sctx->workers_pending, 0);
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100488 atomic_set(&sctx->cancel_req, 0);
489 sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
490 INIT_LIST_HEAD(&sctx->csum_list);
Arne Jansena2de7332011-03-08 14:14:00 +0100491
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100492 spin_lock_init(&sctx->list_lock);
493 spin_lock_init(&sctx->stat_lock);
494 init_waitqueue_head(&sctx->list_wait);
Stefan Behrensff023aa2012-11-06 11:43:11 +0100495
496 ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
497 fs_info->dev_replace.tgtdev, is_dev_replace);
498 if (ret) {
499 scrub_free_ctx(sctx);
500 return ERR_PTR(ret);
501 }
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100502 return sctx;
Arne Jansena2de7332011-03-08 14:14:00 +0100503
504nomem:
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100505 scrub_free_ctx(sctx);
Arne Jansena2de7332011-03-08 14:14:00 +0100506 return ERR_PTR(-ENOMEM);
507}
508
Stefan Behrensff023aa2012-11-06 11:43:11 +0100509static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
510 void *warn_ctx)
Jan Schmidt558540c2011-06-13 19:59:12 +0200511{
512 u64 isize;
513 u32 nlink;
514 int ret;
515 int i;
516 struct extent_buffer *eb;
517 struct btrfs_inode_item *inode_item;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100518 struct scrub_warning *swarn = warn_ctx;
Jan Schmidt558540c2011-06-13 19:59:12 +0200519 struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
520 struct inode_fs_paths *ipath = NULL;
521 struct btrfs_root *local_root;
522 struct btrfs_key root_key;
523
524 root_key.objectid = root;
525 root_key.type = BTRFS_ROOT_ITEM_KEY;
526 root_key.offset = (u64)-1;
527 local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
528 if (IS_ERR(local_root)) {
529 ret = PTR_ERR(local_root);
530 goto err;
531 }
532
533 ret = inode_item_info(inum, 0, local_root, swarn->path);
534 if (ret) {
535 btrfs_release_path(swarn->path);
536 goto err;
537 }
538
539 eb = swarn->path->nodes[0];
540 inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
541 struct btrfs_inode_item);
542 isize = btrfs_inode_size(eb, inode_item);
543 nlink = btrfs_inode_nlink(eb, inode_item);
544 btrfs_release_path(swarn->path);
545
546 ipath = init_ipath(4096, local_root, swarn->path);
Dan Carpenter26bdef52011-11-16 11:28:01 +0300547 if (IS_ERR(ipath)) {
548 ret = PTR_ERR(ipath);
549 ipath = NULL;
550 goto err;
551 }
Jan Schmidt558540c2011-06-13 19:59:12 +0200552 ret = paths_from_inode(inum, ipath);
553
554 if (ret < 0)
555 goto err;
556
557 /*
558 * we deliberately ignore the bit ipath might have been too small to
559 * hold all of the paths here
560 */
561 for (i = 0; i < ipath->fspath->elem_cnt; ++i)
Frank Holtonefe120a2013-12-20 11:37:06 -0500562 printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
Jan Schmidt558540c2011-06-13 19:59:12 +0200563 "%s, sector %llu, root %llu, inode %llu, offset %llu, "
564 "length %llu, links %u (path: %s)\n", swarn->errstr,
Josef Bacik606686e2012-06-04 14:03:51 -0400565 swarn->logical, rcu_str_deref(swarn->dev->name),
Jan Schmidt558540c2011-06-13 19:59:12 +0200566 (unsigned long long)swarn->sector, root, inum, offset,
567 min(isize - offset, (u64)PAGE_SIZE), nlink,
Jeff Mahoney745c4d82011-11-20 07:31:57 -0500568 (char *)(unsigned long)ipath->fspath->val[i]);
Jan Schmidt558540c2011-06-13 19:59:12 +0200569
570 free_ipath(ipath);
571 return 0;
572
573err:
Frank Holtonefe120a2013-12-20 11:37:06 -0500574 printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
Jan Schmidt558540c2011-06-13 19:59:12 +0200575 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
576 "resolving failed with ret=%d\n", swarn->errstr,
Josef Bacik606686e2012-06-04 14:03:51 -0400577 swarn->logical, rcu_str_deref(swarn->dev->name),
Jan Schmidt558540c2011-06-13 19:59:12 +0200578 (unsigned long long)swarn->sector, root, inum, offset, ret);
579
580 free_ipath(ipath);
581 return 0;
582}
583
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400584static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
Jan Schmidt558540c2011-06-13 19:59:12 +0200585{
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100586 struct btrfs_device *dev;
587 struct btrfs_fs_info *fs_info;
Jan Schmidt558540c2011-06-13 19:59:12 +0200588 struct btrfs_path *path;
589 struct btrfs_key found_key;
590 struct extent_buffer *eb;
591 struct btrfs_extent_item *ei;
592 struct scrub_warning swarn;
Jan Schmidt558540c2011-06-13 19:59:12 +0200593 unsigned long ptr = 0;
Jan Schmidt4692cf52011-12-02 14:56:41 +0100594 u64 extent_item_pos;
Liu Bo69917e42012-09-07 20:01:28 -0600595 u64 flags = 0;
596 u64 ref_root;
597 u32 item_size;
598 u8 ref_level;
Liu Bo69917e42012-09-07 20:01:28 -0600599 int ret;
Jan Schmidt558540c2011-06-13 19:59:12 +0200600
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100601 WARN_ON(sblock->page_count < 1);
Stefan Behrens7a9e9982012-11-02 14:58:04 +0100602 dev = sblock->pagev[0]->dev;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100603 fs_info = sblock->sctx->dev_root->fs_info;
604
Jan Schmidt558540c2011-06-13 19:59:12 +0200605 path = btrfs_alloc_path();
David Sterba8b9456d2014-07-30 01:25:30 +0200606 if (!path)
607 return;
Jan Schmidt558540c2011-06-13 19:59:12 +0200608
Stefan Behrens7a9e9982012-11-02 14:58:04 +0100609 swarn.sector = (sblock->pagev[0]->physical) >> 9;
610 swarn.logical = sblock->pagev[0]->logical;
Jan Schmidt558540c2011-06-13 19:59:12 +0200611 swarn.errstr = errstr;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100612 swarn.dev = NULL;
Jan Schmidt558540c2011-06-13 19:59:12 +0200613
Liu Bo69917e42012-09-07 20:01:28 -0600614 ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
615 &flags);
Jan Schmidt558540c2011-06-13 19:59:12 +0200616 if (ret < 0)
617 goto out;
618
Jan Schmidt4692cf52011-12-02 14:56:41 +0100619 extent_item_pos = swarn.logical - found_key.objectid;
Jan Schmidt558540c2011-06-13 19:59:12 +0200620 swarn.extent_item_size = found_key.offset;
621
622 eb = path->nodes[0];
623 ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
624 item_size = btrfs_item_size_nr(eb, path->slots[0]);
625
Liu Bo69917e42012-09-07 20:01:28 -0600626 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
Jan Schmidt558540c2011-06-13 19:59:12 +0200627 do {
Liu Bo6eda71d2014-06-09 10:54:07 +0800628 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
629 item_size, &ref_root,
630 &ref_level);
Josef Bacik606686e2012-06-04 14:03:51 -0400631 printk_in_rcu(KERN_WARNING
Frank Holtonefe120a2013-12-20 11:37:06 -0500632 "BTRFS: %s at logical %llu on dev %s, "
Jan Schmidt558540c2011-06-13 19:59:12 +0200633 "sector %llu: metadata %s (level %d) in tree "
Josef Bacik606686e2012-06-04 14:03:51 -0400634 "%llu\n", errstr, swarn.logical,
635 rcu_str_deref(dev->name),
Jan Schmidt558540c2011-06-13 19:59:12 +0200636 (unsigned long long)swarn.sector,
637 ref_level ? "node" : "leaf",
638 ret < 0 ? -1 : ref_level,
639 ret < 0 ? -1 : ref_root);
640 } while (ret != 1);
Josef Bacikd8fe29e2013-03-29 08:09:34 -0600641 btrfs_release_path(path);
Jan Schmidt558540c2011-06-13 19:59:12 +0200642 } else {
Josef Bacikd8fe29e2013-03-29 08:09:34 -0600643 btrfs_release_path(path);
Jan Schmidt558540c2011-06-13 19:59:12 +0200644 swarn.path = path;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100645 swarn.dev = dev;
Jan Schmidt7a3ae2f2012-03-23 17:32:28 +0100646 iterate_extent_inodes(fs_info, found_key.objectid,
647 extent_item_pos, 1,
Jan Schmidt558540c2011-06-13 19:59:12 +0200648 scrub_print_warning_inode, &swarn);
649 }
650
651out:
652 btrfs_free_path(path);
Jan Schmidt558540c2011-06-13 19:59:12 +0200653}
654
Stefan Behrensff023aa2012-11-06 11:43:11 +0100655static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200656{
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200657 struct page *page = NULL;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200658 unsigned long index;
Stefan Behrensff023aa2012-11-06 11:43:11 +0100659 struct scrub_fixup_nodatasum *fixup = fixup_ctx;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200660 int ret;
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200661 int corrected = 0;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200662 struct btrfs_key key;
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200663 struct inode *inode = NULL;
Liu Bo6f1c3602013-01-29 03:22:10 +0000664 struct btrfs_fs_info *fs_info;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200665 u64 end = offset + PAGE_SIZE - 1;
666 struct btrfs_root *local_root;
Liu Bo6f1c3602013-01-29 03:22:10 +0000667 int srcu_index;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200668
669 key.objectid = root;
670 key.type = BTRFS_ROOT_ITEM_KEY;
671 key.offset = (u64)-1;
Liu Bo6f1c3602013-01-29 03:22:10 +0000672
673 fs_info = fixup->root->fs_info;
674 srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
675
676 local_root = btrfs_read_fs_root_no_name(fs_info, &key);
677 if (IS_ERR(local_root)) {
678 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200679 return PTR_ERR(local_root);
Liu Bo6f1c3602013-01-29 03:22:10 +0000680 }
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200681
682 key.type = BTRFS_INODE_ITEM_KEY;
683 key.objectid = inum;
684 key.offset = 0;
Liu Bo6f1c3602013-01-29 03:22:10 +0000685 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
686 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200687 if (IS_ERR(inode))
688 return PTR_ERR(inode);
689
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200690 index = offset >> PAGE_CACHE_SHIFT;
691
692 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200693 if (!page) {
694 ret = -ENOMEM;
695 goto out;
696 }
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200697
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200698 if (PageUptodate(page)) {
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200699 if (PageDirty(page)) {
700 /*
701 * we need to write the data to the defect sector. the
702 * data that was in that sector is not in memory,
703 * because the page was modified. we must not write the
704 * modified page to that sector.
705 *
706 * TODO: what could be done here: wait for the delalloc
707 * runner to write out that page (might involve
708 * COW) and see whether the sector is still
709 * referenced afterwards.
710 *
711 * For the meantime, we'll treat this error
712 * incorrectable, although there is a chance that a
713 * later scrub will find the bad sector again and that
714 * there's no dirty page in memory, then.
715 */
716 ret = -EIO;
717 goto out;
718 }
Miao Xie1203b682014-09-12 18:44:01 +0800719 ret = repair_io_failure(inode, offset, PAGE_SIZE,
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200720 fixup->logical, page,
Miao Xieffdd2012014-09-12 18:44:00 +0800721 offset - page_offset(page),
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200722 fixup->mirror_num);
723 unlock_page(page);
724 corrected = !ret;
725 } else {
726 /*
727 * we need to get good data first. the general readpage path
728 * will call repair_io_failure for us, we just have to make
729 * sure we read the bad mirror.
730 */
731 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
732 EXTENT_DAMAGED, GFP_NOFS);
733 if (ret) {
734 /* set_extent_bits should give proper error */
735 WARN_ON(ret > 0);
736 if (ret > 0)
737 ret = -EFAULT;
738 goto out;
739 }
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200740
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200741 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
742 btrfs_get_extent,
743 fixup->mirror_num);
744 wait_on_page_locked(page);
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200745
Jan Schmidt5da6fcb2011-08-04 18:11:04 +0200746 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
747 end, EXTENT_DAMAGED, 0, NULL);
748 if (!corrected)
749 clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
750 EXTENT_DAMAGED, GFP_NOFS);
751 }
752
753out:
754 if (page)
755 put_page(page);
Tobias Klauser7fb18a02014-04-25 14:58:05 +0200756
757 iput(inode);
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200758
759 if (ret < 0)
760 return ret;
761
762 if (ret == 0 && corrected) {
763 /*
764 * we only need to call readpage for one of the inodes belonging
765 * to this extent. so make iterate_extent_inodes stop
766 */
767 return 1;
768 }
769
770 return -EIO;
771}
772
773static void scrub_fixup_nodatasum(struct btrfs_work *work)
774{
775 int ret;
776 struct scrub_fixup_nodatasum *fixup;
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100777 struct scrub_ctx *sctx;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200778 struct btrfs_trans_handle *trans = NULL;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200779 struct btrfs_path *path;
780 int uncorrectable = 0;
781
782 fixup = container_of(work, struct scrub_fixup_nodatasum, work);
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100783 sctx = fixup->sctx;
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200784
785 path = btrfs_alloc_path();
786 if (!path) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100787 spin_lock(&sctx->stat_lock);
788 ++sctx->stat.malloc_errors;
789 spin_unlock(&sctx->stat_lock);
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200790 uncorrectable = 1;
791 goto out;
792 }
793
794 trans = btrfs_join_transaction(fixup->root);
795 if (IS_ERR(trans)) {
796 uncorrectable = 1;
797 goto out;
798 }
799
800 /*
801 * the idea is to trigger a regular read through the standard path. we
802 * read a page from the (failed) logical address by specifying the
803 * corresponding copynum of the failed sector. thus, that readpage is
804 * expected to fail.
805 * that is the point where on-the-fly error correction will kick in
806 * (once it's finished) and rewrite the failed sector if a good copy
807 * can be found.
808 */
809 ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
810 path, scrub_fixup_readpage,
811 fixup);
812 if (ret < 0) {
813 uncorrectable = 1;
814 goto out;
815 }
816 WARN_ON(ret != 1);
817
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100818 spin_lock(&sctx->stat_lock);
819 ++sctx->stat.corrected_errors;
820 spin_unlock(&sctx->stat_lock);
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200821
822out:
823 if (trans && !IS_ERR(trans))
824 btrfs_end_transaction(trans, fixup->root);
825 if (uncorrectable) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100826 spin_lock(&sctx->stat_lock);
827 ++sctx->stat.uncorrectable_errors;
828 spin_unlock(&sctx->stat_lock);
Stefan Behrensff023aa2012-11-06 11:43:11 +0100829 btrfs_dev_replace_stats_inc(
830 &sctx->dev_root->fs_info->dev_replace.
831 num_uncorrectable_read_errors);
Frank Holtonefe120a2013-12-20 11:37:06 -0500832 printk_ratelimited_in_rcu(KERN_ERR "BTRFS: "
833 "unable to fixup (nodatasum) error at logical %llu on dev %s\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200834 fixup->logical, rcu_str_deref(fixup->dev->name));
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200835 }
836
837 btrfs_free_path(path);
838 kfree(fixup);
839
Stefan Behrensb6bfebc2012-11-02 16:44:58 +0100840 scrub_pending_trans_workers_dec(sctx);
Jan Schmidt0ef8e452011-06-13 20:04:15 +0200841}
842
Miao Xieaf8e2d12014-10-23 14:42:50 +0800843static inline void scrub_get_recover(struct scrub_recover *recover)
844{
845 atomic_inc(&recover->refs);
846}
847
848static inline void scrub_put_recover(struct scrub_recover *recover)
849{
850 if (atomic_dec_and_test(&recover->refs)) {
851 kfree(recover->bbio);
852 kfree(recover->raid_map);
853 kfree(recover);
854 }
855}
856
Arne Jansena2de7332011-03-08 14:14:00 +0100857/*
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400858 * scrub_handle_errored_block gets called when either verification of the
859 * pages failed or the bio failed to read, e.g. with EIO. In the latter
860 * case, this function handles all pages in the bio, even though only one
861 * may be bad.
862 * The goal of this function is to repair the errored block by using the
863 * contents of one of the mirrors.
Arne Jansena2de7332011-03-08 14:14:00 +0100864 */
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400865static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
Arne Jansena2de7332011-03-08 14:14:00 +0100866{
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100867 struct scrub_ctx *sctx = sblock_to_check->sctx;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100868 struct btrfs_device *dev;
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400869 struct btrfs_fs_info *fs_info;
Arne Jansena2de7332011-03-08 14:14:00 +0100870 u64 length;
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400871 u64 logical;
872 u64 generation;
873 unsigned int failed_mirror_index;
874 unsigned int is_metadata;
875 unsigned int have_csum;
876 u8 *csum;
877 struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
878 struct scrub_block *sblock_bad;
Arne Jansena2de7332011-03-08 14:14:00 +0100879 int ret;
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400880 int mirror_index;
881 int page_num;
882 int success;
883 static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
884 DEFAULT_RATELIMIT_BURST);
Arne Jansena2de7332011-03-08 14:14:00 +0100885
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400886 BUG_ON(sblock_to_check->page_count < 1);
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100887 fs_info = sctx->dev_root->fs_info;
Stefan Behrens4ded4f62012-11-14 18:57:29 +0000888 if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
889 /*
890 * if we find an error in a super block, we just report it.
891 * They will get written with the next transaction commit
892 * anyway
893 */
894 spin_lock(&sctx->stat_lock);
895 ++sctx->stat.super_errors;
896 spin_unlock(&sctx->stat_lock);
897 return 0;
898 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400899 length = sblock_to_check->page_count * PAGE_SIZE;
Stefan Behrens7a9e9982012-11-02 14:58:04 +0100900 logical = sblock_to_check->pagev[0]->logical;
901 generation = sblock_to_check->pagev[0]->generation;
902 BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
903 failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
904 is_metadata = !(sblock_to_check->pagev[0]->flags &
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400905 BTRFS_EXTENT_FLAG_DATA);
Stefan Behrens7a9e9982012-11-02 14:58:04 +0100906 have_csum = sblock_to_check->pagev[0]->have_csum;
907 csum = sblock_to_check->pagev[0]->csum;
908 dev = sblock_to_check->pagev[0]->dev;
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400909
Stefan Behrensff023aa2012-11-06 11:43:11 +0100910 if (sctx->is_dev_replace && !is_metadata && !have_csum) {
911 sblocks_for_recheck = NULL;
912 goto nodatasum_case;
913 }
914
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400915 /*
916 * read all mirrors one after the other. This includes to
917 * re-read the extent or metadata block that failed (that was
918 * the cause that this fixup code is called) another time,
919 * page by page this time in order to know which pages
920 * caused I/O errors and which ones are good (for all mirrors).
921 * It is the goal to handle the situation when more than one
922 * mirror contains I/O errors, but the errors do not
923 * overlap, i.e. the data can be repaired by selecting the
924 * pages from those mirrors without I/O error on the
925 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
926 * would be that mirror #1 has an I/O error on the first page,
927 * the second page is good, and mirror #2 has an I/O error on
928 * the second page, but the first page is good.
929 * Then the first page of the first mirror can be repaired by
930 * taking the first page of the second mirror, and the
931 * second page of the second mirror can be repaired by
932 * copying the contents of the 2nd page of the 1st mirror.
933 * One more note: if the pages of one mirror contain I/O
934 * errors, the checksum cannot be verified. In order to get
935 * the best data for repairing, the first attempt is to find
936 * a mirror without I/O errors and with a validated checksum.
937 * Only if this is not possible, the pages are picked from
938 * mirrors with I/O errors without considering the checksum.
939 * If the latter is the case, at the end, the checksum of the
940 * repaired area is verified in order to correctly maintain
941 * the statistics.
942 */
943
944 sblocks_for_recheck = kzalloc(BTRFS_MAX_MIRRORS *
945 sizeof(*sblocks_for_recheck),
946 GFP_NOFS);
947 if (!sblocks_for_recheck) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100948 spin_lock(&sctx->stat_lock);
949 sctx->stat.malloc_errors++;
950 sctx->stat.read_errors++;
951 sctx->stat.uncorrectable_errors++;
952 spin_unlock(&sctx->stat_lock);
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100953 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400954 goto out;
955 }
956
957 /* setup the context, map the logical blocks and alloc the pages */
Stefan Behrensff023aa2012-11-06 11:43:11 +0100958 ret = scrub_setup_recheck_block(sctx, fs_info, sblock_to_check, length,
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400959 logical, sblocks_for_recheck);
960 if (ret) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100961 spin_lock(&sctx->stat_lock);
962 sctx->stat.read_errors++;
963 sctx->stat.uncorrectable_errors++;
964 spin_unlock(&sctx->stat_lock);
Stefan Behrensa36cf8b2012-11-02 13:26:57 +0100965 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400966 goto out;
967 }
968 BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
969 sblock_bad = sblocks_for_recheck + failed_mirror_index;
970
971 /* build and submit the bios for the failed mirror, check checksums */
Stefan Behrens34f5c8e2012-11-02 16:16:26 +0100972 scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
Miao Xieaf8e2d12014-10-23 14:42:50 +0800973 csum, generation, sctx->csum_size, 1);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400974
975 if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
976 sblock_bad->no_io_error_seen) {
977 /*
978 * the error disappeared after reading page by page, or
979 * the area was part of a huge bio and other parts of the
980 * bio caused I/O errors, or the block layer merged several
981 * read requests into one and the error is caused by a
982 * different bio (usually one of the two latter cases is
983 * the cause)
984 */
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100985 spin_lock(&sctx->stat_lock);
986 sctx->stat.unverified_errors++;
Miao Xie5a6ac9e2014-11-06 17:20:58 +0800987 sblock_to_check->data_corrected = 1;
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100988 spin_unlock(&sctx->stat_lock);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400989
Stefan Behrensff023aa2012-11-06 11:43:11 +0100990 if (sctx->is_dev_replace)
991 scrub_write_block_to_dev_replace(sblock_bad);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400992 goto out;
993 }
994
995 if (!sblock_bad->no_io_error_seen) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +0100996 spin_lock(&sctx->stat_lock);
997 sctx->stat.read_errors++;
998 spin_unlock(&sctx->stat_lock);
Stefan Behrensb5d67f62012-03-27 14:21:27 -0400999 if (__ratelimit(&_rs))
1000 scrub_print_warning("i/o error", sblock_to_check);
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01001001 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001002 } else if (sblock_bad->checksum_error) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001003 spin_lock(&sctx->stat_lock);
1004 sctx->stat.csum_errors++;
1005 spin_unlock(&sctx->stat_lock);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001006 if (__ratelimit(&_rs))
1007 scrub_print_warning("checksum error", sblock_to_check);
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01001008 btrfs_dev_stat_inc_and_print(dev,
Stefan Behrens442a4f62012-05-25 16:06:08 +02001009 BTRFS_DEV_STAT_CORRUPTION_ERRS);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001010 } else if (sblock_bad->header_error) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001011 spin_lock(&sctx->stat_lock);
1012 sctx->stat.verify_errors++;
1013 spin_unlock(&sctx->stat_lock);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001014 if (__ratelimit(&_rs))
1015 scrub_print_warning("checksum/header error",
1016 sblock_to_check);
Stefan Behrens442a4f62012-05-25 16:06:08 +02001017 if (sblock_bad->generation_error)
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01001018 btrfs_dev_stat_inc_and_print(dev,
Stefan Behrens442a4f62012-05-25 16:06:08 +02001019 BTRFS_DEV_STAT_GENERATION_ERRS);
1020 else
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01001021 btrfs_dev_stat_inc_and_print(dev,
Stefan Behrens442a4f62012-05-25 16:06:08 +02001022 BTRFS_DEV_STAT_CORRUPTION_ERRS);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001023 }
1024
Ilya Dryomov33ef30a2013-11-03 19:06:38 +02001025 if (sctx->readonly) {
1026 ASSERT(!sctx->is_dev_replace);
1027 goto out;
1028 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001029
1030 if (!is_metadata && !have_csum) {
1031 struct scrub_fixup_nodatasum *fixup_nodatasum;
1032
Stefan Behrensff023aa2012-11-06 11:43:11 +01001033nodatasum_case:
1034 WARN_ON(sctx->is_dev_replace);
1035
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001036 /*
1037 * !is_metadata and !have_csum, this means that the data
1038 * might not be COW'ed, that it might be modified
1039 * concurrently. The general strategy to work on the
1040 * commit root does not help in the case when COW is not
1041 * used.
1042 */
1043 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
1044 if (!fixup_nodatasum)
1045 goto did_not_correct_error;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001046 fixup_nodatasum->sctx = sctx;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01001047 fixup_nodatasum->dev = dev;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001048 fixup_nodatasum->logical = logical;
1049 fixup_nodatasum->root = fs_info->extent_root;
1050 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01001051 scrub_pending_trans_workers_inc(sctx);
Liu Bo9e0af232014-08-15 23:36:53 +08001052 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1053 scrub_fixup_nodatasum, NULL, NULL);
Qu Wenruo0339ef22014-02-28 10:46:17 +08001054 btrfs_queue_work(fs_info->scrub_workers,
1055 &fixup_nodatasum->work);
Arne Jansena2de7332011-03-08 14:14:00 +01001056 goto out;
1057 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001058
1059 /*
1060 * now build and submit the bios for the other mirrors, check
Stefan Behrenscb2ced72012-11-02 16:14:21 +01001061 * checksums.
1062 * First try to pick the mirror which is completely without I/O
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001063 * errors and also does not have a checksum error.
1064 * If one is found, and if a checksum is present, the full block
1065 * that is known to contain an error is rewritten. Afterwards
1066 * the block is known to be corrected.
1067 * If a mirror is found which is completely correct, and no
1068 * checksum is present, only those pages are rewritten that had
1069 * an I/O error in the block to be repaired, since it cannot be
1070 * determined, which copy of the other pages is better (and it
1071 * could happen otherwise that a correct page would be
1072 * overwritten by a bad one).
1073 */
1074 for (mirror_index = 0;
1075 mirror_index < BTRFS_MAX_MIRRORS &&
1076 sblocks_for_recheck[mirror_index].page_count > 0;
1077 mirror_index++) {
Stefan Behrenscb2ced72012-11-02 16:14:21 +01001078 struct scrub_block *sblock_other;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001079
Stefan Behrenscb2ced72012-11-02 16:14:21 +01001080 if (mirror_index == failed_mirror_index)
1081 continue;
1082 sblock_other = sblocks_for_recheck + mirror_index;
1083
1084 /* build and submit the bios, check checksums */
Stefan Behrens34f5c8e2012-11-02 16:16:26 +01001085 scrub_recheck_block(fs_info, sblock_other, is_metadata,
1086 have_csum, csum, generation,
Miao Xieaf8e2d12014-10-23 14:42:50 +08001087 sctx->csum_size, 0);
Stefan Behrens34f5c8e2012-11-02 16:16:26 +01001088
1089 if (!sblock_other->header_error &&
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001090 !sblock_other->checksum_error &&
1091 sblock_other->no_io_error_seen) {
Stefan Behrensff023aa2012-11-06 11:43:11 +01001092 if (sctx->is_dev_replace) {
1093 scrub_write_block_to_dev_replace(sblock_other);
1094 } else {
1095 int force_write = is_metadata || have_csum;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001096
Stefan Behrensff023aa2012-11-06 11:43:11 +01001097 ret = scrub_repair_block_from_good_copy(
1098 sblock_bad, sblock_other,
1099 force_write);
1100 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001101 if (0 == ret)
1102 goto corrected_error;
Arne Jansena2de7332011-03-08 14:14:00 +01001103 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001104 }
1105
1106 /*
Stefan Behrensff023aa2012-11-06 11:43:11 +01001107 * for dev_replace, pick good pages and write to the target device.
1108 */
1109 if (sctx->is_dev_replace) {
1110 success = 1;
1111 for (page_num = 0; page_num < sblock_bad->page_count;
1112 page_num++) {
1113 int sub_success;
1114
1115 sub_success = 0;
1116 for (mirror_index = 0;
1117 mirror_index < BTRFS_MAX_MIRRORS &&
1118 sblocks_for_recheck[mirror_index].page_count > 0;
1119 mirror_index++) {
1120 struct scrub_block *sblock_other =
1121 sblocks_for_recheck + mirror_index;
1122 struct scrub_page *page_other =
1123 sblock_other->pagev[page_num];
1124
1125 if (!page_other->io_error) {
1126 ret = scrub_write_page_to_dev_replace(
1127 sblock_other, page_num);
1128 if (ret == 0) {
1129 /* succeeded for this page */
1130 sub_success = 1;
1131 break;
1132 } else {
1133 btrfs_dev_replace_stats_inc(
1134 &sctx->dev_root->
1135 fs_info->dev_replace.
1136 num_write_errors);
1137 }
1138 }
1139 }
1140
1141 if (!sub_success) {
1142 /*
1143 * did not find a mirror to fetch the page
1144 * from. scrub_write_page_to_dev_replace()
1145 * handles this case (page->io_error), by
1146 * filling the block with zeros before
1147 * submitting the write request
1148 */
1149 success = 0;
1150 ret = scrub_write_page_to_dev_replace(
1151 sblock_bad, page_num);
1152 if (ret)
1153 btrfs_dev_replace_stats_inc(
1154 &sctx->dev_root->fs_info->
1155 dev_replace.num_write_errors);
1156 }
1157 }
1158
1159 goto out;
1160 }
1161
1162 /*
1163 * for regular scrub, repair those pages that are errored.
1164 * In case of I/O errors in the area that is supposed to be
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001165 * repaired, continue by picking good copies of those pages.
1166 * Select the good pages from mirrors to rewrite bad pages from
1167 * the area to fix. Afterwards verify the checksum of the block
1168 * that is supposed to be repaired. This verification step is
1169 * only done for the purpose of statistic counting and for the
1170 * final scrub report, whether errors remain.
1171 * A perfect algorithm could make use of the checksum and try
1172 * all possible combinations of pages from the different mirrors
1173 * until the checksum verification succeeds. For example, when
1174 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1175 * of mirror #2 is readable but the final checksum test fails,
1176 * then the 2nd page of mirror #3 could be tried, whether now
1177 * the final checksum succeedes. But this would be a rare
1178 * exception and is therefore not implemented. At least it is
1179 * avoided that the good copy is overwritten.
1180 * A more useful improvement would be to pick the sectors
1181 * without I/O error based on sector sizes (512 bytes on legacy
1182 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1183 * mirror could be repaired by taking 512 byte of a different
1184 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1185 * area are unreadable.
1186 */
1187
1188 /* can only fix I/O errors from here on */
1189 if (sblock_bad->no_io_error_seen)
1190 goto did_not_correct_error;
1191
1192 success = 1;
1193 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001194 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001195
1196 if (!page_bad->io_error)
1197 continue;
1198
1199 for (mirror_index = 0;
1200 mirror_index < BTRFS_MAX_MIRRORS &&
1201 sblocks_for_recheck[mirror_index].page_count > 0;
1202 mirror_index++) {
1203 struct scrub_block *sblock_other = sblocks_for_recheck +
1204 mirror_index;
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001205 struct scrub_page *page_other = sblock_other->pagev[
1206 page_num];
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001207
1208 if (!page_other->io_error) {
1209 ret = scrub_repair_page_from_good_copy(
1210 sblock_bad, sblock_other, page_num, 0);
1211 if (0 == ret) {
1212 page_bad->io_error = 0;
1213 break; /* succeeded for this page */
1214 }
Jan Schmidt13db62b2011-06-13 19:56:13 +02001215 }
1216 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001217
1218 if (page_bad->io_error) {
1219 /* did not find a mirror to copy the page from */
1220 success = 0;
1221 }
1222 }
1223
1224 if (success) {
1225 if (is_metadata || have_csum) {
1226 /*
1227 * need to verify the checksum now that all
1228 * sectors on disk are repaired (the write
1229 * request for data to be repaired is on its way).
1230 * Just be lazy and use scrub_recheck_block()
1231 * which re-reads the data before the checksum
1232 * is verified, but most likely the data comes out
1233 * of the page cache.
1234 */
Stefan Behrens34f5c8e2012-11-02 16:16:26 +01001235 scrub_recheck_block(fs_info, sblock_bad,
1236 is_metadata, have_csum, csum,
Miao Xieaf8e2d12014-10-23 14:42:50 +08001237 generation, sctx->csum_size, 1);
Stefan Behrens34f5c8e2012-11-02 16:16:26 +01001238 if (!sblock_bad->header_error &&
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001239 !sblock_bad->checksum_error &&
1240 sblock_bad->no_io_error_seen)
1241 goto corrected_error;
1242 else
1243 goto did_not_correct_error;
1244 } else {
1245corrected_error:
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001246 spin_lock(&sctx->stat_lock);
1247 sctx->stat.corrected_errors++;
Miao Xie5a6ac9e2014-11-06 17:20:58 +08001248 sblock_to_check->data_corrected = 1;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001249 spin_unlock(&sctx->stat_lock);
Josef Bacik606686e2012-06-04 14:03:51 -04001250 printk_ratelimited_in_rcu(KERN_ERR
Frank Holtonefe120a2013-12-20 11:37:06 -05001251 "BTRFS: fixed up error at logical %llu on dev %s\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001252 logical, rcu_str_deref(dev->name));
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001253 }
1254 } else {
1255did_not_correct_error:
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001256 spin_lock(&sctx->stat_lock);
1257 sctx->stat.uncorrectable_errors++;
1258 spin_unlock(&sctx->stat_lock);
Josef Bacik606686e2012-06-04 14:03:51 -04001259 printk_ratelimited_in_rcu(KERN_ERR
Frank Holtonefe120a2013-12-20 11:37:06 -05001260 "BTRFS: unable to fixup (regular) error at logical %llu on dev %s\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001261 logical, rcu_str_deref(dev->name));
Arne Jansena2de7332011-03-08 14:14:00 +01001262 }
1263
1264out:
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001265 if (sblocks_for_recheck) {
1266 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1267 mirror_index++) {
1268 struct scrub_block *sblock = sblocks_for_recheck +
1269 mirror_index;
Miao Xieaf8e2d12014-10-23 14:42:50 +08001270 struct scrub_recover *recover;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001271 int page_index;
1272
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001273 for (page_index = 0; page_index < sblock->page_count;
1274 page_index++) {
1275 sblock->pagev[page_index]->sblock = NULL;
Miao Xieaf8e2d12014-10-23 14:42:50 +08001276 recover = sblock->pagev[page_index]->recover;
1277 if (recover) {
1278 scrub_put_recover(recover);
1279 sblock->pagev[page_index]->recover =
1280 NULL;
1281 }
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001282 scrub_page_put(sblock->pagev[page_index]);
1283 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001284 }
1285 kfree(sblocks_for_recheck);
1286 }
1287
1288 return 0;
Arne Jansena2de7332011-03-08 14:14:00 +01001289}
1290
Miao Xieaf8e2d12014-10-23 14:42:50 +08001291static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio, u64 *raid_map)
1292{
1293 if (raid_map) {
1294 if (raid_map[bbio->num_stripes - 1] == RAID6_Q_STRIPE)
1295 return 3;
1296 else
1297 return 2;
1298 } else {
1299 return (int)bbio->num_stripes;
1300 }
1301}
1302
1303static inline void scrub_stripe_index_and_offset(u64 logical, u64 *raid_map,
1304 u64 mapped_length,
1305 int nstripes, int mirror,
1306 int *stripe_index,
1307 u64 *stripe_offset)
1308{
1309 int i;
1310
1311 if (raid_map) {
1312 /* RAID5/6 */
1313 for (i = 0; i < nstripes; i++) {
1314 if (raid_map[i] == RAID6_Q_STRIPE ||
1315 raid_map[i] == RAID5_P_STRIPE)
1316 continue;
1317
1318 if (logical >= raid_map[i] &&
1319 logical < raid_map[i] + mapped_length)
1320 break;
1321 }
1322
1323 *stripe_index = i;
1324 *stripe_offset = logical - raid_map[i];
1325 } else {
1326 /* The other RAID type */
1327 *stripe_index = mirror;
1328 *stripe_offset = 0;
1329 }
1330}
1331
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001332static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
Stefan Behrens3ec706c2012-11-05 15:46:42 +01001333 struct btrfs_fs_info *fs_info,
Stefan Behrensff023aa2012-11-06 11:43:11 +01001334 struct scrub_block *original_sblock,
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001335 u64 length, u64 logical,
1336 struct scrub_block *sblocks_for_recheck)
Arne Jansena2de7332011-03-08 14:14:00 +01001337{
Miao Xieaf8e2d12014-10-23 14:42:50 +08001338 struct scrub_recover *recover;
1339 struct btrfs_bio *bbio;
1340 u64 *raid_map;
1341 u64 sublen;
1342 u64 mapped_length;
1343 u64 stripe_offset;
1344 int stripe_index;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001345 int page_index;
1346 int mirror_index;
Miao Xieaf8e2d12014-10-23 14:42:50 +08001347 int nmirrors;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001348 int ret;
1349
1350 /*
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001351 * note: the two members ref_count and outstanding_pages
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001352 * are not used (and not set) in the blocks that are used for
1353 * the recheck procedure
1354 */
1355
1356 page_index = 0;
1357 while (length > 0) {
Miao Xieaf8e2d12014-10-23 14:42:50 +08001358 sublen = min_t(u64, length, PAGE_SIZE);
1359 mapped_length = sublen;
1360 bbio = NULL;
1361 raid_map = NULL;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001362
1363 /*
1364 * with a length of PAGE_SIZE, each returned stripe
1365 * represents one mirror
1366 */
Miao Xieaf8e2d12014-10-23 14:42:50 +08001367 ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical,
1368 &mapped_length, &bbio, 0, &raid_map);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001369 if (ret || !bbio || mapped_length < sublen) {
1370 kfree(bbio);
Miao Xieaf8e2d12014-10-23 14:42:50 +08001371 kfree(raid_map);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001372 return -EIO;
1373 }
1374
Miao Xieaf8e2d12014-10-23 14:42:50 +08001375 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1376 if (!recover) {
1377 kfree(bbio);
1378 kfree(raid_map);
1379 return -ENOMEM;
1380 }
1381
1382 atomic_set(&recover->refs, 1);
1383 recover->bbio = bbio;
1384 recover->raid_map = raid_map;
1385 recover->map_length = mapped_length;
1386
Stefan Behrensff023aa2012-11-06 11:43:11 +01001387 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
Miao Xieaf8e2d12014-10-23 14:42:50 +08001388
1389 nmirrors = scrub_nr_raid_mirrors(bbio, raid_map);
1390 for (mirror_index = 0; mirror_index < nmirrors;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001391 mirror_index++) {
1392 struct scrub_block *sblock;
1393 struct scrub_page *page;
1394
1395 if (mirror_index >= BTRFS_MAX_MIRRORS)
1396 continue;
1397
1398 sblock = sblocks_for_recheck + mirror_index;
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001399 sblock->sctx = sctx;
1400 page = kzalloc(sizeof(*page), GFP_NOFS);
1401 if (!page) {
1402leave_nomem:
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001403 spin_lock(&sctx->stat_lock);
1404 sctx->stat.malloc_errors++;
1405 spin_unlock(&sctx->stat_lock);
Miao Xieaf8e2d12014-10-23 14:42:50 +08001406 scrub_put_recover(recover);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001407 return -ENOMEM;
1408 }
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001409 scrub_page_get(page);
1410 sblock->pagev[page_index] = page;
1411 page->logical = logical;
Miao Xieaf8e2d12014-10-23 14:42:50 +08001412
1413 scrub_stripe_index_and_offset(logical, raid_map,
1414 mapped_length,
1415 bbio->num_stripes,
1416 mirror_index,
1417 &stripe_index,
1418 &stripe_offset);
1419 page->physical = bbio->stripes[stripe_index].physical +
1420 stripe_offset;
1421 page->dev = bbio->stripes[stripe_index].dev;
1422
Stefan Behrensff023aa2012-11-06 11:43:11 +01001423 BUG_ON(page_index >= original_sblock->page_count);
1424 page->physical_for_dev_replace =
1425 original_sblock->pagev[page_index]->
1426 physical_for_dev_replace;
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001427 /* for missing devices, dev->bdev is NULL */
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001428 page->mirror_num = mirror_index + 1;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001429 sblock->page_count++;
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001430 page->page = alloc_page(GFP_NOFS);
1431 if (!page->page)
1432 goto leave_nomem;
Miao Xieaf8e2d12014-10-23 14:42:50 +08001433
1434 scrub_get_recover(recover);
1435 page->recover = recover;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001436 }
Miao Xieaf8e2d12014-10-23 14:42:50 +08001437 scrub_put_recover(recover);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001438 length -= sublen;
1439 logical += sublen;
1440 page_index++;
1441 }
1442
1443 return 0;
1444}
1445
Miao Xieaf8e2d12014-10-23 14:42:50 +08001446struct scrub_bio_ret {
1447 struct completion event;
1448 int error;
1449};
1450
1451static void scrub_bio_wait_endio(struct bio *bio, int error)
1452{
1453 struct scrub_bio_ret *ret = bio->bi_private;
1454
1455 ret->error = error;
1456 complete(&ret->event);
1457}
1458
1459static inline int scrub_is_page_on_raid56(struct scrub_page *page)
1460{
1461 return page->recover && page->recover->raid_map;
1462}
1463
1464static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1465 struct bio *bio,
1466 struct scrub_page *page)
1467{
1468 struct scrub_bio_ret done;
1469 int ret;
1470
1471 init_completion(&done.event);
1472 done.error = 0;
1473 bio->bi_iter.bi_sector = page->logical >> 9;
1474 bio->bi_private = &done;
1475 bio->bi_end_io = scrub_bio_wait_endio;
1476
1477 ret = raid56_parity_recover(fs_info->fs_root, bio, page->recover->bbio,
1478 page->recover->raid_map,
1479 page->recover->map_length,
Miao Xie42452152014-11-25 16:39:28 +08001480 page->mirror_num, 0);
Miao Xieaf8e2d12014-10-23 14:42:50 +08001481 if (ret)
1482 return ret;
1483
1484 wait_for_completion(&done.event);
1485 if (done.error)
1486 return -EIO;
1487
1488 return 0;
1489}
1490
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001491/*
1492 * this function will check the on disk data for checksum errors, header
1493 * errors and read I/O errors. If any I/O errors happen, the exact pages
1494 * which are errored are marked as being bad. The goal is to enable scrub
1495 * to take those pages that are not errored from all the mirrors so that
1496 * the pages that are errored in the just handled mirror can be repaired.
1497 */
Stefan Behrens34f5c8e2012-11-02 16:16:26 +01001498static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1499 struct scrub_block *sblock, int is_metadata,
1500 int have_csum, u8 *csum, u64 generation,
Miao Xieaf8e2d12014-10-23 14:42:50 +08001501 u16 csum_size, int retry_failed_mirror)
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001502{
1503 int page_num;
1504
1505 sblock->no_io_error_seen = 1;
1506 sblock->header_error = 0;
1507 sblock->checksum_error = 0;
1508
1509 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1510 struct bio *bio;
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001511 struct scrub_page *page = sblock->pagev[page_num];
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001512
Stefan Behrens442a4f62012-05-25 16:06:08 +02001513 if (page->dev->bdev == NULL) {
Stefan Behrensea9947b2012-05-04 15:16:07 -04001514 page->io_error = 1;
1515 sblock->no_io_error_seen = 0;
1516 continue;
1517 }
1518
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001519 WARN_ON(!page->page);
Chris Mason9be33952013-05-17 18:30:14 -04001520 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Stefan Behrens34f5c8e2012-11-02 16:16:26 +01001521 if (!bio) {
1522 page->io_error = 1;
1523 sblock->no_io_error_seen = 0;
1524 continue;
1525 }
Stefan Behrens442a4f62012-05-25 16:06:08 +02001526 bio->bi_bdev = page->dev->bdev;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001527
Stefan Behrens34f5c8e2012-11-02 16:16:26 +01001528 bio_add_page(bio, page->page, PAGE_SIZE, 0);
Miao Xieaf8e2d12014-10-23 14:42:50 +08001529 if (!retry_failed_mirror && scrub_is_page_on_raid56(page)) {
1530 if (scrub_submit_raid56_bio_wait(fs_info, bio, page))
1531 sblock->no_io_error_seen = 0;
1532 } else {
1533 bio->bi_iter.bi_sector = page->physical >> 9;
1534
1535 if (btrfsic_submit_bio_wait(READ, bio))
1536 sblock->no_io_error_seen = 0;
1537 }
Kent Overstreet33879d42013-11-23 22:33:32 -08001538
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001539 bio_put(bio);
1540 }
1541
1542 if (sblock->no_io_error_seen)
1543 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1544 have_csum, csum, generation,
1545 csum_size);
1546
Stefan Behrens34f5c8e2012-11-02 16:16:26 +01001547 return;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001548}
1549
Miao Xie17a9be22014-07-24 11:37:08 +08001550static inline int scrub_check_fsid(u8 fsid[],
1551 struct scrub_page *spage)
1552{
1553 struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1554 int ret;
1555
1556 ret = memcmp(fsid, fs_devices->fsid, BTRFS_UUID_SIZE);
1557 return !ret;
1558}
1559
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001560static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1561 struct scrub_block *sblock,
1562 int is_metadata, int have_csum,
1563 const u8 *csum, u64 generation,
1564 u16 csum_size)
1565{
1566 int page_num;
1567 u8 calculated_csum[BTRFS_CSUM_SIZE];
1568 u32 crc = ~(u32)0;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001569 void *mapped_buffer;
1570
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001571 WARN_ON(!sblock->pagev[0]->page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001572 if (is_metadata) {
1573 struct btrfs_header *h;
1574
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001575 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001576 h = (struct btrfs_header *)mapped_buffer;
1577
Qu Wenruo3cae2102013-07-16 11:19:18 +08001578 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||
Miao Xie17a9be22014-07-24 11:37:08 +08001579 !scrub_check_fsid(h->fsid, sblock->pagev[0]) ||
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001580 memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
Stefan Behrens442a4f62012-05-25 16:06:08 +02001581 BTRFS_UUID_SIZE)) {
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001582 sblock->header_error = 1;
Qu Wenruo3cae2102013-07-16 11:19:18 +08001583 } else if (generation != btrfs_stack_header_generation(h)) {
Stefan Behrens442a4f62012-05-25 16:06:08 +02001584 sblock->header_error = 1;
1585 sblock->generation_error = 1;
1586 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001587 csum = h->csum;
1588 } else {
1589 if (!have_csum)
1590 return;
1591
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001592 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001593 }
1594
1595 for (page_num = 0;;) {
1596 if (page_num == 0 && is_metadata)
Liu Bob0496682013-03-14 14:57:45 +00001597 crc = btrfs_csum_data(
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001598 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1599 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1600 else
Liu Bob0496682013-03-14 14:57:45 +00001601 crc = btrfs_csum_data(mapped_buffer, crc, PAGE_SIZE);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001602
Linus Torvalds9613beb2012-03-30 12:44:29 -07001603 kunmap_atomic(mapped_buffer);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001604 page_num++;
1605 if (page_num >= sblock->page_count)
1606 break;
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001607 WARN_ON(!sblock->pagev[page_num]->page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001608
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001609 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001610 }
1611
1612 btrfs_csum_final(crc, calculated_csum);
1613 if (memcmp(calculated_csum, csum, csum_size))
1614 sblock->checksum_error = 1;
1615}
1616
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001617static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1618 struct scrub_block *sblock_good,
1619 int force_write)
1620{
1621 int page_num;
1622 int ret = 0;
1623
1624 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1625 int ret_sub;
1626
1627 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1628 sblock_good,
1629 page_num,
1630 force_write);
1631 if (ret_sub)
1632 ret = ret_sub;
1633 }
1634
1635 return ret;
1636}
1637
1638static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1639 struct scrub_block *sblock_good,
1640 int page_num, int force_write)
1641{
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001642 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1643 struct scrub_page *page_good = sblock_good->pagev[page_num];
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001644
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001645 BUG_ON(page_bad->page == NULL);
1646 BUG_ON(page_good->page == NULL);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001647 if (force_write || sblock_bad->header_error ||
1648 sblock_bad->checksum_error || page_bad->io_error) {
1649 struct bio *bio;
1650 int ret;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001651
Stefan Behrensff023aa2012-11-06 11:43:11 +01001652 if (!page_bad->dev->bdev) {
Frank Holtonefe120a2013-12-20 11:37:06 -05001653 printk_ratelimited(KERN_WARNING "BTRFS: "
1654 "scrub_repair_page_from_good_copy(bdev == NULL) "
1655 "is unexpected!\n");
Stefan Behrensff023aa2012-11-06 11:43:11 +01001656 return -EIO;
1657 }
1658
Chris Mason9be33952013-05-17 18:30:14 -04001659 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Tsutomu Itohe627ee72012-04-12 16:03:56 -04001660 if (!bio)
1661 return -EIO;
Stefan Behrens442a4f62012-05-25 16:06:08 +02001662 bio->bi_bdev = page_bad->dev->bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001663 bio->bi_iter.bi_sector = page_bad->physical >> 9;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001664
1665 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1666 if (PAGE_SIZE != ret) {
1667 bio_put(bio);
1668 return -EIO;
1669 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001670
Kent Overstreet33879d42013-11-23 22:33:32 -08001671 if (btrfsic_submit_bio_wait(WRITE, bio)) {
Stefan Behrens442a4f62012-05-25 16:06:08 +02001672 btrfs_dev_stat_inc_and_print(page_bad->dev,
1673 BTRFS_DEV_STAT_WRITE_ERRS);
Stefan Behrensff023aa2012-11-06 11:43:11 +01001674 btrfs_dev_replace_stats_inc(
1675 &sblock_bad->sctx->dev_root->fs_info->
1676 dev_replace.num_write_errors);
Stefan Behrens442a4f62012-05-25 16:06:08 +02001677 bio_put(bio);
1678 return -EIO;
1679 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001680 bio_put(bio);
1681 }
1682
1683 return 0;
1684}
1685
Stefan Behrensff023aa2012-11-06 11:43:11 +01001686static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1687{
1688 int page_num;
1689
Miao Xie5a6ac9e2014-11-06 17:20:58 +08001690 /*
1691 * This block is used for the check of the parity on the source device,
1692 * so the data needn't be written into the destination device.
1693 */
1694 if (sblock->sparity)
1695 return;
1696
Stefan Behrensff023aa2012-11-06 11:43:11 +01001697 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1698 int ret;
1699
1700 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1701 if (ret)
1702 btrfs_dev_replace_stats_inc(
1703 &sblock->sctx->dev_root->fs_info->dev_replace.
1704 num_write_errors);
1705 }
1706}
1707
1708static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1709 int page_num)
1710{
1711 struct scrub_page *spage = sblock->pagev[page_num];
1712
1713 BUG_ON(spage->page == NULL);
1714 if (spage->io_error) {
1715 void *mapped_buffer = kmap_atomic(spage->page);
1716
1717 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1718 flush_dcache_page(spage->page);
1719 kunmap_atomic(mapped_buffer);
1720 }
1721 return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1722}
1723
1724static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1725 struct scrub_page *spage)
1726{
1727 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1728 struct scrub_bio *sbio;
1729 int ret;
1730
1731 mutex_lock(&wr_ctx->wr_lock);
1732again:
1733 if (!wr_ctx->wr_curr_bio) {
1734 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1735 GFP_NOFS);
1736 if (!wr_ctx->wr_curr_bio) {
1737 mutex_unlock(&wr_ctx->wr_lock);
1738 return -ENOMEM;
1739 }
1740 wr_ctx->wr_curr_bio->sctx = sctx;
1741 wr_ctx->wr_curr_bio->page_count = 0;
1742 }
1743 sbio = wr_ctx->wr_curr_bio;
1744 if (sbio->page_count == 0) {
1745 struct bio *bio;
1746
1747 sbio->physical = spage->physical_for_dev_replace;
1748 sbio->logical = spage->logical;
1749 sbio->dev = wr_ctx->tgtdev;
1750 bio = sbio->bio;
1751 if (!bio) {
Chris Mason9be33952013-05-17 18:30:14 -04001752 bio = btrfs_io_bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
Stefan Behrensff023aa2012-11-06 11:43:11 +01001753 if (!bio) {
1754 mutex_unlock(&wr_ctx->wr_lock);
1755 return -ENOMEM;
1756 }
1757 sbio->bio = bio;
1758 }
1759
1760 bio->bi_private = sbio;
1761 bio->bi_end_io = scrub_wr_bio_end_io;
1762 bio->bi_bdev = sbio->dev->bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07001763 bio->bi_iter.bi_sector = sbio->physical >> 9;
Stefan Behrensff023aa2012-11-06 11:43:11 +01001764 sbio->err = 0;
1765 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1766 spage->physical_for_dev_replace ||
1767 sbio->logical + sbio->page_count * PAGE_SIZE !=
1768 spage->logical) {
1769 scrub_wr_submit(sctx);
1770 goto again;
1771 }
1772
1773 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1774 if (ret != PAGE_SIZE) {
1775 if (sbio->page_count < 1) {
1776 bio_put(sbio->bio);
1777 sbio->bio = NULL;
1778 mutex_unlock(&wr_ctx->wr_lock);
1779 return -EIO;
1780 }
1781 scrub_wr_submit(sctx);
1782 goto again;
1783 }
1784
1785 sbio->pagev[sbio->page_count] = spage;
1786 scrub_page_get(spage);
1787 sbio->page_count++;
1788 if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1789 scrub_wr_submit(sctx);
1790 mutex_unlock(&wr_ctx->wr_lock);
1791
1792 return 0;
1793}
1794
1795static void scrub_wr_submit(struct scrub_ctx *sctx)
1796{
1797 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1798 struct scrub_bio *sbio;
1799
1800 if (!wr_ctx->wr_curr_bio)
1801 return;
1802
1803 sbio = wr_ctx->wr_curr_bio;
1804 wr_ctx->wr_curr_bio = NULL;
1805 WARN_ON(!sbio->bio->bi_bdev);
1806 scrub_pending_bio_inc(sctx);
1807 /* process all writes in a single worker thread. Then the block layer
1808 * orders the requests before sending them to the driver which
1809 * doubled the write performance on spinning disks when measured
1810 * with Linux 3.5 */
1811 btrfsic_submit_bio(WRITE, sbio->bio);
1812}
1813
1814static void scrub_wr_bio_end_io(struct bio *bio, int err)
1815{
1816 struct scrub_bio *sbio = bio->bi_private;
1817 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1818
1819 sbio->err = err;
1820 sbio->bio = bio;
1821
Liu Bo9e0af232014-08-15 23:36:53 +08001822 btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
1823 scrub_wr_bio_end_io_worker, NULL, NULL);
Qu Wenruo0339ef22014-02-28 10:46:17 +08001824 btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
Stefan Behrensff023aa2012-11-06 11:43:11 +01001825}
1826
1827static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1828{
1829 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1830 struct scrub_ctx *sctx = sbio->sctx;
1831 int i;
1832
1833 WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1834 if (sbio->err) {
1835 struct btrfs_dev_replace *dev_replace =
1836 &sbio->sctx->dev_root->fs_info->dev_replace;
1837
1838 for (i = 0; i < sbio->page_count; i++) {
1839 struct scrub_page *spage = sbio->pagev[i];
1840
1841 spage->io_error = 1;
1842 btrfs_dev_replace_stats_inc(&dev_replace->
1843 num_write_errors);
1844 }
1845 }
1846
1847 for (i = 0; i < sbio->page_count; i++)
1848 scrub_page_put(sbio->pagev[i]);
1849
1850 bio_put(sbio->bio);
1851 kfree(sbio);
1852 scrub_pending_bio_dec(sctx);
1853}
1854
1855static int scrub_checksum(struct scrub_block *sblock)
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001856{
1857 u64 flags;
1858 int ret;
1859
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001860 WARN_ON(sblock->page_count < 1);
1861 flags = sblock->pagev[0]->flags;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001862 ret = 0;
1863 if (flags & BTRFS_EXTENT_FLAG_DATA)
1864 ret = scrub_checksum_data(sblock);
1865 else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1866 ret = scrub_checksum_tree_block(sblock);
1867 else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1868 (void)scrub_checksum_super(sblock);
1869 else
1870 WARN_ON(1);
1871 if (ret)
1872 scrub_handle_errored_block(sblock);
Stefan Behrensff023aa2012-11-06 11:43:11 +01001873
1874 return ret;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001875}
1876
1877static int scrub_checksum_data(struct scrub_block *sblock)
1878{
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001879 struct scrub_ctx *sctx = sblock->sctx;
Arne Jansena2de7332011-03-08 14:14:00 +01001880 u8 csum[BTRFS_CSUM_SIZE];
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001881 u8 *on_disk_csum;
1882 struct page *page;
1883 void *buffer;
Arne Jansena2de7332011-03-08 14:14:00 +01001884 u32 crc = ~(u32)0;
1885 int fail = 0;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001886 u64 len;
1887 int index;
Arne Jansena2de7332011-03-08 14:14:00 +01001888
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001889 BUG_ON(sblock->page_count < 1);
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001890 if (!sblock->pagev[0]->have_csum)
Arne Jansena2de7332011-03-08 14:14:00 +01001891 return 0;
1892
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001893 on_disk_csum = sblock->pagev[0]->csum;
1894 page = sblock->pagev[0]->page;
Linus Torvalds9613beb2012-03-30 12:44:29 -07001895 buffer = kmap_atomic(page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001896
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001897 len = sctx->sectorsize;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001898 index = 0;
1899 for (;;) {
1900 u64 l = min_t(u64, len, PAGE_SIZE);
1901
Liu Bob0496682013-03-14 14:57:45 +00001902 crc = btrfs_csum_data(buffer, crc, l);
Linus Torvalds9613beb2012-03-30 12:44:29 -07001903 kunmap_atomic(buffer);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001904 len -= l;
1905 if (len == 0)
1906 break;
1907 index++;
1908 BUG_ON(index >= sblock->page_count);
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001909 BUG_ON(!sblock->pagev[index]->page);
1910 page = sblock->pagev[index]->page;
Linus Torvalds9613beb2012-03-30 12:44:29 -07001911 buffer = kmap_atomic(page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001912 }
1913
Arne Jansena2de7332011-03-08 14:14:00 +01001914 btrfs_csum_final(crc, csum);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001915 if (memcmp(csum, on_disk_csum, sctx->csum_size))
Arne Jansena2de7332011-03-08 14:14:00 +01001916 fail = 1;
1917
Arne Jansena2de7332011-03-08 14:14:00 +01001918 return fail;
1919}
1920
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001921static int scrub_checksum_tree_block(struct scrub_block *sblock)
Arne Jansena2de7332011-03-08 14:14:00 +01001922{
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001923 struct scrub_ctx *sctx = sblock->sctx;
Arne Jansena2de7332011-03-08 14:14:00 +01001924 struct btrfs_header *h;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01001925 struct btrfs_root *root = sctx->dev_root;
Arne Jansena2de7332011-03-08 14:14:00 +01001926 struct btrfs_fs_info *fs_info = root->fs_info;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001927 u8 calculated_csum[BTRFS_CSUM_SIZE];
1928 u8 on_disk_csum[BTRFS_CSUM_SIZE];
1929 struct page *page;
1930 void *mapped_buffer;
1931 u64 mapped_size;
1932 void *p;
Arne Jansena2de7332011-03-08 14:14:00 +01001933 u32 crc = ~(u32)0;
1934 int fail = 0;
1935 int crc_fail = 0;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001936 u64 len;
1937 int index;
1938
1939 BUG_ON(sblock->page_count < 1);
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001940 page = sblock->pagev[0]->page;
Linus Torvalds9613beb2012-03-30 12:44:29 -07001941 mapped_buffer = kmap_atomic(page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001942 h = (struct btrfs_header *)mapped_buffer;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001943 memcpy(on_disk_csum, h->csum, sctx->csum_size);
Arne Jansena2de7332011-03-08 14:14:00 +01001944
1945 /*
1946 * we don't use the getter functions here, as we
1947 * a) don't have an extent buffer and
1948 * b) the page is already kmapped
1949 */
Arne Jansena2de7332011-03-08 14:14:00 +01001950
Qu Wenruo3cae2102013-07-16 11:19:18 +08001951 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
Arne Jansena2de7332011-03-08 14:14:00 +01001952 ++fail;
1953
Qu Wenruo3cae2102013-07-16 11:19:18 +08001954 if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))
Arne Jansena2de7332011-03-08 14:14:00 +01001955 ++fail;
1956
Miao Xie17a9be22014-07-24 11:37:08 +08001957 if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
Arne Jansena2de7332011-03-08 14:14:00 +01001958 ++fail;
1959
1960 if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1961 BTRFS_UUID_SIZE))
1962 ++fail;
1963
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001964 len = sctx->nodesize - BTRFS_CSUM_SIZE;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001965 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1966 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1967 index = 0;
1968 for (;;) {
1969 u64 l = min_t(u64, len, mapped_size);
1970
Liu Bob0496682013-03-14 14:57:45 +00001971 crc = btrfs_csum_data(p, crc, l);
Linus Torvalds9613beb2012-03-30 12:44:29 -07001972 kunmap_atomic(mapped_buffer);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001973 len -= l;
1974 if (len == 0)
1975 break;
1976 index++;
1977 BUG_ON(index >= sblock->page_count);
Stefan Behrens7a9e9982012-11-02 14:58:04 +01001978 BUG_ON(!sblock->pagev[index]->page);
1979 page = sblock->pagev[index]->page;
Linus Torvalds9613beb2012-03-30 12:44:29 -07001980 mapped_buffer = kmap_atomic(page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001981 mapped_size = PAGE_SIZE;
1982 p = mapped_buffer;
1983 }
1984
1985 btrfs_csum_final(crc, calculated_csum);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001986 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
Arne Jansena2de7332011-03-08 14:14:00 +01001987 ++crc_fail;
1988
Arne Jansena2de7332011-03-08 14:14:00 +01001989 return fail || crc_fail;
1990}
1991
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001992static int scrub_checksum_super(struct scrub_block *sblock)
Arne Jansena2de7332011-03-08 14:14:00 +01001993{
1994 struct btrfs_super_block *s;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01001995 struct scrub_ctx *sctx = sblock->sctx;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04001996 u8 calculated_csum[BTRFS_CSUM_SIZE];
1997 u8 on_disk_csum[BTRFS_CSUM_SIZE];
1998 struct page *page;
1999 void *mapped_buffer;
2000 u64 mapped_size;
2001 void *p;
Arne Jansena2de7332011-03-08 14:14:00 +01002002 u32 crc = ~(u32)0;
Stefan Behrens442a4f62012-05-25 16:06:08 +02002003 int fail_gen = 0;
2004 int fail_cor = 0;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002005 u64 len;
2006 int index;
Arne Jansena2de7332011-03-08 14:14:00 +01002007
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002008 BUG_ON(sblock->page_count < 1);
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002009 page = sblock->pagev[0]->page;
Linus Torvalds9613beb2012-03-30 12:44:29 -07002010 mapped_buffer = kmap_atomic(page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002011 s = (struct btrfs_super_block *)mapped_buffer;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002012 memcpy(on_disk_csum, s->csum, sctx->csum_size);
Arne Jansena2de7332011-03-08 14:14:00 +01002013
Qu Wenruo3cae2102013-07-16 11:19:18 +08002014 if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
Stefan Behrens442a4f62012-05-25 16:06:08 +02002015 ++fail_cor;
Arne Jansena2de7332011-03-08 14:14:00 +01002016
Qu Wenruo3cae2102013-07-16 11:19:18 +08002017 if (sblock->pagev[0]->generation != btrfs_super_generation(s))
Stefan Behrens442a4f62012-05-25 16:06:08 +02002018 ++fail_gen;
Arne Jansena2de7332011-03-08 14:14:00 +01002019
Miao Xie17a9be22014-07-24 11:37:08 +08002020 if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
Stefan Behrens442a4f62012-05-25 16:06:08 +02002021 ++fail_cor;
Arne Jansena2de7332011-03-08 14:14:00 +01002022
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002023 len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
2024 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2025 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2026 index = 0;
2027 for (;;) {
2028 u64 l = min_t(u64, len, mapped_size);
2029
Liu Bob0496682013-03-14 14:57:45 +00002030 crc = btrfs_csum_data(p, crc, l);
Linus Torvalds9613beb2012-03-30 12:44:29 -07002031 kunmap_atomic(mapped_buffer);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002032 len -= l;
2033 if (len == 0)
2034 break;
2035 index++;
2036 BUG_ON(index >= sblock->page_count);
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002037 BUG_ON(!sblock->pagev[index]->page);
2038 page = sblock->pagev[index]->page;
Linus Torvalds9613beb2012-03-30 12:44:29 -07002039 mapped_buffer = kmap_atomic(page);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002040 mapped_size = PAGE_SIZE;
2041 p = mapped_buffer;
2042 }
2043
2044 btrfs_csum_final(crc, calculated_csum);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002045 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
Stefan Behrens442a4f62012-05-25 16:06:08 +02002046 ++fail_cor;
Arne Jansena2de7332011-03-08 14:14:00 +01002047
Stefan Behrens442a4f62012-05-25 16:06:08 +02002048 if (fail_cor + fail_gen) {
Arne Jansena2de7332011-03-08 14:14:00 +01002049 /*
2050 * if we find an error in a super block, we just report it.
2051 * They will get written with the next transaction commit
2052 * anyway
2053 */
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002054 spin_lock(&sctx->stat_lock);
2055 ++sctx->stat.super_errors;
2056 spin_unlock(&sctx->stat_lock);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002057 if (fail_cor)
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002058 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
Stefan Behrens442a4f62012-05-25 16:06:08 +02002059 BTRFS_DEV_STAT_CORRUPTION_ERRS);
2060 else
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002061 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
Stefan Behrens442a4f62012-05-25 16:06:08 +02002062 BTRFS_DEV_STAT_GENERATION_ERRS);
Arne Jansena2de7332011-03-08 14:14:00 +01002063 }
2064
Stefan Behrens442a4f62012-05-25 16:06:08 +02002065 return fail_cor + fail_gen;
Arne Jansena2de7332011-03-08 14:14:00 +01002066}
2067
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002068static void scrub_block_get(struct scrub_block *sblock)
2069{
2070 atomic_inc(&sblock->ref_count);
2071}
2072
2073static void scrub_block_put(struct scrub_block *sblock)
2074{
2075 if (atomic_dec_and_test(&sblock->ref_count)) {
2076 int i;
2077
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002078 if (sblock->sparity)
2079 scrub_parity_put(sblock->sparity);
2080
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002081 for (i = 0; i < sblock->page_count; i++)
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002082 scrub_page_put(sblock->pagev[i]);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002083 kfree(sblock);
2084 }
2085}
2086
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002087static void scrub_page_get(struct scrub_page *spage)
2088{
2089 atomic_inc(&spage->ref_count);
2090}
2091
2092static void scrub_page_put(struct scrub_page *spage)
2093{
2094 if (atomic_dec_and_test(&spage->ref_count)) {
2095 if (spage->page)
2096 __free_page(spage->page);
2097 kfree(spage);
2098 }
2099}
2100
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002101static void scrub_submit(struct scrub_ctx *sctx)
Arne Jansena2de7332011-03-08 14:14:00 +01002102{
2103 struct scrub_bio *sbio;
2104
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002105 if (sctx->curr == -1)
Stefan Behrens1623ede2012-03-27 14:21:26 -04002106 return;
Arne Jansena2de7332011-03-08 14:14:00 +01002107
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002108 sbio = sctx->bios[sctx->curr];
2109 sctx->curr = -1;
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01002110 scrub_pending_bio_inc(sctx);
Arne Jansena2de7332011-03-08 14:14:00 +01002111
Stefan Behrensff023aa2012-11-06 11:43:11 +01002112 if (!sbio->bio->bi_bdev) {
2113 /*
2114 * this case should not happen. If btrfs_map_block() is
2115 * wrong, it could happen for dev-replace operations on
2116 * missing devices when no mirrors are available, but in
2117 * this case it should already fail the mount.
2118 * This case is handled correctly (but _very_ slowly).
2119 */
2120 printk_ratelimited(KERN_WARNING
Frank Holtonefe120a2013-12-20 11:37:06 -05002121 "BTRFS: scrub_submit(bio bdev == NULL) is unexpected!\n");
Stefan Behrensff023aa2012-11-06 11:43:11 +01002122 bio_endio(sbio->bio, -EIO);
2123 } else {
2124 btrfsic_submit_bio(READ, sbio->bio);
2125 }
Arne Jansena2de7332011-03-08 14:14:00 +01002126}
2127
Stefan Behrensff023aa2012-11-06 11:43:11 +01002128static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2129 struct scrub_page *spage)
Arne Jansena2de7332011-03-08 14:14:00 +01002130{
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002131 struct scrub_block *sblock = spage->sblock;
Arne Jansena2de7332011-03-08 14:14:00 +01002132 struct scrub_bio *sbio;
Arne Jansen69f4cb52011-11-11 08:17:10 -05002133 int ret;
Arne Jansena2de7332011-03-08 14:14:00 +01002134
2135again:
2136 /*
2137 * grab a fresh bio or wait for one to become available
2138 */
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002139 while (sctx->curr == -1) {
2140 spin_lock(&sctx->list_lock);
2141 sctx->curr = sctx->first_free;
2142 if (sctx->curr != -1) {
2143 sctx->first_free = sctx->bios[sctx->curr]->next_free;
2144 sctx->bios[sctx->curr]->next_free = -1;
2145 sctx->bios[sctx->curr]->page_count = 0;
2146 spin_unlock(&sctx->list_lock);
Arne Jansena2de7332011-03-08 14:14:00 +01002147 } else {
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002148 spin_unlock(&sctx->list_lock);
2149 wait_event(sctx->list_wait, sctx->first_free != -1);
Arne Jansena2de7332011-03-08 14:14:00 +01002150 }
2151 }
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002152 sbio = sctx->bios[sctx->curr];
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002153 if (sbio->page_count == 0) {
Arne Jansen69f4cb52011-11-11 08:17:10 -05002154 struct bio *bio;
2155
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002156 sbio->physical = spage->physical;
2157 sbio->logical = spage->logical;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002158 sbio->dev = spage->dev;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002159 bio = sbio->bio;
2160 if (!bio) {
Chris Mason9be33952013-05-17 18:30:14 -04002161 bio = btrfs_io_bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002162 if (!bio)
2163 return -ENOMEM;
2164 sbio->bio = bio;
2165 }
Arne Jansen69f4cb52011-11-11 08:17:10 -05002166
2167 bio->bi_private = sbio;
2168 bio->bi_end_io = scrub_bio_end_io;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002169 bio->bi_bdev = sbio->dev->bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002170 bio->bi_iter.bi_sector = sbio->physical >> 9;
Arne Jansen69f4cb52011-11-11 08:17:10 -05002171 sbio->err = 0;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002172 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2173 spage->physical ||
2174 sbio->logical + sbio->page_count * PAGE_SIZE !=
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002175 spage->logical ||
2176 sbio->dev != spage->dev) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002177 scrub_submit(sctx);
Arne Jansen69f4cb52011-11-11 08:17:10 -05002178 goto again;
2179 }
2180
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002181 sbio->pagev[sbio->page_count] = spage;
2182 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2183 if (ret != PAGE_SIZE) {
2184 if (sbio->page_count < 1) {
2185 bio_put(sbio->bio);
2186 sbio->bio = NULL;
2187 return -EIO;
2188 }
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002189 scrub_submit(sctx);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002190 goto again;
Arne Jansena2de7332011-03-08 14:14:00 +01002191 }
Arne Jansen1bc87792011-05-28 21:57:55 +02002192
Stefan Behrensff023aa2012-11-06 11:43:11 +01002193 scrub_block_get(sblock); /* one for the page added to the bio */
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002194 atomic_inc(&sblock->outstanding_pages);
2195 sbio->page_count++;
Stefan Behrensff023aa2012-11-06 11:43:11 +01002196 if (sbio->page_count == sctx->pages_per_rd_bio)
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002197 scrub_submit(sctx);
Arne Jansena2de7332011-03-08 14:14:00 +01002198
2199 return 0;
2200}
2201
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002202static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002203 u64 physical, struct btrfs_device *dev, u64 flags,
Stefan Behrensff023aa2012-11-06 11:43:11 +01002204 u64 gen, int mirror_num, u8 *csum, int force,
2205 u64 physical_for_dev_replace)
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002206{
2207 struct scrub_block *sblock;
2208 int index;
2209
2210 sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2211 if (!sblock) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002212 spin_lock(&sctx->stat_lock);
2213 sctx->stat.malloc_errors++;
2214 spin_unlock(&sctx->stat_lock);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002215 return -ENOMEM;
2216 }
2217
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002218 /* one ref inside this function, plus one for each page added to
2219 * a bio later on */
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002220 atomic_set(&sblock->ref_count, 1);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002221 sblock->sctx = sctx;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002222 sblock->no_io_error_seen = 1;
2223
2224 for (index = 0; len > 0; index++) {
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002225 struct scrub_page *spage;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002226 u64 l = min_t(u64, len, PAGE_SIZE);
2227
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002228 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2229 if (!spage) {
2230leave_nomem:
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002231 spin_lock(&sctx->stat_lock);
2232 sctx->stat.malloc_errors++;
2233 spin_unlock(&sctx->stat_lock);
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002234 scrub_block_put(sblock);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002235 return -ENOMEM;
2236 }
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002237 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2238 scrub_page_get(spage);
2239 sblock->pagev[index] = spage;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002240 spage->sblock = sblock;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002241 spage->dev = dev;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002242 spage->flags = flags;
2243 spage->generation = gen;
2244 spage->logical = logical;
2245 spage->physical = physical;
Stefan Behrensff023aa2012-11-06 11:43:11 +01002246 spage->physical_for_dev_replace = physical_for_dev_replace;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002247 spage->mirror_num = mirror_num;
2248 if (csum) {
2249 spage->have_csum = 1;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002250 memcpy(spage->csum, csum, sctx->csum_size);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002251 } else {
2252 spage->have_csum = 0;
2253 }
2254 sblock->page_count++;
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002255 spage->page = alloc_page(GFP_NOFS);
2256 if (!spage->page)
2257 goto leave_nomem;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002258 len -= l;
2259 logical += l;
2260 physical += l;
Stefan Behrensff023aa2012-11-06 11:43:11 +01002261 physical_for_dev_replace += l;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002262 }
2263
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002264 WARN_ON(sblock->page_count == 0);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002265 for (index = 0; index < sblock->page_count; index++) {
Stefan Behrens7a9e9982012-11-02 14:58:04 +01002266 struct scrub_page *spage = sblock->pagev[index];
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002267 int ret;
2268
Stefan Behrensff023aa2012-11-06 11:43:11 +01002269 ret = scrub_add_page_to_rd_bio(sctx, spage);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002270 if (ret) {
2271 scrub_block_put(sblock);
2272 return ret;
2273 }
2274 }
2275
2276 if (force)
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002277 scrub_submit(sctx);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002278
2279 /* last one frees, either here or in bio completion for last page */
2280 scrub_block_put(sblock);
2281 return 0;
2282}
2283
2284static void scrub_bio_end_io(struct bio *bio, int err)
2285{
2286 struct scrub_bio *sbio = bio->bi_private;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002287 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002288
2289 sbio->err = err;
2290 sbio->bio = bio;
2291
Qu Wenruo0339ef22014-02-28 10:46:17 +08002292 btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002293}
2294
2295static void scrub_bio_end_io_worker(struct btrfs_work *work)
2296{
2297 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002298 struct scrub_ctx *sctx = sbio->sctx;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002299 int i;
2300
Stefan Behrensff023aa2012-11-06 11:43:11 +01002301 BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002302 if (sbio->err) {
2303 for (i = 0; i < sbio->page_count; i++) {
2304 struct scrub_page *spage = sbio->pagev[i];
2305
2306 spage->io_error = 1;
2307 spage->sblock->no_io_error_seen = 0;
2308 }
2309 }
2310
2311 /* now complete the scrub_block items that have all pages completed */
2312 for (i = 0; i < sbio->page_count; i++) {
2313 struct scrub_page *spage = sbio->pagev[i];
2314 struct scrub_block *sblock = spage->sblock;
2315
2316 if (atomic_dec_and_test(&sblock->outstanding_pages))
2317 scrub_block_complete(sblock);
2318 scrub_block_put(sblock);
2319 }
2320
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002321 bio_put(sbio->bio);
2322 sbio->bio = NULL;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002323 spin_lock(&sctx->list_lock);
2324 sbio->next_free = sctx->first_free;
2325 sctx->first_free = sbio->index;
2326 spin_unlock(&sctx->list_lock);
Stefan Behrensff023aa2012-11-06 11:43:11 +01002327
2328 if (sctx->is_dev_replace &&
2329 atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2330 mutex_lock(&sctx->wr_ctx.wr_lock);
2331 scrub_wr_submit(sctx);
2332 mutex_unlock(&sctx->wr_ctx.wr_lock);
2333 }
2334
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01002335 scrub_pending_bio_dec(sctx);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002336}
2337
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002338static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2339 unsigned long *bitmap,
2340 u64 start, u64 len)
2341{
2342 int offset;
2343 int nsectors;
2344 int sectorsize = sparity->sctx->dev_root->sectorsize;
2345
2346 if (len >= sparity->stripe_len) {
2347 bitmap_set(bitmap, 0, sparity->nsectors);
2348 return;
2349 }
2350
2351 start -= sparity->logic_start;
2352 offset = (int)do_div(start, sparity->stripe_len);
2353 offset /= sectorsize;
2354 nsectors = (int)len / sectorsize;
2355
2356 if (offset + nsectors <= sparity->nsectors) {
2357 bitmap_set(bitmap, offset, nsectors);
2358 return;
2359 }
2360
2361 bitmap_set(bitmap, offset, sparity->nsectors - offset);
2362 bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2363}
2364
2365static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2366 u64 start, u64 len)
2367{
2368 __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2369}
2370
2371static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2372 u64 start, u64 len)
2373{
2374 __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2375}
2376
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002377static void scrub_block_complete(struct scrub_block *sblock)
2378{
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002379 int corrupted = 0;
2380
Stefan Behrensff023aa2012-11-06 11:43:11 +01002381 if (!sblock->no_io_error_seen) {
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002382 corrupted = 1;
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002383 scrub_handle_errored_block(sblock);
Stefan Behrensff023aa2012-11-06 11:43:11 +01002384 } else {
2385 /*
2386 * if has checksum error, write via repair mechanism in
2387 * dev replace case, otherwise write here in dev replace
2388 * case.
2389 */
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002390 corrupted = scrub_checksum(sblock);
2391 if (!corrupted && sblock->sctx->is_dev_replace)
Stefan Behrensff023aa2012-11-06 11:43:11 +01002392 scrub_write_block_to_dev_replace(sblock);
2393 }
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002394
2395 if (sblock->sparity && corrupted && !sblock->data_corrected) {
2396 u64 start = sblock->pagev[0]->logical;
2397 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2398 PAGE_SIZE;
2399
2400 scrub_parity_mark_sectors_error(sblock->sparity,
2401 start, end - start);
2402 }
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002403}
2404
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002405static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
Arne Jansena2de7332011-03-08 14:14:00 +01002406 u8 *csum)
2407{
2408 struct btrfs_ordered_sum *sum = NULL;
Miao Xief51a4a12013-06-19 10:36:09 +08002409 unsigned long index;
Arne Jansena2de7332011-03-08 14:14:00 +01002410 unsigned long num_sectors;
Arne Jansena2de7332011-03-08 14:14:00 +01002411
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002412 while (!list_empty(&sctx->csum_list)) {
2413 sum = list_first_entry(&sctx->csum_list,
Arne Jansena2de7332011-03-08 14:14:00 +01002414 struct btrfs_ordered_sum, list);
2415 if (sum->bytenr > logical)
2416 return 0;
2417 if (sum->bytenr + sum->len > logical)
2418 break;
2419
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002420 ++sctx->stat.csum_discards;
Arne Jansena2de7332011-03-08 14:14:00 +01002421 list_del(&sum->list);
2422 kfree(sum);
2423 sum = NULL;
2424 }
2425 if (!sum)
2426 return 0;
2427
Miao Xief51a4a12013-06-19 10:36:09 +08002428 index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002429 num_sectors = sum->len / sctx->sectorsize;
Miao Xief51a4a12013-06-19 10:36:09 +08002430 memcpy(csum, sum->sums + index, sctx->csum_size);
2431 if (index == num_sectors - 1) {
Arne Jansena2de7332011-03-08 14:14:00 +01002432 list_del(&sum->list);
2433 kfree(sum);
2434 }
Miao Xief51a4a12013-06-19 10:36:09 +08002435 return 1;
Arne Jansena2de7332011-03-08 14:14:00 +01002436}
2437
2438/* scrub extent tries to collect up to 64 kB for each bio */
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002439static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002440 u64 physical, struct btrfs_device *dev, u64 flags,
Stefan Behrensff023aa2012-11-06 11:43:11 +01002441 u64 gen, int mirror_num, u64 physical_for_dev_replace)
Arne Jansena2de7332011-03-08 14:14:00 +01002442{
2443 int ret;
2444 u8 csum[BTRFS_CSUM_SIZE];
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002445 u32 blocksize;
2446
2447 if (flags & BTRFS_EXTENT_FLAG_DATA) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002448 blocksize = sctx->sectorsize;
2449 spin_lock(&sctx->stat_lock);
2450 sctx->stat.data_extents_scrubbed++;
2451 sctx->stat.data_bytes_scrubbed += len;
2452 spin_unlock(&sctx->stat_lock);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002453 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002454 blocksize = sctx->nodesize;
2455 spin_lock(&sctx->stat_lock);
2456 sctx->stat.tree_extents_scrubbed++;
2457 sctx->stat.tree_bytes_scrubbed += len;
2458 spin_unlock(&sctx->stat_lock);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002459 } else {
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002460 blocksize = sctx->sectorsize;
Stefan Behrensff023aa2012-11-06 11:43:11 +01002461 WARN_ON(1);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002462 }
Arne Jansena2de7332011-03-08 14:14:00 +01002463
2464 while (len) {
Stefan Behrensb5d67f62012-03-27 14:21:27 -04002465 u64 l = min_t(u64, len, blocksize);
Arne Jansena2de7332011-03-08 14:14:00 +01002466 int have_csum = 0;
2467
2468 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2469 /* push csums to sbio */
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002470 have_csum = scrub_find_csum(sctx, logical, l, csum);
Arne Jansena2de7332011-03-08 14:14:00 +01002471 if (have_csum == 0)
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002472 ++sctx->stat.no_csum;
Stefan Behrensff023aa2012-11-06 11:43:11 +01002473 if (sctx->is_dev_replace && !have_csum) {
2474 ret = copy_nocow_pages(sctx, logical, l,
2475 mirror_num,
2476 physical_for_dev_replace);
2477 goto behind_scrub_pages;
2478 }
Arne Jansena2de7332011-03-08 14:14:00 +01002479 }
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002480 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
Stefan Behrensff023aa2012-11-06 11:43:11 +01002481 mirror_num, have_csum ? csum : NULL, 0,
2482 physical_for_dev_replace);
2483behind_scrub_pages:
Arne Jansena2de7332011-03-08 14:14:00 +01002484 if (ret)
2485 return ret;
2486 len -= l;
2487 logical += l;
2488 physical += l;
Stefan Behrensff023aa2012-11-06 11:43:11 +01002489 physical_for_dev_replace += l;
Arne Jansena2de7332011-03-08 14:14:00 +01002490 }
2491 return 0;
2492}
2493
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002494static int scrub_pages_for_parity(struct scrub_parity *sparity,
2495 u64 logical, u64 len,
2496 u64 physical, struct btrfs_device *dev,
2497 u64 flags, u64 gen, int mirror_num, u8 *csum)
2498{
2499 struct scrub_ctx *sctx = sparity->sctx;
2500 struct scrub_block *sblock;
2501 int index;
2502
2503 sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2504 if (!sblock) {
2505 spin_lock(&sctx->stat_lock);
2506 sctx->stat.malloc_errors++;
2507 spin_unlock(&sctx->stat_lock);
2508 return -ENOMEM;
2509 }
2510
2511 /* one ref inside this function, plus one for each page added to
2512 * a bio later on */
2513 atomic_set(&sblock->ref_count, 1);
2514 sblock->sctx = sctx;
2515 sblock->no_io_error_seen = 1;
2516 sblock->sparity = sparity;
2517 scrub_parity_get(sparity);
2518
2519 for (index = 0; len > 0; index++) {
2520 struct scrub_page *spage;
2521 u64 l = min_t(u64, len, PAGE_SIZE);
2522
2523 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2524 if (!spage) {
2525leave_nomem:
2526 spin_lock(&sctx->stat_lock);
2527 sctx->stat.malloc_errors++;
2528 spin_unlock(&sctx->stat_lock);
2529 scrub_block_put(sblock);
2530 return -ENOMEM;
2531 }
2532 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2533 /* For scrub block */
2534 scrub_page_get(spage);
2535 sblock->pagev[index] = spage;
2536 /* For scrub parity */
2537 scrub_page_get(spage);
2538 list_add_tail(&spage->list, &sparity->spages);
2539 spage->sblock = sblock;
2540 spage->dev = dev;
2541 spage->flags = flags;
2542 spage->generation = gen;
2543 spage->logical = logical;
2544 spage->physical = physical;
2545 spage->mirror_num = mirror_num;
2546 if (csum) {
2547 spage->have_csum = 1;
2548 memcpy(spage->csum, csum, sctx->csum_size);
2549 } else {
2550 spage->have_csum = 0;
2551 }
2552 sblock->page_count++;
2553 spage->page = alloc_page(GFP_NOFS);
2554 if (!spage->page)
2555 goto leave_nomem;
2556 len -= l;
2557 logical += l;
2558 physical += l;
2559 }
2560
2561 WARN_ON(sblock->page_count == 0);
2562 for (index = 0; index < sblock->page_count; index++) {
2563 struct scrub_page *spage = sblock->pagev[index];
2564 int ret;
2565
2566 ret = scrub_add_page_to_rd_bio(sctx, spage);
2567 if (ret) {
2568 scrub_block_put(sblock);
2569 return ret;
2570 }
2571 }
2572
2573 /* last one frees, either here or in bio completion for last page */
2574 scrub_block_put(sblock);
2575 return 0;
2576}
2577
2578static int scrub_extent_for_parity(struct scrub_parity *sparity,
2579 u64 logical, u64 len,
2580 u64 physical, struct btrfs_device *dev,
2581 u64 flags, u64 gen, int mirror_num)
2582{
2583 struct scrub_ctx *sctx = sparity->sctx;
2584 int ret;
2585 u8 csum[BTRFS_CSUM_SIZE];
2586 u32 blocksize;
2587
2588 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2589 blocksize = sctx->sectorsize;
2590 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2591 blocksize = sctx->nodesize;
2592 } else {
2593 blocksize = sctx->sectorsize;
2594 WARN_ON(1);
2595 }
2596
2597 while (len) {
2598 u64 l = min_t(u64, len, blocksize);
2599 int have_csum = 0;
2600
2601 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2602 /* push csums to sbio */
2603 have_csum = scrub_find_csum(sctx, logical, l, csum);
2604 if (have_csum == 0)
2605 goto skip;
2606 }
2607 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2608 flags, gen, mirror_num,
2609 have_csum ? csum : NULL);
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002610 if (ret)
2611 return ret;
Dan Carpenter6b6d24b2014-12-12 22:30:00 +03002612skip:
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002613 len -= l;
2614 logical += l;
2615 physical += l;
2616 }
2617 return 0;
2618}
2619
Wang Shilong3b080b22014-04-01 18:01:43 +08002620/*
2621 * Given a physical address, this will calculate it's
2622 * logical offset. if this is a parity stripe, it will return
2623 * the most left data stripe's logical offset.
2624 *
2625 * return 0 if it is a data stripe, 1 means parity stripe.
2626 */
2627static int get_raid56_logic_offset(u64 physical, int num,
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002628 struct map_lookup *map, u64 *offset,
2629 u64 *stripe_start)
Wang Shilong3b080b22014-04-01 18:01:43 +08002630{
2631 int i;
2632 int j = 0;
2633 u64 stripe_nr;
2634 u64 last_offset;
2635 int stripe_index;
2636 int rot;
2637
2638 last_offset = (physical - map->stripes[num].physical) *
2639 nr_data_stripes(map);
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002640 if (stripe_start)
2641 *stripe_start = last_offset;
2642
Wang Shilong3b080b22014-04-01 18:01:43 +08002643 *offset = last_offset;
2644 for (i = 0; i < nr_data_stripes(map); i++) {
2645 *offset = last_offset + i * map->stripe_len;
2646
2647 stripe_nr = *offset;
2648 do_div(stripe_nr, map->stripe_len);
2649 do_div(stripe_nr, nr_data_stripes(map));
2650
2651 /* Work out the disk rotation on this stripe-set */
2652 rot = do_div(stripe_nr, map->num_stripes);
2653 /* calculate which stripe this data locates */
2654 rot += i;
Wang Shilonge4fbaee2014-04-11 18:32:25 +08002655 stripe_index = rot % map->num_stripes;
Wang Shilong3b080b22014-04-01 18:01:43 +08002656 if (stripe_index == num)
2657 return 0;
2658 if (stripe_index < num)
2659 j++;
2660 }
2661 *offset = last_offset + j * map->stripe_len;
2662 return 1;
2663}
2664
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002665static void scrub_free_parity(struct scrub_parity *sparity)
2666{
2667 struct scrub_ctx *sctx = sparity->sctx;
2668 struct scrub_page *curr, *next;
2669 int nbits;
2670
2671 nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
2672 if (nbits) {
2673 spin_lock(&sctx->stat_lock);
2674 sctx->stat.read_errors += nbits;
2675 sctx->stat.uncorrectable_errors += nbits;
2676 spin_unlock(&sctx->stat_lock);
2677 }
2678
2679 list_for_each_entry_safe(curr, next, &sparity->spages, list) {
2680 list_del_init(&curr->list);
2681 scrub_page_put(curr);
2682 }
2683
2684 kfree(sparity);
2685}
2686
2687static void scrub_parity_bio_endio(struct bio *bio, int error)
2688{
2689 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
2690 struct scrub_ctx *sctx = sparity->sctx;
2691
2692 if (error)
2693 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2694 sparity->nsectors);
2695
2696 scrub_free_parity(sparity);
2697 scrub_pending_bio_dec(sctx);
2698 bio_put(bio);
2699}
2700
2701static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
2702{
2703 struct scrub_ctx *sctx = sparity->sctx;
2704 struct bio *bio;
2705 struct btrfs_raid_bio *rbio;
2706 struct scrub_page *spage;
2707 struct btrfs_bio *bbio = NULL;
2708 u64 *raid_map = NULL;
2709 u64 length;
2710 int ret;
2711
2712 if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
2713 sparity->nsectors))
2714 goto out;
2715
2716 length = sparity->logic_end - sparity->logic_start + 1;
Miao Xie76035972014-11-14 17:45:42 +08002717 ret = btrfs_map_sblock(sctx->dev_root->fs_info, WRITE,
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002718 sparity->logic_start,
2719 &length, &bbio, 0, &raid_map);
2720 if (ret || !bbio || !raid_map)
2721 goto bbio_out;
2722
2723 bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2724 if (!bio)
2725 goto bbio_out;
2726
2727 bio->bi_iter.bi_sector = sparity->logic_start >> 9;
2728 bio->bi_private = sparity;
2729 bio->bi_end_io = scrub_parity_bio_endio;
2730
2731 rbio = raid56_parity_alloc_scrub_rbio(sctx->dev_root, bio, bbio,
2732 raid_map, length,
2733 sparity->scrub_dev,
2734 sparity->dbitmap,
2735 sparity->nsectors);
2736 if (!rbio)
2737 goto rbio_out;
2738
2739 list_for_each_entry(spage, &sparity->spages, list)
2740 raid56_parity_add_scrub_pages(rbio, spage->page,
2741 spage->logical);
2742
2743 scrub_pending_bio_inc(sctx);
2744 raid56_parity_submit_scrub_rbio(rbio);
2745 return;
2746
2747rbio_out:
2748 bio_put(bio);
2749bbio_out:
2750 kfree(bbio);
2751 kfree(raid_map);
2752 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2753 sparity->nsectors);
2754 spin_lock(&sctx->stat_lock);
2755 sctx->stat.malloc_errors++;
2756 spin_unlock(&sctx->stat_lock);
2757out:
2758 scrub_free_parity(sparity);
2759}
2760
2761static inline int scrub_calc_parity_bitmap_len(int nsectors)
2762{
2763 return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * (BITS_PER_LONG / 8);
2764}
2765
2766static void scrub_parity_get(struct scrub_parity *sparity)
2767{
2768 atomic_inc(&sparity->ref_count);
2769}
2770
2771static void scrub_parity_put(struct scrub_parity *sparity)
2772{
2773 if (!atomic_dec_and_test(&sparity->ref_count))
2774 return;
2775
2776 scrub_parity_check_and_repair(sparity);
2777}
2778
2779static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
2780 struct map_lookup *map,
2781 struct btrfs_device *sdev,
2782 struct btrfs_path *path,
2783 u64 logic_start,
2784 u64 logic_end)
2785{
2786 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2787 struct btrfs_root *root = fs_info->extent_root;
2788 struct btrfs_root *csum_root = fs_info->csum_root;
2789 struct btrfs_extent_item *extent;
2790 u64 flags;
2791 int ret;
2792 int slot;
2793 struct extent_buffer *l;
2794 struct btrfs_key key;
2795 u64 generation;
2796 u64 extent_logical;
2797 u64 extent_physical;
2798 u64 extent_len;
2799 struct btrfs_device *extent_dev;
2800 struct scrub_parity *sparity;
2801 int nsectors;
2802 int bitmap_len;
2803 int extent_mirror_num;
2804 int stop_loop = 0;
2805
2806 nsectors = map->stripe_len / root->sectorsize;
2807 bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
2808 sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
2809 GFP_NOFS);
2810 if (!sparity) {
2811 spin_lock(&sctx->stat_lock);
2812 sctx->stat.malloc_errors++;
2813 spin_unlock(&sctx->stat_lock);
2814 return -ENOMEM;
2815 }
2816
2817 sparity->stripe_len = map->stripe_len;
2818 sparity->nsectors = nsectors;
2819 sparity->sctx = sctx;
2820 sparity->scrub_dev = sdev;
2821 sparity->logic_start = logic_start;
2822 sparity->logic_end = logic_end;
2823 atomic_set(&sparity->ref_count, 1);
2824 INIT_LIST_HEAD(&sparity->spages);
2825 sparity->dbitmap = sparity->bitmap;
2826 sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
2827
2828 ret = 0;
2829 while (logic_start < logic_end) {
2830 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2831 key.type = BTRFS_METADATA_ITEM_KEY;
2832 else
2833 key.type = BTRFS_EXTENT_ITEM_KEY;
2834 key.objectid = logic_start;
2835 key.offset = (u64)-1;
2836
2837 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2838 if (ret < 0)
2839 goto out;
2840
2841 if (ret > 0) {
2842 ret = btrfs_previous_extent_item(root, path, 0);
2843 if (ret < 0)
2844 goto out;
2845 if (ret > 0) {
2846 btrfs_release_path(path);
2847 ret = btrfs_search_slot(NULL, root, &key,
2848 path, 0, 0);
2849 if (ret < 0)
2850 goto out;
2851 }
2852 }
2853
2854 stop_loop = 0;
2855 while (1) {
2856 u64 bytes;
2857
2858 l = path->nodes[0];
2859 slot = path->slots[0];
2860 if (slot >= btrfs_header_nritems(l)) {
2861 ret = btrfs_next_leaf(root, path);
2862 if (ret == 0)
2863 continue;
2864 if (ret < 0)
2865 goto out;
2866
2867 stop_loop = 1;
2868 break;
2869 }
2870 btrfs_item_key_to_cpu(l, &key, slot);
2871
2872 if (key.type == BTRFS_METADATA_ITEM_KEY)
2873 bytes = root->nodesize;
2874 else
2875 bytes = key.offset;
2876
2877 if (key.objectid + bytes <= logic_start)
2878 goto next;
2879
2880 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2881 key.type != BTRFS_METADATA_ITEM_KEY)
2882 goto next;
2883
2884 if (key.objectid > logic_end) {
2885 stop_loop = 1;
2886 break;
2887 }
2888
2889 while (key.objectid >= logic_start + map->stripe_len)
2890 logic_start += map->stripe_len;
2891
2892 extent = btrfs_item_ptr(l, slot,
2893 struct btrfs_extent_item);
2894 flags = btrfs_extent_flags(l, extent);
2895 generation = btrfs_extent_generation(l, extent);
2896
2897 if (key.objectid < logic_start &&
2898 (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
2899 btrfs_err(fs_info,
2900 "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
2901 key.objectid, logic_start);
2902 goto next;
2903 }
2904again:
2905 extent_logical = key.objectid;
2906 extent_len = bytes;
2907
2908 if (extent_logical < logic_start) {
2909 extent_len -= logic_start - extent_logical;
2910 extent_logical = logic_start;
2911 }
2912
2913 if (extent_logical + extent_len >
2914 logic_start + map->stripe_len)
2915 extent_len = logic_start + map->stripe_len -
2916 extent_logical;
2917
2918 scrub_parity_mark_sectors_data(sparity, extent_logical,
2919 extent_len);
2920
2921 scrub_remap_extent(fs_info, extent_logical,
2922 extent_len, &extent_physical,
2923 &extent_dev,
2924 &extent_mirror_num);
2925
2926 ret = btrfs_lookup_csums_range(csum_root,
2927 extent_logical,
2928 extent_logical + extent_len - 1,
2929 &sctx->csum_list, 1);
2930 if (ret)
2931 goto out;
2932
2933 ret = scrub_extent_for_parity(sparity, extent_logical,
2934 extent_len,
2935 extent_physical,
2936 extent_dev, flags,
2937 generation,
2938 extent_mirror_num);
2939 if (ret)
2940 goto out;
2941
2942 scrub_free_csums(sctx);
2943 if (extent_logical + extent_len <
2944 key.objectid + bytes) {
2945 logic_start += map->stripe_len;
2946
2947 if (logic_start >= logic_end) {
2948 stop_loop = 1;
2949 break;
2950 }
2951
2952 if (logic_start < key.objectid + bytes) {
2953 cond_resched();
2954 goto again;
2955 }
2956 }
2957next:
2958 path->slots[0]++;
2959 }
2960
2961 btrfs_release_path(path);
2962
2963 if (stop_loop)
2964 break;
2965
2966 logic_start += map->stripe_len;
2967 }
2968out:
2969 if (ret < 0)
2970 scrub_parity_mark_sectors_error(sparity, logic_start,
2971 logic_end - logic_start + 1);
2972 scrub_parity_put(sparity);
2973 scrub_submit(sctx);
2974 mutex_lock(&sctx->wr_ctx.wr_lock);
2975 scrub_wr_submit(sctx);
2976 mutex_unlock(&sctx->wr_ctx.wr_lock);
2977
2978 btrfs_release_path(path);
2979 return ret < 0 ? ret : 0;
2980}
2981
Stefan Behrensd9d181c2012-11-02 09:58:09 +01002982static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002983 struct map_lookup *map,
2984 struct btrfs_device *scrub_dev,
Stefan Behrensff023aa2012-11-06 11:43:11 +01002985 int num, u64 base, u64 length,
2986 int is_dev_replace)
Arne Jansena2de7332011-03-08 14:14:00 +01002987{
Miao Xie5a6ac9e2014-11-06 17:20:58 +08002988 struct btrfs_path *path, *ppath;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01002989 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
Arne Jansena2de7332011-03-08 14:14:00 +01002990 struct btrfs_root *root = fs_info->extent_root;
2991 struct btrfs_root *csum_root = fs_info->csum_root;
2992 struct btrfs_extent_item *extent;
Arne Jansene7786c32011-05-28 20:58:38 +00002993 struct blk_plug plug;
Arne Jansena2de7332011-03-08 14:14:00 +01002994 u64 flags;
2995 int ret;
2996 int slot;
Arne Jansena2de7332011-03-08 14:14:00 +01002997 u64 nstripes;
Arne Jansena2de7332011-03-08 14:14:00 +01002998 struct extent_buffer *l;
2999 struct btrfs_key key;
3000 u64 physical;
3001 u64 logical;
Liu Bo625f1c8d2013-04-27 02:56:57 +00003002 u64 logic_end;
Wang Shilong3b080b22014-04-01 18:01:43 +08003003 u64 physical_end;
Arne Jansena2de7332011-03-08 14:14:00 +01003004 u64 generation;
Jan Schmidte12fa9c2011-06-17 15:55:21 +02003005 int mirror_num;
Arne Jansen7a262852011-06-10 12:39:23 +02003006 struct reada_control *reada1;
3007 struct reada_control *reada2;
3008 struct btrfs_key key_start;
3009 struct btrfs_key key_end;
Arne Jansena2de7332011-03-08 14:14:00 +01003010 u64 increment = map->stripe_len;
3011 u64 offset;
Stefan Behrensff023aa2012-11-06 11:43:11 +01003012 u64 extent_logical;
3013 u64 extent_physical;
3014 u64 extent_len;
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003015 u64 stripe_logical;
3016 u64 stripe_end;
Stefan Behrensff023aa2012-11-06 11:43:11 +01003017 struct btrfs_device *extent_dev;
3018 int extent_mirror_num;
Wang Shilong3b080b22014-04-01 18:01:43 +08003019 int stop_loop = 0;
David Woodhouse53b381b2013-01-29 18:40:14 -05003020
Arne Jansena2de7332011-03-08 14:14:00 +01003021 nstripes = length;
Wang Shilong3b080b22014-04-01 18:01:43 +08003022 physical = map->stripes[num].physical;
Arne Jansena2de7332011-03-08 14:14:00 +01003023 offset = 0;
3024 do_div(nstripes, map->stripe_len);
3025 if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3026 offset = map->stripe_len * num;
3027 increment = map->stripe_len * map->num_stripes;
Jan Schmidt193ea742011-06-13 19:56:54 +02003028 mirror_num = 1;
Arne Jansena2de7332011-03-08 14:14:00 +01003029 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3030 int factor = map->num_stripes / map->sub_stripes;
3031 offset = map->stripe_len * (num / map->sub_stripes);
3032 increment = map->stripe_len * factor;
Jan Schmidt193ea742011-06-13 19:56:54 +02003033 mirror_num = num % map->sub_stripes + 1;
Arne Jansena2de7332011-03-08 14:14:00 +01003034 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
3035 increment = map->stripe_len;
Jan Schmidt193ea742011-06-13 19:56:54 +02003036 mirror_num = num % map->num_stripes + 1;
Arne Jansena2de7332011-03-08 14:14:00 +01003037 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
3038 increment = map->stripe_len;
Jan Schmidt193ea742011-06-13 19:56:54 +02003039 mirror_num = num % map->num_stripes + 1;
Wang Shilong3b080b22014-04-01 18:01:43 +08003040 } else if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
3041 BTRFS_BLOCK_GROUP_RAID6)) {
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003042 get_raid56_logic_offset(physical, num, map, &offset, NULL);
Wang Shilong3b080b22014-04-01 18:01:43 +08003043 increment = map->stripe_len * nr_data_stripes(map);
3044 mirror_num = 1;
Arne Jansena2de7332011-03-08 14:14:00 +01003045 } else {
3046 increment = map->stripe_len;
Jan Schmidt193ea742011-06-13 19:56:54 +02003047 mirror_num = 1;
Arne Jansena2de7332011-03-08 14:14:00 +01003048 }
3049
3050 path = btrfs_alloc_path();
3051 if (!path)
3052 return -ENOMEM;
3053
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003054 ppath = btrfs_alloc_path();
3055 if (!ppath) {
3056 btrfs_free_path(ppath);
3057 return -ENOMEM;
3058 }
3059
Stefan Behrensb5d67f62012-03-27 14:21:27 -04003060 /*
3061 * work on commit root. The related disk blocks are static as
3062 * long as COW is applied. This means, it is save to rewrite
3063 * them to repair disk errors without any race conditions
3064 */
Arne Jansena2de7332011-03-08 14:14:00 +01003065 path->search_commit_root = 1;
3066 path->skip_locking = 1;
3067
3068 /*
Arne Jansen7a262852011-06-10 12:39:23 +02003069 * trigger the readahead for extent tree csum tree and wait for
3070 * completion. During readahead, the scrub is officially paused
3071 * to not hold off transaction commits
Arne Jansena2de7332011-03-08 14:14:00 +01003072 */
3073 logical = base + offset;
Wang Shilong3b080b22014-04-01 18:01:43 +08003074 physical_end = physical + nstripes * map->stripe_len;
3075 if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
3076 BTRFS_BLOCK_GROUP_RAID6)) {
3077 get_raid56_logic_offset(physical_end, num,
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003078 map, &logic_end, NULL);
Wang Shilong3b080b22014-04-01 18:01:43 +08003079 logic_end += base;
3080 } else {
3081 logic_end = logical + increment * nstripes;
3082 }
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003083 wait_event(sctx->list_wait,
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01003084 atomic_read(&sctx->bios_in_flight) == 0);
Wang Shilongcb7ab022013-12-04 21:16:53 +08003085 scrub_blocked_if_needed(fs_info);
Arne Jansena2de7332011-03-08 14:14:00 +01003086
Arne Jansen7a262852011-06-10 12:39:23 +02003087 /* FIXME it might be better to start readahead at commit root */
3088 key_start.objectid = logical;
3089 key_start.type = BTRFS_EXTENT_ITEM_KEY;
3090 key_start.offset = (u64)0;
Wang Shilong3b080b22014-04-01 18:01:43 +08003091 key_end.objectid = logic_end;
Josef Bacik3173a182013-03-07 14:22:04 -05003092 key_end.type = BTRFS_METADATA_ITEM_KEY;
3093 key_end.offset = (u64)-1;
Arne Jansen7a262852011-06-10 12:39:23 +02003094 reada1 = btrfs_reada_add(root, &key_start, &key_end);
Arne Jansena2de7332011-03-08 14:14:00 +01003095
Arne Jansen7a262852011-06-10 12:39:23 +02003096 key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3097 key_start.type = BTRFS_EXTENT_CSUM_KEY;
3098 key_start.offset = logical;
3099 key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3100 key_end.type = BTRFS_EXTENT_CSUM_KEY;
Wang Shilong3b080b22014-04-01 18:01:43 +08003101 key_end.offset = logic_end;
Arne Jansen7a262852011-06-10 12:39:23 +02003102 reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
Arne Jansena2de7332011-03-08 14:14:00 +01003103
Arne Jansen7a262852011-06-10 12:39:23 +02003104 if (!IS_ERR(reada1))
3105 btrfs_reada_wait(reada1);
3106 if (!IS_ERR(reada2))
3107 btrfs_reada_wait(reada2);
Arne Jansena2de7332011-03-08 14:14:00 +01003108
Arne Jansena2de7332011-03-08 14:14:00 +01003109
3110 /*
3111 * collect all data csums for the stripe to avoid seeking during
3112 * the scrub. This might currently (crc32) end up to be about 1MB
3113 */
Arne Jansene7786c32011-05-28 20:58:38 +00003114 blk_start_plug(&plug);
Arne Jansena2de7332011-03-08 14:14:00 +01003115
Arne Jansena2de7332011-03-08 14:14:00 +01003116 /*
3117 * now find all extents for each stripe and scrub them
3118 */
Arne Jansena2de7332011-03-08 14:14:00 +01003119 ret = 0;
Wang Shilong3b080b22014-04-01 18:01:43 +08003120 while (physical < physical_end) {
3121 /* for raid56, we skip parity stripe */
3122 if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
3123 BTRFS_BLOCK_GROUP_RAID6)) {
3124 ret = get_raid56_logic_offset(physical, num,
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003125 map, &logical, &stripe_logical);
Wang Shilong3b080b22014-04-01 18:01:43 +08003126 logical += base;
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003127 if (ret) {
3128 stripe_logical += base;
3129 stripe_end = stripe_logical + increment - 1;
3130 ret = scrub_raid56_parity(sctx, map, scrub_dev,
3131 ppath, stripe_logical,
3132 stripe_end);
3133 if (ret)
3134 goto out;
Wang Shilong3b080b22014-04-01 18:01:43 +08003135 goto skip;
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003136 }
Wang Shilong3b080b22014-04-01 18:01:43 +08003137 }
Arne Jansena2de7332011-03-08 14:14:00 +01003138 /*
3139 * canceled?
3140 */
3141 if (atomic_read(&fs_info->scrub_cancel_req) ||
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003142 atomic_read(&sctx->cancel_req)) {
Arne Jansena2de7332011-03-08 14:14:00 +01003143 ret = -ECANCELED;
3144 goto out;
3145 }
3146 /*
3147 * check to see if we have to pause
3148 */
3149 if (atomic_read(&fs_info->scrub_pause_req)) {
3150 /* push queued extents */
Stefan Behrensff023aa2012-11-06 11:43:11 +01003151 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003152 scrub_submit(sctx);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003153 mutex_lock(&sctx->wr_ctx.wr_lock);
3154 scrub_wr_submit(sctx);
3155 mutex_unlock(&sctx->wr_ctx.wr_lock);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003156 wait_event(sctx->list_wait,
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01003157 atomic_read(&sctx->bios_in_flight) == 0);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003158 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
Wang Shilong3cb09292013-12-04 21:15:19 +08003159 scrub_blocked_if_needed(fs_info);
Arne Jansena2de7332011-03-08 14:14:00 +01003160 }
3161
Wang Shilong7c76edb2014-01-12 21:38:32 +08003162 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3163 key.type = BTRFS_METADATA_ITEM_KEY;
3164 else
3165 key.type = BTRFS_EXTENT_ITEM_KEY;
Arne Jansena2de7332011-03-08 14:14:00 +01003166 key.objectid = logical;
Liu Bo625f1c8d2013-04-27 02:56:57 +00003167 key.offset = (u64)-1;
Arne Jansena2de7332011-03-08 14:14:00 +01003168
3169 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3170 if (ret < 0)
3171 goto out;
Josef Bacik3173a182013-03-07 14:22:04 -05003172
Arne Jansen8c510322011-06-03 10:09:26 +02003173 if (ret > 0) {
Wang Shilongade2e0b2014-01-12 21:38:33 +08003174 ret = btrfs_previous_extent_item(root, path, 0);
Arne Jansena2de7332011-03-08 14:14:00 +01003175 if (ret < 0)
3176 goto out;
Arne Jansen8c510322011-06-03 10:09:26 +02003177 if (ret > 0) {
3178 /* there's no smaller item, so stick with the
3179 * larger one */
3180 btrfs_release_path(path);
3181 ret = btrfs_search_slot(NULL, root, &key,
3182 path, 0, 0);
3183 if (ret < 0)
3184 goto out;
3185 }
Arne Jansena2de7332011-03-08 14:14:00 +01003186 }
3187
Liu Bo625f1c8d2013-04-27 02:56:57 +00003188 stop_loop = 0;
Arne Jansena2de7332011-03-08 14:14:00 +01003189 while (1) {
Josef Bacik3173a182013-03-07 14:22:04 -05003190 u64 bytes;
3191
Arne Jansena2de7332011-03-08 14:14:00 +01003192 l = path->nodes[0];
3193 slot = path->slots[0];
3194 if (slot >= btrfs_header_nritems(l)) {
3195 ret = btrfs_next_leaf(root, path);
3196 if (ret == 0)
3197 continue;
3198 if (ret < 0)
3199 goto out;
3200
Liu Bo625f1c8d2013-04-27 02:56:57 +00003201 stop_loop = 1;
Arne Jansena2de7332011-03-08 14:14:00 +01003202 break;
3203 }
3204 btrfs_item_key_to_cpu(l, &key, slot);
3205
Josef Bacik3173a182013-03-07 14:22:04 -05003206 if (key.type == BTRFS_METADATA_ITEM_KEY)
David Sterba707e8a02014-06-04 19:22:26 +02003207 bytes = root->nodesize;
Josef Bacik3173a182013-03-07 14:22:04 -05003208 else
3209 bytes = key.offset;
3210
3211 if (key.objectid + bytes <= logical)
Arne Jansena2de7332011-03-08 14:14:00 +01003212 goto next;
3213
Liu Bo625f1c8d2013-04-27 02:56:57 +00003214 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3215 key.type != BTRFS_METADATA_ITEM_KEY)
3216 goto next;
Arne Jansena2de7332011-03-08 14:14:00 +01003217
Liu Bo625f1c8d2013-04-27 02:56:57 +00003218 if (key.objectid >= logical + map->stripe_len) {
3219 /* out of this device extent */
3220 if (key.objectid >= logic_end)
3221 stop_loop = 1;
3222 break;
3223 }
Arne Jansena2de7332011-03-08 14:14:00 +01003224
3225 extent = btrfs_item_ptr(l, slot,
3226 struct btrfs_extent_item);
3227 flags = btrfs_extent_flags(l, extent);
3228 generation = btrfs_extent_generation(l, extent);
3229
3230 if (key.objectid < logical &&
3231 (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
Frank Holtonefe120a2013-12-20 11:37:06 -05003232 btrfs_err(fs_info,
3233 "scrub: tree block %llu spanning "
3234 "stripes, ignored. logical=%llu",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02003235 key.objectid, logical);
Arne Jansena2de7332011-03-08 14:14:00 +01003236 goto next;
3237 }
3238
Liu Bo625f1c8d2013-04-27 02:56:57 +00003239again:
3240 extent_logical = key.objectid;
3241 extent_len = bytes;
3242
Arne Jansena2de7332011-03-08 14:14:00 +01003243 /*
3244 * trim extent to this stripe
3245 */
Liu Bo625f1c8d2013-04-27 02:56:57 +00003246 if (extent_logical < logical) {
3247 extent_len -= logical - extent_logical;
3248 extent_logical = logical;
Arne Jansena2de7332011-03-08 14:14:00 +01003249 }
Liu Bo625f1c8d2013-04-27 02:56:57 +00003250 if (extent_logical + extent_len >
Arne Jansena2de7332011-03-08 14:14:00 +01003251 logical + map->stripe_len) {
Liu Bo625f1c8d2013-04-27 02:56:57 +00003252 extent_len = logical + map->stripe_len -
3253 extent_logical;
Arne Jansena2de7332011-03-08 14:14:00 +01003254 }
3255
Liu Bo625f1c8d2013-04-27 02:56:57 +00003256 extent_physical = extent_logical - logical + physical;
Stefan Behrensff023aa2012-11-06 11:43:11 +01003257 extent_dev = scrub_dev;
3258 extent_mirror_num = mirror_num;
3259 if (is_dev_replace)
3260 scrub_remap_extent(fs_info, extent_logical,
3261 extent_len, &extent_physical,
3262 &extent_dev,
3263 &extent_mirror_num);
Liu Bo625f1c8d2013-04-27 02:56:57 +00003264
3265 ret = btrfs_lookup_csums_range(csum_root, logical,
3266 logical + map->stripe_len - 1,
3267 &sctx->csum_list, 1);
Arne Jansena2de7332011-03-08 14:14:00 +01003268 if (ret)
3269 goto out;
3270
Liu Bo625f1c8d2013-04-27 02:56:57 +00003271 ret = scrub_extent(sctx, extent_logical, extent_len,
3272 extent_physical, extent_dev, flags,
3273 generation, extent_mirror_num,
Stefan Behrens115930c2013-07-04 16:14:23 +02003274 extent_logical - logical + physical);
Liu Bo625f1c8d2013-04-27 02:56:57 +00003275 if (ret)
3276 goto out;
3277
Josef Bacikd88d46c2013-06-10 12:59:04 +00003278 scrub_free_csums(sctx);
Liu Bo625f1c8d2013-04-27 02:56:57 +00003279 if (extent_logical + extent_len <
3280 key.objectid + bytes) {
Wang Shilong3b080b22014-04-01 18:01:43 +08003281 if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
3282 BTRFS_BLOCK_GROUP_RAID6)) {
3283 /*
3284 * loop until we find next data stripe
3285 * or we have finished all stripes.
3286 */
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003287loop:
3288 physical += map->stripe_len;
3289 ret = get_raid56_logic_offset(physical,
3290 num, map, &logical,
3291 &stripe_logical);
3292 logical += base;
3293
3294 if (ret && physical < physical_end) {
3295 stripe_logical += base;
3296 stripe_end = stripe_logical +
3297 increment - 1;
3298 ret = scrub_raid56_parity(sctx,
3299 map, scrub_dev, ppath,
3300 stripe_logical,
3301 stripe_end);
3302 if (ret)
3303 goto out;
3304 goto loop;
3305 }
Wang Shilong3b080b22014-04-01 18:01:43 +08003306 } else {
3307 physical += map->stripe_len;
3308 logical += increment;
3309 }
Liu Bo625f1c8d2013-04-27 02:56:57 +00003310 if (logical < key.objectid + bytes) {
3311 cond_resched();
3312 goto again;
3313 }
3314
Wang Shilong3b080b22014-04-01 18:01:43 +08003315 if (physical >= physical_end) {
Liu Bo625f1c8d2013-04-27 02:56:57 +00003316 stop_loop = 1;
3317 break;
3318 }
3319 }
Arne Jansena2de7332011-03-08 14:14:00 +01003320next:
3321 path->slots[0]++;
3322 }
Chris Mason71267332011-05-23 06:30:52 -04003323 btrfs_release_path(path);
Wang Shilong3b080b22014-04-01 18:01:43 +08003324skip:
Arne Jansena2de7332011-03-08 14:14:00 +01003325 logical += increment;
3326 physical += map->stripe_len;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003327 spin_lock(&sctx->stat_lock);
Liu Bo625f1c8d2013-04-27 02:56:57 +00003328 if (stop_loop)
3329 sctx->stat.last_physical = map->stripes[num].physical +
3330 length;
3331 else
3332 sctx->stat.last_physical = physical;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003333 spin_unlock(&sctx->stat_lock);
Liu Bo625f1c8d2013-04-27 02:56:57 +00003334 if (stop_loop)
3335 break;
Arne Jansena2de7332011-03-08 14:14:00 +01003336 }
Stefan Behrensff023aa2012-11-06 11:43:11 +01003337out:
Arne Jansena2de7332011-03-08 14:14:00 +01003338 /* push queued extents */
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003339 scrub_submit(sctx);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003340 mutex_lock(&sctx->wr_ctx.wr_lock);
3341 scrub_wr_submit(sctx);
3342 mutex_unlock(&sctx->wr_ctx.wr_lock);
Arne Jansena2de7332011-03-08 14:14:00 +01003343
Arne Jansene7786c32011-05-28 20:58:38 +00003344 blk_finish_plug(&plug);
Arne Jansena2de7332011-03-08 14:14:00 +01003345 btrfs_free_path(path);
Miao Xie5a6ac9e2014-11-06 17:20:58 +08003346 btrfs_free_path(ppath);
Arne Jansena2de7332011-03-08 14:14:00 +01003347 return ret < 0 ? ret : 0;
3348}
3349
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003350static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003351 struct btrfs_device *scrub_dev,
3352 u64 chunk_tree, u64 chunk_objectid,
3353 u64 chunk_offset, u64 length,
Stefan Behrensff023aa2012-11-06 11:43:11 +01003354 u64 dev_offset, int is_dev_replace)
Arne Jansena2de7332011-03-08 14:14:00 +01003355{
3356 struct btrfs_mapping_tree *map_tree =
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003357 &sctx->dev_root->fs_info->mapping_tree;
Arne Jansena2de7332011-03-08 14:14:00 +01003358 struct map_lookup *map;
3359 struct extent_map *em;
3360 int i;
Stefan Behrensff023aa2012-11-06 11:43:11 +01003361 int ret = 0;
Arne Jansena2de7332011-03-08 14:14:00 +01003362
3363 read_lock(&map_tree->map_tree.lock);
3364 em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
3365 read_unlock(&map_tree->map_tree.lock);
3366
3367 if (!em)
3368 return -EINVAL;
3369
3370 map = (struct map_lookup *)em->bdev;
3371 if (em->start != chunk_offset)
3372 goto out;
3373
3374 if (em->len < length)
3375 goto out;
3376
3377 for (i = 0; i < map->num_stripes; ++i) {
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003378 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
Arne Jansen859acaf2012-02-09 15:09:02 +01003379 map->stripes[i].physical == dev_offset) {
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003380 ret = scrub_stripe(sctx, map, scrub_dev, i,
Stefan Behrensff023aa2012-11-06 11:43:11 +01003381 chunk_offset, length,
3382 is_dev_replace);
Arne Jansena2de7332011-03-08 14:14:00 +01003383 if (ret)
3384 goto out;
3385 }
3386 }
3387out:
3388 free_extent_map(em);
3389
3390 return ret;
3391}
3392
3393static noinline_for_stack
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003394int scrub_enumerate_chunks(struct scrub_ctx *sctx,
Stefan Behrensff023aa2012-11-06 11:43:11 +01003395 struct btrfs_device *scrub_dev, u64 start, u64 end,
3396 int is_dev_replace)
Arne Jansena2de7332011-03-08 14:14:00 +01003397{
3398 struct btrfs_dev_extent *dev_extent = NULL;
3399 struct btrfs_path *path;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003400 struct btrfs_root *root = sctx->dev_root;
Arne Jansena2de7332011-03-08 14:14:00 +01003401 struct btrfs_fs_info *fs_info = root->fs_info;
3402 u64 length;
3403 u64 chunk_tree;
3404 u64 chunk_objectid;
3405 u64 chunk_offset;
3406 int ret;
3407 int slot;
3408 struct extent_buffer *l;
3409 struct btrfs_key key;
3410 struct btrfs_key found_key;
3411 struct btrfs_block_group_cache *cache;
Stefan Behrensff023aa2012-11-06 11:43:11 +01003412 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
Arne Jansena2de7332011-03-08 14:14:00 +01003413
3414 path = btrfs_alloc_path();
3415 if (!path)
3416 return -ENOMEM;
3417
3418 path->reada = 2;
3419 path->search_commit_root = 1;
3420 path->skip_locking = 1;
3421
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003422 key.objectid = scrub_dev->devid;
Arne Jansena2de7332011-03-08 14:14:00 +01003423 key.offset = 0ull;
3424 key.type = BTRFS_DEV_EXTENT_KEY;
3425
Arne Jansena2de7332011-03-08 14:14:00 +01003426 while (1) {
3427 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3428 if (ret < 0)
Arne Jansen8c510322011-06-03 10:09:26 +02003429 break;
3430 if (ret > 0) {
3431 if (path->slots[0] >=
3432 btrfs_header_nritems(path->nodes[0])) {
3433 ret = btrfs_next_leaf(root, path);
3434 if (ret)
3435 break;
3436 }
3437 }
Arne Jansena2de7332011-03-08 14:14:00 +01003438
3439 l = path->nodes[0];
3440 slot = path->slots[0];
3441
3442 btrfs_item_key_to_cpu(l, &found_key, slot);
3443
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003444 if (found_key.objectid != scrub_dev->devid)
Arne Jansena2de7332011-03-08 14:14:00 +01003445 break;
3446
David Sterba962a2982014-06-04 18:41:45 +02003447 if (found_key.type != BTRFS_DEV_EXTENT_KEY)
Arne Jansena2de7332011-03-08 14:14:00 +01003448 break;
3449
3450 if (found_key.offset >= end)
3451 break;
3452
3453 if (found_key.offset < key.offset)
3454 break;
3455
3456 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
3457 length = btrfs_dev_extent_length(l, dev_extent);
3458
Qu Wenruoced96ed2014-06-19 10:42:51 +08003459 if (found_key.offset + length <= start)
3460 goto skip;
Arne Jansena2de7332011-03-08 14:14:00 +01003461
3462 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
3463 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
3464 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
3465
3466 /*
3467 * get a reference on the corresponding block group to prevent
3468 * the chunk from going away while we scrub it
3469 */
3470 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
Qu Wenruoced96ed2014-06-19 10:42:51 +08003471
3472 /* some chunks are removed but not committed to disk yet,
3473 * continue scrubbing */
3474 if (!cache)
3475 goto skip;
3476
Stefan Behrensff023aa2012-11-06 11:43:11 +01003477 dev_replace->cursor_right = found_key.offset + length;
3478 dev_replace->cursor_left = found_key.offset;
3479 dev_replace->item_needs_writeback = 1;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003480 ret = scrub_chunk(sctx, scrub_dev, chunk_tree, chunk_objectid,
Stefan Behrensff023aa2012-11-06 11:43:11 +01003481 chunk_offset, length, found_key.offset,
3482 is_dev_replace);
3483
3484 /*
3485 * flush, submit all pending read and write bios, afterwards
3486 * wait for them.
3487 * Note that in the dev replace case, a read request causes
3488 * write requests that are submitted in the read completion
3489 * worker. Therefore in the current situation, it is required
3490 * that all write requests are flushed, so that all read and
3491 * write requests are really completed when bios_in_flight
3492 * changes to 0.
3493 */
3494 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
3495 scrub_submit(sctx);
3496 mutex_lock(&sctx->wr_ctx.wr_lock);
3497 scrub_wr_submit(sctx);
3498 mutex_unlock(&sctx->wr_ctx.wr_lock);
3499
3500 wait_event(sctx->list_wait,
3501 atomic_read(&sctx->bios_in_flight) == 0);
Wang Shilong12cf9372014-02-19 19:24:17 +08003502 atomic_inc(&fs_info->scrubs_paused);
3503 wake_up(&fs_info->scrub_pause_wait);
3504
3505 /*
3506 * must be called before we decrease @scrub_paused.
3507 * make sure we don't block transaction commit while
3508 * we are waiting pending workers finished.
3509 */
Stefan Behrensff023aa2012-11-06 11:43:11 +01003510 wait_event(sctx->list_wait,
3511 atomic_read(&sctx->workers_pending) == 0);
Wang Shilong12cf9372014-02-19 19:24:17 +08003512 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3513
3514 mutex_lock(&fs_info->scrub_lock);
3515 __scrub_blocked_if_needed(fs_info);
3516 atomic_dec(&fs_info->scrubs_paused);
3517 mutex_unlock(&fs_info->scrub_lock);
3518 wake_up(&fs_info->scrub_pause_wait);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003519
Arne Jansena2de7332011-03-08 14:14:00 +01003520 btrfs_put_block_group(cache);
3521 if (ret)
3522 break;
Stefan Behrensaf1be4f2012-11-27 17:39:51 +00003523 if (is_dev_replace &&
3524 atomic64_read(&dev_replace->num_write_errors) > 0) {
Stefan Behrensff023aa2012-11-06 11:43:11 +01003525 ret = -EIO;
3526 break;
3527 }
3528 if (sctx->stat.malloc_errors > 0) {
3529 ret = -ENOMEM;
3530 break;
3531 }
Arne Jansena2de7332011-03-08 14:14:00 +01003532
Ilya Dryomov539f3582013-10-07 13:42:57 +03003533 dev_replace->cursor_left = dev_replace->cursor_right;
3534 dev_replace->item_needs_writeback = 1;
Qu Wenruoced96ed2014-06-19 10:42:51 +08003535skip:
Arne Jansena2de7332011-03-08 14:14:00 +01003536 key.offset = found_key.offset + length;
Chris Mason71267332011-05-23 06:30:52 -04003537 btrfs_release_path(path);
Arne Jansena2de7332011-03-08 14:14:00 +01003538 }
3539
Arne Jansena2de7332011-03-08 14:14:00 +01003540 btrfs_free_path(path);
Arne Jansen8c510322011-06-03 10:09:26 +02003541
3542 /*
3543 * ret can still be 1 from search_slot or next_leaf,
3544 * that's not an error
3545 */
3546 return ret < 0 ? ret : 0;
Arne Jansena2de7332011-03-08 14:14:00 +01003547}
3548
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003549static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
3550 struct btrfs_device *scrub_dev)
Arne Jansena2de7332011-03-08 14:14:00 +01003551{
3552 int i;
3553 u64 bytenr;
3554 u64 gen;
3555 int ret;
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003556 struct btrfs_root *root = sctx->dev_root;
Arne Jansena2de7332011-03-08 14:14:00 +01003557
Miao Xie87533c42013-01-29 10:14:48 +00003558 if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003559 return -EIO;
3560
Miao Xie5f546062014-07-24 11:37:09 +08003561 /* Seed devices of a new filesystem has their own generation. */
3562 if (scrub_dev->fs_devices != root->fs_info->fs_devices)
3563 gen = scrub_dev->generation;
3564 else
3565 gen = root->fs_info->last_trans_committed;
Arne Jansena2de7332011-03-08 14:14:00 +01003566
3567 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
3568 bytenr = btrfs_sb_offset(i);
Miao Xie935e5cc2014-09-03 21:35:33 +08003569 if (bytenr + BTRFS_SUPER_INFO_SIZE >
3570 scrub_dev->commit_total_bytes)
Arne Jansena2de7332011-03-08 14:14:00 +01003571 break;
3572
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003573 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
Stefan Behrensa36cf8b2012-11-02 13:26:57 +01003574 scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
Stefan Behrensff023aa2012-11-06 11:43:11 +01003575 NULL, 1, bytenr);
Arne Jansena2de7332011-03-08 14:14:00 +01003576 if (ret)
3577 return ret;
3578 }
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01003579 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
Arne Jansena2de7332011-03-08 14:14:00 +01003580
3581 return 0;
3582}
3583
3584/*
3585 * get a reference count on fs_info->scrub_workers. start worker if necessary
3586 */
Stefan Behrensff023aa2012-11-06 11:43:11 +01003587static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
3588 int is_dev_replace)
Arne Jansena2de7332011-03-08 14:14:00 +01003589{
Josef Bacik0dc3b842011-11-18 14:37:27 -05003590 int ret = 0;
Qu Wenruo0339ef22014-02-28 10:46:17 +08003591 int flags = WQ_FREEZABLE | WQ_UNBOUND;
3592 int max_active = fs_info->thread_pool_size;
Arne Jansena2de7332011-03-08 14:14:00 +01003593
Arne Jansen632dd772011-06-10 12:07:07 +02003594 if (fs_info->scrub_workers_refcnt == 0) {
Stefan Behrensff023aa2012-11-06 11:43:11 +01003595 if (is_dev_replace)
Qu Wenruo0339ef22014-02-28 10:46:17 +08003596 fs_info->scrub_workers =
3597 btrfs_alloc_workqueue("btrfs-scrub", flags,
3598 1, 4);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003599 else
Qu Wenruo0339ef22014-02-28 10:46:17 +08003600 fs_info->scrub_workers =
3601 btrfs_alloc_workqueue("btrfs-scrub", flags,
3602 max_active, 4);
3603 if (!fs_info->scrub_workers) {
3604 ret = -ENOMEM;
Josef Bacik0dc3b842011-11-18 14:37:27 -05003605 goto out;
Qu Wenruo0339ef22014-02-28 10:46:17 +08003606 }
3607 fs_info->scrub_wr_completion_workers =
3608 btrfs_alloc_workqueue("btrfs-scrubwrc", flags,
3609 max_active, 2);
3610 if (!fs_info->scrub_wr_completion_workers) {
3611 ret = -ENOMEM;
Stefan Behrensff023aa2012-11-06 11:43:11 +01003612 goto out;
Qu Wenruo0339ef22014-02-28 10:46:17 +08003613 }
3614 fs_info->scrub_nocow_workers =
3615 btrfs_alloc_workqueue("btrfs-scrubnc", flags, 1, 0);
3616 if (!fs_info->scrub_nocow_workers) {
3617 ret = -ENOMEM;
Stefan Behrensff023aa2012-11-06 11:43:11 +01003618 goto out;
Qu Wenruo0339ef22014-02-28 10:46:17 +08003619 }
Arne Jansen632dd772011-06-10 12:07:07 +02003620 }
Arne Jansena2de7332011-03-08 14:14:00 +01003621 ++fs_info->scrub_workers_refcnt;
Josef Bacik0dc3b842011-11-18 14:37:27 -05003622out:
Josef Bacik0dc3b842011-11-18 14:37:27 -05003623 return ret;
Arne Jansena2de7332011-03-08 14:14:00 +01003624}
3625
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003626static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
Arne Jansena2de7332011-03-08 14:14:00 +01003627{
Stefan Behrensff023aa2012-11-06 11:43:11 +01003628 if (--fs_info->scrub_workers_refcnt == 0) {
Qu Wenruo0339ef22014-02-28 10:46:17 +08003629 btrfs_destroy_workqueue(fs_info->scrub_workers);
3630 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3631 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003632 }
Arne Jansena2de7332011-03-08 14:14:00 +01003633 WARN_ON(fs_info->scrub_workers_refcnt < 0);
Arne Jansena2de7332011-03-08 14:14:00 +01003634}
3635
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003636int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
3637 u64 end, struct btrfs_scrub_progress *progress,
Stefan Behrens63a212a2012-11-05 18:29:28 +01003638 int readonly, int is_dev_replace)
Arne Jansena2de7332011-03-08 14:14:00 +01003639{
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003640 struct scrub_ctx *sctx;
Arne Jansena2de7332011-03-08 14:14:00 +01003641 int ret;
3642 struct btrfs_device *dev;
Miao Xie5d68da32014-07-24 11:37:07 +08003643 struct rcu_string *name;
Arne Jansena2de7332011-03-08 14:14:00 +01003644
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003645 if (btrfs_fs_closing(fs_info))
Arne Jansena2de7332011-03-08 14:14:00 +01003646 return -EINVAL;
3647
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003648 if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
Stefan Behrensb5d67f62012-03-27 14:21:27 -04003649 /*
3650 * in this case scrub is unable to calculate the checksum
3651 * the way scrub is implemented. Do not handle this
3652 * situation at all because it won't ever happen.
3653 */
Frank Holtonefe120a2013-12-20 11:37:06 -05003654 btrfs_err(fs_info,
3655 "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003656 fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
Stefan Behrensb5d67f62012-03-27 14:21:27 -04003657 return -EINVAL;
3658 }
3659
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003660 if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
Stefan Behrensb5d67f62012-03-27 14:21:27 -04003661 /* not supported for data w/o checksums */
Frank Holtonefe120a2013-12-20 11:37:06 -05003662 btrfs_err(fs_info,
3663 "scrub: size assumption sectorsize != PAGE_SIZE "
3664 "(%d != %lu) fails",
Geert Uytterhoeven27f9f022013-08-20 13:20:09 +02003665 fs_info->chunk_root->sectorsize, PAGE_SIZE);
Arne Jansena2de7332011-03-08 14:14:00 +01003666 return -EINVAL;
3667 }
3668
Stefan Behrens7a9e9982012-11-02 14:58:04 +01003669 if (fs_info->chunk_root->nodesize >
3670 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
3671 fs_info->chunk_root->sectorsize >
3672 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
3673 /*
3674 * would exhaust the array bounds of pagev member in
3675 * struct scrub_block
3676 */
Frank Holtonefe120a2013-12-20 11:37:06 -05003677 btrfs_err(fs_info, "scrub: size assumption nodesize and sectorsize "
3678 "<= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
Stefan Behrens7a9e9982012-11-02 14:58:04 +01003679 fs_info->chunk_root->nodesize,
3680 SCRUB_MAX_PAGES_PER_BLOCK,
3681 fs_info->chunk_root->sectorsize,
3682 SCRUB_MAX_PAGES_PER_BLOCK);
3683 return -EINVAL;
3684 }
3685
Arne Jansena2de7332011-03-08 14:14:00 +01003686
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003687 mutex_lock(&fs_info->fs_devices->device_list_mutex);
3688 dev = btrfs_find_device(fs_info, devid, NULL, NULL);
Stefan Behrens63a212a2012-11-05 18:29:28 +01003689 if (!dev || (dev->missing && !is_dev_replace)) {
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003690 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
Arne Jansena2de7332011-03-08 14:14:00 +01003691 return -ENODEV;
3692 }
Arne Jansena2de7332011-03-08 14:14:00 +01003693
Miao Xie5d68da32014-07-24 11:37:07 +08003694 if (!is_dev_replace && !readonly && !dev->writeable) {
3695 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3696 rcu_read_lock();
3697 name = rcu_dereference(dev->name);
3698 btrfs_err(fs_info, "scrub: device %s is not writable",
3699 name->str);
3700 rcu_read_unlock();
3701 return -EROFS;
3702 }
3703
Wang Shilong3b7a0162013-10-12 02:11:12 +08003704 mutex_lock(&fs_info->scrub_lock);
Stefan Behrens63a212a2012-11-05 18:29:28 +01003705 if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
Arne Jansena2de7332011-03-08 14:14:00 +01003706 mutex_unlock(&fs_info->scrub_lock);
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003707 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003708 return -EIO;
Arne Jansena2de7332011-03-08 14:14:00 +01003709 }
3710
Stefan Behrens8dabb742012-11-06 13:15:27 +01003711 btrfs_dev_replace_lock(&fs_info->dev_replace);
3712 if (dev->scrub_device ||
3713 (!is_dev_replace &&
3714 btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
3715 btrfs_dev_replace_unlock(&fs_info->dev_replace);
Arne Jansena2de7332011-03-08 14:14:00 +01003716 mutex_unlock(&fs_info->scrub_lock);
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003717 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
Arne Jansena2de7332011-03-08 14:14:00 +01003718 return -EINPROGRESS;
3719 }
Stefan Behrens8dabb742012-11-06 13:15:27 +01003720 btrfs_dev_replace_unlock(&fs_info->dev_replace);
Wang Shilong3b7a0162013-10-12 02:11:12 +08003721
3722 ret = scrub_workers_get(fs_info, is_dev_replace);
3723 if (ret) {
3724 mutex_unlock(&fs_info->scrub_lock);
3725 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3726 return ret;
3727 }
3728
Stefan Behrens63a212a2012-11-05 18:29:28 +01003729 sctx = scrub_setup_ctx(dev, is_dev_replace);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003730 if (IS_ERR(sctx)) {
Arne Jansena2de7332011-03-08 14:14:00 +01003731 mutex_unlock(&fs_info->scrub_lock);
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003732 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3733 scrub_workers_put(fs_info);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003734 return PTR_ERR(sctx);
Arne Jansena2de7332011-03-08 14:14:00 +01003735 }
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003736 sctx->readonly = readonly;
3737 dev->scrub_device = sctx;
Wang Shilong3cb09292013-12-04 21:15:19 +08003738 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
Arne Jansena2de7332011-03-08 14:14:00 +01003739
Wang Shilong3cb09292013-12-04 21:15:19 +08003740 /*
3741 * checking @scrub_pause_req here, we can avoid
3742 * race between committing transaction and scrubbing.
3743 */
Wang Shilongcb7ab022013-12-04 21:16:53 +08003744 __scrub_blocked_if_needed(fs_info);
Arne Jansena2de7332011-03-08 14:14:00 +01003745 atomic_inc(&fs_info->scrubs_running);
3746 mutex_unlock(&fs_info->scrub_lock);
Arne Jansena2de7332011-03-08 14:14:00 +01003747
Stefan Behrensff023aa2012-11-06 11:43:11 +01003748 if (!is_dev_replace) {
Wang Shilong9b011ad2013-10-25 19:12:02 +08003749 /*
3750 * by holding device list mutex, we can
3751 * kick off writing super in log tree sync.
3752 */
Wang Shilong3cb09292013-12-04 21:15:19 +08003753 mutex_lock(&fs_info->fs_devices->device_list_mutex);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003754 ret = scrub_supers(sctx, dev);
Wang Shilong3cb09292013-12-04 21:15:19 +08003755 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003756 }
Arne Jansena2de7332011-03-08 14:14:00 +01003757
3758 if (!ret)
Stefan Behrensff023aa2012-11-06 11:43:11 +01003759 ret = scrub_enumerate_chunks(sctx, dev, start, end,
3760 is_dev_replace);
Arne Jansena2de7332011-03-08 14:14:00 +01003761
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01003762 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
Arne Jansena2de7332011-03-08 14:14:00 +01003763 atomic_dec(&fs_info->scrubs_running);
3764 wake_up(&fs_info->scrub_pause_wait);
3765
Stefan Behrensb6bfebc2012-11-02 16:44:58 +01003766 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
Jan Schmidt0ef8e452011-06-13 20:04:15 +02003767
Arne Jansena2de7332011-03-08 14:14:00 +01003768 if (progress)
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003769 memcpy(progress, &sctx->stat, sizeof(*progress));
Arne Jansena2de7332011-03-08 14:14:00 +01003770
3771 mutex_lock(&fs_info->scrub_lock);
3772 dev->scrub_device = NULL;
Wang Shilong3b7a0162013-10-12 02:11:12 +08003773 scrub_workers_put(fs_info);
Arne Jansena2de7332011-03-08 14:14:00 +01003774 mutex_unlock(&fs_info->scrub_lock);
3775
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003776 scrub_free_ctx(sctx);
Arne Jansena2de7332011-03-08 14:14:00 +01003777
3778 return ret;
3779}
3780
Jeff Mahoney143bede2012-03-01 14:56:26 +01003781void btrfs_scrub_pause(struct btrfs_root *root)
Arne Jansena2de7332011-03-08 14:14:00 +01003782{
3783 struct btrfs_fs_info *fs_info = root->fs_info;
3784
3785 mutex_lock(&fs_info->scrub_lock);
3786 atomic_inc(&fs_info->scrub_pause_req);
3787 while (atomic_read(&fs_info->scrubs_paused) !=
3788 atomic_read(&fs_info->scrubs_running)) {
3789 mutex_unlock(&fs_info->scrub_lock);
3790 wait_event(fs_info->scrub_pause_wait,
3791 atomic_read(&fs_info->scrubs_paused) ==
3792 atomic_read(&fs_info->scrubs_running));
3793 mutex_lock(&fs_info->scrub_lock);
3794 }
3795 mutex_unlock(&fs_info->scrub_lock);
Arne Jansena2de7332011-03-08 14:14:00 +01003796}
3797
Jeff Mahoney143bede2012-03-01 14:56:26 +01003798void btrfs_scrub_continue(struct btrfs_root *root)
Arne Jansena2de7332011-03-08 14:14:00 +01003799{
3800 struct btrfs_fs_info *fs_info = root->fs_info;
3801
3802 atomic_dec(&fs_info->scrub_pause_req);
3803 wake_up(&fs_info->scrub_pause_wait);
Arne Jansena2de7332011-03-08 14:14:00 +01003804}
3805
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003806int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
Arne Jansena2de7332011-03-08 14:14:00 +01003807{
Arne Jansena2de7332011-03-08 14:14:00 +01003808 mutex_lock(&fs_info->scrub_lock);
3809 if (!atomic_read(&fs_info->scrubs_running)) {
3810 mutex_unlock(&fs_info->scrub_lock);
3811 return -ENOTCONN;
3812 }
3813
3814 atomic_inc(&fs_info->scrub_cancel_req);
3815 while (atomic_read(&fs_info->scrubs_running)) {
3816 mutex_unlock(&fs_info->scrub_lock);
3817 wait_event(fs_info->scrub_pause_wait,
3818 atomic_read(&fs_info->scrubs_running) == 0);
3819 mutex_lock(&fs_info->scrub_lock);
3820 }
3821 atomic_dec(&fs_info->scrub_cancel_req);
3822 mutex_unlock(&fs_info->scrub_lock);
3823
3824 return 0;
3825}
3826
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003827int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
3828 struct btrfs_device *dev)
Jeff Mahoney49b25e02012-03-01 17:24:58 +01003829{
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003830 struct scrub_ctx *sctx;
Arne Jansena2de7332011-03-08 14:14:00 +01003831
3832 mutex_lock(&fs_info->scrub_lock);
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003833 sctx = dev->scrub_device;
3834 if (!sctx) {
Arne Jansena2de7332011-03-08 14:14:00 +01003835 mutex_unlock(&fs_info->scrub_lock);
3836 return -ENOTCONN;
3837 }
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003838 atomic_inc(&sctx->cancel_req);
Arne Jansena2de7332011-03-08 14:14:00 +01003839 while (dev->scrub_device) {
3840 mutex_unlock(&fs_info->scrub_lock);
3841 wait_event(fs_info->scrub_pause_wait,
3842 dev->scrub_device == NULL);
3843 mutex_lock(&fs_info->scrub_lock);
3844 }
3845 mutex_unlock(&fs_info->scrub_lock);
3846
3847 return 0;
3848}
Stefan Behrens1623ede2012-03-27 14:21:26 -04003849
Arne Jansena2de7332011-03-08 14:14:00 +01003850int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
3851 struct btrfs_scrub_progress *progress)
3852{
3853 struct btrfs_device *dev;
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003854 struct scrub_ctx *sctx = NULL;
Arne Jansena2de7332011-03-08 14:14:00 +01003855
3856 mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
Stefan Behrensaa1b8cd2012-11-05 17:03:39 +01003857 dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
Arne Jansena2de7332011-03-08 14:14:00 +01003858 if (dev)
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003859 sctx = dev->scrub_device;
3860 if (sctx)
3861 memcpy(progress, &sctx->stat, sizeof(*progress));
Arne Jansena2de7332011-03-08 14:14:00 +01003862 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
3863
Stefan Behrensd9d181c2012-11-02 09:58:09 +01003864 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
Arne Jansena2de7332011-03-08 14:14:00 +01003865}
Stefan Behrensff023aa2012-11-06 11:43:11 +01003866
3867static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
3868 u64 extent_logical, u64 extent_len,
3869 u64 *extent_physical,
3870 struct btrfs_device **extent_dev,
3871 int *extent_mirror_num)
3872{
3873 u64 mapped_length;
3874 struct btrfs_bio *bbio = NULL;
3875 int ret;
3876
3877 mapped_length = extent_len;
3878 ret = btrfs_map_block(fs_info, READ, extent_logical,
3879 &mapped_length, &bbio, 0);
3880 if (ret || !bbio || mapped_length < extent_len ||
3881 !bbio->stripes[0].dev->bdev) {
3882 kfree(bbio);
3883 return;
3884 }
3885
3886 *extent_physical = bbio->stripes[0].physical;
3887 *extent_mirror_num = bbio->mirror_num;
3888 *extent_dev = bbio->stripes[0].dev;
3889 kfree(bbio);
3890}
3891
3892static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
3893 struct scrub_wr_ctx *wr_ctx,
3894 struct btrfs_fs_info *fs_info,
3895 struct btrfs_device *dev,
3896 int is_dev_replace)
3897{
3898 WARN_ON(wr_ctx->wr_curr_bio != NULL);
3899
3900 mutex_init(&wr_ctx->wr_lock);
3901 wr_ctx->wr_curr_bio = NULL;
3902 if (!is_dev_replace)
3903 return 0;
3904
3905 WARN_ON(!dev->bdev);
3906 wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO,
3907 bio_get_nr_vecs(dev->bdev));
3908 wr_ctx->tgtdev = dev;
3909 atomic_set(&wr_ctx->flush_all_writes, 0);
3910 return 0;
3911}
3912
3913static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
3914{
3915 mutex_lock(&wr_ctx->wr_lock);
3916 kfree(wr_ctx->wr_curr_bio);
3917 wr_ctx->wr_curr_bio = NULL;
3918 mutex_unlock(&wr_ctx->wr_lock);
3919}
3920
3921static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
3922 int mirror_num, u64 physical_for_dev_replace)
3923{
3924 struct scrub_copy_nocow_ctx *nocow_ctx;
3925 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3926
3927 nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
3928 if (!nocow_ctx) {
3929 spin_lock(&sctx->stat_lock);
3930 sctx->stat.malloc_errors++;
3931 spin_unlock(&sctx->stat_lock);
3932 return -ENOMEM;
3933 }
3934
3935 scrub_pending_trans_workers_inc(sctx);
3936
3937 nocow_ctx->sctx = sctx;
3938 nocow_ctx->logical = logical;
3939 nocow_ctx->len = len;
3940 nocow_ctx->mirror_num = mirror_num;
3941 nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
Liu Bo9e0af232014-08-15 23:36:53 +08003942 btrfs_init_work(&nocow_ctx->work, btrfs_scrubnc_helper,
3943 copy_nocow_pages_worker, NULL, NULL);
Josef Bacik652f25a2013-09-12 16:58:28 -04003944 INIT_LIST_HEAD(&nocow_ctx->inodes);
Qu Wenruo0339ef22014-02-28 10:46:17 +08003945 btrfs_queue_work(fs_info->scrub_nocow_workers,
3946 &nocow_ctx->work);
Stefan Behrensff023aa2012-11-06 11:43:11 +01003947
3948 return 0;
3949}
3950
Josef Bacik652f25a2013-09-12 16:58:28 -04003951static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
3952{
3953 struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
3954 struct scrub_nocow_inode *nocow_inode;
3955
3956 nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
3957 if (!nocow_inode)
3958 return -ENOMEM;
3959 nocow_inode->inum = inum;
3960 nocow_inode->offset = offset;
3961 nocow_inode->root = root;
3962 list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
3963 return 0;
3964}
3965
3966#define COPY_COMPLETE 1
3967
Stefan Behrensff023aa2012-11-06 11:43:11 +01003968static void copy_nocow_pages_worker(struct btrfs_work *work)
3969{
3970 struct scrub_copy_nocow_ctx *nocow_ctx =
3971 container_of(work, struct scrub_copy_nocow_ctx, work);
3972 struct scrub_ctx *sctx = nocow_ctx->sctx;
3973 u64 logical = nocow_ctx->logical;
3974 u64 len = nocow_ctx->len;
3975 int mirror_num = nocow_ctx->mirror_num;
3976 u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3977 int ret;
3978 struct btrfs_trans_handle *trans = NULL;
3979 struct btrfs_fs_info *fs_info;
3980 struct btrfs_path *path;
3981 struct btrfs_root *root;
3982 int not_written = 0;
3983
3984 fs_info = sctx->dev_root->fs_info;
3985 root = fs_info->extent_root;
3986
3987 path = btrfs_alloc_path();
3988 if (!path) {
3989 spin_lock(&sctx->stat_lock);
3990 sctx->stat.malloc_errors++;
3991 spin_unlock(&sctx->stat_lock);
3992 not_written = 1;
3993 goto out;
3994 }
3995
3996 trans = btrfs_join_transaction(root);
3997 if (IS_ERR(trans)) {
3998 not_written = 1;
3999 goto out;
4000 }
4001
4002 ret = iterate_inodes_from_logical(logical, fs_info, path,
Josef Bacik652f25a2013-09-12 16:58:28 -04004003 record_inode_for_nocow, nocow_ctx);
Stefan Behrensff023aa2012-11-06 11:43:11 +01004004 if (ret != 0 && ret != -ENOENT) {
Frank Holtonefe120a2013-12-20 11:37:06 -05004005 btrfs_warn(fs_info, "iterate_inodes_from_logical() failed: log %llu, "
4006 "phys %llu, len %llu, mir %u, ret %d",
Geert Uytterhoeven118a0a22013-08-20 13:20:10 +02004007 logical, physical_for_dev_replace, len, mirror_num,
4008 ret);
Stefan Behrensff023aa2012-11-06 11:43:11 +01004009 not_written = 1;
4010 goto out;
4011 }
4012
Josef Bacik652f25a2013-09-12 16:58:28 -04004013 btrfs_end_transaction(trans, root);
4014 trans = NULL;
4015 while (!list_empty(&nocow_ctx->inodes)) {
4016 struct scrub_nocow_inode *entry;
4017 entry = list_first_entry(&nocow_ctx->inodes,
4018 struct scrub_nocow_inode,
4019 list);
4020 list_del_init(&entry->list);
4021 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
4022 entry->root, nocow_ctx);
4023 kfree(entry);
4024 if (ret == COPY_COMPLETE) {
4025 ret = 0;
4026 break;
4027 } else if (ret) {
4028 break;
4029 }
4030 }
Stefan Behrensff023aa2012-11-06 11:43:11 +01004031out:
Josef Bacik652f25a2013-09-12 16:58:28 -04004032 while (!list_empty(&nocow_ctx->inodes)) {
4033 struct scrub_nocow_inode *entry;
4034 entry = list_first_entry(&nocow_ctx->inodes,
4035 struct scrub_nocow_inode,
4036 list);
4037 list_del_init(&entry->list);
4038 kfree(entry);
4039 }
Stefan Behrensff023aa2012-11-06 11:43:11 +01004040 if (trans && !IS_ERR(trans))
4041 btrfs_end_transaction(trans, root);
4042 if (not_written)
4043 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
4044 num_uncorrectable_read_errors);
4045
4046 btrfs_free_path(path);
4047 kfree(nocow_ctx);
4048
4049 scrub_pending_trans_workers_dec(sctx);
4050}
4051
Gui Hecheng32159242014-11-10 15:36:08 +08004052static int check_extent_to_block(struct inode *inode, u64 start, u64 len,
4053 u64 logical)
4054{
4055 struct extent_state *cached_state = NULL;
4056 struct btrfs_ordered_extent *ordered;
4057 struct extent_io_tree *io_tree;
4058 struct extent_map *em;
4059 u64 lockstart = start, lockend = start + len - 1;
4060 int ret = 0;
4061
4062 io_tree = &BTRFS_I(inode)->io_tree;
4063
4064 lock_extent_bits(io_tree, lockstart, lockend, 0, &cached_state);
4065 ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
4066 if (ordered) {
4067 btrfs_put_ordered_extent(ordered);
4068 ret = 1;
4069 goto out_unlock;
4070 }
4071
4072 em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
4073 if (IS_ERR(em)) {
4074 ret = PTR_ERR(em);
4075 goto out_unlock;
4076 }
4077
4078 /*
4079 * This extent does not actually cover the logical extent anymore,
4080 * move on to the next inode.
4081 */
4082 if (em->block_start > logical ||
4083 em->block_start + em->block_len < logical + len) {
4084 free_extent_map(em);
4085 ret = 1;
4086 goto out_unlock;
4087 }
4088 free_extent_map(em);
4089
4090out_unlock:
4091 unlock_extent_cached(io_tree, lockstart, lockend, &cached_state,
4092 GFP_NOFS);
4093 return ret;
4094}
4095
Josef Bacik652f25a2013-09-12 16:58:28 -04004096static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
4097 struct scrub_copy_nocow_ctx *nocow_ctx)
Stefan Behrensff023aa2012-11-06 11:43:11 +01004098{
Miao Xie826aa0a2013-06-27 18:50:59 +08004099 struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004100 struct btrfs_key key;
Miao Xie826aa0a2013-06-27 18:50:59 +08004101 struct inode *inode;
4102 struct page *page;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004103 struct btrfs_root *local_root;
Josef Bacik652f25a2013-09-12 16:58:28 -04004104 struct extent_io_tree *io_tree;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004105 u64 physical_for_dev_replace;
Gui Hecheng32159242014-11-10 15:36:08 +08004106 u64 nocow_ctx_logical;
Josef Bacik652f25a2013-09-12 16:58:28 -04004107 u64 len = nocow_ctx->len;
Miao Xie826aa0a2013-06-27 18:50:59 +08004108 unsigned long index;
Liu Bo6f1c3602013-01-29 03:22:10 +00004109 int srcu_index;
Josef Bacik652f25a2013-09-12 16:58:28 -04004110 int ret = 0;
4111 int err = 0;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004112
4113 key.objectid = root;
4114 key.type = BTRFS_ROOT_ITEM_KEY;
4115 key.offset = (u64)-1;
Liu Bo6f1c3602013-01-29 03:22:10 +00004116
4117 srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
4118
Stefan Behrensff023aa2012-11-06 11:43:11 +01004119 local_root = btrfs_read_fs_root_no_name(fs_info, &key);
Liu Bo6f1c3602013-01-29 03:22:10 +00004120 if (IS_ERR(local_root)) {
4121 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
Stefan Behrensff023aa2012-11-06 11:43:11 +01004122 return PTR_ERR(local_root);
Liu Bo6f1c3602013-01-29 03:22:10 +00004123 }
Stefan Behrensff023aa2012-11-06 11:43:11 +01004124
4125 key.type = BTRFS_INODE_ITEM_KEY;
4126 key.objectid = inum;
4127 key.offset = 0;
4128 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
Liu Bo6f1c3602013-01-29 03:22:10 +00004129 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
Stefan Behrensff023aa2012-11-06 11:43:11 +01004130 if (IS_ERR(inode))
4131 return PTR_ERR(inode);
4132
Miao Xieedd14002013-06-27 18:51:00 +08004133 /* Avoid truncate/dio/punch hole.. */
4134 mutex_lock(&inode->i_mutex);
4135 inode_dio_wait(inode);
4136
Stefan Behrensff023aa2012-11-06 11:43:11 +01004137 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
Josef Bacik652f25a2013-09-12 16:58:28 -04004138 io_tree = &BTRFS_I(inode)->io_tree;
Gui Hecheng32159242014-11-10 15:36:08 +08004139 nocow_ctx_logical = nocow_ctx->logical;
Josef Bacik652f25a2013-09-12 16:58:28 -04004140
Gui Hecheng32159242014-11-10 15:36:08 +08004141 ret = check_extent_to_block(inode, offset, len, nocow_ctx_logical);
4142 if (ret) {
4143 ret = ret > 0 ? 0 : ret;
4144 goto out;
Josef Bacik652f25a2013-09-12 16:58:28 -04004145 }
4146
Stefan Behrensff023aa2012-11-06 11:43:11 +01004147 while (len >= PAGE_CACHE_SIZE) {
Stefan Behrensff023aa2012-11-06 11:43:11 +01004148 index = offset >> PAGE_CACHE_SHIFT;
Miao Xieedd14002013-06-27 18:51:00 +08004149again:
Stefan Behrensff023aa2012-11-06 11:43:11 +01004150 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
4151 if (!page) {
Frank Holtonefe120a2013-12-20 11:37:06 -05004152 btrfs_err(fs_info, "find_or_create_page() failed");
Stefan Behrensff023aa2012-11-06 11:43:11 +01004153 ret = -ENOMEM;
Miao Xie826aa0a2013-06-27 18:50:59 +08004154 goto out;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004155 }
4156
4157 if (PageUptodate(page)) {
4158 if (PageDirty(page))
4159 goto next_page;
4160 } else {
4161 ClearPageError(page);
Gui Hecheng32159242014-11-10 15:36:08 +08004162 err = extent_read_full_page(io_tree, page,
Josef Bacik652f25a2013-09-12 16:58:28 -04004163 btrfs_get_extent,
4164 nocow_ctx->mirror_num);
Miao Xie826aa0a2013-06-27 18:50:59 +08004165 if (err) {
4166 ret = err;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004167 goto next_page;
4168 }
Miao Xieedd14002013-06-27 18:51:00 +08004169
Miao Xie26b258912013-06-27 18:50:58 +08004170 lock_page(page);
Miao Xieedd14002013-06-27 18:51:00 +08004171 /*
4172 * If the page has been remove from the page cache,
4173 * the data on it is meaningless, because it may be
4174 * old one, the new data may be written into the new
4175 * page in the page cache.
4176 */
4177 if (page->mapping != inode->i_mapping) {
Josef Bacik652f25a2013-09-12 16:58:28 -04004178 unlock_page(page);
Miao Xieedd14002013-06-27 18:51:00 +08004179 page_cache_release(page);
4180 goto again;
4181 }
Stefan Behrensff023aa2012-11-06 11:43:11 +01004182 if (!PageUptodate(page)) {
4183 ret = -EIO;
4184 goto next_page;
4185 }
4186 }
Gui Hecheng32159242014-11-10 15:36:08 +08004187
4188 ret = check_extent_to_block(inode, offset, len,
4189 nocow_ctx_logical);
4190 if (ret) {
4191 ret = ret > 0 ? 0 : ret;
4192 goto next_page;
4193 }
4194
Miao Xie826aa0a2013-06-27 18:50:59 +08004195 err = write_page_nocow(nocow_ctx->sctx,
4196 physical_for_dev_replace, page);
4197 if (err)
4198 ret = err;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004199next_page:
Miao Xie826aa0a2013-06-27 18:50:59 +08004200 unlock_page(page);
4201 page_cache_release(page);
4202
4203 if (ret)
4204 break;
4205
Stefan Behrensff023aa2012-11-06 11:43:11 +01004206 offset += PAGE_CACHE_SIZE;
4207 physical_for_dev_replace += PAGE_CACHE_SIZE;
Gui Hecheng32159242014-11-10 15:36:08 +08004208 nocow_ctx_logical += PAGE_CACHE_SIZE;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004209 len -= PAGE_CACHE_SIZE;
4210 }
Josef Bacik652f25a2013-09-12 16:58:28 -04004211 ret = COPY_COMPLETE;
Miao Xie826aa0a2013-06-27 18:50:59 +08004212out:
Miao Xieedd14002013-06-27 18:51:00 +08004213 mutex_unlock(&inode->i_mutex);
Miao Xie826aa0a2013-06-27 18:50:59 +08004214 iput(inode);
Stefan Behrensff023aa2012-11-06 11:43:11 +01004215 return ret;
4216}
4217
4218static int write_page_nocow(struct scrub_ctx *sctx,
4219 u64 physical_for_dev_replace, struct page *page)
4220{
4221 struct bio *bio;
4222 struct btrfs_device *dev;
4223 int ret;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004224
4225 dev = sctx->wr_ctx.tgtdev;
4226 if (!dev)
4227 return -EIO;
4228 if (!dev->bdev) {
4229 printk_ratelimited(KERN_WARNING
Frank Holtonefe120a2013-12-20 11:37:06 -05004230 "BTRFS: scrub write_page_nocow(bdev == NULL) is unexpected!\n");
Stefan Behrensff023aa2012-11-06 11:43:11 +01004231 return -EIO;
4232 }
Chris Mason9be33952013-05-17 18:30:14 -04004233 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Stefan Behrensff023aa2012-11-06 11:43:11 +01004234 if (!bio) {
4235 spin_lock(&sctx->stat_lock);
4236 sctx->stat.malloc_errors++;
4237 spin_unlock(&sctx->stat_lock);
4238 return -ENOMEM;
4239 }
Kent Overstreet4f024f32013-10-11 15:44:27 -07004240 bio->bi_iter.bi_size = 0;
4241 bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
Stefan Behrensff023aa2012-11-06 11:43:11 +01004242 bio->bi_bdev = dev->bdev;
4243 ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
4244 if (ret != PAGE_CACHE_SIZE) {
4245leave_with_eio:
4246 bio_put(bio);
4247 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
4248 return -EIO;
4249 }
Stefan Behrensff023aa2012-11-06 11:43:11 +01004250
Kent Overstreet33879d42013-11-23 22:33:32 -08004251 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio))
Stefan Behrensff023aa2012-11-06 11:43:11 +01004252 goto leave_with_eio;
4253
4254 bio_put(bio);
4255 return 0;
4256}