scrub.c 106.9 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
A
Arne Jansen 已提交
2
/*
3
 * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
A
Arne Jansen 已提交
4 5 6
 */

#include <linux/blkdev.h>
7
#include <linux/ratelimit.h>
8
#include <linux/sched/mm.h>
9
#include <crypto/hash.h>
A
Arne Jansen 已提交
10 11 12 13
#include "ctree.h"
#include "volumes.h"
#include "disk-io.h"
#include "ordered-data.h"
14
#include "transaction.h"
15
#include "backref.h"
16
#include "extent_io.h"
17
#include "dev-replace.h"
18
#include "check-integrity.h"
19
#include "rcu-string.h"
D
David Woodhouse 已提交
20
#include "raid56.h"
21
#include "block-group.h"
A
Arne Jansen 已提交
22 23 24 25 26 27 28 29 30 31 32 33 34 35

/*
 * This is only the first step towards a full-features scrub. It reads all
 * extent and super block and verifies the checksums. In case a bad checksum
 * is found or the extent cannot be read, good data will be written back if
 * any can be found.
 *
 * Future enhancements:
 *  - In case an unrepairable extent is encountered, track which files are
 *    affected and report them
 *  - track and record media errors, throw out bad devices
 *  - add a mode to also read unallocated space
 */

36
struct scrub_block;
37
struct scrub_ctx;
A
Arne Jansen 已提交
38

39 40 41 42 43 44 45 46 47
/*
 * the following three values only influence the performance.
 * The last one configures the number of parallel and outstanding I/O
 * operations. The first two values configure an upper limit for the number
 * of (dynamically allocated) pages that are added to a bio.
 */
#define SCRUB_PAGES_PER_RD_BIO	32	/* 128k per bio */
#define SCRUB_PAGES_PER_WR_BIO	32	/* 128k per bio */
#define SCRUB_BIOS_PER_SCTX	64	/* 8MB per device in flight */
48 49 50 51 52 53

/*
 * the following value times PAGE_SIZE needs to be large enough to match the
 * largest node/leaf/sector size that shall be supported.
 * Values larger than BTRFS_STRIPE_LEN are not supported.
 */
54
#define SCRUB_MAX_PAGES_PER_BLOCK	16	/* 64k per node/leaf/sector */
A
Arne Jansen 已提交
55

56
struct scrub_recover {
57
	refcount_t		refs;
58 59 60 61
	struct btrfs_bio	*bbio;
	u64			map_length;
};

A
Arne Jansen 已提交
62
struct scrub_page {
63 64
	struct scrub_block	*sblock;
	struct page		*page;
65
	struct btrfs_device	*dev;
66
	struct list_head	list;
A
Arne Jansen 已提交
67 68
	u64			flags;  /* extent flags */
	u64			generation;
69 70
	u64			logical;
	u64			physical;
71
	u64			physical_for_dev_replace;
72
	atomic_t		refs;
73 74 75 76 77
	struct {
		unsigned int	mirror_num:8;
		unsigned int	have_csum:1;
		unsigned int	io_error:1;
	};
A
Arne Jansen 已提交
78
	u8			csum[BTRFS_CSUM_SIZE];
79 80

	struct scrub_recover	*recover;
A
Arne Jansen 已提交
81 82 83 84
};

struct scrub_bio {
	int			index;
85
	struct scrub_ctx	*sctx;
86
	struct btrfs_device	*dev;
A
Arne Jansen 已提交
87
	struct bio		*bio;
88
	blk_status_t		status;
A
Arne Jansen 已提交
89 90
	u64			logical;
	u64			physical;
91 92 93 94 95
#if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
	struct scrub_page	*pagev[SCRUB_PAGES_PER_WR_BIO];
#else
	struct scrub_page	*pagev[SCRUB_PAGES_PER_RD_BIO];
#endif
96
	int			page_count;
A
Arne Jansen 已提交
97 98 99 100
	int			next_free;
	struct btrfs_work	work;
};

101
struct scrub_block {
102
	struct scrub_page	*pagev[SCRUB_MAX_PAGES_PER_BLOCK];
103 104
	int			page_count;
	atomic_t		outstanding_pages;
105
	refcount_t		refs; /* free mem on transition to zero */
106
	struct scrub_ctx	*sctx;
107
	struct scrub_parity	*sparity;
108 109 110 111
	struct {
		unsigned int	header_error:1;
		unsigned int	checksum_error:1;
		unsigned int	no_io_error_seen:1;
112
		unsigned int	generation_error:1; /* also sets header_error */
113 114 115 116

		/* The following is for the data used to check parity */
		/* It is for the data with checksum */
		unsigned int	data_corrected:1;
117
	};
118
	struct btrfs_work	work;
119 120
};

121 122 123 124 125 126 127 128 129 130 131 132
/* Used for the chunks with parity stripe such RAID5/6 */
struct scrub_parity {
	struct scrub_ctx	*sctx;

	struct btrfs_device	*scrub_dev;

	u64			logic_start;

	u64			logic_end;

	int			nsectors;

133
	u64			stripe_len;
134

135
	refcount_t		refs;
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153

	struct list_head	spages;

	/* Work of parity check and repair */
	struct btrfs_work	work;

	/* Mark the parity blocks which have data */
	unsigned long		*dbitmap;

	/*
	 * Mark the parity blocks which have data, but errors happen when
	 * read data or check data
	 */
	unsigned long		*ebitmap;

	unsigned long		bitmap[0];
};

154
struct scrub_ctx {
155
	struct scrub_bio	*bios[SCRUB_BIOS_PER_SCTX];
156
	struct btrfs_fs_info	*fs_info;
A
Arne Jansen 已提交
157 158
	int			first_free;
	int			curr;
159 160
	atomic_t		bios_in_flight;
	atomic_t		workers_pending;
A
Arne Jansen 已提交
161 162 163 164 165
	spinlock_t		list_lock;
	wait_queue_head_t	list_wait;
	u16			csum_size;
	struct list_head	csum_list;
	atomic_t		cancel_req;
A
Arne Jansen 已提交
166
	int			readonly;
167
	int			pages_per_rd_bio;
168 169

	int			is_dev_replace;
170 171 172 173 174

	struct scrub_bio        *wr_curr_bio;
	struct mutex            wr_lock;
	int                     pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
	struct btrfs_device     *wr_tgtdev;
175
	bool                    flush_all_writes;
176

A
Arne Jansen 已提交
177 178 179 180 181
	/*
	 * statistics
	 */
	struct btrfs_scrub_progress stat;
	spinlock_t		stat_lock;
182 183 184 185 186 187 188 189

	/*
	 * Use a ref counter to avoid use-after-free issues. Scrub workers
	 * decrement bios_in_flight and workers_pending and then do a wakeup
	 * on the list_wait wait queue. We must ensure the main scrub task
	 * doesn't free the scrub context before or while the workers are
	 * doing the wakeup() call.
	 */
190
	refcount_t              refs;
A
Arne Jansen 已提交
191 192
};

193 194 195 196
struct scrub_warning {
	struct btrfs_path	*path;
	u64			extent_item_size;
	const char		*errstr;
D
David Sterba 已提交
197
	u64			physical;
198 199 200 201
	u64			logical;
	struct btrfs_device	*dev;
};

202 203 204 205 206 207 208
struct full_stripe_lock {
	struct rb_node node;
	u64 logical;
	u64 refs;
	struct mutex mutex;
};

209 210
static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
211
static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
212
static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
213
				     struct scrub_block *sblocks_for_recheck);
214
static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
215 216
				struct scrub_block *sblock,
				int retry_failed_mirror);
217
static void scrub_recheck_block_checksum(struct scrub_block *sblock);
218
static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
219
					     struct scrub_block *sblock_good);
220 221 222
static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
					    struct scrub_block *sblock_good,
					    int page_num, int force_write);
223 224 225
static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
					   int page_num);
226 227 228 229 230
static int scrub_checksum_data(struct scrub_block *sblock);
static int scrub_checksum_tree_block(struct scrub_block *sblock);
static int scrub_checksum_super(struct scrub_block *sblock);
static void scrub_block_get(struct scrub_block *sblock);
static void scrub_block_put(struct scrub_block *sblock);
231 232
static void scrub_page_get(struct scrub_page *spage);
static void scrub_page_put(struct scrub_page *spage);
233 234
static void scrub_parity_get(struct scrub_parity *sparity);
static void scrub_parity_put(struct scrub_parity *sparity);
235 236
static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
				    struct scrub_page *spage);
237
static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
238
		       u64 physical, struct btrfs_device *dev, u64 flags,
239 240
		       u64 gen, int mirror_num, u8 *csum, int force,
		       u64 physical_for_dev_replace);
241
static void scrub_bio_end_io(struct bio *bio);
242 243
static void scrub_bio_end_io_worker(struct btrfs_work *work);
static void scrub_block_complete(struct scrub_block *sblock);
244 245 246 247 248 249 250 251
static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
			       u64 extent_logical, u64 extent_len,
			       u64 *extent_physical,
			       struct btrfs_device **extent_dev,
			       int *extent_mirror_num);
static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
				    struct scrub_page *spage);
static void scrub_wr_submit(struct scrub_ctx *sctx);
252
static void scrub_wr_bio_end_io(struct bio *bio);
253
static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
254
static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
255
static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
256
static void scrub_put_ctx(struct scrub_ctx *sctx);
S
Stefan Behrens 已提交
257

258 259 260 261 262
static inline int scrub_is_page_on_raid56(struct scrub_page *page)
{
	return page->recover &&
	       (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
}
S
Stefan Behrens 已提交
263

264 265
static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
{
266
	refcount_inc(&sctx->refs);
267 268 269 270 271 272 273
	atomic_inc(&sctx->bios_in_flight);
}

static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
{
	atomic_dec(&sctx->bios_in_flight);
	wake_up(&sctx->list_wait);
274
	scrub_put_ctx(sctx);
275 276
}

277
static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
278 279 280 281 282 283 284 285 286
{
	while (atomic_read(&fs_info->scrub_pause_req)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
		   atomic_read(&fs_info->scrub_pause_req) == 0);
		mutex_lock(&fs_info->scrub_lock);
	}
}

287
static void scrub_pause_on(struct btrfs_fs_info *fs_info)
288 289 290
{
	atomic_inc(&fs_info->scrubs_paused);
	wake_up(&fs_info->scrub_pause_wait);
291
}
292

293 294
static void scrub_pause_off(struct btrfs_fs_info *fs_info)
{
295 296 297 298 299 300 301 302
	mutex_lock(&fs_info->scrub_lock);
	__scrub_blocked_if_needed(fs_info);
	atomic_dec(&fs_info->scrubs_paused);
	mutex_unlock(&fs_info->scrub_lock);

	wake_up(&fs_info->scrub_pause_wait);
}

303 304 305 306 307 308
static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
{
	scrub_pause_on(fs_info);
	scrub_pause_off(fs_info);
}

309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
/*
 * Insert new full stripe lock into full stripe locks tree
 *
 * Return pointer to existing or newly inserted full_stripe_lock structure if
 * everything works well.
 * Return ERR_PTR(-ENOMEM) if we failed to allocate memory
 *
 * NOTE: caller must hold full_stripe_locks_root->lock before calling this
 * function
 */
static struct full_stripe_lock *insert_full_stripe_lock(
		struct btrfs_full_stripe_locks_tree *locks_root,
		u64 fstripe_logical)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct full_stripe_lock *entry;
	struct full_stripe_lock *ret;

328
	lockdep_assert_held(&locks_root->lock);
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343

	p = &locks_root->root.rb_node;
	while (*p) {
		parent = *p;
		entry = rb_entry(parent, struct full_stripe_lock, node);
		if (fstripe_logical < entry->logical) {
			p = &(*p)->rb_left;
		} else if (fstripe_logical > entry->logical) {
			p = &(*p)->rb_right;
		} else {
			entry->refs++;
			return entry;
		}
	}

344 345 346
	/*
	 * Insert new lock.
	 */
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
	ret = kmalloc(sizeof(*ret), GFP_KERNEL);
	if (!ret)
		return ERR_PTR(-ENOMEM);
	ret->logical = fstripe_logical;
	ret->refs = 1;
	mutex_init(&ret->mutex);

	rb_link_node(&ret->node, parent, p);
	rb_insert_color(&ret->node, &locks_root->root);
	return ret;
}

/*
 * Search for a full stripe lock of a block group
 *
 * Return pointer to existing full stripe lock if found
 * Return NULL if not found
 */
static struct full_stripe_lock *search_full_stripe_lock(
		struct btrfs_full_stripe_locks_tree *locks_root,
		u64 fstripe_logical)
{
	struct rb_node *node;
	struct full_stripe_lock *entry;

372
	lockdep_assert_held(&locks_root->lock);
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526

	node = locks_root->root.rb_node;
	while (node) {
		entry = rb_entry(node, struct full_stripe_lock, node);
		if (fstripe_logical < entry->logical)
			node = node->rb_left;
		else if (fstripe_logical > entry->logical)
			node = node->rb_right;
		else
			return entry;
	}
	return NULL;
}

/*
 * Helper to get full stripe logical from a normal bytenr.
 *
 * Caller must ensure @cache is a RAID56 block group.
 */
static u64 get_full_stripe_logical(struct btrfs_block_group_cache *cache,
				   u64 bytenr)
{
	u64 ret;

	/*
	 * Due to chunk item size limit, full stripe length should not be
	 * larger than U32_MAX. Just a sanity check here.
	 */
	WARN_ON_ONCE(cache->full_stripe_len >= U32_MAX);

	/*
	 * round_down() can only handle power of 2, while RAID56 full
	 * stripe length can be 64KiB * n, so we need to manually round down.
	 */
	ret = div64_u64(bytenr - cache->key.objectid, cache->full_stripe_len) *
		cache->full_stripe_len + cache->key.objectid;
	return ret;
}

/*
 * Lock a full stripe to avoid concurrency of recovery and read
 *
 * It's only used for profiles with parities (RAID5/6), for other profiles it
 * does nothing.
 *
 * Return 0 if we locked full stripe covering @bytenr, with a mutex held.
 * So caller must call unlock_full_stripe() at the same context.
 *
 * Return <0 if encounters error.
 */
static int lock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
			    bool *locked_ret)
{
	struct btrfs_block_group_cache *bg_cache;
	struct btrfs_full_stripe_locks_tree *locks_root;
	struct full_stripe_lock *existing;
	u64 fstripe_start;
	int ret = 0;

	*locked_ret = false;
	bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
	if (!bg_cache) {
		ASSERT(0);
		return -ENOENT;
	}

	/* Profiles not based on parity don't need full stripe lock */
	if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
		goto out;
	locks_root = &bg_cache->full_stripe_locks_root;

	fstripe_start = get_full_stripe_logical(bg_cache, bytenr);

	/* Now insert the full stripe lock */
	mutex_lock(&locks_root->lock);
	existing = insert_full_stripe_lock(locks_root, fstripe_start);
	mutex_unlock(&locks_root->lock);
	if (IS_ERR(existing)) {
		ret = PTR_ERR(existing);
		goto out;
	}
	mutex_lock(&existing->mutex);
	*locked_ret = true;
out:
	btrfs_put_block_group(bg_cache);
	return ret;
}

/*
 * Unlock a full stripe.
 *
 * NOTE: Caller must ensure it's the same context calling corresponding
 * lock_full_stripe().
 *
 * Return 0 if we unlock full stripe without problem.
 * Return <0 for error
 */
static int unlock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
			      bool locked)
{
	struct btrfs_block_group_cache *bg_cache;
	struct btrfs_full_stripe_locks_tree *locks_root;
	struct full_stripe_lock *fstripe_lock;
	u64 fstripe_start;
	bool freeit = false;
	int ret = 0;

	/* If we didn't acquire full stripe lock, no need to continue */
	if (!locked)
		return 0;

	bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
	if (!bg_cache) {
		ASSERT(0);
		return -ENOENT;
	}
	if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
		goto out;

	locks_root = &bg_cache->full_stripe_locks_root;
	fstripe_start = get_full_stripe_logical(bg_cache, bytenr);

	mutex_lock(&locks_root->lock);
	fstripe_lock = search_full_stripe_lock(locks_root, fstripe_start);
	/* Unpaired unlock_full_stripe() detected */
	if (!fstripe_lock) {
		WARN_ON(1);
		ret = -ENOENT;
		mutex_unlock(&locks_root->lock);
		goto out;
	}

	if (fstripe_lock->refs == 0) {
		WARN_ON(1);
		btrfs_warn(fs_info, "full stripe lock at %llu refcount underflow",
			fstripe_lock->logical);
	} else {
		fstripe_lock->refs--;
	}

	if (fstripe_lock->refs == 0) {
		rb_erase(&fstripe_lock->node, &locks_root->root);
		freeit = true;
	}
	mutex_unlock(&locks_root->lock);

	mutex_unlock(&fstripe_lock->mutex);
	if (freeit)
		kfree(fstripe_lock);
out:
	btrfs_put_block_group(bg_cache);
	return ret;
}

527
static void scrub_free_csums(struct scrub_ctx *sctx)
A
Arne Jansen 已提交
528
{
529
	while (!list_empty(&sctx->csum_list)) {
A
Arne Jansen 已提交
530
		struct btrfs_ordered_sum *sum;
531
		sum = list_first_entry(&sctx->csum_list,
A
Arne Jansen 已提交
532 533 534 535 536 537
				       struct btrfs_ordered_sum, list);
		list_del(&sum->list);
		kfree(sum);
	}
}

538
static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
A
Arne Jansen 已提交
539 540 541
{
	int i;

542
	if (!sctx)
A
Arne Jansen 已提交
543 544
		return;

545
	/* this can happen when scrub is cancelled */
546 547
	if (sctx->curr != -1) {
		struct scrub_bio *sbio = sctx->bios[sctx->curr];
548 549

		for (i = 0; i < sbio->page_count; i++) {
550
			WARN_ON(!sbio->pagev[i]->page);
551 552 553 554 555
			scrub_block_put(sbio->pagev[i]->sblock);
		}
		bio_put(sbio->bio);
	}

556
	for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
557
		struct scrub_bio *sbio = sctx->bios[i];
A
Arne Jansen 已提交
558 559 560 561 562 563

		if (!sbio)
			break;
		kfree(sbio);
	}

564
	kfree(sctx->wr_curr_bio);
565 566
	scrub_free_csums(sctx);
	kfree(sctx);
A
Arne Jansen 已提交
567 568
}

569 570
static void scrub_put_ctx(struct scrub_ctx *sctx)
{
571
	if (refcount_dec_and_test(&sctx->refs))
572 573 574
		scrub_free_ctx(sctx);
}

575 576
static noinline_for_stack struct scrub_ctx *scrub_setup_ctx(
		struct btrfs_fs_info *fs_info, int is_dev_replace)
A
Arne Jansen 已提交
577
{
578
	struct scrub_ctx *sctx;
A
Arne Jansen 已提交
579 580
	int		i;

581
	sctx = kzalloc(sizeof(*sctx), GFP_KERNEL);
582
	if (!sctx)
A
Arne Jansen 已提交
583
		goto nomem;
584
	refcount_set(&sctx->refs, 1);
585
	sctx->is_dev_replace = is_dev_replace;
586
	sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
587
	sctx->curr = -1;
588
	sctx->fs_info = fs_info;
589
	INIT_LIST_HEAD(&sctx->csum_list);
590
	for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
A
Arne Jansen 已提交
591 592
		struct scrub_bio *sbio;

593
		sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);
A
Arne Jansen 已提交
594 595
		if (!sbio)
			goto nomem;
596
		sctx->bios[i] = sbio;
A
Arne Jansen 已提交
597 598

		sbio->index = i;
599
		sbio->sctx = sctx;
600
		sbio->page_count = 0;
601 602
		btrfs_init_work(&sbio->work, scrub_bio_end_io_worker, NULL,
				NULL);
A
Arne Jansen 已提交
603

604
		if (i != SCRUB_BIOS_PER_SCTX - 1)
605
			sctx->bios[i]->next_free = i + 1;
606
		else
607 608 609
			sctx->bios[i]->next_free = -1;
	}
	sctx->first_free = 0;
610 611
	atomic_set(&sctx->bios_in_flight, 0);
	atomic_set(&sctx->workers_pending, 0);
612 613 614 615 616 617
	atomic_set(&sctx->cancel_req, 0);
	sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);

	spin_lock_init(&sctx->list_lock);
	spin_lock_init(&sctx->stat_lock);
	init_waitqueue_head(&sctx->list_wait);
618

619 620 621
	WARN_ON(sctx->wr_curr_bio != NULL);
	mutex_init(&sctx->wr_lock);
	sctx->wr_curr_bio = NULL;
622
	if (is_dev_replace) {
623
		WARN_ON(!fs_info->dev_replace.tgtdev);
624
		sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO;
625
		sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;
626
		sctx->flush_all_writes = false;
627
	}
628

629
	return sctx;
A
Arne Jansen 已提交
630 631

nomem:
632
	scrub_free_ctx(sctx);
A
Arne Jansen 已提交
633 634 635
	return ERR_PTR(-ENOMEM);
}

636 637
static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
				     void *warn_ctx)
638 639 640 641 642
{
	u64 isize;
	u32 nlink;
	int ret;
	int i;
643
	unsigned nofs_flag;
644 645
	struct extent_buffer *eb;
	struct btrfs_inode_item *inode_item;
646
	struct scrub_warning *swarn = warn_ctx;
647
	struct btrfs_fs_info *fs_info = swarn->dev->fs_info;
648 649 650
	struct inode_fs_paths *ipath = NULL;
	struct btrfs_root *local_root;
	struct btrfs_key root_key;
651
	struct btrfs_key key;
652 653 654 655 656 657 658 659 660 661

	root_key.objectid = root;
	root_key.type = BTRFS_ROOT_ITEM_KEY;
	root_key.offset = (u64)-1;
	local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
	if (IS_ERR(local_root)) {
		ret = PTR_ERR(local_root);
		goto err;
	}

662 663 664
	/*
	 * this makes the path point to (inum INODE_ITEM ioff)
	 */
665 666 667 668 669
	key.objectid = inum;
	key.type = BTRFS_INODE_ITEM_KEY;
	key.offset = 0;

	ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
670 671 672 673 674 675 676 677 678 679 680 681
	if (ret) {
		btrfs_release_path(swarn->path);
		goto err;
	}

	eb = swarn->path->nodes[0];
	inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
					struct btrfs_inode_item);
	isize = btrfs_inode_size(eb, inode_item);
	nlink = btrfs_inode_nlink(eb, inode_item);
	btrfs_release_path(swarn->path);

682 683 684 685 686 687
	/*
	 * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub
	 * uses GFP_NOFS in this context, so we keep it consistent but it does
	 * not seem to be strictly necessary.
	 */
	nofs_flag = memalloc_nofs_save();
688
	ipath = init_ipath(4096, local_root, swarn->path);
689
	memalloc_nofs_restore(nofs_flag);
690 691 692 693 694
	if (IS_ERR(ipath)) {
		ret = PTR_ERR(ipath);
		ipath = NULL;
		goto err;
	}
695 696 697 698 699 700 701 702 703 704
	ret = paths_from_inode(inum, ipath);

	if (ret < 0)
		goto err;

	/*
	 * we deliberately ignore the bit ipath might have been too small to
	 * hold all of the paths here
	 */
	for (i = 0; i < ipath->fspath->elem_cnt; ++i)
J
Jeff Mahoney 已提交
705
		btrfs_warn_in_rcu(fs_info,
D
David Sterba 已提交
706
"%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)",
J
Jeff Mahoney 已提交
707 708
				  swarn->errstr, swarn->logical,
				  rcu_str_deref(swarn->dev->name),
D
David Sterba 已提交
709
				  swarn->physical,
J
Jeff Mahoney 已提交
710 711 712
				  root, inum, offset,
				  min(isize - offset, (u64)PAGE_SIZE), nlink,
				  (char *)(unsigned long)ipath->fspath->val[i]);
713 714 715 716 717

	free_ipath(ipath);
	return 0;

err:
J
Jeff Mahoney 已提交
718
	btrfs_warn_in_rcu(fs_info,
D
David Sterba 已提交
719
			  "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d",
J
Jeff Mahoney 已提交
720 721
			  swarn->errstr, swarn->logical,
			  rcu_str_deref(swarn->dev->name),
D
David Sterba 已提交
722
			  swarn->physical,
J
Jeff Mahoney 已提交
723
			  root, inum, offset, ret);
724 725 726 727 728

	free_ipath(ipath);
	return 0;
}

729
static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
730
{
731 732
	struct btrfs_device *dev;
	struct btrfs_fs_info *fs_info;
733 734 735 736 737
	struct btrfs_path *path;
	struct btrfs_key found_key;
	struct extent_buffer *eb;
	struct btrfs_extent_item *ei;
	struct scrub_warning swarn;
738 739 740
	unsigned long ptr = 0;
	u64 extent_item_pos;
	u64 flags = 0;
741
	u64 ref_root;
742
	u32 item_size;
743
	u8 ref_level = 0;
744
	int ret;
745

746
	WARN_ON(sblock->page_count < 1);
747
	dev = sblock->pagev[0]->dev;
748
	fs_info = sblock->sctx->fs_info;
749

750
	path = btrfs_alloc_path();
751 752
	if (!path)
		return;
753

D
David Sterba 已提交
754
	swarn.physical = sblock->pagev[0]->physical;
755
	swarn.logical = sblock->pagev[0]->logical;
756
	swarn.errstr = errstr;
757
	swarn.dev = NULL;
758

759 760
	ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
				  &flags);
761 762 763
	if (ret < 0)
		goto out;

J
Jan Schmidt 已提交
764
	extent_item_pos = swarn.logical - found_key.objectid;
765 766 767 768 769 770
	swarn.extent_item_size = found_key.offset;

	eb = path->nodes[0];
	ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
	item_size = btrfs_item_size_nr(eb, path->slots[0]);

771
	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
772
		do {
773 774 775
			ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
						      item_size, &ref_root,
						      &ref_level);
776
			btrfs_warn_in_rcu(fs_info,
D
David Sterba 已提交
777
"%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",
J
Jeff Mahoney 已提交
778
				errstr, swarn.logical,
779
				rcu_str_deref(dev->name),
D
David Sterba 已提交
780
				swarn.physical,
781 782 783 784
				ref_level ? "node" : "leaf",
				ret < 0 ? -1 : ref_level,
				ret < 0 ? -1 : ref_root);
		} while (ret != 1);
785
		btrfs_release_path(path);
786
	} else {
787
		btrfs_release_path(path);
788
		swarn.path = path;
789
		swarn.dev = dev;
790 791
		iterate_extent_inodes(fs_info, found_key.objectid,
					extent_item_pos, 1,
792
					scrub_print_warning_inode, &swarn, false);
793 794 795 796 797 798
	}

out:
	btrfs_free_path(path);
}

799 800
static inline void scrub_get_recover(struct scrub_recover *recover)
{
801
	refcount_inc(&recover->refs);
802 803
}

804 805
static inline void scrub_put_recover(struct btrfs_fs_info *fs_info,
				     struct scrub_recover *recover)
806
{
807
	if (refcount_dec_and_test(&recover->refs)) {
808
		btrfs_bio_counter_dec(fs_info);
809
		btrfs_put_bbio(recover->bbio);
810 811 812 813
		kfree(recover);
	}
}

A
Arne Jansen 已提交
814
/*
815 816 817 818 819 820
 * scrub_handle_errored_block gets called when either verification of the
 * pages failed or the bio failed to read, e.g. with EIO. In the latter
 * case, this function handles all pages in the bio, even though only one
 * may be bad.
 * The goal of this function is to repair the errored block by using the
 * contents of one of the mirrors.
A
Arne Jansen 已提交
821
 */
822
static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
A
Arne Jansen 已提交
823
{
824
	struct scrub_ctx *sctx = sblock_to_check->sctx;
825
	struct btrfs_device *dev;
826 827 828 829 830 831 832 833 834 835 836
	struct btrfs_fs_info *fs_info;
	u64 logical;
	unsigned int failed_mirror_index;
	unsigned int is_metadata;
	unsigned int have_csum;
	struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
	struct scrub_block *sblock_bad;
	int ret;
	int mirror_index;
	int page_num;
	int success;
837
	bool full_stripe_locked;
838
	unsigned int nofs_flag;
839
	static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
840 841 842
				      DEFAULT_RATELIMIT_BURST);

	BUG_ON(sblock_to_check->page_count < 1);
843
	fs_info = sctx->fs_info;
844 845 846 847 848 849 850 851 852 853 854
	if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
		/*
		 * if we find an error in a super block, we just report it.
		 * They will get written with the next transaction commit
		 * anyway
		 */
		spin_lock(&sctx->stat_lock);
		++sctx->stat.super_errors;
		spin_unlock(&sctx->stat_lock);
		return 0;
	}
855 856 857 858
	logical = sblock_to_check->pagev[0]->logical;
	BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
	failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
	is_metadata = !(sblock_to_check->pagev[0]->flags &
859
			BTRFS_EXTENT_FLAG_DATA);
860 861
	have_csum = sblock_to_check->pagev[0]->have_csum;
	dev = sblock_to_check->pagev[0]->dev;
862

863 864 865 866 867 868 869 870 871 872
	/*
	 * We must use GFP_NOFS because the scrub task might be waiting for a
	 * worker task executing this function and in turn a transaction commit
	 * might be waiting the scrub task to pause (which needs to wait for all
	 * the worker tasks to complete before pausing).
	 * We do allocations in the workers through insert_full_stripe_lock()
	 * and scrub_add_page_to_wr_bio(), which happens down the call chain of
	 * this function.
	 */
	nofs_flag = memalloc_nofs_save();
873 874 875 876 877 878 879 880 881
	/*
	 * For RAID5/6, race can happen for a different device scrub thread.
	 * For data corruption, Parity and Data threads will both try
	 * to recovery the data.
	 * Race can lead to doubly added csum error, or even unrecoverable
	 * error.
	 */
	ret = lock_full_stripe(fs_info, logical, &full_stripe_locked);
	if (ret < 0) {
882
		memalloc_nofs_restore(nofs_flag);
883 884 885 886 887 888 889 890 891
		spin_lock(&sctx->stat_lock);
		if (ret == -ENOMEM)
			sctx->stat.malloc_errors++;
		sctx->stat.read_errors++;
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
		return ret;
	}

892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
	/*
	 * read all mirrors one after the other. This includes to
	 * re-read the extent or metadata block that failed (that was
	 * the cause that this fixup code is called) another time,
	 * page by page this time in order to know which pages
	 * caused I/O errors and which ones are good (for all mirrors).
	 * It is the goal to handle the situation when more than one
	 * mirror contains I/O errors, but the errors do not
	 * overlap, i.e. the data can be repaired by selecting the
	 * pages from those mirrors without I/O error on the
	 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
	 * would be that mirror #1 has an I/O error on the first page,
	 * the second page is good, and mirror #2 has an I/O error on
	 * the second page, but the first page is good.
	 * Then the first page of the first mirror can be repaired by
	 * taking the first page of the second mirror, and the
	 * second page of the second mirror can be repaired by
	 * copying the contents of the 2nd page of the 1st mirror.
	 * One more note: if the pages of one mirror contain I/O
	 * errors, the checksum cannot be verified. In order to get
	 * the best data for repairing, the first attempt is to find
	 * a mirror without I/O errors and with a validated checksum.
	 * Only if this is not possible, the pages are picked from
	 * mirrors with I/O errors without considering the checksum.
	 * If the latter is the case, at the end, the checksum of the
	 * repaired area is verified in order to correctly maintain
	 * the statistics.
	 */

921
	sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
922
				      sizeof(*sblocks_for_recheck), GFP_KERNEL);
923
	if (!sblocks_for_recheck) {
924 925 926 927 928
		spin_lock(&sctx->stat_lock);
		sctx->stat.malloc_errors++;
		sctx->stat.read_errors++;
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
929
		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
930
		goto out;
A
Arne Jansen 已提交
931 932
	}

933
	/* setup the context, map the logical blocks and alloc the pages */
934
	ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
935
	if (ret) {
936 937 938 939
		spin_lock(&sctx->stat_lock);
		sctx->stat.read_errors++;
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
940
		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
941 942 943 944
		goto out;
	}
	BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
	sblock_bad = sblocks_for_recheck + failed_mirror_index;
945

946
	/* build and submit the bios for the failed mirror, check checksums */
947
	scrub_recheck_block(fs_info, sblock_bad, 1);
A
Arne Jansen 已提交
948

949 950 951 952 953 954 955 956 957 958
	if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
	    sblock_bad->no_io_error_seen) {
		/*
		 * the error disappeared after reading page by page, or
		 * the area was part of a huge bio and other parts of the
		 * bio caused I/O errors, or the block layer merged several
		 * read requests into one and the error is caused by a
		 * different bio (usually one of the two latter cases is
		 * the cause)
		 */
959 960
		spin_lock(&sctx->stat_lock);
		sctx->stat.unverified_errors++;
961
		sblock_to_check->data_corrected = 1;
962
		spin_unlock(&sctx->stat_lock);
A
Arne Jansen 已提交
963

964 965
		if (sctx->is_dev_replace)
			scrub_write_block_to_dev_replace(sblock_bad);
966
		goto out;
A
Arne Jansen 已提交
967 968
	}

969
	if (!sblock_bad->no_io_error_seen) {
970 971 972
		spin_lock(&sctx->stat_lock);
		sctx->stat.read_errors++;
		spin_unlock(&sctx->stat_lock);
973 974
		if (__ratelimit(&_rs))
			scrub_print_warning("i/o error", sblock_to_check);
975
		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
976
	} else if (sblock_bad->checksum_error) {
977 978 979
		spin_lock(&sctx->stat_lock);
		sctx->stat.csum_errors++;
		spin_unlock(&sctx->stat_lock);
980 981
		if (__ratelimit(&_rs))
			scrub_print_warning("checksum error", sblock_to_check);
982
		btrfs_dev_stat_inc_and_print(dev,
983
					     BTRFS_DEV_STAT_CORRUPTION_ERRS);
984
	} else if (sblock_bad->header_error) {
985 986 987
		spin_lock(&sctx->stat_lock);
		sctx->stat.verify_errors++;
		spin_unlock(&sctx->stat_lock);
988 989 990
		if (__ratelimit(&_rs))
			scrub_print_warning("checksum/header error",
					    sblock_to_check);
991
		if (sblock_bad->generation_error)
992
			btrfs_dev_stat_inc_and_print(dev,
993 994
				BTRFS_DEV_STAT_GENERATION_ERRS);
		else
995
			btrfs_dev_stat_inc_and_print(dev,
996
				BTRFS_DEV_STAT_CORRUPTION_ERRS);
997
	}
A
Arne Jansen 已提交
998

999 1000 1001 1002
	if (sctx->readonly) {
		ASSERT(!sctx->is_dev_replace);
		goto out;
	}
A
Arne Jansen 已提交
1003

1004 1005
	/*
	 * now build and submit the bios for the other mirrors, check
1006 1007
	 * checksums.
	 * First try to pick the mirror which is completely without I/O
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
	 * errors and also does not have a checksum error.
	 * If one is found, and if a checksum is present, the full block
	 * that is known to contain an error is rewritten. Afterwards
	 * the block is known to be corrected.
	 * If a mirror is found which is completely correct, and no
	 * checksum is present, only those pages are rewritten that had
	 * an I/O error in the block to be repaired, since it cannot be
	 * determined, which copy of the other pages is better (and it
	 * could happen otherwise that a correct page would be
	 * overwritten by a bad one).
	 */
1019
	for (mirror_index = 0; ;mirror_index++) {
1020
		struct scrub_block *sblock_other;
1021

1022 1023
		if (mirror_index == failed_mirror_index)
			continue;
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046

		/* raid56's mirror can be more than BTRFS_MAX_MIRRORS */
		if (!scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
			if (mirror_index >= BTRFS_MAX_MIRRORS)
				break;
			if (!sblocks_for_recheck[mirror_index].page_count)
				break;

			sblock_other = sblocks_for_recheck + mirror_index;
		} else {
			struct scrub_recover *r = sblock_bad->pagev[0]->recover;
			int max_allowed = r->bbio->num_stripes -
						r->bbio->num_tgtdevs;

			if (mirror_index >= max_allowed)
				break;
			if (!sblocks_for_recheck[1].page_count)
				break;

			ASSERT(failed_mirror_index == 0);
			sblock_other = sblocks_for_recheck + 1;
			sblock_other->pagev[0]->mirror_num = 1 + mirror_index;
		}
1047 1048

		/* build and submit the bios, check checksums */
1049
		scrub_recheck_block(fs_info, sblock_other, 0);
1050 1051

		if (!sblock_other->header_error &&
1052 1053
		    !sblock_other->checksum_error &&
		    sblock_other->no_io_error_seen) {
1054 1055
			if (sctx->is_dev_replace) {
				scrub_write_block_to_dev_replace(sblock_other);
1056
				goto corrected_error;
1057 1058
			} else {
				ret = scrub_repair_block_from_good_copy(
1059 1060 1061
						sblock_bad, sblock_other);
				if (!ret)
					goto corrected_error;
1062
			}
1063 1064
		}
	}
A
Arne Jansen 已提交
1065

1066 1067
	if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
		goto did_not_correct_error;
1068 1069 1070

	/*
	 * In case of I/O errors in the area that is supposed to be
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
	 * repaired, continue by picking good copies of those pages.
	 * Select the good pages from mirrors to rewrite bad pages from
	 * the area to fix. Afterwards verify the checksum of the block
	 * that is supposed to be repaired. This verification step is
	 * only done for the purpose of statistic counting and for the
	 * final scrub report, whether errors remain.
	 * A perfect algorithm could make use of the checksum and try
	 * all possible combinations of pages from the different mirrors
	 * until the checksum verification succeeds. For example, when
	 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
	 * of mirror #2 is readable but the final checksum test fails,
	 * then the 2nd page of mirror #3 could be tried, whether now
1083
	 * the final checksum succeeds. But this would be a rare
1084 1085 1086 1087 1088 1089 1090 1091
	 * exception and is therefore not implemented. At least it is
	 * avoided that the good copy is overwritten.
	 * A more useful improvement would be to pick the sectors
	 * without I/O error based on sector sizes (512 bytes on legacy
	 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
	 * mirror could be repaired by taking 512 byte of a different
	 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
	 * area are unreadable.
A
Arne Jansen 已提交
1092
	 */
1093
	success = 1;
1094 1095
	for (page_num = 0; page_num < sblock_bad->page_count;
	     page_num++) {
1096
		struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1097
		struct scrub_block *sblock_other = NULL;
1098

1099 1100
		/* skip no-io-error page in scrub */
		if (!page_bad->io_error && !sctx->is_dev_replace)
A
Arne Jansen 已提交
1101
			continue;
1102

1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
		if (scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
			/*
			 * In case of dev replace, if raid56 rebuild process
			 * didn't work out correct data, then copy the content
			 * in sblock_bad to make sure target device is identical
			 * to source device, instead of writing garbage data in
			 * sblock_for_recheck array to target device.
			 */
			sblock_other = NULL;
		} else if (page_bad->io_error) {
			/* try to find no-io-error page in mirrors */
1114 1115 1116 1117 1118 1119 1120 1121 1122
			for (mirror_index = 0;
			     mirror_index < BTRFS_MAX_MIRRORS &&
			     sblocks_for_recheck[mirror_index].page_count > 0;
			     mirror_index++) {
				if (!sblocks_for_recheck[mirror_index].
				    pagev[page_num]->io_error) {
					sblock_other = sblocks_for_recheck +
						       mirror_index;
					break;
1123 1124
				}
			}
1125 1126
			if (!sblock_other)
				success = 0;
I
Ilya Dryomov 已提交
1127
		}
A
Arne Jansen 已提交
1128

1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
		if (sctx->is_dev_replace) {
			/*
			 * did not find a mirror to fetch the page
			 * from. scrub_write_page_to_dev_replace()
			 * handles this case (page->io_error), by
			 * filling the block with zeros before
			 * submitting the write request
			 */
			if (!sblock_other)
				sblock_other = sblock_bad;

			if (scrub_write_page_to_dev_replace(sblock_other,
							    page_num) != 0) {
1142
				atomic64_inc(
1143
					&fs_info->dev_replace.num_write_errors);
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
				success = 0;
			}
		} else if (sblock_other) {
			ret = scrub_repair_page_from_good_copy(sblock_bad,
							       sblock_other,
							       page_num, 0);
			if (0 == ret)
				page_bad->io_error = 0;
			else
				success = 0;
1154
		}
A
Arne Jansen 已提交
1155 1156
	}

1157
	if (success && !sctx->is_dev_replace) {
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
		if (is_metadata || have_csum) {
			/*
			 * need to verify the checksum now that all
			 * sectors on disk are repaired (the write
			 * request for data to be repaired is on its way).
			 * Just be lazy and use scrub_recheck_block()
			 * which re-reads the data before the checksum
			 * is verified, but most likely the data comes out
			 * of the page cache.
			 */
1168
			scrub_recheck_block(fs_info, sblock_bad, 1);
1169
			if (!sblock_bad->header_error &&
1170 1171 1172 1173 1174 1175 1176
			    !sblock_bad->checksum_error &&
			    sblock_bad->no_io_error_seen)
				goto corrected_error;
			else
				goto did_not_correct_error;
		} else {
corrected_error:
1177 1178
			spin_lock(&sctx->stat_lock);
			sctx->stat.corrected_errors++;
1179
			sblock_to_check->data_corrected = 1;
1180
			spin_unlock(&sctx->stat_lock);
1181 1182
			btrfs_err_rl_in_rcu(fs_info,
				"fixed up error at logical %llu on dev %s",
1183
				logical, rcu_str_deref(dev->name));
A
Arne Jansen 已提交
1184
		}
1185 1186
	} else {
did_not_correct_error:
1187 1188 1189
		spin_lock(&sctx->stat_lock);
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
1190 1191
		btrfs_err_rl_in_rcu(fs_info,
			"unable to fixup (regular) error at logical %llu on dev %s",
1192
			logical, rcu_str_deref(dev->name));
I
Ilya Dryomov 已提交
1193
	}
A
Arne Jansen 已提交
1194

1195 1196 1197 1198 1199 1200
out:
	if (sblocks_for_recheck) {
		for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
		     mirror_index++) {
			struct scrub_block *sblock = sblocks_for_recheck +
						     mirror_index;
1201
			struct scrub_recover *recover;
1202 1203
			int page_index;

1204 1205 1206
			for (page_index = 0; page_index < sblock->page_count;
			     page_index++) {
				sblock->pagev[page_index]->sblock = NULL;
1207 1208
				recover = sblock->pagev[page_index]->recover;
				if (recover) {
1209
					scrub_put_recover(fs_info, recover);
1210 1211 1212
					sblock->pagev[page_index]->recover =
									NULL;
				}
1213 1214
				scrub_page_put(sblock->pagev[page_index]);
			}
1215 1216 1217
		}
		kfree(sblocks_for_recheck);
	}
A
Arne Jansen 已提交
1218

1219
	ret = unlock_full_stripe(fs_info, logical, full_stripe_locked);
1220
	memalloc_nofs_restore(nofs_flag);
1221 1222
	if (ret < 0)
		return ret;
1223 1224
	return 0;
}
A
Arne Jansen 已提交
1225

1226
static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1227
{
Z
Zhao Lei 已提交
1228 1229 1230 1231 1232
	if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
		return 2;
	else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
		return 3;
	else
1233 1234 1235
		return (int)bbio->num_stripes;
}

Z
Zhao Lei 已提交
1236 1237
static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
						 u64 *raid_map,
1238 1239 1240 1241 1242 1243 1244
						 u64 mapped_length,
						 int nstripes, int mirror,
						 int *stripe_index,
						 u64 *stripe_offset)
{
	int i;

1245
	if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
		/* RAID5/6 */
		for (i = 0; i < nstripes; i++) {
			if (raid_map[i] == RAID6_Q_STRIPE ||
			    raid_map[i] == RAID5_P_STRIPE)
				continue;

			if (logical >= raid_map[i] &&
			    logical < raid_map[i] + mapped_length)
				break;
		}

		*stripe_index = i;
		*stripe_offset = logical - raid_map[i];
	} else {
		/* The other RAID type */
		*stripe_index = mirror;
		*stripe_offset = 0;
	}
}

1266
static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1267 1268
				     struct scrub_block *sblocks_for_recheck)
{
1269
	struct scrub_ctx *sctx = original_sblock->sctx;
1270
	struct btrfs_fs_info *fs_info = sctx->fs_info;
1271 1272
	u64 length = original_sblock->page_count * PAGE_SIZE;
	u64 logical = original_sblock->pagev[0]->logical;
1273 1274 1275
	u64 generation = original_sblock->pagev[0]->generation;
	u64 flags = original_sblock->pagev[0]->flags;
	u64 have_csum = original_sblock->pagev[0]->have_csum;
1276 1277 1278 1279 1280 1281
	struct scrub_recover *recover;
	struct btrfs_bio *bbio;
	u64 sublen;
	u64 mapped_length;
	u64 stripe_offset;
	int stripe_index;
1282
	int page_index = 0;
1283
	int mirror_index;
1284
	int nmirrors;
1285 1286 1287
	int ret;

	/*
1288
	 * note: the two members refs and outstanding_pages
1289 1290 1291 1292 1293
	 * are not used (and not set) in the blocks that are used for
	 * the recheck procedure
	 */

	while (length > 0) {
1294 1295 1296
		sublen = min_t(u64, length, PAGE_SIZE);
		mapped_length = sublen;
		bbio = NULL;
A
Arne Jansen 已提交
1297

1298 1299 1300 1301
		/*
		 * with a length of PAGE_SIZE, each returned stripe
		 * represents one mirror
		 */
1302
		btrfs_bio_counter_inc_blocked(fs_info);
1303
		ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
1304
				logical, &mapped_length, &bbio);
1305
		if (ret || !bbio || mapped_length < sublen) {
1306
			btrfs_put_bbio(bbio);
1307
			btrfs_bio_counter_dec(fs_info);
1308 1309
			return -EIO;
		}
A
Arne Jansen 已提交
1310

1311 1312
		recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
		if (!recover) {
1313
			btrfs_put_bbio(bbio);
1314
			btrfs_bio_counter_dec(fs_info);
1315 1316 1317
			return -ENOMEM;
		}

1318
		refcount_set(&recover->refs, 1);
1319 1320 1321
		recover->bbio = bbio;
		recover->map_length = mapped_length;

1322
		BUG_ON(page_index >= SCRUB_MAX_PAGES_PER_BLOCK);
1323

1324
		nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
Z
Zhao Lei 已提交
1325

1326
		for (mirror_index = 0; mirror_index < nmirrors;
1327 1328 1329 1330 1331
		     mirror_index++) {
			struct scrub_block *sblock;
			struct scrub_page *page;

			sblock = sblocks_for_recheck + mirror_index;
1332
			sblock->sctx = sctx;
1333

1334 1335 1336
			page = kzalloc(sizeof(*page), GFP_NOFS);
			if (!page) {
leave_nomem:
1337 1338 1339
				spin_lock(&sctx->stat_lock);
				sctx->stat.malloc_errors++;
				spin_unlock(&sctx->stat_lock);
1340
				scrub_put_recover(fs_info, recover);
1341 1342
				return -ENOMEM;
			}
1343 1344
			scrub_page_get(page);
			sblock->pagev[page_index] = page;
1345 1346 1347
			page->sblock = sblock;
			page->flags = flags;
			page->generation = generation;
1348
			page->logical = logical;
1349 1350 1351 1352 1353
			page->have_csum = have_csum;
			if (have_csum)
				memcpy(page->csum,
				       original_sblock->pagev[0]->csum,
				       sctx->csum_size);
1354

Z
Zhao Lei 已提交
1355 1356 1357
			scrub_stripe_index_and_offset(logical,
						      bbio->map_type,
						      bbio->raid_map,
1358
						      mapped_length,
1359 1360
						      bbio->num_stripes -
						      bbio->num_tgtdevs,
1361 1362 1363 1364 1365 1366 1367
						      mirror_index,
						      &stripe_index,
						      &stripe_offset);
			page->physical = bbio->stripes[stripe_index].physical +
					 stripe_offset;
			page->dev = bbio->stripes[stripe_index].dev;

1368 1369 1370 1371
			BUG_ON(page_index >= original_sblock->page_count);
			page->physical_for_dev_replace =
				original_sblock->pagev[page_index]->
				physical_for_dev_replace;
1372 1373
			/* for missing devices, dev->bdev is NULL */
			page->mirror_num = mirror_index + 1;
1374
			sblock->page_count++;
1375 1376 1377
			page->page = alloc_page(GFP_NOFS);
			if (!page->page)
				goto leave_nomem;
1378 1379 1380

			scrub_get_recover(recover);
			page->recover = recover;
1381
		}
1382
		scrub_put_recover(fs_info, recover);
1383 1384 1385 1386 1387 1388
		length -= sublen;
		logical += sublen;
		page_index++;
	}

	return 0;
I
Ilya Dryomov 已提交
1389 1390
}

1391
static void scrub_bio_wait_endio(struct bio *bio)
1392
{
1393
	complete(bio->bi_private);
1394 1395 1396 1397 1398 1399
}

static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
					struct bio *bio,
					struct scrub_page *page)
{
1400
	DECLARE_COMPLETION_ONSTACK(done);
1401
	int ret;
1402
	int mirror_num;
1403 1404 1405 1406 1407

	bio->bi_iter.bi_sector = page->logical >> 9;
	bio->bi_private = &done;
	bio->bi_end_io = scrub_bio_wait_endio;

1408
	mirror_num = page->sblock->pagev[0]->mirror_num;
1409
	ret = raid56_parity_recover(fs_info, bio, page->recover->bbio,
1410
				    page->recover->map_length,
1411
				    mirror_num, 0);
1412 1413 1414
	if (ret)
		return ret;

1415 1416
	wait_for_completion_io(&done);
	return blk_status_to_errno(bio->bi_status);
1417 1418
}

L
Liu Bo 已提交
1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info,
					  struct scrub_block *sblock)
{
	struct scrub_page *first_page = sblock->pagev[0];
	struct bio *bio;
	int page_num;

	/* All pages in sblock belong to the same stripe on the same device. */
	ASSERT(first_page->dev);
	if (!first_page->dev->bdev)
		goto out;

	bio = btrfs_io_bio_alloc(BIO_MAX_PAGES);
	bio_set_dev(bio, first_page->dev->bdev);

	for (page_num = 0; page_num < sblock->page_count; page_num++) {
		struct scrub_page *page = sblock->pagev[page_num];

		WARN_ON(!page->page);
		bio_add_page(bio, page->page, PAGE_SIZE, 0);
	}

	if (scrub_submit_raid56_bio_wait(fs_info, bio, first_page)) {
		bio_put(bio);
		goto out;
	}

	bio_put(bio);

	scrub_recheck_block_checksum(sblock);

	return;
out:
	for (page_num = 0; page_num < sblock->page_count; page_num++)
		sblock->pagev[page_num]->io_error = 1;

	sblock->no_io_error_seen = 0;
}

1458 1459 1460 1461 1462 1463 1464
/*
 * this function will check the on disk data for checksum errors, header
 * errors and read I/O errors. If any I/O errors happen, the exact pages
 * which are errored are marked as being bad. The goal is to enable scrub
 * to take those pages that are not errored from all the mirrors so that
 * the pages that are errored in the just handled mirror can be repaired.
 */
1465
static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1466 1467
				struct scrub_block *sblock,
				int retry_failed_mirror)
I
Ilya Dryomov 已提交
1468
{
1469
	int page_num;
I
Ilya Dryomov 已提交
1470

1471
	sblock->no_io_error_seen = 1;
I
Ilya Dryomov 已提交
1472

L
Liu Bo 已提交
1473 1474 1475 1476
	/* short cut for raid56 */
	if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))
		return scrub_recheck_block_on_raid56(fs_info, sblock);

1477 1478
	for (page_num = 0; page_num < sblock->page_count; page_num++) {
		struct bio *bio;
1479
		struct scrub_page *page = sblock->pagev[page_num];
1480

1481
		if (page->dev->bdev == NULL) {
1482 1483 1484 1485 1486
			page->io_error = 1;
			sblock->no_io_error_seen = 0;
			continue;
		}

1487
		WARN_ON(!page->page);
1488
		bio = btrfs_io_bio_alloc(1);
1489
		bio_set_dev(bio, page->dev->bdev);
1490

1491
		bio_add_page(bio, page->page, PAGE_SIZE, 0);
L
Liu Bo 已提交
1492 1493
		bio->bi_iter.bi_sector = page->physical >> 9;
		bio->bi_opf = REQ_OP_READ;
1494

L
Liu Bo 已提交
1495 1496 1497
		if (btrfsic_submit_bio_wait(bio)) {
			page->io_error = 1;
			sblock->no_io_error_seen = 0;
1498
		}
1499

1500 1501
		bio_put(bio);
	}
I
Ilya Dryomov 已提交
1502

1503
	if (sblock->no_io_error_seen)
1504
		scrub_recheck_block_checksum(sblock);
A
Arne Jansen 已提交
1505 1506
}

M
Miao Xie 已提交
1507 1508 1509 1510 1511 1512
static inline int scrub_check_fsid(u8 fsid[],
				   struct scrub_page *spage)
{
	struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
	int ret;

1513
	ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
M
Miao Xie 已提交
1514 1515 1516
	return !ret;
}

1517
static void scrub_recheck_block_checksum(struct scrub_block *sblock)
A
Arne Jansen 已提交
1518
{
1519 1520 1521
	sblock->header_error = 0;
	sblock->checksum_error = 0;
	sblock->generation_error = 0;
1522

1523 1524 1525 1526
	if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)
		scrub_checksum_data(sblock);
	else
		scrub_checksum_tree_block(sblock);
A
Arne Jansen 已提交
1527 1528
}

1529
static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1530
					     struct scrub_block *sblock_good)
1531 1532 1533
{
	int page_num;
	int ret = 0;
I
Ilya Dryomov 已提交
1534

1535 1536
	for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
		int ret_sub;
I
Ilya Dryomov 已提交
1537

1538 1539
		ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
							   sblock_good,
1540
							   page_num, 1);
1541 1542
		if (ret_sub)
			ret = ret_sub;
A
Arne Jansen 已提交
1543
	}
1544 1545 1546 1547 1548 1549 1550 1551

	return ret;
}

static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
					    struct scrub_block *sblock_good,
					    int page_num, int force_write)
{
1552 1553
	struct scrub_page *page_bad = sblock_bad->pagev[page_num];
	struct scrub_page *page_good = sblock_good->pagev[page_num];
1554
	struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info;
1555

1556 1557
	BUG_ON(page_bad->page == NULL);
	BUG_ON(page_good->page == NULL);
1558 1559 1560 1561 1562
	if (force_write || sblock_bad->header_error ||
	    sblock_bad->checksum_error || page_bad->io_error) {
		struct bio *bio;
		int ret;

1563
		if (!page_bad->dev->bdev) {
1564
			btrfs_warn_rl(fs_info,
J
Jeff Mahoney 已提交
1565
				"scrub_repair_page_from_good_copy(bdev == NULL) is unexpected");
1566 1567 1568
			return -EIO;
		}

1569
		bio = btrfs_io_bio_alloc(1);
1570
		bio_set_dev(bio, page_bad->dev->bdev);
1571
		bio->bi_iter.bi_sector = page_bad->physical >> 9;
D
David Sterba 已提交
1572
		bio->bi_opf = REQ_OP_WRITE;
1573 1574 1575 1576 1577

		ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
		if (PAGE_SIZE != ret) {
			bio_put(bio);
			return -EIO;
1578
		}
1579

1580
		if (btrfsic_submit_bio_wait(bio)) {
1581 1582
			btrfs_dev_stat_inc_and_print(page_bad->dev,
				BTRFS_DEV_STAT_WRITE_ERRS);
1583
			atomic64_inc(&fs_info->dev_replace.num_write_errors);
1584 1585 1586
			bio_put(bio);
			return -EIO;
		}
1587
		bio_put(bio);
A
Arne Jansen 已提交
1588 1589
	}

1590 1591 1592
	return 0;
}

1593 1594
static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
{
1595
	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
1596 1597
	int page_num;

1598 1599 1600 1601 1602 1603 1604
	/*
	 * This block is used for the check of the parity on the source device,
	 * so the data needn't be written into the destination device.
	 */
	if (sblock->sparity)
		return;

1605 1606 1607 1608 1609
	for (page_num = 0; page_num < sblock->page_count; page_num++) {
		int ret;

		ret = scrub_write_page_to_dev_replace(sblock, page_num);
		if (ret)
1610
			atomic64_inc(&fs_info->dev_replace.num_write_errors);
1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
	}
}

static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
					   int page_num)
{
	struct scrub_page *spage = sblock->pagev[page_num];

	BUG_ON(spage->page == NULL);
	if (spage->io_error) {
		void *mapped_buffer = kmap_atomic(spage->page);

1623
		clear_page(mapped_buffer);
1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
		flush_dcache_page(spage->page);
		kunmap_atomic(mapped_buffer);
	}
	return scrub_add_page_to_wr_bio(sblock->sctx, spage);
}

static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
				    struct scrub_page *spage)
{
	struct scrub_bio *sbio;
	int ret;

1636
	mutex_lock(&sctx->wr_lock);
1637
again:
1638 1639
	if (!sctx->wr_curr_bio) {
		sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio),
1640
					      GFP_KERNEL);
1641 1642
		if (!sctx->wr_curr_bio) {
			mutex_unlock(&sctx->wr_lock);
1643 1644
			return -ENOMEM;
		}
1645 1646
		sctx->wr_curr_bio->sctx = sctx;
		sctx->wr_curr_bio->page_count = 0;
1647
	}
1648
	sbio = sctx->wr_curr_bio;
1649 1650 1651 1652 1653
	if (sbio->page_count == 0) {
		struct bio *bio;

		sbio->physical = spage->physical_for_dev_replace;
		sbio->logical = spage->logical;
1654
		sbio->dev = sctx->wr_tgtdev;
1655 1656
		bio = sbio->bio;
		if (!bio) {
1657
			bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio);
1658 1659 1660 1661 1662
			sbio->bio = bio;
		}

		bio->bi_private = sbio;
		bio->bi_end_io = scrub_wr_bio_end_io;
1663
		bio_set_dev(bio, sbio->dev->bdev);
1664
		bio->bi_iter.bi_sector = sbio->physical >> 9;
D
David Sterba 已提交
1665
		bio->bi_opf = REQ_OP_WRITE;
1666
		sbio->status = 0;
1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
		   spage->physical_for_dev_replace ||
		   sbio->logical + sbio->page_count * PAGE_SIZE !=
		   spage->logical) {
		scrub_wr_submit(sctx);
		goto again;
	}

	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
	if (ret != PAGE_SIZE) {
		if (sbio->page_count < 1) {
			bio_put(sbio->bio);
			sbio->bio = NULL;
1680
			mutex_unlock(&sctx->wr_lock);
1681 1682 1683 1684 1685 1686 1687 1688 1689
			return -EIO;
		}
		scrub_wr_submit(sctx);
		goto again;
	}

	sbio->pagev[sbio->page_count] = spage;
	scrub_page_get(spage);
	sbio->page_count++;
1690
	if (sbio->page_count == sctx->pages_per_wr_bio)
1691
		scrub_wr_submit(sctx);
1692
	mutex_unlock(&sctx->wr_lock);
1693 1694 1695 1696 1697 1698 1699 1700

	return 0;
}

static void scrub_wr_submit(struct scrub_ctx *sctx)
{
	struct scrub_bio *sbio;

1701
	if (!sctx->wr_curr_bio)
1702 1703
		return;

1704 1705
	sbio = sctx->wr_curr_bio;
	sctx->wr_curr_bio = NULL;
1706
	WARN_ON(!sbio->bio->bi_disk);
1707 1708 1709 1710 1711
	scrub_pending_bio_inc(sctx);
	/* process all writes in a single worker thread. Then the block layer
	 * orders the requests before sending them to the driver which
	 * doubled the write performance on spinning disks when measured
	 * with Linux 3.5 */
1712
	btrfsic_submit_bio(sbio->bio);
1713 1714
}

1715
static void scrub_wr_bio_end_io(struct bio *bio)
1716 1717
{
	struct scrub_bio *sbio = bio->bi_private;
1718
	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
1719

1720
	sbio->status = bio->bi_status;
1721 1722
	sbio->bio = bio;

1723
	btrfs_init_work(&sbio->work, scrub_wr_bio_end_io_worker, NULL, NULL);
1724
	btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
1725 1726 1727 1728 1729 1730 1731 1732 1733
}

static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
{
	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
	struct scrub_ctx *sctx = sbio->sctx;
	int i;

	WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1734
	if (sbio->status) {
1735
		struct btrfs_dev_replace *dev_replace =
1736
			&sbio->sctx->fs_info->dev_replace;
1737 1738 1739 1740 1741

		for (i = 0; i < sbio->page_count; i++) {
			struct scrub_page *spage = sbio->pagev[i];

			spage->io_error = 1;
1742
			atomic64_inc(&dev_replace->num_write_errors);
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
		}
	}

	for (i = 0; i < sbio->page_count; i++)
		scrub_page_put(sbio->pagev[i]);

	bio_put(sbio->bio);
	kfree(sbio);
	scrub_pending_bio_dec(sctx);
}

static int scrub_checksum(struct scrub_block *sblock)
1755 1756 1757 1758
{
	u64 flags;
	int ret;

1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770
	/*
	 * No need to initialize these stats currently,
	 * because this function only use return value
	 * instead of these stats value.
	 *
	 * Todo:
	 * always use stats
	 */
	sblock->header_error = 0;
	sblock->generation_error = 0;
	sblock->checksum_error = 0;

1771 1772
	WARN_ON(sblock->page_count < 1);
	flags = sblock->pagev[0]->flags;
1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783
	ret = 0;
	if (flags & BTRFS_EXTENT_FLAG_DATA)
		ret = scrub_checksum_data(sblock);
	else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
		ret = scrub_checksum_tree_block(sblock);
	else if (flags & BTRFS_EXTENT_FLAG_SUPER)
		(void)scrub_checksum_super(sblock);
	else
		WARN_ON(1);
	if (ret)
		scrub_handle_errored_block(sblock);
1784 1785

	return ret;
A
Arne Jansen 已提交
1786 1787
}

1788
static int scrub_checksum_data(struct scrub_block *sblock)
A
Arne Jansen 已提交
1789
{
1790
	struct scrub_ctx *sctx = sblock->sctx;
1791 1792
	struct btrfs_fs_info *fs_info = sctx->fs_info;
	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
A
Arne Jansen 已提交
1793
	u8 csum[BTRFS_CSUM_SIZE];
1794 1795 1796 1797 1798
	u8 *on_disk_csum;
	struct page *page;
	void *buffer;
	u64 len;
	int index;
A
Arne Jansen 已提交
1799

1800
	BUG_ON(sblock->page_count < 1);
1801
	if (!sblock->pagev[0]->have_csum)
A
Arne Jansen 已提交
1802 1803
		return 0;

1804 1805 1806
	shash->tfm = fs_info->csum_shash;
	crypto_shash_init(shash);

1807 1808
	on_disk_csum = sblock->pagev[0]->csum;
	page = sblock->pagev[0]->page;
1809
	buffer = kmap_atomic(page);
1810

1811
	len = sctx->fs_info->sectorsize;
1812 1813 1814 1815
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, PAGE_SIZE);

1816
		crypto_shash_update(shash, buffer, l);
1817
		kunmap_atomic(buffer);
1818 1819 1820 1821 1822
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
1823 1824
		BUG_ON(!sblock->pagev[index]->page);
		page = sblock->pagev[index]->page;
1825
		buffer = kmap_atomic(page);
1826 1827
	}

1828
	crypto_shash_final(shash, csum);
1829
	if (memcmp(csum, on_disk_csum, sctx->csum_size))
1830
		sblock->checksum_error = 1;
A
Arne Jansen 已提交
1831

1832
	return sblock->checksum_error;
A
Arne Jansen 已提交
1833 1834
}

1835
static int scrub_checksum_tree_block(struct scrub_block *sblock)
A
Arne Jansen 已提交
1836
{
1837
	struct scrub_ctx *sctx = sblock->sctx;
A
Arne Jansen 已提交
1838
	struct btrfs_header *h;
1839
	struct btrfs_fs_info *fs_info = sctx->fs_info;
1840
	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
1841 1842 1843 1844 1845 1846 1847 1848 1849
	u8 calculated_csum[BTRFS_CSUM_SIZE];
	u8 on_disk_csum[BTRFS_CSUM_SIZE];
	struct page *page;
	void *mapped_buffer;
	u64 mapped_size;
	void *p;
	u64 len;
	int index;

1850 1851 1852
	shash->tfm = fs_info->csum_shash;
	crypto_shash_init(shash);

1853
	BUG_ON(sblock->page_count < 1);
1854
	page = sblock->pagev[0]->page;
1855
	mapped_buffer = kmap_atomic(page);
1856
	h = (struct btrfs_header *)mapped_buffer;
1857
	memcpy(on_disk_csum, h->csum, sctx->csum_size);
A
Arne Jansen 已提交
1858 1859 1860 1861 1862 1863

	/*
	 * we don't use the getter functions here, as we
	 * a) don't have an extent buffer and
	 * b) the page is already kmapped
	 */
1864
	if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
1865
		sblock->header_error = 1;
A
Arne Jansen 已提交
1866

1867 1868 1869 1870
	if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) {
		sblock->header_error = 1;
		sblock->generation_error = 1;
	}
A
Arne Jansen 已提交
1871

M
Miao Xie 已提交
1872
	if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
1873
		sblock->header_error = 1;
A
Arne Jansen 已提交
1874 1875 1876

	if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
		   BTRFS_UUID_SIZE))
1877
		sblock->header_error = 1;
A
Arne Jansen 已提交
1878

1879
	len = sctx->fs_info->nodesize - BTRFS_CSUM_SIZE;
1880 1881 1882 1883 1884 1885
	mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
	p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, mapped_size);

1886
		crypto_shash_update(shash, p, l);
1887
		kunmap_atomic(mapped_buffer);
1888 1889 1890 1891 1892
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
1893 1894
		BUG_ON(!sblock->pagev[index]->page);
		page = sblock->pagev[index]->page;
1895
		mapped_buffer = kmap_atomic(page);
1896 1897 1898 1899
		mapped_size = PAGE_SIZE;
		p = mapped_buffer;
	}

1900
	crypto_shash_final(shash, calculated_csum);
1901
	if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1902
		sblock->checksum_error = 1;
A
Arne Jansen 已提交
1903

1904
	return sblock->header_error || sblock->checksum_error;
A
Arne Jansen 已提交
1905 1906
}

1907
static int scrub_checksum_super(struct scrub_block *sblock)
A
Arne Jansen 已提交
1908 1909
{
	struct btrfs_super_block *s;
1910
	struct scrub_ctx *sctx = sblock->sctx;
1911 1912
	struct btrfs_fs_info *fs_info = sctx->fs_info;
	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
1913 1914 1915 1916 1917 1918
	u8 calculated_csum[BTRFS_CSUM_SIZE];
	u8 on_disk_csum[BTRFS_CSUM_SIZE];
	struct page *page;
	void *mapped_buffer;
	u64 mapped_size;
	void *p;
1919 1920
	int fail_gen = 0;
	int fail_cor = 0;
1921 1922
	u64 len;
	int index;
A
Arne Jansen 已提交
1923

1924 1925 1926
	shash->tfm = fs_info->csum_shash;
	crypto_shash_init(shash);

1927
	BUG_ON(sblock->page_count < 1);
1928
	page = sblock->pagev[0]->page;
1929
	mapped_buffer = kmap_atomic(page);
1930
	s = (struct btrfs_super_block *)mapped_buffer;
1931
	memcpy(on_disk_csum, s->csum, sctx->csum_size);
A
Arne Jansen 已提交
1932

1933
	if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
1934
		++fail_cor;
A
Arne Jansen 已提交
1935

1936
	if (sblock->pagev[0]->generation != btrfs_super_generation(s))
1937
		++fail_gen;
A
Arne Jansen 已提交
1938

M
Miao Xie 已提交
1939
	if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
1940
		++fail_cor;
A
Arne Jansen 已提交
1941

1942 1943 1944 1945 1946 1947 1948
	len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
	mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
	p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, mapped_size);

1949
		crypto_shash_update(shash, p, l);
1950
		kunmap_atomic(mapped_buffer);
1951 1952 1953 1954 1955
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
1956 1957
		BUG_ON(!sblock->pagev[index]->page);
		page = sblock->pagev[index]->page;
1958
		mapped_buffer = kmap_atomic(page);
1959 1960 1961 1962
		mapped_size = PAGE_SIZE;
		p = mapped_buffer;
	}

1963
	crypto_shash_final(shash, calculated_csum);
1964
	if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1965
		++fail_cor;
A
Arne Jansen 已提交
1966

1967
	if (fail_cor + fail_gen) {
A
Arne Jansen 已提交
1968 1969 1970 1971 1972
		/*
		 * if we find an error in a super block, we just report it.
		 * They will get written with the next transaction commit
		 * anyway
		 */
1973 1974 1975
		spin_lock(&sctx->stat_lock);
		++sctx->stat.super_errors;
		spin_unlock(&sctx->stat_lock);
1976
		if (fail_cor)
1977
			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1978 1979
				BTRFS_DEV_STAT_CORRUPTION_ERRS);
		else
1980
			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
1981
				BTRFS_DEV_STAT_GENERATION_ERRS);
A
Arne Jansen 已提交
1982 1983
	}

1984
	return fail_cor + fail_gen;
A
Arne Jansen 已提交
1985 1986
}

1987 1988
static void scrub_block_get(struct scrub_block *sblock)
{
1989
	refcount_inc(&sblock->refs);
1990 1991 1992 1993
}

static void scrub_block_put(struct scrub_block *sblock)
{
1994
	if (refcount_dec_and_test(&sblock->refs)) {
1995 1996
		int i;

1997 1998 1999
		if (sblock->sparity)
			scrub_parity_put(sblock->sparity);

2000
		for (i = 0; i < sblock->page_count; i++)
2001
			scrub_page_put(sblock->pagev[i]);
2002 2003 2004 2005
		kfree(sblock);
	}
}

2006 2007
static void scrub_page_get(struct scrub_page *spage)
{
2008
	atomic_inc(&spage->refs);
2009 2010 2011 2012
}

static void scrub_page_put(struct scrub_page *spage)
{
2013
	if (atomic_dec_and_test(&spage->refs)) {
2014 2015 2016 2017 2018 2019
		if (spage->page)
			__free_page(spage->page);
		kfree(spage);
	}
}

2020
static void scrub_submit(struct scrub_ctx *sctx)
A
Arne Jansen 已提交
2021 2022 2023
{
	struct scrub_bio *sbio;

2024
	if (sctx->curr == -1)
S
Stefan Behrens 已提交
2025
		return;
A
Arne Jansen 已提交
2026

2027 2028
	sbio = sctx->bios[sctx->curr];
	sctx->curr = -1;
2029
	scrub_pending_bio_inc(sctx);
2030
	btrfsic_submit_bio(sbio->bio);
A
Arne Jansen 已提交
2031 2032
}

2033 2034
static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
				    struct scrub_page *spage)
A
Arne Jansen 已提交
2035
{
2036
	struct scrub_block *sblock = spage->sblock;
A
Arne Jansen 已提交
2037
	struct scrub_bio *sbio;
2038
	int ret;
A
Arne Jansen 已提交
2039 2040 2041 2042 2043

again:
	/*
	 * grab a fresh bio or wait for one to become available
	 */
2044 2045 2046 2047 2048 2049 2050 2051
	while (sctx->curr == -1) {
		spin_lock(&sctx->list_lock);
		sctx->curr = sctx->first_free;
		if (sctx->curr != -1) {
			sctx->first_free = sctx->bios[sctx->curr]->next_free;
			sctx->bios[sctx->curr]->next_free = -1;
			sctx->bios[sctx->curr]->page_count = 0;
			spin_unlock(&sctx->list_lock);
A
Arne Jansen 已提交
2052
		} else {
2053 2054
			spin_unlock(&sctx->list_lock);
			wait_event(sctx->list_wait, sctx->first_free != -1);
A
Arne Jansen 已提交
2055 2056
		}
	}
2057
	sbio = sctx->bios[sctx->curr];
2058
	if (sbio->page_count == 0) {
2059 2060
		struct bio *bio;

2061 2062
		sbio->physical = spage->physical;
		sbio->logical = spage->logical;
2063
		sbio->dev = spage->dev;
2064 2065
		bio = sbio->bio;
		if (!bio) {
2066
			bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio);
2067 2068
			sbio->bio = bio;
		}
2069 2070 2071

		bio->bi_private = sbio;
		bio->bi_end_io = scrub_bio_end_io;
2072
		bio_set_dev(bio, sbio->dev->bdev);
2073
		bio->bi_iter.bi_sector = sbio->physical >> 9;
D
David Sterba 已提交
2074
		bio->bi_opf = REQ_OP_READ;
2075
		sbio->status = 0;
2076 2077 2078
	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
		   spage->physical ||
		   sbio->logical + sbio->page_count * PAGE_SIZE !=
2079 2080
		   spage->logical ||
		   sbio->dev != spage->dev) {
2081
		scrub_submit(sctx);
A
Arne Jansen 已提交
2082 2083
		goto again;
	}
2084

2085 2086 2087 2088 2089 2090 2091 2092
	sbio->pagev[sbio->page_count] = spage;
	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
	if (ret != PAGE_SIZE) {
		if (sbio->page_count < 1) {
			bio_put(sbio->bio);
			sbio->bio = NULL;
			return -EIO;
		}
2093
		scrub_submit(sctx);
2094 2095 2096
		goto again;
	}

2097
	scrub_block_get(sblock); /* one for the page added to the bio */
2098 2099
	atomic_inc(&sblock->outstanding_pages);
	sbio->page_count++;
2100
	if (sbio->page_count == sctx->pages_per_rd_bio)
2101
		scrub_submit(sctx);
2102 2103 2104 2105

	return 0;
}

2106
static void scrub_missing_raid56_end_io(struct bio *bio)
2107 2108
{
	struct scrub_block *sblock = bio->bi_private;
2109
	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
2110

2111
	if (bio->bi_status)
2112 2113
		sblock->no_io_error_seen = 0;

2114 2115
	bio_put(bio);

2116 2117 2118 2119 2120 2121 2122
	btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
}

static void scrub_missing_raid56_worker(struct btrfs_work *work)
{
	struct scrub_block *sblock = container_of(work, struct scrub_block, work);
	struct scrub_ctx *sctx = sblock->sctx;
2123
	struct btrfs_fs_info *fs_info = sctx->fs_info;
2124 2125 2126 2127 2128 2129
	u64 logical;
	struct btrfs_device *dev;

	logical = sblock->pagev[0]->logical;
	dev = sblock->pagev[0]->dev;

2130
	if (sblock->no_io_error_seen)
2131
		scrub_recheck_block_checksum(sblock);
2132 2133 2134 2135 2136

	if (!sblock->no_io_error_seen) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.read_errors++;
		spin_unlock(&sctx->stat_lock);
2137
		btrfs_err_rl_in_rcu(fs_info,
2138
			"IO error rebuilding logical %llu for dev %s",
2139 2140 2141 2142 2143
			logical, rcu_str_deref(dev->name));
	} else if (sblock->header_error || sblock->checksum_error) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
2144
		btrfs_err_rl_in_rcu(fs_info,
2145
			"failed to rebuild valid logical %llu for dev %s",
2146 2147 2148 2149 2150
			logical, rcu_str_deref(dev->name));
	} else {
		scrub_write_block_to_dev_replace(sblock);
	}

2151
	if (sctx->is_dev_replace && sctx->flush_all_writes) {
2152
		mutex_lock(&sctx->wr_lock);
2153
		scrub_wr_submit(sctx);
2154
		mutex_unlock(&sctx->wr_lock);
2155 2156
	}

2157
	scrub_block_put(sblock);
2158 2159 2160 2161 2162 2163
	scrub_pending_bio_dec(sctx);
}

static void scrub_missing_raid56_pages(struct scrub_block *sblock)
{
	struct scrub_ctx *sctx = sblock->sctx;
2164
	struct btrfs_fs_info *fs_info = sctx->fs_info;
2165 2166
	u64 length = sblock->page_count * PAGE_SIZE;
	u64 logical = sblock->pagev[0]->logical;
2167
	struct btrfs_bio *bbio = NULL;
2168 2169 2170 2171 2172
	struct bio *bio;
	struct btrfs_raid_bio *rbio;
	int ret;
	int i;

2173
	btrfs_bio_counter_inc_blocked(fs_info);
2174
	ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical,
2175
			&length, &bbio);
2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189
	if (ret || !bbio || !bbio->raid_map)
		goto bbio_out;

	if (WARN_ON(!sctx->is_dev_replace ||
		    !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
		/*
		 * We shouldn't be scrubbing a missing device. Even for dev
		 * replace, we should only get here for RAID 5/6. We either
		 * managed to mount something with no mirrors remaining or
		 * there's a bug in scrub_remap_extent()/btrfs_map_block().
		 */
		goto bbio_out;
	}

2190
	bio = btrfs_io_bio_alloc(0);
2191 2192 2193 2194
	bio->bi_iter.bi_sector = logical >> 9;
	bio->bi_private = sblock;
	bio->bi_end_io = scrub_missing_raid56_end_io;

2195
	rbio = raid56_alloc_missing_rbio(fs_info, bio, bbio, length);
2196 2197 2198 2199 2200 2201 2202 2203 2204
	if (!rbio)
		goto rbio_out;

	for (i = 0; i < sblock->page_count; i++) {
		struct scrub_page *spage = sblock->pagev[i];

		raid56_add_scrub_pages(rbio, spage->page, spage->logical);
	}

2205
	btrfs_init_work(&sblock->work, scrub_missing_raid56_worker, NULL, NULL);
2206 2207 2208 2209 2210 2211 2212 2213
	scrub_block_get(sblock);
	scrub_pending_bio_inc(sctx);
	raid56_submit_missing_rbio(rbio);
	return;

rbio_out:
	bio_put(bio);
bbio_out:
2214
	btrfs_bio_counter_dec(fs_info);
2215 2216 2217 2218 2219 2220
	btrfs_put_bbio(bbio);
	spin_lock(&sctx->stat_lock);
	sctx->stat.malloc_errors++;
	spin_unlock(&sctx->stat_lock);
}

2221
static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2222
		       u64 physical, struct btrfs_device *dev, u64 flags,
2223 2224
		       u64 gen, int mirror_num, u8 *csum, int force,
		       u64 physical_for_dev_replace)
2225 2226 2227 2228
{
	struct scrub_block *sblock;
	int index;

2229
	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2230
	if (!sblock) {
2231 2232 2233
		spin_lock(&sctx->stat_lock);
		sctx->stat.malloc_errors++;
		spin_unlock(&sctx->stat_lock);
2234
		return -ENOMEM;
A
Arne Jansen 已提交
2235
	}
2236

2237 2238
	/* one ref inside this function, plus one for each page added to
	 * a bio later on */
2239
	refcount_set(&sblock->refs, 1);
2240
	sblock->sctx = sctx;
2241 2242 2243
	sblock->no_io_error_seen = 1;

	for (index = 0; len > 0; index++) {
2244
		struct scrub_page *spage;
2245 2246
		u64 l = min_t(u64, len, PAGE_SIZE);

2247
		spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2248 2249
		if (!spage) {
leave_nomem:
2250 2251 2252
			spin_lock(&sctx->stat_lock);
			sctx->stat.malloc_errors++;
			spin_unlock(&sctx->stat_lock);
2253
			scrub_block_put(sblock);
2254 2255
			return -ENOMEM;
		}
2256 2257 2258
		BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
		scrub_page_get(spage);
		sblock->pagev[index] = spage;
2259
		spage->sblock = sblock;
2260
		spage->dev = dev;
2261 2262 2263 2264
		spage->flags = flags;
		spage->generation = gen;
		spage->logical = logical;
		spage->physical = physical;
2265
		spage->physical_for_dev_replace = physical_for_dev_replace;
2266 2267 2268
		spage->mirror_num = mirror_num;
		if (csum) {
			spage->have_csum = 1;
2269
			memcpy(spage->csum, csum, sctx->csum_size);
2270 2271 2272 2273
		} else {
			spage->have_csum = 0;
		}
		sblock->page_count++;
2274
		spage->page = alloc_page(GFP_KERNEL);
2275 2276
		if (!spage->page)
			goto leave_nomem;
2277 2278 2279
		len -= l;
		logical += l;
		physical += l;
2280
		physical_for_dev_replace += l;
2281 2282
	}

2283
	WARN_ON(sblock->page_count == 0);
2284
	if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2285 2286 2287 2288 2289 2290 2291 2292 2293
		/*
		 * This case should only be hit for RAID 5/6 device replace. See
		 * the comment in scrub_missing_raid56_pages() for details.
		 */
		scrub_missing_raid56_pages(sblock);
	} else {
		for (index = 0; index < sblock->page_count; index++) {
			struct scrub_page *spage = sblock->pagev[index];
			int ret;
2294

2295 2296 2297 2298 2299
			ret = scrub_add_page_to_rd_bio(sctx, spage);
			if (ret) {
				scrub_block_put(sblock);
				return ret;
			}
2300
		}
A
Arne Jansen 已提交
2301

2302 2303 2304
		if (force)
			scrub_submit(sctx);
	}
A
Arne Jansen 已提交
2305

2306 2307
	/* last one frees, either here or in bio completion for last page */
	scrub_block_put(sblock);
A
Arne Jansen 已提交
2308 2309 2310
	return 0;
}

2311
static void scrub_bio_end_io(struct bio *bio)
2312 2313
{
	struct scrub_bio *sbio = bio->bi_private;
2314
	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
2315

2316
	sbio->status = bio->bi_status;
2317 2318
	sbio->bio = bio;

2319
	btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2320 2321 2322 2323 2324
}

static void scrub_bio_end_io_worker(struct btrfs_work *work)
{
	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2325
	struct scrub_ctx *sctx = sbio->sctx;
2326 2327
	int i;

2328
	BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2329
	if (sbio->status) {
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349
		for (i = 0; i < sbio->page_count; i++) {
			struct scrub_page *spage = sbio->pagev[i];

			spage->io_error = 1;
			spage->sblock->no_io_error_seen = 0;
		}
	}

	/* now complete the scrub_block items that have all pages completed */
	for (i = 0; i < sbio->page_count; i++) {
		struct scrub_page *spage = sbio->pagev[i];
		struct scrub_block *sblock = spage->sblock;

		if (atomic_dec_and_test(&sblock->outstanding_pages))
			scrub_block_complete(sblock);
		scrub_block_put(sblock);
	}

	bio_put(sbio->bio);
	sbio->bio = NULL;
2350 2351 2352 2353
	spin_lock(&sctx->list_lock);
	sbio->next_free = sctx->first_free;
	sctx->first_free = sbio->index;
	spin_unlock(&sctx->list_lock);
2354

2355
	if (sctx->is_dev_replace && sctx->flush_all_writes) {
2356
		mutex_lock(&sctx->wr_lock);
2357
		scrub_wr_submit(sctx);
2358
		mutex_unlock(&sctx->wr_lock);
2359 2360
	}

2361
	scrub_pending_bio_dec(sctx);
2362 2363
}

2364 2365 2366 2367
static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
				       unsigned long *bitmap,
				       u64 start, u64 len)
{
2368
	u64 offset;
2369 2370
	u64 nsectors64;
	u32 nsectors;
2371
	int sectorsize = sparity->sctx->fs_info->sectorsize;
2372 2373 2374 2375 2376 2377 2378

	if (len >= sparity->stripe_len) {
		bitmap_set(bitmap, 0, sparity->nsectors);
		return;
	}

	start -= sparity->logic_start;
2379 2380
	start = div64_u64_rem(start, sparity->stripe_len, &offset);
	offset = div_u64(offset, sectorsize);
2381 2382 2383 2384
	nsectors64 = div_u64(len, sectorsize);

	ASSERT(nsectors64 < UINT_MAX);
	nsectors = (u32)nsectors64;
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406

	if (offset + nsectors <= sparity->nsectors) {
		bitmap_set(bitmap, offset, nsectors);
		return;
	}

	bitmap_set(bitmap, offset, sparity->nsectors - offset);
	bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
}

static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
						   u64 start, u64 len)
{
	__scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
}

static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
						  u64 start, u64 len)
{
	__scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
}

2407 2408
static void scrub_block_complete(struct scrub_block *sblock)
{
2409 2410
	int corrupted = 0;

2411
	if (!sblock->no_io_error_seen) {
2412
		corrupted = 1;
2413
		scrub_handle_errored_block(sblock);
2414 2415 2416 2417 2418 2419
	} else {
		/*
		 * if has checksum error, write via repair mechanism in
		 * dev replace case, otherwise write here in dev replace
		 * case.
		 */
2420 2421
		corrupted = scrub_checksum(sblock);
		if (!corrupted && sblock->sctx->is_dev_replace)
2422 2423
			scrub_write_block_to_dev_replace(sblock);
	}
2424 2425 2426 2427 2428 2429 2430 2431 2432

	if (sblock->sparity && corrupted && !sblock->data_corrected) {
		u64 start = sblock->pagev[0]->logical;
		u64 end = sblock->pagev[sblock->page_count - 1]->logical +
			  PAGE_SIZE;

		scrub_parity_mark_sectors_error(sblock->sparity,
						start, end - start);
	}
2433 2434
}

2435
static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum)
A
Arne Jansen 已提交
2436 2437
{
	struct btrfs_ordered_sum *sum = NULL;
2438
	unsigned long index;
A
Arne Jansen 已提交
2439 2440
	unsigned long num_sectors;

2441 2442
	while (!list_empty(&sctx->csum_list)) {
		sum = list_first_entry(&sctx->csum_list,
A
Arne Jansen 已提交
2443 2444 2445 2446 2447 2448
				       struct btrfs_ordered_sum, list);
		if (sum->bytenr > logical)
			return 0;
		if (sum->bytenr + sum->len > logical)
			break;

2449
		++sctx->stat.csum_discards;
A
Arne Jansen 已提交
2450 2451 2452 2453 2454 2455 2456
		list_del(&sum->list);
		kfree(sum);
		sum = NULL;
	}
	if (!sum)
		return 0;

2457 2458 2459
	index = div_u64(logical - sum->bytenr, sctx->fs_info->sectorsize);
	ASSERT(index < UINT_MAX);

2460
	num_sectors = sum->len / sctx->fs_info->sectorsize;
2461
	memcpy(csum, sum->sums + index * sctx->csum_size, sctx->csum_size);
2462
	if (index == num_sectors - 1) {
A
Arne Jansen 已提交
2463 2464 2465
		list_del(&sum->list);
		kfree(sum);
	}
2466
	return 1;
A
Arne Jansen 已提交
2467 2468 2469
}

/* scrub extent tries to collect up to 64 kB for each bio */
L
Liu Bo 已提交
2470 2471
static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map,
			u64 logical, u64 len,
2472
			u64 physical, struct btrfs_device *dev, u64 flags,
2473
			u64 gen, int mirror_num, u64 physical_for_dev_replace)
A
Arne Jansen 已提交
2474 2475 2476
{
	int ret;
	u8 csum[BTRFS_CSUM_SIZE];
2477 2478 2479
	u32 blocksize;

	if (flags & BTRFS_EXTENT_FLAG_DATA) {
L
Liu Bo 已提交
2480 2481 2482 2483
		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
			blocksize = map->stripe_len;
		else
			blocksize = sctx->fs_info->sectorsize;
2484 2485 2486 2487
		spin_lock(&sctx->stat_lock);
		sctx->stat.data_extents_scrubbed++;
		sctx->stat.data_bytes_scrubbed += len;
		spin_unlock(&sctx->stat_lock);
2488
	} else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
L
Liu Bo 已提交
2489 2490 2491 2492
		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
			blocksize = map->stripe_len;
		else
			blocksize = sctx->fs_info->nodesize;
2493 2494 2495 2496
		spin_lock(&sctx->stat_lock);
		sctx->stat.tree_extents_scrubbed++;
		sctx->stat.tree_bytes_scrubbed += len;
		spin_unlock(&sctx->stat_lock);
2497
	} else {
2498
		blocksize = sctx->fs_info->sectorsize;
2499
		WARN_ON(1);
2500
	}
A
Arne Jansen 已提交
2501 2502

	while (len) {
2503
		u64 l = min_t(u64, len, blocksize);
A
Arne Jansen 已提交
2504 2505 2506 2507
		int have_csum = 0;

		if (flags & BTRFS_EXTENT_FLAG_DATA) {
			/* push csums to sbio */
2508
			have_csum = scrub_find_csum(sctx, logical, csum);
A
Arne Jansen 已提交
2509
			if (have_csum == 0)
2510
				++sctx->stat.no_csum;
A
Arne Jansen 已提交
2511
		}
2512
		ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2513 2514
				  mirror_num, have_csum ? csum : NULL, 0,
				  physical_for_dev_replace);
A
Arne Jansen 已提交
2515 2516 2517 2518 2519
		if (ret)
			return ret;
		len -= l;
		logical += l;
		physical += l;
2520
		physical_for_dev_replace += l;
A
Arne Jansen 已提交
2521 2522 2523 2524
	}
	return 0;
}

2525 2526 2527 2528 2529 2530 2531 2532 2533
static int scrub_pages_for_parity(struct scrub_parity *sparity,
				  u64 logical, u64 len,
				  u64 physical, struct btrfs_device *dev,
				  u64 flags, u64 gen, int mirror_num, u8 *csum)
{
	struct scrub_ctx *sctx = sparity->sctx;
	struct scrub_block *sblock;
	int index;

2534
	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2535 2536 2537 2538 2539 2540 2541 2542 2543
	if (!sblock) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.malloc_errors++;
		spin_unlock(&sctx->stat_lock);
		return -ENOMEM;
	}

	/* one ref inside this function, plus one for each page added to
	 * a bio later on */
2544
	refcount_set(&sblock->refs, 1);
2545 2546 2547 2548 2549 2550 2551 2552 2553
	sblock->sctx = sctx;
	sblock->no_io_error_seen = 1;
	sblock->sparity = sparity;
	scrub_parity_get(sparity);

	for (index = 0; len > 0; index++) {
		struct scrub_page *spage;
		u64 l = min_t(u64, len, PAGE_SIZE);

2554
		spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583
		if (!spage) {
leave_nomem:
			spin_lock(&sctx->stat_lock);
			sctx->stat.malloc_errors++;
			spin_unlock(&sctx->stat_lock);
			scrub_block_put(sblock);
			return -ENOMEM;
		}
		BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
		/* For scrub block */
		scrub_page_get(spage);
		sblock->pagev[index] = spage;
		/* For scrub parity */
		scrub_page_get(spage);
		list_add_tail(&spage->list, &sparity->spages);
		spage->sblock = sblock;
		spage->dev = dev;
		spage->flags = flags;
		spage->generation = gen;
		spage->logical = logical;
		spage->physical = physical;
		spage->mirror_num = mirror_num;
		if (csum) {
			spage->have_csum = 1;
			memcpy(spage->csum, csum, sctx->csum_size);
		} else {
			spage->have_csum = 0;
		}
		sblock->page_count++;
2584
		spage->page = alloc_page(GFP_KERNEL);
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618
		if (!spage->page)
			goto leave_nomem;
		len -= l;
		logical += l;
		physical += l;
	}

	WARN_ON(sblock->page_count == 0);
	for (index = 0; index < sblock->page_count; index++) {
		struct scrub_page *spage = sblock->pagev[index];
		int ret;

		ret = scrub_add_page_to_rd_bio(sctx, spage);
		if (ret) {
			scrub_block_put(sblock);
			return ret;
		}
	}

	/* last one frees, either here or in bio completion for last page */
	scrub_block_put(sblock);
	return 0;
}

static int scrub_extent_for_parity(struct scrub_parity *sparity,
				   u64 logical, u64 len,
				   u64 physical, struct btrfs_device *dev,
				   u64 flags, u64 gen, int mirror_num)
{
	struct scrub_ctx *sctx = sparity->sctx;
	int ret;
	u8 csum[BTRFS_CSUM_SIZE];
	u32 blocksize;

2619
	if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2620 2621 2622 2623
		scrub_parity_mark_sectors_error(sparity, logical, len);
		return 0;
	}

2624
	if (flags & BTRFS_EXTENT_FLAG_DATA) {
L
Liu Bo 已提交
2625
		blocksize = sparity->stripe_len;
2626
	} else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
L
Liu Bo 已提交
2627
		blocksize = sparity->stripe_len;
2628
	} else {
2629
		blocksize = sctx->fs_info->sectorsize;
2630 2631 2632 2633 2634 2635 2636 2637 2638
		WARN_ON(1);
	}

	while (len) {
		u64 l = min_t(u64, len, blocksize);
		int have_csum = 0;

		if (flags & BTRFS_EXTENT_FLAG_DATA) {
			/* push csums to sbio */
2639
			have_csum = scrub_find_csum(sctx, logical, csum);
2640 2641 2642 2643 2644 2645 2646 2647
			if (have_csum == 0)
				goto skip;
		}
		ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
					     flags, gen, mirror_num,
					     have_csum ? csum : NULL);
		if (ret)
			return ret;
2648
skip:
2649 2650 2651 2652 2653 2654 2655
		len -= l;
		logical += l;
		physical += l;
	}
	return 0;
}

2656 2657 2658 2659 2660 2661 2662 2663
/*
 * Given a physical address, this will calculate it's
 * logical offset. if this is a parity stripe, it will return
 * the most left data stripe's logical offset.
 *
 * return 0 if it is a data stripe, 1 means parity stripe.
 */
static int get_raid56_logic_offset(u64 physical, int num,
2664 2665
				   struct map_lookup *map, u64 *offset,
				   u64 *stripe_start)
2666 2667 2668 2669 2670
{
	int i;
	int j = 0;
	u64 stripe_nr;
	u64 last_offset;
2671 2672
	u32 stripe_index;
	u32 rot;
2673
	const int data_stripes = nr_data_stripes(map);
2674

2675
	last_offset = (physical - map->stripes[num].physical) * data_stripes;
2676 2677 2678
	if (stripe_start)
		*stripe_start = last_offset;

2679
	*offset = last_offset;
2680
	for (i = 0; i < data_stripes; i++) {
2681 2682
		*offset = last_offset + i * map->stripe_len;

2683
		stripe_nr = div64_u64(*offset, map->stripe_len);
2684
		stripe_nr = div_u64(stripe_nr, data_stripes);
2685 2686

		/* Work out the disk rotation on this stripe-set */
2687
		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2688 2689
		/* calculate which stripe this data locates */
		rot += i;
2690
		stripe_index = rot % map->num_stripes;
2691 2692 2693 2694 2695 2696 2697 2698 2699
		if (stripe_index == num)
			return 0;
		if (stripe_index < num)
			j++;
	}
	*offset = last_offset + j * map->stripe_len;
	return 1;
}

2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721
static void scrub_free_parity(struct scrub_parity *sparity)
{
	struct scrub_ctx *sctx = sparity->sctx;
	struct scrub_page *curr, *next;
	int nbits;

	nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
	if (nbits) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.read_errors += nbits;
		sctx->stat.uncorrectable_errors += nbits;
		spin_unlock(&sctx->stat_lock);
	}

	list_for_each_entry_safe(curr, next, &sparity->spages, list) {
		list_del_init(&curr->list);
		scrub_page_put(curr);
	}

	kfree(sparity);
}

2722 2723 2724 2725 2726 2727 2728 2729 2730 2731
static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
{
	struct scrub_parity *sparity = container_of(work, struct scrub_parity,
						    work);
	struct scrub_ctx *sctx = sparity->sctx;

	scrub_free_parity(sparity);
	scrub_pending_bio_dec(sctx);
}

2732
static void scrub_parity_bio_endio(struct bio *bio)
2733 2734
{
	struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
2735
	struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;
2736

2737
	if (bio->bi_status)
2738 2739 2740 2741
		bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
			  sparity->nsectors);

	bio_put(bio);
2742

2743 2744
	btrfs_init_work(&sparity->work, scrub_parity_bio_endio_worker, NULL,
			NULL);
2745
	btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work);
2746 2747 2748 2749 2750
}

static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
{
	struct scrub_ctx *sctx = sparity->sctx;
2751
	struct btrfs_fs_info *fs_info = sctx->fs_info;
2752 2753 2754 2755 2756 2757 2758 2759 2760 2761
	struct bio *bio;
	struct btrfs_raid_bio *rbio;
	struct btrfs_bio *bbio = NULL;
	u64 length;
	int ret;

	if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
			   sparity->nsectors))
		goto out;

2762
	length = sparity->logic_end - sparity->logic_start;
2763 2764

	btrfs_bio_counter_inc_blocked(fs_info);
2765
	ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start,
2766
			       &length, &bbio);
2767
	if (ret || !bbio || !bbio->raid_map)
2768 2769
		goto bbio_out;

2770
	bio = btrfs_io_bio_alloc(0);
2771 2772 2773 2774
	bio->bi_iter.bi_sector = sparity->logic_start >> 9;
	bio->bi_private = sparity;
	bio->bi_end_io = scrub_parity_bio_endio;

2775
	rbio = raid56_parity_alloc_scrub_rbio(fs_info, bio, bbio,
2776
					      length, sparity->scrub_dev,
2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788
					      sparity->dbitmap,
					      sparity->nsectors);
	if (!rbio)
		goto rbio_out;

	scrub_pending_bio_inc(sctx);
	raid56_parity_submit_scrub_rbio(rbio);
	return;

rbio_out:
	bio_put(bio);
bbio_out:
2789
	btrfs_bio_counter_dec(fs_info);
2790
	btrfs_put_bbio(bbio);
2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801
	bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
		  sparity->nsectors);
	spin_lock(&sctx->stat_lock);
	sctx->stat.malloc_errors++;
	spin_unlock(&sctx->stat_lock);
out:
	scrub_free_parity(sparity);
}

static inline int scrub_calc_parity_bitmap_len(int nsectors)
{
2802
	return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long);
2803 2804 2805 2806
}

static void scrub_parity_get(struct scrub_parity *sparity)
{
2807
	refcount_inc(&sparity->refs);
2808 2809 2810 2811
}

static void scrub_parity_put(struct scrub_parity *sparity)
{
2812
	if (!refcount_dec_and_test(&sparity->refs))
2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824
		return;

	scrub_parity_check_and_repair(sparity);
}

static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
						  struct map_lookup *map,
						  struct btrfs_device *sdev,
						  struct btrfs_path *path,
						  u64 logic_start,
						  u64 logic_end)
{
2825
	struct btrfs_fs_info *fs_info = sctx->fs_info;
2826 2827 2828
	struct btrfs_root *root = fs_info->extent_root;
	struct btrfs_root *csum_root = fs_info->csum_root;
	struct btrfs_extent_item *extent;
2829
	struct btrfs_bio *bbio = NULL;
2830 2831 2832 2833 2834 2835 2836 2837 2838
	u64 flags;
	int ret;
	int slot;
	struct extent_buffer *l;
	struct btrfs_key key;
	u64 generation;
	u64 extent_logical;
	u64 extent_physical;
	u64 extent_len;
2839
	u64 mapped_length;
2840 2841 2842 2843 2844 2845 2846
	struct btrfs_device *extent_dev;
	struct scrub_parity *sparity;
	int nsectors;
	int bitmap_len;
	int extent_mirror_num;
	int stop_loop = 0;

2847
	nsectors = div_u64(map->stripe_len, fs_info->sectorsize);
2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863
	bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
	sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
			  GFP_NOFS);
	if (!sparity) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.malloc_errors++;
		spin_unlock(&sctx->stat_lock);
		return -ENOMEM;
	}

	sparity->stripe_len = map->stripe_len;
	sparity->nsectors = nsectors;
	sparity->sctx = sctx;
	sparity->scrub_dev = sdev;
	sparity->logic_start = logic_start;
	sparity->logic_end = logic_end;
2864
	refcount_set(&sparity->refs, 1);
2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912
	INIT_LIST_HEAD(&sparity->spages);
	sparity->dbitmap = sparity->bitmap;
	sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;

	ret = 0;
	while (logic_start < logic_end) {
		if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
			key.type = BTRFS_METADATA_ITEM_KEY;
		else
			key.type = BTRFS_EXTENT_ITEM_KEY;
		key.objectid = logic_start;
		key.offset = (u64)-1;

		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		if (ret < 0)
			goto out;

		if (ret > 0) {
			ret = btrfs_previous_extent_item(root, path, 0);
			if (ret < 0)
				goto out;
			if (ret > 0) {
				btrfs_release_path(path);
				ret = btrfs_search_slot(NULL, root, &key,
							path, 0, 0);
				if (ret < 0)
					goto out;
			}
		}

		stop_loop = 0;
		while (1) {
			u64 bytes;

			l = path->nodes[0];
			slot = path->slots[0];
			if (slot >= btrfs_header_nritems(l)) {
				ret = btrfs_next_leaf(root, path);
				if (ret == 0)
					continue;
				if (ret < 0)
					goto out;

				stop_loop = 1;
				break;
			}
			btrfs_item_key_to_cpu(l, &key, slot);

2913 2914 2915 2916
			if (key.type != BTRFS_EXTENT_ITEM_KEY &&
			    key.type != BTRFS_METADATA_ITEM_KEY)
				goto next;

2917
			if (key.type == BTRFS_METADATA_ITEM_KEY)
2918
				bytes = fs_info->nodesize;
2919 2920 2921 2922 2923 2924
			else
				bytes = key.offset;

			if (key.objectid + bytes <= logic_start)
				goto next;

2925
			if (key.objectid >= logic_end) {
2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937
				stop_loop = 1;
				break;
			}

			while (key.objectid >= logic_start + map->stripe_len)
				logic_start += map->stripe_len;

			extent = btrfs_item_ptr(l, slot,
						struct btrfs_extent_item);
			flags = btrfs_extent_flags(l, extent);
			generation = btrfs_extent_generation(l, extent);

2938 2939 2940 2941
			if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
			    (key.objectid < logic_start ||
			     key.objectid + bytes >
			     logic_start + map->stripe_len)) {
J
Jeff Mahoney 已提交
2942 2943
				btrfs_err(fs_info,
					  "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
2944
					  key.objectid, logic_start);
2945 2946 2947
				spin_lock(&sctx->stat_lock);
				sctx->stat.uncorrectable_errors++;
				spin_unlock(&sctx->stat_lock);
2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966
				goto next;
			}
again:
			extent_logical = key.objectid;
			extent_len = bytes;

			if (extent_logical < logic_start) {
				extent_len -= logic_start - extent_logical;
				extent_logical = logic_start;
			}

			if (extent_logical + extent_len >
			    logic_start + map->stripe_len)
				extent_len = logic_start + map->stripe_len -
					     extent_logical;

			scrub_parity_mark_sectors_data(sparity, extent_logical,
						       extent_len);

2967
			mapped_length = extent_len;
2968
			bbio = NULL;
2969 2970 2971
			ret = btrfs_map_block(fs_info, BTRFS_MAP_READ,
					extent_logical, &mapped_length, &bbio,
					0);
2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983
			if (!ret) {
				if (!bbio || mapped_length < extent_len)
					ret = -EIO;
			}
			if (ret) {
				btrfs_put_bbio(bbio);
				goto out;
			}
			extent_physical = bbio->stripes[0].physical;
			extent_mirror_num = bbio->mirror_num;
			extent_dev = bbio->stripes[0].dev;
			btrfs_put_bbio(bbio);
2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997

			ret = btrfs_lookup_csums_range(csum_root,
						extent_logical,
						extent_logical + extent_len - 1,
						&sctx->csum_list, 1);
			if (ret)
				goto out;

			ret = scrub_extent_for_parity(sparity, extent_logical,
						      extent_len,
						      extent_physical,
						      extent_dev, flags,
						      generation,
						      extent_mirror_num);
2998 2999 3000

			scrub_free_csums(sctx);

3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031
			if (ret)
				goto out;

			if (extent_logical + extent_len <
			    key.objectid + bytes) {
				logic_start += map->stripe_len;

				if (logic_start >= logic_end) {
					stop_loop = 1;
					break;
				}

				if (logic_start < key.objectid + bytes) {
					cond_resched();
					goto again;
				}
			}
next:
			path->slots[0]++;
		}

		btrfs_release_path(path);

		if (stop_loop)
			break;

		logic_start += map->stripe_len;
	}
out:
	if (ret < 0)
		scrub_parity_mark_sectors_error(sparity, logic_start,
3032
						logic_end - logic_start);
3033 3034
	scrub_parity_put(sparity);
	scrub_submit(sctx);
3035
	mutex_lock(&sctx->wr_lock);
3036
	scrub_wr_submit(sctx);
3037
	mutex_unlock(&sctx->wr_lock);
3038 3039 3040 3041 3042

	btrfs_release_path(path);
	return ret < 0 ? ret : 0;
}

3043
static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
3044 3045
					   struct map_lookup *map,
					   struct btrfs_device *scrub_dev,
3046
					   int num, u64 base, u64 length)
A
Arne Jansen 已提交
3047
{
3048
	struct btrfs_path *path, *ppath;
3049
	struct btrfs_fs_info *fs_info = sctx->fs_info;
A
Arne Jansen 已提交
3050 3051 3052
	struct btrfs_root *root = fs_info->extent_root;
	struct btrfs_root *csum_root = fs_info->csum_root;
	struct btrfs_extent_item *extent;
3053
	struct blk_plug plug;
A
Arne Jansen 已提交
3054 3055 3056 3057 3058 3059 3060
	u64 flags;
	int ret;
	int slot;
	u64 nstripes;
	struct extent_buffer *l;
	u64 physical;
	u64 logical;
L
Liu Bo 已提交
3061
	u64 logic_end;
3062
	u64 physical_end;
A
Arne Jansen 已提交
3063
	u64 generation;
3064
	int mirror_num;
A
Arne Jansen 已提交
3065 3066
	struct reada_control *reada1;
	struct reada_control *reada2;
3067
	struct btrfs_key key;
A
Arne Jansen 已提交
3068
	struct btrfs_key key_end;
A
Arne Jansen 已提交
3069 3070
	u64 increment = map->stripe_len;
	u64 offset;
3071 3072 3073
	u64 extent_logical;
	u64 extent_physical;
	u64 extent_len;
3074 3075
	u64 stripe_logical;
	u64 stripe_end;
3076 3077
	struct btrfs_device *extent_dev;
	int extent_mirror_num;
3078
	int stop_loop = 0;
D
David Woodhouse 已提交
3079

3080
	physical = map->stripes[num].physical;
A
Arne Jansen 已提交
3081
	offset = 0;
3082
	nstripes = div64_u64(length, map->stripe_len);
A
Arne Jansen 已提交
3083 3084 3085
	if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
		offset = map->stripe_len * num;
		increment = map->stripe_len * map->num_stripes;
3086
		mirror_num = 1;
A
Arne Jansen 已提交
3087 3088 3089 3090
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
		int factor = map->num_stripes / map->sub_stripes;
		offset = map->stripe_len * (num / map->sub_stripes);
		increment = map->stripe_len * factor;
3091
		mirror_num = num % map->sub_stripes + 1;
3092
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) {
A
Arne Jansen 已提交
3093
		increment = map->stripe_len;
3094
		mirror_num = num % map->num_stripes + 1;
A
Arne Jansen 已提交
3095 3096
	} else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
		increment = map->stripe_len;
3097
		mirror_num = num % map->num_stripes + 1;
3098
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3099
		get_raid56_logic_offset(physical, num, map, &offset, NULL);
3100 3101
		increment = map->stripe_len * nr_data_stripes(map);
		mirror_num = 1;
A
Arne Jansen 已提交
3102 3103
	} else {
		increment = map->stripe_len;
3104
		mirror_num = 1;
A
Arne Jansen 已提交
3105 3106 3107 3108 3109 3110
	}

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3111 3112
	ppath = btrfs_alloc_path();
	if (!ppath) {
3113
		btrfs_free_path(path);
3114 3115 3116
		return -ENOMEM;
	}

3117 3118 3119 3120 3121
	/*
	 * work on commit root. The related disk blocks are static as
	 * long as COW is applied. This means, it is save to rewrite
	 * them to repair disk errors without any race conditions
	 */
A
Arne Jansen 已提交
3122 3123 3124
	path->search_commit_root = 1;
	path->skip_locking = 1;

3125 3126
	ppath->search_commit_root = 1;
	ppath->skip_locking = 1;
A
Arne Jansen 已提交
3127
	/*
A
Arne Jansen 已提交
3128 3129 3130
	 * trigger the readahead for extent tree csum tree and wait for
	 * completion. During readahead, the scrub is officially paused
	 * to not hold off transaction commits
A
Arne Jansen 已提交
3131 3132
	 */
	logical = base + offset;
3133
	physical_end = physical + nstripes * map->stripe_len;
3134
	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3135
		get_raid56_logic_offset(physical_end, num,
3136
					map, &logic_end, NULL);
3137 3138 3139 3140
		logic_end += base;
	} else {
		logic_end = logical + increment * nstripes;
	}
3141
	wait_event(sctx->list_wait,
3142
		   atomic_read(&sctx->bios_in_flight) == 0);
3143
	scrub_blocked_if_needed(fs_info);
A
Arne Jansen 已提交
3144 3145

	/* FIXME it might be better to start readahead at commit root */
3146 3147 3148
	key.objectid = logical;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = (u64)0;
3149
	key_end.objectid = logic_end;
3150 3151
	key_end.type = BTRFS_METADATA_ITEM_KEY;
	key_end.offset = (u64)-1;
3152
	reada1 = btrfs_reada_add(root, &key, &key_end);
A
Arne Jansen 已提交
3153

3154 3155 3156
	key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
	key.type = BTRFS_EXTENT_CSUM_KEY;
	key.offset = logical;
A
Arne Jansen 已提交
3157 3158
	key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
	key_end.type = BTRFS_EXTENT_CSUM_KEY;
3159
	key_end.offset = logic_end;
3160
	reada2 = btrfs_reada_add(csum_root, &key, &key_end);
A
Arne Jansen 已提交
3161 3162 3163 3164 3165 3166

	if (!IS_ERR(reada1))
		btrfs_reada_wait(reada1);
	if (!IS_ERR(reada2))
		btrfs_reada_wait(reada2);

A
Arne Jansen 已提交
3167 3168 3169 3170 3171

	/*
	 * collect all data csums for the stripe to avoid seeking during
	 * the scrub. This might currently (crc32) end up to be about 1MB
	 */
3172
	blk_start_plug(&plug);
A
Arne Jansen 已提交
3173 3174 3175 3176 3177

	/*
	 * now find all extents for each stripe and scrub them
	 */
	ret = 0;
3178
	while (physical < physical_end) {
A
Arne Jansen 已提交
3179 3180 3181 3182
		/*
		 * canceled?
		 */
		if (atomic_read(&fs_info->scrub_cancel_req) ||
3183
		    atomic_read(&sctx->cancel_req)) {
A
Arne Jansen 已提交
3184 3185 3186 3187 3188 3189 3190 3191
			ret = -ECANCELED;
			goto out;
		}
		/*
		 * check to see if we have to pause
		 */
		if (atomic_read(&fs_info->scrub_pause_req)) {
			/* push queued extents */
3192
			sctx->flush_all_writes = true;
3193
			scrub_submit(sctx);
3194
			mutex_lock(&sctx->wr_lock);
3195
			scrub_wr_submit(sctx);
3196
			mutex_unlock(&sctx->wr_lock);
3197
			wait_event(sctx->list_wait,
3198
				   atomic_read(&sctx->bios_in_flight) == 0);
3199
			sctx->flush_all_writes = false;
3200
			scrub_blocked_if_needed(fs_info);
A
Arne Jansen 已提交
3201 3202
		}

3203 3204 3205 3206 3207 3208
		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
			ret = get_raid56_logic_offset(physical, num, map,
						      &logical,
						      &stripe_logical);
			logical += base;
			if (ret) {
3209
				/* it is parity strip */
3210
				stripe_logical += base;
3211
				stripe_end = stripe_logical + increment;
3212 3213 3214 3215 3216 3217 3218 3219 3220
				ret = scrub_raid56_parity(sctx, map, scrub_dev,
							  ppath, stripe_logical,
							  stripe_end);
				if (ret)
					goto out;
				goto skip;
			}
		}

3221 3222 3223 3224
		if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
			key.type = BTRFS_METADATA_ITEM_KEY;
		else
			key.type = BTRFS_EXTENT_ITEM_KEY;
A
Arne Jansen 已提交
3225
		key.objectid = logical;
L
Liu Bo 已提交
3226
		key.offset = (u64)-1;
A
Arne Jansen 已提交
3227 3228 3229 3230

		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		if (ret < 0)
			goto out;
3231

3232
		if (ret > 0) {
3233
			ret = btrfs_previous_extent_item(root, path, 0);
A
Arne Jansen 已提交
3234 3235
			if (ret < 0)
				goto out;
3236 3237 3238 3239 3240 3241 3242 3243 3244
			if (ret > 0) {
				/* there's no smaller item, so stick with the
				 * larger one */
				btrfs_release_path(path);
				ret = btrfs_search_slot(NULL, root, &key,
							path, 0, 0);
				if (ret < 0)
					goto out;
			}
A
Arne Jansen 已提交
3245 3246
		}

L
Liu Bo 已提交
3247
		stop_loop = 0;
A
Arne Jansen 已提交
3248
		while (1) {
3249 3250
			u64 bytes;

A
Arne Jansen 已提交
3251 3252 3253 3254 3255 3256 3257 3258 3259
			l = path->nodes[0];
			slot = path->slots[0];
			if (slot >= btrfs_header_nritems(l)) {
				ret = btrfs_next_leaf(root, path);
				if (ret == 0)
					continue;
				if (ret < 0)
					goto out;

L
Liu Bo 已提交
3260
				stop_loop = 1;
A
Arne Jansen 已提交
3261 3262 3263 3264
				break;
			}
			btrfs_item_key_to_cpu(l, &key, slot);

3265 3266 3267 3268
			if (key.type != BTRFS_EXTENT_ITEM_KEY &&
			    key.type != BTRFS_METADATA_ITEM_KEY)
				goto next;

3269
			if (key.type == BTRFS_METADATA_ITEM_KEY)
3270
				bytes = fs_info->nodesize;
3271 3272 3273 3274
			else
				bytes = key.offset;

			if (key.objectid + bytes <= logical)
A
Arne Jansen 已提交
3275 3276
				goto next;

L
Liu Bo 已提交
3277 3278 3279 3280 3281 3282
			if (key.objectid >= logical + map->stripe_len) {
				/* out of this device extent */
				if (key.objectid >= logic_end)
					stop_loop = 1;
				break;
			}
A
Arne Jansen 已提交
3283 3284 3285 3286 3287 3288

			extent = btrfs_item_ptr(l, slot,
						struct btrfs_extent_item);
			flags = btrfs_extent_flags(l, extent);
			generation = btrfs_extent_generation(l, extent);

3289 3290 3291 3292
			if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
			    (key.objectid < logical ||
			     key.objectid + bytes >
			     logical + map->stripe_len)) {
3293
				btrfs_err(fs_info,
J
Jeff Mahoney 已提交
3294
					   "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3295
				       key.objectid, logical);
3296 3297 3298
				spin_lock(&sctx->stat_lock);
				sctx->stat.uncorrectable_errors++;
				spin_unlock(&sctx->stat_lock);
A
Arne Jansen 已提交
3299 3300 3301
				goto next;
			}

L
Liu Bo 已提交
3302 3303 3304 3305
again:
			extent_logical = key.objectid;
			extent_len = bytes;

A
Arne Jansen 已提交
3306 3307 3308
			/*
			 * trim extent to this stripe
			 */
L
Liu Bo 已提交
3309 3310 3311
			if (extent_logical < logical) {
				extent_len -= logical - extent_logical;
				extent_logical = logical;
A
Arne Jansen 已提交
3312
			}
L
Liu Bo 已提交
3313
			if (extent_logical + extent_len >
A
Arne Jansen 已提交
3314
			    logical + map->stripe_len) {
L
Liu Bo 已提交
3315 3316
				extent_len = logical + map->stripe_len -
					     extent_logical;
A
Arne Jansen 已提交
3317 3318
			}

L
Liu Bo 已提交
3319
			extent_physical = extent_logical - logical + physical;
3320 3321
			extent_dev = scrub_dev;
			extent_mirror_num = mirror_num;
3322
			if (sctx->is_dev_replace)
3323 3324 3325 3326
				scrub_remap_extent(fs_info, extent_logical,
						   extent_len, &extent_physical,
						   &extent_dev,
						   &extent_mirror_num);
L
Liu Bo 已提交
3327

3328 3329 3330 3331 3332
			ret = btrfs_lookup_csums_range(csum_root,
						       extent_logical,
						       extent_logical +
						       extent_len - 1,
						       &sctx->csum_list, 1);
L
Liu Bo 已提交
3333 3334 3335
			if (ret)
				goto out;

L
Liu Bo 已提交
3336
			ret = scrub_extent(sctx, map, extent_logical, extent_len,
3337 3338
					   extent_physical, extent_dev, flags,
					   generation, extent_mirror_num,
3339
					   extent_logical - logical + physical);
3340 3341 3342

			scrub_free_csums(sctx);

A
Arne Jansen 已提交
3343 3344 3345
			if (ret)
				goto out;

L
Liu Bo 已提交
3346 3347
			if (extent_logical + extent_len <
			    key.objectid + bytes) {
3348
				if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3349 3350 3351 3352
					/*
					 * loop until we find next data stripe
					 * or we have finished all stripes.
					 */
3353 3354 3355 3356 3357 3358 3359 3360 3361 3362
loop:
					physical += map->stripe_len;
					ret = get_raid56_logic_offset(physical,
							num, map, &logical,
							&stripe_logical);
					logical += base;

					if (ret && physical < physical_end) {
						stripe_logical += base;
						stripe_end = stripe_logical +
3363
								increment;
3364 3365 3366 3367 3368 3369 3370 3371
						ret = scrub_raid56_parity(sctx,
							map, scrub_dev, ppath,
							stripe_logical,
							stripe_end);
						if (ret)
							goto out;
						goto loop;
					}
3372 3373 3374 3375
				} else {
					physical += map->stripe_len;
					logical += increment;
				}
L
Liu Bo 已提交
3376 3377 3378 3379 3380
				if (logical < key.objectid + bytes) {
					cond_resched();
					goto again;
				}

3381
				if (physical >= physical_end) {
L
Liu Bo 已提交
3382 3383 3384 3385
					stop_loop = 1;
					break;
				}
			}
A
Arne Jansen 已提交
3386 3387 3388
next:
			path->slots[0]++;
		}
C
Chris Mason 已提交
3389
		btrfs_release_path(path);
3390
skip:
A
Arne Jansen 已提交
3391 3392
		logical += increment;
		physical += map->stripe_len;
3393
		spin_lock(&sctx->stat_lock);
L
Liu Bo 已提交
3394 3395 3396 3397 3398
		if (stop_loop)
			sctx->stat.last_physical = map->stripes[num].physical +
						   length;
		else
			sctx->stat.last_physical = physical;
3399
		spin_unlock(&sctx->stat_lock);
L
Liu Bo 已提交
3400 3401
		if (stop_loop)
			break;
A
Arne Jansen 已提交
3402
	}
3403
out:
A
Arne Jansen 已提交
3404
	/* push queued extents */
3405
	scrub_submit(sctx);
3406
	mutex_lock(&sctx->wr_lock);
3407
	scrub_wr_submit(sctx);
3408
	mutex_unlock(&sctx->wr_lock);
A
Arne Jansen 已提交
3409

3410
	blk_finish_plug(&plug);
A
Arne Jansen 已提交
3411
	btrfs_free_path(path);
3412
	btrfs_free_path(ppath);
A
Arne Jansen 已提交
3413 3414 3415
	return ret < 0 ? ret : 0;
}

3416
static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
3417 3418
					  struct btrfs_device *scrub_dev,
					  u64 chunk_offset, u64 length,
3419
					  u64 dev_offset,
3420
					  struct btrfs_block_group_cache *cache)
A
Arne Jansen 已提交
3421
{
3422
	struct btrfs_fs_info *fs_info = sctx->fs_info;
3423
	struct extent_map_tree *map_tree = &fs_info->mapping_tree;
A
Arne Jansen 已提交
3424 3425 3426
	struct map_lookup *map;
	struct extent_map *em;
	int i;
3427
	int ret = 0;
A
Arne Jansen 已提交
3428

3429 3430 3431
	read_lock(&map_tree->lock);
	em = lookup_extent_mapping(map_tree, chunk_offset, 1);
	read_unlock(&map_tree->lock);
A
Arne Jansen 已提交
3432

3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444
	if (!em) {
		/*
		 * Might have been an unused block group deleted by the cleaner
		 * kthread or relocation.
		 */
		spin_lock(&cache->lock);
		if (!cache->removed)
			ret = -EINVAL;
		spin_unlock(&cache->lock);

		return ret;
	}
A
Arne Jansen 已提交
3445

3446
	map = em->map_lookup;
A
Arne Jansen 已提交
3447 3448 3449 3450 3451 3452 3453
	if (em->start != chunk_offset)
		goto out;

	if (em->len < length)
		goto out;

	for (i = 0; i < map->num_stripes; ++i) {
3454
		if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
3455
		    map->stripes[i].physical == dev_offset) {
3456
			ret = scrub_stripe(sctx, map, scrub_dev, i,
3457
					   chunk_offset, length);
A
Arne Jansen 已提交
3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468
			if (ret)
				goto out;
		}
	}
out:
	free_extent_map(em);

	return ret;
}

static noinline_for_stack
3469
int scrub_enumerate_chunks(struct scrub_ctx *sctx,
3470
			   struct btrfs_device *scrub_dev, u64 start, u64 end)
A
Arne Jansen 已提交
3471 3472 3473
{
	struct btrfs_dev_extent *dev_extent = NULL;
	struct btrfs_path *path;
3474 3475
	struct btrfs_fs_info *fs_info = sctx->fs_info;
	struct btrfs_root *root = fs_info->dev_root;
A
Arne Jansen 已提交
3476 3477
	u64 length;
	u64 chunk_offset;
3478
	int ret = 0;
3479
	int ro_set;
A
Arne Jansen 已提交
3480 3481 3482 3483 3484
	int slot;
	struct extent_buffer *l;
	struct btrfs_key key;
	struct btrfs_key found_key;
	struct btrfs_block_group_cache *cache;
3485
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
A
Arne Jansen 已提交
3486 3487 3488 3489 3490

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3491
	path->reada = READA_FORWARD;
A
Arne Jansen 已提交
3492 3493 3494
	path->search_commit_root = 1;
	path->skip_locking = 1;

3495
	key.objectid = scrub_dev->devid;
A
Arne Jansen 已提交
3496 3497 3498 3499 3500 3501
	key.offset = 0ull;
	key.type = BTRFS_DEV_EXTENT_KEY;

	while (1) {
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		if (ret < 0)
3502 3503 3504 3505 3506
			break;
		if (ret > 0) {
			if (path->slots[0] >=
			    btrfs_header_nritems(path->nodes[0])) {
				ret = btrfs_next_leaf(root, path);
3507 3508 3509 3510
				if (ret < 0)
					break;
				if (ret > 0) {
					ret = 0;
3511
					break;
3512 3513 3514
				}
			} else {
				ret = 0;
3515 3516
			}
		}
A
Arne Jansen 已提交
3517 3518 3519 3520 3521 3522

		l = path->nodes[0];
		slot = path->slots[0];

		btrfs_item_key_to_cpu(l, &found_key, slot);

3523
		if (found_key.objectid != scrub_dev->devid)
A
Arne Jansen 已提交
3524 3525
			break;

3526
		if (found_key.type != BTRFS_DEV_EXTENT_KEY)
A
Arne Jansen 已提交
3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537
			break;

		if (found_key.offset >= end)
			break;

		if (found_key.offset < key.offset)
			break;

		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
		length = btrfs_dev_extent_length(l, dev_extent);

3538 3539
		if (found_key.offset + length <= start)
			goto skip;
A
Arne Jansen 已提交
3540 3541 3542 3543 3544 3545 3546 3547

		chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);

		/*
		 * get a reference on the corresponding block group to prevent
		 * the chunk from going away while we scrub it
		 */
		cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3548 3549 3550 3551 3552 3553

		/* some chunks are removed but not committed to disk yet,
		 * continue scrubbing */
		if (!cache)
			goto skip;

3554 3555 3556 3557 3558 3559 3560 3561 3562
		/*
		 * we need call btrfs_inc_block_group_ro() with scrubs_paused,
		 * to avoid deadlock caused by:
		 * btrfs_inc_block_group_ro()
		 * -> btrfs_wait_for_commit()
		 * -> btrfs_commit_transaction()
		 * -> btrfs_scrub_pause()
		 */
		scrub_pause_on(fs_info);
3563
		ret = btrfs_inc_block_group_ro(cache);
3564
		if (!ret && sctx->is_dev_replace) {
3565 3566
			/*
			 * If we are doing a device replace wait for any tasks
3567
			 * that started delalloc right before we set the block
3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584
			 * group to RO mode, as they might have just allocated
			 * an extent from it or decided they could do a nocow
			 * write. And if any such tasks did that, wait for their
			 * ordered extents to complete and then commit the
			 * current transaction, so that we can later see the new
			 * extent items in the extent tree - the ordered extents
			 * create delayed data references (for cow writes) when
			 * they complete, which will be run and insert the
			 * corresponding extent items into the extent tree when
			 * we commit the transaction they used when running
			 * inode.c:btrfs_finish_ordered_io(). We later use
			 * the commit root of the extent tree to find extents
			 * to copy from the srcdev into the tgtdev, and we don't
			 * want to miss any new extents.
			 */
			btrfs_wait_block_group_reservations(cache);
			btrfs_wait_nocow_writers(cache);
3585
			ret = btrfs_wait_ordered_roots(fs_info, U64_MAX,
3586 3587 3588 3589 3590 3591 3592 3593 3594
						       cache->key.objectid,
						       cache->key.offset);
			if (ret > 0) {
				struct btrfs_trans_handle *trans;

				trans = btrfs_join_transaction(root);
				if (IS_ERR(trans))
					ret = PTR_ERR(trans);
				else
3595
					ret = btrfs_commit_transaction(trans);
3596 3597 3598 3599 3600 3601 3602
				if (ret) {
					scrub_pause_off(fs_info);
					btrfs_put_block_group(cache);
					break;
				}
			}
		}
3603
		scrub_pause_off(fs_info);
3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616

		if (ret == 0) {
			ro_set = 1;
		} else if (ret == -ENOSPC) {
			/*
			 * btrfs_inc_block_group_ro return -ENOSPC when it
			 * failed in creating new chunk for metadata.
			 * It is not a problem for scrub/replace, because
			 * metadata are always cowed, and our scrub paused
			 * commit_transactions.
			 */
			ro_set = 0;
		} else {
J
Jeff Mahoney 已提交
3617
			btrfs_warn(fs_info,
3618
				   "failed setting block group ro: %d", ret);
3619 3620 3621 3622
			btrfs_put_block_group(cache);
			break;
		}

3623
		down_write(&fs_info->dev_replace.rwsem);
3624 3625 3626
		dev_replace->cursor_right = found_key.offset + length;
		dev_replace->cursor_left = found_key.offset;
		dev_replace->item_needs_writeback = 1;
3627 3628
		up_write(&dev_replace->rwsem);

3629
		ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length,
3630
				  found_key.offset, cache);
3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641

		/*
		 * flush, submit all pending read and write bios, afterwards
		 * wait for them.
		 * Note that in the dev replace case, a read request causes
		 * write requests that are submitted in the read completion
		 * worker. Therefore in the current situation, it is required
		 * that all write requests are flushed, so that all read and
		 * write requests are really completed when bios_in_flight
		 * changes to 0.
		 */
3642
		sctx->flush_all_writes = true;
3643
		scrub_submit(sctx);
3644
		mutex_lock(&sctx->wr_lock);
3645
		scrub_wr_submit(sctx);
3646
		mutex_unlock(&sctx->wr_lock);
3647 3648 3649

		wait_event(sctx->list_wait,
			   atomic_read(&sctx->bios_in_flight) == 0);
3650 3651

		scrub_pause_on(fs_info);
3652 3653 3654 3655 3656 3657

		/*
		 * must be called before we decrease @scrub_paused.
		 * make sure we don't block transaction commit while
		 * we are waiting pending workers finished.
		 */
3658 3659
		wait_event(sctx->list_wait,
			   atomic_read(&sctx->workers_pending) == 0);
3660
		sctx->flush_all_writes = false;
3661

3662
		scrub_pause_off(fs_info);
3663

3664
		down_write(&fs_info->dev_replace.rwsem);
3665 3666
		dev_replace->cursor_left = dev_replace->cursor_right;
		dev_replace->item_needs_writeback = 1;
3667
		up_write(&fs_info->dev_replace.rwsem);
3668

3669
		if (ro_set)
3670
			btrfs_dec_block_group_ro(cache);
3671

3672 3673 3674 3675 3676 3677 3678 3679 3680
		/*
		 * We might have prevented the cleaner kthread from deleting
		 * this block group if it was already unused because we raced
		 * and set it to RO mode first. So add it back to the unused
		 * list, otherwise it might not ever be deleted unless a manual
		 * balance is triggered or it becomes used and unused again.
		 */
		spin_lock(&cache->lock);
		if (!cache->removed && !cache->ro && cache->reserved == 0 &&
3681
		    cache->used == 0) {
3682
			spin_unlock(&cache->lock);
3683
			btrfs_mark_bg_unused(cache);
3684 3685 3686 3687
		} else {
			spin_unlock(&cache->lock);
		}

A
Arne Jansen 已提交
3688 3689 3690
		btrfs_put_block_group(cache);
		if (ret)
			break;
3691
		if (sctx->is_dev_replace &&
3692
		    atomic64_read(&dev_replace->num_write_errors) > 0) {
3693 3694 3695 3696 3697 3698 3699
			ret = -EIO;
			break;
		}
		if (sctx->stat.malloc_errors > 0) {
			ret = -ENOMEM;
			break;
		}
3700
skip:
A
Arne Jansen 已提交
3701
		key.offset = found_key.offset + length;
C
Chris Mason 已提交
3702
		btrfs_release_path(path);
A
Arne Jansen 已提交
3703 3704 3705
	}

	btrfs_free_path(path);
3706

3707
	return ret;
A
Arne Jansen 已提交
3708 3709
}

3710 3711
static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
					   struct btrfs_device *scrub_dev)
A
Arne Jansen 已提交
3712 3713 3714 3715 3716
{
	int	i;
	u64	bytenr;
	u64	gen;
	int	ret;
3717
	struct btrfs_fs_info *fs_info = sctx->fs_info;
A
Arne Jansen 已提交
3718

3719
	if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
3720 3721
		return -EIO;

3722
	/* Seed devices of a new filesystem has their own generation. */
3723
	if (scrub_dev->fs_devices != fs_info->fs_devices)
3724 3725
		gen = scrub_dev->generation;
	else
3726
		gen = fs_info->last_trans_committed;
A
Arne Jansen 已提交
3727 3728 3729

	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
3730 3731
		if (bytenr + BTRFS_SUPER_INFO_SIZE >
		    scrub_dev->commit_total_bytes)
A
Arne Jansen 已提交
3732 3733
			break;

3734
		ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
3735
				  scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
3736
				  NULL, 1, bytenr);
A
Arne Jansen 已提交
3737 3738 3739
		if (ret)
			return ret;
	}
3740
	wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
A
Arne Jansen 已提交
3741 3742 3743 3744 3745 3746 3747

	return 0;
}

/*
 * get a reference count on fs_info->scrub_workers. start worker if necessary
 */
3748 3749
static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
						int is_dev_replace)
A
Arne Jansen 已提交
3750
{
3751
	unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
3752
	int max_active = fs_info->thread_pool_size;
A
Arne Jansen 已提交
3753

3754 3755
	lockdep_assert_held(&fs_info->scrub_lock);

3756
	if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) {
3757
		ASSERT(fs_info->scrub_workers == NULL);
3758 3759
		fs_info->scrub_workers = btrfs_alloc_workqueue(fs_info, "scrub",
				flags, is_dev_replace ? 1 : max_active, 4);
3760 3761 3762
		if (!fs_info->scrub_workers)
			goto fail_scrub_workers;

3763
		ASSERT(fs_info->scrub_wr_completion_workers == NULL);
3764
		fs_info->scrub_wr_completion_workers =
3765
			btrfs_alloc_workqueue(fs_info, "scrubwrc", flags,
3766
					      max_active, 2);
3767 3768 3769
		if (!fs_info->scrub_wr_completion_workers)
			goto fail_scrub_wr_completion_workers;

3770
		ASSERT(fs_info->scrub_parity_workers == NULL);
3771
		fs_info->scrub_parity_workers =
3772
			btrfs_alloc_workqueue(fs_info, "scrubparity", flags,
3773
					      max_active, 2);
3774 3775
		if (!fs_info->scrub_parity_workers)
			goto fail_scrub_parity_workers;
3776 3777 3778 3779

		refcount_set(&fs_info->scrub_workers_refcnt, 1);
	} else {
		refcount_inc(&fs_info->scrub_workers_refcnt);
A
Arne Jansen 已提交
3780
	}
3781 3782 3783 3784 3785 3786 3787 3788
	return 0;

fail_scrub_parity_workers:
	btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
fail_scrub_wr_completion_workers:
	btrfs_destroy_workqueue(fs_info->scrub_workers);
fail_scrub_workers:
	return -ENOMEM;
A
Arne Jansen 已提交
3789 3790
}

3791 3792
int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
		    u64 end, struct btrfs_scrub_progress *progress,
3793
		    int readonly, int is_dev_replace)
A
Arne Jansen 已提交
3794
{
3795
	struct scrub_ctx *sctx;
A
Arne Jansen 已提交
3796 3797
	int ret;
	struct btrfs_device *dev;
3798
	unsigned int nofs_flag;
3799 3800 3801
	struct btrfs_workqueue *scrub_workers = NULL;
	struct btrfs_workqueue *scrub_wr_comp = NULL;
	struct btrfs_workqueue *scrub_parity = NULL;
A
Arne Jansen 已提交
3802

3803
	if (btrfs_fs_closing(fs_info))
3804
		return -EAGAIN;
A
Arne Jansen 已提交
3805

3806
	if (fs_info->nodesize > BTRFS_STRIPE_LEN) {
3807 3808 3809 3810 3811
		/*
		 * in this case scrub is unable to calculate the checksum
		 * the way scrub is implemented. Do not handle this
		 * situation at all because it won't ever happen.
		 */
3812 3813
		btrfs_err(fs_info,
			   "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
3814 3815
		       fs_info->nodesize,
		       BTRFS_STRIPE_LEN);
3816 3817 3818
		return -EINVAL;
	}

3819
	if (fs_info->sectorsize != PAGE_SIZE) {
3820
		/* not supported for data w/o checksums */
3821
		btrfs_err_rl(fs_info,
J
Jeff Mahoney 已提交
3822
			   "scrub: size assumption sectorsize != PAGE_SIZE (%d != %lu) fails",
3823
		       fs_info->sectorsize, PAGE_SIZE);
A
Arne Jansen 已提交
3824 3825 3826
		return -EINVAL;
	}

3827
	if (fs_info->nodesize >
3828
	    PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
3829
	    fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
3830 3831 3832 3833
		/*
		 * would exhaust the array bounds of pagev member in
		 * struct scrub_block
		 */
J
Jeff Mahoney 已提交
3834 3835
		btrfs_err(fs_info,
			  "scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
3836
		       fs_info->nodesize,
3837
		       SCRUB_MAX_PAGES_PER_BLOCK,
3838
		       fs_info->sectorsize,
3839 3840 3841 3842
		       SCRUB_MAX_PAGES_PER_BLOCK);
		return -EINVAL;
	}

3843 3844 3845 3846
	/* Allocate outside of device_list_mutex */
	sctx = scrub_setup_ctx(fs_info, is_dev_replace);
	if (IS_ERR(sctx))
		return PTR_ERR(sctx);
A
Arne Jansen 已提交
3847

3848
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
3849
	dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL, true);
3850 3851
	if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) &&
		     !is_dev_replace)) {
3852
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3853 3854
		ret = -ENODEV;
		goto out_free_ctx;
A
Arne Jansen 已提交
3855 3856
	}

3857 3858
	if (!is_dev_replace && !readonly &&
	    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
3859
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3860 3861
		btrfs_err_in_rcu(fs_info, "scrub: device %s is not writable",
				rcu_str_deref(dev->name));
3862 3863
		ret = -EROFS;
		goto out_free_ctx;
3864 3865
	}

3866
	mutex_lock(&fs_info->scrub_lock);
3867
	if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3868
	    test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state)) {
A
Arne Jansen 已提交
3869
		mutex_unlock(&fs_info->scrub_lock);
3870
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3871 3872
		ret = -EIO;
		goto out_free_ctx;
A
Arne Jansen 已提交
3873 3874
	}

3875
	down_read(&fs_info->dev_replace.rwsem);
3876
	if (dev->scrub_ctx ||
3877 3878
	    (!is_dev_replace &&
	     btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
3879
		up_read(&fs_info->dev_replace.rwsem);
A
Arne Jansen 已提交
3880
		mutex_unlock(&fs_info->scrub_lock);
3881
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3882 3883
		ret = -EINPROGRESS;
		goto out_free_ctx;
A
Arne Jansen 已提交
3884
	}
3885
	up_read(&fs_info->dev_replace.rwsem);
3886 3887 3888 3889 3890

	ret = scrub_workers_get(fs_info, is_dev_replace);
	if (ret) {
		mutex_unlock(&fs_info->scrub_lock);
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3891
		goto out_free_ctx;
3892 3893
	}

3894
	sctx->readonly = readonly;
3895
	dev->scrub_ctx = sctx;
3896
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
A
Arne Jansen 已提交
3897

3898 3899 3900 3901
	/*
	 * checking @scrub_pause_req here, we can avoid
	 * race between committing transaction and scrubbing.
	 */
3902
	__scrub_blocked_if_needed(fs_info);
A
Arne Jansen 已提交
3903 3904 3905
	atomic_inc(&fs_info->scrubs_running);
	mutex_unlock(&fs_info->scrub_lock);

3906 3907 3908 3909 3910 3911 3912 3913 3914 3915
	/*
	 * In order to avoid deadlock with reclaim when there is a transaction
	 * trying to pause scrub, make sure we use GFP_NOFS for all the
	 * allocations done at btrfs_scrub_pages() and scrub_pages_for_parity()
	 * invoked by our callees. The pausing request is done when the
	 * transaction commit starts, and it blocks the transaction until scrub
	 * is paused (done at specific points at scrub_stripe() or right above
	 * before incrementing fs_info->scrubs_running).
	 */
	nofs_flag = memalloc_nofs_save();
3916
	if (!is_dev_replace) {
3917
		btrfs_info(fs_info, "scrub: started on devid %llu", devid);
3918 3919 3920 3921
		/*
		 * by holding device list mutex, we can
		 * kick off writing super in log tree sync.
		 */
3922
		mutex_lock(&fs_info->fs_devices->device_list_mutex);
3923
		ret = scrub_supers(sctx, dev);
3924
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3925
	}
A
Arne Jansen 已提交
3926 3927

	if (!ret)
3928
		ret = scrub_enumerate_chunks(sctx, dev, start, end);
3929
	memalloc_nofs_restore(nofs_flag);
A
Arne Jansen 已提交
3930

3931
	wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
A
Arne Jansen 已提交
3932 3933 3934
	atomic_dec(&fs_info->scrubs_running);
	wake_up(&fs_info->scrub_pause_wait);

3935
	wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
3936

A
Arne Jansen 已提交
3937
	if (progress)
3938
		memcpy(progress, &sctx->stat, sizeof(*progress));
A
Arne Jansen 已提交
3939

3940 3941 3942 3943
	if (!is_dev_replace)
		btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d",
			ret ? "not finished" : "finished", devid, ret);

A
Arne Jansen 已提交
3944
	mutex_lock(&fs_info->scrub_lock);
3945
	dev->scrub_ctx = NULL;
3946
	if (refcount_dec_and_test(&fs_info->scrub_workers_refcnt)) {
3947 3948 3949
		scrub_workers = fs_info->scrub_workers;
		scrub_wr_comp = fs_info->scrub_wr_completion_workers;
		scrub_parity = fs_info->scrub_parity_workers;
3950 3951 3952 3953

		fs_info->scrub_workers = NULL;
		fs_info->scrub_wr_completion_workers = NULL;
		fs_info->scrub_parity_workers = NULL;
3954
	}
A
Arne Jansen 已提交
3955 3956
	mutex_unlock(&fs_info->scrub_lock);

3957 3958 3959
	btrfs_destroy_workqueue(scrub_workers);
	btrfs_destroy_workqueue(scrub_wr_comp);
	btrfs_destroy_workqueue(scrub_parity);
3960
	scrub_put_ctx(sctx);
A
Arne Jansen 已提交
3961

3962 3963 3964 3965 3966
	return ret;

out_free_ctx:
	scrub_free_ctx(sctx);

A
Arne Jansen 已提交
3967 3968 3969
	return ret;
}

3970
void btrfs_scrub_pause(struct btrfs_fs_info *fs_info)
A
Arne Jansen 已提交
3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984
{
	mutex_lock(&fs_info->scrub_lock);
	atomic_inc(&fs_info->scrub_pause_req);
	while (atomic_read(&fs_info->scrubs_paused) !=
	       atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
			   atomic_read(&fs_info->scrubs_paused) ==
			   atomic_read(&fs_info->scrubs_running));
		mutex_lock(&fs_info->scrub_lock);
	}
	mutex_unlock(&fs_info->scrub_lock);
}

3985
void btrfs_scrub_continue(struct btrfs_fs_info *fs_info)
A
Arne Jansen 已提交
3986 3987 3988 3989 3990
{
	atomic_dec(&fs_info->scrub_pause_req);
	wake_up(&fs_info->scrub_pause_wait);
}

3991
int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
A
Arne Jansen 已提交
3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011
{
	mutex_lock(&fs_info->scrub_lock);
	if (!atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		return -ENOTCONN;
	}

	atomic_inc(&fs_info->scrub_cancel_req);
	while (atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
			   atomic_read(&fs_info->scrubs_running) == 0);
		mutex_lock(&fs_info->scrub_lock);
	}
	atomic_dec(&fs_info->scrub_cancel_req);
	mutex_unlock(&fs_info->scrub_lock);

	return 0;
}

4012
int btrfs_scrub_cancel_dev(struct btrfs_device *dev)
4013
{
4014
	struct btrfs_fs_info *fs_info = dev->fs_info;
4015
	struct scrub_ctx *sctx;
A
Arne Jansen 已提交
4016 4017

	mutex_lock(&fs_info->scrub_lock);
4018
	sctx = dev->scrub_ctx;
4019
	if (!sctx) {
A
Arne Jansen 已提交
4020 4021 4022
		mutex_unlock(&fs_info->scrub_lock);
		return -ENOTCONN;
	}
4023
	atomic_inc(&sctx->cancel_req);
4024
	while (dev->scrub_ctx) {
A
Arne Jansen 已提交
4025 4026
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
4027
			   dev->scrub_ctx == NULL);
A
Arne Jansen 已提交
4028 4029 4030 4031 4032 4033
		mutex_lock(&fs_info->scrub_lock);
	}
	mutex_unlock(&fs_info->scrub_lock);

	return 0;
}
S
Stefan Behrens 已提交
4034

4035
int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid,
A
Arne Jansen 已提交
4036 4037 4038
			 struct btrfs_scrub_progress *progress)
{
	struct btrfs_device *dev;
4039
	struct scrub_ctx *sctx = NULL;
A
Arne Jansen 已提交
4040

4041
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
4042
	dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL, true);
A
Arne Jansen 已提交
4043
	if (dev)
4044
		sctx = dev->scrub_ctx;
4045 4046
	if (sctx)
		memcpy(progress, &sctx->stat, sizeof(*progress));
4047
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
A
Arne Jansen 已提交
4048

4049
	return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
A
Arne Jansen 已提交
4050
}
4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062

static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
			       u64 extent_logical, u64 extent_len,
			       u64 *extent_physical,
			       struct btrfs_device **extent_dev,
			       int *extent_mirror_num)
{
	u64 mapped_length;
	struct btrfs_bio *bbio = NULL;
	int ret;

	mapped_length = extent_len;
4063
	ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_logical,
4064 4065 4066
			      &mapped_length, &bbio, 0);
	if (ret || !bbio || mapped_length < extent_len ||
	    !bbio->stripes[0].dev->bdev) {
4067
		btrfs_put_bbio(bbio);
4068 4069 4070 4071 4072 4073
		return;
	}

	*extent_physical = bbio->stripes[0].physical;
	*extent_mirror_num = bbio->mirror_num;
	*extent_dev = bbio->stripes[0].dev;
4074
	btrfs_put_bbio(bbio);
4075
}