scrub.c 113.4 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
A
Arne Jansen 已提交
2
/*
3
 * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
A
Arne Jansen 已提交
4 5 6
 */

#include <linux/blkdev.h>
7
#include <linux/ratelimit.h>
8
#include <linux/sched/mm.h>
A
Arne Jansen 已提交
9 10 11 12
#include "ctree.h"
#include "volumes.h"
#include "disk-io.h"
#include "ordered-data.h"
13
#include "transaction.h"
14
#include "backref.h"
15
#include "extent_io.h"
16
#include "dev-replace.h"
17
#include "check-integrity.h"
18
#include "rcu-string.h"
D
David Woodhouse 已提交
19
#include "raid56.h"
A
Arne Jansen 已提交
20 21 22 23 24 25 26 27 28 29 30 31 32 33

/*
 * This is only the first step towards a full-features scrub. It reads all
 * extent and super block and verifies the checksums. In case a bad checksum
 * is found or the extent cannot be read, good data will be written back if
 * any can be found.
 *
 * Future enhancements:
 *  - In case an unrepairable extent is encountered, track which files are
 *    affected and report them
 *  - track and record media errors, throw out bad devices
 *  - add a mode to also read unallocated space
 */

34
struct scrub_block;
35
struct scrub_ctx;
A
Arne Jansen 已提交
36

37 38 39 40 41 42 43 44 45
/*
 * the following three values only influence the performance.
 * The last one configures the number of parallel and outstanding I/O
 * operations. The first two values configure an upper limit for the number
 * of (dynamically allocated) pages that are added to a bio.
 */
#define SCRUB_PAGES_PER_RD_BIO	32	/* 128k per bio */
#define SCRUB_PAGES_PER_WR_BIO	32	/* 128k per bio */
#define SCRUB_BIOS_PER_SCTX	64	/* 8MB per device in flight */
46 47 48 49 50 51

/*
 * the following value times PAGE_SIZE needs to be large enough to match the
 * largest node/leaf/sector size that shall be supported.
 * Values larger than BTRFS_STRIPE_LEN are not supported.
 */
52
#define SCRUB_MAX_PAGES_PER_BLOCK	16	/* 64k per node/leaf/sector */
A
Arne Jansen 已提交
53

54
struct scrub_recover {
55
	refcount_t		refs;
56 57 58 59
	struct btrfs_bio	*bbio;
	u64			map_length;
};

A
Arne Jansen 已提交
60
struct scrub_page {
61 62
	struct scrub_block	*sblock;
	struct page		*page;
63
	struct btrfs_device	*dev;
64
	struct list_head	list;
A
Arne Jansen 已提交
65 66
	u64			flags;  /* extent flags */
	u64			generation;
67 68
	u64			logical;
	u64			physical;
69
	u64			physical_for_dev_replace;
70
	atomic_t		refs;
71 72 73 74 75
	struct {
		unsigned int	mirror_num:8;
		unsigned int	have_csum:1;
		unsigned int	io_error:1;
	};
A
Arne Jansen 已提交
76
	u8			csum[BTRFS_CSUM_SIZE];
77 78

	struct scrub_recover	*recover;
A
Arne Jansen 已提交
79 80 81 82
};

struct scrub_bio {
	int			index;
83
	struct scrub_ctx	*sctx;
84
	struct btrfs_device	*dev;
A
Arne Jansen 已提交
85
	struct bio		*bio;
86
	blk_status_t		status;
A
Arne Jansen 已提交
87 88
	u64			logical;
	u64			physical;
89 90 91 92 93
#if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
	struct scrub_page	*pagev[SCRUB_PAGES_PER_WR_BIO];
#else
	struct scrub_page	*pagev[SCRUB_PAGES_PER_RD_BIO];
#endif
94
	int			page_count;
A
Arne Jansen 已提交
95 96 97 98
	int			next_free;
	struct btrfs_work	work;
};

99
struct scrub_block {
100
	struct scrub_page	*pagev[SCRUB_MAX_PAGES_PER_BLOCK];
101 102
	int			page_count;
	atomic_t		outstanding_pages;
103
	refcount_t		refs; /* free mem on transition to zero */
104
	struct scrub_ctx	*sctx;
105
	struct scrub_parity	*sparity;
106 107 108 109
	struct {
		unsigned int	header_error:1;
		unsigned int	checksum_error:1;
		unsigned int	no_io_error_seen:1;
110
		unsigned int	generation_error:1; /* also sets header_error */
111 112 113 114

		/* The following is for the data used to check parity */
		/* It is for the data with checksum */
		unsigned int	data_corrected:1;
115
	};
116
	struct btrfs_work	work;
117 118
};

119 120 121 122 123 124 125 126 127 128 129 130
/* Used for the chunks with parity stripe such RAID5/6 */
struct scrub_parity {
	struct scrub_ctx	*sctx;

	struct btrfs_device	*scrub_dev;

	u64			logic_start;

	u64			logic_end;

	int			nsectors;

131
	u64			stripe_len;
132

133
	refcount_t		refs;
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151

	struct list_head	spages;

	/* Work of parity check and repair */
	struct btrfs_work	work;

	/* Mark the parity blocks which have data */
	unsigned long		*dbitmap;

	/*
	 * Mark the parity blocks which have data, but errors happen when
	 * read data or check data
	 */
	unsigned long		*ebitmap;

	unsigned long		bitmap[0];
};

152
struct scrub_ctx {
153
	struct scrub_bio	*bios[SCRUB_BIOS_PER_SCTX];
154
	struct btrfs_fs_info	*fs_info;
A
Arne Jansen 已提交
155 156
	int			first_free;
	int			curr;
157 158
	atomic_t		bios_in_flight;
	atomic_t		workers_pending;
A
Arne Jansen 已提交
159 160 161 162 163
	spinlock_t		list_lock;
	wait_queue_head_t	list_wait;
	u16			csum_size;
	struct list_head	csum_list;
	atomic_t		cancel_req;
A
Arne Jansen 已提交
164
	int			readonly;
165
	int			pages_per_rd_bio;
166 167

	int			is_dev_replace;
168 169 170 171 172

	struct scrub_bio        *wr_curr_bio;
	struct mutex            wr_lock;
	int                     pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
	struct btrfs_device     *wr_tgtdev;
173
	bool                    flush_all_writes;
174

A
Arne Jansen 已提交
175 176 177 178 179
	/*
	 * statistics
	 */
	struct btrfs_scrub_progress stat;
	spinlock_t		stat_lock;
180 181 182 183 184 185 186 187

	/*
	 * Use a ref counter to avoid use-after-free issues. Scrub workers
	 * decrement bios_in_flight and workers_pending and then do a wakeup
	 * on the list_wait wait queue. We must ensure the main scrub task
	 * doesn't free the scrub context before or while the workers are
	 * doing the wakeup() call.
	 */
188
	refcount_t              refs;
A
Arne Jansen 已提交
189 190
};

191
struct scrub_fixup_nodatasum {
192
	struct scrub_ctx	*sctx;
193
	struct btrfs_device	*dev;
194 195 196 197 198 199
	u64			logical;
	struct btrfs_root	*root;
	struct btrfs_work	work;
	int			mirror_num;
};

200 201 202 203
struct scrub_warning {
	struct btrfs_path	*path;
	u64			extent_item_size;
	const char		*errstr;
D
David Sterba 已提交
204
	u64			physical;
205 206 207 208
	u64			logical;
	struct btrfs_device	*dev;
};

209 210 211 212 213 214 215
struct full_stripe_lock {
	struct rb_node node;
	u64 logical;
	u64 refs;
	struct mutex mutex;
};

216 217 218 219
static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
220
static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
221
static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
222
				     struct scrub_block *sblocks_for_recheck);
223
static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
224 225
				struct scrub_block *sblock,
				int retry_failed_mirror);
226
static void scrub_recheck_block_checksum(struct scrub_block *sblock);
227
static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
228
					     struct scrub_block *sblock_good);
229 230 231
static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
					    struct scrub_block *sblock_good,
					    int page_num, int force_write);
232 233 234
static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
					   int page_num);
235 236 237 238 239
static int scrub_checksum_data(struct scrub_block *sblock);
static int scrub_checksum_tree_block(struct scrub_block *sblock);
static int scrub_checksum_super(struct scrub_block *sblock);
static void scrub_block_get(struct scrub_block *sblock);
static void scrub_block_put(struct scrub_block *sblock);
240 241
static void scrub_page_get(struct scrub_page *spage);
static void scrub_page_put(struct scrub_page *spage);
242 243
static void scrub_parity_get(struct scrub_parity *sparity);
static void scrub_parity_put(struct scrub_parity *sparity);
244 245
static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
				    struct scrub_page *spage);
246
static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
247
		       u64 physical, struct btrfs_device *dev, u64 flags,
248 249
		       u64 gen, int mirror_num, u8 *csum, int force,
		       u64 physical_for_dev_replace);
250
static void scrub_bio_end_io(struct bio *bio);
251 252
static void scrub_bio_end_io_worker(struct btrfs_work *work);
static void scrub_block_complete(struct scrub_block *sblock);
253 254 255 256 257 258 259 260
static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
			       u64 extent_logical, u64 extent_len,
			       u64 *extent_physical,
			       struct btrfs_device **extent_dev,
			       int *extent_mirror_num);
static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
				    struct scrub_page *spage);
static void scrub_wr_submit(struct scrub_ctx *sctx);
261
static void scrub_wr_bio_end_io(struct bio *bio);
262
static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
263
static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
264
static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
265
static void scrub_put_ctx(struct scrub_ctx *sctx);
S
Stefan Behrens 已提交
266

267 268 269 270 271
static inline int scrub_is_page_on_raid56(struct scrub_page *page)
{
	return page->recover &&
	       (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
}
S
Stefan Behrens 已提交
272

273 274
static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
{
275
	refcount_inc(&sctx->refs);
276 277 278 279 280 281 282
	atomic_inc(&sctx->bios_in_flight);
}

static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
{
	atomic_dec(&sctx->bios_in_flight);
	wake_up(&sctx->list_wait);
283
	scrub_put_ctx(sctx);
284 285
}

286
static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
287 288 289 290 291 292 293 294 295
{
	while (atomic_read(&fs_info->scrub_pause_req)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
		   atomic_read(&fs_info->scrub_pause_req) == 0);
		mutex_lock(&fs_info->scrub_lock);
	}
}

296
static void scrub_pause_on(struct btrfs_fs_info *fs_info)
297 298 299
{
	atomic_inc(&fs_info->scrubs_paused);
	wake_up(&fs_info->scrub_pause_wait);
300
}
301

302 303
static void scrub_pause_off(struct btrfs_fs_info *fs_info)
{
304 305 306 307 308 309 310 311
	mutex_lock(&fs_info->scrub_lock);
	__scrub_blocked_if_needed(fs_info);
	atomic_dec(&fs_info->scrubs_paused);
	mutex_unlock(&fs_info->scrub_lock);

	wake_up(&fs_info->scrub_pause_wait);
}

312 313 314 315 316 317
static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
{
	scrub_pause_on(fs_info);
	scrub_pause_off(fs_info);
}

318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
/*
 * Insert new full stripe lock into full stripe locks tree
 *
 * Return pointer to existing or newly inserted full_stripe_lock structure if
 * everything works well.
 * Return ERR_PTR(-ENOMEM) if we failed to allocate memory
 *
 * NOTE: caller must hold full_stripe_locks_root->lock before calling this
 * function
 */
static struct full_stripe_lock *insert_full_stripe_lock(
		struct btrfs_full_stripe_locks_tree *locks_root,
		u64 fstripe_logical)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct full_stripe_lock *entry;
	struct full_stripe_lock *ret;

337
	lockdep_assert_held(&locks_root->lock);
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378

	p = &locks_root->root.rb_node;
	while (*p) {
		parent = *p;
		entry = rb_entry(parent, struct full_stripe_lock, node);
		if (fstripe_logical < entry->logical) {
			p = &(*p)->rb_left;
		} else if (fstripe_logical > entry->logical) {
			p = &(*p)->rb_right;
		} else {
			entry->refs++;
			return entry;
		}
	}

	/* Insert new lock */
	ret = kmalloc(sizeof(*ret), GFP_KERNEL);
	if (!ret)
		return ERR_PTR(-ENOMEM);
	ret->logical = fstripe_logical;
	ret->refs = 1;
	mutex_init(&ret->mutex);

	rb_link_node(&ret->node, parent, p);
	rb_insert_color(&ret->node, &locks_root->root);
	return ret;
}

/*
 * Search for a full stripe lock of a block group
 *
 * Return pointer to existing full stripe lock if found
 * Return NULL if not found
 */
static struct full_stripe_lock *search_full_stripe_lock(
		struct btrfs_full_stripe_locks_tree *locks_root,
		u64 fstripe_logical)
{
	struct rb_node *node;
	struct full_stripe_lock *entry;

379
	lockdep_assert_held(&locks_root->lock);
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533

	node = locks_root->root.rb_node;
	while (node) {
		entry = rb_entry(node, struct full_stripe_lock, node);
		if (fstripe_logical < entry->logical)
			node = node->rb_left;
		else if (fstripe_logical > entry->logical)
			node = node->rb_right;
		else
			return entry;
	}
	return NULL;
}

/*
 * Helper to get full stripe logical from a normal bytenr.
 *
 * Caller must ensure @cache is a RAID56 block group.
 */
static u64 get_full_stripe_logical(struct btrfs_block_group_cache *cache,
				   u64 bytenr)
{
	u64 ret;

	/*
	 * Due to chunk item size limit, full stripe length should not be
	 * larger than U32_MAX. Just a sanity check here.
	 */
	WARN_ON_ONCE(cache->full_stripe_len >= U32_MAX);

	/*
	 * round_down() can only handle power of 2, while RAID56 full
	 * stripe length can be 64KiB * n, so we need to manually round down.
	 */
	ret = div64_u64(bytenr - cache->key.objectid, cache->full_stripe_len) *
		cache->full_stripe_len + cache->key.objectid;
	return ret;
}

/*
 * Lock a full stripe to avoid concurrency of recovery and read
 *
 * It's only used for profiles with parities (RAID5/6), for other profiles it
 * does nothing.
 *
 * Return 0 if we locked full stripe covering @bytenr, with a mutex held.
 * So caller must call unlock_full_stripe() at the same context.
 *
 * Return <0 if encounters error.
 */
static int lock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
			    bool *locked_ret)
{
	struct btrfs_block_group_cache *bg_cache;
	struct btrfs_full_stripe_locks_tree *locks_root;
	struct full_stripe_lock *existing;
	u64 fstripe_start;
	int ret = 0;

	*locked_ret = false;
	bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
	if (!bg_cache) {
		ASSERT(0);
		return -ENOENT;
	}

	/* Profiles not based on parity don't need full stripe lock */
	if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
		goto out;
	locks_root = &bg_cache->full_stripe_locks_root;

	fstripe_start = get_full_stripe_logical(bg_cache, bytenr);

	/* Now insert the full stripe lock */
	mutex_lock(&locks_root->lock);
	existing = insert_full_stripe_lock(locks_root, fstripe_start);
	mutex_unlock(&locks_root->lock);
	if (IS_ERR(existing)) {
		ret = PTR_ERR(existing);
		goto out;
	}
	mutex_lock(&existing->mutex);
	*locked_ret = true;
out:
	btrfs_put_block_group(bg_cache);
	return ret;
}

/*
 * Unlock a full stripe.
 *
 * NOTE: Caller must ensure it's the same context calling corresponding
 * lock_full_stripe().
 *
 * Return 0 if we unlock full stripe without problem.
 * Return <0 for error
 */
static int unlock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
			      bool locked)
{
	struct btrfs_block_group_cache *bg_cache;
	struct btrfs_full_stripe_locks_tree *locks_root;
	struct full_stripe_lock *fstripe_lock;
	u64 fstripe_start;
	bool freeit = false;
	int ret = 0;

	/* If we didn't acquire full stripe lock, no need to continue */
	if (!locked)
		return 0;

	bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
	if (!bg_cache) {
		ASSERT(0);
		return -ENOENT;
	}
	if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
		goto out;

	locks_root = &bg_cache->full_stripe_locks_root;
	fstripe_start = get_full_stripe_logical(bg_cache, bytenr);

	mutex_lock(&locks_root->lock);
	fstripe_lock = search_full_stripe_lock(locks_root, fstripe_start);
	/* Unpaired unlock_full_stripe() detected */
	if (!fstripe_lock) {
		WARN_ON(1);
		ret = -ENOENT;
		mutex_unlock(&locks_root->lock);
		goto out;
	}

	if (fstripe_lock->refs == 0) {
		WARN_ON(1);
		btrfs_warn(fs_info, "full stripe lock at %llu refcount underflow",
			fstripe_lock->logical);
	} else {
		fstripe_lock->refs--;
	}

	if (fstripe_lock->refs == 0) {
		rb_erase(&fstripe_lock->node, &locks_root->root);
		freeit = true;
	}
	mutex_unlock(&locks_root->lock);

	mutex_unlock(&fstripe_lock->mutex);
	if (freeit)
		kfree(fstripe_lock);
out:
	btrfs_put_block_group(bg_cache);
	return ret;
}

534 535 536 537 538 539
/*
 * used for workers that require transaction commits (i.e., for the
 * NOCOW case)
 */
static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
{
540
	struct btrfs_fs_info *fs_info = sctx->fs_info;
541

542
	refcount_inc(&sctx->refs);
543 544 545 546 547 548 549 550 551 552 553 554 555
	/*
	 * increment scrubs_running to prevent cancel requests from
	 * completing as long as a worker is running. we must also
	 * increment scrubs_paused to prevent deadlocking on pause
	 * requests used for transactions commits (as the worker uses a
	 * transaction context). it is safe to regard the worker
	 * as paused for all matters practical. effectively, we only
	 * avoid cancellation requests from completing.
	 */
	mutex_lock(&fs_info->scrub_lock);
	atomic_inc(&fs_info->scrubs_running);
	atomic_inc(&fs_info->scrubs_paused);
	mutex_unlock(&fs_info->scrub_lock);
556 557 558 559 560 561 562 563 564 565

	/*
	 * check if @scrubs_running=@scrubs_paused condition
	 * inside wait_event() is not an atomic operation.
	 * which means we may inc/dec @scrub_running/paused
	 * at any time. Let's wake up @scrub_pause_wait as
	 * much as we can to let commit transaction blocked less.
	 */
	wake_up(&fs_info->scrub_pause_wait);

566 567 568 569 570 571
	atomic_inc(&sctx->workers_pending);
}

/* used for workers that require transaction commits */
static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
{
572
	struct btrfs_fs_info *fs_info = sctx->fs_info;
573 574 575 576 577 578 579 580 581 582 583 584

	/*
	 * see scrub_pending_trans_workers_inc() why we're pretending
	 * to be paused in the scrub counters
	 */
	mutex_lock(&fs_info->scrub_lock);
	atomic_dec(&fs_info->scrubs_running);
	atomic_dec(&fs_info->scrubs_paused);
	mutex_unlock(&fs_info->scrub_lock);
	atomic_dec(&sctx->workers_pending);
	wake_up(&fs_info->scrub_pause_wait);
	wake_up(&sctx->list_wait);
585
	scrub_put_ctx(sctx);
586 587
}

588
static void scrub_free_csums(struct scrub_ctx *sctx)
A
Arne Jansen 已提交
589
{
590
	while (!list_empty(&sctx->csum_list)) {
A
Arne Jansen 已提交
591
		struct btrfs_ordered_sum *sum;
592
		sum = list_first_entry(&sctx->csum_list,
A
Arne Jansen 已提交
593 594 595 596 597 598
				       struct btrfs_ordered_sum, list);
		list_del(&sum->list);
		kfree(sum);
	}
}

599
static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
A
Arne Jansen 已提交
600 601 602
{
	int i;

603
	if (!sctx)
A
Arne Jansen 已提交
604 605
		return;

606
	/* this can happen when scrub is cancelled */
607 608
	if (sctx->curr != -1) {
		struct scrub_bio *sbio = sctx->bios[sctx->curr];
609 610

		for (i = 0; i < sbio->page_count; i++) {
611
			WARN_ON(!sbio->pagev[i]->page);
612 613 614 615 616
			scrub_block_put(sbio->pagev[i]->sblock);
		}
		bio_put(sbio->bio);
	}

617
	for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
618
		struct scrub_bio *sbio = sctx->bios[i];
A
Arne Jansen 已提交
619 620 621 622 623 624

		if (!sbio)
			break;
		kfree(sbio);
	}

625
	kfree(sctx->wr_curr_bio);
626 627
	scrub_free_csums(sctx);
	kfree(sctx);
A
Arne Jansen 已提交
628 629
}

630 631
static void scrub_put_ctx(struct scrub_ctx *sctx)
{
632
	if (refcount_dec_and_test(&sctx->refs))
633 634 635
		scrub_free_ctx(sctx);
}

A
Arne Jansen 已提交
636
static noinline_for_stack
637
struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
A
Arne Jansen 已提交
638
{
639
	struct scrub_ctx *sctx;
A
Arne Jansen 已提交
640
	int		i;
641
	struct btrfs_fs_info *fs_info = dev->fs_info;
A
Arne Jansen 已提交
642

643
	sctx = kzalloc(sizeof(*sctx), GFP_KERNEL);
644
	if (!sctx)
A
Arne Jansen 已提交
645
		goto nomem;
646
	refcount_set(&sctx->refs, 1);
647
	sctx->is_dev_replace = is_dev_replace;
648
	sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
649
	sctx->curr = -1;
650
	sctx->fs_info = dev->fs_info;
651
	for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
A
Arne Jansen 已提交
652 653
		struct scrub_bio *sbio;

654
		sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);
A
Arne Jansen 已提交
655 656
		if (!sbio)
			goto nomem;
657
		sctx->bios[i] = sbio;
A
Arne Jansen 已提交
658 659

		sbio->index = i;
660
		sbio->sctx = sctx;
661
		sbio->page_count = 0;
662 663
		btrfs_init_work(&sbio->work, btrfs_scrub_helper,
				scrub_bio_end_io_worker, NULL, NULL);
A
Arne Jansen 已提交
664

665
		if (i != SCRUB_BIOS_PER_SCTX - 1)
666
			sctx->bios[i]->next_free = i + 1;
667
		else
668 669 670
			sctx->bios[i]->next_free = -1;
	}
	sctx->first_free = 0;
671 672
	atomic_set(&sctx->bios_in_flight, 0);
	atomic_set(&sctx->workers_pending, 0);
673 674 675 676 677 678 679
	atomic_set(&sctx->cancel_req, 0);
	sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
	INIT_LIST_HEAD(&sctx->csum_list);

	spin_lock_init(&sctx->list_lock);
	spin_lock_init(&sctx->stat_lock);
	init_waitqueue_head(&sctx->list_wait);
680

681 682 683
	WARN_ON(sctx->wr_curr_bio != NULL);
	mutex_init(&sctx->wr_lock);
	sctx->wr_curr_bio = NULL;
684
	if (is_dev_replace) {
685
		WARN_ON(!fs_info->dev_replace.tgtdev);
686
		sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO;
687
		sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;
688
		sctx->flush_all_writes = false;
689
	}
690

691
	return sctx;
A
Arne Jansen 已提交
692 693

nomem:
694
	scrub_free_ctx(sctx);
A
Arne Jansen 已提交
695 696 697
	return ERR_PTR(-ENOMEM);
}

698 699
static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
				     void *warn_ctx)
700 701 702 703 704
{
	u64 isize;
	u32 nlink;
	int ret;
	int i;
705
	unsigned nofs_flag;
706 707
	struct extent_buffer *eb;
	struct btrfs_inode_item *inode_item;
708
	struct scrub_warning *swarn = warn_ctx;
709
	struct btrfs_fs_info *fs_info = swarn->dev->fs_info;
710 711 712
	struct inode_fs_paths *ipath = NULL;
	struct btrfs_root *local_root;
	struct btrfs_key root_key;
713
	struct btrfs_key key;
714 715 716 717 718 719 720 721 722 723

	root_key.objectid = root;
	root_key.type = BTRFS_ROOT_ITEM_KEY;
	root_key.offset = (u64)-1;
	local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
	if (IS_ERR(local_root)) {
		ret = PTR_ERR(local_root);
		goto err;
	}

724 725 726
	/*
	 * this makes the path point to (inum INODE_ITEM ioff)
	 */
727 728 729 730 731
	key.objectid = inum;
	key.type = BTRFS_INODE_ITEM_KEY;
	key.offset = 0;

	ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
732 733 734 735 736 737 738 739 740 741 742 743
	if (ret) {
		btrfs_release_path(swarn->path);
		goto err;
	}

	eb = swarn->path->nodes[0];
	inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
					struct btrfs_inode_item);
	isize = btrfs_inode_size(eb, inode_item);
	nlink = btrfs_inode_nlink(eb, inode_item);
	btrfs_release_path(swarn->path);

744 745 746 747 748 749
	/*
	 * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub
	 * uses GFP_NOFS in this context, so we keep it consistent but it does
	 * not seem to be strictly necessary.
	 */
	nofs_flag = memalloc_nofs_save();
750
	ipath = init_ipath(4096, local_root, swarn->path);
751
	memalloc_nofs_restore(nofs_flag);
752 753 754 755 756
	if (IS_ERR(ipath)) {
		ret = PTR_ERR(ipath);
		ipath = NULL;
		goto err;
	}
757 758 759 760 761 762 763 764 765 766
	ret = paths_from_inode(inum, ipath);

	if (ret < 0)
		goto err;

	/*
	 * we deliberately ignore the bit ipath might have been too small to
	 * hold all of the paths here
	 */
	for (i = 0; i < ipath->fspath->elem_cnt; ++i)
J
Jeff Mahoney 已提交
767
		btrfs_warn_in_rcu(fs_info,
D
David Sterba 已提交
768
"%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)",
J
Jeff Mahoney 已提交
769 770
				  swarn->errstr, swarn->logical,
				  rcu_str_deref(swarn->dev->name),
D
David Sterba 已提交
771
				  swarn->physical,
J
Jeff Mahoney 已提交
772 773 774
				  root, inum, offset,
				  min(isize - offset, (u64)PAGE_SIZE), nlink,
				  (char *)(unsigned long)ipath->fspath->val[i]);
775 776 777 778 779

	free_ipath(ipath);
	return 0;

err:
J
Jeff Mahoney 已提交
780
	btrfs_warn_in_rcu(fs_info,
D
David Sterba 已提交
781
			  "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d",
J
Jeff Mahoney 已提交
782 783
			  swarn->errstr, swarn->logical,
			  rcu_str_deref(swarn->dev->name),
D
David Sterba 已提交
784
			  swarn->physical,
J
Jeff Mahoney 已提交
785
			  root, inum, offset, ret);
786 787 788 789 790

	free_ipath(ipath);
	return 0;
}

791
static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
792
{
793 794
	struct btrfs_device *dev;
	struct btrfs_fs_info *fs_info;
795 796 797 798 799
	struct btrfs_path *path;
	struct btrfs_key found_key;
	struct extent_buffer *eb;
	struct btrfs_extent_item *ei;
	struct scrub_warning swarn;
800 801 802
	unsigned long ptr = 0;
	u64 extent_item_pos;
	u64 flags = 0;
803
	u64 ref_root;
804
	u32 item_size;
805
	u8 ref_level = 0;
806
	int ret;
807

808
	WARN_ON(sblock->page_count < 1);
809
	dev = sblock->pagev[0]->dev;
810
	fs_info = sblock->sctx->fs_info;
811

812
	path = btrfs_alloc_path();
813 814
	if (!path)
		return;
815

D
David Sterba 已提交
816
	swarn.physical = sblock->pagev[0]->physical;
817
	swarn.logical = sblock->pagev[0]->logical;
818
	swarn.errstr = errstr;
819
	swarn.dev = NULL;
820

821 822
	ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
				  &flags);
823 824 825
	if (ret < 0)
		goto out;

J
Jan Schmidt 已提交
826
	extent_item_pos = swarn.logical - found_key.objectid;
827 828 829 830 831 832
	swarn.extent_item_size = found_key.offset;

	eb = path->nodes[0];
	ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
	item_size = btrfs_item_size_nr(eb, path->slots[0]);

833
	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
834
		do {
835 836 837
			ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
						      item_size, &ref_root,
						      &ref_level);
838
			btrfs_warn_in_rcu(fs_info,
D
David Sterba 已提交
839
"%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",
J
Jeff Mahoney 已提交
840
				errstr, swarn.logical,
841
				rcu_str_deref(dev->name),
D
David Sterba 已提交
842
				swarn.physical,
843 844 845 846
				ref_level ? "node" : "leaf",
				ret < 0 ? -1 : ref_level,
				ret < 0 ? -1 : ref_root);
		} while (ret != 1);
847
		btrfs_release_path(path);
848
	} else {
849
		btrfs_release_path(path);
850
		swarn.path = path;
851
		swarn.dev = dev;
852 853
		iterate_extent_inodes(fs_info, found_key.objectid,
					extent_item_pos, 1,
854
					scrub_print_warning_inode, &swarn, false);
855 856 857 858 859 860
	}

out:
	btrfs_free_path(path);
}

861
static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
862
{
863
	struct page *page = NULL;
864
	unsigned long index;
865
	struct scrub_fixup_nodatasum *fixup = fixup_ctx;
866
	int ret;
867
	int corrected = 0;
868
	struct btrfs_key key;
869
	struct inode *inode = NULL;
870
	struct btrfs_fs_info *fs_info;
871 872
	u64 end = offset + PAGE_SIZE - 1;
	struct btrfs_root *local_root;
873
	int srcu_index;
874 875 876 877

	key.objectid = root;
	key.type = BTRFS_ROOT_ITEM_KEY;
	key.offset = (u64)-1;
878 879 880 881 882 883 884

	fs_info = fixup->root->fs_info;
	srcu_index = srcu_read_lock(&fs_info->subvol_srcu);

	local_root = btrfs_read_fs_root_no_name(fs_info, &key);
	if (IS_ERR(local_root)) {
		srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
885
		return PTR_ERR(local_root);
886
	}
887 888 889 890

	key.type = BTRFS_INODE_ITEM_KEY;
	key.objectid = inum;
	key.offset = 0;
891 892
	inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
	srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
893 894 895
	if (IS_ERR(inode))
		return PTR_ERR(inode);

896
	index = offset >> PAGE_SHIFT;
897 898

	page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924
	if (!page) {
		ret = -ENOMEM;
		goto out;
	}

	if (PageUptodate(page)) {
		if (PageDirty(page)) {
			/*
			 * we need to write the data to the defect sector. the
			 * data that was in that sector is not in memory,
			 * because the page was modified. we must not write the
			 * modified page to that sector.
			 *
			 * TODO: what could be done here: wait for the delalloc
			 *       runner to write out that page (might involve
			 *       COW) and see whether the sector is still
			 *       referenced afterwards.
			 *
			 * For the meantime, we'll treat this error
			 * incorrectable, although there is a chance that a
			 * later scrub will find the bad sector again and that
			 * there's no dirty page in memory, then.
			 */
			ret = -EIO;
			goto out;
		}
925
		ret = repair_io_failure(fs_info, inum, offset, PAGE_SIZE,
926
					fixup->logical, page,
927
					offset - page_offset(page),
928 929 930 931 932 933 934 935 936 937
					fixup->mirror_num);
		unlock_page(page);
		corrected = !ret;
	} else {
		/*
		 * we need to get good data first. the general readpage path
		 * will call repair_io_failure for us, we just have to make
		 * sure we read the bad mirror.
		 */
		ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
938
					EXTENT_DAMAGED);
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
		if (ret) {
			/* set_extent_bits should give proper error */
			WARN_ON(ret > 0);
			if (ret > 0)
				ret = -EFAULT;
			goto out;
		}

		ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
						btrfs_get_extent,
						fixup->mirror_num);
		wait_on_page_locked(page);

		corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
						end, EXTENT_DAMAGED, 0, NULL);
		if (!corrected)
			clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
956
						EXTENT_DAMAGED);
957 958 959 960 961
	}

out:
	if (page)
		put_page(page);
962 963

	iput(inode);
964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980

	if (ret < 0)
		return ret;

	if (ret == 0 && corrected) {
		/*
		 * we only need to call readpage for one of the inodes belonging
		 * to this extent. so make iterate_extent_inodes stop
		 */
		return 1;
	}

	return -EIO;
}

static void scrub_fixup_nodatasum(struct btrfs_work *work)
{
981
	struct btrfs_fs_info *fs_info;
982 983
	int ret;
	struct scrub_fixup_nodatasum *fixup;
984
	struct scrub_ctx *sctx;
985 986 987 988 989
	struct btrfs_trans_handle *trans = NULL;
	struct btrfs_path *path;
	int uncorrectable = 0;

	fixup = container_of(work, struct scrub_fixup_nodatasum, work);
990
	sctx = fixup->sctx;
991
	fs_info = fixup->root->fs_info;
992 993 994

	path = btrfs_alloc_path();
	if (!path) {
995 996 997
		spin_lock(&sctx->stat_lock);
		++sctx->stat.malloc_errors;
		spin_unlock(&sctx->stat_lock);
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
		uncorrectable = 1;
		goto out;
	}

	trans = btrfs_join_transaction(fixup->root);
	if (IS_ERR(trans)) {
		uncorrectable = 1;
		goto out;
	}

	/*
	 * the idea is to trigger a regular read through the standard path. we
	 * read a page from the (failed) logical address by specifying the
	 * corresponding copynum of the failed sector. thus, that readpage is
	 * expected to fail.
	 * that is the point where on-the-fly error correction will kick in
	 * (once it's finished) and rewrite the failed sector if a good copy
	 * can be found.
	 */
1017
	ret = iterate_inodes_from_logical(fixup->logical, fs_info, path,
1018
					  scrub_fixup_readpage, fixup, false);
1019 1020 1021 1022 1023 1024
	if (ret < 0) {
		uncorrectable = 1;
		goto out;
	}
	WARN_ON(ret != 1);

1025 1026 1027
	spin_lock(&sctx->stat_lock);
	++sctx->stat.corrected_errors;
	spin_unlock(&sctx->stat_lock);
1028 1029 1030

out:
	if (trans && !IS_ERR(trans))
1031
		btrfs_end_transaction(trans);
1032
	if (uncorrectable) {
1033 1034 1035
		spin_lock(&sctx->stat_lock);
		++sctx->stat.uncorrectable_errors;
		spin_unlock(&sctx->stat_lock);
1036
		btrfs_dev_replace_stats_inc(
1037 1038
			&fs_info->dev_replace.num_uncorrectable_read_errors);
		btrfs_err_rl_in_rcu(fs_info,
1039
		    "unable to fixup (nodatasum) error at logical %llu on dev %s",
1040
			fixup->logical, rcu_str_deref(fixup->dev->name));
1041 1042 1043 1044 1045
	}

	btrfs_free_path(path);
	kfree(fixup);

1046
	scrub_pending_trans_workers_dec(sctx);
1047 1048
}

1049 1050
static inline void scrub_get_recover(struct scrub_recover *recover)
{
1051
	refcount_inc(&recover->refs);
1052 1053
}

1054 1055
static inline void scrub_put_recover(struct btrfs_fs_info *fs_info,
				     struct scrub_recover *recover)
1056
{
1057
	if (refcount_dec_and_test(&recover->refs)) {
1058
		btrfs_bio_counter_dec(fs_info);
1059
		btrfs_put_bbio(recover->bbio);
1060 1061 1062 1063
		kfree(recover);
	}
}

A
Arne Jansen 已提交
1064
/*
1065 1066 1067 1068 1069 1070
 * scrub_handle_errored_block gets called when either verification of the
 * pages failed or the bio failed to read, e.g. with EIO. In the latter
 * case, this function handles all pages in the bio, even though only one
 * may be bad.
 * The goal of this function is to repair the errored block by using the
 * contents of one of the mirrors.
A
Arne Jansen 已提交
1071
 */
1072
static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
A
Arne Jansen 已提交
1073
{
1074
	struct scrub_ctx *sctx = sblock_to_check->sctx;
1075
	struct btrfs_device *dev;
1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
	struct btrfs_fs_info *fs_info;
	u64 logical;
	unsigned int failed_mirror_index;
	unsigned int is_metadata;
	unsigned int have_csum;
	struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
	struct scrub_block *sblock_bad;
	int ret;
	int mirror_index;
	int page_num;
	int success;
1087
	bool full_stripe_locked;
1088
	static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
1089 1090 1091
				      DEFAULT_RATELIMIT_BURST);

	BUG_ON(sblock_to_check->page_count < 1);
1092
	fs_info = sctx->fs_info;
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
	if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
		/*
		 * if we find an error in a super block, we just report it.
		 * They will get written with the next transaction commit
		 * anyway
		 */
		spin_lock(&sctx->stat_lock);
		++sctx->stat.super_errors;
		spin_unlock(&sctx->stat_lock);
		return 0;
	}
1104 1105 1106 1107
	logical = sblock_to_check->pagev[0]->logical;
	BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
	failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
	is_metadata = !(sblock_to_check->pagev[0]->flags &
1108
			BTRFS_EXTENT_FLAG_DATA);
1109 1110
	have_csum = sblock_to_check->pagev[0]->have_csum;
	dev = sblock_to_check->pagev[0]->dev;
1111

1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	/*
	 * For RAID5/6, race can happen for a different device scrub thread.
	 * For data corruption, Parity and Data threads will both try
	 * to recovery the data.
	 * Race can lead to doubly added csum error, or even unrecoverable
	 * error.
	 */
	ret = lock_full_stripe(fs_info, logical, &full_stripe_locked);
	if (ret < 0) {
		spin_lock(&sctx->stat_lock);
		if (ret == -ENOMEM)
			sctx->stat.malloc_errors++;
		sctx->stat.read_errors++;
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
		return ret;
	}

1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158
	/*
	 * read all mirrors one after the other. This includes to
	 * re-read the extent or metadata block that failed (that was
	 * the cause that this fixup code is called) another time,
	 * page by page this time in order to know which pages
	 * caused I/O errors and which ones are good (for all mirrors).
	 * It is the goal to handle the situation when more than one
	 * mirror contains I/O errors, but the errors do not
	 * overlap, i.e. the data can be repaired by selecting the
	 * pages from those mirrors without I/O error on the
	 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
	 * would be that mirror #1 has an I/O error on the first page,
	 * the second page is good, and mirror #2 has an I/O error on
	 * the second page, but the first page is good.
	 * Then the first page of the first mirror can be repaired by
	 * taking the first page of the second mirror, and the
	 * second page of the second mirror can be repaired by
	 * copying the contents of the 2nd page of the 1st mirror.
	 * One more note: if the pages of one mirror contain I/O
	 * errors, the checksum cannot be verified. In order to get
	 * the best data for repairing, the first attempt is to find
	 * a mirror without I/O errors and with a validated checksum.
	 * Only if this is not possible, the pages are picked from
	 * mirrors with I/O errors without considering the checksum.
	 * If the latter is the case, at the end, the checksum of the
	 * repaired area is verified in order to correctly maintain
	 * the statistics.
	 */

1159 1160
	sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
				      sizeof(*sblocks_for_recheck), GFP_NOFS);
1161
	if (!sblocks_for_recheck) {
1162 1163 1164 1165 1166
		spin_lock(&sctx->stat_lock);
		sctx->stat.malloc_errors++;
		sctx->stat.read_errors++;
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
1167
		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1168
		goto out;
A
Arne Jansen 已提交
1169 1170
	}

1171
	/* setup the context, map the logical blocks and alloc the pages */
1172
	ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
1173
	if (ret) {
1174 1175 1176 1177
		spin_lock(&sctx->stat_lock);
		sctx->stat.read_errors++;
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
1178
		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1179 1180 1181 1182
		goto out;
	}
	BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
	sblock_bad = sblocks_for_recheck + failed_mirror_index;
1183

1184
	/* build and submit the bios for the failed mirror, check checksums */
1185
	scrub_recheck_block(fs_info, sblock_bad, 1);
A
Arne Jansen 已提交
1186

1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
	if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
	    sblock_bad->no_io_error_seen) {
		/*
		 * the error disappeared after reading page by page, or
		 * the area was part of a huge bio and other parts of the
		 * bio caused I/O errors, or the block layer merged several
		 * read requests into one and the error is caused by a
		 * different bio (usually one of the two latter cases is
		 * the cause)
		 */
1197 1198
		spin_lock(&sctx->stat_lock);
		sctx->stat.unverified_errors++;
1199
		sblock_to_check->data_corrected = 1;
1200
		spin_unlock(&sctx->stat_lock);
A
Arne Jansen 已提交
1201

1202 1203
		if (sctx->is_dev_replace)
			scrub_write_block_to_dev_replace(sblock_bad);
1204
		goto out;
A
Arne Jansen 已提交
1205 1206
	}

1207
	if (!sblock_bad->no_io_error_seen) {
1208 1209 1210
		spin_lock(&sctx->stat_lock);
		sctx->stat.read_errors++;
		spin_unlock(&sctx->stat_lock);
1211 1212
		if (__ratelimit(&_rs))
			scrub_print_warning("i/o error", sblock_to_check);
1213
		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1214
	} else if (sblock_bad->checksum_error) {
1215 1216 1217
		spin_lock(&sctx->stat_lock);
		sctx->stat.csum_errors++;
		spin_unlock(&sctx->stat_lock);
1218 1219
		if (__ratelimit(&_rs))
			scrub_print_warning("checksum error", sblock_to_check);
1220
		btrfs_dev_stat_inc_and_print(dev,
1221
					     BTRFS_DEV_STAT_CORRUPTION_ERRS);
1222
	} else if (sblock_bad->header_error) {
1223 1224 1225
		spin_lock(&sctx->stat_lock);
		sctx->stat.verify_errors++;
		spin_unlock(&sctx->stat_lock);
1226 1227 1228
		if (__ratelimit(&_rs))
			scrub_print_warning("checksum/header error",
					    sblock_to_check);
1229
		if (sblock_bad->generation_error)
1230
			btrfs_dev_stat_inc_and_print(dev,
1231 1232
				BTRFS_DEV_STAT_GENERATION_ERRS);
		else
1233
			btrfs_dev_stat_inc_and_print(dev,
1234
				BTRFS_DEV_STAT_CORRUPTION_ERRS);
1235
	}
A
Arne Jansen 已提交
1236

1237 1238 1239 1240
	if (sctx->readonly) {
		ASSERT(!sctx->is_dev_replace);
		goto out;
	}
A
Arne Jansen 已提交
1241

1242 1243 1244 1245 1246 1247 1248 1249 1250
	/*
	 * NOTE: Even for nodatasum case, it's still possible that it's a
	 * compressed data extent, thus scrub_fixup_nodatasum(), which write
	 * inode page cache onto disk, could cause serious data corruption.
	 *
	 * So here we could only read from disk, and hope our recovery could
	 * reach disk before the newer write.
	 */
	if (0 && !is_metadata && !have_csum) {
1251
		struct scrub_fixup_nodatasum *fixup_nodatasum;
A
Arne Jansen 已提交
1252

1253 1254
		WARN_ON(sctx->is_dev_replace);

1255 1256
		/*
		 * !is_metadata and !have_csum, this means that the data
1257
		 * might not be COWed, that it might be modified
1258 1259 1260 1261 1262 1263 1264
		 * concurrently. The general strategy to work on the
		 * commit root does not help in the case when COW is not
		 * used.
		 */
		fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
		if (!fixup_nodatasum)
			goto did_not_correct_error;
1265
		fixup_nodatasum->sctx = sctx;
1266
		fixup_nodatasum->dev = dev;
1267 1268 1269
		fixup_nodatasum->logical = logical;
		fixup_nodatasum->root = fs_info->extent_root;
		fixup_nodatasum->mirror_num = failed_mirror_index + 1;
1270
		scrub_pending_trans_workers_inc(sctx);
1271 1272
		btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
				scrub_fixup_nodatasum, NULL, NULL);
1273 1274
		btrfs_queue_work(fs_info->scrub_workers,
				 &fixup_nodatasum->work);
1275
		goto out;
A
Arne Jansen 已提交
1276 1277
	}

1278 1279
	/*
	 * now build and submit the bios for the other mirrors, check
1280 1281
	 * checksums.
	 * First try to pick the mirror which is completely without I/O
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
	 * errors and also does not have a checksum error.
	 * If one is found, and if a checksum is present, the full block
	 * that is known to contain an error is rewritten. Afterwards
	 * the block is known to be corrected.
	 * If a mirror is found which is completely correct, and no
	 * checksum is present, only those pages are rewritten that had
	 * an I/O error in the block to be repaired, since it cannot be
	 * determined, which copy of the other pages is better (and it
	 * could happen otherwise that a correct page would be
	 * overwritten by a bad one).
	 */
1293
	for (mirror_index = 0; ;mirror_index++) {
1294
		struct scrub_block *sblock_other;
1295

1296 1297
		if (mirror_index == failed_mirror_index)
			continue;
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320

		/* raid56's mirror can be more than BTRFS_MAX_MIRRORS */
		if (!scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
			if (mirror_index >= BTRFS_MAX_MIRRORS)
				break;
			if (!sblocks_for_recheck[mirror_index].page_count)
				break;

			sblock_other = sblocks_for_recheck + mirror_index;
		} else {
			struct scrub_recover *r = sblock_bad->pagev[0]->recover;
			int max_allowed = r->bbio->num_stripes -
						r->bbio->num_tgtdevs;

			if (mirror_index >= max_allowed)
				break;
			if (!sblocks_for_recheck[1].page_count)
				break;

			ASSERT(failed_mirror_index == 0);
			sblock_other = sblocks_for_recheck + 1;
			sblock_other->pagev[0]->mirror_num = 1 + mirror_index;
		}
1321 1322

		/* build and submit the bios, check checksums */
1323
		scrub_recheck_block(fs_info, sblock_other, 0);
1324 1325

		if (!sblock_other->header_error &&
1326 1327
		    !sblock_other->checksum_error &&
		    sblock_other->no_io_error_seen) {
1328 1329
			if (sctx->is_dev_replace) {
				scrub_write_block_to_dev_replace(sblock_other);
1330
				goto corrected_error;
1331 1332
			} else {
				ret = scrub_repair_block_from_good_copy(
1333 1334 1335
						sblock_bad, sblock_other);
				if (!ret)
					goto corrected_error;
1336
			}
1337 1338
		}
	}
A
Arne Jansen 已提交
1339

1340 1341
	if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
		goto did_not_correct_error;
1342 1343 1344

	/*
	 * In case of I/O errors in the area that is supposed to be
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
	 * repaired, continue by picking good copies of those pages.
	 * Select the good pages from mirrors to rewrite bad pages from
	 * the area to fix. Afterwards verify the checksum of the block
	 * that is supposed to be repaired. This verification step is
	 * only done for the purpose of statistic counting and for the
	 * final scrub report, whether errors remain.
	 * A perfect algorithm could make use of the checksum and try
	 * all possible combinations of pages from the different mirrors
	 * until the checksum verification succeeds. For example, when
	 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
	 * of mirror #2 is readable but the final checksum test fails,
	 * then the 2nd page of mirror #3 could be tried, whether now
1357
	 * the final checksum succeeds. But this would be a rare
1358 1359 1360 1361 1362 1363 1364 1365
	 * exception and is therefore not implemented. At least it is
	 * avoided that the good copy is overwritten.
	 * A more useful improvement would be to pick the sectors
	 * without I/O error based on sector sizes (512 bytes on legacy
	 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
	 * mirror could be repaired by taking 512 byte of a different
	 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
	 * area are unreadable.
A
Arne Jansen 已提交
1366
	 */
1367
	success = 1;
1368 1369
	for (page_num = 0; page_num < sblock_bad->page_count;
	     page_num++) {
1370
		struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1371
		struct scrub_block *sblock_other = NULL;
1372

1373 1374
		/* skip no-io-error page in scrub */
		if (!page_bad->io_error && !sctx->is_dev_replace)
A
Arne Jansen 已提交
1375
			continue;
1376

1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387
		if (scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
			/*
			 * In case of dev replace, if raid56 rebuild process
			 * didn't work out correct data, then copy the content
			 * in sblock_bad to make sure target device is identical
			 * to source device, instead of writing garbage data in
			 * sblock_for_recheck array to target device.
			 */
			sblock_other = NULL;
		} else if (page_bad->io_error) {
			/* try to find no-io-error page in mirrors */
1388 1389 1390 1391 1392 1393 1394 1395 1396
			for (mirror_index = 0;
			     mirror_index < BTRFS_MAX_MIRRORS &&
			     sblocks_for_recheck[mirror_index].page_count > 0;
			     mirror_index++) {
				if (!sblocks_for_recheck[mirror_index].
				    pagev[page_num]->io_error) {
					sblock_other = sblocks_for_recheck +
						       mirror_index;
					break;
1397 1398
				}
			}
1399 1400
			if (!sblock_other)
				success = 0;
I
Ilya Dryomov 已提交
1401
		}
A
Arne Jansen 已提交
1402

1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416
		if (sctx->is_dev_replace) {
			/*
			 * did not find a mirror to fetch the page
			 * from. scrub_write_page_to_dev_replace()
			 * handles this case (page->io_error), by
			 * filling the block with zeros before
			 * submitting the write request
			 */
			if (!sblock_other)
				sblock_other = sblock_bad;

			if (scrub_write_page_to_dev_replace(sblock_other,
							    page_num) != 0) {
				btrfs_dev_replace_stats_inc(
1417
					&fs_info->dev_replace.num_write_errors);
1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
				success = 0;
			}
		} else if (sblock_other) {
			ret = scrub_repair_page_from_good_copy(sblock_bad,
							       sblock_other,
							       page_num, 0);
			if (0 == ret)
				page_bad->io_error = 0;
			else
				success = 0;
1428
		}
A
Arne Jansen 已提交
1429 1430
	}

1431
	if (success && !sctx->is_dev_replace) {
1432 1433 1434 1435 1436 1437 1438 1439 1440 1441
		if (is_metadata || have_csum) {
			/*
			 * need to verify the checksum now that all
			 * sectors on disk are repaired (the write
			 * request for data to be repaired is on its way).
			 * Just be lazy and use scrub_recheck_block()
			 * which re-reads the data before the checksum
			 * is verified, but most likely the data comes out
			 * of the page cache.
			 */
1442
			scrub_recheck_block(fs_info, sblock_bad, 1);
1443
			if (!sblock_bad->header_error &&
1444 1445 1446 1447 1448 1449 1450
			    !sblock_bad->checksum_error &&
			    sblock_bad->no_io_error_seen)
				goto corrected_error;
			else
				goto did_not_correct_error;
		} else {
corrected_error:
1451 1452
			spin_lock(&sctx->stat_lock);
			sctx->stat.corrected_errors++;
1453
			sblock_to_check->data_corrected = 1;
1454
			spin_unlock(&sctx->stat_lock);
1455 1456
			btrfs_err_rl_in_rcu(fs_info,
				"fixed up error at logical %llu on dev %s",
1457
				logical, rcu_str_deref(dev->name));
A
Arne Jansen 已提交
1458
		}
1459 1460
	} else {
did_not_correct_error:
1461 1462 1463
		spin_lock(&sctx->stat_lock);
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
1464 1465
		btrfs_err_rl_in_rcu(fs_info,
			"unable to fixup (regular) error at logical %llu on dev %s",
1466
			logical, rcu_str_deref(dev->name));
I
Ilya Dryomov 已提交
1467
	}
A
Arne Jansen 已提交
1468

1469 1470 1471 1472 1473 1474
out:
	if (sblocks_for_recheck) {
		for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
		     mirror_index++) {
			struct scrub_block *sblock = sblocks_for_recheck +
						     mirror_index;
1475
			struct scrub_recover *recover;
1476 1477
			int page_index;

1478 1479 1480
			for (page_index = 0; page_index < sblock->page_count;
			     page_index++) {
				sblock->pagev[page_index]->sblock = NULL;
1481 1482
				recover = sblock->pagev[page_index]->recover;
				if (recover) {
1483
					scrub_put_recover(fs_info, recover);
1484 1485 1486
					sblock->pagev[page_index]->recover =
									NULL;
				}
1487 1488
				scrub_page_put(sblock->pagev[page_index]);
			}
1489 1490 1491
		}
		kfree(sblocks_for_recheck);
	}
A
Arne Jansen 已提交
1492

1493 1494 1495
	ret = unlock_full_stripe(fs_info, logical, full_stripe_locked);
	if (ret < 0)
		return ret;
1496 1497
	return 0;
}
A
Arne Jansen 已提交
1498

1499
static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1500
{
Z
Zhao Lei 已提交
1501 1502 1503 1504 1505
	if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
		return 2;
	else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
		return 3;
	else
1506 1507 1508
		return (int)bbio->num_stripes;
}

Z
Zhao Lei 已提交
1509 1510
static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
						 u64 *raid_map,
1511 1512 1513 1514 1515 1516 1517
						 u64 mapped_length,
						 int nstripes, int mirror,
						 int *stripe_index,
						 u64 *stripe_offset)
{
	int i;

1518
	if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
		/* RAID5/6 */
		for (i = 0; i < nstripes; i++) {
			if (raid_map[i] == RAID6_Q_STRIPE ||
			    raid_map[i] == RAID5_P_STRIPE)
				continue;

			if (logical >= raid_map[i] &&
			    logical < raid_map[i] + mapped_length)
				break;
		}

		*stripe_index = i;
		*stripe_offset = logical - raid_map[i];
	} else {
		/* The other RAID type */
		*stripe_index = mirror;
		*stripe_offset = 0;
	}
}

1539
static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1540 1541
				     struct scrub_block *sblocks_for_recheck)
{
1542
	struct scrub_ctx *sctx = original_sblock->sctx;
1543
	struct btrfs_fs_info *fs_info = sctx->fs_info;
1544 1545
	u64 length = original_sblock->page_count * PAGE_SIZE;
	u64 logical = original_sblock->pagev[0]->logical;
1546 1547 1548
	u64 generation = original_sblock->pagev[0]->generation;
	u64 flags = original_sblock->pagev[0]->flags;
	u64 have_csum = original_sblock->pagev[0]->have_csum;
1549 1550 1551 1552 1553 1554
	struct scrub_recover *recover;
	struct btrfs_bio *bbio;
	u64 sublen;
	u64 mapped_length;
	u64 stripe_offset;
	int stripe_index;
1555
	int page_index = 0;
1556
	int mirror_index;
1557
	int nmirrors;
1558 1559 1560
	int ret;

	/*
1561
	 * note: the two members refs and outstanding_pages
1562 1563 1564 1565 1566
	 * are not used (and not set) in the blocks that are used for
	 * the recheck procedure
	 */

	while (length > 0) {
1567 1568 1569
		sublen = min_t(u64, length, PAGE_SIZE);
		mapped_length = sublen;
		bbio = NULL;
A
Arne Jansen 已提交
1570

1571 1572 1573 1574
		/*
		 * with a length of PAGE_SIZE, each returned stripe
		 * represents one mirror
		 */
1575
		btrfs_bio_counter_inc_blocked(fs_info);
1576
		ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
1577
				logical, &mapped_length, &bbio);
1578
		if (ret || !bbio || mapped_length < sublen) {
1579
			btrfs_put_bbio(bbio);
1580
			btrfs_bio_counter_dec(fs_info);
1581 1582
			return -EIO;
		}
A
Arne Jansen 已提交
1583

1584 1585
		recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
		if (!recover) {
1586
			btrfs_put_bbio(bbio);
1587
			btrfs_bio_counter_dec(fs_info);
1588 1589 1590
			return -ENOMEM;
		}

1591
		refcount_set(&recover->refs, 1);
1592 1593 1594
		recover->bbio = bbio;
		recover->map_length = mapped_length;

1595
		BUG_ON(page_index >= SCRUB_MAX_PAGES_PER_BLOCK);
1596

1597
		nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
Z
Zhao Lei 已提交
1598

1599
		for (mirror_index = 0; mirror_index < nmirrors;
1600 1601 1602 1603 1604
		     mirror_index++) {
			struct scrub_block *sblock;
			struct scrub_page *page;

			sblock = sblocks_for_recheck + mirror_index;
1605
			sblock->sctx = sctx;
1606

1607 1608 1609
			page = kzalloc(sizeof(*page), GFP_NOFS);
			if (!page) {
leave_nomem:
1610 1611 1612
				spin_lock(&sctx->stat_lock);
				sctx->stat.malloc_errors++;
				spin_unlock(&sctx->stat_lock);
1613
				scrub_put_recover(fs_info, recover);
1614 1615
				return -ENOMEM;
			}
1616 1617
			scrub_page_get(page);
			sblock->pagev[page_index] = page;
1618 1619 1620
			page->sblock = sblock;
			page->flags = flags;
			page->generation = generation;
1621
			page->logical = logical;
1622 1623 1624 1625 1626
			page->have_csum = have_csum;
			if (have_csum)
				memcpy(page->csum,
				       original_sblock->pagev[0]->csum,
				       sctx->csum_size);
1627

Z
Zhao Lei 已提交
1628 1629 1630
			scrub_stripe_index_and_offset(logical,
						      bbio->map_type,
						      bbio->raid_map,
1631
						      mapped_length,
1632 1633
						      bbio->num_stripes -
						      bbio->num_tgtdevs,
1634 1635 1636 1637 1638 1639 1640
						      mirror_index,
						      &stripe_index,
						      &stripe_offset);
			page->physical = bbio->stripes[stripe_index].physical +
					 stripe_offset;
			page->dev = bbio->stripes[stripe_index].dev;

1641 1642 1643 1644
			BUG_ON(page_index >= original_sblock->page_count);
			page->physical_for_dev_replace =
				original_sblock->pagev[page_index]->
				physical_for_dev_replace;
1645 1646
			/* for missing devices, dev->bdev is NULL */
			page->mirror_num = mirror_index + 1;
1647
			sblock->page_count++;
1648 1649 1650
			page->page = alloc_page(GFP_NOFS);
			if (!page->page)
				goto leave_nomem;
1651 1652 1653

			scrub_get_recover(recover);
			page->recover = recover;
1654
		}
1655
		scrub_put_recover(fs_info, recover);
1656 1657 1658 1659 1660 1661
		length -= sublen;
		logical += sublen;
		page_index++;
	}

	return 0;
I
Ilya Dryomov 已提交
1662 1663
}

1664
static void scrub_bio_wait_endio(struct bio *bio)
1665
{
1666
	complete(bio->bi_private);
1667 1668 1669 1670 1671 1672
}

static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
					struct bio *bio,
					struct scrub_page *page)
{
1673
	DECLARE_COMPLETION_ONSTACK(done);
1674
	int ret;
1675
	int mirror_num;
1676 1677 1678 1679 1680

	bio->bi_iter.bi_sector = page->logical >> 9;
	bio->bi_private = &done;
	bio->bi_end_io = scrub_bio_wait_endio;

1681
	mirror_num = page->sblock->pagev[0]->mirror_num;
1682
	ret = raid56_parity_recover(fs_info, bio, page->recover->bbio,
1683
				    page->recover->map_length,
1684
				    mirror_num, 0);
1685 1686 1687
	if (ret)
		return ret;

1688 1689
	wait_for_completion_io(&done);
	return blk_status_to_errno(bio->bi_status);
1690 1691
}

L
Liu Bo 已提交
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info,
					  struct scrub_block *sblock)
{
	struct scrub_page *first_page = sblock->pagev[0];
	struct bio *bio;
	int page_num;

	/* All pages in sblock belong to the same stripe on the same device. */
	ASSERT(first_page->dev);
	if (!first_page->dev->bdev)
		goto out;

	bio = btrfs_io_bio_alloc(BIO_MAX_PAGES);
	bio_set_dev(bio, first_page->dev->bdev);

	for (page_num = 0; page_num < sblock->page_count; page_num++) {
		struct scrub_page *page = sblock->pagev[page_num];

		WARN_ON(!page->page);
		bio_add_page(bio, page->page, PAGE_SIZE, 0);
	}

	if (scrub_submit_raid56_bio_wait(fs_info, bio, first_page)) {
		bio_put(bio);
		goto out;
	}

	bio_put(bio);

	scrub_recheck_block_checksum(sblock);

	return;
out:
	for (page_num = 0; page_num < sblock->page_count; page_num++)
		sblock->pagev[page_num]->io_error = 1;

	sblock->no_io_error_seen = 0;
}

1731 1732 1733 1734 1735 1736 1737
/*
 * this function will check the on disk data for checksum errors, header
 * errors and read I/O errors. If any I/O errors happen, the exact pages
 * which are errored are marked as being bad. The goal is to enable scrub
 * to take those pages that are not errored from all the mirrors so that
 * the pages that are errored in the just handled mirror can be repaired.
 */
1738
static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1739 1740
				struct scrub_block *sblock,
				int retry_failed_mirror)
I
Ilya Dryomov 已提交
1741
{
1742
	int page_num;
I
Ilya Dryomov 已提交
1743

1744
	sblock->no_io_error_seen = 1;
I
Ilya Dryomov 已提交
1745

L
Liu Bo 已提交
1746 1747 1748 1749
	/* short cut for raid56 */
	if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))
		return scrub_recheck_block_on_raid56(fs_info, sblock);

1750 1751
	for (page_num = 0; page_num < sblock->page_count; page_num++) {
		struct bio *bio;
1752
		struct scrub_page *page = sblock->pagev[page_num];
1753

1754
		if (page->dev->bdev == NULL) {
1755 1756 1757 1758 1759
			page->io_error = 1;
			sblock->no_io_error_seen = 0;
			continue;
		}

1760
		WARN_ON(!page->page);
1761
		bio = btrfs_io_bio_alloc(1);
1762
		bio_set_dev(bio, page->dev->bdev);
1763

1764
		bio_add_page(bio, page->page, PAGE_SIZE, 0);
L
Liu Bo 已提交
1765 1766
		bio->bi_iter.bi_sector = page->physical >> 9;
		bio->bi_opf = REQ_OP_READ;
1767

L
Liu Bo 已提交
1768 1769 1770
		if (btrfsic_submit_bio_wait(bio)) {
			page->io_error = 1;
			sblock->no_io_error_seen = 0;
1771
		}
1772

1773 1774
		bio_put(bio);
	}
I
Ilya Dryomov 已提交
1775

1776
	if (sblock->no_io_error_seen)
1777
		scrub_recheck_block_checksum(sblock);
A
Arne Jansen 已提交
1778 1779
}

M
Miao Xie 已提交
1780 1781 1782 1783 1784 1785
static inline int scrub_check_fsid(u8 fsid[],
				   struct scrub_page *spage)
{
	struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
	int ret;

1786
	ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
M
Miao Xie 已提交
1787 1788 1789
	return !ret;
}

1790
static void scrub_recheck_block_checksum(struct scrub_block *sblock)
A
Arne Jansen 已提交
1791
{
1792 1793 1794
	sblock->header_error = 0;
	sblock->checksum_error = 0;
	sblock->generation_error = 0;
1795

1796 1797 1798 1799
	if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)
		scrub_checksum_data(sblock);
	else
		scrub_checksum_tree_block(sblock);
A
Arne Jansen 已提交
1800 1801
}

1802
static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1803
					     struct scrub_block *sblock_good)
1804 1805 1806
{
	int page_num;
	int ret = 0;
I
Ilya Dryomov 已提交
1807

1808 1809
	for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
		int ret_sub;
I
Ilya Dryomov 已提交
1810

1811 1812
		ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
							   sblock_good,
1813
							   page_num, 1);
1814 1815
		if (ret_sub)
			ret = ret_sub;
A
Arne Jansen 已提交
1816
	}
1817 1818 1819 1820 1821 1822 1823 1824

	return ret;
}

static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
					    struct scrub_block *sblock_good,
					    int page_num, int force_write)
{
1825 1826
	struct scrub_page *page_bad = sblock_bad->pagev[page_num];
	struct scrub_page *page_good = sblock_good->pagev[page_num];
1827
	struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info;
1828

1829 1830
	BUG_ON(page_bad->page == NULL);
	BUG_ON(page_good->page == NULL);
1831 1832 1833 1834 1835
	if (force_write || sblock_bad->header_error ||
	    sblock_bad->checksum_error || page_bad->io_error) {
		struct bio *bio;
		int ret;

1836
		if (!page_bad->dev->bdev) {
1837
			btrfs_warn_rl(fs_info,
J
Jeff Mahoney 已提交
1838
				"scrub_repair_page_from_good_copy(bdev == NULL) is unexpected");
1839 1840 1841
			return -EIO;
		}

1842
		bio = btrfs_io_bio_alloc(1);
1843
		bio_set_dev(bio, page_bad->dev->bdev);
1844
		bio->bi_iter.bi_sector = page_bad->physical >> 9;
M
Mike Christie 已提交
1845
		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1846 1847 1848 1849 1850

		ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
		if (PAGE_SIZE != ret) {
			bio_put(bio);
			return -EIO;
1851
		}
1852

1853
		if (btrfsic_submit_bio_wait(bio)) {
1854 1855
			btrfs_dev_stat_inc_and_print(page_bad->dev,
				BTRFS_DEV_STAT_WRITE_ERRS);
1856
			btrfs_dev_replace_stats_inc(
1857
				&fs_info->dev_replace.num_write_errors);
1858 1859 1860
			bio_put(bio);
			return -EIO;
		}
1861
		bio_put(bio);
A
Arne Jansen 已提交
1862 1863
	}

1864 1865 1866
	return 0;
}

1867 1868
static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
{
1869
	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
1870 1871
	int page_num;

1872 1873 1874 1875 1876 1877 1878
	/*
	 * This block is used for the check of the parity on the source device,
	 * so the data needn't be written into the destination device.
	 */
	if (sblock->sparity)
		return;

1879 1880 1881 1882 1883 1884
	for (page_num = 0; page_num < sblock->page_count; page_num++) {
		int ret;

		ret = scrub_write_page_to_dev_replace(sblock, page_num);
		if (ret)
			btrfs_dev_replace_stats_inc(
1885
				&fs_info->dev_replace.num_write_errors);
1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897
	}
}

static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
					   int page_num)
{
	struct scrub_page *spage = sblock->pagev[page_num];

	BUG_ON(spage->page == NULL);
	if (spage->io_error) {
		void *mapped_buffer = kmap_atomic(spage->page);

1898
		clear_page(mapped_buffer);
1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910
		flush_dcache_page(spage->page);
		kunmap_atomic(mapped_buffer);
	}
	return scrub_add_page_to_wr_bio(sblock->sctx, spage);
}

static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
				    struct scrub_page *spage)
{
	struct scrub_bio *sbio;
	int ret;

1911
	mutex_lock(&sctx->wr_lock);
1912
again:
1913 1914
	if (!sctx->wr_curr_bio) {
		sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio),
1915
					      GFP_KERNEL);
1916 1917
		if (!sctx->wr_curr_bio) {
			mutex_unlock(&sctx->wr_lock);
1918 1919
			return -ENOMEM;
		}
1920 1921
		sctx->wr_curr_bio->sctx = sctx;
		sctx->wr_curr_bio->page_count = 0;
1922
	}
1923
	sbio = sctx->wr_curr_bio;
1924 1925 1926 1927 1928
	if (sbio->page_count == 0) {
		struct bio *bio;

		sbio->physical = spage->physical_for_dev_replace;
		sbio->logical = spage->logical;
1929
		sbio->dev = sctx->wr_tgtdev;
1930 1931
		bio = sbio->bio;
		if (!bio) {
1932
			bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio);
1933 1934 1935 1936 1937
			sbio->bio = bio;
		}

		bio->bi_private = sbio;
		bio->bi_end_io = scrub_wr_bio_end_io;
1938
		bio_set_dev(bio, sbio->dev->bdev);
1939
		bio->bi_iter.bi_sector = sbio->physical >> 9;
M
Mike Christie 已提交
1940
		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1941
		sbio->status = 0;
1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954
	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
		   spage->physical_for_dev_replace ||
		   sbio->logical + sbio->page_count * PAGE_SIZE !=
		   spage->logical) {
		scrub_wr_submit(sctx);
		goto again;
	}

	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
	if (ret != PAGE_SIZE) {
		if (sbio->page_count < 1) {
			bio_put(sbio->bio);
			sbio->bio = NULL;
1955
			mutex_unlock(&sctx->wr_lock);
1956 1957 1958 1959 1960 1961 1962 1963 1964
			return -EIO;
		}
		scrub_wr_submit(sctx);
		goto again;
	}

	sbio->pagev[sbio->page_count] = spage;
	scrub_page_get(spage);
	sbio->page_count++;
1965
	if (sbio->page_count == sctx->pages_per_wr_bio)
1966
		scrub_wr_submit(sctx);
1967
	mutex_unlock(&sctx->wr_lock);
1968 1969 1970 1971 1972 1973 1974 1975

	return 0;
}

static void scrub_wr_submit(struct scrub_ctx *sctx)
{
	struct scrub_bio *sbio;

1976
	if (!sctx->wr_curr_bio)
1977 1978
		return;

1979 1980
	sbio = sctx->wr_curr_bio;
	sctx->wr_curr_bio = NULL;
1981
	WARN_ON(!sbio->bio->bi_disk);
1982 1983 1984 1985 1986
	scrub_pending_bio_inc(sctx);
	/* process all writes in a single worker thread. Then the block layer
	 * orders the requests before sending them to the driver which
	 * doubled the write performance on spinning disks when measured
	 * with Linux 3.5 */
1987
	btrfsic_submit_bio(sbio->bio);
1988 1989
}

1990
static void scrub_wr_bio_end_io(struct bio *bio)
1991 1992
{
	struct scrub_bio *sbio = bio->bi_private;
1993
	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
1994

1995
	sbio->status = bio->bi_status;
1996 1997
	sbio->bio = bio;

1998 1999
	btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
			 scrub_wr_bio_end_io_worker, NULL, NULL);
2000
	btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
2001 2002 2003 2004 2005 2006 2007 2008 2009
}

static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
{
	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
	struct scrub_ctx *sctx = sbio->sctx;
	int i;

	WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
2010
	if (sbio->status) {
2011
		struct btrfs_dev_replace *dev_replace =
2012
			&sbio->sctx->fs_info->dev_replace;
2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031

		for (i = 0; i < sbio->page_count; i++) {
			struct scrub_page *spage = sbio->pagev[i];

			spage->io_error = 1;
			btrfs_dev_replace_stats_inc(&dev_replace->
						    num_write_errors);
		}
	}

	for (i = 0; i < sbio->page_count; i++)
		scrub_page_put(sbio->pagev[i]);

	bio_put(sbio->bio);
	kfree(sbio);
	scrub_pending_bio_dec(sctx);
}

static int scrub_checksum(struct scrub_block *sblock)
2032 2033 2034 2035
{
	u64 flags;
	int ret;

2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047
	/*
	 * No need to initialize these stats currently,
	 * because this function only use return value
	 * instead of these stats value.
	 *
	 * Todo:
	 * always use stats
	 */
	sblock->header_error = 0;
	sblock->generation_error = 0;
	sblock->checksum_error = 0;

2048 2049
	WARN_ON(sblock->page_count < 1);
	flags = sblock->pagev[0]->flags;
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060
	ret = 0;
	if (flags & BTRFS_EXTENT_FLAG_DATA)
		ret = scrub_checksum_data(sblock);
	else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
		ret = scrub_checksum_tree_block(sblock);
	else if (flags & BTRFS_EXTENT_FLAG_SUPER)
		(void)scrub_checksum_super(sblock);
	else
		WARN_ON(1);
	if (ret)
		scrub_handle_errored_block(sblock);
2061 2062

	return ret;
A
Arne Jansen 已提交
2063 2064
}

2065
static int scrub_checksum_data(struct scrub_block *sblock)
A
Arne Jansen 已提交
2066
{
2067
	struct scrub_ctx *sctx = sblock->sctx;
A
Arne Jansen 已提交
2068
	u8 csum[BTRFS_CSUM_SIZE];
2069 2070 2071
	u8 *on_disk_csum;
	struct page *page;
	void *buffer;
A
Arne Jansen 已提交
2072
	u32 crc = ~(u32)0;
2073 2074
	u64 len;
	int index;
A
Arne Jansen 已提交
2075

2076
	BUG_ON(sblock->page_count < 1);
2077
	if (!sblock->pagev[0]->have_csum)
A
Arne Jansen 已提交
2078 2079
		return 0;

2080 2081
	on_disk_csum = sblock->pagev[0]->csum;
	page = sblock->pagev[0]->page;
2082
	buffer = kmap_atomic(page);
2083

2084
	len = sctx->fs_info->sectorsize;
2085 2086 2087 2088
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, PAGE_SIZE);

2089
		crc = btrfs_csum_data(buffer, crc, l);
2090
		kunmap_atomic(buffer);
2091 2092 2093 2094 2095
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
2096 2097
		BUG_ON(!sblock->pagev[index]->page);
		page = sblock->pagev[index]->page;
2098
		buffer = kmap_atomic(page);
2099 2100
	}

A
Arne Jansen 已提交
2101
	btrfs_csum_final(crc, csum);
2102
	if (memcmp(csum, on_disk_csum, sctx->csum_size))
2103
		sblock->checksum_error = 1;
A
Arne Jansen 已提交
2104

2105
	return sblock->checksum_error;
A
Arne Jansen 已提交
2106 2107
}

2108
static int scrub_checksum_tree_block(struct scrub_block *sblock)
A
Arne Jansen 已提交
2109
{
2110
	struct scrub_ctx *sctx = sblock->sctx;
A
Arne Jansen 已提交
2111
	struct btrfs_header *h;
2112
	struct btrfs_fs_info *fs_info = sctx->fs_info;
2113 2114 2115 2116 2117 2118
	u8 calculated_csum[BTRFS_CSUM_SIZE];
	u8 on_disk_csum[BTRFS_CSUM_SIZE];
	struct page *page;
	void *mapped_buffer;
	u64 mapped_size;
	void *p;
A
Arne Jansen 已提交
2119
	u32 crc = ~(u32)0;
2120 2121 2122 2123
	u64 len;
	int index;

	BUG_ON(sblock->page_count < 1);
2124
	page = sblock->pagev[0]->page;
2125
	mapped_buffer = kmap_atomic(page);
2126
	h = (struct btrfs_header *)mapped_buffer;
2127
	memcpy(on_disk_csum, h->csum, sctx->csum_size);
A
Arne Jansen 已提交
2128 2129 2130 2131 2132 2133

	/*
	 * we don't use the getter functions here, as we
	 * a) don't have an extent buffer and
	 * b) the page is already kmapped
	 */
2134
	if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
2135
		sblock->header_error = 1;
A
Arne Jansen 已提交
2136

2137 2138 2139 2140
	if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) {
		sblock->header_error = 1;
		sblock->generation_error = 1;
	}
A
Arne Jansen 已提交
2141

M
Miao Xie 已提交
2142
	if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
2143
		sblock->header_error = 1;
A
Arne Jansen 已提交
2144 2145 2146

	if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
		   BTRFS_UUID_SIZE))
2147
		sblock->header_error = 1;
A
Arne Jansen 已提交
2148

2149
	len = sctx->fs_info->nodesize - BTRFS_CSUM_SIZE;
2150 2151 2152 2153 2154 2155
	mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
	p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, mapped_size);

2156
		crc = btrfs_csum_data(p, crc, l);
2157
		kunmap_atomic(mapped_buffer);
2158 2159 2160 2161 2162
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
2163 2164
		BUG_ON(!sblock->pagev[index]->page);
		page = sblock->pagev[index]->page;
2165
		mapped_buffer = kmap_atomic(page);
2166 2167 2168 2169 2170
		mapped_size = PAGE_SIZE;
		p = mapped_buffer;
	}

	btrfs_csum_final(crc, calculated_csum);
2171
	if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2172
		sblock->checksum_error = 1;
A
Arne Jansen 已提交
2173

2174
	return sblock->header_error || sblock->checksum_error;
A
Arne Jansen 已提交
2175 2176
}

2177
static int scrub_checksum_super(struct scrub_block *sblock)
A
Arne Jansen 已提交
2178 2179
{
	struct btrfs_super_block *s;
2180
	struct scrub_ctx *sctx = sblock->sctx;
2181 2182 2183 2184 2185 2186
	u8 calculated_csum[BTRFS_CSUM_SIZE];
	u8 on_disk_csum[BTRFS_CSUM_SIZE];
	struct page *page;
	void *mapped_buffer;
	u64 mapped_size;
	void *p;
A
Arne Jansen 已提交
2187
	u32 crc = ~(u32)0;
2188 2189
	int fail_gen = 0;
	int fail_cor = 0;
2190 2191
	u64 len;
	int index;
A
Arne Jansen 已提交
2192

2193
	BUG_ON(sblock->page_count < 1);
2194
	page = sblock->pagev[0]->page;
2195
	mapped_buffer = kmap_atomic(page);
2196
	s = (struct btrfs_super_block *)mapped_buffer;
2197
	memcpy(on_disk_csum, s->csum, sctx->csum_size);
A
Arne Jansen 已提交
2198

2199
	if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
2200
		++fail_cor;
A
Arne Jansen 已提交
2201

2202
	if (sblock->pagev[0]->generation != btrfs_super_generation(s))
2203
		++fail_gen;
A
Arne Jansen 已提交
2204

M
Miao Xie 已提交
2205
	if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
2206
		++fail_cor;
A
Arne Jansen 已提交
2207

2208 2209 2210 2211 2212 2213 2214
	len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
	mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
	p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, mapped_size);

2215
		crc = btrfs_csum_data(p, crc, l);
2216
		kunmap_atomic(mapped_buffer);
2217 2218 2219 2220 2221
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
2222 2223
		BUG_ON(!sblock->pagev[index]->page);
		page = sblock->pagev[index]->page;
2224
		mapped_buffer = kmap_atomic(page);
2225 2226 2227 2228 2229
		mapped_size = PAGE_SIZE;
		p = mapped_buffer;
	}

	btrfs_csum_final(crc, calculated_csum);
2230
	if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2231
		++fail_cor;
A
Arne Jansen 已提交
2232

2233
	if (fail_cor + fail_gen) {
A
Arne Jansen 已提交
2234 2235 2236 2237 2238
		/*
		 * if we find an error in a super block, we just report it.
		 * They will get written with the next transaction commit
		 * anyway
		 */
2239 2240 2241
		spin_lock(&sctx->stat_lock);
		++sctx->stat.super_errors;
		spin_unlock(&sctx->stat_lock);
2242
		if (fail_cor)
2243
			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2244 2245
				BTRFS_DEV_STAT_CORRUPTION_ERRS);
		else
2246
			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2247
				BTRFS_DEV_STAT_GENERATION_ERRS);
A
Arne Jansen 已提交
2248 2249
	}

2250
	return fail_cor + fail_gen;
A
Arne Jansen 已提交
2251 2252
}

2253 2254
static void scrub_block_get(struct scrub_block *sblock)
{
2255
	refcount_inc(&sblock->refs);
2256 2257 2258 2259
}

static void scrub_block_put(struct scrub_block *sblock)
{
2260
	if (refcount_dec_and_test(&sblock->refs)) {
2261 2262
		int i;

2263 2264 2265
		if (sblock->sparity)
			scrub_parity_put(sblock->sparity);

2266
		for (i = 0; i < sblock->page_count; i++)
2267
			scrub_page_put(sblock->pagev[i]);
2268 2269 2270 2271
		kfree(sblock);
	}
}

2272 2273
static void scrub_page_get(struct scrub_page *spage)
{
2274
	atomic_inc(&spage->refs);
2275 2276 2277 2278
}

static void scrub_page_put(struct scrub_page *spage)
{
2279
	if (atomic_dec_and_test(&spage->refs)) {
2280 2281 2282 2283 2284 2285
		if (spage->page)
			__free_page(spage->page);
		kfree(spage);
	}
}

2286
static void scrub_submit(struct scrub_ctx *sctx)
A
Arne Jansen 已提交
2287 2288 2289
{
	struct scrub_bio *sbio;

2290
	if (sctx->curr == -1)
S
Stefan Behrens 已提交
2291
		return;
A
Arne Jansen 已提交
2292

2293 2294
	sbio = sctx->bios[sctx->curr];
	sctx->curr = -1;
2295
	scrub_pending_bio_inc(sctx);
2296
	btrfsic_submit_bio(sbio->bio);
A
Arne Jansen 已提交
2297 2298
}

2299 2300
static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
				    struct scrub_page *spage)
A
Arne Jansen 已提交
2301
{
2302
	struct scrub_block *sblock = spage->sblock;
A
Arne Jansen 已提交
2303
	struct scrub_bio *sbio;
2304
	int ret;
A
Arne Jansen 已提交
2305 2306 2307 2308 2309

again:
	/*
	 * grab a fresh bio or wait for one to become available
	 */
2310 2311 2312 2313 2314 2315 2316 2317
	while (sctx->curr == -1) {
		spin_lock(&sctx->list_lock);
		sctx->curr = sctx->first_free;
		if (sctx->curr != -1) {
			sctx->first_free = sctx->bios[sctx->curr]->next_free;
			sctx->bios[sctx->curr]->next_free = -1;
			sctx->bios[sctx->curr]->page_count = 0;
			spin_unlock(&sctx->list_lock);
A
Arne Jansen 已提交
2318
		} else {
2319 2320
			spin_unlock(&sctx->list_lock);
			wait_event(sctx->list_wait, sctx->first_free != -1);
A
Arne Jansen 已提交
2321 2322
		}
	}
2323
	sbio = sctx->bios[sctx->curr];
2324
	if (sbio->page_count == 0) {
2325 2326
		struct bio *bio;

2327 2328
		sbio->physical = spage->physical;
		sbio->logical = spage->logical;
2329
		sbio->dev = spage->dev;
2330 2331
		bio = sbio->bio;
		if (!bio) {
2332
			bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio);
2333 2334
			sbio->bio = bio;
		}
2335 2336 2337

		bio->bi_private = sbio;
		bio->bi_end_io = scrub_bio_end_io;
2338
		bio_set_dev(bio, sbio->dev->bdev);
2339
		bio->bi_iter.bi_sector = sbio->physical >> 9;
M
Mike Christie 已提交
2340
		bio_set_op_attrs(bio, REQ_OP_READ, 0);
2341
		sbio->status = 0;
2342 2343 2344
	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
		   spage->physical ||
		   sbio->logical + sbio->page_count * PAGE_SIZE !=
2345 2346
		   spage->logical ||
		   sbio->dev != spage->dev) {
2347
		scrub_submit(sctx);
A
Arne Jansen 已提交
2348 2349
		goto again;
	}
2350

2351 2352 2353 2354 2355 2356 2357 2358
	sbio->pagev[sbio->page_count] = spage;
	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
	if (ret != PAGE_SIZE) {
		if (sbio->page_count < 1) {
			bio_put(sbio->bio);
			sbio->bio = NULL;
			return -EIO;
		}
2359
		scrub_submit(sctx);
2360 2361 2362
		goto again;
	}

2363
	scrub_block_get(sblock); /* one for the page added to the bio */
2364 2365
	atomic_inc(&sblock->outstanding_pages);
	sbio->page_count++;
2366
	if (sbio->page_count == sctx->pages_per_rd_bio)
2367
		scrub_submit(sctx);
2368 2369 2370 2371

	return 0;
}

2372
static void scrub_missing_raid56_end_io(struct bio *bio)
2373 2374
{
	struct scrub_block *sblock = bio->bi_private;
2375
	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
2376

2377
	if (bio->bi_status)
2378 2379
		sblock->no_io_error_seen = 0;

2380 2381
	bio_put(bio);

2382 2383 2384 2385 2386 2387 2388
	btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
}

static void scrub_missing_raid56_worker(struct btrfs_work *work)
{
	struct scrub_block *sblock = container_of(work, struct scrub_block, work);
	struct scrub_ctx *sctx = sblock->sctx;
2389
	struct btrfs_fs_info *fs_info = sctx->fs_info;
2390 2391 2392 2393 2394 2395
	u64 logical;
	struct btrfs_device *dev;

	logical = sblock->pagev[0]->logical;
	dev = sblock->pagev[0]->dev;

2396
	if (sblock->no_io_error_seen)
2397
		scrub_recheck_block_checksum(sblock);
2398 2399 2400 2401 2402

	if (!sblock->no_io_error_seen) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.read_errors++;
		spin_unlock(&sctx->stat_lock);
2403
		btrfs_err_rl_in_rcu(fs_info,
2404
			"IO error rebuilding logical %llu for dev %s",
2405 2406 2407 2408 2409
			logical, rcu_str_deref(dev->name));
	} else if (sblock->header_error || sblock->checksum_error) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.uncorrectable_errors++;
		spin_unlock(&sctx->stat_lock);
2410
		btrfs_err_rl_in_rcu(fs_info,
2411
			"failed to rebuild valid logical %llu for dev %s",
2412 2413 2414 2415 2416 2417 2418
			logical, rcu_str_deref(dev->name));
	} else {
		scrub_write_block_to_dev_replace(sblock);
	}

	scrub_block_put(sblock);

2419
	if (sctx->is_dev_replace && sctx->flush_all_writes) {
2420
		mutex_lock(&sctx->wr_lock);
2421
		scrub_wr_submit(sctx);
2422
		mutex_unlock(&sctx->wr_lock);
2423 2424 2425 2426 2427 2428 2429 2430
	}

	scrub_pending_bio_dec(sctx);
}

static void scrub_missing_raid56_pages(struct scrub_block *sblock)
{
	struct scrub_ctx *sctx = sblock->sctx;
2431
	struct btrfs_fs_info *fs_info = sctx->fs_info;
2432 2433
	u64 length = sblock->page_count * PAGE_SIZE;
	u64 logical = sblock->pagev[0]->logical;
2434
	struct btrfs_bio *bbio = NULL;
2435 2436 2437 2438 2439
	struct bio *bio;
	struct btrfs_raid_bio *rbio;
	int ret;
	int i;

2440
	btrfs_bio_counter_inc_blocked(fs_info);
2441
	ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical,
2442
			&length, &bbio);
2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456
	if (ret || !bbio || !bbio->raid_map)
		goto bbio_out;

	if (WARN_ON(!sctx->is_dev_replace ||
		    !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
		/*
		 * We shouldn't be scrubbing a missing device. Even for dev
		 * replace, we should only get here for RAID 5/6. We either
		 * managed to mount something with no mirrors remaining or
		 * there's a bug in scrub_remap_extent()/btrfs_map_block().
		 */
		goto bbio_out;
	}

2457
	bio = btrfs_io_bio_alloc(0);
2458 2459 2460 2461
	bio->bi_iter.bi_sector = logical >> 9;
	bio->bi_private = sblock;
	bio->bi_end_io = scrub_missing_raid56_end_io;

2462
	rbio = raid56_alloc_missing_rbio(fs_info, bio, bbio, length);
2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481
	if (!rbio)
		goto rbio_out;

	for (i = 0; i < sblock->page_count; i++) {
		struct scrub_page *spage = sblock->pagev[i];

		raid56_add_scrub_pages(rbio, spage->page, spage->logical);
	}

	btrfs_init_work(&sblock->work, btrfs_scrub_helper,
			scrub_missing_raid56_worker, NULL, NULL);
	scrub_block_get(sblock);
	scrub_pending_bio_inc(sctx);
	raid56_submit_missing_rbio(rbio);
	return;

rbio_out:
	bio_put(bio);
bbio_out:
2482
	btrfs_bio_counter_dec(fs_info);
2483 2484 2485 2486 2487 2488
	btrfs_put_bbio(bbio);
	spin_lock(&sctx->stat_lock);
	sctx->stat.malloc_errors++;
	spin_unlock(&sctx->stat_lock);
}

2489
static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2490
		       u64 physical, struct btrfs_device *dev, u64 flags,
2491 2492
		       u64 gen, int mirror_num, u8 *csum, int force,
		       u64 physical_for_dev_replace)
2493 2494 2495 2496
{
	struct scrub_block *sblock;
	int index;

2497
	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2498
	if (!sblock) {
2499 2500 2501
		spin_lock(&sctx->stat_lock);
		sctx->stat.malloc_errors++;
		spin_unlock(&sctx->stat_lock);
2502
		return -ENOMEM;
A
Arne Jansen 已提交
2503
	}
2504

2505 2506
	/* one ref inside this function, plus one for each page added to
	 * a bio later on */
2507
	refcount_set(&sblock->refs, 1);
2508
	sblock->sctx = sctx;
2509 2510 2511
	sblock->no_io_error_seen = 1;

	for (index = 0; len > 0; index++) {
2512
		struct scrub_page *spage;
2513 2514
		u64 l = min_t(u64, len, PAGE_SIZE);

2515
		spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2516 2517
		if (!spage) {
leave_nomem:
2518 2519 2520
			spin_lock(&sctx->stat_lock);
			sctx->stat.malloc_errors++;
			spin_unlock(&sctx->stat_lock);
2521
			scrub_block_put(sblock);
2522 2523
			return -ENOMEM;
		}
2524 2525 2526
		BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
		scrub_page_get(spage);
		sblock->pagev[index] = spage;
2527
		spage->sblock = sblock;
2528
		spage->dev = dev;
2529 2530 2531 2532
		spage->flags = flags;
		spage->generation = gen;
		spage->logical = logical;
		spage->physical = physical;
2533
		spage->physical_for_dev_replace = physical_for_dev_replace;
2534 2535 2536
		spage->mirror_num = mirror_num;
		if (csum) {
			spage->have_csum = 1;
2537
			memcpy(spage->csum, csum, sctx->csum_size);
2538 2539 2540 2541
		} else {
			spage->have_csum = 0;
		}
		sblock->page_count++;
2542
		spage->page = alloc_page(GFP_KERNEL);
2543 2544
		if (!spage->page)
			goto leave_nomem;
2545 2546 2547
		len -= l;
		logical += l;
		physical += l;
2548
		physical_for_dev_replace += l;
2549 2550
	}

2551
	WARN_ON(sblock->page_count == 0);
2552
	if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2553 2554 2555 2556 2557 2558 2559 2560 2561
		/*
		 * This case should only be hit for RAID 5/6 device replace. See
		 * the comment in scrub_missing_raid56_pages() for details.
		 */
		scrub_missing_raid56_pages(sblock);
	} else {
		for (index = 0; index < sblock->page_count; index++) {
			struct scrub_page *spage = sblock->pagev[index];
			int ret;
2562

2563 2564 2565 2566 2567
			ret = scrub_add_page_to_rd_bio(sctx, spage);
			if (ret) {
				scrub_block_put(sblock);
				return ret;
			}
2568
		}
A
Arne Jansen 已提交
2569

2570 2571 2572
		if (force)
			scrub_submit(sctx);
	}
A
Arne Jansen 已提交
2573

2574 2575
	/* last one frees, either here or in bio completion for last page */
	scrub_block_put(sblock);
A
Arne Jansen 已提交
2576 2577 2578
	return 0;
}

2579
static void scrub_bio_end_io(struct bio *bio)
2580 2581
{
	struct scrub_bio *sbio = bio->bi_private;
2582
	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
2583

2584
	sbio->status = bio->bi_status;
2585 2586
	sbio->bio = bio;

2587
	btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2588 2589 2590 2591 2592
}

static void scrub_bio_end_io_worker(struct btrfs_work *work)
{
	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2593
	struct scrub_ctx *sctx = sbio->sctx;
2594 2595
	int i;

2596
	BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2597
	if (sbio->status) {
2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
		for (i = 0; i < sbio->page_count; i++) {
			struct scrub_page *spage = sbio->pagev[i];

			spage->io_error = 1;
			spage->sblock->no_io_error_seen = 0;
		}
	}

	/* now complete the scrub_block items that have all pages completed */
	for (i = 0; i < sbio->page_count; i++) {
		struct scrub_page *spage = sbio->pagev[i];
		struct scrub_block *sblock = spage->sblock;

		if (atomic_dec_and_test(&sblock->outstanding_pages))
			scrub_block_complete(sblock);
		scrub_block_put(sblock);
	}

	bio_put(sbio->bio);
	sbio->bio = NULL;
2618 2619 2620 2621
	spin_lock(&sctx->list_lock);
	sbio->next_free = sctx->first_free;
	sctx->first_free = sbio->index;
	spin_unlock(&sctx->list_lock);
2622

2623
	if (sctx->is_dev_replace && sctx->flush_all_writes) {
2624
		mutex_lock(&sctx->wr_lock);
2625
		scrub_wr_submit(sctx);
2626
		mutex_unlock(&sctx->wr_lock);
2627 2628
	}

2629
	scrub_pending_bio_dec(sctx);
2630 2631
}

2632 2633 2634 2635
static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
				       unsigned long *bitmap,
				       u64 start, u64 len)
{
2636
	u64 offset;
2637 2638
	u64 nsectors64;
	u32 nsectors;
2639
	int sectorsize = sparity->sctx->fs_info->sectorsize;
2640 2641 2642 2643 2644 2645 2646

	if (len >= sparity->stripe_len) {
		bitmap_set(bitmap, 0, sparity->nsectors);
		return;
	}

	start -= sparity->logic_start;
2647 2648
	start = div64_u64_rem(start, sparity->stripe_len, &offset);
	offset = div_u64(offset, sectorsize);
2649 2650 2651 2652
	nsectors64 = div_u64(len, sectorsize);

	ASSERT(nsectors64 < UINT_MAX);
	nsectors = (u32)nsectors64;
2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674

	if (offset + nsectors <= sparity->nsectors) {
		bitmap_set(bitmap, offset, nsectors);
		return;
	}

	bitmap_set(bitmap, offset, sparity->nsectors - offset);
	bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
}

static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
						   u64 start, u64 len)
{
	__scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
}

static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
						  u64 start, u64 len)
{
	__scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
}

2675 2676
static void scrub_block_complete(struct scrub_block *sblock)
{
2677 2678
	int corrupted = 0;

2679
	if (!sblock->no_io_error_seen) {
2680
		corrupted = 1;
2681
		scrub_handle_errored_block(sblock);
2682 2683 2684 2685 2686 2687
	} else {
		/*
		 * if has checksum error, write via repair mechanism in
		 * dev replace case, otherwise write here in dev replace
		 * case.
		 */
2688 2689
		corrupted = scrub_checksum(sblock);
		if (!corrupted && sblock->sctx->is_dev_replace)
2690 2691
			scrub_write_block_to_dev_replace(sblock);
	}
2692 2693 2694 2695 2696 2697 2698 2699 2700

	if (sblock->sparity && corrupted && !sblock->data_corrected) {
		u64 start = sblock->pagev[0]->logical;
		u64 end = sblock->pagev[sblock->page_count - 1]->logical +
			  PAGE_SIZE;

		scrub_parity_mark_sectors_error(sblock->sparity,
						start, end - start);
	}
2701 2702
}

2703
static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum)
A
Arne Jansen 已提交
2704 2705
{
	struct btrfs_ordered_sum *sum = NULL;
2706
	unsigned long index;
A
Arne Jansen 已提交
2707 2708
	unsigned long num_sectors;

2709 2710
	while (!list_empty(&sctx->csum_list)) {
		sum = list_first_entry(&sctx->csum_list,
A
Arne Jansen 已提交
2711 2712 2713 2714 2715 2716
				       struct btrfs_ordered_sum, list);
		if (sum->bytenr > logical)
			return 0;
		if (sum->bytenr + sum->len > logical)
			break;

2717
		++sctx->stat.csum_discards;
A
Arne Jansen 已提交
2718 2719 2720 2721 2722 2723 2724
		list_del(&sum->list);
		kfree(sum);
		sum = NULL;
	}
	if (!sum)
		return 0;

2725 2726 2727
	index = div_u64(logical - sum->bytenr, sctx->fs_info->sectorsize);
	ASSERT(index < UINT_MAX);

2728
	num_sectors = sum->len / sctx->fs_info->sectorsize;
2729 2730
	memcpy(csum, sum->sums + index, sctx->csum_size);
	if (index == num_sectors - 1) {
A
Arne Jansen 已提交
2731 2732 2733
		list_del(&sum->list);
		kfree(sum);
	}
2734
	return 1;
A
Arne Jansen 已提交
2735 2736 2737
}

/* scrub extent tries to collect up to 64 kB for each bio */
L
Liu Bo 已提交
2738 2739
static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map,
			u64 logical, u64 len,
2740
			u64 physical, struct btrfs_device *dev, u64 flags,
2741
			u64 gen, int mirror_num, u64 physical_for_dev_replace)
A
Arne Jansen 已提交
2742 2743 2744
{
	int ret;
	u8 csum[BTRFS_CSUM_SIZE];
2745 2746 2747
	u32 blocksize;

	if (flags & BTRFS_EXTENT_FLAG_DATA) {
L
Liu Bo 已提交
2748 2749 2750 2751
		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
			blocksize = map->stripe_len;
		else
			blocksize = sctx->fs_info->sectorsize;
2752 2753 2754 2755
		spin_lock(&sctx->stat_lock);
		sctx->stat.data_extents_scrubbed++;
		sctx->stat.data_bytes_scrubbed += len;
		spin_unlock(&sctx->stat_lock);
2756
	} else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
L
Liu Bo 已提交
2757 2758 2759 2760
		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
			blocksize = map->stripe_len;
		else
			blocksize = sctx->fs_info->nodesize;
2761 2762 2763 2764
		spin_lock(&sctx->stat_lock);
		sctx->stat.tree_extents_scrubbed++;
		sctx->stat.tree_bytes_scrubbed += len;
		spin_unlock(&sctx->stat_lock);
2765
	} else {
2766
		blocksize = sctx->fs_info->sectorsize;
2767
		WARN_ON(1);
2768
	}
A
Arne Jansen 已提交
2769 2770

	while (len) {
2771
		u64 l = min_t(u64, len, blocksize);
A
Arne Jansen 已提交
2772 2773 2774 2775
		int have_csum = 0;

		if (flags & BTRFS_EXTENT_FLAG_DATA) {
			/* push csums to sbio */
2776
			have_csum = scrub_find_csum(sctx, logical, csum);
A
Arne Jansen 已提交
2777
			if (have_csum == 0)
2778
				++sctx->stat.no_csum;
A
Arne Jansen 已提交
2779
		}
2780
		ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2781 2782
				  mirror_num, have_csum ? csum : NULL, 0,
				  physical_for_dev_replace);
A
Arne Jansen 已提交
2783 2784 2785 2786 2787
		if (ret)
			return ret;
		len -= l;
		logical += l;
		physical += l;
2788
		physical_for_dev_replace += l;
A
Arne Jansen 已提交
2789 2790 2791 2792
	}
	return 0;
}

2793 2794 2795 2796 2797 2798 2799 2800 2801
static int scrub_pages_for_parity(struct scrub_parity *sparity,
				  u64 logical, u64 len,
				  u64 physical, struct btrfs_device *dev,
				  u64 flags, u64 gen, int mirror_num, u8 *csum)
{
	struct scrub_ctx *sctx = sparity->sctx;
	struct scrub_block *sblock;
	int index;

2802
	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2803 2804 2805 2806 2807 2808 2809 2810 2811
	if (!sblock) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.malloc_errors++;
		spin_unlock(&sctx->stat_lock);
		return -ENOMEM;
	}

	/* one ref inside this function, plus one for each page added to
	 * a bio later on */
2812
	refcount_set(&sblock->refs, 1);
2813 2814 2815 2816 2817 2818 2819 2820 2821
	sblock->sctx = sctx;
	sblock->no_io_error_seen = 1;
	sblock->sparity = sparity;
	scrub_parity_get(sparity);

	for (index = 0; len > 0; index++) {
		struct scrub_page *spage;
		u64 l = min_t(u64, len, PAGE_SIZE);

2822
		spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851
		if (!spage) {
leave_nomem:
			spin_lock(&sctx->stat_lock);
			sctx->stat.malloc_errors++;
			spin_unlock(&sctx->stat_lock);
			scrub_block_put(sblock);
			return -ENOMEM;
		}
		BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
		/* For scrub block */
		scrub_page_get(spage);
		sblock->pagev[index] = spage;
		/* For scrub parity */
		scrub_page_get(spage);
		list_add_tail(&spage->list, &sparity->spages);
		spage->sblock = sblock;
		spage->dev = dev;
		spage->flags = flags;
		spage->generation = gen;
		spage->logical = logical;
		spage->physical = physical;
		spage->mirror_num = mirror_num;
		if (csum) {
			spage->have_csum = 1;
			memcpy(spage->csum, csum, sctx->csum_size);
		} else {
			spage->have_csum = 0;
		}
		sblock->page_count++;
2852
		spage->page = alloc_page(GFP_KERNEL);
2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886
		if (!spage->page)
			goto leave_nomem;
		len -= l;
		logical += l;
		physical += l;
	}

	WARN_ON(sblock->page_count == 0);
	for (index = 0; index < sblock->page_count; index++) {
		struct scrub_page *spage = sblock->pagev[index];
		int ret;

		ret = scrub_add_page_to_rd_bio(sctx, spage);
		if (ret) {
			scrub_block_put(sblock);
			return ret;
		}
	}

	/* last one frees, either here or in bio completion for last page */
	scrub_block_put(sblock);
	return 0;
}

static int scrub_extent_for_parity(struct scrub_parity *sparity,
				   u64 logical, u64 len,
				   u64 physical, struct btrfs_device *dev,
				   u64 flags, u64 gen, int mirror_num)
{
	struct scrub_ctx *sctx = sparity->sctx;
	int ret;
	u8 csum[BTRFS_CSUM_SIZE];
	u32 blocksize;

2887
	if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2888 2889 2890 2891
		scrub_parity_mark_sectors_error(sparity, logical, len);
		return 0;
	}

2892
	if (flags & BTRFS_EXTENT_FLAG_DATA) {
L
Liu Bo 已提交
2893
		blocksize = sparity->stripe_len;
2894
	} else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
L
Liu Bo 已提交
2895
		blocksize = sparity->stripe_len;
2896
	} else {
2897
		blocksize = sctx->fs_info->sectorsize;
2898 2899 2900 2901 2902 2903 2904 2905 2906
		WARN_ON(1);
	}

	while (len) {
		u64 l = min_t(u64, len, blocksize);
		int have_csum = 0;

		if (flags & BTRFS_EXTENT_FLAG_DATA) {
			/* push csums to sbio */
2907
			have_csum = scrub_find_csum(sctx, logical, csum);
2908 2909 2910 2911 2912 2913 2914 2915
			if (have_csum == 0)
				goto skip;
		}
		ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
					     flags, gen, mirror_num,
					     have_csum ? csum : NULL);
		if (ret)
			return ret;
2916
skip:
2917 2918 2919 2920 2921 2922 2923
		len -= l;
		logical += l;
		physical += l;
	}
	return 0;
}

2924 2925 2926 2927 2928 2929 2930 2931
/*
 * Given a physical address, this will calculate it's
 * logical offset. if this is a parity stripe, it will return
 * the most left data stripe's logical offset.
 *
 * return 0 if it is a data stripe, 1 means parity stripe.
 */
static int get_raid56_logic_offset(u64 physical, int num,
2932 2933
				   struct map_lookup *map, u64 *offset,
				   u64 *stripe_start)
2934 2935 2936 2937 2938
{
	int i;
	int j = 0;
	u64 stripe_nr;
	u64 last_offset;
2939 2940
	u32 stripe_index;
	u32 rot;
2941 2942 2943

	last_offset = (physical - map->stripes[num].physical) *
		      nr_data_stripes(map);
2944 2945 2946
	if (stripe_start)
		*stripe_start = last_offset;

2947 2948 2949 2950
	*offset = last_offset;
	for (i = 0; i < nr_data_stripes(map); i++) {
		*offset = last_offset + i * map->stripe_len;

2951
		stripe_nr = div64_u64(*offset, map->stripe_len);
2952
		stripe_nr = div_u64(stripe_nr, nr_data_stripes(map));
2953 2954

		/* Work out the disk rotation on this stripe-set */
2955
		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2956 2957
		/* calculate which stripe this data locates */
		rot += i;
2958
		stripe_index = rot % map->num_stripes;
2959 2960 2961 2962 2963 2964 2965 2966 2967
		if (stripe_index == num)
			return 0;
		if (stripe_index < num)
			j++;
	}
	*offset = last_offset + j * map->stripe_len;
	return 1;
}

2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989
static void scrub_free_parity(struct scrub_parity *sparity)
{
	struct scrub_ctx *sctx = sparity->sctx;
	struct scrub_page *curr, *next;
	int nbits;

	nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
	if (nbits) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.read_errors += nbits;
		sctx->stat.uncorrectable_errors += nbits;
		spin_unlock(&sctx->stat_lock);
	}

	list_for_each_entry_safe(curr, next, &sparity->spages, list) {
		list_del_init(&curr->list);
		scrub_page_put(curr);
	}

	kfree(sparity);
}

2990 2991 2992 2993 2994 2995 2996 2997 2998 2999
static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
{
	struct scrub_parity *sparity = container_of(work, struct scrub_parity,
						    work);
	struct scrub_ctx *sctx = sparity->sctx;

	scrub_free_parity(sparity);
	scrub_pending_bio_dec(sctx);
}

3000
static void scrub_parity_bio_endio(struct bio *bio)
3001 3002
{
	struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
3003
	struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;
3004

3005
	if (bio->bi_status)
3006 3007 3008 3009
		bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
			  sparity->nsectors);

	bio_put(bio);
3010 3011 3012

	btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,
			scrub_parity_bio_endio_worker, NULL, NULL);
3013
	btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work);
3014 3015 3016 3017 3018
}

static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
{
	struct scrub_ctx *sctx = sparity->sctx;
3019
	struct btrfs_fs_info *fs_info = sctx->fs_info;
3020 3021 3022 3023 3024 3025 3026 3027 3028 3029
	struct bio *bio;
	struct btrfs_raid_bio *rbio;
	struct btrfs_bio *bbio = NULL;
	u64 length;
	int ret;

	if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
			   sparity->nsectors))
		goto out;

3030
	length = sparity->logic_end - sparity->logic_start;
3031 3032

	btrfs_bio_counter_inc_blocked(fs_info);
3033
	ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start,
3034
			       &length, &bbio);
3035
	if (ret || !bbio || !bbio->raid_map)
3036 3037
		goto bbio_out;

3038
	bio = btrfs_io_bio_alloc(0);
3039 3040 3041 3042
	bio->bi_iter.bi_sector = sparity->logic_start >> 9;
	bio->bi_private = sparity;
	bio->bi_end_io = scrub_parity_bio_endio;

3043
	rbio = raid56_parity_alloc_scrub_rbio(fs_info, bio, bbio,
3044
					      length, sparity->scrub_dev,
3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056
					      sparity->dbitmap,
					      sparity->nsectors);
	if (!rbio)
		goto rbio_out;

	scrub_pending_bio_inc(sctx);
	raid56_parity_submit_scrub_rbio(rbio);
	return;

rbio_out:
	bio_put(bio);
bbio_out:
3057
	btrfs_bio_counter_dec(fs_info);
3058
	btrfs_put_bbio(bbio);
3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069
	bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
		  sparity->nsectors);
	spin_lock(&sctx->stat_lock);
	sctx->stat.malloc_errors++;
	spin_unlock(&sctx->stat_lock);
out:
	scrub_free_parity(sparity);
}

static inline int scrub_calc_parity_bitmap_len(int nsectors)
{
3070
	return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long);
3071 3072 3073 3074
}

static void scrub_parity_get(struct scrub_parity *sparity)
{
3075
	refcount_inc(&sparity->refs);
3076 3077 3078 3079
}

static void scrub_parity_put(struct scrub_parity *sparity)
{
3080
	if (!refcount_dec_and_test(&sparity->refs))
3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092
		return;

	scrub_parity_check_and_repair(sparity);
}

static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
						  struct map_lookup *map,
						  struct btrfs_device *sdev,
						  struct btrfs_path *path,
						  u64 logic_start,
						  u64 logic_end)
{
3093
	struct btrfs_fs_info *fs_info = sctx->fs_info;
3094 3095 3096
	struct btrfs_root *root = fs_info->extent_root;
	struct btrfs_root *csum_root = fs_info->csum_root;
	struct btrfs_extent_item *extent;
3097
	struct btrfs_bio *bbio = NULL;
3098 3099 3100 3101 3102 3103 3104 3105 3106
	u64 flags;
	int ret;
	int slot;
	struct extent_buffer *l;
	struct btrfs_key key;
	u64 generation;
	u64 extent_logical;
	u64 extent_physical;
	u64 extent_len;
3107
	u64 mapped_length;
3108 3109 3110 3111 3112 3113 3114
	struct btrfs_device *extent_dev;
	struct scrub_parity *sparity;
	int nsectors;
	int bitmap_len;
	int extent_mirror_num;
	int stop_loop = 0;

3115
	nsectors = div_u64(map->stripe_len, fs_info->sectorsize);
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131
	bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
	sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
			  GFP_NOFS);
	if (!sparity) {
		spin_lock(&sctx->stat_lock);
		sctx->stat.malloc_errors++;
		spin_unlock(&sctx->stat_lock);
		return -ENOMEM;
	}

	sparity->stripe_len = map->stripe_len;
	sparity->nsectors = nsectors;
	sparity->sctx = sctx;
	sparity->scrub_dev = sdev;
	sparity->logic_start = logic_start;
	sparity->logic_end = logic_end;
3132
	refcount_set(&sparity->refs, 1);
3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180
	INIT_LIST_HEAD(&sparity->spages);
	sparity->dbitmap = sparity->bitmap;
	sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;

	ret = 0;
	while (logic_start < logic_end) {
		if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
			key.type = BTRFS_METADATA_ITEM_KEY;
		else
			key.type = BTRFS_EXTENT_ITEM_KEY;
		key.objectid = logic_start;
		key.offset = (u64)-1;

		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		if (ret < 0)
			goto out;

		if (ret > 0) {
			ret = btrfs_previous_extent_item(root, path, 0);
			if (ret < 0)
				goto out;
			if (ret > 0) {
				btrfs_release_path(path);
				ret = btrfs_search_slot(NULL, root, &key,
							path, 0, 0);
				if (ret < 0)
					goto out;
			}
		}

		stop_loop = 0;
		while (1) {
			u64 bytes;

			l = path->nodes[0];
			slot = path->slots[0];
			if (slot >= btrfs_header_nritems(l)) {
				ret = btrfs_next_leaf(root, path);
				if (ret == 0)
					continue;
				if (ret < 0)
					goto out;

				stop_loop = 1;
				break;
			}
			btrfs_item_key_to_cpu(l, &key, slot);

3181 3182 3183 3184
			if (key.type != BTRFS_EXTENT_ITEM_KEY &&
			    key.type != BTRFS_METADATA_ITEM_KEY)
				goto next;

3185
			if (key.type == BTRFS_METADATA_ITEM_KEY)
3186
				bytes = fs_info->nodesize;
3187 3188 3189 3190 3191 3192
			else
				bytes = key.offset;

			if (key.objectid + bytes <= logic_start)
				goto next;

3193
			if (key.objectid >= logic_end) {
3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205
				stop_loop = 1;
				break;
			}

			while (key.objectid >= logic_start + map->stripe_len)
				logic_start += map->stripe_len;

			extent = btrfs_item_ptr(l, slot,
						struct btrfs_extent_item);
			flags = btrfs_extent_flags(l, extent);
			generation = btrfs_extent_generation(l, extent);

3206 3207 3208 3209
			if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
			    (key.objectid < logic_start ||
			     key.objectid + bytes >
			     logic_start + map->stripe_len)) {
J
Jeff Mahoney 已提交
3210 3211
				btrfs_err(fs_info,
					  "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3212
					  key.objectid, logic_start);
3213 3214 3215
				spin_lock(&sctx->stat_lock);
				sctx->stat.uncorrectable_errors++;
				spin_unlock(&sctx->stat_lock);
3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234
				goto next;
			}
again:
			extent_logical = key.objectid;
			extent_len = bytes;

			if (extent_logical < logic_start) {
				extent_len -= logic_start - extent_logical;
				extent_logical = logic_start;
			}

			if (extent_logical + extent_len >
			    logic_start + map->stripe_len)
				extent_len = logic_start + map->stripe_len -
					     extent_logical;

			scrub_parity_mark_sectors_data(sparity, extent_logical,
						       extent_len);

3235
			mapped_length = extent_len;
3236
			bbio = NULL;
3237 3238 3239
			ret = btrfs_map_block(fs_info, BTRFS_MAP_READ,
					extent_logical, &mapped_length, &bbio,
					0);
3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251
			if (!ret) {
				if (!bbio || mapped_length < extent_len)
					ret = -EIO;
			}
			if (ret) {
				btrfs_put_bbio(bbio);
				goto out;
			}
			extent_physical = bbio->stripes[0].physical;
			extent_mirror_num = bbio->mirror_num;
			extent_dev = bbio->stripes[0].dev;
			btrfs_put_bbio(bbio);
3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265

			ret = btrfs_lookup_csums_range(csum_root,
						extent_logical,
						extent_logical + extent_len - 1,
						&sctx->csum_list, 1);
			if (ret)
				goto out;

			ret = scrub_extent_for_parity(sparity, extent_logical,
						      extent_len,
						      extent_physical,
						      extent_dev, flags,
						      generation,
						      extent_mirror_num);
3266 3267 3268

			scrub_free_csums(sctx);

3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299
			if (ret)
				goto out;

			if (extent_logical + extent_len <
			    key.objectid + bytes) {
				logic_start += map->stripe_len;

				if (logic_start >= logic_end) {
					stop_loop = 1;
					break;
				}

				if (logic_start < key.objectid + bytes) {
					cond_resched();
					goto again;
				}
			}
next:
			path->slots[0]++;
		}

		btrfs_release_path(path);

		if (stop_loop)
			break;

		logic_start += map->stripe_len;
	}
out:
	if (ret < 0)
		scrub_parity_mark_sectors_error(sparity, logic_start,
3300
						logic_end - logic_start);
3301 3302
	scrub_parity_put(sparity);
	scrub_submit(sctx);
3303
	mutex_lock(&sctx->wr_lock);
3304
	scrub_wr_submit(sctx);
3305
	mutex_unlock(&sctx->wr_lock);
3306 3307 3308 3309 3310

	btrfs_release_path(path);
	return ret < 0 ? ret : 0;
}

3311
static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
3312 3313
					   struct map_lookup *map,
					   struct btrfs_device *scrub_dev,
3314 3315
					   int num, u64 base, u64 length,
					   int is_dev_replace)
A
Arne Jansen 已提交
3316
{
3317
	struct btrfs_path *path, *ppath;
3318
	struct btrfs_fs_info *fs_info = sctx->fs_info;
A
Arne Jansen 已提交
3319 3320 3321
	struct btrfs_root *root = fs_info->extent_root;
	struct btrfs_root *csum_root = fs_info->csum_root;
	struct btrfs_extent_item *extent;
3322
	struct blk_plug plug;
A
Arne Jansen 已提交
3323 3324 3325 3326 3327 3328 3329
	u64 flags;
	int ret;
	int slot;
	u64 nstripes;
	struct extent_buffer *l;
	u64 physical;
	u64 logical;
L
Liu Bo 已提交
3330
	u64 logic_end;
3331
	u64 physical_end;
A
Arne Jansen 已提交
3332
	u64 generation;
3333
	int mirror_num;
A
Arne Jansen 已提交
3334 3335
	struct reada_control *reada1;
	struct reada_control *reada2;
3336
	struct btrfs_key key;
A
Arne Jansen 已提交
3337
	struct btrfs_key key_end;
A
Arne Jansen 已提交
3338 3339
	u64 increment = map->stripe_len;
	u64 offset;
3340 3341 3342
	u64 extent_logical;
	u64 extent_physical;
	u64 extent_len;
3343 3344
	u64 stripe_logical;
	u64 stripe_end;
3345 3346
	struct btrfs_device *extent_dev;
	int extent_mirror_num;
3347
	int stop_loop = 0;
D
David Woodhouse 已提交
3348

3349
	physical = map->stripes[num].physical;
A
Arne Jansen 已提交
3350
	offset = 0;
3351
	nstripes = div64_u64(length, map->stripe_len);
A
Arne Jansen 已提交
3352 3353 3354
	if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
		offset = map->stripe_len * num;
		increment = map->stripe_len * map->num_stripes;
3355
		mirror_num = 1;
A
Arne Jansen 已提交
3356 3357 3358 3359
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
		int factor = map->num_stripes / map->sub_stripes;
		offset = map->stripe_len * (num / map->sub_stripes);
		increment = map->stripe_len * factor;
3360
		mirror_num = num % map->sub_stripes + 1;
A
Arne Jansen 已提交
3361 3362
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
		increment = map->stripe_len;
3363
		mirror_num = num % map->num_stripes + 1;
A
Arne Jansen 已提交
3364 3365
	} else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
		increment = map->stripe_len;
3366
		mirror_num = num % map->num_stripes + 1;
3367
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3368
		get_raid56_logic_offset(physical, num, map, &offset, NULL);
3369 3370
		increment = map->stripe_len * nr_data_stripes(map);
		mirror_num = 1;
A
Arne Jansen 已提交
3371 3372
	} else {
		increment = map->stripe_len;
3373
		mirror_num = 1;
A
Arne Jansen 已提交
3374 3375 3376 3377 3378 3379
	}

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3380 3381
	ppath = btrfs_alloc_path();
	if (!ppath) {
3382
		btrfs_free_path(path);
3383 3384 3385
		return -ENOMEM;
	}

3386 3387 3388 3389 3390
	/*
	 * work on commit root. The related disk blocks are static as
	 * long as COW is applied. This means, it is save to rewrite
	 * them to repair disk errors without any race conditions
	 */
A
Arne Jansen 已提交
3391 3392 3393
	path->search_commit_root = 1;
	path->skip_locking = 1;

3394 3395
	ppath->search_commit_root = 1;
	ppath->skip_locking = 1;
A
Arne Jansen 已提交
3396
	/*
A
Arne Jansen 已提交
3397 3398 3399
	 * trigger the readahead for extent tree csum tree and wait for
	 * completion. During readahead, the scrub is officially paused
	 * to not hold off transaction commits
A
Arne Jansen 已提交
3400 3401
	 */
	logical = base + offset;
3402
	physical_end = physical + nstripes * map->stripe_len;
3403
	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3404
		get_raid56_logic_offset(physical_end, num,
3405
					map, &logic_end, NULL);
3406 3407 3408 3409
		logic_end += base;
	} else {
		logic_end = logical + increment * nstripes;
	}
3410
	wait_event(sctx->list_wait,
3411
		   atomic_read(&sctx->bios_in_flight) == 0);
3412
	scrub_blocked_if_needed(fs_info);
A
Arne Jansen 已提交
3413 3414

	/* FIXME it might be better to start readahead at commit root */
3415 3416 3417
	key.objectid = logical;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = (u64)0;
3418
	key_end.objectid = logic_end;
3419 3420
	key_end.type = BTRFS_METADATA_ITEM_KEY;
	key_end.offset = (u64)-1;
3421
	reada1 = btrfs_reada_add(root, &key, &key_end);
A
Arne Jansen 已提交
3422

3423 3424 3425
	key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
	key.type = BTRFS_EXTENT_CSUM_KEY;
	key.offset = logical;
A
Arne Jansen 已提交
3426 3427
	key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
	key_end.type = BTRFS_EXTENT_CSUM_KEY;
3428
	key_end.offset = logic_end;
3429
	reada2 = btrfs_reada_add(csum_root, &key, &key_end);
A
Arne Jansen 已提交
3430 3431 3432 3433 3434 3435

	if (!IS_ERR(reada1))
		btrfs_reada_wait(reada1);
	if (!IS_ERR(reada2))
		btrfs_reada_wait(reada2);

A
Arne Jansen 已提交
3436 3437 3438 3439 3440

	/*
	 * collect all data csums for the stripe to avoid seeking during
	 * the scrub. This might currently (crc32) end up to be about 1MB
	 */
3441
	blk_start_plug(&plug);
A
Arne Jansen 已提交
3442 3443 3444 3445 3446

	/*
	 * now find all extents for each stripe and scrub them
	 */
	ret = 0;
3447
	while (physical < physical_end) {
A
Arne Jansen 已提交
3448 3449 3450 3451
		/*
		 * canceled?
		 */
		if (atomic_read(&fs_info->scrub_cancel_req) ||
3452
		    atomic_read(&sctx->cancel_req)) {
A
Arne Jansen 已提交
3453 3454 3455 3456 3457 3458 3459 3460
			ret = -ECANCELED;
			goto out;
		}
		/*
		 * check to see if we have to pause
		 */
		if (atomic_read(&fs_info->scrub_pause_req)) {
			/* push queued extents */
3461
			sctx->flush_all_writes = true;
3462
			scrub_submit(sctx);
3463
			mutex_lock(&sctx->wr_lock);
3464
			scrub_wr_submit(sctx);
3465
			mutex_unlock(&sctx->wr_lock);
3466
			wait_event(sctx->list_wait,
3467
				   atomic_read(&sctx->bios_in_flight) == 0);
3468
			sctx->flush_all_writes = false;
3469
			scrub_blocked_if_needed(fs_info);
A
Arne Jansen 已提交
3470 3471
		}

3472 3473 3474 3475 3476 3477
		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
			ret = get_raid56_logic_offset(physical, num, map,
						      &logical,
						      &stripe_logical);
			logical += base;
			if (ret) {
3478
				/* it is parity strip */
3479
				stripe_logical += base;
3480
				stripe_end = stripe_logical + increment;
3481 3482 3483 3484 3485 3486 3487 3488 3489
				ret = scrub_raid56_parity(sctx, map, scrub_dev,
							  ppath, stripe_logical,
							  stripe_end);
				if (ret)
					goto out;
				goto skip;
			}
		}

3490 3491 3492 3493
		if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
			key.type = BTRFS_METADATA_ITEM_KEY;
		else
			key.type = BTRFS_EXTENT_ITEM_KEY;
A
Arne Jansen 已提交
3494
		key.objectid = logical;
L
Liu Bo 已提交
3495
		key.offset = (u64)-1;
A
Arne Jansen 已提交
3496 3497 3498 3499

		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		if (ret < 0)
			goto out;
3500

3501
		if (ret > 0) {
3502
			ret = btrfs_previous_extent_item(root, path, 0);
A
Arne Jansen 已提交
3503 3504
			if (ret < 0)
				goto out;
3505 3506 3507 3508 3509 3510 3511 3512 3513
			if (ret > 0) {
				/* there's no smaller item, so stick with the
				 * larger one */
				btrfs_release_path(path);
				ret = btrfs_search_slot(NULL, root, &key,
							path, 0, 0);
				if (ret < 0)
					goto out;
			}
A
Arne Jansen 已提交
3514 3515
		}

L
Liu Bo 已提交
3516
		stop_loop = 0;
A
Arne Jansen 已提交
3517
		while (1) {
3518 3519
			u64 bytes;

A
Arne Jansen 已提交
3520 3521 3522 3523 3524 3525 3526 3527 3528
			l = path->nodes[0];
			slot = path->slots[0];
			if (slot >= btrfs_header_nritems(l)) {
				ret = btrfs_next_leaf(root, path);
				if (ret == 0)
					continue;
				if (ret < 0)
					goto out;

L
Liu Bo 已提交
3529
				stop_loop = 1;
A
Arne Jansen 已提交
3530 3531 3532 3533
				break;
			}
			btrfs_item_key_to_cpu(l, &key, slot);

3534 3535 3536 3537
			if (key.type != BTRFS_EXTENT_ITEM_KEY &&
			    key.type != BTRFS_METADATA_ITEM_KEY)
				goto next;

3538
			if (key.type == BTRFS_METADATA_ITEM_KEY)
3539
				bytes = fs_info->nodesize;
3540 3541 3542 3543
			else
				bytes = key.offset;

			if (key.objectid + bytes <= logical)
A
Arne Jansen 已提交
3544 3545
				goto next;

L
Liu Bo 已提交
3546 3547 3548 3549 3550 3551
			if (key.objectid >= logical + map->stripe_len) {
				/* out of this device extent */
				if (key.objectid >= logic_end)
					stop_loop = 1;
				break;
			}
A
Arne Jansen 已提交
3552 3553 3554 3555 3556 3557

			extent = btrfs_item_ptr(l, slot,
						struct btrfs_extent_item);
			flags = btrfs_extent_flags(l, extent);
			generation = btrfs_extent_generation(l, extent);

3558 3559 3560 3561
			if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
			    (key.objectid < logical ||
			     key.objectid + bytes >
			     logical + map->stripe_len)) {
3562
				btrfs_err(fs_info,
J
Jeff Mahoney 已提交
3563
					   "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3564
				       key.objectid, logical);
3565 3566 3567
				spin_lock(&sctx->stat_lock);
				sctx->stat.uncorrectable_errors++;
				spin_unlock(&sctx->stat_lock);
A
Arne Jansen 已提交
3568 3569 3570
				goto next;
			}

L
Liu Bo 已提交
3571 3572 3573 3574
again:
			extent_logical = key.objectid;
			extent_len = bytes;

A
Arne Jansen 已提交
3575 3576 3577
			/*
			 * trim extent to this stripe
			 */
L
Liu Bo 已提交
3578 3579 3580
			if (extent_logical < logical) {
				extent_len -= logical - extent_logical;
				extent_logical = logical;
A
Arne Jansen 已提交
3581
			}
L
Liu Bo 已提交
3582
			if (extent_logical + extent_len >
A
Arne Jansen 已提交
3583
			    logical + map->stripe_len) {
L
Liu Bo 已提交
3584 3585
				extent_len = logical + map->stripe_len -
					     extent_logical;
A
Arne Jansen 已提交
3586 3587
			}

L
Liu Bo 已提交
3588
			extent_physical = extent_logical - logical + physical;
3589 3590 3591 3592 3593 3594 3595
			extent_dev = scrub_dev;
			extent_mirror_num = mirror_num;
			if (is_dev_replace)
				scrub_remap_extent(fs_info, extent_logical,
						   extent_len, &extent_physical,
						   &extent_dev,
						   &extent_mirror_num);
L
Liu Bo 已提交
3596

3597 3598 3599 3600 3601
			ret = btrfs_lookup_csums_range(csum_root,
						       extent_logical,
						       extent_logical +
						       extent_len - 1,
						       &sctx->csum_list, 1);
L
Liu Bo 已提交
3602 3603 3604
			if (ret)
				goto out;

L
Liu Bo 已提交
3605
			ret = scrub_extent(sctx, map, extent_logical, extent_len,
3606 3607
					   extent_physical, extent_dev, flags,
					   generation, extent_mirror_num,
3608
					   extent_logical - logical + physical);
3609 3610 3611

			scrub_free_csums(sctx);

A
Arne Jansen 已提交
3612 3613 3614
			if (ret)
				goto out;

L
Liu Bo 已提交
3615 3616
			if (extent_logical + extent_len <
			    key.objectid + bytes) {
3617
				if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3618 3619 3620 3621
					/*
					 * loop until we find next data stripe
					 * or we have finished all stripes.
					 */
3622 3623 3624 3625 3626 3627 3628 3629 3630 3631
loop:
					physical += map->stripe_len;
					ret = get_raid56_logic_offset(physical,
							num, map, &logical,
							&stripe_logical);
					logical += base;

					if (ret && physical < physical_end) {
						stripe_logical += base;
						stripe_end = stripe_logical +
3632
								increment;
3633 3634 3635 3636 3637 3638 3639 3640
						ret = scrub_raid56_parity(sctx,
							map, scrub_dev, ppath,
							stripe_logical,
							stripe_end);
						if (ret)
							goto out;
						goto loop;
					}
3641 3642 3643 3644
				} else {
					physical += map->stripe_len;
					logical += increment;
				}
L
Liu Bo 已提交
3645 3646 3647 3648 3649
				if (logical < key.objectid + bytes) {
					cond_resched();
					goto again;
				}

3650
				if (physical >= physical_end) {
L
Liu Bo 已提交
3651 3652 3653 3654
					stop_loop = 1;
					break;
				}
			}
A
Arne Jansen 已提交
3655 3656 3657
next:
			path->slots[0]++;
		}
C
Chris Mason 已提交
3658
		btrfs_release_path(path);
3659
skip:
A
Arne Jansen 已提交
3660 3661
		logical += increment;
		physical += map->stripe_len;
3662
		spin_lock(&sctx->stat_lock);
L
Liu Bo 已提交
3663 3664 3665 3666 3667
		if (stop_loop)
			sctx->stat.last_physical = map->stripes[num].physical +
						   length;
		else
			sctx->stat.last_physical = physical;
3668
		spin_unlock(&sctx->stat_lock);
L
Liu Bo 已提交
3669 3670
		if (stop_loop)
			break;
A
Arne Jansen 已提交
3671
	}
3672
out:
A
Arne Jansen 已提交
3673
	/* push queued extents */
3674
	scrub_submit(sctx);
3675
	mutex_lock(&sctx->wr_lock);
3676
	scrub_wr_submit(sctx);
3677
	mutex_unlock(&sctx->wr_lock);
A
Arne Jansen 已提交
3678

3679
	blk_finish_plug(&plug);
A
Arne Jansen 已提交
3680
	btrfs_free_path(path);
3681
	btrfs_free_path(ppath);
A
Arne Jansen 已提交
3682 3683 3684
	return ret < 0 ? ret : 0;
}

3685
static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
3686 3687
					  struct btrfs_device *scrub_dev,
					  u64 chunk_offset, u64 length,
3688 3689 3690
					  u64 dev_offset,
					  struct btrfs_block_group_cache *cache,
					  int is_dev_replace)
A
Arne Jansen 已提交
3691
{
3692 3693
	struct btrfs_fs_info *fs_info = sctx->fs_info;
	struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
A
Arne Jansen 已提交
3694 3695 3696
	struct map_lookup *map;
	struct extent_map *em;
	int i;
3697
	int ret = 0;
A
Arne Jansen 已提交
3698 3699 3700 3701 3702

	read_lock(&map_tree->map_tree.lock);
	em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
	read_unlock(&map_tree->map_tree.lock);

3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714
	if (!em) {
		/*
		 * Might have been an unused block group deleted by the cleaner
		 * kthread or relocation.
		 */
		spin_lock(&cache->lock);
		if (!cache->removed)
			ret = -EINVAL;
		spin_unlock(&cache->lock);

		return ret;
	}
A
Arne Jansen 已提交
3715

3716
	map = em->map_lookup;
A
Arne Jansen 已提交
3717 3718 3719 3720 3721 3722 3723
	if (em->start != chunk_offset)
		goto out;

	if (em->len < length)
		goto out;

	for (i = 0; i < map->num_stripes; ++i) {
3724
		if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
3725
		    map->stripes[i].physical == dev_offset) {
3726
			ret = scrub_stripe(sctx, map, scrub_dev, i,
3727 3728
					   chunk_offset, length,
					   is_dev_replace);
A
Arne Jansen 已提交
3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739
			if (ret)
				goto out;
		}
	}
out:
	free_extent_map(em);

	return ret;
}

static noinline_for_stack
3740
int scrub_enumerate_chunks(struct scrub_ctx *sctx,
3741 3742
			   struct btrfs_device *scrub_dev, u64 start, u64 end,
			   int is_dev_replace)
A
Arne Jansen 已提交
3743 3744 3745
{
	struct btrfs_dev_extent *dev_extent = NULL;
	struct btrfs_path *path;
3746 3747
	struct btrfs_fs_info *fs_info = sctx->fs_info;
	struct btrfs_root *root = fs_info->dev_root;
A
Arne Jansen 已提交
3748 3749
	u64 length;
	u64 chunk_offset;
3750
	int ret = 0;
3751
	int ro_set;
A
Arne Jansen 已提交
3752 3753 3754 3755 3756
	int slot;
	struct extent_buffer *l;
	struct btrfs_key key;
	struct btrfs_key found_key;
	struct btrfs_block_group_cache *cache;
3757
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
A
Arne Jansen 已提交
3758 3759 3760 3761 3762

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3763
	path->reada = READA_FORWARD;
A
Arne Jansen 已提交
3764 3765 3766
	path->search_commit_root = 1;
	path->skip_locking = 1;

3767
	key.objectid = scrub_dev->devid;
A
Arne Jansen 已提交
3768 3769 3770 3771 3772 3773
	key.offset = 0ull;
	key.type = BTRFS_DEV_EXTENT_KEY;

	while (1) {
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		if (ret < 0)
3774 3775 3776 3777 3778
			break;
		if (ret > 0) {
			if (path->slots[0] >=
			    btrfs_header_nritems(path->nodes[0])) {
				ret = btrfs_next_leaf(root, path);
3779 3780 3781 3782
				if (ret < 0)
					break;
				if (ret > 0) {
					ret = 0;
3783
					break;
3784 3785 3786
				}
			} else {
				ret = 0;
3787 3788
			}
		}
A
Arne Jansen 已提交
3789 3790 3791 3792 3793 3794

		l = path->nodes[0];
		slot = path->slots[0];

		btrfs_item_key_to_cpu(l, &found_key, slot);

3795
		if (found_key.objectid != scrub_dev->devid)
A
Arne Jansen 已提交
3796 3797
			break;

3798
		if (found_key.type != BTRFS_DEV_EXTENT_KEY)
A
Arne Jansen 已提交
3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809
			break;

		if (found_key.offset >= end)
			break;

		if (found_key.offset < key.offset)
			break;

		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
		length = btrfs_dev_extent_length(l, dev_extent);

3810 3811
		if (found_key.offset + length <= start)
			goto skip;
A
Arne Jansen 已提交
3812 3813 3814 3815 3816 3817 3818 3819

		chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);

		/*
		 * get a reference on the corresponding block group to prevent
		 * the chunk from going away while we scrub it
		 */
		cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3820 3821 3822 3823 3824 3825

		/* some chunks are removed but not committed to disk yet,
		 * continue scrubbing */
		if (!cache)
			goto skip;

3826 3827 3828 3829 3830 3831 3832 3833 3834
		/*
		 * we need call btrfs_inc_block_group_ro() with scrubs_paused,
		 * to avoid deadlock caused by:
		 * btrfs_inc_block_group_ro()
		 * -> btrfs_wait_for_commit()
		 * -> btrfs_commit_transaction()
		 * -> btrfs_scrub_pause()
		 */
		scrub_pause_on(fs_info);
3835
		ret = btrfs_inc_block_group_ro(fs_info, cache);
3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856
		if (!ret && is_dev_replace) {
			/*
			 * If we are doing a device replace wait for any tasks
			 * that started dellaloc right before we set the block
			 * group to RO mode, as they might have just allocated
			 * an extent from it or decided they could do a nocow
			 * write. And if any such tasks did that, wait for their
			 * ordered extents to complete and then commit the
			 * current transaction, so that we can later see the new
			 * extent items in the extent tree - the ordered extents
			 * create delayed data references (for cow writes) when
			 * they complete, which will be run and insert the
			 * corresponding extent items into the extent tree when
			 * we commit the transaction they used when running
			 * inode.c:btrfs_finish_ordered_io(). We later use
			 * the commit root of the extent tree to find extents
			 * to copy from the srcdev into the tgtdev, and we don't
			 * want to miss any new extents.
			 */
			btrfs_wait_block_group_reservations(cache);
			btrfs_wait_nocow_writers(cache);
3857
			ret = btrfs_wait_ordered_roots(fs_info, U64_MAX,
3858 3859 3860 3861 3862 3863 3864 3865 3866
						       cache->key.objectid,
						       cache->key.offset);
			if (ret > 0) {
				struct btrfs_trans_handle *trans;

				trans = btrfs_join_transaction(root);
				if (IS_ERR(trans))
					ret = PTR_ERR(trans);
				else
3867
					ret = btrfs_commit_transaction(trans);
3868 3869 3870 3871 3872 3873 3874
				if (ret) {
					scrub_pause_off(fs_info);
					btrfs_put_block_group(cache);
					break;
				}
			}
		}
3875
		scrub_pause_off(fs_info);
3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888

		if (ret == 0) {
			ro_set = 1;
		} else if (ret == -ENOSPC) {
			/*
			 * btrfs_inc_block_group_ro return -ENOSPC when it
			 * failed in creating new chunk for metadata.
			 * It is not a problem for scrub/replace, because
			 * metadata are always cowed, and our scrub paused
			 * commit_transactions.
			 */
			ro_set = 0;
		} else {
J
Jeff Mahoney 已提交
3889
			btrfs_warn(fs_info,
3890
				   "failed setting block group ro: %d", ret);
3891 3892 3893 3894
			btrfs_put_block_group(cache);
			break;
		}

3895
		btrfs_dev_replace_write_lock(&fs_info->dev_replace);
3896 3897 3898
		dev_replace->cursor_right = found_key.offset + length;
		dev_replace->cursor_left = found_key.offset;
		dev_replace->item_needs_writeback = 1;
3899
		btrfs_dev_replace_write_unlock(&fs_info->dev_replace);
3900
		ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length,
3901
				  found_key.offset, cache, is_dev_replace);
3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912

		/*
		 * flush, submit all pending read and write bios, afterwards
		 * wait for them.
		 * Note that in the dev replace case, a read request causes
		 * write requests that are submitted in the read completion
		 * worker. Therefore in the current situation, it is required
		 * that all write requests are flushed, so that all read and
		 * write requests are really completed when bios_in_flight
		 * changes to 0.
		 */
3913
		sctx->flush_all_writes = true;
3914
		scrub_submit(sctx);
3915
		mutex_lock(&sctx->wr_lock);
3916
		scrub_wr_submit(sctx);
3917
		mutex_unlock(&sctx->wr_lock);
3918 3919 3920

		wait_event(sctx->list_wait,
			   atomic_read(&sctx->bios_in_flight) == 0);
3921 3922

		scrub_pause_on(fs_info);
3923 3924 3925 3926 3927 3928

		/*
		 * must be called before we decrease @scrub_paused.
		 * make sure we don't block transaction commit while
		 * we are waiting pending workers finished.
		 */
3929 3930
		wait_event(sctx->list_wait,
			   atomic_read(&sctx->workers_pending) == 0);
3931
		sctx->flush_all_writes = false;
3932

3933
		scrub_pause_off(fs_info);
3934

3935
		btrfs_dev_replace_write_lock(&fs_info->dev_replace);
3936 3937
		dev_replace->cursor_left = dev_replace->cursor_right;
		dev_replace->item_needs_writeback = 1;
3938
		btrfs_dev_replace_write_unlock(&fs_info->dev_replace);
3939

3940
		if (ro_set)
3941
			btrfs_dec_block_group_ro(cache);
3942

3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956
		/*
		 * We might have prevented the cleaner kthread from deleting
		 * this block group if it was already unused because we raced
		 * and set it to RO mode first. So add it back to the unused
		 * list, otherwise it might not ever be deleted unless a manual
		 * balance is triggered or it becomes used and unused again.
		 */
		spin_lock(&cache->lock);
		if (!cache->removed && !cache->ro && cache->reserved == 0 &&
		    btrfs_block_group_used(&cache->item) == 0) {
			spin_unlock(&cache->lock);
			spin_lock(&fs_info->unused_bgs_lock);
			if (list_empty(&cache->bg_list)) {
				btrfs_get_block_group(cache);
3957
				trace_btrfs_add_unused_block_group(cache);
3958 3959 3960 3961 3962 3963 3964 3965
				list_add_tail(&cache->bg_list,
					      &fs_info->unused_bgs);
			}
			spin_unlock(&fs_info->unused_bgs_lock);
		} else {
			spin_unlock(&cache->lock);
		}

A
Arne Jansen 已提交
3966 3967 3968
		btrfs_put_block_group(cache);
		if (ret)
			break;
3969 3970
		if (is_dev_replace &&
		    atomic64_read(&dev_replace->num_write_errors) > 0) {
3971 3972 3973 3974 3975 3976 3977
			ret = -EIO;
			break;
		}
		if (sctx->stat.malloc_errors > 0) {
			ret = -ENOMEM;
			break;
		}
3978
skip:
A
Arne Jansen 已提交
3979
		key.offset = found_key.offset + length;
C
Chris Mason 已提交
3980
		btrfs_release_path(path);
A
Arne Jansen 已提交
3981 3982 3983
	}

	btrfs_free_path(path);
3984

3985
	return ret;
A
Arne Jansen 已提交
3986 3987
}

3988 3989
static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
					   struct btrfs_device *scrub_dev)
A
Arne Jansen 已提交
3990 3991 3992 3993 3994
{
	int	i;
	u64	bytenr;
	u64	gen;
	int	ret;
3995
	struct btrfs_fs_info *fs_info = sctx->fs_info;
A
Arne Jansen 已提交
3996

3997
	if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
3998 3999
		return -EIO;

4000
	/* Seed devices of a new filesystem has their own generation. */
4001
	if (scrub_dev->fs_devices != fs_info->fs_devices)
4002 4003
		gen = scrub_dev->generation;
	else
4004
		gen = fs_info->last_trans_committed;
A
Arne Jansen 已提交
4005 4006 4007

	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
4008 4009
		if (bytenr + BTRFS_SUPER_INFO_SIZE >
		    scrub_dev->commit_total_bytes)
A
Arne Jansen 已提交
4010 4011
			break;

4012
		ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
4013
				  scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
4014
				  NULL, 1, bytenr);
A
Arne Jansen 已提交
4015 4016 4017
		if (ret)
			return ret;
	}
4018
	wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
A
Arne Jansen 已提交
4019 4020 4021 4022 4023 4024 4025

	return 0;
}

/*
 * get a reference count on fs_info->scrub_workers. start worker if necessary
 */
4026 4027
static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
						int is_dev_replace)
A
Arne Jansen 已提交
4028
{
4029
	unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
4030
	int max_active = fs_info->thread_pool_size;
A
Arne Jansen 已提交
4031

A
Arne Jansen 已提交
4032
	if (fs_info->scrub_workers_refcnt == 0) {
4033 4034
		fs_info->scrub_workers = btrfs_alloc_workqueue(fs_info, "scrub",
				flags, is_dev_replace ? 1 : max_active, 4);
4035 4036 4037
		if (!fs_info->scrub_workers)
			goto fail_scrub_workers;

4038
		fs_info->scrub_wr_completion_workers =
4039
			btrfs_alloc_workqueue(fs_info, "scrubwrc", flags,
4040
					      max_active, 2);
4041 4042 4043
		if (!fs_info->scrub_wr_completion_workers)
			goto fail_scrub_wr_completion_workers;

4044
		fs_info->scrub_parity_workers =
4045
			btrfs_alloc_workqueue(fs_info, "scrubparity", flags,
4046
					      max_active, 2);
4047 4048
		if (!fs_info->scrub_parity_workers)
			goto fail_scrub_parity_workers;
A
Arne Jansen 已提交
4049
	}
A
Arne Jansen 已提交
4050
	++fs_info->scrub_workers_refcnt;
4051 4052 4053 4054 4055 4056 4057 4058
	return 0;

fail_scrub_parity_workers:
	btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
fail_scrub_wr_completion_workers:
	btrfs_destroy_workqueue(fs_info->scrub_workers);
fail_scrub_workers:
	return -ENOMEM;
A
Arne Jansen 已提交
4059 4060
}

4061
static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
A
Arne Jansen 已提交
4062
{
4063
	if (--fs_info->scrub_workers_refcnt == 0) {
4064 4065
		btrfs_destroy_workqueue(fs_info->scrub_workers);
		btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
4066
		btrfs_destroy_workqueue(fs_info->scrub_parity_workers);
4067
	}
A
Arne Jansen 已提交
4068 4069 4070
	WARN_ON(fs_info->scrub_workers_refcnt < 0);
}

4071 4072
int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
		    u64 end, struct btrfs_scrub_progress *progress,
4073
		    int readonly, int is_dev_replace)
A
Arne Jansen 已提交
4074
{
4075
	struct scrub_ctx *sctx;
A
Arne Jansen 已提交
4076 4077
	int ret;
	struct btrfs_device *dev;
4078
	struct rcu_string *name;
A
Arne Jansen 已提交
4079

4080
	if (btrfs_fs_closing(fs_info))
A
Arne Jansen 已提交
4081 4082
		return -EINVAL;

4083
	if (fs_info->nodesize > BTRFS_STRIPE_LEN) {
4084 4085 4086 4087 4088
		/*
		 * in this case scrub is unable to calculate the checksum
		 * the way scrub is implemented. Do not handle this
		 * situation at all because it won't ever happen.
		 */
4089 4090
		btrfs_err(fs_info,
			   "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
4091 4092
		       fs_info->nodesize,
		       BTRFS_STRIPE_LEN);
4093 4094 4095
		return -EINVAL;
	}

4096
	if (fs_info->sectorsize != PAGE_SIZE) {
4097
		/* not supported for data w/o checksums */
4098
		btrfs_err_rl(fs_info,
J
Jeff Mahoney 已提交
4099
			   "scrub: size assumption sectorsize != PAGE_SIZE (%d != %lu) fails",
4100
		       fs_info->sectorsize, PAGE_SIZE);
A
Arne Jansen 已提交
4101 4102 4103
		return -EINVAL;
	}

4104
	if (fs_info->nodesize >
4105
	    PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
4106
	    fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
4107 4108 4109 4110
		/*
		 * would exhaust the array bounds of pagev member in
		 * struct scrub_block
		 */
J
Jeff Mahoney 已提交
4111 4112
		btrfs_err(fs_info,
			  "scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
4113
		       fs_info->nodesize,
4114
		       SCRUB_MAX_PAGES_PER_BLOCK,
4115
		       fs_info->sectorsize,
4116 4117 4118 4119
		       SCRUB_MAX_PAGES_PER_BLOCK);
		return -EINVAL;
	}

A
Arne Jansen 已提交
4120

4121 4122
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
	dev = btrfs_find_device(fs_info, devid, NULL, NULL);
4123 4124
	if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) &&
		     !is_dev_replace)) {
4125
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
A
Arne Jansen 已提交
4126 4127 4128
		return -ENODEV;
	}

4129 4130
	if (!is_dev_replace && !readonly &&
	    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
4131 4132 4133 4134 4135 4136 4137 4138 4139
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
		rcu_read_lock();
		name = rcu_dereference(dev->name);
		btrfs_err(fs_info, "scrub: device %s is not writable",
			  name->str);
		rcu_read_unlock();
		return -EROFS;
	}

4140
	mutex_lock(&fs_info->scrub_lock);
4141
	if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4142
	    test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state)) {
A
Arne Jansen 已提交
4143
		mutex_unlock(&fs_info->scrub_lock);
4144 4145
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
		return -EIO;
A
Arne Jansen 已提交
4146 4147
	}

4148
	btrfs_dev_replace_read_lock(&fs_info->dev_replace);
4149
	if (dev->scrub_ctx ||
4150 4151
	    (!is_dev_replace &&
	     btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
4152
		btrfs_dev_replace_read_unlock(&fs_info->dev_replace);
A
Arne Jansen 已提交
4153
		mutex_unlock(&fs_info->scrub_lock);
4154
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
A
Arne Jansen 已提交
4155 4156
		return -EINPROGRESS;
	}
4157
	btrfs_dev_replace_read_unlock(&fs_info->dev_replace);
4158 4159 4160 4161 4162 4163 4164 4165

	ret = scrub_workers_get(fs_info, is_dev_replace);
	if (ret) {
		mutex_unlock(&fs_info->scrub_lock);
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
		return ret;
	}

4166
	sctx = scrub_setup_ctx(dev, is_dev_replace);
4167
	if (IS_ERR(sctx)) {
A
Arne Jansen 已提交
4168
		mutex_unlock(&fs_info->scrub_lock);
4169 4170
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
		scrub_workers_put(fs_info);
4171
		return PTR_ERR(sctx);
A
Arne Jansen 已提交
4172
	}
4173
	sctx->readonly = readonly;
4174
	dev->scrub_ctx = sctx;
4175
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
A
Arne Jansen 已提交
4176

4177 4178 4179 4180
	/*
	 * checking @scrub_pause_req here, we can avoid
	 * race between committing transaction and scrubbing.
	 */
4181
	__scrub_blocked_if_needed(fs_info);
A
Arne Jansen 已提交
4182 4183 4184
	atomic_inc(&fs_info->scrubs_running);
	mutex_unlock(&fs_info->scrub_lock);

4185
	if (!is_dev_replace) {
4186 4187 4188 4189
		/*
		 * by holding device list mutex, we can
		 * kick off writing super in log tree sync.
		 */
4190
		mutex_lock(&fs_info->fs_devices->device_list_mutex);
4191
		ret = scrub_supers(sctx, dev);
4192
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4193
	}
A
Arne Jansen 已提交
4194 4195

	if (!ret)
4196 4197
		ret = scrub_enumerate_chunks(sctx, dev, start, end,
					     is_dev_replace);
A
Arne Jansen 已提交
4198

4199
	wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
A
Arne Jansen 已提交
4200 4201 4202
	atomic_dec(&fs_info->scrubs_running);
	wake_up(&fs_info->scrub_pause_wait);

4203
	wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
4204

A
Arne Jansen 已提交
4205
	if (progress)
4206
		memcpy(progress, &sctx->stat, sizeof(*progress));
A
Arne Jansen 已提交
4207 4208

	mutex_lock(&fs_info->scrub_lock);
4209
	dev->scrub_ctx = NULL;
4210
	scrub_workers_put(fs_info);
A
Arne Jansen 已提交
4211 4212
	mutex_unlock(&fs_info->scrub_lock);

4213
	scrub_put_ctx(sctx);
A
Arne Jansen 已提交
4214 4215 4216 4217

	return ret;
}

4218
void btrfs_scrub_pause(struct btrfs_fs_info *fs_info)
A
Arne Jansen 已提交
4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232
{
	mutex_lock(&fs_info->scrub_lock);
	atomic_inc(&fs_info->scrub_pause_req);
	while (atomic_read(&fs_info->scrubs_paused) !=
	       atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
			   atomic_read(&fs_info->scrubs_paused) ==
			   atomic_read(&fs_info->scrubs_running));
		mutex_lock(&fs_info->scrub_lock);
	}
	mutex_unlock(&fs_info->scrub_lock);
}

4233
void btrfs_scrub_continue(struct btrfs_fs_info *fs_info)
A
Arne Jansen 已提交
4234 4235 4236 4237 4238
{
	atomic_dec(&fs_info->scrub_pause_req);
	wake_up(&fs_info->scrub_pause_wait);
}

4239
int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
A
Arne Jansen 已提交
4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259
{
	mutex_lock(&fs_info->scrub_lock);
	if (!atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		return -ENOTCONN;
	}

	atomic_inc(&fs_info->scrub_cancel_req);
	while (atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
			   atomic_read(&fs_info->scrubs_running) == 0);
		mutex_lock(&fs_info->scrub_lock);
	}
	atomic_dec(&fs_info->scrub_cancel_req);
	mutex_unlock(&fs_info->scrub_lock);

	return 0;
}

4260 4261
int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
			   struct btrfs_device *dev)
4262
{
4263
	struct scrub_ctx *sctx;
A
Arne Jansen 已提交
4264 4265

	mutex_lock(&fs_info->scrub_lock);
4266
	sctx = dev->scrub_ctx;
4267
	if (!sctx) {
A
Arne Jansen 已提交
4268 4269 4270
		mutex_unlock(&fs_info->scrub_lock);
		return -ENOTCONN;
	}
4271
	atomic_inc(&sctx->cancel_req);
4272
	while (dev->scrub_ctx) {
A
Arne Jansen 已提交
4273 4274
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
4275
			   dev->scrub_ctx == NULL);
A
Arne Jansen 已提交
4276 4277 4278 4279 4280 4281
		mutex_lock(&fs_info->scrub_lock);
	}
	mutex_unlock(&fs_info->scrub_lock);

	return 0;
}
S
Stefan Behrens 已提交
4282

4283
int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid,
A
Arne Jansen 已提交
4284 4285 4286
			 struct btrfs_scrub_progress *progress)
{
	struct btrfs_device *dev;
4287
	struct scrub_ctx *sctx = NULL;
A
Arne Jansen 已提交
4288

4289 4290
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
	dev = btrfs_find_device(fs_info, devid, NULL, NULL);
A
Arne Jansen 已提交
4291
	if (dev)
4292
		sctx = dev->scrub_ctx;
4293 4294
	if (sctx)
		memcpy(progress, &sctx->stat, sizeof(*progress));
4295
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
A
Arne Jansen 已提交
4296

4297
	return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
A
Arne Jansen 已提交
4298
}
4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310

static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
			       u64 extent_logical, u64 extent_len,
			       u64 *extent_physical,
			       struct btrfs_device **extent_dev,
			       int *extent_mirror_num)
{
	u64 mapped_length;
	struct btrfs_bio *bbio = NULL;
	int ret;

	mapped_length = extent_len;
4311
	ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_logical,
4312 4313 4314
			      &mapped_length, &bbio, 0);
	if (ret || !bbio || mapped_length < extent_len ||
	    !bbio->stripes[0].dev->bdev) {
4315
		btrfs_put_bbio(bbio);
4316 4317 4318 4319 4320 4321
		return;
	}

	*extent_physical = bbio->stripes[0].physical;
	*extent_mirror_num = bbio->mirror_num;
	*extent_dev = bbio->stripes[0].dev;
4322
	btrfs_put_bbio(bbio);
4323
}