scrub.c 65.5 KB
Newer Older
A
Arne Jansen 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Copyright (C) 2011 STRATO.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */

#include <linux/blkdev.h>
20
#include <linux/ratelimit.h>
A
Arne Jansen 已提交
21 22 23 24
#include "ctree.h"
#include "volumes.h"
#include "disk-io.h"
#include "ordered-data.h"
25
#include "transaction.h"
26
#include "backref.h"
27
#include "extent_io.h"
28
#include "check-integrity.h"
29
#include "rcu-string.h"
A
Arne Jansen 已提交
30 31 32 33 34 35 36 37 38 39 40 41 42 43

/*
 * This is only the first step towards a full-features scrub. It reads all
 * extent and super block and verifies the checksums. In case a bad checksum
 * is found or the extent cannot be read, good data will be written back if
 * any can be found.
 *
 * Future enhancements:
 *  - In case an unrepairable extent is encountered, track which files are
 *    affected and report them
 *  - track and record media errors, throw out bad devices
 *  - add a mode to also read unallocated space
 */

44
struct scrub_block;
A
Arne Jansen 已提交
45 46 47 48
struct scrub_dev;

#define SCRUB_PAGES_PER_BIO	16	/* 64k per bio */
#define SCRUB_BIOS_PER_DEV	16	/* 1 MB per device in flight */
49
#define SCRUB_MAX_PAGES_PER_BLOCK	16	/* 64k per node/leaf/sector */
A
Arne Jansen 已提交
50 51

struct scrub_page {
52 53
	struct scrub_block	*sblock;
	struct page		*page;
54
	struct btrfs_device	*dev;
A
Arne Jansen 已提交
55 56
	u64			flags;  /* extent flags */
	u64			generation;
57 58 59 60 61 62 63
	u64			logical;
	u64			physical;
	struct {
		unsigned int	mirror_num:8;
		unsigned int	have_csum:1;
		unsigned int	io_error:1;
	};
A
Arne Jansen 已提交
64 65 66 67 68 69 70 71 72 73
	u8			csum[BTRFS_CSUM_SIZE];
};

struct scrub_bio {
	int			index;
	struct scrub_dev	*sdev;
	struct bio		*bio;
	int			err;
	u64			logical;
	u64			physical;
74 75
	struct scrub_page	*pagev[SCRUB_PAGES_PER_BIO];
	int			page_count;
A
Arne Jansen 已提交
76 77 78 79
	int			next_free;
	struct btrfs_work	work;
};

80 81 82 83 84 85 86 87 88 89
struct scrub_block {
	struct scrub_page	pagev[SCRUB_MAX_PAGES_PER_BLOCK];
	int			page_count;
	atomic_t		outstanding_pages;
	atomic_t		ref_count; /* free mem on transition to zero */
	struct scrub_dev	*sdev;
	struct {
		unsigned int	header_error:1;
		unsigned int	checksum_error:1;
		unsigned int	no_io_error_seen:1;
90
		unsigned int	generation_error:1; /* also sets header_error */
91 92 93
	};
};

A
Arne Jansen 已提交
94 95 96 97 98 99
struct scrub_dev {
	struct scrub_bio	*bios[SCRUB_BIOS_PER_DEV];
	struct btrfs_device	*dev;
	int			first_free;
	int			curr;
	atomic_t		in_flight;
100
	atomic_t		fixup_cnt;
A
Arne Jansen 已提交
101 102 103 104 105
	spinlock_t		list_lock;
	wait_queue_head_t	list_wait;
	u16			csum_size;
	struct list_head	csum_list;
	atomic_t		cancel_req;
A
Arne Jansen 已提交
106
	int			readonly;
107 108 109 110
	int			pages_per_bio; /* <= SCRUB_PAGES_PER_BIO */
	u32			sectorsize;
	u32			nodesize;
	u32			leafsize;
A
Arne Jansen 已提交
111 112 113 114 115 116 117
	/*
	 * statistics
	 */
	struct btrfs_scrub_progress stat;
	spinlock_t		stat_lock;
};

118 119 120 121 122 123 124 125
struct scrub_fixup_nodatasum {
	struct scrub_dev	*sdev;
	u64			logical;
	struct btrfs_root	*root;
	struct btrfs_work	work;
	int			mirror_num;
};

126 127 128 129 130 131 132 133 134 135 136 137 138
struct scrub_warning {
	struct btrfs_path	*path;
	u64			extent_item_size;
	char			*scratch_buf;
	char			*msg_buf;
	const char		*errstr;
	sector_t		sector;
	u64			logical;
	struct btrfs_device	*dev;
	int			msg_bufsize;
	int			scratch_bufsize;
};

139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170

static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
static int scrub_setup_recheck_block(struct scrub_dev *sdev,
				     struct btrfs_mapping_tree *map_tree,
				     u64 length, u64 logical,
				     struct scrub_block *sblock);
static int scrub_recheck_block(struct btrfs_fs_info *fs_info,
			       struct scrub_block *sblock, int is_metadata,
			       int have_csum, u8 *csum, u64 generation,
			       u16 csum_size);
static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
					 struct scrub_block *sblock,
					 int is_metadata, int have_csum,
					 const u8 *csum, u64 generation,
					 u16 csum_size);
static void scrub_complete_bio_end_io(struct bio *bio, int err);
static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
					     struct scrub_block *sblock_good,
					     int force_write);
static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
					    struct scrub_block *sblock_good,
					    int page_num, int force_write);
static int scrub_checksum_data(struct scrub_block *sblock);
static int scrub_checksum_tree_block(struct scrub_block *sblock);
static int scrub_checksum_super(struct scrub_block *sblock);
static void scrub_block_get(struct scrub_block *sblock);
static void scrub_block_put(struct scrub_block *sblock);
static int scrub_add_page_to_bio(struct scrub_dev *sdev,
				 struct scrub_page *spage);
static int scrub_pages(struct scrub_dev *sdev, u64 logical, u64 len,
		       u64 physical, u64 flags, u64 gen, int mirror_num,
		       u8 *csum, int force);
S
Stefan Behrens 已提交
171
static void scrub_bio_end_io(struct bio *bio, int err);
172 173
static void scrub_bio_end_io_worker(struct btrfs_work *work);
static void scrub_block_complete(struct scrub_block *sblock);
S
Stefan Behrens 已提交
174 175


A
Arne Jansen 已提交
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
static void scrub_free_csums(struct scrub_dev *sdev)
{
	while (!list_empty(&sdev->csum_list)) {
		struct btrfs_ordered_sum *sum;
		sum = list_first_entry(&sdev->csum_list,
				       struct btrfs_ordered_sum, list);
		list_del(&sum->list);
		kfree(sum);
	}
}

static noinline_for_stack void scrub_free_dev(struct scrub_dev *sdev)
{
	int i;

	if (!sdev)
		return;

194 195 196 197 198 199 200 201 202 203 204 205
	/* this can happen when scrub is cancelled */
	if (sdev->curr != -1) {
		struct scrub_bio *sbio = sdev->bios[sdev->curr];

		for (i = 0; i < sbio->page_count; i++) {
			BUG_ON(!sbio->pagev[i]);
			BUG_ON(!sbio->pagev[i]->page);
			scrub_block_put(sbio->pagev[i]->sblock);
		}
		bio_put(sbio->bio);
	}

A
Arne Jansen 已提交
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
	for (i = 0; i < SCRUB_BIOS_PER_DEV; ++i) {
		struct scrub_bio *sbio = sdev->bios[i];

		if (!sbio)
			break;
		kfree(sbio);
	}

	scrub_free_csums(sdev);
	kfree(sdev);
}

static noinline_for_stack
struct scrub_dev *scrub_setup_dev(struct btrfs_device *dev)
{
	struct scrub_dev *sdev;
	int		i;
	struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
224
	int pages_per_bio;
A
Arne Jansen 已提交
225

226 227
	pages_per_bio = min_t(int, SCRUB_PAGES_PER_BIO,
			      bio_get_nr_vecs(dev->bdev));
A
Arne Jansen 已提交
228 229 230 231
	sdev = kzalloc(sizeof(*sdev), GFP_NOFS);
	if (!sdev)
		goto nomem;
	sdev->dev = dev;
232 233
	sdev->pages_per_bio = pages_per_bio;
	sdev->curr = -1;
A
Arne Jansen 已提交
234 235 236 237 238 239 240 241 242 243
	for (i = 0; i < SCRUB_BIOS_PER_DEV; ++i) {
		struct scrub_bio *sbio;

		sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
		if (!sbio)
			goto nomem;
		sdev->bios[i] = sbio;

		sbio->index = i;
		sbio->sdev = sdev;
244 245
		sbio->page_count = 0;
		sbio->work.func = scrub_bio_end_io_worker;
A
Arne Jansen 已提交
246 247 248

		if (i != SCRUB_BIOS_PER_DEV-1)
			sdev->bios[i]->next_free = i + 1;
249
		else
A
Arne Jansen 已提交
250 251 252
			sdev->bios[i]->next_free = -1;
	}
	sdev->first_free = 0;
253 254 255
	sdev->nodesize = dev->dev_root->nodesize;
	sdev->leafsize = dev->dev_root->leafsize;
	sdev->sectorsize = dev->dev_root->sectorsize;
A
Arne Jansen 已提交
256
	atomic_set(&sdev->in_flight, 0);
257
	atomic_set(&sdev->fixup_cnt, 0);
A
Arne Jansen 已提交
258
	atomic_set(&sdev->cancel_req, 0);
259
	sdev->csum_size = btrfs_super_csum_size(fs_info->super_copy);
A
Arne Jansen 已提交
260 261 262 263 264 265 266 267 268 269 270 271
	INIT_LIST_HEAD(&sdev->csum_list);

	spin_lock_init(&sdev->list_lock);
	spin_lock_init(&sdev->stat_lock);
	init_waitqueue_head(&sdev->list_wait);
	return sdev;

nomem:
	scrub_free_dev(sdev);
	return ERR_PTR(-ENOMEM);
}

272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root, void *ctx)
{
	u64 isize;
	u32 nlink;
	int ret;
	int i;
	struct extent_buffer *eb;
	struct btrfs_inode_item *inode_item;
	struct scrub_warning *swarn = ctx;
	struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
	struct inode_fs_paths *ipath = NULL;
	struct btrfs_root *local_root;
	struct btrfs_key root_key;

	root_key.objectid = root;
	root_key.type = BTRFS_ROOT_ITEM_KEY;
	root_key.offset = (u64)-1;
	local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
	if (IS_ERR(local_root)) {
		ret = PTR_ERR(local_root);
		goto err;
	}

	ret = inode_item_info(inum, 0, local_root, swarn->path);
	if (ret) {
		btrfs_release_path(swarn->path);
		goto err;
	}

	eb = swarn->path->nodes[0];
	inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
					struct btrfs_inode_item);
	isize = btrfs_inode_size(eb, inode_item);
	nlink = btrfs_inode_nlink(eb, inode_item);
	btrfs_release_path(swarn->path);

	ipath = init_ipath(4096, local_root, swarn->path);
309 310 311 312 313
	if (IS_ERR(ipath)) {
		ret = PTR_ERR(ipath);
		ipath = NULL;
		goto err;
	}
314 315 316 317 318 319 320 321 322 323
	ret = paths_from_inode(inum, ipath);

	if (ret < 0)
		goto err;

	/*
	 * we deliberately ignore the bit ipath might have been too small to
	 * hold all of the paths here
	 */
	for (i = 0; i < ipath->fspath->elem_cnt; ++i)
324
		printk_in_rcu(KERN_WARNING "btrfs: %s at logical %llu on dev "
325 326
			"%s, sector %llu, root %llu, inode %llu, offset %llu, "
			"length %llu, links %u (path: %s)\n", swarn->errstr,
327
			swarn->logical, rcu_str_deref(swarn->dev->name),
328 329
			(unsigned long long)swarn->sector, root, inum, offset,
			min(isize - offset, (u64)PAGE_SIZE), nlink,
330
			(char *)(unsigned long)ipath->fspath->val[i]);
331 332 333 334 335

	free_ipath(ipath);
	return 0;

err:
336
	printk_in_rcu(KERN_WARNING "btrfs: %s at logical %llu on dev "
337 338
		"%s, sector %llu, root %llu, inode %llu, offset %llu: path "
		"resolving failed with ret=%d\n", swarn->errstr,
339
		swarn->logical, rcu_str_deref(swarn->dev->name),
340 341 342 343 344 345
		(unsigned long long)swarn->sector, root, inum, offset, ret);

	free_ipath(ipath);
	return 0;
}

346
static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
347
{
348
	struct btrfs_device *dev = sblock->sdev->dev;
349 350 351 352 353 354
	struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
	struct btrfs_path *path;
	struct btrfs_key found_key;
	struct extent_buffer *eb;
	struct btrfs_extent_item *ei;
	struct scrub_warning swarn;
355 356 357
	unsigned long ptr = 0;
	u64 extent_item_pos;
	u64 flags = 0;
358
	u64 ref_root;
359
	u32 item_size;
360 361
	u8 ref_level;
	const int bufsize = 4096;
362
	int ret;
363 364 365 366 367

	path = btrfs_alloc_path();

	swarn.scratch_buf = kmalloc(bufsize, GFP_NOFS);
	swarn.msg_buf = kmalloc(bufsize, GFP_NOFS);
368 369 370
	BUG_ON(sblock->page_count < 1);
	swarn.sector = (sblock->pagev[0].physical) >> 9;
	swarn.logical = sblock->pagev[0].logical;
371 372 373 374 375 376 377 378
	swarn.errstr = errstr;
	swarn.dev = dev;
	swarn.msg_bufsize = bufsize;
	swarn.scratch_bufsize = bufsize;

	if (!path || !swarn.scratch_buf || !swarn.msg_buf)
		goto out;

379 380
	ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
				  &flags);
381 382 383
	if (ret < 0)
		goto out;

J
Jan Schmidt 已提交
384
	extent_item_pos = swarn.logical - found_key.objectid;
385 386 387 388 389
	swarn.extent_item_size = found_key.offset;

	eb = path->nodes[0];
	ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
	item_size = btrfs_item_size_nr(eb, path->slots[0]);
J
Jan Schmidt 已提交
390
	btrfs_release_path(path);
391

392
	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
393 394 395
		do {
			ret = tree_backref_for_extent(&ptr, eb, ei, item_size,
							&ref_root, &ref_level);
396
			printk_in_rcu(KERN_WARNING
S
Stefan Behrens 已提交
397
				"btrfs: %s at logical %llu on dev %s, "
398
				"sector %llu: metadata %s (level %d) in tree "
399 400
				"%llu\n", errstr, swarn.logical,
				rcu_str_deref(dev->name),
401 402 403 404 405 406 407
				(unsigned long long)swarn.sector,
				ref_level ? "node" : "leaf",
				ret < 0 ? -1 : ref_level,
				ret < 0 ? -1 : ref_root);
		} while (ret != 1);
	} else {
		swarn.path = path;
408 409
		iterate_extent_inodes(fs_info, found_key.objectid,
					extent_item_pos, 1,
410 411 412 413 414 415 416 417 418
					scrub_print_warning_inode, &swarn);
	}

out:
	btrfs_free_path(path);
	kfree(swarn.scratch_buf);
	kfree(swarn.msg_buf);
}

419 420
static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *ctx)
{
421
	struct page *page = NULL;
422 423 424
	unsigned long index;
	struct scrub_fixup_nodatasum *fixup = ctx;
	int ret;
425
	int corrected = 0;
426
	struct btrfs_key key;
427
	struct inode *inode = NULL;
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
	u64 end = offset + PAGE_SIZE - 1;
	struct btrfs_root *local_root;

	key.objectid = root;
	key.type = BTRFS_ROOT_ITEM_KEY;
	key.offset = (u64)-1;
	local_root = btrfs_read_fs_root_no_name(fixup->root->fs_info, &key);
	if (IS_ERR(local_root))
		return PTR_ERR(local_root);

	key.type = BTRFS_INODE_ITEM_KEY;
	key.objectid = inum;
	key.offset = 0;
	inode = btrfs_iget(fixup->root->fs_info->sb, &key, local_root, NULL);
	if (IS_ERR(inode))
		return PTR_ERR(inode);

	index = offset >> PAGE_CACHE_SHIFT;

	page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
	if (!page) {
		ret = -ENOMEM;
		goto out;
	}

	if (PageUptodate(page)) {
		struct btrfs_mapping_tree *map_tree;
		if (PageDirty(page)) {
			/*
			 * we need to write the data to the defect sector. the
			 * data that was in that sector is not in memory,
			 * because the page was modified. we must not write the
			 * modified page to that sector.
			 *
			 * TODO: what could be done here: wait for the delalloc
			 *       runner to write out that page (might involve
			 *       COW) and see whether the sector is still
			 *       referenced afterwards.
			 *
			 * For the meantime, we'll treat this error
			 * incorrectable, although there is a chance that a
			 * later scrub will find the bad sector again and that
			 * there's no dirty page in memory, then.
			 */
			ret = -EIO;
			goto out;
		}
		map_tree = &BTRFS_I(inode)->root->fs_info->mapping_tree;
		ret = repair_io_failure(map_tree, offset, PAGE_SIZE,
					fixup->logical, page,
					fixup->mirror_num);
		unlock_page(page);
		corrected = !ret;
	} else {
		/*
		 * we need to get good data first. the general readpage path
		 * will call repair_io_failure for us, we just have to make
		 * sure we read the bad mirror.
		 */
		ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
					EXTENT_DAMAGED, GFP_NOFS);
		if (ret) {
			/* set_extent_bits should give proper error */
			WARN_ON(ret > 0);
			if (ret > 0)
				ret = -EFAULT;
			goto out;
		}

		ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
						btrfs_get_extent,
						fixup->mirror_num);
		wait_on_page_locked(page);

		corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
						end, EXTENT_DAMAGED, 0, NULL);
		if (!corrected)
			clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
						EXTENT_DAMAGED, GFP_NOFS);
	}

out:
	if (page)
		put_page(page);
	if (inode)
		iput(inode);
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586

	if (ret < 0)
		return ret;

	if (ret == 0 && corrected) {
		/*
		 * we only need to call readpage for one of the inodes belonging
		 * to this extent. so make iterate_extent_inodes stop
		 */
		return 1;
	}

	return -EIO;
}

static void scrub_fixup_nodatasum(struct btrfs_work *work)
{
	int ret;
	struct scrub_fixup_nodatasum *fixup;
	struct scrub_dev *sdev;
	struct btrfs_trans_handle *trans = NULL;
	struct btrfs_fs_info *fs_info;
	struct btrfs_path *path;
	int uncorrectable = 0;

	fixup = container_of(work, struct scrub_fixup_nodatasum, work);
	sdev = fixup->sdev;
	fs_info = fixup->root->fs_info;

	path = btrfs_alloc_path();
	if (!path) {
		spin_lock(&sdev->stat_lock);
		++sdev->stat.malloc_errors;
		spin_unlock(&sdev->stat_lock);
		uncorrectable = 1;
		goto out;
	}

	trans = btrfs_join_transaction(fixup->root);
	if (IS_ERR(trans)) {
		uncorrectable = 1;
		goto out;
	}

	/*
	 * the idea is to trigger a regular read through the standard path. we
	 * read a page from the (failed) logical address by specifying the
	 * corresponding copynum of the failed sector. thus, that readpage is
	 * expected to fail.
	 * that is the point where on-the-fly error correction will kick in
	 * (once it's finished) and rewrite the failed sector if a good copy
	 * can be found.
	 */
	ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
						path, scrub_fixup_readpage,
						fixup);
	if (ret < 0) {
		uncorrectable = 1;
		goto out;
	}
	WARN_ON(ret != 1);

	spin_lock(&sdev->stat_lock);
	++sdev->stat.corrected_errors;
	spin_unlock(&sdev->stat_lock);

out:
	if (trans && !IS_ERR(trans))
		btrfs_end_transaction(trans, fixup->root);
	if (uncorrectable) {
		spin_lock(&sdev->stat_lock);
		++sdev->stat.uncorrectable_errors;
		spin_unlock(&sdev->stat_lock);
587 588

		printk_ratelimited_in_rcu(KERN_ERR
589
			"btrfs: unable to fixup (nodatasum) error at logical %llu on dev %s\n",
590 591
			(unsigned long long)fixup->logical,
			rcu_str_deref(sdev->dev->name));
592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
	}

	btrfs_free_path(path);
	kfree(fixup);

	/* see caller why we're pretending to be paused in the scrub counters */
	mutex_lock(&fs_info->scrub_lock);
	atomic_dec(&fs_info->scrubs_running);
	atomic_dec(&fs_info->scrubs_paused);
	mutex_unlock(&fs_info->scrub_lock);
	atomic_dec(&sdev->fixup_cnt);
	wake_up(&fs_info->scrub_pause_wait);
	wake_up(&sdev->list_wait);
}

A
Arne Jansen 已提交
607
/*
608 609 610 611 612 613
 * scrub_handle_errored_block gets called when either verification of the
 * pages failed or the bio failed to read, e.g. with EIO. In the latter
 * case, this function handles all pages in the bio, even though only one
 * may be bad.
 * The goal of this function is to repair the errored block by using the
 * contents of one of the mirrors.
A
Arne Jansen 已提交
614
 */
615
static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
A
Arne Jansen 已提交
616
{
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
	struct scrub_dev *sdev = sblock_to_check->sdev;
	struct btrfs_fs_info *fs_info;
	u64 length;
	u64 logical;
	u64 generation;
	unsigned int failed_mirror_index;
	unsigned int is_metadata;
	unsigned int have_csum;
	u8 *csum;
	struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
	struct scrub_block *sblock_bad;
	int ret;
	int mirror_index;
	int page_num;
	int success;
632
	static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
633 634 635 636 637 638 639 640 641 642 643 644 645
				      DEFAULT_RATELIMIT_BURST);

	BUG_ON(sblock_to_check->page_count < 1);
	fs_info = sdev->dev->dev_root->fs_info;
	length = sblock_to_check->page_count * PAGE_SIZE;
	logical = sblock_to_check->pagev[0].logical;
	generation = sblock_to_check->pagev[0].generation;
	BUG_ON(sblock_to_check->pagev[0].mirror_num < 1);
	failed_mirror_index = sblock_to_check->pagev[0].mirror_num - 1;
	is_metadata = !(sblock_to_check->pagev[0].flags &
			BTRFS_EXTENT_FLAG_DATA);
	have_csum = sblock_to_check->pagev[0].have_csum;
	csum = sblock_to_check->pagev[0].csum;
646

647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
	/*
	 * read all mirrors one after the other. This includes to
	 * re-read the extent or metadata block that failed (that was
	 * the cause that this fixup code is called) another time,
	 * page by page this time in order to know which pages
	 * caused I/O errors and which ones are good (for all mirrors).
	 * It is the goal to handle the situation when more than one
	 * mirror contains I/O errors, but the errors do not
	 * overlap, i.e. the data can be repaired by selecting the
	 * pages from those mirrors without I/O error on the
	 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
	 * would be that mirror #1 has an I/O error on the first page,
	 * the second page is good, and mirror #2 has an I/O error on
	 * the second page, but the first page is good.
	 * Then the first page of the first mirror can be repaired by
	 * taking the first page of the second mirror, and the
	 * second page of the second mirror can be repaired by
	 * copying the contents of the 2nd page of the 1st mirror.
	 * One more note: if the pages of one mirror contain I/O
	 * errors, the checksum cannot be verified. In order to get
	 * the best data for repairing, the first attempt is to find
	 * a mirror without I/O errors and with a validated checksum.
	 * Only if this is not possible, the pages are picked from
	 * mirrors with I/O errors without considering the checksum.
	 * If the latter is the case, at the end, the checksum of the
	 * repaired area is verified in order to correctly maintain
	 * the statistics.
	 */

	sblocks_for_recheck = kzalloc(BTRFS_MAX_MIRRORS *
				     sizeof(*sblocks_for_recheck),
				     GFP_NOFS);
	if (!sblocks_for_recheck) {
		spin_lock(&sdev->stat_lock);
		sdev->stat.malloc_errors++;
		sdev->stat.read_errors++;
		sdev->stat.uncorrectable_errors++;
		spin_unlock(&sdev->stat_lock);
685 686
		btrfs_dev_stat_inc_and_print(sdev->dev,
					     BTRFS_DEV_STAT_READ_ERRS);
687
		goto out;
A
Arne Jansen 已提交
688 689
	}

690 691 692 693 694 695 696 697
	/* setup the context, map the logical blocks and alloc the pages */
	ret = scrub_setup_recheck_block(sdev, &fs_info->mapping_tree, length,
					logical, sblocks_for_recheck);
	if (ret) {
		spin_lock(&sdev->stat_lock);
		sdev->stat.read_errors++;
		sdev->stat.uncorrectable_errors++;
		spin_unlock(&sdev->stat_lock);
698 699
		btrfs_dev_stat_inc_and_print(sdev->dev,
					     BTRFS_DEV_STAT_READ_ERRS);
700 701 702 703
		goto out;
	}
	BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
	sblock_bad = sblocks_for_recheck + failed_mirror_index;
704

705 706 707 708 709 710 711 712
	/* build and submit the bios for the failed mirror, check checksums */
	ret = scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
				  csum, generation, sdev->csum_size);
	if (ret) {
		spin_lock(&sdev->stat_lock);
		sdev->stat.read_errors++;
		sdev->stat.uncorrectable_errors++;
		spin_unlock(&sdev->stat_lock);
713 714
		btrfs_dev_stat_inc_and_print(sdev->dev,
					     BTRFS_DEV_STAT_READ_ERRS);
715 716
		goto out;
	}
A
Arne Jansen 已提交
717

718 719 720 721 722 723 724 725 726 727 728 729 730
	if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
	    sblock_bad->no_io_error_seen) {
		/*
		 * the error disappeared after reading page by page, or
		 * the area was part of a huge bio and other parts of the
		 * bio caused I/O errors, or the block layer merged several
		 * read requests into one and the error is caused by a
		 * different bio (usually one of the two latter cases is
		 * the cause)
		 */
		spin_lock(&sdev->stat_lock);
		sdev->stat.unverified_errors++;
		spin_unlock(&sdev->stat_lock);
A
Arne Jansen 已提交
731

732
		goto out;
A
Arne Jansen 已提交
733 734
	}

735 736 737 738 739 740
	if (!sblock_bad->no_io_error_seen) {
		spin_lock(&sdev->stat_lock);
		sdev->stat.read_errors++;
		spin_unlock(&sdev->stat_lock);
		if (__ratelimit(&_rs))
			scrub_print_warning("i/o error", sblock_to_check);
741 742
		btrfs_dev_stat_inc_and_print(sdev->dev,
					     BTRFS_DEV_STAT_READ_ERRS);
743 744 745 746 747 748
	} else if (sblock_bad->checksum_error) {
		spin_lock(&sdev->stat_lock);
		sdev->stat.csum_errors++;
		spin_unlock(&sdev->stat_lock);
		if (__ratelimit(&_rs))
			scrub_print_warning("checksum error", sblock_to_check);
749 750
		btrfs_dev_stat_inc_and_print(sdev->dev,
					     BTRFS_DEV_STAT_CORRUPTION_ERRS);
751 752 753 754 755 756 757
	} else if (sblock_bad->header_error) {
		spin_lock(&sdev->stat_lock);
		sdev->stat.verify_errors++;
		spin_unlock(&sdev->stat_lock);
		if (__ratelimit(&_rs))
			scrub_print_warning("checksum/header error",
					    sblock_to_check);
758 759 760 761 762 763
		if (sblock_bad->generation_error)
			btrfs_dev_stat_inc_and_print(sdev->dev,
				BTRFS_DEV_STAT_GENERATION_ERRS);
		else
			btrfs_dev_stat_inc_and_print(sdev->dev,
				BTRFS_DEV_STAT_CORRUPTION_ERRS);
764
	}
A
Arne Jansen 已提交
765

766 767
	if (sdev->readonly)
		goto did_not_correct_error;
A
Arne Jansen 已提交
768

769 770
	if (!is_metadata && !have_csum) {
		struct scrub_fixup_nodatasum *fixup_nodatasum;
A
Arne Jansen 已提交
771

772 773 774 775 776 777 778 779 780 781 782 783 784 785
		/*
		 * !is_metadata and !have_csum, this means that the data
		 * might not be COW'ed, that it might be modified
		 * concurrently. The general strategy to work on the
		 * commit root does not help in the case when COW is not
		 * used.
		 */
		fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
		if (!fixup_nodatasum)
			goto did_not_correct_error;
		fixup_nodatasum->sdev = sdev;
		fixup_nodatasum->logical = logical;
		fixup_nodatasum->root = fs_info->extent_root;
		fixup_nodatasum->mirror_num = failed_mirror_index + 1;
A
Arne Jansen 已提交
786
		/*
787 788 789 790 791 792 793
		 * increment scrubs_running to prevent cancel requests from
		 * completing as long as a fixup worker is running. we must also
		 * increment scrubs_paused to prevent deadlocking on pause
		 * requests used for transactions commits (as the worker uses a
		 * transaction context). it is safe to regard the fixup worker
		 * as paused for all matters practical. effectively, we only
		 * avoid cancellation requests from completing.
A
Arne Jansen 已提交
794
		 */
795 796 797 798 799
		mutex_lock(&fs_info->scrub_lock);
		atomic_inc(&fs_info->scrubs_running);
		atomic_inc(&fs_info->scrubs_paused);
		mutex_unlock(&fs_info->scrub_lock);
		atomic_inc(&sdev->fixup_cnt);
800 801 802 803
		fixup_nodatasum->work.func = scrub_fixup_nodatasum;
		btrfs_queue_worker(&fs_info->scrub_workers,
				   &fixup_nodatasum->work);
		goto out;
A
Arne Jansen 已提交
804 805
	}

806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823
	/*
	 * now build and submit the bios for the other mirrors, check
	 * checksums
	 */
	for (mirror_index = 0;
	     mirror_index < BTRFS_MAX_MIRRORS &&
	     sblocks_for_recheck[mirror_index].page_count > 0;
	     mirror_index++) {
		if (mirror_index == failed_mirror_index)
			continue;

		/* build and submit the bios, check checksums */
		ret = scrub_recheck_block(fs_info,
					  sblocks_for_recheck + mirror_index,
					  is_metadata, have_csum, csum,
					  generation, sdev->csum_size);
		if (ret)
			goto did_not_correct_error;
A
Arne Jansen 已提交
824 825
	}

826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857
	/*
	 * first try to pick the mirror which is completely without I/O
	 * errors and also does not have a checksum error.
	 * If one is found, and if a checksum is present, the full block
	 * that is known to contain an error is rewritten. Afterwards
	 * the block is known to be corrected.
	 * If a mirror is found which is completely correct, and no
	 * checksum is present, only those pages are rewritten that had
	 * an I/O error in the block to be repaired, since it cannot be
	 * determined, which copy of the other pages is better (and it
	 * could happen otherwise that a correct page would be
	 * overwritten by a bad one).
	 */
	for (mirror_index = 0;
	     mirror_index < BTRFS_MAX_MIRRORS &&
	     sblocks_for_recheck[mirror_index].page_count > 0;
	     mirror_index++) {
		struct scrub_block *sblock_other = sblocks_for_recheck +
						   mirror_index;

		if (!sblock_other->header_error &&
		    !sblock_other->checksum_error &&
		    sblock_other->no_io_error_seen) {
			int force_write = is_metadata || have_csum;

			ret = scrub_repair_block_from_good_copy(sblock_bad,
								sblock_other,
								force_write);
			if (0 == ret)
				goto corrected_error;
		}
	}
A
Arne Jansen 已提交
858 859

	/*
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
	 * in case of I/O errors in the area that is supposed to be
	 * repaired, continue by picking good copies of those pages.
	 * Select the good pages from mirrors to rewrite bad pages from
	 * the area to fix. Afterwards verify the checksum of the block
	 * that is supposed to be repaired. This verification step is
	 * only done for the purpose of statistic counting and for the
	 * final scrub report, whether errors remain.
	 * A perfect algorithm could make use of the checksum and try
	 * all possible combinations of pages from the different mirrors
	 * until the checksum verification succeeds. For example, when
	 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
	 * of mirror #2 is readable but the final checksum test fails,
	 * then the 2nd page of mirror #3 could be tried, whether now
	 * the final checksum succeedes. But this would be a rare
	 * exception and is therefore not implemented. At least it is
	 * avoided that the good copy is overwritten.
	 * A more useful improvement would be to pick the sectors
	 * without I/O error based on sector sizes (512 bytes on legacy
	 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
	 * mirror could be repaired by taking 512 byte of a different
	 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
	 * area are unreadable.
A
Arne Jansen 已提交
882 883
	 */

884 885 886 887 888 889 890 891 892
	/* can only fix I/O errors from here on */
	if (sblock_bad->no_io_error_seen)
		goto did_not_correct_error;

	success = 1;
	for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
		struct scrub_page *page_bad = sblock_bad->pagev + page_num;

		if (!page_bad->io_error)
A
Arne Jansen 已提交
893
			continue;
894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911

		for (mirror_index = 0;
		     mirror_index < BTRFS_MAX_MIRRORS &&
		     sblocks_for_recheck[mirror_index].page_count > 0;
		     mirror_index++) {
			struct scrub_block *sblock_other = sblocks_for_recheck +
							   mirror_index;
			struct scrub_page *page_other = sblock_other->pagev +
							page_num;

			if (!page_other->io_error) {
				ret = scrub_repair_page_from_good_copy(
					sblock_bad, sblock_other, page_num, 0);
				if (0 == ret) {
					page_bad->io_error = 0;
					break; /* succeeded for this page */
				}
			}
I
Ilya Dryomov 已提交
912
		}
A
Arne Jansen 已提交
913

914 915 916 917
		if (page_bad->io_error) {
			/* did not find a mirror to copy the page from */
			success = 0;
		}
A
Arne Jansen 已提交
918 919
	}

920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944
	if (success) {
		if (is_metadata || have_csum) {
			/*
			 * need to verify the checksum now that all
			 * sectors on disk are repaired (the write
			 * request for data to be repaired is on its way).
			 * Just be lazy and use scrub_recheck_block()
			 * which re-reads the data before the checksum
			 * is verified, but most likely the data comes out
			 * of the page cache.
			 */
			ret = scrub_recheck_block(fs_info, sblock_bad,
						  is_metadata, have_csum, csum,
						  generation, sdev->csum_size);
			if (!ret && !sblock_bad->header_error &&
			    !sblock_bad->checksum_error &&
			    sblock_bad->no_io_error_seen)
				goto corrected_error;
			else
				goto did_not_correct_error;
		} else {
corrected_error:
			spin_lock(&sdev->stat_lock);
			sdev->stat.corrected_errors++;
			spin_unlock(&sdev->stat_lock);
945
			printk_ratelimited_in_rcu(KERN_ERR
946
				"btrfs: fixed up error at logical %llu on dev %s\n",
947 948
				(unsigned long long)logical,
				rcu_str_deref(sdev->dev->name));
A
Arne Jansen 已提交
949
		}
950 951 952 953 954
	} else {
did_not_correct_error:
		spin_lock(&sdev->stat_lock);
		sdev->stat.uncorrectable_errors++;
		spin_unlock(&sdev->stat_lock);
955
		printk_ratelimited_in_rcu(KERN_ERR
956
			"btrfs: unable to fixup (regular) error at logical %llu on dev %s\n",
957 958
			(unsigned long long)logical,
			rcu_str_deref(sdev->dev->name));
I
Ilya Dryomov 已提交
959
	}
A
Arne Jansen 已提交
960

961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
out:
	if (sblocks_for_recheck) {
		for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
		     mirror_index++) {
			struct scrub_block *sblock = sblocks_for_recheck +
						     mirror_index;
			int page_index;

			for (page_index = 0; page_index < SCRUB_PAGES_PER_BIO;
			     page_index++)
				if (sblock->pagev[page_index].page)
					__free_page(
						sblock->pagev[page_index].page);
		}
		kfree(sblocks_for_recheck);
	}
A
Arne Jansen 已提交
977

978 979
	return 0;
}
A
Arne Jansen 已提交
980

981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
static int scrub_setup_recheck_block(struct scrub_dev *sdev,
				     struct btrfs_mapping_tree *map_tree,
				     u64 length, u64 logical,
				     struct scrub_block *sblocks_for_recheck)
{
	int page_index;
	int mirror_index;
	int ret;

	/*
	 * note: the three members sdev, ref_count and outstanding_pages
	 * are not used (and not set) in the blocks that are used for
	 * the recheck procedure
	 */

	page_index = 0;
	while (length > 0) {
		u64 sublen = min_t(u64, length, PAGE_SIZE);
		u64 mapped_length = sublen;
		struct btrfs_bio *bbio = NULL;
A
Arne Jansen 已提交
1001

1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
		/*
		 * with a length of PAGE_SIZE, each returned stripe
		 * represents one mirror
		 */
		ret = btrfs_map_block(map_tree, WRITE, logical, &mapped_length,
				      &bbio, 0);
		if (ret || !bbio || mapped_length < sublen) {
			kfree(bbio);
			return -EIO;
		}
A
Arne Jansen 已提交
1012

1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
		BUG_ON(page_index >= SCRUB_PAGES_PER_BIO);
		for (mirror_index = 0; mirror_index < (int)bbio->num_stripes;
		     mirror_index++) {
			struct scrub_block *sblock;
			struct scrub_page *page;

			if (mirror_index >= BTRFS_MAX_MIRRORS)
				continue;

			sblock = sblocks_for_recheck + mirror_index;
			page = sblock->pagev + page_index;
			page->logical = logical;
			page->physical = bbio->stripes[mirror_index].physical;
1026 1027
			/* for missing devices, dev->bdev is NULL */
			page->dev = bbio->stripes[mirror_index].dev;
1028 1029 1030 1031 1032 1033
			page->mirror_num = mirror_index + 1;
			page->page = alloc_page(GFP_NOFS);
			if (!page->page) {
				spin_lock(&sdev->stat_lock);
				sdev->stat.malloc_errors++;
				spin_unlock(&sdev->stat_lock);
1034
				kfree(bbio);
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
				return -ENOMEM;
			}
			sblock->page_count++;
		}
		kfree(bbio);
		length -= sublen;
		logical += sublen;
		page_index++;
	}

	return 0;
I
Ilya Dryomov 已提交
1046 1047
}

1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
/*
 * this function will check the on disk data for checksum errors, header
 * errors and read I/O errors. If any I/O errors happen, the exact pages
 * which are errored are marked as being bad. The goal is to enable scrub
 * to take those pages that are not errored from all the mirrors so that
 * the pages that are errored in the just handled mirror can be repaired.
 */
static int scrub_recheck_block(struct btrfs_fs_info *fs_info,
			       struct scrub_block *sblock, int is_metadata,
			       int have_csum, u8 *csum, u64 generation,
			       u16 csum_size)
I
Ilya Dryomov 已提交
1059
{
1060
	int page_num;
I
Ilya Dryomov 已提交
1061

1062 1063 1064
	sblock->no_io_error_seen = 1;
	sblock->header_error = 0;
	sblock->checksum_error = 0;
I
Ilya Dryomov 已提交
1065

1066 1067 1068 1069 1070 1071
	for (page_num = 0; page_num < sblock->page_count; page_num++) {
		struct bio *bio;
		int ret;
		struct scrub_page *page = sblock->pagev + page_num;
		DECLARE_COMPLETION_ONSTACK(complete);

1072
		if (page->dev->bdev == NULL) {
1073 1074 1075 1076 1077
			page->io_error = 1;
			sblock->no_io_error_seen = 0;
			continue;
		}

1078 1079
		BUG_ON(!page->page);
		bio = bio_alloc(GFP_NOFS, 1);
1080 1081
		if (!bio)
			return -EIO;
1082
		bio->bi_bdev = page->dev->bdev;
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
		bio->bi_sector = page->physical >> 9;
		bio->bi_end_io = scrub_complete_bio_end_io;
		bio->bi_private = &complete;

		ret = bio_add_page(bio, page->page, PAGE_SIZE, 0);
		if (PAGE_SIZE != ret) {
			bio_put(bio);
			return -EIO;
		}
		btrfsic_submit_bio(READ, bio);
I
Ilya Dryomov 已提交
1093

1094 1095
		/* this will also unplug the queue */
		wait_for_completion(&complete);
I
Ilya Dryomov 已提交
1096

1097 1098 1099 1100 1101
		page->io_error = !test_bit(BIO_UPTODATE, &bio->bi_flags);
		if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
			sblock->no_io_error_seen = 0;
		bio_put(bio);
	}
I
Ilya Dryomov 已提交
1102

1103 1104 1105 1106 1107 1108
	if (sblock->no_io_error_seen)
		scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
					     have_csum, csum, generation,
					     csum_size);

	return 0;
A
Arne Jansen 已提交
1109 1110
}

1111 1112 1113 1114 1115
static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
					 struct scrub_block *sblock,
					 int is_metadata, int have_csum,
					 const u8 *csum, u64 generation,
					 u16 csum_size)
A
Arne Jansen 已提交
1116
{
1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
	int page_num;
	u8 calculated_csum[BTRFS_CSUM_SIZE];
	u32 crc = ~(u32)0;
	struct btrfs_root *root = fs_info->extent_root;
	void *mapped_buffer;

	BUG_ON(!sblock->pagev[0].page);
	if (is_metadata) {
		struct btrfs_header *h;

1127
		mapped_buffer = kmap_atomic(sblock->pagev[0].page);
1128 1129 1130 1131 1132
		h = (struct btrfs_header *)mapped_buffer;

		if (sblock->pagev[0].logical != le64_to_cpu(h->bytenr) ||
		    memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE) ||
		    memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1133
			   BTRFS_UUID_SIZE)) {
1134
			sblock->header_error = 1;
1135 1136 1137 1138
		} else if (generation != le64_to_cpu(h->generation)) {
			sblock->header_error = 1;
			sblock->generation_error = 1;
		}
1139 1140 1141 1142
		csum = h->csum;
	} else {
		if (!have_csum)
			return;
A
Arne Jansen 已提交
1143

1144
		mapped_buffer = kmap_atomic(sblock->pagev[0].page);
1145
	}
A
Arne Jansen 已提交
1146

1147 1148 1149 1150 1151 1152 1153 1154 1155
	for (page_num = 0;;) {
		if (page_num == 0 && is_metadata)
			crc = btrfs_csum_data(root,
				((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
				crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
		else
			crc = btrfs_csum_data(root, mapped_buffer, crc,
					      PAGE_SIZE);

1156
		kunmap_atomic(mapped_buffer);
1157 1158 1159 1160 1161
		page_num++;
		if (page_num >= sblock->page_count)
			break;
		BUG_ON(!sblock->pagev[page_num].page);

1162
		mapped_buffer = kmap_atomic(sblock->pagev[page_num].page);
1163 1164 1165 1166 1167
	}

	btrfs_csum_final(crc, calculated_csum);
	if (memcmp(calculated_csum, csum, csum_size))
		sblock->checksum_error = 1;
A
Arne Jansen 已提交
1168 1169
}

1170
static void scrub_complete_bio_end_io(struct bio *bio, int err)
A
Arne Jansen 已提交
1171
{
1172 1173
	complete((struct completion *)bio->bi_private);
}
A
Arne Jansen 已提交
1174

1175 1176 1177 1178 1179 1180
static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
					     struct scrub_block *sblock_good,
					     int force_write)
{
	int page_num;
	int ret = 0;
I
Ilya Dryomov 已提交
1181

1182 1183
	for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
		int ret_sub;
I
Ilya Dryomov 已提交
1184

1185 1186 1187 1188 1189 1190
		ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
							   sblock_good,
							   page_num,
							   force_write);
		if (ret_sub)
			ret = ret_sub;
A
Arne Jansen 已提交
1191
	}
1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211

	return ret;
}

static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
					    struct scrub_block *sblock_good,
					    int page_num, int force_write)
{
	struct scrub_page *page_bad = sblock_bad->pagev + page_num;
	struct scrub_page *page_good = sblock_good->pagev + page_num;

	BUG_ON(sblock_bad->pagev[page_num].page == NULL);
	BUG_ON(sblock_good->pagev[page_num].page == NULL);
	if (force_write || sblock_bad->header_error ||
	    sblock_bad->checksum_error || page_bad->io_error) {
		struct bio *bio;
		int ret;
		DECLARE_COMPLETION_ONSTACK(complete);

		bio = bio_alloc(GFP_NOFS, 1);
1212 1213
		if (!bio)
			return -EIO;
1214
		bio->bi_bdev = page_bad->dev->bdev;
1215 1216 1217 1218 1219 1220 1221 1222
		bio->bi_sector = page_bad->physical >> 9;
		bio->bi_end_io = scrub_complete_bio_end_io;
		bio->bi_private = &complete;

		ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
		if (PAGE_SIZE != ret) {
			bio_put(bio);
			return -EIO;
1223
		}
1224 1225 1226 1227
		btrfsic_submit_bio(WRITE, bio);

		/* this will also unplug the queue */
		wait_for_completion(&complete);
1228 1229 1230 1231 1232 1233
		if (!bio_flagged(bio, BIO_UPTODATE)) {
			btrfs_dev_stat_inc_and_print(page_bad->dev,
				BTRFS_DEV_STAT_WRITE_ERRS);
			bio_put(bio);
			return -EIO;
		}
1234
		bio_put(bio);
A
Arne Jansen 已提交
1235 1236
	}

1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257
	return 0;
}

static void scrub_checksum(struct scrub_block *sblock)
{
	u64 flags;
	int ret;

	BUG_ON(sblock->page_count < 1);
	flags = sblock->pagev[0].flags;
	ret = 0;
	if (flags & BTRFS_EXTENT_FLAG_DATA)
		ret = scrub_checksum_data(sblock);
	else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
		ret = scrub_checksum_tree_block(sblock);
	else if (flags & BTRFS_EXTENT_FLAG_SUPER)
		(void)scrub_checksum_super(sblock);
	else
		WARN_ON(1);
	if (ret)
		scrub_handle_errored_block(sblock);
A
Arne Jansen 已提交
1258 1259
}

1260
static int scrub_checksum_data(struct scrub_block *sblock)
A
Arne Jansen 已提交
1261
{
1262
	struct scrub_dev *sdev = sblock->sdev;
A
Arne Jansen 已提交
1263
	u8 csum[BTRFS_CSUM_SIZE];
1264 1265 1266
	u8 *on_disk_csum;
	struct page *page;
	void *buffer;
A
Arne Jansen 已提交
1267 1268 1269
	u32 crc = ~(u32)0;
	int fail = 0;
	struct btrfs_root *root = sdev->dev->dev_root;
1270 1271
	u64 len;
	int index;
A
Arne Jansen 已提交
1272

1273 1274
	BUG_ON(sblock->page_count < 1);
	if (!sblock->pagev[0].have_csum)
A
Arne Jansen 已提交
1275 1276
		return 0;

1277 1278
	on_disk_csum = sblock->pagev[0].csum;
	page = sblock->pagev[0].page;
1279
	buffer = kmap_atomic(page);
1280 1281 1282 1283 1284 1285 1286

	len = sdev->sectorsize;
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, PAGE_SIZE);

		crc = btrfs_csum_data(root, buffer, crc, l);
1287
		kunmap_atomic(buffer);
1288 1289 1290 1291 1292 1293 1294
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
		BUG_ON(!sblock->pagev[index].page);
		page = sblock->pagev[index].page;
1295
		buffer = kmap_atomic(page);
1296 1297
	}

A
Arne Jansen 已提交
1298
	btrfs_csum_final(crc, csum);
1299
	if (memcmp(csum, on_disk_csum, sdev->csum_size))
A
Arne Jansen 已提交
1300 1301 1302 1303 1304
		fail = 1;

	return fail;
}

1305
static int scrub_checksum_tree_block(struct scrub_block *sblock)
A
Arne Jansen 已提交
1306
{
1307
	struct scrub_dev *sdev = sblock->sdev;
A
Arne Jansen 已提交
1308 1309 1310
	struct btrfs_header *h;
	struct btrfs_root *root = sdev->dev->dev_root;
	struct btrfs_fs_info *fs_info = root->fs_info;
1311 1312 1313 1314 1315 1316
	u8 calculated_csum[BTRFS_CSUM_SIZE];
	u8 on_disk_csum[BTRFS_CSUM_SIZE];
	struct page *page;
	void *mapped_buffer;
	u64 mapped_size;
	void *p;
A
Arne Jansen 已提交
1317 1318 1319
	u32 crc = ~(u32)0;
	int fail = 0;
	int crc_fail = 0;
1320 1321 1322 1323 1324
	u64 len;
	int index;

	BUG_ON(sblock->page_count < 1);
	page = sblock->pagev[0].page;
1325
	mapped_buffer = kmap_atomic(page);
1326 1327
	h = (struct btrfs_header *)mapped_buffer;
	memcpy(on_disk_csum, h->csum, sdev->csum_size);
A
Arne Jansen 已提交
1328 1329 1330 1331 1332 1333 1334

	/*
	 * we don't use the getter functions here, as we
	 * a) don't have an extent buffer and
	 * b) the page is already kmapped
	 */

1335
	if (sblock->pagev[0].logical != le64_to_cpu(h->bytenr))
A
Arne Jansen 已提交
1336 1337
		++fail;

1338
	if (sblock->pagev[0].generation != le64_to_cpu(h->generation))
A
Arne Jansen 已提交
1339 1340 1341 1342 1343 1344 1345 1346 1347
		++fail;

	if (memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
		++fail;

	if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
		   BTRFS_UUID_SIZE))
		++fail;

1348 1349 1350 1351 1352 1353 1354 1355 1356
	BUG_ON(sdev->nodesize != sdev->leafsize);
	len = sdev->nodesize - BTRFS_CSUM_SIZE;
	mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
	p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, mapped_size);

		crc = btrfs_csum_data(root, p, crc, l);
1357
		kunmap_atomic(mapped_buffer);
1358 1359 1360 1361 1362 1363 1364
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
		BUG_ON(!sblock->pagev[index].page);
		page = sblock->pagev[index].page;
1365
		mapped_buffer = kmap_atomic(page);
1366 1367 1368 1369 1370 1371
		mapped_size = PAGE_SIZE;
		p = mapped_buffer;
	}

	btrfs_csum_final(crc, calculated_csum);
	if (memcmp(calculated_csum, on_disk_csum, sdev->csum_size))
A
Arne Jansen 已提交
1372 1373 1374 1375 1376
		++crc_fail;

	return fail || crc_fail;
}

1377
static int scrub_checksum_super(struct scrub_block *sblock)
A
Arne Jansen 已提交
1378 1379
{
	struct btrfs_super_block *s;
1380
	struct scrub_dev *sdev = sblock->sdev;
A
Arne Jansen 已提交
1381 1382
	struct btrfs_root *root = sdev->dev->dev_root;
	struct btrfs_fs_info *fs_info = root->fs_info;
1383 1384 1385 1386 1387 1388
	u8 calculated_csum[BTRFS_CSUM_SIZE];
	u8 on_disk_csum[BTRFS_CSUM_SIZE];
	struct page *page;
	void *mapped_buffer;
	u64 mapped_size;
	void *p;
A
Arne Jansen 已提交
1389
	u32 crc = ~(u32)0;
1390 1391
	int fail_gen = 0;
	int fail_cor = 0;
1392 1393
	u64 len;
	int index;
A
Arne Jansen 已提交
1394

1395 1396
	BUG_ON(sblock->page_count < 1);
	page = sblock->pagev[0].page;
1397
	mapped_buffer = kmap_atomic(page);
1398 1399
	s = (struct btrfs_super_block *)mapped_buffer;
	memcpy(on_disk_csum, s->csum, sdev->csum_size);
A
Arne Jansen 已提交
1400

1401
	if (sblock->pagev[0].logical != le64_to_cpu(s->bytenr))
1402
		++fail_cor;
A
Arne Jansen 已提交
1403

1404
	if (sblock->pagev[0].generation != le64_to_cpu(s->generation))
1405
		++fail_gen;
A
Arne Jansen 已提交
1406 1407

	if (memcmp(s->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
1408
		++fail_cor;
A
Arne Jansen 已提交
1409

1410 1411 1412 1413 1414 1415 1416 1417
	len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
	mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
	p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
	index = 0;
	for (;;) {
		u64 l = min_t(u64, len, mapped_size);

		crc = btrfs_csum_data(root, p, crc, l);
1418
		kunmap_atomic(mapped_buffer);
1419 1420 1421 1422 1423 1424 1425
		len -= l;
		if (len == 0)
			break;
		index++;
		BUG_ON(index >= sblock->page_count);
		BUG_ON(!sblock->pagev[index].page);
		page = sblock->pagev[index].page;
1426
		mapped_buffer = kmap_atomic(page);
1427 1428 1429 1430 1431 1432
		mapped_size = PAGE_SIZE;
		p = mapped_buffer;
	}

	btrfs_csum_final(crc, calculated_csum);
	if (memcmp(calculated_csum, on_disk_csum, sdev->csum_size))
1433
		++fail_cor;
A
Arne Jansen 已提交
1434

1435
	if (fail_cor + fail_gen) {
A
Arne Jansen 已提交
1436 1437 1438 1439 1440 1441 1442 1443
		/*
		 * if we find an error in a super block, we just report it.
		 * They will get written with the next transaction commit
		 * anyway
		 */
		spin_lock(&sdev->stat_lock);
		++sdev->stat.super_errors;
		spin_unlock(&sdev->stat_lock);
1444 1445 1446 1447 1448 1449
		if (fail_cor)
			btrfs_dev_stat_inc_and_print(sdev->dev,
				BTRFS_DEV_STAT_CORRUPTION_ERRS);
		else
			btrfs_dev_stat_inc_and_print(sdev->dev,
				BTRFS_DEV_STAT_GENERATION_ERRS);
A
Arne Jansen 已提交
1450 1451
	}

1452
	return fail_cor + fail_gen;
A
Arne Jansen 已提交
1453 1454
}

1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
static void scrub_block_get(struct scrub_block *sblock)
{
	atomic_inc(&sblock->ref_count);
}

static void scrub_block_put(struct scrub_block *sblock)
{
	if (atomic_dec_and_test(&sblock->ref_count)) {
		int i;

		for (i = 0; i < sblock->page_count; i++)
			if (sblock->pagev[i].page)
				__free_page(sblock->pagev[i].page);
		kfree(sblock);
	}
}

S
Stefan Behrens 已提交
1472
static void scrub_submit(struct scrub_dev *sdev)
A
Arne Jansen 已提交
1473 1474 1475 1476
{
	struct scrub_bio *sbio;

	if (sdev->curr == -1)
S
Stefan Behrens 已提交
1477
		return;
A
Arne Jansen 已提交
1478 1479 1480 1481 1482

	sbio = sdev->bios[sdev->curr];
	sdev->curr = -1;
	atomic_inc(&sdev->in_flight);

1483
	btrfsic_submit_bio(READ, sbio->bio);
A
Arne Jansen 已提交
1484 1485
}

1486 1487
static int scrub_add_page_to_bio(struct scrub_dev *sdev,
				 struct scrub_page *spage)
A
Arne Jansen 已提交
1488
{
1489
	struct scrub_block *sblock = spage->sblock;
A
Arne Jansen 已提交
1490
	struct scrub_bio *sbio;
1491
	int ret;
A
Arne Jansen 已提交
1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502

again:
	/*
	 * grab a fresh bio or wait for one to become available
	 */
	while (sdev->curr == -1) {
		spin_lock(&sdev->list_lock);
		sdev->curr = sdev->first_free;
		if (sdev->curr != -1) {
			sdev->first_free = sdev->bios[sdev->curr]->next_free;
			sdev->bios[sdev->curr]->next_free = -1;
1503
			sdev->bios[sdev->curr]->page_count = 0;
A
Arne Jansen 已提交
1504 1505 1506 1507 1508 1509 1510
			spin_unlock(&sdev->list_lock);
		} else {
			spin_unlock(&sdev->list_lock);
			wait_event(sdev->list_wait, sdev->first_free != -1);
		}
	}
	sbio = sdev->bios[sdev->curr];
1511
	if (sbio->page_count == 0) {
1512 1513
		struct bio *bio;

1514 1515 1516 1517 1518 1519 1520 1521 1522
		sbio->physical = spage->physical;
		sbio->logical = spage->logical;
		bio = sbio->bio;
		if (!bio) {
			bio = bio_alloc(GFP_NOFS, sdev->pages_per_bio);
			if (!bio)
				return -ENOMEM;
			sbio->bio = bio;
		}
1523 1524 1525 1526

		bio->bi_private = sbio;
		bio->bi_end_io = scrub_bio_end_io;
		bio->bi_bdev = sdev->dev->bdev;
1527
		bio->bi_sector = spage->physical >> 9;
1528
		sbio->err = 0;
1529 1530 1531 1532
	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
		   spage->physical ||
		   sbio->logical + sbio->page_count * PAGE_SIZE !=
		   spage->logical) {
S
Stefan Behrens 已提交
1533
		scrub_submit(sdev);
A
Arne Jansen 已提交
1534 1535
		goto again;
	}
1536

1537 1538 1539 1540 1541 1542 1543 1544
	sbio->pagev[sbio->page_count] = spage;
	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
	if (ret != PAGE_SIZE) {
		if (sbio->page_count < 1) {
			bio_put(sbio->bio);
			sbio->bio = NULL;
			return -EIO;
		}
S
Stefan Behrens 已提交
1545
		scrub_submit(sdev);
1546 1547 1548
		goto again;
	}

1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	scrub_block_get(sblock); /* one for the added page */
	atomic_inc(&sblock->outstanding_pages);
	sbio->page_count++;
	if (sbio->page_count == sdev->pages_per_bio)
		scrub_submit(sdev);

	return 0;
}

static int scrub_pages(struct scrub_dev *sdev, u64 logical, u64 len,
		       u64 physical, u64 flags, u64 gen, int mirror_num,
		       u8 *csum, int force)
{
	struct scrub_block *sblock;
	int index;

	sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
	if (!sblock) {
		spin_lock(&sdev->stat_lock);
		sdev->stat.malloc_errors++;
		spin_unlock(&sdev->stat_lock);
		return -ENOMEM;
A
Arne Jansen 已提交
1571
	}
1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595

	/* one ref inside this function, plus one for each page later on */
	atomic_set(&sblock->ref_count, 1);
	sblock->sdev = sdev;
	sblock->no_io_error_seen = 1;

	for (index = 0; len > 0; index++) {
		struct scrub_page *spage = sblock->pagev + index;
		u64 l = min_t(u64, len, PAGE_SIZE);

		BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
		spage->page = alloc_page(GFP_NOFS);
		if (!spage->page) {
			spin_lock(&sdev->stat_lock);
			sdev->stat.malloc_errors++;
			spin_unlock(&sdev->stat_lock);
			while (index > 0) {
				index--;
				__free_page(sblock->pagev[index].page);
			}
			kfree(sblock);
			return -ENOMEM;
		}
		spage->sblock = sblock;
1596
		spage->dev = sdev->dev;
1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
		spage->flags = flags;
		spage->generation = gen;
		spage->logical = logical;
		spage->physical = physical;
		spage->mirror_num = mirror_num;
		if (csum) {
			spage->have_csum = 1;
			memcpy(spage->csum, csum, sdev->csum_size);
		} else {
			spage->have_csum = 0;
		}
		sblock->page_count++;
		len -= l;
		logical += l;
		physical += l;
	}

	BUG_ON(sblock->page_count == 0);
	for (index = 0; index < sblock->page_count; index++) {
		struct scrub_page *spage = sblock->pagev + index;
1617 1618
		int ret;

1619 1620 1621
		ret = scrub_add_page_to_bio(sdev, spage);
		if (ret) {
			scrub_block_put(sblock);
1622
			return ret;
1623
		}
1624
	}
A
Arne Jansen 已提交
1625

1626
	if (force)
S
Stefan Behrens 已提交
1627
		scrub_submit(sdev);
A
Arne Jansen 已提交
1628

1629 1630
	/* last one frees, either here or in bio completion for last page */
	scrub_block_put(sblock);
A
Arne Jansen 已提交
1631 1632 1633
	return 0;
}

1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
static void scrub_bio_end_io(struct bio *bio, int err)
{
	struct scrub_bio *sbio = bio->bi_private;
	struct scrub_dev *sdev = sbio->sdev;
	struct btrfs_fs_info *fs_info = sdev->dev->dev_root->fs_info;

	sbio->err = err;
	sbio->bio = bio;

	btrfs_queue_worker(&fs_info->scrub_workers, &sbio->work);
}

static void scrub_bio_end_io_worker(struct btrfs_work *work)
{
	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
	struct scrub_dev *sdev = sbio->sdev;
	int i;

	BUG_ON(sbio->page_count > SCRUB_PAGES_PER_BIO);
	if (sbio->err) {
		for (i = 0; i < sbio->page_count; i++) {
			struct scrub_page *spage = sbio->pagev[i];

			spage->io_error = 1;
			spage->sblock->no_io_error_seen = 0;
		}
	}

	/* now complete the scrub_block items that have all pages completed */
	for (i = 0; i < sbio->page_count; i++) {
		struct scrub_page *spage = sbio->pagev[i];
		struct scrub_block *sblock = spage->sblock;

		if (atomic_dec_and_test(&sblock->outstanding_pages))
			scrub_block_complete(sblock);
		scrub_block_put(sblock);
	}

	bio_put(sbio->bio);
	sbio->bio = NULL;
	spin_lock(&sdev->list_lock);
	sbio->next_free = sdev->first_free;
	sdev->first_free = sbio->index;
	spin_unlock(&sdev->list_lock);
	atomic_dec(&sdev->in_flight);
	wake_up(&sdev->list_wait);
}

static void scrub_block_complete(struct scrub_block *sblock)
{
	if (!sblock->no_io_error_seen)
		scrub_handle_errored_block(sblock);
	else
		scrub_checksum(sblock);
}

A
Arne Jansen 已提交
1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
static int scrub_find_csum(struct scrub_dev *sdev, u64 logical, u64 len,
			   u8 *csum)
{
	struct btrfs_ordered_sum *sum = NULL;
	int ret = 0;
	unsigned long i;
	unsigned long num_sectors;

	while (!list_empty(&sdev->csum_list)) {
		sum = list_first_entry(&sdev->csum_list,
				       struct btrfs_ordered_sum, list);
		if (sum->bytenr > logical)
			return 0;
		if (sum->bytenr + sum->len > logical)
			break;

		++sdev->stat.csum_discards;
		list_del(&sum->list);
		kfree(sum);
		sum = NULL;
	}
	if (!sum)
		return 0;

1714
	num_sectors = sum->len / sdev->sectorsize;
A
Arne Jansen 已提交
1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
	for (i = 0; i < num_sectors; ++i) {
		if (sum->sums[i].bytenr == logical) {
			memcpy(csum, &sum->sums[i].sum, sdev->csum_size);
			ret = 1;
			break;
		}
	}
	if (ret && i == num_sectors - 1) {
		list_del(&sum->list);
		kfree(sum);
	}
	return ret;
}

/* scrub extent tries to collect up to 64 kB for each bio */
static int scrub_extent(struct scrub_dev *sdev, u64 logical, u64 len,
1731
			u64 physical, u64 flags, u64 gen, int mirror_num)
A
Arne Jansen 已提交
1732 1733 1734
{
	int ret;
	u8 csum[BTRFS_CSUM_SIZE];
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753
	u32 blocksize;

	if (flags & BTRFS_EXTENT_FLAG_DATA) {
		blocksize = sdev->sectorsize;
		spin_lock(&sdev->stat_lock);
		sdev->stat.data_extents_scrubbed++;
		sdev->stat.data_bytes_scrubbed += len;
		spin_unlock(&sdev->stat_lock);
	} else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
		BUG_ON(sdev->nodesize != sdev->leafsize);
		blocksize = sdev->nodesize;
		spin_lock(&sdev->stat_lock);
		sdev->stat.tree_extents_scrubbed++;
		sdev->stat.tree_bytes_scrubbed += len;
		spin_unlock(&sdev->stat_lock);
	} else {
		blocksize = sdev->sectorsize;
		BUG_ON(1);
	}
A
Arne Jansen 已提交
1754 1755

	while (len) {
1756
		u64 l = min_t(u64, len, blocksize);
A
Arne Jansen 已提交
1757 1758 1759 1760 1761 1762 1763 1764
		int have_csum = 0;

		if (flags & BTRFS_EXTENT_FLAG_DATA) {
			/* push csums to sbio */
			have_csum = scrub_find_csum(sdev, logical, l, csum);
			if (have_csum == 0)
				++sdev->stat.no_csum;
		}
1765 1766
		ret = scrub_pages(sdev, logical, l, physical, flags, gen,
				  mirror_num, have_csum ? csum : NULL, 0);
A
Arne Jansen 已提交
1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783
		if (ret)
			return ret;
		len -= l;
		logical += l;
		physical += l;
	}
	return 0;
}

static noinline_for_stack int scrub_stripe(struct scrub_dev *sdev,
	struct map_lookup *map, int num, u64 base, u64 length)
{
	struct btrfs_path *path;
	struct btrfs_fs_info *fs_info = sdev->dev->dev_root->fs_info;
	struct btrfs_root *root = fs_info->extent_root;
	struct btrfs_root *csum_root = fs_info->csum_root;
	struct btrfs_extent_item *extent;
1784
	struct blk_plug plug;
A
Arne Jansen 已提交
1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
	u64 flags;
	int ret;
	int slot;
	int i;
	u64 nstripes;
	struct extent_buffer *l;
	struct btrfs_key key;
	u64 physical;
	u64 logical;
	u64 generation;
1795
	int mirror_num;
A
Arne Jansen 已提交
1796 1797 1798 1799
	struct reada_control *reada1;
	struct reada_control *reada2;
	struct btrfs_key key_start;
	struct btrfs_key key_end;
A
Arne Jansen 已提交
1800 1801 1802 1803 1804 1805 1806 1807 1808 1809

	u64 increment = map->stripe_len;
	u64 offset;

	nstripes = length;
	offset = 0;
	do_div(nstripes, map->stripe_len);
	if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
		offset = map->stripe_len * num;
		increment = map->stripe_len * map->num_stripes;
1810
		mirror_num = 1;
A
Arne Jansen 已提交
1811 1812 1813 1814
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
		int factor = map->num_stripes / map->sub_stripes;
		offset = map->stripe_len * (num / map->sub_stripes);
		increment = map->stripe_len * factor;
1815
		mirror_num = num % map->sub_stripes + 1;
A
Arne Jansen 已提交
1816 1817
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
		increment = map->stripe_len;
1818
		mirror_num = num % map->num_stripes + 1;
A
Arne Jansen 已提交
1819 1820
	} else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
		increment = map->stripe_len;
1821
		mirror_num = num % map->num_stripes + 1;
A
Arne Jansen 已提交
1822 1823
	} else {
		increment = map->stripe_len;
1824
		mirror_num = 1;
A
Arne Jansen 已提交
1825 1826 1827 1828 1829 1830
	}

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

1831 1832 1833 1834 1835
	/*
	 * work on commit root. The related disk blocks are static as
	 * long as COW is applied. This means, it is save to rewrite
	 * them to repair disk errors without any race conditions
	 */
A
Arne Jansen 已提交
1836 1837 1838 1839
	path->search_commit_root = 1;
	path->skip_locking = 1;

	/*
A
Arne Jansen 已提交
1840 1841 1842
	 * trigger the readahead for extent tree csum tree and wait for
	 * completion. During readahead, the scrub is officially paused
	 * to not hold off transaction commits
A
Arne Jansen 已提交
1843 1844 1845
	 */
	logical = base + offset;

A
Arne Jansen 已提交
1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
	wait_event(sdev->list_wait,
		   atomic_read(&sdev->in_flight) == 0);
	atomic_inc(&fs_info->scrubs_paused);
	wake_up(&fs_info->scrub_pause_wait);

	/* FIXME it might be better to start readahead at commit root */
	key_start.objectid = logical;
	key_start.type = BTRFS_EXTENT_ITEM_KEY;
	key_start.offset = (u64)0;
	key_end.objectid = base + offset + nstripes * increment;
	key_end.type = BTRFS_EXTENT_ITEM_KEY;
	key_end.offset = (u64)0;
	reada1 = btrfs_reada_add(root, &key_start, &key_end);

	key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
	key_start.type = BTRFS_EXTENT_CSUM_KEY;
	key_start.offset = logical;
	key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
	key_end.type = BTRFS_EXTENT_CSUM_KEY;
	key_end.offset = base + offset + nstripes * increment;
	reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);

	if (!IS_ERR(reada1))
		btrfs_reada_wait(reada1);
	if (!IS_ERR(reada2))
		btrfs_reada_wait(reada2);

	mutex_lock(&fs_info->scrub_lock);
	while (atomic_read(&fs_info->scrub_pause_req)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
		   atomic_read(&fs_info->scrub_pause_req) == 0);
		mutex_lock(&fs_info->scrub_lock);
A
Arne Jansen 已提交
1879
	}
A
Arne Jansen 已提交
1880 1881 1882
	atomic_dec(&fs_info->scrubs_paused);
	mutex_unlock(&fs_info->scrub_lock);
	wake_up(&fs_info->scrub_pause_wait);
A
Arne Jansen 已提交
1883 1884 1885 1886 1887

	/*
	 * collect all data csums for the stripe to avoid seeking during
	 * the scrub. This might currently (crc32) end up to be about 1MB
	 */
1888
	blk_start_plug(&plug);
A
Arne Jansen 已提交
1889 1890 1891 1892

	/*
	 * now find all extents for each stripe and scrub them
	 */
A
Arne Jansen 已提交
1893 1894
	logical = base + offset;
	physical = map->stripes[num].physical;
A
Arne Jansen 已提交
1895
	ret = 0;
A
Arne Jansen 已提交
1896
	for (i = 0; i < nstripes; ++i) {
A
Arne Jansen 已提交
1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926
		/*
		 * canceled?
		 */
		if (atomic_read(&fs_info->scrub_cancel_req) ||
		    atomic_read(&sdev->cancel_req)) {
			ret = -ECANCELED;
			goto out;
		}
		/*
		 * check to see if we have to pause
		 */
		if (atomic_read(&fs_info->scrub_pause_req)) {
			/* push queued extents */
			scrub_submit(sdev);
			wait_event(sdev->list_wait,
				   atomic_read(&sdev->in_flight) == 0);
			atomic_inc(&fs_info->scrubs_paused);
			wake_up(&fs_info->scrub_pause_wait);
			mutex_lock(&fs_info->scrub_lock);
			while (atomic_read(&fs_info->scrub_pause_req)) {
				mutex_unlock(&fs_info->scrub_lock);
				wait_event(fs_info->scrub_pause_wait,
				   atomic_read(&fs_info->scrub_pause_req) == 0);
				mutex_lock(&fs_info->scrub_lock);
			}
			atomic_dec(&fs_info->scrubs_paused);
			mutex_unlock(&fs_info->scrub_lock);
			wake_up(&fs_info->scrub_pause_wait);
		}

A
Arne Jansen 已提交
1927 1928 1929 1930 1931 1932
		ret = btrfs_lookup_csums_range(csum_root, logical,
					       logical + map->stripe_len - 1,
					       &sdev->csum_list, 1);
		if (ret)
			goto out;

A
Arne Jansen 已提交
1933 1934 1935 1936 1937 1938 1939
		key.objectid = logical;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = (u64)0;

		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		if (ret < 0)
			goto out;
1940
		if (ret > 0) {
A
Arne Jansen 已提交
1941 1942 1943 1944
			ret = btrfs_previous_item(root, path, 0,
						  BTRFS_EXTENT_ITEM_KEY);
			if (ret < 0)
				goto out;
1945 1946 1947 1948 1949 1950 1951 1952 1953
			if (ret > 0) {
				/* there's no smaller item, so stick with the
				 * larger one */
				btrfs_release_path(path);
				ret = btrfs_search_slot(NULL, root, &key,
							path, 0, 0);
				if (ret < 0)
					goto out;
			}
A
Arne Jansen 已提交
1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015
		}

		while (1) {
			l = path->nodes[0];
			slot = path->slots[0];
			if (slot >= btrfs_header_nritems(l)) {
				ret = btrfs_next_leaf(root, path);
				if (ret == 0)
					continue;
				if (ret < 0)
					goto out;

				break;
			}
			btrfs_item_key_to_cpu(l, &key, slot);

			if (key.objectid + key.offset <= logical)
				goto next;

			if (key.objectid >= logical + map->stripe_len)
				break;

			if (btrfs_key_type(&key) != BTRFS_EXTENT_ITEM_KEY)
				goto next;

			extent = btrfs_item_ptr(l, slot,
						struct btrfs_extent_item);
			flags = btrfs_extent_flags(l, extent);
			generation = btrfs_extent_generation(l, extent);

			if (key.objectid < logical &&
			    (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
				printk(KERN_ERR
				       "btrfs scrub: tree block %llu spanning "
				       "stripes, ignored. logical=%llu\n",
				       (unsigned long long)key.objectid,
				       (unsigned long long)logical);
				goto next;
			}

			/*
			 * trim extent to this stripe
			 */
			if (key.objectid < logical) {
				key.offset -= logical - key.objectid;
				key.objectid = logical;
			}
			if (key.objectid + key.offset >
			    logical + map->stripe_len) {
				key.offset = logical + map->stripe_len -
					     key.objectid;
			}

			ret = scrub_extent(sdev, key.objectid, key.offset,
					   key.objectid - logical + physical,
					   flags, generation, mirror_num);
			if (ret)
				goto out;

next:
			path->slots[0]++;
		}
C
Chris Mason 已提交
2016
		btrfs_release_path(path);
A
Arne Jansen 已提交
2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
		logical += increment;
		physical += map->stripe_len;
		spin_lock(&sdev->stat_lock);
		sdev->stat.last_physical = physical;
		spin_unlock(&sdev->stat_lock);
	}
	/* push queued extents */
	scrub_submit(sdev);

out:
2027
	blk_finish_plug(&plug);
A
Arne Jansen 已提交
2028 2029 2030 2031 2032
	btrfs_free_path(path);
	return ret < 0 ? ret : 0;
}

static noinline_for_stack int scrub_chunk(struct scrub_dev *sdev,
2033 2034
	u64 chunk_tree, u64 chunk_objectid, u64 chunk_offset, u64 length,
	u64 dev_offset)
A
Arne Jansen 已提交
2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057
{
	struct btrfs_mapping_tree *map_tree =
		&sdev->dev->dev_root->fs_info->mapping_tree;
	struct map_lookup *map;
	struct extent_map *em;
	int i;
	int ret = -EINVAL;

	read_lock(&map_tree->map_tree.lock);
	em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
	read_unlock(&map_tree->map_tree.lock);

	if (!em)
		return -EINVAL;

	map = (struct map_lookup *)em->bdev;
	if (em->start != chunk_offset)
		goto out;

	if (em->len < length)
		goto out;

	for (i = 0; i < map->num_stripes; ++i) {
2058 2059
		if (map->stripes[i].dev == sdev->dev &&
		    map->stripes[i].physical == dev_offset) {
A
Arne Jansen 已提交
2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104
			ret = scrub_stripe(sdev, map, i, chunk_offset, length);
			if (ret)
				goto out;
		}
	}
out:
	free_extent_map(em);

	return ret;
}

static noinline_for_stack
int scrub_enumerate_chunks(struct scrub_dev *sdev, u64 start, u64 end)
{
	struct btrfs_dev_extent *dev_extent = NULL;
	struct btrfs_path *path;
	struct btrfs_root *root = sdev->dev->dev_root;
	struct btrfs_fs_info *fs_info = root->fs_info;
	u64 length;
	u64 chunk_tree;
	u64 chunk_objectid;
	u64 chunk_offset;
	int ret;
	int slot;
	struct extent_buffer *l;
	struct btrfs_key key;
	struct btrfs_key found_key;
	struct btrfs_block_group_cache *cache;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 2;
	path->search_commit_root = 1;
	path->skip_locking = 1;

	key.objectid = sdev->dev->devid;
	key.offset = 0ull;
	key.type = BTRFS_DEV_EXTENT_KEY;


	while (1) {
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		if (ret < 0)
2105 2106 2107 2108 2109 2110 2111 2112 2113
			break;
		if (ret > 0) {
			if (path->slots[0] >=
			    btrfs_header_nritems(path->nodes[0])) {
				ret = btrfs_next_leaf(root, path);
				if (ret)
					break;
			}
		}
A
Arne Jansen 已提交
2114 2115 2116 2117 2118 2119 2120 2121 2122

		l = path->nodes[0];
		slot = path->slots[0];

		btrfs_item_key_to_cpu(l, &found_key, slot);

		if (found_key.objectid != sdev->dev->devid)
			break;

2123
		if (btrfs_key_type(&found_key) != BTRFS_DEV_EXTENT_KEY)
A
Arne Jansen 已提交
2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136
			break;

		if (found_key.offset >= end)
			break;

		if (found_key.offset < key.offset)
			break;

		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
		length = btrfs_dev_extent_length(l, dev_extent);

		if (found_key.offset + length <= start) {
			key.offset = found_key.offset + length;
C
Chris Mason 已提交
2137
			btrfs_release_path(path);
A
Arne Jansen 已提交
2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
			continue;
		}

		chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
		chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
		chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);

		/*
		 * get a reference on the corresponding block group to prevent
		 * the chunk from going away while we scrub it
		 */
		cache = btrfs_lookup_block_group(fs_info, chunk_offset);
		if (!cache) {
			ret = -ENOENT;
2152
			break;
A
Arne Jansen 已提交
2153 2154
		}
		ret = scrub_chunk(sdev, chunk_tree, chunk_objectid,
2155
				  chunk_offset, length, found_key.offset);
A
Arne Jansen 已提交
2156 2157 2158 2159 2160
		btrfs_put_block_group(cache);
		if (ret)
			break;

		key.offset = found_key.offset + length;
C
Chris Mason 已提交
2161
		btrfs_release_path(path);
A
Arne Jansen 已提交
2162 2163 2164
	}

	btrfs_free_path(path);
2165 2166 2167 2168 2169 2170

	/*
	 * ret can still be 1 from search_slot or next_leaf,
	 * that's not an error
	 */
	return ret < 0 ? ret : 0;
A
Arne Jansen 已提交
2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181
}

static noinline_for_stack int scrub_supers(struct scrub_dev *sdev)
{
	int	i;
	u64	bytenr;
	u64	gen;
	int	ret;
	struct btrfs_device *device = sdev->dev;
	struct btrfs_root *root = device->dev_root;

2182 2183 2184
	if (root->fs_info->fs_state & BTRFS_SUPER_FLAG_ERROR)
		return -EIO;

A
Arne Jansen 已提交
2185 2186 2187 2188
	gen = root->fs_info->last_trans_committed;

	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
S
Stefan Behrens 已提交
2189
		if (bytenr + BTRFS_SUPER_INFO_SIZE > device->total_bytes)
A
Arne Jansen 已提交
2190 2191
			break;

2192 2193
		ret = scrub_pages(sdev, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
				     BTRFS_EXTENT_FLAG_SUPER, gen, i, NULL, 1);
A
Arne Jansen 已提交
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207
		if (ret)
			return ret;
	}
	wait_event(sdev->list_wait, atomic_read(&sdev->in_flight) == 0);

	return 0;
}

/*
 * get a reference count on fs_info->scrub_workers. start worker if necessary
 */
static noinline_for_stack int scrub_workers_get(struct btrfs_root *root)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
2208
	int ret = 0;
A
Arne Jansen 已提交
2209 2210

	mutex_lock(&fs_info->scrub_lock);
A
Arne Jansen 已提交
2211 2212 2213 2214
	if (fs_info->scrub_workers_refcnt == 0) {
		btrfs_init_workers(&fs_info->scrub_workers, "scrub",
			   fs_info->thread_pool_size, &fs_info->generic_worker);
		fs_info->scrub_workers.idle_thresh = 4;
2215 2216 2217
		ret = btrfs_start_workers(&fs_info->scrub_workers);
		if (ret)
			goto out;
A
Arne Jansen 已提交
2218
	}
A
Arne Jansen 已提交
2219
	++fs_info->scrub_workers_refcnt;
2220
out:
A
Arne Jansen 已提交
2221 2222
	mutex_unlock(&fs_info->scrub_lock);

2223
	return ret;
A
Arne Jansen 已提交
2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238
}

static noinline_for_stack void scrub_workers_put(struct btrfs_root *root)
{
	struct btrfs_fs_info *fs_info = root->fs_info;

	mutex_lock(&fs_info->scrub_lock);
	if (--fs_info->scrub_workers_refcnt == 0)
		btrfs_stop_workers(&fs_info->scrub_workers);
	WARN_ON(fs_info->scrub_workers_refcnt < 0);
	mutex_unlock(&fs_info->scrub_lock);
}


int btrfs_scrub_dev(struct btrfs_root *root, u64 devid, u64 start, u64 end,
A
Arne Jansen 已提交
2239
		    struct btrfs_scrub_progress *progress, int readonly)
A
Arne Jansen 已提交
2240 2241 2242 2243 2244 2245
{
	struct scrub_dev *sdev;
	struct btrfs_fs_info *fs_info = root->fs_info;
	int ret;
	struct btrfs_device *dev;

2246
	if (btrfs_fs_closing(root->fs_info))
A
Arne Jansen 已提交
2247 2248 2249 2250 2251
		return -EINVAL;

	/*
	 * check some assumptions
	 */
2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275
	if (root->nodesize != root->leafsize) {
		printk(KERN_ERR
		       "btrfs_scrub: size assumption nodesize == leafsize (%d == %d) fails\n",
		       root->nodesize, root->leafsize);
		return -EINVAL;
	}

	if (root->nodesize > BTRFS_STRIPE_LEN) {
		/*
		 * in this case scrub is unable to calculate the checksum
		 * the way scrub is implemented. Do not handle this
		 * situation at all because it won't ever happen.
		 */
		printk(KERN_ERR
		       "btrfs_scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails\n",
		       root->nodesize, BTRFS_STRIPE_LEN);
		return -EINVAL;
	}

	if (root->sectorsize != PAGE_SIZE) {
		/* not supported for data w/o checksums */
		printk(KERN_ERR
		       "btrfs_scrub: size assumption sectorsize != PAGE_SIZE (%d != %lld) fails\n",
		       root->sectorsize, (unsigned long long)PAGE_SIZE);
A
Arne Jansen 已提交
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311
		return -EINVAL;
	}

	ret = scrub_workers_get(root);
	if (ret)
		return ret;

	mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
	dev = btrfs_find_device(root, devid, NULL, NULL);
	if (!dev || dev->missing) {
		mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
		scrub_workers_put(root);
		return -ENODEV;
	}
	mutex_lock(&fs_info->scrub_lock);

	if (!dev->in_fs_metadata) {
		mutex_unlock(&fs_info->scrub_lock);
		mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
		scrub_workers_put(root);
		return -ENODEV;
	}

	if (dev->scrub_device) {
		mutex_unlock(&fs_info->scrub_lock);
		mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
		scrub_workers_put(root);
		return -EINPROGRESS;
	}
	sdev = scrub_setup_dev(dev);
	if (IS_ERR(sdev)) {
		mutex_unlock(&fs_info->scrub_lock);
		mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
		scrub_workers_put(root);
		return PTR_ERR(sdev);
	}
A
Arne Jansen 已提交
2312
	sdev->readonly = readonly;
A
Arne Jansen 已提交
2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329
	dev->scrub_device = sdev;

	atomic_inc(&fs_info->scrubs_running);
	mutex_unlock(&fs_info->scrub_lock);
	mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);

	down_read(&fs_info->scrub_super_lock);
	ret = scrub_supers(sdev);
	up_read(&fs_info->scrub_super_lock);

	if (!ret)
		ret = scrub_enumerate_chunks(sdev, start, end);

	wait_event(sdev->list_wait, atomic_read(&sdev->in_flight) == 0);
	atomic_dec(&fs_info->scrubs_running);
	wake_up(&fs_info->scrub_pause_wait);

2330 2331
	wait_event(sdev->list_wait, atomic_read(&sdev->fixup_cnt) == 0);

A
Arne Jansen 已提交
2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344
	if (progress)
		memcpy(progress, &sdev->stat, sizeof(*progress));

	mutex_lock(&fs_info->scrub_lock);
	dev->scrub_device = NULL;
	mutex_unlock(&fs_info->scrub_lock);

	scrub_free_dev(sdev);
	scrub_workers_put(root);

	return ret;
}

2345
void btrfs_scrub_pause(struct btrfs_root *root)
A
Arne Jansen 已提交
2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361
{
	struct btrfs_fs_info *fs_info = root->fs_info;

	mutex_lock(&fs_info->scrub_lock);
	atomic_inc(&fs_info->scrub_pause_req);
	while (atomic_read(&fs_info->scrubs_paused) !=
	       atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
			   atomic_read(&fs_info->scrubs_paused) ==
			   atomic_read(&fs_info->scrubs_running));
		mutex_lock(&fs_info->scrub_lock);
	}
	mutex_unlock(&fs_info->scrub_lock);
}

2362
void btrfs_scrub_continue(struct btrfs_root *root)
A
Arne Jansen 已提交
2363 2364 2365 2366 2367 2368 2369
{
	struct btrfs_fs_info *fs_info = root->fs_info;

	atomic_dec(&fs_info->scrub_pause_req);
	wake_up(&fs_info->scrub_pause_wait);
}

2370
void btrfs_scrub_pause_super(struct btrfs_root *root)
A
Arne Jansen 已提交
2371 2372 2373 2374
{
	down_write(&root->fs_info->scrub_super_lock);
}

2375
void btrfs_scrub_continue_super(struct btrfs_root *root)
A
Arne Jansen 已提交
2376 2377 2378 2379
{
	up_write(&root->fs_info->scrub_super_lock);
}

2380
int __btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
A
Arne Jansen 已提交
2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
{

	mutex_lock(&fs_info->scrub_lock);
	if (!atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		return -ENOTCONN;
	}

	atomic_inc(&fs_info->scrub_cancel_req);
	while (atomic_read(&fs_info->scrubs_running)) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
			   atomic_read(&fs_info->scrubs_running) == 0);
		mutex_lock(&fs_info->scrub_lock);
	}
	atomic_dec(&fs_info->scrub_cancel_req);
	mutex_unlock(&fs_info->scrub_lock);

	return 0;
}

2402 2403 2404 2405 2406
int btrfs_scrub_cancel(struct btrfs_root *root)
{
	return __btrfs_scrub_cancel(root->fs_info);
}

A
Arne Jansen 已提交
2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428
int btrfs_scrub_cancel_dev(struct btrfs_root *root, struct btrfs_device *dev)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct scrub_dev *sdev;

	mutex_lock(&fs_info->scrub_lock);
	sdev = dev->scrub_device;
	if (!sdev) {
		mutex_unlock(&fs_info->scrub_lock);
		return -ENOTCONN;
	}
	atomic_inc(&sdev->cancel_req);
	while (dev->scrub_device) {
		mutex_unlock(&fs_info->scrub_lock);
		wait_event(fs_info->scrub_pause_wait,
			   dev->scrub_device == NULL);
		mutex_lock(&fs_info->scrub_lock);
	}
	mutex_unlock(&fs_info->scrub_lock);

	return 0;
}
S
Stefan Behrens 已提交
2429

A
Arne Jansen 已提交
2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467
int btrfs_scrub_cancel_devid(struct btrfs_root *root, u64 devid)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_device *dev;
	int ret;

	/*
	 * we have to hold the device_list_mutex here so the device
	 * does not go away in cancel_dev. FIXME: find a better solution
	 */
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
	dev = btrfs_find_device(root, devid, NULL, NULL);
	if (!dev) {
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
		return -ENODEV;
	}
	ret = btrfs_scrub_cancel_dev(root, dev);
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);

	return ret;
}

int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
			 struct btrfs_scrub_progress *progress)
{
	struct btrfs_device *dev;
	struct scrub_dev *sdev = NULL;

	mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
	dev = btrfs_find_device(root, devid, NULL, NULL);
	if (dev)
		sdev = dev->scrub_device;
	if (sdev)
		memcpy(progress, &sdev->stat, sizeof(*progress));
	mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);

	return dev ? (sdev ? 0 : -ENOTCONN) : -ENODEV;
}