dev-replace.c 31.7 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4
/*
 * Copyright (C) STRATO AG 2012.  All rights reserved.
 */
5

6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
#include <linux/sched.h>
#include <linux/bio.h>
#include <linux/slab.h>
#include <linux/buffer_head.h>
#include <linux/blkdev.h>
#include <linux/random.h>
#include <linux/iocontext.h>
#include <linux/capability.h>
#include <linux/kthread.h>
#include <linux/math64.h>
#include <asm/div64.h>
#include "ctree.h"
#include "extent_map.h"
#include "disk-io.h"
#include "transaction.h"
#include "print-tree.h"
#include "volumes.h"
#include "async-thread.h"
#include "check-integrity.h"
#include "rcu-string.h"
#include "dev-replace.h"
27
#include "sysfs.h"
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87

static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
				       int scrub_ret);
static void btrfs_dev_replace_update_device_in_mapping_tree(
						struct btrfs_fs_info *fs_info,
						struct btrfs_device *srcdev,
						struct btrfs_device *tgtdev);
static int btrfs_dev_replace_kthread(void *data);

int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)
{
	struct btrfs_key key;
	struct btrfs_root *dev_root = fs_info->dev_root;
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
	struct extent_buffer *eb;
	int slot;
	int ret = 0;
	struct btrfs_path *path = NULL;
	int item_size;
	struct btrfs_dev_replace_item *ptr;
	u64 src_devid;

	path = btrfs_alloc_path();
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}

	key.objectid = 0;
	key.type = BTRFS_DEV_REPLACE_KEY;
	key.offset = 0;
	ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
	if (ret) {
no_valid_dev_replace_entry_found:
		ret = 0;
		dev_replace->replace_state =
			BTRFS_DEV_REPLACE_ITEM_STATE_NEVER_STARTED;
		dev_replace->cont_reading_from_srcdev_mode =
		    BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS;
		dev_replace->replace_state = 0;
		dev_replace->time_started = 0;
		dev_replace->time_stopped = 0;
		atomic64_set(&dev_replace->num_write_errors, 0);
		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
		dev_replace->cursor_left = 0;
		dev_replace->committed_cursor_left = 0;
		dev_replace->cursor_left_last_write_of_item = 0;
		dev_replace->cursor_right = 0;
		dev_replace->srcdev = NULL;
		dev_replace->tgtdev = NULL;
		dev_replace->is_valid = 0;
		dev_replace->item_needs_writeback = 0;
		goto out;
	}
	slot = path->slots[0];
	eb = path->nodes[0];
	item_size = btrfs_item_size_nr(eb, slot);
	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_replace_item);

	if (item_size != sizeof(struct btrfs_dev_replace_item)) {
88 89
		btrfs_warn(fs_info,
			"dev_replace entry found has unexpected size, ignore entry");
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
		goto no_valid_dev_replace_entry_found;
	}

	src_devid = btrfs_dev_replace_src_devid(eb, ptr);
	dev_replace->cont_reading_from_srcdev_mode =
		btrfs_dev_replace_cont_reading_from_srcdev_mode(eb, ptr);
	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);
	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);
	dev_replace->time_stopped =
		btrfs_dev_replace_time_stopped(eb, ptr);
	atomic64_set(&dev_replace->num_write_errors,
		     btrfs_dev_replace_num_write_errors(eb, ptr));
	atomic64_set(&dev_replace->num_uncorrectable_read_errors,
		     btrfs_dev_replace_num_uncorrectable_read_errors(eb, ptr));
	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);
	dev_replace->committed_cursor_left = dev_replace->cursor_left;
	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;
	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);
	dev_replace->is_valid = 1;

	dev_replace->item_needs_writeback = 0;
	switch (dev_replace->replace_state) {
	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
		dev_replace->srcdev = NULL;
		dev_replace->tgtdev = NULL;
		break;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
		dev_replace->srcdev = btrfs_find_device(fs_info, src_devid,
							NULL, NULL);
		dev_replace->tgtdev = btrfs_find_device(fs_info,
							BTRFS_DEV_REPLACE_DEVID,
							NULL, NULL);
		/*
		 * allow 'btrfs dev replace_cancel' if src/tgt device is
		 * missing
		 */
		if (!dev_replace->srcdev &&
130
		    !btrfs_test_opt(fs_info, DEGRADED)) {
131
			ret = -EIO;
132 133 134 135 136
			btrfs_warn(fs_info,
			   "cannot mount because device replace operation is ongoing and");
			btrfs_warn(fs_info,
			   "srcdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
			   src_devid);
137 138
		}
		if (!dev_replace->tgtdev &&
139
		    !btrfs_test_opt(fs_info, DEGRADED)) {
140
			ret = -EIO;
141 142 143 144
			btrfs_warn(fs_info,
			   "cannot mount because device replace operation is ongoing and");
			btrfs_warn(fs_info,
			   "tgtdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
145
				BTRFS_DEV_REPLACE_DEVID);
146 147 148 149 150 151 152
		}
		if (dev_replace->tgtdev) {
			if (dev_replace->srcdev) {
				dev_replace->tgtdev->total_bytes =
					dev_replace->srcdev->total_bytes;
				dev_replace->tgtdev->disk_total_bytes =
					dev_replace->srcdev->disk_total_bytes;
153 154
				dev_replace->tgtdev->commit_total_bytes =
					dev_replace->srcdev->commit_total_bytes;
155 156
				dev_replace->tgtdev->bytes_used =
					dev_replace->srcdev->bytes_used;
157 158
				dev_replace->tgtdev->commit_bytes_used =
					dev_replace->srcdev->commit_bytes_used;
159
			}
160 161
			set_bit(BTRFS_DEV_STATE_REPLACE_TGT,
				&dev_replace->tgtdev->dev_state);
162 163 164 165 166 167 168 169

			WARN_ON(fs_info->fs_devices->rw_devices == 0);
			dev_replace->tgtdev->io_width = fs_info->sectorsize;
			dev_replace->tgtdev->io_align = fs_info->sectorsize;
			dev_replace->tgtdev->sector_size = fs_info->sectorsize;
			dev_replace->tgtdev->fs_info = fs_info;
			set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
				&dev_replace->tgtdev->dev_state);
170 171 172 173 174
		}
		break;
	}

out:
175
	btrfs_free_path(path);
176 177 178
	return ret;
}

179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
/*
 * Initialize a new device for device replace target from a given source dev
 * and path.
 *
 * Return 0 and new device in @device_out, otherwise return < 0
 */
static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
				  const char *device_path,
				  struct btrfs_device *srcdev,
				  struct btrfs_device **device_out)
{
	struct btrfs_device *device;
	struct block_device *bdev;
	struct list_head *devices;
	struct rcu_string *name;
	u64 devid = BTRFS_DEV_REPLACE_DEVID;
	int ret = 0;

	*device_out = NULL;
	if (fs_info->fs_devices->seeding) {
		btrfs_err(fs_info, "the filesystem is a seed filesystem!");
		return -EINVAL;
	}

	bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
				  fs_info->bdev_holder);
	if (IS_ERR(bdev)) {
		btrfs_err(fs_info, "target device %s is invalid!", device_path);
		return PTR_ERR(bdev);
	}

	filemap_write_and_wait(bdev->bd_inode->i_mapping);

	devices = &fs_info->fs_devices->devices;
	list_for_each_entry(device, devices, dev_list) {
		if (device->bdev == bdev) {
			btrfs_err(fs_info,
				  "target device is in the filesystem!");
			ret = -EEXIST;
			goto error;
		}
	}


	if (i_size_read(bdev->bd_inode) <
	    btrfs_device_get_total_bytes(srcdev)) {
		btrfs_err(fs_info,
			  "target device is smaller than source device!");
		ret = -EINVAL;
		goto error;
	}


	device = btrfs_alloc_device(NULL, &devid, NULL);
	if (IS_ERR(device)) {
		ret = PTR_ERR(device);
		goto error;
	}

	name = rcu_string_strdup(device_path, GFP_KERNEL);
	if (!name) {
		btrfs_free_device(device);
		ret = -ENOMEM;
		goto error;
	}
	rcu_assign_pointer(device->name, name);

	mutex_lock(&fs_info->fs_devices->device_list_mutex);
	set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
	device->generation = 0;
	device->io_width = fs_info->sectorsize;
	device->io_align = fs_info->sectorsize;
	device->sector_size = fs_info->sectorsize;
	device->total_bytes = btrfs_device_get_total_bytes(srcdev);
	device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev);
	device->bytes_used = btrfs_device_get_bytes_used(srcdev);
	device->commit_total_bytes = srcdev->commit_total_bytes;
	device->commit_bytes_used = device->bytes_used;
	device->fs_info = fs_info;
	device->bdev = bdev;
	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
	set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
	device->mode = FMODE_EXCL;
	device->dev_stats_valid = 1;
	set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
	device->fs_devices = fs_info->fs_devices;
	list_add(&device->dev_list, &fs_info->fs_devices->devices);
	fs_info->fs_devices->num_devices++;
	fs_info->fs_devices->open_devices++;
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);

	*device_out = device;
	return 0;

error:
	blkdev_put(bdev, FMODE_EXCL);
	return ret;
}

278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
/*
 * called from commit_transaction. Writes changed device replace state to
 * disk.
 */
int btrfs_run_dev_replace(struct btrfs_trans_handle *trans,
			  struct btrfs_fs_info *fs_info)
{
	int ret;
	struct btrfs_root *dev_root = fs_info->dev_root;
	struct btrfs_path *path;
	struct btrfs_key key;
	struct extent_buffer *eb;
	struct btrfs_dev_replace_item *ptr;
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;

293
	btrfs_dev_replace_read_lock(dev_replace);
294 295
	if (!dev_replace->is_valid ||
	    !dev_replace->item_needs_writeback) {
296
		btrfs_dev_replace_read_unlock(dev_replace);
297 298
		return 0;
	}
299
	btrfs_dev_replace_read_unlock(dev_replace);
300 301 302 303 304 305 306 307 308 309 310 311

	key.objectid = 0;
	key.type = BTRFS_DEV_REPLACE_KEY;
	key.offset = 0;

	path = btrfs_alloc_path();
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
	ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
	if (ret < 0) {
J
Jeff Mahoney 已提交
312 313 314
		btrfs_warn(fs_info,
			   "error %d while searching for dev_replace item!",
			   ret);
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
		goto out;
	}

	if (ret == 0 &&
	    btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
		/*
		 * need to delete old one and insert a new one.
		 * Since no attempt is made to recover any old state, if the
		 * dev_replace state is 'running', the data on the target
		 * drive is lost.
		 * It would be possible to recover the state: just make sure
		 * that the beginning of the item is never changed and always
		 * contains all the essential information. Then read this
		 * minimal set of information and use it as a base for the
		 * new state.
		 */
		ret = btrfs_del_item(trans, dev_root, path);
		if (ret != 0) {
J
Jeff Mahoney 已提交
333 334 335
			btrfs_warn(fs_info,
				   "delete too small dev_replace item failed %d!",
				   ret);
336 337 338 339 340 341 342 343 344 345 346
			goto out;
		}
		ret = 1;
	}

	if (ret == 1) {
		/* need to insert a new item */
		btrfs_release_path(path);
		ret = btrfs_insert_empty_item(trans, dev_root, path,
					      &key, sizeof(*ptr));
		if (ret < 0) {
J
Jeff Mahoney 已提交
347 348
			btrfs_warn(fs_info,
				   "insert dev_replace item failed %d!", ret);
349 350 351 352 353 354 355 356
			goto out;
		}
	}

	eb = path->nodes[0];
	ptr = btrfs_item_ptr(eb, path->slots[0],
			     struct btrfs_dev_replace_item);

357
	btrfs_dev_replace_write_lock(dev_replace);
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
	if (dev_replace->srcdev)
		btrfs_set_dev_replace_src_devid(eb, ptr,
			dev_replace->srcdev->devid);
	else
		btrfs_set_dev_replace_src_devid(eb, ptr, (u64)-1);
	btrfs_set_dev_replace_cont_reading_from_srcdev_mode(eb, ptr,
		dev_replace->cont_reading_from_srcdev_mode);
	btrfs_set_dev_replace_replace_state(eb, ptr,
		dev_replace->replace_state);
	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);
	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);
	btrfs_set_dev_replace_num_write_errors(eb, ptr,
		atomic64_read(&dev_replace->num_write_errors));
	btrfs_set_dev_replace_num_uncorrectable_read_errors(eb, ptr,
		atomic64_read(&dev_replace->num_uncorrectable_read_errors));
	dev_replace->cursor_left_last_write_of_item =
		dev_replace->cursor_left;
	btrfs_set_dev_replace_cursor_left(eb, ptr,
		dev_replace->cursor_left_last_write_of_item);
	btrfs_set_dev_replace_cursor_right(eb, ptr,
		dev_replace->cursor_right);
	dev_replace->item_needs_writeback = 0;
380
	btrfs_dev_replace_write_unlock(dev_replace);
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397

	btrfs_mark_buffer_dirty(eb);

out:
	btrfs_free_path(path);

	return ret;
}

void btrfs_after_dev_replace_commit(struct btrfs_fs_info *fs_info)
{
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;

	dev_replace->committed_cursor_left =
		dev_replace->cursor_left_last_write_of_item;
}

398 399
static char* btrfs_dev_name(struct btrfs_device *device)
{
400
	if (!device || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
401 402 403 404 405
		return "<missing disk>";
	else
		return rcu_str_deref(device->name);
}

406 407 408
int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
		const char *tgtdev_name, u64 srcdevid, const char *srcdev_name,
		int read_src)
409
{
410
	struct btrfs_root *root = fs_info->dev_root;
411 412 413 414 415 416
	struct btrfs_trans_handle *trans;
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
	int ret;
	struct btrfs_device *tgt_device = NULL;
	struct btrfs_device *src_device = NULL;

417
	/* the disk copy procedure reuses the scrub code */
418
	mutex_lock(&fs_info->volume_mutex);
419
	ret = btrfs_find_device_by_devspec(fs_info, srcdevid,
420
					    srcdev_name, &src_device);
421
	if (ret) {
422 423
		mutex_unlock(&fs_info->volume_mutex);
		return ret;
424 425
	}

426
	ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,
427 428 429 430
					    src_device, &tgt_device);
	mutex_unlock(&fs_info->volume_mutex);
	if (ret)
		return ret;
431

432 433 434 435 436 437
	/*
	 * Here we commit the transaction to make sure commit_total_bytes
	 * of all the devices are updated.
	 */
	trans = btrfs_attach_transaction(root);
	if (!IS_ERR(trans)) {
438
		ret = btrfs_commit_transaction(trans);
439 440 441 442 443 444
		if (ret)
			return ret;
	} else if (PTR_ERR(trans) != -ENOENT) {
		return PTR_ERR(trans);
	}

445
	btrfs_dev_replace_write_lock(dev_replace);
446 447 448 449 450 451 452
	switch (dev_replace->replace_state) {
	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
		break;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
453
		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED;
454 455 456
		goto leave;
	}

457
	dev_replace->cont_reading_from_srcdev_mode = read_src;
458 459 460 461
	WARN_ON(!src_device);
	dev_replace->srcdev = src_device;
	dev_replace->tgtdev = tgt_device;

A
Anand Jain 已提交
462
	btrfs_info_in_rcu(fs_info,
463
		      "dev_replace from %s (devid %llu) to %s started",
464
		      btrfs_dev_name(src_device),
465 466 467 468 469 470 471 472
		      src_device->devid,
		      rcu_str_deref(tgt_device->name));

	/*
	 * from now on, the writes to the srcdev are all duplicated to
	 * go to the tgtdev as well (refer to btrfs_map_block()).
	 */
	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
473
	dev_replace->time_started = get_seconds();
474 475 476 477 478 479
	dev_replace->cursor_left = 0;
	dev_replace->committed_cursor_left = 0;
	dev_replace->cursor_left_last_write_of_item = 0;
	dev_replace->cursor_right = 0;
	dev_replace->is_valid = 1;
	dev_replace->item_needs_writeback = 1;
480 481
	atomic64_set(&dev_replace->num_write_errors, 0);
	atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
482
	btrfs_dev_replace_write_unlock(dev_replace);
483

484 485
	ret = btrfs_sysfs_add_device_link(tgt_device->fs_devices, tgt_device);
	if (ret)
486
		btrfs_err(fs_info, "kobj add dev failed %d", ret);
487

488
	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
489 490 491 492 493

	/* force writing the updated state information to disk */
	trans = btrfs_start_transaction(root, 0);
	if (IS_ERR(trans)) {
		ret = PTR_ERR(trans);
494
		btrfs_dev_replace_write_lock(dev_replace);
495 496 497
		goto leave;
	}

498
	ret = btrfs_commit_transaction(trans);
499 500 501 502
	WARN_ON(ret);

	/* the disk copy procedure reuses the scrub code */
	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,
503
			      btrfs_device_get_total_bytes(src_device),
504 505
			      &dev_replace->scrub_progress, 0, 1);

A
Anand Jain 已提交
506
	ret = btrfs_dev_replace_finishing(fs_info, ret);
507
	if (ret == -EINPROGRESS) {
508
		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS;
509 510 511
	} else {
		WARN_ON(ret);
	}
512

513
	return ret;
514 515 516 517

leave:
	dev_replace->srcdev = NULL;
	dev_replace->tgtdev = NULL;
518
	btrfs_dev_replace_write_unlock(dev_replace);
519
	btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);
520 521 522
	return ret;
}

523
int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,
524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
			    struct btrfs_ioctl_dev_replace_args *args)
{
	int ret;

	switch (args->start.cont_reading_from_srcdev_mode) {
	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_ALWAYS:
	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_AVOID:
		break;
	default:
		return -EINVAL;
	}

	if ((args->start.srcdevid == 0 && args->start.srcdev_name[0] == '\0') ||
	    args->start.tgtdev_name[0] == '\0')
		return -EINVAL;

540
	ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,
541 542 543 544 545 546 547 548 549 550 551
					args->start.srcdevid,
					args->start.srcdev_name,
					args->start.cont_reading_from_srcdev_mode);
	args->result = ret;
	/* don't warn if EINPROGRESS, someone else might be running scrub */
	if (ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS)
		ret = 0;

	return ret;
}

552
/*
553
 * blocked until all in-flight bios operations are finished.
554 555 556 557
 */
static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)
{
	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
558 559
	wait_event(fs_info->replace_wait, !percpu_counter_sum(
		   &fs_info->bio_counter));
560 561 562 563 564 565 566 567
}

/*
 * we have removed target device, it is safe to allow new bios request.
 */
static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)
{
	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
568
	wake_up(&fs_info->replace_wait);
569 570
}

571 572 573 574 575 576 577 578 579 580 581 582 583 584
static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
				       int scrub_ret)
{
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
	struct btrfs_device *tgt_device;
	struct btrfs_device *src_device;
	struct btrfs_root *root = fs_info->tree_root;
	u8 uuid_tmp[BTRFS_UUID_SIZE];
	struct btrfs_trans_handle *trans;
	int ret = 0;

	/* don't allow cancel or unmount to disturb the finishing procedure */
	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);

585
	btrfs_dev_replace_read_lock(dev_replace);
586 587 588
	/* was the operation canceled, or is it finished? */
	if (dev_replace->replace_state !=
	    BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED) {
589
		btrfs_dev_replace_read_unlock(dev_replace);
590 591 592 593 594 595
		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
		return 0;
	}

	tgt_device = dev_replace->tgtdev;
	src_device = dev_replace->srcdev;
596
	btrfs_dev_replace_read_unlock(dev_replace);
597 598 599 600 601

	/*
	 * flush all outstanding I/O and inode extent mappings before the
	 * copy operation is declared as being finished
	 */
602
	ret = btrfs_start_delalloc_roots(fs_info, 0, -1);
603 604 605 606
	if (ret) {
		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
		return ret;
	}
607
	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
608 609 610 611 612 613

	trans = btrfs_start_transaction(root, 0);
	if (IS_ERR(trans)) {
		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
		return PTR_ERR(trans);
	}
614
	ret = btrfs_commit_transaction(trans);
615 616
	WARN_ON(ret);

617
	mutex_lock(&uuid_mutex);
618
	/* keep away write_all_supers() during the finishing procedure */
619 620
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
	mutex_lock(&fs_info->chunk_mutex);
621
	btrfs_dev_replace_write_lock(dev_replace);
622 623 624 625 626
	dev_replace->replace_state =
		scrub_ret ? BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED
			  : BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED;
	dev_replace->tgtdev = NULL;
	dev_replace->srcdev = NULL;
627
	dev_replace->time_stopped = get_seconds();
628 629
	dev_replace->item_needs_writeback = 1;

630 631 632 633 634 635
	/* replace old device with new one in mapping tree */
	if (!scrub_ret) {
		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,
								src_device,
								tgt_device);
	} else {
636 637
		btrfs_err_in_rcu(fs_info,
				 "btrfs_scrub_dev(%s, %llu, %s) failed %d",
638
				 btrfs_dev_name(src_device),
639 640
				 src_device->devid,
				 rcu_str_deref(tgt_device->name), scrub_ret);
641
		btrfs_dev_replace_write_unlock(dev_replace);
642 643
		mutex_unlock(&fs_info->chunk_mutex);
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
644
		mutex_unlock(&uuid_mutex);
645
		btrfs_rm_dev_replace_blocked(fs_info);
646 647
		if (tgt_device)
			btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);
648
		btrfs_rm_dev_replace_unblocked(fs_info);
649 650
		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);

651
		return scrub_ret;
652 653
	}

654 655
	btrfs_info_in_rcu(fs_info,
			  "dev_replace from %s (devid %llu) to %s finished",
656
			  btrfs_dev_name(src_device),
657 658
			  src_device->devid,
			  rcu_str_deref(tgt_device->name));
659
	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &tgt_device->dev_state);
660 661 662 663 664
	tgt_device->devid = src_device->devid;
	src_device->devid = BTRFS_DEV_REPLACE_DEVID;
	memcpy(uuid_tmp, tgt_device->uuid, sizeof(uuid_tmp));
	memcpy(tgt_device->uuid, src_device->uuid, sizeof(tgt_device->uuid));
	memcpy(src_device->uuid, uuid_tmp, sizeof(src_device->uuid));
665 666 667 668
	btrfs_device_set_total_bytes(tgt_device, src_device->total_bytes);
	btrfs_device_set_disk_total_bytes(tgt_device,
					  src_device->disk_total_bytes);
	btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used);
669 670
	ASSERT(list_empty(&src_device->resized_list));
	tgt_device->commit_total_bytes = src_device->commit_total_bytes;
671
	tgt_device->commit_bytes_used = src_device->bytes_used;
672 673 674

	btrfs_assign_next_active_device(fs_info, src_device, tgt_device);

675
	list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list);
676
	fs_info->fs_devices->rw_devices++;
677

678
	btrfs_dev_replace_write_unlock(dev_replace);
679

680 681
	btrfs_rm_dev_replace_blocked(fs_info);

682
	btrfs_rm_dev_replace_remove_srcdev(fs_info, src_device);
683

684 685
	btrfs_rm_dev_replace_unblocked(fs_info);

686 687 688 689 690 691 692
	/*
	 * this is again a consistent state where no dev_replace procedure
	 * is running, the target device is part of the filesystem, the
	 * source device is not part of the filesystem anymore and its 1st
	 * superblock is scratched out so that it is no longer marked to
	 * belong to this filesystem.
	 */
693 694
	mutex_unlock(&fs_info->chunk_mutex);
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
695
	mutex_unlock(&uuid_mutex);
696

697
	/* replace the sysfs entry */
698
	btrfs_sysfs_rm_device_link(fs_info->fs_devices, src_device);
699 700
	btrfs_rm_dev_replace_free_srcdev(fs_info, src_device);

701 702 703
	/* write back the superblocks */
	trans = btrfs_start_transaction(root, 0);
	if (!IS_ERR(trans))
704
		btrfs_commit_transaction(trans);
705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726

	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);

	return 0;
}

static void btrfs_dev_replace_update_device_in_mapping_tree(
						struct btrfs_fs_info *fs_info,
						struct btrfs_device *srcdev,
						struct btrfs_device *tgtdev)
{
	struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree;
	struct extent_map *em;
	struct map_lookup *map;
	u64 start = 0;
	int i;

	write_lock(&em_tree->lock);
	do {
		em = lookup_extent_mapping(em_tree, start, (u64)-1);
		if (!em)
			break;
727
		map = em->map_lookup;
728 729 730 731 732 733 734 735 736
		for (i = 0; i < map->num_stripes; i++)
			if (srcdev == map->stripes[i].dev)
				map->stripes[i].dev = tgtdev;
		start = em->start + em->len;
		free_extent_map(em);
	} while (start);
	write_unlock(&em_tree->lock);
}

737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765
/*
 * Read progress of device replace status according to the state and last
 * stored position. The value format is the same as for
 * btrfs_dev_replace::progress_1000
 */
static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)
{
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
	u64 ret = 0;

	switch (dev_replace->replace_state) {
	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
		ret = 0;
		break;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
		ret = 1000;
		break;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
		ret = div64_u64(dev_replace->cursor_left,
				div_u64(btrfs_device_get_total_bytes(
						dev_replace->srcdev), 1000));
		break;
	}

	return ret;
}

766 767 768 769 770
void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,
			      struct btrfs_ioctl_dev_replace_args *args)
{
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;

771
	btrfs_dev_replace_read_lock(dev_replace);
772 773 774 775 776 777 778 779 780 781
	/* even if !dev_replace_is_valid, the values are good enough for
	 * the replace_status ioctl */
	args->result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
	args->status.replace_state = dev_replace->replace_state;
	args->status.time_started = dev_replace->time_started;
	args->status.time_stopped = dev_replace->time_stopped;
	args->status.num_write_errors =
		atomic64_read(&dev_replace->num_write_errors);
	args->status.num_uncorrectable_read_errors =
		atomic64_read(&dev_replace->num_uncorrectable_read_errors);
782
	args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);
783
	btrfs_dev_replace_read_unlock(dev_replace);
784 785
}

786
int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)
787 788 789
{
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
	struct btrfs_device *tgt_device = NULL;
790
	struct btrfs_device *src_device = NULL;
791 792
	struct btrfs_trans_handle *trans;
	struct btrfs_root *root = fs_info->tree_root;
793
	int result;
794 795
	int ret;

796
	if (sb_rdonly(fs_info->sb))
797 798
		return -EROFS;

799
	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
800
	btrfs_dev_replace_write_lock(dev_replace);
801 802 803 804 805
	switch (dev_replace->replace_state) {
	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
806
		btrfs_dev_replace_write_unlock(dev_replace);
807 808 809 810 811
		goto leave;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
		tgt_device = dev_replace->tgtdev;
812
		src_device = dev_replace->srcdev;
813 814 815 816 817
		dev_replace->tgtdev = NULL;
		dev_replace->srcdev = NULL;
		break;
	}
	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED;
818
	dev_replace->time_stopped = get_seconds();
819
	dev_replace->item_needs_writeback = 1;
820
	btrfs_dev_replace_write_unlock(dev_replace);
821 822 823 824 825 826 827
	btrfs_scrub_cancel(fs_info);

	trans = btrfs_start_transaction(root, 0);
	if (IS_ERR(trans)) {
		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
		return PTR_ERR(trans);
	}
828
	ret = btrfs_commit_transaction(trans);
829
	WARN_ON(ret);
830 831 832 833 834 835

	btrfs_info_in_rcu(fs_info,
		"dev_replace from %s (devid %llu) to %s canceled",
		btrfs_dev_name(src_device), src_device->devid,
		btrfs_dev_name(tgt_device));

836 837 838 839 840 841 842 843 844 845 846 847 848
	if (tgt_device)
		btrfs_destroy_dev_replace_tgtdev(fs_info, tgt_device);

leave:
	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
	return result;
}

void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)
{
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;

	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
849
	btrfs_dev_replace_write_lock(dev_replace);
850 851 852 853 854 855 856 857 858
	switch (dev_replace->replace_state) {
	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
		break;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
		dev_replace->replace_state =
			BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
859
		dev_replace->time_stopped = get_seconds();
860
		dev_replace->item_needs_writeback = 1;
861
		btrfs_info(fs_info, "suspending dev_replace for unmount");
862 863 864
		break;
	}

865
	btrfs_dev_replace_write_unlock(dev_replace);
866 867 868 869 870 871 872 873 874
	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
}

/* resume dev_replace procedure that was interrupted by unmount */
int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)
{
	struct task_struct *task;
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;

875
	btrfs_dev_replace_write_lock(dev_replace);
876 877 878 879
	switch (dev_replace->replace_state) {
	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
880
		btrfs_dev_replace_write_unlock(dev_replace);
881 882 883 884 885 886 887 888 889
		return 0;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
		break;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
		dev_replace->replace_state =
			BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
		break;
	}
	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {
890
		btrfs_info(fs_info,
J
Jeff Mahoney 已提交
891 892 893
			   "cannot continue dev_replace, tgtdev is missing");
		btrfs_info(fs_info,
			   "you may cancel the operation after 'mount -o degraded'");
894
		btrfs_dev_replace_write_unlock(dev_replace);
895 896
		return 0;
	}
897
	btrfs_dev_replace_write_unlock(dev_replace);
898

899 900 901 902 903 904 905 906 907 908 909
	/*
	 * This could collide with a paused balance, but the exclusive op logic
	 * should never allow both to start and pause. We don't want to allow
	 * dev-replace to start anyway.
	 */
	if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
		btrfs_info(fs_info,
		"cannot resume dev-replace, other exclusive operation running");
		return 0;
	}

910
	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");
911
	return PTR_ERR_OR_ZERO(task);
912 913 914 915 916 917 918
}

static int btrfs_dev_replace_kthread(void *data)
{
	struct btrfs_fs_info *fs_info = data;
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
	u64 progress;
919
	int ret;
920

921 922 923
	progress = btrfs_dev_replace_progress(fs_info);
	progress = div_u64(progress, 10);
	btrfs_info_in_rcu(fs_info,
924 925
		"continuing dev_replace from %s (devid %llu) to target %s @%u%%",
		btrfs_dev_name(dev_replace->srcdev),
926
		dev_replace->srcdev->devid,
927
		btrfs_dev_name(dev_replace->tgtdev),
928 929
		(unsigned int)progress);

930 931
	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,
			      dev_replace->committed_cursor_left,
932
			      btrfs_device_get_total_bytes(dev_replace->srcdev),
933 934 935
			      &dev_replace->scrub_progress, 0, 1);
	ret = btrfs_dev_replace_finishing(fs_info, ret);
	WARN_ON(ret);
936 937

	clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960
	return 0;
}

int btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)
{
	if (!dev_replace->is_valid)
		return 0;

	switch (dev_replace->replace_state) {
	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
		return 0;
	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
		/*
		 * return true even if tgtdev is missing (this is
		 * something that can happen if the dev_replace
		 * procedure is suspended by an umount and then
		 * the tgtdev is missing (or "btrfs dev scan") was
		 * not called and the the filesystem is remounted
		 * in degraded state. This does not stop the
		 * dev_replace procedure. It needs to be canceled
961
		 * manually if the cancellation is wanted.
962 963 964 965 966 967
		 */
		break;
	}
	return 1;
}

968
void btrfs_dev_replace_read_lock(struct btrfs_dev_replace *dev_replace)
969
{
970 971 972 973 974 975 976 977 978
	read_lock(&dev_replace->lock);
	atomic_inc(&dev_replace->read_locks);
}

void btrfs_dev_replace_read_unlock(struct btrfs_dev_replace *dev_replace)
{
	ASSERT(atomic_read(&dev_replace->read_locks) > 0);
	atomic_dec(&dev_replace->read_locks);
	read_unlock(&dev_replace->lock);
979
}
980

981
void btrfs_dev_replace_write_lock(struct btrfs_dev_replace *dev_replace)
982
{
983 984 985 986 987
again:
	wait_event(dev_replace->read_lock_wq,
		   atomic_read(&dev_replace->blocking_readers) == 0);
	write_lock(&dev_replace->lock);
	if (atomic_read(&dev_replace->blocking_readers)) {
988
		write_unlock(&dev_replace->lock);
989
		goto again;
990
	}
991
}
992

993 994 995 996 997 998
void btrfs_dev_replace_write_unlock(struct btrfs_dev_replace *dev_replace)
{
	ASSERT(atomic_read(&dev_replace->blocking_readers) == 0);
	write_unlock(&dev_replace->lock);
}

999 1000 1001 1002 1003 1004 1005 1006
/* inc blocking cnt and release read lock */
void btrfs_dev_replace_set_lock_blocking(
					struct btrfs_dev_replace *dev_replace)
{
	/* only set blocking for read lock */
	ASSERT(atomic_read(&dev_replace->read_locks) > 0);
	atomic_inc(&dev_replace->blocking_readers);
	read_unlock(&dev_replace->lock);
1007 1008
}

1009 1010 1011
/* acquire read lock and dec blocking cnt */
void btrfs_dev_replace_clear_lock_blocking(
					struct btrfs_dev_replace *dev_replace)
1012
{
1013 1014 1015 1016 1017 1018 1019
	/* only set blocking for read lock */
	ASSERT(atomic_read(&dev_replace->read_locks) > 0);
	ASSERT(atomic_read(&dev_replace->blocking_readers) > 0);
	read_lock(&dev_replace->lock);
	if (atomic_dec_and_test(&dev_replace->blocking_readers) &&
	    waitqueue_active(&dev_replace->read_lock_wq))
		wake_up(&dev_replace->read_lock_wq);
1020
}
1021 1022 1023 1024 1025 1026

void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info)
{
	percpu_counter_inc(&fs_info->bio_counter);
}

1027
void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)
1028
{
1029
	percpu_counter_sub(&fs_info->bio_counter, amount);
1030 1031 1032 1033 1034 1035 1036

	if (waitqueue_active(&fs_info->replace_wait))
		wake_up(&fs_info->replace_wait);
}

void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)
{
1037 1038 1039 1040 1041 1042
	while (1) {
		percpu_counter_inc(&fs_info->bio_counter);
		if (likely(!test_bit(BTRFS_FS_STATE_DEV_REPLACING,
				     &fs_info->fs_state)))
			break;

1043 1044 1045 1046 1047 1048
		btrfs_bio_counter_dec(fs_info);
		wait_event(fs_info->replace_wait,
			   !test_bit(BTRFS_FS_STATE_DEV_REPLACING,
				     &fs_info->fs_state));
	}
}