volumes.c 186.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
#include <linux/sched.h>
#include <linux/bio.h>
20
#include <linux/slab.h>
21
#include <linux/buffer_head.h>
22
#include <linux/blkdev.h>
23
#include <linux/iocontext.h>
24
#include <linux/capability.h>
25
#include <linux/ratelimit.h>
I
Ilya Dryomov 已提交
26
#include <linux/kthread.h>
D
David Woodhouse 已提交
27
#include <linux/raid/pq.h>
S
Stefan Behrens 已提交
28
#include <linux/semaphore.h>
29
#include <linux/uuid.h>
D
David Woodhouse 已提交
30
#include <asm/div64.h>
31 32 33 34 35 36
#include "ctree.h"
#include "extent_map.h"
#include "disk-io.h"
#include "transaction.h"
#include "print-tree.h"
#include "volumes.h"
D
David Woodhouse 已提交
37
#include "raid56.h"
38
#include "async-thread.h"
39
#include "check-integrity.h"
40
#include "rcu-string.h"
41
#include "math.h"
42
#include "dev-replace.h"
43
#include "sysfs.h"
44

Z
Zhao Lei 已提交
45 46 47 48 49 50
const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
	[BTRFS_RAID_RAID10] = {
		.sub_stripes	= 2,
		.dev_stripes	= 1,
		.devs_max	= 0,	/* 0 == as many as possible */
		.devs_min	= 4,
51
		.tolerated_failures = 1,
Z
Zhao Lei 已提交
52 53 54 55 56 57 58 59
		.devs_increment	= 2,
		.ncopies	= 2,
	},
	[BTRFS_RAID_RAID1] = {
		.sub_stripes	= 1,
		.dev_stripes	= 1,
		.devs_max	= 2,
		.devs_min	= 2,
60
		.tolerated_failures = 1,
Z
Zhao Lei 已提交
61 62 63 64 65 66 67 68
		.devs_increment	= 2,
		.ncopies	= 2,
	},
	[BTRFS_RAID_DUP] = {
		.sub_stripes	= 1,
		.dev_stripes	= 2,
		.devs_max	= 1,
		.devs_min	= 1,
69
		.tolerated_failures = 0,
Z
Zhao Lei 已提交
70 71 72 73 74 75 76 77
		.devs_increment	= 1,
		.ncopies	= 2,
	},
	[BTRFS_RAID_RAID0] = {
		.sub_stripes	= 1,
		.dev_stripes	= 1,
		.devs_max	= 0,
		.devs_min	= 2,
78
		.tolerated_failures = 0,
Z
Zhao Lei 已提交
79 80 81 82 83 84 85 86
		.devs_increment	= 1,
		.ncopies	= 1,
	},
	[BTRFS_RAID_SINGLE] = {
		.sub_stripes	= 1,
		.dev_stripes	= 1,
		.devs_max	= 1,
		.devs_min	= 1,
87
		.tolerated_failures = 0,
Z
Zhao Lei 已提交
88 89 90 91 92 93 94 95
		.devs_increment	= 1,
		.ncopies	= 1,
	},
	[BTRFS_RAID_RAID5] = {
		.sub_stripes	= 1,
		.dev_stripes	= 1,
		.devs_max	= 0,
		.devs_min	= 2,
96
		.tolerated_failures = 1,
Z
Zhao Lei 已提交
97 98 99 100 101 102 103 104
		.devs_increment	= 1,
		.ncopies	= 2,
	},
	[BTRFS_RAID_RAID6] = {
		.sub_stripes	= 1,
		.dev_stripes	= 1,
		.devs_max	= 0,
		.devs_min	= 3,
105
		.tolerated_failures = 2,
Z
Zhao Lei 已提交
106 107 108 109 110
		.devs_increment	= 1,
		.ncopies	= 3,
	},
};

111
const u64 btrfs_raid_group[BTRFS_NR_RAID_TYPES] = {
Z
Zhao Lei 已提交
112 113 114 115 116 117 118 119 120
	[BTRFS_RAID_RAID10] = BTRFS_BLOCK_GROUP_RAID10,
	[BTRFS_RAID_RAID1]  = BTRFS_BLOCK_GROUP_RAID1,
	[BTRFS_RAID_DUP]    = BTRFS_BLOCK_GROUP_DUP,
	[BTRFS_RAID_RAID0]  = BTRFS_BLOCK_GROUP_RAID0,
	[BTRFS_RAID_SINGLE] = 0,
	[BTRFS_RAID_RAID5]  = BTRFS_BLOCK_GROUP_RAID5,
	[BTRFS_RAID_RAID6]  = BTRFS_BLOCK_GROUP_RAID6,
};

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
/*
 * Table to convert BTRFS_RAID_* to the error code if minimum number of devices
 * condition is not met. Zero means there's no corresponding
 * BTRFS_ERROR_DEV_*_NOT_MET value.
 */
const int btrfs_raid_mindev_error[BTRFS_NR_RAID_TYPES] = {
	[BTRFS_RAID_RAID10] = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
	[BTRFS_RAID_RAID1]  = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
	[BTRFS_RAID_DUP]    = 0,
	[BTRFS_RAID_RAID0]  = 0,
	[BTRFS_RAID_SINGLE] = 0,
	[BTRFS_RAID_RAID5]  = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
	[BTRFS_RAID_RAID6]  = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
};

Y
Yan Zheng 已提交
136
static int init_first_rw_device(struct btrfs_trans_handle *trans,
137
				struct btrfs_fs_info *fs_info,
Y
Yan Zheng 已提交
138
				struct btrfs_device *device);
139
static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
140
static void __btrfs_reset_dev_stats(struct btrfs_device *dev);
141
static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
142
static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
Y
Yan Zheng 已提交
143

144
DEFINE_MUTEX(uuid_mutex);
145
static LIST_HEAD(fs_uuids);
146 147 148 149
struct list_head *btrfs_get_fs_uuids(void)
{
	return &fs_uuids;
}
150

151 152 153 154
static struct btrfs_fs_devices *__alloc_fs_devices(void)
{
	struct btrfs_fs_devices *fs_devs;

155
	fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
156 157 158 159 160 161
	if (!fs_devs)
		return ERR_PTR(-ENOMEM);

	mutex_init(&fs_devs->device_list_mutex);

	INIT_LIST_HEAD(&fs_devs->devices);
162
	INIT_LIST_HEAD(&fs_devs->resized_devices);
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
	INIT_LIST_HEAD(&fs_devs->alloc_list);
	INIT_LIST_HEAD(&fs_devs->list);

	return fs_devs;
}

/**
 * alloc_fs_devices - allocate struct btrfs_fs_devices
 * @fsid:	a pointer to UUID for this FS.  If NULL a new UUID is
 *		generated.
 *
 * Return: a pointer to a new &struct btrfs_fs_devices on success;
 * ERR_PTR() on error.  Returned struct is not linked onto any lists and
 * can be destroyed with kfree() right away.
 */
static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid)
{
	struct btrfs_fs_devices *fs_devs;

	fs_devs = __alloc_fs_devices();
	if (IS_ERR(fs_devs))
		return fs_devs;

	if (fsid)
		memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
	else
		generate_random_uuid(fs_devs->fsid);

	return fs_devs;
}

Y
Yan Zheng 已提交
194 195 196 197 198 199 200 201
static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
{
	struct btrfs_device *device;
	WARN_ON(fs_devices->opened);
	while (!list_empty(&fs_devices->devices)) {
		device = list_entry(fs_devices->devices.next,
				    struct btrfs_device, dev_list);
		list_del(&device->dev_list);
202
		rcu_string_free(device->name);
Y
Yan Zheng 已提交
203 204 205 206 207
		kfree(device);
	}
	kfree(fs_devices);
}

208 209 210 211 212 213 214
static void btrfs_kobject_uevent(struct block_device *bdev,
				 enum kobject_action action)
{
	int ret;

	ret = kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, action);
	if (ret)
215
		pr_warn("BTRFS: Sending event '%d' to kobject: '%s' (%p): failed\n",
216 217 218 219 220
			action,
			kobject_name(&disk_to_dev(bdev->bd_disk)->kobj),
			&disk_to_dev(bdev->bd_disk)->kobj);
}

221
void btrfs_cleanup_fs_uuids(void)
222 223 224
{
	struct btrfs_fs_devices *fs_devices;

Y
Yan Zheng 已提交
225 226 227 228
	while (!list_empty(&fs_uuids)) {
		fs_devices = list_entry(fs_uuids.next,
					struct btrfs_fs_devices, list);
		list_del(&fs_devices->list);
Y
Yan Zheng 已提交
229
		free_fs_devices(fs_devices);
230 231 232
	}
}

233 234 235 236
static struct btrfs_device *__alloc_device(void)
{
	struct btrfs_device *dev;

237
	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
238 239 240 241 242
	if (!dev)
		return ERR_PTR(-ENOMEM);

	INIT_LIST_HEAD(&dev->dev_list);
	INIT_LIST_HEAD(&dev->dev_alloc_list);
243
	INIT_LIST_HEAD(&dev->resized_list);
244 245 246 247 248

	spin_lock_init(&dev->io_lock);

	spin_lock_init(&dev->reada_lock);
	atomic_set(&dev->reada_in_flight, 0);
249
	atomic_set(&dev->dev_stats_ccnt, 0);
250
	btrfs_device_data_ordered_init(dev);
251 252
	INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
	INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
253 254 255 256

	return dev;
}

257 258
static noinline struct btrfs_device *__find_device(struct list_head *head,
						   u64 devid, u8 *uuid)
259 260 261
{
	struct btrfs_device *dev;

Q
Qinghuang Feng 已提交
262
	list_for_each_entry(dev, head, dev_list) {
263
		if (dev->devid == devid &&
264
		    (!uuid || !memcmp(dev->uuid, uuid, BTRFS_UUID_SIZE))) {
265
			return dev;
266
		}
267 268 269 270
	}
	return NULL;
}

271
static noinline struct btrfs_fs_devices *find_fsid(u8 *fsid)
272 273 274
{
	struct btrfs_fs_devices *fs_devices;

Q
Qinghuang Feng 已提交
275
	list_for_each_entry(fs_devices, &fs_uuids, list) {
276 277 278 279 280 281
		if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
			return fs_devices;
	}
	return NULL;
}

282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
static int
btrfs_get_bdev_and_sb(const char *device_path, fmode_t flags, void *holder,
		      int flush, struct block_device **bdev,
		      struct buffer_head **bh)
{
	int ret;

	*bdev = blkdev_get_by_path(device_path, flags, holder);

	if (IS_ERR(*bdev)) {
		ret = PTR_ERR(*bdev);
		goto error;
	}

	if (flush)
		filemap_write_and_wait((*bdev)->bd_inode->i_mapping);
	ret = set_blocksize(*bdev, 4096);
	if (ret) {
		blkdev_put(*bdev, flags);
		goto error;
	}
	invalidate_bdev(*bdev);
	*bh = btrfs_read_dev_super(*bdev);
305 306
	if (IS_ERR(*bh)) {
		ret = PTR_ERR(*bh);
307 308 309 310 311 312 313 314 315 316 317 318
		blkdev_put(*bdev, flags);
		goto error;
	}

	return 0;

error:
	*bdev = NULL;
	*bh = NULL;
	return ret;
}

319 320 321 322 323 324 325 326 327 328 329 330 331 332
static void requeue_list(struct btrfs_pending_bios *pending_bios,
			struct bio *head, struct bio *tail)
{

	struct bio *old_head;

	old_head = pending_bios->head;
	pending_bios->head = head;
	if (pending_bios->tail)
		tail->bi_next = old_head;
	else
		pending_bios->tail = tail;
}

333 334 335 336 337 338 339 340 341 342 343
/*
 * we try to collect pending bios for a device so we don't get a large
 * number of procs sending bios down to the same device.  This greatly
 * improves the schedulers ability to collect and merge the bios.
 *
 * But, it also turns into a long list of bios to process and that is sure
 * to eventually make the worker thread block.  The solution here is to
 * make some progress and then put this work struct back at the end of
 * the list if the block device is congested.  This way, multiple devices
 * can make progress from a single worker thread.
 */
344
static noinline void run_scheduled_bios(struct btrfs_device *device)
345
{
346
	struct btrfs_fs_info *fs_info = device->fs_info;
347 348
	struct bio *pending;
	struct backing_dev_info *bdi;
349
	struct btrfs_pending_bios *pending_bios;
350 351 352
	struct bio *tail;
	struct bio *cur;
	int again = 0;
353
	unsigned long num_run;
354
	unsigned long batch_run = 0;
355
	unsigned long limit;
356
	unsigned long last_waited = 0;
357
	int force_reg = 0;
M
Miao Xie 已提交
358
	int sync_pending = 0;
359 360 361 362 363 364 365 366 367
	struct blk_plug plug;

	/*
	 * this function runs all the bios we've collected for
	 * a particular device.  We don't want to wander off to
	 * another device without first sending all of these down.
	 * So, setup a plug here and finish it off before we return
	 */
	blk_start_plug(&plug);
368

369
	bdi = blk_get_backing_dev_info(device->bdev);
370 371 372
	limit = btrfs_async_submit_limit(fs_info);
	limit = limit * 2 / 3;

373 374 375
loop:
	spin_lock(&device->io_lock);

376
loop_lock:
377
	num_run = 0;
378

379 380 381 382 383
	/* take all the bios off the list at once and process them
	 * later on (without the lock held).  But, remember the
	 * tail and other pointers so the bios can be properly reinserted
	 * into the list if we hit congestion
	 */
384
	if (!force_reg && device->pending_sync_bios.head) {
385
		pending_bios = &device->pending_sync_bios;
386 387
		force_reg = 1;
	} else {
388
		pending_bios = &device->pending_bios;
389 390
		force_reg = 0;
	}
391 392 393

	pending = pending_bios->head;
	tail = pending_bios->tail;
394 395 396 397 398 399 400 401 402 403
	WARN_ON(pending && !tail);

	/*
	 * if pending was null this time around, no bios need processing
	 * at all and we can stop.  Otherwise it'll loop back up again
	 * and do an additional check so no bios are missed.
	 *
	 * device->running_pending is used to synchronize with the
	 * schedule_bio code.
	 */
404 405
	if (device->pending_sync_bios.head == NULL &&
	    device->pending_bios.head == NULL) {
406 407
		again = 0;
		device->running_pending = 0;
408 409 410
	} else {
		again = 1;
		device->running_pending = 1;
411
	}
412 413 414 415

	pending_bios->head = NULL;
	pending_bios->tail = NULL;

416 417
	spin_unlock(&device->io_lock);

C
Chris Mason 已提交
418
	while (pending) {
419 420

		rmb();
421 422 423 424 425 426 427 428
		/* we want to work on both lists, but do more bios on the
		 * sync list than the regular list
		 */
		if ((num_run > 32 &&
		    pending_bios != &device->pending_sync_bios &&
		    device->pending_sync_bios.head) ||
		   (num_run > 64 && pending_bios == &device->pending_sync_bios &&
		    device->pending_bios.head)) {
429 430 431 432 433
			spin_lock(&device->io_lock);
			requeue_list(pending_bios, pending, tail);
			goto loop_lock;
		}

434 435 436
		cur = pending;
		pending = pending->bi_next;
		cur->bi_next = NULL;
437

438 439 440
		/*
		 * atomic_dec_return implies a barrier for waitqueue_active
		 */
441
		if (atomic_dec_return(&fs_info->nr_async_bios) < limit &&
442 443
		    waitqueue_active(&fs_info->async_submit_wait))
			wake_up(&fs_info->async_submit_wait);
444

445
		BUG_ON(atomic_read(&cur->__bi_cnt) == 0);
446

447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
		/*
		 * if we're doing the sync list, record that our
		 * plug has some sync requests on it
		 *
		 * If we're doing the regular list and there are
		 * sync requests sitting around, unplug before
		 * we add more
		 */
		if (pending_bios == &device->pending_sync_bios) {
			sync_pending = 1;
		} else if (sync_pending) {
			blk_finish_plug(&plug);
			blk_start_plug(&plug);
			sync_pending = 0;
		}

463
		btrfsic_submit_bio(cur);
464 465
		num_run++;
		batch_run++;
466 467

		cond_resched();
468 469 470 471 472 473

		/*
		 * we made progress, there is more work to do and the bdi
		 * is now congested.  Back off and let other work structs
		 * run instead
		 */
C
Chris Mason 已提交
474
		if (pending && bdi_write_congested(bdi) && batch_run > 8 &&
475
		    fs_info->fs_devices->open_devices > 1) {
476
			struct io_context *ioc;
477

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
			ioc = current->io_context;

			/*
			 * the main goal here is that we don't want to
			 * block if we're going to be able to submit
			 * more requests without blocking.
			 *
			 * This code does two great things, it pokes into
			 * the elevator code from a filesystem _and_
			 * it makes assumptions about how batching works.
			 */
			if (ioc && ioc->nr_batch_requests > 0 &&
			    time_before(jiffies, ioc->last_waited + HZ/50UL) &&
			    (last_waited == 0 ||
			     ioc->last_waited == last_waited)) {
				/*
				 * we want to go through our batch of
				 * requests and stop.  So, we copy out
				 * the ioc->last_waited time and test
				 * against it before looping
				 */
				last_waited = ioc->last_waited;
500
				cond_resched();
501 502
				continue;
			}
503
			spin_lock(&device->io_lock);
504
			requeue_list(pending_bios, pending, tail);
505
			device->running_pending = 1;
506 507

			spin_unlock(&device->io_lock);
508 509
			btrfs_queue_work(fs_info->submit_workers,
					 &device->work);
510 511
			goto done;
		}
C
Chris Mason 已提交
512 513 514 515 516 517
		/* unplug every 64 requests just for good measure */
		if (batch_run % 64 == 0) {
			blk_finish_plug(&plug);
			blk_start_plug(&plug);
			sync_pending = 0;
		}
518
	}
519

520 521 522 523 524 525 526 527 528
	cond_resched();
	if (again)
		goto loop;

	spin_lock(&device->io_lock);
	if (device->pending_bios.head || device->pending_sync_bios.head)
		goto loop_lock;
	spin_unlock(&device->io_lock);

529
done:
530
	blk_finish_plug(&plug);
531 532
}

533
static void pending_bios_fn(struct btrfs_work *work)
534 535 536 537 538 539 540
{
	struct btrfs_device *device;

	device = container_of(work, struct btrfs_device, work);
	run_scheduled_bios(device);
}

A
Anand Jain 已提交
541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595

void btrfs_free_stale_device(struct btrfs_device *cur_dev)
{
	struct btrfs_fs_devices *fs_devs;
	struct btrfs_device *dev;

	if (!cur_dev->name)
		return;

	list_for_each_entry(fs_devs, &fs_uuids, list) {
		int del = 1;

		if (fs_devs->opened)
			continue;
		if (fs_devs->seeding)
			continue;

		list_for_each_entry(dev, &fs_devs->devices, dev_list) {

			if (dev == cur_dev)
				continue;
			if (!dev->name)
				continue;

			/*
			 * Todo: This won't be enough. What if the same device
			 * comes back (with new uuid and) with its mapper path?
			 * But for now, this does help as mostly an admin will
			 * either use mapper or non mapper path throughout.
			 */
			rcu_read_lock();
			del = strcmp(rcu_str_deref(dev->name),
						rcu_str_deref(cur_dev->name));
			rcu_read_unlock();
			if (!del)
				break;
		}

		if (!del) {
			/* delete the stale device */
			if (fs_devs->num_devices == 1) {
				btrfs_sysfs_remove_fsid(fs_devs);
				list_del(&fs_devs->list);
				free_fs_devices(fs_devs);
			} else {
				fs_devs->num_devices--;
				list_del(&dev->dev_list);
				rcu_string_free(dev->name);
				kfree(dev);
			}
			break;
		}
	}
}

596 597 598 599 600 601 602 603
/*
 * Add new device to list of registered devices
 *
 * Returns:
 * 1   - first time device is seen
 * 0   - device already known
 * < 0 - error
 */
604
static noinline int device_list_add(const char *path,
605 606 607 608 609
			   struct btrfs_super_block *disk_super,
			   u64 devid, struct btrfs_fs_devices **fs_devices_ret)
{
	struct btrfs_device *device;
	struct btrfs_fs_devices *fs_devices;
610
	struct rcu_string *name;
611
	int ret = 0;
612 613 614 615
	u64 found_transid = btrfs_super_generation(disk_super);

	fs_devices = find_fsid(disk_super->fsid);
	if (!fs_devices) {
616 617 618 619
		fs_devices = alloc_fs_devices(disk_super->fsid);
		if (IS_ERR(fs_devices))
			return PTR_ERR(fs_devices);

620
		list_add(&fs_devices->list, &fs_uuids);
621

622 623
		device = NULL;
	} else {
624 625
		device = __find_device(&fs_devices->devices, devid,
				       disk_super->dev_item.uuid);
626
	}
627

628
	if (!device) {
Y
Yan Zheng 已提交
629 630 631
		if (fs_devices->opened)
			return -EBUSY;

632 633 634
		device = btrfs_alloc_device(NULL, &devid,
					    disk_super->dev_item.uuid);
		if (IS_ERR(device)) {
635
			/* we can safely leave the fs_devices entry around */
636
			return PTR_ERR(device);
637
		}
638 639 640

		name = rcu_string_strdup(path, GFP_NOFS);
		if (!name) {
641 642 643
			kfree(device);
			return -ENOMEM;
		}
644
		rcu_assign_pointer(device->name, name);
645

646
		mutex_lock(&fs_devices->device_list_mutex);
647
		list_add_rcu(&device->dev_list, &fs_devices->devices);
648
		fs_devices->num_devices++;
649 650
		mutex_unlock(&fs_devices->device_list_mutex);

651
		ret = 1;
Y
Yan Zheng 已提交
652
		device->fs_devices = fs_devices;
653
	} else if (!device->name || strcmp(device->name->str, path)) {
654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
		/*
		 * When FS is already mounted.
		 * 1. If you are here and if the device->name is NULL that
		 *    means this device was missing at time of FS mount.
		 * 2. If you are here and if the device->name is different
		 *    from 'path' that means either
		 *      a. The same device disappeared and reappeared with
		 *         different name. or
		 *      b. The missing-disk-which-was-replaced, has
		 *         reappeared now.
		 *
		 * We must allow 1 and 2a above. But 2b would be a spurious
		 * and unintentional.
		 *
		 * Further in case of 1 and 2a above, the disk at 'path'
		 * would have missed some transaction when it was away and
		 * in case of 2a the stale bdev has to be updated as well.
		 * 2b must not be allowed at all time.
		 */

		/*
675 676 677 678
		 * For now, we do allow update to btrfs_fs_device through the
		 * btrfs dev scan cli after FS has been mounted.  We're still
		 * tracking a problem where systems fail mount by subvolume id
		 * when we reject replacement on a mounted FS.
679
		 */
680
		if (!fs_devices->opened && found_transid < device->generation) {
681 682 683 684 685 686 687
			/*
			 * That is if the FS is _not_ mounted and if you
			 * are here, that means there is more than one
			 * disk with same uuid and devid.We keep the one
			 * with larger generation number or the last-in if
			 * generation are equal.
			 */
688
			return -EEXIST;
689
		}
690

691
		name = rcu_string_strdup(path, GFP_NOFS);
692 693
		if (!name)
			return -ENOMEM;
694 695
		rcu_string_free(device->name);
		rcu_assign_pointer(device->name, name);
696 697 698 699
		if (device->missing) {
			fs_devices->missing_devices--;
			device->missing = 0;
		}
700 701
	}

702 703 704 705 706 707 708 709 710
	/*
	 * Unmount does not free the btrfs_device struct but would zero
	 * generation along with most of the other members. So just update
	 * it back. We need it to pick the disk with largest generation
	 * (as above).
	 */
	if (!fs_devices->opened)
		device->generation = found_transid;

A
Anand Jain 已提交
711 712 713 714
	/*
	 * if there is new btrfs on an already registered device,
	 * then remove the stale device entry.
	 */
715 716
	if (ret > 0)
		btrfs_free_stale_device(device);
A
Anand Jain 已提交
717

718
	*fs_devices_ret = fs_devices;
719 720

	return ret;
721 722
}

Y
Yan Zheng 已提交
723 724 725 726 727 728
static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
{
	struct btrfs_fs_devices *fs_devices;
	struct btrfs_device *device;
	struct btrfs_device *orig_dev;

729 730 731
	fs_devices = alloc_fs_devices(orig->fsid);
	if (IS_ERR(fs_devices))
		return fs_devices;
Y
Yan Zheng 已提交
732

733
	mutex_lock(&orig->device_list_mutex);
J
Josef Bacik 已提交
734
	fs_devices->total_devices = orig->total_devices;
Y
Yan Zheng 已提交
735

736
	/* We have held the volume lock, it is safe to get the devices. */
Y
Yan Zheng 已提交
737
	list_for_each_entry(orig_dev, &orig->devices, dev_list) {
738 739
		struct rcu_string *name;

740 741 742
		device = btrfs_alloc_device(NULL, &orig_dev->devid,
					    orig_dev->uuid);
		if (IS_ERR(device))
Y
Yan Zheng 已提交
743 744
			goto error;

745 746 747 748
		/*
		 * This is ok to do without rcu read locked because we hold the
		 * uuid mutex so nothing we touch in here is going to disappear.
		 */
749
		if (orig_dev->name) {
750 751
			name = rcu_string_strdup(orig_dev->name->str,
					GFP_KERNEL);
752 753 754 755 756
			if (!name) {
				kfree(device);
				goto error;
			}
			rcu_assign_pointer(device->name, name);
J
Julia Lawall 已提交
757
		}
Y
Yan Zheng 已提交
758 759 760 761 762

		list_add(&device->dev_list, &fs_devices->devices);
		device->fs_devices = fs_devices;
		fs_devices->num_devices++;
	}
763
	mutex_unlock(&orig->device_list_mutex);
Y
Yan Zheng 已提交
764 765
	return fs_devices;
error:
766
	mutex_unlock(&orig->device_list_mutex);
Y
Yan Zheng 已提交
767 768 769 770
	free_fs_devices(fs_devices);
	return ERR_PTR(-ENOMEM);
}

771
void btrfs_close_extra_devices(struct btrfs_fs_devices *fs_devices, int step)
772
{
Q
Qinghuang Feng 已提交
773
	struct btrfs_device *device, *next;
774
	struct btrfs_device *latest_dev = NULL;
775

776 777
	mutex_lock(&uuid_mutex);
again:
778
	/* This is the initialized path, it is safe to release the devices. */
Q
Qinghuang Feng 已提交
779
	list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
780
		if (device->in_fs_metadata) {
781
			if (!device->is_tgtdev_for_dev_replace &&
782 783 784
			    (!latest_dev ||
			     device->generation > latest_dev->generation)) {
				latest_dev = device;
785
			}
Y
Yan Zheng 已提交
786
			continue;
787
		}
Y
Yan Zheng 已提交
788

789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
		if (device->devid == BTRFS_DEV_REPLACE_DEVID) {
			/*
			 * In the first step, keep the device which has
			 * the correct fsid and the devid that is used
			 * for the dev_replace procedure.
			 * In the second step, the dev_replace state is
			 * read from the device tree and it is known
			 * whether the procedure is really active or
			 * not, which means whether this device is
			 * used or whether it should be removed.
			 */
			if (step == 0 || device->is_tgtdev_for_dev_replace) {
				continue;
			}
		}
Y
Yan Zheng 已提交
804
		if (device->bdev) {
805
			blkdev_put(device->bdev, device->mode);
Y
Yan Zheng 已提交
806 807 808 809 810 811
			device->bdev = NULL;
			fs_devices->open_devices--;
		}
		if (device->writeable) {
			list_del_init(&device->dev_alloc_list);
			device->writeable = 0;
812 813
			if (!device->is_tgtdev_for_dev_replace)
				fs_devices->rw_devices--;
Y
Yan Zheng 已提交
814
		}
Y
Yan Zheng 已提交
815 816
		list_del_init(&device->dev_list);
		fs_devices->num_devices--;
817
		rcu_string_free(device->name);
Y
Yan Zheng 已提交
818
		kfree(device);
819
	}
Y
Yan Zheng 已提交
820 821 822 823 824 825

	if (fs_devices->seed) {
		fs_devices = fs_devices->seed;
		goto again;
	}

826
	fs_devices->latest_bdev = latest_dev->bdev;
827

828 829
	mutex_unlock(&uuid_mutex);
}
830

831 832 833 834 835
static void __free_device(struct work_struct *work)
{
	struct btrfs_device *device;

	device = container_of(work, struct btrfs_device, rcu_work);
836
	rcu_string_free(device->name);
837 838 839 840 841 842 843 844 845 846 847 848 849
	kfree(device);
}

static void free_device(struct rcu_head *head)
{
	struct btrfs_device *device;

	device = container_of(head, struct btrfs_device, rcu);

	INIT_WORK(&device->rcu_work, __free_device);
	schedule_work(&device->rcu_work);
}

850 851 852 853 854 855 856 857 858 859 860
static void btrfs_close_bdev(struct btrfs_device *device)
{
	if (device->bdev && device->writeable) {
		sync_blockdev(device->bdev);
		invalidate_bdev(device->bdev);
	}

	if (device->bdev)
		blkdev_put(device->bdev, device->mode);
}

861
static void btrfs_prepare_close_one_device(struct btrfs_device *device)
862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
{
	struct btrfs_fs_devices *fs_devices = device->fs_devices;
	struct btrfs_device *new_device;
	struct rcu_string *name;

	if (device->bdev)
		fs_devices->open_devices--;

	if (device->writeable &&
	    device->devid != BTRFS_DEV_REPLACE_DEVID) {
		list_del_init(&device->dev_alloc_list);
		fs_devices->rw_devices--;
	}

	if (device->missing)
		fs_devices->missing_devices--;

	new_device = btrfs_alloc_device(NULL, &device->devid,
					device->uuid);
	BUG_ON(IS_ERR(new_device)); /* -ENOMEM */

	/* Safe because we are under uuid_mutex */
	if (device->name) {
		name = rcu_string_strdup(device->name->str, GFP_NOFS);
		BUG_ON(!name); /* -ENOMEM */
		rcu_assign_pointer(new_device->name, name);
	}

	list_replace_rcu(&device->dev_list, &new_device->dev_list);
	new_device->fs_devices = device->fs_devices;
}

Y
Yan Zheng 已提交
894
static int __btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
895
{
896
	struct btrfs_device *device, *tmp;
897 898 899
	struct list_head pending_put;

	INIT_LIST_HEAD(&pending_put);
Y
Yan Zheng 已提交
900

Y
Yan Zheng 已提交
901 902
	if (--fs_devices->opened > 0)
		return 0;
903

904
	mutex_lock(&fs_devices->device_list_mutex);
905
	list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list) {
906 907
		btrfs_prepare_close_one_device(device);
		list_add(&device->dev_list, &pending_put);
908
	}
909 910
	mutex_unlock(&fs_devices->device_list_mutex);

911 912 913 914 915 916 917 918 919 920 921 922 923 924
	/*
	 * btrfs_show_devname() is using the device_list_mutex,
	 * sometimes call to blkdev_put() leads vfs calling
	 * into this func. So do put outside of device_list_mutex,
	 * as of now.
	 */
	while (!list_empty(&pending_put)) {
		device = list_first_entry(&pending_put,
				struct btrfs_device, dev_list);
		list_del(&device->dev_list);
		btrfs_close_bdev(device);
		call_rcu(&device->rcu, free_device);
	}

Y
Yan Zheng 已提交
925 926
	WARN_ON(fs_devices->open_devices);
	WARN_ON(fs_devices->rw_devices);
Y
Yan Zheng 已提交
927 928 929
	fs_devices->opened = 0;
	fs_devices->seeding = 0;

930 931 932
	return 0;
}

Y
Yan Zheng 已提交
933 934
int btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
{
Y
Yan Zheng 已提交
935
	struct btrfs_fs_devices *seed_devices = NULL;
Y
Yan Zheng 已提交
936 937 938 939
	int ret;

	mutex_lock(&uuid_mutex);
	ret = __btrfs_close_devices(fs_devices);
Y
Yan Zheng 已提交
940 941 942 943
	if (!fs_devices->opened) {
		seed_devices = fs_devices->seed;
		fs_devices->seed = NULL;
	}
Y
Yan Zheng 已提交
944
	mutex_unlock(&uuid_mutex);
Y
Yan Zheng 已提交
945 946 947 948 949 950 951

	while (seed_devices) {
		fs_devices = seed_devices;
		seed_devices = fs_devices->seed;
		__btrfs_close_devices(fs_devices);
		free_fs_devices(fs_devices);
	}
952 953 954 955 956 957
	/*
	 * Wait for rcu kworkers under __btrfs_close_devices
	 * to finish all blkdev_puts so device is really
	 * free when umount is done.
	 */
	rcu_barrier();
Y
Yan Zheng 已提交
958 959 960
	return ret;
}

Y
Yan Zheng 已提交
961 962
static int __btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
				fmode_t flags, void *holder)
963
{
964
	struct request_queue *q;
965 966 967
	struct block_device *bdev;
	struct list_head *head = &fs_devices->devices;
	struct btrfs_device *device;
968
	struct btrfs_device *latest_dev = NULL;
969 970 971
	struct buffer_head *bh;
	struct btrfs_super_block *disk_super;
	u64 devid;
Y
Yan Zheng 已提交
972
	int seeding = 1;
973
	int ret = 0;
974

975 976
	flags |= FMODE_EXCL;

Q
Qinghuang Feng 已提交
977
	list_for_each_entry(device, head, dev_list) {
978 979
		if (device->bdev)
			continue;
980 981 982
		if (!device->name)
			continue;

983 984 985
		/* Just open everything we can; ignore failures here */
		if (btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
					    &bdev, &bh))
986
			continue;
987 988

		disk_super = (struct btrfs_super_block *)bh->b_data;
989
		devid = btrfs_stack_device_id(&disk_super->dev_item);
990 991 992
		if (devid != device->devid)
			goto error_brelse;

Y
Yan Zheng 已提交
993 994 995 996 997
		if (memcmp(device->uuid, disk_super->dev_item.uuid,
			   BTRFS_UUID_SIZE))
			goto error_brelse;

		device->generation = btrfs_super_generation(disk_super);
998 999 1000
		if (!latest_dev ||
		    device->generation > latest_dev->generation)
			latest_dev = device;
1001

Y
Yan Zheng 已提交
1002 1003 1004 1005 1006 1007 1008
		if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
			device->writeable = 0;
		} else {
			device->writeable = !bdev_read_only(bdev);
			seeding = 0;
		}

1009
		q = bdev_get_queue(bdev);
1010
		if (blk_queue_discard(q))
1011 1012
			device->can_discard = 1;

1013
		device->bdev = bdev;
1014
		device->in_fs_metadata = 0;
1015 1016
		device->mode = flags;

C
Chris Mason 已提交
1017 1018 1019
		if (!blk_queue_nonrot(bdev_get_queue(bdev)))
			fs_devices->rotating = 1;

1020
		fs_devices->open_devices++;
1021 1022
		if (device->writeable &&
		    device->devid != BTRFS_DEV_REPLACE_DEVID) {
Y
Yan Zheng 已提交
1023 1024 1025 1026
			fs_devices->rw_devices++;
			list_add(&device->dev_alloc_list,
				 &fs_devices->alloc_list);
		}
1027
		brelse(bh);
1028
		continue;
1029

1030 1031
error_brelse:
		brelse(bh);
1032
		blkdev_put(bdev, flags);
1033
		continue;
1034
	}
1035
	if (fs_devices->open_devices == 0) {
1036
		ret = -EINVAL;
1037 1038
		goto out;
	}
Y
Yan Zheng 已提交
1039 1040
	fs_devices->seeding = seeding;
	fs_devices->opened = 1;
1041
	fs_devices->latest_bdev = latest_dev->bdev;
Y
Yan Zheng 已提交
1042
	fs_devices->total_rw_bytes = 0;
1043
out:
Y
Yan Zheng 已提交
1044 1045 1046 1047
	return ret;
}

int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1048
		       fmode_t flags, void *holder)
Y
Yan Zheng 已提交
1049 1050 1051 1052 1053
{
	int ret;

	mutex_lock(&uuid_mutex);
	if (fs_devices->opened) {
Y
Yan Zheng 已提交
1054 1055
		fs_devices->opened++;
		ret = 0;
Y
Yan Zheng 已提交
1056
	} else {
1057
		ret = __btrfs_open_devices(fs_devices, flags, holder);
Y
Yan Zheng 已提交
1058
	}
1059 1060 1061 1062
	mutex_unlock(&uuid_mutex);
	return ret;
}

1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
void btrfs_release_disk_super(struct page *page)
{
	kunmap(page);
	put_page(page);
}

int btrfs_read_disk_super(struct block_device *bdev, u64 bytenr,
		struct page **page, struct btrfs_super_block **disk_super)
{
	void *p;
	pgoff_t index;

	/* make sure our super fits in the device */
	if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode))
		return 1;

	/* make sure our super fits in the page */
	if (sizeof(**disk_super) > PAGE_SIZE)
		return 1;

	/* make sure our super doesn't straddle pages on disk */
	index = bytenr >> PAGE_SHIFT;
	if ((bytenr + sizeof(**disk_super) - 1) >> PAGE_SHIFT != index)
		return 1;

	/* pull in the page with our super */
	*page = read_cache_page_gfp(bdev->bd_inode->i_mapping,
				   index, GFP_KERNEL);

	if (IS_ERR_OR_NULL(*page))
		return 1;

	p = kmap(*page);

	/* align our pointer to the offset of the super block */
	*disk_super = p + (bytenr & ~PAGE_MASK);

	if (btrfs_super_bytenr(*disk_super) != bytenr ||
	    btrfs_super_magic(*disk_super) != BTRFS_MAGIC) {
		btrfs_release_disk_super(*page);
		return 1;
	}

	if ((*disk_super)->label[0] &&
		(*disk_super)->label[BTRFS_LABEL_SIZE - 1])
		(*disk_super)->label[BTRFS_LABEL_SIZE - 1] = '\0';

	return 0;
}

1113 1114 1115 1116 1117
/*
 * Look for a btrfs signature on a device. This may be called out of the mount path
 * and we are not allowed to call set_blocksize during the scan. The superblock
 * is read via pagecache
 */
1118
int btrfs_scan_one_device(const char *path, fmode_t flags, void *holder,
1119 1120 1121 1122
			  struct btrfs_fs_devices **fs_devices_ret)
{
	struct btrfs_super_block *disk_super;
	struct block_device *bdev;
1123 1124
	struct page *page;
	int ret = -EINVAL;
1125
	u64 devid;
1126
	u64 transid;
J
Josef Bacik 已提交
1127
	u64 total_devices;
1128
	u64 bytenr;
1129

1130 1131 1132 1133 1134 1135 1136
	/*
	 * we would like to check all the supers, but that would make
	 * a btrfs mount succeed after a mkfs from a different FS.
	 * So, we need to add a special mount option to scan for
	 * later supers, using BTRFS_SUPER_MIRROR_MAX instead
	 */
	bytenr = btrfs_sb_offset(0);
1137
	flags |= FMODE_EXCL;
1138
	mutex_lock(&uuid_mutex);
1139 1140 1141 1142

	bdev = blkdev_get_by_path(path, flags, holder);
	if (IS_ERR(bdev)) {
		ret = PTR_ERR(bdev);
1143
		goto error;
1144 1145
	}

1146
	if (btrfs_read_disk_super(bdev, bytenr, &page, &disk_super))
1147 1148
		goto error_bdev_put;

1149
	devid = btrfs_stack_device_id(&disk_super->dev_item);
1150
	transid = btrfs_super_generation(disk_super);
J
Josef Bacik 已提交
1151
	total_devices = btrfs_super_num_devices(disk_super);
1152

1153
	ret = device_list_add(path, disk_super, devid, fs_devices_ret);
1154 1155
	if (ret > 0) {
		if (disk_super->label[0]) {
1156
			pr_info("BTRFS: device label %s ", disk_super->label);
1157
		} else {
1158
			pr_info("BTRFS: device fsid %pU ", disk_super->fsid);
1159 1160
		}

1161
		pr_cont("devid %llu transid %llu %s\n", devid, transid, path);
1162 1163
		ret = 0;
	}
J
Josef Bacik 已提交
1164 1165
	if (!ret && fs_devices_ret)
		(*fs_devices_ret)->total_devices = total_devices;
1166

1167
	btrfs_release_disk_super(page);
1168 1169

error_bdev_put:
1170
	blkdev_put(bdev, flags);
1171
error:
1172
	mutex_unlock(&uuid_mutex);
1173 1174
	return ret;
}
1175

1176 1177 1178 1179 1180
/* helper to account the used device space in the range */
int btrfs_account_dev_extents_size(struct btrfs_device *device, u64 start,
				   u64 end, u64 *length)
{
	struct btrfs_key key;
1181
	struct btrfs_root *root = device->fs_info->dev_root;
1182 1183 1184 1185 1186 1187 1188 1189 1190
	struct btrfs_dev_extent *dev_extent;
	struct btrfs_path *path;
	u64 extent_end;
	int ret;
	int slot;
	struct extent_buffer *l;

	*length = 0;

1191
	if (start >= device->total_bytes || device->is_tgtdev_for_dev_replace)
1192 1193 1194 1195 1196
		return 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
1197
	path->reada = READA_FORWARD;
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231

	key.objectid = device->devid;
	key.offset = start;
	key.type = BTRFS_DEV_EXTENT_KEY;

	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
	if (ret > 0) {
		ret = btrfs_previous_item(root, path, key.objectid, key.type);
		if (ret < 0)
			goto out;
	}

	while (1) {
		l = path->nodes[0];
		slot = path->slots[0];
		if (slot >= btrfs_header_nritems(l)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
				goto out;

			break;
		}
		btrfs_item_key_to_cpu(l, &key, slot);

		if (key.objectid < device->devid)
			goto next;

		if (key.objectid > device->devid)
			break;

1232
		if (key.type != BTRFS_DEV_EXTENT_KEY)
1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
			goto next;

		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
		extent_end = key.offset + btrfs_dev_extent_length(l,
								  dev_extent);
		if (key.offset <= start && extent_end > end) {
			*length = end - start + 1;
			break;
		} else if (key.offset <= start && extent_end > start)
			*length += extent_end - start;
		else if (key.offset > start && extent_end <= end)
			*length += extent_end - key.offset;
		else if (key.offset > start && key.offset <= end) {
			*length += end - key.offset + 1;
			break;
		} else if (key.offset > end)
			break;

next:
		path->slots[0]++;
	}
	ret = 0;
out:
	btrfs_free_path(path);
	return ret;
}

1260
static int contains_pending_extent(struct btrfs_transaction *transaction,
1261 1262 1263
				   struct btrfs_device *device,
				   u64 *start, u64 len)
{
1264
	struct btrfs_fs_info *fs_info = device->fs_info;
1265
	struct extent_map *em;
1266
	struct list_head *search_list = &fs_info->pinned_chunks;
1267
	int ret = 0;
1268
	u64 physical_start = *start;
1269

1270 1271
	if (transaction)
		search_list = &transaction->pending_chunks;
1272 1273
again:
	list_for_each_entry(em, search_list, list) {
1274 1275 1276
		struct map_lookup *map;
		int i;

1277
		map = em->map_lookup;
1278
		for (i = 0; i < map->num_stripes; i++) {
1279 1280
			u64 end;

1281 1282
			if (map->stripes[i].dev != device)
				continue;
1283
			if (map->stripes[i].physical >= physical_start + len ||
1284
			    map->stripes[i].physical + em->orig_block_len <=
1285
			    physical_start)
1286
				continue;
1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
			/*
			 * Make sure that while processing the pinned list we do
			 * not override our *start with a lower value, because
			 * we can have pinned chunks that fall within this
			 * device hole and that have lower physical addresses
			 * than the pending chunks we processed before. If we
			 * do not take this special care we can end up getting
			 * 2 pending chunks that start at the same physical
			 * device offsets because the end offset of a pinned
			 * chunk can be equal to the start offset of some
			 * pending chunk.
			 */
			end = map->stripes[i].physical + em->orig_block_len;
			if (end > *start) {
				*start = end;
				ret = 1;
			}
1304 1305
		}
	}
1306 1307
	if (search_list != &fs_info->pinned_chunks) {
		search_list = &fs_info->pinned_chunks;
1308 1309
		goto again;
	}
1310 1311 1312 1313 1314

	return ret;
}


1315
/*
1316 1317 1318 1319 1320 1321 1322
 * find_free_dev_extent_start - find free space in the specified device
 * @device:	  the device which we search the free space in
 * @num_bytes:	  the size of the free space that we need
 * @search_start: the position from which to begin the search
 * @start:	  store the start of the free space.
 * @len:	  the size of the free space. that we find, or the size
 *		  of the max free space if we don't find suitable free space
1323
 *
1324 1325 1326
 * this uses a pretty simple search, the expectation is that it is
 * called very infrequently and that a given device has a small number
 * of extents
1327 1328 1329 1330 1331 1332 1333 1334
 *
 * @start is used to store the start of the free space if we find. But if we
 * don't find suitable free space, it will be used to store the start position
 * of the max free space.
 *
 * @len is used to store the size of the free space that we find.
 * But if we don't find suitable free space, it is used to store the size of
 * the max free space.
1335
 */
1336 1337 1338
int find_free_dev_extent_start(struct btrfs_transaction *transaction,
			       struct btrfs_device *device, u64 num_bytes,
			       u64 search_start, u64 *start, u64 *len)
1339
{
1340 1341
	struct btrfs_fs_info *fs_info = device->fs_info;
	struct btrfs_root *root = fs_info->dev_root;
1342
	struct btrfs_key key;
1343
	struct btrfs_dev_extent *dev_extent;
Y
Yan Zheng 已提交
1344
	struct btrfs_path *path;
1345 1346 1347 1348
	u64 hole_size;
	u64 max_hole_start;
	u64 max_hole_size;
	u64 extent_end;
1349 1350
	u64 search_end = device->total_bytes;
	int ret;
1351
	int slot;
1352
	struct extent_buffer *l;
1353 1354 1355 1356 1357 1358 1359
	u64 min_search_start;

	/*
	 * We don't want to overwrite the superblock on the drive nor any area
	 * used by the boot loader (grub for example), so we make sure to start
	 * at an offset of at least 1MB.
	 */
1360
	min_search_start = max(fs_info->alloc_start, 1024ull * 1024);
1361
	search_start = max(search_start, min_search_start);
1362

1363 1364 1365
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
1366

1367 1368 1369
	max_hole_start = search_start;
	max_hole_size = 0;

1370
again:
1371
	if (search_start >= search_end || device->is_tgtdev_for_dev_replace) {
1372
		ret = -ENOSPC;
1373
		goto out;
1374 1375
	}

1376
	path->reada = READA_FORWARD;
1377 1378
	path->search_commit_root = 1;
	path->skip_locking = 1;
1379

1380 1381 1382
	key.objectid = device->devid;
	key.offset = search_start;
	key.type = BTRFS_DEV_EXTENT_KEY;
1383

1384
	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1385
	if (ret < 0)
1386
		goto out;
1387 1388 1389
	if (ret > 0) {
		ret = btrfs_previous_item(root, path, key.objectid, key.type);
		if (ret < 0)
1390
			goto out;
1391
	}
1392

1393 1394 1395 1396 1397 1398 1399 1400
	while (1) {
		l = path->nodes[0];
		slot = path->slots[0];
		if (slot >= btrfs_header_nritems(l)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
1401 1402 1403
				goto out;

			break;
1404 1405 1406 1407 1408 1409 1410
		}
		btrfs_item_key_to_cpu(l, &key, slot);

		if (key.objectid < device->devid)
			goto next;

		if (key.objectid > device->devid)
1411
			break;
1412

1413
		if (key.type != BTRFS_DEV_EXTENT_KEY)
1414
			goto next;
1415

1416 1417
		if (key.offset > search_start) {
			hole_size = key.offset - search_start;
1418

1419 1420 1421 1422
			/*
			 * Have to check before we set max_hole_start, otherwise
			 * we could end up sending back this offset anyway.
			 */
1423
			if (contains_pending_extent(transaction, device,
1424
						    &search_start,
1425 1426 1427 1428 1429 1430 1431 1432
						    hole_size)) {
				if (key.offset >= search_start) {
					hole_size = key.offset - search_start;
				} else {
					WARN_ON_ONCE(1);
					hole_size = 0;
				}
			}
1433

1434 1435 1436 1437
			if (hole_size > max_hole_size) {
				max_hole_start = search_start;
				max_hole_size = hole_size;
			}
1438

1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450
			/*
			 * If this free space is greater than which we need,
			 * it must be the max free space that we have found
			 * until now, so max_hole_start must point to the start
			 * of this free space and the length of this free space
			 * is stored in max_hole_size. Thus, we return
			 * max_hole_start and max_hole_size and go back to the
			 * caller.
			 */
			if (hole_size >= num_bytes) {
				ret = 0;
				goto out;
1451 1452 1453 1454
			}
		}

		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1455 1456 1457 1458
		extent_end = key.offset + btrfs_dev_extent_length(l,
								  dev_extent);
		if (extent_end > search_start)
			search_start = extent_end;
1459 1460 1461 1462 1463
next:
		path->slots[0]++;
		cond_resched();
	}

1464 1465 1466 1467 1468
	/*
	 * At this point, search_start should be the end of
	 * allocated dev extents, and when shrinking the device,
	 * search_end may be smaller than search_start.
	 */
1469
	if (search_end > search_start) {
1470 1471
		hole_size = search_end - search_start;

1472
		if (contains_pending_extent(transaction, device, &search_start,
1473 1474 1475 1476
					    hole_size)) {
			btrfs_release_path(path);
			goto again;
		}
1477

1478 1479 1480 1481
		if (hole_size > max_hole_size) {
			max_hole_start = search_start;
			max_hole_size = hole_size;
		}
1482 1483
	}

1484
	/* See above. */
1485
	if (max_hole_size < num_bytes)
1486 1487 1488 1489 1490
		ret = -ENOSPC;
	else
		ret = 0;

out:
Y
Yan Zheng 已提交
1491
	btrfs_free_path(path);
1492
	*start = max_hole_start;
1493
	if (len)
1494
		*len = max_hole_size;
1495 1496 1497
	return ret;
}

1498 1499 1500 1501 1502 1503
int find_free_dev_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_device *device, u64 num_bytes,
			 u64 *start, u64 *len)
{
	/* FIXME use last free of some kind */
	return find_free_dev_extent_start(trans->transaction, device,
1504
					  num_bytes, 0, start, len);
1505 1506
}

1507
static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1508
			  struct btrfs_device *device,
M
Miao Xie 已提交
1509
			  u64 start, u64 *dev_extent_len)
1510
{
1511 1512
	struct btrfs_fs_info *fs_info = device->fs_info;
	struct btrfs_root *root = fs_info->dev_root;
1513 1514 1515
	int ret;
	struct btrfs_path *path;
	struct btrfs_key key;
1516 1517 1518
	struct btrfs_key found_key;
	struct extent_buffer *leaf = NULL;
	struct btrfs_dev_extent *extent = NULL;
1519 1520 1521 1522 1523 1524 1525 1526

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = device->devid;
	key.offset = start;
	key.type = BTRFS_DEV_EXTENT_KEY;
M
Miao Xie 已提交
1527
again:
1528
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1529 1530 1531
	if (ret > 0) {
		ret = btrfs_previous_item(root, path, key.objectid,
					  BTRFS_DEV_EXTENT_KEY);
1532 1533
		if (ret)
			goto out;
1534 1535 1536 1537 1538 1539
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
		extent = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_dev_extent);
		BUG_ON(found_key.offset > start || found_key.offset +
		       btrfs_dev_extent_length(leaf, extent) < start);
M
Miao Xie 已提交
1540 1541 1542
		key = found_key;
		btrfs_release_path(path);
		goto again;
1543 1544 1545 1546
	} else if (ret == 0) {
		leaf = path->nodes[0];
		extent = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_dev_extent);
1547
	} else {
1548
		btrfs_handle_fs_error(fs_info, ret, "Slot search failed");
1549
		goto out;
1550
	}
1551

M
Miao Xie 已提交
1552 1553
	*dev_extent_len = btrfs_dev_extent_length(leaf, extent);

1554
	ret = btrfs_del_item(trans, root, path);
1555
	if (ret) {
1556 1557
		btrfs_handle_fs_error(fs_info, ret,
				      "Failed to remove dev extent item");
Z
Zhao Lei 已提交
1558
	} else {
1559
		set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags);
1560
	}
1561
out:
1562 1563 1564 1565
	btrfs_free_path(path);
	return ret;
}

1566 1567 1568 1569
static int btrfs_alloc_dev_extent(struct btrfs_trans_handle *trans,
				  struct btrfs_device *device,
				  u64 chunk_tree, u64 chunk_objectid,
				  u64 chunk_offset, u64 start, u64 num_bytes)
1570 1571 1572
{
	int ret;
	struct btrfs_path *path;
1573 1574
	struct btrfs_fs_info *fs_info = device->fs_info;
	struct btrfs_root *root = fs_info->dev_root;
1575 1576 1577 1578
	struct btrfs_dev_extent *extent;
	struct extent_buffer *leaf;
	struct btrfs_key key;

1579
	WARN_ON(!device->in_fs_metadata);
1580
	WARN_ON(device->is_tgtdev_for_dev_replace);
1581 1582 1583 1584 1585
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = device->devid;
Y
Yan Zheng 已提交
1586
	key.offset = start;
1587 1588 1589
	key.type = BTRFS_DEV_EXTENT_KEY;
	ret = btrfs_insert_empty_item(trans, root, path, &key,
				      sizeof(*extent));
1590 1591
	if (ret)
		goto out;
1592 1593 1594 1595

	leaf = path->nodes[0];
	extent = btrfs_item_ptr(leaf, path->slots[0],
				struct btrfs_dev_extent);
1596 1597 1598 1599
	btrfs_set_dev_extent_chunk_tree(leaf, extent, chunk_tree);
	btrfs_set_dev_extent_chunk_objectid(leaf, extent, chunk_objectid);
	btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset);

1600
	write_extent_buffer_chunk_tree_uuid(leaf, fs_info->chunk_tree_uuid);
1601

1602 1603
	btrfs_set_dev_extent_length(leaf, extent, num_bytes);
	btrfs_mark_buffer_dirty(leaf);
1604
out:
1605 1606 1607 1608
	btrfs_free_path(path);
	return ret;
}

1609
static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1610
{
1611 1612 1613 1614
	struct extent_map_tree *em_tree;
	struct extent_map *em;
	struct rb_node *n;
	u64 ret = 0;
1615

1616 1617 1618 1619 1620 1621
	em_tree = &fs_info->mapping_tree.map_tree;
	read_lock(&em_tree->lock);
	n = rb_last(&em_tree->map);
	if (n) {
		em = rb_entry(n, struct extent_map, rb_node);
		ret = em->start + em->len;
1622
	}
1623 1624
	read_unlock(&em_tree->lock);

1625 1626 1627
	return ret;
}

1628 1629
static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
				    u64 *devid_ret)
1630 1631 1632 1633
{
	int ret;
	struct btrfs_key key;
	struct btrfs_key found_key;
Y
Yan Zheng 已提交
1634 1635 1636 1637 1638
	struct btrfs_path *path;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
1639 1640 1641 1642 1643

	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
	key.type = BTRFS_DEV_ITEM_KEY;
	key.offset = (u64)-1;

1644
	ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1645 1646 1647
	if (ret < 0)
		goto error;

1648
	BUG_ON(ret == 0); /* Corruption */
1649

1650 1651
	ret = btrfs_previous_item(fs_info->chunk_root, path,
				  BTRFS_DEV_ITEMS_OBJECTID,
1652 1653
				  BTRFS_DEV_ITEM_KEY);
	if (ret) {
1654
		*devid_ret = 1;
1655 1656 1657
	} else {
		btrfs_item_key_to_cpu(path->nodes[0], &found_key,
				      path->slots[0]);
1658
		*devid_ret = found_key.offset + 1;
1659 1660 1661
	}
	ret = 0;
error:
Y
Yan Zheng 已提交
1662
	btrfs_free_path(path);
1663 1664 1665 1666 1667 1668 1669
	return ret;
}

/*
 * the device information is stored in the chunk root
 * the btrfs_device struct should be fully filled in
 */
1670
static int btrfs_add_device(struct btrfs_trans_handle *trans,
1671
			    struct btrfs_fs_info *fs_info,
1672
			    struct btrfs_device *device)
1673
{
1674
	struct btrfs_root *root = fs_info->chunk_root;
1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687
	int ret;
	struct btrfs_path *path;
	struct btrfs_dev_item *dev_item;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	unsigned long ptr;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
	key.type = BTRFS_DEV_ITEM_KEY;
Y
Yan Zheng 已提交
1688
	key.offset = device->devid;
1689 1690

	ret = btrfs_insert_empty_item(trans, root, path, &key,
1691
				      sizeof(*dev_item));
1692 1693 1694 1695 1696 1697 1698
	if (ret)
		goto out;

	leaf = path->nodes[0];
	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);

	btrfs_set_device_id(leaf, dev_item, device->devid);
Y
Yan Zheng 已提交
1699
	btrfs_set_device_generation(leaf, dev_item, 0);
1700 1701 1702 1703
	btrfs_set_device_type(leaf, dev_item, device->type);
	btrfs_set_device_io_align(leaf, dev_item, device->io_align);
	btrfs_set_device_io_width(leaf, dev_item, device->io_width);
	btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1704 1705 1706 1707
	btrfs_set_device_total_bytes(leaf, dev_item,
				     btrfs_device_get_disk_total_bytes(device));
	btrfs_set_device_bytes_used(leaf, dev_item,
				    btrfs_device_get_bytes_used(device));
1708 1709 1710
	btrfs_set_device_group(leaf, dev_item, 0);
	btrfs_set_device_seek_speed(leaf, dev_item, 0);
	btrfs_set_device_bandwidth(leaf, dev_item, 0);
1711
	btrfs_set_device_start_offset(leaf, dev_item, 0);
1712

1713
	ptr = btrfs_device_uuid(dev_item);
1714
	write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1715
	ptr = btrfs_device_fsid(dev_item);
1716
	write_extent_buffer(leaf, fs_info->fsid, ptr, BTRFS_UUID_SIZE);
1717 1718
	btrfs_mark_buffer_dirty(leaf);

Y
Yan Zheng 已提交
1719
	ret = 0;
1720 1721 1722 1723
out:
	btrfs_free_path(path);
	return ret;
}
1724

1725 1726 1727 1728 1729 1730 1731 1732 1733
/*
 * Function to update ctime/mtime for a given device path.
 * Mainly used for ctime/mtime based probe like libblkid.
 */
static void update_dev_time(char *path_name)
{
	struct file *filp;

	filp = filp_open(path_name, O_RDWR, 0);
1734
	if (IS_ERR(filp))
1735 1736 1737 1738 1739
		return;
	file_update_time(filp);
	filp_close(filp, NULL);
}

1740
static int btrfs_rm_dev_item(struct btrfs_fs_info *fs_info,
1741 1742
			     struct btrfs_device *device)
{
1743
	struct btrfs_root *root = fs_info->chunk_root;
1744 1745 1746 1747 1748 1749 1750 1751 1752
	int ret;
	struct btrfs_path *path;
	struct btrfs_key key;
	struct btrfs_trans_handle *trans;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

1753
	trans = btrfs_start_transaction(root, 0);
1754 1755 1756 1757
	if (IS_ERR(trans)) {
		btrfs_free_path(path);
		return PTR_ERR(trans);
	}
1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779
	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
	key.type = BTRFS_DEV_ITEM_KEY;
	key.offset = device->devid;

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0)
		goto out;

	if (ret > 0) {
		ret = -ENOENT;
		goto out;
	}

	ret = btrfs_del_item(trans, root, path);
	if (ret)
		goto out;
out:
	btrfs_free_path(path);
	btrfs_commit_transaction(trans, root);
	return ret;
}

1780 1781 1782 1783 1784 1785 1786
/*
 * Verify that @num_devices satisfies the RAID profile constraints in the whole
 * filesystem. It's up to the caller to adjust that number regarding eg. device
 * replace.
 */
static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
		u64 num_devices)
1787 1788
{
	u64 all_avail;
1789
	unsigned seq;
1790
	int i;
1791

1792
	do {
1793
		seq = read_seqbegin(&fs_info->profiles_lock);
1794

1795 1796 1797 1798
		all_avail = fs_info->avail_data_alloc_bits |
			    fs_info->avail_system_alloc_bits |
			    fs_info->avail_metadata_alloc_bits;
	} while (read_seqretry(&fs_info->profiles_lock, seq));
1799

1800 1801 1802
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
		if (!(all_avail & btrfs_raid_group[i]))
			continue;
1803

1804 1805
		if (num_devices < btrfs_raid_array[i].devs_min) {
			int ret = btrfs_raid_mindev_error[i];
1806

1807 1808 1809
			if (ret)
				return ret;
		}
D
David Woodhouse 已提交
1810 1811
	}

1812
	return 0;
1813 1814
}

1815 1816
struct btrfs_device *btrfs_find_next_active_device(struct btrfs_fs_devices *fs_devs,
					struct btrfs_device *device)
1817
{
Y
Yan Zheng 已提交
1818
	struct btrfs_device *next_device;
1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854

	list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
		if (next_device != device &&
			!next_device->missing && next_device->bdev)
			return next_device;
	}

	return NULL;
}

/*
 * Helper function to check if the given device is part of s_bdev / latest_bdev
 * and replace it with the provided or the next active device, in the context
 * where this function called, there should be always be another device (or
 * this_dev) which is active.
 */
void btrfs_assign_next_active_device(struct btrfs_fs_info *fs_info,
		struct btrfs_device *device, struct btrfs_device *this_dev)
{
	struct btrfs_device *next_device;

	if (this_dev)
		next_device = this_dev;
	else
		next_device = btrfs_find_next_active_device(fs_info->fs_devices,
								device);
	ASSERT(next_device);

	if (fs_info->sb->s_bdev &&
			(fs_info->sb->s_bdev == device->bdev))
		fs_info->sb->s_bdev = next_device->bdev;

	if (fs_info->fs_devices->latest_bdev == device->bdev)
		fs_info->fs_devices->latest_bdev = next_device->bdev;
}

1855
int btrfs_rm_device(struct btrfs_fs_info *fs_info, char *device_path, u64 devid)
1856 1857
{
	struct btrfs_device *device;
1858
	struct btrfs_fs_devices *cur_devices;
Y
Yan Zheng 已提交
1859
	u64 num_devices;
1860
	int ret = 0;
1861
	bool clear_super = false;
1862 1863 1864

	mutex_lock(&uuid_mutex);

1865 1866 1867
	num_devices = fs_info->fs_devices->num_devices;
	btrfs_dev_replace_lock(&fs_info->dev_replace, 0);
	if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
1868 1869 1870
		WARN_ON(num_devices < 1);
		num_devices--;
	}
1871
	btrfs_dev_replace_unlock(&fs_info->dev_replace, 0);
1872

1873
	ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
1874
	if (ret)
1875 1876
		goto out;

1877 1878
	ret = btrfs_find_device_by_devspec(fs_info, devid, device_path,
					   &device);
1879
	if (ret)
D
David Woodhouse 已提交
1880
		goto out;
1881

1882
	if (device->is_tgtdev_for_dev_replace) {
1883
		ret = BTRFS_ERROR_DEV_TGT_REPLACE;
1884
		goto out;
1885 1886
	}

1887
	if (device->writeable && fs_info->fs_devices->rw_devices == 1) {
1888
		ret = BTRFS_ERROR_DEV_ONLY_WRITABLE;
1889
		goto out;
Y
Yan Zheng 已提交
1890 1891 1892
	}

	if (device->writeable) {
1893
		lock_chunks(fs_info);
Y
Yan Zheng 已提交
1894
		list_del_init(&device->dev_alloc_list);
1895
		device->fs_devices->rw_devices--;
1896
		unlock_chunks(fs_info);
1897
		clear_super = true;
1898
	}
1899

1900
	mutex_unlock(&uuid_mutex);
1901
	ret = btrfs_shrink_device(device, 0);
1902
	mutex_lock(&uuid_mutex);
1903
	if (ret)
1904
		goto error_undo;
1905

1906 1907 1908 1909 1910
	/*
	 * TODO: the superblock still includes this device in its num_devices
	 * counter although write_all_supers() is not locked out. This
	 * could give a filesystem state which requires a degraded mount.
	 */
1911
	ret = btrfs_rm_dev_item(fs_info, device);
1912
	if (ret)
1913
		goto error_undo;
1914

Y
Yan Zheng 已提交
1915
	device->in_fs_metadata = 0;
1916
	btrfs_scrub_cancel_dev(fs_info, device);
1917 1918 1919 1920

	/*
	 * the device list mutex makes sure that we don't change
	 * the device list while someone else is writing out all
1921 1922 1923 1924 1925
	 * the device supers. Whoever is writing all supers, should
	 * lock the device list mutex before getting the number of
	 * devices in the super block (super_copy). Conversely,
	 * whoever updates the number of devices in the super block
	 * (super_copy) should hold the device list mutex.
1926
	 */
1927 1928

	cur_devices = device->fs_devices;
1929
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
1930
	list_del_rcu(&device->dev_list);
1931

Y
Yan Zheng 已提交
1932
	device->fs_devices->num_devices--;
J
Josef Bacik 已提交
1933
	device->fs_devices->total_devices--;
Y
Yan Zheng 已提交
1934

1935
	if (device->missing)
1936
		device->fs_devices->missing_devices--;
1937

1938
	btrfs_assign_next_active_device(fs_info, device, NULL);
Y
Yan Zheng 已提交
1939

1940
	if (device->bdev) {
Y
Yan Zheng 已提交
1941
		device->fs_devices->open_devices--;
1942
		/* remove sysfs entry */
1943
		btrfs_sysfs_rm_device_link(fs_info->fs_devices, device);
1944
	}
1945

1946 1947 1948
	num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
	btrfs_set_super_num_devices(fs_info->super_copy, num_devices);
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
Y
Yan Zheng 已提交
1949

1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960
	/*
	 * at this point, the device is zero sized and detached from
	 * the devices list.  All that's left is to zero out the old
	 * supers and free the device.
	 */
	if (device->writeable)
		btrfs_scratch_superblocks(device->bdev, device->name->str);

	btrfs_close_bdev(device);
	call_rcu(&device->rcu, free_device);

1961
	if (cur_devices->open_devices == 0) {
Y
Yan Zheng 已提交
1962
		struct btrfs_fs_devices *fs_devices;
1963
		fs_devices = fs_info->fs_devices;
Y
Yan Zheng 已提交
1964
		while (fs_devices) {
1965 1966
			if (fs_devices->seed == cur_devices) {
				fs_devices->seed = cur_devices->seed;
Y
Yan Zheng 已提交
1967
				break;
1968
			}
Y
Yan Zheng 已提交
1969
			fs_devices = fs_devices->seed;
Y
Yan Zheng 已提交
1970
		}
1971 1972 1973
		cur_devices->seed = NULL;
		__btrfs_close_devices(cur_devices);
		free_fs_devices(cur_devices);
Y
Yan Zheng 已提交
1974 1975
	}

1976 1977
	fs_info->num_tolerated_disk_barrier_failures =
		btrfs_calc_num_tolerated_disk_barrier_failures(fs_info);
1978

1979 1980 1981
out:
	mutex_unlock(&uuid_mutex);
	return ret;
1982

1983 1984
error_undo:
	if (device->writeable) {
1985
		lock_chunks(fs_info);
1986
		list_add(&device->dev_alloc_list,
1987
			 &fs_info->fs_devices->alloc_list);
1988
		device->fs_devices->rw_devices++;
1989
		unlock_chunks(fs_info);
1990
	}
1991
	goto out;
1992 1993
}

1994 1995
void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_fs_info *fs_info,
					struct btrfs_device *srcdev)
1996
{
1997 1998
	struct btrfs_fs_devices *fs_devices;

1999
	WARN_ON(!mutex_is_locked(&fs_info->fs_devices->device_list_mutex));
2000

2001 2002 2003 2004 2005 2006 2007
	/*
	 * in case of fs with no seed, srcdev->fs_devices will point
	 * to fs_devices of fs_info. However when the dev being replaced is
	 * a seed dev it will point to the seed's local fs_devices. In short
	 * srcdev will have its correct fs_devices in both the cases.
	 */
	fs_devices = srcdev->fs_devices;
2008

2009 2010
	list_del_rcu(&srcdev->dev_list);
	list_del_rcu(&srcdev->dev_alloc_list);
2011
	fs_devices->num_devices--;
2012
	if (srcdev->missing)
2013
		fs_devices->missing_devices--;
2014

2015
	if (srcdev->writeable)
2016
		fs_devices->rw_devices--;
2017

2018
	if (srcdev->bdev)
2019
		fs_devices->open_devices--;
2020 2021 2022 2023 2024 2025
}

void btrfs_rm_dev_replace_free_srcdev(struct btrfs_fs_info *fs_info,
				      struct btrfs_device *srcdev)
{
	struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2026

2027 2028 2029 2030
	if (srcdev->writeable) {
		/* zero out the old super if it is writable */
		btrfs_scratch_superblocks(srcdev->bdev, srcdev->name->str);
	}
2031 2032 2033

	btrfs_close_bdev(srcdev);

2034
	call_rcu(&srcdev->rcu, free_device);
2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054

	/*
	 * unless fs_devices is seed fs, num_devices shouldn't go
	 * zero
	 */
	BUG_ON(!fs_devices->num_devices && !fs_devices->seeding);

	/* if this is no devs we rather delete the fs_devices */
	if (!fs_devices->num_devices) {
		struct btrfs_fs_devices *tmp_fs_devices;

		tmp_fs_devices = fs_info->fs_devices;
		while (tmp_fs_devices) {
			if (tmp_fs_devices->seed == fs_devices) {
				tmp_fs_devices->seed = fs_devices->seed;
				break;
			}
			tmp_fs_devices = tmp_fs_devices->seed;
		}
		fs_devices->seed = NULL;
2055 2056
		__btrfs_close_devices(fs_devices);
		free_fs_devices(fs_devices);
2057
	}
2058 2059 2060 2061 2062
}

void btrfs_destroy_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
				      struct btrfs_device *tgtdev)
{
2063
	mutex_lock(&uuid_mutex);
2064 2065
	WARN_ON(!tgtdev);
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
2066

2067
	btrfs_sysfs_rm_device_link(fs_info->fs_devices, tgtdev);
2068

2069
	if (tgtdev->bdev)
2070
		fs_info->fs_devices->open_devices--;
2071

2072 2073
	fs_info->fs_devices->num_devices--;

2074
	btrfs_assign_next_active_device(fs_info, tgtdev, NULL);
2075 2076 2077 2078

	list_del_rcu(&tgtdev->dev_list);

	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2079
	mutex_unlock(&uuid_mutex);
2080 2081 2082 2083 2084 2085 2086 2087 2088

	/*
	 * The update_dev_time() with in btrfs_scratch_superblocks()
	 * may lead to a call to btrfs_show_devname() which will try
	 * to hold device_list_mutex. And here this device
	 * is already out of device list, so we don't have to hold
	 * the device_list_mutex lock.
	 */
	btrfs_scratch_superblocks(tgtdev->bdev, tgtdev->name->str);
2089 2090

	btrfs_close_bdev(tgtdev);
2091
	call_rcu(&tgtdev->rcu, free_device);
2092 2093
}

2094 2095
static int btrfs_find_device_by_path(struct btrfs_fs_info *fs_info,
				     char *device_path,
2096
				     struct btrfs_device **device)
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
{
	int ret = 0;
	struct btrfs_super_block *disk_super;
	u64 devid;
	u8 *dev_uuid;
	struct block_device *bdev;
	struct buffer_head *bh;

	*device = NULL;
	ret = btrfs_get_bdev_and_sb(device_path, FMODE_READ,
2107
				    fs_info->bdev_holder, 0, &bdev, &bh);
2108 2109 2110 2111 2112
	if (ret)
		return ret;
	disk_super = (struct btrfs_super_block *)bh->b_data;
	devid = btrfs_stack_device_id(&disk_super->dev_item);
	dev_uuid = disk_super->dev_item.uuid;
2113
	*device = btrfs_find_device(fs_info, devid, dev_uuid, disk_super->fsid);
2114 2115 2116 2117 2118 2119 2120
	brelse(bh);
	if (!*device)
		ret = -ENOENT;
	blkdev_put(bdev, FMODE_READ);
	return ret;
}

2121
int btrfs_find_device_missing_or_by_path(struct btrfs_fs_info *fs_info,
2122 2123 2124 2125 2126 2127 2128 2129
					 char *device_path,
					 struct btrfs_device **device)
{
	*device = NULL;
	if (strcmp(device_path, "missing") == 0) {
		struct list_head *devices;
		struct btrfs_device *tmp;

2130
		devices = &fs_info->fs_devices->devices;
2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
		/*
		 * It is safe to read the devices since the volume_mutex
		 * is held by the caller.
		 */
		list_for_each_entry(tmp, devices, dev_list) {
			if (tmp->in_fs_metadata && !tmp->bdev) {
				*device = tmp;
				break;
			}
		}

2142 2143
		if (!*device)
			return BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2144 2145 2146

		return 0;
	} else {
2147
		return btrfs_find_device_by_path(fs_info, device_path, device);
2148 2149 2150
	}
}

2151 2152 2153
/*
 * Lookup a device given by device id, or the path if the id is 0.
 */
2154 2155
int btrfs_find_device_by_devspec(struct btrfs_fs_info *fs_info, u64 devid,
				 char *devpath, struct btrfs_device **device)
2156 2157 2158
{
	int ret;

2159
	if (devid) {
2160
		ret = 0;
2161
		*device = btrfs_find_device(fs_info, devid, NULL, NULL);
2162 2163 2164
		if (!*device)
			ret = -ENOENT;
	} else {
2165
		if (!devpath || !devpath[0])
2166 2167
			return -EINVAL;

2168
		ret = btrfs_find_device_missing_or_by_path(fs_info, devpath,
2169 2170 2171 2172 2173
							   device);
	}
	return ret;
}

Y
Yan Zheng 已提交
2174 2175 2176
/*
 * does all the dirty work required for changing file system's UUID.
 */
2177
static int btrfs_prepare_sprout(struct btrfs_fs_info *fs_info)
Y
Yan Zheng 已提交
2178
{
2179
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
Y
Yan Zheng 已提交
2180
	struct btrfs_fs_devices *old_devices;
Y
Yan Zheng 已提交
2181
	struct btrfs_fs_devices *seed_devices;
2182
	struct btrfs_super_block *disk_super = fs_info->super_copy;
Y
Yan Zheng 已提交
2183 2184 2185 2186
	struct btrfs_device *device;
	u64 super_flags;

	BUG_ON(!mutex_is_locked(&uuid_mutex));
Y
Yan Zheng 已提交
2187
	if (!fs_devices->seeding)
Y
Yan Zheng 已提交
2188 2189
		return -EINVAL;

2190 2191 2192
	seed_devices = __alloc_fs_devices();
	if (IS_ERR(seed_devices))
		return PTR_ERR(seed_devices);
Y
Yan Zheng 已提交
2193

Y
Yan Zheng 已提交
2194 2195 2196 2197
	old_devices = clone_fs_devices(fs_devices);
	if (IS_ERR(old_devices)) {
		kfree(seed_devices);
		return PTR_ERR(old_devices);
Y
Yan Zheng 已提交
2198
	}
Y
Yan Zheng 已提交
2199

Y
Yan Zheng 已提交
2200 2201
	list_add(&old_devices->list, &fs_uuids);

Y
Yan Zheng 已提交
2202 2203 2204 2205
	memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
	seed_devices->opened = 1;
	INIT_LIST_HEAD(&seed_devices->devices);
	INIT_LIST_HEAD(&seed_devices->alloc_list);
2206
	mutex_init(&seed_devices->device_list_mutex);
2207

2208
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
2209 2210
	list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
			      synchronize_rcu);
M
Miao Xie 已提交
2211 2212
	list_for_each_entry(device, &seed_devices->devices, dev_list)
		device->fs_devices = seed_devices;
2213

2214
	lock_chunks(fs_info);
Y
Yan Zheng 已提交
2215
	list_splice_init(&fs_devices->alloc_list, &seed_devices->alloc_list);
2216
	unlock_chunks(fs_info);
Y
Yan Zheng 已提交
2217

Y
Yan Zheng 已提交
2218 2219 2220
	fs_devices->seeding = 0;
	fs_devices->num_devices = 0;
	fs_devices->open_devices = 0;
2221 2222
	fs_devices->missing_devices = 0;
	fs_devices->rotating = 0;
Y
Yan Zheng 已提交
2223
	fs_devices->seed = seed_devices;
Y
Yan Zheng 已提交
2224 2225

	generate_random_uuid(fs_devices->fsid);
2226
	memcpy(fs_info->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
Y
Yan Zheng 已提交
2227
	memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2228
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2229

Y
Yan Zheng 已提交
2230 2231 2232 2233 2234 2235 2236 2237
	super_flags = btrfs_super_flags(disk_super) &
		      ~BTRFS_SUPER_FLAG_SEEDING;
	btrfs_set_super_flags(disk_super, super_flags);

	return 0;
}

/*
2238
 * Store the expected generation for seed devices in device items.
Y
Yan Zheng 已提交
2239 2240
 */
static int btrfs_finish_sprout(struct btrfs_trans_handle *trans,
2241
			       struct btrfs_fs_info *fs_info)
Y
Yan Zheng 已提交
2242
{
2243
	struct btrfs_root *root = fs_info->chunk_root;
Y
Yan Zheng 已提交
2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_dev_item *dev_item;
	struct btrfs_device *device;
	struct btrfs_key key;
	u8 fs_uuid[BTRFS_UUID_SIZE];
	u8 dev_uuid[BTRFS_UUID_SIZE];
	u64 devid;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
	key.offset = 0;
	key.type = BTRFS_DEV_ITEM_KEY;

	while (1) {
		ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
		if (ret < 0)
			goto error;

		leaf = path->nodes[0];
next_slot:
		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret > 0)
				break;
			if (ret < 0)
				goto error;
			leaf = path->nodes[0];
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2277
			btrfs_release_path(path);
Y
Yan Zheng 已提交
2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288
			continue;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
		    key.type != BTRFS_DEV_ITEM_KEY)
			break;

		dev_item = btrfs_item_ptr(leaf, path->slots[0],
					  struct btrfs_dev_item);
		devid = btrfs_device_id(leaf, dev_item);
2289
		read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
Y
Yan Zheng 已提交
2290
				   BTRFS_UUID_SIZE);
2291
		read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
Y
Yan Zheng 已提交
2292
				   BTRFS_UUID_SIZE);
2293
		device = btrfs_find_device(fs_info, devid, dev_uuid, fs_uuid);
2294
		BUG_ON(!device); /* Logic error */
Y
Yan Zheng 已提交
2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310

		if (device->fs_devices->seeding) {
			btrfs_set_device_generation(leaf, dev_item,
						    device->generation);
			btrfs_mark_buffer_dirty(leaf);
		}

		path->slots[0]++;
		goto next_slot;
	}
	ret = 0;
error:
	btrfs_free_path(path);
	return ret;
}

2311
int btrfs_init_new_device(struct btrfs_fs_info *fs_info, char *device_path)
2312
{
2313
	struct btrfs_root *root = fs_info->dev_root;
2314
	struct request_queue *q;
2315 2316 2317 2318
	struct btrfs_trans_handle *trans;
	struct btrfs_device *device;
	struct block_device *bdev;
	struct list_head *devices;
2319
	struct super_block *sb = fs_info->sb;
2320
	struct rcu_string *name;
2321
	u64 tmp;
Y
Yan Zheng 已提交
2322
	int seeding_dev = 0;
2323 2324
	int ret = 0;

2325
	if ((sb->s_flags & MS_RDONLY) && !fs_info->fs_devices->seeding)
2326
		return -EROFS;
2327

2328
	bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2329
				  fs_info->bdev_holder);
2330 2331
	if (IS_ERR(bdev))
		return PTR_ERR(bdev);
2332

2333
	if (fs_info->fs_devices->seeding) {
Y
Yan Zheng 已提交
2334 2335 2336 2337 2338
		seeding_dev = 1;
		down_write(&sb->s_umount);
		mutex_lock(&uuid_mutex);
	}

2339
	filemap_write_and_wait(bdev->bd_inode->i_mapping);
2340

2341
	devices = &fs_info->fs_devices->devices;
2342

2343
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
Q
Qinghuang Feng 已提交
2344
	list_for_each_entry(device, devices, dev_list) {
2345 2346
		if (device->bdev == bdev) {
			ret = -EEXIST;
2347
			mutex_unlock(
2348
				&fs_info->fs_devices->device_list_mutex);
Y
Yan Zheng 已提交
2349
			goto error;
2350 2351
		}
	}
2352
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2353

2354
	device = btrfs_alloc_device(fs_info, NULL, NULL);
2355
	if (IS_ERR(device)) {
2356
		/* we can safely leave the fs_devices entry around */
2357
		ret = PTR_ERR(device);
Y
Yan Zheng 已提交
2358
		goto error;
2359 2360
	}

2361
	name = rcu_string_strdup(device_path, GFP_KERNEL);
2362
	if (!name) {
2363
		kfree(device);
Y
Yan Zheng 已提交
2364 2365
		ret = -ENOMEM;
		goto error;
2366
	}
2367
	rcu_assign_pointer(device->name, name);
Y
Yan Zheng 已提交
2368

2369
	trans = btrfs_start_transaction(root, 0);
2370
	if (IS_ERR(trans)) {
2371
		rcu_string_free(device->name);
2372 2373 2374 2375 2376
		kfree(device);
		ret = PTR_ERR(trans);
		goto error;
	}

2377 2378 2379
	q = bdev_get_queue(bdev);
	if (blk_queue_discard(q))
		device->can_discard = 1;
Y
Yan Zheng 已提交
2380 2381
	device->writeable = 1;
	device->generation = trans->transid;
2382 2383 2384
	device->io_width = fs_info->sectorsize;
	device->io_align = fs_info->sectorsize;
	device->sector_size = fs_info->sectorsize;
2385
	device->total_bytes = i_size_read(bdev->bd_inode);
2386
	device->disk_total_bytes = device->total_bytes;
2387
	device->commit_total_bytes = device->total_bytes;
2388
	device->fs_info = fs_info;
2389
	device->bdev = bdev;
2390
	device->in_fs_metadata = 1;
2391
	device->is_tgtdev_for_dev_replace = 0;
2392
	device->mode = FMODE_EXCL;
2393
	device->dev_stats_valid = 1;
Y
Yan Zheng 已提交
2394
	set_blocksize(device->bdev, 4096);
2395

Y
Yan Zheng 已提交
2396 2397
	if (seeding_dev) {
		sb->s_flags &= ~MS_RDONLY;
2398
		ret = btrfs_prepare_sprout(fs_info);
2399
		BUG_ON(ret); /* -ENOMEM */
Y
Yan Zheng 已提交
2400
	}
2401

2402
	device->fs_devices = fs_info->fs_devices;
2403

2404 2405 2406
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
	lock_chunks(fs_info);
	list_add_rcu(&device->dev_list, &fs_info->fs_devices->devices);
Y
Yan Zheng 已提交
2407
	list_add(&device->dev_alloc_list,
2408 2409 2410 2411 2412 2413
		 &fs_info->fs_devices->alloc_list);
	fs_info->fs_devices->num_devices++;
	fs_info->fs_devices->open_devices++;
	fs_info->fs_devices->rw_devices++;
	fs_info->fs_devices->total_devices++;
	fs_info->fs_devices->total_rw_bytes += device->total_bytes;
2414

2415 2416 2417
	spin_lock(&fs_info->free_chunk_lock);
	fs_info->free_chunk_space += device->total_bytes;
	spin_unlock(&fs_info->free_chunk_lock);
2418

C
Chris Mason 已提交
2419
	if (!blk_queue_nonrot(bdev_get_queue(bdev)))
2420
		fs_info->fs_devices->rotating = 1;
C
Chris Mason 已提交
2421

2422 2423
	tmp = btrfs_super_total_bytes(fs_info->super_copy);
	btrfs_set_super_total_bytes(fs_info->super_copy,
2424
				    tmp + device->total_bytes);
2425

2426 2427
	tmp = btrfs_super_num_devices(fs_info->super_copy);
	btrfs_set_super_num_devices(fs_info->super_copy, tmp + 1);
2428 2429

	/* add sysfs device entry */
2430
	btrfs_sysfs_add_device_link(fs_info->fs_devices, device);
2431

M
Miao Xie 已提交
2432 2433 2434 2435
	/*
	 * we've got more storage, clear any full flags on the space
	 * infos
	 */
2436
	btrfs_clear_space_info_full(fs_info);
M
Miao Xie 已提交
2437

2438 2439
	unlock_chunks(fs_info);
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2440

Y
Yan Zheng 已提交
2441
	if (seeding_dev) {
2442
		lock_chunks(fs_info);
2443
		ret = init_first_rw_device(trans, fs_info, device);
2444
		unlock_chunks(fs_info);
2445
		if (ret) {
2446
			btrfs_abort_transaction(trans, ret);
2447
			goto error_trans;
2448
		}
M
Miao Xie 已提交
2449 2450
	}

2451
	ret = btrfs_add_device(trans, fs_info, device);
M
Miao Xie 已提交
2452
	if (ret) {
2453
		btrfs_abort_transaction(trans, ret);
M
Miao Xie 已提交
2454 2455 2456 2457 2458 2459
		goto error_trans;
	}

	if (seeding_dev) {
		char fsid_buf[BTRFS_UUID_UNPARSED_SIZE];

2460
		ret = btrfs_finish_sprout(trans, fs_info);
2461
		if (ret) {
2462
			btrfs_abort_transaction(trans, ret);
2463
			goto error_trans;
2464
		}
2465 2466 2467 2468 2469

		/* Sprouting would change fsid of the mounted root,
		 * so rename the fsid on the sysfs
		 */
		snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
2470 2471 2472 2473
						fs_info->fsid);
		if (kobject_rename(&fs_info->fs_devices->fsid_kobj, fsid_buf))
			btrfs_warn(fs_info,
				   "sysfs: failed to create fsid for sprout");
Y
Yan Zheng 已提交
2474 2475
	}

2476 2477
	fs_info->num_tolerated_disk_barrier_failures =
		btrfs_calc_num_tolerated_disk_barrier_failures(fs_info);
2478
	ret = btrfs_commit_transaction(trans, root);
2479

Y
Yan Zheng 已提交
2480 2481 2482
	if (seeding_dev) {
		mutex_unlock(&uuid_mutex);
		up_write(&sb->s_umount);
2483

2484 2485 2486
		if (ret) /* transaction commit */
			return ret;

2487
		ret = btrfs_relocate_sys_chunks(fs_info);
2488
		if (ret < 0)
2489
			btrfs_handle_fs_error(fs_info, ret,
J
Jeff Mahoney 已提交
2490
				    "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2491 2492 2493 2494 2495 2496 2497
		trans = btrfs_attach_transaction(root);
		if (IS_ERR(trans)) {
			if (PTR_ERR(trans) == -ENOENT)
				return 0;
			return PTR_ERR(trans);
		}
		ret = btrfs_commit_transaction(trans, root);
Y
Yan Zheng 已提交
2498
	}
2499

2500 2501
	/* Update ctime/mtime for libblkid */
	update_dev_time(device_path);
Y
Yan Zheng 已提交
2502
	return ret;
2503 2504 2505

error_trans:
	btrfs_end_transaction(trans, root);
2506
	rcu_string_free(device->name);
2507
	btrfs_sysfs_rm_device_link(fs_info->fs_devices, device);
2508
	kfree(device);
Y
Yan Zheng 已提交
2509
error:
2510
	blkdev_put(bdev, FMODE_EXCL);
Y
Yan Zheng 已提交
2511 2512 2513 2514
	if (seeding_dev) {
		mutex_unlock(&uuid_mutex);
		up_write(&sb->s_umount);
	}
2515
	return ret;
2516 2517
}

2518 2519
int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
				  char *device_path,
2520
				  struct btrfs_device *srcdev,
2521 2522 2523 2524 2525 2526 2527
				  struct btrfs_device **device_out)
{
	struct request_queue *q;
	struct btrfs_device *device;
	struct block_device *bdev;
	struct list_head *devices;
	struct rcu_string *name;
2528
	u64 devid = BTRFS_DEV_REPLACE_DEVID;
2529 2530 2531
	int ret = 0;

	*device_out = NULL;
2532 2533
	if (fs_info->fs_devices->seeding) {
		btrfs_err(fs_info, "the filesystem is a seed filesystem!");
2534
		return -EINVAL;
2535
	}
2536 2537 2538

	bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
				  fs_info->bdev_holder);
2539 2540
	if (IS_ERR(bdev)) {
		btrfs_err(fs_info, "target device %s is invalid!", device_path);
2541
		return PTR_ERR(bdev);
2542
	}
2543 2544 2545 2546 2547 2548

	filemap_write_and_wait(bdev->bd_inode->i_mapping);

	devices = &fs_info->fs_devices->devices;
	list_for_each_entry(device, devices, dev_list) {
		if (device->bdev == bdev) {
J
Jeff Mahoney 已提交
2549 2550
			btrfs_err(fs_info,
				  "target device is in the filesystem!");
2551 2552 2553 2554 2555
			ret = -EEXIST;
			goto error;
		}
	}

2556

2557 2558
	if (i_size_read(bdev->bd_inode) <
	    btrfs_device_get_total_bytes(srcdev)) {
J
Jeff Mahoney 已提交
2559 2560
		btrfs_err(fs_info,
			  "target device is smaller than source device!");
2561 2562 2563 2564 2565
		ret = -EINVAL;
		goto error;
	}


2566 2567 2568
	device = btrfs_alloc_device(NULL, &devid, NULL);
	if (IS_ERR(device)) {
		ret = PTR_ERR(device);
2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582
		goto error;
	}

	name = rcu_string_strdup(device_path, GFP_NOFS);
	if (!name) {
		kfree(device);
		ret = -ENOMEM;
		goto error;
	}
	rcu_assign_pointer(device->name, name);

	q = bdev_get_queue(bdev);
	if (blk_queue_discard(q))
		device->can_discard = 1;
2583
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
2584 2585
	device->writeable = 1;
	device->generation = 0;
2586 2587 2588
	device->io_width = fs_info->sectorsize;
	device->io_align = fs_info->sectorsize;
	device->sector_size = fs_info->sectorsize;
2589 2590 2591
	device->total_bytes = btrfs_device_get_total_bytes(srcdev);
	device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev);
	device->bytes_used = btrfs_device_get_bytes_used(srcdev);
2592 2593
	ASSERT(list_empty(&srcdev->resized_list));
	device->commit_total_bytes = srcdev->commit_total_bytes;
2594
	device->commit_bytes_used = device->bytes_used;
2595
	device->fs_info = fs_info;
2596 2597 2598 2599
	device->bdev = bdev;
	device->in_fs_metadata = 1;
	device->is_tgtdev_for_dev_replace = 1;
	device->mode = FMODE_EXCL;
2600
	device->dev_stats_valid = 1;
2601 2602 2603 2604 2605
	set_blocksize(device->bdev, 4096);
	device->fs_devices = fs_info->fs_devices;
	list_add(&device->dev_list, &fs_info->fs_devices->devices);
	fs_info->fs_devices->num_devices++;
	fs_info->fs_devices->open_devices++;
2606
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618

	*device_out = device;
	return ret;

error:
	blkdev_put(bdev, FMODE_EXCL);
	return ret;
}

void btrfs_init_dev_replace_tgtdev_for_resume(struct btrfs_fs_info *fs_info,
					      struct btrfs_device *tgtdev)
{
2619 2620
	u32 sectorsize = fs_info->sectorsize;

2621
	WARN_ON(fs_info->fs_devices->rw_devices == 0);
2622 2623 2624
	tgtdev->io_width = sectorsize;
	tgtdev->io_align = sectorsize;
	tgtdev->sector_size = sectorsize;
2625
	tgtdev->fs_info = fs_info;
2626 2627 2628
	tgtdev->in_fs_metadata = 1;
}

C
Chris Mason 已提交
2629 2630
static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
					struct btrfs_device *device)
2631 2632 2633
{
	int ret;
	struct btrfs_path *path;
2634
	struct btrfs_root *root = device->fs_info->chunk_root;
2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663
	struct btrfs_dev_item *dev_item;
	struct extent_buffer *leaf;
	struct btrfs_key key;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
	key.type = BTRFS_DEV_ITEM_KEY;
	key.offset = device->devid;

	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
	if (ret < 0)
		goto out;

	if (ret > 0) {
		ret = -ENOENT;
		goto out;
	}

	leaf = path->nodes[0];
	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);

	btrfs_set_device_id(leaf, dev_item, device->devid);
	btrfs_set_device_type(leaf, dev_item, device->type);
	btrfs_set_device_io_align(leaf, dev_item, device->io_align);
	btrfs_set_device_io_width(leaf, dev_item, device->io_width);
	btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2664 2665 2666 2667
	btrfs_set_device_total_bytes(leaf, dev_item,
				     btrfs_device_get_disk_total_bytes(device));
	btrfs_set_device_bytes_used(leaf, dev_item,
				    btrfs_device_get_bytes_used(device));
2668 2669 2670 2671 2672 2673 2674
	btrfs_mark_buffer_dirty(leaf);

out:
	btrfs_free_path(path);
	return ret;
}

M
Miao Xie 已提交
2675
int btrfs_grow_device(struct btrfs_trans_handle *trans,
2676 2677
		      struct btrfs_device *device, u64 new_size)
{
2678 2679
	struct btrfs_fs_info *fs_info = device->fs_info;
	struct btrfs_super_block *super_copy = fs_info->super_copy;
2680
	struct btrfs_fs_devices *fs_devices;
M
Miao Xie 已提交
2681 2682
	u64 old_total;
	u64 diff;
2683

Y
Yan Zheng 已提交
2684 2685
	if (!device->writeable)
		return -EACCES;
M
Miao Xie 已提交
2686

2687
	lock_chunks(fs_info);
M
Miao Xie 已提交
2688 2689 2690
	old_total = btrfs_super_total_bytes(super_copy);
	diff = new_size - device->total_bytes;

2691
	if (new_size <= device->total_bytes ||
M
Miao Xie 已提交
2692
	    device->is_tgtdev_for_dev_replace) {
2693
		unlock_chunks(fs_info);
Y
Yan Zheng 已提交
2694
		return -EINVAL;
M
Miao Xie 已提交
2695
	}
Y
Yan Zheng 已提交
2696

2697
	fs_devices = fs_info->fs_devices;
Y
Yan Zheng 已提交
2698

2699
	btrfs_set_super_total_bytes(super_copy, old_total + diff);
Y
Yan Zheng 已提交
2700 2701
	device->fs_devices->total_rw_bytes += diff;

2702 2703
	btrfs_device_set_total_bytes(device, new_size);
	btrfs_device_set_disk_total_bytes(device, new_size);
2704
	btrfs_clear_space_info_full(device->fs_info);
2705 2706 2707
	if (list_empty(&device->resized_list))
		list_add_tail(&device->resized_list,
			      &fs_devices->resized_devices);
2708
	unlock_chunks(fs_info);
2709

2710 2711 2712 2713
	return btrfs_update_device(trans, device);
}

static int btrfs_free_chunk(struct btrfs_trans_handle *trans,
2714
			    struct btrfs_fs_info *fs_info, u64 chunk_objectid,
2715 2716
			    u64 chunk_offset)
{
2717
	struct btrfs_root *root = fs_info->chunk_root;
2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730
	int ret;
	struct btrfs_path *path;
	struct btrfs_key key;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = chunk_objectid;
	key.offset = chunk_offset;
	key.type = BTRFS_CHUNK_ITEM_KEY;

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2731 2732 2733
	if (ret < 0)
		goto out;
	else if (ret > 0) { /* Logic error or corruption */
2734 2735
		btrfs_handle_fs_error(fs_info, -ENOENT,
				      "Failed lookup while freeing chunk.");
2736 2737 2738
		ret = -ENOENT;
		goto out;
	}
2739 2740

	ret = btrfs_del_item(trans, root, path);
2741
	if (ret < 0)
2742 2743
		btrfs_handle_fs_error(fs_info, ret,
				      "Failed to delete chunk item.");
2744
out:
2745
	btrfs_free_path(path);
2746
	return ret;
2747 2748
}

2749 2750
static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info,
			       u64 chunk_objectid, u64 chunk_offset)
2751
{
2752
	struct btrfs_super_block *super_copy = fs_info->super_copy;
2753 2754 2755 2756 2757 2758 2759 2760 2761 2762
	struct btrfs_disk_key *disk_key;
	struct btrfs_chunk *chunk;
	u8 *ptr;
	int ret = 0;
	u32 num_stripes;
	u32 array_size;
	u32 len = 0;
	u32 cur;
	struct btrfs_key key;

2763
	lock_chunks(fs_info);
2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792
	array_size = btrfs_super_sys_array_size(super_copy);

	ptr = super_copy->sys_chunk_array;
	cur = 0;

	while (cur < array_size) {
		disk_key = (struct btrfs_disk_key *)ptr;
		btrfs_disk_key_to_cpu(&key, disk_key);

		len = sizeof(*disk_key);

		if (key.type == BTRFS_CHUNK_ITEM_KEY) {
			chunk = (struct btrfs_chunk *)(ptr + len);
			num_stripes = btrfs_stack_chunk_num_stripes(chunk);
			len += btrfs_chunk_item_size(num_stripes);
		} else {
			ret = -EIO;
			break;
		}
		if (key.objectid == chunk_objectid &&
		    key.offset == chunk_offset) {
			memmove(ptr, ptr + len, array_size - (cur + len));
			array_size -= len;
			btrfs_set_super_sys_array_size(super_copy, array_size);
		} else {
			ptr += len;
			cur += len;
		}
	}
2793
	unlock_chunks(fs_info);
2794 2795 2796
	return ret;
}

2797
int btrfs_remove_chunk(struct btrfs_trans_handle *trans,
2798
		       struct btrfs_fs_info *fs_info, u64 chunk_offset)
2799 2800 2801 2802
{
	struct extent_map_tree *em_tree;
	struct extent_map *em;
	struct map_lookup *map;
M
Miao Xie 已提交
2803
	u64 dev_extent_len = 0;
2804 2805
	u64 chunk_objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
	int i, ret = 0;
2806
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2807

2808
	em_tree = &fs_info->mapping_tree.map_tree;
2809

2810
	read_lock(&em_tree->lock);
2811
	em = lookup_extent_mapping(em_tree, chunk_offset, 1);
2812
	read_unlock(&em_tree->lock);
2813

2814 2815 2816 2817
	if (!em || em->start > chunk_offset ||
	    em->start + em->len < chunk_offset) {
		/*
		 * This is a logic error, but we don't want to just rely on the
2818
		 * user having built with ASSERT enabled, so if ASSERT doesn't
2819 2820 2821 2822 2823 2824 2825
		 * do anything we still error out.
		 */
		ASSERT(0);
		if (em)
			free_extent_map(em);
		return -EINVAL;
	}
2826
	map = em->map_lookup;
2827
	lock_chunks(fs_info);
2828
	check_system_chunk(trans, fs_info, map->type);
2829
	unlock_chunks(fs_info);
2830

2831 2832 2833 2834 2835 2836
	/*
	 * Take the device list mutex to prevent races with the final phase of
	 * a device replace operation that replaces the device object associated
	 * with map stripes (dev-replace.c:btrfs_dev_replace_finishing()).
	 */
	mutex_lock(&fs_devices->device_list_mutex);
2837
	for (i = 0; i < map->num_stripes; i++) {
2838
		struct btrfs_device *device = map->stripes[i].dev;
M
Miao Xie 已提交
2839 2840 2841
		ret = btrfs_free_dev_extent(trans, device,
					    map->stripes[i].physical,
					    &dev_extent_len);
2842
		if (ret) {
2843
			mutex_unlock(&fs_devices->device_list_mutex);
2844
			btrfs_abort_transaction(trans, ret);
2845 2846
			goto out;
		}
2847

M
Miao Xie 已提交
2848
		if (device->bytes_used > 0) {
2849
			lock_chunks(fs_info);
M
Miao Xie 已提交
2850 2851
			btrfs_device_set_bytes_used(device,
					device->bytes_used - dev_extent_len);
2852 2853 2854 2855 2856
			spin_lock(&fs_info->free_chunk_lock);
			fs_info->free_chunk_space += dev_extent_len;
			spin_unlock(&fs_info->free_chunk_lock);
			btrfs_clear_space_info_full(fs_info);
			unlock_chunks(fs_info);
M
Miao Xie 已提交
2857
		}
2858

2859 2860
		if (map->stripes[i].dev) {
			ret = btrfs_update_device(trans, map->stripes[i].dev);
2861
			if (ret) {
2862
				mutex_unlock(&fs_devices->device_list_mutex);
2863
				btrfs_abort_transaction(trans, ret);
2864 2865
				goto out;
			}
2866
		}
2867
	}
2868 2869
	mutex_unlock(&fs_devices->device_list_mutex);

2870
	ret = btrfs_free_chunk(trans, fs_info, chunk_objectid, chunk_offset);
2871
	if (ret) {
2872
		btrfs_abort_transaction(trans, ret);
2873 2874
		goto out;
	}
2875

2876
	trace_btrfs_chunk_free(fs_info, map, chunk_offset, em->len);
2877

2878
	if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
2879 2880
		ret = btrfs_del_sys_chunk(fs_info, chunk_objectid,
					  chunk_offset);
2881
		if (ret) {
2882
			btrfs_abort_transaction(trans, ret);
2883 2884
			goto out;
		}
2885 2886
	}

2887
	ret = btrfs_remove_block_group(trans, fs_info, chunk_offset, em);
2888
	if (ret) {
2889
		btrfs_abort_transaction(trans, ret);
2890 2891
		goto out;
	}
Y
Yan Zheng 已提交
2892

2893
out:
Y
Yan Zheng 已提交
2894 2895
	/* once for us */
	free_extent_map(em);
2896 2897
	return ret;
}
Y
Yan Zheng 已提交
2898

2899
static int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
2900
{
2901 2902
	struct btrfs_root *root = fs_info->chunk_root;
	struct btrfs_root *extent_root = fs_info->extent_root;
2903
	struct btrfs_trans_handle *trans;
2904
	int ret;
Y
Yan Zheng 已提交
2905

2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917
	/*
	 * Prevent races with automatic removal of unused block groups.
	 * After we relocate and before we remove the chunk with offset
	 * chunk_offset, automatic removal of the block group can kick in,
	 * resulting in a failure when calling btrfs_remove_chunk() below.
	 *
	 * Make sure to acquire this mutex before doing a tree search (dev
	 * or chunk trees) to find chunks. Otherwise the cleaner kthread might
	 * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
	 * we release the path used to search the chunk/dev tree and before
	 * the current task acquires this mutex and calls us.
	 */
2918
	ASSERT(mutex_is_locked(&fs_info->delete_unused_bgs_mutex));
2919

2920
	ret = btrfs_can_relocate(fs_info, chunk_offset);
2921 2922 2923 2924
	if (ret)
		return -ENOSPC;

	/* step one, relocate all the extents inside this chunk */
2925
	btrfs_scrub_pause(fs_info);
2926
	ret = btrfs_relocate_block_group(fs_info, chunk_offset);
2927
	btrfs_scrub_continue(fs_info);
2928 2929 2930
	if (ret)
		return ret;

2931 2932 2933 2934 2935 2936 2937 2938
	trans = btrfs_start_trans_remove_block_group(root->fs_info,
						     chunk_offset);
	if (IS_ERR(trans)) {
		ret = PTR_ERR(trans);
		btrfs_handle_fs_error(root->fs_info, ret, NULL);
		return ret;
	}

2939
	/*
2940 2941
	 * step two, delete the device extents and the
	 * chunk tree entries
2942
	 */
2943
	ret = btrfs_remove_chunk(trans, fs_info, chunk_offset);
2944 2945
	btrfs_end_transaction(trans, extent_root);
	return ret;
Y
Yan Zheng 已提交
2946 2947
}

2948
static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
Y
Yan Zheng 已提交
2949
{
2950
	struct btrfs_root *chunk_root = fs_info->chunk_root;
Y
Yan Zheng 已提交
2951 2952 2953 2954 2955 2956
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_chunk *chunk;
	struct btrfs_key key;
	struct btrfs_key found_key;
	u64 chunk_type;
2957 2958
	bool retried = false;
	int failed = 0;
Y
Yan Zheng 已提交
2959 2960 2961 2962 2963 2964
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

2965
again:
Y
Yan Zheng 已提交
2966 2967 2968 2969 2970
	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
	key.offset = (u64)-1;
	key.type = BTRFS_CHUNK_ITEM_KEY;

	while (1) {
2971
		mutex_lock(&fs_info->delete_unused_bgs_mutex);
Y
Yan Zheng 已提交
2972
		ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
2973
		if (ret < 0) {
2974
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
Y
Yan Zheng 已提交
2975
			goto error;
2976
		}
2977
		BUG_ON(ret == 0); /* Corruption */
Y
Yan Zheng 已提交
2978 2979 2980

		ret = btrfs_previous_item(chunk_root, path, key.objectid,
					  key.type);
2981
		if (ret)
2982
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
Y
Yan Zheng 已提交
2983 2984 2985 2986
		if (ret < 0)
			goto error;
		if (ret > 0)
			break;
Z
Zheng Yan 已提交
2987

Y
Yan Zheng 已提交
2988 2989
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
Z
Zheng Yan 已提交
2990

Y
Yan Zheng 已提交
2991 2992 2993
		chunk = btrfs_item_ptr(leaf, path->slots[0],
				       struct btrfs_chunk);
		chunk_type = btrfs_chunk_type(leaf, chunk);
2994
		btrfs_release_path(path);
2995

Y
Yan Zheng 已提交
2996
		if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
2997
			ret = btrfs_relocate_chunk(fs_info, found_key.offset);
2998 2999
			if (ret == -ENOSPC)
				failed++;
H
HIMANGI SARAOGI 已提交
3000 3001
			else
				BUG_ON(ret);
Y
Yan Zheng 已提交
3002
		}
3003
		mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3004

Y
Yan Zheng 已提交
3005 3006 3007 3008 3009
		if (found_key.offset == 0)
			break;
		key.offset = found_key.offset - 1;
	}
	ret = 0;
3010 3011 3012 3013
	if (failed && !retried) {
		failed = 0;
		retried = true;
		goto again;
3014
	} else if (WARN_ON(failed && retried)) {
3015 3016
		ret = -ENOSPC;
	}
Y
Yan Zheng 已提交
3017 3018 3019
error:
	btrfs_free_path(path);
	return ret;
3020 3021
}

3022
static int insert_balance_item(struct btrfs_fs_info *fs_info,
3023 3024
			       struct btrfs_balance_control *bctl)
{
3025
	struct btrfs_root *root = fs_info->tree_root;
3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044
	struct btrfs_trans_handle *trans;
	struct btrfs_balance_item *item;
	struct btrfs_disk_balance_args disk_bargs;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	int ret, err;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	trans = btrfs_start_transaction(root, 0);
	if (IS_ERR(trans)) {
		btrfs_free_path(path);
		return PTR_ERR(trans);
	}

	key.objectid = BTRFS_BALANCE_OBJECTID;
3045
	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3046 3047 3048 3049 3050 3051 3052 3053 3054 3055
	key.offset = 0;

	ret = btrfs_insert_empty_item(trans, root, path, &key,
				      sizeof(*item));
	if (ret)
		goto out;

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);

3056
	memzero_extent_buffer(leaf, (unsigned long)item, sizeof(*item));
3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075

	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data);
	btrfs_set_balance_data(leaf, item, &disk_bargs);
	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta);
	btrfs_set_balance_meta(leaf, item, &disk_bargs);
	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys);
	btrfs_set_balance_sys(leaf, item, &disk_bargs);

	btrfs_set_balance_flags(leaf, item, bctl->flags);

	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	err = btrfs_commit_transaction(trans, root);
	if (err && !ret)
		ret = err;
	return ret;
}

3076
static int del_balance_item(struct btrfs_fs_info *fs_info)
3077
{
3078
	struct btrfs_root *root = fs_info->tree_root;
3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094
	struct btrfs_trans_handle *trans;
	struct btrfs_path *path;
	struct btrfs_key key;
	int ret, err;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	trans = btrfs_start_transaction(root, 0);
	if (IS_ERR(trans)) {
		btrfs_free_path(path);
		return PTR_ERR(trans);
	}

	key.objectid = BTRFS_BALANCE_OBJECTID;
3095
	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114
	key.offset = 0;

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0) {
		ret = -ENOENT;
		goto out;
	}

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	err = btrfs_commit_transaction(trans, root);
	if (err && !ret)
		ret = err;
	return ret;
}

I
Ilya Dryomov 已提交
3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138
/*
 * This is a heuristic used to reduce the number of chunks balanced on
 * resume after balance was interrupted.
 */
static void update_balance_args(struct btrfs_balance_control *bctl)
{
	/*
	 * Turn on soft mode for chunk types that were being converted.
	 */
	if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)
		bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT;
	if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)
		bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT;
	if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)
		bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT;

	/*
	 * Turn on usage filter if is not already used.  The idea is
	 * that chunks that we have already balanced should be
	 * reasonably full.  Don't do it for chunks that are being
	 * converted - that will keep us from relocating unconverted
	 * (albeit full) chunks.
	 */
	if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3139
	    !(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
I
Ilya Dryomov 已提交
3140 3141 3142 3143 3144
	    !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
		bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE;
		bctl->data.usage = 90;
	}
	if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3145
	    !(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
I
Ilya Dryomov 已提交
3146 3147 3148 3149 3150
	    !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
		bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE;
		bctl->sys.usage = 90;
	}
	if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3151
	    !(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
I
Ilya Dryomov 已提交
3152 3153 3154 3155 3156 3157
	    !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
		bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE;
		bctl->meta.usage = 90;
	}
}

3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186
/*
 * Should be called with both balance and volume mutexes held to
 * serialize other volume operations (add_dev/rm_dev/resize) with
 * restriper.  Same goes for unset_balance_control.
 */
static void set_balance_control(struct btrfs_balance_control *bctl)
{
	struct btrfs_fs_info *fs_info = bctl->fs_info;

	BUG_ON(fs_info->balance_ctl);

	spin_lock(&fs_info->balance_lock);
	fs_info->balance_ctl = bctl;
	spin_unlock(&fs_info->balance_lock);
}

static void unset_balance_control(struct btrfs_fs_info *fs_info)
{
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;

	BUG_ON(!fs_info->balance_ctl);

	spin_lock(&fs_info->balance_lock);
	fs_info->balance_ctl = NULL;
	spin_unlock(&fs_info->balance_lock);

	kfree(bctl);
}

I
Ilya Dryomov 已提交
3187 3188 3189 3190
/*
 * Balance filters.  Return 1 if chunk should be filtered out
 * (should not be balanced).
 */
3191
static int chunk_profiles_filter(u64 chunk_type,
I
Ilya Dryomov 已提交
3192 3193
				 struct btrfs_balance_args *bargs)
{
3194 3195
	chunk_type = chunk_to_extended(chunk_type) &
				BTRFS_EXTENDED_PROFILE_MASK;
I
Ilya Dryomov 已提交
3196

3197
	if (bargs->profiles & chunk_type)
I
Ilya Dryomov 已提交
3198 3199 3200 3201 3202
		return 0;

	return 1;
}

3203
static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
I
Ilya Dryomov 已提交
3204
			      struct btrfs_balance_args *bargs)
3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
{
	struct btrfs_block_group_cache *cache;
	u64 chunk_used;
	u64 user_thresh_min;
	u64 user_thresh_max;
	int ret = 1;

	cache = btrfs_lookup_block_group(fs_info, chunk_offset);
	chunk_used = btrfs_block_group_used(&cache->item);

	if (bargs->usage_min == 0)
		user_thresh_min = 0;
	else
		user_thresh_min = div_factor_fine(cache->key.offset,
					bargs->usage_min);

	if (bargs->usage_max == 0)
		user_thresh_max = 1;
	else if (bargs->usage_max > 100)
		user_thresh_max = cache->key.offset;
	else
		user_thresh_max = div_factor_fine(cache->key.offset,
					bargs->usage_max);

	if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max)
		ret = 0;

	btrfs_put_block_group(cache);
	return ret;
}

3236
static int chunk_usage_filter(struct btrfs_fs_info *fs_info,
3237
		u64 chunk_offset, struct btrfs_balance_args *bargs)
I
Ilya Dryomov 已提交
3238 3239 3240 3241 3242 3243 3244 3245
{
	struct btrfs_block_group_cache *cache;
	u64 chunk_used, user_thresh;
	int ret = 1;

	cache = btrfs_lookup_block_group(fs_info, chunk_offset);
	chunk_used = btrfs_block_group_used(&cache->item);

3246
	if (bargs->usage_min == 0)
3247
		user_thresh = 1;
3248 3249 3250 3251 3252 3253
	else if (bargs->usage > 100)
		user_thresh = cache->key.offset;
	else
		user_thresh = div_factor_fine(cache->key.offset,
					      bargs->usage);

I
Ilya Dryomov 已提交
3254 3255 3256 3257 3258 3259 3260
	if (chunk_used < user_thresh)
		ret = 0;

	btrfs_put_block_group(cache);
	return ret;
}

I
Ilya Dryomov 已提交
3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277
static int chunk_devid_filter(struct extent_buffer *leaf,
			      struct btrfs_chunk *chunk,
			      struct btrfs_balance_args *bargs)
{
	struct btrfs_stripe *stripe;
	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
	int i;

	for (i = 0; i < num_stripes; i++) {
		stripe = btrfs_stripe_nr(chunk, i);
		if (btrfs_stripe_devid(leaf, stripe) == bargs->devid)
			return 0;
	}

	return 1;
}

I
Ilya Dryomov 已提交
3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294
/* [pstart, pend) */
static int chunk_drange_filter(struct extent_buffer *leaf,
			       struct btrfs_chunk *chunk,
			       u64 chunk_offset,
			       struct btrfs_balance_args *bargs)
{
	struct btrfs_stripe *stripe;
	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
	u64 stripe_offset;
	u64 stripe_length;
	int factor;
	int i;

	if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID))
		return 0;

	if (btrfs_chunk_type(leaf, chunk) & (BTRFS_BLOCK_GROUP_DUP |
D
David Woodhouse 已提交
3295 3296 3297 3298 3299 3300 3301 3302 3303
	     BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10)) {
		factor = num_stripes / 2;
	} else if (btrfs_chunk_type(leaf, chunk) & BTRFS_BLOCK_GROUP_RAID5) {
		factor = num_stripes - 1;
	} else if (btrfs_chunk_type(leaf, chunk) & BTRFS_BLOCK_GROUP_RAID6) {
		factor = num_stripes - 2;
	} else {
		factor = num_stripes;
	}
I
Ilya Dryomov 已提交
3304 3305 3306 3307 3308 3309 3310 3311

	for (i = 0; i < num_stripes; i++) {
		stripe = btrfs_stripe_nr(chunk, i);
		if (btrfs_stripe_devid(leaf, stripe) != bargs->devid)
			continue;

		stripe_offset = btrfs_stripe_offset(leaf, stripe);
		stripe_length = btrfs_chunk_length(leaf, chunk);
3312
		stripe_length = div_u64(stripe_length, factor);
I
Ilya Dryomov 已提交
3313 3314 3315 3316 3317 3318 3319 3320 3321

		if (stripe_offset < bargs->pend &&
		    stripe_offset + stripe_length > bargs->pstart)
			return 0;
	}

	return 1;
}

3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335
/* [vstart, vend) */
static int chunk_vrange_filter(struct extent_buffer *leaf,
			       struct btrfs_chunk *chunk,
			       u64 chunk_offset,
			       struct btrfs_balance_args *bargs)
{
	if (chunk_offset < bargs->vend &&
	    chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart)
		/* at least part of the chunk is inside this vrange */
		return 0;

	return 1;
}

3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348
static int chunk_stripes_range_filter(struct extent_buffer *leaf,
			       struct btrfs_chunk *chunk,
			       struct btrfs_balance_args *bargs)
{
	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);

	if (bargs->stripes_min <= num_stripes
			&& num_stripes <= bargs->stripes_max)
		return 0;

	return 1;
}

3349
static int chunk_soft_convert_filter(u64 chunk_type,
3350 3351 3352 3353 3354
				     struct btrfs_balance_args *bargs)
{
	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
		return 0;

3355 3356
	chunk_type = chunk_to_extended(chunk_type) &
				BTRFS_EXTENDED_PROFILE_MASK;
3357

3358
	if (bargs->target == chunk_type)
3359 3360 3361 3362 3363
		return 1;

	return 0;
}

3364
static int should_balance_chunk(struct btrfs_fs_info *fs_info,
3365 3366 3367
				struct extent_buffer *leaf,
				struct btrfs_chunk *chunk, u64 chunk_offset)
{
3368
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384
	struct btrfs_balance_args *bargs = NULL;
	u64 chunk_type = btrfs_chunk_type(leaf, chunk);

	/* type filter */
	if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) &
	      (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) {
		return 0;
	}

	if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
		bargs = &bctl->data;
	else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
		bargs = &bctl->sys;
	else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
		bargs = &bctl->meta;

I
Ilya Dryomov 已提交
3385 3386 3387 3388
	/* profiles filter */
	if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) &&
	    chunk_profiles_filter(chunk_type, bargs)) {
		return 0;
I
Ilya Dryomov 已提交
3389 3390 3391 3392
	}

	/* usage filter */
	if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) &&
3393
	    chunk_usage_filter(fs_info, chunk_offset, bargs)) {
I
Ilya Dryomov 已提交
3394
		return 0;
3395
	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3396
	    chunk_usage_range_filter(fs_info, chunk_offset, bargs)) {
3397
		return 0;
I
Ilya Dryomov 已提交
3398 3399 3400 3401 3402 3403
	}

	/* devid filter */
	if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) &&
	    chunk_devid_filter(leaf, chunk, bargs)) {
		return 0;
I
Ilya Dryomov 已提交
3404 3405 3406 3407 3408 3409
	}

	/* drange filter, makes sense only with devid filter */
	if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) &&
	    chunk_drange_filter(leaf, chunk, chunk_offset, bargs)) {
		return 0;
3410 3411 3412 3413 3414 3415
	}

	/* vrange filter */
	if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) &&
	    chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) {
		return 0;
I
Ilya Dryomov 已提交
3416 3417
	}

3418 3419 3420 3421 3422 3423
	/* stripes filter */
	if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) &&
	    chunk_stripes_range_filter(leaf, chunk, bargs)) {
		return 0;
	}

3424 3425 3426 3427 3428 3429
	/* soft profile changing mode */
	if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) &&
	    chunk_soft_convert_filter(chunk_type, bargs)) {
		return 0;
	}

3430 3431 3432 3433 3434 3435 3436 3437
	/*
	 * limited by count, must be the last filter
	 */
	if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) {
		if (bargs->limit == 0)
			return 0;
		else
			bargs->limit--;
3438 3439 3440
	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) {
		/*
		 * Same logic as the 'limit' filter; the minimum cannot be
3441
		 * determined here because we do not have the global information
3442 3443 3444 3445 3446 3447
		 * about the count of all chunks that satisfy the filters.
		 */
		if (bargs->limit_max == 0)
			return 0;
		else
			bargs->limit_max--;
3448 3449
	}

3450 3451 3452
	return 1;
}

3453
static int __btrfs_balance(struct btrfs_fs_info *fs_info)
3454
{
3455
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3456 3457 3458
	struct btrfs_root *chunk_root = fs_info->chunk_root;
	struct btrfs_root *dev_root = fs_info->dev_root;
	struct list_head *devices;
3459 3460 3461
	struct btrfs_device *device;
	u64 old_size;
	u64 size_to_free;
3462
	u64 chunk_type;
3463
	struct btrfs_chunk *chunk;
3464
	struct btrfs_path *path = NULL;
3465 3466
	struct btrfs_key key;
	struct btrfs_key found_key;
3467
	struct btrfs_trans_handle *trans;
3468 3469
	struct extent_buffer *leaf;
	int slot;
3470 3471
	int ret;
	int enospc_errors = 0;
3472
	bool counting = true;
3473
	/* The single value limit and min/max limits use the same bytes in the */
3474 3475 3476
	u64 limit_data = bctl->data.limit;
	u64 limit_meta = bctl->meta.limit;
	u64 limit_sys = bctl->sys.limit;
3477 3478 3479
	u32 count_data = 0;
	u32 count_meta = 0;
	u32 count_sys = 0;
3480
	int chunk_reserved = 0;
3481
	u64 bytes_used = 0;
3482 3483

	/* step one make some room on all the devices */
3484
	devices = &fs_info->fs_devices->devices;
Q
Qinghuang Feng 已提交
3485
	list_for_each_entry(device, devices, dev_list) {
3486
		old_size = btrfs_device_get_total_bytes(device);
3487
		size_to_free = div_factor(old_size, 1);
3488
		size_to_free = min_t(u64, size_to_free, SZ_1M);
Y
Yan Zheng 已提交
3489
		if (!device->writeable ||
3490 3491
		    btrfs_device_get_total_bytes(device) -
		    btrfs_device_get_bytes_used(device) > size_to_free ||
3492
		    device->is_tgtdev_for_dev_replace)
3493 3494 3495
			continue;

		ret = btrfs_shrink_device(device, old_size - size_to_free);
3496 3497
		if (ret == -ENOSPC)
			break;
3498 3499 3500 3501 3502
		if (ret) {
			/* btrfs_shrink_device never returns ret > 0 */
			WARN_ON(ret > 0);
			goto error;
		}
3503

3504
		trans = btrfs_start_transaction(dev_root, 0);
3505 3506 3507 3508 3509 3510 3511 3512
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			btrfs_info_in_rcu(fs_info,
		 "resize: unable to start transaction after shrinking device %s (error %d), old size %llu, new size %llu",
					  rcu_str_deref(device->name), ret,
					  old_size, old_size - size_to_free);
			goto error;
		}
3513 3514

		ret = btrfs_grow_device(trans, device, old_size);
3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
		if (ret) {
			btrfs_end_transaction(trans, dev_root);
			/* btrfs_grow_device never returns ret > 0 */
			WARN_ON(ret > 0);
			btrfs_info_in_rcu(fs_info,
		 "resize: unable to grow device after shrinking device %s (error %d), old size %llu, new size %llu",
					  rcu_str_deref(device->name), ret,
					  old_size, old_size - size_to_free);
			goto error;
		}
3525 3526 3527 3528 3529 3530

		btrfs_end_transaction(trans, dev_root);
	}

	/* step two, relocate all the chunks */
	path = btrfs_alloc_path();
3531 3532 3533 3534
	if (!path) {
		ret = -ENOMEM;
		goto error;
	}
3535 3536 3537 3538 3539 3540

	/* zero out stat counters */
	spin_lock(&fs_info->balance_lock);
	memset(&bctl->stat, 0, sizeof(bctl->stat));
	spin_unlock(&fs_info->balance_lock);
again:
3541
	if (!counting) {
3542 3543 3544 3545
		/*
		 * The single value limit and min/max limits use the same bytes
		 * in the
		 */
3546 3547 3548 3549
		bctl->data.limit = limit_data;
		bctl->meta.limit = limit_meta;
		bctl->sys.limit = limit_sys;
	}
3550 3551 3552 3553
	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
	key.offset = (u64)-1;
	key.type = BTRFS_CHUNK_ITEM_KEY;

C
Chris Mason 已提交
3554
	while (1) {
3555
		if ((!counting && atomic_read(&fs_info->balance_pause_req)) ||
3556
		    atomic_read(&fs_info->balance_cancel_req)) {
3557 3558 3559 3560
			ret = -ECANCELED;
			goto error;
		}

3561
		mutex_lock(&fs_info->delete_unused_bgs_mutex);
3562
		ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3563 3564
		if (ret < 0) {
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3565
			goto error;
3566
		}
3567 3568 3569 3570 3571 3572

		/*
		 * this shouldn't happen, it means the last relocate
		 * failed
		 */
		if (ret == 0)
3573
			BUG(); /* FIXME break ? */
3574 3575 3576

		ret = btrfs_previous_item(chunk_root, path, 0,
					  BTRFS_CHUNK_ITEM_KEY);
3577
		if (ret) {
3578
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3579
			ret = 0;
3580
			break;
3581
		}
3582

3583 3584 3585
		leaf = path->nodes[0];
		slot = path->slots[0];
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
3586

3587 3588
		if (found_key.objectid != key.objectid) {
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3589
			break;
3590
		}
3591

3592
		chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
3593
		chunk_type = btrfs_chunk_type(leaf, chunk);
3594

3595 3596 3597 3598 3599 3600
		if (!counting) {
			spin_lock(&fs_info->balance_lock);
			bctl->stat.considered++;
			spin_unlock(&fs_info->balance_lock);
		}

3601
		ret = should_balance_chunk(fs_info, leaf, chunk,
3602
					   found_key.offset);
3603

3604
		btrfs_release_path(path);
3605 3606
		if (!ret) {
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3607
			goto loop;
3608
		}
3609

3610
		if (counting) {
3611
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3612 3613 3614
			spin_lock(&fs_info->balance_lock);
			bctl->stat.expected++;
			spin_unlock(&fs_info->balance_lock);
3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636

			if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
				count_data++;
			else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
				count_sys++;
			else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
				count_meta++;

			goto loop;
		}

		/*
		 * Apply limit_min filter, no need to check if the LIMITS
		 * filter is used, limit_min is 0 by default
		 */
		if (((chunk_type & BTRFS_BLOCK_GROUP_DATA) &&
					count_data < bctl->data.limit_min)
				|| ((chunk_type & BTRFS_BLOCK_GROUP_METADATA) &&
					count_meta < bctl->meta.limit_min)
				|| ((chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) &&
					count_sys < bctl->sys.limit_min)) {
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3637 3638 3639
			goto loop;
		}

3640 3641 3642 3643 3644 3645 3646
		ASSERT(fs_info->data_sinfo);
		spin_lock(&fs_info->data_sinfo->lock);
		bytes_used = fs_info->data_sinfo->bytes_used;
		spin_unlock(&fs_info->data_sinfo->lock);

		if ((chunk_type & BTRFS_BLOCK_GROUP_DATA) &&
		    !chunk_reserved && !bytes_used) {
3647 3648 3649 3650 3651 3652 3653
			trans = btrfs_start_transaction(chunk_root, 0);
			if (IS_ERR(trans)) {
				mutex_unlock(&fs_info->delete_unused_bgs_mutex);
				ret = PTR_ERR(trans);
				goto error;
			}

3654
			ret = btrfs_force_chunk_alloc(trans, fs_info,
3655
						      BTRFS_BLOCK_GROUP_DATA);
3656
			btrfs_end_transaction(trans, chunk_root);
3657 3658 3659 3660 3661 3662 3663
			if (ret < 0) {
				mutex_unlock(&fs_info->delete_unused_bgs_mutex);
				goto error;
			}
			chunk_reserved = 1;
		}

3664
		ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3665
		mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3666 3667
		if (ret && ret != -ENOSPC)
			goto error;
3668
		if (ret == -ENOSPC) {
3669
			enospc_errors++;
3670 3671 3672 3673 3674
		} else {
			spin_lock(&fs_info->balance_lock);
			bctl->stat.completed++;
			spin_unlock(&fs_info->balance_lock);
		}
3675
loop:
3676 3677
		if (found_key.offset == 0)
			break;
3678
		key.offset = found_key.offset - 1;
3679
	}
3680

3681 3682 3683 3684 3685
	if (counting) {
		btrfs_release_path(path);
		counting = false;
		goto again;
	}
3686 3687
error:
	btrfs_free_path(path);
3688
	if (enospc_errors) {
3689
		btrfs_info(fs_info, "%d enospc errors during balance",
J
Jeff Mahoney 已提交
3690
			   enospc_errors);
3691 3692 3693 3694
		if (!ret)
			ret = -ENOSPC;
	}

3695 3696 3697
	return ret;
}

3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721
/**
 * alloc_profile_is_valid - see if a given profile is valid and reduced
 * @flags: profile to validate
 * @extended: if true @flags is treated as an extended profile
 */
static int alloc_profile_is_valid(u64 flags, int extended)
{
	u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK :
			       BTRFS_BLOCK_GROUP_PROFILE_MASK);

	flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK;

	/* 1) check that all other bits are zeroed */
	if (flags & ~mask)
		return 0;

	/* 2) see if profile is reduced */
	if (flags == 0)
		return !extended; /* "0" is valid for usual profiles */

	/* true if exactly one bit set */
	return (flags & (flags - 1)) == 0;
}

3722 3723
static inline int balance_need_close(struct btrfs_fs_info *fs_info)
{
3724 3725 3726 3727
	/* cancel requested || normal exit path */
	return atomic_read(&fs_info->balance_cancel_req) ||
		(atomic_read(&fs_info->balance_pause_req) == 0 &&
		 atomic_read(&fs_info->balance_cancel_req) == 0);
3728 3729
}

3730 3731
static void __cancel_balance(struct btrfs_fs_info *fs_info)
{
3732 3733
	int ret;

3734
	unset_balance_control(fs_info);
3735
	ret = del_balance_item(fs_info);
3736
	if (ret)
3737
		btrfs_handle_fs_error(fs_info, ret, NULL);
3738 3739

	atomic_set(&fs_info->mutually_exclusive_operation_running, 0);
3740 3741
}

3742 3743 3744 3745 3746 3747 3748 3749 3750
/* Non-zero return value signifies invalidity */
static inline int validate_convert_profile(struct btrfs_balance_args *bctl_arg,
		u64 allowed)
{
	return ((bctl_arg->flags & BTRFS_BALANCE_ARGS_CONVERT) &&
		(!alloc_profile_is_valid(bctl_arg->target, 1) ||
		 (bctl_arg->target & ~allowed)));
}

3751 3752 3753 3754 3755 3756 3757
/*
 * Should be called with both balance and volume mutexes held
 */
int btrfs_balance(struct btrfs_balance_control *bctl,
		  struct btrfs_ioctl_balance_args *bargs)
{
	struct btrfs_fs_info *fs_info = bctl->fs_info;
3758
	u64 allowed;
3759
	int mixed = 0;
3760
	int ret;
3761
	u64 num_devices;
3762
	unsigned seq;
3763

3764
	if (btrfs_fs_closing(fs_info) ||
3765 3766
	    atomic_read(&fs_info->balance_pause_req) ||
	    atomic_read(&fs_info->balance_cancel_req)) {
3767 3768 3769 3770
		ret = -EINVAL;
		goto out;
	}

3771 3772 3773 3774
	allowed = btrfs_super_incompat_flags(fs_info->super_copy);
	if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;

3775 3776 3777 3778
	/*
	 * In case of mixed groups both data and meta should be picked,
	 * and identical options should be given for both of them.
	 */
3779 3780
	allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA;
	if (mixed && (bctl->flags & allowed)) {
3781 3782 3783
		if (!(bctl->flags & BTRFS_BALANCE_DATA) ||
		    !(bctl->flags & BTRFS_BALANCE_METADATA) ||
		    memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) {
J
Jeff Mahoney 已提交
3784 3785
			btrfs_err(fs_info,
				  "with mixed groups data and metadata balance options must be the same");
3786 3787 3788 3789 3790
			ret = -EINVAL;
			goto out;
		}
	}

3791
	num_devices = fs_info->fs_devices->num_devices;
3792
	btrfs_dev_replace_lock(&fs_info->dev_replace, 0);
3793 3794 3795 3796
	if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
		BUG_ON(num_devices < 1);
		num_devices--;
	}
3797
	btrfs_dev_replace_unlock(&fs_info->dev_replace, 0);
3798 3799
	allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE | BTRFS_BLOCK_GROUP_DUP;
	if (num_devices > 1)
3800
		allowed |= (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID1);
3801 3802 3803 3804 3805
	if (num_devices > 2)
		allowed |= BTRFS_BLOCK_GROUP_RAID5;
	if (num_devices > 3)
		allowed |= (BTRFS_BLOCK_GROUP_RAID10 |
			    BTRFS_BLOCK_GROUP_RAID6);
3806
	if (validate_convert_profile(&bctl->data, allowed)) {
J
Jeff Mahoney 已提交
3807 3808 3809
		btrfs_err(fs_info,
			  "unable to start balance with target data profile %llu",
			  bctl->data.target);
3810 3811 3812
		ret = -EINVAL;
		goto out;
	}
3813
	if (validate_convert_profile(&bctl->meta, allowed)) {
3814
		btrfs_err(fs_info,
J
Jeff Mahoney 已提交
3815 3816
			  "unable to start balance with target metadata profile %llu",
			  bctl->meta.target);
3817 3818 3819
		ret = -EINVAL;
		goto out;
	}
3820
	if (validate_convert_profile(&bctl->sys, allowed)) {
3821
		btrfs_err(fs_info,
J
Jeff Mahoney 已提交
3822 3823
			  "unable to start balance with target system profile %llu",
			  bctl->sys.target);
3824 3825 3826 3827 3828 3829
		ret = -EINVAL;
		goto out;
	}

	/* allow to reduce meta or sys integrity only if force set */
	allowed = BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
D
David Woodhouse 已提交
3830 3831 3832
			BTRFS_BLOCK_GROUP_RAID10 |
			BTRFS_BLOCK_GROUP_RAID5 |
			BTRFS_BLOCK_GROUP_RAID6;
3833 3834 3835 3836 3837 3838 3839 3840 3841 3842
	do {
		seq = read_seqbegin(&fs_info->profiles_lock);

		if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
		     (fs_info->avail_system_alloc_bits & allowed) &&
		     !(bctl->sys.target & allowed)) ||
		    ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
		     (fs_info->avail_metadata_alloc_bits & allowed) &&
		     !(bctl->meta.target & allowed))) {
			if (bctl->flags & BTRFS_BALANCE_FORCE) {
J
Jeff Mahoney 已提交
3843 3844
				btrfs_info(fs_info,
					   "force reducing metadata integrity");
3845
			} else {
J
Jeff Mahoney 已提交
3846 3847
				btrfs_err(fs_info,
					  "balance will reduce metadata integrity, use force if you want this");
3848 3849 3850
				ret = -EINVAL;
				goto out;
			}
3851
		}
3852
	} while (read_seqretry(&fs_info->profiles_lock, seq));
3853

3854 3855 3856
	if (btrfs_get_num_tolerated_disk_barrier_failures(bctl->meta.target) <
		btrfs_get_num_tolerated_disk_barrier_failures(bctl->data.target)) {
		btrfs_warn(fs_info,
J
Jeff Mahoney 已提交
3857 3858
			   "metadata profile 0x%llx has lower redundancy than data profile 0x%llx",
			   bctl->meta.target, bctl->data.target);
3859 3860
	}

3861
	if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3862 3863 3864 3865
		fs_info->num_tolerated_disk_barrier_failures = min(
			btrfs_calc_num_tolerated_disk_barrier_failures(fs_info),
			btrfs_get_num_tolerated_disk_barrier_failures(
				bctl->sys.target));
3866 3867
	}

3868
	ret = insert_balance_item(fs_info, bctl);
I
Ilya Dryomov 已提交
3869
	if (ret && ret != -EEXIST)
3870 3871
		goto out;

I
Ilya Dryomov 已提交
3872 3873 3874 3875 3876 3877 3878 3879 3880
	if (!(bctl->flags & BTRFS_BALANCE_RESUME)) {
		BUG_ON(ret == -EEXIST);
		set_balance_control(bctl);
	} else {
		BUG_ON(ret != -EEXIST);
		spin_lock(&fs_info->balance_lock);
		update_balance_args(bctl);
		spin_unlock(&fs_info->balance_lock);
	}
3881

3882
	atomic_inc(&fs_info->balance_running);
3883 3884 3885 3886 3887
	mutex_unlock(&fs_info->balance_mutex);

	ret = __btrfs_balance(fs_info);

	mutex_lock(&fs_info->balance_mutex);
3888
	atomic_dec(&fs_info->balance_running);
3889

3890 3891 3892 3893 3894
	if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		fs_info->num_tolerated_disk_barrier_failures =
			btrfs_calc_num_tolerated_disk_barrier_failures(fs_info);
	}

3895 3896
	if (bargs) {
		memset(bargs, 0, sizeof(*bargs));
3897
		update_ioctl_balance_args(fs_info, 0, bargs);
3898 3899
	}

3900 3901 3902 3903 3904
	if ((ret && ret != -ECANCELED && ret != -ENOSPC) ||
	    balance_need_close(fs_info)) {
		__cancel_balance(fs_info);
	}

3905
	wake_up(&fs_info->balance_wait_q);
3906 3907 3908

	return ret;
out:
I
Ilya Dryomov 已提交
3909 3910
	if (bctl->flags & BTRFS_BALANCE_RESUME)
		__cancel_balance(fs_info);
3911
	else {
I
Ilya Dryomov 已提交
3912
		kfree(bctl);
3913 3914
		atomic_set(&fs_info->mutually_exclusive_operation_running, 0);
	}
I
Ilya Dryomov 已提交
3915 3916 3917 3918 3919
	return ret;
}

static int balance_kthread(void *data)
{
3920
	struct btrfs_fs_info *fs_info = data;
3921
	int ret = 0;
I
Ilya Dryomov 已提交
3922 3923 3924 3925

	mutex_lock(&fs_info->volume_mutex);
	mutex_lock(&fs_info->balance_mutex);

3926
	if (fs_info->balance_ctl) {
3927
		btrfs_info(fs_info, "continuing balance");
3928
		ret = btrfs_balance(fs_info->balance_ctl, NULL);
3929
	}
I
Ilya Dryomov 已提交
3930 3931 3932

	mutex_unlock(&fs_info->balance_mutex);
	mutex_unlock(&fs_info->volume_mutex);
3933

I
Ilya Dryomov 已提交
3934 3935 3936
	return ret;
}

3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947
int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info)
{
	struct task_struct *tsk;

	spin_lock(&fs_info->balance_lock);
	if (!fs_info->balance_ctl) {
		spin_unlock(&fs_info->balance_lock);
		return 0;
	}
	spin_unlock(&fs_info->balance_lock);

3948
	if (btrfs_test_opt(fs_info, SKIP_BALANCE)) {
3949
		btrfs_info(fs_info, "force skipping balance");
3950 3951 3952 3953
		return 0;
	}

	tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance");
3954
	return PTR_ERR_OR_ZERO(tsk);
3955 3956
}

3957
int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
I
Ilya Dryomov 已提交
3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971
{
	struct btrfs_balance_control *bctl;
	struct btrfs_balance_item *item;
	struct btrfs_disk_balance_args disk_bargs;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = BTRFS_BALANCE_OBJECTID;
3972
	key.type = BTRFS_TEMPORARY_ITEM_KEY;
I
Ilya Dryomov 已提交
3973 3974
	key.offset = 0;

3975
	ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
I
Ilya Dryomov 已提交
3976
	if (ret < 0)
3977
		goto out;
I
Ilya Dryomov 已提交
3978 3979
	if (ret > 0) { /* ret = -ENOENT; */
		ret = 0;
3980 3981 3982 3983 3984 3985 3986
		goto out;
	}

	bctl = kzalloc(sizeof(*bctl), GFP_NOFS);
	if (!bctl) {
		ret = -ENOMEM;
		goto out;
I
Ilya Dryomov 已提交
3987 3988 3989 3990 3991
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);

3992 3993 3994
	bctl->fs_info = fs_info;
	bctl->flags = btrfs_balance_flags(leaf, item);
	bctl->flags |= BTRFS_BALANCE_RESUME;
I
Ilya Dryomov 已提交
3995 3996 3997 3998 3999 4000 4001 4002

	btrfs_balance_data(leaf, item, &disk_bargs);
	btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs);
	btrfs_balance_meta(leaf, item, &disk_bargs);
	btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs);
	btrfs_balance_sys(leaf, item, &disk_bargs);
	btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs);

4003 4004
	WARN_ON(atomic_xchg(&fs_info->mutually_exclusive_operation_running, 1));

4005 4006
	mutex_lock(&fs_info->volume_mutex);
	mutex_lock(&fs_info->balance_mutex);
I
Ilya Dryomov 已提交
4007

4008 4009 4010 4011
	set_balance_control(bctl);

	mutex_unlock(&fs_info->balance_mutex);
	mutex_unlock(&fs_info->volume_mutex);
I
Ilya Dryomov 已提交
4012 4013
out:
	btrfs_free_path(path);
4014 4015 4016
	return ret;
}

4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045
int btrfs_pause_balance(struct btrfs_fs_info *fs_info)
{
	int ret = 0;

	mutex_lock(&fs_info->balance_mutex);
	if (!fs_info->balance_ctl) {
		mutex_unlock(&fs_info->balance_mutex);
		return -ENOTCONN;
	}

	if (atomic_read(&fs_info->balance_running)) {
		atomic_inc(&fs_info->balance_pause_req);
		mutex_unlock(&fs_info->balance_mutex);

		wait_event(fs_info->balance_wait_q,
			   atomic_read(&fs_info->balance_running) == 0);

		mutex_lock(&fs_info->balance_mutex);
		/* we are good with balance_ctl ripped off from under us */
		BUG_ON(atomic_read(&fs_info->balance_running));
		atomic_dec(&fs_info->balance_pause_req);
	} else {
		ret = -ENOTCONN;
	}

	mutex_unlock(&fs_info->balance_mutex);
	return ret;
}

4046 4047
int btrfs_cancel_balance(struct btrfs_fs_info *fs_info)
{
4048 4049 4050
	if (fs_info->sb->s_flags & MS_RDONLY)
		return -EROFS;

4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084
	mutex_lock(&fs_info->balance_mutex);
	if (!fs_info->balance_ctl) {
		mutex_unlock(&fs_info->balance_mutex);
		return -ENOTCONN;
	}

	atomic_inc(&fs_info->balance_cancel_req);
	/*
	 * if we are running just wait and return, balance item is
	 * deleted in btrfs_balance in this case
	 */
	if (atomic_read(&fs_info->balance_running)) {
		mutex_unlock(&fs_info->balance_mutex);
		wait_event(fs_info->balance_wait_q,
			   atomic_read(&fs_info->balance_running) == 0);
		mutex_lock(&fs_info->balance_mutex);
	} else {
		/* __cancel_balance needs volume_mutex */
		mutex_unlock(&fs_info->balance_mutex);
		mutex_lock(&fs_info->volume_mutex);
		mutex_lock(&fs_info->balance_mutex);

		if (fs_info->balance_ctl)
			__cancel_balance(fs_info);

		mutex_unlock(&fs_info->volume_mutex);
	}

	BUG_ON(fs_info->balance_ctl || atomic_read(&fs_info->balance_running));
	atomic_dec(&fs_info->balance_cancel_req);
	mutex_unlock(&fs_info->balance_mutex);
	return 0;
}

S
Stefan Behrens 已提交
4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096
static int btrfs_uuid_scan_kthread(void *data)
{
	struct btrfs_fs_info *fs_info = data;
	struct btrfs_root *root = fs_info->tree_root;
	struct btrfs_key key;
	struct btrfs_key max_key;
	struct btrfs_path *path = NULL;
	int ret = 0;
	struct extent_buffer *eb;
	int slot;
	struct btrfs_root_item root_item;
	u32 item_size;
4097
	struct btrfs_trans_handle *trans = NULL;
S
Stefan Behrens 已提交
4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113

	path = btrfs_alloc_path();
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}

	key.objectid = 0;
	key.type = BTRFS_ROOT_ITEM_KEY;
	key.offset = 0;

	max_key.objectid = (u64)-1;
	max_key.type = BTRFS_ROOT_ITEM_KEY;
	max_key.offset = (u64)-1;

	while (1) {
4114
		ret = btrfs_search_forward(root, &key, path, 0);
S
Stefan Behrens 已提交
4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137
		if (ret) {
			if (ret > 0)
				ret = 0;
			break;
		}

		if (key.type != BTRFS_ROOT_ITEM_KEY ||
		    (key.objectid < BTRFS_FIRST_FREE_OBJECTID &&
		     key.objectid != BTRFS_FS_TREE_OBJECTID) ||
		    key.objectid > BTRFS_LAST_FREE_OBJECTID)
			goto skip;

		eb = path->nodes[0];
		slot = path->slots[0];
		item_size = btrfs_item_size_nr(eb, slot);
		if (item_size < sizeof(root_item))
			goto skip;

		read_extent_buffer(eb, &root_item,
				   btrfs_item_ptr_offset(eb, slot),
				   (int)sizeof(root_item));
		if (btrfs_root_refs(&root_item) == 0)
			goto skip;
4138 4139 4140 4141 4142 4143 4144

		if (!btrfs_is_empty_uuid(root_item.uuid) ||
		    !btrfs_is_empty_uuid(root_item.received_uuid)) {
			if (trans)
				goto update_tree;

			btrfs_release_path(path);
S
Stefan Behrens 已提交
4145 4146 4147 4148 4149 4150 4151 4152 4153
			/*
			 * 1 - subvol uuid item
			 * 1 - received_subvol uuid item
			 */
			trans = btrfs_start_transaction(fs_info->uuid_root, 2);
			if (IS_ERR(trans)) {
				ret = PTR_ERR(trans);
				break;
			}
4154 4155 4156 4157 4158 4159
			continue;
		} else {
			goto skip;
		}
update_tree:
		if (!btrfs_is_empty_uuid(root_item.uuid)) {
4160
			ret = btrfs_uuid_tree_add(trans, fs_info,
S
Stefan Behrens 已提交
4161 4162 4163 4164
						  root_item.uuid,
						  BTRFS_UUID_KEY_SUBVOL,
						  key.objectid);
			if (ret < 0) {
4165
				btrfs_warn(fs_info, "uuid_tree_add failed %d",
S
Stefan Behrens 已提交
4166 4167 4168 4169 4170 4171
					ret);
				break;
			}
		}

		if (!btrfs_is_empty_uuid(root_item.received_uuid)) {
4172
			ret = btrfs_uuid_tree_add(trans, fs_info,
S
Stefan Behrens 已提交
4173 4174 4175 4176
						  root_item.received_uuid,
						 BTRFS_UUID_KEY_RECEIVED_SUBVOL,
						  key.objectid);
			if (ret < 0) {
4177
				btrfs_warn(fs_info, "uuid_tree_add failed %d",
S
Stefan Behrens 已提交
4178 4179 4180 4181 4182
					ret);
				break;
			}
		}

4183
skip:
S
Stefan Behrens 已提交
4184 4185
		if (trans) {
			ret = btrfs_end_transaction(trans, fs_info->uuid_root);
4186
			trans = NULL;
S
Stefan Behrens 已提交
4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208
			if (ret)
				break;
		}

		btrfs_release_path(path);
		if (key.offset < (u64)-1) {
			key.offset++;
		} else if (key.type < BTRFS_ROOT_ITEM_KEY) {
			key.offset = 0;
			key.type = BTRFS_ROOT_ITEM_KEY;
		} else if (key.objectid < (u64)-1) {
			key.offset = 0;
			key.type = BTRFS_ROOT_ITEM_KEY;
			key.objectid++;
		} else {
			break;
		}
		cond_resched();
	}

out:
	btrfs_free_path(path);
4209 4210
	if (trans && !IS_ERR(trans))
		btrfs_end_transaction(trans, fs_info->uuid_root);
S
Stefan Behrens 已提交
4211
	if (ret)
4212
		btrfs_warn(fs_info, "btrfs_uuid_scan_kthread failed %d", ret);
4213
	else
4214
		set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
S
Stefan Behrens 已提交
4215 4216 4217 4218
	up(&fs_info->uuid_tree_rescan_sem);
	return 0;
}

4219 4220 4221 4222
/*
 * Callback for btrfs_uuid_tree_iterate().
 * returns:
 * 0	check succeeded, the entry is not outdated.
4223
 * < 0	if an error occurred.
4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275
 * > 0	if the check failed, which means the caller shall remove the entry.
 */
static int btrfs_check_uuid_tree_entry(struct btrfs_fs_info *fs_info,
				       u8 *uuid, u8 type, u64 subid)
{
	struct btrfs_key key;
	int ret = 0;
	struct btrfs_root *subvol_root;

	if (type != BTRFS_UUID_KEY_SUBVOL &&
	    type != BTRFS_UUID_KEY_RECEIVED_SUBVOL)
		goto out;

	key.objectid = subid;
	key.type = BTRFS_ROOT_ITEM_KEY;
	key.offset = (u64)-1;
	subvol_root = btrfs_read_fs_root_no_name(fs_info, &key);
	if (IS_ERR(subvol_root)) {
		ret = PTR_ERR(subvol_root);
		if (ret == -ENOENT)
			ret = 1;
		goto out;
	}

	switch (type) {
	case BTRFS_UUID_KEY_SUBVOL:
		if (memcmp(uuid, subvol_root->root_item.uuid, BTRFS_UUID_SIZE))
			ret = 1;
		break;
	case BTRFS_UUID_KEY_RECEIVED_SUBVOL:
		if (memcmp(uuid, subvol_root->root_item.received_uuid,
			   BTRFS_UUID_SIZE))
			ret = 1;
		break;
	}

out:
	return ret;
}

static int btrfs_uuid_rescan_kthread(void *data)
{
	struct btrfs_fs_info *fs_info = (struct btrfs_fs_info *)data;
	int ret;

	/*
	 * 1st step is to iterate through the existing UUID tree and
	 * to delete all entries that contain outdated data.
	 * 2nd step is to add all missing entries to the UUID tree.
	 */
	ret = btrfs_uuid_tree_iterate(fs_info, btrfs_check_uuid_tree_entry);
	if (ret < 0) {
4276
		btrfs_warn(fs_info, "iterating uuid_tree failed %d", ret);
4277 4278 4279 4280 4281 4282
		up(&fs_info->uuid_tree_rescan_sem);
		return ret;
	}
	return btrfs_uuid_scan_kthread(data);
}

4283 4284 4285 4286 4287
int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info)
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = fs_info->tree_root;
	struct btrfs_root *uuid_root;
S
Stefan Behrens 已提交
4288 4289
	struct task_struct *task;
	int ret;
4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301

	/*
	 * 1 - root node
	 * 1 - root item
	 */
	trans = btrfs_start_transaction(tree_root, 2);
	if (IS_ERR(trans))
		return PTR_ERR(trans);

	uuid_root = btrfs_create_tree(trans, fs_info,
				      BTRFS_UUID_TREE_OBJECTID);
	if (IS_ERR(uuid_root)) {
4302
		ret = PTR_ERR(uuid_root);
4303
		btrfs_abort_transaction(trans, ret);
4304
		btrfs_end_transaction(trans, tree_root);
4305
		return ret;
4306 4307 4308 4309
	}

	fs_info->uuid_root = uuid_root;

S
Stefan Behrens 已提交
4310 4311 4312 4313 4314 4315 4316
	ret = btrfs_commit_transaction(trans, tree_root);
	if (ret)
		return ret;

	down(&fs_info->uuid_tree_rescan_sem);
	task = kthread_run(btrfs_uuid_scan_kthread, fs_info, "btrfs-uuid");
	if (IS_ERR(task)) {
4317
		/* fs_info->update_uuid_tree_gen remains 0 in all error case */
4318
		btrfs_warn(fs_info, "failed to start uuid_scan task");
S
Stefan Behrens 已提交
4319 4320 4321 4322 4323
		up(&fs_info->uuid_tree_rescan_sem);
		return PTR_ERR(task);
	}

	return 0;
4324
}
S
Stefan Behrens 已提交
4325

4326 4327 4328 4329 4330 4331 4332 4333
int btrfs_check_uuid_tree(struct btrfs_fs_info *fs_info)
{
	struct task_struct *task;

	down(&fs_info->uuid_tree_rescan_sem);
	task = kthread_run(btrfs_uuid_rescan_kthread, fs_info, "btrfs-uuid");
	if (IS_ERR(task)) {
		/* fs_info->update_uuid_tree_gen remains 0 in all error case */
4334
		btrfs_warn(fs_info, "failed to start uuid_rescan task");
4335 4336 4337 4338 4339 4340 4341
		up(&fs_info->uuid_tree_rescan_sem);
		return PTR_ERR(task);
	}

	return 0;
}

4342 4343 4344 4345 4346 4347 4348
/*
 * shrinking a device means finding all of the device extents past
 * the new size, and then following the back refs to the chunks.
 * The chunk relocation code actually frees the device extent
 */
int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
{
4349 4350
	struct btrfs_fs_info *fs_info = device->fs_info;
	struct btrfs_root *root = fs_info->dev_root;
4351 4352 4353 4354 4355 4356 4357
	struct btrfs_trans_handle *trans;
	struct btrfs_dev_extent *dev_extent = NULL;
	struct btrfs_path *path;
	u64 length;
	u64 chunk_offset;
	int ret;
	int slot;
4358 4359
	int failed = 0;
	bool retried = false;
4360
	bool checked_pending_chunks = false;
4361 4362
	struct extent_buffer *l;
	struct btrfs_key key;
4363
	struct btrfs_super_block *super_copy = fs_info->super_copy;
4364
	u64 old_total = btrfs_super_total_bytes(super_copy);
4365 4366
	u64 old_size = btrfs_device_get_total_bytes(device);
	u64 diff = old_size - new_size;
4367

4368 4369 4370
	if (device->is_tgtdev_for_dev_replace)
		return -EINVAL;

4371 4372 4373 4374
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

4375
	path->reada = READA_FORWARD;
4376

4377
	lock_chunks(fs_info);
4378

4379
	btrfs_device_set_total_bytes(device, new_size);
4380
	if (device->writeable) {
Y
Yan Zheng 已提交
4381
		device->fs_devices->total_rw_bytes -= diff;
4382 4383 4384
		spin_lock(&fs_info->free_chunk_lock);
		fs_info->free_chunk_space -= diff;
		spin_unlock(&fs_info->free_chunk_lock);
4385
	}
4386
	unlock_chunks(fs_info);
4387

4388
again:
4389 4390 4391 4392
	key.objectid = device->devid;
	key.offset = (u64)-1;
	key.type = BTRFS_DEV_EXTENT_KEY;

4393
	do {
4394
		mutex_lock(&fs_info->delete_unused_bgs_mutex);
4395
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
4396
		if (ret < 0) {
4397
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4398
			goto done;
4399
		}
4400 4401

		ret = btrfs_previous_item(root, path, 0, key.type);
4402
		if (ret)
4403
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4404 4405 4406 4407
		if (ret < 0)
			goto done;
		if (ret) {
			ret = 0;
4408
			btrfs_release_path(path);
4409
			break;
4410 4411 4412 4413 4414 4415
		}

		l = path->nodes[0];
		slot = path->slots[0];
		btrfs_item_key_to_cpu(l, &key, path->slots[0]);

4416
		if (key.objectid != device->devid) {
4417
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4418
			btrfs_release_path(path);
4419
			break;
4420
		}
4421 4422 4423 4424

		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
		length = btrfs_dev_extent_length(l, dev_extent);

4425
		if (key.offset + length <= new_size) {
4426
			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4427
			btrfs_release_path(path);
4428
			break;
4429
		}
4430 4431

		chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
4432
		btrfs_release_path(path);
4433

4434 4435
		ret = btrfs_relocate_chunk(fs_info, chunk_offset);
		mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4436
		if (ret && ret != -ENOSPC)
4437
			goto done;
4438 4439
		if (ret == -ENOSPC)
			failed++;
4440
	} while (key.offset-- > 0);
4441 4442 4443 4444 4445 4446 4447 4448

	if (failed && !retried) {
		failed = 0;
		retried = true;
		goto again;
	} else if (failed && retried) {
		ret = -ENOSPC;
		goto done;
4449 4450
	}

4451
	/* Shrinking succeeded, else we would be at "done". */
4452
	trans = btrfs_start_transaction(root, 0);
4453 4454 4455 4456 4457
	if (IS_ERR(trans)) {
		ret = PTR_ERR(trans);
		goto done;
	}

4458
	lock_chunks(fs_info);
4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475

	/*
	 * We checked in the above loop all device extents that were already in
	 * the device tree. However before we have updated the device's
	 * total_bytes to the new size, we might have had chunk allocations that
	 * have not complete yet (new block groups attached to transaction
	 * handles), and therefore their device extents were not yet in the
	 * device tree and we missed them in the loop above. So if we have any
	 * pending chunk using a device extent that overlaps the device range
	 * that we can not use anymore, commit the current transaction and
	 * repeat the search on the device tree - this way we guarantee we will
	 * not have chunks using device extents that end beyond 'new_size'.
	 */
	if (!checked_pending_chunks) {
		u64 start = new_size;
		u64 len = old_size - new_size;

4476 4477
		if (contains_pending_extent(trans->transaction, device,
					    &start, len)) {
4478
			unlock_chunks(fs_info);
4479 4480 4481 4482 4483 4484 4485 4486 4487 4488
			checked_pending_chunks = true;
			failed = 0;
			retried = false;
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				goto done;
			goto again;
		}
	}

4489
	btrfs_device_set_disk_total_bytes(device, new_size);
4490 4491
	if (list_empty(&device->resized_list))
		list_add_tail(&device->resized_list,
4492
			      &fs_info->fs_devices->resized_devices);
4493 4494 4495

	WARN_ON(diff > old_total);
	btrfs_set_super_total_bytes(super_copy, old_total - diff);
4496
	unlock_chunks(fs_info);
M
Miao Xie 已提交
4497 4498 4499

	/* Now btrfs_update_device() will change the on-disk size. */
	ret = btrfs_update_device(trans, device);
4500
	btrfs_end_transaction(trans, root);
4501 4502
done:
	btrfs_free_path(path);
4503
	if (ret) {
4504
		lock_chunks(fs_info);
4505 4506 4507
		btrfs_device_set_total_bytes(device, old_size);
		if (device->writeable)
			device->fs_devices->total_rw_bytes += diff;
4508 4509 4510 4511
		spin_lock(&fs_info->free_chunk_lock);
		fs_info->free_chunk_space += diff;
		spin_unlock(&fs_info->free_chunk_lock);
		unlock_chunks(fs_info);
4512
	}
4513 4514 4515
	return ret;
}

4516
static int btrfs_add_system_chunk(struct btrfs_fs_info *fs_info,
4517 4518 4519
			   struct btrfs_key *key,
			   struct btrfs_chunk *chunk, int item_size)
{
4520
	struct btrfs_super_block *super_copy = fs_info->super_copy;
4521 4522 4523 4524
	struct btrfs_disk_key disk_key;
	u32 array_size;
	u8 *ptr;

4525
	lock_chunks(fs_info);
4526
	array_size = btrfs_super_sys_array_size(super_copy);
4527
	if (array_size + item_size + sizeof(disk_key)
4528
			> BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
4529
		unlock_chunks(fs_info);
4530
		return -EFBIG;
4531
	}
4532 4533 4534 4535 4536 4537 4538 4539

	ptr = super_copy->sys_chunk_array + array_size;
	btrfs_cpu_key_to_disk(&disk_key, key);
	memcpy(ptr, &disk_key, sizeof(disk_key));
	ptr += sizeof(disk_key);
	memcpy(ptr, chunk, item_size);
	item_size += sizeof(disk_key);
	btrfs_set_super_sys_array_size(super_copy, array_size + item_size);
4540
	unlock_chunks(fs_info);
4541

4542 4543 4544
	return 0;
}

4545 4546 4547 4548
/*
 * sort the devices in descending order by max_avail, total_avail
 */
static int btrfs_cmp_device_info(const void *a, const void *b)
4549
{
4550 4551
	const struct btrfs_device_info *di_a = a;
	const struct btrfs_device_info *di_b = b;
4552

4553
	if (di_a->max_avail > di_b->max_avail)
4554
		return -1;
4555
	if (di_a->max_avail < di_b->max_avail)
4556
		return 1;
4557 4558 4559 4560 4561
	if (di_a->total_avail > di_b->total_avail)
		return -1;
	if (di_a->total_avail < di_b->total_avail)
		return 1;
	return 0;
4562
}
4563

D
David Woodhouse 已提交
4564 4565 4566
static u32 find_raid56_stripe_len(u32 data_devices, u32 dev_stripe_target)
{
	/* TODO allow them to set a preferred stripe size */
4567
	return SZ_64K;
D
David Woodhouse 已提交
4568 4569 4570 4571
}

static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
{
4572
	if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
D
David Woodhouse 已提交
4573 4574
		return;

4575
	btrfs_set_fs_incompat(info, RAID56);
D
David Woodhouse 已提交
4576 4577
}

4578
#define BTRFS_MAX_DEVS(r) ((BTRFS_MAX_ITEM_SIZE(r->fs_info)		\
4579 4580 4581 4582 4583 4584 4585 4586
			- sizeof(struct btrfs_chunk))		\
			/ sizeof(struct btrfs_stripe) + 1)

#define BTRFS_MAX_DEVS_SYS_CHUNK ((BTRFS_SYSTEM_CHUNK_ARRAY_SIZE	\
				- 2 * sizeof(struct btrfs_disk_key)	\
				- 2 * sizeof(struct btrfs_chunk))	\
				/ sizeof(struct btrfs_stripe) + 1)

4587
static int __btrfs_alloc_chunk(struct btrfs_trans_handle *trans,
4588
			       struct btrfs_fs_info *fs_info, u64 start,
4589
			       u64 type)
4590
{
4591
	struct btrfs_fs_info *info = trans->fs_info;
4592 4593 4594 4595 4596 4597 4598 4599
	struct btrfs_fs_devices *fs_devices = info->fs_devices;
	struct list_head *cur;
	struct map_lookup *map = NULL;
	struct extent_map_tree *em_tree;
	struct extent_map *em;
	struct btrfs_device_info *devices_info = NULL;
	u64 total_avail;
	int num_stripes;	/* total number of stripes to allocate */
D
David Woodhouse 已提交
4600 4601
	int data_stripes;	/* number of stripes that count for
				   block group size */
4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612
	int sub_stripes;	/* sub_stripes info for map */
	int dev_stripes;	/* stripes per dev */
	int devs_max;		/* max devs to use */
	int devs_min;		/* min devs needed */
	int devs_increment;	/* ndevs has to be a multiple of this */
	int ncopies;		/* how many copies to data has */
	int ret;
	u64 max_stripe_size;
	u64 max_chunk_size;
	u64 stripe_size;
	u64 num_bytes;
D
David Woodhouse 已提交
4613
	u64 raid_stripe_len = BTRFS_STRIPE_LEN;
4614 4615 4616
	int ndevs;
	int i;
	int j;
4617
	int index;
4618

4619
	BUG_ON(!alloc_profile_is_valid(type, 0));
4620

4621 4622
	if (list_empty(&fs_devices->alloc_list))
		return -ENOSPC;
4623

4624
	index = __get_raid_index(type);
4625

4626 4627 4628 4629 4630 4631
	sub_stripes = btrfs_raid_array[index].sub_stripes;
	dev_stripes = btrfs_raid_array[index].dev_stripes;
	devs_max = btrfs_raid_array[index].devs_max;
	devs_min = btrfs_raid_array[index].devs_min;
	devs_increment = btrfs_raid_array[index].devs_increment;
	ncopies = btrfs_raid_array[index].ncopies;
4632

4633
	if (type & BTRFS_BLOCK_GROUP_DATA) {
4634
		max_stripe_size = SZ_1G;
4635
		max_chunk_size = 10 * max_stripe_size;
4636 4637
		if (!devs_max)
			devs_max = BTRFS_MAX_DEVS(info->chunk_root);
4638
	} else if (type & BTRFS_BLOCK_GROUP_METADATA) {
4639
		/* for larger filesystems, use larger metadata chunks */
4640 4641
		if (fs_devices->total_rw_bytes > 50ULL * SZ_1G)
			max_stripe_size = SZ_1G;
4642
		else
4643
			max_stripe_size = SZ_256M;
4644
		max_chunk_size = max_stripe_size;
4645 4646
		if (!devs_max)
			devs_max = BTRFS_MAX_DEVS(info->chunk_root);
4647
	} else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
4648
		max_stripe_size = SZ_32M;
4649
		max_chunk_size = 2 * max_stripe_size;
4650 4651
		if (!devs_max)
			devs_max = BTRFS_MAX_DEVS_SYS_CHUNK;
4652
	} else {
4653
		btrfs_err(info, "invalid chunk type 0x%llx requested",
4654 4655
		       type);
		BUG_ON(1);
4656 4657
	}

Y
Yan Zheng 已提交
4658 4659 4660
	/* we don't want a chunk larger than 10% of writeable space */
	max_chunk_size = min(div_factor(fs_devices->total_rw_bytes, 1),
			     max_chunk_size);
4661

4662
	devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info),
4663 4664 4665
			       GFP_NOFS);
	if (!devices_info)
		return -ENOMEM;
4666

4667
	cur = fs_devices->alloc_list.next;
4668

4669
	/*
4670 4671
	 * in the first pass through the devices list, we gather information
	 * about the available holes on each device.
4672
	 */
4673 4674 4675 4676 4677
	ndevs = 0;
	while (cur != &fs_devices->alloc_list) {
		struct btrfs_device *device;
		u64 max_avail;
		u64 dev_offset;
4678

4679
		device = list_entry(cur, struct btrfs_device, dev_alloc_list);
4680

4681
		cur = cur->next;
4682

4683
		if (!device->writeable) {
J
Julia Lawall 已提交
4684
			WARN(1, KERN_ERR
4685
			       "BTRFS: read-only device in alloc_list\n");
4686 4687
			continue;
		}
4688

4689 4690
		if (!device->in_fs_metadata ||
		    device->is_tgtdev_for_dev_replace)
4691
			continue;
4692

4693 4694 4695 4696
		if (device->total_bytes > device->bytes_used)
			total_avail = device->total_bytes - device->bytes_used;
		else
			total_avail = 0;
4697 4698 4699 4700

		/* If there is no space on this device, skip it. */
		if (total_avail == 0)
			continue;
4701

4702
		ret = find_free_dev_extent(trans, device,
4703 4704 4705 4706
					   max_stripe_size * dev_stripes,
					   &dev_offset, &max_avail);
		if (ret && ret != -ENOSPC)
			goto error;
4707

4708 4709
		if (ret == 0)
			max_avail = max_stripe_size * dev_stripes;
4710

4711 4712
		if (max_avail < BTRFS_STRIPE_LEN * dev_stripes)
			continue;
4713

4714 4715 4716 4717 4718
		if (ndevs == fs_devices->rw_devices) {
			WARN(1, "%s: found more than %llu devices\n",
			     __func__, fs_devices->rw_devices);
			break;
		}
4719 4720 4721 4722 4723 4724
		devices_info[ndevs].dev_offset = dev_offset;
		devices_info[ndevs].max_avail = max_avail;
		devices_info[ndevs].total_avail = total_avail;
		devices_info[ndevs].dev = device;
		++ndevs;
	}
4725

4726 4727 4728 4729 4730
	/*
	 * now sort the devices by hole size / available space
	 */
	sort(devices_info, ndevs, sizeof(struct btrfs_device_info),
	     btrfs_cmp_device_info, NULL);
4731

4732 4733
	/* round down to number of usable stripes */
	ndevs -= ndevs % devs_increment;
4734

4735 4736 4737
	if (ndevs < devs_increment * sub_stripes || ndevs < devs_min) {
		ret = -ENOSPC;
		goto error;
4738
	}
4739

4740 4741 4742 4743 4744 4745 4746 4747
	if (devs_max && ndevs > devs_max)
		ndevs = devs_max;
	/*
	 * the primary goal is to maximize the number of stripes, so use as many
	 * devices as possible, even if the stripes are not maximum sized.
	 */
	stripe_size = devices_info[ndevs-1].max_avail;
	num_stripes = ndevs * dev_stripes;
4748

D
David Woodhouse 已提交
4749 4750 4751 4752 4753 4754 4755 4756
	/*
	 * this will have to be fixed for RAID1 and RAID10 over
	 * more drives
	 */
	data_stripes = num_stripes / ncopies;

	if (type & BTRFS_BLOCK_GROUP_RAID5) {
		raid_stripe_len = find_raid56_stripe_len(ndevs - 1,
4757
							 info->stripesize);
D
David Woodhouse 已提交
4758 4759 4760 4761
		data_stripes = num_stripes - 1;
	}
	if (type & BTRFS_BLOCK_GROUP_RAID6) {
		raid_stripe_len = find_raid56_stripe_len(ndevs - 2,
4762
							 info->stripesize);
D
David Woodhouse 已提交
4763 4764
		data_stripes = num_stripes - 2;
	}
4765 4766 4767 4768 4769 4770 4771 4772

	/*
	 * Use the number of data stripes to figure out how big this chunk
	 * is really going to be in terms of logical address space,
	 * and compare that answer with the max chunk size
	 */
	if (stripe_size * data_stripes > max_chunk_size) {
		u64 mask = (1ULL << 24) - 1;
4773 4774

		stripe_size = div_u64(max_chunk_size, data_stripes);
4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785

		/* bump the answer up to a 16MB boundary */
		stripe_size = (stripe_size + mask) & ~mask;

		/* but don't go higher than the limits we found
		 * while searching for free extents
		 */
		if (stripe_size > devices_info[ndevs-1].max_avail)
			stripe_size = devices_info[ndevs-1].max_avail;
	}

4786
	stripe_size = div_u64(stripe_size, dev_stripes);
4787 4788

	/* align to BTRFS_STRIPE_LEN */
4789
	stripe_size = div_u64(stripe_size, raid_stripe_len);
D
David Woodhouse 已提交
4790
	stripe_size *= raid_stripe_len;
4791 4792 4793 4794 4795 4796 4797

	map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
	if (!map) {
		ret = -ENOMEM;
		goto error;
	}
	map->num_stripes = num_stripes;
4798

4799 4800 4801 4802 4803 4804
	for (i = 0; i < ndevs; ++i) {
		for (j = 0; j < dev_stripes; ++j) {
			int s = i * dev_stripes + j;
			map->stripes[s].dev = devices_info[i].dev;
			map->stripes[s].physical = devices_info[i].dev_offset +
						   j * stripe_size;
4805 4806
		}
	}
4807
	map->sector_size = info->sectorsize;
D
David Woodhouse 已提交
4808 4809 4810
	map->stripe_len = raid_stripe_len;
	map->io_align = raid_stripe_len;
	map->io_width = raid_stripe_len;
Y
Yan Zheng 已提交
4811 4812
	map->type = type;
	map->sub_stripes = sub_stripes;
4813

D
David Woodhouse 已提交
4814
	num_bytes = stripe_size * data_stripes;
4815

4816
	trace_btrfs_chunk_alloc(info, map, start, num_bytes);
4817

4818
	em = alloc_extent_map();
Y
Yan Zheng 已提交
4819
	if (!em) {
4820
		kfree(map);
4821 4822
		ret = -ENOMEM;
		goto error;
4823
	}
4824
	set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
4825
	em->map_lookup = map;
Y
Yan Zheng 已提交
4826
	em->start = start;
4827
	em->len = num_bytes;
Y
Yan Zheng 已提交
4828 4829
	em->block_start = 0;
	em->block_len = em->len;
4830
	em->orig_block_len = stripe_size;
4831

4832
	em_tree = &info->mapping_tree.map_tree;
4833
	write_lock(&em_tree->lock);
J
Josef Bacik 已提交
4834
	ret = add_extent_mapping(em_tree, em, 0);
4835 4836 4837 4838
	if (!ret) {
		list_add_tail(&em->list, &trans->transaction->pending_chunks);
		atomic_inc(&em->refs);
	}
4839
	write_unlock(&em_tree->lock);
4840 4841
	if (ret) {
		free_extent_map(em);
4842
		goto error;
4843
	}
4844

4845
	ret = btrfs_make_block_group(trans, info, 0, type,
4846 4847
				     BTRFS_FIRST_CHUNK_TREE_OBJECTID,
				     start, num_bytes);
4848 4849
	if (ret)
		goto error_del_extent;
Y
Yan Zheng 已提交
4850

4851 4852 4853 4854
	for (i = 0; i < map->num_stripes; i++) {
		num_bytes = map->stripes[i].dev->bytes_used + stripe_size;
		btrfs_device_set_bytes_used(map->stripes[i].dev, num_bytes);
	}
4855

4856 4857 4858
	spin_lock(&info->free_chunk_lock);
	info->free_chunk_space -= (stripe_size * map->num_stripes);
	spin_unlock(&info->free_chunk_lock);
4859

4860
	free_extent_map(em);
4861
	check_raid56_incompat_flag(info, type);
D
David Woodhouse 已提交
4862

4863
	kfree(devices_info);
Y
Yan Zheng 已提交
4864
	return 0;
4865

4866
error_del_extent:
4867 4868 4869 4870 4871 4872 4873 4874
	write_lock(&em_tree->lock);
	remove_extent_mapping(em_tree, em);
	write_unlock(&em_tree->lock);

	/* One for our allocation */
	free_extent_map(em);
	/* One for the tree reference */
	free_extent_map(em);
4875 4876
	/* One for the pending_chunks list reference */
	free_extent_map(em);
4877 4878 4879
error:
	kfree(devices_info);
	return ret;
Y
Yan Zheng 已提交
4880 4881
}

4882
int btrfs_finish_chunk_alloc(struct btrfs_trans_handle *trans,
4883
				struct btrfs_fs_info *fs_info,
4884
				u64 chunk_offset, u64 chunk_size)
Y
Yan Zheng 已提交
4885
{
4886 4887
	struct btrfs_root *extent_root = fs_info->extent_root;
	struct btrfs_root *chunk_root = fs_info->chunk_root;
Y
Yan Zheng 已提交
4888 4889 4890 4891
	struct btrfs_key key;
	struct btrfs_device *device;
	struct btrfs_chunk *chunk;
	struct btrfs_stripe *stripe;
4892 4893 4894 4895 4896 4897 4898
	struct extent_map_tree *em_tree;
	struct extent_map *em;
	struct map_lookup *map;
	size_t item_size;
	u64 dev_offset;
	u64 stripe_size;
	int i = 0;
4899
	int ret = 0;
Y
Yan Zheng 已提交
4900

4901
	em_tree = &fs_info->mapping_tree.map_tree;
4902 4903 4904 4905 4906
	read_lock(&em_tree->lock);
	em = lookup_extent_mapping(em_tree, chunk_offset, chunk_size);
	read_unlock(&em_tree->lock);

	if (!em) {
4907
		btrfs_crit(fs_info, "unable to find logical %Lu len %Lu",
J
Jeff Mahoney 已提交
4908
			   chunk_offset, chunk_size);
4909 4910 4911 4912
		return -EINVAL;
	}

	if (em->start != chunk_offset || em->len != chunk_size) {
4913
		btrfs_crit(fs_info,
J
Jeff Mahoney 已提交
4914 4915
			   "found a bad mapping, wanted %Lu-%Lu, found %Lu-%Lu",
			    chunk_offset, chunk_size, em->start, em->len);
4916 4917 4918 4919
		free_extent_map(em);
		return -EINVAL;
	}

4920
	map = em->map_lookup;
4921 4922 4923
	item_size = btrfs_chunk_item_size(map->num_stripes);
	stripe_size = em->orig_block_len;

Y
Yan Zheng 已提交
4924
	chunk = kzalloc(item_size, GFP_NOFS);
4925 4926 4927 4928 4929
	if (!chunk) {
		ret = -ENOMEM;
		goto out;
	}

4930 4931 4932 4933 4934 4935 4936
	/*
	 * Take the device list mutex to prevent races with the final phase of
	 * a device replace operation that replaces the device object associated
	 * with the map's stripes, because the device object's id can change
	 * at any time during that final phase of the device replace operation
	 * (dev-replace.c:btrfs_dev_replace_finishing()).
	 */
4937
	mutex_lock(&fs_info->fs_devices->device_list_mutex);
4938 4939 4940
	for (i = 0; i < map->num_stripes; i++) {
		device = map->stripes[i].dev;
		dev_offset = map->stripes[i].physical;
Y
Yan Zheng 已提交
4941

4942
		ret = btrfs_update_device(trans, device);
4943
		if (ret)
4944
			break;
4945 4946 4947 4948 4949 4950
		ret = btrfs_alloc_dev_extent(trans, device,
					     chunk_root->root_key.objectid,
					     BTRFS_FIRST_CHUNK_TREE_OBJECTID,
					     chunk_offset, dev_offset,
					     stripe_size);
		if (ret)
4951 4952 4953
			break;
	}
	if (ret) {
4954
		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4955
		goto out;
Y
Yan Zheng 已提交
4956 4957 4958
	}

	stripe = &chunk->stripe;
4959 4960 4961
	for (i = 0; i < map->num_stripes; i++) {
		device = map->stripes[i].dev;
		dev_offset = map->stripes[i].physical;
4962

4963 4964 4965
		btrfs_set_stack_stripe_devid(stripe, device->devid);
		btrfs_set_stack_stripe_offset(stripe, dev_offset);
		memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
Y
Yan Zheng 已提交
4966
		stripe++;
4967
	}
4968
	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4969

Y
Yan Zheng 已提交
4970
	btrfs_set_stack_chunk_length(chunk, chunk_size);
4971
	btrfs_set_stack_chunk_owner(chunk, extent_root->root_key.objectid);
Y
Yan Zheng 已提交
4972 4973 4974 4975 4976
	btrfs_set_stack_chunk_stripe_len(chunk, map->stripe_len);
	btrfs_set_stack_chunk_type(chunk, map->type);
	btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes);
	btrfs_set_stack_chunk_io_align(chunk, map->stripe_len);
	btrfs_set_stack_chunk_io_width(chunk, map->stripe_len);
4977
	btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize);
Y
Yan Zheng 已提交
4978
	btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes);
4979

Y
Yan Zheng 已提交
4980 4981 4982
	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
	key.type = BTRFS_CHUNK_ITEM_KEY;
	key.offset = chunk_offset;
4983

Y
Yan Zheng 已提交
4984
	ret = btrfs_insert_item(trans, chunk_root, &key, chunk, item_size);
4985 4986 4987 4988 4989
	if (ret == 0 && map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
		/*
		 * TODO: Cleanup of inserted chunk root in case of
		 * failure.
		 */
4990
		ret = btrfs_add_system_chunk(fs_info, &key, chunk, item_size);
4991
	}
4992

4993
out:
4994
	kfree(chunk);
4995
	free_extent_map(em);
4996
	return ret;
Y
Yan Zheng 已提交
4997
}
4998

Y
Yan Zheng 已提交
4999 5000 5001 5002 5003 5004 5005 5006
/*
 * Chunk allocation falls into two parts. The first part does works
 * that make the new allocated chunk useable, but not do any operation
 * that modifies the chunk tree. The second part does the works that
 * require modifying the chunk tree. This division is important for the
 * bootstrap process of adding storage to a seed btrfs.
 */
int btrfs_alloc_chunk(struct btrfs_trans_handle *trans,
5007
		      struct btrfs_fs_info *fs_info, u64 type)
Y
Yan Zheng 已提交
5008 5009 5010
{
	u64 chunk_offset;

5011 5012
	ASSERT(mutex_is_locked(&fs_info->chunk_mutex));
	chunk_offset = find_next_chunk(fs_info);
5013
	return __btrfs_alloc_chunk(trans, fs_info, chunk_offset, type);
Y
Yan Zheng 已提交
5014 5015
}

C
Chris Mason 已提交
5016
static noinline int init_first_rw_device(struct btrfs_trans_handle *trans,
5017
					 struct btrfs_fs_info *fs_info,
Y
Yan Zheng 已提交
5018 5019
					 struct btrfs_device *device)
{
5020
	struct btrfs_root *extent_root = fs_info->extent_root;
Y
Yan Zheng 已提交
5021 5022 5023 5024 5025
	u64 chunk_offset;
	u64 sys_chunk_offset;
	u64 alloc_profile;
	int ret;

5026
	chunk_offset = find_next_chunk(fs_info);
5027
	alloc_profile = btrfs_get_alloc_profile(extent_root, 0);
5028
	ret = __btrfs_alloc_chunk(trans, fs_info, chunk_offset, alloc_profile);
5029 5030
	if (ret)
		return ret;
Y
Yan Zheng 已提交
5031

5032
	sys_chunk_offset = find_next_chunk(fs_info);
5033
	alloc_profile = btrfs_get_alloc_profile(fs_info->chunk_root, 0);
5034
	ret = __btrfs_alloc_chunk(trans, fs_info, sys_chunk_offset,
5035
				  alloc_profile);
5036
	return ret;
Y
Yan Zheng 已提交
5037 5038
}

5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051
static inline int btrfs_chunk_max_errors(struct map_lookup *map)
{
	int max_errors;

	if (map->type & (BTRFS_BLOCK_GROUP_RAID1 |
			 BTRFS_BLOCK_GROUP_RAID10 |
			 BTRFS_BLOCK_GROUP_RAID5 |
			 BTRFS_BLOCK_GROUP_DUP)) {
		max_errors = 1;
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID6) {
		max_errors = 2;
	} else {
		max_errors = 0;
5052
	}
Y
Yan Zheng 已提交
5053

5054
	return max_errors;
Y
Yan Zheng 已提交
5055 5056
}

5057
int btrfs_chunk_readonly(struct btrfs_fs_info *fs_info, u64 chunk_offset)
Y
Yan Zheng 已提交
5058 5059 5060
{
	struct extent_map *em;
	struct map_lookup *map;
5061
	struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
Y
Yan Zheng 已提交
5062
	int readonly = 0;
5063
	int miss_ndevs = 0;
Y
Yan Zheng 已提交
5064 5065
	int i;

5066
	read_lock(&map_tree->map_tree.lock);
Y
Yan Zheng 已提交
5067
	em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
5068
	read_unlock(&map_tree->map_tree.lock);
Y
Yan Zheng 已提交
5069 5070 5071
	if (!em)
		return 1;

5072
	map = em->map_lookup;
Y
Yan Zheng 已提交
5073
	for (i = 0; i < map->num_stripes; i++) {
5074 5075 5076 5077 5078
		if (map->stripes[i].dev->missing) {
			miss_ndevs++;
			continue;
		}

Y
Yan Zheng 已提交
5079 5080
		if (!map->stripes[i].dev->writeable) {
			readonly = 1;
5081
			goto end;
Y
Yan Zheng 已提交
5082 5083
		}
	}
5084 5085 5086 5087 5088 5089 5090 5091 5092

	/*
	 * If the number of missing devices is larger than max errors,
	 * we can not write the data into that chunk successfully, so
	 * set it readonly.
	 */
	if (miss_ndevs > btrfs_chunk_max_errors(map))
		readonly = 1;
end:
5093
	free_extent_map(em);
Y
Yan Zheng 已提交
5094
	return readonly;
5095 5096 5097 5098
}

void btrfs_mapping_init(struct btrfs_mapping_tree *tree)
{
5099
	extent_map_tree_init(&tree->map_tree);
5100 5101 5102 5103 5104 5105
}

void btrfs_mapping_tree_free(struct btrfs_mapping_tree *tree)
{
	struct extent_map *em;

C
Chris Mason 已提交
5106
	while (1) {
5107
		write_lock(&tree->map_tree.lock);
5108 5109 5110
		em = lookup_extent_mapping(&tree->map_tree, 0, (u64)-1);
		if (em)
			remove_extent_mapping(&tree->map_tree, em);
5111
		write_unlock(&tree->map_tree.lock);
5112 5113 5114 5115 5116 5117 5118 5119 5120
		if (!em)
			break;
		/* once for us */
		free_extent_map(em);
		/* once for the tree */
		free_extent_map(em);
	}
}

5121
int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5122
{
5123
	struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
5124 5125 5126 5127 5128
	struct extent_map *em;
	struct map_lookup *map;
	struct extent_map_tree *em_tree = &map_tree->map_tree;
	int ret;

5129
	read_lock(&em_tree->lock);
5130
	em = lookup_extent_mapping(em_tree, logical, len);
5131
	read_unlock(&em_tree->lock);
5132

5133 5134 5135 5136 5137 5138
	/*
	 * We could return errors for these cases, but that could get ugly and
	 * we'd probably do the same thing which is just not do anything else
	 * and exit, so return 1 so the callers don't try to use other copies.
	 */
	if (!em) {
5139
		btrfs_crit(fs_info, "No mapping for %Lu-%Lu", logical,
5140 5141 5142 5143 5144
			    logical+len);
		return 1;
	}

	if (em->start > logical || em->start + em->len < logical) {
J
Jeff Mahoney 已提交
5145 5146 5147
		btrfs_crit(fs_info, "Invalid mapping for %Lu-%Lu, got %Lu-%Lu",
			   logical, logical+len, em->start,
			   em->start + em->len);
5148
		free_extent_map(em);
5149 5150 5151
		return 1;
	}

5152
	map = em->map_lookup;
5153 5154
	if (map->type & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1))
		ret = map->num_stripes;
C
Chris Mason 已提交
5155 5156
	else if (map->type & BTRFS_BLOCK_GROUP_RAID10)
		ret = map->sub_stripes;
D
David Woodhouse 已提交
5157 5158 5159 5160
	else if (map->type & BTRFS_BLOCK_GROUP_RAID5)
		ret = 2;
	else if (map->type & BTRFS_BLOCK_GROUP_RAID6)
		ret = 3;
5161 5162 5163
	else
		ret = 1;
	free_extent_map(em);
5164

5165
	btrfs_dev_replace_lock(&fs_info->dev_replace, 0);
5166 5167
	if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))
		ret++;
5168
	btrfs_dev_replace_unlock(&fs_info->dev_replace, 0);
5169

5170 5171 5172
	return ret;
}

5173
unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
D
David Woodhouse 已提交
5174 5175 5176 5177 5178 5179
				    struct btrfs_mapping_tree *map_tree,
				    u64 logical)
{
	struct extent_map *em;
	struct map_lookup *map;
	struct extent_map_tree *em_tree = &map_tree->map_tree;
5180
	unsigned long len = fs_info->sectorsize;
D
David Woodhouse 已提交
5181 5182 5183 5184 5185 5186 5187

	read_lock(&em_tree->lock);
	em = lookup_extent_mapping(em_tree, logical, len);
	read_unlock(&em_tree->lock);
	BUG_ON(!em);

	BUG_ON(em->start > logical || em->start + em->len < logical);
5188
	map = em->map_lookup;
5189
	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
D
David Woodhouse 已提交
5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208
		len = map->stripe_len * nr_data_stripes(map);
	free_extent_map(em);
	return len;
}

int btrfs_is_parity_mirror(struct btrfs_mapping_tree *map_tree,
			   u64 logical, u64 len, int mirror_num)
{
	struct extent_map *em;
	struct map_lookup *map;
	struct extent_map_tree *em_tree = &map_tree->map_tree;
	int ret = 0;

	read_lock(&em_tree->lock);
	em = lookup_extent_mapping(em_tree, logical, len);
	read_unlock(&em_tree->lock);
	BUG_ON(!em);

	BUG_ON(em->start > logical || em->start + em->len < logical);
5209
	map = em->map_lookup;
5210
	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
D
David Woodhouse 已提交
5211 5212 5213 5214 5215
		ret = 1;
	free_extent_map(em);
	return ret;
}

5216 5217 5218
static int find_live_mirror(struct btrfs_fs_info *fs_info,
			    struct map_lookup *map, int first, int num,
			    int optimal, int dev_replace_is_ongoing)
5219 5220
{
	int i;
5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244
	int tolerance;
	struct btrfs_device *srcdev;

	if (dev_replace_is_ongoing &&
	    fs_info->dev_replace.cont_reading_from_srcdev_mode ==
	     BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID)
		srcdev = fs_info->dev_replace.srcdev;
	else
		srcdev = NULL;

	/*
	 * try to avoid the drive that is the source drive for a
	 * dev-replace procedure, only choose it if no other non-missing
	 * mirror is available
	 */
	for (tolerance = 0; tolerance < 2; tolerance++) {
		if (map->stripes[optimal].dev->bdev &&
		    (tolerance || map->stripes[optimal].dev != srcdev))
			return optimal;
		for (i = first; i < first + num; i++) {
			if (map->stripes[i].dev->bdev &&
			    (tolerance || map->stripes[i].dev != srcdev))
				return i;
		}
5245
	}
5246

5247 5248 5249 5250 5251 5252
	/* we couldn't find one that doesn't fail.  Just return something
	 * and the io error handling code will clean up eventually
	 */
	return optimal;
}

D
David Woodhouse 已提交
5253 5254 5255 5256 5257 5258
static inline int parity_smaller(u64 a, u64 b)
{
	return a > b;
}

/* Bubble-sort the stripe set to put the parity/syndrome stripes last */
5259
static void sort_parity_stripes(struct btrfs_bio *bbio, int num_stripes)
D
David Woodhouse 已提交
5260 5261 5262 5263 5264 5265 5266 5267
{
	struct btrfs_bio_stripe s;
	int i;
	u64 l;
	int again = 1;

	while (again) {
		again = 0;
5268
		for (i = 0; i < num_stripes - 1; i++) {
5269 5270
			if (parity_smaller(bbio->raid_map[i],
					   bbio->raid_map[i+1])) {
D
David Woodhouse 已提交
5271
				s = bbio->stripes[i];
5272
				l = bbio->raid_map[i];
D
David Woodhouse 已提交
5273
				bbio->stripes[i] = bbio->stripes[i+1];
5274
				bbio->raid_map[i] = bbio->raid_map[i+1];
D
David Woodhouse 已提交
5275
				bbio->stripes[i+1] = s;
5276
				bbio->raid_map[i+1] = l;
5277

D
David Woodhouse 已提交
5278 5279 5280 5281 5282 5283
				again = 1;
			}
		}
	}
}

5284 5285 5286
static struct btrfs_bio *alloc_btrfs_bio(int total_stripes, int real_stripes)
{
	struct btrfs_bio *bbio = kzalloc(
5287
		 /* the size of the btrfs_bio */
5288
		sizeof(struct btrfs_bio) +
5289
		/* plus the variable array for the stripes */
5290
		sizeof(struct btrfs_bio_stripe) * (total_stripes) +
5291
		/* plus the variable array for the tgt dev */
5292
		sizeof(int) * (real_stripes) +
5293 5294 5295 5296 5297
		/*
		 * plus the raid_map, which includes both the tgt dev
		 * and the stripes
		 */
		sizeof(u64) * (total_stripes),
5298
		GFP_NOFS|__GFP_NOFAIL);
5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319

	atomic_set(&bbio->error, 0);
	atomic_set(&bbio->refs, 1);

	return bbio;
}

void btrfs_get_bbio(struct btrfs_bio *bbio)
{
	WARN_ON(!atomic_read(&bbio->refs));
	atomic_inc(&bbio->refs);
}

void btrfs_put_bbio(struct btrfs_bio *bbio)
{
	if (!bbio)
		return;
	if (atomic_dec_and_test(&bbio->refs))
		kfree(bbio);
}

5320 5321
static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
			     enum btrfs_map_op op,
5322
			     u64 logical, u64 *length,
5323
			     struct btrfs_bio **bbio_ret,
5324
			     int mirror_num, int need_raid_map)
5325 5326 5327
{
	struct extent_map *em;
	struct map_lookup *map;
5328
	struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
5329 5330
	struct extent_map_tree *em_tree = &map_tree->map_tree;
	u64 offset;
5331
	u64 stripe_offset;
5332
	u64 stripe_end_offset;
5333
	u64 stripe_nr;
5334 5335
	u64 stripe_nr_orig;
	u64 stripe_nr_end;
D
David Woodhouse 已提交
5336
	u64 stripe_len;
5337
	u32 stripe_index;
5338
	int i;
L
Li Zefan 已提交
5339
	int ret = 0;
5340
	int num_stripes;
5341
	int max_errors = 0;
5342
	int tgtdev_indexes = 0;
5343
	struct btrfs_bio *bbio = NULL;
5344 5345 5346
	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
	int dev_replace_is_ongoing = 0;
	int num_alloc_stripes;
5347 5348
	int patch_the_first_stripe_for_dev_replace = 0;
	u64 physical_to_patch_in_first_stripe = 0;
D
David Woodhouse 已提交
5349
	u64 raid56_full_stripe_start = (u64)-1;
5350

5351
	read_lock(&em_tree->lock);
5352
	em = lookup_extent_mapping(em_tree, logical, *length);
5353
	read_unlock(&em_tree->lock);
5354

5355
	if (!em) {
5356
		btrfs_crit(fs_info, "unable to find logical %llu len %llu",
5357
			logical, *length);
5358 5359 5360 5361
		return -EINVAL;
	}

	if (em->start > logical || em->start + em->len < logical) {
J
Jeff Mahoney 已提交
5362 5363 5364
		btrfs_crit(fs_info,
			   "found a bad mapping, wanted %Lu, found %Lu-%Lu",
			   logical, em->start, em->start + em->len);
5365
		free_extent_map(em);
5366
		return -EINVAL;
5367
	}
5368

5369
	map = em->map_lookup;
5370
	offset = logical - em->start;
5371

D
David Woodhouse 已提交
5372
	stripe_len = map->stripe_len;
5373 5374 5375 5376 5377
	stripe_nr = offset;
	/*
	 * stripe_nr counts the total number of stripes we have to stride
	 * to get to this block
	 */
5378
	stripe_nr = div64_u64(stripe_nr, stripe_len);
5379

D
David Woodhouse 已提交
5380
	stripe_offset = stripe_nr * stripe_len;
5381
	if (offset < stripe_offset) {
J
Jeff Mahoney 已提交
5382 5383
		btrfs_crit(fs_info,
			   "stripe math has gone wrong, stripe_offset=%llu, offset=%llu, start=%llu, logical=%llu, stripe_len=%llu",
5384 5385 5386 5387 5388
			   stripe_offset, offset, em->start, logical,
			   stripe_len);
		free_extent_map(em);
		return -EINVAL;
	}
5389 5390 5391 5392

	/* stripe_offset is the offset of this block in its stripe*/
	stripe_offset = offset - stripe_offset;

D
David Woodhouse 已提交
5393
	/* if we're here for raid56, we need to know the stripe aligned start */
5394
	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
D
David Woodhouse 已提交
5395 5396 5397 5398 5399 5400
		unsigned long full_stripe_len = stripe_len * nr_data_stripes(map);
		raid56_full_stripe_start = offset;

		/* allow a write of a full stripe, but make sure we don't
		 * allow straddling of stripes
		 */
5401 5402
		raid56_full_stripe_start = div64_u64(raid56_full_stripe_start,
				full_stripe_len);
D
David Woodhouse 已提交
5403 5404 5405
		raid56_full_stripe_start *= full_stripe_len;
	}

5406
	if (op == BTRFS_MAP_DISCARD) {
D
David Woodhouse 已提交
5407
		/* we don't discard raid56 yet */
5408
		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
D
David Woodhouse 已提交
5409 5410 5411
			ret = -EOPNOTSUPP;
			goto out;
		}
5412
		*length = min_t(u64, em->len - offset, *length);
D
David Woodhouse 已提交
5413 5414 5415 5416 5417
	} else if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
		u64 max_len;
		/* For writes to RAID[56], allow a full stripeset across all disks.
		   For other RAID types and for RAID[56] reads, just allow a single
		   stripe (on a single disk). */
5418
		if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&
5419
		    (op == BTRFS_MAP_WRITE)) {
D
David Woodhouse 已提交
5420 5421 5422 5423 5424 5425 5426
			max_len = stripe_len * nr_data_stripes(map) -
				(offset - raid56_full_stripe_start);
		} else {
			/* we limit the length of each bio to what fits in a stripe */
			max_len = stripe_len - stripe_offset;
		}
		*length = min_t(u64, em->len - offset, max_len);
5427 5428 5429
	} else {
		*length = em->len - offset;
	}
5430

D
David Woodhouse 已提交
5431 5432
	/* This is for when we're called from btrfs_merge_bio_hook() and all
	   it cares about is the length */
5433
	if (!bbio_ret)
5434 5435
		goto out;

5436
	btrfs_dev_replace_lock(dev_replace, 0);
5437 5438
	dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
	if (!dev_replace_is_ongoing)
5439 5440 5441
		btrfs_dev_replace_unlock(dev_replace, 0);
	else
		btrfs_dev_replace_set_lock_blocking(dev_replace);
5442

5443
	if (dev_replace_is_ongoing && mirror_num == map->num_stripes + 1 &&
5444 5445
	    op != BTRFS_MAP_WRITE && op != BTRFS_MAP_DISCARD &&
	    op != BTRFS_MAP_GET_READ_MIRRORS && dev_replace->tgtdev != NULL) {
5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465
		/*
		 * in dev-replace case, for repair case (that's the only
		 * case where the mirror is selected explicitly when
		 * calling btrfs_map_block), blocks left of the left cursor
		 * can also be read from the target drive.
		 * For REQ_GET_READ_MIRRORS, the target drive is added as
		 * the last one to the array of stripes. For READ, it also
		 * needs to be supported using the same mirror number.
		 * If the requested block is not left of the left cursor,
		 * EIO is returned. This can happen because btrfs_num_copies()
		 * returns one more in the dev-replace case.
		 */
		u64 tmp_length = *length;
		struct btrfs_bio *tmp_bbio = NULL;
		int tmp_num_stripes;
		u64 srcdev_devid = dev_replace->srcdev->devid;
		int index_srcdev = 0;
		int found = 0;
		u64 physical_of_found = 0;

5466
		ret = __btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
5467
			     logical, &tmp_length, &tmp_bbio, 0, 0);
5468 5469 5470 5471 5472 5473 5474 5475
		if (ret) {
			WARN_ON(tmp_bbio != NULL);
			goto out;
		}

		tmp_num_stripes = tmp_bbio->num_stripes;
		if (mirror_num > tmp_num_stripes) {
			/*
5476
			 * BTRFS_MAP_GET_READ_MIRRORS does not contain this
5477 5478 5479 5480
			 * mirror, that means that the requested area
			 * is not left of the left cursor
			 */
			ret = -EIO;
5481
			btrfs_put_bbio(tmp_bbio);
5482 5483 5484 5485 5486 5487 5488 5489 5490 5491
			goto out;
		}

		/*
		 * process the rest of the function using the mirror_num
		 * of the source drive. Therefore look it up first.
		 * At the end, patch the device pointer to the one of the
		 * target drive.
		 */
		for (i = 0; i < tmp_num_stripes; i++) {
5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505
			if (tmp_bbio->stripes[i].dev->devid != srcdev_devid)
				continue;

			/*
			 * In case of DUP, in order to keep it simple, only add
			 * the mirror with the lowest physical address
			 */
			if (found &&
			    physical_of_found <= tmp_bbio->stripes[i].physical)
				continue;

			index_srcdev = i;
			found = 1;
			physical_of_found = tmp_bbio->stripes[i].physical;
5506 5507
		}

5508 5509 5510
		btrfs_put_bbio(tmp_bbio);

		if (!found) {
5511 5512 5513 5514 5515
			WARN_ON(1);
			ret = -EIO;
			goto out;
		}

5516 5517 5518
		mirror_num = index_srcdev + 1;
		patch_the_first_stripe_for_dev_replace = 1;
		physical_to_patch_in_first_stripe = physical_of_found;
5519 5520 5521 5522
	} else if (mirror_num > map->num_stripes) {
		mirror_num = 0;
	}

5523
	num_stripes = 1;
5524
	stripe_index = 0;
5525
	stripe_nr_orig = stripe_nr;
5526
	stripe_nr_end = ALIGN(offset + *length, map->stripe_len);
5527
	stripe_nr_end = div_u64(stripe_nr_end, map->stripe_len);
5528 5529
	stripe_end_offset = stripe_nr_end * map->stripe_len -
			    (offset + *length);
D
David Woodhouse 已提交
5530

5531
	if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
5532
		if (op == BTRFS_MAP_DISCARD)
5533 5534
			num_stripes = min_t(u64, map->num_stripes,
					    stripe_nr_end - stripe_nr_orig);
5535 5536
		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
				&stripe_index);
5537 5538
		if (op != BTRFS_MAP_WRITE && op != BTRFS_MAP_DISCARD &&
		    op != BTRFS_MAP_GET_READ_MIRRORS)
5539
			mirror_num = 1;
5540
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
5541 5542
		if (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_DISCARD ||
		    op == BTRFS_MAP_GET_READ_MIRRORS)
5543
			num_stripes = map->num_stripes;
5544
		else if (mirror_num)
5545
			stripe_index = mirror_num - 1;
5546
		else {
5547
			stripe_index = find_live_mirror(fs_info, map, 0,
5548
					    map->num_stripes,
5549 5550
					    current->pid % map->num_stripes,
					    dev_replace_is_ongoing);
5551
			mirror_num = stripe_index + 1;
5552
		}
5553

5554
	} else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
5555 5556
		if (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_DISCARD ||
		    op == BTRFS_MAP_GET_READ_MIRRORS) {
5557
			num_stripes = map->num_stripes;
5558
		} else if (mirror_num) {
5559
			stripe_index = mirror_num - 1;
5560 5561 5562
		} else {
			mirror_num = 1;
		}
5563

C
Chris Mason 已提交
5564
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
5565
		u32 factor = map->num_stripes / map->sub_stripes;
C
Chris Mason 已提交
5566

5567
		stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
C
Chris Mason 已提交
5568 5569
		stripe_index *= map->sub_stripes;

5570
		if (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS)
5571
			num_stripes = map->sub_stripes;
5572
		else if (op == BTRFS_MAP_DISCARD)
5573 5574 5575
			num_stripes = min_t(u64, map->sub_stripes *
					    (stripe_nr_end - stripe_nr_orig),
					    map->num_stripes);
C
Chris Mason 已提交
5576 5577
		else if (mirror_num)
			stripe_index += mirror_num - 1;
5578
		else {
J
Jan Schmidt 已提交
5579
			int old_stripe_index = stripe_index;
5580 5581
			stripe_index = find_live_mirror(fs_info, map,
					      stripe_index,
5582
					      map->sub_stripes, stripe_index +
5583 5584
					      current->pid % map->sub_stripes,
					      dev_replace_is_ongoing);
J
Jan Schmidt 已提交
5585
			mirror_num = stripe_index - old_stripe_index + 1;
5586
		}
D
David Woodhouse 已提交
5587

5588
	} else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5589
		if (need_raid_map &&
5590
		    (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS ||
5591
		     mirror_num > 1)) {
D
David Woodhouse 已提交
5592
			/* push stripe_nr back to the start of the full stripe */
5593 5594
			stripe_nr = div_u64(raid56_full_stripe_start,
					stripe_len * nr_data_stripes(map));
D
David Woodhouse 已提交
5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608

			/* RAID[56] write or recovery. Return all stripes */
			num_stripes = map->num_stripes;
			max_errors = nr_parity_stripes(map);

			*length = map->stripe_len;
			stripe_index = 0;
			stripe_offset = 0;
		} else {
			/*
			 * Mirror #0 or #1 means the original data block.
			 * Mirror #2 is RAID5 parity block.
			 * Mirror #3 is RAID6 Q block.
			 */
5609 5610
			stripe_nr = div_u64_rem(stripe_nr,
					nr_data_stripes(map), &stripe_index);
D
David Woodhouse 已提交
5611 5612 5613 5614 5615
			if (mirror_num > 1)
				stripe_index = nr_data_stripes(map) +
						mirror_num - 2;

			/* We distribute the parity blocks across stripes */
5616 5617
			div_u64_rem(stripe_nr + stripe_index, map->num_stripes,
					&stripe_index);
5618 5619
			if ((op != BTRFS_MAP_WRITE && op != BTRFS_MAP_DISCARD &&
			    op != BTRFS_MAP_GET_READ_MIRRORS) && mirror_num <= 1)
5620
				mirror_num = 1;
D
David Woodhouse 已提交
5621
		}
5622 5623
	} else {
		/*
5624 5625 5626
		 * after this, stripe_nr is the number of stripes on this
		 * device we have to walk to find the data, and stripe_index is
		 * the number of our device in the stripe array
5627
		 */
5628 5629
		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
				&stripe_index);
5630
		mirror_num = stripe_index + 1;
5631
	}
5632
	if (stripe_index >= map->num_stripes) {
J
Jeff Mahoney 已提交
5633 5634
		btrfs_crit(fs_info,
			   "stripe index math went horribly wrong, got stripe_index=%u, num_stripes=%u",
5635 5636 5637 5638
			   stripe_index, map->num_stripes);
		ret = -EINVAL;
		goto out;
	}
5639

5640
	num_alloc_stripes = num_stripes;
5641
	if (dev_replace_is_ongoing) {
5642
		if (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_DISCARD)
5643
			num_alloc_stripes <<= 1;
5644
		if (op == BTRFS_MAP_GET_READ_MIRRORS)
5645
			num_alloc_stripes++;
5646
		tgtdev_indexes = num_stripes;
5647
	}
5648

5649
	bbio = alloc_btrfs_bio(num_alloc_stripes, tgtdev_indexes);
L
Li Zefan 已提交
5650 5651 5652 5653
	if (!bbio) {
		ret = -ENOMEM;
		goto out;
	}
5654 5655
	if (dev_replace_is_ongoing)
		bbio->tgtdev_map = (int *)(bbio->stripes + num_alloc_stripes);
L
Li Zefan 已提交
5656

5657
	/* build raid_map */
5658
	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK &&
5659
	    need_raid_map &&
5660
	    ((op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS) ||
5661 5662
	    mirror_num > 1)) {
		u64 tmp;
5663
		unsigned rot;
5664 5665 5666 5667 5668 5669 5670

		bbio->raid_map = (u64 *)((void *)bbio->stripes +
				 sizeof(struct btrfs_bio_stripe) *
				 num_alloc_stripes +
				 sizeof(int) * tgtdev_indexes);

		/* Work out the disk rotation on this stripe-set */
5671
		div_u64_rem(stripe_nr, num_stripes, &rot);
5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684

		/* Fill in the logical address of each stripe */
		tmp = stripe_nr * nr_data_stripes(map);
		for (i = 0; i < nr_data_stripes(map); i++)
			bbio->raid_map[(i+rot) % num_stripes] =
				em->start + (tmp + i) * map->stripe_len;

		bbio->raid_map[(i+rot) % map->num_stripes] = RAID5_P_STRIPE;
		if (map->type & BTRFS_BLOCK_GROUP_RAID6)
			bbio->raid_map[(i+rot+1) % num_stripes] =
				RAID6_Q_STRIPE;
	}

5685
	if (op == BTRFS_MAP_DISCARD) {
5686 5687
		u32 factor = 0;
		u32 sub_stripes = 0;
5688 5689
		u64 stripes_per_dev = 0;
		u32 remaining_stripes = 0;
L
Liu Bo 已提交
5690
		u32 last_stripe = 0;
5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703

		if (map->type &
		    (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)) {
			if (map->type & BTRFS_BLOCK_GROUP_RAID0)
				sub_stripes = 1;
			else
				sub_stripes = map->sub_stripes;

			factor = map->num_stripes / sub_stripes;
			stripes_per_dev = div_u64_rem(stripe_nr_end -
						      stripe_nr_orig,
						      factor,
						      &remaining_stripes);
L
Liu Bo 已提交
5704 5705
			div_u64_rem(stripe_nr_end - 1, factor, &last_stripe);
			last_stripe *= sub_stripes;
5706 5707
		}

5708
		for (i = 0; i < num_stripes; i++) {
5709
			bbio->stripes[i].physical =
5710 5711
				map->stripes[stripe_index].physical +
				stripe_offset + stripe_nr * map->stripe_len;
5712
			bbio->stripes[i].dev = map->stripes[stripe_index].dev;
5713

5714 5715 5716 5717
			if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
					 BTRFS_BLOCK_GROUP_RAID10)) {
				bbio->stripes[i].length = stripes_per_dev *
							  map->stripe_len;
L
Liu Bo 已提交
5718

5719 5720 5721
				if (i / sub_stripes < remaining_stripes)
					bbio->stripes[i].length +=
						map->stripe_len;
L
Liu Bo 已提交
5722 5723 5724 5725 5726 5727 5728 5729 5730

				/*
				 * Special for the first stripe and
				 * the last stripe:
				 *
				 * |-------|...|-------|
				 *     |----------|
				 *    off     end_off
				 */
5731
				if (i < sub_stripes)
5732
					bbio->stripes[i].length -=
5733
						stripe_offset;
L
Liu Bo 已提交
5734 5735 5736 5737

				if (stripe_index >= last_stripe &&
				    stripe_index <= (last_stripe +
						     sub_stripes - 1))
5738
					bbio->stripes[i].length -=
5739
						stripe_end_offset;
L
Liu Bo 已提交
5740

5741 5742
				if (i == sub_stripes - 1)
					stripe_offset = 0;
5743
			} else
5744
				bbio->stripes[i].length = *length;
5745 5746 5747 5748 5749 5750 5751 5752 5753 5754

			stripe_index++;
			if (stripe_index == map->num_stripes) {
				/* This could only happen for RAID0/10 */
				stripe_index = 0;
				stripe_nr++;
			}
		}
	} else {
		for (i = 0; i < num_stripes; i++) {
5755
			bbio->stripes[i].physical =
5756 5757 5758
				map->stripes[stripe_index].physical +
				stripe_offset +
				stripe_nr * map->stripe_len;
5759
			bbio->stripes[i].dev =
5760
				map->stripes[stripe_index].dev;
5761
			stripe_index++;
5762
		}
5763
	}
L
Li Zefan 已提交
5764

5765
	if (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS)
5766
		max_errors = btrfs_chunk_max_errors(map);
L
Li Zefan 已提交
5767

5768 5769
	if (bbio->raid_map)
		sort_parity_stripes(bbio, num_stripes);
5770

5771
	tgtdev_indexes = 0;
5772
	if (dev_replace_is_ongoing &&
5773
	   (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_DISCARD) &&
5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800
	    dev_replace->tgtdev != NULL) {
		int index_where_to_add;
		u64 srcdev_devid = dev_replace->srcdev->devid;

		/*
		 * duplicate the write operations while the dev replace
		 * procedure is running. Since the copying of the old disk
		 * to the new disk takes place at run time while the
		 * filesystem is mounted writable, the regular write
		 * operations to the old disk have to be duplicated to go
		 * to the new disk as well.
		 * Note that device->missing is handled by the caller, and
		 * that the write to the old disk is already set up in the
		 * stripes array.
		 */
		index_where_to_add = num_stripes;
		for (i = 0; i < num_stripes; i++) {
			if (bbio->stripes[i].dev->devid == srcdev_devid) {
				/* write to new disk, too */
				struct btrfs_bio_stripe *new =
					bbio->stripes + index_where_to_add;
				struct btrfs_bio_stripe *old =
					bbio->stripes + i;

				new->physical = old->physical;
				new->length = old->length;
				new->dev = dev_replace->tgtdev;
5801
				bbio->tgtdev_map[i] = index_where_to_add;
5802 5803
				index_where_to_add++;
				max_errors++;
5804
				tgtdev_indexes++;
5805 5806 5807
			}
		}
		num_stripes = index_where_to_add;
5808 5809
	} else if (dev_replace_is_ongoing &&
		   op == BTRFS_MAP_GET_READ_MIRRORS &&
5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839
		   dev_replace->tgtdev != NULL) {
		u64 srcdev_devid = dev_replace->srcdev->devid;
		int index_srcdev = 0;
		int found = 0;
		u64 physical_of_found = 0;

		/*
		 * During the dev-replace procedure, the target drive can
		 * also be used to read data in case it is needed to repair
		 * a corrupt block elsewhere. This is possible if the
		 * requested area is left of the left cursor. In this area,
		 * the target drive is a full copy of the source drive.
		 */
		for (i = 0; i < num_stripes; i++) {
			if (bbio->stripes[i].dev->devid == srcdev_devid) {
				/*
				 * In case of DUP, in order to keep it
				 * simple, only add the mirror with the
				 * lowest physical address
				 */
				if (found &&
				    physical_of_found <=
				     bbio->stripes[i].physical)
					continue;
				index_srcdev = i;
				found = 1;
				physical_of_found = bbio->stripes[i].physical;
			}
		}
		if (found) {
5840 5841
			struct btrfs_bio_stripe *tgtdev_stripe =
				bbio->stripes + num_stripes;
5842

5843 5844 5845 5846 5847
			tgtdev_stripe->physical = physical_of_found;
			tgtdev_stripe->length =
				bbio->stripes[index_srcdev].length;
			tgtdev_stripe->dev = dev_replace->tgtdev;
			bbio->tgtdev_map[index_srcdev] = num_stripes;
5848

5849 5850
			tgtdev_indexes++;
			num_stripes++;
5851
		}
5852 5853
	}

L
Li Zefan 已提交
5854
	*bbio_ret = bbio;
Z
Zhao Lei 已提交
5855
	bbio->map_type = map->type;
L
Li Zefan 已提交
5856 5857 5858
	bbio->num_stripes = num_stripes;
	bbio->max_errors = max_errors;
	bbio->mirror_num = mirror_num;
5859
	bbio->num_tgtdevs = tgtdev_indexes;
5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871

	/*
	 * this is the case that REQ_READ && dev_replace_is_ongoing &&
	 * mirror_num == num_stripes + 1 && dev_replace target drive is
	 * available as a mirror
	 */
	if (patch_the_first_stripe_for_dev_replace && num_stripes > 0) {
		WARN_ON(num_stripes > 1);
		bbio->stripes[0].dev = dev_replace->tgtdev;
		bbio->stripes[0].physical = physical_to_patch_in_first_stripe;
		bbio->mirror_num = map->num_stripes + 1;
	}
5872
out:
5873 5874 5875 5876
	if (dev_replace_is_ongoing) {
		btrfs_dev_replace_clear_lock_blocking(dev_replace);
		btrfs_dev_replace_unlock(dev_replace, 0);
	}
5877
	free_extent_map(em);
L
Li Zefan 已提交
5878
	return ret;
5879 5880
}

5881
int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
5882
		      u64 logical, u64 *length,
5883
		      struct btrfs_bio **bbio_ret, int mirror_num)
5884
{
5885
	return __btrfs_map_block(fs_info, op, logical, length, bbio_ret,
5886
				 mirror_num, 0);
5887 5888
}

5889
/* For Scrub/replace */
5890
int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
5891 5892
		     u64 logical, u64 *length,
		     struct btrfs_bio **bbio_ret, int mirror_num,
5893
		     int need_raid_map)
5894
{
5895
	return __btrfs_map_block(fs_info, op, logical, length, bbio_ret,
5896
				 mirror_num, need_raid_map);
5897 5898
}

5899
int btrfs_rmap_block(struct btrfs_fs_info *fs_info,
Y
Yan Zheng 已提交
5900 5901 5902
		     u64 chunk_start, u64 physical, u64 devid,
		     u64 **logical, int *naddrs, int *stripe_len)
{
5903
	struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
Y
Yan Zheng 已提交
5904 5905 5906 5907 5908 5909 5910
	struct extent_map_tree *em_tree = &map_tree->map_tree;
	struct extent_map *em;
	struct map_lookup *map;
	u64 *buf;
	u64 bytenr;
	u64 length;
	u64 stripe_nr;
D
David Woodhouse 已提交
5911
	u64 rmap_len;
Y
Yan Zheng 已提交
5912 5913
	int i, j, nr = 0;

5914
	read_lock(&em_tree->lock);
Y
Yan Zheng 已提交
5915
	em = lookup_extent_mapping(em_tree, chunk_start, 1);
5916
	read_unlock(&em_tree->lock);
Y
Yan Zheng 已提交
5917

5918
	if (!em) {
5919 5920
		btrfs_err(fs_info, "couldn't find em for chunk %Lu",
			chunk_start);
5921 5922 5923 5924
		return -EIO;
	}

	if (em->start != chunk_start) {
5925
		btrfs_err(fs_info, "bad chunk start, em=%Lu, wanted=%Lu",
5926 5927 5928 5929
		       em->start, chunk_start);
		free_extent_map(em);
		return -EIO;
	}
5930
	map = em->map_lookup;
Y
Yan Zheng 已提交
5931 5932

	length = em->len;
D
David Woodhouse 已提交
5933 5934
	rmap_len = map->stripe_len;

Y
Yan Zheng 已提交
5935
	if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5936
		length = div_u64(length, map->num_stripes / map->sub_stripes);
Y
Yan Zheng 已提交
5937
	else if (map->type & BTRFS_BLOCK_GROUP_RAID0)
5938
		length = div_u64(length, map->num_stripes);
5939
	else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5940
		length = div_u64(length, nr_data_stripes(map));
D
David Woodhouse 已提交
5941 5942
		rmap_len = map->stripe_len * nr_data_stripes(map);
	}
Y
Yan Zheng 已提交
5943

5944
	buf = kcalloc(map->num_stripes, sizeof(u64), GFP_NOFS);
5945
	BUG_ON(!buf); /* -ENOMEM */
Y
Yan Zheng 已提交
5946 5947 5948 5949 5950 5951 5952 5953 5954

	for (i = 0; i < map->num_stripes; i++) {
		if (devid && map->stripes[i].dev->devid != devid)
			continue;
		if (map->stripes[i].physical > physical ||
		    map->stripes[i].physical + length <= physical)
			continue;

		stripe_nr = physical - map->stripes[i].physical;
5955
		stripe_nr = div_u64(stripe_nr, map->stripe_len);
Y
Yan Zheng 已提交
5956 5957 5958

		if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
			stripe_nr = stripe_nr * map->num_stripes + i;
5959
			stripe_nr = div_u64(stripe_nr, map->sub_stripes);
Y
Yan Zheng 已提交
5960 5961
		} else if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
			stripe_nr = stripe_nr * map->num_stripes + i;
D
David Woodhouse 已提交
5962 5963 5964 5965 5966
		} /* else if RAID[56], multiply by nr_data_stripes().
		   * Alternatively, just use rmap_len below instead of
		   * map->stripe_len */

		bytenr = chunk_start + stripe_nr * rmap_len;
5967
		WARN_ON(nr >= map->num_stripes);
Y
Yan Zheng 已提交
5968 5969 5970 5971
		for (j = 0; j < nr; j++) {
			if (buf[j] == bytenr)
				break;
		}
5972 5973
		if (j == nr) {
			WARN_ON(nr >= map->num_stripes);
Y
Yan Zheng 已提交
5974
			buf[nr++] = bytenr;
5975
		}
Y
Yan Zheng 已提交
5976 5977 5978 5979
	}

	*logical = buf;
	*naddrs = nr;
D
David Woodhouse 已提交
5980
	*stripe_len = rmap_len;
Y
Yan Zheng 已提交
5981 5982 5983

	free_extent_map(em);
	return 0;
5984 5985
}

5986
static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio)
5987
{
5988 5989
	bio->bi_private = bbio->private;
	bio->bi_end_io = bbio->end_io;
5990
	bio_endio(bio);
5991

5992
	btrfs_put_bbio(bbio);
5993 5994
}

5995
static void btrfs_end_bio(struct bio *bio)
5996
{
5997
	struct btrfs_bio *bbio = bio->bi_private;
5998
	int is_orig_bio = 0;
5999

6000
	if (bio->bi_error) {
6001
		atomic_inc(&bbio->error);
6002
		if (bio->bi_error == -EIO || bio->bi_error == -EREMOTEIO) {
6003
			unsigned int stripe_index =
6004
				btrfs_io_bio(bio)->stripe_index;
6005
			struct btrfs_device *dev;
6006 6007 6008

			BUG_ON(stripe_index >= bbio->num_stripes);
			dev = bbio->stripes[stripe_index].dev;
6009
			if (dev->bdev) {
M
Mike Christie 已提交
6010
				if (bio_op(bio) == REQ_OP_WRITE)
6011 6012 6013 6014 6015
					btrfs_dev_stat_inc(dev,
						BTRFS_DEV_STAT_WRITE_ERRS);
				else
					btrfs_dev_stat_inc(dev,
						BTRFS_DEV_STAT_READ_ERRS);
J
Jens Axboe 已提交
6016
				if ((bio->bi_opf & WRITE_FLUSH) == WRITE_FLUSH)
6017 6018 6019 6020
					btrfs_dev_stat_inc(dev,
						BTRFS_DEV_STAT_FLUSH_ERRS);
				btrfs_dev_stat_print_on_error(dev);
			}
6021 6022
		}
	}
6023

6024
	if (bio == bbio->orig_bio)
6025 6026
		is_orig_bio = 1;

6027 6028
	btrfs_bio_counter_dec(bbio->fs_info);

6029
	if (atomic_dec_and_test(&bbio->stripes_pending)) {
6030 6031
		if (!is_orig_bio) {
			bio_put(bio);
6032
			bio = bbio->orig_bio;
6033
		}
6034

6035
		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
6036
		/* only send an error to the higher layers if it is
D
David Woodhouse 已提交
6037
		 * beyond the tolerance of the btrfs bio
6038
		 */
6039
		if (atomic_read(&bbio->error) > bbio->max_errors) {
6040
			bio->bi_error = -EIO;
6041
		} else {
6042 6043 6044 6045
			/*
			 * this bio is actually up to date, we didn't
			 * go over the max number of errors
			 */
6046
			bio->bi_error = 0;
6047
		}
6048

6049
		btrfs_end_bbio(bbio, bio);
6050
	} else if (!is_orig_bio) {
6051 6052 6053 6054
		bio_put(bio);
	}
}

6055 6056 6057 6058 6059 6060 6061
/*
 * see run_scheduled_bios for a description of why bios are collected for
 * async submit.
 *
 * This will add one bio to the pending list for a device and make sure
 * the work struct is scheduled.
 */
6062
static noinline void btrfs_schedule_bio(struct btrfs_device *device,
6063
					struct bio *bio)
6064
{
6065
	struct btrfs_fs_info *fs_info = device->fs_info;
6066
	int should_queue = 1;
6067
	struct btrfs_pending_bios *pending_bios;
6068

D
David Woodhouse 已提交
6069
	if (device->missing || !device->bdev) {
6070
		bio_io_error(bio);
D
David Woodhouse 已提交
6071 6072 6073
		return;
	}

6074
	/* don't bother with additional async steps for reads, right now */
M
Mike Christie 已提交
6075
	if (bio_op(bio) == REQ_OP_READ) {
6076
		bio_get(bio);
6077
		btrfsic_submit_bio(bio);
6078
		bio_put(bio);
6079
		return;
6080 6081 6082
	}

	/*
6083
	 * nr_async_bios allows us to reliably return congestion to the
6084 6085 6086 6087
	 * higher layers.  Otherwise, the async bio makes it appear we have
	 * made progress against dirty pages when we've really just put it
	 * on a queue for later
	 */
6088
	atomic_inc(&fs_info->nr_async_bios);
6089
	WARN_ON(bio->bi_next);
6090 6091 6092
	bio->bi_next = NULL;

	spin_lock(&device->io_lock);
J
Jens Axboe 已提交
6093
	if (bio->bi_opf & REQ_SYNC)
6094 6095 6096
		pending_bios = &device->pending_sync_bios;
	else
		pending_bios = &device->pending_bios;
6097

6098 6099
	if (pending_bios->tail)
		pending_bios->tail->bi_next = bio;
6100

6101 6102 6103
	pending_bios->tail = bio;
	if (!pending_bios->head)
		pending_bios->head = bio;
6104 6105 6106 6107 6108 6109
	if (device->running_pending)
		should_queue = 0;

	spin_unlock(&device->io_lock);

	if (should_queue)
6110
		btrfs_queue_work(fs_info->submit_workers, &device->work);
6111 6112
}

6113 6114
static void submit_stripe_bio(struct btrfs_bio *bbio, struct bio *bio,
			      u64 physical, int dev_nr, int async)
6115 6116
{
	struct btrfs_device *dev = bbio->stripes[dev_nr].dev;
6117
	struct btrfs_fs_info *fs_info = bbio->fs_info;
6118 6119

	bio->bi_private = bbio;
6120
	btrfs_io_bio(bio)->stripe_index = dev_nr;
6121
	bio->bi_end_io = btrfs_end_bio;
6122
	bio->bi_iter.bi_sector = physical >> 9;
6123 6124 6125 6126 6127 6128
#ifdef DEBUG
	{
		struct rcu_string *name;

		rcu_read_lock();
		name = rcu_dereference(dev->name);
6129 6130 6131 6132 6133 6134
		btrfs_debug(fs_info,
			"btrfs_map_bio: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u",
			bio_op(bio), bio->bi_opf,
			(u64)bio->bi_iter.bi_sector,
			(u_long)dev->bdev->bd_dev, name->str, dev->devid,
			bio->bi_iter.bi_size);
6135 6136 6137 6138
		rcu_read_unlock();
	}
#endif
	bio->bi_bdev = dev->bdev;
6139

6140
	btrfs_bio_counter_inc_noblocked(fs_info);
6141

6142
	if (async)
6143
		btrfs_schedule_bio(dev, bio);
6144
	else
6145
		btrfsic_submit_bio(bio);
6146 6147 6148 6149 6150 6151
}

static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)
{
	atomic_inc(&bbio->error);
	if (atomic_dec_and_test(&bbio->stripes_pending)) {
6152
		/* Should be the original bio. */
6153 6154
		WARN_ON(bio != bbio->orig_bio);

6155
		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
6156
		bio->bi_iter.bi_sector = logical >> 9;
6157 6158
		bio->bi_error = -EIO;
		btrfs_end_bbio(bbio, bio);
6159 6160 6161
	}
}

6162
int btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio,
6163
		  int mirror_num, int async_submit)
6164 6165
{
	struct btrfs_device *dev;
6166
	struct bio *first_bio = bio;
6167
	u64 logical = (u64)bio->bi_iter.bi_sector << 9;
6168 6169 6170
	u64 length = 0;
	u64 map_length;
	int ret;
6171 6172
	int dev_nr;
	int total_devs;
6173
	struct btrfs_bio *bbio = NULL;
6174

6175
	length = bio->bi_iter.bi_size;
6176
	map_length = length;
6177

6178 6179
	btrfs_bio_counter_inc_blocked(fs_info);
	ret = __btrfs_map_block(fs_info, bio_op(bio), logical,
M
Mike Christie 已提交
6180
				&map_length, &bbio, mirror_num, 1);
6181
	if (ret) {
6182
		btrfs_bio_counter_dec(fs_info);
6183
		return ret;
6184
	}
6185

6186
	total_devs = bbio->num_stripes;
D
David Woodhouse 已提交
6187 6188 6189
	bbio->orig_bio = first_bio;
	bbio->private = first_bio->bi_private;
	bbio->end_io = first_bio->bi_end_io;
6190
	bbio->fs_info = fs_info;
D
David Woodhouse 已提交
6191 6192
	atomic_set(&bbio->stripes_pending, bbio->num_stripes);

6193
	if ((bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&
M
Mike Christie 已提交
6194
	    ((bio_op(bio) == REQ_OP_WRITE) || (mirror_num > 1))) {
D
David Woodhouse 已提交
6195 6196
		/* In this case, map_length has been set to the length of
		   a single stripe; not the whole write */
M
Mike Christie 已提交
6197
		if (bio_op(bio) == REQ_OP_WRITE) {
6198 6199
			ret = raid56_parity_write(fs_info, bio, bbio,
						  map_length);
D
David Woodhouse 已提交
6200
		} else {
6201 6202
			ret = raid56_parity_recover(fs_info, bio, bbio,
						    map_length, mirror_num, 1);
D
David Woodhouse 已提交
6203
		}
6204

6205
		btrfs_bio_counter_dec(fs_info);
6206
		return ret;
D
David Woodhouse 已提交
6207 6208
	}

6209
	if (map_length < length) {
6210
		btrfs_crit(fs_info,
J
Jeff Mahoney 已提交
6211 6212
			   "mapping failed logical %llu bio len %llu len %llu",
			   logical, length, map_length);
6213 6214
		BUG();
	}
6215

6216
	for (dev_nr = 0; dev_nr < total_devs; dev_nr++) {
6217
		dev = bbio->stripes[dev_nr].dev;
M
Mike Christie 已提交
6218 6219
		if (!dev || !dev->bdev ||
		    (bio_op(bio) == REQ_OP_WRITE && !dev->writeable)) {
6220 6221 6222 6223
			bbio_error(bbio, first_bio, logical);
			continue;
		}

6224
		if (dev_nr < total_devs - 1) {
6225
			bio = btrfs_bio_clone(first_bio, GFP_NOFS);
6226
			BUG_ON(!bio); /* -ENOMEM */
6227
		} else
6228
			bio = first_bio;
6229

6230 6231
		submit_stripe_bio(bbio, bio, bbio->stripes[dev_nr].physical,
				  dev_nr, async_submit);
6232
	}
6233
	btrfs_bio_counter_dec(fs_info);
6234 6235 6236
	return 0;
}

6237
struct btrfs_device *btrfs_find_device(struct btrfs_fs_info *fs_info, u64 devid,
Y
Yan Zheng 已提交
6238
				       u8 *uuid, u8 *fsid)
6239
{
Y
Yan Zheng 已提交
6240 6241 6242
	struct btrfs_device *device;
	struct btrfs_fs_devices *cur_devices;

6243
	cur_devices = fs_info->fs_devices;
Y
Yan Zheng 已提交
6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254
	while (cur_devices) {
		if (!fsid ||
		    !memcmp(cur_devices->fsid, fsid, BTRFS_UUID_SIZE)) {
			device = __find_device(&cur_devices->devices,
					       devid, uuid);
			if (device)
				return device;
		}
		cur_devices = cur_devices->seed;
	}
	return NULL;
6255 6256
}

6257
static struct btrfs_device *add_missing_dev(struct btrfs_fs_devices *fs_devices,
6258 6259 6260 6261
					    u64 devid, u8 *dev_uuid)
{
	struct btrfs_device *device;

6262 6263
	device = btrfs_alloc_device(NULL, &devid, dev_uuid);
	if (IS_ERR(device))
6264
		return NULL;
6265 6266

	list_add(&device->dev_list, &fs_devices->devices);
Y
Yan Zheng 已提交
6267
	device->fs_devices = fs_devices;
6268
	fs_devices->num_devices++;
6269 6270

	device->missing = 1;
6271
	fs_devices->missing_devices++;
6272

6273 6274 6275
	return device;
}

6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295
/**
 * btrfs_alloc_device - allocate struct btrfs_device
 * @fs_info:	used only for generating a new devid, can be NULL if
 *		devid is provided (i.e. @devid != NULL).
 * @devid:	a pointer to devid for this device.  If NULL a new devid
 *		is generated.
 * @uuid:	a pointer to UUID for this device.  If NULL a new UUID
 *		is generated.
 *
 * Return: a pointer to a new &struct btrfs_device on success; ERR_PTR()
 * on error.  Returned struct is not linked onto any lists and can be
 * destroyed with kfree() right away.
 */
struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
					const u64 *devid,
					const u8 *uuid)
{
	struct btrfs_device *dev;
	u64 tmp;

6296
	if (WARN_ON(!devid && !fs_info))
6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320
		return ERR_PTR(-EINVAL);

	dev = __alloc_device();
	if (IS_ERR(dev))
		return dev;

	if (devid)
		tmp = *devid;
	else {
		int ret;

		ret = find_next_devid(fs_info, &tmp);
		if (ret) {
			kfree(dev);
			return ERR_PTR(ret);
		}
	}
	dev->devid = tmp;

	if (uuid)
		memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE);
	else
		generate_random_uuid(dev->uuid);

6321 6322
	btrfs_init_work(&dev->work, btrfs_submit_helper,
			pending_bios_fn, NULL, NULL);
6323 6324 6325 6326

	return dev;
}

6327
/* Return -EIO if any error, otherwise return 0. */
6328
static int btrfs_check_chunk_valid(struct btrfs_fs_info *fs_info,
6329 6330
				   struct extent_buffer *leaf,
				   struct btrfs_chunk *chunk, u64 logical)
6331 6332
{
	u64 length;
6333
	u64 stripe_len;
6334 6335 6336
	u16 num_stripes;
	u16 sub_stripes;
	u64 type;
6337

6338
	length = btrfs_chunk_length(leaf, chunk);
6339 6340
	stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
	num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
6341 6342 6343
	sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);
	type = btrfs_chunk_type(leaf, chunk);

6344
	if (!num_stripes) {
6345
		btrfs_err(fs_info, "invalid chunk num_stripes: %u",
6346 6347 6348
			  num_stripes);
		return -EIO;
	}
6349 6350
	if (!IS_ALIGNED(logical, fs_info->sectorsize)) {
		btrfs_err(fs_info, "invalid chunk logical %llu", logical);
6351 6352
		return -EIO;
	}
6353 6354
	if (btrfs_chunk_sector_size(leaf, chunk) != fs_info->sectorsize) {
		btrfs_err(fs_info, "invalid chunk sectorsize %u",
6355 6356 6357
			  btrfs_chunk_sector_size(leaf, chunk));
		return -EIO;
	}
6358 6359
	if (!length || !IS_ALIGNED(length, fs_info->sectorsize)) {
		btrfs_err(fs_info, "invalid chunk length %llu", length);
6360 6361
		return -EIO;
	}
6362
	if (!is_power_of_2(stripe_len) || stripe_len != BTRFS_STRIPE_LEN) {
6363
		btrfs_err(fs_info, "invalid chunk stripe length: %llu",
6364 6365 6366 6367
			  stripe_len);
		return -EIO;
	}
	if (~(BTRFS_BLOCK_GROUP_TYPE_MASK | BTRFS_BLOCK_GROUP_PROFILE_MASK) &
6368
	    type) {
6369
		btrfs_err(fs_info, "unrecognized chunk type: %llu",
6370 6371 6372 6373 6374
			  ~(BTRFS_BLOCK_GROUP_TYPE_MASK |
			    BTRFS_BLOCK_GROUP_PROFILE_MASK) &
			  btrfs_chunk_type(leaf, chunk));
		return -EIO;
	}
6375 6376 6377 6378 6379 6380 6381
	if ((type & BTRFS_BLOCK_GROUP_RAID10 && sub_stripes != 2) ||
	    (type & BTRFS_BLOCK_GROUP_RAID1 && num_stripes < 1) ||
	    (type & BTRFS_BLOCK_GROUP_RAID5 && num_stripes < 2) ||
	    (type & BTRFS_BLOCK_GROUP_RAID6 && num_stripes < 3) ||
	    (type & BTRFS_BLOCK_GROUP_DUP && num_stripes > 2) ||
	    ((type & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 &&
	     num_stripes != 1)) {
6382
		btrfs_err(fs_info,
6383 6384 6385 6386 6387 6388 6389 6390 6391
			"invalid num_stripes:sub_stripes %u:%u for profile %llu",
			num_stripes, sub_stripes,
			type & BTRFS_BLOCK_GROUP_PROFILE_MASK);
		return -EIO;
	}

	return 0;
}

6392
static int read_one_chunk(struct btrfs_fs_info *fs_info, struct btrfs_key *key,
6393 6394 6395
			  struct extent_buffer *leaf,
			  struct btrfs_chunk *chunk)
{
6396
	struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412
	struct map_lookup *map;
	struct extent_map *em;
	u64 logical;
	u64 length;
	u64 stripe_len;
	u64 devid;
	u8 uuid[BTRFS_UUID_SIZE];
	int num_stripes;
	int ret;
	int i;

	logical = key->offset;
	length = btrfs_chunk_length(leaf, chunk);
	stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
	num_stripes = btrfs_chunk_num_stripes(leaf, chunk);

6413
	ret = btrfs_check_chunk_valid(fs_info, leaf, chunk, logical);
6414 6415
	if (ret)
		return ret;
6416

6417
	read_lock(&map_tree->map_tree.lock);
6418
	em = lookup_extent_mapping(&map_tree->map_tree, logical, 1);
6419
	read_unlock(&map_tree->map_tree.lock);
6420 6421 6422 6423 6424 6425 6426 6427 6428

	/* already mapped? */
	if (em && em->start <= logical && em->start + em->len > logical) {
		free_extent_map(em);
		return 0;
	} else if (em) {
		free_extent_map(em);
	}

6429
	em = alloc_extent_map();
6430 6431
	if (!em)
		return -ENOMEM;
6432
	map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
6433 6434 6435 6436 6437
	if (!map) {
		free_extent_map(em);
		return -ENOMEM;
	}

6438
	set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
6439
	em->map_lookup = map;
6440 6441
	em->start = logical;
	em->len = length;
6442
	em->orig_start = 0;
6443
	em->block_start = 0;
C
Chris Mason 已提交
6444
	em->block_len = em->len;
6445

6446 6447 6448 6449 6450 6451
	map->num_stripes = num_stripes;
	map->io_width = btrfs_chunk_io_width(leaf, chunk);
	map->io_align = btrfs_chunk_io_align(leaf, chunk);
	map->sector_size = btrfs_chunk_sector_size(leaf, chunk);
	map->stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
	map->type = btrfs_chunk_type(leaf, chunk);
C
Chris Mason 已提交
6452
	map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);
6453 6454 6455 6456
	for (i = 0; i < num_stripes; i++) {
		map->stripes[i].physical =
			btrfs_stripe_offset_nr(leaf, chunk, i);
		devid = btrfs_stripe_devid_nr(leaf, chunk, i);
6457 6458 6459
		read_extent_buffer(leaf, uuid, (unsigned long)
				   btrfs_stripe_dev_uuid_nr(chunk, i),
				   BTRFS_UUID_SIZE);
6460
		map->stripes[i].dev = btrfs_find_device(fs_info, devid,
6461
							uuid, NULL);
6462
		if (!map->stripes[i].dev &&
6463
		    !btrfs_test_opt(fs_info, DEGRADED)) {
6464 6465 6466
			free_extent_map(em);
			return -EIO;
		}
6467 6468
		if (!map->stripes[i].dev) {
			map->stripes[i].dev =
6469 6470
				add_missing_dev(fs_info->fs_devices, devid,
						uuid);
6471 6472 6473 6474
			if (!map->stripes[i].dev) {
				free_extent_map(em);
				return -EIO;
			}
6475
			btrfs_warn(fs_info, "devid %llu uuid %pU is missing",
J
Jeff Mahoney 已提交
6476
				   devid, uuid);
6477 6478
		}
		map->stripes[i].dev->in_fs_metadata = 1;
6479 6480
	}

6481
	write_lock(&map_tree->map_tree.lock);
J
Josef Bacik 已提交
6482
	ret = add_extent_mapping(&map_tree->map_tree, em, 0);
6483
	write_unlock(&map_tree->map_tree.lock);
6484
	BUG_ON(ret); /* Tree corruption */
6485 6486 6487 6488 6489
	free_extent_map(em);

	return 0;
}

6490
static void fill_device_from_item(struct extent_buffer *leaf,
6491 6492 6493 6494 6495 6496
				 struct btrfs_dev_item *dev_item,
				 struct btrfs_device *device)
{
	unsigned long ptr;

	device->devid = btrfs_device_id(leaf, dev_item);
6497 6498
	device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item);
	device->total_bytes = device->disk_total_bytes;
6499
	device->commit_total_bytes = device->disk_total_bytes;
6500
	device->bytes_used = btrfs_device_bytes_used(leaf, dev_item);
6501
	device->commit_bytes_used = device->bytes_used;
6502 6503 6504 6505
	device->type = btrfs_device_type(leaf, dev_item);
	device->io_align = btrfs_device_io_align(leaf, dev_item);
	device->io_width = btrfs_device_io_width(leaf, dev_item);
	device->sector_size = btrfs_device_sector_size(leaf, dev_item);
6506
	WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID);
6507
	device->is_tgtdev_for_dev_replace = 0;
6508

6509
	ptr = btrfs_device_uuid(dev_item);
6510
	read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
6511 6512
}

6513
static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info,
6514
						  u8 *fsid)
Y
Yan Zheng 已提交
6515 6516 6517 6518
{
	struct btrfs_fs_devices *fs_devices;
	int ret;

6519
	BUG_ON(!mutex_is_locked(&uuid_mutex));
Y
Yan Zheng 已提交
6520

6521
	fs_devices = fs_info->fs_devices->seed;
Y
Yan Zheng 已提交
6522
	while (fs_devices) {
6523 6524 6525
		if (!memcmp(fs_devices->fsid, fsid, BTRFS_UUID_SIZE))
			return fs_devices;

Y
Yan Zheng 已提交
6526 6527 6528 6529 6530
		fs_devices = fs_devices->seed;
	}

	fs_devices = find_fsid(fsid);
	if (!fs_devices) {
6531
		if (!btrfs_test_opt(fs_info, DEGRADED))
6532 6533 6534 6535 6536 6537 6538 6539 6540
			return ERR_PTR(-ENOENT);

		fs_devices = alloc_fs_devices(fsid);
		if (IS_ERR(fs_devices))
			return fs_devices;

		fs_devices->seeding = 1;
		fs_devices->opened = 1;
		return fs_devices;
Y
Yan Zheng 已提交
6541
	}
Y
Yan Zheng 已提交
6542 6543

	fs_devices = clone_fs_devices(fs_devices);
6544 6545
	if (IS_ERR(fs_devices))
		return fs_devices;
Y
Yan Zheng 已提交
6546

6547
	ret = __btrfs_open_devices(fs_devices, FMODE_READ,
6548
				   fs_info->bdev_holder);
6549 6550
	if (ret) {
		free_fs_devices(fs_devices);
6551
		fs_devices = ERR_PTR(ret);
Y
Yan Zheng 已提交
6552
		goto out;
6553
	}
Y
Yan Zheng 已提交
6554 6555 6556

	if (!fs_devices->seeding) {
		__btrfs_close_devices(fs_devices);
Y
Yan Zheng 已提交
6557
		free_fs_devices(fs_devices);
6558
		fs_devices = ERR_PTR(-EINVAL);
Y
Yan Zheng 已提交
6559 6560 6561
		goto out;
	}

6562 6563
	fs_devices->seed = fs_info->fs_devices->seed;
	fs_info->fs_devices->seed = fs_devices;
Y
Yan Zheng 已提交
6564
out:
6565
	return fs_devices;
Y
Yan Zheng 已提交
6566 6567
}

6568
static int read_one_dev(struct btrfs_fs_info *fs_info,
6569 6570 6571
			struct extent_buffer *leaf,
			struct btrfs_dev_item *dev_item)
{
6572
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
6573 6574 6575
	struct btrfs_device *device;
	u64 devid;
	int ret;
Y
Yan Zheng 已提交
6576
	u8 fs_uuid[BTRFS_UUID_SIZE];
6577 6578
	u8 dev_uuid[BTRFS_UUID_SIZE];

6579
	devid = btrfs_device_id(leaf, dev_item);
6580
	read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
6581
			   BTRFS_UUID_SIZE);
6582
	read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
Y
Yan Zheng 已提交
6583 6584
			   BTRFS_UUID_SIZE);

6585
	if (memcmp(fs_uuid, fs_info->fsid, BTRFS_UUID_SIZE)) {
6586
		fs_devices = open_seed_devices(fs_info, fs_uuid);
6587 6588
		if (IS_ERR(fs_devices))
			return PTR_ERR(fs_devices);
Y
Yan Zheng 已提交
6589 6590
	}

6591
	device = btrfs_find_device(fs_info, devid, dev_uuid, fs_uuid);
6592
	if (!device) {
6593
		if (!btrfs_test_opt(fs_info, DEGRADED))
Y
Yan Zheng 已提交
6594 6595
			return -EIO;

6596
		device = add_missing_dev(fs_devices, devid, dev_uuid);
6597 6598
		if (!device)
			return -ENOMEM;
6599
		btrfs_warn(fs_info, "devid %llu uuid %pU missing",
6600
				devid, dev_uuid);
6601
	} else {
6602
		if (!device->bdev && !btrfs_test_opt(fs_info, DEGRADED))
6603 6604 6605
			return -EIO;

		if(!device->bdev && !device->missing) {
6606 6607 6608 6609 6610 6611
			/*
			 * this happens when a device that was properly setup
			 * in the device info lists suddenly goes bad.
			 * device->bdev is NULL, and so we have to set
			 * device->missing to one here
			 */
6612
			device->fs_devices->missing_devices++;
6613
			device->missing = 1;
Y
Yan Zheng 已提交
6614
		}
6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628

		/* Move the device to its own fs_devices */
		if (device->fs_devices != fs_devices) {
			ASSERT(device->missing);

			list_move(&device->dev_list, &fs_devices->devices);
			device->fs_devices->num_devices--;
			fs_devices->num_devices++;

			device->fs_devices->missing_devices--;
			fs_devices->missing_devices++;

			device->fs_devices = fs_devices;
		}
Y
Yan Zheng 已提交
6629 6630
	}

6631
	if (device->fs_devices != fs_info->fs_devices) {
Y
Yan Zheng 已提交
6632 6633 6634 6635
		BUG_ON(device->writeable);
		if (device->generation !=
		    btrfs_device_generation(leaf, dev_item))
			return -EINVAL;
6636
	}
6637 6638

	fill_device_from_item(leaf, dev_item, device);
6639
	device->in_fs_metadata = 1;
6640
	if (device->writeable && !device->is_tgtdev_for_dev_replace) {
Y
Yan Zheng 已提交
6641
		device->fs_devices->total_rw_bytes += device->total_bytes;
6642 6643
		spin_lock(&fs_info->free_chunk_lock);
		fs_info->free_chunk_space += device->total_bytes -
6644
			device->bytes_used;
6645
		spin_unlock(&fs_info->free_chunk_lock);
6646
	}
6647 6648 6649 6650
	ret = 0;
	return ret;
}

6651
int btrfs_read_sys_array(struct btrfs_fs_info *fs_info)
6652
{
6653
	struct btrfs_root *root = fs_info->tree_root;
6654
	struct btrfs_super_block *super_copy = fs_info->super_copy;
6655
	struct extent_buffer *sb;
6656 6657
	struct btrfs_disk_key *disk_key;
	struct btrfs_chunk *chunk;
6658 6659
	u8 *array_ptr;
	unsigned long sb_array_offset;
6660
	int ret = 0;
6661 6662 6663
	u32 num_stripes;
	u32 array_size;
	u32 len = 0;
6664
	u32 cur_offset;
6665
	u64 type;
6666
	struct btrfs_key key;
6667

6668
	ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize);
6669 6670 6671 6672 6673
	/*
	 * This will create extent buffer of nodesize, superblock size is
	 * fixed to BTRFS_SUPER_INFO_SIZE. If nodesize > sb size, this will
	 * overallocate but we can keep it as-is, only the first page is used.
	 */
6674
	sb = btrfs_find_create_tree_block(fs_info, BTRFS_SUPER_INFO_OFFSET);
6675 6676
	if (IS_ERR(sb))
		return PTR_ERR(sb);
6677
	set_extent_buffer_uptodate(sb);
6678
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, sb, 0);
6679
	/*
6680
	 * The sb extent buffer is artificial and just used to read the system array.
6681
	 * set_extent_buffer_uptodate() call does not properly mark all it's
6682 6683 6684 6685 6686 6687 6688 6689 6690
	 * pages up-to-date when the page is larger: extent does not cover the
	 * whole page and consequently check_page_uptodate does not find all
	 * the page's extents up-to-date (the hole beyond sb),
	 * write_extent_buffer then triggers a WARN_ON.
	 *
	 * Regular short extents go through mark_extent_buffer_dirty/writeback cycle,
	 * but sb spans only this function. Add an explicit SetPageUptodate call
	 * to silence the warning eg. on PowerPC 64.
	 */
6691
	if (PAGE_SIZE > BTRFS_SUPER_INFO_SIZE)
6692
		SetPageUptodate(sb->pages[0]);
6693

6694
	write_extent_buffer(sb, super_copy, 0, BTRFS_SUPER_INFO_SIZE);
6695 6696
	array_size = btrfs_super_sys_array_size(super_copy);

6697 6698 6699
	array_ptr = super_copy->sys_chunk_array;
	sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array);
	cur_offset = 0;
6700

6701 6702
	while (cur_offset < array_size) {
		disk_key = (struct btrfs_disk_key *)array_ptr;
6703 6704 6705 6706
		len = sizeof(*disk_key);
		if (cur_offset + len > array_size)
			goto out_short_read;

6707 6708
		btrfs_disk_key_to_cpu(&key, disk_key);

6709 6710 6711
		array_ptr += len;
		sb_array_offset += len;
		cur_offset += len;
6712

6713
		if (key.type == BTRFS_CHUNK_ITEM_KEY) {
6714
			chunk = (struct btrfs_chunk *)sb_array_offset;
6715 6716 6717 6718 6719 6720 6721 6722 6723
			/*
			 * At least one btrfs_chunk with one stripe must be
			 * present, exact stripe count check comes afterwards
			 */
			len = btrfs_chunk_item_size(1);
			if (cur_offset + len > array_size)
				goto out_short_read;

			num_stripes = btrfs_chunk_num_stripes(sb, chunk);
6724
			if (!num_stripes) {
6725 6726
				btrfs_err(fs_info,
					"invalid number of stripes %u in sys_array at offset %u",
6727 6728 6729 6730 6731
					num_stripes, cur_offset);
				ret = -EIO;
				break;
			}

6732 6733
			type = btrfs_chunk_type(sb, chunk);
			if ((type & BTRFS_BLOCK_GROUP_SYSTEM) == 0) {
6734
				btrfs_err(fs_info,
6735 6736 6737 6738 6739 6740
			    "invalid chunk type %llu in sys_array at offset %u",
					type, cur_offset);
				ret = -EIO;
				break;
			}

6741 6742 6743 6744
			len = btrfs_chunk_item_size(num_stripes);
			if (cur_offset + len > array_size)
				goto out_short_read;

6745
			ret = read_one_chunk(fs_info, &key, sb, chunk);
6746 6747
			if (ret)
				break;
6748
		} else {
6749 6750 6751
			btrfs_err(fs_info,
			    "unexpected item type %u in sys_array at offset %u",
				  (u32)key.type, cur_offset);
6752 6753
			ret = -EIO;
			break;
6754
		}
6755 6756 6757
		array_ptr += len;
		sb_array_offset += len;
		cur_offset += len;
6758
	}
6759
	clear_extent_buffer_uptodate(sb);
6760
	free_extent_buffer_stale(sb);
6761
	return ret;
6762 6763

out_short_read:
6764
	btrfs_err(fs_info, "sys_array too short to read %u bytes at offset %u",
6765
			len, cur_offset);
6766
	clear_extent_buffer_uptodate(sb);
6767
	free_extent_buffer_stale(sb);
6768
	return -EIO;
6769 6770
}

6771
int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
6772
{
6773
	struct btrfs_root *root = fs_info->chunk_root;
6774 6775 6776 6777 6778 6779
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	struct btrfs_key found_key;
	int ret;
	int slot;
6780
	u64 total_dev = 0;
6781 6782 6783 6784 6785

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

6786
	mutex_lock(&uuid_mutex);
6787
	lock_chunks(fs_info);
6788

6789 6790 6791 6792 6793
	/*
	 * Read all device items, and then all the chunk items. All
	 * device items are found before any chunk item (their object id
	 * is smaller than the lowest possible object id for a chunk
	 * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID).
6794 6795 6796 6797 6798
	 */
	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
	key.offset = 0;
	key.type = 0;
	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6799 6800
	if (ret < 0)
		goto error;
C
Chris Mason 已提交
6801
	while (1) {
6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812
		leaf = path->nodes[0];
		slot = path->slots[0];
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
				goto error;
			break;
		}
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
6813 6814 6815
		if (found_key.type == BTRFS_DEV_ITEM_KEY) {
			struct btrfs_dev_item *dev_item;
			dev_item = btrfs_item_ptr(leaf, slot,
6816
						  struct btrfs_dev_item);
6817
			ret = read_one_dev(fs_info, leaf, dev_item);
6818 6819
			if (ret)
				goto error;
6820
			total_dev++;
6821 6822 6823
		} else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
			struct btrfs_chunk *chunk;
			chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
6824
			ret = read_one_chunk(fs_info, &found_key, leaf, chunk);
Y
Yan Zheng 已提交
6825 6826
			if (ret)
				goto error;
6827 6828 6829
		}
		path->slots[0]++;
	}
6830 6831 6832 6833 6834

	/*
	 * After loading chunk tree, we've got all device information,
	 * do another round of validation checks.
	 */
6835 6836
	if (total_dev != fs_info->fs_devices->total_devices) {
		btrfs_err(fs_info,
6837
	   "super_num_devices %llu mismatch with num_devices %llu found here",
6838
			  btrfs_super_num_devices(fs_info->super_copy),
6839 6840 6841 6842
			  total_dev);
		ret = -EINVAL;
		goto error;
	}
6843 6844 6845
	if (btrfs_super_total_bytes(fs_info->super_copy) <
	    fs_info->fs_devices->total_rw_bytes) {
		btrfs_err(fs_info,
6846
	"super_total_bytes %llu mismatch with fs_devices total_rw_bytes %llu",
6847 6848
			  btrfs_super_total_bytes(fs_info->super_copy),
			  fs_info->fs_devices->total_rw_bytes);
6849 6850 6851
		ret = -EINVAL;
		goto error;
	}
6852 6853
	ret = 0;
error:
6854
	unlock_chunks(fs_info);
6855 6856
	mutex_unlock(&uuid_mutex);

Y
Yan Zheng 已提交
6857
	btrfs_free_path(path);
6858 6859
	return ret;
}
6860

6861 6862 6863 6864 6865
void btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
{
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	struct btrfs_device *device;

6866 6867 6868
	while (fs_devices) {
		mutex_lock(&fs_devices->device_list_mutex);
		list_for_each_entry(device, &fs_devices->devices, dev_list)
6869
			device->fs_info = fs_info;
6870 6871 6872 6873
		mutex_unlock(&fs_devices->device_list_mutex);

		fs_devices = fs_devices->seed;
	}
6874 6875
}

6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907
static void __btrfs_reset_dev_stats(struct btrfs_device *dev)
{
	int i;

	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
		btrfs_dev_stat_reset(dev, i);
}

int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
{
	struct btrfs_key key;
	struct btrfs_key found_key;
	struct btrfs_root *dev_root = fs_info->dev_root;
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	struct extent_buffer *eb;
	int slot;
	int ret = 0;
	struct btrfs_device *device;
	struct btrfs_path *path = NULL;
	int i;

	path = btrfs_alloc_path();
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}

	mutex_lock(&fs_devices->device_list_mutex);
	list_for_each_entry(device, &fs_devices->devices, dev_list) {
		int item_size;
		struct btrfs_dev_stats_item *ptr;

6908 6909
		key.objectid = BTRFS_DEV_STATS_OBJECTID;
		key.type = BTRFS_PERSISTENT_ITEM_KEY;
6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945
		key.offset = device->devid;
		ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
		if (ret) {
			__btrfs_reset_dev_stats(device);
			device->dev_stats_valid = 1;
			btrfs_release_path(path);
			continue;
		}
		slot = path->slots[0];
		eb = path->nodes[0];
		btrfs_item_key_to_cpu(eb, &found_key, slot);
		item_size = btrfs_item_size_nr(eb, slot);

		ptr = btrfs_item_ptr(eb, slot,
				     struct btrfs_dev_stats_item);

		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
			if (item_size >= (1 + i) * sizeof(__le64))
				btrfs_dev_stat_set(device, i,
					btrfs_dev_stats_value(eb, ptr, i));
			else
				btrfs_dev_stat_reset(device, i);
		}

		device->dev_stats_valid = 1;
		btrfs_dev_stat_print_on_load(device);
		btrfs_release_path(path);
	}
	mutex_unlock(&fs_devices->device_list_mutex);

out:
	btrfs_free_path(path);
	return ret < 0 ? ret : 0;
}

static int update_dev_stat_item(struct btrfs_trans_handle *trans,
6946
				struct btrfs_fs_info *fs_info,
6947 6948
				struct btrfs_device *device)
{
6949
	struct btrfs_root *dev_root = fs_info->dev_root;
6950 6951 6952 6953 6954 6955 6956
	struct btrfs_path *path;
	struct btrfs_key key;
	struct extent_buffer *eb;
	struct btrfs_dev_stats_item *ptr;
	int ret;
	int i;

6957 6958
	key.objectid = BTRFS_DEV_STATS_OBJECTID;
	key.type = BTRFS_PERSISTENT_ITEM_KEY;
6959 6960 6961 6962 6963 6964
	key.offset = device->devid;

	path = btrfs_alloc_path();
	BUG_ON(!path);
	ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
	if (ret < 0) {
6965
		btrfs_warn_in_rcu(fs_info,
6966
			"error %d while searching for dev_stats item for device %s",
6967
			      ret, rcu_str_deref(device->name));
6968 6969 6970 6971 6972 6973 6974 6975
		goto out;
	}

	if (ret == 0 &&
	    btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
		/* need to delete old one and insert a new one */
		ret = btrfs_del_item(trans, dev_root, path);
		if (ret != 0) {
6976
			btrfs_warn_in_rcu(fs_info,
6977
				"delete too small dev_stats item for device %s failed %d",
6978
				      rcu_str_deref(device->name), ret);
6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989
			goto out;
		}
		ret = 1;
	}

	if (ret == 1) {
		/* need to insert a new item */
		btrfs_release_path(path);
		ret = btrfs_insert_empty_item(trans, dev_root, path,
					      &key, sizeof(*ptr));
		if (ret < 0) {
6990
			btrfs_warn_in_rcu(fs_info,
6991 6992
				"insert dev_stats item for device %s failed %d",
				rcu_str_deref(device->name), ret);
6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016
			goto out;
		}
	}

	eb = path->nodes[0];
	ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item);
	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
		btrfs_set_dev_stats_value(eb, ptr, i,
					  btrfs_dev_stat_read(device, i));
	btrfs_mark_buffer_dirty(eb);

out:
	btrfs_free_path(path);
	return ret;
}

/*
 * called from commit_transaction. Writes all changed device stats to disk.
 */
int btrfs_run_dev_stats(struct btrfs_trans_handle *trans,
			struct btrfs_fs_info *fs_info)
{
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	struct btrfs_device *device;
7017
	int stats_cnt;
7018 7019 7020 7021
	int ret = 0;

	mutex_lock(&fs_devices->device_list_mutex);
	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7022
		if (!device->dev_stats_valid || !btrfs_dev_stats_dirty(device))
7023 7024
			continue;

7025
		stats_cnt = atomic_read(&device->dev_stats_ccnt);
7026
		ret = update_dev_stat_item(trans, fs_info, device);
7027
		if (!ret)
7028
			atomic_sub(stats_cnt, &device->dev_stats_ccnt);
7029 7030 7031 7032 7033 7034
	}
	mutex_unlock(&fs_devices->device_list_mutex);

	return ret;
}

7035 7036 7037 7038 7039 7040
void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
{
	btrfs_dev_stat_inc(dev, index);
	btrfs_dev_stat_print_on_error(dev);
}

7041
static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev)
7042
{
7043 7044
	if (!dev->dev_stats_valid)
		return;
7045
	btrfs_err_rl_in_rcu(dev->fs_info,
7046
		"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7047
			   rcu_str_deref(dev->name),
7048 7049 7050
			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7051 7052
			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7053
}
7054

7055 7056
static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev)
{
7057 7058 7059 7060 7061 7062 7063 7064
	int i;

	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
		if (btrfs_dev_stat_read(dev, i) != 0)
			break;
	if (i == BTRFS_DEV_STAT_VALUES_MAX)
		return; /* all values == 0, suppress message */

7065
	btrfs_info_in_rcu(dev->fs_info,
7066
		"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7067
	       rcu_str_deref(dev->name),
7068 7069 7070 7071 7072 7073 7074
	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
}

7075
int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info,
7076
			struct btrfs_ioctl_get_dev_stats *stats)
7077 7078
{
	struct btrfs_device *dev;
7079
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7080 7081 7082
	int i;

	mutex_lock(&fs_devices->device_list_mutex);
7083
	dev = btrfs_find_device(fs_info, stats->devid, NULL, NULL);
7084 7085 7086
	mutex_unlock(&fs_devices->device_list_mutex);

	if (!dev) {
7087
		btrfs_warn(fs_info, "get dev_stats failed, device not found");
7088
		return -ENODEV;
7089
	} else if (!dev->dev_stats_valid) {
7090
		btrfs_warn(fs_info, "get dev_stats failed, not yet valid");
7091
		return -ENODEV;
7092
	} else if (stats->flags & BTRFS_DEV_STATS_RESET) {
7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108
		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
			if (stats->nr_items > i)
				stats->values[i] =
					btrfs_dev_stat_read_and_reset(dev, i);
			else
				btrfs_dev_stat_reset(dev, i);
		}
	} else {
		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
			if (stats->nr_items > i)
				stats->values[i] = btrfs_dev_stat_read(dev, i);
	}
	if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX)
		stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX;
	return 0;
}
7109

7110
void btrfs_scratch_superblocks(struct block_device *bdev, char *device_path)
7111 7112 7113
{
	struct buffer_head *bh;
	struct btrfs_super_block *disk_super;
7114
	int copy_num;
7115

7116 7117
	if (!bdev)
		return;
7118

7119 7120
	for (copy_num = 0; copy_num < BTRFS_SUPER_MIRROR_MAX;
		copy_num++) {
7121

7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137
		if (btrfs_read_dev_one_super(bdev, copy_num, &bh))
			continue;

		disk_super = (struct btrfs_super_block *)bh->b_data;

		memset(&disk_super->magic, 0, sizeof(disk_super->magic));
		set_buffer_dirty(bh);
		sync_dirty_buffer(bh);
		brelse(bh);
	}

	/* Notify udev that device has changed */
	btrfs_kobject_uevent(bdev, KOBJ_CHANGE);

	/* Update ctime/mtime for device path for libblkid */
	update_dev_time(device_path);
7138
}
7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152

/*
 * Update the size of all devices, which is used for writing out the
 * super blocks.
 */
void btrfs_update_commit_device_size(struct btrfs_fs_info *fs_info)
{
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	struct btrfs_device *curr, *next;

	if (list_empty(&fs_devices->resized_devices))
		return;

	mutex_lock(&fs_devices->device_list_mutex);
7153
	lock_chunks(fs_info);
7154 7155 7156 7157 7158
	list_for_each_entry_safe(curr, next, &fs_devices->resized_devices,
				 resized_list) {
		list_del_init(&curr->resized_list);
		curr->commit_total_bytes = curr->disk_total_bytes;
	}
7159
	unlock_chunks(fs_info);
7160 7161
	mutex_unlock(&fs_devices->device_list_mutex);
}
7162 7163

/* Must be invoked during the transaction commit */
7164
void btrfs_update_commit_device_bytes_used(struct btrfs_fs_info *fs_info,
7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175
					struct btrfs_transaction *transaction)
{
	struct extent_map *em;
	struct map_lookup *map;
	struct btrfs_device *dev;
	int i;

	if (list_empty(&transaction->pending_chunks))
		return;

	/* In order to kick the device replace finish process */
7176
	lock_chunks(fs_info);
7177
	list_for_each_entry(em, &transaction->pending_chunks, list) {
7178
		map = em->map_lookup;
7179 7180 7181 7182 7183 7184

		for (i = 0; i < map->num_stripes; i++) {
			dev = map->stripes[i].dev;
			dev->commit_bytes_used = dev->bytes_used;
		}
	}
7185
	unlock_chunks(fs_info);
7186
}
7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204

void btrfs_set_fs_info_ptr(struct btrfs_fs_info *fs_info)
{
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	while (fs_devices) {
		fs_devices->fs_info = fs_info;
		fs_devices = fs_devices->seed;
	}
}

void btrfs_reset_fs_info_ptr(struct btrfs_fs_info *fs_info)
{
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	while (fs_devices) {
		fs_devices->fs_info = NULL;
		fs_devices = fs_devices->seed;
	}
}