extent-tree.c 228.0 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
C
Chris Mason 已提交
26
#include "compat.h"
27
#include "hash.h"
28 29 30
#include "ctree.h"
#include "disk-io.h"
#include "print-tree.h"
31
#include "transaction.h"
32
#include "volumes.h"
33
#include "locking.h"
34
#include "free-space-cache.h"
35

36 37
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
38 39 40
			      u64 bytenr, u64 num_bytes, int alloc);
static int update_reserved_bytes(struct btrfs_block_group_cache *cache,
				 u64 num_bytes, int reserve, int sinfo);
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extra_op);
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins);
J
Josef Bacik 已提交
60 61 62
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
			  u64 flags, int force);
63 64
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
65 66
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
J
Josef Bacik 已提交
67

J
Josef Bacik 已提交
68 69 70 71 72 73 74
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
	return cache->cached == BTRFS_CACHE_FINISHED;
}

J
Josef Bacik 已提交
75 76 77 78 79
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

80 81 82 83 84 85 86
void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
87 88 89 90
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
		WARN_ON(cache->reserved_pinned > 0);
91
		kfree(cache);
92
	}
93 94
}

J
Josef Bacik 已提交
95 96 97 98
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
99
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
167
	if (ret)
168
		btrfs_get_block_group(ret);
J
Josef Bacik 已提交
169 170 171 172 173
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

174 175
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
176
{
177 178 179 180 181 182 183
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
J
Josef Bacik 已提交
184

185 186 187 188
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
189

190 191 192 193 194 195 196
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
J
Josef Bacik 已提交
197 198
}

199 200
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
201 202 203 204 205 206
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

207 208 209 210 211 212 213 214
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
		BUG_ON(ret);
	}

J
Josef Bacik 已提交
215 216 217 218 219 220
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
		BUG_ON(ret);
221

J
Josef Bacik 已提交
222
		while (nr--) {
223
			cache->bytes_super += stripe_len;
224 225 226
			ret = add_excluded_extent(root, logical[nr],
						  stripe_len);
			BUG_ON(ret);
J
Josef Bacik 已提交
227
		}
228

J
Josef Bacik 已提交
229 230 231 232 233
		kfree(logical);
	}
	return 0;
}

234 235 236 237 238 239 240 241 242 243 244
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_STARTED) {
		spin_unlock(&cache->lock);
		return NULL;
	}

245 246 247
	/* We're loading it the fast way, so we don't have a caching_ctl. */
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

J
Josef Bacik 已提交
263 264 265 266 267
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
J
Josef Bacik 已提交
268
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
J
Josef Bacik 已提交
269 270
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
J
Josef Bacik 已提交
271
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
272 273 274
	int ret;

	while (start < end) {
275
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
276
					    &extent_start, &extent_end,
277
					    EXTENT_DIRTY | EXTENT_UPTODATE);
J
Josef Bacik 已提交
278 279 280
		if (ret)
			break;

281
		if (extent_start <= start) {
J
Josef Bacik 已提交
282 283 284
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
285
			total_added += size;
286 287
			ret = btrfs_add_free_space(block_group, start,
						   size);
J
Josef Bacik 已提交
288 289 290 291 292 293 294 295 296
			BUG_ON(ret);
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
297
		total_added += size;
298
		ret = btrfs_add_free_space(block_group, start, size);
J
Josef Bacik 已提交
299 300 301
		BUG_ON(ret);
	}

J
Josef Bacik 已提交
302
	return total_added;
J
Josef Bacik 已提交
303 304
}

J
Josef Bacik 已提交
305
static int caching_kthread(void *data)
306
{
J
Josef Bacik 已提交
307 308
	struct btrfs_block_group_cache *block_group = data;
	struct btrfs_fs_info *fs_info = block_group->fs_info;
309 310
	struct btrfs_caching_control *caching_ctl = block_group->caching_ctl;
	struct btrfs_root *extent_root = fs_info->extent_root;
311
	struct btrfs_path *path;
312
	struct extent_buffer *leaf;
313
	struct btrfs_key key;
J
Josef Bacik 已提交
314
	u64 total_found = 0;
315 316 317
	u64 last = 0;
	u32 nritems;
	int ret = 0;
318

319 320 321
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
322

J
Josef Bacik 已提交
323
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
324

325
	/*
J
Josef Bacik 已提交
326 327 328 329
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
330 331
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
332 333 334
	path->search_commit_root = 1;
	path->reada = 2;

Y
Yan Zheng 已提交
335
	key.objectid = last;
336
	key.offset = 0;
337
	key.type = BTRFS_EXTENT_ITEM_KEY;
338
again:
339
	mutex_lock(&caching_ctl->mutex);
340 341 342
	/* need to make sure the commit_root doesn't disappear */
	down_read(&fs_info->extent_commit_sem);

343
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
344
	if (ret < 0)
345
		goto err;
Y
Yan Zheng 已提交
346

347 348 349
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
350
	while (1) {
J
Josef Bacik 已提交
351
		smp_mb();
352
		if (fs_info->closing > 1) {
353
			last = (u64)-1;
J
Josef Bacik 已提交
354
			break;
355
		}
J
Josef Bacik 已提交
356

357 358 359 360 361
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
362
				break;
J
Josef Bacik 已提交
363

364 365 366 367 368
			caching_ctl->progress = last;
			btrfs_release_path(extent_root, path);
			up_read(&fs_info->extent_commit_sem);
			mutex_unlock(&caching_ctl->mutex);
			if (btrfs_transaction_in_commit(fs_info))
369
				schedule_timeout(1);
370 371 372 373
			else
				cond_resched();
			goto again;
		}
J
Josef Bacik 已提交
374

375 376
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
377
			continue;
378
		}
J
Josef Bacik 已提交
379

380
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
381
		    block_group->key.offset)
382
			break;
383

384
		if (key.type == BTRFS_EXTENT_ITEM_KEY) {
J
Josef Bacik 已提交
385 386 387
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
388
			last = key.objectid + key.offset;
J
Josef Bacik 已提交
389

390 391 392 393
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
J
Josef Bacik 已提交
394
		}
395 396
		path->slots[0]++;
	}
J
Josef Bacik 已提交
397
	ret = 0;
398

J
Josef Bacik 已提交
399 400 401
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
402
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
403 404

	spin_lock(&block_group->lock);
405
	block_group->caching_ctl = NULL;
J
Josef Bacik 已提交
406 407
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
408

409
err:
410
	btrfs_free_path(path);
411
	up_read(&fs_info->extent_commit_sem);
J
Josef Bacik 已提交
412

413 414 415 416 417 418 419
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
	atomic_dec(&block_group->space_info->caching_threads);
420 421
	btrfs_put_block_group(block_group);

J
Josef Bacik 已提交
422 423 424
	return 0;
}

425 426
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     struct btrfs_trans_handle *trans,
427
			     struct btrfs_root *root,
428
			     int load_cache_only)
J
Josef Bacik 已提交
429
{
430 431
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
432 433 434
	struct task_struct *tsk;
	int ret = 0;

435 436 437 438
	smp_mb();
	if (cache->cached != BTRFS_CACHE_NO)
		return 0;

439 440
	/*
	 * We can't do the read from on-disk cache during a commit since we need
441 442 443
	 * to have the normal tree locking.  Also if we are currently trying to
	 * allocate blocks for the tree root we can't do the fast caching since
	 * we likely hold important locks.
444
	 */
445 446
	if (!trans->transaction->in_commit &&
	    (root && root != root->fs_info->tree_root)) {
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
		spin_lock(&cache->lock);
		if (cache->cached != BTRFS_CACHE_NO) {
			spin_unlock(&cache->lock);
			return 0;
		}
		cache->cached = BTRFS_CACHE_STARTED;
		spin_unlock(&cache->lock);

		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
		} else {
			cache->cached = BTRFS_CACHE_NO;
		}
		spin_unlock(&cache->lock);
465 466
		if (ret == 1) {
			free_excluded_extents(fs_info->extent_root, cache);
467
			return 0;
468
		}
469 470 471 472 473
	}

	if (load_cache_only)
		return 0;

474 475 476 477 478 479 480 481 482 483 484
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_KERNEL);
	BUG_ON(!caching_ctl);

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	/* one for caching kthread, one for caching block group list */
	atomic_set(&caching_ctl->count, 2);

J
Josef Bacik 已提交
485 486 487
	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
488 489
		kfree(caching_ctl);
		return 0;
J
Josef Bacik 已提交
490
	}
491
	cache->caching_ctl = caching_ctl;
J
Josef Bacik 已提交
492 493 494
	cache->cached = BTRFS_CACHE_STARTED;
	spin_unlock(&cache->lock);

495 496 497 498 499
	down_write(&fs_info->extent_commit_sem);
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
	up_write(&fs_info->extent_commit_sem);

	atomic_inc(&cache->space_info->caching_threads);
500
	btrfs_get_block_group(cache);
501

J
Josef Bacik 已提交
502 503 504 505 506 507 508 509
	tsk = kthread_run(caching_kthread, cache, "btrfs-cache-%llu\n",
			  cache->key.objectid);
	if (IS_ERR(tsk)) {
		ret = PTR_ERR(tsk);
		printk(KERN_ERR "error running thread %d\n", ret);
		BUG();
	}

510
	return ret;
511 512
}

J
Josef Bacik 已提交
513 514 515
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
516 517
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
518
{
J
Josef Bacik 已提交
519
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
520

J
Josef Bacik 已提交
521
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
522

J
Josef Bacik 已提交
523
	return cache;
C
Chris Mason 已提交
524 525
}

J
Josef Bacik 已提交
526
/*
527
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
528
 */
C
Chris Mason 已提交
529 530 531
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
532
{
J
Josef Bacik 已提交
533
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
534

J
Josef Bacik 已提交
535
	cache = block_group_cache_tree_search(info, bytenr, 1);
536

J
Josef Bacik 已提交
537
	return cache;
C
Chris Mason 已提交
538
}
539

J
Josef Bacik 已提交
540 541
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
542
{
J
Josef Bacik 已提交
543 544
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
545

546 547 548
	flags &= BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_SYSTEM |
		 BTRFS_BLOCK_GROUP_METADATA;

549 550
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
551
		if (found->flags & flags) {
552
			rcu_read_unlock();
J
Josef Bacik 已提交
553
			return found;
554
		}
J
Josef Bacik 已提交
555
	}
556
	rcu_read_unlock();
J
Josef Bacik 已提交
557
	return NULL;
558 559
}

560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

575 576 577 578 579 580 581 582 583
static u64 div_factor(u64 num, int factor)
{
	if (factor == 10)
		return num;
	num *= factor;
	do_div(num, 10);
	return num;
}

584 585 586 587 588 589 590 591 592
static u64 div_factor_fine(u64 num, int factor)
{
	if (factor == 100)
		return num;
	num *= factor;
	do_div(num, 100);
	return num;
}

593 594
u64 btrfs_find_block_group(struct btrfs_root *root,
			   u64 search_start, u64 search_hint, int owner)
C
Chris Mason 已提交
595
{
596
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
597
	u64 used;
598 599
	u64 last = max(search_hint, search_start);
	u64 group_start = 0;
600
	int full_search = 0;
601
	int factor = 9;
C
Chris Mason 已提交
602
	int wrapped = 0;
603
again:
604 605
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
J
Josef Bacik 已提交
606 607
		if (!cache)
			break;
608

609
		spin_lock(&cache->lock);
610 611 612
		last = cache->key.objectid + cache->key.offset;
		used = btrfs_block_group_used(&cache->item);

613 614
		if ((full_search || !cache->ro) &&
		    block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
615
			if (used + cache->pinned + cache->reserved <
616 617
			    div_factor(cache->key.offset, factor)) {
				group_start = cache->key.objectid;
618
				spin_unlock(&cache->lock);
619
				btrfs_put_block_group(cache);
620 621
				goto found;
			}
622
		}
623
		spin_unlock(&cache->lock);
624
		btrfs_put_block_group(cache);
625
		cond_resched();
C
Chris Mason 已提交
626
	}
C
Chris Mason 已提交
627 628 629 630 631 632
	if (!wrapped) {
		last = search_start;
		wrapped = 1;
		goto again;
	}
	if (!full_search && factor < 10) {
C
Chris Mason 已提交
633
		last = search_start;
634
		full_search = 1;
C
Chris Mason 已提交
635
		factor = 10;
636 637
		goto again;
	}
C
Chris Mason 已提交
638
found:
639
	return group_start;
640
}
J
Josef Bacik 已提交
641

642
/* simple helper to search for an existing extent at a given offset */
Z
Zheng Yan 已提交
643
int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
644 645 646
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
647
	struct btrfs_path *path;
648

Z
Zheng Yan 已提交
649 650
	path = btrfs_alloc_path();
	BUG_ON(!path);
651 652 653 654 655
	key.objectid = start;
	key.offset = len;
	btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
656
	btrfs_free_path(path);
657 658 659
	return ret;
}

660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766
/*
 * helper function to lookup reference count and flags of extent.
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
			     u64 num_bytes, u64 *refs, u64 *flags)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = num_bytes;
	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
again:
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

			btrfs_release_path(root->fs_info->extent_root, path);

			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
			goto again;
		}
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

767 768 769 770 771 772 773 774 775 776 777 778 779 780
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
826 827 828
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
829
 * - different files inside a single subvolume
830 831
 * - different offsets inside a file (bookend extents in file.c)
 *
832
 * The extent ref structure for the implicit back refs has fields for:
833 834 835
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
836 837
 * - original offset in the file
 * - how many bookend extents
838
 *
839 840
 * The key offset for the implicit back refs is hash of the first
 * three fields.
841
 *
842
 * The extent ref structure for the full back refs has field for:
843
 *
844
 * - number of pointers in the tree leaf
845
 *
846 847
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
848
 *
849 850
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
851
 *
852
 *     (root_key.objectid, inode objectid, offset in file, 1)
853
 *
854 855
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
856
 *
857
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
858
 *
859
 * Btree extents can be referenced by:
860
 *
861
 * - Different subvolumes
862
 *
863 864 865 866
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
867
 *
868 869 870
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
871
 */
Z
Zheng Yan 已提交
872

873 874 875 876 877
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
878
{
879 880 881 882 883
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
884
	struct btrfs_key key;
885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
				BUG_ON(ret > 0);
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
	btrfs_release_path(root, path);

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
	BUG_ON(ret);

	ret = btrfs_extend_item(trans, root, path, new_size);
	BUG_ON(ret);

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
963
	high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
964
	lenum = cpu_to_le64(owner);
965
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
966
	lenum = cpu_to_le64(offset);
967
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1000
	struct extent_buffer *leaf;
1001
	u32 nritems;
1002
	int ret;
1003 1004
	int recow;
	int err = -ENOENT;
1005

Z
Zheng Yan 已提交
1006
	key.objectid = bytenr;
1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1022

1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		btrfs_release_path(root, path);
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1038 1039 1040
	}

	leaf = path->nodes[0];
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
				btrfs_release_path(root, path);
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1073
	}
1074 1075
fail:
	return err;
Z
Zheng Yan 已提交
1076 1077
}

1078 1079 1080 1081 1082 1083
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1084 1085 1086
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1087
	u32 size;
Z
Zheng Yan 已提交
1088 1089
	u32 num_refs;
	int ret;
1090 1091

	key.objectid = bytenr;
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1102

1103 1104 1105 1106 1107 1108 1109
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1110
		ref = btrfs_item_ptr(leaf, path->slots[0],
1111 1112 1113 1114 1115 1116 1117
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1118
		}
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
			btrfs_release_path(root, path);
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1133

1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1148 1149
		}
	}
1150 1151 1152
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1153 1154
	btrfs_release_path(root, path);
	return ret;
1155 1156
}

1157 1158 1159 1160
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   int refs_to_drop)
Z
Zheng Yan 已提交
1161
{
1162 1163 1164
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1165
	struct extent_buffer *leaf;
1166
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1167 1168 1169
	int ret = 0;

	leaf = path->nodes[0];
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1191 1192
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1193

Z
Zheng Yan 已提交
1194 1195 1196
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
	} else {
1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1209 1210 1211 1212 1213
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1214 1215 1216
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1217
{
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1249
#endif
1250 1251 1252 1253 1254
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1255

1256 1257 1258 1259 1260
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1261
{
1262
	struct btrfs_key key;
1263 1264
	int ret;

1265 1266 1267 1268 1269 1270 1271
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1272 1273
	}

1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
		btrfs_release_path(root, path);
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1285
#endif
1286
	return ret;
1287 1288
}

1289 1290 1291 1292 1293
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1294
{
1295
	struct btrfs_key key;
Z
Zheng Yan 已提交
1296 1297
	int ret;

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
	btrfs_release_path(root, path);
Z
Zheng Yan 已提交
1309 1310 1311
	return ret;
}

1312
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1313
{
1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1327
}
1328

1329 1330
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1331

C
Chris Mason 已提交
1332
{
1333
	for (; level < BTRFS_MAX_LEVEL; level++) {
1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1349

1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1385

1386
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1387
	key.type = BTRFS_EXTENT_ITEM_KEY;
1388
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1389

1390 1391 1392
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1393
		path->keep_locks = 1;
1394 1395 1396
	} else
		extra_size = -1;
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1397
	if (ret < 0) {
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
		err = ret;
		goto out;
	}
	BUG_ON(ret);

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	} else {
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1495 1496
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1497
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1498 1499 1500 1501 1502 1503
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1504
	if (insert) {
1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
int setup_inline_extent_backref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_path *path,
				struct btrfs_extent_inline_ref *iref,
				u64 parent, u64 root_objectid,
				u64 owner, u64 offset, int refs_to_add,
				struct btrfs_delayed_extent_op *extent_op)
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;
	int ret;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

	ret = btrfs_extend_item(trans, root, path, size);
	BUG_ON(ret);

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1592
		return ret;
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602

	btrfs_release_path(root, path);
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1603
	}
1604 1605
	return ret;
}
Z
Zheng Yan 已提交
1606

1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
int update_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_mod,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	int ret;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1650
	}
Z
Zheng Yan 已提交
1651

1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
		ret = btrfs_truncate_item(trans, root, path, item_size, 1);
		BUG_ON(ret);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
		ret = update_inline_extent_backref(trans, root, path, iref,
						   refs_to_add, extent_op);
	} else if (ret == -ENOENT) {
		ret = setup_inline_extent_backref(trans, root, path, iref,
						  parent, root_objectid,
						  owner, offset, refs_to_add,
						  extent_op);
1700
	}
1701 1702
	return ret;
}
Z
Zheng Yan 已提交
1703

1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1722

1723 1724 1725 1726 1727 1728 1729
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_drop, int is_data)
{
	int ret;
1730

1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
		ret = update_inline_extent_backref(trans, root, path, iref,
						   -refs_to_drop, NULL);
	} else if (is_data) {
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
	} else {
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

static void btrfs_issue_discard(struct block_device *bdev,
				u64 start, u64 len)
{
1746
	blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_KERNEL, 0);
1747 1748 1749 1750 1751 1752 1753 1754 1755
}

static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
				u64 num_bytes)
{
	int ret;
	u64 map_length = num_bytes;
	struct btrfs_multi_bio *multi = NULL;

C
Christoph Hellwig 已提交
1756 1757 1758
	if (!btrfs_test_opt(root, DISCARD))
		return 0;

1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839
	/* Tell the block device(s) that the sectors can be discarded */
	ret = btrfs_map_block(&root->fs_info->mapping_tree, READ,
			      bytenr, &map_length, &multi, 0);
	if (!ret) {
		struct btrfs_bio_stripe *stripe = multi->stripes;
		int i;

		if (map_length > num_bytes)
			map_length = num_bytes;

		for (i = 0; i < multi->num_stripes; i++, stripe++) {
			btrfs_issue_discard(stripe->dev->bdev,
					    stripe->physical,
					    map_length);
		}
		kfree(multi);
	}

	return ret;
}

int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
			 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
					parent, root_objectid, (int)owner,
					BTRFS_ADD_DELAYED_REF, NULL);
	} else {
		ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
					parent, root_objectid, owner, offset,
					BTRFS_ADD_DELAYED_REF, NULL);
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
	u64 refs;
	int ret;
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
	ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
					   path, bytenr, num_bytes, parent,
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
	if (ret == 0)
		goto out;

	if (ret != -EAGAIN) {
		err = ret;
		goto out;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
1840

1841
	btrfs_mark_buffer_dirty(leaf);
1842 1843 1844
	btrfs_release_path(root->fs_info->extent_root, path);

	path->reada = 1;
1845 1846
	path->leave_spinning = 1;

1847 1848
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
1849 1850
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
1851
	BUG_ON(ret);
1852
out:
1853
	btrfs_free_path(path);
1854
	return err;
1855 1856
}

1857 1858 1859 1860 1861
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
1862
{
1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		if (extent_op) {
			BUG_ON(extent_op->update_key);
			flags |= extent_op->flags_to_set;
		}
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
					     extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
					  extent_op);
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
1935
	int ret;
1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = node->num_bytes;

	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
		err = -EIO;
		goto out;
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
1976

1977 1978 1979 1980
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
1981 1982
}

1983 1984 1985 1986 1987
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
1988 1989
{
	int ret = 0;
1990 1991 1992 1993
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
1994

1995 1996 1997
	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;
1998

1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024
	ref = btrfs_delayed_node_to_tree_ref(node);
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		BUG_ON(!extent_op || !extent_op->update_flags ||
		       !extent_op->update_key);
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
						ref->level, &ins);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
					     ref->level, 0, 1, extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
					  ref->level, 0, 1, extent_op);
	} else {
		BUG();
	}
2025 2026 2027 2028
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2029 2030 2031 2032 2033
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2034 2035
{
	int ret;
2036
	if (btrfs_delayed_ref_is_head(node)) {
2037 2038 2039 2040 2041 2042 2043
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2044 2045
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2046
		if (insert_reserved) {
2047 2048
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2049 2050 2051 2052 2053 2054
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
				BUG_ON(ret);
			}
2055 2056 2057 2058 2059
		}
		mutex_unlock(&head->mutex);
		return 0;
	}

2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_node *ref;
	int action = BTRFS_ADD_DELAYED_REF;
again:
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
	node = rb_prev(&head->node.rb_node);
	while (1) {
		if (!node)
			break;
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
		if (ref->bytenr != head->node.bytenr)
			break;
2093
		if (ref->action == action)
2094 2095 2096 2097 2098 2099 2100 2101 2102 2103
			return ref;
		node = rb_prev(node);
	}
	if (action == BTRFS_ADD_DELAYED_REF) {
		action = BTRFS_DROP_DELAYED_REF;
		goto again;
	}
	return NULL;
}

2104 2105 2106
static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root,
				       struct list_head *cluster)
2107 2108 2109 2110
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2111
	struct btrfs_delayed_extent_op *extent_op;
2112
	int ret;
2113
	int count = 0;
2114 2115 2116 2117 2118
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2119 2120
			/* pick a new head ref from the cluster list */
			if (list_empty(cluster))
2121 2122
				break;

2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139
			locked_ref = list_entry(cluster->next,
				     struct btrfs_delayed_ref_head, cluster);

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);

			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2140 2141
			}
		}
2142

2143 2144 2145 2146 2147 2148
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2149

2150 2151 2152
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);
		if (!ref) {
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182

			if (extent_op && must_insert_reserved) {
				kfree(extent_op);
				extent_op = NULL;
			}

			if (extent_op) {
				spin_unlock(&delayed_refs->lock);

				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
				BUG_ON(ret);
				kfree(extent_op);

				cond_resched();
				spin_lock(&delayed_refs->lock);
				continue;
			}

2183
			list_del_init(&locked_ref->cluster);
2184 2185
			locked_ref = NULL;
		}
C
Chris Mason 已提交
2186

2187 2188 2189
		ref->in_tree = 0;
		rb_erase(&ref->rb_node, &delayed_refs->root);
		delayed_refs->num_entries--;
2190

2191
		spin_unlock(&delayed_refs->lock);
2192

2193
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2194 2195
					  must_insert_reserved);
		BUG_ON(ret);
2196

2197 2198
		btrfs_put_delayed_ref(ref);
		kfree(extent_op);
2199
		count++;
2200

2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239
		cond_resched();
		spin_lock(&delayed_refs->lock);
	}
	return count;
}

/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct list_head cluster;
	int ret;
	int run_all = count == (unsigned long)-1;
	int run_most = 0;

	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

	delayed_refs = &trans->transaction->delayed_refs;
	INIT_LIST_HEAD(&cluster);
again:
	spin_lock(&delayed_refs->lock);
	if (count == 0) {
		count = delayed_refs->num_entries * 2;
		run_most = 1;
	}
	while (1) {
		if (!(run_all || run_most) &&
		    delayed_refs->num_heads_ready < 64)
			break;
2240

2241
		/*
2242 2243 2244 2245
		 * go find something we can process in the rbtree.  We start at
		 * the beginning of the tree, and then build a cluster
		 * of refs to process starting at the first one we are able to
		 * lock
2246
		 */
2247 2248 2249
		ret = btrfs_find_ref_cluster(trans, &cluster,
					     delayed_refs->run_delayed_start);
		if (ret)
2250 2251
			break;

2252 2253 2254 2255 2256 2257 2258
		ret = run_clustered_refs(trans, root, &cluster);
		BUG_ON(ret < 0);

		count -= min_t(unsigned long, ret, count);

		if (count == 0)
			break;
2259
	}
2260

2261 2262
	if (run_all) {
		node = rb_first(&delayed_refs->root);
2263
		if (!node)
2264
			goto out;
2265
		count = (unsigned long)-1;
2266

2267 2268 2269 2270 2271
		while (node) {
			ref = rb_entry(node, struct btrfs_delayed_ref_node,
				       rb_node);
			if (btrfs_delayed_ref_is_head(ref)) {
				struct btrfs_delayed_ref_head *head;
2272

2273 2274 2275 2276 2277 2278 2279 2280
				head = btrfs_delayed_node_to_head(ref);
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2281
				cond_resched();
2282 2283 2284 2285 2286 2287 2288
				goto again;
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
		schedule_timeout(1);
		goto again;
2289
	}
2290
out:
2291
	spin_unlock(&delayed_refs->lock);
2292 2293 2294
	return 0;
}

2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
				int is_data)
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

	extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;

	ret = btrfs_add_delayed_extent_op(trans, bytenr, num_bytes, extent_op);
	if (ret)
		kfree(extent_op);
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	ret = -ENOENT;
	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

		btrfs_release_path(root->fs_info->extent_root, path);

		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out_unlock;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	if (ref->bytenr != bytenr)
		goto out_unlock;

	ret = 1;
	if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
		goto out_unlock;

	data_ref = btrfs_delayed_node_to_data_ref(ref);

	node = rb_prev(node);
	if (node) {
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
		if (ref->bytenr == bytenr)
			goto out_unlock;
	}

	if (data_ref->root != root->root_key.objectid ||
	    data_ref->objectid != objectid || data_ref->offset != offset)
		goto out_unlock;

	ret = 0;
out_unlock:
	mutex_unlock(&head->mutex);
out:
	spin_unlock(&delayed_refs->lock);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2387 2388
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2389
	struct extent_buffer *leaf;
2390 2391 2392
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2393
	struct btrfs_key key;
2394
	u32 item_size;
2395
	int ret;
2396

2397
	key.objectid = bytenr;
Z
Zheng Yan 已提交
2398
	key.offset = (u64)-1;
2399
	key.type = BTRFS_EXTENT_ITEM_KEY;
2400 2401 2402 2403 2404

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
	BUG_ON(ret == 0);
Y
Yan Zheng 已提交
2405 2406 2407

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
2408
		goto out;
2409

Z
Zheng Yan 已提交
2410
	path->slots[0]--;
2411
	leaf = path->nodes[0];
2412
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2413

2414
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2415
		goto out;
2416

2417 2418 2419 2420 2421 2422 2423 2424 2425
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2426

2427 2428 2429
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2430

2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
2470
			goto out;
Y
Yan Zheng 已提交
2471

2472 2473 2474 2475 2476 2477 2478
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
2479
	}
2480 2481 2482

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
2483
out:
Y
Yan Zheng 已提交
2484
	btrfs_free_path(path);
2485 2486
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
2487
	return ret;
2488
}
C
Chris Mason 已提交
2489

2490
#if 0
Z
Zheng Yan 已提交
2491 2492
int btrfs_cache_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		    struct extent_buffer *buf, u32 nr_extents)
C
Chris Mason 已提交
2493
{
2494
	struct btrfs_key key;
2495
	struct btrfs_file_extent_item *fi;
Z
Zheng Yan 已提交
2496 2497
	u64 root_gen;
	u32 nritems;
C
Chris Mason 已提交
2498
	int i;
2499
	int level;
Z
Zheng Yan 已提交
2500
	int ret = 0;
Z
Zheng Yan 已提交
2501
	int shared = 0;
2502

2503
	if (!root->ref_cows)
2504
		return 0;
2505

Z
Zheng Yan 已提交
2506 2507 2508 2509 2510 2511 2512 2513
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
		shared = 0;
		root_gen = root->root_key.offset;
	} else {
		shared = 1;
		root_gen = trans->transid - 1;
	}

2514
	level = btrfs_header_level(buf);
2515
	nritems = btrfs_header_nritems(buf);
C
Chris Mason 已提交
2516

Z
Zheng Yan 已提交
2517
	if (level == 0) {
Y
Yan Zheng 已提交
2518 2519 2520
		struct btrfs_leaf_ref *ref;
		struct btrfs_extent_info *info;

Z
Zheng Yan 已提交
2521
		ref = btrfs_alloc_leaf_ref(root, nr_extents);
Y
Yan Zheng 已提交
2522
		if (!ref) {
Z
Zheng Yan 已提交
2523
			ret = -ENOMEM;
Y
Yan Zheng 已提交
2524 2525 2526
			goto out;
		}

Z
Zheng Yan 已提交
2527
		ref->root_gen = root_gen;
Y
Yan Zheng 已提交
2528 2529 2530
		ref->bytenr = buf->start;
		ref->owner = btrfs_header_owner(buf);
		ref->generation = btrfs_header_generation(buf);
Z
Zheng Yan 已提交
2531
		ref->nritems = nr_extents;
Y
Yan Zheng 已提交
2532
		info = ref->extents;
2533

Z
Zheng Yan 已提交
2534
		for (i = 0; nr_extents > 0 && i < nritems; i++) {
Y
Yan Zheng 已提交
2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555
			u64 disk_bytenr;
			btrfs_item_key_to_cpu(buf, &key, i);
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
			fi = btrfs_item_ptr(buf, i,
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			disk_bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (disk_bytenr == 0)
				continue;

			info->bytenr = disk_bytenr;
			info->num_bytes =
				btrfs_file_extent_disk_num_bytes(buf, fi);
			info->objectid = key.objectid;
			info->offset = key.offset;
			info++;
		}

Z
Zheng Yan 已提交
2556
		ret = btrfs_add_leaf_ref(root, ref, shared);
Y
Yan Zheng 已提交
2557 2558 2559 2560 2561 2562 2563 2564
		if (ret == -EEXIST && shared) {
			struct btrfs_leaf_ref *old;
			old = btrfs_lookup_leaf_ref(root, ref->bytenr);
			BUG_ON(!old);
			btrfs_remove_leaf_ref(root, old);
			btrfs_free_leaf_ref(root, old);
			ret = btrfs_add_leaf_ref(root, ref, shared);
		}
Y
Yan Zheng 已提交
2565
		WARN_ON(ret);
2566
		btrfs_free_leaf_ref(root, ref);
Y
Yan Zheng 已提交
2567 2568
	}
out:
Z
Zheng Yan 已提交
2569 2570 2571
	return ret;
}

2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582
/* when a block goes through cow, we update the reference counts of
 * everything that block points to.  The internal pointers of the block
 * can be in just about any order, and it is likely to have clusters of
 * things that are close together and clusters of things that are not.
 *
 * To help reduce the seeks that come with updating all of these reference
 * counts, sort them by byte number before actual updates are done.
 *
 * struct refsort is used to match byte number to slot in the btree block.
 * we sort based on the byte number and then use the slot to actually
 * find the item.
2583 2584 2585 2586 2587
 *
 * struct refsort is smaller than strcut btrfs_item and smaller than
 * struct btrfs_key_ptr.  Since we're currently limited to the page size
 * for a btree block, there's no way for a kmalloc of refsorts for a
 * single node to be bigger than a page.
2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607
 */
struct refsort {
	u64 bytenr;
	u32 slot;
};

/*
 * for passing into sort()
 */
static int refsort_cmp(const void *a_void, const void *b_void)
{
	const struct refsort *a = a_void;
	const struct refsort *b = b_void;

	if (a->bytenr < b->bytenr)
		return -1;
	if (a->bytenr > b->bytenr)
		return 1;
	return 0;
}
2608
#endif
2609

2610
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2611
			   struct btrfs_root *root,
2612 2613
			   struct extent_buffer *buf,
			   int full_backref, int inc)
Z
Zheng Yan 已提交
2614 2615
{
	u64 bytenr;
2616 2617
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
2618 2619 2620 2621 2622 2623 2624 2625
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
2626
			    u64, u64, u64, u64, u64, u64);
Z
Zheng Yan 已提交
2627 2628 2629 2630 2631

	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

2632 2633
	if (!root->ref_cows && level == 0)
		return 0;
Z
Zheng Yan 已提交
2634

2635 2636 2637 2638
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
2639

2640 2641 2642 2643 2644 2645
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
2646
		if (level == 0) {
2647
			btrfs_item_key_to_cpu(buf, &key, i);
Z
Zheng Yan 已提交
2648 2649
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
2650
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
2651 2652 2653 2654 2655 2656 2657
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
2658 2659 2660 2661 2662 2663

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
					   key.offset);
Z
Zheng Yan 已提交
2664 2665 2666
			if (ret)
				goto fail;
		} else {
2667 2668 2669 2670
			bytenr = btrfs_node_blockptr(buf, i);
			num_bytes = btrfs_level_size(root, level - 1);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, level - 1, 0);
Z
Zheng Yan 已提交
2671 2672 2673 2674 2675 2676
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690
	BUG();
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		  struct extent_buffer *buf, int full_backref)
{
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		  struct extent_buffer *buf, int full_backref)
{
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
Z
Zheng Yan 已提交
2691 2692
}

C
Chris Mason 已提交
2693 2694 2695 2696 2697 2698 2699
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2700 2701
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
2702 2703

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2704 2705
	if (ret < 0)
		goto fail;
C
Chris Mason 已提交
2706
	BUG_ON(ret);
2707 2708 2709 2710 2711

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
C
Chris Mason 已提交
2712
	btrfs_release_path(extent_root, path);
2713
fail:
C
Chris Mason 已提交
2714 2715 2716 2717 2718 2719
	if (ret)
		return ret;
	return 0;

}

2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
	spin_lock(&root->fs_info->block_group_cache_lock);
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
2731
		btrfs_get_block_group(cache);
2732 2733 2734 2735 2736 2737
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

2738 2739 2740 2741 2742 2743 2744
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
2745
	int dcs = BTRFS_DC_ERROR;
2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799
	int num_pages = 0;
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
		btrfs_release_path(root, path);
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
		ret = btrfs_truncate_free_space_cache(root, trans, path,
						      inode);
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
	if (block_group->cached != BTRFS_CACHE_FINISHED) {
2800 2801
		/* We're not cached, don't bother trying to write stuff out */
		dcs = BTRFS_DC_WRITTEN;
2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

	num_pages = (int)div64_u64(block_group->key.offset, 1024 * 1024 * 1024);
	if (!num_pages)
		num_pages = 1;

	/*
	 * Just to make absolutely sure we have enough space, we're going to
	 * preallocate 12 pages worth of space for each block group.  In
	 * practice we ought to use at most 8, but we need extra space so we can
	 * add our header and have a terminator between the extents and the
	 * bitmaps.
	 */
	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
2828 2829
	if (!ret)
		dcs = BTRFS_DC_SETUP;
2830 2831 2832 2833 2834 2835 2836
	btrfs_free_reserved_data_space(inode, num_pages);
out_put:
	iput(inode);
out_free:
	btrfs_release_path(root, path);
out:
	spin_lock(&block_group->lock);
2837
	block_group->disk_cache_state = dcs;
2838 2839 2840 2841 2842
	spin_unlock(&block_group->lock);

	return ret;
}

2843 2844
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
C
Chris Mason 已提交
2845
{
2846
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
2847 2848
	int err = 0;
	struct btrfs_path *path;
2849
	u64 last = 0;
C
Chris Mason 已提交
2850 2851 2852 2853 2854

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873
again:
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			if (cache->disk_cache_state == BTRFS_DC_CLEAR)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
		err = cache_save_setup(cache, trans, path);
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
2874
	while (1) {
2875 2876 2877 2878
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
			BUG_ON(err);
J
Josef Bacik 已提交
2879
		}
2880

2881 2882
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
2883 2884 2885 2886 2887
			if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
				btrfs_put_block_group(cache);
				goto again;
			}

2888 2889 2890 2891 2892 2893 2894 2895 2896 2897
			if (cache->dirty)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
J
Josef Bacik 已提交
2898

J
Josef Bacik 已提交
2899 2900
		if (cache->disk_cache_state == BTRFS_DC_SETUP)
			cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
2901
		cache->dirty = 0;
2902
		last = cache->key.objectid + cache->key.offset;
J
Josef Bacik 已提交
2903

2904 2905 2906
		err = write_one_cache_group(trans, root, path, cache);
		BUG_ON(err);
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
2907
	}
2908

J
Josef Bacik 已提交
2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954
	while (1) {
		/*
		 * I don't think this is needed since we're just marking our
		 * preallocated extent as written, but just in case it can't
		 * hurt.
		 */
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
			BUG_ON(err);
		}

		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			/*
			 * Really this shouldn't happen, but it could if we
			 * couldn't write the entire preallocated extent and
			 * splitting the extent resulted in a new block.
			 */
			if (cache->dirty) {
				btrfs_put_block_group(cache);
				goto again;
			}
			if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		btrfs_write_out_cache(root, trans, cache, path);

		/*
		 * If we didn't have an error then the cache state is still
		 * NEED_WRITE, so we can set it to WRITTEN.
		 */
		if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
			cache->disk_cache_state = BTRFS_DC_WRITTEN;
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
2955
	btrfs_free_path(path);
2956
	return 0;
C
Chris Mason 已提交
2957 2958
}

2959 2960 2961 2962 2963 2964 2965 2966 2967
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
2968
		btrfs_put_block_group(block_group);
2969 2970 2971
	return readonly;
}

2972 2973 2974 2975 2976
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
2977 2978 2979 2980 2981 2982 2983 2984
	int i;
	int factor;

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
2985 2986 2987

	found = __find_space_info(info, flags);
	if (found) {
2988
		spin_lock(&found->lock);
2989
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
2990
		found->disk_total += total_bytes * factor;
2991
		found->bytes_used += bytes_used;
2992
		found->disk_used += bytes_used * factor;
2993
		found->full = 0;
2994
		spin_unlock(&found->lock);
2995 2996 2997
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
2998
	found = kzalloc(sizeof(*found), GFP_NOFS);
2999 3000 3001
	if (!found)
		return -ENOMEM;

3002 3003
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		INIT_LIST_HEAD(&found->block_groups[i]);
3004
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
3005
	spin_lock_init(&found->lock);
3006 3007 3008
	found->flags = flags & (BTRFS_BLOCK_GROUP_DATA |
				BTRFS_BLOCK_GROUP_SYSTEM |
				BTRFS_BLOCK_GROUP_METADATA);
3009
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
3010
	found->disk_total = total_bytes * factor;
3011
	found->bytes_used = bytes_used;
3012
	found->disk_used = bytes_used * factor;
3013
	found->bytes_pinned = 0;
3014
	found->bytes_reserved = 0;
Y
Yan Zheng 已提交
3015
	found->bytes_readonly = 0;
3016
	found->bytes_may_use = 0;
3017
	found->full = 0;
C
Chris Mason 已提交
3018
	found->force_alloc = 0;
3019
	*space_info = found;
3020
	list_add_rcu(&found->list, &info->space_info);
J
Josef Bacik 已提交
3021
	atomic_set(&found->caching_threads, 0);
3022 3023 3024
	return 0;
}

3025 3026 3027
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
	u64 extra_flags = flags & (BTRFS_BLOCK_GROUP_RAID0 |
3028
				   BTRFS_BLOCK_GROUP_RAID1 |
C
Chris Mason 已提交
3029
				   BTRFS_BLOCK_GROUP_RAID10 |
3030
				   BTRFS_BLOCK_GROUP_DUP);
3031 3032 3033 3034 3035 3036 3037 3038 3039
	if (extra_flags) {
		if (flags & BTRFS_BLOCK_GROUP_DATA)
			fs_info->avail_data_alloc_bits |= extra_flags;
		if (flags & BTRFS_BLOCK_GROUP_METADATA)
			fs_info->avail_metadata_alloc_bits |= extra_flags;
		if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
			fs_info->avail_system_alloc_bits |= extra_flags;
	}
}
3040

Y
Yan Zheng 已提交
3041
u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3042
{
3043 3044 3045 3046 3047 3048 3049
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	u64 num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;
3050 3051 3052 3053 3054 3055

	if (num_devices == 1)
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

3056 3057
	if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
	    (flags & (BTRFS_BLOCK_GROUP_RAID1 |
3058
		      BTRFS_BLOCK_GROUP_RAID10))) {
3059
		flags &= ~BTRFS_BLOCK_GROUP_DUP;
3060
	}
3061 3062

	if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
3063
	    (flags & BTRFS_BLOCK_GROUP_RAID10)) {
3064
		flags &= ~BTRFS_BLOCK_GROUP_RAID1;
3065
	}
3066 3067 3068 3069 3070 3071 3072 3073 3074

	if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
	    ((flags & BTRFS_BLOCK_GROUP_RAID1) |
	     (flags & BTRFS_BLOCK_GROUP_RAID10) |
	     (flags & BTRFS_BLOCK_GROUP_DUP)))
		flags &= ~BTRFS_BLOCK_GROUP_RAID0;
	return flags;
}

3075
static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
J
Josef Bacik 已提交
3076
{
3077 3078 3079 3080 3081 3082 3083 3084 3085 3086
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		flags |= root->fs_info->avail_data_alloc_bits &
			 root->fs_info->data_alloc_profile;
	else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		flags |= root->fs_info->avail_system_alloc_bits &
			 root->fs_info->system_alloc_profile;
	else if (flags & BTRFS_BLOCK_GROUP_METADATA)
		flags |= root->fs_info->avail_metadata_alloc_bits &
			 root->fs_info->metadata_alloc_profile;
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
3087 3088
}

3089
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
3090
{
3091
	u64 flags;
J
Josef Bacik 已提交
3092

3093 3094 3095 3096
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
3097
	else
3098
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
3099

3100
	return get_alloc_profile(root, flags);
J
Josef Bacik 已提交
3101
}
J
Josef Bacik 已提交
3102

J
Josef Bacik 已提交
3103 3104 3105
void btrfs_set_inode_space_info(struct btrfs_root *root, struct inode *inode)
{
	BTRFS_I(inode)->space_info = __find_space_info(root->fs_info,
3106
						       BTRFS_BLOCK_GROUP_DATA);
J
Josef Bacik 已提交
3107 3108
}

J
Josef Bacik 已提交
3109 3110 3111 3112
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3113
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
3114 3115
{
	struct btrfs_space_info *data_sinfo;
3116
	struct btrfs_root *root = BTRFS_I(inode)->root;
3117
	u64 used;
3118
	int ret = 0, committed = 0, alloc_chunk = 1;
J
Josef Bacik 已提交
3119 3120 3121 3122

	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);

3123 3124 3125 3126 3127
	if (root == root->fs_info->tree_root) {
		alloc_chunk = 0;
		committed = 1;
	}

J
Josef Bacik 已提交
3128
	data_sinfo = BTRFS_I(inode)->space_info;
C
Chris Mason 已提交
3129 3130
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
3131

J
Josef Bacik 已提交
3132 3133 3134
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3135 3136 3137
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3138 3139

	if (used + bytes > data_sinfo->total_bytes) {
3140
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3141

J
Josef Bacik 已提交
3142 3143 3144 3145
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3146
		if (!data_sinfo->full && alloc_chunk) {
J
Josef Bacik 已提交
3147
			u64 alloc_target;
J
Josef Bacik 已提交
3148

J
Josef Bacik 已提交
3149 3150
			data_sinfo->force_alloc = 1;
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
3151
alloc:
J
Josef Bacik 已提交
3152
			alloc_target = btrfs_get_alloc_profile(root, 1);
3153 3154 3155
			trans = btrfs_join_transaction(root, 1);
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
3156

J
Josef Bacik 已提交
3157 3158 3159 3160
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
					     bytes + 2 * 1024 * 1024,
					     alloc_target, 0);
			btrfs_end_transaction(trans, root);
3161 3162 3163 3164 3165 3166
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
J
Josef Bacik 已提交
3167

C
Chris Mason 已提交
3168 3169 3170 3171
			if (!data_sinfo) {
				btrfs_set_inode_space_info(root, inode);
				data_sinfo = BTRFS_I(inode)->space_info;
			}
J
Josef Bacik 已提交
3172 3173 3174 3175
			goto again;
		}
		spin_unlock(&data_sinfo->lock);

3176
		/* commit the current transaction and try again */
3177
commit_trans:
3178
		if (!committed && !root->fs_info->open_ioctl_trans) {
3179 3180
			committed = 1;
			trans = btrfs_join_transaction(root, 1);
3181 3182
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3183 3184 3185 3186 3187
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
J
Josef Bacik 已提交
3188

C
Chris Mason 已提交
3189
#if 0 /* I hope we never need this code again, just in case */
3190 3191 3192 3193
		printk(KERN_ERR "no space left, need %llu, %llu bytes_used, "
		       "%llu bytes_reserved, " "%llu bytes_pinned, "
		       "%llu bytes_readonly, %llu may use %llu total\n",
		       (unsigned long long)bytes,
3194 3195 3196 3197 3198 3199
		       (unsigned long long)data_sinfo->bytes_used,
		       (unsigned long long)data_sinfo->bytes_reserved,
		       (unsigned long long)data_sinfo->bytes_pinned,
		       (unsigned long long)data_sinfo->bytes_readonly,
		       (unsigned long long)data_sinfo->bytes_may_use,
		       (unsigned long long)data_sinfo->total_bytes);
C
Chris Mason 已提交
3200
#endif
J
Josef Bacik 已提交
3201 3202 3203 3204 3205 3206
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
	BTRFS_I(inode)->reserved_bytes += bytes;
	spin_unlock(&data_sinfo->lock);

J
Josef Bacik 已提交
3207 3208
	return 0;
}
J
Josef Bacik 已提交
3209 3210

/*
3211 3212 3213
 * called when we are clearing an delalloc extent from the
 * inode's io_tree or there was an error for whatever reason
 * after calling btrfs_check_data_free_space
J
Josef Bacik 已提交
3214
 */
3215
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3216
{
3217
	struct btrfs_root *root = BTRFS_I(inode)->root;
J
Josef Bacik 已提交
3218
	struct btrfs_space_info *data_sinfo;
3219

J
Josef Bacik 已提交
3220 3221
	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3222

J
Josef Bacik 已提交
3223 3224 3225 3226 3227
	data_sinfo = BTRFS_I(inode)->space_info;
	spin_lock(&data_sinfo->lock);
	data_sinfo->bytes_may_use -= bytes;
	BTRFS_I(inode)->reserved_bytes -= bytes;
	spin_unlock(&data_sinfo->lock);
3228 3229
}

3230
static void force_metadata_allocation(struct btrfs_fs_info *info)
3231
{
3232 3233
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3234

3235 3236 3237 3238
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
			found->force_alloc = 1;
3239
	}
3240
	rcu_read_unlock();
3241 3242
}

3243 3244
static int should_alloc_chunk(struct btrfs_root *root,
			      struct btrfs_space_info *sinfo, u64 alloc_bytes)
3245
{
3246
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3247
	u64 thresh;
3248

3249 3250 3251
	if (sinfo->bytes_used + sinfo->bytes_reserved +
	    alloc_bytes + 256 * 1024 * 1024 < num_bytes)
		return 0;
3252

3253 3254 3255
	if (sinfo->bytes_used + sinfo->bytes_reserved +
	    alloc_bytes < div_factor(num_bytes, 8))
		return 0;
3256

3257 3258 3259 3260
	thresh = btrfs_super_total_bytes(&root->fs_info->super_copy);
	thresh = max_t(u64, 256 * 1024 * 1024, div_factor_fine(thresh, 5));

	if (num_bytes > thresh && sinfo->bytes_used < div_factor(num_bytes, 3))
3261 3262
		return 0;

3263
	return 1;
3264 3265
}

3266 3267
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
C
Chris Mason 已提交
3268
			  u64 flags, int force)
J
Josef Bacik 已提交
3269
{
3270
	struct btrfs_space_info *space_info;
3271
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
J
Josef Bacik 已提交
3272 3273
	int ret = 0;

3274
	mutex_lock(&fs_info->chunk_mutex);
J
Josef Bacik 已提交
3275

Y
Yan Zheng 已提交
3276
	flags = btrfs_reduce_alloc_profile(extent_root, flags);
3277

3278
	space_info = __find_space_info(extent_root->fs_info, flags);
3279 3280 3281 3282
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
		BUG_ON(ret);
J
Josef Bacik 已提交
3283
	}
3284
	BUG_ON(!space_info);
J
Josef Bacik 已提交
3285

3286
	spin_lock(&space_info->lock);
J
Josef Bacik 已提交
3287
	if (space_info->force_alloc)
C
Chris Mason 已提交
3288
		force = 1;
3289 3290
	if (space_info->full) {
		spin_unlock(&space_info->lock);
3291
		goto out;
J
Josef Bacik 已提交
3292 3293
	}

3294 3295
	if (!force && !should_alloc_chunk(extent_root, space_info,
					  alloc_bytes)) {
3296
		spin_unlock(&space_info->lock);
3297
		goto out;
J
Josef Bacik 已提交
3298
	}
3299
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3300

3301 3302 3303 3304 3305 3306 3307
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3308 3309 3310 3311 3312
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
3313
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3314 3315 3316 3317
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
3318 3319
	}

Y
Yan Zheng 已提交
3320
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
J
Josef Bacik 已提交
3321 3322
	spin_lock(&space_info->lock);
	if (ret)
3323
		space_info->full = 1;
3324 3325
	else
		ret = 1;
J
Josef Bacik 已提交
3326 3327 3328
	space_info->force_alloc = 0;
	spin_unlock(&space_info->lock);
out:
Y
Yan Zheng 已提交
3329
	mutex_unlock(&extent_root->fs_info->chunk_mutex);
J
Josef Bacik 已提交
3330
	return ret;
3331
}
J
Josef Bacik 已提交
3332 3333

/*
3334
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
3335
 */
3336
static int shrink_delalloc(struct btrfs_trans_handle *trans,
J
Josef Bacik 已提交
3337
			   struct btrfs_root *root, u64 to_reclaim, int sync)
3338
{
3339
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
3340
	struct btrfs_space_info *space_info;
3341 3342 3343
	u64 reserved;
	u64 max_reclaim;
	u64 reclaimed = 0;
3344
	long time_left;
3345
	int pause = 1;
3346
	int nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3347
	int loops = 0;
3348

3349
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
3350
	space_info = block_rsv->space_info;
3351 3352

	smp_mb();
J
Josef Bacik 已提交
3353
	reserved = space_info->bytes_reserved;
3354 3355 3356 3357 3358 3359

	if (reserved == 0)
		return 0;

	max_reclaim = min(reserved, to_reclaim);

3360
	while (loops < 1024) {
3361 3362 3363 3364 3365
		/* have the flusher threads jump in and do some IO */
		smp_mb();
		nr_pages = min_t(unsigned long, nr_pages,
		       root->fs_info->delalloc_bytes >> PAGE_CACHE_SHIFT);
		writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages);
3366

J
Josef Bacik 已提交
3367
		spin_lock(&space_info->lock);
3368 3369
		if (reserved > space_info->bytes_reserved) {
			loops = 0;
J
Josef Bacik 已提交
3370
			reclaimed += reserved - space_info->bytes_reserved;
3371 3372 3373
		} else {
			loops++;
		}
J
Josef Bacik 已提交
3374 3375
		reserved = space_info->bytes_reserved;
		spin_unlock(&space_info->lock);
3376 3377 3378 3379 3380 3381

		if (reserved == 0 || reclaimed >= max_reclaim)
			break;

		if (trans && trans->transaction->blocked)
			return -EAGAIN;
3382 3383

		__set_current_state(TASK_INTERRUPTIBLE);
3384 3385 3386 3387 3388 3389
		time_left = schedule_timeout(pause);

		/* We were interrupted, exit */
		if (time_left)
			break;

3390 3391 3392 3393
		pause <<= 1;
		if (pause > HZ / 10)
			pause = HZ / 10;

3394 3395 3396 3397
	}
	return reclaimed >= to_reclaim;
}

3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412
/*
 * Retries tells us how many times we've called reserve_metadata_bytes.  The
 * idea is if this is the first call (retries == 0) then we will add to our
 * reserved count if we can't make the allocation in order to hold our place
 * while we go and try and free up space.  That way for retries > 1 we don't try
 * and add space, we just check to see if the amount of unused space is >= the
 * total space, meaning that our reservation is valid.
 *
 * However if we don't intend to retry this reservation, pass -1 as retries so
 * that it short circuits this logic.
 */
static int reserve_metadata_bytes(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_block_rsv *block_rsv,
				  u64 orig_bytes, int flush)
J
Josef Bacik 已提交
3413
{
3414
	struct btrfs_space_info *space_info = block_rsv->space_info;
3415 3416 3417 3418 3419
	u64 unused;
	u64 num_bytes = orig_bytes;
	int retries = 0;
	int ret = 0;
	bool reserved = false;
3420
	bool committed = false;
J
Josef Bacik 已提交
3421

3422 3423 3424 3425
again:
	ret = -ENOSPC;
	if (reserved)
		num_bytes = 0;
J
Josef Bacik 已提交
3426

3427 3428 3429 3430
	spin_lock(&space_info->lock);
	unused = space_info->bytes_used + space_info->bytes_reserved +
		 space_info->bytes_pinned + space_info->bytes_readonly +
		 space_info->bytes_may_use;
J
Josef Bacik 已提交
3431

3432 3433 3434 3435 3436 3437 3438 3439
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
	if (unused <= space_info->total_bytes) {
3440
		unused = space_info->total_bytes - unused;
3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461
		if (unused >= num_bytes) {
			if (!reserved)
				space_info->bytes_reserved += orig_bytes;
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
		num_bytes = unused - space_info->total_bytes +
			(orig_bytes * (retries + 1));
	}
J
Josef Bacik 已提交
3462

3463 3464 3465 3466 3467 3468 3469 3470 3471
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
	 */
	if (ret && !reserved) {
		space_info->bytes_reserved += orig_bytes;
		reserved = true;
	}
J
Josef Bacik 已提交
3472

3473
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3474

3475 3476
	if (!ret)
		return 0;
J
Josef Bacik 已提交
3477

3478 3479
	if (!flush)
		goto out;
3480

3481 3482 3483 3484 3485 3486 3487 3488 3489
	/*
	 * We do synchronous shrinking since we don't actually unreserve
	 * metadata until after the IO is completed.
	 */
	ret = shrink_delalloc(trans, root, num_bytes, 1);
	if (ret > 0)
		return 0;
	else if (ret < 0)
		goto out;
3490

3491 3492 3493 3494 3495 3496 3497 3498 3499
	/*
	 * So if we were overcommitted it's possible that somebody else flushed
	 * out enough space and we simply didn't have enough space to reclaim,
	 * so go back around and try again.
	 */
	if (retries < 2) {
		retries++;
		goto again;
	}
3500 3501

	spin_lock(&space_info->lock);
3502 3503 3504 3505 3506 3507 3508 3509 3510
	/*
	 * Not enough space to be reclaimed, don't bother committing the
	 * transaction.
	 */
	if (space_info->bytes_pinned < orig_bytes)
		ret = -ENOSPC;
	spin_unlock(&space_info->lock);
	if (ret)
		goto out;
3511

3512
	ret = -EAGAIN;
3513
	if (trans || committed)
3514
		goto out;
3515

3516 3517 3518 3519 3520
	ret = -ENOSPC;
	trans = btrfs_join_transaction(root, 1);
	if (IS_ERR(trans))
		goto out;
	ret = btrfs_commit_transaction(trans, root);
3521 3522 3523
	if (!ret) {
		trans = NULL;
		committed = true;
3524
		goto again;
3525
	}
3526 3527 3528 3529 3530 3531

out:
	if (reserved) {
		spin_lock(&space_info->lock);
		space_info->bytes_reserved -= orig_bytes;
		spin_unlock(&space_info->lock);
3532
	}
3533

3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598
	return ret;
}

static struct btrfs_block_rsv *get_block_rsv(struct btrfs_trans_handle *trans,
					     struct btrfs_root *root)
{
	struct btrfs_block_rsv *block_rsv;
	if (root->ref_cows)
		block_rsv = trans->block_rsv;
	else
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

void block_rsv_release_bytes(struct btrfs_block_rsv *block_rsv,
			     struct btrfs_block_rsv *dest, u64 num_bytes)
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
3613 3614 3615
			spin_lock(&space_info->lock);
			space_info->bytes_reserved -= num_bytes;
			spin_unlock(&space_info->lock);
3616
		}
J
Josef Bacik 已提交
3617
	}
3618
}
3619

3620 3621 3622 3623
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
J
Josef Bacik 已提交
3624

3625 3626 3627
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
3628

3629
	block_rsv_add_bytes(dst, num_bytes, 1);
J
Josef Bacik 已提交
3630 3631 3632
	return 0;
}

3633
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv)
J
Josef Bacik 已提交
3634
{
3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
	atomic_set(&rsv->usage, 1);
	rsv->priority = 6;
	INIT_LIST_HEAD(&rsv->list);
}

struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root)
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
3646

3647 3648 3649
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
3650

3651 3652 3653 3654 3655
	btrfs_init_block_rsv(block_rsv);
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
3656

3657 3658 3659 3660 3661 3662 3663 3664
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
	if (rsv && atomic_dec_and_test(&rsv->usage)) {
		btrfs_block_rsv_release(root, rsv, (u64)-1);
		if (!rsv->durable)
			kfree(rsv);
	}
J
Josef Bacik 已提交
3665 3666 3667
}

/*
3668 3669 3670
 * make the block_rsv struct be able to capture freed space.
 * the captured space will re-add to the the block_rsv struct
 * after transaction commit
J
Josef Bacik 已提交
3671
 */
3672 3673
void btrfs_add_durable_block_rsv(struct btrfs_fs_info *fs_info,
				 struct btrfs_block_rsv *block_rsv)
J
Josef Bacik 已提交
3674
{
3675 3676 3677 3678 3679
	block_rsv->durable = 1;
	mutex_lock(&fs_info->durable_block_rsv_mutex);
	list_add_tail(&block_rsv->list, &fs_info->durable_block_rsv_list);
	mutex_unlock(&fs_info->durable_block_rsv_mutex);
}
J
Josef Bacik 已提交
3680

3681 3682 3683
int btrfs_block_rsv_add(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv,
3684
			u64 num_bytes)
3685 3686
{
	int ret;
J
Josef Bacik 已提交
3687

3688 3689
	if (num_bytes == 0)
		return 0;
3690 3691

	ret = reserve_metadata_bytes(trans, root, block_rsv, num_bytes, 1);
3692 3693 3694 3695
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
3696

3697 3698
	return ret;
}
J
Josef Bacik 已提交
3699

3700 3701 3702 3703 3704 3705 3706 3707
int btrfs_block_rsv_check(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  struct btrfs_block_rsv *block_rsv,
			  u64 min_reserved, int min_factor)
{
	u64 num_bytes = 0;
	int commit_trans = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
3708

3709 3710
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
3711

3712 3713 3714 3715 3716
	spin_lock(&block_rsv->lock);
	if (min_factor > 0)
		num_bytes = div_factor(block_rsv->size, min_factor);
	if (min_reserved > num_bytes)
		num_bytes = min_reserved;
J
Josef Bacik 已提交
3717

3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730
	if (block_rsv->reserved >= num_bytes) {
		ret = 0;
	} else {
		num_bytes -= block_rsv->reserved;
		if (block_rsv->durable &&
		    block_rsv->freed[0] + block_rsv->freed[1] >= num_bytes)
			commit_trans = 1;
	}
	spin_unlock(&block_rsv->lock);
	if (!ret)
		return 0;

	if (block_rsv->refill_used) {
3731 3732
		ret = reserve_metadata_bytes(trans, root, block_rsv,
					     num_bytes, 0);
3733 3734 3735
		if (!ret) {
			block_rsv_add_bytes(block_rsv, num_bytes, 0);
			return 0;
3736
		}
3737
	}
J
Josef Bacik 已提交
3738

3739 3740 3741 3742 3743 3744 3745 3746
	if (commit_trans) {
		if (trans)
			return -EAGAIN;

		trans = btrfs_join_transaction(root, 1);
		BUG_ON(IS_ERR(trans));
		ret = btrfs_commit_transaction(trans, root);
		return 0;
J
Josef Bacik 已提交
3747
	}
J
Josef Bacik 已提交
3748

3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767
	return -ENOSPC;
}

int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
	if (global_rsv->full || global_rsv == block_rsv ||
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
	block_rsv_release_bytes(block_rsv, global_rsv, num_bytes);
J
Josef Bacik 已提交
3768 3769 3770
}

/*
3771 3772 3773
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
3774
 */
3775
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3776
{
3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
	int csum_size = btrfs_super_csum_size(&fs_info->super_copy);
#if 0
	/*
	 * per tree used space accounting can be inaccuracy, so we
	 * can't rely on it.
	 */
	spin_lock(&fs_info->extent_root->accounting_lock);
	num_bytes = btrfs_root_used(&fs_info->extent_root->root_item);
	spin_unlock(&fs_info->extent_root->accounting_lock);
J
Josef Bacik 已提交
3790

3791 3792 3793
	spin_lock(&fs_info->csum_root->accounting_lock);
	num_bytes += btrfs_root_used(&fs_info->csum_root->root_item);
	spin_unlock(&fs_info->csum_root->accounting_lock);
J
Josef Bacik 已提交
3794

3795 3796 3797 3798 3799 3800 3801 3802
	spin_lock(&fs_info->tree_root->accounting_lock);
	num_bytes += btrfs_root_used(&fs_info->tree_root->root_item);
	spin_unlock(&fs_info->tree_root->accounting_lock);
#endif
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
3803

3804 3805
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
3806 3807
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
3808 3809
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
3810

3811 3812 3813
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
	num_bytes += div64_u64(data_used + meta_used, 50);
3814

3815 3816
	if (num_bytes * 3 > meta_used)
		num_bytes = div64_u64(meta_used, 3);
3817

3818 3819
	return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
}
J
Josef Bacik 已提交
3820

3821 3822 3823 3824 3825
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
3826

3827
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
3828

3829 3830
	spin_lock(&block_rsv->lock);
	spin_lock(&sinfo->lock);
3831

3832
	block_rsv->size = num_bytes;
3833

3834
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
3835 3836
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
3837 3838 3839 3840 3841

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
		sinfo->bytes_reserved += num_bytes;
J
Josef Bacik 已提交
3842 3843
	}

3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		sinfo->bytes_reserved -= num_bytes;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
#if 0
	printk(KERN_INFO"global block rsv size %llu reserved %llu\n",
		block_rsv->size, block_rsv->reserved);
#endif
	spin_unlock(&sinfo->lock);
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
3856 3857
}

3858
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3859
{
3860
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
3861

3862 3863 3864
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
	fs_info->chunk_block_rsv.priority = 10;
J
Josef Bacik 已提交
3865

3866
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
3867 3868 3869 3870
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->global_block_rsv.priority = 10;
	fs_info->global_block_rsv.refill_used = 1;
	fs_info->delalloc_block_rsv.space_info = space_info;
3871 3872 3873 3874
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.priority = 10;

3875 3876 3877 3878
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
3879
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
3880 3881 3882 3883 3884 3885

	btrfs_add_durable_block_rsv(fs_info, &fs_info->global_block_rsv);

	btrfs_add_durable_block_rsv(fs_info, &fs_info->delalloc_block_rsv);

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
3886 3887
}

3888
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3889
{
3890 3891 3892 3893 3894 3895 3896
	block_rsv_release_bytes(&fs_info->global_block_rsv, NULL, (u64)-1);
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
3897
}
J
Josef Bacik 已提交
3898

3899 3900 3901 3902 3903
static u64 calc_trans_metadata_size(struct btrfs_root *root, int num_items)
{
	return (root->leafsize + root->nodesize * (BTRFS_MAX_LEVEL - 1)) *
		3 * num_items;
}
J
Josef Bacik 已提交
3904

3905 3906
int btrfs_trans_reserve_metadata(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
3907
				 int num_items)
3908 3909 3910
{
	u64 num_bytes;
	int ret;
J
Josef Bacik 已提交
3911

3912 3913
	if (num_items == 0 || root->fs_info->chunk_root == root)
		return 0;
J
Josef Bacik 已提交
3914

3915 3916
	num_bytes = calc_trans_metadata_size(root, num_items);
	ret = btrfs_block_rsv_add(trans, root, &root->fs_info->trans_block_rsv,
3917
				  num_bytes);
3918 3919 3920 3921 3922
	if (!ret) {
		trans->bytes_reserved += num_bytes;
		trans->block_rsv = &root->fs_info->trans_block_rsv;
	}
	return ret;
J
Josef Bacik 已提交
3923 3924
}

3925 3926
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
3927
{
3928 3929
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
3930

3931 3932 3933 3934 3935
	BUG_ON(trans->block_rsv != &root->fs_info->trans_block_rsv);
	btrfs_block_rsv_release(root, trans->block_rsv,
				trans->bytes_reserved);
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
3936

3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
	 * one for deleting orphan item, one for updating inode and
	 * two for calling btrfs_truncate_inode_items.
	 *
	 * btrfs_truncate_inode_items is a delete operation, it frees
	 * more space than it uses in most cases. So two units of
	 * metadata space should be enough for calling it many times.
	 * If all of the metadata space is used, we can commit
	 * transaction and use space it freed.
	 */
	u64 num_bytes = calc_trans_metadata_size(root, 4);
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
J
Josef Bacik 已提交
3956 3957
}

3958
void btrfs_orphan_release_metadata(struct inode *inode)
3959
{
3960 3961 3962 3963
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 num_bytes = calc_trans_metadata_size(root, 4);
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
3964

3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977
int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
				struct btrfs_pending_snapshot *pending)
{
	struct btrfs_root *root = pending->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
	/*
	 * two for root back/forward refs, two for directory entries
	 * and one for root of the snapshot.
	 */
	u64 num_bytes = calc_trans_metadata_size(root, 5);
	dst_rsv->space_info = src_rsv->space_info;
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
3978 3979
}

3980
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes)
3981
{
3982 3983
	return num_bytes >>= 3;
}
Y
Yan Zheng 已提交
3984

3985 3986 3987 3988 3989 3990 3991
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
	u64 to_reserve;
	int nr_extents;
	int ret;
3992

3993 3994
	if (btrfs_transaction_in_commit(root->fs_info))
		schedule_timeout(1);
3995

3996
	num_bytes = ALIGN(num_bytes, root->sectorsize);
3997

3998 3999 4000 4001 4002 4003 4004 4005
	spin_lock(&BTRFS_I(inode)->accounting_lock);
	nr_extents = atomic_read(&BTRFS_I(inode)->outstanding_extents) + 1;
	if (nr_extents > BTRFS_I(inode)->reserved_extents) {
		nr_extents -= BTRFS_I(inode)->reserved_extents;
		to_reserve = calc_trans_metadata_size(root, nr_extents);
	} else {
		nr_extents = 0;
		to_reserve = 0;
4006
	}
4007
	spin_unlock(&BTRFS_I(inode)->accounting_lock);
4008

4009
	to_reserve += calc_csum_metadata_size(inode, num_bytes);
4010 4011
	ret = reserve_metadata_bytes(NULL, root, block_rsv, to_reserve, 1);
	if (ret)
4012
		return ret;
4013

4014
	spin_lock(&BTRFS_I(inode)->accounting_lock);
4015 4016 4017
	BTRFS_I(inode)->reserved_extents += nr_extents;
	atomic_inc(&BTRFS_I(inode)->outstanding_extents);
	spin_unlock(&BTRFS_I(inode)->accounting_lock);
4018

4019 4020 4021
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	if (block_rsv->size > 512 * 1024 * 1024)
J
Josef Bacik 已提交
4022
		shrink_delalloc(NULL, root, to_reserve, 0);
4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034

	return 0;
}

void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 to_free;
	int nr_extents;

	num_bytes = ALIGN(num_bytes, root->sectorsize);
	atomic_dec(&BTRFS_I(inode)->outstanding_extents);
4035
	WARN_ON(atomic_read(&BTRFS_I(inode)->outstanding_extents) < 0);
4036 4037 4038 4039 4040 4041 4042 4043

	spin_lock(&BTRFS_I(inode)->accounting_lock);
	nr_extents = atomic_read(&BTRFS_I(inode)->outstanding_extents);
	if (nr_extents < BTRFS_I(inode)->reserved_extents) {
		nr_extents = BTRFS_I(inode)->reserved_extents - nr_extents;
		BTRFS_I(inode)->reserved_extents -= nr_extents;
	} else {
		nr_extents = 0;
4044
	}
4045
	spin_unlock(&BTRFS_I(inode)->accounting_lock);
4046

4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059
	to_free = calc_csum_metadata_size(inode, num_bytes);
	if (nr_extents > 0)
		to_free += calc_trans_metadata_size(root, nr_extents);

	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
C
Chris Mason 已提交
4060
	if (ret)
4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
4076 4077
}

C
Chris Mason 已提交
4078 4079
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
4080
			      u64 bytenr, u64 num_bytes, int alloc)
C
Chris Mason 已提交
4081
{
4082
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
4083
	struct btrfs_fs_info *info = root->fs_info;
4084
	u64 total = num_bytes;
C
Chris Mason 已提交
4085
	u64 old_val;
4086
	u64 byte_in_group;
4087
	int factor;
C
Chris Mason 已提交
4088

4089 4090 4091 4092 4093 4094 4095 4096 4097 4098
	/* block accounting for super block */
	spin_lock(&info->delalloc_lock);
	old_val = btrfs_super_bytes_used(&info->super_copy);
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
	btrfs_set_super_bytes_used(&info->super_copy, old_val);
	spin_unlock(&info->delalloc_lock);

C
Chris Mason 已提交
4099
	while (total) {
4100
		cache = btrfs_lookup_block_group(info, bytenr);
4101
		if (!cache)
C
Chris Mason 已提交
4102
			return -1;
4103 4104 4105 4106 4107 4108
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
4109 4110 4111 4112 4113 4114 4115
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
4116
			cache_block_group(cache, trans, NULL, 1);
4117

4118 4119
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
4120

4121
		spin_lock(&cache->space_info->lock);
4122
		spin_lock(&cache->lock);
4123 4124 4125 4126 4127

		if (btrfs_super_cache_generation(&info->super_copy) != 0 &&
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

J
Josef Bacik 已提交
4128
		cache->dirty = 1;
C
Chris Mason 已提交
4129
		old_val = btrfs_block_group_used(&cache->item);
4130
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
4131
		if (alloc) {
4132
			old_val += num_bytes;
4133 4134 4135
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
4136 4137
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
4138
			spin_unlock(&cache->lock);
4139
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4140
		} else {
4141
			old_val -= num_bytes;
4142
			btrfs_set_block_group_used(&cache->item, old_val);
4143 4144
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
4145
			cache->space_info->bytes_used -= num_bytes;
4146
			cache->space_info->disk_used -= num_bytes * factor;
4147
			spin_unlock(&cache->lock);
4148
			spin_unlock(&cache->space_info->lock);
4149

4150 4151 4152
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
C
Chris Mason 已提交
4153
		}
4154
		btrfs_put_block_group(cache);
4155 4156
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
4157 4158 4159
	}
	return 0;
}
4160

4161 4162
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
4163
	struct btrfs_block_group_cache *cache;
4164
	u64 bytenr;
J
Josef Bacik 已提交
4165 4166 4167

	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
4168
		return 0;
J
Josef Bacik 已提交
4169

4170
	bytenr = cache->key.objectid;
4171
	btrfs_put_block_group(cache);
4172 4173

	return bytenr;
4174 4175
}

4176 4177 4178
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
4179
{
4180 4181 4182 4183 4184 4185 4186 4187 4188 4189
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
4190

4191 4192 4193 4194
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
	return 0;
}
J
Josef Bacik 已提交
4195

4196 4197 4198 4199 4200 4201 4202
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
4203

4204 4205 4206 4207 4208 4209
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
	BUG_ON(!cache);

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
4210 4211 4212
	return 0;
}

4213 4214 4215 4216 4217 4218
/*
 * update size of reserved extents. this function may return -EAGAIN
 * if 'reserve' is true or 'sinfo' is false.
 */
static int update_reserved_bytes(struct btrfs_block_group_cache *cache,
				 u64 num_bytes, int reserve, int sinfo)
4219
{
4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239
	int ret = 0;
	if (sinfo) {
		struct btrfs_space_info *space_info = cache->space_info;
		spin_lock(&space_info->lock);
		spin_lock(&cache->lock);
		if (reserve) {
			if (cache->ro) {
				ret = -EAGAIN;
			} else {
				cache->reserved += num_bytes;
				space_info->bytes_reserved += num_bytes;
			}
		} else {
			if (cache->ro)
				space_info->bytes_readonly += num_bytes;
			cache->reserved -= num_bytes;
			space_info->bytes_reserved -= num_bytes;
		}
		spin_unlock(&cache->lock);
		spin_unlock(&space_info->lock);
4240
	} else {
4241 4242 4243 4244 4245 4246 4247 4248 4249 4250
		spin_lock(&cache->lock);
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
			if (reserve)
				cache->reserved += num_bytes;
			else
				cache->reserved -= num_bytes;
		}
		spin_unlock(&cache->lock);
4251
	}
4252
	return ret;
4253
}
C
Chris Mason 已提交
4254

4255 4256
int btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
				struct btrfs_root *root)
4257 4258
{
	struct btrfs_fs_info *fs_info = root->fs_info;
4259 4260 4261
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
4262

4263
	down_write(&fs_info->extent_commit_sem);
4264

4265 4266 4267 4268 4269 4270 4271
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
4272
		} else {
4273
			cache->last_byte_to_unpin = caching_ctl->progress;
4274 4275
		}
	}
4276 4277 4278 4279 4280 4281 4282

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

	up_write(&fs_info->extent_commit_sem);
4283 4284

	update_global_block_rsv(fs_info);
4285 4286 4287
	return 0;
}

4288
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
C
Chris Mason 已提交
4289
{
4290 4291 4292
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 len;
C
Chris Mason 已提交
4293

4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310
	while (start <= end) {
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
			BUG_ON(!cache);
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
			btrfs_add_free_space(cache, start, len);
		}

4311 4312
		start += len;

4313 4314 4315 4316
		spin_lock(&cache->space_info->lock);
		spin_lock(&cache->lock);
		cache->pinned -= len;
		cache->space_info->bytes_pinned -= len;
4317 4318 4319 4320 4321 4322 4323
		if (cache->ro) {
			cache->space_info->bytes_readonly += len;
		} else if (cache->reserved_pinned > 0) {
			len = min(len, cache->reserved_pinned);
			cache->reserved_pinned -= len;
			cache->space_info->bytes_reserved += len;
		}
4324 4325
		spin_unlock(&cache->lock);
		spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4326
	}
4327 4328 4329

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
4330 4331 4332 4333
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
4334
			       struct btrfs_root *root)
4335
{
4336 4337
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
4338 4339
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_block_rsv *next_rsv;
4340 4341
	u64 start;
	u64 end;
4342
	int idx;
4343 4344
	int ret;

4345 4346 4347 4348 4349
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

C
Chris Mason 已提交
4350
	while (1) {
4351 4352 4353
		ret = find_first_extent_bit(unpin, 0, &start, &end,
					    EXTENT_DIRTY);
		if (ret)
4354
			break;
4355 4356 4357

		ret = btrfs_discard_extent(root, start, end + 1 - start);

4358
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
4359
		unpin_extent_range(root, start, end);
4360
		cond_resched();
4361
	}
J
Josef Bacik 已提交
4362

4363 4364 4365
	mutex_lock(&fs_info->durable_block_rsv_mutex);
	list_for_each_entry_safe(block_rsv, next_rsv,
				 &fs_info->durable_block_rsv_list, list) {
4366

4367 4368 4369 4370 4371 4372 4373 4374
		idx = trans->transid & 0x1;
		if (block_rsv->freed[idx] > 0) {
			block_rsv_add_bytes(block_rsv,
					    block_rsv->freed[idx], 0);
			block_rsv->freed[idx] = 0;
		}
		if (atomic_read(&block_rsv->usage) == 0) {
			btrfs_block_rsv_release(root, block_rsv, (u64)-1);
Z
Zheng Yan 已提交
4375

4376 4377 4378 4379 4380 4381 4382
			if (block_rsv->freed[0] == 0 &&
			    block_rsv->freed[1] == 0) {
				list_del_init(&block_rsv->list);
				kfree(block_rsv);
			}
		} else {
			btrfs_block_rsv_release(root, block_rsv, 0);
C
Chris Mason 已提交
4383
		}
C
Chris Mason 已提交
4384
	}
4385
	mutex_unlock(&fs_info->durable_block_rsv_mutex);
Z
Zheng Yan 已提交
4386

C
Chris Mason 已提交
4387 4388 4389
	return 0;
}

4390 4391 4392 4393 4394 4395
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extent_op)
4396
{
C
Chris Mason 已提交
4397
	struct btrfs_key key;
4398
	struct btrfs_path *path;
4399 4400
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
4401
	struct extent_buffer *leaf;
4402 4403
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
4404
	int ret;
4405
	int is_data;
4406 4407 4408
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
4409 4410
	u32 item_size;
	u64 refs;
C
Chris Mason 已提交
4411

4412
	path = btrfs_alloc_path();
4413 4414
	if (!path)
		return -ENOMEM;
4415

4416
	path->reada = 1;
4417
	path->leave_spinning = 1;
4418 4419 4420 4421 4422 4423 4424 4425

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
4426
	if (ret == 0) {
4427
		extent_slot = path->slots[0];
4428 4429
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
4430
					      extent_slot);
4431
			if (key.objectid != bytenr)
4432
				break;
4433 4434
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
4435 4436 4437 4438 4439
				found_extent = 1;
				break;
			}
			if (path->slots[0] - extent_slot > 5)
				break;
4440
			extent_slot--;
4441
		}
4442 4443 4444 4445 4446
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
4447
		if (!found_extent) {
4448
			BUG_ON(iref);
4449
			ret = remove_extent_backref(trans, extent_root, path,
4450 4451
						    NULL, refs_to_drop,
						    is_data);
Z
Zheng Yan 已提交
4452 4453
			BUG_ON(ret);
			btrfs_release_path(extent_root, path);
4454
			path->leave_spinning = 1;
4455 4456 4457 4458 4459

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

Z
Zheng Yan 已提交
4460 4461
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
4462 4463
			if (ret) {
				printk(KERN_ERR "umm, got %d back from search"
C
Chris Mason 已提交
4464 4465
				       ", was looking for %llu\n", ret,
				       (unsigned long long)bytenr);
4466 4467
				btrfs_print_leaf(extent_root, path->nodes[0]);
			}
Z
Zheng Yan 已提交
4468 4469 4470
			BUG_ON(ret);
			extent_slot = path->slots[0];
		}
4471 4472 4473
	} else {
		btrfs_print_leaf(extent_root, path->nodes[0]);
		WARN_ON(1);
C
Chris Mason 已提交
4474
		printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
4475
		       "parent %llu root %llu  owner %llu offset %llu\n",
C
Chris Mason 已提交
4476
		       (unsigned long long)bytenr,
4477
		       (unsigned long long)parent,
C
Chris Mason 已提交
4478
		       (unsigned long long)root_objectid,
4479 4480
		       (unsigned long long)owner_objectid,
		       (unsigned long long)owner_offset);
4481
	}
4482 4483

	leaf = path->nodes[0];
4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
		BUG_ON(ret < 0);

		btrfs_release_path(extent_root, path);
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
			printk(KERN_ERR "umm, got %d back from search"
			       ", was looking for %llu\n", ret,
			       (unsigned long long)bytenr);
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
		BUG_ON(ret);
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
4514
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
4515
			    struct btrfs_extent_item);
4516 4517 4518 4519 4520 4521
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
4522

4523
	refs = btrfs_extent_refs(leaf, ei);
4524 4525
	BUG_ON(refs < refs_to_drop);
	refs -= refs_to_drop;
4526

4527 4528 4529 4530 4531 4532
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
4533
		 */
4534 4535 4536 4537 4538 4539 4540 4541 4542 4543
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
						    is_data);
4544 4545
			BUG_ON(ret);
		}
4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
4557
		}
4558

4559 4560
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
Z
Zheng Yan 已提交
4561
		BUG_ON(ret);
4562
		btrfs_release_path(extent_root, path);
4563

4564
		if (is_data) {
4565 4566
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
			BUG_ON(ret);
4567 4568 4569 4570
		} else {
			invalidate_mapping_pages(info->btree_inode->i_mapping,
			     bytenr >> PAGE_CACHE_SHIFT,
			     (bytenr + num_bytes - 1) >> PAGE_CACHE_SHIFT);
4571 4572
		}

4573
		ret = update_block_group(trans, root, bytenr, num_bytes, 0);
4574
		BUG_ON(ret);
4575
	}
4576
	btrfs_free_path(path);
4577 4578 4579
	return ret;
}

4580
/*
4581
 * when we free an block, it is possible (and likely) that we free the last
4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct rb_node *node;
4593
	int ret = 0;
4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	/* there are still entries for this ref, we can't drop it */
	if (ref->bytenr == bytenr)
		goto out;

4611 4612 4613 4614 4615 4616 4617
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
		kfree(head->extent_op);
		head->extent_op = NULL;
	}

4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
	rb_erase(&head->node.rb_node, &delayed_refs->root);
4631

4632 4633 4634 4635 4636 4637
	delayed_refs->num_entries--;

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
4638 4639 4640 4641 4642
	delayed_refs->num_heads--;
	if (list_empty(&head->cluster))
		delayed_refs->num_heads_ready--;

	list_del_init(&head->cluster);
4643 4644
	spin_unlock(&delayed_refs->lock);

4645 4646 4647 4648 4649
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
4650
	btrfs_put_delayed_ref(&head->node);
4651
	return ret;
4652 4653 4654 4655 4656
out:
	spin_unlock(&delayed_refs->lock);
	return 0;
}

4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
			   u64 parent, int last_ref)
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_block_group_cache *cache = NULL;
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, buf->start, buf->len,
						parent, root->root_key.objectid,
						btrfs_header_level(buf),
						BTRFS_DROP_DELAYED_REF, NULL);
		BUG_ON(ret);
	}

	if (!last_ref)
		return;

	block_rsv = get_block_rsv(trans, root);
	cache = btrfs_lookup_block_group(root->fs_info, buf->start);
4679 4680
	if (block_rsv->space_info != cache->space_info)
		goto out;
4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738

	if (btrfs_header_generation(buf) == trans->transid) {
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
				goto pin;
		}

		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
			goto pin;
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
		ret = update_reserved_bytes(cache, buf->len, 0, 0);
		if (ret == -EAGAIN) {
			/* block group became read-only */
			update_reserved_bytes(cache, buf->len, 0, 1);
			goto out;
		}

		ret = 1;
		spin_lock(&block_rsv->lock);
		if (block_rsv->reserved < block_rsv->size) {
			block_rsv->reserved += buf->len;
			ret = 0;
		}
		spin_unlock(&block_rsv->lock);

		if (ret) {
			spin_lock(&cache->space_info->lock);
			cache->space_info->bytes_reserved -= buf->len;
			spin_unlock(&cache->space_info->lock);
		}
		goto out;
	}
pin:
	if (block_rsv->durable && !cache->ro) {
		ret = 0;
		spin_lock(&cache->lock);
		if (!cache->ro) {
			cache->reserved_pinned += buf->len;
			ret = 1;
		}
		spin_unlock(&cache->lock);

		if (ret) {
			spin_lock(&block_rsv->lock);
			block_rsv->freed[trans->transid & 0x1] += buf->len;
			spin_unlock(&block_rsv->lock);
		}
	}
out:
	btrfs_put_block_group(cache);
}

4739
int btrfs_free_extent(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
4740 4741
		      struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent,
4742
		      u64 root_objectid, u64 owner, u64 offset)
4743 4744 4745
{
	int ret;

4746 4747 4748 4749
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
4750 4751
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
4752
		/* unlocks the pinned mutex */
4753
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
4754
		ret = 0;
4755 4756 4757 4758
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
					parent, root_objectid, (int)owner,
					BTRFS_DROP_DELAYED_REF, NULL);
4759
		BUG_ON(ret);
4760 4761 4762 4763 4764
	} else {
		ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
					parent, root_objectid, owner,
					offset, BTRFS_DROP_DELAYED_REF, NULL);
		BUG_ON(ret);
4765
	}
4766 4767 4768
	return ret;
}

4769 4770 4771 4772 4773 4774 4775
static u64 stripe_align(struct btrfs_root *root, u64 val)
{
	u64 mask = ((u64)root->stripesize - 1);
	u64 ret = (val + mask) & ~mask;
	return ret;
}

J
Josef Bacik 已提交
4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
 */
static noinline int
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
4791
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
4792 4793
	DEFINE_WAIT(wait);

4794 4795
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
J
Josef Bacik 已提交
4796 4797
		return 0;

4798
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
J
Josef Bacik 已提交
4799
		   (cache->free_space >= num_bytes));
4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817

	put_caching_control(caching_ctl);
	return 0;
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;
	DEFINE_WAIT(wait);

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
		return 0;

	wait_event(caching_ctl->wait, block_group_cache_done(cache));

	put_caching_control(caching_ctl);
J
Josef Bacik 已提交
4818 4819 4820
	return 0;
}

4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836
static int get_block_group_index(struct btrfs_block_group_cache *cache)
{
	int index;
	if (cache->flags & BTRFS_BLOCK_GROUP_RAID10)
		index = 0;
	else if (cache->flags & BTRFS_BLOCK_GROUP_RAID1)
		index = 1;
	else if (cache->flags & BTRFS_BLOCK_GROUP_DUP)
		index = 2;
	else if (cache->flags & BTRFS_BLOCK_GROUP_RAID0)
		index = 3;
	else
		index = 4;
	return index;
}

J
Josef Bacik 已提交
4837
enum btrfs_loop_type {
4838
	LOOP_FIND_IDEAL = 0,
J
Josef Bacik 已提交
4839 4840 4841 4842 4843 4844
	LOOP_CACHING_NOWAIT = 1,
	LOOP_CACHING_WAIT = 2,
	LOOP_ALLOC_CHUNK = 3,
	LOOP_NO_EMPTY_SIZE = 4,
};

4845 4846 4847 4848
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
 * ins->objectid == block start
4849
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
4850 4851 4852
 * ins->offset == number of blocks
 * Any available blocks before search_start are skipped.
 */
C
Chris Mason 已提交
4853
static noinline int find_free_extent(struct btrfs_trans_handle *trans,
4854 4855 4856 4857 4858
				     struct btrfs_root *orig_root,
				     u64 num_bytes, u64 empty_size,
				     u64 search_start, u64 search_end,
				     u64 hint_byte, struct btrfs_key *ins,
				     int data)
4859
{
4860
	int ret = 0;
C
Chris Mason 已提交
4861
	struct btrfs_root *root = orig_root->fs_info->extent_root;
4862
	struct btrfs_free_cluster *last_ptr = NULL;
4863
	struct btrfs_block_group_cache *block_group = NULL;
4864
	int empty_cluster = 2 * 1024 * 1024;
C
Chris Mason 已提交
4865
	int allowed_chunk_alloc = 0;
4866
	int done_chunk_alloc = 0;
4867
	struct btrfs_space_info *space_info;
4868 4869
	int last_ptr_loop = 0;
	int loop = 0;
4870
	int index = 0;
J
Josef Bacik 已提交
4871
	bool found_uncached_bg = false;
4872
	bool failed_cluster_refill = false;
4873
	bool failed_alloc = false;
4874
	bool use_cluster = true;
4875 4876
	u64 ideal_cache_percent = 0;
	u64 ideal_cache_offset = 0;
4877

4878
	WARN_ON(num_bytes < root->sectorsize);
4879
	btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
4880 4881
	ins->objectid = 0;
	ins->offset = 0;
4882

J
Josef Bacik 已提交
4883
	space_info = __find_space_info(root->fs_info, data);
4884 4885 4886 4887
	if (!space_info) {
		printk(KERN_ERR "No space info for %d\n", data);
		return -ENOSPC;
	}
J
Josef Bacik 已提交
4888

4889 4890 4891 4892 4893 4894 4895
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

C
Chris Mason 已提交
4896 4897 4898
	if (orig_root->ref_cows || empty_size)
		allowed_chunk_alloc = 1;

4899
	if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
4900
		last_ptr = &root->fs_info->meta_alloc_cluster;
4901 4902
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
4903 4904
	}

4905 4906
	if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
	    btrfs_test_opt(root, SSD)) {
4907 4908
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
J
Josef Bacik 已提交
4909

4910
	if (last_ptr) {
4911 4912 4913 4914
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
4915
	}
4916

4917
	search_start = max(search_start, first_logical_byte(root, 0));
4918
	search_start = max(search_start, hint_byte);
4919

J
Josef Bacik 已提交
4920
	if (!last_ptr)
4921 4922
		empty_cluster = 0;

J
Josef Bacik 已提交
4923
	if (search_start == hint_byte) {
4924
ideal_cache:
J
Josef Bacik 已提交
4925 4926
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
J
Josef Bacik 已提交
4927 4928 4929
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
4930 4931 4932
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
4933 4934
		 */
		if (block_group && block_group_bits(block_group, data) &&
4935 4936
		    (block_group->cached != BTRFS_CACHE_NO ||
		     search_start == ideal_cache_offset)) {
J
Josef Bacik 已提交
4937
			down_read(&space_info->groups_sem);
4938 4939 4940 4941 4942 4943 4944 4945 4946 4947
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
4948
			} else {
4949
				index = get_block_group_index(block_group);
4950
				goto have_block_group;
4951
			}
J
Josef Bacik 已提交
4952
		} else if (block_group) {
4953
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
4954
		}
4955
	}
J
Josef Bacik 已提交
4956
search:
4957
	down_read(&space_info->groups_sem);
4958 4959
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
4960
		u64 offset;
J
Josef Bacik 已提交
4961
		int cached;
4962

4963
		btrfs_get_block_group(block_group);
J
Josef Bacik 已提交
4964
		search_start = block_group->key.objectid;
4965

4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
		if (!block_group_bits(block_group, data)) {
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
			if ((data & extra) && !(block_group->flags & extra))
				goto loop;
		}

J
Josef Bacik 已提交
4985
have_block_group:
J
Josef Bacik 已提交
4986
		if (unlikely(block_group->cached == BTRFS_CACHE_NO)) {
4987 4988
			u64 free_percent;

4989 4990
			ret = cache_block_group(block_group, trans,
						orig_root, 1);
4991 4992 4993
			if (block_group->cached == BTRFS_CACHE_FINISHED)
				goto have_block_group;

4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004
			free_percent = btrfs_block_group_used(&block_group->item);
			free_percent *= 100;
			free_percent = div64_u64(free_percent,
						 block_group->key.offset);
			free_percent = 100 - free_percent;
			if (free_percent > ideal_cache_percent &&
			    likely(!block_group->ro)) {
				ideal_cache_offset = block_group->key.objectid;
				ideal_cache_percent = free_percent;
			}

J
Josef Bacik 已提交
5005
			/*
5006 5007 5008 5009
			 * We only want to start kthread caching if we are at
			 * the point where we will wait for caching to make
			 * progress, or if our ideal search is over and we've
			 * found somebody to start caching.
J
Josef Bacik 已提交
5010 5011
			 */
			if (loop > LOOP_CACHING_NOWAIT ||
5012 5013
			    (loop > LOOP_FIND_IDEAL &&
			     atomic_read(&space_info->caching_threads) < 2)) {
5014 5015
				ret = cache_block_group(block_group, trans,
							orig_root, 0);
J
Josef Bacik 已提交
5016
				BUG_ON(ret);
J
Josef Bacik 已提交
5017
			}
J
Josef Bacik 已提交
5018 5019
			found_uncached_bg = true;

5020 5021 5022 5023 5024
			/*
			 * If loop is set for cached only, try the next block
			 * group.
			 */
			if (loop == LOOP_FIND_IDEAL)
J
Josef Bacik 已提交
5025 5026 5027
				goto loop;
		}

5028 5029 5030 5031
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached))
			found_uncached_bg = true;

5032
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
5033
			goto loop;
J
Josef Bacik 已提交
5034

5035 5036 5037 5038 5039 5040 5041 5042 5043 5044
		/*
		 * Ok we want to try and use the cluster allocator, so lets look
		 * there, unless we are on LOOP_NO_EMPTY_SIZE, since we will
		 * have tried the cluster allocator plenty of times at this
		 * point and not have found anything, so we are likely way too
		 * fragmented for the clustering stuff to find anything, so lets
		 * just skip it and let the allocator find whatever block it can
		 * find
		 */
		if (last_ptr && loop < LOOP_NO_EMPTY_SIZE) {
5045 5046 5047 5048 5049
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
			spin_lock(&last_ptr->refill_lock);
5050 5051 5052 5053 5054 5055 5056
			if (last_ptr->block_group &&
			    (last_ptr->block_group->ro ||
			    !block_group_bits(last_ptr->block_group, data))) {
				offset = 0;
				goto refill_cluster;
			}

5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075
			offset = btrfs_alloc_from_cluster(block_group, last_ptr,
						 num_bytes, search_start);
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
				goto checks;
			}

			spin_lock(&last_ptr->lock);
			/*
			 * whoops, this cluster doesn't actually point to
			 * this block group.  Get a ref on the block
			 * group is does point to and try again
			 */
			if (!last_ptr_loop && last_ptr->block_group &&
			    last_ptr->block_group != block_group) {

				btrfs_put_block_group(block_group);
				block_group = last_ptr->block_group;
5076
				btrfs_get_block_group(block_group);
5077 5078 5079 5080 5081
				spin_unlock(&last_ptr->lock);
				spin_unlock(&last_ptr->refill_lock);

				last_ptr_loop = 1;
				search_start = block_group->key.objectid;
5082 5083 5084 5085 5086 5087 5088
				/*
				 * we know this block group is properly
				 * in the list because
				 * btrfs_remove_block_group, drops the
				 * cluster before it removes the block
				 * group from the list
				 */
5089 5090 5091
				goto have_block_group;
			}
			spin_unlock(&last_ptr->lock);
5092
refill_cluster:
5093 5094 5095 5096 5097 5098 5099 5100 5101
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

			last_ptr_loop = 0;

			/* allocate a cluster in this block group */
5102
			ret = btrfs_find_space_cluster(trans, root,
5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118
					       block_group, last_ptr,
					       offset, num_bytes,
					       empty_cluster + empty_size);
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
						  last_ptr, num_bytes,
						  search_start);
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
					goto checks;
				}
5119 5120
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
5121 5122
				spin_unlock(&last_ptr->refill_lock);

5123
				failed_cluster_refill = true;
J
Josef Bacik 已提交
5124 5125 5126
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
5127
			}
J
Josef Bacik 已提交
5128

5129 5130 5131 5132 5133 5134
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
5135
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
5136
			spin_unlock(&last_ptr->refill_lock);
5137
			goto loop;
5138 5139
		}

5140 5141
		offset = btrfs_find_space_for_alloc(block_group, search_start,
						    num_bytes, empty_size);
5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
5153
			wait_block_group_cache_progress(block_group,
5154 5155
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
5156
			goto have_block_group;
5157 5158
		} else if (!offset) {
			goto loop;
J
Josef Bacik 已提交
5159
		}
5160
checks:
5161
		search_start = stripe_align(root, offset);
J
Josef Bacik 已提交
5162
		/* move on to the next group */
5163 5164
		if (search_start + num_bytes >= search_end) {
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5165
			goto loop;
5166
		}
5167

J
Josef Bacik 已提交
5168 5169
		/* move on to the next group */
		if (search_start + num_bytes >
5170 5171
		    block_group->key.objectid + block_group->key.offset) {
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5172
			goto loop;
5173
		}
5174

5175 5176
		ins->objectid = search_start;
		ins->offset = num_bytes;
J
Josef Bacik 已提交
5177

5178 5179 5180 5181
		if (offset < search_start)
			btrfs_add_free_space(block_group, offset,
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
5182

5183 5184 5185
		ret = update_reserved_bytes(block_group, num_bytes, 1,
					    (data & BTRFS_BLOCK_GROUP_DATA));
		if (ret == -EAGAIN) {
5186
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5187
			goto loop;
J
Josef Bacik 已提交
5188
		}
5189

5190
		/* we are all good, lets return */
J
Josef Bacik 已提交
5191 5192
		ins->objectid = search_start;
		ins->offset = num_bytes;
5193

5194 5195 5196 5197
		if (offset < search_start)
			btrfs_add_free_space(block_group, offset,
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
5198 5199
		break;
loop:
5200
		failed_cluster_refill = false;
5201
		failed_alloc = false;
5202
		BUG_ON(index != get_block_group_index(block_group));
5203
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5204 5205 5206
	}
	up_read(&space_info->groups_sem);

5207 5208 5209
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

5210 5211 5212 5213 5214
	/* LOOP_FIND_IDEAL, only search caching/cached bg's, and don't wait for
	 *			for them to make caching progress.  Also
	 *			determine the best possible bg to cache
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
5215 5216 5217 5218
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
5219
	 */
J
Josef Bacik 已提交
5220 5221 5222
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE &&
	    (found_uncached_bg || empty_size || empty_cluster ||
	     allowed_chunk_alloc)) {
5223
		index = 0;
5224
		if (loop == LOOP_FIND_IDEAL && found_uncached_bg) {
J
Josef Bacik 已提交
5225
			found_uncached_bg = false;
5226 5227 5228
			loop++;
			if (!ideal_cache_percent &&
			    atomic_read(&space_info->caching_threads))
J
Josef Bacik 已提交
5229
				goto search;
5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264

			/*
			 * 1 of the following 2 things have happened so far
			 *
			 * 1) We found an ideal block group for caching that
			 * is mostly full and will cache quickly, so we might
			 * as well wait for it.
			 *
			 * 2) We searched for cached only and we didn't find
			 * anything, and we didn't start any caching kthreads
			 * either, so chances are we will loop through and
			 * start a couple caching kthreads, and then come back
			 * around and just wait for them.  This will be slower
			 * because we will have 2 caching kthreads reading at
			 * the same time when we could have just started one
			 * and waited for it to get far enough to give us an
			 * allocation, so go ahead and go to the wait caching
			 * loop.
			 */
			loop = LOOP_CACHING_WAIT;
			search_start = ideal_cache_offset;
			ideal_cache_percent = 0;
			goto ideal_cache;
		} else if (loop == LOOP_FIND_IDEAL) {
			/*
			 * Didn't find a uncached bg, wait on anything we find
			 * next.
			 */
			loop = LOOP_CACHING_WAIT;
			goto search;
		}

		if (loop < LOOP_CACHING_WAIT) {
			loop++;
			goto search;
J
Josef Bacik 已提交
5265 5266 5267
		}

		if (loop == LOOP_ALLOC_CHUNK) {
5268 5269 5270
			empty_size = 0;
			empty_cluster = 0;
		}
J
Josef Bacik 已提交
5271 5272 5273 5274 5275

		if (allowed_chunk_alloc) {
			ret = do_chunk_alloc(trans, root, num_bytes +
					     2 * 1024 * 1024, data, 1);
			allowed_chunk_alloc = 0;
5276 5277
			done_chunk_alloc = 1;
		} else if (!done_chunk_alloc) {
J
Josef Bacik 已提交
5278 5279 5280
			space_info->force_alloc = 1;
		}

J
Josef Bacik 已提交
5281
		if (loop < LOOP_NO_EMPTY_SIZE) {
5282
			loop++;
J
Josef Bacik 已提交
5283
			goto search;
5284
		}
J
Josef Bacik 已提交
5285 5286 5287
		ret = -ENOSPC;
	} else if (!ins->objectid) {
		ret = -ENOSPC;
5288
	}
5289

5290 5291 5292
	/* we found what we needed */
	if (ins->objectid) {
		if (!(data & BTRFS_BLOCK_GROUP_DATA))
5293
			trans->block_group = block_group->key.objectid;
J
Josef Bacik 已提交
5294

5295
		btrfs_put_block_group(block_group);
5296
		ret = 0;
C
Chris Mason 已提交
5297 5298
	}

C
Chris Mason 已提交
5299
	return ret;
5300
}
5301

J
Josef Bacik 已提交
5302 5303
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
5304 5305
{
	struct btrfs_block_group_cache *cache;
5306
	int index = 0;
J
Josef Bacik 已提交
5307

J
Josef Bacik 已提交
5308
	spin_lock(&info->lock);
C
Chris Mason 已提交
5309 5310
	printk(KERN_INFO "space_info has %llu free, is %sfull\n",
	       (unsigned long long)(info->total_bytes - info->bytes_used -
J
Josef Bacik 已提交
5311
				    info->bytes_pinned - info->bytes_reserved -
5312
				    info->bytes_readonly),
C
Chris Mason 已提交
5313
	       (info->full) ? "" : "not ");
5314 5315
	printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
5316
	       (unsigned long long)info->total_bytes,
5317
	       (unsigned long long)info->bytes_used,
5318
	       (unsigned long long)info->bytes_pinned,
5319
	       (unsigned long long)info->bytes_reserved,
5320
	       (unsigned long long)info->bytes_may_use,
5321
	       (unsigned long long)info->bytes_readonly);
J
Josef Bacik 已提交
5322 5323 5324 5325
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
5326

5327
	down_read(&info->groups_sem);
5328 5329
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
5330
		spin_lock(&cache->lock);
C
Chris Mason 已提交
5331 5332 5333 5334 5335 5336 5337
		printk(KERN_INFO "block group %llu has %llu bytes, %llu used "
		       "%llu pinned %llu reserved\n",
		       (unsigned long long)cache->key.objectid,
		       (unsigned long long)cache->key.offset,
		       (unsigned long long)btrfs_block_group_used(&cache->item),
		       (unsigned long long)cache->pinned,
		       (unsigned long long)cache->reserved);
J
Josef Bacik 已提交
5338 5339 5340
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
5341 5342
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
5343
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
5344
}
5345

5346 5347 5348 5349 5350 5351
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
			 u64 search_end, struct btrfs_key *ins,
			 u64 data)
5352 5353
{
	int ret;
5354
	u64 search_start = 0;
5355

J
Josef Bacik 已提交
5356
	data = btrfs_get_alloc_profile(root, data);
5357
again:
C
Chris Mason 已提交
5358 5359 5360 5361
	/*
	 * the only place that sets empty_size is btrfs_realloc_node, which
	 * is not called recursively on allocations
	 */
J
Josef Bacik 已提交
5362
	if (empty_size || root->ref_cows)
5363
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
C
Chris Mason 已提交
5364
				     num_bytes + 2 * 1024 * 1024, data, 0);
5365

5366 5367
	WARN_ON(num_bytes < root->sectorsize);
	ret = find_free_extent(trans, root, num_bytes, empty_size,
5368 5369
			       search_start, search_end, hint_byte,
			       ins, data);
5370

5371 5372
	if (ret == -ENOSPC && num_bytes > min_alloc_size) {
		num_bytes = num_bytes >> 1;
J
Josef Bacik 已提交
5373
		num_bytes = num_bytes & ~(root->sectorsize - 1);
5374
		num_bytes = max(num_bytes, min_alloc_size);
C
Chris Mason 已提交
5375 5376
		do_chunk_alloc(trans, root->fs_info->extent_root,
			       num_bytes, data, 1);
5377 5378
		goto again;
	}
5379
	if (ret == -ENOSPC && btrfs_test_opt(root, ENOSPC_DEBUG)) {
J
Josef Bacik 已提交
5380 5381 5382
		struct btrfs_space_info *sinfo;

		sinfo = __find_space_info(root->fs_info, data);
C
Chris Mason 已提交
5383 5384 5385
		printk(KERN_ERR "btrfs allocation failed flags %llu, "
		       "wanted %llu\n", (unsigned long long)data,
		       (unsigned long long)num_bytes);
J
Josef Bacik 已提交
5386
		dump_space_info(sinfo, num_bytes, 1);
5387
	}
J
Josef Bacik 已提交
5388 5389

	return ret;
5390 5391
}

5392 5393
int btrfs_free_reserved_extent(struct btrfs_root *root, u64 start, u64 len)
{
J
Josef Bacik 已提交
5394
	struct btrfs_block_group_cache *cache;
5395
	int ret = 0;
J
Josef Bacik 已提交
5396 5397 5398

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
C
Chris Mason 已提交
5399 5400
		printk(KERN_ERR "Unable to find block group for %llu\n",
		       (unsigned long long)start);
J
Josef Bacik 已提交
5401 5402
		return -ENOSPC;
	}
5403 5404 5405

	ret = btrfs_discard_extent(root, start, len);

J
Josef Bacik 已提交
5406
	btrfs_add_free_space(cache, start, len);
5407
	update_reserved_bytes(cache, len, 0, 1);
5408
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
5409

5410 5411 5412
	return ret;
}

5413 5414 5415 5416 5417
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
5418 5419
{
	int ret;
5420
	struct btrfs_fs_info *fs_info = root->fs_info;
5421
	struct btrfs_extent_item *extent_item;
5422
	struct btrfs_extent_inline_ref *iref;
5423
	struct btrfs_path *path;
5424 5425 5426
	struct extent_buffer *leaf;
	int type;
	u32 size;
5427

5428 5429 5430 5431
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
5432

5433
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
5434 5435 5436

	path = btrfs_alloc_path();
	BUG_ON(!path);
5437

5438
	path->leave_spinning = 1;
5439 5440
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
C
Chris Mason 已提交
5441
	BUG_ON(ret);
J
Josef Bacik 已提交
5442

5443 5444
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
5445
				     struct btrfs_extent_item);
5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
5466 5467

	btrfs_mark_buffer_dirty(path->nodes[0]);
5468
	btrfs_free_path(path);
5469

5470
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5471
	if (ret) {
C
Chris Mason 已提交
5472 5473 5474
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
5475 5476
		BUG();
	}
5477 5478 5479
	return ret;
}

5480 5481 5482 5483 5484
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins)
5485 5486
{
	int ret;
5487 5488 5489 5490 5491 5492 5493
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
5494

5495 5496
	path = btrfs_alloc_path();
	BUG_ON(!path);
5497

5498 5499 5500
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
5501
	BUG_ON(ret);
5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
	block_info = (struct btrfs_tree_block_info *)(extent_item + 1);

	btrfs_set_tree_block_key(leaf, block_info, key);
	btrfs_set_tree_block_level(leaf, block_info, level);

	iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

5530
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551
	if (ret) {
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
		BUG();
	}
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

	ret = btrfs_add_delayed_data_ref(trans, ins->objectid, ins->offset,
					 0, root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL);
5552 5553
	return ret;
}
5554 5555 5556 5557 5558 5559

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
5560 5561 5562 5563
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
5564 5565 5566
{
	int ret;
	struct btrfs_block_group_cache *block_group;
5567 5568 5569
	struct btrfs_caching_control *caching_ctl;
	u64 start = ins->objectid;
	u64 num_bytes = ins->offset;
5570 5571

	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
5572
	cache_block_group(block_group, trans, NULL, 0);
5573
	caching_ctl = get_caching_control(block_group);
5574

5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605
	if (!caching_ctl) {
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
		BUG_ON(ret);
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
			BUG_ON(ret);
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			BUG_ON(ret);
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			BUG_ON(ret);

			start = caching_ctl->progress;
			num_bytes = ins->objectid + ins->offset -
				    caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
			BUG_ON(ret);
		}

		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}

5606 5607
	ret = update_reserved_bytes(block_group, ins->offset, 1, 1);
	BUG_ON(ret);
5608
	btrfs_put_block_group(block_group);
5609 5610
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
5611 5612 5613
	return ret;
}

5614 5615
struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
					    struct btrfs_root *root,
5616 5617
					    u64 bytenr, u32 blocksize,
					    int level)
5618 5619 5620 5621 5622 5623 5624
{
	struct extent_buffer *buf;

	buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
5625
	btrfs_set_buffer_lockdep_class(buf, level);
5626 5627
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
5628 5629

	btrfs_set_lock_blocking(buf);
5630
	btrfs_set_buffer_uptodate(buf);
5631

5632
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
5633 5634 5635 5636 5637 5638 5639 5640 5641 5642
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
		if (root->log_transid % 2 == 0)
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
5643 5644
	} else {
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
5645
			 buf->start + buf->len - 1, GFP_NOFS);
5646
	}
5647
	trans->blocks_used++;
5648
	/* this returns a buffer locked for blocking */
5649 5650 5651
	return buf;
}

5652 5653 5654 5655 5656
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
5657
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
5658 5659 5660 5661 5662
	int ret;

	block_rsv = get_block_rsv(trans, root);

	if (block_rsv->size == 0) {
5663 5664
		ret = reserve_metadata_bytes(trans, root, block_rsv,
					     blocksize, 0);
5665 5666 5667 5668 5669 5670 5671 5672
		/*
		 * If we couldn't reserve metadata bytes try and use some from
		 * the global reserve.
		 */
		if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
5673
			return ERR_PTR(ret);
5674
		} else if (ret) {
5675
			return ERR_PTR(ret);
5676
		}
5677 5678 5679 5680 5681 5682
		return block_rsv;
	}

	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;
5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697
	if (ret) {
		WARN_ON(1);
		ret = reserve_metadata_bytes(trans, root, block_rsv, blocksize,
					     0);
		if (!ret) {
			spin_lock(&block_rsv->lock);
			block_rsv->size += blocksize;
			spin_unlock(&block_rsv->lock);
			return block_rsv;
		} else if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
		}
	}
5698 5699 5700 5701 5702 5703 5704 5705 5706 5707

	return ERR_PTR(-ENOSPC);
}

static void unuse_block_rsv(struct btrfs_block_rsv *block_rsv, u32 blocksize)
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
	block_rsv_release_bytes(block_rsv, NULL, 0);
}

5708
/*
5709 5710 5711 5712
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
5713 5714
 * returns the tree buffer or NULL.
 */
5715
struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
5716 5717 5718 5719
					struct btrfs_root *root, u32 blocksize,
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
					u64 hint, u64 empty_size)
5720
{
C
Chris Mason 已提交
5721
	struct btrfs_key ins;
5722
	struct btrfs_block_rsv *block_rsv;
5723
	struct extent_buffer *buf;
5724 5725 5726
	u64 flags = 0;
	int ret;

5727

5728 5729 5730 5731 5732 5733
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

	ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
				   empty_size, hint, (u64)-1, &ins, 0);
5734
	if (ret) {
5735
		unuse_block_rsv(block_rsv, blocksize);
5736
		return ERR_PTR(ret);
5737
	}
5738

5739 5740
	buf = btrfs_init_new_buffer(trans, root, ins.objectid,
				    blocksize, level);
5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768
	BUG_ON(IS_ERR(buf));

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
		extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
		BUG_ON(!extent_op);
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
		extent_op->update_key = 1;
		extent_op->update_flags = 1;
		extent_op->is_data = 0;

		ret = btrfs_add_delayed_tree_ref(trans, ins.objectid,
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
					extent_op);
		BUG_ON(ret);
	}
5769 5770
	return buf;
}
5771

5772 5773 5774 5775 5776 5777 5778 5779 5780
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
5781 5782
	int reada_slot;
	int reada_count;
5783 5784 5785 5786 5787
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
5788 5789 5790 5791
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
5792
{
Y
Yan, Zheng 已提交
5793 5794 5795
	u64 bytenr;
	u64 generation;
	u64 refs;
5796
	u64 flags;
5797
	u32 nritems;
Y
Yan, Zheng 已提交
5798 5799 5800
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
5801
	int ret;
Y
Yan, Zheng 已提交
5802 5803
	int slot;
	int nread = 0;
5804

Y
Yan, Zheng 已提交
5805 5806 5807 5808 5809 5810 5811 5812
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
5813

Y
Yan, Zheng 已提交
5814 5815 5816
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
	blocksize = btrfs_level_size(root, wc->level - 1);
5817

Y
Yan, Zheng 已提交
5818 5819 5820
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
5821

C
Chris Mason 已提交
5822
		cond_resched();
Y
Yan, Zheng 已提交
5823 5824
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
5825

Y
Yan, Zheng 已提交
5826 5827
		if (slot == path->slots[wc->level])
			goto reada;
5828

Y
Yan, Zheng 已提交
5829 5830
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
5831 5832
			continue;

5833 5834 5835 5836 5837 5838
		/* We don't lock the tree block, it's OK to be racy here */
		ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
					       &refs, &flags);
		BUG_ON(ret);
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
5839 5840 5841
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
5842

5843 5844 5845
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
5846 5847 5848 5849 5850 5851 5852 5853
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
5854 5855 5856 5857
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
5858
		}
Y
Yan, Zheng 已提交
5859 5860 5861 5862
reada:
		ret = readahead_tree_block(root, bytenr, blocksize,
					   generation);
		if (ret)
5863
			break;
Y
Yan, Zheng 已提交
5864
		nread++;
C
Chris Mason 已提交
5865
	}
Y
Yan, Zheng 已提交
5866
	wc->reada_slot = slot;
C
Chris Mason 已提交
5867
}
5868

Y
Yan Zheng 已提交
5869
/*
5870 5871 5872 5873 5874 5875
 * hepler to process tree block while walking down the tree.
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
5876
 */
5877
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
5878
				   struct btrfs_root *root,
5879
				   struct btrfs_path *path,
5880
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
5881
{
5882 5883 5884
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
5885 5886
	int ret;

5887 5888 5889
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
5890

5891 5892 5893 5894
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
5895 5896 5897
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
5898 5899 5900 5901 5902 5903 5904 5905
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
					       eb->start, eb->len,
					       &wc->refs[level],
					       &wc->flags[level]);
		BUG_ON(ret);
		BUG_ON(wc->refs[level] == 0);
	}
5906

5907 5908 5909
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
5910

5911 5912 5913 5914 5915 5916
		if (path->locks[level] && !wc->keep_locks) {
			btrfs_tree_unlock(eb);
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
5917

5918 5919 5920 5921
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
		ret = btrfs_inc_ref(trans, root, eb, 1);
Y
Yan Zheng 已提交
5922
		BUG_ON(ret);
5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941
		ret = btrfs_dec_ref(trans, root, eb, 0);
		BUG_ON(ret);
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
						  eb->len, flag, 0);
		BUG_ON(ret);
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
		btrfs_tree_unlock(eb);
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957
/*
 * hepler to process tree block pointer.
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
5958
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
5978 5979
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
5980
		return 1;
5981
	}
Y
Yan, Zheng 已提交
5982 5983 5984 5985 5986 5987 5988

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
	blocksize = btrfs_level_size(root, level - 1);

	next = btrfs_find_tree_block(root, bytenr, blocksize);
	if (!next) {
		next = btrfs_find_create_tree_block(root, bytenr, blocksize);
5989 5990
		if (!next)
			return -ENOMEM;
Y
Yan, Zheng 已提交
5991 5992 5993 5994 5995
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

5996 5997 5998 5999 6000 6001
	ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
	BUG_ON(ret);
	BUG_ON(wc->refs[level - 1] == 0);
	*lookup_info = 0;
Y
Yan, Zheng 已提交
6002

6003
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
6004
		if (wc->refs[level - 1] > 1) {
6005 6006 6007 6008
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
6022 6023 6024 6025
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
6026 6027 6028 6029 6030 6031
	}

	if (!btrfs_buffer_uptodate(next, generation)) {
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
6032
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
		next = read_tree_block(root, bytenr, blocksize, generation);
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
	path->locks[level] = 1;
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
6055 6056 6057 6058 6059 6060 6061 6062
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
6063

6064 6065 6066
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
					root->root_key.objectid, level - 1, 0);
		BUG_ON(ret);
Y
Yan, Zheng 已提交
6067 6068 6069
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
6070
	*lookup_info = 1;
Y
Yan, Zheng 已提交
6071 6072 6073
	return 1;
}

6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090
/*
 * hepler to process tree block while walking up the tree.
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
6091
	int ret;
6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
			path->locks[level] = 1;

			ret = btrfs_lookup_extent_info(trans, root,
						       eb->start, eb->len,
						       &wc->refs[level],
						       &wc->flags[level]);
Y
Yan Zheng 已提交
6124
			BUG_ON(ret);
6125 6126 6127 6128 6129 6130
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
				btrfs_tree_unlock(eb);
				path->locks[level] = 0;
				return 1;
			}
Y
Yan Zheng 已提交
6131
		}
6132
	}
Y
Yan Zheng 已提交
6133

6134 6135
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6136

6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
				ret = btrfs_dec_ref(trans, root, eb, 1);
			else
				ret = btrfs_dec_ref(trans, root, eb, 0);
			BUG_ON(ret);
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
			path->locks[level] = 1;
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
6167 6168
	}

6169
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
6170 6171 6172
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
6173
	return 0;
6174 6175 6176 6177 6178 6179 6180 6181
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
6182
	int lookup_info = 1;
6183 6184 6185
	int ret;

	while (level >= 0) {
6186
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
6187 6188 6189 6190 6191 6192
		if (ret > 0)
			break;

		if (level == 0)
			break;

6193 6194 6195 6196
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

6197
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
6198 6199 6200
		if (ret > 0) {
			path->slots[level]++;
			continue;
6201 6202
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
6203
		level = wc->level;
Y
Yan Zheng 已提交
6204 6205 6206 6207
	}
	return 0;
}

C
Chris Mason 已提交
6208
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6209
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
6210
				 struct btrfs_path *path,
6211
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
6212
{
6213
	int level = wc->level;
C
Chris Mason 已提交
6214
	int ret;
6215

6216 6217 6218 6219 6220 6221
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
6222 6223
			return 0;
		} else {
6224 6225 6226
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
6227

6228 6229 6230
			if (path->locks[level]) {
				btrfs_tree_unlock(path->nodes[level]);
				path->locks[level] = 0;
Y
Yan Zheng 已提交
6231
			}
6232 6233 6234
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
6235 6236 6237 6238 6239
		}
	}
	return 1;
}

C
Chris Mason 已提交
6240
/*
6241 6242 6243 6244 6245 6246 6247 6248 6249
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
C
Chris Mason 已提交
6250
 */
6251 6252
int btrfs_drop_snapshot(struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv, int update_ref)
C
Chris Mason 已提交
6253
{
6254
	struct btrfs_path *path;
6255 6256
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
6257
	struct btrfs_root_item *root_item = &root->root_item;
6258 6259 6260 6261 6262
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
C
Chris Mason 已提交
6263

6264 6265
	path = btrfs_alloc_path();
	BUG_ON(!path);
C
Chris Mason 已提交
6266

6267 6268 6269
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
	BUG_ON(!wc);

6270
	trans = btrfs_start_transaction(tree_root, 0);
6271 6272
	BUG_ON(IS_ERR(trans));

6273 6274
	if (block_rsv)
		trans->block_rsv = block_rsv;
6275

6276
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
6277
		level = btrfs_header_level(root->node);
6278 6279
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
6280
		path->slots[level] = 0;
6281
		path->locks[level] = 1;
6282 6283
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
6284 6285
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
6286 6287 6288
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

6289
		level = root_item->drop_level;
6290
		BUG_ON(level == 0);
6291
		path->lowest_level = level;
6292 6293 6294 6295
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
6296 6297
			goto out;
		}
Y
Yan, Zheng 已提交
6298
		WARN_ON(ret > 0);
6299

6300 6301 6302 6303
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
6304
		btrfs_unlock_up_safe(path, 0);
6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
						path->nodes[level]->len,
						&wc->refs[level],
						&wc->flags[level]);
			BUG_ON(ret);
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
6326
	}
6327 6328 6329 6330 6331 6332

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
Y
Yan, Zheng 已提交
6333
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
6334

C
Chris Mason 已提交
6335
	while (1) {
6336 6337 6338
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6339
			break;
6340
		}
C
Chris Mason 已提交
6341

6342 6343 6344
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6345
			break;
6346 6347 6348 6349
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
6350 6351
			break;
		}
6352 6353 6354 6355 6356 6357 6358 6359 6360 6361

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
6362
		if (btrfs_should_end_transaction(trans, tree_root)) {
6363 6364 6365 6366 6367
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
			BUG_ON(ret);

6368
			btrfs_end_transaction_throttle(trans, tree_root);
6369
			trans = btrfs_start_transaction(tree_root, 0);
6370
			BUG_ON(IS_ERR(trans));
6371 6372
			if (block_rsv)
				trans->block_rsv = block_rsv;
6373
		}
C
Chris Mason 已提交
6374
	}
6375 6376 6377 6378 6379 6380
	btrfs_release_path(root, path);
	BUG_ON(err);

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
	BUG_ON(ret);

6381 6382 6383 6384 6385
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
		ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
					   NULL, NULL);
		BUG_ON(ret < 0);
		if (ret > 0) {
6386 6387 6388 6389 6390 6391 6392
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
6393 6394 6395 6396 6397 6398 6399 6400 6401 6402
		}
	}

	if (root->in_radix) {
		btrfs_free_fs_root(tree_root->fs_info, root);
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
		kfree(root);
	}
6403
out:
6404
	btrfs_end_transaction_throttle(trans, tree_root);
6405
	kfree(wc);
6406
	btrfs_free_path(path);
6407
	return err;
C
Chris Mason 已提交
6408
}
C
Chris Mason 已提交
6409

6410 6411 6412 6413 6414
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
 */
Y
Yan Zheng 已提交
6415 6416 6417 6418 6419 6420
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
6421
	struct walk_control *wc;
Y
Yan Zheng 已提交
6422 6423 6424 6425 6426
	int level;
	int parent_level;
	int ret = 0;
	int wret;

6427 6428
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
6429 6430 6431
	path = btrfs_alloc_path();
	BUG_ON(!path);

6432 6433 6434
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
	BUG_ON(!wc);

6435
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
6436 6437 6438 6439 6440
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

6441
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
6442 6443 6444
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
6445 6446 6447 6448 6449 6450 6451 6452 6453
	path->locks[level] = 1;

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
Y
Yan, Zheng 已提交
6454
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
6455 6456

	while (1) {
6457 6458
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
6459 6460
			ret = wret;
			break;
6461
		}
Y
Yan Zheng 已提交
6462

6463
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
6464 6465 6466 6467 6468 6469
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

6470
	kfree(wc);
Y
Yan Zheng 已提交
6471 6472 6473 6474
	btrfs_free_path(path);
	return ret;
}

6475
#if 0
6476 6477 6478 6479 6480 6481
static unsigned long calc_ra(unsigned long start, unsigned long last,
			     unsigned long nr)
{
	return min(last, start + nr - 1);
}

C
Chris Mason 已提交
6482
static noinline int relocate_inode_pages(struct inode *inode, u64 start,
6483
					 u64 len)
6484 6485 6486
{
	u64 page_start;
	u64 page_end;
Z
Zheng Yan 已提交
6487
	unsigned long first_index;
6488 6489 6490
	unsigned long last_index;
	unsigned long i;
	struct page *page;
6491
	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
6492
	struct file_ra_state *ra;
6493
	struct btrfs_ordered_extent *ordered;
Z
Zheng Yan 已提交
6494 6495 6496
	unsigned int total_read = 0;
	unsigned int total_dirty = 0;
	int ret = 0;
6497 6498

	ra = kzalloc(sizeof(*ra), GFP_NOFS);
6499 6500
	if (!ra)
		return -ENOMEM;
6501 6502

	mutex_lock(&inode->i_mutex);
Z
Zheng Yan 已提交
6503
	first_index = start >> PAGE_CACHE_SHIFT;
6504 6505
	last_index = (start + len - 1) >> PAGE_CACHE_SHIFT;

Z
Zheng Yan 已提交
6506 6507 6508 6509 6510
	/* make sure the dirty trick played by the caller work */
	ret = invalidate_inode_pages2_range(inode->i_mapping,
					    first_index, last_index);
	if (ret)
		goto out_unlock;
6511

6512
	file_ra_state_init(ra, inode->i_mapping);
6513

Z
Zheng Yan 已提交
6514 6515
	for (i = first_index ; i <= last_index; i++) {
		if (total_read % ra->ra_pages == 0) {
6516
			btrfs_force_ra(inode->i_mapping, ra, NULL, i,
Z
Zheng Yan 已提交
6517
				       calc_ra(i, last_index, ra->ra_pages));
6518 6519
		}
		total_read++;
6520 6521
again:
		if (((u64)i << PAGE_CACHE_SHIFT) > i_size_read(inode))
Z
Zheng Yan 已提交
6522
			BUG_ON(1);
6523
		page = grab_cache_page(inode->i_mapping, i);
6524
		if (!page) {
Z
Zheng Yan 已提交
6525
			ret = -ENOMEM;
6526
			goto out_unlock;
6527
		}
6528 6529 6530 6531 6532 6533
		if (!PageUptodate(page)) {
			btrfs_readpage(NULL, page);
			lock_page(page);
			if (!PageUptodate(page)) {
				unlock_page(page);
				page_cache_release(page);
Z
Zheng Yan 已提交
6534
				ret = -EIO;
6535 6536 6537
				goto out_unlock;
			}
		}
6538
		wait_on_page_writeback(page);
6539

6540 6541
		page_start = (u64)page->index << PAGE_CACHE_SHIFT;
		page_end = page_start + PAGE_CACHE_SIZE - 1;
6542
		lock_extent(io_tree, page_start, page_end, GFP_NOFS);
6543

6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554
		ordered = btrfs_lookup_ordered_extent(inode, page_start);
		if (ordered) {
			unlock_extent(io_tree, page_start, page_end, GFP_NOFS);
			unlock_page(page);
			page_cache_release(page);
			btrfs_start_ordered_extent(inode, ordered, 1);
			btrfs_put_ordered_extent(ordered);
			goto again;
		}
		set_page_extent_mapped(page);

Z
Zheng Yan 已提交
6555 6556 6557
		if (i == first_index)
			set_extent_bits(io_tree, page_start, page_end,
					EXTENT_BOUNDARY, GFP_NOFS);
6558
		btrfs_set_extent_delalloc(inode, page_start, page_end);
Z
Zheng Yan 已提交
6559

6560
		set_page_dirty(page);
Z
Zheng Yan 已提交
6561
		total_dirty++;
6562

6563
		unlock_extent(io_tree, page_start, page_end, GFP_NOFS);
6564 6565 6566 6567 6568
		unlock_page(page);
		page_cache_release(page);
	}

out_unlock:
6569
	kfree(ra);
6570
	mutex_unlock(&inode->i_mutex);
Z
Zheng Yan 已提交
6571 6572
	balance_dirty_pages_ratelimited_nr(inode->i_mapping, total_dirty);
	return ret;
6573 6574
}

C
Chris Mason 已提交
6575
static noinline int relocate_data_extent(struct inode *reloc_inode,
Z
Zheng Yan 已提交
6576 6577 6578 6579 6580 6581
					 struct btrfs_key *extent_key,
					 u64 offset)
{
	struct btrfs_root *root = BTRFS_I(reloc_inode)->root;
	struct extent_map_tree *em_tree = &BTRFS_I(reloc_inode)->extent_tree;
	struct extent_map *em;
Y
Yan Zheng 已提交
6582 6583
	u64 start = extent_key->objectid - offset;
	u64 end = start + extent_key->offset - 1;
6584

Z
Zheng Yan 已提交
6585
	em = alloc_extent_map(GFP_NOFS);
6586
	BUG_ON(!em);
6587

Y
Yan Zheng 已提交
6588
	em->start = start;
Z
Zheng Yan 已提交
6589
	em->len = extent_key->offset;
C
Chris Mason 已提交
6590
	em->block_len = extent_key->offset;
Z
Zheng Yan 已提交
6591 6592 6593 6594 6595
	em->block_start = extent_key->objectid;
	em->bdev = root->fs_info->fs_devices->latest_bdev;
	set_bit(EXTENT_FLAG_PINNED, &em->flags);

	/* setup extent map to cheat btrfs_readpage */
Y
Yan Zheng 已提交
6596
	lock_extent(&BTRFS_I(reloc_inode)->io_tree, start, end, GFP_NOFS);
Z
Zheng Yan 已提交
6597 6598
	while (1) {
		int ret;
6599
		write_lock(&em_tree->lock);
Z
Zheng Yan 已提交
6600
		ret = add_extent_mapping(em_tree, em);
6601
		write_unlock(&em_tree->lock);
Z
Zheng Yan 已提交
6602 6603
		if (ret != -EEXIST) {
			free_extent_map(em);
6604 6605
			break;
		}
Y
Yan Zheng 已提交
6606
		btrfs_drop_extent_cache(reloc_inode, start, end, 0);
6607
	}
Y
Yan Zheng 已提交
6608
	unlock_extent(&BTRFS_I(reloc_inode)->io_tree, start, end, GFP_NOFS);
6609

Y
Yan Zheng 已提交
6610
	return relocate_inode_pages(reloc_inode, start, extent_key->offset);
Z
Zheng Yan 已提交
6611
}
6612

Z
Zheng Yan 已提交
6613 6614 6615 6616 6617 6618 6619 6620 6621
struct btrfs_ref_path {
	u64 extent_start;
	u64 nodes[BTRFS_MAX_LEVEL];
	u64 root_objectid;
	u64 root_generation;
	u64 owner_objectid;
	u32 num_refs;
	int lowest_level;
	int current_level;
Y
Yan Zheng 已提交
6622 6623 6624 6625
	int shared_level;

	struct btrfs_key node_keys[BTRFS_MAX_LEVEL];
	u64 new_nodes[BTRFS_MAX_LEVEL];
Z
Zheng Yan 已提交
6626
};
6627

Z
Zheng Yan 已提交
6628
struct disk_extent {
C
Chris Mason 已提交
6629
	u64 ram_bytes;
Z
Zheng Yan 已提交
6630 6631 6632 6633
	u64 disk_bytenr;
	u64 disk_num_bytes;
	u64 offset;
	u64 num_bytes;
C
Chris Mason 已提交
6634 6635 6636
	u8 compression;
	u8 encryption;
	u16 other_encoding;
Z
Zheng Yan 已提交
6637
};
6638

Z
Zheng Yan 已提交
6639 6640 6641 6642 6643 6644
static int is_cowonly_root(u64 root_objectid)
{
	if (root_objectid == BTRFS_ROOT_TREE_OBJECTID ||
	    root_objectid == BTRFS_EXTENT_TREE_OBJECTID ||
	    root_objectid == BTRFS_CHUNK_TREE_OBJECTID ||
	    root_objectid == BTRFS_DEV_TREE_OBJECTID ||
6645 6646
	    root_objectid == BTRFS_TREE_LOG_OBJECTID ||
	    root_objectid == BTRFS_CSUM_TREE_OBJECTID)
Z
Zheng Yan 已提交
6647 6648 6649
		return 1;
	return 0;
}
6650

C
Chris Mason 已提交
6651
static noinline int __next_ref_path(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664
				    struct btrfs_root *extent_root,
				    struct btrfs_ref_path *ref_path,
				    int first_time)
{
	struct extent_buffer *leaf;
	struct btrfs_path *path;
	struct btrfs_extent_ref *ref;
	struct btrfs_key key;
	struct btrfs_key found_key;
	u64 bytenr;
	u32 nritems;
	int level;
	int ret = 1;
6665

Z
Zheng Yan 已提交
6666 6667 6668
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
6669

Z
Zheng Yan 已提交
6670 6671 6672
	if (first_time) {
		ref_path->lowest_level = -1;
		ref_path->current_level = -1;
Y
Yan Zheng 已提交
6673
		ref_path->shared_level = -1;
Z
Zheng Yan 已提交
6674 6675 6676 6677 6678 6679 6680 6681
		goto walk_up;
	}
walk_down:
	level = ref_path->current_level - 1;
	while (level >= -1) {
		u64 parent;
		if (level < ref_path->lowest_level)
			break;
6682

C
Chris Mason 已提交
6683
		if (level >= 0)
Z
Zheng Yan 已提交
6684
			bytenr = ref_path->nodes[level];
C
Chris Mason 已提交
6685
		else
Z
Zheng Yan 已提交
6686 6687
			bytenr = ref_path->extent_start;
		BUG_ON(bytenr == 0);
6688

Z
Zheng Yan 已提交
6689 6690 6691 6692
		parent = ref_path->nodes[level + 1];
		ref_path->nodes[level + 1] = 0;
		ref_path->current_level = level;
		BUG_ON(parent == 0);
C
Chris Mason 已提交
6693

Z
Zheng Yan 已提交
6694 6695 6696
		key.objectid = bytenr;
		key.offset = parent + 1;
		key.type = BTRFS_EXTENT_REF_KEY;
6697

Z
Zheng Yan 已提交
6698 6699
		ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 0);
		if (ret < 0)
6700
			goto out;
Z
Zheng Yan 已提交
6701
		BUG_ON(ret == 0);
6702

Z
Zheng Yan 已提交
6703 6704 6705 6706 6707 6708 6709 6710 6711 6712
		leaf = path->nodes[0];
		nritems = btrfs_header_nritems(leaf);
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(extent_root, path);
			if (ret < 0)
				goto out;
			if (ret > 0)
				goto next;
			leaf = path->nodes[0];
		}
C
Chris Mason 已提交
6713

Z
Zheng Yan 已提交
6714 6715
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
		if (found_key.objectid == bytenr &&
Y
Yan Zheng 已提交
6716 6717 6718
		    found_key.type == BTRFS_EXTENT_REF_KEY) {
			if (level < ref_path->shared_level)
				ref_path->shared_level = level;
Z
Zheng Yan 已提交
6719
			goto found;
Y
Yan Zheng 已提交
6720
		}
Z
Zheng Yan 已提交
6721 6722 6723
next:
		level--;
		btrfs_release_path(extent_root, path);
Y
Yan Zheng 已提交
6724
		cond_resched();
Z
Zheng Yan 已提交
6725 6726 6727 6728 6729 6730 6731 6732
	}
	/* reached lowest level */
	ret = 1;
	goto out;
walk_up:
	level = ref_path->current_level;
	while (level < BTRFS_MAX_LEVEL - 1) {
		u64 ref_objectid;
C
Chris Mason 已提交
6733 6734

		if (level >= 0)
Z
Zheng Yan 已提交
6735
			bytenr = ref_path->nodes[level];
C
Chris Mason 已提交
6736
		else
Z
Zheng Yan 已提交
6737
			bytenr = ref_path->extent_start;
C
Chris Mason 已提交
6738

Z
Zheng Yan 已提交
6739
		BUG_ON(bytenr == 0);
6740

Z
Zheng Yan 已提交
6741 6742 6743
		key.objectid = bytenr;
		key.offset = 0;
		key.type = BTRFS_EXTENT_REF_KEY;
6744

Z
Zheng Yan 已提交
6745 6746 6747
		ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 0);
		if (ret < 0)
			goto out;
6748

Z
Zheng Yan 已提交
6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763
		leaf = path->nodes[0];
		nritems = btrfs_header_nritems(leaf);
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(extent_root, path);
			if (ret < 0)
				goto out;
			if (ret > 0) {
				/* the extent was freed by someone */
				if (ref_path->lowest_level == level)
					goto out;
				btrfs_release_path(extent_root, path);
				goto walk_down;
			}
			leaf = path->nodes[0];
		}
6764

Z
Zheng Yan 已提交
6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
		if (found_key.objectid != bytenr ||
				found_key.type != BTRFS_EXTENT_REF_KEY) {
			/* the extent was freed by someone */
			if (ref_path->lowest_level == level) {
				ret = 1;
				goto out;
			}
			btrfs_release_path(extent_root, path);
			goto walk_down;
		}
found:
		ref = btrfs_item_ptr(leaf, path->slots[0],
				struct btrfs_extent_ref);
		ref_objectid = btrfs_ref_objectid(leaf, ref);
		if (ref_objectid < BTRFS_FIRST_FREE_OBJECTID) {
			if (first_time) {
				level = (int)ref_objectid;
				BUG_ON(level >= BTRFS_MAX_LEVEL);
				ref_path->lowest_level = level;
				ref_path->current_level = level;
				ref_path->nodes[level] = bytenr;
			} else {
				WARN_ON(ref_objectid != level);
			}
		} else {
			WARN_ON(level != -1);
		}
		first_time = 0;
6794

Z
Zheng Yan 已提交
6795 6796 6797 6798
		if (ref_path->lowest_level == level) {
			ref_path->owner_objectid = ref_objectid;
			ref_path->num_refs = btrfs_ref_num_refs(leaf, ref);
		}
6799

6800
		/*
Z
Zheng Yan 已提交
6801 6802
		 * the block is tree root or the block isn't in reference
		 * counted tree.
6803
		 */
Z
Zheng Yan 已提交
6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816
		if (found_key.objectid == found_key.offset ||
		    is_cowonly_root(btrfs_ref_root(leaf, ref))) {
			ref_path->root_objectid = btrfs_ref_root(leaf, ref);
			ref_path->root_generation =
				btrfs_ref_generation(leaf, ref);
			if (level < 0) {
				/* special reference from the tree log */
				ref_path->nodes[0] = found_key.offset;
				ref_path->current_level = 0;
			}
			ret = 0;
			goto out;
		}
6817

Z
Zheng Yan 已提交
6818 6819 6820 6821
		level++;
		BUG_ON(ref_path->nodes[level] != 0);
		ref_path->nodes[level] = found_key.offset;
		ref_path->current_level = level;
6822

Z
Zheng Yan 已提交
6823 6824 6825 6826 6827 6828 6829 6830 6831 6832
		/*
		 * the reference was created in the running transaction,
		 * no need to continue walking up.
		 */
		if (btrfs_ref_generation(leaf, ref) == trans->transid) {
			ref_path->root_objectid = btrfs_ref_root(leaf, ref);
			ref_path->root_generation =
				btrfs_ref_generation(leaf, ref);
			ret = 0;
			goto out;
6833 6834
		}

Z
Zheng Yan 已提交
6835
		btrfs_release_path(extent_root, path);
Y
Yan Zheng 已提交
6836
		cond_resched();
6837
	}
Z
Zheng Yan 已提交
6838 6839
	/* reached max tree level, but no tree root found. */
	BUG();
6840
out:
Z
Zheng Yan 已提交
6841 6842
	btrfs_free_path(path);
	return ret;
6843 6844
}

Z
Zheng Yan 已提交
6845 6846 6847 6848
static int btrfs_first_ref_path(struct btrfs_trans_handle *trans,
				struct btrfs_root *extent_root,
				struct btrfs_ref_path *ref_path,
				u64 extent_start)
6849
{
Z
Zheng Yan 已提交
6850 6851
	memset(ref_path, 0, sizeof(*ref_path));
	ref_path->extent_start = extent_start;
6852

Z
Zheng Yan 已提交
6853
	return __next_ref_path(trans, extent_root, ref_path, 1);
6854 6855
}

Z
Zheng Yan 已提交
6856 6857 6858
static int btrfs_next_ref_path(struct btrfs_trans_handle *trans,
			       struct btrfs_root *extent_root,
			       struct btrfs_ref_path *ref_path)
6859
{
Z
Zheng Yan 已提交
6860 6861 6862
	return __next_ref_path(trans, extent_root, ref_path, 0);
}

C
Chris Mason 已提交
6863
static noinline int get_new_locations(struct inode *reloc_inode,
Z
Zheng Yan 已提交
6864 6865 6866 6867 6868 6869 6870 6871
				      struct btrfs_key *extent_key,
				      u64 offset, int no_fragment,
				      struct disk_extent **extents,
				      int *nr_extents)
{
	struct btrfs_root *root = BTRFS_I(reloc_inode)->root;
	struct btrfs_path *path;
	struct btrfs_file_extent_item *fi;
6872
	struct extent_buffer *leaf;
Z
Zheng Yan 已提交
6873 6874 6875 6876
	struct disk_extent *exts = *extents;
	struct btrfs_key found_key;
	u64 cur_pos;
	u64 last_byte;
6877
	u32 nritems;
Z
Zheng Yan 已提交
6878 6879 6880
	int nr = 0;
	int max = *nr_extents;
	int ret;
6881

Z
Zheng Yan 已提交
6882 6883 6884 6885 6886 6887
	WARN_ON(!no_fragment && *extents);
	if (!exts) {
		max = 1;
		exts = kmalloc(sizeof(*exts) * max, GFP_NOFS);
		if (!exts)
			return -ENOMEM;
6888
	}
6889

Z
Zheng Yan 已提交
6890 6891
	path = btrfs_alloc_path();
	BUG_ON(!path);
6892

Z
Zheng Yan 已提交
6893 6894 6895 6896 6897 6898 6899 6900 6901 6902
	cur_pos = extent_key->objectid - offset;
	last_byte = extent_key->objectid + extent_key->offset;
	ret = btrfs_lookup_file_extent(NULL, root, path, reloc_inode->i_ino,
				       cur_pos, 0);
	if (ret < 0)
		goto out;
	if (ret > 0) {
		ret = -ENOENT;
		goto out;
	}
6903

Z
Zheng Yan 已提交
6904
	while (1) {
6905 6906
		leaf = path->nodes[0];
		nritems = btrfs_header_nritems(leaf);
Z
Zheng Yan 已提交
6907 6908
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
6909 6910
			if (ret < 0)
				goto out;
Z
Zheng Yan 已提交
6911 6912
			if (ret > 0)
				break;
6913
			leaf = path->nodes[0];
6914
		}
6915 6916

		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
Z
Zheng Yan 已提交
6917 6918 6919
		if (found_key.offset != cur_pos ||
		    found_key.type != BTRFS_EXTENT_DATA_KEY ||
		    found_key.objectid != reloc_inode->i_ino)
6920 6921
			break;

Z
Zheng Yan 已提交
6922 6923 6924 6925 6926
		fi = btrfs_item_ptr(leaf, path->slots[0],
				    struct btrfs_file_extent_item);
		if (btrfs_file_extent_type(leaf, fi) !=
		    BTRFS_FILE_EXTENT_REG ||
		    btrfs_file_extent_disk_bytenr(leaf, fi) == 0)
6927
			break;
Z
Zheng Yan 已提交
6928 6929 6930 6931 6932 6933 6934 6935

		if (nr == max) {
			struct disk_extent *old = exts;
			max *= 2;
			exts = kzalloc(sizeof(*exts) * max, GFP_NOFS);
			memcpy(exts, old, sizeof(*exts) * nr);
			if (old != *extents)
				kfree(old);
6936
		}
6937

Z
Zheng Yan 已提交
6938 6939 6940 6941 6942 6943
		exts[nr].disk_bytenr =
			btrfs_file_extent_disk_bytenr(leaf, fi);
		exts[nr].disk_num_bytes =
			btrfs_file_extent_disk_num_bytes(leaf, fi);
		exts[nr].offset = btrfs_file_extent_offset(leaf, fi);
		exts[nr].num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
C
Chris Mason 已提交
6944 6945 6946 6947 6948
		exts[nr].ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
		exts[nr].compression = btrfs_file_extent_compression(leaf, fi);
		exts[nr].encryption = btrfs_file_extent_encryption(leaf, fi);
		exts[nr].other_encoding = btrfs_file_extent_other_encoding(leaf,
									   fi);
Y
Yan Zheng 已提交
6949 6950 6951
		BUG_ON(exts[nr].offset > 0);
		BUG_ON(exts[nr].compression || exts[nr].encryption);
		BUG_ON(exts[nr].num_bytes != exts[nr].disk_num_bytes);
6952

Z
Zheng Yan 已提交
6953 6954 6955 6956 6957 6958 6959 6960
		cur_pos += exts[nr].num_bytes;
		nr++;

		if (cur_pos + offset >= last_byte)
			break;

		if (no_fragment) {
			ret = 1;
6961
			goto out;
Z
Zheng Yan 已提交
6962 6963 6964 6965
		}
		path->slots[0]++;
	}

6966
	BUG_ON(cur_pos + offset > last_byte);
Z
Zheng Yan 已提交
6967 6968 6969
	if (cur_pos + offset < last_byte) {
		ret = -ENOENT;
		goto out;
6970 6971 6972
	}
	ret = 0;
out:
Z
Zheng Yan 已提交
6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983
	btrfs_free_path(path);
	if (ret) {
		if (exts != *extents)
			kfree(exts);
	} else {
		*extents = exts;
		*nr_extents = nr;
	}
	return ret;
}

C
Chris Mason 已提交
6984
static noinline int replace_one_extent(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000
					struct btrfs_root *root,
					struct btrfs_path *path,
					struct btrfs_key *extent_key,
					struct btrfs_key *leaf_key,
					struct btrfs_ref_path *ref_path,
					struct disk_extent *new_extents,
					int nr_extents)
{
	struct extent_buffer *leaf;
	struct btrfs_file_extent_item *fi;
	struct inode *inode = NULL;
	struct btrfs_key key;
	u64 lock_start = 0;
	u64 lock_end = 0;
	u64 num_bytes;
	u64 ext_offset;
7001
	u64 search_end = (u64)-1;
Z
Zheng Yan 已提交
7002
	u32 nritems;
7003
	int nr_scaned = 0;
Z
Zheng Yan 已提交
7004
	int extent_locked = 0;
Y
Yan Zheng 已提交
7005
	int extent_type;
Z
Zheng Yan 已提交
7006 7007
	int ret;

7008
	memcpy(&key, leaf_key, sizeof(key));
Z
Zheng Yan 已提交
7009
	if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS) {
7010 7011 7012 7013 7014 7015 7016
		if (key.objectid < ref_path->owner_objectid ||
		    (key.objectid == ref_path->owner_objectid &&
		     key.type < BTRFS_EXTENT_DATA_KEY)) {
			key.objectid = ref_path->owner_objectid;
			key.type = BTRFS_EXTENT_DATA_KEY;
			key.offset = 0;
		}
Z
Zheng Yan 已提交
7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037
	}

	while (1) {
		ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
		if (ret < 0)
			goto out;

		leaf = path->nodes[0];
		nritems = btrfs_header_nritems(leaf);
next:
		if (extent_locked && ret > 0) {
			/*
			 * the file extent item was modified by someone
			 * before the extent got locked.
			 */
			unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
				      lock_end, GFP_NOFS);
			extent_locked = 0;
		}

		if (path->slots[0] >= nritems) {
7038
			if (++nr_scaned > 2)
Z
Zheng Yan 已提交
7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056
				break;

			BUG_ON(extent_locked);
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				goto out;
			if (ret > 0)
				break;
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

		if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS) {
			if ((key.objectid > ref_path->owner_objectid) ||
			    (key.objectid == ref_path->owner_objectid &&
			     key.type > BTRFS_EXTENT_DATA_KEY) ||
7057
			    key.offset >= search_end)
Z
Zheng Yan 已提交
7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076
				break;
		}

		if (inode && key.objectid != inode->i_ino) {
			BUG_ON(extent_locked);
			btrfs_release_path(root, path);
			mutex_unlock(&inode->i_mutex);
			iput(inode);
			inode = NULL;
			continue;
		}

		if (key.type != BTRFS_EXTENT_DATA_KEY) {
			path->slots[0]++;
			ret = 1;
			goto next;
		}
		fi = btrfs_item_ptr(leaf, path->slots[0],
				    struct btrfs_file_extent_item);
Y
Yan Zheng 已提交
7077 7078 7079
		extent_type = btrfs_file_extent_type(leaf, fi);
		if ((extent_type != BTRFS_FILE_EXTENT_REG &&
		     extent_type != BTRFS_FILE_EXTENT_PREALLOC) ||
Z
Zheng Yan 已提交
7080 7081 7082 7083 7084 7085 7086 7087 7088 7089
		    (btrfs_file_extent_disk_bytenr(leaf, fi) !=
		     extent_key->objectid)) {
			path->slots[0]++;
			ret = 1;
			goto next;
		}

		num_bytes = btrfs_file_extent_num_bytes(leaf, fi);
		ext_offset = btrfs_file_extent_offset(leaf, fi);

7090 7091 7092 7093
		if (search_end == (u64)-1) {
			search_end = key.offset - ext_offset +
				btrfs_file_extent_ram_bytes(leaf, fi);
		}
Z
Zheng Yan 已提交
7094 7095 7096 7097 7098

		if (!extent_locked) {
			lock_start = key.offset;
			lock_end = lock_start + num_bytes - 1;
		} else {
Y
Yan Zheng 已提交
7099 7100 7101 7102 7103 7104
			if (lock_start > key.offset ||
			    lock_end + 1 < key.offset + num_bytes) {
				unlock_extent(&BTRFS_I(inode)->io_tree,
					      lock_start, lock_end, GFP_NOFS);
				extent_locked = 0;
			}
Z
Zheng Yan 已提交
7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178
		}

		if (!inode) {
			btrfs_release_path(root, path);

			inode = btrfs_iget_locked(root->fs_info->sb,
						  key.objectid, root);
			if (inode->i_state & I_NEW) {
				BTRFS_I(inode)->root = root;
				BTRFS_I(inode)->location.objectid =
					key.objectid;
				BTRFS_I(inode)->location.type =
					BTRFS_INODE_ITEM_KEY;
				BTRFS_I(inode)->location.offset = 0;
				btrfs_read_locked_inode(inode);
				unlock_new_inode(inode);
			}
			/*
			 * some code call btrfs_commit_transaction while
			 * holding the i_mutex, so we can't use mutex_lock
			 * here.
			 */
			if (is_bad_inode(inode) ||
			    !mutex_trylock(&inode->i_mutex)) {
				iput(inode);
				inode = NULL;
				key.offset = (u64)-1;
				goto skip;
			}
		}

		if (!extent_locked) {
			struct btrfs_ordered_extent *ordered;

			btrfs_release_path(root, path);

			lock_extent(&BTRFS_I(inode)->io_tree, lock_start,
				    lock_end, GFP_NOFS);
			ordered = btrfs_lookup_first_ordered_extent(inode,
								    lock_end);
			if (ordered &&
			    ordered->file_offset <= lock_end &&
			    ordered->file_offset + ordered->len > lock_start) {
				unlock_extent(&BTRFS_I(inode)->io_tree,
					      lock_start, lock_end, GFP_NOFS);
				btrfs_start_ordered_extent(inode, ordered, 1);
				btrfs_put_ordered_extent(ordered);
				key.offset += num_bytes;
				goto skip;
			}
			if (ordered)
				btrfs_put_ordered_extent(ordered);

			extent_locked = 1;
			continue;
		}

		if (nr_extents == 1) {
			/* update extent pointer in place */
			btrfs_set_file_extent_disk_bytenr(leaf, fi,
						new_extents[0].disk_bytenr);
			btrfs_set_file_extent_disk_num_bytes(leaf, fi,
						new_extents[0].disk_num_bytes);
			btrfs_mark_buffer_dirty(leaf);

			btrfs_drop_extent_cache(inode, key.offset,
						key.offset + num_bytes - 1, 0);

			ret = btrfs_inc_extent_ref(trans, root,
						new_extents[0].disk_bytenr,
						new_extents[0].disk_num_bytes,
						leaf->start,
						root->root_key.objectid,
						trans->transid,
7179
						key.objectid);
Z
Zheng Yan 已提交
7180 7181 7182 7183 7184 7185 7186 7187
			BUG_ON(ret);

			ret = btrfs_free_extent(trans, root,
						extent_key->objectid,
						extent_key->offset,
						leaf->start,
						btrfs_header_owner(leaf),
						btrfs_header_generation(leaf),
7188
						key.objectid, 0);
Z
Zheng Yan 已提交
7189 7190 7191 7192 7193
			BUG_ON(ret);

			btrfs_release_path(root, path);
			key.offset += num_bytes;
		} else {
Y
Yan Zheng 已提交
7194 7195
			BUG_ON(1);
#if 0
Z
Zheng Yan 已提交
7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232
			u64 alloc_hint;
			u64 extent_len;
			int i;
			/*
			 * drop old extent pointer at first, then insert the
			 * new pointers one bye one
			 */
			btrfs_release_path(root, path);
			ret = btrfs_drop_extents(trans, root, inode, key.offset,
						 key.offset + num_bytes,
						 key.offset, &alloc_hint);
			BUG_ON(ret);

			for (i = 0; i < nr_extents; i++) {
				if (ext_offset >= new_extents[i].num_bytes) {
					ext_offset -= new_extents[i].num_bytes;
					continue;
				}
				extent_len = min(new_extents[i].num_bytes -
						 ext_offset, num_bytes);

				ret = btrfs_insert_empty_item(trans, root,
							      path, &key,
							      sizeof(*fi));
				BUG_ON(ret);

				leaf = path->nodes[0];
				fi = btrfs_item_ptr(leaf, path->slots[0],
						struct btrfs_file_extent_item);
				btrfs_set_file_extent_generation(leaf, fi,
							trans->transid);
				btrfs_set_file_extent_type(leaf, fi,
							BTRFS_FILE_EXTENT_REG);
				btrfs_set_file_extent_disk_bytenr(leaf, fi,
						new_extents[i].disk_bytenr);
				btrfs_set_file_extent_disk_num_bytes(leaf, fi,
						new_extents[i].disk_num_bytes);
C
Chris Mason 已提交
7233 7234 7235 7236 7237 7238 7239 7240 7241 7242
				btrfs_set_file_extent_ram_bytes(leaf, fi,
						new_extents[i].ram_bytes);

				btrfs_set_file_extent_compression(leaf, fi,
						new_extents[i].compression);
				btrfs_set_file_extent_encryption(leaf, fi,
						new_extents[i].encryption);
				btrfs_set_file_extent_other_encoding(leaf, fi,
						new_extents[i].other_encoding);

Z
Zheng Yan 已提交
7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257
				btrfs_set_file_extent_num_bytes(leaf, fi,
							extent_len);
				ext_offset += new_extents[i].offset;
				btrfs_set_file_extent_offset(leaf, fi,
							ext_offset);
				btrfs_mark_buffer_dirty(leaf);

				btrfs_drop_extent_cache(inode, key.offset,
						key.offset + extent_len - 1, 0);

				ret = btrfs_inc_extent_ref(trans, root,
						new_extents[i].disk_bytenr,
						new_extents[i].disk_num_bytes,
						leaf->start,
						root->root_key.objectid,
7258
						trans->transid, key.objectid);
Z
Zheng Yan 已提交
7259 7260 7261
				BUG_ON(ret);
				btrfs_release_path(root, path);

7262
				inode_add_bytes(inode, extent_len);
Z
Zheng Yan 已提交
7263 7264 7265 7266 7267 7268 7269 7270 7271

				ext_offset = 0;
				num_bytes -= extent_len;
				key.offset += extent_len;

				if (num_bytes == 0)
					break;
			}
			BUG_ON(i >= nr_extents);
Y
Yan Zheng 已提交
7272
#endif
Z
Zheng Yan 已提交
7273 7274 7275 7276 7277 7278 7279 7280 7281
		}

		if (extent_locked) {
			unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
				      lock_end, GFP_NOFS);
			extent_locked = 0;
		}
skip:
		if (ref_path->owner_objectid != BTRFS_MULTIPLE_OBJECTIDS &&
7282
		    key.offset >= search_end)
Z
Zheng Yan 已提交
7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335
			break;

		cond_resched();
	}
	ret = 0;
out:
	btrfs_release_path(root, path);
	if (inode) {
		mutex_unlock(&inode->i_mutex);
		if (extent_locked) {
			unlock_extent(&BTRFS_I(inode)->io_tree, lock_start,
				      lock_end, GFP_NOFS);
		}
		iput(inode);
	}
	return ret;
}

int btrfs_reloc_tree_cache_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct extent_buffer *buf, u64 orig_start)
{
	int level;
	int ret;

	BUG_ON(btrfs_header_generation(buf) != trans->transid);
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

	level = btrfs_header_level(buf);
	if (level == 0) {
		struct btrfs_leaf_ref *ref;
		struct btrfs_leaf_ref *orig_ref;

		orig_ref = btrfs_lookup_leaf_ref(root, orig_start);
		if (!orig_ref)
			return -ENOENT;

		ref = btrfs_alloc_leaf_ref(root, orig_ref->nritems);
		if (!ref) {
			btrfs_free_leaf_ref(root, orig_ref);
			return -ENOMEM;
		}

		ref->nritems = orig_ref->nritems;
		memcpy(ref->extents, orig_ref->extents,
			sizeof(ref->extents[0]) * ref->nritems);

		btrfs_free_leaf_ref(root, orig_ref);

		ref->root_gen = trans->transid;
		ref->bytenr = buf->start;
		ref->owner = btrfs_header_owner(buf);
		ref->generation = btrfs_header_generation(buf);
7336

Z
Zheng Yan 已提交
7337 7338 7339 7340 7341 7342 7343
		ret = btrfs_add_leaf_ref(root, ref, 0);
		WARN_ON(ret);
		btrfs_free_leaf_ref(root, ref);
	}
	return 0;
}

C
Chris Mason 已提交
7344
static noinline int invalidate_extent_cache(struct btrfs_root *root,
Z
Zheng Yan 已提交
7345 7346 7347 7348 7349 7350 7351
					struct extent_buffer *leaf,
					struct btrfs_block_group_cache *group,
					struct btrfs_root *target_root)
{
	struct btrfs_key key;
	struct inode *inode = NULL;
	struct btrfs_file_extent_item *fi;
7352
	struct extent_state *cached_state = NULL;
Z
Zheng Yan 已提交
7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380
	u64 num_bytes;
	u64 skip_objectid = 0;
	u32 nritems;
	u32 i;

	nritems = btrfs_header_nritems(leaf);
	for (i = 0; i < nritems; i++) {
		btrfs_item_key_to_cpu(leaf, &key, i);
		if (key.objectid == skip_objectid ||
		    key.type != BTRFS_EXTENT_DATA_KEY)
			continue;
		fi = btrfs_item_ptr(leaf, i, struct btrfs_file_extent_item);
		if (btrfs_file_extent_type(leaf, fi) ==
		    BTRFS_FILE_EXTENT_INLINE)
			continue;
		if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0)
			continue;
		if (!inode || inode->i_ino != key.objectid) {
			iput(inode);
			inode = btrfs_ilookup(target_root->fs_info->sb,
					      key.objectid, target_root, 1);
		}
		if (!inode) {
			skip_objectid = key.objectid;
			continue;
		}
		num_bytes = btrfs_file_extent_num_bytes(leaf, fi);

7381 7382 7383
		lock_extent_bits(&BTRFS_I(inode)->io_tree, key.offset,
				 key.offset + num_bytes - 1, 0, &cached_state,
				 GFP_NOFS);
Z
Zheng Yan 已提交
7384 7385
		btrfs_drop_extent_cache(inode, key.offset,
					key.offset + num_bytes - 1, 1);
7386 7387 7388
		unlock_extent_cached(&BTRFS_I(inode)->io_tree, key.offset,
				     key.offset + num_bytes - 1, &cached_state,
				     GFP_NOFS);
Z
Zheng Yan 已提交
7389 7390 7391 7392 7393 7394
		cond_resched();
	}
	iput(inode);
	return 0;
}

C
Chris Mason 已提交
7395
static noinline int replace_extents_in_leaf(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466
					struct btrfs_root *root,
					struct extent_buffer *leaf,
					struct btrfs_block_group_cache *group,
					struct inode *reloc_inode)
{
	struct btrfs_key key;
	struct btrfs_key extent_key;
	struct btrfs_file_extent_item *fi;
	struct btrfs_leaf_ref *ref;
	struct disk_extent *new_extent;
	u64 bytenr;
	u64 num_bytes;
	u32 nritems;
	u32 i;
	int ext_index;
	int nr_extent;
	int ret;

	new_extent = kmalloc(sizeof(*new_extent), GFP_NOFS);
	BUG_ON(!new_extent);

	ref = btrfs_lookup_leaf_ref(root, leaf->start);
	BUG_ON(!ref);

	ext_index = -1;
	nritems = btrfs_header_nritems(leaf);
	for (i = 0; i < nritems; i++) {
		btrfs_item_key_to_cpu(leaf, &key, i);
		if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
			continue;
		fi = btrfs_item_ptr(leaf, i, struct btrfs_file_extent_item);
		if (btrfs_file_extent_type(leaf, fi) ==
		    BTRFS_FILE_EXTENT_INLINE)
			continue;
		bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
		num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
		if (bytenr == 0)
			continue;

		ext_index++;
		if (bytenr >= group->key.objectid + group->key.offset ||
		    bytenr + num_bytes <= group->key.objectid)
			continue;

		extent_key.objectid = bytenr;
		extent_key.offset = num_bytes;
		extent_key.type = BTRFS_EXTENT_ITEM_KEY;
		nr_extent = 1;
		ret = get_new_locations(reloc_inode, &extent_key,
					group->key.objectid, 1,
					&new_extent, &nr_extent);
		if (ret > 0)
			continue;
		BUG_ON(ret < 0);

		BUG_ON(ref->extents[ext_index].bytenr != bytenr);
		BUG_ON(ref->extents[ext_index].num_bytes != num_bytes);
		ref->extents[ext_index].bytenr = new_extent->disk_bytenr;
		ref->extents[ext_index].num_bytes = new_extent->disk_num_bytes;

		btrfs_set_file_extent_disk_bytenr(leaf, fi,
						new_extent->disk_bytenr);
		btrfs_set_file_extent_disk_num_bytes(leaf, fi,
						new_extent->disk_num_bytes);
		btrfs_mark_buffer_dirty(leaf);

		ret = btrfs_inc_extent_ref(trans, root,
					new_extent->disk_bytenr,
					new_extent->disk_num_bytes,
					leaf->start,
					root->root_key.objectid,
7467
					trans->transid, key.objectid);
Z
Zheng Yan 已提交
7468
		BUG_ON(ret);
7469

Z
Zheng Yan 已提交
7470 7471 7472 7473
		ret = btrfs_free_extent(trans, root,
					bytenr, num_bytes, leaf->start,
					btrfs_header_owner(leaf),
					btrfs_header_generation(leaf),
7474
					key.objectid, 0);
Z
Zheng Yan 已提交
7475 7476 7477 7478 7479 7480 7481 7482 7483
		BUG_ON(ret);
		cond_resched();
	}
	kfree(new_extent);
	BUG_ON(ext_index + 1 != ref->nritems);
	btrfs_free_leaf_ref(root, ref);
	return 0;
}

Y
Yan Zheng 已提交
7484 7485
int btrfs_free_reloc_root(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root)
Z
Zheng Yan 已提交
7486 7487
{
	struct btrfs_root *reloc_root;
Y
Yan Zheng 已提交
7488
	int ret;
Z
Zheng Yan 已提交
7489 7490 7491 7492 7493 7494

	if (root->reloc_root) {
		reloc_root = root->reloc_root;
		root->reloc_root = NULL;
		list_add(&reloc_root->dead_list,
			 &root->fs_info->dead_reloc_roots);
Y
Yan Zheng 已提交
7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507

		btrfs_set_root_bytenr(&reloc_root->root_item,
				      reloc_root->node->start);
		btrfs_set_root_level(&root->root_item,
				     btrfs_header_level(reloc_root->node));
		memset(&reloc_root->root_item.drop_progress, 0,
			sizeof(struct btrfs_disk_key));
		reloc_root->root_item.drop_level = 0;

		ret = btrfs_update_root(trans, root->fs_info->tree_root,
					&reloc_root->root_key,
					&reloc_root->root_item);
		BUG_ON(ret);
Z
Zheng Yan 已提交
7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531
	}
	return 0;
}

int btrfs_drop_dead_reloc_roots(struct btrfs_root *root)
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *reloc_root;
	struct btrfs_root *prev_root = NULL;
	struct list_head dead_roots;
	int ret;
	unsigned long nr;

	INIT_LIST_HEAD(&dead_roots);
	list_splice_init(&root->fs_info->dead_reloc_roots, &dead_roots);

	while (!list_empty(&dead_roots)) {
		reloc_root = list_entry(dead_roots.prev,
					struct btrfs_root, dead_list);
		list_del_init(&reloc_root->dead_list);

		BUG_ON(reloc_root->commit_root != NULL);
		while (1) {
			trans = btrfs_join_transaction(root, 1);
7532
			BUG_ON(IS_ERR(trans));
Z
Zheng Yan 已提交
7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589

			mutex_lock(&root->fs_info->drop_mutex);
			ret = btrfs_drop_snapshot(trans, reloc_root);
			if (ret != -EAGAIN)
				break;
			mutex_unlock(&root->fs_info->drop_mutex);

			nr = trans->blocks_used;
			ret = btrfs_end_transaction(trans, root);
			BUG_ON(ret);
			btrfs_btree_balance_dirty(root, nr);
		}

		free_extent_buffer(reloc_root->node);

		ret = btrfs_del_root(trans, root->fs_info->tree_root,
				     &reloc_root->root_key);
		BUG_ON(ret);
		mutex_unlock(&root->fs_info->drop_mutex);

		nr = trans->blocks_used;
		ret = btrfs_end_transaction(trans, root);
		BUG_ON(ret);
		btrfs_btree_balance_dirty(root, nr);

		kfree(prev_root);
		prev_root = reloc_root;
	}
	if (prev_root) {
		btrfs_remove_leaf_refs(prev_root, (u64)-1, 0);
		kfree(prev_root);
	}
	return 0;
}

int btrfs_add_dead_reloc_root(struct btrfs_root *root)
{
	list_add(&root->dead_list, &root->fs_info->dead_reloc_roots);
	return 0;
}

int btrfs_cleanup_reloc_trees(struct btrfs_root *root)
{
	struct btrfs_root *reloc_root;
	struct btrfs_trans_handle *trans;
	struct btrfs_key location;
	int found;
	int ret;

	mutex_lock(&root->fs_info->tree_reloc_mutex);
	ret = btrfs_find_dead_roots(root, BTRFS_TREE_RELOC_OBJECTID, NULL);
	BUG_ON(ret);
	found = !list_empty(&root->fs_info->dead_reloc_roots);
	mutex_unlock(&root->fs_info->tree_reloc_mutex);

	if (found) {
		trans = btrfs_start_transaction(root, 1);
7590
		BUG_ON(IS_ERR(trans));
Z
Zheng Yan 已提交
7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604
		ret = btrfs_commit_transaction(trans, root);
		BUG_ON(ret);
	}

	location.objectid = BTRFS_DATA_RELOC_TREE_OBJECTID;
	location.offset = (u64)-1;
	location.type = BTRFS_ROOT_ITEM_KEY;

	reloc_root = btrfs_read_fs_root_no_name(root->fs_info, &location);
	BUG_ON(!reloc_root);
	btrfs_orphan_cleanup(reloc_root);
	return 0;
}

C
Chris Mason 已提交
7605
static noinline int init_reloc_tree(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632
				    struct btrfs_root *root)
{
	struct btrfs_root *reloc_root;
	struct extent_buffer *eb;
	struct btrfs_root_item *root_item;
	struct btrfs_key root_key;
	int ret;

	BUG_ON(!root->ref_cows);
	if (root->reloc_root)
		return 0;

	root_item = kmalloc(sizeof(*root_item), GFP_NOFS);
	BUG_ON(!root_item);

	ret = btrfs_copy_root(trans, root, root->commit_root,
			      &eb, BTRFS_TREE_RELOC_OBJECTID);
	BUG_ON(ret);

	root_key.objectid = BTRFS_TREE_RELOC_OBJECTID;
	root_key.offset = root->root_key.objectid;
	root_key.type = BTRFS_ROOT_ITEM_KEY;

	memcpy(root_item, &root->root_item, sizeof(root_item));
	btrfs_set_root_refs(root_item, 0);
	btrfs_set_root_bytenr(root_item, eb->start);
	btrfs_set_root_level(root_item, btrfs_header_level(eb));
7633
	btrfs_set_root_generation(root_item, trans->transid);
Z
Zheng Yan 已提交
7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657

	btrfs_tree_unlock(eb);
	free_extent_buffer(eb);

	ret = btrfs_insert_root(trans, root->fs_info->tree_root,
				&root_key, root_item);
	BUG_ON(ret);
	kfree(root_item);

	reloc_root = btrfs_read_fs_root_no_radix(root->fs_info->tree_root,
						 &root_key);
	BUG_ON(!reloc_root);
	reloc_root->last_trans = trans->transid;
	reloc_root->commit_root = NULL;
	reloc_root->ref_tree = &root->fs_info->reloc_ref_tree;

	root->reloc_root = reloc_root;
	return 0;
}

/*
 * Core function of space balance.
 *
 * The idea is using reloc trees to relocate tree blocks in reference
Y
Yan Zheng 已提交
7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670
 * counted roots. There is one reloc tree for each subvol, and all
 * reloc trees share same root key objectid. Reloc trees are snapshots
 * of the latest committed roots of subvols (root->commit_root).
 *
 * To relocate a tree block referenced by a subvol, there are two steps.
 * COW the block through subvol's reloc tree, then update block pointer
 * in the subvol to point to the new block. Since all reloc trees share
 * same root key objectid, doing special handing for tree blocks owned
 * by them is easy. Once a tree block has been COWed in one reloc tree,
 * we can use the resulting new block directly when the same block is
 * required to COW again through other reloc trees. By this way, relocated
 * tree blocks are shared between reloc trees, so they are also shared
 * between subvols.
Z
Zheng Yan 已提交
7671
 */
C
Chris Mason 已提交
7672
static noinline int relocate_one_path(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      struct btrfs_key *first_key,
				      struct btrfs_ref_path *ref_path,
				      struct btrfs_block_group_cache *group,
				      struct inode *reloc_inode)
{
	struct btrfs_root *reloc_root;
	struct extent_buffer *eb = NULL;
	struct btrfs_key *keys;
	u64 *nodes;
	int level;
Y
Yan Zheng 已提交
7685
	int shared_level;
Z
Zheng Yan 已提交
7686 7687 7688 7689 7690 7691
	int lowest_level = 0;
	int ret;

	if (ref_path->owner_objectid < BTRFS_FIRST_FREE_OBJECTID)
		lowest_level = ref_path->owner_objectid;

Y
Yan Zheng 已提交
7692
	if (!root->ref_cows) {
Z
Zheng Yan 已提交
7693 7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705
		path->lowest_level = lowest_level;
		ret = btrfs_search_slot(trans, root, first_key, path, 0, 1);
		BUG_ON(ret < 0);
		path->lowest_level = 0;
		btrfs_release_path(root, path);
		return 0;
	}

	mutex_lock(&root->fs_info->tree_reloc_mutex);
	ret = init_reloc_tree(trans, root);
	BUG_ON(ret);
	reloc_root = root->reloc_root;

Y
Yan Zheng 已提交
7706 7707
	shared_level = ref_path->shared_level;
	ref_path->shared_level = BTRFS_MAX_LEVEL - 1;
Z
Zheng Yan 已提交
7708

Y
Yan Zheng 已提交
7709 7710 7711 7712 7713 7714
	keys = ref_path->node_keys;
	nodes = ref_path->new_nodes;
	memset(&keys[shared_level + 1], 0,
	       sizeof(*keys) * (BTRFS_MAX_LEVEL - shared_level - 1));
	memset(&nodes[shared_level + 1], 0,
	       sizeof(*nodes) * (BTRFS_MAX_LEVEL - shared_level - 1));
Z
Zheng Yan 已提交
7715

Y
Yan Zheng 已提交
7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730
	if (nodes[lowest_level] == 0) {
		path->lowest_level = lowest_level;
		ret = btrfs_search_slot(trans, reloc_root, first_key, path,
					0, 1);
		BUG_ON(ret);
		for (level = lowest_level; level < BTRFS_MAX_LEVEL; level++) {
			eb = path->nodes[level];
			if (!eb || eb == reloc_root->node)
				break;
			nodes[level] = eb->start;
			if (level == 0)
				btrfs_item_key_to_cpu(eb, &keys[level], 0);
			else
				btrfs_node_key_to_cpu(eb, &keys[level], 0);
		}
Y
Yan Zheng 已提交
7731 7732
		if (nodes[0] &&
		    ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
Y
Yan Zheng 已提交
7733 7734 7735 7736 7737 7738 7739
			eb = path->nodes[0];
			ret = replace_extents_in_leaf(trans, reloc_root, eb,
						      group, reloc_inode);
			BUG_ON(ret);
		}
		btrfs_release_path(reloc_root, path);
	} else {
Z
Zheng Yan 已提交
7740
		ret = btrfs_merge_path(trans, reloc_root, keys, nodes,
Y
Yan Zheng 已提交
7741
				       lowest_level);
Z
Zheng Yan 已提交
7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752
		BUG_ON(ret);
	}

	/*
	 * replace tree blocks in the fs tree with tree blocks in
	 * the reloc tree.
	 */
	ret = btrfs_merge_path(trans, root, keys, nodes, lowest_level);
	BUG_ON(ret < 0);

	if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
Y
Yan Zheng 已提交
7753 7754 7755 7756 7757 7758
		ret = btrfs_search_slot(trans, reloc_root, first_key, path,
					0, 0);
		BUG_ON(ret);
		extent_buffer_get(path->nodes[0]);
		eb = path->nodes[0];
		btrfs_release_path(reloc_root, path);
Z
Zheng Yan 已提交
7759 7760 7761 7762 7763
		ret = invalidate_extent_cache(reloc_root, eb, group, root);
		BUG_ON(ret);
		free_extent_buffer(eb);
	}

Y
Yan Zheng 已提交
7764
	mutex_unlock(&root->fs_info->tree_reloc_mutex);
Z
Zheng Yan 已提交
7765 7766 7767 7768
	path->lowest_level = 0;
	return 0;
}

C
Chris Mason 已提交
7769
static noinline int relocate_tree_block(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783
					struct btrfs_root *root,
					struct btrfs_path *path,
					struct btrfs_key *first_key,
					struct btrfs_ref_path *ref_path)
{
	int ret;

	ret = relocate_one_path(trans, root, path, first_key,
				ref_path, NULL, NULL);
	BUG_ON(ret);

	return 0;
}

C
Chris Mason 已提交
7784
static noinline int del_extent_zero(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799
				    struct btrfs_root *extent_root,
				    struct btrfs_path *path,
				    struct btrfs_key *extent_key)
{
	int ret;

	ret = btrfs_search_slot(trans, extent_root, extent_key, path, -1, 1);
	if (ret)
		goto out;
	ret = btrfs_del_item(trans, extent_root, path);
out:
	btrfs_release_path(extent_root, path);
	return ret;
}

C
Chris Mason 已提交
7800
static noinline struct btrfs_root *read_ref_root(struct btrfs_fs_info *fs_info,
Z
Zheng Yan 已提交
7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814
						struct btrfs_ref_path *ref_path)
{
	struct btrfs_key root_key;

	root_key.objectid = ref_path->root_objectid;
	root_key.type = BTRFS_ROOT_ITEM_KEY;
	if (is_cowonly_root(ref_path->root_objectid))
		root_key.offset = 0;
	else
		root_key.offset = (u64)-1;

	return btrfs_read_fs_root_no_name(fs_info, &root_key);
}

C
Chris Mason 已提交
7815
static noinline int relocate_one_extent(struct btrfs_root *extent_root,
Z
Zheng Yan 已提交
7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833
					struct btrfs_path *path,
					struct btrfs_key *extent_key,
					struct btrfs_block_group_cache *group,
					struct inode *reloc_inode, int pass)
{
	struct btrfs_trans_handle *trans;
	struct btrfs_root *found_root;
	struct btrfs_ref_path *ref_path = NULL;
	struct disk_extent *new_extents = NULL;
	int nr_extents = 0;
	int loops;
	int ret;
	int level;
	struct btrfs_key first_key;
	u64 prev_block = 0;


	trans = btrfs_start_transaction(extent_root, 1);
7834
	BUG_ON(IS_ERR(trans));
Z
Zheng Yan 已提交
7835 7836 7837 7838 7839 7840 7841 7842

	if (extent_key->objectid == 0) {
		ret = del_extent_zero(trans, extent_root, path, extent_key);
		goto out;
	}

	ref_path = kmalloc(sizeof(*ref_path), GFP_NOFS);
	if (!ref_path) {
C
Chris Mason 已提交
7843 7844
		ret = -ENOMEM;
		goto out;
Z
Zheng Yan 已提交
7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910
	}

	for (loops = 0; ; loops++) {
		if (loops == 0) {
			ret = btrfs_first_ref_path(trans, extent_root, ref_path,
						   extent_key->objectid);
		} else {
			ret = btrfs_next_ref_path(trans, extent_root, ref_path);
		}
		if (ret < 0)
			goto out;
		if (ret > 0)
			break;

		if (ref_path->root_objectid == BTRFS_TREE_LOG_OBJECTID ||
		    ref_path->root_objectid == BTRFS_TREE_RELOC_OBJECTID)
			continue;

		found_root = read_ref_root(extent_root->fs_info, ref_path);
		BUG_ON(!found_root);
		/*
		 * for reference counted tree, only process reference paths
		 * rooted at the latest committed root.
		 */
		if (found_root->ref_cows &&
		    ref_path->root_generation != found_root->root_key.offset)
			continue;

		if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
			if (pass == 0) {
				/*
				 * copy data extents to new locations
				 */
				u64 group_start = group->key.objectid;
				ret = relocate_data_extent(reloc_inode,
							   extent_key,
							   group_start);
				if (ret < 0)
					goto out;
				break;
			}
			level = 0;
		} else {
			level = ref_path->owner_objectid;
		}

		if (prev_block != ref_path->nodes[level]) {
			struct extent_buffer *eb;
			u64 block_start = ref_path->nodes[level];
			u64 block_size = btrfs_level_size(found_root, level);

			eb = read_tree_block(found_root, block_start,
					     block_size, 0);
			btrfs_tree_lock(eb);
			BUG_ON(level != btrfs_header_level(eb));

			if (level == 0)
				btrfs_item_key_to_cpu(eb, &first_key, 0);
			else
				btrfs_node_key_to_cpu(eb, &first_key, 0);

			btrfs_tree_unlock(eb);
			free_extent_buffer(eb);
			prev_block = block_start;
		}

7911
		mutex_lock(&extent_root->fs_info->trans_mutex);
Y
Yan Zheng 已提交
7912
		btrfs_record_root_in_trans(found_root);
7913
		mutex_unlock(&extent_root->fs_info->trans_mutex);
Y
Yan Zheng 已提交
7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926
		if (ref_path->owner_objectid >= BTRFS_FIRST_FREE_OBJECTID) {
			/*
			 * try to update data extent references while
			 * keeping metadata shared between snapshots.
			 */
			if (pass == 1) {
				ret = relocate_one_path(trans, found_root,
						path, &first_key, ref_path,
						group, reloc_inode);
				if (ret < 0)
					goto out;
				continue;
			}
Z
Zheng Yan 已提交
7927 7928 7929 7930 7931 7932
			/*
			 * use fallback method to process the remaining
			 * references.
			 */
			if (!new_extents) {
				u64 group_start = group->key.objectid;
Y
Yan Zheng 已提交
7933 7934 7935
				new_extents = kmalloc(sizeof(*new_extents),
						      GFP_NOFS);
				nr_extents = 1;
Z
Zheng Yan 已提交
7936 7937
				ret = get_new_locations(reloc_inode,
							extent_key,
Y
Yan Zheng 已提交
7938
							group_start, 1,
Z
Zheng Yan 已提交
7939 7940
							&new_extents,
							&nr_extents);
Y
Yan Zheng 已提交
7941
				if (ret)
Z
Zheng Yan 已提交
7942 7943 7944 7945 7946 7947
					goto out;
			}
			ret = replace_one_extent(trans, found_root,
						path, extent_key,
						&first_key, ref_path,
						new_extents, nr_extents);
Y
Yan Zheng 已提交
7948
		} else {
Z
Zheng Yan 已提交
7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961
			ret = relocate_tree_block(trans, found_root, path,
						  &first_key, ref_path);
		}
		if (ret < 0)
			goto out;
	}
	ret = 0;
out:
	btrfs_end_transaction(trans, extent_root);
	kfree(new_extents);
	kfree(ref_path);
	return ret;
}
7962
#endif
Z
Zheng Yan 已提交
7963

7964 7965 7966 7967 7968 7969
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
	u64 stripped = BTRFS_BLOCK_GROUP_RAID0 |
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

7970 7971 7972 7973 7974 7975 7976 7977
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;

7978 7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989 7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
		return flags;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

		/* turn single device chunks into raid0 */
		return stripped | BTRFS_BLOCK_GROUP_RAID0;
	}
	return flags;
}

8009
static int set_block_group_ro(struct btrfs_block_group_cache *cache)
C
Chris Mason 已提交
8010
{
8011 8012 8013
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
	int ret = -ENOSPC;
C
Chris Mason 已提交
8014

8015 8016
	if (cache->ro)
		return 0;
8017

8018 8019 8020 8021 8022 8023 8024
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
	    sinfo->bytes_may_use + sinfo->bytes_readonly +
8025
	    cache->reserved_pinned + num_bytes <= sinfo->total_bytes) {
8026 8027 8028 8029 8030 8031
		sinfo->bytes_readonly += num_bytes;
		sinfo->bytes_reserved += cache->reserved_pinned;
		cache->reserved_pinned = 0;
		cache->ro = 1;
		ret = 0;
	}
8032

8033 8034 8035 8036
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
8037

8038 8039
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
8040

8041 8042 8043 8044
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
8045

8046
	BUG_ON(cache->ro);
C
Chris Mason 已提交
8047

8048 8049
	trans = btrfs_join_transaction(root, 1);
	BUG_ON(IS_ERR(trans));
8050

8051 8052 8053
	alloc_flags = update_block_group_flags(root, cache->flags);
	if (alloc_flags != cache->flags)
		do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags, 1);
8054

8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066
	ret = set_block_group_ro(cache);
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
	ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags, 1);
	if (ret < 0)
		goto out;
	ret = set_block_group_ro(cache);
out:
	btrfs_end_transaction(trans, root);
	return ret;
}
8067

8068 8069 8070 8071 8072 8073 8074
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
	return do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags, 1);
}

8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130
/*
 * helper to account the unused space of all the readonly block group in the
 * list. takes mirrors into account.
 */
static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

	list_for_each_entry(block_group, groups_list, list) {
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}

	return free_bytes;
}

/*
 * helper to account the unused space of all the readonly block group in the
 * space_info. takes mirrors into account.
 */
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
{
	int i;
	u64 free_bytes = 0;

	spin_lock(&sinfo->lock);

	for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		if (!list_empty(&sinfo->block_groups[i]))
			free_bytes += __btrfs_get_ro_block_group_free_space(
						&sinfo->block_groups[i]);

	spin_unlock(&sinfo->lock);

	return free_bytes;
}

8131 8132
int btrfs_set_block_group_rw(struct btrfs_root *root,
			      struct btrfs_block_group_cache *cache)
8133
{
8134 8135 8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
8147 8148 8149
	return 0;
}

8150 8151 8152 8153 8154 8155 8156
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
8157
{
8158 8159 8160 8161 8162 8163
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
8164

8165
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
8166

8167 8168 8169
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
Z
Zheng Yan 已提交
8170

8171 8172
	/* no bytes used, we're good */
	if (!btrfs_block_group_used(&block_group->item))
Z
Zheng Yan 已提交
8173 8174
		goto out;

8175 8176
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
8177

8178
	full = space_info->full;
8179

8180 8181
	/*
	 * if this is the last block group we have in this space, we can't
8182 8183 8184 8185
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
8186
	 */
8187 8188
	if ((space_info->total_bytes != block_group->key.offset) &&
	   (space_info->bytes_used + space_info->bytes_reserved +
8189 8190
	    space_info->bytes_pinned + space_info->bytes_readonly +
	    btrfs_block_group_used(&block_group->item) <
8191
	    space_info->total_bytes)) {
8192 8193
		spin_unlock(&space_info->lock);
		goto out;
8194
	}
8195
	spin_unlock(&space_info->lock);
8196

8197 8198 8199 8200 8201 8202 8203 8204 8205 8206
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
	 * alloc devices and guess if we have enough space.  However, if we
	 * were marked as full, then we know there aren't enough chunks, and we
	 * can just return.
	 */
	ret = -1;
	if (full)
		goto out;
8207

8208 8209 8210
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
		u64 min_free = btrfs_block_group_used(&block_group->item);
8211
		u64 dev_offset;
8212

8213 8214 8215 8216 8217 8218
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
		if (device->total_bytes > device->bytes_used + min_free) {
			ret = find_free_dev_extent(NULL, device, min_free,
8219
						   &dev_offset, NULL);
8220
			if (!ret)
8221
				break;
8222
			ret = -1;
8223
		}
8224
	}
8225
	mutex_unlock(&root->fs_info->chunk_mutex);
8226
out:
8227
	btrfs_put_block_group(block_group);
8228 8229 8230
	return ret;
}

8231 8232
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
8233
{
8234
	int ret = 0;
8235 8236 8237
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
8238

8239 8240
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
8241 8242
		goto out;

C
Chris Mason 已提交
8243
	while (1) {
8244
		slot = path->slots[0];
8245
		leaf = path->nodes[0];
8246 8247 8248 8249 8250
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
8251
				goto out;
8252
			break;
8253
		}
8254
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
8255

8256
		if (found_key.objectid >= key->objectid &&
8257 8258 8259 8260
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
8261
		path->slots[0]++;
8262
	}
8263
out:
8264
	return ret;
8265 8266
}

8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
8301 8302 8303
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
8304
	struct btrfs_space_info *space_info;
8305
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
8306 8307
	struct rb_node *n;

8308 8309 8310 8311 8312 8313 8314 8315 8316
	down_write(&info->extent_commit_sem);
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
	up_write(&info->extent_commit_sem);

Z
Zheng Yan 已提交
8317 8318 8319 8320 8321 8322
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
Y
Yan Zheng 已提交
8323 8324
		spin_unlock(&info->block_group_cache_lock);

8325
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
8326
		list_del(&block_group->list);
8327
		up_write(&block_group->space_info->groups_sem);
8328

J
Josef Bacik 已提交
8329
		if (block_group->cached == BTRFS_CACHE_STARTED)
8330
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
8331

8332 8333 8334 8335 8336 8337 8338
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
		if (block_group->cached == BTRFS_CACHE_NO)
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
8339
		btrfs_remove_free_space_cache(block_group);
8340
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
8341 8342

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
8343 8344
	}
	spin_unlock(&info->block_group_cache_lock);
8345 8346 8347 8348 8349 8350 8351 8352 8353

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

8354 8355
	release_global_block_rsv(info);

8356 8357 8358 8359
	while(!list_empty(&info->space_info)) {
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
8360 8361 8362 8363 8364
		if (space_info->bytes_pinned > 0 ||
		    space_info->bytes_reserved > 0) {
			WARN_ON(1);
			dump_space_info(space_info, 0, 0);
		}
8365 8366 8367
		list_del(&space_info->list);
		kfree(space_info);
	}
Z
Zheng Yan 已提交
8368 8369 8370
	return 0;
}

8371 8372 8373 8374 8375 8376 8377 8378 8379 8380
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);

	down_write(&space_info->groups_sem);
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);
}

C
Chris Mason 已提交
8381 8382 8383 8384 8385
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
8386
	struct btrfs_fs_info *info = root->fs_info;
8387
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
8388 8389
	struct btrfs_key key;
	struct btrfs_key found_key;
8390
	struct extent_buffer *leaf;
8391 8392
	int need_clear = 0;
	u64 cache_gen;
8393

C
Chris Mason 已提交
8394
	root = info->extent_root;
C
Chris Mason 已提交
8395
	key.objectid = 0;
8396
	key.offset = 0;
C
Chris Mason 已提交
8397 8398 8399 8400 8401
	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

8402 8403 8404 8405
	cache_gen = btrfs_super_cache_generation(&root->fs_info->super_copy);
	if (cache_gen != 0 &&
	    btrfs_super_generation(&root->fs_info->super_copy) != cache_gen)
		need_clear = 1;
8406 8407
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
8408 8409
	if (!btrfs_test_opt(root, SPACE_CACHE) && cache_gen)
		printk(KERN_INFO "btrfs: disk space caching is enabled\n");
8410

C
Chris Mason 已提交
8411
	while (1) {
8412
		ret = find_first_block_group(root, path, &key);
8413 8414
		if (ret > 0)
			break;
8415 8416
		if (ret != 0)
			goto error;
8417 8418
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
8419
		cache = kzalloc(sizeof(*cache), GFP_NOFS);
C
Chris Mason 已提交
8420
		if (!cache) {
8421
			ret = -ENOMEM;
8422
			goto error;
C
Chris Mason 已提交
8423
		}
C
Chris Mason 已提交
8424

8425
		atomic_set(&cache->count, 1);
8426
		spin_lock_init(&cache->lock);
8427
		spin_lock_init(&cache->tree_lock);
J
Josef Bacik 已提交
8428
		cache->fs_info = info;
J
Josef Bacik 已提交
8429
		INIT_LIST_HEAD(&cache->list);
8430
		INIT_LIST_HEAD(&cache->cluster_list);
8431

8432 8433 8434
		if (need_clear)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

8435 8436 8437 8438 8439 8440 8441 8442
		/*
		 * we only want to have 32k of ram per block group for keeping
		 * track of free space, and if we pass 1/2 of that we want to
		 * start converting things over to using bitmaps
		 */
		cache->extents_thresh = ((1024 * 32) / 2) /
			sizeof(struct btrfs_free_space);

8443 8444 8445
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
C
Chris Mason 已提交
8446
		memcpy(&cache->key, &found_key, sizeof(found_key));
8447

C
Chris Mason 已提交
8448 8449
		key.objectid = found_key.objectid + found_key.offset;
		btrfs_release_path(root, path);
8450
		cache->flags = btrfs_block_group_flags(&cache->item);
J
Josef Bacik 已提交
8451 8452
		cache->sectorsize = root->sectorsize;

8453 8454 8455 8456 8457 8458 8459
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
		exclude_super_stripes(root, cache);

J
Josef Bacik 已提交
8460 8461 8462 8463 8464 8465 8466 8467
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
8468
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
8469
			cache->cached = BTRFS_CACHE_FINISHED;
8470
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
8471
		} else if (btrfs_block_group_used(&cache->item) == 0) {
8472
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
8473 8474 8475 8476 8477
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
8478
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
8479
		}
8480

8481 8482 8483 8484 8485
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
		BUG_ON(ret);
		cache->space_info = space_info;
8486
		spin_lock(&cache->space_info->lock);
8487
		cache->space_info->bytes_readonly += cache->bytes_super;
8488 8489
		spin_unlock(&cache->space_info->lock);

8490
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
8491 8492 8493

		ret = btrfs_add_block_group_cache(root->fs_info, cache);
		BUG_ON(ret);
8494 8495

		set_avail_alloc_bits(root->fs_info, cache->flags);
Y
Yan Zheng 已提交
8496
		if (btrfs_chunk_readonly(root, cache->key.objectid))
8497
			set_block_group_ro(cache);
C
Chris Mason 已提交
8498
	}
8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
		list_for_each_entry(cache, &space_info->block_groups[3], list)
8511
			set_block_group_ro(cache);
8512
		list_for_each_entry(cache, &space_info->block_groups[4], list)
8513
			set_block_group_ro(cache);
C
Chris Mason 已提交
8514
	}
8515 8516

	init_global_block_rsv(info);
8517 8518
	ret = 0;
error:
C
Chris Mason 已提交
8519
	btrfs_free_path(path);
8520
	return ret;
C
Chris Mason 已提交
8521
}
8522 8523 8524

int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
8525
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
8526 8527 8528 8529 8530 8531 8532 8533
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

8534
	root->fs_info->last_trans_log_full_commit = trans->transid;
8535

8536
	cache = kzalloc(sizeof(*cache), GFP_NOFS);
J
Josef Bacik 已提交
8537 8538 8539
	if (!cache)
		return -ENOMEM;

8540
	cache->key.objectid = chunk_offset;
8541
	cache->key.offset = size;
8542
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
8543
	cache->sectorsize = root->sectorsize;
8544
	cache->fs_info = root->fs_info;
8545 8546 8547 8548 8549 8550 8551 8552

	/*
	 * we only want to have 32k of ram per block group for keeping track
	 * of free space, and if we pass 1/2 of that we want to start
	 * converting things over to using bitmaps
	 */
	cache->extents_thresh = ((1024 * 32) / 2) /
		sizeof(struct btrfs_free_space);
8553
	atomic_set(&cache->count, 1);
8554
	spin_lock_init(&cache->lock);
8555
	spin_lock_init(&cache->tree_lock);
J
Josef Bacik 已提交
8556
	INIT_LIST_HEAD(&cache->list);
8557
	INIT_LIST_HEAD(&cache->cluster_list);
C
Chris Mason 已提交
8558

8559 8560 8561 8562 8563
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	cache->flags = type;
	btrfs_set_block_group_flags(&cache->item, type);

8564
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
8565
	cache->cached = BTRFS_CACHE_FINISHED;
8566
	exclude_super_stripes(root, cache);
8567

J
Josef Bacik 已提交
8568 8569 8570
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

8571 8572
	free_excluded_extents(root, cache);

8573 8574 8575
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
	BUG_ON(ret);
8576 8577

	spin_lock(&cache->space_info->lock);
8578
	cache->space_info->bytes_readonly += cache->bytes_super;
8579 8580
	spin_unlock(&cache->space_info->lock);

8581
	__link_block_group(cache->space_info, cache);
8582

J
Josef Bacik 已提交
8583 8584
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
	BUG_ON(ret);
8585

8586 8587 8588 8589
	ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
				sizeof(cache->item));
	BUG_ON(ret);

C
Chris Mason 已提交
8590
	set_avail_alloc_bits(extent_root->fs_info, type);
8591

8592 8593
	return 0;
}
Z
Zheng Yan 已提交
8594 8595 8596 8597 8598 8599

int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 group_start)
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
8600
	struct btrfs_free_cluster *cluster;
8601
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
8602
	struct btrfs_key key;
8603
	struct inode *inode;
Z
Zheng Yan 已提交
8604
	int ret;
J
Josef Bacik 已提交
8605
	int factor;
Z
Zheng Yan 已提交
8606 8607 8608 8609 8610

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
8611
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
8612 8613

	memcpy(&key, &block_group->key, sizeof(key));
J
Josef Bacik 已提交
8614 8615 8616 8617 8618 8619
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
8620

8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
8636 8637 8638
	path = btrfs_alloc_path();
	BUG_ON(!path);

8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672
	inode = lookup_free_space_inode(root, block_group, path);
	if (!IS_ERR(inode)) {
		btrfs_orphan_add(trans, inode);
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
		iput(inode);
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
		btrfs_release_path(tree_root, path);
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
		btrfs_release_path(tree_root, path);
	}

8673
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
8674 8675
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
8676
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
8677

8678
	down_write(&block_group->space_info->groups_sem);
8679 8680 8681 8682 8683
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
8684
	up_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
8685

J
Josef Bacik 已提交
8686
	if (block_group->cached == BTRFS_CACHE_STARTED)
8687
		wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
8688 8689 8690

	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
8691 8692 8693
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
8694
	block_group->space_info->disk_total -= block_group->key.offset * factor;
Y
Yan Zheng 已提交
8695
	spin_unlock(&block_group->space_info->lock);
8696

8697 8698
	memcpy(&key, &block_group->key, sizeof(key));

8699
	btrfs_clear_space_info_full(root->fs_info);
Y
Yan Zheng 已提交
8700

8701 8702
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
8715 8716 8717 8718 8719 8720 8721 8722 8723 8724 8725

int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
	return unpin_extent_range(root, start, end);
}

int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
			       u64 num_bytes)
{
	return btrfs_discard_extent(root, bytenr, num_bytes);
}