extent-tree.c 261.0 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
26
#include <linux/ratelimit.h>
27
#include <linux/percpu_counter.h>
28
#include "hash.h"
29
#include "tree-log.h"
30 31
#include "disk-io.h"
#include "print-tree.h"
32
#include "volumes.h"
D
David Woodhouse 已提交
33
#include "raid56.h"
34
#include "locking.h"
35
#include "free-space-cache.h"
36
#include "math.h"
37
#include "sysfs.h"
J
Josef Bacik 已提交
38
#include "qgroup.h"
39

40 41
#undef SCRAMBLE_DELAYED_REFS

42 43
/*
 * control flags for do_chunk_alloc's force field
44 45 46 47 48 49 50 51 52
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
53 54
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
55 56 57
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
58 59
	CHUNK_ALLOC_LIMITED = 1,
	CHUNK_ALLOC_FORCE = 2,
60 61
};

62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
/*
 * Control how reservations are dealt with.
 *
 * RESERVE_FREE - freeing a reservation.
 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
 *   ENOSPC accounting
 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
 *   bytes_may_use as the ENOSPC accounting is done elsewhere
 */
enum {
	RESERVE_FREE = 0,
	RESERVE_ALLOC = 1,
	RESERVE_ALLOC_NO_ACCOUNT = 2,
};

77 78 79
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root, u64 bytenr,
			      u64 num_bytes, int alloc);
80 81 82 83 84
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
J
Josef Bacik 已提交
85 86
				struct btrfs_delayed_extent_op *extra_op,
				int no_quota);
87 88 89 90 91 92 93 94 95 96 97 98
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
J
Josef Bacik 已提交
99 100
				     int level, struct btrfs_key *ins,
				     int no_quota);
J
Josef Bacik 已提交
101
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
102 103
			  struct btrfs_root *extent_root, u64 flags,
			  int force);
104 105
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
106 107
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
108
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
109 110
				       u64 num_bytes, int reserve,
				       int delalloc);
111 112
static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes);
113 114
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved);
J
Josef Bacik 已提交
115

J
Josef Bacik 已提交
116 117 118 119
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
120 121
	return cache->cached == BTRFS_CACHE_FINISHED ||
		cache->cached == BTRFS_CACHE_ERROR;
J
Josef Bacik 已提交
122 123
}

J
Josef Bacik 已提交
124 125 126 127 128
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

129
static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
130 131 132 133 134 135
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
136 137 138
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
139
		kfree(cache->free_space_ctl);
140
		kfree(cache);
141
	}
142 143
}

J
Josef Bacik 已提交
144 145 146 147
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
148
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
175 176 177 178

	if (info->first_logical_byte > block_group->key.objectid)
		info->first_logical_byte = block_group->key.objectid;

J
Josef Bacik 已提交
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
220
	if (ret) {
221
		btrfs_get_block_group(ret);
222 223 224
		if (bytenr == 0 && info->first_logical_byte > ret->key.objectid)
			info->first_logical_byte = ret->key.objectid;
	}
J
Josef Bacik 已提交
225 226 227 228 229
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

230 231
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
232
{
233 234 235 236 237 238 239
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
J
Josef Bacik 已提交
240

241 242 243 244
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
245

246 247 248 249 250 251 252
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
J
Josef Bacik 已提交
253 254
}

255 256
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
257 258 259 260 261 262
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

263 264 265 266 267
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
268 269
		if (ret)
			return ret;
270 271
	}

J
Josef Bacik 已提交
272 273 274 275 276
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
277 278
		if (ret)
			return ret;
279

J
Josef Bacik 已提交
280
		while (nr--) {
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
			u64 start, len;

			if (logical[nr] > cache->key.objectid +
			    cache->key.offset)
				continue;

			if (logical[nr] + stripe_len <= cache->key.objectid)
				continue;

			start = logical[nr];
			if (start < cache->key.objectid) {
				start = cache->key.objectid;
				len = (logical[nr] + stripe_len) - start;
			} else {
				len = min_t(u64, stripe_len,
					    cache->key.objectid +
					    cache->key.offset - start);
			}

			cache->bytes_super += len;
			ret = add_excluded_extent(root, start, len);
302 303 304 305
			if (ret) {
				kfree(logical);
				return ret;
			}
J
Josef Bacik 已提交
306
		}
307

J
Josef Bacik 已提交
308 309 310 311 312
		kfree(logical);
	}
	return 0;
}

313 314 315 316 317 318
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
319 320
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

J
Josef Bacik 已提交
336 337 338 339 340
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
J
Josef Bacik 已提交
341
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
J
Josef Bacik 已提交
342 343
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
J
Josef Bacik 已提交
344
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
345 346 347
	int ret;

	while (start < end) {
348
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
349
					    &extent_start, &extent_end,
350 351
					    EXTENT_DIRTY | EXTENT_UPTODATE,
					    NULL);
J
Josef Bacik 已提交
352 353 354
		if (ret)
			break;

355
		if (extent_start <= start) {
J
Josef Bacik 已提交
356 357 358
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
359
			total_added += size;
360 361
			ret = btrfs_add_free_space(block_group, start,
						   size);
362
			BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
363 364 365 366 367 368 369 370
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
371
		total_added += size;
372
		ret = btrfs_add_free_space(block_group, start, size);
373
		BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
374 375
	}

J
Josef Bacik 已提交
376
	return total_added;
J
Josef Bacik 已提交
377 378
}

379
static noinline void caching_thread(struct btrfs_work *work)
380
{
381 382 383 384
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_root *extent_root;
385
	struct btrfs_path *path;
386
	struct extent_buffer *leaf;
387
	struct btrfs_key key;
J
Josef Bacik 已提交
388
	u64 total_found = 0;
389 390
	u64 last = 0;
	u32 nritems;
391
	int ret = -ENOMEM;
392

393 394 395 396 397
	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

398 399
	path = btrfs_alloc_path();
	if (!path)
400
		goto out;
401

J
Josef Bacik 已提交
402
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
403

404
	/*
J
Josef Bacik 已提交
405 406 407 408
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
409 410
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
411
	path->search_commit_root = 1;
J
Josef Bacik 已提交
412
	path->reada = 1;
J
Josef Bacik 已提交
413

Y
Yan Zheng 已提交
414
	key.objectid = last;
415
	key.offset = 0;
416
	key.type = BTRFS_EXTENT_ITEM_KEY;
417
again:
418
	mutex_lock(&caching_ctl->mutex);
419
	/* need to make sure the commit_root doesn't disappear */
420
	down_read(&fs_info->commit_root_sem);
421

422
next:
423
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
424
	if (ret < 0)
425
		goto err;
Y
Yan Zheng 已提交
426

427 428 429
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
430
	while (1) {
431
		if (btrfs_fs_closing(fs_info) > 1) {
432
			last = (u64)-1;
J
Josef Bacik 已提交
433
			break;
434
		}
J
Josef Bacik 已提交
435

436 437 438 439 440
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
441
				break;
J
Josef Bacik 已提交
442

443
			if (need_resched() ||
444
			    rwsem_is_contended(&fs_info->commit_root_sem)) {
445
				caching_ctl->progress = last;
C
Chris Mason 已提交
446
				btrfs_release_path(path);
447
				up_read(&fs_info->commit_root_sem);
448
				mutex_unlock(&caching_ctl->mutex);
449
				cond_resched();
450 451
				goto again;
			}
452 453 454 455 456 457

			ret = btrfs_next_leaf(extent_root, path);
			if (ret < 0)
				goto err;
			if (ret)
				break;
458 459 460
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
461
		}
J
Josef Bacik 已提交
462

463 464 465 466 467 468 469 470 471 472
		if (key.objectid < last) {
			key.objectid = last;
			key.offset = 0;
			key.type = BTRFS_EXTENT_ITEM_KEY;

			caching_ctl->progress = last;
			btrfs_release_path(path);
			goto next;
		}

473 474
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
475
			continue;
476
		}
J
Josef Bacik 已提交
477

478
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
479
		    block_group->key.offset)
480
			break;
481

482 483
		if (key.type == BTRFS_EXTENT_ITEM_KEY ||
		    key.type == BTRFS_METADATA_ITEM_KEY) {
J
Josef Bacik 已提交
484 485 486
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
487 488
			if (key.type == BTRFS_METADATA_ITEM_KEY)
				last = key.objectid +
489
					fs_info->tree_root->nodesize;
490 491
			else
				last = key.objectid + key.offset;
J
Josef Bacik 已提交
492

493 494 495 496
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
J
Josef Bacik 已提交
497
		}
498 499
		path->slots[0]++;
	}
J
Josef Bacik 已提交
500
	ret = 0;
501

J
Josef Bacik 已提交
502 503 504
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
505
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
506 507

	spin_lock(&block_group->lock);
508
	block_group->caching_ctl = NULL;
J
Josef Bacik 已提交
509 510
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
511

512
err:
513
	btrfs_free_path(path);
514
	up_read(&fs_info->commit_root_sem);
J
Josef Bacik 已提交
515

516 517 518
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
519
out:
520 521 522 523 524 525
	if (ret) {
		spin_lock(&block_group->lock);
		block_group->caching_ctl = NULL;
		block_group->cached = BTRFS_CACHE_ERROR;
		spin_unlock(&block_group->lock);
	}
526 527 528
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
529
	btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
530 531
}

532 533
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     int load_cache_only)
J
Josef Bacik 已提交
534
{
535
	DEFINE_WAIT(wait);
536 537
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
538 539
	int ret = 0;

540
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
541 542
	if (!caching_ctl)
		return -ENOMEM;
543 544 545 546 547 548 549

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	atomic_set(&caching_ctl->count, 1);
550 551
	btrfs_init_work(&caching_ctl->work, btrfs_cache_helper,
			caching_thread, NULL, NULL);
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583

	spin_lock(&cache->lock);
	/*
	 * This should be a rare occasion, but this could happen I think in the
	 * case where one thread starts to load the space cache info, and then
	 * some other thread starts a transaction commit which tries to do an
	 * allocation while the other thread is still loading the space cache
	 * info.  The previous loop should have kept us from choosing this block
	 * group, but if we've moved to the state where we will wait on caching
	 * block groups we need to first check if we're doing a fast load here,
	 * so we can wait for it to finish, otherwise we could end up allocating
	 * from a block group who's cache gets evicted for one reason or
	 * another.
	 */
	while (cache->cached == BTRFS_CACHE_FAST) {
		struct btrfs_caching_control *ctl;

		ctl = cache->caching_ctl;
		atomic_inc(&ctl->count);
		prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&cache->lock);

		schedule();

		finish_wait(&ctl->wait, &wait);
		put_caching_control(ctl);
		spin_lock(&cache->lock);
	}

	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
		kfree(caching_ctl);
584
		return 0;
585 586 587 588 589
	}
	WARN_ON(cache->caching_ctl);
	cache->caching_ctl = caching_ctl;
	cache->cached = BTRFS_CACHE_FAST;
	spin_unlock(&cache->lock);
590

591
	if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
592
		mutex_lock(&caching_ctl->mutex);
593 594 595 596
		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
597
			cache->caching_ctl = NULL;
598 599
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
600
			caching_ctl->progress = (u64)-1;
601
		} else {
602 603 604 605 606
			if (load_cache_only) {
				cache->caching_ctl = NULL;
				cache->cached = BTRFS_CACHE_NO;
			} else {
				cache->cached = BTRFS_CACHE_STARTED;
607
				cache->has_caching_ctl = 1;
608
			}
609 610
		}
		spin_unlock(&cache->lock);
611 612
		mutex_unlock(&caching_ctl->mutex);

613
		wake_up(&caching_ctl->wait);
614
		if (ret == 1) {
615
			put_caching_control(caching_ctl);
616
			free_excluded_extents(fs_info->extent_root, cache);
617
			return 0;
618
		}
619 620 621 622 623 624 625 626 627 628 629
	} else {
		/*
		 * We are not going to do the fast caching, set cached to the
		 * appropriate value and wakeup any waiters.
		 */
		spin_lock(&cache->lock);
		if (load_cache_only) {
			cache->caching_ctl = NULL;
			cache->cached = BTRFS_CACHE_NO;
		} else {
			cache->cached = BTRFS_CACHE_STARTED;
630
			cache->has_caching_ctl = 1;
631 632 633
		}
		spin_unlock(&cache->lock);
		wake_up(&caching_ctl->wait);
634 635
	}

636 637
	if (load_cache_only) {
		put_caching_control(caching_ctl);
638
		return 0;
J
Josef Bacik 已提交
639 640
	}

641
	down_write(&fs_info->commit_root_sem);
642
	atomic_inc(&caching_ctl->count);
643
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
644
	up_write(&fs_info->commit_root_sem);
645

646
	btrfs_get_block_group(cache);
647

648
	btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work);
J
Josef Bacik 已提交
649

650
	return ret;
651 652
}

J
Josef Bacik 已提交
653 654 655
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
656 657
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
658
{
J
Josef Bacik 已提交
659
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
660

J
Josef Bacik 已提交
661
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
662

J
Josef Bacik 已提交
663
	return cache;
C
Chris Mason 已提交
664 665
}

J
Josef Bacik 已提交
666
/*
667
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
668
 */
C
Chris Mason 已提交
669 670 671
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
672
{
J
Josef Bacik 已提交
673
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
674

J
Josef Bacik 已提交
675
	cache = block_group_cache_tree_search(info, bytenr, 1);
676

J
Josef Bacik 已提交
677
	return cache;
C
Chris Mason 已提交
678
}
679

J
Josef Bacik 已提交
680 681
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
682
{
J
Josef Bacik 已提交
683 684
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
685

686
	flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
687

688 689
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
690
		if (found->flags & flags) {
691
			rcu_read_unlock();
J
Josef Bacik 已提交
692
			return found;
693
		}
J
Josef Bacik 已提交
694
	}
695
	rcu_read_unlock();
J
Josef Bacik 已提交
696
	return NULL;
697 698
}

699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

714 715
/* simple helper to search for an existing data extent at a given offset */
int btrfs_lookup_data_extent(struct btrfs_root *root, u64 start, u64 len)
716 717 718
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
719
	struct btrfs_path *path;
720

Z
Zheng Yan 已提交
721
	path = btrfs_alloc_path();
722 723 724
	if (!path)
		return -ENOMEM;

725 726
	key.objectid = start;
	key.offset = len;
727
	key.type = BTRFS_EXTENT_ITEM_KEY;
728 729
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
730
	btrfs_free_path(path);
731 732 733
	return ret;
}

734
/*
735
 * helper function to lookup reference count and flags of a tree block.
736 737 738 739 740 741 742 743 744
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
745
			     u64 offset, int metadata, u64 *refs, u64 *flags)
746 747 748 749 750 751 752 753 754 755 756 757
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

758 759 760 761 762
	/*
	 * If we don't have skinny metadata, don't bother doing anything
	 * different
	 */
	if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA)) {
763
		offset = root->nodesize;
764 765 766
		metadata = 0;
	}

767 768 769 770 771 772 773 774
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
775 776 777 778 779 780 781 782 783

search_again:
	key.objectid = bytenr;
	key.offset = offset;
	if (metadata)
		key.type = BTRFS_METADATA_ITEM_KEY;
	else
		key.type = BTRFS_EXTENT_ITEM_KEY;

784 785 786 787 788
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

789
	if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) {
790 791 792 793 794 795
		if (path->slots[0]) {
			path->slots[0]--;
			btrfs_item_key_to_cpu(path->nodes[0], &key,
					      path->slots[0]);
			if (key.objectid == bytenr &&
			    key.type == BTRFS_EXTENT_ITEM_KEY &&
796
			    key.offset == root->nodesize)
797 798
				ret = 0;
		}
799 800
	}

801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839
	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

840
			btrfs_release_path(path);
841

842 843 844 845
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
846 847 848
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
849
			goto search_again;
850
		}
851
		spin_lock(&head->lock);
852 853 854 855 856 857
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
858
		spin_unlock(&head->lock);
859 860 861 862 863 864 865 866 867 868 869 870 871 872
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

873 874 875 876 877 878 879 880 881 882 883 884 885 886
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
932 933 934
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
935
 * - different files inside a single subvolume
936 937
 * - different offsets inside a file (bookend extents in file.c)
 *
938
 * The extent ref structure for the implicit back refs has fields for:
939 940 941
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
942 943
 * - original offset in the file
 * - how many bookend extents
944
 *
945 946
 * The key offset for the implicit back refs is hash of the first
 * three fields.
947
 *
948
 * The extent ref structure for the full back refs has field for:
949
 *
950
 * - number of pointers in the tree leaf
951
 *
952 953
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
954
 *
955 956
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
957
 *
958
 *     (root_key.objectid, inode objectid, offset in file, 1)
959
 *
960 961
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
962
 *
963
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
964
 *
965
 * Btree extents can be referenced by:
966
 *
967
 * - Different subvolumes
968
 *
969 970 971 972
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
973
 *
974 975 976
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
977
 */
Z
Zheng Yan 已提交
978

979 980 981 982 983
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
984
{
985 986 987 988 989
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
990
	struct btrfs_key key;
991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
1010
				BUG_ON(ret > 0); /* Corruption */
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
1026
	btrfs_release_path(path);
1027 1028 1029 1030 1031 1032 1033 1034 1035

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
1036
	BUG_ON(ret); /* Corruption */
1037

1038
	btrfs_extend_item(root, path, new_size);
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
1068
	high_crc = btrfs_crc32c(high_crc, &lenum, sizeof(lenum));
1069
	lenum = cpu_to_le64(owner);
1070
	low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum));
1071
	lenum = cpu_to_le64(offset);
1072
	low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum));
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1105
	struct extent_buffer *leaf;
1106
	u32 nritems;
1107
	int ret;
1108 1109
	int recow;
	int err = -ENOENT;
1110

Z
Zheng Yan 已提交
1111
	key.objectid = bytenr;
1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1127

1128 1129 1130 1131 1132
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1133
		btrfs_release_path(path);
1134 1135 1136 1137 1138 1139 1140 1141 1142
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1143 1144 1145
	}

	leaf = path->nodes[0];
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1171
				btrfs_release_path(path);
1172 1173 1174 1175 1176 1177
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1178
	}
1179 1180
fail:
	return err;
Z
Zheng Yan 已提交
1181 1182
}

1183 1184 1185 1186 1187 1188
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1189 1190 1191
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1192
	u32 size;
Z
Zheng Yan 已提交
1193 1194
	u32 num_refs;
	int ret;
1195 1196

	key.objectid = bytenr;
1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1207

1208 1209 1210 1211 1212 1213 1214
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1215
		ref = btrfs_item_ptr(leaf, path->slots[0],
1216 1217 1218 1219 1220 1221 1222
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1223
		}
1224 1225 1226 1227 1228 1229 1230 1231
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1232
			btrfs_release_path(path);
1233 1234 1235 1236 1237
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1238

1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1253 1254
		}
	}
1255 1256 1257
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1258
	btrfs_release_path(path);
1259
	return ret;
1260 1261
}

1262 1263 1264
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
J
Josef Bacik 已提交
1265
					   int refs_to_drop, int *last_ref)
Z
Zheng Yan 已提交
1266
{
1267 1268 1269
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1270
	struct extent_buffer *leaf;
1271
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1272 1273 1274
	int ret = 0;

	leaf = path->nodes[0];
1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1296 1297
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1298

Z
Zheng Yan 已提交
1299 1300
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
J
Josef Bacik 已提交
1301
		*last_ref = 1;
Z
Zheng Yan 已提交
1302
	} else {
1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1315 1316 1317 1318 1319
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1320 1321 1322
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1323
{
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1355
#endif
1356 1357 1358 1359 1360
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1361

1362 1363 1364 1365 1366
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1367
{
1368
	struct btrfs_key key;
1369 1370
	int ret;

1371 1372 1373 1374 1375 1376 1377
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1378 1379
	}

1380 1381 1382 1383 1384
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1385
		btrfs_release_path(path);
1386 1387 1388 1389 1390
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1391
#endif
1392
	return ret;
1393 1394
}

1395 1396 1397 1398 1399
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1400
{
1401
	struct btrfs_key key;
Z
Zheng Yan 已提交
1402 1403
	int ret;

1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1414
	btrfs_release_path(path);
Z
Zheng Yan 已提交
1415 1416 1417
	return ret;
}

1418
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1419
{
1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1433
}
1434

1435 1436
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1437

C
Chris Mason 已提交
1438
{
1439
	for (; level < BTRFS_MAX_LEVEL; level++) {
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1455

1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1491 1492
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
1493

1494
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1495
	key.type = BTRFS_EXTENT_ITEM_KEY;
1496
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1497

1498 1499 1500
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1501
		path->keep_locks = 1;
1502 1503
	} else
		extra_size = -1;
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514

	/*
	 * Owner is our parent level, so we can just add one to get the level
	 * for the block we are interested in.
	 */
	if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) {
		key.type = BTRFS_METADATA_ITEM_KEY;
		key.offset = owner;
	}

again:
1515
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1516
	if (ret < 0) {
1517 1518 1519
		err = ret;
		goto out;
	}
1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536

	/*
	 * We may be a newly converted file system which still has the old fat
	 * extent entries for metadata, so try and see if we have one of those.
	 */
	if (ret > 0 && skinny_metadata) {
		skinny_metadata = false;
		if (path->slots[0]) {
			path->slots[0]--;
			btrfs_item_key_to_cpu(path->nodes[0], &key,
					      path->slots[0]);
			if (key.objectid == bytenr &&
			    key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes)
				ret = 0;
		}
		if (ret) {
1537
			key.objectid = bytenr;
1538 1539 1540 1541 1542 1543 1544
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;
			btrfs_release_path(path);
			goto again;
		}
	}

1545 1546 1547
	if (ret && !insert) {
		err = -ENOENT;
		goto out;
1548
	} else if (WARN_ON(ret)) {
1549 1550
		err = -EIO;
		goto out;
1551
	}
1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

1579
	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) {
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1643 1644
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1645
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1646 1647 1648 1649 1650 1651
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1652
	if (insert) {
1653 1654 1655 1656 1657 1658 1659 1660 1661 1662
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
1663
void setup_inline_extent_backref(struct btrfs_root *root,
1664 1665 1666 1667 1668
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

1686
	btrfs_extend_item(root, path, size);
1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1736
		return ret;
1737

1738
	btrfs_release_path(path);
1739 1740 1741 1742 1743 1744 1745 1746
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1747
	}
1748 1749
	return ret;
}
Z
Zheng Yan 已提交
1750

1751 1752 1753 1754
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
1755
void update_inline_extent_backref(struct btrfs_root *root,
1756 1757 1758
				  struct btrfs_path *path,
				  struct btrfs_extent_inline_ref *iref,
				  int refs_to_mod,
J
Josef Bacik 已提交
1759 1760
				  struct btrfs_delayed_extent_op *extent_op,
				  int *last_ref)
1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1793
	}
Z
Zheng Yan 已提交
1794

1795 1796 1797 1798 1799 1800 1801 1802 1803
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
J
Josef Bacik 已提交
1804
		*last_ref = 1;
1805 1806 1807 1808 1809 1810 1811 1812
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
1813
		btrfs_truncate_item(root, path, item_size, 1);
1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834
	}
	btrfs_mark_buffer_dirty(leaf);
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1835
		update_inline_extent_backref(root, path, iref,
J
Josef Bacik 已提交
1836
					     refs_to_add, extent_op, NULL);
1837
	} else if (ret == -ENOENT) {
1838
		setup_inline_extent_backref(root, path, iref, parent,
1839 1840 1841
					    root_objectid, owner, offset,
					    refs_to_add, extent_op);
		ret = 0;
1842
	}
1843 1844
	return ret;
}
Z
Zheng Yan 已提交
1845

1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1864

1865 1866 1867 1868
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
J
Josef Bacik 已提交
1869
				 int refs_to_drop, int is_data, int *last_ref)
1870
{
1871
	int ret = 0;
1872

1873 1874
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
1875
		update_inline_extent_backref(root, path, iref,
J
Josef Bacik 已提交
1876
					     -refs_to_drop, NULL, last_ref);
1877
	} else if (is_data) {
J
Josef Bacik 已提交
1878 1879
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop,
					     last_ref);
1880
	} else {
J
Josef Bacik 已提交
1881
		*last_ref = 1;
1882 1883 1884 1885 1886
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1887
static int btrfs_issue_discard(struct block_device *bdev,
1888 1889
				u64 start, u64 len)
{
1890
	return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1891 1892
}

1893 1894
int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
			 u64 num_bytes, u64 *actual_bytes)
1895 1896
{
	int ret;
1897
	u64 discarded_bytes = 0;
1898
	struct btrfs_bio *bbio = NULL;
1899

C
Christoph Hellwig 已提交
1900

1901
	/* Tell the block device(s) that the sectors can be discarded */
1902
	ret = btrfs_map_block(root->fs_info, REQ_DISCARD,
1903
			      bytenr, &num_bytes, &bbio, 0);
1904
	/* Error condition is -ENOMEM */
1905
	if (!ret) {
1906
		struct btrfs_bio_stripe *stripe = bbio->stripes;
1907 1908 1909
		int i;


1910
		for (i = 0; i < bbio->num_stripes; i++, stripe++) {
1911 1912 1913
			if (!stripe->dev->can_discard)
				continue;

1914 1915 1916 1917 1918 1919
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
						  stripe->length);
			if (!ret)
				discarded_bytes += stripe->length;
			else if (ret != -EOPNOTSUPP)
1920
				break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
1921 1922 1923 1924 1925 1926 1927

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
1928
		}
1929
		btrfs_put_bbio(bbio);
1930
	}
1931 1932 1933 1934

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

1935

D
David Woodhouse 已提交
1936 1937
	if (ret == -EOPNOTSUPP)
		ret = 0;
1938 1939 1940
	return ret;
}

1941
/* Can return -ENOMEM */
1942 1943 1944
int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
J
Josef Bacik 已提交
1945 1946
			 u64 root_objectid, u64 owner, u64 offset,
			 int no_quota)
1947 1948
{
	int ret;
A
Arne Jansen 已提交
1949 1950
	struct btrfs_fs_info *fs_info = root->fs_info;

1951 1952 1953 1954
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
1955 1956
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
1957
					parent, root_objectid, (int)owner,
J
Josef Bacik 已提交
1958
					BTRFS_ADD_DELAYED_REF, NULL, no_quota);
1959
	} else {
A
Arne Jansen 已提交
1960 1961
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
					num_bytes,
1962
					parent, root_objectid, owner, offset,
J
Josef Bacik 已提交
1963
					BTRFS_ADD_DELAYED_REF, NULL, no_quota);
1964 1965 1966 1967 1968 1969 1970 1971 1972
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
J
Josef Bacik 已提交
1973
				  int no_quota,
1974 1975
				  struct btrfs_delayed_extent_op *extent_op)
{
J
Josef Bacik 已提交
1976
	struct btrfs_fs_info *fs_info = root->fs_info;
1977 1978 1979
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
J
Josef Bacik 已提交
1980
	struct btrfs_key key;
1981 1982
	u64 refs;
	int ret;
J
Josef Bacik 已提交
1983
	enum btrfs_qgroup_operation_type type = BTRFS_QGROUP_OPER_ADD_EXCL;
1984 1985 1986 1987 1988

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

J
Josef Bacik 已提交
1989 1990 1991
	if (!is_fstree(root_objectid) || !root->fs_info->quota_enabled)
		no_quota = 1;

1992 1993 1994
	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
J
Josef Bacik 已提交
1995 1996
	ret = insert_inline_extent_backref(trans, fs_info->extent_root, path,
					   bytenr, num_bytes, parent,
1997 1998
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
J
Josef Bacik 已提交
1999
	if ((ret < 0 && ret != -EAGAIN) || (!ret && no_quota))
2000
		goto out;
J
Josef Bacik 已提交
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013
	/*
	 * Ok we were able to insert an inline extent and it appears to be a new
	 * reference, deal with the qgroup accounting.
	 */
	if (!ret && !no_quota) {
		ASSERT(root->fs_info->quota_enabled);
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		item = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_item);
		if (btrfs_extent_refs(leaf, item) > (u64)refs_to_add)
			type = BTRFS_QGROUP_OPER_ADD_SHARED;
		btrfs_release_path(path);
2014

J
Josef Bacik 已提交
2015 2016 2017 2018 2019 2020 2021 2022 2023 2024
		ret = btrfs_qgroup_record_ref(trans, fs_info, root_objectid,
					      bytenr, num_bytes, type, 0);
		goto out;
	}

	/*
	 * Ok we had -EAGAIN which means we didn't have space to insert and
	 * inline extent ref, so just update the reference count and add a
	 * normal backref.
	 */
2025
	leaf = path->nodes[0];
J
Josef Bacik 已提交
2026
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2027 2028
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
J
Josef Bacik 已提交
2029 2030
	if (refs)
		type = BTRFS_QGROUP_OPER_ADD_SHARED;
2031 2032 2033
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
2034

2035
	btrfs_mark_buffer_dirty(leaf);
2036
	btrfs_release_path(path);
2037

J
Josef Bacik 已提交
2038 2039 2040 2041 2042 2043 2044
	if (!no_quota) {
		ret = btrfs_qgroup_record_ref(trans, fs_info, root_objectid,
					      bytenr, num_bytes, type, 0);
		if (ret)
			goto out;
	}

2045
	path->reada = 1;
2046
	path->leave_spinning = 1;
2047 2048
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
2049 2050
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
2051 2052
	if (ret)
		btrfs_abort_transaction(trans, root, ret);
2053
out:
2054
	btrfs_free_path(path);
2055
	return ret;
2056 2057
}

2058 2059 2060 2061 2062
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2063
{
2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
2076 2077
	trace_run_delayed_data_ref(node, ref, node->action);

2078 2079
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
J
Josef Bacik 已提交
2080
	ref_root = ref->root;
2081 2082

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2083
		if (extent_op)
2084 2085 2086 2087 2088 2089 2090 2091 2092 2093
			flags |= extent_op->flags_to_set;
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
J
Josef Bacik 已提交
2094
					     node->no_quota, extent_op);
2095 2096 2097 2098 2099
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
J
Josef Bacik 已提交
2100
					  extent_op, node->no_quota);
2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
2135
	int ret;
2136
	int err = 0;
2137
	int metadata = !extent_op->is_data;
2138

2139 2140 2141
	if (trans->aborted)
		return 0;

2142 2143 2144
	if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA))
		metadata = 0;

2145 2146 2147 2148 2149 2150
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;

2151 2152
	if (metadata) {
		key.type = BTRFS_METADATA_ITEM_KEY;
2153
		key.offset = extent_op->level;
2154 2155 2156 2157 2158 2159
	} else {
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = node->num_bytes;
	}

again:
2160 2161 2162 2163 2164 2165 2166 2167 2168
	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
2169
		if (metadata) {
2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181
			if (path->slots[0] > 0) {
				path->slots[0]--;
				btrfs_item_key_to_cpu(path->nodes[0], &key,
						      path->slots[0]);
				if (key.objectid == node->bytenr &&
				    key.type == BTRFS_EXTENT_ITEM_KEY &&
				    key.offset == node->num_bytes)
					ret = 0;
			}
			if (ret > 0) {
				btrfs_release_path(path);
				metadata = 0;
2182

2183 2184 2185 2186 2187 2188 2189 2190
				key.objectid = node->bytenr;
				key.offset = node->num_bytes;
				key.type = BTRFS_EXTENT_ITEM_KEY;
				goto again;
			}
		} else {
			err = -EIO;
			goto out;
2191
		}
2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2211

2212 2213 2214 2215
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2216 2217
}

2218 2219 2220 2221 2222
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2223 2224
{
	int ret = 0;
2225 2226 2227 2228
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2229 2230
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
2231

2232
	ref = btrfs_delayed_node_to_tree_ref(node);
2233 2234
	trace_run_delayed_tree_ref(node, ref, node->action);

2235 2236
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
J
Josef Bacik 已提交
2237
	ref_root = ref->root;
2238

2239 2240 2241 2242 2243 2244 2245 2246 2247
	ins.objectid = node->bytenr;
	if (skinny_metadata) {
		ins.offset = ref->level;
		ins.type = BTRFS_METADATA_ITEM_KEY;
	} else {
		ins.offset = node->num_bytes;
		ins.type = BTRFS_EXTENT_ITEM_KEY;
	}

2248 2249
	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2250
		BUG_ON(!extent_op || !extent_op->update_flags);
2251 2252 2253 2254
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
J
Josef Bacik 已提交
2255 2256
						ref->level, &ins,
						node->no_quota);
2257 2258 2259
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
J
Josef Bacik 已提交
2260 2261
					     ref->level, 0, 1, node->no_quota,
					     extent_op);
2262 2263 2264
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
J
Josef Bacik 已提交
2265 2266
					  ref->level, 0, 1, extent_op,
					  node->no_quota);
2267 2268 2269
	} else {
		BUG();
	}
2270 2271 2272 2273
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2274 2275 2276 2277 2278
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2279
{
2280 2281
	int ret = 0;

2282 2283 2284 2285
	if (trans->aborted) {
		if (insert_reserved)
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2286
		return 0;
2287
	}
2288

2289
	if (btrfs_delayed_ref_is_head(node)) {
2290 2291 2292 2293 2294 2295 2296
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2297 2298
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2299 2300
		trace_run_delayed_ref_head(node, head, node->action);

2301
		if (insert_reserved) {
2302 2303
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2304 2305 2306 2307 2308
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
			}
2309
		}
2310
		return ret;
2311 2312
	}

2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2324 2325 2326 2327 2328 2329
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
2330 2331
	struct btrfs_delayed_ref_node *ref, *last = NULL;;

2332 2333 2334 2335 2336
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
2337 2338
	node = rb_first(&head->ref_root);
	while (node) {
2339 2340
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
2341
		if (ref->action == BTRFS_ADD_DELAYED_REF)
2342
			return ref;
2343 2344 2345
		else if (last == NULL)
			last = ref;
		node = rb_next(node);
2346
	}
2347
	return last;
2348 2349
}

2350 2351 2352 2353
/*
 * Returns 0 on success or if called with an already aborted transaction.
 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
 */
2354 2355 2356
static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
					     struct btrfs_root *root,
					     unsigned long nr)
2357 2358 2359 2360
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2361
	struct btrfs_delayed_extent_op *extent_op;
2362
	struct btrfs_fs_info *fs_info = root->fs_info;
2363
	ktime_t start = ktime_get();
2364
	int ret;
2365
	unsigned long count = 0;
2366
	unsigned long actual_count = 0;
2367 2368 2369 2370 2371
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2372
			if (count >= nr)
2373 2374
				break;

2375 2376 2377 2378 2379 2380
			spin_lock(&delayed_refs->lock);
			locked_ref = btrfs_select_ref_head(trans);
			if (!locked_ref) {
				spin_unlock(&delayed_refs->lock);
				break;
			}
2381 2382 2383 2384

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);
2385
			spin_unlock(&delayed_refs->lock);
2386 2387 2388 2389 2390 2391 2392 2393 2394 2395
			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2396 2397
			}
		}
2398

2399 2400 2401 2402 2403 2404 2405
		/*
		 * We need to try and merge add/drops of the same ref since we
		 * can run into issues with relocate dropping the implicit ref
		 * and then it being added back again before the drop can
		 * finish.  If we merged anything we need to re-loop so we can
		 * get a good ref.
		 */
2406
		spin_lock(&locked_ref->lock);
2407 2408 2409
		btrfs_merge_delayed_refs(trans, fs_info, delayed_refs,
					 locked_ref);

2410 2411 2412 2413 2414 2415 2416
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);

		if (ref && ref->seq &&
2417
		    btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) {
2418
			spin_unlock(&locked_ref->lock);
2419
			btrfs_delayed_ref_unlock(locked_ref);
2420 2421
			spin_lock(&delayed_refs->lock);
			locked_ref->processing = 0;
2422 2423
			delayed_refs->num_heads_ready++;
			spin_unlock(&delayed_refs->lock);
2424
			locked_ref = NULL;
2425
			cond_resched();
2426
			count++;
2427 2428 2429
			continue;
		}

2430 2431 2432 2433 2434 2435
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2436

2437 2438 2439
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2440
		if (!ref) {
2441 2442


2443 2444 2445 2446 2447
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2448 2449

			if (extent_op && must_insert_reserved) {
2450
				btrfs_free_delayed_extent_op(extent_op);
2451 2452 2453 2454
				extent_op = NULL;
			}

			if (extent_op) {
2455
				spin_unlock(&locked_ref->lock);
2456 2457
				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
2458
				btrfs_free_delayed_extent_op(extent_op);
2459

2460
				if (ret) {
2461 2462 2463 2464 2465 2466 2467 2468
					/*
					 * Need to reset must_insert_reserved if
					 * there was an error so the abort stuff
					 * can cleanup the reserved space
					 * properly.
					 */
					if (must_insert_reserved)
						locked_ref->must_insert_reserved = 1;
2469
					locked_ref->processing = 0;
2470
					btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret);
2471
					btrfs_delayed_ref_unlock(locked_ref);
2472 2473
					return ret;
				}
2474
				continue;
2475
			}
C
Chris Mason 已提交
2476

2477 2478 2479 2480 2481 2482 2483 2484
			/*
			 * Need to drop our head ref lock and re-aqcuire the
			 * delayed ref lock and then re-check to make sure
			 * nobody got added.
			 */
			spin_unlock(&locked_ref->lock);
			spin_lock(&delayed_refs->lock);
			spin_lock(&locked_ref->lock);
2485 2486
			if (rb_first(&locked_ref->ref_root) ||
			    locked_ref->extent_op) {
2487 2488 2489 2490 2491 2492
				spin_unlock(&locked_ref->lock);
				spin_unlock(&delayed_refs->lock);
				continue;
			}
			ref->in_tree = 0;
			delayed_refs->num_heads--;
L
Liu Bo 已提交
2493 2494
			rb_erase(&locked_ref->href_node,
				 &delayed_refs->href_root);
2495 2496
			spin_unlock(&delayed_refs->lock);
		} else {
2497
			actual_count++;
2498 2499
			ref->in_tree = 0;
			rb_erase(&ref->rb_node, &locked_ref->ref_root);
L
Liu Bo 已提交
2500
		}
2501 2502
		atomic_dec(&delayed_refs->num_entries);

2503
		if (!btrfs_delayed_ref_is_head(ref)) {
2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519
			/*
			 * when we play the delayed ref, also correct the
			 * ref_mod on head
			 */
			switch (ref->action) {
			case BTRFS_ADD_DELAYED_REF:
			case BTRFS_ADD_DELAYED_EXTENT:
				locked_ref->node.ref_mod -= ref->ref_mod;
				break;
			case BTRFS_DROP_DELAYED_REF:
				locked_ref->node.ref_mod += ref->ref_mod;
				break;
			default:
				WARN_ON(1);
			}
		}
2520
		spin_unlock(&locked_ref->lock);
2521

2522
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2523
					  must_insert_reserved);
2524

2525
		btrfs_free_delayed_extent_op(extent_op);
2526
		if (ret) {
2527
			locked_ref->processing = 0;
2528 2529
			btrfs_delayed_ref_unlock(locked_ref);
			btrfs_put_delayed_ref(ref);
2530
			btrfs_debug(fs_info, "run_one_delayed_ref returned %d", ret);
2531 2532 2533
			return ret;
		}

2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545
		/*
		 * If this node is a head, that means all the refs in this head
		 * have been dealt with, and we will pick the next head to deal
		 * with, so we must unlock the head and drop it from the cluster
		 * list before we release it.
		 */
		if (btrfs_delayed_ref_is_head(ref)) {
			btrfs_delayed_ref_unlock(locked_ref);
			locked_ref = NULL;
		}
		btrfs_put_delayed_ref(ref);
		count++;
2546 2547
		cond_resched();
	}
2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563

	/*
	 * We don't want to include ref heads since we can have empty ref heads
	 * and those will drastically skew our runtime down since we just do
	 * accounting, no actual extent tree updates.
	 */
	if (actual_count > 0) {
		u64 runtime = ktime_to_ns(ktime_sub(ktime_get(), start));
		u64 avg;

		/*
		 * We weigh the current average higher than our current runtime
		 * to avoid large swings in the average.
		 */
		spin_lock(&delayed_refs->lock);
		avg = fs_info->avg_delayed_ref_runtime * 3 + runtime;
2564
		fs_info->avg_delayed_ref_runtime = avg >> 2;	/* div by 4 */
2565 2566
		spin_unlock(&delayed_refs->lock);
	}
2567
	return 0;
2568 2569
}

2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612
#ifdef SCRAMBLE_DELAYED_REFS
/*
 * Normally delayed refs get processed in ascending bytenr order. This
 * correlates in most cases to the order added. To expose dependencies on this
 * order, we start to process the tree in the middle instead of the beginning
 */
static u64 find_middle(struct rb_root *root)
{
	struct rb_node *n = root->rb_node;
	struct btrfs_delayed_ref_node *entry;
	int alt = 1;
	u64 middle;
	u64 first = 0, last = 0;

	n = rb_first(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		first = entry->bytenr;
	}
	n = rb_last(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		last = entry->bytenr;
	}
	n = root->rb_node;

	while (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		WARN_ON(!entry->in_tree);

		middle = entry->bytenr;

		if (alt)
			n = n->rb_left;
		else
			n = n->rb_right;

		alt = 1 - alt;
	}
	return middle;
}
#endif

2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
static inline u64 heads_to_leaves(struct btrfs_root *root, u64 heads)
{
	u64 num_bytes;

	num_bytes = heads * (sizeof(struct btrfs_extent_item) +
			     sizeof(struct btrfs_extent_inline_ref));
	if (!btrfs_fs_incompat(root->fs_info, SKINNY_METADATA))
		num_bytes += heads * sizeof(struct btrfs_tree_block_info);

	/*
	 * We don't ever fill up leaves all the way so multiply by 2 just to be
	 * closer to what we're really going to want to ouse.
	 */
2626
	return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(root));
2627 2628
}

2629
int btrfs_check_space_for_delayed_refs(struct btrfs_trans_handle *trans,
2630 2631 2632 2633 2634 2635 2636 2637 2638 2639
				       struct btrfs_root *root)
{
	struct btrfs_block_rsv *global_rsv;
	u64 num_heads = trans->transaction->delayed_refs.num_heads_ready;
	u64 num_bytes;
	int ret = 0;

	num_bytes = btrfs_calc_trans_metadata_size(root, 1);
	num_heads = heads_to_leaves(root, num_heads);
	if (num_heads > 1)
2640
		num_bytes += (num_heads - 1) * root->nodesize;
2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657
	num_bytes <<= 1;
	global_rsv = &root->fs_info->global_block_rsv;

	/*
	 * If we can't allocate any more chunks lets make sure we have _lots_ of
	 * wiggle room since running delayed refs can create more delayed refs.
	 */
	if (global_rsv->space_info->full)
		num_bytes <<= 1;

	spin_lock(&global_rsv->lock);
	if (global_rsv->reserved <= num_bytes)
		ret = 1;
	spin_unlock(&global_rsv->lock);
	return ret;
}

2658 2659 2660 2661 2662 2663 2664
int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	u64 num_entries =
		atomic_read(&trans->transaction->delayed_refs.num_entries);
	u64 avg_runtime;
C
Chris Mason 已提交
2665
	u64 val;
2666 2667 2668

	smp_mb();
	avg_runtime = fs_info->avg_delayed_ref_runtime;
C
Chris Mason 已提交
2669
	val = num_entries * avg_runtime;
2670 2671
	if (num_entries * avg_runtime >= NSEC_PER_SEC)
		return 1;
C
Chris Mason 已提交
2672 2673
	if (val >= NSEC_PER_SEC / 2)
		return 2;
2674 2675 2676 2677

	return btrfs_check_space_for_delayed_refs(trans, root);
}

C
Chris Mason 已提交
2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738
struct async_delayed_refs {
	struct btrfs_root *root;
	int count;
	int error;
	int sync;
	struct completion wait;
	struct btrfs_work work;
};

static void delayed_ref_async_start(struct btrfs_work *work)
{
	struct async_delayed_refs *async;
	struct btrfs_trans_handle *trans;
	int ret;

	async = container_of(work, struct async_delayed_refs, work);

	trans = btrfs_join_transaction(async->root);
	if (IS_ERR(trans)) {
		async->error = PTR_ERR(trans);
		goto done;
	}

	/*
	 * trans->sync means that when we call end_transaciton, we won't
	 * wait on delayed refs
	 */
	trans->sync = true;
	ret = btrfs_run_delayed_refs(trans, async->root, async->count);
	if (ret)
		async->error = ret;

	ret = btrfs_end_transaction(trans, async->root);
	if (ret && !async->error)
		async->error = ret;
done:
	if (async->sync)
		complete(&async->wait);
	else
		kfree(async);
}

int btrfs_async_run_delayed_refs(struct btrfs_root *root,
				 unsigned long count, int wait)
{
	struct async_delayed_refs *async;
	int ret;

	async = kmalloc(sizeof(*async), GFP_NOFS);
	if (!async)
		return -ENOMEM;

	async->root = root->fs_info->tree_root;
	async->count = count;
	async->error = 0;
	if (wait)
		async->sync = 1;
	else
		async->sync = 0;
	init_completion(&async->wait);

2739 2740
	btrfs_init_work(&async->work, btrfs_extent_refs_helper,
			delayed_ref_async_start, NULL, NULL);
C
Chris Mason 已提交
2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752

	btrfs_queue_work(root->fs_info->extent_workers, &async->work);

	if (wait) {
		wait_for_completion(&async->wait);
		ret = async->error;
		kfree(async);
		return ret;
	}
	return 0;
}

2753 2754 2755 2756 2757 2758
/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
2759 2760 2761
 *
 * Returns 0 on success or if called with an aborted transaction
 * Returns <0 on error and aborts the transaction
2762 2763 2764 2765 2766 2767
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
L
Liu Bo 已提交
2768
	struct btrfs_delayed_ref_head *head;
2769 2770 2771
	int ret;
	int run_all = count == (unsigned long)-1;

2772 2773 2774 2775
	/* We'll clean this up in btrfs_cleanup_transaction */
	if (trans->aborted)
		return 0;

2776 2777 2778 2779
	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

	delayed_refs = &trans->transaction->delayed_refs;
L
Liu Bo 已提交
2780
	if (count == 0)
2781
		count = atomic_read(&delayed_refs->num_entries) * 2;
2782

2783
again:
2784 2785 2786
#ifdef SCRAMBLE_DELAYED_REFS
	delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
#endif
2787 2788 2789 2790
	ret = __btrfs_run_delayed_refs(trans, root, count);
	if (ret < 0) {
		btrfs_abort_transaction(trans, root, ret);
		return ret;
2791
	}
2792

2793
	if (run_all) {
2794
		if (!list_empty(&trans->new_bgs))
2795 2796
			btrfs_create_pending_block_groups(trans, root);

2797
		spin_lock(&delayed_refs->lock);
L
Liu Bo 已提交
2798
		node = rb_first(&delayed_refs->href_root);
2799 2800
		if (!node) {
			spin_unlock(&delayed_refs->lock);
2801
			goto out;
2802
		}
2803
		count = (unsigned long)-1;
2804

2805
		while (node) {
L
Liu Bo 已提交
2806 2807 2808 2809
			head = rb_entry(node, struct btrfs_delayed_ref_head,
					href_node);
			if (btrfs_delayed_ref_is_head(&head->node)) {
				struct btrfs_delayed_ref_node *ref;
2810

L
Liu Bo 已提交
2811
				ref = &head->node;
2812 2813 2814
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2815 2816 2817 2818
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
2819 2820 2821 2822
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2823
				cond_resched();
2824
				goto again;
L
Liu Bo 已提交
2825 2826
			} else {
				WARN_ON(1);
2827 2828 2829 2830
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
2831
		cond_resched();
2832
		goto again;
2833
	}
2834
out:
J
Josef Bacik 已提交
2835 2836 2837
	ret = btrfs_delayed_qgroup_accounting(trans, root->fs_info);
	if (ret)
		return ret;
2838
	assert_qgroups_uptodate(trans);
2839 2840 2841
	return 0;
}

2842 2843 2844
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
2845
				int level, int is_data)
2846 2847 2848 2849
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

2850
	extent_op = btrfs_alloc_delayed_extent_op();
2851 2852 2853 2854 2855 2856 2857
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;
2858
	extent_op->level = level;
2859

A
Arne Jansen 已提交
2860 2861
	ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
					  num_bytes, extent_op);
2862
	if (ret)
2863
		btrfs_free_delayed_extent_op(extent_op);
2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
2882 2883 2884 2885
	if (!head) {
		spin_unlock(&delayed_refs->lock);
		return 0;
	}
2886 2887 2888 2889 2890

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

2891
		btrfs_release_path(path);
2892

2893 2894 2895 2896
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
2897 2898 2899 2900 2901
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}
2902
	spin_unlock(&delayed_refs->lock);
2903

2904 2905 2906 2907 2908
	spin_lock(&head->lock);
	node = rb_first(&head->ref_root);
	while (node) {
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
		node = rb_next(node);
2909

2910 2911 2912 2913 2914
		/* If it's a shared ref we know a cross reference exists */
		if (ref->type != BTRFS_EXTENT_DATA_REF_KEY) {
			ret = 1;
			break;
		}
2915

2916
		data_ref = btrfs_delayed_node_to_data_ref(ref);
2917

2918 2919 2920 2921 2922 2923 2924 2925 2926 2927
		/*
		 * If our ref doesn't match the one we're currently looking at
		 * then we have a cross reference.
		 */
		if (data_ref->root != root->root_key.objectid ||
		    data_ref->objectid != objectid ||
		    data_ref->offset != offset) {
			ret = 1;
			break;
		}
2928
	}
2929
	spin_unlock(&head->lock);
2930 2931 2932 2933 2934 2935 2936 2937
	mutex_unlock(&head->mutex);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2938 2939
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2940
	struct extent_buffer *leaf;
2941 2942 2943
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2944
	struct btrfs_key key;
2945
	u32 item_size;
2946
	int ret;
2947

2948
	key.objectid = bytenr;
Z
Zheng Yan 已提交
2949
	key.offset = (u64)-1;
2950
	key.type = BTRFS_EXTENT_ITEM_KEY;
2951 2952 2953 2954

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
2955
	BUG_ON(ret == 0); /* Corruption */
Y
Yan Zheng 已提交
2956 2957 2958

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
2959
		goto out;
2960

Z
Zheng Yan 已提交
2961
	path->slots[0]--;
2962
	leaf = path->nodes[0];
2963
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2964

2965
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2966
		goto out;
2967

2968 2969 2970 2971 2972 2973 2974 2975 2976
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2977

2978 2979 2980
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2981

2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
3021
			goto out;
Y
Yan Zheng 已提交
3022

3023 3024 3025 3026 3027 3028 3029
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
3030
	}
3031 3032 3033

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
3034
out:
Y
Yan Zheng 已提交
3035
	btrfs_free_path(path);
3036 3037
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
3038
	return ret;
3039
}
C
Chris Mason 已提交
3040

3041
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
3042
			   struct btrfs_root *root,
3043
			   struct extent_buffer *buf,
3044
			   int full_backref, int inc)
Z
Zheng Yan 已提交
3045 3046
{
	u64 bytenr;
3047 3048
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
3049 3050 3051 3052 3053 3054 3055 3056
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
A
Arne Jansen 已提交
3057
			    u64, u64, u64, u64, u64, u64, int);
Z
Zheng Yan 已提交
3058

3059 3060

	if (btrfs_test_is_dummy_root(root))
3061
		return 0;
3062

Z
Zheng Yan 已提交
3063 3064 3065 3066
	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

3067
	if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state) && level == 0)
3068
		return 0;
Z
Zheng Yan 已提交
3069

3070 3071 3072 3073
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
3074

3075 3076 3077 3078 3079 3080
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
3081
		if (level == 0) {
3082
			btrfs_item_key_to_cpu(buf, &key, i);
3083
			if (key.type != BTRFS_EXTENT_DATA_KEY)
Z
Zheng Yan 已提交
3084
				continue;
3085
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
3086 3087 3088 3089 3090 3091 3092
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
3093 3094 3095 3096 3097

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
3098
					   key.offset, 1);
Z
Zheng Yan 已提交
3099 3100 3101
			if (ret)
				goto fail;
		} else {
3102
			bytenr = btrfs_node_blockptr(buf, i);
3103
			num_bytes = root->nodesize;
3104
			ret = process_func(trans, root, bytenr, num_bytes,
A
Arne Jansen 已提交
3105
					   parent, ref_root, level - 1, 0,
3106
					   1);
Z
Zheng Yan 已提交
3107 3108 3109 3110 3111 3112
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
3113 3114 3115 3116
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3117
		  struct extent_buffer *buf, int full_backref)
3118
{
3119
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
3120 3121 3122
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3123
		  struct extent_buffer *buf, int full_backref)
3124
{
3125
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
Z
Zheng Yan 已提交
3126 3127
}

C
Chris Mason 已提交
3128 3129 3130 3131 3132 3133 3134
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
3135 3136
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
3137 3138

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
3139 3140 3141
	if (ret) {
		if (ret > 0)
			ret = -ENOENT;
3142
		goto fail;
3143
	}
3144 3145 3146 3147 3148

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
3149
	btrfs_release_path(path);
3150
fail:
3151
	if (ret)
3152
		btrfs_abort_transaction(trans, root, ret);
3153
	return ret;
C
Chris Mason 已提交
3154 3155 3156

}

3157 3158 3159 3160 3161
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
3162

3163
	spin_lock(&root->fs_info->block_group_cache_lock);
3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174

	/* If our block group was removed, we need a full search. */
	if (RB_EMPTY_NODE(&cache->cache_node)) {
		const u64 next_bytenr = cache->key.objectid + cache->key.offset;

		spin_unlock(&root->fs_info->block_group_cache_lock);
		btrfs_put_block_group(cache);
		cache = btrfs_lookup_first_block_group(root->fs_info,
						       next_bytenr);
		return cache;
	}
3175 3176 3177 3178 3179
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
3180
		btrfs_get_block_group(cache);
3181 3182 3183 3184 3185 3186
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

3187 3188 3189 3190 3191 3192 3193
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
3194
	int dcs = BTRFS_DC_ERROR;
3195
	u64 num_pages = 0;
3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
3214
		btrfs_release_path(path);
3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

3231 3232 3233 3234 3235 3236 3237
	/* We've already setup this transaction, go ahead and exit */
	if (block_group->cache_generation == trans->transid &&
	    i_size_read(inode)) {
		dcs = BTRFS_DC_SETUP;
		goto out_put;
	}

3238 3239 3240 3241 3242 3243 3244 3245 3246 3247
	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
3248 3249 3250 3251 3252
		ret = btrfs_check_trunc_cache_free_space(root,
					&root->fs_info->global_block_rsv);
		if (ret)
			goto out_put;

3253
		ret = btrfs_truncate_free_space_cache(root, trans, inode);
3254 3255 3256 3257 3258
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
3259
	if (block_group->cached != BTRFS_CACHE_FINISHED ||
3260 3261
	    !btrfs_test_opt(root, SPACE_CACHE) ||
	    block_group->delalloc_bytes) {
3262 3263 3264 3265 3266
		/*
		 * don't bother trying to write stuff out _if_
		 * a) we're not cached,
		 * b) we're with nospace_cache mount option.
		 */
3267
		dcs = BTRFS_DC_WRITTEN;
3268 3269 3270 3271 3272
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

3273 3274 3275 3276 3277 3278
	/*
	 * Try to preallocate enough space based on how big the block group is.
	 * Keep in mind this has to include any pinned space which could end up
	 * taking up quite a bit since it's not folded into the other space
	 * cache.
	 */
3279
	num_pages = div_u64(block_group->key.offset, 256 * 1024 * 1024);
3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292
	if (!num_pages)
		num_pages = 1;

	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
3293 3294
	if (!ret)
		dcs = BTRFS_DC_SETUP;
3295
	btrfs_free_reserved_data_space(inode, num_pages);
3296

3297 3298 3299
out_put:
	iput(inode);
out_free:
3300
	btrfs_release_path(path);
3301 3302
out:
	spin_lock(&block_group->lock);
3303
	if (!ret && dcs == BTRFS_DC_SETUP)
3304
		block_group->cache_generation = trans->transid;
3305
	block_group->disk_cache_state = dcs;
3306 3307 3308 3309 3310
	spin_unlock(&block_group->lock);

	return ret;
}

3311 3312
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
C
Chris Mason 已提交
3313
{
3314
	struct btrfs_block_group_cache *cache;
3315 3316
	struct btrfs_transaction *cur_trans = trans->transaction;
	int ret = 0;
C
Chris Mason 已提交
3317
	struct btrfs_path *path;
3318 3319 3320

	if (list_empty(&cur_trans->dirty_bgs))
		return 0;
C
Chris Mason 已提交
3321 3322 3323 3324 3325

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345
	/*
	 * We don't need the lock here since we are protected by the transaction
	 * commit.  We want to do the cache_save_setup first and then run the
	 * delayed refs to make sure we have the best chance at doing this all
	 * in one shot.
	 */
	while (!list_empty(&cur_trans->dirty_bgs)) {
		cache = list_first_entry(&cur_trans->dirty_bgs,
					 struct btrfs_block_group_cache,
					 dirty_list);
		list_del_init(&cache->dirty_list);
		if (cache->disk_cache_state == BTRFS_DC_CLEAR)
			cache_save_setup(cache, trans, path);
		if (!ret)
			ret = btrfs_run_delayed_refs(trans, root,
						     (unsigned long) -1);
		if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP)
			btrfs_write_out_cache(root, trans, cache, path);
		if (!ret)
			ret = write_one_cache_group(trans, root, path, cache);
J
Josef Bacik 已提交
3346 3347 3348
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
3349
	btrfs_free_path(path);
3350
	return ret;
C
Chris Mason 已提交
3351 3352
}

3353 3354 3355 3356 3357 3358 3359 3360 3361
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
3362
		btrfs_put_block_group(block_group);
3363 3364 3365
	return readonly;
}

3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382
static const char *alloc_name(u64 flags)
{
	switch (flags) {
	case BTRFS_BLOCK_GROUP_METADATA|BTRFS_BLOCK_GROUP_DATA:
		return "mixed";
	case BTRFS_BLOCK_GROUP_METADATA:
		return "metadata";
	case BTRFS_BLOCK_GROUP_DATA:
		return "data";
	case BTRFS_BLOCK_GROUP_SYSTEM:
		return "system";
	default:
		WARN_ON(1);
		return "invalid-combination";
	};
}

3383 3384 3385 3386 3387
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
3388 3389
	int i;
	int factor;
3390
	int ret;
3391 3392 3393 3394 3395 3396

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
3397 3398 3399

	found = __find_space_info(info, flags);
	if (found) {
3400
		spin_lock(&found->lock);
3401
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
3402
		found->disk_total += total_bytes * factor;
3403
		found->bytes_used += bytes_used;
3404
		found->disk_used += bytes_used * factor;
3405
		found->full = 0;
3406
		spin_unlock(&found->lock);
3407 3408 3409
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
3410
	found = kzalloc(sizeof(*found), GFP_NOFS);
3411 3412 3413
	if (!found)
		return -ENOMEM;

3414
	ret = percpu_counter_init(&found->total_bytes_pinned, 0, GFP_KERNEL);
3415 3416 3417 3418 3419
	if (ret) {
		kfree(found);
		return ret;
	}

3420
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
3421
		INIT_LIST_HEAD(&found->block_groups[i]);
3422
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
3423
	spin_lock_init(&found->lock);
3424
	found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3425
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
3426
	found->disk_total = total_bytes * factor;
3427
	found->bytes_used = bytes_used;
3428
	found->disk_used = bytes_used * factor;
3429
	found->bytes_pinned = 0;
3430
	found->bytes_reserved = 0;
Y
Yan Zheng 已提交
3431
	found->bytes_readonly = 0;
3432
	found->bytes_may_use = 0;
3433
	found->full = 0;
3434
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3435
	found->chunk_alloc = 0;
3436 3437
	found->flush = 0;
	init_waitqueue_head(&found->wait);
3438
	INIT_LIST_HEAD(&found->ro_bgs);
3439 3440 3441 3442 3443 3444 3445 3446 3447

	ret = kobject_init_and_add(&found->kobj, &space_info_ktype,
				    info->space_info_kobj, "%s",
				    alloc_name(found->flags));
	if (ret) {
		kfree(found);
		return ret;
	}

3448
	*space_info = found;
3449
	list_add_rcu(&found->list, &info->space_info);
3450 3451
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		info->data_sinfo = found;
3452 3453

	return ret;
3454 3455
}

3456 3457
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
3458 3459
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
3460

3461
	write_seqlock(&fs_info->profiles_lock);
3462 3463 3464 3465 3466 3467
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits |= extra_flags;
3468
	write_sequnlock(&fs_info->profiles_lock);
3469
}
3470

3471 3472 3473
/*
 * returns target flags in extended format or 0 if restripe for this
 * chunk_type is not in progress
3474 3475
 *
 * should be called with either volume_mutex or balance_lock held
3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498
 */
static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
{
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
	u64 target = 0;

	if (!bctl)
		return 0;

	if (flags & BTRFS_BLOCK_GROUP_DATA &&
	    bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
	} else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
		   bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
	} else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
		   bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
	}

	return target;
}

3499 3500 3501
/*
 * @flags: available profiles in extended format (see ctree.h)
 *
3502 3503 3504
 * Returns reduced profile in chunk format.  If profile changing is in
 * progress (either running or paused) picks the target profile (if it's
 * already available), otherwise falls back to plain reducing.
3505
 */
3506
static u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3507
{
3508
	u64 num_devices = root->fs_info->fs_devices->rw_devices;
3509
	u64 target;
D
David Woodhouse 已提交
3510
	u64 tmp;
3511

3512 3513 3514 3515
	/*
	 * see if restripe for this chunk_type is in progress, if so
	 * try to reduce to the target profile
	 */
3516
	spin_lock(&root->fs_info->balance_lock);
3517 3518 3519 3520
	target = get_restripe_target(root->fs_info, flags);
	if (target) {
		/* pick target profile only if it's already available */
		if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
3521
			spin_unlock(&root->fs_info->balance_lock);
3522
			return extended_to_chunk(target);
3523 3524 3525 3526
		}
	}
	spin_unlock(&root->fs_info->balance_lock);

D
David Woodhouse 已提交
3527
	/* First, mask out the RAID levels which aren't possible */
3528
	if (num_devices == 1)
D
David Woodhouse 已提交
3529 3530 3531 3532
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0 |
			   BTRFS_BLOCK_GROUP_RAID5);
	if (num_devices < 3)
		flags &= ~BTRFS_BLOCK_GROUP_RAID6;
3533 3534 3535
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

D
David Woodhouse 已提交
3536 3537 3538 3539
	tmp = flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID0 |
		       BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID5 |
		       BTRFS_BLOCK_GROUP_RAID6 | BTRFS_BLOCK_GROUP_RAID10);
	flags &= ~tmp;
3540

D
David Woodhouse 已提交
3541 3542 3543 3544 3545 3546 3547 3548 3549 3550
	if (tmp & BTRFS_BLOCK_GROUP_RAID6)
		tmp = BTRFS_BLOCK_GROUP_RAID6;
	else if (tmp & BTRFS_BLOCK_GROUP_RAID5)
		tmp = BTRFS_BLOCK_GROUP_RAID5;
	else if (tmp & BTRFS_BLOCK_GROUP_RAID10)
		tmp = BTRFS_BLOCK_GROUP_RAID10;
	else if (tmp & BTRFS_BLOCK_GROUP_RAID1)
		tmp = BTRFS_BLOCK_GROUP_RAID1;
	else if (tmp & BTRFS_BLOCK_GROUP_RAID0)
		tmp = BTRFS_BLOCK_GROUP_RAID0;
3551

D
David Woodhouse 已提交
3552
	return extended_to_chunk(flags | tmp);
3553 3554
}

3555
static u64 get_alloc_profile(struct btrfs_root *root, u64 orig_flags)
J
Josef Bacik 已提交
3556
{
3557
	unsigned seq;
3558
	u64 flags;
3559 3560

	do {
3561
		flags = orig_flags;
3562 3563 3564 3565 3566 3567 3568 3569 3570
		seq = read_seqbegin(&root->fs_info->profiles_lock);

		if (flags & BTRFS_BLOCK_GROUP_DATA)
			flags |= root->fs_info->avail_data_alloc_bits;
		else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
			flags |= root->fs_info->avail_system_alloc_bits;
		else if (flags & BTRFS_BLOCK_GROUP_METADATA)
			flags |= root->fs_info->avail_metadata_alloc_bits;
	} while (read_seqretry(&root->fs_info->profiles_lock, seq));
3571

3572
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
3573 3574
}

3575
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
3576
{
3577
	u64 flags;
D
David Woodhouse 已提交
3578
	u64 ret;
J
Josef Bacik 已提交
3579

3580 3581 3582 3583
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
3584
	else
3585
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
3586

D
David Woodhouse 已提交
3587 3588
	ret = get_alloc_profile(root, flags);
	return ret;
J
Josef Bacik 已提交
3589
}
J
Josef Bacik 已提交
3590

J
Josef Bacik 已提交
3591 3592 3593 3594
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3595
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
3596 3597
{
	struct btrfs_space_info *data_sinfo;
3598
	struct btrfs_root *root = BTRFS_I(inode)->root;
3599
	struct btrfs_fs_info *fs_info = root->fs_info;
3600
	u64 used;
3601
	int ret = 0, committed = 0, alloc_chunk = 1;
J
Josef Bacik 已提交
3602 3603

	/* make sure bytes are sectorsize aligned */
3604
	bytes = ALIGN(bytes, root->sectorsize);
J
Josef Bacik 已提交
3605

3606
	if (btrfs_is_free_space_inode(inode)) {
3607
		committed = 1;
3608
		ASSERT(current->journal_info);
3609 3610
	}

3611
	data_sinfo = fs_info->data_sinfo;
C
Chris Mason 已提交
3612 3613
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
3614

J
Josef Bacik 已提交
3615 3616 3617
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3618 3619 3620
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3621 3622

	if (used + bytes > data_sinfo->total_bytes) {
3623
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3624

J
Josef Bacik 已提交
3625 3626 3627 3628
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3629
		if (!data_sinfo->full && alloc_chunk) {
J
Josef Bacik 已提交
3630
			u64 alloc_target;
J
Josef Bacik 已提交
3631

3632
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
J
Josef Bacik 已提交
3633
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
3634
alloc:
J
Josef Bacik 已提交
3635
			alloc_target = btrfs_get_alloc_profile(root, 1);
3636 3637 3638 3639 3640 3641 3642 3643 3644 3645
			/*
			 * It is ugly that we don't call nolock join
			 * transaction for the free space inode case here.
			 * But it is safe because we only do the data space
			 * reservation for the free space cache in the
			 * transaction context, the common join transaction
			 * just increase the counter of the current transaction
			 * handler, doesn't try to acquire the trans_lock of
			 * the fs.
			 */
3646
			trans = btrfs_join_transaction(root);
3647 3648
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
3649

J
Josef Bacik 已提交
3650
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
3651 3652
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
J
Josef Bacik 已提交
3653
			btrfs_end_transaction(trans, root);
3654 3655 3656 3657 3658 3659
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
J
Josef Bacik 已提交
3660

3661 3662 3663
			if (!data_sinfo)
				data_sinfo = fs_info->data_sinfo;

J
Josef Bacik 已提交
3664 3665
			goto again;
		}
3666 3667

		/*
3668 3669
		 * If we don't have enough pinned space to deal with this
		 * allocation don't bother committing the transaction.
3670
		 */
3671 3672
		if (percpu_counter_compare(&data_sinfo->total_bytes_pinned,
					   bytes) < 0)
3673
			committed = 1;
J
Josef Bacik 已提交
3674 3675
		spin_unlock(&data_sinfo->lock);

3676
		/* commit the current transaction and try again */
3677
commit_trans:
J
Josef Bacik 已提交
3678 3679
		if (!committed &&
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
3680
			committed = 1;
3681

3682
			trans = btrfs_join_transaction(root);
3683 3684
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3685 3686 3687 3688 3689
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
J
Josef Bacik 已提交
3690

3691 3692 3693
		trace_btrfs_space_reservation(root->fs_info,
					      "space_info:enospc",
					      data_sinfo->flags, bytes, 1);
J
Josef Bacik 已提交
3694 3695 3696
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
J
Josef Bacik 已提交
3697
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3698
				      data_sinfo->flags, bytes, 1);
J
Josef Bacik 已提交
3699 3700
	spin_unlock(&data_sinfo->lock);

J
Josef Bacik 已提交
3701 3702
	return 0;
}
J
Josef Bacik 已提交
3703 3704

/*
3705
 * Called if we need to clear a data reservation for this inode.
J
Josef Bacik 已提交
3706
 */
3707
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3708
{
3709
	struct btrfs_root *root = BTRFS_I(inode)->root;
J
Josef Bacik 已提交
3710
	struct btrfs_space_info *data_sinfo;
3711

J
Josef Bacik 已提交
3712
	/* make sure bytes are sectorsize aligned */
3713
	bytes = ALIGN(bytes, root->sectorsize);
3714

3715
	data_sinfo = root->fs_info->data_sinfo;
J
Josef Bacik 已提交
3716
	spin_lock(&data_sinfo->lock);
3717
	WARN_ON(data_sinfo->bytes_may_use < bytes);
J
Josef Bacik 已提交
3718
	data_sinfo->bytes_may_use -= bytes;
J
Josef Bacik 已提交
3719
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3720
				      data_sinfo->flags, bytes, 0);
J
Josef Bacik 已提交
3721
	spin_unlock(&data_sinfo->lock);
3722 3723
}

3724
static void force_metadata_allocation(struct btrfs_fs_info *info)
3725
{
3726 3727
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3728

3729 3730 3731
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3732
			found->force_alloc = CHUNK_ALLOC_FORCE;
3733
	}
3734
	rcu_read_unlock();
3735 3736
}

3737 3738 3739 3740 3741
static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global)
{
	return (global->size << 1);
}

3742
static int should_alloc_chunk(struct btrfs_root *root,
3743
			      struct btrfs_space_info *sinfo, int force)
3744
{
3745
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3746
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3747
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3748
	u64 thresh;
3749

3750 3751 3752
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

3753 3754 3755 3756 3757
	/*
	 * We need to take into account the global rsv because for all intents
	 * and purposes it's used space.  Don't worry about locking the
	 * global_rsv, it doesn't change except when the transaction commits.
	 */
3758
	if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)
3759
		num_allocated += calc_global_rsv_need_space(global_rsv);
3760

3761 3762 3763 3764 3765
	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
3766
		thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3767 3768 3769 3770 3771 3772 3773
		thresh = max_t(u64, 64 * 1024 * 1024,
			       div_factor_fine(thresh, 1));

		if (num_bytes - num_allocated < thresh)
			return 1;
	}

3774
	if (num_allocated + 2 * 1024 * 1024 < div_factor(num_bytes, 8))
3775
		return 0;
3776
	return 1;
3777 3778
}

3779 3780 3781 3782
static u64 get_system_chunk_thresh(struct btrfs_root *root, u64 type)
{
	u64 num_dev;

D
David Woodhouse 已提交
3783 3784 3785 3786
	if (type & (BTRFS_BLOCK_GROUP_RAID10 |
		    BTRFS_BLOCK_GROUP_RAID0 |
		    BTRFS_BLOCK_GROUP_RAID5 |
		    BTRFS_BLOCK_GROUP_RAID6))
3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811
		num_dev = root->fs_info->fs_devices->rw_devices;
	else if (type & BTRFS_BLOCK_GROUP_RAID1)
		num_dev = 2;
	else
		num_dev = 1;	/* DUP or single */

	/* metadata for updaing devices and chunk tree */
	return btrfs_calc_trans_metadata_size(root, num_dev + 1);
}

static void check_system_chunk(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root, u64 type)
{
	struct btrfs_space_info *info;
	u64 left;
	u64 thresh;

	info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	spin_lock(&info->lock);
	left = info->total_bytes - info->bytes_used - info->bytes_pinned -
		info->bytes_reserved - info->bytes_readonly;
	spin_unlock(&info->lock);

	thresh = get_system_chunk_thresh(root, type);
	if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
3812 3813
		btrfs_info(root->fs_info, "left=%llu, need=%llu, flags=%llu",
			left, thresh, type);
3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824
		dump_space_info(info, 0, 0);
	}

	if (left < thresh) {
		u64 flags;

		flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
		btrfs_alloc_chunk(trans, root, flags);
	}
}

3825
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
3826
			  struct btrfs_root *extent_root, u64 flags, int force)
J
Josef Bacik 已提交
3827
{
3828
	struct btrfs_space_info *space_info;
3829
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
3830
	int wait_for_alloc = 0;
J
Josef Bacik 已提交
3831 3832
	int ret = 0;

3833 3834 3835 3836
	/* Don't re-enter if we're already allocating a chunk */
	if (trans->allocating_chunk)
		return -ENOSPC;

3837
	space_info = __find_space_info(extent_root->fs_info, flags);
3838 3839 3840
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
3841
		BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
3842
	}
3843
	BUG_ON(!space_info); /* Logic error */
J
Josef Bacik 已提交
3844

3845
again:
3846
	spin_lock(&space_info->lock);
3847
	if (force < space_info->force_alloc)
3848
		force = space_info->force_alloc;
3849
	if (space_info->full) {
3850 3851 3852 3853
		if (should_alloc_chunk(extent_root, space_info, force))
			ret = -ENOSPC;
		else
			ret = 0;
3854
		spin_unlock(&space_info->lock);
3855
		return ret;
J
Josef Bacik 已提交
3856 3857
	}

3858
	if (!should_alloc_chunk(extent_root, space_info, force)) {
3859
		spin_unlock(&space_info->lock);
3860 3861 3862 3863 3864
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
J
Josef Bacik 已提交
3865
	}
3866

3867
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3868

3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

3883 3884
	trans->allocating_chunk = true;

3885 3886 3887 3888 3889 3890 3891
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3892 3893 3894 3895 3896
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
3897
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3898 3899 3900 3901
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
3902 3903
	}

3904 3905 3906 3907 3908 3909
	/*
	 * Check if we have enough space in SYSTEM chunk because we may need
	 * to update devices.
	 */
	check_system_chunk(trans, extent_root, flags);

Y
Yan Zheng 已提交
3910
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
3911
	trans->allocating_chunk = false;
3912

J
Josef Bacik 已提交
3913
	spin_lock(&space_info->lock);
3914 3915
	if (ret < 0 && ret != -ENOSPC)
		goto out;
J
Josef Bacik 已提交
3916
	if (ret)
3917
		space_info->full = 1;
3918 3919
	else
		ret = 1;
3920

3921
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3922
out:
3923
	space_info->chunk_alloc = 0;
J
Josef Bacik 已提交
3924
	spin_unlock(&space_info->lock);
3925
	mutex_unlock(&fs_info->chunk_mutex);
J
Josef Bacik 已提交
3926
	return ret;
3927
}
J
Josef Bacik 已提交
3928

J
Josef Bacik 已提交
3929 3930
static int can_overcommit(struct btrfs_root *root,
			  struct btrfs_space_info *space_info, u64 bytes,
M
Miao Xie 已提交
3931
			  enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
3932
{
3933
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
J
Josef Bacik 已提交
3934
	u64 profile = btrfs_get_alloc_profile(root, 0);
3935
	u64 space_size;
J
Josef Bacik 已提交
3936 3937 3938 3939
	u64 avail;
	u64 used;

	used = space_info->bytes_used + space_info->bytes_reserved +
3940 3941 3942 3943 3944 3945 3946 3947
		space_info->bytes_pinned + space_info->bytes_readonly;

	/*
	 * We only want to allow over committing if we have lots of actual space
	 * free, but if we don't have enough space to handle the global reserve
	 * space then we could end up having a real enospc problem when trying
	 * to allocate a chunk or some other such important allocation.
	 */
3948 3949 3950 3951
	spin_lock(&global_rsv->lock);
	space_size = calc_global_rsv_need_space(global_rsv);
	spin_unlock(&global_rsv->lock);
	if (used + space_size >= space_info->total_bytes)
3952 3953 3954
		return 0;

	used += space_info->bytes_may_use;
J
Josef Bacik 已提交
3955 3956 3957 3958 3959 3960 3961

	spin_lock(&root->fs_info->free_chunk_lock);
	avail = root->fs_info->free_chunk_space;
	spin_unlock(&root->fs_info->free_chunk_lock);

	/*
	 * If we have dup, raid1 or raid10 then only half of the free
D
David Woodhouse 已提交
3962 3963 3964
	 * space is actually useable.  For raid56, the space info used
	 * doesn't include the parity drive, so we don't have to
	 * change the math
J
Josef Bacik 已提交
3965 3966 3967 3968 3969 3970 3971
	 */
	if (profile & (BTRFS_BLOCK_GROUP_DUP |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_RAID10))
		avail >>= 1;

	/*
3972 3973 3974
	 * If we aren't flushing all things, let us overcommit up to
	 * 1/2th of the space. If we can flush, don't let us overcommit
	 * too much, let it overcommit up to 1/8 of the space.
J
Josef Bacik 已提交
3975
	 */
M
Miao Xie 已提交
3976
	if (flush == BTRFS_RESERVE_FLUSH_ALL)
3977
		avail >>= 3;
J
Josef Bacik 已提交
3978
	else
3979
		avail >>= 1;
J
Josef Bacik 已提交
3980

3981
	if (used + bytes < space_info->total_bytes + avail)
J
Josef Bacik 已提交
3982 3983 3984 3985
		return 1;
	return 0;
}

3986
static void btrfs_writeback_inodes_sb_nr(struct btrfs_root *root,
3987
					 unsigned long nr_pages, int nr_items)
3988 3989 3990
{
	struct super_block *sb = root->fs_info->sb;

3991 3992 3993 3994
	if (down_read_trylock(&sb->s_umount)) {
		writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE);
		up_read(&sb->s_umount);
	} else {
3995 3996 3997 3998 3999 4000 4001
		/*
		 * We needn't worry the filesystem going from r/w to r/o though
		 * we don't acquire ->s_umount mutex, because the filesystem
		 * should guarantee the delalloc inodes list be empty after
		 * the filesystem is readonly(all dirty pages are written to
		 * the disk).
		 */
4002
		btrfs_start_delalloc_roots(root->fs_info, 0, nr_items);
4003
		if (!current->journal_info)
4004
			btrfs_wait_ordered_roots(root->fs_info, nr_items);
4005 4006 4007
	}
}

4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019
static inline int calc_reclaim_items_nr(struct btrfs_root *root, u64 to_reclaim)
{
	u64 bytes;
	int nr;

	bytes = btrfs_calc_trans_metadata_size(root, 1);
	nr = (int)div64_u64(to_reclaim, bytes);
	if (!nr)
		nr = 1;
	return nr;
}

4020 4021
#define EXTENT_SIZE_PER_ITEM	(256 * 1024)

J
Josef Bacik 已提交
4022
/*
4023
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
4024
 */
J
Josef Bacik 已提交
4025 4026
static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig,
			    bool wait_ordered)
4027
{
4028
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
4029
	struct btrfs_space_info *space_info;
4030
	struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
4031
	u64 delalloc_bytes;
4032
	u64 max_reclaim;
4033
	long time_left;
4034 4035
	unsigned long nr_pages;
	int loops;
4036
	int items;
M
Miao Xie 已提交
4037
	enum btrfs_reserve_flush_enum flush;
4038

4039
	/* Calc the number of the pages we need flush for space reservation */
4040 4041
	items = calc_reclaim_items_nr(root, to_reclaim);
	to_reclaim = items * EXTENT_SIZE_PER_ITEM;
4042

4043
	trans = (struct btrfs_trans_handle *)current->journal_info;
4044
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
4045
	space_info = block_rsv->space_info;
4046

4047 4048
	delalloc_bytes = percpu_counter_sum_positive(
						&root->fs_info->delalloc_bytes);
J
Josef Bacik 已提交
4049
	if (delalloc_bytes == 0) {
4050
		if (trans)
J
Josef Bacik 已提交
4051
			return;
4052
		if (wait_ordered)
4053
			btrfs_wait_ordered_roots(root->fs_info, items);
J
Josef Bacik 已提交
4054
		return;
4055 4056
	}

4057
	loops = 0;
J
Josef Bacik 已提交
4058 4059 4060
	while (delalloc_bytes && loops < 3) {
		max_reclaim = min(delalloc_bytes, to_reclaim);
		nr_pages = max_reclaim >> PAGE_CACHE_SHIFT;
4061
		btrfs_writeback_inodes_sb_nr(root, nr_pages, items);
4062 4063 4064 4065
		/*
		 * We need to wait for the async pages to actually start before
		 * we do anything.
		 */
4066 4067 4068 4069 4070 4071 4072 4073
		max_reclaim = atomic_read(&root->fs_info->async_delalloc_pages);
		if (!max_reclaim)
			goto skip_async;

		if (max_reclaim <= nr_pages)
			max_reclaim = 0;
		else
			max_reclaim -= nr_pages;
4074

4075 4076 4077 4078
		wait_event(root->fs_info->async_submit_wait,
			   atomic_read(&root->fs_info->async_delalloc_pages) <=
			   (int)max_reclaim);
skip_async:
M
Miao Xie 已提交
4079 4080 4081 4082
		if (!trans)
			flush = BTRFS_RESERVE_FLUSH_ALL;
		else
			flush = BTRFS_RESERVE_NO_FLUSH;
J
Josef Bacik 已提交
4083
		spin_lock(&space_info->lock);
M
Miao Xie 已提交
4084
		if (can_overcommit(root, space_info, orig, flush)) {
J
Josef Bacik 已提交
4085 4086 4087
			spin_unlock(&space_info->lock);
			break;
		}
J
Josef Bacik 已提交
4088
		spin_unlock(&space_info->lock);
4089

4090
		loops++;
4091
		if (wait_ordered && !trans) {
4092
			btrfs_wait_ordered_roots(root->fs_info, items);
4093
		} else {
J
Josef Bacik 已提交
4094
			time_left = schedule_timeout_killable(1);
4095 4096 4097
			if (time_left)
				break;
		}
4098 4099
		delalloc_bytes = percpu_counter_sum_positive(
						&root->fs_info->delalloc_bytes);
4100 4101 4102
	}
}

4103 4104 4105 4106 4107
/**
 * maybe_commit_transaction - possibly commit the transaction if its ok to
 * @root - the root we're allocating for
 * @bytes - the number of bytes we want to reserve
 * @force - force the commit
4108
 *
4109 4110 4111
 * This will check to make sure that committing the transaction will actually
 * get us somewhere and then commit the transaction if it does.  Otherwise it
 * will return -ENOSPC.
4112
 */
4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127
static int may_commit_transaction(struct btrfs_root *root,
				  struct btrfs_space_info *space_info,
				  u64 bytes, int force)
{
	struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
	struct btrfs_trans_handle *trans;

	trans = (struct btrfs_trans_handle *)current->journal_info;
	if (trans)
		return -EAGAIN;

	if (force)
		goto commit;

	/* See if there is enough pinned space to make this reservation */
4128
	if (percpu_counter_compare(&space_info->total_bytes_pinned,
4129
				   bytes) >= 0)
4130 4131 4132 4133 4134 4135 4136 4137 4138 4139
		goto commit;

	/*
	 * See if there is some space in the delayed insertion reservation for
	 * this reservation.
	 */
	if (space_info != delayed_rsv->space_info)
		return -ENOSPC;

	spin_lock(&delayed_rsv->lock);
4140 4141
	if (percpu_counter_compare(&space_info->total_bytes_pinned,
				   bytes - delayed_rsv->size) >= 0) {
4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154
		spin_unlock(&delayed_rsv->lock);
		return -ENOSPC;
	}
	spin_unlock(&delayed_rsv->lock);

commit:
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans))
		return -ENOSPC;

	return btrfs_commit_transaction(trans, root);
}

4155
enum flush_state {
4156 4157 4158 4159
	FLUSH_DELAYED_ITEMS_NR	=	1,
	FLUSH_DELAYED_ITEMS	=	2,
	FLUSH_DELALLOC		=	3,
	FLUSH_DELALLOC_WAIT	=	4,
4160 4161
	ALLOC_CHUNK		=	5,
	COMMIT_TRANS		=	6,
4162 4163 4164 4165 4166 4167 4168 4169
};

static int flush_space(struct btrfs_root *root,
		       struct btrfs_space_info *space_info, u64 num_bytes,
		       u64 orig_bytes, int state)
{
	struct btrfs_trans_handle *trans;
	int nr;
J
Josef Bacik 已提交
4170
	int ret = 0;
4171 4172 4173 4174

	switch (state) {
	case FLUSH_DELAYED_ITEMS_NR:
	case FLUSH_DELAYED_ITEMS:
4175 4176 4177
		if (state == FLUSH_DELAYED_ITEMS_NR)
			nr = calc_reclaim_items_nr(root, num_bytes) * 2;
		else
4178
			nr = -1;
4179

4180 4181 4182 4183 4184 4185 4186 4187
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = btrfs_run_delayed_items_nr(trans, root, nr);
		btrfs_end_transaction(trans, root);
		break;
4188 4189
	case FLUSH_DELALLOC:
	case FLUSH_DELALLOC_WAIT:
4190
		shrink_delalloc(root, num_bytes * 2, orig_bytes,
4191 4192
				state == FLUSH_DELALLOC_WAIT);
		break;
4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205
	case ALLOC_CHUNK:
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
				     btrfs_get_alloc_profile(root, 0),
				     CHUNK_ALLOC_NO_FORCE);
		btrfs_end_transaction(trans, root);
		if (ret == -ENOSPC)
			ret = 0;
		break;
4206 4207 4208 4209 4210 4211 4212 4213 4214 4215
	case COMMIT_TRANS:
		ret = may_commit_transaction(root, space_info, orig_bytes, 0);
		break;
	default:
		ret = -ENOSPC;
		break;
	}

	return ret;
}
4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263

static inline u64
btrfs_calc_reclaim_metadata_size(struct btrfs_root *root,
				 struct btrfs_space_info *space_info)
{
	u64 used;
	u64 expected;
	u64 to_reclaim;

	to_reclaim = min_t(u64, num_online_cpus() * 1024 * 1024,
				16 * 1024 * 1024);
	spin_lock(&space_info->lock);
	if (can_overcommit(root, space_info, to_reclaim,
			   BTRFS_RESERVE_FLUSH_ALL)) {
		to_reclaim = 0;
		goto out;
	}

	used = space_info->bytes_used + space_info->bytes_reserved +
	       space_info->bytes_pinned + space_info->bytes_readonly +
	       space_info->bytes_may_use;
	if (can_overcommit(root, space_info, 1024 * 1024,
			   BTRFS_RESERVE_FLUSH_ALL))
		expected = div_factor_fine(space_info->total_bytes, 95);
	else
		expected = div_factor_fine(space_info->total_bytes, 90);

	if (used > expected)
		to_reclaim = used - expected;
	else
		to_reclaim = 0;
	to_reclaim = min(to_reclaim, space_info->bytes_may_use +
				     space_info->bytes_reserved);
out:
	spin_unlock(&space_info->lock);

	return to_reclaim;
}

static inline int need_do_async_reclaim(struct btrfs_space_info *space_info,
					struct btrfs_fs_info *fs_info, u64 used)
{
	return (used >= div_factor_fine(space_info->total_bytes, 98) &&
		!btrfs_fs_closing(fs_info) &&
		!test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state));
}

static int btrfs_need_do_async_reclaim(struct btrfs_space_info *space_info,
4264 4265
				       struct btrfs_fs_info *fs_info,
				       int flush_state)
4266 4267 4268 4269
{
	u64 used;

	spin_lock(&space_info->lock);
4270 4271 4272 4273 4274 4275 4276 4277 4278
	/*
	 * We run out of space and have not got any free space via flush_space,
	 * so don't bother doing async reclaim.
	 */
	if (flush_state > COMMIT_TRANS && space_info->full) {
		spin_unlock(&space_info->lock);
		return 0;
	}

4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310
	used = space_info->bytes_used + space_info->bytes_reserved +
	       space_info->bytes_pinned + space_info->bytes_readonly +
	       space_info->bytes_may_use;
	if (need_do_async_reclaim(space_info, fs_info, used)) {
		spin_unlock(&space_info->lock);
		return 1;
	}
	spin_unlock(&space_info->lock);

	return 0;
}

static void btrfs_async_reclaim_metadata_space(struct work_struct *work)
{
	struct btrfs_fs_info *fs_info;
	struct btrfs_space_info *space_info;
	u64 to_reclaim;
	int flush_state;

	fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work);
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);

	to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info->fs_root,
						      space_info);
	if (!to_reclaim)
		return;

	flush_state = FLUSH_DELAYED_ITEMS_NR;
	do {
		flush_space(fs_info->fs_root, space_info, to_reclaim,
			    to_reclaim, flush_state);
		flush_state++;
4311 4312
		if (!btrfs_need_do_async_reclaim(space_info, fs_info,
						 flush_state))
4313 4314 4315
			return;
	} while (flush_state <= COMMIT_TRANS);

4316
	if (btrfs_need_do_async_reclaim(space_info, fs_info, flush_state))
4317 4318 4319 4320 4321 4322 4323 4324
		queue_work(system_unbound_wq, work);
}

void btrfs_init_async_reclaim_work(struct work_struct *work)
{
	INIT_WORK(work, btrfs_async_reclaim_metadata_space);
}

4325 4326 4327 4328 4329
/**
 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
 * @root - the root we're allocating for
 * @block_rsv - the block_rsv we're allocating for
 * @orig_bytes - the number of bytes we want
4330
 * @flush - whether or not we can flush to make our reservation
4331
 *
4332 4333 4334 4335 4336 4337
 * This will reserve orgi_bytes number of bytes from the space info associated
 * with the block_rsv.  If there is not enough space it will make an attempt to
 * flush out space to make room.  It will do this by flushing delalloc if
 * possible or committing the transaction.  If flush is 0 then no attempts to
 * regain reservations will be made and this will fail if there is not enough
 * space already.
4338
 */
4339
static int reserve_metadata_bytes(struct btrfs_root *root,
4340
				  struct btrfs_block_rsv *block_rsv,
M
Miao Xie 已提交
4341 4342
				  u64 orig_bytes,
				  enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
4343
{
4344
	struct btrfs_space_info *space_info = block_rsv->space_info;
4345
	u64 used;
4346
	u64 num_bytes = orig_bytes;
4347
	int flush_state = FLUSH_DELAYED_ITEMS_NR;
4348
	int ret = 0;
4349
	bool flushing = false;
J
Josef Bacik 已提交
4350

4351
again:
4352
	ret = 0;
4353
	spin_lock(&space_info->lock);
4354
	/*
M
Miao Xie 已提交
4355 4356
	 * We only want to wait if somebody other than us is flushing and we
	 * are actually allowed to flush all things.
4357
	 */
M
Miao Xie 已提交
4358 4359
	while (flush == BTRFS_RESERVE_FLUSH_ALL && !flushing &&
	       space_info->flush) {
4360 4361 4362 4363 4364 4365 4366
		spin_unlock(&space_info->lock);
		/*
		 * If we have a trans handle we can't wait because the flusher
		 * may have to commit the transaction, which would mean we would
		 * deadlock since we are waiting for the flusher to finish, but
		 * hold the current transaction open.
		 */
4367
		if (current->journal_info)
4368
			return -EAGAIN;
A
Arne Jansen 已提交
4369 4370 4371
		ret = wait_event_killable(space_info->wait, !space_info->flush);
		/* Must have been killed, return */
		if (ret)
4372 4373 4374 4375 4376 4377
			return -EINTR;

		spin_lock(&space_info->lock);
	}

	ret = -ENOSPC;
4378 4379 4380
	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;
J
Josef Bacik 已提交
4381

4382 4383 4384 4385 4386 4387 4388
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
4389 4390
	if (used <= space_info->total_bytes) {
		if (used + orig_bytes <= space_info->total_bytes) {
4391
			space_info->bytes_may_use += orig_bytes;
J
Josef Bacik 已提交
4392
			trace_btrfs_space_reservation(root->fs_info,
4393
				"space_info", space_info->flags, orig_bytes, 1);
4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
4409
		num_bytes = used - space_info->total_bytes +
4410
			(orig_bytes * 2);
4411
	}
J
Josef Bacik 已提交
4412

4413 4414 4415 4416 4417 4418
	if (ret && can_overcommit(root, space_info, orig_bytes, flush)) {
		space_info->bytes_may_use += orig_bytes;
		trace_btrfs_space_reservation(root->fs_info, "space_info",
					      space_info->flags, orig_bytes,
					      1);
		ret = 0;
4419 4420
	}

4421 4422 4423 4424
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
M
Miao Xie 已提交
4425 4426 4427
	 *
	 * We make the other tasks wait for the flush only when we can flush
	 * all things.
4428
	 */
4429
	if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
4430 4431
		flushing = true;
		space_info->flush = 1;
4432 4433
	} else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) {
		used += orig_bytes;
4434 4435 4436 4437 4438 4439 4440
		/*
		 * We will do the space reservation dance during log replay,
		 * which means we won't have fs_info->fs_root set, so don't do
		 * the async reclaim as we will panic.
		 */
		if (!root->fs_info->log_root_recovering &&
		    need_do_async_reclaim(space_info, root->fs_info, used) &&
4441 4442 4443
		    !work_busy(&root->fs_info->async_reclaim_work))
			queue_work(system_unbound_wq,
				   &root->fs_info->async_reclaim_work);
4444
	}
4445
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
4446

M
Miao Xie 已提交
4447
	if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
4448
		goto out;
4449

4450 4451 4452
	ret = flush_space(root, space_info, num_bytes, orig_bytes,
			  flush_state);
	flush_state++;
M
Miao Xie 已提交
4453 4454 4455 4456 4457 4458 4459 4460 4461 4462

	/*
	 * If we are FLUSH_LIMIT, we can not flush delalloc, or the deadlock
	 * would happen. So skip delalloc flush.
	 */
	if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
	    (flush_state == FLUSH_DELALLOC ||
	     flush_state == FLUSH_DELALLOC_WAIT))
		flush_state = ALLOC_CHUNK;

4463
	if (!ret)
4464
		goto again;
M
Miao Xie 已提交
4465 4466 4467 4468 4469
	else if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
		 flush_state < COMMIT_TRANS)
		goto again;
	else if (flush == BTRFS_RESERVE_FLUSH_ALL &&
		 flush_state <= COMMIT_TRANS)
4470 4471 4472
		goto again;

out:
4473 4474 4475 4476 4477 4478 4479 4480 4481
	if (ret == -ENOSPC &&
	    unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) {
		struct btrfs_block_rsv *global_rsv =
			&root->fs_info->global_block_rsv;

		if (block_rsv != global_rsv &&
		    !block_rsv_use_bytes(global_rsv, orig_bytes))
			ret = 0;
	}
4482 4483 4484 4485
	if (ret == -ENOSPC)
		trace_btrfs_space_reservation(root->fs_info,
					      "space_info:enospc",
					      space_info->flags, orig_bytes, 1);
4486
	if (flushing) {
4487
		spin_lock(&space_info->lock);
4488 4489
		space_info->flush = 0;
		wake_up_all(&space_info->wait);
4490
		spin_unlock(&space_info->lock);
4491 4492 4493 4494
	}
	return ret;
}

4495 4496 4497
static struct btrfs_block_rsv *get_block_rsv(
					const struct btrfs_trans_handle *trans,
					const struct btrfs_root *root)
4498
{
4499 4500
	struct btrfs_block_rsv *block_rsv = NULL;

4501
	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4502 4503 4504
		block_rsv = trans->block_rsv;

	if (root == root->fs_info->csum_root && trans->adding_csums)
4505
		block_rsv = trans->block_rsv;
4506

4507 4508 4509
	if (root == root->fs_info->uuid_root)
		block_rsv = trans->block_rsv;

4510
	if (!block_rsv)
4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570
int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info,
			     struct btrfs_block_rsv *dest, u64 num_bytes,
			     int min_factor)
{
	struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
	u64 min_bytes;

	if (global_rsv->space_info != dest->space_info)
		return -ENOSPC;

	spin_lock(&global_rsv->lock);
	min_bytes = div_factor(global_rsv->size, min_factor);
	if (global_rsv->reserved < min_bytes + num_bytes) {
		spin_unlock(&global_rsv->lock);
		return -ENOSPC;
	}
	global_rsv->reserved -= num_bytes;
	if (global_rsv->reserved < global_rsv->size)
		global_rsv->full = 0;
	spin_unlock(&global_rsv->lock);

	block_rsv_add_bytes(dest, num_bytes, 1);
	return 0;
}

J
Josef Bacik 已提交
4571 4572
static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
				    struct btrfs_block_rsv *block_rsv,
4573
				    struct btrfs_block_rsv *dest, u64 num_bytes)
4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
4606
			spin_lock(&space_info->lock);
4607
			space_info->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4608
			trace_btrfs_space_reservation(fs_info, "space_info",
4609
					space_info->flags, num_bytes, 0);
4610
			spin_unlock(&space_info->lock);
4611
		}
J
Josef Bacik 已提交
4612
	}
4613
}
4614

4615 4616 4617 4618
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
J
Josef Bacik 已提交
4619

4620 4621 4622
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
4623

4624
	block_rsv_add_bytes(dst, num_bytes, 1);
J
Josef Bacik 已提交
4625 4626 4627
	return 0;
}

4628
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
J
Josef Bacik 已提交
4629
{
4630 4631
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
4632
	rsv->type = type;
4633 4634
}

4635 4636
struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root,
					      unsigned short type)
4637 4638 4639
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
4640

4641 4642 4643
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
4644

4645
	btrfs_init_block_rsv(block_rsv, type);
4646 4647 4648 4649
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
4650

4651 4652 4653
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
J
Josef Bacik 已提交
4654 4655
	if (!rsv)
		return;
4656 4657
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	kfree(rsv);
J
Josef Bacik 已提交
4658 4659
}

M
Miao Xie 已提交
4660 4661 4662
int btrfs_block_rsv_add(struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv, u64 num_bytes,
			enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
4663
{
4664
	int ret;
J
Josef Bacik 已提交
4665

4666 4667
	if (num_bytes == 0)
		return 0;
4668

4669
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4670 4671 4672 4673
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
4674

4675 4676
	return ret;
}
J
Josef Bacik 已提交
4677

4678
int btrfs_block_rsv_check(struct btrfs_root *root,
4679
			  struct btrfs_block_rsv *block_rsv, int min_factor)
4680 4681 4682
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
4683

4684 4685
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
4686

4687
	spin_lock(&block_rsv->lock);
4688 4689 4690 4691
	num_bytes = div_factor(block_rsv->size, min_factor);
	if (block_rsv->reserved >= num_bytes)
		ret = 0;
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
4692

4693 4694 4695
	return ret;
}

M
Miao Xie 已提交
4696 4697 4698
int btrfs_block_rsv_refill(struct btrfs_root *root,
			   struct btrfs_block_rsv *block_rsv, u64 min_reserved,
			   enum btrfs_reserve_flush_enum flush)
4699 4700 4701 4702 4703 4704 4705 4706 4707
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;

	if (!block_rsv)
		return 0;

	spin_lock(&block_rsv->lock);
	num_bytes = min_reserved;
4708
	if (block_rsv->reserved >= num_bytes)
4709
		ret = 0;
4710
	else
4711 4712
		num_bytes -= block_rsv->reserved;
	spin_unlock(&block_rsv->lock);
4713

4714 4715 4716
	if (!ret)
		return 0;

4717
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4718 4719
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 0);
4720
		return 0;
J
Josef Bacik 已提交
4721
	}
J
Josef Bacik 已提交
4722

4723
	return ret;
4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737
}

int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
4738
	if (global_rsv == block_rsv ||
4739 4740
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
J
Josef Bacik 已提交
4741 4742
	block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
				num_bytes);
J
Josef Bacik 已提交
4743 4744 4745
}

/*
4746 4747 4748
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
4749
 */
4750
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4751
{
4752 4753 4754 4755
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
4756
	int csum_size = btrfs_super_csum_size(fs_info->super_copy);
J
Josef Bacik 已提交
4757

4758 4759 4760 4761
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
4762

4763 4764
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
4765 4766
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
4767 4768
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
4769

4770 4771
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
4772
	num_bytes += div_u64(data_used + meta_used, 50);
4773

4774
	if (num_bytes * 3 > meta_used)
4775
		num_bytes = div_u64(meta_used, 3);
4776

4777
	return ALIGN(num_bytes, fs_info->extent_root->nodesize << 10);
4778
}
J
Josef Bacik 已提交
4779

4780 4781 4782 4783 4784
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
4785

4786
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
4787

4788
	spin_lock(&sinfo->lock);
4789
	spin_lock(&block_rsv->lock);
4790

4791
	block_rsv->size = min_t(u64, num_bytes, 512 * 1024 * 1024);
4792

4793
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
4794 4795
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
4796 4797 4798 4799

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
4800
		sinfo->bytes_may_use += num_bytes;
J
Josef Bacik 已提交
4801
		trace_btrfs_space_reservation(fs_info, "space_info",
4802
				      sinfo->flags, num_bytes, 1);
J
Josef Bacik 已提交
4803 4804
	}

4805 4806
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
4807
		sinfo->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4808
		trace_btrfs_space_reservation(fs_info, "space_info",
4809
				      sinfo->flags, num_bytes, 0);
4810 4811 4812
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
4813

4814
	spin_unlock(&block_rsv->lock);
4815
	spin_unlock(&sinfo->lock);
J
Josef Bacik 已提交
4816 4817
}

4818
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4819
{
4820
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
4821

4822 4823
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
J
Josef Bacik 已提交
4824

4825
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4826 4827
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->delalloc_block_rsv.space_info = space_info;
4828 4829
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
4830
	fs_info->delayed_block_rsv.space_info = space_info;
4831

4832 4833 4834 4835
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
4836 4837
	if (fs_info->quota_root)
		fs_info->quota_root->block_rsv = &fs_info->global_block_rsv;
4838
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
4839 4840

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
4841 4842
}

4843
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4844
{
J
Josef Bacik 已提交
4845 4846
	block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
				(u64)-1);
4847 4848 4849 4850 4851 4852
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
4853 4854
	WARN_ON(fs_info->delayed_block_rsv.size > 0);
	WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
4855 4856
}

4857 4858
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
4859
{
4860 4861 4862
	if (!trans->block_rsv)
		return;

4863 4864
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
4865

4866
	trace_btrfs_space_reservation(root->fs_info, "transaction",
4867
				      trans->transid, trans->bytes_reserved, 0);
4868
	btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
4869 4870
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
4871

4872
/* Can only return 0 or -ENOSPC */
4873 4874 4875 4876 4877 4878 4879 4880
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
4881 4882 4883
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
4884
	 */
C
Chris Mason 已提交
4885
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4886 4887
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 1);
4888
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
J
Josef Bacik 已提交
4889 4890
}

4891
void btrfs_orphan_release_metadata(struct inode *inode)
4892
{
4893
	struct btrfs_root *root = BTRFS_I(inode)->root;
C
Chris Mason 已提交
4894
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4895 4896
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 0);
4897 4898
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
4899

4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916
/*
 * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation
 * root: the root of the parent directory
 * rsv: block reservation
 * items: the number of items that we need do reservation
 * qgroup_reserved: used to return the reserved size in qgroup
 *
 * This function is used to reserve the space for snapshot/subvolume
 * creation and deletion. Those operations are different with the
 * common file/directory operations, they change two fs/file trees
 * and root tree, the number of items that the qgroup reserves is
 * different with the free space reservation. So we can not use
 * the space reseravtion mechanism in start_transaction().
 */
int btrfs_subvolume_reserve_metadata(struct btrfs_root *root,
				     struct btrfs_block_rsv *rsv,
				     int items,
4917 4918
				     u64 *qgroup_reserved,
				     bool use_global_rsv)
4919
{
4920 4921
	u64 num_bytes;
	int ret;
4922
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
4923 4924 4925

	if (root->fs_info->quota_enabled) {
		/* One for parent inode, two for dir entries */
4926
		num_bytes = 3 * root->nodesize;
4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940
		ret = btrfs_qgroup_reserve(root, num_bytes);
		if (ret)
			return ret;
	} else {
		num_bytes = 0;
	}

	*qgroup_reserved = num_bytes;

	num_bytes = btrfs_calc_trans_metadata_size(root, items);
	rsv->space_info = __find_space_info(root->fs_info,
					    BTRFS_BLOCK_GROUP_METADATA);
	ret = btrfs_block_rsv_add(root, rsv, num_bytes,
				  BTRFS_RESERVE_FLUSH_ALL);
4941 4942 4943 4944

	if (ret == -ENOSPC && use_global_rsv)
		ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes);

4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959
	if (ret) {
		if (*qgroup_reserved)
			btrfs_qgroup_free(root, *qgroup_reserved);
	}

	return ret;
}

void btrfs_subvolume_release_metadata(struct btrfs_root *root,
				      struct btrfs_block_rsv *rsv,
				      u64 qgroup_reserved)
{
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	if (qgroup_reserved)
		btrfs_qgroup_free(root, qgroup_reserved);
4960 4961
}

4962 4963 4964
/**
 * drop_outstanding_extent - drop an outstanding extent
 * @inode: the inode we're dropping the extent for
4965
 * @num_bytes: the number of bytes we're relaseing.
4966 4967 4968 4969 4970 4971
 *
 * This is called when we are freeing up an outstanding extent, either called
 * after an error or after an extent is written.  This will return the number of
 * reserved extents that need to be freed.  This must be called with
 * BTRFS_I(inode)->lock held.
 */
4972
static unsigned drop_outstanding_extent(struct inode *inode, u64 num_bytes)
4973
{
4974
	unsigned drop_inode_space = 0;
4975
	unsigned dropped_extents = 0;
4976
	unsigned num_extents = 0;
4977

4978 4979 4980 4981 4982 4983
	num_extents = (unsigned)div64_u64(num_bytes +
					  BTRFS_MAX_EXTENT_SIZE - 1,
					  BTRFS_MAX_EXTENT_SIZE);
	ASSERT(num_extents);
	ASSERT(BTRFS_I(inode)->outstanding_extents >= num_extents);
	BTRFS_I(inode)->outstanding_extents -= num_extents;
4984

4985
	if (BTRFS_I(inode)->outstanding_extents == 0 &&
4986 4987
	    test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			       &BTRFS_I(inode)->runtime_flags))
4988 4989
		drop_inode_space = 1;

4990 4991 4992 4993 4994 4995
	/*
	 * If we have more or the same amount of outsanding extents than we have
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
4996
		return drop_inode_space;
4997 4998 4999 5000

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
5001
	return dropped_extents + drop_inode_space;
5002 5003
}

5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023
/**
 * calc_csum_metadata_size - return the amount of metada space that must be
 *	reserved/free'd for the given bytes.
 * @inode: the inode we're manipulating
 * @num_bytes: the number of bytes in question
 * @reserve: 1 if we are reserving space, 0 if we are freeing space
 *
 * This adjusts the number of csum_bytes in the inode and then returns the
 * correct amount of metadata that must either be reserved or freed.  We
 * calculate how many checksums we can fit into one leaf and then divide the
 * number of bytes that will need to be checksumed by this value to figure out
 * how many checksums will be required.  If we are adding bytes then the number
 * may go up and we will return the number of additional bytes that must be
 * reserved.  If it is going down we will return the number of bytes that must
 * be freed.
 *
 * This must be called with BTRFS_I(inode)->lock held.
 */
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
				   int reserve)
5024
{
5025 5026 5027 5028 5029 5030 5031 5032 5033 5034
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 csum_size;
	int num_csums_per_leaf;
	int num_csums;
	int old_csums;

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
	    BTRFS_I(inode)->csum_bytes == 0)
		return 0;

5035
	old_csums = (int)div_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
5036 5037 5038 5039 5040
	if (reserve)
		BTRFS_I(inode)->csum_bytes += num_bytes;
	else
		BTRFS_I(inode)->csum_bytes -= num_bytes;
	csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
5041
	num_csums_per_leaf = (int)div_u64(csum_size,
5042 5043
					    sizeof(struct btrfs_csum_item) +
					    sizeof(struct btrfs_disk_key));
5044
	num_csums = (int)div_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059
	num_csums = num_csums + num_csums_per_leaf - 1;
	num_csums = num_csums / num_csums_per_leaf;

	old_csums = old_csums + num_csums_per_leaf - 1;
	old_csums = old_csums / num_csums_per_leaf;

	/* No change, no need to reserve more */
	if (old_csums == num_csums)
		return 0;

	if (reserve)
		return btrfs_calc_trans_metadata_size(root,
						      num_csums - old_csums);

	return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
5060
}
Y
Yan Zheng 已提交
5061

5062 5063 5064 5065
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
5066
	u64 to_reserve = 0;
5067
	u64 csum_bytes;
5068
	unsigned nr_extents = 0;
5069
	int extra_reserve = 0;
M
Miao Xie 已提交
5070
	enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
5071
	int ret = 0;
5072
	bool delalloc_lock = true;
5073 5074
	u64 to_free = 0;
	unsigned dropped;
5075

5076 5077 5078 5079 5080 5081
	/* If we are a free space inode we need to not flush since we will be in
	 * the middle of a transaction commit.  We also don't need the delalloc
	 * mutex since we won't race with anybody.  We need this mostly to make
	 * lockdep shut its filthy mouth.
	 */
	if (btrfs_is_free_space_inode(inode)) {
M
Miao Xie 已提交
5082
		flush = BTRFS_RESERVE_NO_FLUSH;
5083 5084
		delalloc_lock = false;
	}
5085

M
Miao Xie 已提交
5086 5087
	if (flush != BTRFS_RESERVE_NO_FLUSH &&
	    btrfs_transaction_in_commit(root->fs_info))
5088
		schedule_timeout(1);
5089

5090 5091 5092
	if (delalloc_lock)
		mutex_lock(&BTRFS_I(inode)->delalloc_mutex);

5093
	num_bytes = ALIGN(num_bytes, root->sectorsize);
5094

5095 5096 5097 5098
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;

	if (BTRFS_I(inode)->outstanding_extents >
5099
	    BTRFS_I(inode)->reserved_extents)
5100 5101
		nr_extents = BTRFS_I(inode)->outstanding_extents -
			BTRFS_I(inode)->reserved_extents;
5102

5103 5104 5105 5106
	/*
	 * Add an item to reserve for updating the inode when we complete the
	 * delalloc io.
	 */
5107 5108
	if (!test_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
		      &BTRFS_I(inode)->runtime_flags)) {
5109
		nr_extents++;
5110
		extra_reserve = 1;
5111
	}
5112 5113

	to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
5114
	to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
5115
	csum_bytes = BTRFS_I(inode)->csum_bytes;
5116
	spin_unlock(&BTRFS_I(inode)->lock);
5117

5118
	if (root->fs_info->quota_enabled) {
5119
		ret = btrfs_qgroup_reserve(root, num_bytes +
5120
					   nr_extents * root->nodesize);
5121 5122 5123
		if (ret)
			goto out_fail;
	}
5124

5125 5126 5127
	ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush);
	if (unlikely(ret)) {
		if (root->fs_info->quota_enabled)
5128
			btrfs_qgroup_free(root, num_bytes +
5129
						nr_extents * root->nodesize);
5130
		goto out_fail;
5131
	}
5132

5133 5134
	spin_lock(&BTRFS_I(inode)->lock);
	if (extra_reserve) {
5135 5136
		set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			&BTRFS_I(inode)->runtime_flags);
5137 5138 5139 5140
		nr_extents--;
	}
	BTRFS_I(inode)->reserved_extents += nr_extents;
	spin_unlock(&BTRFS_I(inode)->lock);
5141 5142 5143

	if (delalloc_lock)
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
5144

J
Josef Bacik 已提交
5145
	if (to_reserve)
5146
		trace_btrfs_space_reservation(root->fs_info, "delalloc",
J
Josef Bacik 已提交
5147
					      btrfs_ino(inode), to_reserve, 1);
5148 5149 5150
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	return 0;
5151 5152 5153

out_fail:
	spin_lock(&BTRFS_I(inode)->lock);
5154
	dropped = drop_outstanding_extent(inode, num_bytes);
5155 5156 5157 5158 5159
	/*
	 * If the inodes csum_bytes is the same as the original
	 * csum_bytes then we know we haven't raced with any free()ers
	 * so we can just reduce our inodes csum bytes and carry on.
	 */
5160
	if (BTRFS_I(inode)->csum_bytes == csum_bytes) {
5161
		calc_csum_metadata_size(inode, num_bytes, 0);
5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201
	} else {
		u64 orig_csum_bytes = BTRFS_I(inode)->csum_bytes;
		u64 bytes;

		/*
		 * This is tricky, but first we need to figure out how much we
		 * free'd from any free-ers that occured during this
		 * reservation, so we reset ->csum_bytes to the csum_bytes
		 * before we dropped our lock, and then call the free for the
		 * number of bytes that were freed while we were trying our
		 * reservation.
		 */
		bytes = csum_bytes - BTRFS_I(inode)->csum_bytes;
		BTRFS_I(inode)->csum_bytes = csum_bytes;
		to_free = calc_csum_metadata_size(inode, bytes, 0);


		/*
		 * Now we need to see how much we would have freed had we not
		 * been making this reservation and our ->csum_bytes were not
		 * artificially inflated.
		 */
		BTRFS_I(inode)->csum_bytes = csum_bytes - num_bytes;
		bytes = csum_bytes - orig_csum_bytes;
		bytes = calc_csum_metadata_size(inode, bytes, 0);

		/*
		 * Now reset ->csum_bytes to what it should be.  If bytes is
		 * more than to_free then we would have free'd more space had we
		 * not had an artificially high ->csum_bytes, so we need to free
		 * the remainder.  If bytes is the same or less then we don't
		 * need to do anything, the other free-ers did the correct
		 * thing.
		 */
		BTRFS_I(inode)->csum_bytes = orig_csum_bytes - num_bytes;
		if (bytes > to_free)
			to_free = bytes - to_free;
		else
			to_free = 0;
	}
5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213
	spin_unlock(&BTRFS_I(inode)->lock);
	if (dropped)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);

	if (to_free) {
		btrfs_block_rsv_release(root, block_rsv, to_free);
		trace_btrfs_space_reservation(root->fs_info, "delalloc",
					      btrfs_ino(inode), to_free, 0);
	}
	if (delalloc_lock)
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
	return ret;
5214 5215
}

5216 5217 5218 5219 5220 5221 5222 5223 5224
/**
 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
 * @inode: the inode to release the reservation for
 * @num_bytes: the number of bytes we're releasing
 *
 * This will release the metadata reservation for an inode.  This can be called
 * once we complete IO for a given set of bytes to release their metadata
 * reservations.
 */
5225 5226 5227
void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
5228 5229
	u64 to_free = 0;
	unsigned dropped;
5230 5231

	num_bytes = ALIGN(num_bytes, root->sectorsize);
5232
	spin_lock(&BTRFS_I(inode)->lock);
5233
	dropped = drop_outstanding_extent(inode, num_bytes);
5234

5235 5236
	if (num_bytes)
		to_free = calc_csum_metadata_size(inode, num_bytes, 0);
5237
	spin_unlock(&BTRFS_I(inode)->lock);
5238 5239
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
5240

J
Josef Bacik 已提交
5241 5242
	trace_btrfs_space_reservation(root->fs_info, "delalloc",
				      btrfs_ino(inode), to_free, 0);
5243 5244
	if (root->fs_info->quota_enabled) {
		btrfs_qgroup_free(root, num_bytes +
5245
					dropped * root->nodesize);
5246 5247
	}

5248 5249 5250 5251
	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266
/**
 * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
 * @inode: inode we're writing to
 * @num_bytes: the number of bytes we want to allocate
 *
 * This will do the following things
 *
 * o reserve space in the data space info for num_bytes
 * o reserve space in the metadata space info based on number of outstanding
 *   extents and how much csums will be needed
 * o add to the inodes ->delalloc_bytes
 * o add it to the fs_info's delalloc inodes list.
 *
 * This will return 0 for success and -ENOSPC if there is no space left.
 */
5267 5268 5269 5270 5271
int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
C
Chris Mason 已提交
5272
	if (ret)
5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296
/**
 * btrfs_delalloc_release_space - release data and metadata space for delalloc
 * @inode: inode we're releasing space for
 * @num_bytes: the number of bytes we want to free up
 *
 * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
 * called in the case that we don't need the metadata AND data reservations
 * anymore.  So if there is an error or we insert an inline extent.
 *
 * This function will release the metadata space that was not used and will
 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
 * list if there are no delalloc bytes left.
 */
5297 5298 5299 5300
void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
5301 5302
}

5303 5304 5305
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root, u64 bytenr,
			      u64 num_bytes, int alloc)
C
Chris Mason 已提交
5306
{
5307
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
5308
	struct btrfs_fs_info *info = root->fs_info;
5309
	u64 total = num_bytes;
C
Chris Mason 已提交
5310
	u64 old_val;
5311
	u64 byte_in_group;
5312
	int factor;
C
Chris Mason 已提交
5313

5314
	/* block accounting for super block */
5315
	spin_lock(&info->delalloc_root_lock);
5316
	old_val = btrfs_super_bytes_used(info->super_copy);
5317 5318 5319 5320
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
5321
	btrfs_set_super_bytes_used(info->super_copy, old_val);
5322
	spin_unlock(&info->delalloc_root_lock);
5323

C
Chris Mason 已提交
5324
	while (total) {
5325
		cache = btrfs_lookup_block_group(info, bytenr);
5326
		if (!cache)
5327
			return -ENOENT;
5328 5329 5330 5331 5332 5333
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
5334 5335 5336 5337 5338 5339 5340
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
5341
			cache_block_group(cache, 1);
5342

5343 5344 5345 5346 5347 5348 5349 5350
		spin_lock(&trans->transaction->dirty_bgs_lock);
		if (list_empty(&cache->dirty_list)) {
			list_add_tail(&cache->dirty_list,
				      &trans->transaction->dirty_bgs);
			btrfs_get_block_group(cache);
		}
		spin_unlock(&trans->transaction->dirty_bgs_lock);

5351 5352
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
5353

5354
		spin_lock(&cache->space_info->lock);
5355
		spin_lock(&cache->lock);
5356

5357
		if (btrfs_test_opt(root, SPACE_CACHE) &&
5358 5359 5360
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

C
Chris Mason 已提交
5361
		old_val = btrfs_block_group_used(&cache->item);
5362
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
5363
		if (alloc) {
5364
			old_val += num_bytes;
5365 5366 5367
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
5368 5369
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
5370
			spin_unlock(&cache->lock);
5371
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
5372
		} else {
5373
			old_val -= num_bytes;
5374 5375 5376 5377 5378 5379 5380
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
			cache->space_info->bytes_used -= num_bytes;
			cache->space_info->disk_used -= num_bytes * factor;
			spin_unlock(&cache->lock);
			spin_unlock(&cache->space_info->lock);
5381

5382 5383 5384
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397
			/*
			 * No longer have used bytes in this block group, queue
			 * it for deletion.
			 */
			if (old_val == 0) {
				spin_lock(&info->unused_bgs_lock);
				if (list_empty(&cache->bg_list)) {
					btrfs_get_block_group(cache);
					list_add_tail(&cache->bg_list,
						      &info->unused_bgs);
				}
				spin_unlock(&info->unused_bgs_lock);
			}
C
Chris Mason 已提交
5398
		}
5399
		btrfs_put_block_group(cache);
5400 5401
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
5402 5403 5404
	}
	return 0;
}
5405

5406 5407
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
5408
	struct btrfs_block_group_cache *cache;
5409
	u64 bytenr;
J
Josef Bacik 已提交
5410

5411 5412 5413 5414 5415 5416 5417
	spin_lock(&root->fs_info->block_group_cache_lock);
	bytenr = root->fs_info->first_logical_byte;
	spin_unlock(&root->fs_info->block_group_cache_lock);

	if (bytenr < (u64)-1)
		return bytenr;

J
Josef Bacik 已提交
5418 5419
	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
5420
		return 0;
J
Josef Bacik 已提交
5421

5422
	bytenr = cache->key.objectid;
5423
	btrfs_put_block_group(cache);
5424 5425

	return bytenr;
5426 5427
}

5428 5429 5430
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
5431
{
5432 5433 5434 5435 5436 5437 5438 5439 5440 5441
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
5442

5443 5444
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
J
Josef Bacik 已提交
5445 5446
	if (reserved)
		trace_btrfs_reserved_extent_free(root, bytenr, num_bytes);
5447 5448
	return 0;
}
J
Josef Bacik 已提交
5449

5450 5451 5452 5453 5454 5455 5456
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
5457

5458
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
5459
	BUG_ON(!cache); /* Logic error */
5460 5461 5462 5463

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
5464 5465 5466
	return 0;
}

5467
/*
5468 5469
 * this function must be called within transaction
 */
5470
int btrfs_pin_extent_for_log_replay(struct btrfs_root *root,
5471 5472 5473
				    u64 bytenr, u64 num_bytes)
{
	struct btrfs_block_group_cache *cache;
5474
	int ret;
5475 5476

	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
5477 5478
	if (!cache)
		return -EINVAL;
5479 5480 5481 5482 5483 5484 5485

	/*
	 * pull in the free space cache (if any) so that our pin
	 * removes the free space from the cache.  We have load_only set
	 * to one because the slow code to read in the free extents does check
	 * the pinned extents.
	 */
5486
	cache_block_group(cache, 1);
5487 5488 5489 5490

	pin_down_extent(root, cache, bytenr, num_bytes, 0);

	/* remove us from the free space cache (if we're there at all) */
5491
	ret = btrfs_remove_free_space(cache, bytenr, num_bytes);
5492
	btrfs_put_block_group(cache);
5493
	return ret;
5494 5495
}

5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569
static int __exclude_logged_extent(struct btrfs_root *root, u64 start, u64 num_bytes)
{
	int ret;
	struct btrfs_block_group_cache *block_group;
	struct btrfs_caching_control *caching_ctl;

	block_group = btrfs_lookup_block_group(root->fs_info, start);
	if (!block_group)
		return -EINVAL;

	cache_block_group(block_group, 0);
	caching_ctl = get_caching_control(block_group);

	if (!caching_ctl) {
		/* Logic error */
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			if (ret)
				goto out_lock;

			num_bytes = (start + num_bytes) -
				caching_ctl->progress;
			start = caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
		}
out_lock:
		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}
	btrfs_put_block_group(block_group);
	return ret;
}

int btrfs_exclude_logged_extents(struct btrfs_root *log,
				 struct extent_buffer *eb)
{
	struct btrfs_file_extent_item *item;
	struct btrfs_key key;
	int found_type;
	int i;

	if (!btrfs_fs_incompat(log->fs_info, MIXED_GROUPS))
		return 0;

	for (i = 0; i < btrfs_header_nritems(eb); i++) {
		btrfs_item_key_to_cpu(eb, &key, i);
		if (key.type != BTRFS_EXTENT_DATA_KEY)
			continue;
		item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
		found_type = btrfs_file_extent_type(eb, item);
		if (found_type == BTRFS_FILE_EXTENT_INLINE)
			continue;
		if (btrfs_file_extent_disk_bytenr(eb, item) == 0)
			continue;
		key.objectid = btrfs_file_extent_disk_bytenr(eb, item);
		key.offset = btrfs_file_extent_disk_num_bytes(eb, item);
		__exclude_logged_extent(log, key.objectid, key.offset);
	}

	return 0;
}

5570 5571 5572 5573 5574
/**
 * btrfs_update_reserved_bytes - update the block_group and space info counters
 * @cache:	The cache we are manipulating
 * @num_bytes:	The number of bytes in question
 * @reserve:	One of the reservation enums
5575
 * @delalloc:   The blocks are allocated for the delalloc write
5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591
 *
 * This is called by the allocator when it reserves space, or by somebody who is
 * freeing space that was never actually used on disk.  For example if you
 * reserve some space for a new leaf in transaction A and before transaction A
 * commits you free that leaf, you call this with reserve set to 0 in order to
 * clear the reservation.
 *
 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
 * ENOSPC accounting.  For data we handle the reservation through clearing the
 * delalloc bits in the io_tree.  We have to do this since we could end up
 * allocating less disk space for the amount of data we have reserved in the
 * case of compression.
 *
 * If this is a reservation and the block group has become read only we cannot
 * make the reservation and return -EAGAIN, otherwise this function always
 * succeeds.
5592
 */
5593
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
5594
				       u64 num_bytes, int reserve, int delalloc)
5595
{
5596
	struct btrfs_space_info *space_info = cache->space_info;
5597
	int ret = 0;
5598

5599 5600 5601
	spin_lock(&space_info->lock);
	spin_lock(&cache->lock);
	if (reserve != RESERVE_FREE) {
5602 5603 5604
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
5605 5606 5607
			cache->reserved += num_bytes;
			space_info->bytes_reserved += num_bytes;
			if (reserve == RESERVE_ALLOC) {
J
Josef Bacik 已提交
5608
				trace_btrfs_space_reservation(cache->fs_info,
5609 5610
						"space_info", space_info->flags,
						num_bytes, 0);
5611 5612
				space_info->bytes_may_use -= num_bytes;
			}
5613 5614 5615

			if (delalloc)
				cache->delalloc_bytes += num_bytes;
5616
		}
5617 5618 5619 5620 5621
	} else {
		if (cache->ro)
			space_info->bytes_readonly += num_bytes;
		cache->reserved -= num_bytes;
		space_info->bytes_reserved -= num_bytes;
5622 5623 5624

		if (delalloc)
			cache->delalloc_bytes -= num_bytes;
5625
	}
5626 5627
	spin_unlock(&cache->lock);
	spin_unlock(&space_info->lock);
5628
	return ret;
5629
}
C
Chris Mason 已提交
5630

5631
void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
5632
				struct btrfs_root *root)
5633 5634
{
	struct btrfs_fs_info *fs_info = root->fs_info;
5635 5636 5637
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
5638

5639
	down_write(&fs_info->commit_root_sem);
5640

5641 5642 5643 5644 5645 5646 5647
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
5648
		} else {
5649
			cache->last_byte_to_unpin = caching_ctl->progress;
5650 5651
		}
	}
5652 5653 5654 5655 5656 5657

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

5658
	up_write(&fs_info->commit_root_sem);
5659 5660

	update_global_block_rsv(fs_info);
5661 5662
}

5663 5664
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end,
			      const bool return_free_space)
C
Chris Mason 已提交
5665
{
5666 5667
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
5668 5669
	struct btrfs_space_info *space_info;
	struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5670
	u64 len;
5671
	bool readonly;
C
Chris Mason 已提交
5672

5673
	while (start <= end) {
5674
		readonly = false;
5675 5676 5677 5678 5679
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
5680
			BUG_ON(!cache); /* Logic error */
5681 5682 5683 5684 5685 5686 5687
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
5688 5689
			if (return_free_space)
				btrfs_add_free_space(cache, start, len);
5690 5691
		}

5692
		start += len;
5693
		space_info = cache->space_info;
5694

5695
		spin_lock(&space_info->lock);
5696 5697
		spin_lock(&cache->lock);
		cache->pinned -= len;
5698
		space_info->bytes_pinned -= len;
5699
		percpu_counter_add(&space_info->total_bytes_pinned, -len);
5700 5701 5702 5703
		if (cache->ro) {
			space_info->bytes_readonly += len;
			readonly = true;
		}
5704
		spin_unlock(&cache->lock);
5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717
		if (!readonly && global_rsv->space_info == space_info) {
			spin_lock(&global_rsv->lock);
			if (!global_rsv->full) {
				len = min(len, global_rsv->size -
					  global_rsv->reserved);
				global_rsv->reserved += len;
				space_info->bytes_may_use += len;
				if (global_rsv->reserved >= global_rsv->size)
					global_rsv->full = 1;
			}
			spin_unlock(&global_rsv->lock);
		}
		spin_unlock(&space_info->lock);
C
Chris Mason 已提交
5718
	}
5719 5720 5721

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
5722 5723 5724 5725
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
5726
			       struct btrfs_root *root)
5727
{
5728 5729
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
5730 5731
	u64 start;
	u64 end;
5732 5733
	int ret;

5734 5735 5736
	if (trans->aborted)
		return 0;

5737 5738 5739 5740 5741
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

C
Chris Mason 已提交
5742
	while (1) {
5743
		mutex_lock(&fs_info->unused_bg_unpin_mutex);
5744
		ret = find_first_extent_bit(unpin, 0, &start, &end,
5745
					    EXTENT_DIRTY, NULL);
5746 5747
		if (ret) {
			mutex_unlock(&fs_info->unused_bg_unpin_mutex);
5748
			break;
5749
		}
5750

5751 5752 5753
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
5754

5755
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
5756
		unpin_extent_range(root, start, end, true);
5757
		mutex_unlock(&fs_info->unused_bg_unpin_mutex);
5758
		cond_resched();
5759
	}
J
Josef Bacik 已提交
5760

C
Chris Mason 已提交
5761 5762 5763
	return 0;
}

5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784
static void add_pinned_bytes(struct btrfs_fs_info *fs_info, u64 num_bytes,
			     u64 owner, u64 root_objectid)
{
	struct btrfs_space_info *space_info;
	u64 flags;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (root_objectid == BTRFS_CHUNK_TREE_OBJECTID)
			flags = BTRFS_BLOCK_GROUP_SYSTEM;
		else
			flags = BTRFS_BLOCK_GROUP_METADATA;
	} else {
		flags = BTRFS_BLOCK_GROUP_DATA;
	}

	space_info = __find_space_info(fs_info, flags);
	BUG_ON(!space_info); /* Logic bug */
	percpu_counter_add(&space_info->total_bytes_pinned, num_bytes);
}


5785 5786 5787 5788 5789
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
J
Josef Bacik 已提交
5790 5791
				struct btrfs_delayed_extent_op *extent_op,
				int no_quota)
5792
{
C
Chris Mason 已提交
5793
	struct btrfs_key key;
5794
	struct btrfs_path *path;
5795 5796
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
5797
	struct extent_buffer *leaf;
5798 5799
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
5800
	int ret;
5801
	int is_data;
5802 5803 5804
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
5805 5806
	u32 item_size;
	u64 refs;
J
Josef Bacik 已提交
5807 5808
	int last_ref = 0;
	enum btrfs_qgroup_operation_type type = BTRFS_QGROUP_OPER_SUB_EXCL;
5809 5810
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
C
Chris Mason 已提交
5811

J
Josef Bacik 已提交
5812 5813 5814
	if (!info->quota_enabled || !is_fstree(root_objectid))
		no_quota = 1;

5815
	path = btrfs_alloc_path();
5816 5817
	if (!path)
		return -ENOMEM;
5818

5819
	path->reada = 1;
5820
	path->leave_spinning = 1;
5821 5822 5823 5824

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

5825 5826 5827
	if (is_data)
		skinny_metadata = 0;

5828 5829 5830 5831
	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
5832
	if (ret == 0) {
5833
		extent_slot = path->slots[0];
5834 5835
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
5836
					      extent_slot);
5837
			if (key.objectid != bytenr)
5838
				break;
5839 5840
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
5841 5842 5843
				found_extent = 1;
				break;
			}
5844 5845 5846 5847 5848
			if (key.type == BTRFS_METADATA_ITEM_KEY &&
			    key.offset == owner_objectid) {
				found_extent = 1;
				break;
			}
5849 5850
			if (path->slots[0] - extent_slot > 5)
				break;
5851
			extent_slot--;
5852
		}
5853 5854 5855 5856 5857
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
5858
		if (!found_extent) {
5859
			BUG_ON(iref);
5860
			ret = remove_extent_backref(trans, extent_root, path,
5861
						    NULL, refs_to_drop,
J
Josef Bacik 已提交
5862
						    is_data, &last_ref);
5863 5864 5865 5866
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5867
			btrfs_release_path(path);
5868
			path->leave_spinning = 1;
5869 5870 5871 5872 5873

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

5874 5875 5876 5877 5878
			if (!is_data && skinny_metadata) {
				key.type = BTRFS_METADATA_ITEM_KEY;
				key.offset = owner_objectid;
			}

Z
Zheng Yan 已提交
5879 5880
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896
			if (ret > 0 && skinny_metadata && path->slots[0]) {
				/*
				 * Couldn't find our skinny metadata item,
				 * see if we have ye olde extent item.
				 */
				path->slots[0]--;
				btrfs_item_key_to_cpu(path->nodes[0], &key,
						      path->slots[0]);
				if (key.objectid == bytenr &&
				    key.type == BTRFS_EXTENT_ITEM_KEY &&
				    key.offset == num_bytes)
					ret = 0;
			}

			if (ret > 0 && skinny_metadata) {
				skinny_metadata = false;
5897
				key.objectid = bytenr;
5898 5899 5900 5901 5902 5903 5904
				key.type = BTRFS_EXTENT_ITEM_KEY;
				key.offset = num_bytes;
				btrfs_release_path(path);
				ret = btrfs_search_slot(trans, extent_root,
							&key, path, -1, 1);
			}

5905
			if (ret) {
5906
				btrfs_err(info, "umm, got %d back from search, was looking for %llu",
5907
					ret, bytenr);
5908 5909 5910
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
5911
			}
5912 5913 5914 5915
			if (ret < 0) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
Z
Zheng Yan 已提交
5916 5917
			extent_slot = path->slots[0];
		}
5918
	} else if (WARN_ON(ret == -ENOENT)) {
5919
		btrfs_print_leaf(extent_root, path->nodes[0]);
5920 5921
		btrfs_err(info,
			"unable to find ref byte nr %llu parent %llu root %llu  owner %llu offset %llu",
5922 5923
			bytenr, parent, root_objectid, owner_objectid,
			owner_offset);
5924 5925
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
5926
	} else {
5927 5928
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
5929
	}
5930 5931

	leaf = path->nodes[0];
5932 5933 5934 5935 5936 5937
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
5938 5939 5940 5941
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5942

5943
		btrfs_release_path(path);
5944 5945 5946 5947 5948 5949 5950 5951 5952
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
5953
			btrfs_err(info, "umm, got %d back from search, was looking for %llu",
5954
				ret, bytenr);
5955 5956
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
5957 5958 5959 5960 5961
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}

5962 5963 5964 5965 5966 5967
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
5968
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
5969
			    struct btrfs_extent_item);
5970 5971
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID &&
	    key.type == BTRFS_EXTENT_ITEM_KEY) {
5972 5973 5974 5975 5976
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
5977

5978
	refs = btrfs_extent_refs(leaf, ei);
5979 5980
	if (refs < refs_to_drop) {
		btrfs_err(info, "trying to drop %d refs but we only have %Lu "
5981
			  "for bytenr %Lu", refs_to_drop, refs, bytenr);
5982 5983 5984 5985
		ret = -EINVAL;
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
	}
5986
	refs -= refs_to_drop;
5987

5988
	if (refs > 0) {
J
Josef Bacik 已提交
5989
		type = BTRFS_QGROUP_OPER_SUB_SHARED;
5990 5991 5992 5993 5994
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
5995
		 */
5996 5997 5998 5999 6000 6001 6002 6003 6004
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
J
Josef Bacik 已提交
6005
						    is_data, &last_ref);
6006 6007 6008 6009
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
6010
		}
6011 6012
		add_pinned_bytes(root->fs_info, -num_bytes, owner_objectid,
				 root_objectid);
6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
6024
		}
6025

J
Josef Bacik 已提交
6026
		last_ref = 1;
6027 6028
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
6029 6030 6031 6032
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
6033
		btrfs_release_path(path);
6034

6035
		if (is_data) {
6036
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
6037 6038 6039 6040
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
6041 6042
		}

6043
		ret = update_block_group(trans, root, bytenr, num_bytes, 0);
6044 6045 6046 6047
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
6048
	}
J
Josef Bacik 已提交
6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062
	btrfs_release_path(path);

	/* Deal with the quota accounting */
	if (!ret && last_ref && !no_quota) {
		int mod_seq = 0;

		if (owner_objectid >= BTRFS_FIRST_FREE_OBJECTID &&
		    type == BTRFS_QGROUP_OPER_SUB_SHARED)
			mod_seq = 1;

		ret = btrfs_qgroup_record_ref(trans, info, root_objectid,
					      bytenr, num_bytes, type,
					      mod_seq);
	}
6063
out:
6064
	btrfs_free_path(path);
6065 6066 6067
	return ret;
}

6068
/*
6069
 * when we free an block, it is possible (and likely) that we free the last
6070 6071 6072 6073 6074 6075 6076 6077 6078
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
6079
	int ret = 0;
6080 6081 6082 6083 6084

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
6085
		goto out_delayed_unlock;
6086

6087 6088
	spin_lock(&head->lock);
	if (rb_first(&head->ref_root))
6089 6090
		goto out;

6091 6092 6093
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
6094
		btrfs_free_delayed_extent_op(head->extent_op);
6095 6096 6097
		head->extent_op = NULL;
	}

6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
L
Liu Bo 已提交
6110
	rb_erase(&head->href_node, &delayed_refs->href_root);
6111

6112
	atomic_dec(&delayed_refs->num_entries);
6113 6114 6115 6116 6117

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
6118
	delayed_refs->num_heads--;
6119
	if (head->processing == 0)
6120
		delayed_refs->num_heads_ready--;
6121 6122
	head->processing = 0;
	spin_unlock(&head->lock);
6123 6124
	spin_unlock(&delayed_refs->lock);

6125 6126 6127 6128 6129
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
6130
	btrfs_put_delayed_ref(&head->node);
6131
	return ret;
6132
out:
6133
	spin_unlock(&head->lock);
6134 6135

out_delayed_unlock:
6136 6137 6138 6139
	spin_unlock(&delayed_refs->lock);
	return 0;
}

6140 6141 6142
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
6143
			   u64 parent, int last_ref)
6144
{
6145
	int pin = 1;
6146 6147 6148
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
A
Arne Jansen 已提交
6149 6150 6151 6152
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					buf->start, buf->len,
					parent, root->root_key.objectid,
					btrfs_header_level(buf),
6153
					BTRFS_DROP_DELAYED_REF, NULL, 0);
6154
		BUG_ON(ret); /* -ENOMEM */
6155 6156 6157 6158 6159 6160
	}

	if (!last_ref)
		return;

	if (btrfs_header_generation(buf) == trans->transid) {
6161 6162
		struct btrfs_block_group_cache *cache;

6163 6164 6165
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
6166
				goto out;
6167 6168
		}

6169 6170
		cache = btrfs_lookup_block_group(root->fs_info, buf->start);

6171 6172
		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
6173
			btrfs_put_block_group(cache);
6174
			goto out;
6175 6176 6177 6178 6179
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
6180
		btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE, 0);
6181
		btrfs_put_block_group(cache);
J
Josef Bacik 已提交
6182
		trace_btrfs_reserved_extent_free(root, buf->start, buf->len);
6183
		pin = 0;
6184 6185
	}
out:
6186 6187 6188 6189 6190
	if (pin)
		add_pinned_bytes(root->fs_info, buf->len,
				 btrfs_header_level(buf),
				 root->root_key.objectid);

6191 6192 6193 6194 6195
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
6196 6197
}

6198
/* Can return -ENOMEM */
A
Arne Jansen 已提交
6199 6200
int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
J
Josef Bacik 已提交
6201
		      u64 owner, u64 offset, int no_quota)
6202 6203
{
	int ret;
A
Arne Jansen 已提交
6204
	struct btrfs_fs_info *fs_info = root->fs_info;
6205

6206
	if (btrfs_test_is_dummy_root(root))
6207
		return 0;
6208

6209 6210
	add_pinned_bytes(root->fs_info, num_bytes, owner, root_objectid);

6211 6212 6213 6214
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
6215 6216
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
6217
		/* unlocks the pinned mutex */
6218
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
6219
		ret = 0;
6220
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
6221 6222
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
6223
					parent, root_objectid, (int)owner,
J
Josef Bacik 已提交
6224
					BTRFS_DROP_DELAYED_REF, NULL, no_quota);
6225
	} else {
A
Arne Jansen 已提交
6226 6227 6228 6229
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
						num_bytes,
						parent, root_objectid, owner,
						offset, BTRFS_DROP_DELAYED_REF,
J
Josef Bacik 已提交
6230
						NULL, no_quota);
6231
	}
6232 6233 6234
	return ret;
}

J
Josef Bacik 已提交
6235 6236 6237 6238 6239 6240 6241 6242 6243 6244
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
6245 6246 6247
 *
 * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using
 * any of the information in this block group.
J
Josef Bacik 已提交
6248
 */
6249
static noinline void
J
Josef Bacik 已提交
6250 6251 6252
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
6253
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
6254

6255 6256
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
6257
		return;
J
Josef Bacik 已提交
6258

6259
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
6260
		   (cache->free_space_ctl->free_space >= num_bytes));
6261 6262 6263 6264 6265 6266 6267 6268

	put_caching_control(caching_ctl);
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;
6269
	int ret = 0;
6270 6271 6272

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
6273
		return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0;
6274 6275

	wait_event(caching_ctl->wait, block_group_cache_done(cache));
6276 6277
	if (cache->cached == BTRFS_CACHE_ERROR)
		ret = -EIO;
6278
	put_caching_control(caching_ctl);
6279
	return ret;
J
Josef Bacik 已提交
6280 6281
}

6282
int __get_raid_index(u64 flags)
6283
{
6284
	if (flags & BTRFS_BLOCK_GROUP_RAID10)
6285
		return BTRFS_RAID_RAID10;
6286
	else if (flags & BTRFS_BLOCK_GROUP_RAID1)
6287
		return BTRFS_RAID_RAID1;
6288
	else if (flags & BTRFS_BLOCK_GROUP_DUP)
6289
		return BTRFS_RAID_DUP;
6290
	else if (flags & BTRFS_BLOCK_GROUP_RAID0)
6291
		return BTRFS_RAID_RAID0;
D
David Woodhouse 已提交
6292
	else if (flags & BTRFS_BLOCK_GROUP_RAID5)
6293
		return BTRFS_RAID_RAID5;
D
David Woodhouse 已提交
6294
	else if (flags & BTRFS_BLOCK_GROUP_RAID6)
6295
		return BTRFS_RAID_RAID6;
6296

6297
	return BTRFS_RAID_SINGLE; /* BTRFS_BLOCK_GROUP_SINGLE */
6298 6299
}

6300
int get_block_group_index(struct btrfs_block_group_cache *cache)
6301
{
6302
	return __get_raid_index(cache->flags);
6303 6304
}

6305 6306 6307 6308 6309 6310 6311 6312 6313 6314
static const char *btrfs_raid_type_names[BTRFS_NR_RAID_TYPES] = {
	[BTRFS_RAID_RAID10]	= "raid10",
	[BTRFS_RAID_RAID1]	= "raid1",
	[BTRFS_RAID_DUP]	= "dup",
	[BTRFS_RAID_RAID0]	= "raid0",
	[BTRFS_RAID_SINGLE]	= "single",
	[BTRFS_RAID_RAID5]	= "raid5",
	[BTRFS_RAID_RAID6]	= "raid6",
};

6315
static const char *get_raid_name(enum btrfs_raid_types type)
6316 6317 6318 6319 6320 6321 6322
{
	if (type >= BTRFS_NR_RAID_TYPES)
		return NULL;

	return btrfs_raid_type_names[type];
}

J
Josef Bacik 已提交
6323
enum btrfs_loop_type {
6324 6325 6326 6327
	LOOP_CACHING_NOWAIT = 0,
	LOOP_CACHING_WAIT = 1,
	LOOP_ALLOC_CHUNK = 2,
	LOOP_NO_EMPTY_SIZE = 3,
J
Josef Bacik 已提交
6328 6329
};

6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393
static inline void
btrfs_lock_block_group(struct btrfs_block_group_cache *cache,
		       int delalloc)
{
	if (delalloc)
		down_read(&cache->data_rwsem);
}

static inline void
btrfs_grab_block_group(struct btrfs_block_group_cache *cache,
		       int delalloc)
{
	btrfs_get_block_group(cache);
	if (delalloc)
		down_read(&cache->data_rwsem);
}

static struct btrfs_block_group_cache *
btrfs_lock_cluster(struct btrfs_block_group_cache *block_group,
		   struct btrfs_free_cluster *cluster,
		   int delalloc)
{
	struct btrfs_block_group_cache *used_bg;
	bool locked = false;
again:
	spin_lock(&cluster->refill_lock);
	if (locked) {
		if (used_bg == cluster->block_group)
			return used_bg;

		up_read(&used_bg->data_rwsem);
		btrfs_put_block_group(used_bg);
	}

	used_bg = cluster->block_group;
	if (!used_bg)
		return NULL;

	if (used_bg == block_group)
		return used_bg;

	btrfs_get_block_group(used_bg);

	if (!delalloc)
		return used_bg;

	if (down_read_trylock(&used_bg->data_rwsem))
		return used_bg;

	spin_unlock(&cluster->refill_lock);
	down_read(&used_bg->data_rwsem);
	locked = true;
	goto again;
}

static inline void
btrfs_release_block_group(struct btrfs_block_group_cache *cache,
			 int delalloc)
{
	if (delalloc)
		up_read(&cache->data_rwsem);
	btrfs_put_block_group(cache);
}

6394 6395 6396
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
6397
 * ins->objectid == start position
6398
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
6399
 * ins->offset == the size of the hole.
6400
 * Any available blocks before search_start are skipped.
6401 6402 6403
 *
 * If there is no suitable free space, we will record the max size of
 * the free space extent currently.
6404
 */
6405
static noinline int find_free_extent(struct btrfs_root *orig_root,
6406 6407
				     u64 num_bytes, u64 empty_size,
				     u64 hint_byte, struct btrfs_key *ins,
6408
				     u64 flags, int delalloc)
6409
{
6410
	int ret = 0;
C
Chris Mason 已提交
6411
	struct btrfs_root *root = orig_root->fs_info->extent_root;
6412
	struct btrfs_free_cluster *last_ptr = NULL;
6413
	struct btrfs_block_group_cache *block_group = NULL;
6414
	u64 search_start = 0;
6415
	u64 max_extent_size = 0;
6416
	int empty_cluster = 2 * 1024 * 1024;
6417
	struct btrfs_space_info *space_info;
6418
	int loop = 0;
6419 6420
	int index = __get_raid_index(flags);
	int alloc_type = (flags & BTRFS_BLOCK_GROUP_DATA) ?
6421
		RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
6422
	bool failed_cluster_refill = false;
6423
	bool failed_alloc = false;
6424
	bool use_cluster = true;
6425
	bool have_caching_bg = false;
6426

6427
	WARN_ON(num_bytes < root->sectorsize);
6428
	ins->type = BTRFS_EXTENT_ITEM_KEY;
6429 6430
	ins->objectid = 0;
	ins->offset = 0;
6431

6432
	trace_find_free_extent(orig_root, num_bytes, empty_size, flags);
J
Josef Bacik 已提交
6433

6434
	space_info = __find_space_info(root->fs_info, flags);
6435
	if (!space_info) {
6436
		btrfs_err(root->fs_info, "No space info for %llu", flags);
6437 6438
		return -ENOSPC;
	}
J
Josef Bacik 已提交
6439

6440 6441 6442 6443 6444 6445 6446
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

6447
	if (flags & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
6448
		last_ptr = &root->fs_info->meta_alloc_cluster;
6449 6450
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
6451 6452
	}

6453
	if ((flags & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
6454
	    btrfs_test_opt(root, SSD)) {
6455 6456
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
J
Josef Bacik 已提交
6457

6458
	if (last_ptr) {
6459 6460 6461 6462
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
6463
	}
6464

6465
	search_start = max(search_start, first_logical_byte(root, 0));
6466
	search_start = max(search_start, hint_byte);
6467

J
Josef Bacik 已提交
6468
	if (!last_ptr)
6469 6470
		empty_cluster = 0;

J
Josef Bacik 已提交
6471 6472 6473
	if (search_start == hint_byte) {
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
J
Josef Bacik 已提交
6474 6475 6476
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
6477 6478 6479
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
6480
		 */
6481
		if (block_group && block_group_bits(block_group, flags) &&
6482
		    block_group->cached != BTRFS_CACHE_NO) {
J
Josef Bacik 已提交
6483
			down_read(&space_info->groups_sem);
6484 6485 6486 6487 6488 6489 6490 6491 6492 6493
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
6494
			} else {
6495
				index = get_block_group_index(block_group);
6496
				btrfs_lock_block_group(block_group, delalloc);
6497
				goto have_block_group;
6498
			}
J
Josef Bacik 已提交
6499
		} else if (block_group) {
6500
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
6501
		}
6502
	}
J
Josef Bacik 已提交
6503
search:
6504
	have_caching_bg = false;
6505
	down_read(&space_info->groups_sem);
6506 6507
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
6508
		u64 offset;
J
Josef Bacik 已提交
6509
		int cached;
6510

6511
		btrfs_grab_block_group(block_group, delalloc);
J
Josef Bacik 已提交
6512
		search_start = block_group->key.objectid;
6513

6514 6515 6516 6517 6518
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
6519
		if (!block_group_bits(block_group, flags)) {
6520 6521
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
D
David Woodhouse 已提交
6522 6523
				BTRFS_BLOCK_GROUP_RAID5 |
				BTRFS_BLOCK_GROUP_RAID6 |
6524 6525 6526 6527 6528 6529 6530
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
6531
			if ((flags & extra) && !(block_group->flags & extra))
6532 6533 6534
				goto loop;
		}

J
Josef Bacik 已提交
6535
have_block_group:
6536 6537
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached)) {
6538
			ret = cache_block_group(block_group, 0);
6539 6540
			BUG_ON(ret < 0);
			ret = 0;
J
Josef Bacik 已提交
6541 6542
		}

6543 6544
		if (unlikely(block_group->cached == BTRFS_CACHE_ERROR))
			goto loop;
6545
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
6546
			goto loop;
J
Josef Bacik 已提交
6547

6548
		/*
6549 6550
		 * Ok we want to try and use the cluster allocator, so
		 * lets look there
6551
		 */
6552
		if (last_ptr) {
6553
			struct btrfs_block_group_cache *used_block_group;
6554
			unsigned long aligned_cluster;
6555 6556 6557 6558
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
6559 6560 6561 6562
			used_block_group = btrfs_lock_cluster(block_group,
							      last_ptr,
							      delalloc);
			if (!used_block_group)
6563
				goto refill_cluster;
6564

6565 6566 6567 6568
			if (used_block_group != block_group &&
			    (used_block_group->ro ||
			     !block_group_bits(used_block_group, flags)))
				goto release_cluster;
6569

6570
			offset = btrfs_alloc_from_cluster(used_block_group,
6571 6572 6573 6574
						last_ptr,
						num_bytes,
						used_block_group->key.objectid,
						&max_extent_size);
6575 6576 6577
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
6578
				trace_btrfs_reserve_extent_cluster(root,
6579 6580
						used_block_group,
						search_start, num_bytes);
6581
				if (used_block_group != block_group) {
6582 6583
					btrfs_release_block_group(block_group,
								  delalloc);
6584 6585
					block_group = used_block_group;
				}
6586 6587 6588
				goto checks;
			}

6589
			WARN_ON(last_ptr->block_group != used_block_group);
6590
release_cluster:
6591 6592 6593 6594 6595 6596 6597 6598
			/* If we are on LOOP_NO_EMPTY_SIZE, we can't
			 * set up a new clusters, so lets just skip it
			 * and let the allocator find whatever block
			 * it can find.  If we reach this point, we
			 * will have tried the cluster allocator
			 * plenty of times and not have found
			 * anything, so we are likely way too
			 * fragmented for the clustering stuff to find
6599 6600 6601 6602 6603 6604 6605 6606
			 * anything.
			 *
			 * However, if the cluster is taken from the
			 * current block group, release the cluster
			 * first, so that we stand a better chance of
			 * succeeding in the unclustered
			 * allocation.  */
			if (loop >= LOOP_NO_EMPTY_SIZE &&
6607
			    used_block_group != block_group) {
6608
				spin_unlock(&last_ptr->refill_lock);
6609 6610
				btrfs_release_block_group(used_block_group,
							  delalloc);
6611 6612 6613
				goto unclustered_alloc;
			}

6614 6615 6616 6617 6618 6619
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

6620 6621 6622 6623
			if (used_block_group != block_group)
				btrfs_release_block_group(used_block_group,
							  delalloc);
refill_cluster:
6624 6625 6626 6627 6628
			if (loop >= LOOP_NO_EMPTY_SIZE) {
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

6629 6630 6631 6632
			aligned_cluster = max_t(unsigned long,
						empty_cluster + empty_size,
					      block_group->full_stripe_len);

6633
			/* allocate a cluster in this block group */
6634 6635 6636 6637
			ret = btrfs_find_space_cluster(root, block_group,
						       last_ptr, search_start,
						       num_bytes,
						       aligned_cluster);
6638 6639 6640 6641 6642 6643
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
6644 6645 6646 6647
							last_ptr,
							num_bytes,
							search_start,
							&max_extent_size);
6648 6649 6650
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
6651 6652 6653
					trace_btrfs_reserve_extent_cluster(root,
						block_group, search_start,
						num_bytes);
6654 6655
					goto checks;
				}
6656 6657
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
6658 6659
				spin_unlock(&last_ptr->refill_lock);

6660
				failed_cluster_refill = true;
J
Josef Bacik 已提交
6661 6662 6663
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
6664
			}
J
Josef Bacik 已提交
6665

6666 6667 6668 6669 6670 6671
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
6672
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
6673
			spin_unlock(&last_ptr->refill_lock);
6674
			goto loop;
6675 6676
		}

6677
unclustered_alloc:
6678 6679 6680 6681
		spin_lock(&block_group->free_space_ctl->tree_lock);
		if (cached &&
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_cluster + empty_size) {
6682 6683 6684 6685
			if (block_group->free_space_ctl->free_space >
			    max_extent_size)
				max_extent_size =
					block_group->free_space_ctl->free_space;
6686 6687 6688 6689 6690
			spin_unlock(&block_group->free_space_ctl->tree_lock);
			goto loop;
		}
		spin_unlock(&block_group->free_space_ctl->tree_lock);

6691
		offset = btrfs_find_space_for_alloc(block_group, search_start,
6692 6693
						    num_bytes, empty_size,
						    &max_extent_size);
6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
6705
			wait_block_group_cache_progress(block_group,
6706 6707
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
6708
			goto have_block_group;
6709
		} else if (!offset) {
6710 6711
			if (!cached)
				have_caching_bg = true;
6712
			goto loop;
J
Josef Bacik 已提交
6713
		}
6714
checks:
6715
		search_start = ALIGN(offset, root->stripesize);
6716

J
Josef Bacik 已提交
6717 6718
		/* move on to the next group */
		if (search_start + num_bytes >
6719 6720
		    block_group->key.objectid + block_group->key.offset) {
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
6721
			goto loop;
6722
		}
6723

6724
		if (offset < search_start)
6725
			btrfs_add_free_space(block_group, offset,
6726 6727
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
6728

6729
		ret = btrfs_update_reserved_bytes(block_group, num_bytes,
6730
						  alloc_type, delalloc);
6731
		if (ret == -EAGAIN) {
6732
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
6733
			goto loop;
J
Josef Bacik 已提交
6734
		}
6735

6736
		/* we are all good, lets return */
J
Josef Bacik 已提交
6737 6738
		ins->objectid = search_start;
		ins->offset = num_bytes;
6739

J
Josef Bacik 已提交
6740 6741
		trace_btrfs_reserve_extent(orig_root, block_group,
					   search_start, num_bytes);
6742
		btrfs_release_block_group(block_group, delalloc);
J
Josef Bacik 已提交
6743 6744
		break;
loop:
6745
		failed_cluster_refill = false;
6746
		failed_alloc = false;
6747
		BUG_ON(index != get_block_group_index(block_group));
6748
		btrfs_release_block_group(block_group, delalloc);
J
Josef Bacik 已提交
6749 6750 6751
	}
	up_read(&space_info->groups_sem);

6752 6753 6754
	if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
		goto search;

6755 6756 6757
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

6758
	/*
6759 6760
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
6761 6762 6763 6764
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
6765
	 */
6766
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
6767
		index = 0;
6768
		loop++;
J
Josef Bacik 已提交
6769
		if (loop == LOOP_ALLOC_CHUNK) {
6770
			struct btrfs_trans_handle *trans;
6771 6772 6773 6774 6775 6776 6777
			int exist = 0;

			trans = current->journal_info;
			if (trans)
				exist = 1;
			else
				trans = btrfs_join_transaction(root);
6778 6779 6780 6781 6782 6783

			if (IS_ERR(trans)) {
				ret = PTR_ERR(trans);
				goto out;
			}

6784
			ret = do_chunk_alloc(trans, root, flags,
6785 6786 6787 6788 6789
					     CHUNK_ALLOC_FORCE);
			/*
			 * Do not bail out on ENOSPC since we
			 * can do more things.
			 */
6790
			if (ret < 0 && ret != -ENOSPC)
6791 6792
				btrfs_abort_transaction(trans,
							root, ret);
6793 6794
			else
				ret = 0;
6795 6796
			if (!exist)
				btrfs_end_transaction(trans, root);
6797
			if (ret)
6798
				goto out;
J
Josef Bacik 已提交
6799 6800
		}

6801 6802 6803
		if (loop == LOOP_NO_EMPTY_SIZE) {
			empty_size = 0;
			empty_cluster = 0;
6804
		}
6805 6806

		goto search;
J
Josef Bacik 已提交
6807 6808
	} else if (!ins->objectid) {
		ret = -ENOSPC;
6809
	} else if (ins->objectid) {
6810
		ret = 0;
C
Chris Mason 已提交
6811
	}
6812
out:
6813 6814
	if (ret == -ENOSPC)
		ins->offset = max_extent_size;
C
Chris Mason 已提交
6815
	return ret;
6816
}
6817

J
Josef Bacik 已提交
6818 6819
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
6820 6821
{
	struct btrfs_block_group_cache *cache;
6822
	int index = 0;
J
Josef Bacik 已提交
6823

J
Josef Bacik 已提交
6824
	spin_lock(&info->lock);
6825
	printk(KERN_INFO "BTRFS: space_info %llu has %llu free, is %sfull\n",
6826 6827 6828
	       info->flags,
	       info->total_bytes - info->bytes_used - info->bytes_pinned -
	       info->bytes_reserved - info->bytes_readonly,
C
Chris Mason 已提交
6829
	       (info->full) ? "" : "not ");
6830
	printk(KERN_INFO "BTRFS: space_info total=%llu, used=%llu, pinned=%llu, "
6831
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
6832 6833 6834
	       info->total_bytes, info->bytes_used, info->bytes_pinned,
	       info->bytes_reserved, info->bytes_may_use,
	       info->bytes_readonly);
J
Josef Bacik 已提交
6835 6836 6837 6838
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
6839

6840
	down_read(&info->groups_sem);
6841 6842
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
6843
		spin_lock(&cache->lock);
6844 6845 6846
		printk(KERN_INFO "BTRFS: "
			   "block group %llu has %llu bytes, "
			   "%llu used %llu pinned %llu reserved %s\n",
6847 6848 6849
		       cache->key.objectid, cache->key.offset,
		       btrfs_block_group_used(&cache->item), cache->pinned,
		       cache->reserved, cache->ro ? "[readonly]" : "");
J
Josef Bacik 已提交
6850 6851 6852
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
6853 6854
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
6855
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
6856
}
6857

6858
int btrfs_reserve_extent(struct btrfs_root *root,
6859 6860
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
6861
			 struct btrfs_key *ins, int is_data, int delalloc)
6862
{
6863
	bool final_tried = false;
6864
	u64 flags;
6865
	int ret;
6866

6867
	flags = btrfs_get_alloc_profile(root, is_data);
6868
again:
6869
	WARN_ON(num_bytes < root->sectorsize);
6870
	ret = find_free_extent(root, num_bytes, empty_size, hint_byte, ins,
6871
			       flags, delalloc);
6872

6873
	if (ret == -ENOSPC) {
6874 6875
		if (!final_tried && ins->offset) {
			num_bytes = min(num_bytes >> 1, ins->offset);
6876
			num_bytes = round_down(num_bytes, root->sectorsize);
6877 6878 6879 6880 6881 6882 6883
			num_bytes = max(num_bytes, min_alloc_size);
			if (num_bytes == min_alloc_size)
				final_tried = true;
			goto again;
		} else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
			struct btrfs_space_info *sinfo;

6884
			sinfo = __find_space_info(root->fs_info, flags);
6885
			btrfs_err(root->fs_info, "allocation failed flags %llu, wanted %llu",
6886
				flags, num_bytes);
6887 6888
			if (sinfo)
				dump_space_info(sinfo, num_bytes, 1);
6889
		}
6890
	}
J
Josef Bacik 已提交
6891 6892

	return ret;
6893 6894
}

6895
static int __btrfs_free_reserved_extent(struct btrfs_root *root,
6896 6897
					u64 start, u64 len,
					int pin, int delalloc)
6898
{
J
Josef Bacik 已提交
6899
	struct btrfs_block_group_cache *cache;
6900
	int ret = 0;
J
Josef Bacik 已提交
6901 6902 6903

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
6904
		btrfs_err(root->fs_info, "Unable to find block group for %llu",
6905
			start);
J
Josef Bacik 已提交
6906 6907
		return -ENOSPC;
	}
6908

6909 6910
	if (btrfs_test_opt(root, DISCARD))
		ret = btrfs_discard_extent(root, start, len, NULL);
6911

6912 6913 6914 6915
	if (pin)
		pin_down_extent(root, cache, start, len, 1);
	else {
		btrfs_add_free_space(cache, start, len);
6916
		btrfs_update_reserved_bytes(cache, len, RESERVE_FREE, delalloc);
6917
	}
6918
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
6919

6920 6921
	trace_btrfs_reserved_extent_free(root, start, len);

6922 6923 6924
	return ret;
}

6925
int btrfs_free_reserved_extent(struct btrfs_root *root,
6926
			       u64 start, u64 len, int delalloc)
6927
{
6928
	return __btrfs_free_reserved_extent(root, start, len, 0, delalloc);
6929 6930 6931 6932 6933
}

int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
				       u64 start, u64 len)
{
6934
	return __btrfs_free_reserved_extent(root, start, len, 1, 0);
6935 6936
}

6937 6938 6939 6940 6941
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
6942 6943
{
	int ret;
6944
	struct btrfs_fs_info *fs_info = root->fs_info;
6945
	struct btrfs_extent_item *extent_item;
6946
	struct btrfs_extent_inline_ref *iref;
6947
	struct btrfs_path *path;
6948 6949 6950
	struct extent_buffer *leaf;
	int type;
	u32 size;
6951

6952 6953 6954 6955
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
6956

6957
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
6958 6959

	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
6960 6961
	if (!path)
		return -ENOMEM;
6962

6963
	path->leave_spinning = 1;
6964 6965
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6966 6967 6968 6969
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
J
Josef Bacik 已提交
6970

6971 6972
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
6973
				     struct btrfs_extent_item);
6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
6994 6995

	btrfs_mark_buffer_dirty(path->nodes[0]);
6996
	btrfs_free_path(path);
6997

J
Josef Bacik 已提交
6998 6999 7000 7001 7002 7003 7004
	/* Always set parent to 0 here since its exclusive anyway. */
	ret = btrfs_qgroup_record_ref(trans, fs_info, root_objectid,
				      ins->objectid, ins->offset,
				      BTRFS_QGROUP_OPER_ADD_EXCL, 0);
	if (ret)
		return ret;

7005
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
7006
	if (ret) { /* -ENOENT, logic error */
7007
		btrfs_err(fs_info, "update block group failed for %llu %llu",
7008
			ins->objectid, ins->offset);
7009 7010
		BUG();
	}
J
Josef Bacik 已提交
7011
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);
7012 7013 7014
	return ret;
}

7015 7016 7017 7018
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
J
Josef Bacik 已提交
7019 7020
				     int level, struct btrfs_key *ins,
				     int no_quota)
7021 7022
{
	int ret;
7023 7024 7025 7026 7027 7028
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
7029
	u32 size = sizeof(*extent_item) + sizeof(*iref);
J
Josef Bacik 已提交
7030
	u64 num_bytes = ins->offset;
7031 7032 7033 7034 7035
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);

	if (!skinny_metadata)
		size += sizeof(*block_info);
7036

7037
	path = btrfs_alloc_path();
7038 7039
	if (!path) {
		btrfs_free_and_pin_reserved_extent(root, ins->objectid,
7040
						   root->nodesize);
7041
		return -ENOMEM;
7042
	}
7043

7044 7045 7046
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
7047
	if (ret) {
7048
		btrfs_free_and_pin_reserved_extent(root, ins->objectid,
7049
						   root->nodesize);
7050 7051 7052
		btrfs_free_path(path);
		return ret;
	}
7053 7054 7055 7056 7057 7058 7059 7060 7061

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);

7062 7063
	if (skinny_metadata) {
		iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
7064
		num_bytes = root->nodesize;
7065 7066 7067 7068 7069 7070
	} else {
		block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
		btrfs_set_tree_block_key(leaf, block_info, key);
		btrfs_set_tree_block_level(leaf, block_info, level);
		iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	}
7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085

	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

J
Josef Bacik 已提交
7086 7087 7088 7089 7090 7091 7092 7093
	if (!no_quota) {
		ret = btrfs_qgroup_record_ref(trans, fs_info, root_objectid,
					      ins->objectid, num_bytes,
					      BTRFS_QGROUP_OPER_ADD_EXCL, 0);
		if (ret)
			return ret;
	}

7094 7095
	ret = update_block_group(trans, root, ins->objectid, root->nodesize,
				 1);
7096
	if (ret) { /* -ENOENT, logic error */
7097
		btrfs_err(fs_info, "update block group failed for %llu %llu",
7098
			ins->objectid, ins->offset);
7099 7100
		BUG();
	}
J
Josef Bacik 已提交
7101

7102
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, root->nodesize);
7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

A
Arne Jansen 已提交
7115 7116 7117 7118
	ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
					 ins->offset, 0,
					 root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL, 0);
7119 7120
	return ret;
}
7121 7122 7123 7124 7125 7126

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
7127 7128 7129 7130
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
7131 7132 7133
{
	int ret;
	struct btrfs_block_group_cache *block_group;
7134

7135 7136 7137 7138 7139 7140
	/*
	 * Mixed block groups will exclude before processing the log so we only
	 * need to do the exlude dance if this fs isn't mixed.
	 */
	if (!btrfs_fs_incompat(root->fs_info, MIXED_GROUPS)) {
		ret = __exclude_logged_extent(root, ins->objectid, ins->offset);
7141
		if (ret)
7142
			return ret;
7143 7144
	}

7145 7146 7147 7148
	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
	if (!block_group)
		return -EINVAL;

7149
	ret = btrfs_update_reserved_bytes(block_group, ins->offset,
7150
					  RESERVE_ALLOC_NO_ACCOUNT, 0);
7151
	BUG_ON(ret); /* logic error */
7152 7153
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
7154
	btrfs_put_block_group(block_group);
7155 7156 7157
	return ret;
}

7158 7159
static struct extent_buffer *
btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root,
7160
		      u64 bytenr, int level)
7161 7162 7163
{
	struct extent_buffer *buf;

7164
	buf = btrfs_find_create_tree_block(root, bytenr);
7165 7166 7167
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
7168
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
7169 7170
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
7171
	clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
7172 7173

	btrfs_set_lock_blocking(buf);
7174
	btrfs_set_buffer_uptodate(buf);
7175

7176
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
7177
		buf->log_index = root->log_transid % 2;
7178 7179 7180 7181
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
7182
		if (buf->log_index == 0)
7183 7184 7185 7186 7187
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
7188
	} else {
7189
		buf->log_index = -1;
7190
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
7191
			 buf->start + buf->len - 1, GFP_NOFS);
7192
	}
7193
	trans->blocks_used++;
7194
	/* this returns a buffer locked for blocking */
7195 7196 7197
	return buf;
}

7198 7199 7200 7201 7202
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
7203
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
7204
	int ret;
7205
	bool global_updated = false;
7206 7207 7208

	block_rsv = get_block_rsv(trans, root);

7209 7210
	if (unlikely(block_rsv->size == 0))
		goto try_reserve;
7211
again:
7212 7213 7214 7215
	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;

7216 7217 7218
	if (block_rsv->failfast)
		return ERR_PTR(ret);

7219 7220 7221 7222 7223 7224
	if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) {
		global_updated = true;
		update_global_block_rsv(root->fs_info);
		goto again;
	}

7225 7226 7227 7228 7229 7230
	if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
		static DEFINE_RATELIMIT_STATE(_rs,
				DEFAULT_RATELIMIT_INTERVAL * 10,
				/*DEFAULT_RATELIMIT_BURST*/ 1);
		if (__ratelimit(&_rs))
			WARN(1, KERN_DEBUG
7231
				"BTRFS: block rsv returned %d\n", ret);
7232 7233 7234 7235 7236 7237 7238 7239
	}
try_reserve:
	ret = reserve_metadata_bytes(root, block_rsv, blocksize,
				     BTRFS_RESERVE_NO_FLUSH);
	if (!ret)
		return block_rsv;
	/*
	 * If we couldn't reserve metadata bytes try and use some from
7240 7241
	 * the global reserve if its space type is the same as the global
	 * reservation.
7242
	 */
7243 7244
	if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL &&
	    block_rsv->space_info == global_rsv->space_info) {
7245 7246 7247 7248 7249
		ret = block_rsv_use_bytes(global_rsv, blocksize);
		if (!ret)
			return global_rsv;
	}
	return ERR_PTR(ret);
7250 7251
}

J
Josef Bacik 已提交
7252 7253
static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
			    struct btrfs_block_rsv *block_rsv, u32 blocksize)
7254 7255
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
J
Josef Bacik 已提交
7256
	block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
7257 7258
}

7259
/*
7260 7261 7262 7263
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
7264 7265
 * returns the tree buffer or NULL.
 */
7266 7267
struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
7268 7269
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
7270
					u64 hint, u64 empty_size)
7271
{
C
Chris Mason 已提交
7272
	struct btrfs_key ins;
7273
	struct btrfs_block_rsv *block_rsv;
7274
	struct extent_buffer *buf;
7275 7276
	u64 flags = 0;
	int ret;
7277
	u32 blocksize = root->nodesize;
7278 7279
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
7280

7281
	if (btrfs_test_is_dummy_root(root)) {
7282
		buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr,
7283
					    level);
7284 7285 7286 7287
		if (!IS_ERR(buf))
			root->alloc_bytenr += blocksize;
		return buf;
	}
7288

7289 7290 7291 7292
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

7293
	ret = btrfs_reserve_extent(root, blocksize, blocksize,
7294
				   empty_size, hint, &ins, 0, 0);
7295
	if (ret) {
J
Josef Bacik 已提交
7296
		unuse_block_rsv(root->fs_info, block_rsv, blocksize);
7297
		return ERR_PTR(ret);
7298
	}
7299

7300
	buf = btrfs_init_new_buffer(trans, root, ins.objectid, level);
7301
	BUG_ON(IS_ERR(buf)); /* -ENOMEM */
7302 7303 7304 7305 7306 7307 7308 7309 7310 7311

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
7312
		extent_op = btrfs_alloc_delayed_extent_op();
7313
		BUG_ON(!extent_op); /* -ENOMEM */
7314 7315 7316 7317 7318
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
7319 7320 7321 7322
		if (skinny_metadata)
			extent_op->update_key = 0;
		else
			extent_op->update_key = 1;
7323 7324
		extent_op->update_flags = 1;
		extent_op->is_data = 0;
7325
		extent_op->level = level;
7326

A
Arne Jansen 已提交
7327 7328
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					ins.objectid,
7329 7330
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
7331
					extent_op, 0);
7332
		BUG_ON(ret); /* -ENOMEM */
7333
	}
7334 7335
	return buf;
}
7336

7337 7338 7339 7340 7341 7342 7343 7344 7345
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
7346 7347
	int reada_slot;
	int reada_count;
A
Arne Jansen 已提交
7348
	int for_reloc;
7349 7350 7351 7352 7353
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
7354 7355 7356 7357
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
7358
{
Y
Yan, Zheng 已提交
7359 7360 7361
	u64 bytenr;
	u64 generation;
	u64 refs;
7362
	u64 flags;
7363
	u32 nritems;
Y
Yan, Zheng 已提交
7364 7365 7366
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
7367
	int ret;
Y
Yan, Zheng 已提交
7368 7369
	int slot;
	int nread = 0;
7370

Y
Yan, Zheng 已提交
7371 7372 7373 7374 7375 7376 7377 7378
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
7379

Y
Yan, Zheng 已提交
7380 7381
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
7382
	blocksize = root->nodesize;
7383

Y
Yan, Zheng 已提交
7384 7385 7386
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
7387

C
Chris Mason 已提交
7388
		cond_resched();
Y
Yan, Zheng 已提交
7389 7390
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
7391

Y
Yan, Zheng 已提交
7392 7393
		if (slot == path->slots[wc->level])
			goto reada;
7394

Y
Yan, Zheng 已提交
7395 7396
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
7397 7398
			continue;

7399
		/* We don't lock the tree block, it's OK to be racy here */
7400 7401 7402
		ret = btrfs_lookup_extent_info(trans, root, bytenr,
					       wc->level - 1, 1, &refs,
					       &flags);
7403 7404 7405
		/* We don't care about errors in readahead. */
		if (ret < 0)
			continue;
7406 7407
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
7408 7409 7410
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
7411

7412 7413 7414
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
7415 7416 7417 7418 7419 7420 7421 7422
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
7423 7424 7425 7426
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
7427
		}
Y
Yan, Zheng 已提交
7428
reada:
7429
		readahead_tree_block(root, bytenr);
Y
Yan, Zheng 已提交
7430
		nread++;
C
Chris Mason 已提交
7431
	}
Y
Yan, Zheng 已提交
7432
	wc->reada_slot = slot;
C
Chris Mason 已提交
7433
}
7434

7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598
static int account_leaf_items(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
			      struct extent_buffer *eb)
{
	int nr = btrfs_header_nritems(eb);
	int i, extent_type, ret;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	u64 bytenr, num_bytes;

	for (i = 0; i < nr; i++) {
		btrfs_item_key_to_cpu(eb, &key, i);

		if (key.type != BTRFS_EXTENT_DATA_KEY)
			continue;

		fi = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
		/* filter out non qgroup-accountable extents  */
		extent_type = btrfs_file_extent_type(eb, fi);

		if (extent_type == BTRFS_FILE_EXTENT_INLINE)
			continue;

		bytenr = btrfs_file_extent_disk_bytenr(eb, fi);
		if (!bytenr)
			continue;

		num_bytes = btrfs_file_extent_disk_num_bytes(eb, fi);

		ret = btrfs_qgroup_record_ref(trans, root->fs_info,
					      root->objectid,
					      bytenr, num_bytes,
					      BTRFS_QGROUP_OPER_SUB_SUBTREE, 0);
		if (ret)
			return ret;
	}
	return 0;
}

/*
 * Walk up the tree from the bottom, freeing leaves and any interior
 * nodes which have had all slots visited. If a node (leaf or
 * interior) is freed, the node above it will have it's slot
 * incremented. The root node will never be freed.
 *
 * At the end of this function, we should have a path which has all
 * slots incremented to the next position for a search. If we need to
 * read a new node it will be NULL and the node above it will have the
 * correct slot selected for a later read.
 *
 * If we increment the root nodes slot counter past the number of
 * elements, 1 is returned to signal completion of the search.
 */
static int adjust_slots_upwards(struct btrfs_root *root,
				struct btrfs_path *path, int root_level)
{
	int level = 0;
	int nr, slot;
	struct extent_buffer *eb;

	if (root_level == 0)
		return 1;

	while (level <= root_level) {
		eb = path->nodes[level];
		nr = btrfs_header_nritems(eb);
		path->slots[level]++;
		slot = path->slots[level];
		if (slot >= nr || level == 0) {
			/*
			 * Don't free the root -  we will detect this
			 * condition after our loop and return a
			 * positive value for caller to stop walking the tree.
			 */
			if (level != root_level) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
				path->locks[level] = 0;

				free_extent_buffer(eb);
				path->nodes[level] = NULL;
				path->slots[level] = 0;
			}
		} else {
			/*
			 * We have a valid slot to walk back down
			 * from. Stop here so caller can process these
			 * new nodes.
			 */
			break;
		}

		level++;
	}

	eb = path->nodes[root_level];
	if (path->slots[root_level] >= btrfs_header_nritems(eb))
		return 1;

	return 0;
}

/*
 * root_eb is the subtree root and is locked before this function is called.
 */
static int account_shared_subtree(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct extent_buffer *root_eb,
				  u64 root_gen,
				  int root_level)
{
	int ret = 0;
	int level;
	struct extent_buffer *eb = root_eb;
	struct btrfs_path *path = NULL;

	BUG_ON(root_level < 0 || root_level > BTRFS_MAX_LEVEL);
	BUG_ON(root_eb == NULL);

	if (!root->fs_info->quota_enabled)
		return 0;

	if (!extent_buffer_uptodate(root_eb)) {
		ret = btrfs_read_buffer(root_eb, root_gen);
		if (ret)
			goto out;
	}

	if (root_level == 0) {
		ret = account_leaf_items(trans, root, root_eb);
		goto out;
	}

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	/*
	 * Walk down the tree.  Missing extent blocks are filled in as
	 * we go. Metadata is accounted every time we read a new
	 * extent block.
	 *
	 * When we reach a leaf, we account for file extent items in it,
	 * walk back up the tree (adjusting slot pointers as we go)
	 * and restart the search process.
	 */
	extent_buffer_get(root_eb); /* For path */
	path->nodes[root_level] = root_eb;
	path->slots[root_level] = 0;
	path->locks[root_level] = 0; /* so release_path doesn't try to unlock */
walk_down:
	level = root_level;
	while (level >= 0) {
		if (path->nodes[level] == NULL) {
			int parent_slot;
			u64 child_gen;
			u64 child_bytenr;

			/* We need to get child blockptr/gen from
			 * parent before we can read it. */
			eb = path->nodes[level + 1];
			parent_slot = path->slots[level + 1];
			child_bytenr = btrfs_node_blockptr(eb, parent_slot);
			child_gen = btrfs_node_ptr_generation(eb, parent_slot);

7599
			eb = read_tree_block(root, child_bytenr, child_gen);
7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614
			if (!eb || !extent_buffer_uptodate(eb)) {
				ret = -EIO;
				goto out;
			}

			path->nodes[level] = eb;
			path->slots[level] = 0;

			btrfs_tree_read_lock(eb);
			btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK);
			path->locks[level] = BTRFS_READ_LOCK_BLOCKING;

			ret = btrfs_qgroup_record_ref(trans, root->fs_info,
						root->objectid,
						child_bytenr,
7615
						root->nodesize,
7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646
						BTRFS_QGROUP_OPER_SUB_SUBTREE,
						0);
			if (ret)
				goto out;

		}

		if (level == 0) {
			ret = account_leaf_items(trans, root, path->nodes[level]);
			if (ret)
				goto out;

			/* Nonzero return here means we completed our search */
			ret = adjust_slots_upwards(root, path, root_level);
			if (ret)
				break;

			/* Restart search with new slots */
			goto walk_down;
		}

		level--;
	}

	ret = 0;
out:
	btrfs_free_path(path);

	return ret;
}

Y
Yan Zheng 已提交
7647
/*
L
Liu Bo 已提交
7648
 * helper to process tree block while walking down the tree.
7649 7650 7651 7652 7653
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
7654
 */
7655
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
7656
				   struct btrfs_root *root,
7657
				   struct btrfs_path *path,
7658
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
7659
{
7660 7661 7662
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
7663 7664
	int ret;

7665 7666 7667
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
7668

7669 7670 7671 7672
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
7673 7674 7675
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
7676 7677
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
7678
					       eb->start, level, 1,
7679 7680
					       &wc->refs[level],
					       &wc->flags[level]);
7681 7682 7683
		BUG_ON(ret == -ENOMEM);
		if (ret)
			return ret;
7684 7685
		BUG_ON(wc->refs[level] == 0);
	}
7686

7687 7688 7689
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
7690

7691
		if (path->locks[level] && !wc->keep_locks) {
7692
			btrfs_tree_unlock_rw(eb, path->locks[level]);
7693 7694 7695 7696
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
7697

7698 7699 7700
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
7701
		ret = btrfs_inc_ref(trans, root, eb, 1);
7702
		BUG_ON(ret); /* -ENOMEM */
7703
		ret = btrfs_dec_ref(trans, root, eb, 0);
7704
		BUG_ON(ret); /* -ENOMEM */
7705
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
7706 7707
						  eb->len, flag,
						  btrfs_header_level(eb), 0);
7708
		BUG_ON(ret); /* -ENOMEM */
7709 7710 7711 7712 7713 7714 7715 7716
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
7717
		btrfs_tree_unlock_rw(eb, path->locks[level]);
7718 7719 7720 7721 7722
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
7723
/*
L
Liu Bo 已提交
7724
 * helper to process tree block pointer.
Y
Yan, Zheng 已提交
7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
7739
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
7740 7741 7742 7743 7744 7745 7746 7747 7748 7749
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;
7750
	bool need_account = false;
Y
Yan, Zheng 已提交
7751 7752 7753 7754 7755 7756 7757 7758 7759

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
7760 7761
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
7762
		return 1;
7763
	}
Y
Yan, Zheng 已提交
7764 7765

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
7766
	blocksize = root->nodesize;
Y
Yan, Zheng 已提交
7767

7768
	next = btrfs_find_tree_block(root, bytenr);
Y
Yan, Zheng 已提交
7769
	if (!next) {
7770
		next = btrfs_find_create_tree_block(root, bytenr);
7771 7772
		if (!next)
			return -ENOMEM;
7773 7774
		btrfs_set_buffer_lockdep_class(root->root_key.objectid, next,
					       level - 1);
Y
Yan, Zheng 已提交
7775 7776 7777 7778 7779
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

7780
	ret = btrfs_lookup_extent_info(trans, root, bytenr, level - 1, 1,
7781 7782
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
7783 7784 7785 7786 7787
	if (ret < 0) {
		btrfs_tree_unlock(next);
		return ret;
	}

7788 7789 7790 7791
	if (unlikely(wc->refs[level - 1] == 0)) {
		btrfs_err(root->fs_info, "Missing references.");
		BUG();
	}
7792
	*lookup_info = 0;
Y
Yan, Zheng 已提交
7793

7794
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
7795
		if (wc->refs[level - 1] > 1) {
7796
			need_account = true;
7797 7798 7799 7800
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
7814 7815 7816 7817
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
7818 7819
	}

7820
	if (!btrfs_buffer_uptodate(next, generation, 0)) {
Y
Yan, Zheng 已提交
7821 7822 7823
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
7824
		*lookup_info = 1;
Y
Yan, Zheng 已提交
7825 7826 7827 7828 7829
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
7830
		next = read_tree_block(root, bytenr, generation);
7831 7832
		if (!next || !extent_buffer_uptodate(next)) {
			free_extent_buffer(next);
7833
			return -EIO;
7834
		}
Y
Yan, Zheng 已提交
7835 7836 7837 7838 7839 7840 7841 7842
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
7843
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Y
Yan, Zheng 已提交
7844 7845 7846 7847 7848 7849 7850
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
7851 7852 7853 7854 7855 7856 7857 7858
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
7859

7860 7861 7862 7863 7864 7865 7866 7867 7868 7869
		if (need_account) {
			ret = account_shared_subtree(trans, root, next,
						     generation, level - 1);
			if (ret) {
				printk_ratelimited(KERN_ERR "BTRFS: %s Error "
					"%d accounting shared subtree. Quota "
					"is out of sync, rescan required.\n",
					root->fs_info->sb->s_id, ret);
			}
		}
7870
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
A
Arne Jansen 已提交
7871
				root->root_key.objectid, level - 1, 0, 0);
7872
		BUG_ON(ret); /* -ENOMEM */
Y
Yan, Zheng 已提交
7873 7874 7875
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
7876
	*lookup_info = 1;
Y
Yan, Zheng 已提交
7877 7878 7879
	return 1;
}

7880
/*
L
Liu Bo 已提交
7881
 * helper to process tree block while walking up the tree.
7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892 7893 7894 7895 7896
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
7897
	int ret;
7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
7924
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7925 7926

			ret = btrfs_lookup_extent_info(trans, root,
7927
						       eb->start, level, 1,
7928 7929
						       &wc->refs[level],
						       &wc->flags[level]);
7930 7931
			if (ret < 0) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
L
Liu Bo 已提交
7932
				path->locks[level] = 0;
7933 7934
				return ret;
			}
7935 7936
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
7937
				btrfs_tree_unlock_rw(eb, path->locks[level]);
L
Liu Bo 已提交
7938
				path->locks[level] = 0;
7939 7940
				return 1;
			}
Y
Yan Zheng 已提交
7941
		}
7942
	}
Y
Yan Zheng 已提交
7943

7944 7945
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
7946

7947 7948 7949
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
7950
				ret = btrfs_dec_ref(trans, root, eb, 1);
7951
			else
7952
				ret = btrfs_dec_ref(trans, root, eb, 0);
7953
			BUG_ON(ret); /* -ENOMEM */
7954 7955 7956 7957 7958 7959 7960
			ret = account_leaf_items(trans, root, eb);
			if (ret) {
				printk_ratelimited(KERN_ERR "BTRFS: %s Error "
					"%d accounting leaf items. Quota "
					"is out of sync, rescan required.\n",
					root->fs_info->sb->s_id, ret);
			}
7961 7962 7963 7964 7965 7966
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
7967
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
7984 7985
	}

7986
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
7987 7988 7989
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
7990
	return 0;
7991 7992 7993 7994 7995 7996 7997 7998
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
7999
	int lookup_info = 1;
8000 8001 8002
	int ret;

	while (level >= 0) {
8003
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
8004 8005 8006 8007 8008 8009
		if (ret > 0)
			break;

		if (level == 0)
			break;

8010 8011 8012 8013
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

8014
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
8015 8016 8017
		if (ret > 0) {
			path->slots[level]++;
			continue;
8018 8019
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
8020
		level = wc->level;
Y
Yan Zheng 已提交
8021 8022 8023 8024
	}
	return 0;
}

C
Chris Mason 已提交
8025
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
8026
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
8027
				 struct btrfs_path *path,
8028
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
8029
{
8030
	int level = wc->level;
C
Chris Mason 已提交
8031
	int ret;
8032

8033 8034 8035 8036 8037 8038
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
8039 8040
			return 0;
		} else {
8041 8042 8043
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
8044

8045
			if (path->locks[level]) {
8046 8047
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
8048
				path->locks[level] = 0;
Y
Yan Zheng 已提交
8049
			}
8050 8051 8052
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
8053 8054 8055 8056 8057
		}
	}
	return 1;
}

C
Chris Mason 已提交
8058
/*
8059 8060 8061 8062 8063 8064 8065 8066 8067
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
D
David Sterba 已提交
8068 8069
 *
 * If called with for_reloc == 0, may exit early with -EAGAIN
C
Chris Mason 已提交
8070
 */
8071
int btrfs_drop_snapshot(struct btrfs_root *root,
A
Arne Jansen 已提交
8072 8073
			 struct btrfs_block_rsv *block_rsv, int update_ref,
			 int for_reloc)
C
Chris Mason 已提交
8074
{
8075
	struct btrfs_path *path;
8076 8077
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
8078
	struct btrfs_root_item *root_item = &root->root_item;
8079 8080 8081 8082 8083
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
8084
	bool root_dropped = false;
C
Chris Mason 已提交
8085

8086 8087
	btrfs_debug(root->fs_info, "Drop subvolume %llu", root->objectid);

8088
	path = btrfs_alloc_path();
8089 8090 8091 8092
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
C
Chris Mason 已提交
8093

8094
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
8095 8096
	if (!wc) {
		btrfs_free_path(path);
8097 8098
		err = -ENOMEM;
		goto out;
8099
	}
8100

8101
	trans = btrfs_start_transaction(tree_root, 0);
8102 8103 8104 8105
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
		goto out_free;
	}
8106

8107 8108
	if (block_rsv)
		trans->block_rsv = block_rsv;
8109

8110
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
8111
		level = btrfs_header_level(root->node);
8112 8113
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
8114
		path->slots[level] = 0;
8115
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
8116 8117
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
8118 8119
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
8120 8121 8122
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

8123
		level = root_item->drop_level;
8124
		BUG_ON(level == 0);
8125
		path->lowest_level = level;
8126 8127 8128 8129
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
8130
			goto out_end_trans;
8131
		}
Y
Yan, Zheng 已提交
8132
		WARN_ON(ret > 0);
8133

8134 8135 8136 8137
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
8138
		btrfs_unlock_up_safe(path, 0);
8139 8140 8141 8142 8143

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);
8144
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
8145 8146 8147

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
8148
						level, 1, &wc->refs[level],
8149
						&wc->flags[level]);
8150 8151 8152 8153
			if (ret < 0) {
				err = ret;
				goto out_end_trans;
			}
8154 8155 8156 8157 8158 8159
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
8160
			path->locks[level] = 0;
8161 8162 8163
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
8164
	}
8165 8166 8167 8168 8169 8170

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
A
Arne Jansen 已提交
8171
	wc->for_reloc = for_reloc;
Y
Yan, Zheng 已提交
8172
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
8173

C
Chris Mason 已提交
8174
	while (1) {
D
David Sterba 已提交
8175

8176 8177 8178
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
8179
			break;
8180
		}
C
Chris Mason 已提交
8181

8182 8183 8184
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
8185
			break;
8186 8187 8188 8189
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
8190 8191
			break;
		}
8192 8193 8194 8195 8196 8197 8198 8199 8200 8201

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
8202 8203
		if (btrfs_should_end_transaction(trans, tree_root) ||
		    (!for_reloc && btrfs_need_cleaner_sleep(root))) {
8204 8205 8206
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
8207 8208 8209 8210 8211
			if (ret) {
				btrfs_abort_transaction(trans, tree_root, ret);
				err = ret;
				goto out_end_trans;
			}
8212

8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230
			/*
			 * Qgroup update accounting is run from
			 * delayed ref handling. This usually works
			 * out because delayed refs are normally the
			 * only way qgroup updates are added. However,
			 * we may have added updates during our tree
			 * walk so run qgroups here to make sure we
			 * don't lose any updates.
			 */
			ret = btrfs_delayed_qgroup_accounting(trans,
							      root->fs_info);
			if (ret)
				printk_ratelimited(KERN_ERR "BTRFS: Failure %d "
						   "running qgroup updates "
						   "during snapshot delete. "
						   "Quota is out of sync, "
						   "rescan required.\n", ret);

8231
			btrfs_end_transaction_throttle(trans, tree_root);
8232
			if (!for_reloc && btrfs_need_cleaner_sleep(root)) {
8233
				pr_debug("BTRFS: drop snapshot early exit\n");
8234 8235 8236 8237
				err = -EAGAIN;
				goto out_free;
			}

8238
			trans = btrfs_start_transaction(tree_root, 0);
8239 8240 8241 8242
			if (IS_ERR(trans)) {
				err = PTR_ERR(trans);
				goto out_free;
			}
8243 8244
			if (block_rsv)
				trans->block_rsv = block_rsv;
8245
		}
C
Chris Mason 已提交
8246
	}
8247
	btrfs_release_path(path);
8248 8249
	if (err)
		goto out_end_trans;
8250 8251

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
8252 8253 8254 8255
	if (ret) {
		btrfs_abort_transaction(trans, tree_root, ret);
		goto out_end_trans;
	}
8256

8257
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
8258 8259
		ret = btrfs_find_root(tree_root, &root->root_key, path,
				      NULL, NULL);
8260 8261 8262 8263 8264
		if (ret < 0) {
			btrfs_abort_transaction(trans, tree_root, ret);
			err = ret;
			goto out_end_trans;
		} else if (ret > 0) {
8265 8266 8267 8268 8269 8270 8271
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
8272 8273 8274
		}
	}

8275
	if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) {
8276
		btrfs_drop_and_free_fs_root(tree_root->fs_info, root);
8277 8278 8279
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
8280
		btrfs_put_fs_root(root);
8281
	}
8282
	root_dropped = true;
8283
out_end_trans:
8284 8285 8286 8287 8288 8289 8290 8291
	ret = btrfs_delayed_qgroup_accounting(trans, tree_root->fs_info);
	if (ret)
		printk_ratelimited(KERN_ERR "BTRFS: Failure %d "
				   "running qgroup updates "
				   "during snapshot delete. "
				   "Quota is out of sync, "
				   "rescan required.\n", ret);

8292
	btrfs_end_transaction_throttle(trans, tree_root);
8293
out_free:
8294
	kfree(wc);
8295
	btrfs_free_path(path);
8296
out:
8297 8298 8299 8300 8301 8302 8303
	/*
	 * So if we need to stop dropping the snapshot for whatever reason we
	 * need to make sure to add it back to the dead root list so that we
	 * keep trying to do the work later.  This also cleans up roots if we
	 * don't have it in the radix (like when we recover after a power fail
	 * or unmount) so we don't leak memory.
	 */
8304
	if (!for_reloc && root_dropped == false)
8305
		btrfs_add_dead_root(root);
8306
	if (err && err != -EAGAIN)
8307
		btrfs_std_error(root->fs_info, err);
8308
	return err;
C
Chris Mason 已提交
8309
}
C
Chris Mason 已提交
8310

8311 8312 8313 8314
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
A
Arne Jansen 已提交
8315
 * only used by relocation code
8316
 */
Y
Yan Zheng 已提交
8317 8318 8319 8320 8321 8322
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
8323
	struct walk_control *wc;
Y
Yan Zheng 已提交
8324 8325 8326 8327 8328
	int level;
	int parent_level;
	int ret = 0;
	int wret;

8329 8330
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
8331
	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
8332 8333
	if (!path)
		return -ENOMEM;
Y
Yan Zheng 已提交
8334

8335
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
T
Tsutomu Itoh 已提交
8336 8337 8338 8339
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
8340

8341
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
8342 8343 8344 8345 8346
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

8347
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
8348 8349 8350
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
8351
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
8352 8353 8354 8355 8356 8357 8358 8359

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
A
Arne Jansen 已提交
8360
	wc->for_reloc = 1;
Y
Yan, Zheng 已提交
8361
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
8362 8363

	while (1) {
8364 8365
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
8366 8367
			ret = wret;
			break;
8368
		}
Y
Yan Zheng 已提交
8369

8370
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
8371 8372 8373 8374 8375 8376
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

8377
	kfree(wc);
Y
Yan Zheng 已提交
8378 8379 8380 8381
	btrfs_free_path(path);
	return ret;
}

8382 8383 8384
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
8385
	u64 stripped;
8386

8387 8388 8389 8390 8391 8392 8393
	/*
	 * if restripe for this chunk_type is on pick target profile and
	 * return, otherwise do the usual balance
	 */
	stripped = get_restripe_target(root->fs_info, flags);
	if (stripped)
		return extended_to_chunk(stripped);
8394

8395
	num_devices = root->fs_info->fs_devices->rw_devices;
8396

8397
	stripped = BTRFS_BLOCK_GROUP_RAID0 |
D
David Woodhouse 已提交
8398
		BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 |
8399 8400
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

8425
		/* this is drive concat, leave it alone */
8426
	}
8427

8428 8429 8430
	return flags;
}

8431
static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
C
Chris Mason 已提交
8432
{
8433 8434
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
8435
	u64 min_allocable_bytes;
8436
	int ret = -ENOSPC;
C
Chris Mason 已提交
8437

8438

8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
		min_allocable_bytes = 1 * 1024 * 1024;
	else
		min_allocable_bytes = 0;

8451 8452
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
8453 8454 8455 8456 8457 8458

	if (cache->ro) {
		ret = 0;
		goto out;
	}

8459 8460 8461 8462
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
8463 8464
	    sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
	    min_allocable_bytes <= sinfo->total_bytes) {
8465 8466
		sinfo->bytes_readonly += num_bytes;
		cache->ro = 1;
8467
		list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
8468 8469
		ret = 0;
	}
8470
out:
8471 8472 8473 8474
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
8475

8476 8477
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
8478

8479 8480 8481 8482
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
8483

8484
	BUG_ON(cache->ro);
C
Chris Mason 已提交
8485

C
Chris Mason 已提交
8486
	trans = btrfs_join_transaction(root);
8487 8488
	if (IS_ERR(trans))
		return PTR_ERR(trans);
8489

8490
	ret = set_block_group_ro(cache, 0);
8491 8492 8493
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
8494
	ret = do_chunk_alloc(trans, root, alloc_flags,
8495
			     CHUNK_ALLOC_FORCE);
8496 8497
	if (ret < 0)
		goto out;
8498
	ret = set_block_group_ro(cache, 0);
8499
out:
8500 8501 8502 8503 8504
	if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) {
		alloc_flags = update_block_group_flags(root, cache->flags);
		check_system_chunk(trans, root, alloc_flags);
	}

8505 8506 8507
	btrfs_end_transaction(trans, root);
	return ret;
}
8508

8509 8510 8511 8512
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
8513
	return do_chunk_alloc(trans, root, alloc_flags,
8514
			      CHUNK_ALLOC_FORCE);
8515 8516
}

8517 8518
/*
 * helper to account the unused space of all the readonly block group in the
8519
 * space_info. takes mirrors into account.
8520
 */
8521
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
8522 8523 8524 8525 8526
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

8527 8528 8529 8530 8531 8532
	/* It's df, we don't care if it's racey */
	if (list_empty(&sinfo->ro_bgs))
		return 0;

	spin_lock(&sinfo->lock);
	list_for_each_entry(block_group, &sinfo->ro_bgs, ro_list) {
8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}
	spin_unlock(&sinfo->lock);

	return free_bytes;
}

8558
void btrfs_set_block_group_rw(struct btrfs_root *root,
8559
			      struct btrfs_block_group_cache *cache)
8560
{
8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
8572
	list_del_init(&cache->ro_list);
8573 8574
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
8575 8576
}

8577 8578 8579 8580 8581 8582 8583
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
8584
{
8585 8586 8587 8588
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
8589
	struct btrfs_trans_handle *trans;
8590
	u64 min_free;
J
Josef Bacik 已提交
8591 8592
	u64 dev_min = 1;
	u64 dev_nr = 0;
8593
	u64 target;
8594
	int index;
8595 8596
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
8597

8598
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
8599

8600 8601 8602
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
Z
Zheng Yan 已提交
8603

8604 8605
	min_free = btrfs_block_group_used(&block_group->item);

8606
	/* no bytes used, we're good */
8607
	if (!min_free)
Z
Zheng Yan 已提交
8608 8609
		goto out;

8610 8611
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
8612

8613
	full = space_info->full;
8614

8615 8616
	/*
	 * if this is the last block group we have in this space, we can't
8617 8618 8619 8620
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
8621
	 */
8622
	if ((space_info->total_bytes != block_group->key.offset) &&
8623 8624 8625
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
8626 8627
		spin_unlock(&space_info->lock);
		goto out;
8628
	}
8629
	spin_unlock(&space_info->lock);
8630

8631 8632 8633
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
8634 8635 8636
	 * alloc devices and guess if we have enough space.  if this block
	 * group is going to be restriped, run checks against the target
	 * profile instead of the current one.
8637 8638
	 */
	ret = -1;
8639

8640 8641 8642 8643 8644 8645 8646 8647
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
8648 8649
	target = get_restripe_target(root->fs_info, block_group->flags);
	if (target) {
8650
		index = __get_raid_index(extended_to_chunk(target));
8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661
	} else {
		/*
		 * this is just a balance, so if we were marked as full
		 * we know there is no space for a new chunk
		 */
		if (full)
			goto out;

		index = get_block_group_index(block_group);
	}

8662
	if (index == BTRFS_RAID_RAID10) {
8663
		dev_min = 4;
J
Josef Bacik 已提交
8664 8665
		/* Divide by 2 */
		min_free >>= 1;
8666
	} else if (index == BTRFS_RAID_RAID1) {
8667
		dev_min = 2;
8668
	} else if (index == BTRFS_RAID_DUP) {
J
Josef Bacik 已提交
8669 8670
		/* Multiply by 2 */
		min_free <<= 1;
8671
	} else if (index == BTRFS_RAID_RAID0) {
8672
		dev_min = fs_devices->rw_devices;
8673
		min_free = div64_u64(min_free, dev_min);
8674 8675
	}

8676 8677 8678 8679 8680 8681 8682
	/* We need to do this so that we can look at pending chunks */
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans)) {
		ret = PTR_ERR(trans);
		goto out;
	}

8683 8684
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
8685
		u64 dev_offset;
8686

8687 8688 8689 8690
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
8691 8692
		if (device->total_bytes > device->bytes_used + min_free &&
		    !device->is_tgtdev_for_dev_replace) {
8693
			ret = find_free_dev_extent(trans, device, min_free,
8694
						   &dev_offset, NULL);
8695
			if (!ret)
8696 8697 8698
				dev_nr++;

			if (dev_nr >= dev_min)
8699
				break;
8700

8701
			ret = -1;
8702
		}
8703
	}
8704
	mutex_unlock(&root->fs_info->chunk_mutex);
8705
	btrfs_end_transaction(trans, root);
8706
out:
8707
	btrfs_put_block_group(block_group);
8708 8709 8710
	return ret;
}

8711 8712
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
8713
{
8714
	int ret = 0;
8715 8716 8717
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
8718

8719 8720
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
8721 8722
		goto out;

C
Chris Mason 已提交
8723
	while (1) {
8724
		slot = path->slots[0];
8725
		leaf = path->nodes[0];
8726 8727 8728 8729 8730
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
8731
				goto out;
8732
			break;
8733
		}
8734
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
8735

8736
		if (found_key.objectid >= key->objectid &&
8737 8738 8739 8740
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
8741
		path->slots[0]++;
8742
	}
8743
out:
8744
	return ret;
8745 8746
}

8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
8781 8782 8783
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
8784
	struct btrfs_space_info *space_info;
8785
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
8786 8787
	struct rb_node *n;

8788
	down_write(&info->commit_root_sem);
8789 8790 8791 8792 8793 8794
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
8795
	up_write(&info->commit_root_sem);
8796

8797 8798 8799 8800 8801 8802 8803 8804 8805 8806
	spin_lock(&info->unused_bgs_lock);
	while (!list_empty(&info->unused_bgs)) {
		block_group = list_first_entry(&info->unused_bgs,
					       struct btrfs_block_group_cache,
					       bg_list);
		list_del_init(&block_group->bg_list);
		btrfs_put_block_group(block_group);
	}
	spin_unlock(&info->unused_bgs_lock);

Z
Zheng Yan 已提交
8807 8808 8809 8810 8811 8812
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
8813
		RB_CLEAR_NODE(&block_group->cache_node);
Y
Yan Zheng 已提交
8814 8815
		spin_unlock(&info->block_group_cache_lock);

8816
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
8817
		list_del(&block_group->list);
8818
		up_write(&block_group->space_info->groups_sem);
8819

J
Josef Bacik 已提交
8820
		if (block_group->cached == BTRFS_CACHE_STARTED)
8821
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
8822

8823 8824 8825 8826
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
8827 8828
		if (block_group->cached == BTRFS_CACHE_NO ||
		    block_group->cached == BTRFS_CACHE_ERROR)
8829 8830
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
8831
		btrfs_remove_free_space_cache(block_group);
8832
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
8833 8834

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
8835 8836
	}
	spin_unlock(&info->block_group_cache_lock);
8837 8838 8839 8840 8841 8842 8843 8844 8845

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

8846 8847
	release_global_block_rsv(info);

8848
	while (!list_empty(&info->space_info)) {
8849 8850
		int i;

8851 8852 8853
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
8854
		if (btrfs_test_opt(info->tree_root, ENOSPC_DEBUG)) {
8855
			if (WARN_ON(space_info->bytes_pinned > 0 ||
8856
			    space_info->bytes_reserved > 0 ||
8857
			    space_info->bytes_may_use > 0)) {
8858 8859
				dump_space_info(space_info, 0, 0);
			}
8860
		}
8861
		list_del(&space_info->list);
8862 8863
		for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
			struct kobject *kobj;
8864 8865 8866
			kobj = space_info->block_group_kobjs[i];
			space_info->block_group_kobjs[i] = NULL;
			if (kobj) {
8867 8868 8869 8870 8871 8872
				kobject_del(kobj);
				kobject_put(kobj);
			}
		}
		kobject_del(&space_info->kobj);
		kobject_put(&space_info->kobj);
8873
	}
Z
Zheng Yan 已提交
8874 8875 8876
	return 0;
}

8877 8878 8879 8880
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);
8881
	bool first = false;
8882 8883

	down_write(&space_info->groups_sem);
8884 8885 8886 8887 8888 8889
	if (list_empty(&space_info->block_groups[index]))
		first = true;
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);

	if (first) {
8890
		struct raid_kobject *rkobj;
8891 8892
		int ret;

8893 8894 8895 8896 8897 8898 8899
		rkobj = kzalloc(sizeof(*rkobj), GFP_NOFS);
		if (!rkobj)
			goto out_err;
		rkobj->raid_type = index;
		kobject_init(&rkobj->kobj, &btrfs_raid_ktype);
		ret = kobject_add(&rkobj->kobj, &space_info->kobj,
				  "%s", get_raid_name(index));
8900
		if (ret) {
8901 8902
			kobject_put(&rkobj->kobj);
			goto out_err;
8903
		}
8904
		space_info->block_group_kobjs[index] = &rkobj->kobj;
8905
	}
8906 8907 8908 8909

	return;
out_err:
	pr_warn("BTRFS: failed to add kobject for block cache. ignoring.\n");
8910 8911
}

8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930 8931 8932 8933 8934 8935 8936 8937 8938
static struct btrfs_block_group_cache *
btrfs_create_block_group_cache(struct btrfs_root *root, u64 start, u64 size)
{
	struct btrfs_block_group_cache *cache;

	cache = kzalloc(sizeof(*cache), GFP_NOFS);
	if (!cache)
		return NULL;

	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return NULL;
	}

	cache->key.objectid = start;
	cache->key.offset = size;
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;

	cache->sectorsize = root->sectorsize;
	cache->fs_info = root->fs_info;
	cache->full_stripe_len = btrfs_full_stripe_len(root,
					       &root->fs_info->mapping_tree,
					       start);
	atomic_set(&cache->count, 1);
	spin_lock_init(&cache->lock);
8939
	init_rwsem(&cache->data_rwsem);
8940 8941
	INIT_LIST_HEAD(&cache->list);
	INIT_LIST_HEAD(&cache->cluster_list);
8942
	INIT_LIST_HEAD(&cache->bg_list);
8943
	INIT_LIST_HEAD(&cache->ro_list);
8944
	INIT_LIST_HEAD(&cache->dirty_list);
8945
	btrfs_init_free_space_ctl(cache);
8946
	atomic_set(&cache->trimming, 0);
8947 8948 8949 8950

	return cache;
}

C
Chris Mason 已提交
8951 8952 8953 8954 8955
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
8956
	struct btrfs_fs_info *info = root->fs_info;
8957
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
8958 8959
	struct btrfs_key key;
	struct btrfs_key found_key;
8960
	struct extent_buffer *leaf;
8961 8962
	int need_clear = 0;
	u64 cache_gen;
8963

C
Chris Mason 已提交
8964
	root = info->extent_root;
C
Chris Mason 已提交
8965
	key.objectid = 0;
8966
	key.offset = 0;
8967
	key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
C
Chris Mason 已提交
8968 8969 8970
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
J
Josef Bacik 已提交
8971
	path->reada = 1;
C
Chris Mason 已提交
8972

8973
	cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
8974
	if (btrfs_test_opt(root, SPACE_CACHE) &&
8975
	    btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
8976
		need_clear = 1;
8977 8978
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
8979

C
Chris Mason 已提交
8980
	while (1) {
8981
		ret = find_first_block_group(root, path, &key);
8982 8983
		if (ret > 0)
			break;
8984 8985
		if (ret != 0)
			goto error;
8986

8987 8988
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
8989 8990 8991

		cache = btrfs_create_block_group_cache(root, found_key.objectid,
						       found_key.offset);
C
Chris Mason 已提交
8992
		if (!cache) {
8993
			ret = -ENOMEM;
8994
			goto error;
C
Chris Mason 已提交
8995
		}
8996

8997 8998 8999 9000 9001 9002 9003 9004 9005 9006 9007 9008
		if (need_clear) {
			/*
			 * When we mount with old space cache, we need to
			 * set BTRFS_DC_CLEAR and set dirty flag.
			 *
			 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
			 *    truncate the old free space cache inode and
			 *    setup a new one.
			 * b) Setting 'dirty flag' makes sure that we flush
			 *    the new space cache info onto disk.
			 */
			if (btrfs_test_opt(root, SPACE_CACHE))
9009
				cache->disk_cache_state = BTRFS_DC_CLEAR;
9010
		}
9011

9012 9013 9014
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
9015
		cache->flags = btrfs_block_group_flags(&cache->item);
9016

C
Chris Mason 已提交
9017
		key.objectid = found_key.objectid + found_key.offset;
9018
		btrfs_release_path(path);
9019

9020 9021 9022 9023 9024
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
9025 9026 9027 9028 9029 9030 9031
		ret = exclude_super_stripes(root, cache);
		if (ret) {
			/*
			 * We may have excluded something, so call this just in
			 * case.
			 */
			free_excluded_extents(root, cache);
9032
			btrfs_put_block_group(cache);
9033 9034
			goto error;
		}
9035

J
Josef Bacik 已提交
9036 9037 9038 9039 9040 9041 9042 9043
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
9044
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
9045
			cache->cached = BTRFS_CACHE_FINISHED;
9046
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
9047
		} else if (btrfs_block_group_used(&cache->item) == 0) {
9048
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
9049 9050 9051 9052 9053
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
9054
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
9055
		}
9056

9057 9058 9059 9060 9061 9062 9063
		ret = btrfs_add_block_group_cache(root->fs_info, cache);
		if (ret) {
			btrfs_remove_free_space_cache(cache);
			btrfs_put_block_group(cache);
			goto error;
		}

9064 9065 9066
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
9067 9068 9069 9070 9071
		if (ret) {
			btrfs_remove_free_space_cache(cache);
			spin_lock(&info->block_group_cache_lock);
			rb_erase(&cache->cache_node,
				 &info->block_group_cache_tree);
9072
			RB_CLEAR_NODE(&cache->cache_node);
9073 9074 9075 9076 9077
			spin_unlock(&info->block_group_cache_lock);
			btrfs_put_block_group(cache);
			goto error;
		}

9078
		cache->space_info = space_info;
9079
		spin_lock(&cache->space_info->lock);
9080
		cache->space_info->bytes_readonly += cache->bytes_super;
9081 9082
		spin_unlock(&cache->space_info->lock);

9083
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
9084

9085
		set_avail_alloc_bits(root->fs_info, cache->flags);
9086
		if (btrfs_chunk_readonly(root, cache->key.objectid)) {
9087
			set_block_group_ro(cache, 1);
9088 9089 9090 9091 9092 9093 9094 9095 9096 9097
		} else if (btrfs_block_group_used(&cache->item) == 0) {
			spin_lock(&info->unused_bgs_lock);
			/* Should always be true but just in case. */
			if (list_empty(&cache->bg_list)) {
				btrfs_get_block_group(cache);
				list_add_tail(&cache->bg_list,
					      &info->unused_bgs);
			}
			spin_unlock(&info->unused_bgs_lock);
		}
C
Chris Mason 已提交
9098
	}
9099 9100 9101 9102 9103

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
D
David Woodhouse 已提交
9104 9105
		       BTRFS_BLOCK_GROUP_RAID5 |
		       BTRFS_BLOCK_GROUP_RAID6 |
9106 9107 9108 9109 9110 9111
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
9112 9113 9114
		list_for_each_entry(cache,
				&space_info->block_groups[BTRFS_RAID_RAID0],
				list)
9115
			set_block_group_ro(cache, 1);
9116 9117 9118
		list_for_each_entry(cache,
				&space_info->block_groups[BTRFS_RAID_SINGLE],
				list)
9119
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
9120
	}
9121 9122

	init_global_block_rsv(info);
9123 9124
	ret = 0;
error:
C
Chris Mason 已提交
9125
	btrfs_free_path(path);
9126
	return ret;
C
Chris Mason 已提交
9127
}
9128

9129 9130 9131 9132 9133 9134 9135 9136 9137
void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root)
{
	struct btrfs_block_group_cache *block_group, *tmp;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
	struct btrfs_block_group_item item;
	struct btrfs_key key;
	int ret = 0;

9138
	list_for_each_entry_safe(block_group, tmp, &trans->new_bgs, bg_list) {
9139
		if (ret)
9140
			goto next;
9141 9142 9143 9144 9145 9146 9147 9148 9149 9150

		spin_lock(&block_group->lock);
		memcpy(&item, &block_group->item, sizeof(item));
		memcpy(&key, &block_group->key, sizeof(key));
		spin_unlock(&block_group->lock);

		ret = btrfs_insert_item(trans, extent_root, &key, &item,
					sizeof(item));
		if (ret)
			btrfs_abort_transaction(trans, extent_root, ret);
9151 9152 9153 9154
		ret = btrfs_finish_chunk_alloc(trans, extent_root,
					       key.objectid, key.offset);
		if (ret)
			btrfs_abort_transaction(trans, extent_root, ret);
9155 9156
next:
		list_del_init(&block_group->bg_list);
9157 9158 9159
	}
}

9160 9161
int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
9162
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
9163 9164 9165 9166 9167 9168 9169 9170
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

9171
	btrfs_set_log_full_commit(root->fs_info, trans);
9172

9173
	cache = btrfs_create_block_group_cache(root, chunk_offset, size);
J
Josef Bacik 已提交
9174 9175
	if (!cache)
		return -ENOMEM;
9176

9177 9178 9179 9180
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	btrfs_set_block_group_flags(&cache->item, type);

9181
	cache->flags = type;
9182
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
9183
	cache->cached = BTRFS_CACHE_FINISHED;
9184 9185 9186 9187 9188 9189 9190
	ret = exclude_super_stripes(root, cache);
	if (ret) {
		/*
		 * We may have excluded something, so call this just in
		 * case.
		 */
		free_excluded_extents(root, cache);
9191
		btrfs_put_block_group(cache);
9192 9193
		return ret;
	}
9194

J
Josef Bacik 已提交
9195 9196 9197
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

9198 9199
	free_excluded_extents(root, cache);

9200 9201 9202 9203 9204 9205 9206
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
	if (ret) {
		btrfs_remove_free_space_cache(cache);
		btrfs_put_block_group(cache);
		return ret;
	}

9207 9208
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
9209 9210 9211 9212 9213
	if (ret) {
		btrfs_remove_free_space_cache(cache);
		spin_lock(&root->fs_info->block_group_cache_lock);
		rb_erase(&cache->cache_node,
			 &root->fs_info->block_group_cache_tree);
9214
		RB_CLEAR_NODE(&cache->cache_node);
9215 9216 9217 9218
		spin_unlock(&root->fs_info->block_group_cache_lock);
		btrfs_put_block_group(cache);
		return ret;
	}
9219
	update_global_block_rsv(root->fs_info);
9220 9221

	spin_lock(&cache->space_info->lock);
9222
	cache->space_info->bytes_readonly += cache->bytes_super;
9223 9224
	spin_unlock(&cache->space_info->lock);

9225
	__link_block_group(cache->space_info, cache);
9226

9227
	list_add_tail(&cache->bg_list, &trans->new_bgs);
9228

C
Chris Mason 已提交
9229
	set_avail_alloc_bits(extent_root->fs_info, type);
9230

9231 9232
	return 0;
}
Z
Zheng Yan 已提交
9233

9234 9235
static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
9236 9237
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
9238

9239
	write_seqlock(&fs_info->profiles_lock);
9240 9241 9242 9243 9244 9245
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits &= ~extra_flags;
9246
	write_sequnlock(&fs_info->profiles_lock);
9247 9248
}

Z
Zheng Yan 已提交
9249
int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
9250 9251
			     struct btrfs_root *root, u64 group_start,
			     struct extent_map *em)
Z
Zheng Yan 已提交
9252 9253 9254
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
9255
	struct btrfs_free_cluster *cluster;
9256
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
9257
	struct btrfs_key key;
9258
	struct inode *inode;
9259
	struct kobject *kobj = NULL;
Z
Zheng Yan 已提交
9260
	int ret;
9261
	int index;
J
Josef Bacik 已提交
9262
	int factor;
9263
	struct btrfs_caching_control *caching_ctl = NULL;
9264
	bool remove_em;
Z
Zheng Yan 已提交
9265 9266 9267 9268 9269

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
9270
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
9271

9272 9273 9274 9275 9276 9277
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

Z
Zheng Yan 已提交
9278
	memcpy(&key, &block_group->key, sizeof(key));
9279
	index = get_block_group_index(block_group);
J
Josef Bacik 已提交
9280 9281 9282 9283 9284 9285
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
9286

9287 9288 9289 9290 9291 9292 9293 9294 9295 9296 9297 9298 9299 9300 9301
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
9302
	path = btrfs_alloc_path();
9303 9304 9305 9306
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
Z
Zheng Yan 已提交
9307

9308
	inode = lookup_free_space_inode(tree_root, block_group, path);
9309
	if (!IS_ERR(inode)) {
9310
		ret = btrfs_orphan_add(trans, inode);
9311 9312 9313 9314
		if (ret) {
			btrfs_add_delayed_iput(inode);
			goto out;
		}
9315 9316 9317 9318 9319 9320 9321 9322 9323 9324 9325 9326
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
9327
		btrfs_add_delayed_iput(inode);
9328 9329 9330 9331 9332 9333 9334 9335 9336 9337
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
9338
		btrfs_release_path(path);
9339 9340 9341 9342
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
9343
		btrfs_release_path(path);
9344 9345
	}

9346
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
9347 9348
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
9349
	RB_CLEAR_NODE(&block_group->cache_node);
9350 9351 9352

	if (root->fs_info->first_logical_byte == block_group->key.objectid)
		root->fs_info->first_logical_byte = (u64)-1;
9353
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
9354

9355
	down_write(&block_group->space_info->groups_sem);
9356 9357 9358 9359 9360
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
9361
	list_del_init(&block_group->ro_list);
9362
	if (list_empty(&block_group->space_info->block_groups[index])) {
9363 9364
		kobj = block_group->space_info->block_group_kobjs[index];
		block_group->space_info->block_group_kobjs[index] = NULL;
9365
		clear_avail_alloc_bits(root->fs_info, block_group->flags);
9366
	}
9367
	up_write(&block_group->space_info->groups_sem);
9368 9369 9370 9371
	if (kobj) {
		kobject_del(kobj);
		kobject_put(kobj);
	}
Z
Zheng Yan 已提交
9372

9373 9374
	if (block_group->has_caching_ctl)
		caching_ctl = get_caching_control(block_group);
J
Josef Bacik 已提交
9375
	if (block_group->cached == BTRFS_CACHE_STARTED)
9376
		wait_block_group_cache_done(block_group);
9377 9378 9379 9380 9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396 9397 9398
	if (block_group->has_caching_ctl) {
		down_write(&root->fs_info->commit_root_sem);
		if (!caching_ctl) {
			struct btrfs_caching_control *ctl;

			list_for_each_entry(ctl,
				    &root->fs_info->caching_block_groups, list)
				if (ctl->block_group == block_group) {
					caching_ctl = ctl;
					atomic_inc(&caching_ctl->count);
					break;
				}
		}
		if (caching_ctl)
			list_del_init(&caching_ctl->list);
		up_write(&root->fs_info->commit_root_sem);
		if (caching_ctl) {
			/* Once for the caching bgs list and once for us. */
			put_caching_control(caching_ctl);
			put_caching_control(caching_ctl);
		}
	}
J
Josef Bacik 已提交
9399

9400 9401 9402 9403 9404 9405 9406
	spin_lock(&trans->transaction->dirty_bgs_lock);
	if (!list_empty(&block_group->dirty_list)) {
		list_del_init(&block_group->dirty_list);
		btrfs_put_block_group(block_group);
	}
	spin_unlock(&trans->transaction->dirty_bgs_lock);

J
Josef Bacik 已提交
9407 9408
	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
9409 9410 9411
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
9412
	block_group->space_info->disk_total -= block_group->key.offset * factor;
Y
Yan Zheng 已提交
9413
	spin_unlock(&block_group->space_info->lock);
9414

9415 9416
	memcpy(&key, &block_group->key, sizeof(key));

9417
	lock_chunks(root);
9418 9419 9420 9421
	if (!list_empty(&em->list)) {
		/* We're in the transaction->pending_chunks list. */
		free_extent_map(em);
	}
9422 9423 9424 9425 9426 9427 9428 9429 9430 9431 9432 9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449 9450 9451 9452 9453 9454 9455 9456 9457 9458 9459 9460 9461 9462 9463 9464 9465 9466 9467 9468
	spin_lock(&block_group->lock);
	block_group->removed = 1;
	/*
	 * At this point trimming can't start on this block group, because we
	 * removed the block group from the tree fs_info->block_group_cache_tree
	 * so no one can't find it anymore and even if someone already got this
	 * block group before we removed it from the rbtree, they have already
	 * incremented block_group->trimming - if they didn't, they won't find
	 * any free space entries because we already removed them all when we
	 * called btrfs_remove_free_space_cache().
	 *
	 * And we must not remove the extent map from the fs_info->mapping_tree
	 * to prevent the same logical address range and physical device space
	 * ranges from being reused for a new block group. This is because our
	 * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
	 * completely transactionless, so while it is trimming a range the
	 * currently running transaction might finish and a new one start,
	 * allowing for new block groups to be created that can reuse the same
	 * physical device locations unless we take this special care.
	 */
	remove_em = (atomic_read(&block_group->trimming) == 0);
	/*
	 * Make sure a trimmer task always sees the em in the pinned_chunks list
	 * if it sees block_group->removed == 1 (needs to lock block_group->lock
	 * before checking block_group->removed).
	 */
	if (!remove_em) {
		/*
		 * Our em might be in trans->transaction->pending_chunks which
		 * is protected by fs_info->chunk_mutex ([lock|unlock]_chunks),
		 * and so is the fs_info->pinned_chunks list.
		 *
		 * So at this point we must be holding the chunk_mutex to avoid
		 * any races with chunk allocation (more specifically at
		 * volumes.c:contains_pending_extent()), to ensure it always
		 * sees the em, either in the pending_chunks list or in the
		 * pinned_chunks list.
		 */
		list_move_tail(&em->list, &root->fs_info->pinned_chunks);
	}
	spin_unlock(&block_group->lock);

	if (remove_em) {
		struct extent_map_tree *em_tree;

		em_tree = &root->fs_info->mapping_tree.map_tree;
		write_lock(&em_tree->lock);
9469 9470 9471 9472 9473
		/*
		 * The em might be in the pending_chunks list, so make sure the
		 * chunk mutex is locked, since remove_extent_mapping() will
		 * delete us from that list.
		 */
9474 9475 9476 9477 9478 9479
		remove_extent_mapping(em_tree, em);
		write_unlock(&em_tree->lock);
		/* once for the tree */
		free_extent_map(em);
	}

9480 9481
	unlock_chunks(root);

9482 9483
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
9496

9497 9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530 9531 9532 9533 9534 9535 9536 9537 9538 9539 9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556
/*
 * Process the unused_bgs list and remove any that don't have any allocated
 * space inside of them.
 */
void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_root *root = fs_info->extent_root;
	struct btrfs_trans_handle *trans;
	int ret = 0;

	if (!fs_info->open)
		return;

	spin_lock(&fs_info->unused_bgs_lock);
	while (!list_empty(&fs_info->unused_bgs)) {
		u64 start, end;

		block_group = list_first_entry(&fs_info->unused_bgs,
					       struct btrfs_block_group_cache,
					       bg_list);
		space_info = block_group->space_info;
		list_del_init(&block_group->bg_list);
		if (ret || btrfs_mixed_space_info(space_info)) {
			btrfs_put_block_group(block_group);
			continue;
		}
		spin_unlock(&fs_info->unused_bgs_lock);

		/* Don't want to race with allocators so take the groups_sem */
		down_write(&space_info->groups_sem);
		spin_lock(&block_group->lock);
		if (block_group->reserved ||
		    btrfs_block_group_used(&block_group->item) ||
		    block_group->ro) {
			/*
			 * We want to bail if we made new allocations or have
			 * outstanding allocations in this block group.  We do
			 * the ro check in case balance is currently acting on
			 * this block group.
			 */
			spin_unlock(&block_group->lock);
			up_write(&space_info->groups_sem);
			goto next;
		}
		spin_unlock(&block_group->lock);

		/* We don't want to force the issue, only flip if it's ok. */
		ret = set_block_group_ro(block_group, 0);
		up_write(&space_info->groups_sem);
		if (ret < 0) {
			ret = 0;
			goto next;
		}

		/*
		 * Want to do this before we do anything else so we can recover
		 * properly if we fail to join the transaction.
		 */
9557 9558
		/* 1 for btrfs_orphan_reserve_metadata() */
		trans = btrfs_start_transaction(root, 1);
9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570
		if (IS_ERR(trans)) {
			btrfs_set_block_group_rw(root, block_group);
			ret = PTR_ERR(trans);
			goto next;
		}

		/*
		 * We could have pending pinned extents for this block group,
		 * just delete them, we don't care about them anymore.
		 */
		start = block_group->key.objectid;
		end = start + block_group->key.offset - 1;
9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582
		/*
		 * Hold the unused_bg_unpin_mutex lock to avoid racing with
		 * btrfs_finish_extent_commit(). If we are at transaction N,
		 * another task might be running finish_extent_commit() for the
		 * previous transaction N - 1, and have seen a range belonging
		 * to the block group in freed_extents[] before we were able to
		 * clear the whole block group range from freed_extents[]. This
		 * means that task can lookup for the block group after we
		 * unpinned it from freed_extents[] and removed it, leading to
		 * a BUG_ON() at btrfs_unpin_extent_range().
		 */
		mutex_lock(&fs_info->unused_bg_unpin_mutex);
9583
		ret = clear_extent_bits(&fs_info->freed_extents[0], start, end,
9584
				  EXTENT_DIRTY, GFP_NOFS);
9585
		if (ret) {
9586
			mutex_unlock(&fs_info->unused_bg_unpin_mutex);
9587 9588 9589 9590
			btrfs_set_block_group_rw(root, block_group);
			goto end_trans;
		}
		ret = clear_extent_bits(&fs_info->freed_extents[1], start, end,
9591
				  EXTENT_DIRTY, GFP_NOFS);
9592
		if (ret) {
9593
			mutex_unlock(&fs_info->unused_bg_unpin_mutex);
9594 9595 9596
			btrfs_set_block_group_rw(root, block_group);
			goto end_trans;
		}
9597
		mutex_unlock(&fs_info->unused_bg_unpin_mutex);
9598 9599 9600 9601 9602 9603 9604 9605 9606 9607

		/* Reset pinned so btrfs_put_block_group doesn't complain */
		block_group->pinned = 0;

		/*
		 * Btrfs_remove_chunk will abort the transaction if things go
		 * horribly wrong.
		 */
		ret = btrfs_remove_chunk(trans, root,
					 block_group->key.objectid);
9608
end_trans:
9609 9610 9611 9612 9613 9614 9615 9616
		btrfs_end_transaction(trans, root);
next:
		btrfs_put_block_group(block_group);
		spin_lock(&fs_info->unused_bgs_lock);
	}
	spin_unlock(&fs_info->unused_bgs_lock);
}

9617 9618 9619
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
9620 9621 9622 9623
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
9624 9625
	int ret;

9626
	disk_super = fs_info->super_copy;
9627 9628
	if (!btrfs_super_root(disk_super))
		return 1;
9629

9630 9631 9632
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
9633

9634 9635
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
	ret = update_space_info(fs_info, flags, 0, 0, &space_info);
9636
	if (ret)
9637
		goto out;
9638

9639 9640 9641 9642 9643 9644 9645 9646 9647 9648 9649 9650 9651
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	}
out:
9652 9653 9654
	return ret;
}

L
liubo 已提交
9655 9656
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
9657
	return unpin_extent_range(root, start, end, false);
L
liubo 已提交
9658 9659
}

9660 9661 9662 9663 9664 9665 9666 9667
int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
9668
	u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
9669 9670
	int ret = 0;

9671 9672 9673 9674 9675 9676 9677
	/*
	 * try to trim all FS space, our block group may start from non-zero.
	 */
	if (range->len == total_bytes)
		cache = btrfs_lookup_first_block_group(fs_info, range->start);
	else
		cache = btrfs_lookup_block_group(fs_info, range->start);
9678 9679 9680 9681 9682 9683 9684 9685 9686 9687 9688 9689 9690

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
9691
				ret = cache_block_group(cache, 0);
9692 9693 9694 9695 9696 9697 9698 9699 9700
				if (ret) {
					btrfs_put_block_group(cache);
					break;
				}
				ret = wait_block_group_cache_done(cache);
				if (ret) {
					btrfs_put_block_group(cache);
					break;
				}
9701 9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719 9720
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

	range->len = trimmed;
	return ret;
}
9721 9722

/*
9723 9724 9725 9726 9727 9728
 * btrfs_{start,end}_write_no_snapshoting() are similar to
 * mnt_{want,drop}_write(), they are used to prevent some tasks from writing
 * data into the page cache through nocow before the subvolume is snapshoted,
 * but flush the data into disk after the snapshot creation, or to prevent
 * operations while snapshoting is ongoing and that cause the snapshot to be
 * inconsistent (writes followed by expanding truncates for example).
9729
 */
9730
void btrfs_end_write_no_snapshoting(struct btrfs_root *root)
9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741
{
	percpu_counter_dec(&root->subv_writers->counter);
	/*
	 * Make sure counter is updated before we wake up
	 * waiters.
	 */
	smp_mb();
	if (waitqueue_active(&root->subv_writers->wait))
		wake_up(&root->subv_writers->wait);
}

9742
int btrfs_start_write_no_snapshoting(struct btrfs_root *root)
9743
{
9744
	if (atomic_read(&root->will_be_snapshoted))
9745 9746 9747 9748 9749 9750 9751
		return 0;

	percpu_counter_inc(&root->subv_writers->counter);
	/*
	 * Make sure counter is updated before we check for snapshot creation.
	 */
	smp_mb();
9752
	if (atomic_read(&root->will_be_snapshoted)) {
9753
		btrfs_end_write_no_snapshoting(root);
9754 9755 9756 9757
		return 0;
	}
	return 1;
}