extent-tree.c 303.1 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
26
#include <linux/ratelimit.h>
27
#include <linux/percpu_counter.h>
28
#include "hash.h"
29
#include "tree-log.h"
30 31
#include "disk-io.h"
#include "print-tree.h"
32
#include "volumes.h"
D
David Woodhouse 已提交
33
#include "raid56.h"
34
#include "locking.h"
35
#include "free-space-cache.h"
36
#include "free-space-tree.h"
37
#include "math.h"
38
#include "sysfs.h"
J
Josef Bacik 已提交
39
#include "qgroup.h"
40

41 42
#undef SCRAMBLE_DELAYED_REFS

43 44
/*
 * control flags for do_chunk_alloc's force field
45 46 47 48 49 50 51 52 53
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
54 55
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
56 57 58
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
59 60
	CHUNK_ALLOC_LIMITED = 1,
	CHUNK_ALLOC_FORCE = 2,
61 62
};

63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
/*
 * Control how reservations are dealt with.
 *
 * RESERVE_FREE - freeing a reservation.
 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
 *   ENOSPC accounting
 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
 *   bytes_may_use as the ENOSPC accounting is done elsewhere
 */
enum {
	RESERVE_FREE = 0,
	RESERVE_ALLOC = 1,
	RESERVE_ALLOC_NO_ACCOUNT = 2,
};

78 79 80
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root, u64 bytenr,
			      u64 num_bytes, int alloc);
81 82
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
83
				struct btrfs_delayed_ref_node *node, u64 parent,
84 85
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
86
				struct btrfs_delayed_extent_op *extra_op);
87 88 89 90 91 92 93 94 95 96 97 98
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
99
				     int level, struct btrfs_key *ins);
J
Josef Bacik 已提交
100
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
101 102
			  struct btrfs_root *extent_root, u64 flags,
			  int force);
103 104
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
105 106
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
107
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
108 109
				       u64 num_bytes, int reserve,
				       int delalloc);
110 111
static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes);
112 113
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved);
114 115 116 117 118 119 120 121 122 123
static int __reserve_metadata_bytes(struct btrfs_root *root,
				    struct btrfs_space_info *space_info,
				    u64 orig_bytes,
				    enum btrfs_reserve_flush_enum flush);
static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info,
				     struct btrfs_space_info *space_info,
				     u64 num_bytes);
static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info,
				     struct btrfs_space_info *space_info,
				     u64 num_bytes);
J
Josef Bacik 已提交
124

J
Josef Bacik 已提交
125 126 127 128
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
129 130
	return cache->cached == BTRFS_CACHE_FINISHED ||
		cache->cached == BTRFS_CACHE_ERROR;
J
Josef Bacik 已提交
131 132
}

J
Josef Bacik 已提交
133 134 135 136 137
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

138
void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
139 140 141 142 143 144
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
145 146 147
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
148
		kfree(cache->free_space_ctl);
149
		kfree(cache);
150
	}
151 152
}

J
Josef Bacik 已提交
153 154 155 156
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
157
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
184 185 186 187

	if (info->first_logical_byte > block_group->key.objectid)
		info->first_logical_byte = block_group->key.objectid;

J
Josef Bacik 已提交
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
229
	if (ret) {
230
		btrfs_get_block_group(ret);
231 232 233
		if (bytenr == 0 && info->first_logical_byte > ret->key.objectid)
			info->first_logical_byte = ret->key.objectid;
	}
J
Josef Bacik 已提交
234 235 236 237 238
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

239 240
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
241
{
242 243
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
244
			start, end, EXTENT_UPTODATE);
245
	set_extent_bits(&root->fs_info->freed_extents[1],
246
			start, end, EXTENT_UPTODATE);
247 248
	return 0;
}
J
Josef Bacik 已提交
249

250 251 252 253
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
254

255 256 257 258
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
259
			  start, end, EXTENT_UPTODATE);
260
	clear_extent_bits(&root->fs_info->freed_extents[1],
261
			  start, end, EXTENT_UPTODATE);
J
Josef Bacik 已提交
262 263
}

264 265
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
266 267 268 269 270 271
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

272 273 274 275 276
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
277 278
		if (ret)
			return ret;
279 280
	}

J
Josef Bacik 已提交
281 282 283 284 285
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
286 287
		if (ret)
			return ret;
288

J
Josef Bacik 已提交
289
		while (nr--) {
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
			u64 start, len;

			if (logical[nr] > cache->key.objectid +
			    cache->key.offset)
				continue;

			if (logical[nr] + stripe_len <= cache->key.objectid)
				continue;

			start = logical[nr];
			if (start < cache->key.objectid) {
				start = cache->key.objectid;
				len = (logical[nr] + stripe_len) - start;
			} else {
				len = min_t(u64, stripe_len,
					    cache->key.objectid +
					    cache->key.offset - start);
			}

			cache->bytes_super += len;
			ret = add_excluded_extent(root, start, len);
311 312 313 314
			if (ret) {
				kfree(logical);
				return ret;
			}
J
Josef Bacik 已提交
315
		}
316

J
Josef Bacik 已提交
317 318 319 320 321
		kfree(logical);
	}
	return 0;
}

322 323 324 325 326 327
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
328 329
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
#ifdef CONFIG_BTRFS_DEBUG
static void fragment_free_space(struct btrfs_root *root,
				struct btrfs_block_group_cache *block_group)
{
	u64 start = block_group->key.objectid;
	u64 len = block_group->key.offset;
	u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ?
		root->nodesize : root->sectorsize;
	u64 step = chunk << 1;

	while (len > chunk) {
		btrfs_remove_free_space(block_group, start, chunk);
		start += step;
		if (len < step)
			len = 0;
		else
			len -= step;
	}
}
#endif

J
Josef Bacik 已提交
366 367 368 369 370
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
371 372
u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
		       struct btrfs_fs_info *info, u64 start, u64 end)
J
Josef Bacik 已提交
373
{
J
Josef Bacik 已提交
374
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
375 376 377
	int ret;

	while (start < end) {
378
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
379
					    &extent_start, &extent_end,
380 381
					    EXTENT_DIRTY | EXTENT_UPTODATE,
					    NULL);
J
Josef Bacik 已提交
382 383 384
		if (ret)
			break;

385
		if (extent_start <= start) {
J
Josef Bacik 已提交
386 387 388
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
389
			total_added += size;
390 391
			ret = btrfs_add_free_space(block_group, start,
						   size);
392
			BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
393 394 395 396 397 398 399 400
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
401
		total_added += size;
402
		ret = btrfs_add_free_space(block_group, start, size);
403
		BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
404 405
	}

J
Josef Bacik 已提交
406
	return total_added;
J
Josef Bacik 已提交
407 408
}

O
Omar Sandoval 已提交
409
static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl)
410
{
411 412 413
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_root *extent_root;
414
	struct btrfs_path *path;
415
	struct extent_buffer *leaf;
416
	struct btrfs_key key;
J
Josef Bacik 已提交
417
	u64 total_found = 0;
418 419
	u64 last = 0;
	u32 nritems;
O
Omar Sandoval 已提交
420
	int ret;
421
	bool wakeup = true;
422

423 424 425 426
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

427 428
	path = btrfs_alloc_path();
	if (!path)
O
Omar Sandoval 已提交
429
		return -ENOMEM;
430

J
Josef Bacik 已提交
431
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
432

433 434 435 436 437 438 439 440 441
#ifdef CONFIG_BTRFS_DEBUG
	/*
	 * If we're fragmenting we don't want to make anybody think we can
	 * allocate from this block group until we've had a chance to fragment
	 * the free space.
	 */
	if (btrfs_should_fragment_free_space(extent_root, block_group))
		wakeup = false;
#endif
442
	/*
J
Josef Bacik 已提交
443 444 445 446
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
447 448
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
449
	path->search_commit_root = 1;
450
	path->reada = READA_FORWARD;
J
Josef Bacik 已提交
451

Y
Yan Zheng 已提交
452
	key.objectid = last;
453
	key.offset = 0;
454
	key.type = BTRFS_EXTENT_ITEM_KEY;
455

456
next:
457
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
458
	if (ret < 0)
O
Omar Sandoval 已提交
459
		goto out;
Y
Yan Zheng 已提交
460

461 462 463
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
464
	while (1) {
465
		if (btrfs_fs_closing(fs_info) > 1) {
466
			last = (u64)-1;
J
Josef Bacik 已提交
467
			break;
468
		}
J
Josef Bacik 已提交
469

470 471 472 473 474
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
475
				break;
J
Josef Bacik 已提交
476

477
			if (need_resched() ||
478
			    rwsem_is_contended(&fs_info->commit_root_sem)) {
479 480
				if (wakeup)
					caching_ctl->progress = last;
C
Chris Mason 已提交
481
				btrfs_release_path(path);
482
				up_read(&fs_info->commit_root_sem);
483
				mutex_unlock(&caching_ctl->mutex);
484
				cond_resched();
O
Omar Sandoval 已提交
485 486 487
				mutex_lock(&caching_ctl->mutex);
				down_read(&fs_info->commit_root_sem);
				goto next;
488
			}
489 490 491

			ret = btrfs_next_leaf(extent_root, path);
			if (ret < 0)
O
Omar Sandoval 已提交
492
				goto out;
493 494
			if (ret)
				break;
495 496 497
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
498
		}
J
Josef Bacik 已提交
499

500 501 502 503 504
		if (key.objectid < last) {
			key.objectid = last;
			key.offset = 0;
			key.type = BTRFS_EXTENT_ITEM_KEY;

505 506
			if (wakeup)
				caching_ctl->progress = last;
507 508 509 510
			btrfs_release_path(path);
			goto next;
		}

511 512
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
513
			continue;
514
		}
J
Josef Bacik 已提交
515

516
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
517
		    block_group->key.offset)
518
			break;
519

520 521
		if (key.type == BTRFS_EXTENT_ITEM_KEY ||
		    key.type == BTRFS_METADATA_ITEM_KEY) {
J
Josef Bacik 已提交
522 523 524
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
525 526
			if (key.type == BTRFS_METADATA_ITEM_KEY)
				last = key.objectid +
527
					fs_info->tree_root->nodesize;
528 529
			else
				last = key.objectid + key.offset;
J
Josef Bacik 已提交
530

O
Omar Sandoval 已提交
531
			if (total_found > CACHING_CTL_WAKE_UP) {
532
				total_found = 0;
533 534
				if (wakeup)
					wake_up(&caching_ctl->wait);
535
			}
J
Josef Bacik 已提交
536
		}
537 538
		path->slots[0]++;
	}
J
Josef Bacik 已提交
539
	ret = 0;
540

J
Josef Bacik 已提交
541 542 543
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
544
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
545

O
Omar Sandoval 已提交
546 547 548 549 550 551 552 553 554 555
out:
	btrfs_free_path(path);
	return ret;
}

static noinline void caching_thread(struct btrfs_work *work)
{
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
556
	struct btrfs_root *extent_root;
O
Omar Sandoval 已提交
557 558 559 560 561
	int ret;

	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
562
	extent_root = fs_info->extent_root;
O
Omar Sandoval 已提交
563 564 565 566

	mutex_lock(&caching_ctl->mutex);
	down_read(&fs_info->commit_root_sem);

567 568 569 570
	if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE))
		ret = load_free_space_tree(caching_ctl);
	else
		ret = load_extent_tree_free(caching_ctl);
O
Omar Sandoval 已提交
571

J
Josef Bacik 已提交
572
	spin_lock(&block_group->lock);
573
	block_group->caching_ctl = NULL;
O
Omar Sandoval 已提交
574
	block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED;
J
Josef Bacik 已提交
575
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
576

577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
#ifdef CONFIG_BTRFS_DEBUG
	if (btrfs_should_fragment_free_space(extent_root, block_group)) {
		u64 bytes_used;

		spin_lock(&block_group->space_info->lock);
		spin_lock(&block_group->lock);
		bytes_used = block_group->key.offset -
			btrfs_block_group_used(&block_group->item);
		block_group->space_info->bytes_used += bytes_used >> 1;
		spin_unlock(&block_group->lock);
		spin_unlock(&block_group->space_info->lock);
		fragment_free_space(extent_root, block_group);
	}
#endif

	caching_ctl->progress = (u64)-1;
593

594
	up_read(&fs_info->commit_root_sem);
O
Omar Sandoval 已提交
595
	free_excluded_extents(fs_info->extent_root, block_group);
596
	mutex_unlock(&caching_ctl->mutex);
O
Omar Sandoval 已提交
597

598 599 600
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
601
	btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
602 603
}

604 605
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     int load_cache_only)
J
Josef Bacik 已提交
606
{
607
	DEFINE_WAIT(wait);
608 609
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
610 611
	int ret = 0;

612
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
613 614
	if (!caching_ctl)
		return -ENOMEM;
615 616 617 618 619 620 621

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	atomic_set(&caching_ctl->count, 1);
622 623
	btrfs_init_work(&caching_ctl->work, btrfs_cache_helper,
			caching_thread, NULL, NULL);
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655

	spin_lock(&cache->lock);
	/*
	 * This should be a rare occasion, but this could happen I think in the
	 * case where one thread starts to load the space cache info, and then
	 * some other thread starts a transaction commit which tries to do an
	 * allocation while the other thread is still loading the space cache
	 * info.  The previous loop should have kept us from choosing this block
	 * group, but if we've moved to the state where we will wait on caching
	 * block groups we need to first check if we're doing a fast load here,
	 * so we can wait for it to finish, otherwise we could end up allocating
	 * from a block group who's cache gets evicted for one reason or
	 * another.
	 */
	while (cache->cached == BTRFS_CACHE_FAST) {
		struct btrfs_caching_control *ctl;

		ctl = cache->caching_ctl;
		atomic_inc(&ctl->count);
		prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&cache->lock);

		schedule();

		finish_wait(&ctl->wait, &wait);
		put_caching_control(ctl);
		spin_lock(&cache->lock);
	}

	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
		kfree(caching_ctl);
656
		return 0;
657 658 659 660 661
	}
	WARN_ON(cache->caching_ctl);
	cache->caching_ctl = caching_ctl;
	cache->cached = BTRFS_CACHE_FAST;
	spin_unlock(&cache->lock);
662

663
	if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
664
		mutex_lock(&caching_ctl->mutex);
665 666 667 668
		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
669
			cache->caching_ctl = NULL;
670 671
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
672
			caching_ctl->progress = (u64)-1;
673
		} else {
674 675 676 677 678
			if (load_cache_only) {
				cache->caching_ctl = NULL;
				cache->cached = BTRFS_CACHE_NO;
			} else {
				cache->cached = BTRFS_CACHE_STARTED;
679
				cache->has_caching_ctl = 1;
680
			}
681 682
		}
		spin_unlock(&cache->lock);
683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698
#ifdef CONFIG_BTRFS_DEBUG
		if (ret == 1 &&
		    btrfs_should_fragment_free_space(fs_info->extent_root,
						     cache)) {
			u64 bytes_used;

			spin_lock(&cache->space_info->lock);
			spin_lock(&cache->lock);
			bytes_used = cache->key.offset -
				btrfs_block_group_used(&cache->item);
			cache->space_info->bytes_used += bytes_used >> 1;
			spin_unlock(&cache->lock);
			spin_unlock(&cache->space_info->lock);
			fragment_free_space(fs_info->extent_root, cache);
		}
#endif
699 700
		mutex_unlock(&caching_ctl->mutex);

701
		wake_up(&caching_ctl->wait);
702
		if (ret == 1) {
703
			put_caching_control(caching_ctl);
704
			free_excluded_extents(fs_info->extent_root, cache);
705
			return 0;
706
		}
707 708
	} else {
		/*
709 710
		 * We're either using the free space tree or no caching at all.
		 * Set cached to the appropriate value and wakeup any waiters.
711 712 713 714 715 716 717
		 */
		spin_lock(&cache->lock);
		if (load_cache_only) {
			cache->caching_ctl = NULL;
			cache->cached = BTRFS_CACHE_NO;
		} else {
			cache->cached = BTRFS_CACHE_STARTED;
718
			cache->has_caching_ctl = 1;
719 720 721
		}
		spin_unlock(&cache->lock);
		wake_up(&caching_ctl->wait);
722 723
	}

724 725
	if (load_cache_only) {
		put_caching_control(caching_ctl);
726
		return 0;
J
Josef Bacik 已提交
727 728
	}

729
	down_write(&fs_info->commit_root_sem);
730
	atomic_inc(&caching_ctl->count);
731
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
732
	up_write(&fs_info->commit_root_sem);
733

734
	btrfs_get_block_group(cache);
735

736
	btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work);
J
Josef Bacik 已提交
737

738
	return ret;
739 740
}

J
Josef Bacik 已提交
741 742 743
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
744 745
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
746
{
J
Josef Bacik 已提交
747
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
748

J
Josef Bacik 已提交
749
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
750

J
Josef Bacik 已提交
751
	return cache;
C
Chris Mason 已提交
752 753
}

J
Josef Bacik 已提交
754
/*
755
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
756
 */
C
Chris Mason 已提交
757 758 759
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
760
{
J
Josef Bacik 已提交
761
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
762

J
Josef Bacik 已提交
763
	cache = block_group_cache_tree_search(info, bytenr, 1);
764

J
Josef Bacik 已提交
765
	return cache;
C
Chris Mason 已提交
766
}
767

J
Josef Bacik 已提交
768 769
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
770
{
J
Josef Bacik 已提交
771 772
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
773

774
	flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
775

776 777
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
778
		if (found->flags & flags) {
779
			rcu_read_unlock();
J
Josef Bacik 已提交
780
			return found;
781
		}
J
Josef Bacik 已提交
782
	}
783
	rcu_read_unlock();
J
Josef Bacik 已提交
784
	return NULL;
785 786
}

787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

802 803
/* simple helper to search for an existing data extent at a given offset */
int btrfs_lookup_data_extent(struct btrfs_root *root, u64 start, u64 len)
804 805 806
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
807
	struct btrfs_path *path;
808

Z
Zheng Yan 已提交
809
	path = btrfs_alloc_path();
810 811 812
	if (!path)
		return -ENOMEM;

813 814
	key.objectid = start;
	key.offset = len;
815
	key.type = BTRFS_EXTENT_ITEM_KEY;
816 817
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
818
	btrfs_free_path(path);
819 820 821
	return ret;
}

822
/*
823
 * helper function to lookup reference count and flags of a tree block.
824 825 826 827 828 829 830 831 832
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
833
			     u64 offset, int metadata, u64 *refs, u64 *flags)
834 835 836 837 838 839 840 841 842 843 844 845
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

846 847 848 849 850
	/*
	 * If we don't have skinny metadata, don't bother doing anything
	 * different
	 */
	if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA)) {
851
		offset = root->nodesize;
852 853 854
		metadata = 0;
	}

855 856 857 858 859 860 861 862
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
863 864 865 866 867 868 869 870 871

search_again:
	key.objectid = bytenr;
	key.offset = offset;
	if (metadata)
		key.type = BTRFS_METADATA_ITEM_KEY;
	else
		key.type = BTRFS_EXTENT_ITEM_KEY;

872 873 874 875 876
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

877
	if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) {
878 879 880 881 882 883
		if (path->slots[0]) {
			path->slots[0]--;
			btrfs_item_key_to_cpu(path->nodes[0], &key,
					      path->slots[0]);
			if (key.objectid == bytenr &&
			    key.type == BTRFS_EXTENT_ITEM_KEY &&
884
			    key.offset == root->nodesize)
885 886
				ret = 0;
		}
887 888
	}

889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927
	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

928
			btrfs_release_path(path);
929

930 931 932 933
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
934 935 936
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
937
			goto search_again;
938
		}
939
		spin_lock(&head->lock);
940 941 942 943 944 945
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
946
		spin_unlock(&head->lock);
947 948 949 950 951 952 953 954 955 956 957 958 959 960
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

961 962 963 964 965 966 967 968 969 970 971 972 973 974
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
993
 * When a tree block is COWed through a tree, there are four cases:
994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
1020 1021 1022
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
1023
 * - different files inside a single subvolume
1024 1025
 * - different offsets inside a file (bookend extents in file.c)
 *
1026
 * The extent ref structure for the implicit back refs has fields for:
1027 1028 1029
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
1030 1031
 * - original offset in the file
 * - how many bookend extents
1032
 *
1033 1034
 * The key offset for the implicit back refs is hash of the first
 * three fields.
1035
 *
1036
 * The extent ref structure for the full back refs has field for:
1037
 *
1038
 * - number of pointers in the tree leaf
1039
 *
1040 1041
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
1042
 *
1043 1044
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
1045
 *
1046
 *     (root_key.objectid, inode objectid, offset in file, 1)
1047
 *
1048 1049
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
1050
 *
1051
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
1052
 *
1053
 * Btree extents can be referenced by:
1054
 *
1055
 * - Different subvolumes
1056
 *
1057 1058 1059 1060
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
1061
 *
1062 1063 1064
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
1065
 */
Z
Zheng Yan 已提交
1066

1067 1068 1069 1070 1071
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
1072
{
1073 1074 1075 1076 1077
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
1078
	struct btrfs_key key;
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
1098
				BUG_ON(ret > 0); /* Corruption */
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
1114
	btrfs_release_path(path);
1115 1116 1117 1118 1119 1120 1121 1122 1123

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
1124
	BUG_ON(ret); /* Corruption */
1125

1126
	btrfs_extend_item(root, path, new_size);
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
1156
	high_crc = btrfs_crc32c(high_crc, &lenum, sizeof(lenum));
1157
	lenum = cpu_to_le64(owner);
1158
	low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum));
1159
	lenum = cpu_to_le64(offset);
1160
	low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum));
1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1193
	struct extent_buffer *leaf;
1194
	u32 nritems;
1195
	int ret;
1196 1197
	int recow;
	int err = -ENOENT;
1198

Z
Zheng Yan 已提交
1199
	key.objectid = bytenr;
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1215

1216 1217 1218 1219 1220
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1221
		btrfs_release_path(path);
1222 1223 1224 1225 1226 1227 1228 1229 1230
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1231 1232 1233
	}

	leaf = path->nodes[0];
1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1259
				btrfs_release_path(path);
1260 1261 1262 1263 1264 1265
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1266
	}
1267 1268
fail:
	return err;
Z
Zheng Yan 已提交
1269 1270
}

1271 1272 1273 1274 1275 1276
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1277 1278 1279
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1280
	u32 size;
Z
Zheng Yan 已提交
1281 1282
	u32 num_refs;
	int ret;
1283 1284

	key.objectid = bytenr;
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1295

1296 1297 1298 1299 1300 1301 1302
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1303
		ref = btrfs_item_ptr(leaf, path->slots[0],
1304 1305 1306 1307 1308 1309 1310
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1311
		}
1312 1313 1314 1315 1316 1317 1318 1319
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1320
			btrfs_release_path(path);
1321 1322 1323 1324 1325
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1326

1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1341 1342
		}
	}
1343 1344 1345
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1346
	btrfs_release_path(path);
1347
	return ret;
1348 1349
}

1350 1351 1352
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
J
Josef Bacik 已提交
1353
					   int refs_to_drop, int *last_ref)
Z
Zheng Yan 已提交
1354
{
1355 1356 1357
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1358
	struct extent_buffer *leaf;
1359
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1360 1361 1362
	int ret = 0;

	leaf = path->nodes[0];
1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1384 1385
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1386

Z
Zheng Yan 已提交
1387 1388
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
J
Josef Bacik 已提交
1389
		*last_ref = 1;
Z
Zheng Yan 已提交
1390
	} else {
1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1403 1404 1405 1406 1407
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1408
static noinline u32 extent_data_ref_count(struct btrfs_path *path,
1409
					  struct btrfs_extent_inline_ref *iref)
1410
{
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1442
#endif
1443 1444 1445 1446 1447
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1448

1449 1450 1451 1452 1453
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1454
{
1455
	struct btrfs_key key;
1456 1457
	int ret;

1458 1459 1460 1461 1462 1463 1464
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1465 1466
	}

1467 1468 1469 1470 1471
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1472
		btrfs_release_path(path);
1473 1474 1475 1476 1477
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1478
#endif
1479
	return ret;
1480 1481
}

1482 1483 1484 1485 1486
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1487
{
1488
	struct btrfs_key key;
Z
Zheng Yan 已提交
1489 1490
	int ret;

1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1501
	btrfs_release_path(path);
Z
Zheng Yan 已提交
1502 1503 1504
	return ret;
}

1505
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1506
{
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1520
}
1521

1522 1523
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1524

C
Chris Mason 已提交
1525
{
1526
	for (; level < BTRFS_MAX_LEVEL; level++) {
1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1542

1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1578 1579
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
1580

1581
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1582
	key.type = BTRFS_EXTENT_ITEM_KEY;
1583
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1584

1585 1586 1587
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1588
		path->keep_locks = 1;
1589 1590
	} else
		extra_size = -1;
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601

	/*
	 * Owner is our parent level, so we can just add one to get the level
	 * for the block we are interested in.
	 */
	if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) {
		key.type = BTRFS_METADATA_ITEM_KEY;
		key.offset = owner;
	}

again:
1602
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1603
	if (ret < 0) {
1604 1605 1606
		err = ret;
		goto out;
	}
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623

	/*
	 * We may be a newly converted file system which still has the old fat
	 * extent entries for metadata, so try and see if we have one of those.
	 */
	if (ret > 0 && skinny_metadata) {
		skinny_metadata = false;
		if (path->slots[0]) {
			path->slots[0]--;
			btrfs_item_key_to_cpu(path->nodes[0], &key,
					      path->slots[0]);
			if (key.objectid == bytenr &&
			    key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes)
				ret = 0;
		}
		if (ret) {
1624
			key.objectid = bytenr;
1625 1626 1627 1628 1629 1630 1631
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;
			btrfs_release_path(path);
			goto again;
		}
	}

1632 1633 1634
	if (ret && !insert) {
		err = -ENOENT;
		goto out;
1635
	} else if (WARN_ON(ret)) {
1636 1637
		err = -EIO;
		goto out;
1638
	}
1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

1666
	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) {
1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1730 1731
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1732
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1733 1734 1735 1736 1737 1738
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1739
	if (insert) {
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
1750
void setup_inline_extent_backref(struct btrfs_root *root,
1751 1752 1753 1754 1755
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

1773
	btrfs_extend_item(root, path, size);
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1823
		return ret;
1824

1825
	btrfs_release_path(path);
1826 1827 1828 1829 1830 1831 1832 1833
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1834
	}
1835 1836
	return ret;
}
Z
Zheng Yan 已提交
1837

1838 1839 1840 1841
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
1842
void update_inline_extent_backref(struct btrfs_root *root,
1843 1844 1845
				  struct btrfs_path *path,
				  struct btrfs_extent_inline_ref *iref,
				  int refs_to_mod,
J
Josef Bacik 已提交
1846 1847
				  struct btrfs_delayed_extent_op *extent_op,
				  int *last_ref)
1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1880
	}
Z
Zheng Yan 已提交
1881

1882 1883 1884 1885 1886 1887 1888 1889 1890
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
J
Josef Bacik 已提交
1891
		*last_ref = 1;
1892 1893 1894 1895 1896 1897 1898 1899
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
1900
		btrfs_truncate_item(root, path, item_size, 1);
1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921
	}
	btrfs_mark_buffer_dirty(leaf);
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1922
		update_inline_extent_backref(root, path, iref,
J
Josef Bacik 已提交
1923
					     refs_to_add, extent_op, NULL);
1924
	} else if (ret == -ENOENT) {
1925
		setup_inline_extent_backref(root, path, iref, parent,
1926 1927 1928
					    root_objectid, owner, offset,
					    refs_to_add, extent_op);
		ret = 0;
1929
	}
1930 1931
	return ret;
}
Z
Zheng Yan 已提交
1932

1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1951

1952 1953 1954 1955
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
J
Josef Bacik 已提交
1956
				 int refs_to_drop, int is_data, int *last_ref)
1957
{
1958
	int ret = 0;
1959

1960 1961
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
1962
		update_inline_extent_backref(root, path, iref,
J
Josef Bacik 已提交
1963
					     -refs_to_drop, NULL, last_ref);
1964
	} else if (is_data) {
J
Josef Bacik 已提交
1965 1966
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop,
					     last_ref);
1967
	} else {
J
Josef Bacik 已提交
1968
		*last_ref = 1;
1969 1970 1971 1972 1973
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1974
#define in_range(b, first, len)        ((b) >= (first) && (b) < (first) + (len))
1975 1976
static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len,
			       u64 *discarded_bytes)
1977
{
1978 1979
	int j, ret = 0;
	u64 bytes_left, end;
1980
	u64 aligned_start = ALIGN(start, 1 << 9);
1981

1982 1983 1984 1985 1986
	if (WARN_ON(start != aligned_start)) {
		len -= aligned_start - start;
		len = round_down(len, 1 << 9);
		start = aligned_start;
	}
1987

1988
	*discarded_bytes = 0;
1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039

	if (!len)
		return 0;

	end = start + len;
	bytes_left = len;

	/* Skip any superblocks on this device. */
	for (j = 0; j < BTRFS_SUPER_MIRROR_MAX; j++) {
		u64 sb_start = btrfs_sb_offset(j);
		u64 sb_end = sb_start + BTRFS_SUPER_INFO_SIZE;
		u64 size = sb_start - start;

		if (!in_range(sb_start, start, bytes_left) &&
		    !in_range(sb_end, start, bytes_left) &&
		    !in_range(start, sb_start, BTRFS_SUPER_INFO_SIZE))
			continue;

		/*
		 * Superblock spans beginning of range.  Adjust start and
		 * try again.
		 */
		if (sb_start <= start) {
			start += sb_end - start;
			if (start > end) {
				bytes_left = 0;
				break;
			}
			bytes_left = end - start;
			continue;
		}

		if (size) {
			ret = blkdev_issue_discard(bdev, start >> 9, size >> 9,
						   GFP_NOFS, 0);
			if (!ret)
				*discarded_bytes += size;
			else if (ret != -EOPNOTSUPP)
				return ret;
		}

		start = sb_end;
		if (start > end) {
			bytes_left = 0;
			break;
		}
		bytes_left = end - start;
	}

	if (bytes_left) {
		ret = blkdev_issue_discard(bdev, start >> 9, bytes_left >> 9,
2040 2041
					   GFP_NOFS, 0);
		if (!ret)
2042
			*discarded_bytes += bytes_left;
2043
	}
2044
	return ret;
2045 2046
}

2047 2048
int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
			 u64 num_bytes, u64 *actual_bytes)
2049 2050
{
	int ret;
2051
	u64 discarded_bytes = 0;
2052
	struct btrfs_bio *bbio = NULL;
2053

C
Christoph Hellwig 已提交
2054

2055 2056 2057 2058 2059
	/*
	 * Avoid races with device replace and make sure our bbio has devices
	 * associated to its stripes that don't go away while we are discarding.
	 */
	btrfs_bio_counter_inc_blocked(root->fs_info);
2060
	/* Tell the block device(s) that the sectors can be discarded */
2061
	ret = btrfs_map_block(root->fs_info, REQ_DISCARD,
2062
			      bytenr, &num_bytes, &bbio, 0);
2063
	/* Error condition is -ENOMEM */
2064
	if (!ret) {
2065
		struct btrfs_bio_stripe *stripe = bbio->stripes;
2066 2067 2068
		int i;


2069
		for (i = 0; i < bbio->num_stripes; i++, stripe++) {
2070
			u64 bytes;
2071 2072 2073
			if (!stripe->dev->can_discard)
				continue;

2074 2075
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
2076 2077
						  stripe->length,
						  &bytes);
2078
			if (!ret)
2079
				discarded_bytes += bytes;
2080
			else if (ret != -EOPNOTSUPP)
2081
				break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
2082 2083 2084 2085 2086 2087 2088

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
2089
		}
2090
		btrfs_put_bbio(bbio);
2091
	}
2092
	btrfs_bio_counter_dec(root->fs_info);
2093 2094 2095 2096

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

2097

D
David Woodhouse 已提交
2098 2099
	if (ret == -EOPNOTSUPP)
		ret = 0;
2100 2101 2102
	return ret;
}

2103
/* Can return -ENOMEM */
2104 2105 2106
int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
2107
			 u64 root_objectid, u64 owner, u64 offset)
2108 2109
{
	int ret;
A
Arne Jansen 已提交
2110 2111
	struct btrfs_fs_info *fs_info = root->fs_info;

2112 2113 2114 2115
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
2116 2117
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
2118
					parent, root_objectid, (int)owner,
2119
					BTRFS_ADD_DELAYED_REF, NULL);
2120
	} else {
A
Arne Jansen 已提交
2121
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
2122 2123
					num_bytes, parent, root_objectid,
					owner, offset, 0,
2124
					BTRFS_ADD_DELAYED_REF, NULL);
2125 2126 2127 2128 2129 2130
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
2131
				  struct btrfs_delayed_ref_node *node,
2132 2133 2134 2135
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
J
Josef Bacik 已提交
2136
	struct btrfs_fs_info *fs_info = root->fs_info;
2137 2138 2139
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
J
Josef Bacik 已提交
2140
	struct btrfs_key key;
2141 2142
	u64 bytenr = node->bytenr;
	u64 num_bytes = node->num_bytes;
2143 2144 2145 2146 2147 2148 2149
	u64 refs;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

2150
	path->reada = READA_FORWARD;
2151 2152
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
J
Josef Bacik 已提交
2153 2154
	ret = insert_inline_extent_backref(trans, fs_info->extent_root, path,
					   bytenr, num_bytes, parent,
2155 2156
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
2157
	if ((ret < 0 && ret != -EAGAIN) || !ret)
2158
		goto out;
J
Josef Bacik 已提交
2159 2160 2161 2162 2163 2164

	/*
	 * Ok we had -EAGAIN which means we didn't have space to insert and
	 * inline extent ref, so just update the reference count and add a
	 * normal backref.
	 */
2165
	leaf = path->nodes[0];
J
Josef Bacik 已提交
2166
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2167 2168 2169 2170 2171
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
2172

2173
	btrfs_mark_buffer_dirty(leaf);
2174
	btrfs_release_path(path);
2175

2176
	path->reada = READA_FORWARD;
2177
	path->leave_spinning = 1;
2178 2179
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
2180 2181
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
2182 2183
	if (ret)
		btrfs_abort_transaction(trans, root, ret);
2184
out:
2185
	btrfs_free_path(path);
2186
	return ret;
2187 2188
}

2189 2190 2191 2192 2193
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2194
{
2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
2207 2208
	trace_run_delayed_data_ref(node, ref, node->action);

2209 2210
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
J
Josef Bacik 已提交
2211
	ref_root = ref->root;
2212 2213

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2214
		if (extent_op)
2215 2216 2217 2218 2219 2220
			flags |= extent_op->flags_to_set;
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
2221
		ret = __btrfs_inc_extent_ref(trans, root, node, parent,
2222 2223
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
2224
					     extent_op);
2225
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
2226
		ret = __btrfs_free_extent(trans, root, node, parent,
2227 2228
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
2229
					  extent_op);
2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
2264
	int ret;
2265
	int err = 0;
2266
	int metadata = !extent_op->is_data;
2267

2268 2269 2270
	if (trans->aborted)
		return 0;

2271 2272 2273
	if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA))
		metadata = 0;

2274 2275 2276 2277 2278 2279
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;

2280 2281
	if (metadata) {
		key.type = BTRFS_METADATA_ITEM_KEY;
2282
		key.offset = extent_op->level;
2283 2284 2285 2286 2287 2288
	} else {
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = node->num_bytes;
	}

again:
2289
	path->reada = READA_FORWARD;
2290 2291 2292 2293 2294 2295 2296 2297
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
2298
		if (metadata) {
2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
			if (path->slots[0] > 0) {
				path->slots[0]--;
				btrfs_item_key_to_cpu(path->nodes[0], &key,
						      path->slots[0]);
				if (key.objectid == node->bytenr &&
				    key.type == BTRFS_EXTENT_ITEM_KEY &&
				    key.offset == node->num_bytes)
					ret = 0;
			}
			if (ret > 0) {
				btrfs_release_path(path);
				metadata = 0;
2311

2312 2313 2314 2315 2316 2317 2318 2319
				key.objectid = node->bytenr;
				key.offset = node->num_bytes;
				key.type = BTRFS_EXTENT_ITEM_KEY;
				goto again;
			}
		} else {
			err = -EIO;
			goto out;
2320
		}
2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2340

2341 2342 2343 2344
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2345 2346
}

2347 2348 2349 2350 2351
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2352 2353
{
	int ret = 0;
2354 2355 2356 2357
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2358 2359
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
2360

2361
	ref = btrfs_delayed_node_to_tree_ref(node);
2362 2363
	trace_run_delayed_tree_ref(node, ref, node->action);

2364 2365
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
J
Josef Bacik 已提交
2366
	ref_root = ref->root;
2367

2368 2369 2370 2371 2372 2373 2374 2375 2376
	ins.objectid = node->bytenr;
	if (skinny_metadata) {
		ins.offset = ref->level;
		ins.type = BTRFS_METADATA_ITEM_KEY;
	} else {
		ins.offset = node->num_bytes;
		ins.type = BTRFS_EXTENT_ITEM_KEY;
	}

2377 2378
	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2379
		BUG_ON(!extent_op || !extent_op->update_flags);
2380 2381 2382 2383
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
2384
						ref->level, &ins);
2385
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
2386 2387 2388
		ret = __btrfs_inc_extent_ref(trans, root, node,
					     parent, ref_root,
					     ref->level, 0, 1,
J
Josef Bacik 已提交
2389
					     extent_op);
2390
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
2391 2392 2393
		ret = __btrfs_free_extent(trans, root, node,
					  parent, ref_root,
					  ref->level, 0, 1, extent_op);
2394 2395 2396
	} else {
		BUG();
	}
2397 2398 2399 2400
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2401 2402 2403 2404 2405
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2406
{
2407 2408
	int ret = 0;

2409 2410 2411 2412
	if (trans->aborted) {
		if (insert_reserved)
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2413
		return 0;
2414
	}
2415

2416
	if (btrfs_delayed_ref_is_head(node)) {
2417 2418 2419 2420 2421 2422 2423
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2424 2425
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2426 2427
		trace_run_delayed_ref_head(node, head, node->action);

2428
		if (insert_reserved) {
2429 2430
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2431 2432 2433 2434 2435
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
			}
2436
		}
2437 2438 2439 2440 2441

		/* Also free its reserved qgroup space */
		btrfs_qgroup_free_delayed_ref(root->fs_info,
					      head->qgroup_ref_root,
					      head->qgroup_reserved);
2442
		return ret;
2443 2444
	}

2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2456 2457
}

2458
static inline struct btrfs_delayed_ref_node *
2459 2460
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
2461 2462
	struct btrfs_delayed_ref_node *ref;

2463 2464
	if (list_empty(&head->ref_list))
		return NULL;
2465

2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476
	/*
	 * Select a delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * This is to prevent a ref count from going down to zero, which deletes
	 * the extent item from the extent tree, when there still are references
	 * to add, which would fail because they would not find the extent item.
	 */
	list_for_each_entry(ref, &head->ref_list, list) {
		if (ref->action == BTRFS_ADD_DELAYED_REF)
			return ref;
	}

2477 2478
	return list_entry(head->ref_list.next, struct btrfs_delayed_ref_node,
			  list);
2479 2480
}

2481 2482 2483 2484
/*
 * Returns 0 on success or if called with an already aborted transaction.
 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
 */
2485 2486 2487
static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
					     struct btrfs_root *root,
					     unsigned long nr)
2488 2489 2490 2491
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2492
	struct btrfs_delayed_extent_op *extent_op;
2493
	struct btrfs_fs_info *fs_info = root->fs_info;
2494
	ktime_t start = ktime_get();
2495
	int ret;
2496
	unsigned long count = 0;
2497
	unsigned long actual_count = 0;
2498 2499 2500 2501 2502
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2503
			if (count >= nr)
2504 2505
				break;

2506 2507 2508 2509 2510 2511
			spin_lock(&delayed_refs->lock);
			locked_ref = btrfs_select_ref_head(trans);
			if (!locked_ref) {
				spin_unlock(&delayed_refs->lock);
				break;
			}
2512 2513 2514 2515

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);
2516
			spin_unlock(&delayed_refs->lock);
2517 2518 2519 2520 2521 2522 2523 2524 2525 2526
			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2527 2528
			}
		}
2529

2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541
		/*
		 * We need to try and merge add/drops of the same ref since we
		 * can run into issues with relocate dropping the implicit ref
		 * and then it being added back again before the drop can
		 * finish.  If we merged anything we need to re-loop so we can
		 * get a good ref.
		 * Or we can get node references of the same type that weren't
		 * merged when created due to bumps in the tree mod seq, and
		 * we need to merge them to prevent adding an inline extent
		 * backref before dropping it (triggering a BUG_ON at
		 * insert_inline_extent_backref()).
		 */
2542
		spin_lock(&locked_ref->lock);
2543 2544
		btrfs_merge_delayed_refs(trans, fs_info, delayed_refs,
					 locked_ref);
2545

2546 2547 2548 2549 2550 2551 2552
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);

		if (ref && ref->seq &&
2553
		    btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) {
2554
			spin_unlock(&locked_ref->lock);
2555
			btrfs_delayed_ref_unlock(locked_ref);
2556 2557
			spin_lock(&delayed_refs->lock);
			locked_ref->processing = 0;
2558 2559
			delayed_refs->num_heads_ready++;
			spin_unlock(&delayed_refs->lock);
2560
			locked_ref = NULL;
2561
			cond_resched();
2562
			count++;
2563 2564 2565
			continue;
		}

2566 2567 2568 2569 2570 2571
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2572

2573 2574 2575
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2576
		if (!ref) {
2577 2578


2579 2580 2581 2582 2583
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2584 2585

			if (extent_op && must_insert_reserved) {
2586
				btrfs_free_delayed_extent_op(extent_op);
2587 2588 2589 2590
				extent_op = NULL;
			}

			if (extent_op) {
2591
				spin_unlock(&locked_ref->lock);
2592 2593
				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
2594
				btrfs_free_delayed_extent_op(extent_op);
2595

2596
				if (ret) {
2597 2598 2599 2600 2601 2602 2603 2604
					/*
					 * Need to reset must_insert_reserved if
					 * there was an error so the abort stuff
					 * can cleanup the reserved space
					 * properly.
					 */
					if (must_insert_reserved)
						locked_ref->must_insert_reserved = 1;
2605
					locked_ref->processing = 0;
2606
					btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret);
2607
					btrfs_delayed_ref_unlock(locked_ref);
2608 2609
					return ret;
				}
2610
				continue;
2611
			}
C
Chris Mason 已提交
2612

2613
			/*
2614
			 * Need to drop our head ref lock and re-acquire the
2615 2616 2617 2618 2619 2620
			 * delayed ref lock and then re-check to make sure
			 * nobody got added.
			 */
			spin_unlock(&locked_ref->lock);
			spin_lock(&delayed_refs->lock);
			spin_lock(&locked_ref->lock);
2621
			if (!list_empty(&locked_ref->ref_list) ||
2622
			    locked_ref->extent_op) {
2623 2624 2625 2626 2627 2628
				spin_unlock(&locked_ref->lock);
				spin_unlock(&delayed_refs->lock);
				continue;
			}
			ref->in_tree = 0;
			delayed_refs->num_heads--;
L
Liu Bo 已提交
2629 2630
			rb_erase(&locked_ref->href_node,
				 &delayed_refs->href_root);
2631 2632
			spin_unlock(&delayed_refs->lock);
		} else {
2633
			actual_count++;
2634
			ref->in_tree = 0;
2635
			list_del(&ref->list);
L
Liu Bo 已提交
2636
		}
2637 2638
		atomic_dec(&delayed_refs->num_entries);

2639
		if (!btrfs_delayed_ref_is_head(ref)) {
2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655
			/*
			 * when we play the delayed ref, also correct the
			 * ref_mod on head
			 */
			switch (ref->action) {
			case BTRFS_ADD_DELAYED_REF:
			case BTRFS_ADD_DELAYED_EXTENT:
				locked_ref->node.ref_mod -= ref->ref_mod;
				break;
			case BTRFS_DROP_DELAYED_REF:
				locked_ref->node.ref_mod += ref->ref_mod;
				break;
			default:
				WARN_ON(1);
			}
		}
2656
		spin_unlock(&locked_ref->lock);
2657

2658
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2659
					  must_insert_reserved);
2660

2661
		btrfs_free_delayed_extent_op(extent_op);
2662
		if (ret) {
2663
			locked_ref->processing = 0;
2664 2665
			btrfs_delayed_ref_unlock(locked_ref);
			btrfs_put_delayed_ref(ref);
2666
			btrfs_debug(fs_info, "run_one_delayed_ref returned %d", ret);
2667 2668 2669
			return ret;
		}

2670 2671 2672 2673 2674 2675 2676
		/*
		 * If this node is a head, that means all the refs in this head
		 * have been dealt with, and we will pick the next head to deal
		 * with, so we must unlock the head and drop it from the cluster
		 * list before we release it.
		 */
		if (btrfs_delayed_ref_is_head(ref)) {
2677 2678 2679 2680 2681 2682
			if (locked_ref->is_data &&
			    locked_ref->total_ref_mod < 0) {
				spin_lock(&delayed_refs->lock);
				delayed_refs->pending_csums -= ref->num_bytes;
				spin_unlock(&delayed_refs->lock);
			}
2683 2684 2685 2686 2687
			btrfs_delayed_ref_unlock(locked_ref);
			locked_ref = NULL;
		}
		btrfs_put_delayed_ref(ref);
		count++;
2688 2689
		cond_resched();
	}
2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705

	/*
	 * We don't want to include ref heads since we can have empty ref heads
	 * and those will drastically skew our runtime down since we just do
	 * accounting, no actual extent tree updates.
	 */
	if (actual_count > 0) {
		u64 runtime = ktime_to_ns(ktime_sub(ktime_get(), start));
		u64 avg;

		/*
		 * We weigh the current average higher than our current runtime
		 * to avoid large swings in the average.
		 */
		spin_lock(&delayed_refs->lock);
		avg = fs_info->avg_delayed_ref_runtime * 3 + runtime;
2706
		fs_info->avg_delayed_ref_runtime = avg >> 2;	/* div by 4 */
2707 2708
		spin_unlock(&delayed_refs->lock);
	}
2709
	return 0;
2710 2711
}

2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754
#ifdef SCRAMBLE_DELAYED_REFS
/*
 * Normally delayed refs get processed in ascending bytenr order. This
 * correlates in most cases to the order added. To expose dependencies on this
 * order, we start to process the tree in the middle instead of the beginning
 */
static u64 find_middle(struct rb_root *root)
{
	struct rb_node *n = root->rb_node;
	struct btrfs_delayed_ref_node *entry;
	int alt = 1;
	u64 middle;
	u64 first = 0, last = 0;

	n = rb_first(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		first = entry->bytenr;
	}
	n = rb_last(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		last = entry->bytenr;
	}
	n = root->rb_node;

	while (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		WARN_ON(!entry->in_tree);

		middle = entry->bytenr;

		if (alt)
			n = n->rb_left;
		else
			n = n->rb_right;

		alt = 1 - alt;
	}
	return middle;
}
#endif

2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765
static inline u64 heads_to_leaves(struct btrfs_root *root, u64 heads)
{
	u64 num_bytes;

	num_bytes = heads * (sizeof(struct btrfs_extent_item) +
			     sizeof(struct btrfs_extent_inline_ref));
	if (!btrfs_fs_incompat(root->fs_info, SKINNY_METADATA))
		num_bytes += heads * sizeof(struct btrfs_tree_block_info);

	/*
	 * We don't ever fill up leaves all the way so multiply by 2 just to be
2766
	 * closer to what we're really going to want to use.
2767
	 */
2768
	return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(root));
2769 2770
}

2771 2772 2773 2774
/*
 * Takes the number of bytes to be csumm'ed and figures out how many leaves it
 * would require to store the csums for that many bytes.
 */
2775
u64 btrfs_csum_bytes_to_leaves(struct btrfs_root *root, u64 csum_bytes)
2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789
{
	u64 csum_size;
	u64 num_csums_per_leaf;
	u64 num_csums;

	csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
	num_csums_per_leaf = div64_u64(csum_size,
			(u64)btrfs_super_csum_size(root->fs_info->super_copy));
	num_csums = div64_u64(csum_bytes, root->sectorsize);
	num_csums += num_csums_per_leaf - 1;
	num_csums = div64_u64(num_csums, num_csums_per_leaf);
	return num_csums;
}

2790
int btrfs_check_space_for_delayed_refs(struct btrfs_trans_handle *trans,
2791 2792 2793 2794
				       struct btrfs_root *root)
{
	struct btrfs_block_rsv *global_rsv;
	u64 num_heads = trans->transaction->delayed_refs.num_heads_ready;
2795
	u64 csum_bytes = trans->transaction->delayed_refs.pending_csums;
2796 2797
	u64 num_dirty_bgs = trans->transaction->num_dirty_bgs;
	u64 num_bytes, num_dirty_bgs_bytes;
2798 2799 2800 2801 2802
	int ret = 0;

	num_bytes = btrfs_calc_trans_metadata_size(root, 1);
	num_heads = heads_to_leaves(root, num_heads);
	if (num_heads > 1)
2803
		num_bytes += (num_heads - 1) * root->nodesize;
2804
	num_bytes <<= 1;
2805
	num_bytes += btrfs_csum_bytes_to_leaves(root, csum_bytes) * root->nodesize;
2806 2807
	num_dirty_bgs_bytes = btrfs_calc_trans_metadata_size(root,
							     num_dirty_bgs);
2808 2809 2810 2811 2812 2813
	global_rsv = &root->fs_info->global_block_rsv;

	/*
	 * If we can't allocate any more chunks lets make sure we have _lots_ of
	 * wiggle room since running delayed refs can create more delayed refs.
	 */
2814 2815
	if (global_rsv->space_info->full) {
		num_dirty_bgs_bytes <<= 1;
2816
		num_bytes <<= 1;
2817
	}
2818 2819

	spin_lock(&global_rsv->lock);
2820
	if (global_rsv->reserved <= num_bytes + num_dirty_bgs_bytes)
2821 2822 2823 2824 2825
		ret = 1;
	spin_unlock(&global_rsv->lock);
	return ret;
}

2826 2827 2828 2829 2830 2831 2832
int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	u64 num_entries =
		atomic_read(&trans->transaction->delayed_refs.num_entries);
	u64 avg_runtime;
C
Chris Mason 已提交
2833
	u64 val;
2834 2835 2836

	smp_mb();
	avg_runtime = fs_info->avg_delayed_ref_runtime;
C
Chris Mason 已提交
2837
	val = num_entries * avg_runtime;
2838 2839
	if (num_entries * avg_runtime >= NSEC_PER_SEC)
		return 1;
C
Chris Mason 已提交
2840 2841
	if (val >= NSEC_PER_SEC / 2)
		return 2;
2842 2843 2844 2845

	return btrfs_check_space_for_delayed_refs(trans, root);
}

C
Chris Mason 已提交
2846 2847
struct async_delayed_refs {
	struct btrfs_root *root;
2848
	u64 transid;
C
Chris Mason 已提交
2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863
	int count;
	int error;
	int sync;
	struct completion wait;
	struct btrfs_work work;
};

static void delayed_ref_async_start(struct btrfs_work *work)
{
	struct async_delayed_refs *async;
	struct btrfs_trans_handle *trans;
	int ret;

	async = container_of(work, struct async_delayed_refs, work);

2864 2865
	/* if the commit is already started, we don't need to wait here */
	if (btrfs_transaction_blocked(async->root->fs_info))
2866 2867
		goto done;

2868 2869 2870
	trans = btrfs_join_transaction(async->root);
	if (IS_ERR(trans)) {
		async->error = PTR_ERR(trans);
C
Chris Mason 已提交
2871 2872 2873 2874
		goto done;
	}

	/*
2875
	 * trans->sync means that when we call end_transaction, we won't
C
Chris Mason 已提交
2876 2877 2878
	 * wait on delayed refs
	 */
	trans->sync = true;
2879 2880 2881 2882 2883

	/* Don't bother flushing if we got into a different transaction */
	if (trans->transid > async->transid)
		goto end;

C
Chris Mason 已提交
2884 2885 2886
	ret = btrfs_run_delayed_refs(trans, async->root, async->count);
	if (ret)
		async->error = ret;
2887
end:
C
Chris Mason 已提交
2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898
	ret = btrfs_end_transaction(trans, async->root);
	if (ret && !async->error)
		async->error = ret;
done:
	if (async->sync)
		complete(&async->wait);
	else
		kfree(async);
}

int btrfs_async_run_delayed_refs(struct btrfs_root *root,
2899
				 unsigned long count, u64 transid, int wait)
C
Chris Mason 已提交
2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910
{
	struct async_delayed_refs *async;
	int ret;

	async = kmalloc(sizeof(*async), GFP_NOFS);
	if (!async)
		return -ENOMEM;

	async->root = root->fs_info->tree_root;
	async->count = count;
	async->error = 0;
2911
	async->transid = transid;
C
Chris Mason 已提交
2912 2913 2914 2915 2916 2917
	if (wait)
		async->sync = 1;
	else
		async->sync = 0;
	init_completion(&async->wait);

2918 2919
	btrfs_init_work(&async->work, btrfs_extent_refs_helper,
			delayed_ref_async_start, NULL, NULL);
C
Chris Mason 已提交
2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931

	btrfs_queue_work(root->fs_info->extent_workers, &async->work);

	if (wait) {
		wait_for_completion(&async->wait);
		ret = async->error;
		kfree(async);
		return ret;
	}
	return 0;
}

2932 2933 2934 2935 2936 2937
/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
2938 2939 2940
 *
 * Returns 0 on success or if called with an aborted transaction
 * Returns <0 on error and aborts the transaction
2941 2942 2943 2944 2945 2946
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
L
Liu Bo 已提交
2947
	struct btrfs_delayed_ref_head *head;
2948 2949
	int ret;
	int run_all = count == (unsigned long)-1;
2950
	bool can_flush_pending_bgs = trans->can_flush_pending_bgs;
2951

2952 2953 2954 2955
	/* We'll clean this up in btrfs_cleanup_transaction */
	if (trans->aborted)
		return 0;

2956 2957 2958
	if (root->fs_info->creating_free_space_tree)
		return 0;

2959 2960 2961 2962
	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

	delayed_refs = &trans->transaction->delayed_refs;
L
Liu Bo 已提交
2963
	if (count == 0)
2964
		count = atomic_read(&delayed_refs->num_entries) * 2;
2965

2966
again:
2967 2968 2969
#ifdef SCRAMBLE_DELAYED_REFS
	delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
#endif
2970
	trans->can_flush_pending_bgs = false;
2971 2972 2973 2974
	ret = __btrfs_run_delayed_refs(trans, root, count);
	if (ret < 0) {
		btrfs_abort_transaction(trans, root, ret);
		return ret;
2975
	}
2976

2977
	if (run_all) {
2978
		if (!list_empty(&trans->new_bgs))
2979 2980
			btrfs_create_pending_block_groups(trans, root);

2981
		spin_lock(&delayed_refs->lock);
L
Liu Bo 已提交
2982
		node = rb_first(&delayed_refs->href_root);
2983 2984
		if (!node) {
			spin_unlock(&delayed_refs->lock);
2985
			goto out;
2986
		}
2987
		count = (unsigned long)-1;
2988

2989
		while (node) {
L
Liu Bo 已提交
2990 2991 2992 2993
			head = rb_entry(node, struct btrfs_delayed_ref_head,
					href_node);
			if (btrfs_delayed_ref_is_head(&head->node)) {
				struct btrfs_delayed_ref_node *ref;
2994

L
Liu Bo 已提交
2995
				ref = &head->node;
2996 2997 2998
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2999 3000 3001 3002
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
3003 3004 3005 3006
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
3007
				cond_resched();
3008
				goto again;
L
Liu Bo 已提交
3009 3010
			} else {
				WARN_ON(1);
3011 3012 3013 3014
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
3015
		cond_resched();
3016
		goto again;
3017
	}
3018
out:
3019
	assert_qgroups_uptodate(trans);
3020
	trans->can_flush_pending_bgs = can_flush_pending_bgs;
3021 3022 3023
	return 0;
}

3024 3025 3026
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
3027
				int level, int is_data)
3028 3029 3030 3031
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

3032
	extent_op = btrfs_alloc_delayed_extent_op();
3033 3034 3035 3036
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
3037 3038 3039
	extent_op->update_flags = true;
	extent_op->update_key = false;
	extent_op->is_data = is_data ? true : false;
3040
	extent_op->level = level;
3041

A
Arne Jansen 已提交
3042 3043
	ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
					  num_bytes, extent_op);
3044
	if (ret)
3045
		btrfs_free_delayed_extent_op(extent_op);
3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	int ret = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
3063 3064 3065 3066
	if (!head) {
		spin_unlock(&delayed_refs->lock);
		return 0;
	}
3067 3068 3069 3070 3071

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

3072
		btrfs_release_path(path);
3073

3074 3075 3076 3077
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
3078 3079 3080 3081 3082
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}
3083
	spin_unlock(&delayed_refs->lock);
3084

3085
	spin_lock(&head->lock);
3086
	list_for_each_entry(ref, &head->ref_list, list) {
3087 3088 3089 3090 3091
		/* If it's a shared ref we know a cross reference exists */
		if (ref->type != BTRFS_EXTENT_DATA_REF_KEY) {
			ret = 1;
			break;
		}
3092

3093
		data_ref = btrfs_delayed_node_to_data_ref(ref);
3094

3095 3096 3097 3098 3099 3100 3101 3102 3103 3104
		/*
		 * If our ref doesn't match the one we're currently looking at
		 * then we have a cross reference.
		 */
		if (data_ref->root != root->root_key.objectid ||
		    data_ref->objectid != objectid ||
		    data_ref->offset != offset) {
			ret = 1;
			break;
		}
3105
	}
3106
	spin_unlock(&head->lock);
3107 3108 3109 3110 3111 3112 3113 3114
	mutex_unlock(&head->mutex);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
3115 3116
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
3117
	struct extent_buffer *leaf;
3118 3119 3120
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
3121
	struct btrfs_key key;
3122
	u32 item_size;
3123
	int ret;
3124

3125
	key.objectid = bytenr;
Z
Zheng Yan 已提交
3126
	key.offset = (u64)-1;
3127
	key.type = BTRFS_EXTENT_ITEM_KEY;
3128 3129 3130 3131

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
3132
	BUG_ON(ret == 0); /* Corruption */
Y
Yan Zheng 已提交
3133 3134 3135

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
3136
		goto out;
3137

Z
Zheng Yan 已提交
3138
	path->slots[0]--;
3139
	leaf = path->nodes[0];
3140
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
3141

3142
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
3143
		goto out;
3144

3145 3146 3147 3148 3149 3150 3151 3152 3153
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
3154

3155 3156 3157
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
3158

3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
3198
			goto out;
Y
Yan Zheng 已提交
3199

3200 3201 3202 3203 3204 3205 3206
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
3207
	}
3208 3209 3210

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
3211
out:
Y
Yan Zheng 已提交
3212
	btrfs_free_path(path);
3213 3214
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
3215
	return ret;
3216
}
C
Chris Mason 已提交
3217

3218
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
3219
			   struct btrfs_root *root,
3220
			   struct extent_buffer *buf,
3221
			   int full_backref, int inc)
Z
Zheng Yan 已提交
3222 3223
{
	u64 bytenr;
3224 3225
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
3226 3227 3228 3229 3230 3231 3232 3233
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
3234
			    u64, u64, u64, u64, u64, u64);
Z
Zheng Yan 已提交
3235

3236 3237

	if (btrfs_test_is_dummy_root(root))
3238
		return 0;
3239

Z
Zheng Yan 已提交
3240 3241 3242 3243
	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

3244
	if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state) && level == 0)
3245
		return 0;
Z
Zheng Yan 已提交
3246

3247 3248 3249 3250
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
3251

3252 3253 3254 3255 3256 3257
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
3258
		if (level == 0) {
3259
			btrfs_item_key_to_cpu(buf, &key, i);
3260
			if (key.type != BTRFS_EXTENT_DATA_KEY)
Z
Zheng Yan 已提交
3261
				continue;
3262
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
3263 3264 3265 3266 3267 3268 3269
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
3270 3271 3272 3273 3274

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
3275
					   key.offset);
Z
Zheng Yan 已提交
3276 3277 3278
			if (ret)
				goto fail;
		} else {
3279
			bytenr = btrfs_node_blockptr(buf, i);
3280
			num_bytes = root->nodesize;
3281
			ret = process_func(trans, root, bytenr, num_bytes,
3282
					   parent, ref_root, level - 1, 0);
Z
Zheng Yan 已提交
3283 3284 3285 3286 3287 3288
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
3289 3290 3291 3292
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3293
		  struct extent_buffer *buf, int full_backref)
3294
{
3295
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
3296 3297 3298
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3299
		  struct extent_buffer *buf, int full_backref)
3300
{
3301
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
Z
Zheng Yan 已提交
3302 3303
}

C
Chris Mason 已提交
3304 3305 3306 3307 3308 3309 3310
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
3311 3312
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
3313 3314

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
3315 3316 3317
	if (ret) {
		if (ret > 0)
			ret = -ENOENT;
3318
		goto fail;
3319
	}
3320 3321 3322 3323 3324

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
3325
fail:
3326
	btrfs_release_path(path);
3327
	return ret;
C
Chris Mason 已提交
3328 3329 3330

}

3331 3332 3333 3334 3335
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
3336

3337
	spin_lock(&root->fs_info->block_group_cache_lock);
3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348

	/* If our block group was removed, we need a full search. */
	if (RB_EMPTY_NODE(&cache->cache_node)) {
		const u64 next_bytenr = cache->key.objectid + cache->key.offset;

		spin_unlock(&root->fs_info->block_group_cache_lock);
		btrfs_put_block_group(cache);
		cache = btrfs_lookup_first_block_group(root->fs_info,
						       next_bytenr);
		return cache;
	}
3349 3350 3351 3352 3353
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
3354
		btrfs_get_block_group(cache);
3355 3356 3357 3358 3359 3360
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

3361 3362 3363 3364 3365 3366 3367
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
3368
	int dcs = BTRFS_DC_ERROR;
3369
	u64 num_pages = 0;
3370 3371 3372 3373 3374 3375 3376
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
3377
	if (block_group->key.offset < (100 * SZ_1M)) {
3378 3379 3380 3381 3382 3383
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

3384 3385
	if (trans->aborted)
		return 0;
3386 3387 3388 3389
again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
3390
		btrfs_release_path(path);
3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

3407 3408 3409 3410 3411 3412 3413
	/* We've already setup this transaction, go ahead and exit */
	if (block_group->cache_generation == trans->transid &&
	    i_size_read(inode)) {
		dcs = BTRFS_DC_SETUP;
		goto out_put;
	}

3414 3415 3416 3417 3418 3419 3420
	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434
	if (ret) {
		/*
		 * So theoretically we could recover from this, simply set the
		 * super cache generation to 0 so we know to invalidate the
		 * cache, but then we'd have to keep track of the block groups
		 * that fail this way so we know we _have_ to reset this cache
		 * before the next commit or risk reading stale cache.  So to
		 * limit our exposure to horrible edge cases lets just abort the
		 * transaction, this only happens in really bad situations
		 * anyway.
		 */
		btrfs_abort_transaction(trans, root, ret);
		goto out_put;
	}
3435 3436 3437
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
3438 3439 3440 3441 3442
		ret = btrfs_check_trunc_cache_free_space(root,
					&root->fs_info->global_block_rsv);
		if (ret)
			goto out_put;

3443
		ret = btrfs_truncate_free_space_cache(root, trans, NULL, inode);
3444 3445 3446 3447 3448
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
3449
	if (block_group->cached != BTRFS_CACHE_FINISHED ||
3450
	    !btrfs_test_opt(root, SPACE_CACHE)) {
3451 3452 3453 3454 3455
		/*
		 * don't bother trying to write stuff out _if_
		 * a) we're not cached,
		 * b) we're with nospace_cache mount option.
		 */
3456
		dcs = BTRFS_DC_WRITTEN;
3457 3458 3459 3460 3461
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

3462 3463 3464 3465 3466 3467 3468 3469 3470
	/*
	 * We hit an ENOSPC when setting up the cache in this transaction, just
	 * skip doing the setup, we've already cleared the cache so we're safe.
	 */
	if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) {
		ret = -ENOSPC;
		goto out_put;
	}

3471 3472 3473 3474 3475 3476
	/*
	 * Try to preallocate enough space based on how big the block group is.
	 * Keep in mind this has to include any pinned space which could end up
	 * taking up quite a bit since it's not folded into the other space
	 * cache.
	 */
3477
	num_pages = div_u64(block_group->key.offset, SZ_256M);
3478 3479 3480 3481
	if (!num_pages)
		num_pages = 1;

	num_pages *= 16;
3482
	num_pages *= PAGE_SIZE;
3483

3484
	ret = btrfs_check_data_free_space(inode, 0, num_pages);
3485 3486 3487 3488 3489 3490
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
3491 3492 3493 3494 3495 3496 3497 3498
	/*
	 * Our cache requires contiguous chunks so that we don't modify a bunch
	 * of metadata or split extents when writing the cache out, which means
	 * we can enospc if we are heavily fragmented in addition to just normal
	 * out of space conditions.  So if we hit this just skip setting up any
	 * other block groups for this transaction, maybe we'll unpin enough
	 * space the next time around.
	 */
3499 3500
	if (!ret)
		dcs = BTRFS_DC_SETUP;
3501 3502
	else if (ret == -ENOSPC)
		set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags);
3503
	btrfs_free_reserved_data_space(inode, 0, num_pages);
3504

3505 3506 3507
out_put:
	iput(inode);
out_free:
3508
	btrfs_release_path(path);
3509 3510
out:
	spin_lock(&block_group->lock);
3511
	if (!ret && dcs == BTRFS_DC_SETUP)
3512
		block_group->cache_generation = trans->transid;
3513
	block_group->disk_cache_state = dcs;
3514 3515 3516 3517 3518
	spin_unlock(&block_group->lock);

	return ret;
}

3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544
int btrfs_setup_space_cache(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root)
{
	struct btrfs_block_group_cache *cache, *tmp;
	struct btrfs_transaction *cur_trans = trans->transaction;
	struct btrfs_path *path;

	if (list_empty(&cur_trans->dirty_bgs) ||
	    !btrfs_test_opt(root, SPACE_CACHE))
		return 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	/* Could add new block groups, use _safe just in case */
	list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs,
				 dirty_list) {
		if (cache->disk_cache_state == BTRFS_DC_CLEAR)
			cache_save_setup(cache, trans, path);
	}

	btrfs_free_path(path);
	return 0;
}

3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557
/*
 * transaction commit does final block group cache writeback during a
 * critical section where nothing is allowed to change the FS.  This is
 * required in order for the cache to actually match the block group,
 * but can introduce a lot of latency into the commit.
 *
 * So, btrfs_start_dirty_block_groups is here to kick off block group
 * cache IO.  There's a chance we'll have to redo some of it if the
 * block group changes again during the commit, but it greatly reduces
 * the commit latency by getting rid of the easy block groups while
 * we're still allowing others to join the commit.
 */
int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans,
3558
				   struct btrfs_root *root)
C
Chris Mason 已提交
3559
{
3560
	struct btrfs_block_group_cache *cache;
3561 3562
	struct btrfs_transaction *cur_trans = trans->transaction;
	int ret = 0;
3563
	int should_put;
3564 3565 3566
	struct btrfs_path *path = NULL;
	LIST_HEAD(dirty);
	struct list_head *io = &cur_trans->io_bgs;
3567
	int num_started = 0;
3568 3569 3570
	int loops = 0;

	spin_lock(&cur_trans->dirty_bgs_lock);
3571 3572 3573
	if (list_empty(&cur_trans->dirty_bgs)) {
		spin_unlock(&cur_trans->dirty_bgs_lock);
		return 0;
3574
	}
3575
	list_splice_init(&cur_trans->dirty_bgs, &dirty);
3576
	spin_unlock(&cur_trans->dirty_bgs_lock);
3577

3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590
again:
	/*
	 * make sure all the block groups on our dirty list actually
	 * exist
	 */
	btrfs_create_pending_block_groups(trans, root);

	if (!path) {
		path = btrfs_alloc_path();
		if (!path)
			return -ENOMEM;
	}

3591 3592 3593 3594 3595 3596
	/*
	 * cache_write_mutex is here only to save us from balance or automatic
	 * removal of empty block groups deleting this block group while we are
	 * writing out the cache
	 */
	mutex_lock(&trans->transaction->cache_write_mutex);
3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650
	while (!list_empty(&dirty)) {
		cache = list_first_entry(&dirty,
					 struct btrfs_block_group_cache,
					 dirty_list);
		/*
		 * this can happen if something re-dirties a block
		 * group that is already under IO.  Just wait for it to
		 * finish and then do it all again
		 */
		if (!list_empty(&cache->io_list)) {
			list_del_init(&cache->io_list);
			btrfs_wait_cache_io(root, trans, cache,
					    &cache->io_ctl, path,
					    cache->key.objectid);
			btrfs_put_block_group(cache);
		}


		/*
		 * btrfs_wait_cache_io uses the cache->dirty_list to decide
		 * if it should update the cache_state.  Don't delete
		 * until after we wait.
		 *
		 * Since we're not running in the commit critical section
		 * we need the dirty_bgs_lock to protect from update_block_group
		 */
		spin_lock(&cur_trans->dirty_bgs_lock);
		list_del_init(&cache->dirty_list);
		spin_unlock(&cur_trans->dirty_bgs_lock);

		should_put = 1;

		cache_save_setup(cache, trans, path);

		if (cache->disk_cache_state == BTRFS_DC_SETUP) {
			cache->io_ctl.inode = NULL;
			ret = btrfs_write_out_cache(root, trans, cache, path);
			if (ret == 0 && cache->io_ctl.inode) {
				num_started++;
				should_put = 0;

				/*
				 * the cache_write_mutex is protecting
				 * the io_list
				 */
				list_add_tail(&cache->io_list, io);
			} else {
				/*
				 * if we failed to write the cache, the
				 * generation will be bad and life goes on
				 */
				ret = 0;
			}
		}
3651
		if (!ret) {
3652
			ret = write_one_cache_group(trans, root, path, cache);
3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674
			/*
			 * Our block group might still be attached to the list
			 * of new block groups in the transaction handle of some
			 * other task (struct btrfs_trans_handle->new_bgs). This
			 * means its block group item isn't yet in the extent
			 * tree. If this happens ignore the error, as we will
			 * try again later in the critical section of the
			 * transaction commit.
			 */
			if (ret == -ENOENT) {
				ret = 0;
				spin_lock(&cur_trans->dirty_bgs_lock);
				if (list_empty(&cache->dirty_list)) {
					list_add_tail(&cache->dirty_list,
						      &cur_trans->dirty_bgs);
					btrfs_get_block_group(cache);
				}
				spin_unlock(&cur_trans->dirty_bgs_lock);
			} else if (ret) {
				btrfs_abort_transaction(trans, root, ret);
			}
		}
3675 3676 3677 3678 3679 3680 3681

		/* if its not on the io list, we need to put the block group */
		if (should_put)
			btrfs_put_block_group(cache);

		if (ret)
			break;
3682 3683 3684 3685 3686 3687 3688 3689

		/*
		 * Avoid blocking other tasks for too long. It might even save
		 * us from writing caches for block groups that are going to be
		 * removed.
		 */
		mutex_unlock(&trans->transaction->cache_write_mutex);
		mutex_lock(&trans->transaction->cache_write_mutex);
3690
	}
3691
	mutex_unlock(&trans->transaction->cache_write_mutex);
3692 3693 3694 3695 3696 3697 3698 3699 3700 3701

	/*
	 * go through delayed refs for all the stuff we've just kicked off
	 * and then loop back (just once)
	 */
	ret = btrfs_run_delayed_refs(trans, root, 0);
	if (!ret && loops == 0) {
		loops++;
		spin_lock(&cur_trans->dirty_bgs_lock);
		list_splice_init(&cur_trans->dirty_bgs, &dirty);
3702 3703 3704 3705 3706 3707 3708 3709
		/*
		 * dirty_bgs_lock protects us from concurrent block group
		 * deletes too (not just cache_write_mutex).
		 */
		if (!list_empty(&dirty)) {
			spin_unlock(&cur_trans->dirty_bgs_lock);
			goto again;
		}
3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726
		spin_unlock(&cur_trans->dirty_bgs_lock);
	}

	btrfs_free_path(path);
	return ret;
}

int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
{
	struct btrfs_block_group_cache *cache;
	struct btrfs_transaction *cur_trans = trans->transaction;
	int ret = 0;
	int should_put;
	struct btrfs_path *path;
	struct list_head *io = &cur_trans->io_bgs;
	int num_started = 0;
C
Chris Mason 已提交
3727 3728 3729 3730 3731

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3732
	/*
3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743
	 * Even though we are in the critical section of the transaction commit,
	 * we can still have concurrent tasks adding elements to this
	 * transaction's list of dirty block groups. These tasks correspond to
	 * endio free space workers started when writeback finishes for a
	 * space cache, which run inode.c:btrfs_finish_ordered_io(), and can
	 * allocate new block groups as a result of COWing nodes of the root
	 * tree when updating the free space inode. The writeback for the space
	 * caches is triggered by an earlier call to
	 * btrfs_start_dirty_block_groups() and iterations of the following
	 * loop.
	 * Also we want to do the cache_save_setup first and then run the
3744 3745 3746
	 * delayed refs to make sure we have the best chance at doing this all
	 * in one shot.
	 */
3747
	spin_lock(&cur_trans->dirty_bgs_lock);
3748 3749 3750 3751
	while (!list_empty(&cur_trans->dirty_bgs)) {
		cache = list_first_entry(&cur_trans->dirty_bgs,
					 struct btrfs_block_group_cache,
					 dirty_list);
3752 3753 3754 3755 3756 3757 3758

		/*
		 * this can happen if cache_save_setup re-dirties a block
		 * group that is already under IO.  Just wait for it to
		 * finish and then do it all again
		 */
		if (!list_empty(&cache->io_list)) {
3759
			spin_unlock(&cur_trans->dirty_bgs_lock);
3760 3761 3762 3763 3764
			list_del_init(&cache->io_list);
			btrfs_wait_cache_io(root, trans, cache,
					    &cache->io_ctl, path,
					    cache->key.objectid);
			btrfs_put_block_group(cache);
3765
			spin_lock(&cur_trans->dirty_bgs_lock);
3766 3767
		}

3768 3769 3770 3771
		/*
		 * don't remove from the dirty list until after we've waited
		 * on any pending IO
		 */
3772
		list_del_init(&cache->dirty_list);
3773
		spin_unlock(&cur_trans->dirty_bgs_lock);
3774 3775
		should_put = 1;

3776
		cache_save_setup(cache, trans, path);
3777

3778
		if (!ret)
3779 3780 3781 3782 3783 3784 3785 3786
			ret = btrfs_run_delayed_refs(trans, root, (unsigned long) -1);

		if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) {
			cache->io_ctl.inode = NULL;
			ret = btrfs_write_out_cache(root, trans, cache, path);
			if (ret == 0 && cache->io_ctl.inode) {
				num_started++;
				should_put = 0;
3787
				list_add_tail(&cache->io_list, io);
3788 3789 3790 3791 3792 3793 3794 3795
			} else {
				/*
				 * if we failed to write the cache, the
				 * generation will be bad and life goes on
				 */
				ret = 0;
			}
		}
3796
		if (!ret) {
3797
			ret = write_one_cache_group(trans, root, path, cache);
3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816
			/*
			 * One of the free space endio workers might have
			 * created a new block group while updating a free space
			 * cache's inode (at inode.c:btrfs_finish_ordered_io())
			 * and hasn't released its transaction handle yet, in
			 * which case the new block group is still attached to
			 * its transaction handle and its creation has not
			 * finished yet (no block group item in the extent tree
			 * yet, etc). If this is the case, wait for all free
			 * space endio workers to finish and retry. This is a
			 * a very rare case so no need for a more efficient and
			 * complex approach.
			 */
			if (ret == -ENOENT) {
				wait_event(cur_trans->writer_wait,
				   atomic_read(&cur_trans->num_writers) == 1);
				ret = write_one_cache_group(trans, root, path,
							    cache);
			}
3817 3818 3819
			if (ret)
				btrfs_abort_transaction(trans, root, ret);
		}
3820 3821 3822 3823

		/* if its not on the io list, we need to put the block group */
		if (should_put)
			btrfs_put_block_group(cache);
3824
		spin_lock(&cur_trans->dirty_bgs_lock);
3825
	}
3826
	spin_unlock(&cur_trans->dirty_bgs_lock);
3827

3828 3829
	while (!list_empty(io)) {
		cache = list_first_entry(io, struct btrfs_block_group_cache,
3830 3831 3832 3833
					 io_list);
		list_del_init(&cache->io_list);
		btrfs_wait_cache_io(root, trans, cache,
				    &cache->io_ctl, path, cache->key.objectid);
J
Josef Bacik 已提交
3834 3835 3836
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
3837
	btrfs_free_path(path);
3838
	return ret;
C
Chris Mason 已提交
3839 3840
}

3841 3842 3843 3844 3845 3846 3847 3848 3849
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
3850
		btrfs_put_block_group(block_group);
3851 3852 3853
	return readonly;
}

3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906
bool btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
{
	struct btrfs_block_group_cache *bg;
	bool ret = true;

	bg = btrfs_lookup_block_group(fs_info, bytenr);
	if (!bg)
		return false;

	spin_lock(&bg->lock);
	if (bg->ro)
		ret = false;
	else
		atomic_inc(&bg->nocow_writers);
	spin_unlock(&bg->lock);

	/* no put on block group, done by btrfs_dec_nocow_writers */
	if (!ret)
		btrfs_put_block_group(bg);

	return ret;

}

void btrfs_dec_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
{
	struct btrfs_block_group_cache *bg;

	bg = btrfs_lookup_block_group(fs_info, bytenr);
	ASSERT(bg);
	if (atomic_dec_and_test(&bg->nocow_writers))
		wake_up_atomic_t(&bg->nocow_writers);
	/*
	 * Once for our lookup and once for the lookup done by a previous call
	 * to btrfs_inc_nocow_writers()
	 */
	btrfs_put_block_group(bg);
	btrfs_put_block_group(bg);
}

static int btrfs_wait_nocow_writers_atomic_t(atomic_t *a)
{
	schedule();
	return 0;
}

void btrfs_wait_nocow_writers(struct btrfs_block_group_cache *bg)
{
	wait_on_atomic_t(&bg->nocow_writers,
			 btrfs_wait_nocow_writers_atomic_t,
			 TASK_UNINTERRUPTIBLE);
}

3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923
static const char *alloc_name(u64 flags)
{
	switch (flags) {
	case BTRFS_BLOCK_GROUP_METADATA|BTRFS_BLOCK_GROUP_DATA:
		return "mixed";
	case BTRFS_BLOCK_GROUP_METADATA:
		return "metadata";
	case BTRFS_BLOCK_GROUP_DATA:
		return "data";
	case BTRFS_BLOCK_GROUP_SYSTEM:
		return "system";
	default:
		WARN_ON(1);
		return "invalid-combination";
	};
}

3924 3925
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
3926
			     u64 bytes_readonly,
3927 3928 3929
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
3930 3931
	int i;
	int factor;
3932
	int ret;
3933 3934 3935 3936 3937 3938

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
3939 3940 3941

	found = __find_space_info(info, flags);
	if (found) {
3942
		spin_lock(&found->lock);
3943
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
3944
		found->disk_total += total_bytes * factor;
3945
		found->bytes_used += bytes_used;
3946
		found->disk_used += bytes_used * factor;
3947
		found->bytes_readonly += bytes_readonly;
3948 3949
		if (total_bytes > 0)
			found->full = 0;
3950 3951
		space_info_add_new_bytes(info, found, total_bytes -
					 bytes_used - bytes_readonly);
3952
		spin_unlock(&found->lock);
3953 3954 3955
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
3956
	found = kzalloc(sizeof(*found), GFP_NOFS);
3957 3958 3959
	if (!found)
		return -ENOMEM;

3960
	ret = percpu_counter_init(&found->total_bytes_pinned, 0, GFP_KERNEL);
3961 3962 3963 3964 3965
	if (ret) {
		kfree(found);
		return ret;
	}

3966
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
3967
		INIT_LIST_HEAD(&found->block_groups[i]);
3968
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
3969
	spin_lock_init(&found->lock);
3970
	found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3971
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
3972
	found->disk_total = total_bytes * factor;
3973
	found->bytes_used = bytes_used;
3974
	found->disk_used = bytes_used * factor;
3975
	found->bytes_pinned = 0;
3976
	found->bytes_reserved = 0;
3977
	found->bytes_readonly = bytes_readonly;
3978
	found->bytes_may_use = 0;
3979
	found->full = 0;
3980
	found->max_extent_size = 0;
3981
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3982
	found->chunk_alloc = 0;
3983 3984
	found->flush = 0;
	init_waitqueue_head(&found->wait);
3985
	INIT_LIST_HEAD(&found->ro_bgs);
3986 3987
	INIT_LIST_HEAD(&found->tickets);
	INIT_LIST_HEAD(&found->priority_tickets);
3988 3989 3990 3991 3992 3993 3994 3995 3996

	ret = kobject_init_and_add(&found->kobj, &space_info_ktype,
				    info->space_info_kobj, "%s",
				    alloc_name(found->flags));
	if (ret) {
		kfree(found);
		return ret;
	}

3997
	*space_info = found;
3998
	list_add_rcu(&found->list, &info->space_info);
3999 4000
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		info->data_sinfo = found;
4001 4002

	return ret;
4003 4004
}

4005 4006
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
4007 4008
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
4009

4010
	write_seqlock(&fs_info->profiles_lock);
4011 4012 4013 4014 4015 4016
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits |= extra_flags;
4017
	write_sequnlock(&fs_info->profiles_lock);
4018
}
4019

4020 4021 4022
/*
 * returns target flags in extended format or 0 if restripe for this
 * chunk_type is not in progress
4023 4024
 *
 * should be called with either volume_mutex or balance_lock held
4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047
 */
static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
{
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
	u64 target = 0;

	if (!bctl)
		return 0;

	if (flags & BTRFS_BLOCK_GROUP_DATA &&
	    bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
	} else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
		   bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
	} else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
		   bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
	}

	return target;
}

4048 4049 4050
/*
 * @flags: available profiles in extended format (see ctree.h)
 *
4051 4052 4053
 * Returns reduced profile in chunk format.  If profile changing is in
 * progress (either running or paused) picks the target profile (if it's
 * already available), otherwise falls back to plain reducing.
4054
 */
4055
static u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
4056
{
4057
	u64 num_devices = root->fs_info->fs_devices->rw_devices;
4058
	u64 target;
4059 4060
	u64 raid_type;
	u64 allowed = 0;
4061

4062 4063 4064 4065
	/*
	 * see if restripe for this chunk_type is in progress, if so
	 * try to reduce to the target profile
	 */
4066
	spin_lock(&root->fs_info->balance_lock);
4067 4068 4069 4070
	target = get_restripe_target(root->fs_info, flags);
	if (target) {
		/* pick target profile only if it's already available */
		if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
4071
			spin_unlock(&root->fs_info->balance_lock);
4072
			return extended_to_chunk(target);
4073 4074 4075 4076
		}
	}
	spin_unlock(&root->fs_info->balance_lock);

D
David Woodhouse 已提交
4077
	/* First, mask out the RAID levels which aren't possible */
4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097
	for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
		if (num_devices >= btrfs_raid_array[raid_type].devs_min)
			allowed |= btrfs_raid_group[raid_type];
	}
	allowed &= flags;

	if (allowed & BTRFS_BLOCK_GROUP_RAID6)
		allowed = BTRFS_BLOCK_GROUP_RAID6;
	else if (allowed & BTRFS_BLOCK_GROUP_RAID5)
		allowed = BTRFS_BLOCK_GROUP_RAID5;
	else if (allowed & BTRFS_BLOCK_GROUP_RAID10)
		allowed = BTRFS_BLOCK_GROUP_RAID10;
	else if (allowed & BTRFS_BLOCK_GROUP_RAID1)
		allowed = BTRFS_BLOCK_GROUP_RAID1;
	else if (allowed & BTRFS_BLOCK_GROUP_RAID0)
		allowed = BTRFS_BLOCK_GROUP_RAID0;

	flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK;

	return extended_to_chunk(flags | allowed);
4098 4099
}

4100
static u64 get_alloc_profile(struct btrfs_root *root, u64 orig_flags)
J
Josef Bacik 已提交
4101
{
4102
	unsigned seq;
4103
	u64 flags;
4104 4105

	do {
4106
		flags = orig_flags;
4107 4108 4109 4110 4111 4112 4113 4114 4115
		seq = read_seqbegin(&root->fs_info->profiles_lock);

		if (flags & BTRFS_BLOCK_GROUP_DATA)
			flags |= root->fs_info->avail_data_alloc_bits;
		else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
			flags |= root->fs_info->avail_system_alloc_bits;
		else if (flags & BTRFS_BLOCK_GROUP_METADATA)
			flags |= root->fs_info->avail_metadata_alloc_bits;
	} while (read_seqretry(&root->fs_info->profiles_lock, seq));
4116

4117
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
4118 4119
}

4120
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
4121
{
4122
	u64 flags;
D
David Woodhouse 已提交
4123
	u64 ret;
J
Josef Bacik 已提交
4124

4125 4126 4127 4128
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
4129
	else
4130
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
4131

D
David Woodhouse 已提交
4132 4133
	ret = get_alloc_profile(root, flags);
	return ret;
J
Josef Bacik 已提交
4134
}
J
Josef Bacik 已提交
4135

4136
int btrfs_alloc_data_chunk_ondemand(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
4137 4138
{
	struct btrfs_space_info *data_sinfo;
4139
	struct btrfs_root *root = BTRFS_I(inode)->root;
4140
	struct btrfs_fs_info *fs_info = root->fs_info;
4141
	u64 used;
4142
	int ret = 0;
4143 4144
	int need_commit = 2;
	int have_pinned_space;
J
Josef Bacik 已提交
4145 4146

	/* make sure bytes are sectorsize aligned */
4147
	bytes = ALIGN(bytes, root->sectorsize);
J
Josef Bacik 已提交
4148

4149
	if (btrfs_is_free_space_inode(inode)) {
4150
		need_commit = 0;
4151
		ASSERT(current->journal_info);
4152 4153
	}

4154
	data_sinfo = fs_info->data_sinfo;
C
Chris Mason 已提交
4155 4156
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
4157

J
Josef Bacik 已提交
4158 4159 4160
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
4161 4162 4163
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
4164 4165

	if (used + bytes > data_sinfo->total_bytes) {
4166
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
4167

J
Josef Bacik 已提交
4168 4169 4170 4171
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
4172
		if (!data_sinfo->full) {
J
Josef Bacik 已提交
4173
			u64 alloc_target;
J
Josef Bacik 已提交
4174

4175
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
J
Josef Bacik 已提交
4176
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
4177
alloc:
J
Josef Bacik 已提交
4178
			alloc_target = btrfs_get_alloc_profile(root, 1);
4179 4180 4181 4182 4183 4184 4185 4186 4187 4188
			/*
			 * It is ugly that we don't call nolock join
			 * transaction for the free space inode case here.
			 * But it is safe because we only do the data space
			 * reservation for the free space cache in the
			 * transaction context, the common join transaction
			 * just increase the counter of the current transaction
			 * handler, doesn't try to acquire the trans_lock of
			 * the fs.
			 */
4189
			trans = btrfs_join_transaction(root);
4190 4191
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
4192

J
Josef Bacik 已提交
4193
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
4194 4195
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
J
Josef Bacik 已提交
4196
			btrfs_end_transaction(trans, root);
4197 4198 4199
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
4200 4201
				else {
					have_pinned_space = 1;
4202
					goto commit_trans;
4203
				}
4204
			}
J
Josef Bacik 已提交
4205

4206 4207 4208
			if (!data_sinfo)
				data_sinfo = fs_info->data_sinfo;

J
Josef Bacik 已提交
4209 4210
			goto again;
		}
4211 4212

		/*
4213
		 * If we don't have enough pinned space to deal with this
4214 4215
		 * allocation, and no removed chunk in current transaction,
		 * don't bother committing the transaction.
4216
		 */
4217 4218 4219
		have_pinned_space = percpu_counter_compare(
			&data_sinfo->total_bytes_pinned,
			used + bytes - data_sinfo->total_bytes);
J
Josef Bacik 已提交
4220 4221
		spin_unlock(&data_sinfo->lock);

4222
		/* commit the current transaction and try again */
4223
commit_trans:
4224
		if (need_commit &&
J
Josef Bacik 已提交
4225
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
4226
			need_commit--;
4227

4228 4229
			if (need_commit > 0) {
				btrfs_start_delalloc_roots(fs_info, 0, -1);
4230
				btrfs_wait_ordered_roots(fs_info, -1, 0, (u64)-1);
4231
			}
4232

4233
			trans = btrfs_join_transaction(root);
4234 4235
			if (IS_ERR(trans))
				return PTR_ERR(trans);
4236
			if (have_pinned_space >= 0 ||
4237 4238
			    test_bit(BTRFS_TRANS_HAVE_FREE_BGS,
				     &trans->transaction->flags) ||
4239
			    need_commit > 0) {
4240 4241 4242
				ret = btrfs_commit_transaction(trans, root);
				if (ret)
					return ret;
4243
				/*
4244 4245 4246
				 * The cleaner kthread might still be doing iput
				 * operations. Wait for it to finish so that
				 * more space is released.
4247
				 */
4248 4249
				mutex_lock(&root->fs_info->cleaner_delayed_iput_mutex);
				mutex_unlock(&root->fs_info->cleaner_delayed_iput_mutex);
4250 4251 4252 4253
				goto again;
			} else {
				btrfs_end_transaction(trans, root);
			}
4254
		}
J
Josef Bacik 已提交
4255

4256 4257 4258
		trace_btrfs_space_reservation(root->fs_info,
					      "space_info:enospc",
					      data_sinfo->flags, bytes, 1);
J
Josef Bacik 已提交
4259 4260 4261
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
J
Josef Bacik 已提交
4262
	trace_btrfs_space_reservation(root->fs_info, "space_info",
4263
				      data_sinfo->flags, bytes, 1);
J
Josef Bacik 已提交
4264 4265
	spin_unlock(&data_sinfo->lock);

4266
	return ret;
J
Josef Bacik 已提交
4267
}
J
Josef Bacik 已提交
4268

4269 4270 4271 4272 4273
/*
 * New check_data_free_space() with ability for precious data reservation
 * Will replace old btrfs_check_data_free_space(), but for patch split,
 * add a new function first and then replace it.
 */
4274
int btrfs_check_data_free_space(struct inode *inode, u64 start, u64 len)
4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	int ret;

	/* align the range */
	len = round_up(start + len, root->sectorsize) -
	      round_down(start, root->sectorsize);
	start = round_down(start, root->sectorsize);

	ret = btrfs_alloc_data_chunk_ondemand(inode, len);
	if (ret < 0)
		return ret;

4288 4289 4290 4291 4292 4293
	/*
	 * Use new btrfs_qgroup_reserve_data to reserve precious data space
	 *
	 * TODO: Find a good method to avoid reserve data space for NOCOW
	 * range, but don't impact performance on quota disable case.
	 */
4294 4295 4296 4297 4298 4299 4300 4301
	ret = btrfs_qgroup_reserve_data(inode, start, len);
	return ret;
}

/*
 * Called if we need to clear a data reservation for this inode
 * Normally in a error case.
 *
4302 4303 4304
 * This one will *NOT* use accurate qgroup reserved space API, just for case
 * which we can't sleep and is sure it won't affect qgroup reserved space.
 * Like clear_bit_hook().
4305
 */
4306 4307
void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start,
					    u64 len)
4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_space_info *data_sinfo;

	/* Make sure the range is aligned to sectorsize */
	len = round_up(start + len, root->sectorsize) -
	      round_down(start, root->sectorsize);
	start = round_down(start, root->sectorsize);

	data_sinfo = root->fs_info->data_sinfo;
	spin_lock(&data_sinfo->lock);
	if (WARN_ON(data_sinfo->bytes_may_use < len))
		data_sinfo->bytes_may_use = 0;
	else
		data_sinfo->bytes_may_use -= len;
	trace_btrfs_space_reservation(root->fs_info, "space_info",
				      data_sinfo->flags, len, 0);
	spin_unlock(&data_sinfo->lock);
}

4328 4329 4330 4331
/*
 * Called if we need to clear a data reservation for this inode
 * Normally in a error case.
 *
4332
 * This one will handle the per-inode data rsv map for accurate reserved
4333 4334 4335 4336 4337 4338 4339 4340
 * space framework.
 */
void btrfs_free_reserved_data_space(struct inode *inode, u64 start, u64 len)
{
	btrfs_free_reserved_data_space_noquota(inode, start, len);
	btrfs_qgroup_free_data(inode, start, len);
}

4341
static void force_metadata_allocation(struct btrfs_fs_info *info)
4342
{
4343 4344
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
4345

4346 4347 4348
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
4349
			found->force_alloc = CHUNK_ALLOC_FORCE;
4350
	}
4351
	rcu_read_unlock();
4352 4353
}

4354 4355 4356 4357 4358
static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global)
{
	return (global->size << 1);
}

4359
static int should_alloc_chunk(struct btrfs_root *root,
4360
			      struct btrfs_space_info *sinfo, int force)
4361
{
4362
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
4363
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
4364
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
4365
	u64 thresh;
4366

4367 4368 4369
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

4370 4371 4372 4373 4374
	/*
	 * We need to take into account the global rsv because for all intents
	 * and purposes it's used space.  Don't worry about locking the
	 * global_rsv, it doesn't change except when the transaction commits.
	 */
4375
	if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)
4376
		num_allocated += calc_global_rsv_need_space(global_rsv);
4377

4378 4379 4380 4381 4382
	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
4383
		thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
4384
		thresh = max_t(u64, SZ_64M, div_factor_fine(thresh, 1));
4385 4386 4387 4388 4389

		if (num_bytes - num_allocated < thresh)
			return 1;
	}

4390
	if (num_allocated + SZ_2M < div_factor(num_bytes, 8))
4391
		return 0;
4392
	return 1;
4393 4394
}

4395
static u64 get_profile_num_devs(struct btrfs_root *root, u64 type)
4396 4397 4398
{
	u64 num_dev;

D
David Woodhouse 已提交
4399 4400 4401 4402
	if (type & (BTRFS_BLOCK_GROUP_RAID10 |
		    BTRFS_BLOCK_GROUP_RAID0 |
		    BTRFS_BLOCK_GROUP_RAID5 |
		    BTRFS_BLOCK_GROUP_RAID6))
4403 4404 4405 4406 4407 4408
		num_dev = root->fs_info->fs_devices->rw_devices;
	else if (type & BTRFS_BLOCK_GROUP_RAID1)
		num_dev = 2;
	else
		num_dev = 1;	/* DUP or single */

4409
	return num_dev;
4410 4411
}

4412 4413 4414 4415 4416 4417 4418
/*
 * If @is_allocation is true, reserve space in the system space info necessary
 * for allocating a chunk, otherwise if it's false, reserve space necessary for
 * removing a chunk.
 */
void check_system_chunk(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
4419
			u64 type)
4420 4421 4422 4423
{
	struct btrfs_space_info *info;
	u64 left;
	u64 thresh;
4424
	int ret = 0;
4425
	u64 num_devs;
4426 4427 4428 4429 4430 4431

	/*
	 * Needed because we can end up allocating a system chunk and for an
	 * atomic and race free space reservation in the chunk block reserve.
	 */
	ASSERT(mutex_is_locked(&root->fs_info->chunk_mutex));
4432 4433 4434 4435

	info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	spin_lock(&info->lock);
	left = info->total_bytes - info->bytes_used - info->bytes_pinned -
4436 4437
		info->bytes_reserved - info->bytes_readonly -
		info->bytes_may_use;
4438 4439
	spin_unlock(&info->lock);

4440 4441 4442
	num_devs = get_profile_num_devs(root, type);

	/* num_devs device items to update and 1 chunk item to add or remove */
4443 4444
	thresh = btrfs_calc_trunc_metadata_size(root, num_devs) +
		btrfs_calc_trans_metadata_size(root, 1);
4445

4446
	if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
4447 4448
		btrfs_info(root->fs_info, "left=%llu, need=%llu, flags=%llu",
			left, thresh, type);
4449 4450 4451 4452 4453 4454 4455
		dump_space_info(info, 0, 0);
	}

	if (left < thresh) {
		u64 flags;

		flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470
		/*
		 * Ignore failure to create system chunk. We might end up not
		 * needing it, as we might not need to COW all nodes/leafs from
		 * the paths we visit in the chunk tree (they were already COWed
		 * or created in the current transaction for example).
		 */
		ret = btrfs_alloc_chunk(trans, root, flags);
	}

	if (!ret) {
		ret = btrfs_block_rsv_add(root->fs_info->chunk_root,
					  &root->fs_info->chunk_block_rsv,
					  thresh, BTRFS_RESERVE_NO_FLUSH);
		if (!ret)
			trans->chunk_bytes_reserved += thresh;
4471 4472 4473
	}
}

4474
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
4475
			  struct btrfs_root *extent_root, u64 flags, int force)
J
Josef Bacik 已提交
4476
{
4477
	struct btrfs_space_info *space_info;
4478
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
4479
	int wait_for_alloc = 0;
J
Josef Bacik 已提交
4480 4481
	int ret = 0;

4482 4483 4484 4485
	/* Don't re-enter if we're already allocating a chunk */
	if (trans->allocating_chunk)
		return -ENOSPC;

4486
	space_info = __find_space_info(extent_root->fs_info, flags);
4487 4488
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
4489
					0, 0, 0, &space_info);
4490
		BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
4491
	}
4492
	BUG_ON(!space_info); /* Logic error */
J
Josef Bacik 已提交
4493

4494
again:
4495
	spin_lock(&space_info->lock);
4496
	if (force < space_info->force_alloc)
4497
		force = space_info->force_alloc;
4498
	if (space_info->full) {
4499 4500 4501 4502
		if (should_alloc_chunk(extent_root, space_info, force))
			ret = -ENOSPC;
		else
			ret = 0;
4503
		spin_unlock(&space_info->lock);
4504
		return ret;
J
Josef Bacik 已提交
4505 4506
	}

4507
	if (!should_alloc_chunk(extent_root, space_info, force)) {
4508
		spin_unlock(&space_info->lock);
4509 4510 4511 4512 4513
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
J
Josef Bacik 已提交
4514
	}
4515

4516
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
4517

4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

4532 4533
	trans->allocating_chunk = true;

4534 4535 4536 4537 4538 4539 4540
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

4541 4542 4543 4544 4545
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
4546
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
4547 4548 4549 4550
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
4551 4552
	}

4553 4554 4555 4556
	/*
	 * Check if we have enough space in SYSTEM chunk because we may need
	 * to update devices.
	 */
4557
	check_system_chunk(trans, extent_root, flags);
4558

Y
Yan Zheng 已提交
4559
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
4560
	trans->allocating_chunk = false;
4561

J
Josef Bacik 已提交
4562
	spin_lock(&space_info->lock);
4563 4564
	if (ret < 0 && ret != -ENOSPC)
		goto out;
J
Josef Bacik 已提交
4565
	if (ret)
4566
		space_info->full = 1;
4567 4568
	else
		ret = 1;
4569

4570
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
4571
out:
4572
	space_info->chunk_alloc = 0;
J
Josef Bacik 已提交
4573
	spin_unlock(&space_info->lock);
4574
	mutex_unlock(&fs_info->chunk_mutex);
4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588
	/*
	 * When we allocate a new chunk we reserve space in the chunk block
	 * reserve to make sure we can COW nodes/leafs in the chunk tree or
	 * add new nodes/leafs to it if we end up needing to do it when
	 * inserting the chunk item and updating device items as part of the
	 * second phase of chunk allocation, performed by
	 * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a
	 * large number of new block groups to create in our transaction
	 * handle's new_bgs list to avoid exhausting the chunk block reserve
	 * in extreme cases - like having a single transaction create many new
	 * block groups when starting to write out the free space caches of all
	 * the block groups that were made dirty during the lifetime of the
	 * transaction.
	 */
4589
	if (trans->can_flush_pending_bgs &&
4590
	    trans->chunk_bytes_reserved >= (u64)SZ_2M) {
4591 4592 4593
		btrfs_create_pending_block_groups(trans, trans->root);
		btrfs_trans_release_chunk_metadata(trans);
	}
J
Josef Bacik 已提交
4594
	return ret;
4595
}
J
Josef Bacik 已提交
4596

J
Josef Bacik 已提交
4597 4598
static int can_overcommit(struct btrfs_root *root,
			  struct btrfs_space_info *space_info, u64 bytes,
M
Miao Xie 已提交
4599
			  enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
4600
{
4601 4602
	struct btrfs_block_rsv *global_rsv;
	u64 profile;
4603
	u64 space_size;
J
Josef Bacik 已提交
4604 4605 4606
	u64 avail;
	u64 used;

4607 4608 4609 4610 4611 4612 4613
	/* Don't overcommit when in mixed mode. */
	if (space_info->flags & BTRFS_BLOCK_GROUP_DATA)
		return 0;

	BUG_ON(root->fs_info == NULL);
	global_rsv = &root->fs_info->global_block_rsv;
	profile = btrfs_get_alloc_profile(root, 0);
J
Josef Bacik 已提交
4614
	used = space_info->bytes_used + space_info->bytes_reserved +
4615 4616 4617 4618 4619 4620 4621 4622
		space_info->bytes_pinned + space_info->bytes_readonly;

	/*
	 * We only want to allow over committing if we have lots of actual space
	 * free, but if we don't have enough space to handle the global reserve
	 * space then we could end up having a real enospc problem when trying
	 * to allocate a chunk or some other such important allocation.
	 */
4623 4624 4625 4626
	spin_lock(&global_rsv->lock);
	space_size = calc_global_rsv_need_space(global_rsv);
	spin_unlock(&global_rsv->lock);
	if (used + space_size >= space_info->total_bytes)
4627 4628 4629
		return 0;

	used += space_info->bytes_may_use;
J
Josef Bacik 已提交
4630 4631 4632 4633 4634 4635 4636

	spin_lock(&root->fs_info->free_chunk_lock);
	avail = root->fs_info->free_chunk_space;
	spin_unlock(&root->fs_info->free_chunk_lock);

	/*
	 * If we have dup, raid1 or raid10 then only half of the free
D
David Woodhouse 已提交
4637 4638 4639
	 * space is actually useable.  For raid56, the space info used
	 * doesn't include the parity drive, so we don't have to
	 * change the math
J
Josef Bacik 已提交
4640 4641 4642 4643 4644 4645 4646
	 */
	if (profile & (BTRFS_BLOCK_GROUP_DUP |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_RAID10))
		avail >>= 1;

	/*
4647 4648 4649
	 * If we aren't flushing all things, let us overcommit up to
	 * 1/2th of the space. If we can flush, don't let us overcommit
	 * too much, let it overcommit up to 1/8 of the space.
J
Josef Bacik 已提交
4650
	 */
M
Miao Xie 已提交
4651
	if (flush == BTRFS_RESERVE_FLUSH_ALL)
4652
		avail >>= 3;
J
Josef Bacik 已提交
4653
	else
4654
		avail >>= 1;
J
Josef Bacik 已提交
4655

4656
	if (used + bytes < space_info->total_bytes + avail)
J
Josef Bacik 已提交
4657 4658 4659 4660
		return 1;
	return 0;
}

4661
static void btrfs_writeback_inodes_sb_nr(struct btrfs_root *root,
4662
					 unsigned long nr_pages, int nr_items)
4663 4664 4665
{
	struct super_block *sb = root->fs_info->sb;

4666 4667 4668 4669
	if (down_read_trylock(&sb->s_umount)) {
		writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE);
		up_read(&sb->s_umount);
	} else {
4670 4671 4672 4673 4674 4675 4676
		/*
		 * We needn't worry the filesystem going from r/w to r/o though
		 * we don't acquire ->s_umount mutex, because the filesystem
		 * should guarantee the delalloc inodes list be empty after
		 * the filesystem is readonly(all dirty pages are written to
		 * the disk).
		 */
4677
		btrfs_start_delalloc_roots(root->fs_info, 0, nr_items);
4678
		if (!current->journal_info)
4679 4680
			btrfs_wait_ordered_roots(root->fs_info, nr_items,
						 0, (u64)-1);
4681 4682 4683
	}
}

4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695
static inline int calc_reclaim_items_nr(struct btrfs_root *root, u64 to_reclaim)
{
	u64 bytes;
	int nr;

	bytes = btrfs_calc_trans_metadata_size(root, 1);
	nr = (int)div64_u64(to_reclaim, bytes);
	if (!nr)
		nr = 1;
	return nr;
}

4696
#define EXTENT_SIZE_PER_ITEM	SZ_256K
4697

J
Josef Bacik 已提交
4698
/*
4699
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
4700
 */
J
Josef Bacik 已提交
4701 4702
static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig,
			    bool wait_ordered)
4703
{
4704
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
4705
	struct btrfs_space_info *space_info;
4706
	struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
4707
	u64 delalloc_bytes;
4708
	u64 max_reclaim;
4709
	long time_left;
4710 4711
	unsigned long nr_pages;
	int loops;
4712
	int items;
M
Miao Xie 已提交
4713
	enum btrfs_reserve_flush_enum flush;
4714

4715
	/* Calc the number of the pages we need flush for space reservation */
4716
	items = calc_reclaim_items_nr(root, to_reclaim);
4717
	to_reclaim = (u64)items * EXTENT_SIZE_PER_ITEM;
4718

4719
	trans = (struct btrfs_trans_handle *)current->journal_info;
4720
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
4721
	space_info = block_rsv->space_info;
4722

4723 4724
	delalloc_bytes = percpu_counter_sum_positive(
						&root->fs_info->delalloc_bytes);
J
Josef Bacik 已提交
4725
	if (delalloc_bytes == 0) {
4726
		if (trans)
J
Josef Bacik 已提交
4727
			return;
4728
		if (wait_ordered)
4729 4730
			btrfs_wait_ordered_roots(root->fs_info, items,
						 0, (u64)-1);
J
Josef Bacik 已提交
4731
		return;
4732 4733
	}

4734
	loops = 0;
J
Josef Bacik 已提交
4735 4736
	while (delalloc_bytes && loops < 3) {
		max_reclaim = min(delalloc_bytes, to_reclaim);
4737
		nr_pages = max_reclaim >> PAGE_SHIFT;
4738
		btrfs_writeback_inodes_sb_nr(root, nr_pages, items);
4739 4740 4741 4742
		/*
		 * We need to wait for the async pages to actually start before
		 * we do anything.
		 */
4743 4744 4745 4746 4747 4748 4749 4750
		max_reclaim = atomic_read(&root->fs_info->async_delalloc_pages);
		if (!max_reclaim)
			goto skip_async;

		if (max_reclaim <= nr_pages)
			max_reclaim = 0;
		else
			max_reclaim -= nr_pages;
4751

4752 4753 4754 4755
		wait_event(root->fs_info->async_submit_wait,
			   atomic_read(&root->fs_info->async_delalloc_pages) <=
			   (int)max_reclaim);
skip_async:
M
Miao Xie 已提交
4756 4757 4758 4759
		if (!trans)
			flush = BTRFS_RESERVE_FLUSH_ALL;
		else
			flush = BTRFS_RESERVE_NO_FLUSH;
J
Josef Bacik 已提交
4760
		spin_lock(&space_info->lock);
M
Miao Xie 已提交
4761
		if (can_overcommit(root, space_info, orig, flush)) {
J
Josef Bacik 已提交
4762 4763 4764
			spin_unlock(&space_info->lock);
			break;
		}
4765 4766 4767 4768 4769
		if (list_empty(&space_info->tickets) &&
		    list_empty(&space_info->priority_tickets)) {
			spin_unlock(&space_info->lock);
			break;
		}
J
Josef Bacik 已提交
4770
		spin_unlock(&space_info->lock);
4771

4772
		loops++;
4773
		if (wait_ordered && !trans) {
4774 4775
			btrfs_wait_ordered_roots(root->fs_info, items,
						 0, (u64)-1);
4776
		} else {
J
Josef Bacik 已提交
4777
			time_left = schedule_timeout_killable(1);
4778 4779 4780
			if (time_left)
				break;
		}
4781 4782
		delalloc_bytes = percpu_counter_sum_positive(
						&root->fs_info->delalloc_bytes);
4783 4784 4785
	}
}

4786 4787 4788 4789 4790
/**
 * maybe_commit_transaction - possibly commit the transaction if its ok to
 * @root - the root we're allocating for
 * @bytes - the number of bytes we want to reserve
 * @force - force the commit
4791
 *
4792 4793 4794
 * This will check to make sure that committing the transaction will actually
 * get us somewhere and then commit the transaction if it does.  Otherwise it
 * will return -ENOSPC.
4795
 */
4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810
static int may_commit_transaction(struct btrfs_root *root,
				  struct btrfs_space_info *space_info,
				  u64 bytes, int force)
{
	struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
	struct btrfs_trans_handle *trans;

	trans = (struct btrfs_trans_handle *)current->journal_info;
	if (trans)
		return -EAGAIN;

	if (force)
		goto commit;

	/* See if there is enough pinned space to make this reservation */
4811
	if (percpu_counter_compare(&space_info->total_bytes_pinned,
4812
				   bytes) >= 0)
4813 4814 4815 4816 4817 4818 4819 4820 4821 4822
		goto commit;

	/*
	 * See if there is some space in the delayed insertion reservation for
	 * this reservation.
	 */
	if (space_info != delayed_rsv->space_info)
		return -ENOSPC;

	spin_lock(&delayed_rsv->lock);
4823 4824
	if (percpu_counter_compare(&space_info->total_bytes_pinned,
				   bytes - delayed_rsv->size) >= 0) {
4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837
		spin_unlock(&delayed_rsv->lock);
		return -ENOSPC;
	}
	spin_unlock(&delayed_rsv->lock);

commit:
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans))
		return -ENOSPC;

	return btrfs_commit_transaction(trans, root);
}

4838
enum flush_state {
4839 4840 4841 4842
	FLUSH_DELAYED_ITEMS_NR	=	1,
	FLUSH_DELAYED_ITEMS	=	2,
	FLUSH_DELALLOC		=	3,
	FLUSH_DELALLOC_WAIT	=	4,
4843 4844
	ALLOC_CHUNK		=	5,
	COMMIT_TRANS		=	6,
4845 4846
};

4847 4848 4849 4850 4851 4852 4853
struct reserve_ticket {
	u64 bytes;
	int error;
	struct list_head list;
	wait_queue_head_t wait;
};

4854 4855 4856 4857 4858 4859
static int flush_space(struct btrfs_root *root,
		       struct btrfs_space_info *space_info, u64 num_bytes,
		       u64 orig_bytes, int state)
{
	struct btrfs_trans_handle *trans;
	int nr;
J
Josef Bacik 已提交
4860
	int ret = 0;
4861 4862 4863 4864

	switch (state) {
	case FLUSH_DELAYED_ITEMS_NR:
	case FLUSH_DELAYED_ITEMS:
4865 4866 4867
		if (state == FLUSH_DELAYED_ITEMS_NR)
			nr = calc_reclaim_items_nr(root, num_bytes) * 2;
		else
4868
			nr = -1;
4869

4870 4871 4872 4873 4874 4875 4876 4877
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = btrfs_run_delayed_items_nr(trans, root, nr);
		btrfs_end_transaction(trans, root);
		break;
4878 4879
	case FLUSH_DELALLOC:
	case FLUSH_DELALLOC_WAIT:
4880
		shrink_delalloc(root, num_bytes * 2, orig_bytes,
4881 4882
				state == FLUSH_DELALLOC_WAIT);
		break;
4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895
	case ALLOC_CHUNK:
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
				     btrfs_get_alloc_profile(root, 0),
				     CHUNK_ALLOC_NO_FORCE);
		btrfs_end_transaction(trans, root);
		if (ret == -ENOSPC)
			ret = 0;
		break;
4896 4897 4898 4899 4900 4901 4902 4903 4904 4905
	case COMMIT_TRANS:
		ret = may_commit_transaction(root, space_info, orig_bytes, 0);
		break;
	default:
		ret = -ENOSPC;
		break;
	}

	return ret;
}
4906 4907 4908 4909 4910

static inline u64
btrfs_calc_reclaim_metadata_size(struct btrfs_root *root,
				 struct btrfs_space_info *space_info)
{
4911
	struct reserve_ticket *ticket;
4912 4913
	u64 used;
	u64 expected;
4914
	u64 to_reclaim = 0;
4915

4916
	to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M);
4917
	if (can_overcommit(root, space_info, to_reclaim,
4918 4919 4920 4921 4922 4923 4924 4925 4926
			   BTRFS_RESERVE_FLUSH_ALL))
		return 0;

	list_for_each_entry(ticket, &space_info->tickets, list)
		to_reclaim += ticket->bytes;
	list_for_each_entry(ticket, &space_info->priority_tickets, list)
		to_reclaim += ticket->bytes;
	if (to_reclaim)
		return to_reclaim;
4927 4928 4929 4930

	used = space_info->bytes_used + space_info->bytes_reserved +
	       space_info->bytes_pinned + space_info->bytes_readonly +
	       space_info->bytes_may_use;
4931
	if (can_overcommit(root, space_info, SZ_1M, BTRFS_RESERVE_FLUSH_ALL))
4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947
		expected = div_factor_fine(space_info->total_bytes, 95);
	else
		expected = div_factor_fine(space_info->total_bytes, 90);

	if (used > expected)
		to_reclaim = used - expected;
	else
		to_reclaim = 0;
	to_reclaim = min(to_reclaim, space_info->bytes_may_use +
				     space_info->bytes_reserved);
	return to_reclaim;
}

static inline int need_do_async_reclaim(struct btrfs_space_info *space_info,
					struct btrfs_fs_info *fs_info, u64 used)
{
4948 4949 4950
	u64 thresh = div_factor_fine(space_info->total_bytes, 98);

	/* If we're just plain full then async reclaim just slows us down. */
4951
	if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh)
4952 4953 4954
		return 0;

	return (used >= thresh && !btrfs_fs_closing(fs_info) &&
4955 4956 4957
		!test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state));
}

4958
static void wake_all_tickets(struct list_head *head)
4959
{
4960
	struct reserve_ticket *ticket;
4961

4962 4963 4964 4965 4966
	while (!list_empty(head)) {
		ticket = list_first_entry(head, struct reserve_ticket, list);
		list_del_init(&ticket->list);
		ticket->error = -ENOSPC;
		wake_up(&ticket->wait);
4967
	}
4968 4969
}

4970 4971 4972 4973 4974
/*
 * This is for normal flushers, we can wait all goddamned day if we want to.  We
 * will loop and continuously try to flush as long as we are making progress.
 * We count progress as clearing off tickets each time we have to loop.
 */
4975 4976
static void btrfs_async_reclaim_metadata_space(struct work_struct *work)
{
4977
	struct reserve_ticket *last_ticket = NULL;
4978 4979 4980 4981
	struct btrfs_fs_info *fs_info;
	struct btrfs_space_info *space_info;
	u64 to_reclaim;
	int flush_state;
4982
	int commit_cycles = 0;
4983 4984 4985 4986

	fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work);
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);

4987
	spin_lock(&space_info->lock);
4988 4989
	to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info->fs_root,
						      space_info);
4990 4991 4992
	if (!to_reclaim) {
		space_info->flush = 0;
		spin_unlock(&space_info->lock);
4993
		return;
4994 4995 4996 4997
	}
	last_ticket = list_first_entry(&space_info->tickets,
				       struct reserve_ticket, list);
	spin_unlock(&space_info->lock);
4998 4999

	flush_state = FLUSH_DELAYED_ITEMS_NR;
5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058
	do {
		struct reserve_ticket *ticket;
		int ret;

		ret = flush_space(fs_info->fs_root, space_info, to_reclaim,
			    to_reclaim, flush_state);
		spin_lock(&space_info->lock);
		if (list_empty(&space_info->tickets)) {
			space_info->flush = 0;
			spin_unlock(&space_info->lock);
			return;
		}
		to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info->fs_root,
							      space_info);
		ticket = list_first_entry(&space_info->tickets,
					  struct reserve_ticket, list);
		if (last_ticket == ticket) {
			flush_state++;
		} else {
			last_ticket = ticket;
			flush_state = FLUSH_DELAYED_ITEMS_NR;
			if (commit_cycles)
				commit_cycles--;
		}

		if (flush_state > COMMIT_TRANS) {
			commit_cycles++;
			if (commit_cycles > 2) {
				wake_all_tickets(&space_info->tickets);
				space_info->flush = 0;
			} else {
				flush_state = FLUSH_DELAYED_ITEMS_NR;
			}
		}
		spin_unlock(&space_info->lock);
	} while (flush_state <= COMMIT_TRANS);
}

void btrfs_init_async_reclaim_work(struct work_struct *work)
{
	INIT_WORK(work, btrfs_async_reclaim_metadata_space);
}

static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info,
					    struct btrfs_space_info *space_info,
					    struct reserve_ticket *ticket)
{
	u64 to_reclaim;
	int flush_state = FLUSH_DELAYED_ITEMS_NR;

	spin_lock(&space_info->lock);
	to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info->fs_root,
						      space_info);
	if (!to_reclaim) {
		spin_unlock(&space_info->lock);
		return;
	}
	spin_unlock(&space_info->lock);

5059 5060 5061 5062
	do {
		flush_space(fs_info->fs_root, space_info, to_reclaim,
			    to_reclaim, flush_state);
		flush_state++;
5063 5064 5065
		spin_lock(&space_info->lock);
		if (ticket->bytes == 0) {
			spin_unlock(&space_info->lock);
5066
			return;
5067 5068 5069 5070 5071 5072 5073 5074 5075 5076
		}
		spin_unlock(&space_info->lock);

		/*
		 * Priority flushers can't wait on delalloc without
		 * deadlocking.
		 */
		if (flush_state == FLUSH_DELALLOC ||
		    flush_state == FLUSH_DELALLOC_WAIT)
			flush_state = ALLOC_CHUNK;
5077
	} while (flush_state < COMMIT_TRANS);
5078 5079
}

5080 5081 5082 5083
static int wait_reserve_ticket(struct btrfs_fs_info *fs_info,
			       struct btrfs_space_info *space_info,
			       struct reserve_ticket *ticket, u64 orig_bytes)

5084
{
5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114
	DEFINE_WAIT(wait);
	int ret = 0;

	spin_lock(&space_info->lock);
	while (ticket->bytes > 0 && ticket->error == 0) {
		ret = prepare_to_wait_event(&ticket->wait, &wait, TASK_KILLABLE);
		if (ret) {
			ret = -EINTR;
			break;
		}
		spin_unlock(&space_info->lock);

		schedule();

		finish_wait(&ticket->wait, &wait);
		spin_lock(&space_info->lock);
	}
	if (!ret)
		ret = ticket->error;
	if (!list_empty(&ticket->list))
		list_del_init(&ticket->list);
	if (ticket->bytes && ticket->bytes < orig_bytes) {
		u64 num_bytes = orig_bytes - ticket->bytes;
		space_info->bytes_may_use -= num_bytes;
		trace_btrfs_space_reservation(fs_info, "space_info",
					      space_info->flags, num_bytes, 0);
	}
	spin_unlock(&space_info->lock);

	return ret;
5115 5116
}

5117 5118 5119
/**
 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
 * @root - the root we're allocating for
5120
 * @space_info - the space info we want to allocate from
5121
 * @orig_bytes - the number of bytes we want
5122
 * @flush - whether or not we can flush to make our reservation
5123
 *
5124
 * This will reserve orig_bytes number of bytes from the space info associated
5125 5126 5127 5128 5129
 * with the block_rsv.  If there is not enough space it will make an attempt to
 * flush out space to make room.  It will do this by flushing delalloc if
 * possible or committing the transaction.  If flush is 0 then no attempts to
 * regain reservations will be made and this will fail if there is not enough
 * space already.
5130
 */
5131 5132 5133 5134
static int __reserve_metadata_bytes(struct btrfs_root *root,
				    struct btrfs_space_info *space_info,
				    u64 orig_bytes,
				    enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
5135
{
5136
	struct reserve_ticket ticket;
5137
	u64 used;
5138
	int ret = 0;
J
Josef Bacik 已提交
5139

5140
	ASSERT(orig_bytes);
5141
	spin_lock(&space_info->lock);
5142
	ret = -ENOSPC;
5143 5144 5145
	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;
J
Josef Bacik 已提交
5146

5147
	/*
5148 5149 5150
	 * If we have enough space then hooray, make our reservation and carry
	 * on.  If not see if we can overcommit, and if we can, hooray carry on.
	 * If not things get more complicated.
5151
	 */
5152 5153 5154 5155 5156 5157 5158
	if (used + orig_bytes <= space_info->total_bytes) {
		space_info->bytes_may_use += orig_bytes;
		trace_btrfs_space_reservation(root->fs_info, "space_info",
					      space_info->flags, orig_bytes,
					      1);
		ret = 0;
	} else if (can_overcommit(root, space_info, orig_bytes, flush)) {
5159 5160 5161 5162 5163
		space_info->bytes_may_use += orig_bytes;
		trace_btrfs_space_reservation(root->fs_info, "space_info",
					      space_info->flags, orig_bytes,
					      1);
		ret = 0;
5164 5165
	}

5166
	/*
5167 5168
	 * If we couldn't make a reservation then setup our reservation ticket
	 * and kick the async worker if it's not already running.
M
Miao Xie 已提交
5169
	 *
5170 5171
	 * If we are a priority flusher then we just need to add our ticket to
	 * the list and we will do our own flushing further down.
5172
	 */
5173
	if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187
		ticket.bytes = orig_bytes;
		ticket.error = 0;
		init_waitqueue_head(&ticket.wait);
		if (flush == BTRFS_RESERVE_FLUSH_ALL) {
			list_add_tail(&ticket.list, &space_info->tickets);
			if (!space_info->flush) {
				space_info->flush = 1;
				queue_work(system_unbound_wq,
					   &root->fs_info->async_reclaim_work);
			}
		} else {
			list_add_tail(&ticket.list,
				      &space_info->priority_tickets);
		}
5188 5189
	} else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) {
		used += orig_bytes;
5190 5191 5192 5193 5194 5195 5196
		/*
		 * We will do the space reservation dance during log replay,
		 * which means we won't have fs_info->fs_root set, so don't do
		 * the async reclaim as we will panic.
		 */
		if (!root->fs_info->log_root_recovering &&
		    need_do_async_reclaim(space_info, root->fs_info, used) &&
5197 5198 5199
		    !work_busy(&root->fs_info->async_reclaim_work))
			queue_work(system_unbound_wq,
				   &root->fs_info->async_reclaim_work);
5200
	}
5201
	spin_unlock(&space_info->lock);
M
Miao Xie 已提交
5202
	if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
5203
		return ret;
5204

5205 5206 5207
	if (flush == BTRFS_RESERVE_FLUSH_ALL)
		return wait_reserve_ticket(root->fs_info, space_info, &ticket,
					   orig_bytes);
M
Miao Xie 已提交
5208

5209 5210 5211 5212 5213 5214 5215 5216 5217 5218
	ret = 0;
	priority_reclaim_metadata_space(root->fs_info, space_info, &ticket);
	spin_lock(&space_info->lock);
	if (ticket.bytes) {
		if (ticket.bytes < orig_bytes) {
			u64 num_bytes = orig_bytes - ticket.bytes;
			space_info->bytes_may_use -= num_bytes;
			trace_btrfs_space_reservation(root->fs_info,
					"space_info", space_info->flags,
					num_bytes, 0);
M
Miao Xie 已提交
5219

5220 5221 5222 5223 5224 5225 5226 5227
		}
		list_del_init(&ticket.list);
		ret = -ENOSPC;
	}
	spin_unlock(&space_info->lock);
	ASSERT(list_empty(&ticket.list));
	return ret;
}
5228

5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251
/**
 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
 * @root - the root we're allocating for
 * @block_rsv - the block_rsv we're allocating for
 * @orig_bytes - the number of bytes we want
 * @flush - whether or not we can flush to make our reservation
 *
 * This will reserve orgi_bytes number of bytes from the space info associated
 * with the block_rsv.  If there is not enough space it will make an attempt to
 * flush out space to make room.  It will do this by flushing delalloc if
 * possible or committing the transaction.  If flush is 0 then no attempts to
 * regain reservations will be made and this will fail if there is not enough
 * space already.
 */
static int reserve_metadata_bytes(struct btrfs_root *root,
				  struct btrfs_block_rsv *block_rsv,
				  u64 orig_bytes,
				  enum btrfs_reserve_flush_enum flush)
{
	int ret;

	ret = __reserve_metadata_bytes(root, block_rsv->space_info, orig_bytes,
				       flush);
5252 5253 5254 5255 5256 5257 5258 5259 5260
	if (ret == -ENOSPC &&
	    unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) {
		struct btrfs_block_rsv *global_rsv =
			&root->fs_info->global_block_rsv;

		if (block_rsv != global_rsv &&
		    !block_rsv_use_bytes(global_rsv, orig_bytes))
			ret = 0;
	}
5261 5262 5263
	if (ret == -ENOSPC)
		trace_btrfs_space_reservation(root->fs_info,
					      "space_info:enospc",
5264 5265
					      block_rsv->space_info->flags,
					      orig_bytes, 1);
5266 5267 5268
	return ret;
}

5269 5270 5271
static struct btrfs_block_rsv *get_block_rsv(
					const struct btrfs_trans_handle *trans,
					const struct btrfs_root *root)
5272
{
5273 5274
	struct btrfs_block_rsv *block_rsv = NULL;

5275 5276 5277
	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
	    (root == root->fs_info->csum_root && trans->adding_csums) ||
	     (root == root->fs_info->uuid_root))
5278 5279
		block_rsv = trans->block_rsv;

5280
	if (!block_rsv)
5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340
int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info,
			     struct btrfs_block_rsv *dest, u64 num_bytes,
			     int min_factor)
{
	struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
	u64 min_bytes;

	if (global_rsv->space_info != dest->space_info)
		return -ENOSPC;

	spin_lock(&global_rsv->lock);
	min_bytes = div_factor(global_rsv->size, min_factor);
	if (global_rsv->reserved < min_bytes + num_bytes) {
		spin_unlock(&global_rsv->lock);
		return -ENOSPC;
	}
	global_rsv->reserved -= num_bytes;
	if (global_rsv->reserved < global_rsv->size)
		global_rsv->full = 0;
	spin_unlock(&global_rsv->lock);

	block_rsv_add_bytes(dest, num_bytes, 1);
	return 0;
}

5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442
/*
 * This is for space we already have accounted in space_info->bytes_may_use, so
 * basically when we're returning space from block_rsv's.
 */
static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info,
				     struct btrfs_space_info *space_info,
				     u64 num_bytes)
{
	struct reserve_ticket *ticket;
	struct list_head *head;
	u64 used;
	enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH;
	bool check_overcommit = false;

	spin_lock(&space_info->lock);
	head = &space_info->priority_tickets;

	/*
	 * If we are over our limit then we need to check and see if we can
	 * overcommit, and if we can't then we just need to free up our space
	 * and not satisfy any requests.
	 */
	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;
	if (used - num_bytes >= space_info->total_bytes)
		check_overcommit = true;
again:
	while (!list_empty(head) && num_bytes) {
		ticket = list_first_entry(head, struct reserve_ticket,
					  list);
		/*
		 * We use 0 bytes because this space is already reserved, so
		 * adding the ticket space would be a double count.
		 */
		if (check_overcommit &&
		    !can_overcommit(fs_info->extent_root, space_info, 0,
				    flush))
			break;
		if (num_bytes >= ticket->bytes) {
			list_del_init(&ticket->list);
			num_bytes -= ticket->bytes;
			ticket->bytes = 0;
			wake_up(&ticket->wait);
		} else {
			ticket->bytes -= num_bytes;
			num_bytes = 0;
		}
	}

	if (num_bytes && head == &space_info->priority_tickets) {
		head = &space_info->tickets;
		flush = BTRFS_RESERVE_FLUSH_ALL;
		goto again;
	}
	space_info->bytes_may_use -= num_bytes;
	trace_btrfs_space_reservation(fs_info, "space_info",
				      space_info->flags, num_bytes, 0);
	spin_unlock(&space_info->lock);
}

/*
 * This is for newly allocated space that isn't accounted in
 * space_info->bytes_may_use yet.  So if we allocate a chunk or unpin an extent
 * we use this helper.
 */
static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info,
				     struct btrfs_space_info *space_info,
				     u64 num_bytes)
{
	struct reserve_ticket *ticket;
	struct list_head *head = &space_info->priority_tickets;

again:
	while (!list_empty(head) && num_bytes) {
		ticket = list_first_entry(head, struct reserve_ticket,
					  list);
		if (num_bytes >= ticket->bytes) {
			trace_btrfs_space_reservation(fs_info, "space_info",
						      space_info->flags,
						      ticket->bytes, 1);
			list_del_init(&ticket->list);
			num_bytes -= ticket->bytes;
			space_info->bytes_may_use += ticket->bytes;
			ticket->bytes = 0;
			wake_up(&ticket->wait);
		} else {
			trace_btrfs_space_reservation(fs_info, "space_info",
						      space_info->flags,
						      num_bytes, 1);
			space_info->bytes_may_use += num_bytes;
			ticket->bytes -= num_bytes;
			num_bytes = 0;
		}
	}

	if (num_bytes && head == &space_info->priority_tickets) {
		head = &space_info->tickets;
		goto again;
	}
}

J
Josef Bacik 已提交
5443 5444
static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
				    struct btrfs_block_rsv *block_rsv,
5445
				    struct btrfs_block_rsv *dest, u64 num_bytes)
5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
5477 5478 5479
		if (num_bytes)
			space_info_add_old_bytes(fs_info, space_info,
						 num_bytes);
J
Josef Bacik 已提交
5480
	}
5481
}
5482

5483 5484 5485
int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src,
			    struct btrfs_block_rsv *dst, u64 num_bytes,
			    int update_size)
5486 5487
{
	int ret;
J
Josef Bacik 已提交
5488

5489 5490 5491
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
5492

5493
	block_rsv_add_bytes(dst, num_bytes, update_size);
J
Josef Bacik 已提交
5494 5495 5496
	return 0;
}

5497
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
J
Josef Bacik 已提交
5498
{
5499 5500
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
5501
	rsv->type = type;
5502 5503
}

5504 5505
struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root,
					      unsigned short type)
5506 5507 5508
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
5509

5510 5511 5512
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
5513

5514
	btrfs_init_block_rsv(block_rsv, type);
5515 5516 5517 5518
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
5519

5520 5521 5522
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
J
Josef Bacik 已提交
5523 5524
	if (!rsv)
		return;
5525 5526
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	kfree(rsv);
J
Josef Bacik 已提交
5527 5528
}

5529 5530 5531 5532 5533
void __btrfs_free_block_rsv(struct btrfs_block_rsv *rsv)
{
	kfree(rsv);
}

M
Miao Xie 已提交
5534 5535 5536
int btrfs_block_rsv_add(struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv, u64 num_bytes,
			enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
5537
{
5538
	int ret;
J
Josef Bacik 已提交
5539

5540 5541
	if (num_bytes == 0)
		return 0;
5542

5543
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
5544 5545 5546 5547
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
5548

5549 5550
	return ret;
}
J
Josef Bacik 已提交
5551

5552
int btrfs_block_rsv_check(struct btrfs_root *root,
5553
			  struct btrfs_block_rsv *block_rsv, int min_factor)
5554 5555 5556
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
5557

5558 5559
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
5560

5561
	spin_lock(&block_rsv->lock);
5562 5563 5564 5565
	num_bytes = div_factor(block_rsv->size, min_factor);
	if (block_rsv->reserved >= num_bytes)
		ret = 0;
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
5566

5567 5568 5569
	return ret;
}

M
Miao Xie 已提交
5570 5571 5572
int btrfs_block_rsv_refill(struct btrfs_root *root,
			   struct btrfs_block_rsv *block_rsv, u64 min_reserved,
			   enum btrfs_reserve_flush_enum flush)
5573 5574 5575 5576 5577 5578 5579 5580 5581
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;

	if (!block_rsv)
		return 0;

	spin_lock(&block_rsv->lock);
	num_bytes = min_reserved;
5582
	if (block_rsv->reserved >= num_bytes)
5583
		ret = 0;
5584
	else
5585 5586
		num_bytes -= block_rsv->reserved;
	spin_unlock(&block_rsv->lock);
5587

5588 5589 5590
	if (!ret)
		return 0;

5591
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
5592 5593
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 0);
5594
		return 0;
J
Josef Bacik 已提交
5595
	}
J
Josef Bacik 已提交
5596

5597
	return ret;
5598 5599 5600 5601 5602 5603 5604
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
5605
	if (global_rsv == block_rsv ||
5606 5607
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
J
Josef Bacik 已提交
5608 5609
	block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
				num_bytes);
J
Josef Bacik 已提交
5610 5611 5612
}

/*
5613 5614 5615
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
5616
 */
5617
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
5618
{
5619 5620 5621 5622
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
5623
	int csum_size = btrfs_super_csum_size(fs_info->super_copy);
J
Josef Bacik 已提交
5624

5625 5626 5627 5628
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
5629

5630 5631
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
5632 5633
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
5634 5635
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
5636

5637 5638
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
5639
	num_bytes += div_u64(data_used + meta_used, 50);
5640

5641
	if (num_bytes * 3 > meta_used)
5642
		num_bytes = div_u64(meta_used, 3);
5643

5644
	return ALIGN(num_bytes, fs_info->extent_root->nodesize << 10);
5645
}
J
Josef Bacik 已提交
5646

5647 5648 5649 5650 5651
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
5652

5653
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
5654

5655
	spin_lock(&sinfo->lock);
5656
	spin_lock(&block_rsv->lock);
5657

5658
	block_rsv->size = min_t(u64, num_bytes, SZ_512M);
5659

5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674
	if (block_rsv->reserved < block_rsv->size) {
		num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
			sinfo->bytes_reserved + sinfo->bytes_readonly +
			sinfo->bytes_may_use;
		if (sinfo->total_bytes > num_bytes) {
			num_bytes = sinfo->total_bytes - num_bytes;
			num_bytes = min(num_bytes,
					block_rsv->size - block_rsv->reserved);
			block_rsv->reserved += num_bytes;
			sinfo->bytes_may_use += num_bytes;
			trace_btrfs_space_reservation(fs_info, "space_info",
						      sinfo->flags, num_bytes,
						      1);
		}
	} else if (block_rsv->reserved > block_rsv->size) {
5675
		num_bytes = block_rsv->reserved - block_rsv->size;
5676
		sinfo->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
5677
		trace_btrfs_space_reservation(fs_info, "space_info",
5678
				      sinfo->flags, num_bytes, 0);
5679 5680
		block_rsv->reserved = block_rsv->size;
	}
5681

5682 5683 5684 5685 5686
	if (block_rsv->reserved == block_rsv->size)
		block_rsv->full = 1;
	else
		block_rsv->full = 0;

5687
	spin_unlock(&block_rsv->lock);
5688
	spin_unlock(&sinfo->lock);
J
Josef Bacik 已提交
5689 5690
}

5691
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
5692
{
5693
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
5694

5695 5696
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
J
Josef Bacik 已提交
5697

5698
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
5699 5700
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->delalloc_block_rsv.space_info = space_info;
5701 5702
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
5703
	fs_info->delayed_block_rsv.space_info = space_info;
5704

5705 5706 5707 5708
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
5709 5710
	if (fs_info->quota_root)
		fs_info->quota_root->block_rsv = &fs_info->global_block_rsv;
5711
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
5712 5713

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
5714 5715
}

5716
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
5717
{
J
Josef Bacik 已提交
5718 5719
	block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
				(u64)-1);
5720 5721 5722 5723 5724 5725
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
5726 5727
	WARN_ON(fs_info->delayed_block_rsv.size > 0);
	WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
5728 5729
}

5730 5731
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
5732
{
5733 5734 5735
	if (!trans->block_rsv)
		return;

5736 5737
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
5738

5739
	trace_btrfs_space_reservation(root->fs_info, "transaction",
5740
				      trans->transid, trans->bytes_reserved, 0);
5741
	btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
5742 5743
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
5744

5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762
/*
 * To be called after all the new block groups attached to the transaction
 * handle have been created (btrfs_create_pending_block_groups()).
 */
void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans)
{
	struct btrfs_fs_info *fs_info = trans->root->fs_info;

	if (!trans->chunk_bytes_reserved)
		return;

	WARN_ON_ONCE(!list_empty(&trans->new_bgs));

	block_rsv_release_bytes(fs_info, &fs_info->chunk_block_rsv, NULL,
				trans->chunk_bytes_reserved);
	trans->chunk_bytes_reserved = 0;
}

5763
/* Can only return 0 or -ENOSPC */
5764 5765 5766 5767 5768 5769 5770 5771
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
5772 5773 5774
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
5775
	 */
C
Chris Mason 已提交
5776
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
5777 5778
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 1);
5779
	return btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes, 1);
J
Josef Bacik 已提交
5780 5781
}

5782
void btrfs_orphan_release_metadata(struct inode *inode)
5783
{
5784
	struct btrfs_root *root = BTRFS_I(inode)->root;
C
Chris Mason 已提交
5785
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
5786 5787
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 0);
5788 5789
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
5790

5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802
/*
 * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation
 * root: the root of the parent directory
 * rsv: block reservation
 * items: the number of items that we need do reservation
 * qgroup_reserved: used to return the reserved size in qgroup
 *
 * This function is used to reserve the space for snapshot/subvolume
 * creation and deletion. Those operations are different with the
 * common file/directory operations, they change two fs/file trees
 * and root tree, the number of items that the qgroup reserves is
 * different with the free space reservation. So we can not use
5803
 * the space reservation mechanism in start_transaction().
5804 5805 5806 5807
 */
int btrfs_subvolume_reserve_metadata(struct btrfs_root *root,
				     struct btrfs_block_rsv *rsv,
				     int items,
5808 5809
				     u64 *qgroup_reserved,
				     bool use_global_rsv)
5810
{
5811 5812
	u64 num_bytes;
	int ret;
5813
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
5814 5815 5816

	if (root->fs_info->quota_enabled) {
		/* One for parent inode, two for dir entries */
5817
		num_bytes = 3 * root->nodesize;
5818
		ret = btrfs_qgroup_reserve_meta(root, num_bytes);
5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831
		if (ret)
			return ret;
	} else {
		num_bytes = 0;
	}

	*qgroup_reserved = num_bytes;

	num_bytes = btrfs_calc_trans_metadata_size(root, items);
	rsv->space_info = __find_space_info(root->fs_info,
					    BTRFS_BLOCK_GROUP_METADATA);
	ret = btrfs_block_rsv_add(root, rsv, num_bytes,
				  BTRFS_RESERVE_FLUSH_ALL);
5832 5833

	if (ret == -ENOSPC && use_global_rsv)
5834
		ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes, 1);
5835

5836 5837
	if (ret && *qgroup_reserved)
		btrfs_qgroup_free_meta(root, *qgroup_reserved);
5838 5839 5840 5841 5842 5843 5844 5845 5846

	return ret;
}

void btrfs_subvolume_release_metadata(struct btrfs_root *root,
				      struct btrfs_block_rsv *rsv,
				      u64 qgroup_reserved)
{
	btrfs_block_rsv_release(root, rsv, (u64)-1);
5847 5848
}

5849 5850 5851
/**
 * drop_outstanding_extent - drop an outstanding extent
 * @inode: the inode we're dropping the extent for
5852
 * @num_bytes: the number of bytes we're releasing.
5853 5854 5855 5856 5857 5858
 *
 * This is called when we are freeing up an outstanding extent, either called
 * after an error or after an extent is written.  This will return the number of
 * reserved extents that need to be freed.  This must be called with
 * BTRFS_I(inode)->lock held.
 */
5859
static unsigned drop_outstanding_extent(struct inode *inode, u64 num_bytes)
5860
{
5861
	unsigned drop_inode_space = 0;
5862
	unsigned dropped_extents = 0;
5863
	unsigned num_extents = 0;
5864

5865 5866 5867 5868 5869 5870
	num_extents = (unsigned)div64_u64(num_bytes +
					  BTRFS_MAX_EXTENT_SIZE - 1,
					  BTRFS_MAX_EXTENT_SIZE);
	ASSERT(num_extents);
	ASSERT(BTRFS_I(inode)->outstanding_extents >= num_extents);
	BTRFS_I(inode)->outstanding_extents -= num_extents;
5871

5872
	if (BTRFS_I(inode)->outstanding_extents == 0 &&
5873 5874
	    test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			       &BTRFS_I(inode)->runtime_flags))
5875 5876
		drop_inode_space = 1;

5877
	/*
5878
	 * If we have more or the same amount of outstanding extents than we have
5879 5880 5881 5882
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
5883
		return drop_inode_space;
5884 5885 5886 5887

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
5888
	return dropped_extents + drop_inode_space;
5889 5890
}

5891
/**
5892 5893
 * calc_csum_metadata_size - return the amount of metadata space that must be
 *	reserved/freed for the given bytes.
5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910
 * @inode: the inode we're manipulating
 * @num_bytes: the number of bytes in question
 * @reserve: 1 if we are reserving space, 0 if we are freeing space
 *
 * This adjusts the number of csum_bytes in the inode and then returns the
 * correct amount of metadata that must either be reserved or freed.  We
 * calculate how many checksums we can fit into one leaf and then divide the
 * number of bytes that will need to be checksumed by this value to figure out
 * how many checksums will be required.  If we are adding bytes then the number
 * may go up and we will return the number of additional bytes that must be
 * reserved.  If it is going down we will return the number of bytes that must
 * be freed.
 *
 * This must be called with BTRFS_I(inode)->lock held.
 */
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
				   int reserve)
5911
{
5912
	struct btrfs_root *root = BTRFS_I(inode)->root;
5913
	u64 old_csums, num_csums;
5914 5915 5916 5917 5918

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
	    BTRFS_I(inode)->csum_bytes == 0)
		return 0;

5919
	old_csums = btrfs_csum_bytes_to_leaves(root, BTRFS_I(inode)->csum_bytes);
5920 5921 5922 5923
	if (reserve)
		BTRFS_I(inode)->csum_bytes += num_bytes;
	else
		BTRFS_I(inode)->csum_bytes -= num_bytes;
5924
	num_csums = btrfs_csum_bytes_to_leaves(root, BTRFS_I(inode)->csum_bytes);
5925 5926 5927 5928 5929 5930 5931 5932 5933 5934

	/* No change, no need to reserve more */
	if (old_csums == num_csums)
		return 0;

	if (reserve)
		return btrfs_calc_trans_metadata_size(root,
						      num_csums - old_csums);

	return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
5935
}
Y
Yan Zheng 已提交
5936

5937 5938 5939 5940
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
5941
	u64 to_reserve = 0;
5942
	u64 csum_bytes;
5943
	unsigned nr_extents = 0;
M
Miao Xie 已提交
5944
	enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
5945
	int ret = 0;
5946
	bool delalloc_lock = true;
5947 5948
	u64 to_free = 0;
	unsigned dropped;
5949
	bool release_extra = false;
5950

5951 5952 5953 5954 5955 5956
	/* If we are a free space inode we need to not flush since we will be in
	 * the middle of a transaction commit.  We also don't need the delalloc
	 * mutex since we won't race with anybody.  We need this mostly to make
	 * lockdep shut its filthy mouth.
	 */
	if (btrfs_is_free_space_inode(inode)) {
M
Miao Xie 已提交
5957
		flush = BTRFS_RESERVE_NO_FLUSH;
5958 5959
		delalloc_lock = false;
	}
5960

M
Miao Xie 已提交
5961 5962
	if (flush != BTRFS_RESERVE_NO_FLUSH &&
	    btrfs_transaction_in_commit(root->fs_info))
5963
		schedule_timeout(1);
5964

5965 5966 5967
	if (delalloc_lock)
		mutex_lock(&BTRFS_I(inode)->delalloc_mutex);

5968
	num_bytes = ALIGN(num_bytes, root->sectorsize);
5969

5970
	spin_lock(&BTRFS_I(inode)->lock);
5971 5972 5973 5974
	nr_extents = (unsigned)div64_u64(num_bytes +
					 BTRFS_MAX_EXTENT_SIZE - 1,
					 BTRFS_MAX_EXTENT_SIZE);
	BTRFS_I(inode)->outstanding_extents += nr_extents;
5975

5976
	nr_extents = 0;
5977
	if (BTRFS_I(inode)->outstanding_extents >
5978
	    BTRFS_I(inode)->reserved_extents)
5979
		nr_extents += BTRFS_I(inode)->outstanding_extents -
5980
			BTRFS_I(inode)->reserved_extents;
5981

5982 5983
	/* We always want to reserve a slot for updating the inode. */
	to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents + 1);
5984
	to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
5985
	csum_bytes = BTRFS_I(inode)->csum_bytes;
5986
	spin_unlock(&BTRFS_I(inode)->lock);
5987

5988
	if (root->fs_info->quota_enabled) {
5989 5990
		ret = btrfs_qgroup_reserve_meta(root,
				nr_extents * root->nodesize);
5991 5992 5993
		if (ret)
			goto out_fail;
	}
5994

5995
	ret = btrfs_block_rsv_add(root, block_rsv, to_reserve, flush);
5996
	if (unlikely(ret)) {
5997
		btrfs_qgroup_free_meta(root, nr_extents * root->nodesize);
5998
		goto out_fail;
5999
	}
6000

6001
	spin_lock(&BTRFS_I(inode)->lock);
6002 6003 6004
	if (test_and_set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			     &BTRFS_I(inode)->runtime_flags))
		release_extra = true;
6005 6006
	BTRFS_I(inode)->reserved_extents += nr_extents;
	spin_unlock(&BTRFS_I(inode)->lock);
6007 6008 6009

	if (delalloc_lock)
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
6010

J
Josef Bacik 已提交
6011
	if (to_reserve)
6012
		trace_btrfs_space_reservation(root->fs_info, "delalloc",
J
Josef Bacik 已提交
6013
					      btrfs_ino(inode), to_reserve, 1);
6014 6015 6016 6017
	if (release_extra)
		btrfs_block_rsv_release(root, block_rsv,
					btrfs_calc_trans_metadata_size(root,
								       1));
6018
	return 0;
6019 6020 6021

out_fail:
	spin_lock(&BTRFS_I(inode)->lock);
6022
	dropped = drop_outstanding_extent(inode, num_bytes);
6023 6024 6025 6026 6027
	/*
	 * If the inodes csum_bytes is the same as the original
	 * csum_bytes then we know we haven't raced with any free()ers
	 * so we can just reduce our inodes csum bytes and carry on.
	 */
6028
	if (BTRFS_I(inode)->csum_bytes == csum_bytes) {
6029
		calc_csum_metadata_size(inode, num_bytes, 0);
6030 6031 6032 6033 6034 6035
	} else {
		u64 orig_csum_bytes = BTRFS_I(inode)->csum_bytes;
		u64 bytes;

		/*
		 * This is tricky, but first we need to figure out how much we
6036
		 * freed from any free-ers that occurred during this
6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057
		 * reservation, so we reset ->csum_bytes to the csum_bytes
		 * before we dropped our lock, and then call the free for the
		 * number of bytes that were freed while we were trying our
		 * reservation.
		 */
		bytes = csum_bytes - BTRFS_I(inode)->csum_bytes;
		BTRFS_I(inode)->csum_bytes = csum_bytes;
		to_free = calc_csum_metadata_size(inode, bytes, 0);


		/*
		 * Now we need to see how much we would have freed had we not
		 * been making this reservation and our ->csum_bytes were not
		 * artificially inflated.
		 */
		BTRFS_I(inode)->csum_bytes = csum_bytes - num_bytes;
		bytes = csum_bytes - orig_csum_bytes;
		bytes = calc_csum_metadata_size(inode, bytes, 0);

		/*
		 * Now reset ->csum_bytes to what it should be.  If bytes is
6058
		 * more than to_free then we would have freed more space had we
6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069
		 * not had an artificially high ->csum_bytes, so we need to free
		 * the remainder.  If bytes is the same or less then we don't
		 * need to do anything, the other free-ers did the correct
		 * thing.
		 */
		BTRFS_I(inode)->csum_bytes = orig_csum_bytes - num_bytes;
		if (bytes > to_free)
			to_free = bytes - to_free;
		else
			to_free = 0;
	}
6070
	spin_unlock(&BTRFS_I(inode)->lock);
6071
	if (dropped)
6072 6073 6074 6075 6076 6077 6078 6079 6080 6081
		to_free += btrfs_calc_trans_metadata_size(root, dropped);

	if (to_free) {
		btrfs_block_rsv_release(root, block_rsv, to_free);
		trace_btrfs_space_reservation(root->fs_info, "delalloc",
					      btrfs_ino(inode), to_free, 0);
	}
	if (delalloc_lock)
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
	return ret;
6082 6083
}

6084 6085 6086 6087 6088 6089 6090 6091 6092
/**
 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
 * @inode: the inode to release the reservation for
 * @num_bytes: the number of bytes we're releasing
 *
 * This will release the metadata reservation for an inode.  This can be called
 * once we complete IO for a given set of bytes to release their metadata
 * reservations.
 */
6093 6094 6095
void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
6096 6097
	u64 to_free = 0;
	unsigned dropped;
6098 6099

	num_bytes = ALIGN(num_bytes, root->sectorsize);
6100
	spin_lock(&BTRFS_I(inode)->lock);
6101
	dropped = drop_outstanding_extent(inode, num_bytes);
6102

6103 6104
	if (num_bytes)
		to_free = calc_csum_metadata_size(inode, num_bytes, 0);
6105
	spin_unlock(&BTRFS_I(inode)->lock);
6106 6107
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
6108

6109 6110 6111
	if (btrfs_test_is_dummy_root(root))
		return;

J
Josef Bacik 已提交
6112 6113
	trace_btrfs_space_reservation(root->fs_info, "delalloc",
				      btrfs_ino(inode), to_free, 0);
6114

6115 6116 6117 6118
	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

6119
/**
6120
 * btrfs_delalloc_reserve_space - reserve data and metadata space for
6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143
 * delalloc
 * @inode: inode we're writing to
 * @start: start range we are writing to
 * @len: how long the range we are writing to
 *
 * TODO: This function will finally replace old btrfs_delalloc_reserve_space()
 *
 * This will do the following things
 *
 * o reserve space in data space info for num bytes
 *   and reserve precious corresponding qgroup space
 *   (Done in check_data_free_space)
 *
 * o reserve space for metadata space, based on the number of outstanding
 *   extents and how much csums will be needed
 *   also reserve metadata space in a per root over-reserve method.
 * o add to the inodes->delalloc_bytes
 * o add it to the fs_info's delalloc inodes list.
 *   (Above 3 all done in delalloc_reserve_metadata)
 *
 * Return 0 for success
 * Return <0 for error(-ENOSPC or -EQUOT)
 */
6144
int btrfs_delalloc_reserve_space(struct inode *inode, u64 start, u64 len)
6145 6146 6147
{
	int ret;

6148
	ret = btrfs_check_data_free_space(inode, start, len);
6149 6150 6151 6152
	if (ret < 0)
		return ret;
	ret = btrfs_delalloc_reserve_metadata(inode, len);
	if (ret < 0)
6153
		btrfs_free_reserved_data_space(inode, start, len);
6154 6155 6156
	return ret;
}

6157
/**
6158
 * btrfs_delalloc_release_space - release data and metadata space for delalloc
6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171
 * @inode: inode we're releasing space for
 * @start: start position of the space already reserved
 * @len: the len of the space already reserved
 *
 * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
 * called in the case that we don't need the metadata AND data reservations
 * anymore.  So if there is an error or we insert an inline extent.
 *
 * This function will release the metadata space that was not used and will
 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
 * list if there are no delalloc bytes left.
 * Also it will handle the qgroup reserved space.
 */
6172
void btrfs_delalloc_release_space(struct inode *inode, u64 start, u64 len)
6173 6174
{
	btrfs_delalloc_release_metadata(inode, len);
6175
	btrfs_free_reserved_data_space(inode, start, len);
6176 6177
}

6178 6179 6180
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root, u64 bytenr,
			      u64 num_bytes, int alloc)
C
Chris Mason 已提交
6181
{
6182
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
6183
	struct btrfs_fs_info *info = root->fs_info;
6184
	u64 total = num_bytes;
C
Chris Mason 已提交
6185
	u64 old_val;
6186
	u64 byte_in_group;
6187
	int factor;
C
Chris Mason 已提交
6188

6189
	/* block accounting for super block */
6190
	spin_lock(&info->delalloc_root_lock);
6191
	old_val = btrfs_super_bytes_used(info->super_copy);
6192 6193 6194 6195
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
6196
	btrfs_set_super_bytes_used(info->super_copy, old_val);
6197
	spin_unlock(&info->delalloc_root_lock);
6198

C
Chris Mason 已提交
6199
	while (total) {
6200
		cache = btrfs_lookup_block_group(info, bytenr);
6201
		if (!cache)
6202
			return -ENOENT;
6203 6204 6205 6206 6207 6208
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
6209 6210 6211 6212 6213 6214 6215
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
6216
			cache_block_group(cache, 1);
6217

6218 6219
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
6220

6221
		spin_lock(&cache->space_info->lock);
6222
		spin_lock(&cache->lock);
6223

6224
		if (btrfs_test_opt(root, SPACE_CACHE) &&
6225 6226 6227
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

C
Chris Mason 已提交
6228
		old_val = btrfs_block_group_used(&cache->item);
6229
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
6230
		if (alloc) {
6231
			old_val += num_bytes;
6232 6233 6234
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
6235 6236
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
6237
			spin_unlock(&cache->lock);
6238
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
6239
		} else {
6240
			old_val -= num_bytes;
6241 6242 6243 6244 6245 6246 6247
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
			cache->space_info->bytes_used -= num_bytes;
			cache->space_info->disk_used -= num_bytes * factor;
			spin_unlock(&cache->lock);
			spin_unlock(&cache->space_info->lock);
6248

J
Josef Bacik 已提交
6249 6250 6251
			trace_btrfs_space_reservation(root->fs_info, "pinned",
						      cache->space_info->flags,
						      num_bytes, 1);
6252 6253 6254
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
C
Chris Mason 已提交
6255
		}
6256 6257 6258 6259 6260 6261 6262 6263 6264 6265

		spin_lock(&trans->transaction->dirty_bgs_lock);
		if (list_empty(&cache->dirty_list)) {
			list_add_tail(&cache->dirty_list,
				      &trans->transaction->dirty_bgs);
				trans->transaction->num_dirty_bgs++;
			btrfs_get_block_group(cache);
		}
		spin_unlock(&trans->transaction->dirty_bgs_lock);

6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281
		/*
		 * No longer have used bytes in this block group, queue it for
		 * deletion. We do this after adding the block group to the
		 * dirty list to avoid races between cleaner kthread and space
		 * cache writeout.
		 */
		if (!alloc && old_val == 0) {
			spin_lock(&info->unused_bgs_lock);
			if (list_empty(&cache->bg_list)) {
				btrfs_get_block_group(cache);
				list_add_tail(&cache->bg_list,
					      &info->unused_bgs);
			}
			spin_unlock(&info->unused_bgs_lock);
		}

6282
		btrfs_put_block_group(cache);
6283 6284
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
6285 6286 6287
	}
	return 0;
}
6288

6289 6290
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
6291
	struct btrfs_block_group_cache *cache;
6292
	u64 bytenr;
J
Josef Bacik 已提交
6293

6294 6295 6296 6297 6298 6299 6300
	spin_lock(&root->fs_info->block_group_cache_lock);
	bytenr = root->fs_info->first_logical_byte;
	spin_unlock(&root->fs_info->block_group_cache_lock);

	if (bytenr < (u64)-1)
		return bytenr;

J
Josef Bacik 已提交
6301 6302
	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
6303
		return 0;
J
Josef Bacik 已提交
6304

6305
	bytenr = cache->key.objectid;
6306
	btrfs_put_block_group(cache);
6307 6308

	return bytenr;
6309 6310
}

6311 6312 6313
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
6314
{
6315 6316 6317 6318 6319 6320 6321 6322 6323 6324
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
6325

J
Josef Bacik 已提交
6326 6327
	trace_btrfs_space_reservation(root->fs_info, "pinned",
				      cache->space_info->flags, num_bytes, 1);
6328 6329
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
6330
	if (reserved)
J
Josef Bacik 已提交
6331
		trace_btrfs_reserved_extent_free(root, bytenr, num_bytes);
6332 6333
	return 0;
}
J
Josef Bacik 已提交
6334

6335 6336 6337 6338 6339 6340 6341
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
6342

6343
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
6344
	BUG_ON(!cache); /* Logic error */
6345 6346 6347 6348

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
6349 6350 6351
	return 0;
}

6352
/*
6353 6354
 * this function must be called within transaction
 */
6355
int btrfs_pin_extent_for_log_replay(struct btrfs_root *root,
6356 6357 6358
				    u64 bytenr, u64 num_bytes)
{
	struct btrfs_block_group_cache *cache;
6359
	int ret;
6360 6361

	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
6362 6363
	if (!cache)
		return -EINVAL;
6364 6365 6366 6367 6368 6369 6370

	/*
	 * pull in the free space cache (if any) so that our pin
	 * removes the free space from the cache.  We have load_only set
	 * to one because the slow code to read in the free extents does check
	 * the pinned extents.
	 */
6371
	cache_block_group(cache, 1);
6372 6373 6374 6375

	pin_down_extent(root, cache, bytenr, num_bytes, 0);

	/* remove us from the free space cache (if we're there at all) */
6376
	ret = btrfs_remove_free_space(cache, bytenr, num_bytes);
6377
	btrfs_put_block_group(cache);
6378
	return ret;
6379 6380
}

6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454
static int __exclude_logged_extent(struct btrfs_root *root, u64 start, u64 num_bytes)
{
	int ret;
	struct btrfs_block_group_cache *block_group;
	struct btrfs_caching_control *caching_ctl;

	block_group = btrfs_lookup_block_group(root->fs_info, start);
	if (!block_group)
		return -EINVAL;

	cache_block_group(block_group, 0);
	caching_ctl = get_caching_control(block_group);

	if (!caching_ctl) {
		/* Logic error */
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			if (ret)
				goto out_lock;

			num_bytes = (start + num_bytes) -
				caching_ctl->progress;
			start = caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
		}
out_lock:
		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}
	btrfs_put_block_group(block_group);
	return ret;
}

int btrfs_exclude_logged_extents(struct btrfs_root *log,
				 struct extent_buffer *eb)
{
	struct btrfs_file_extent_item *item;
	struct btrfs_key key;
	int found_type;
	int i;

	if (!btrfs_fs_incompat(log->fs_info, MIXED_GROUPS))
		return 0;

	for (i = 0; i < btrfs_header_nritems(eb); i++) {
		btrfs_item_key_to_cpu(eb, &key, i);
		if (key.type != BTRFS_EXTENT_DATA_KEY)
			continue;
		item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
		found_type = btrfs_file_extent_type(eb, item);
		if (found_type == BTRFS_FILE_EXTENT_INLINE)
			continue;
		if (btrfs_file_extent_disk_bytenr(eb, item) == 0)
			continue;
		key.objectid = btrfs_file_extent_disk_bytenr(eb, item);
		key.offset = btrfs_file_extent_disk_num_bytes(eb, item);
		__exclude_logged_extent(log, key.objectid, key.offset);
	}

	return 0;
}

6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505
static void
btrfs_inc_block_group_reservations(struct btrfs_block_group_cache *bg)
{
	atomic_inc(&bg->reservations);
}

void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info,
					const u64 start)
{
	struct btrfs_block_group_cache *bg;

	bg = btrfs_lookup_block_group(fs_info, start);
	ASSERT(bg);
	if (atomic_dec_and_test(&bg->reservations))
		wake_up_atomic_t(&bg->reservations);
	btrfs_put_block_group(bg);
}

static int btrfs_wait_bg_reservations_atomic_t(atomic_t *a)
{
	schedule();
	return 0;
}

void btrfs_wait_block_group_reservations(struct btrfs_block_group_cache *bg)
{
	struct btrfs_space_info *space_info = bg->space_info;

	ASSERT(bg->ro);

	if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA))
		return;

	/*
	 * Our block group is read only but before we set it to read only,
	 * some task might have had allocated an extent from it already, but it
	 * has not yet created a respective ordered extent (and added it to a
	 * root's list of ordered extents).
	 * Therefore wait for any task currently allocating extents, since the
	 * block group's reservations counter is incremented while a read lock
	 * on the groups' semaphore is held and decremented after releasing
	 * the read access on that semaphore and creating the ordered extent.
	 */
	down_write(&space_info->groups_sem);
	up_write(&space_info->groups_sem);

	wait_on_atomic_t(&bg->reservations,
			 btrfs_wait_bg_reservations_atomic_t,
			 TASK_UNINTERRUPTIBLE);
}

6506 6507 6508 6509 6510
/**
 * btrfs_update_reserved_bytes - update the block_group and space info counters
 * @cache:	The cache we are manipulating
 * @num_bytes:	The number of bytes in question
 * @reserve:	One of the reservation enums
6511
 * @delalloc:   The blocks are allocated for the delalloc write
6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527
 *
 * This is called by the allocator when it reserves space, or by somebody who is
 * freeing space that was never actually used on disk.  For example if you
 * reserve some space for a new leaf in transaction A and before transaction A
 * commits you free that leaf, you call this with reserve set to 0 in order to
 * clear the reservation.
 *
 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
 * ENOSPC accounting.  For data we handle the reservation through clearing the
 * delalloc bits in the io_tree.  We have to do this since we could end up
 * allocating less disk space for the amount of data we have reserved in the
 * case of compression.
 *
 * If this is a reservation and the block group has become read only we cannot
 * make the reservation and return -EAGAIN, otherwise this function always
 * succeeds.
6528
 */
6529
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
6530
				       u64 num_bytes, int reserve, int delalloc)
6531
{
6532
	struct btrfs_space_info *space_info = cache->space_info;
6533
	int ret = 0;
6534

6535 6536 6537
	spin_lock(&space_info->lock);
	spin_lock(&cache->lock);
	if (reserve != RESERVE_FREE) {
6538 6539 6540
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
6541 6542 6543
			cache->reserved += num_bytes;
			space_info->bytes_reserved += num_bytes;
			if (reserve == RESERVE_ALLOC) {
J
Josef Bacik 已提交
6544
				trace_btrfs_space_reservation(cache->fs_info,
6545 6546
						"space_info", space_info->flags,
						num_bytes, 0);
6547 6548
				space_info->bytes_may_use -= num_bytes;
			}
6549 6550 6551

			if (delalloc)
				cache->delalloc_bytes += num_bytes;
6552
		}
6553 6554 6555 6556 6557
	} else {
		if (cache->ro)
			space_info->bytes_readonly += num_bytes;
		cache->reserved -= num_bytes;
		space_info->bytes_reserved -= num_bytes;
6558 6559 6560

		if (delalloc)
			cache->delalloc_bytes -= num_bytes;
6561
	}
6562 6563
	spin_unlock(&cache->lock);
	spin_unlock(&space_info->lock);
6564
	return ret;
6565
}
C
Chris Mason 已提交
6566

6567
void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
6568
				struct btrfs_root *root)
6569 6570
{
	struct btrfs_fs_info *fs_info = root->fs_info;
6571 6572 6573
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
6574

6575
	down_write(&fs_info->commit_root_sem);
6576

6577 6578 6579 6580 6581 6582 6583
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
6584
		} else {
6585
			cache->last_byte_to_unpin = caching_ctl->progress;
6586 6587
		}
	}
6588 6589 6590 6591 6592 6593

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

6594
	up_write(&fs_info->commit_root_sem);
6595 6596

	update_global_block_rsv(fs_info);
6597 6598
}

6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614
/*
 * Returns the free cluster for the given space info and sets empty_cluster to
 * what it should be based on the mount options.
 */
static struct btrfs_free_cluster *
fetch_cluster_info(struct btrfs_root *root, struct btrfs_space_info *space_info,
		   u64 *empty_cluster)
{
	struct btrfs_free_cluster *ret = NULL;
	bool ssd = btrfs_test_opt(root, SSD);

	*empty_cluster = 0;
	if (btrfs_mixed_space_info(space_info))
		return ret;

	if (ssd)
6615
		*empty_cluster = SZ_2M;
6616 6617 6618
	if (space_info->flags & BTRFS_BLOCK_GROUP_METADATA) {
		ret = &root->fs_info->meta_alloc_cluster;
		if (!ssd)
6619
			*empty_cluster = SZ_64K;
6620 6621 6622 6623 6624 6625 6626
	} else if ((space_info->flags & BTRFS_BLOCK_GROUP_DATA) && ssd) {
		ret = &root->fs_info->data_alloc_cluster;
	}

	return ret;
}

6627 6628
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end,
			      const bool return_free_space)
C
Chris Mason 已提交
6629
{
6630 6631
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
6632 6633
	struct btrfs_space_info *space_info;
	struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
6634
	struct btrfs_free_cluster *cluster = NULL;
6635
	u64 len;
6636 6637
	u64 total_unpinned = 0;
	u64 empty_cluster = 0;
6638
	bool readonly;
C
Chris Mason 已提交
6639

6640
	while (start <= end) {
6641
		readonly = false;
6642 6643 6644 6645
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
6646
			total_unpinned = 0;
6647
			cache = btrfs_lookup_block_group(fs_info, start);
6648
			BUG_ON(!cache); /* Logic error */
6649 6650 6651 6652 6653

			cluster = fetch_cluster_info(root,
						     cache->space_info,
						     &empty_cluster);
			empty_cluster <<= 1;
6654 6655 6656 6657 6658 6659 6660
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
6661 6662
			if (return_free_space)
				btrfs_add_free_space(cache, start, len);
6663 6664
		}

6665
		start += len;
6666
		total_unpinned += len;
6667
		space_info = cache->space_info;
6668

6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681
		/*
		 * If this space cluster has been marked as fragmented and we've
		 * unpinned enough in this block group to potentially allow a
		 * cluster to be created inside of it go ahead and clear the
		 * fragmented check.
		 */
		if (cluster && cluster->fragmented &&
		    total_unpinned > empty_cluster) {
			spin_lock(&cluster->lock);
			cluster->fragmented = 0;
			spin_unlock(&cluster->lock);
		}

6682
		spin_lock(&space_info->lock);
6683 6684
		spin_lock(&cache->lock);
		cache->pinned -= len;
6685
		space_info->bytes_pinned -= len;
J
Josef Bacik 已提交
6686 6687 6688

		trace_btrfs_space_reservation(fs_info, "pinned",
					      space_info->flags, len, 0);
6689
		space_info->max_extent_size = 0;
6690
		percpu_counter_add(&space_info->total_bytes_pinned, -len);
6691 6692 6693 6694
		if (cache->ro) {
			space_info->bytes_readonly += len;
			readonly = true;
		}
6695
		spin_unlock(&cache->lock);
6696 6697 6698 6699
		if (!readonly && return_free_space &&
		    global_rsv->space_info == space_info) {
			u64 to_add = len;
			WARN_ON(!return_free_space);
6700 6701
			spin_lock(&global_rsv->lock);
			if (!global_rsv->full) {
6702 6703 6704 6705
				to_add = min(len, global_rsv->size -
					     global_rsv->reserved);
				global_rsv->reserved += to_add;
				space_info->bytes_may_use += to_add;
6706 6707
				if (global_rsv->reserved >= global_rsv->size)
					global_rsv->full = 1;
6708 6709 6710 6711 6712
				trace_btrfs_space_reservation(fs_info,
							      "space_info",
							      space_info->flags,
							      to_add, 1);
				len -= to_add;
6713 6714
			}
			spin_unlock(&global_rsv->lock);
6715 6716 6717 6718
			/* Add to any tickets we may have */
			if (len)
				space_info_add_new_bytes(fs_info, space_info,
							 len);
6719 6720
		}
		spin_unlock(&space_info->lock);
C
Chris Mason 已提交
6721
	}
6722 6723 6724

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
6725 6726 6727 6728
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
6729
			       struct btrfs_root *root)
6730
{
6731
	struct btrfs_fs_info *fs_info = root->fs_info;
6732 6733
	struct btrfs_block_group_cache *block_group, *tmp;
	struct list_head *deleted_bgs;
6734
	struct extent_io_tree *unpin;
6735 6736
	u64 start;
	u64 end;
6737 6738
	int ret;

6739 6740 6741 6742 6743
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

6744
	while (!trans->aborted) {
6745
		mutex_lock(&fs_info->unused_bg_unpin_mutex);
6746
		ret = find_first_extent_bit(unpin, 0, &start, &end,
6747
					    EXTENT_DIRTY, NULL);
6748 6749
		if (ret) {
			mutex_unlock(&fs_info->unused_bg_unpin_mutex);
6750
			break;
6751
		}
6752

6753 6754 6755
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
6756

6757
		clear_extent_dirty(unpin, start, end);
6758
		unpin_extent_range(root, start, end, true);
6759
		mutex_unlock(&fs_info->unused_bg_unpin_mutex);
6760
		cond_resched();
6761
	}
J
Josef Bacik 已提交
6762

6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790
	/*
	 * Transaction is finished.  We don't need the lock anymore.  We
	 * do need to clean up the block groups in case of a transaction
	 * abort.
	 */
	deleted_bgs = &trans->transaction->deleted_bgs;
	list_for_each_entry_safe(block_group, tmp, deleted_bgs, bg_list) {
		u64 trimmed = 0;

		ret = -EROFS;
		if (!trans->aborted)
			ret = btrfs_discard_extent(root,
						   block_group->key.objectid,
						   block_group->key.offset,
						   &trimmed);

		list_del_init(&block_group->bg_list);
		btrfs_put_block_group_trimming(block_group);
		btrfs_put_block_group(block_group);

		if (ret) {
			const char *errstr = btrfs_decode_error(ret);
			btrfs_warn(fs_info,
				   "Discard failed while removing blockgroup: errno=%d %s\n",
				   ret, errstr);
		}
	}

C
Chris Mason 已提交
6791 6792 6793
	return 0;
}

6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814
static void add_pinned_bytes(struct btrfs_fs_info *fs_info, u64 num_bytes,
			     u64 owner, u64 root_objectid)
{
	struct btrfs_space_info *space_info;
	u64 flags;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (root_objectid == BTRFS_CHUNK_TREE_OBJECTID)
			flags = BTRFS_BLOCK_GROUP_SYSTEM;
		else
			flags = BTRFS_BLOCK_GROUP_METADATA;
	} else {
		flags = BTRFS_BLOCK_GROUP_DATA;
	}

	space_info = __find_space_info(fs_info, flags);
	BUG_ON(!space_info); /* Logic bug */
	percpu_counter_add(&space_info->total_bytes_pinned, num_bytes);
}


6815 6816
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
6817
				struct btrfs_delayed_ref_node *node, u64 parent,
6818 6819
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
6820
				struct btrfs_delayed_extent_op *extent_op)
6821
{
C
Chris Mason 已提交
6822
	struct btrfs_key key;
6823
	struct btrfs_path *path;
6824 6825
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
6826
	struct extent_buffer *leaf;
6827 6828
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
6829
	int ret;
6830
	int is_data;
6831 6832 6833
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
6834 6835
	u32 item_size;
	u64 refs;
6836 6837
	u64 bytenr = node->bytenr;
	u64 num_bytes = node->num_bytes;
J
Josef Bacik 已提交
6838
	int last_ref = 0;
6839 6840
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
C
Chris Mason 已提交
6841

6842
	path = btrfs_alloc_path();
6843 6844
	if (!path)
		return -ENOMEM;
6845

6846
	path->reada = READA_FORWARD;
6847
	path->leave_spinning = 1;
6848 6849 6850 6851

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

6852 6853 6854
	if (is_data)
		skinny_metadata = 0;

6855 6856 6857 6858
	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
6859
	if (ret == 0) {
6860
		extent_slot = path->slots[0];
6861 6862
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
6863
					      extent_slot);
6864
			if (key.objectid != bytenr)
6865
				break;
6866 6867
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
6868 6869 6870
				found_extent = 1;
				break;
			}
6871 6872 6873 6874 6875
			if (key.type == BTRFS_METADATA_ITEM_KEY &&
			    key.offset == owner_objectid) {
				found_extent = 1;
				break;
			}
6876 6877
			if (path->slots[0] - extent_slot > 5)
				break;
6878
			extent_slot--;
6879
		}
6880 6881 6882 6883 6884
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
6885
		if (!found_extent) {
6886
			BUG_ON(iref);
6887
			ret = remove_extent_backref(trans, extent_root, path,
6888
						    NULL, refs_to_drop,
J
Josef Bacik 已提交
6889
						    is_data, &last_ref);
6890 6891 6892 6893
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
6894
			btrfs_release_path(path);
6895
			path->leave_spinning = 1;
6896 6897 6898 6899 6900

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

6901 6902 6903 6904 6905
			if (!is_data && skinny_metadata) {
				key.type = BTRFS_METADATA_ITEM_KEY;
				key.offset = owner_objectid;
			}

Z
Zheng Yan 已提交
6906 6907
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923
			if (ret > 0 && skinny_metadata && path->slots[0]) {
				/*
				 * Couldn't find our skinny metadata item,
				 * see if we have ye olde extent item.
				 */
				path->slots[0]--;
				btrfs_item_key_to_cpu(path->nodes[0], &key,
						      path->slots[0]);
				if (key.objectid == bytenr &&
				    key.type == BTRFS_EXTENT_ITEM_KEY &&
				    key.offset == num_bytes)
					ret = 0;
			}

			if (ret > 0 && skinny_metadata) {
				skinny_metadata = false;
6924
				key.objectid = bytenr;
6925 6926 6927 6928 6929 6930 6931
				key.type = BTRFS_EXTENT_ITEM_KEY;
				key.offset = num_bytes;
				btrfs_release_path(path);
				ret = btrfs_search_slot(trans, extent_root,
							&key, path, -1, 1);
			}

6932
			if (ret) {
6933
				btrfs_err(info, "umm, got %d back from search, was looking for %llu",
6934
					ret, bytenr);
6935 6936 6937
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
6938
			}
6939 6940 6941 6942
			if (ret < 0) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
Z
Zheng Yan 已提交
6943 6944
			extent_slot = path->slots[0];
		}
6945
	} else if (WARN_ON(ret == -ENOENT)) {
6946
		btrfs_print_leaf(extent_root, path->nodes[0]);
6947 6948
		btrfs_err(info,
			"unable to find ref byte nr %llu parent %llu root %llu  owner %llu offset %llu",
6949 6950
			bytenr, parent, root_objectid, owner_objectid,
			owner_offset);
6951 6952
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
6953
	} else {
6954 6955
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
6956
	}
6957 6958

	leaf = path->nodes[0];
6959 6960 6961 6962 6963 6964
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
6965 6966 6967 6968
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
6969

6970
		btrfs_release_path(path);
6971 6972 6973 6974 6975 6976 6977 6978 6979
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
6980
			btrfs_err(info, "umm, got %d back from search, was looking for %llu",
6981
				ret, bytenr);
6982 6983
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
6984 6985 6986 6987 6988
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}

6989 6990 6991 6992 6993 6994
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
6995
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
6996
			    struct btrfs_extent_item);
6997 6998
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID &&
	    key.type == BTRFS_EXTENT_ITEM_KEY) {
6999 7000 7001 7002 7003
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
7004

7005
	refs = btrfs_extent_refs(leaf, ei);
7006 7007
	if (refs < refs_to_drop) {
		btrfs_err(info, "trying to drop %d refs but we only have %Lu "
7008
			  "for bytenr %Lu", refs_to_drop, refs, bytenr);
7009 7010 7011 7012
		ret = -EINVAL;
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
	}
7013
	refs -= refs_to_drop;
7014

7015 7016 7017 7018 7019 7020
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
7021
		 */
7022 7023 7024 7025 7026 7027 7028 7029 7030
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
J
Josef Bacik 已提交
7031
						    is_data, &last_ref);
7032 7033 7034 7035
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
7036
		}
7037 7038
		add_pinned_bytes(root->fs_info, -num_bytes, owner_objectid,
				 root_objectid);
7039 7040 7041
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
7042
			       extent_data_ref_count(path, iref));
7043 7044 7045 7046 7047 7048 7049
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
7050
		}
7051

J
Josef Bacik 已提交
7052
		last_ref = 1;
7053 7054
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
7055 7056 7057 7058
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
7059
		btrfs_release_path(path);
7060

7061
		if (is_data) {
7062
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
7063 7064 7065 7066
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
7067 7068
		}

7069 7070 7071 7072 7073 7074 7075
		ret = add_to_free_space_tree(trans, root->fs_info, bytenr,
					     num_bytes);
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}

7076
		ret = update_block_group(trans, root, bytenr, num_bytes, 0);
7077 7078 7079 7080
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
7081
	}
J
Josef Bacik 已提交
7082 7083
	btrfs_release_path(path);

7084
out:
7085
	btrfs_free_path(path);
7086 7087 7088
	return ret;
}

7089
/*
7090
 * when we free an block, it is possible (and likely) that we free the last
7091 7092 7093 7094 7095 7096 7097 7098 7099
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
7100
	int ret = 0;
7101 7102 7103 7104 7105

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
7106
		goto out_delayed_unlock;
7107

7108
	spin_lock(&head->lock);
7109
	if (!list_empty(&head->ref_list))
7110 7111
		goto out;

7112 7113 7114
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
7115
		btrfs_free_delayed_extent_op(head->extent_op);
7116 7117 7118
		head->extent_op = NULL;
	}

7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
L
Liu Bo 已提交
7131
	rb_erase(&head->href_node, &delayed_refs->href_root);
7132

7133
	atomic_dec(&delayed_refs->num_entries);
7134 7135 7136 7137 7138

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
7139
	delayed_refs->num_heads--;
7140
	if (head->processing == 0)
7141
		delayed_refs->num_heads_ready--;
7142 7143
	head->processing = 0;
	spin_unlock(&head->lock);
7144 7145
	spin_unlock(&delayed_refs->lock);

7146 7147 7148 7149 7150
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
7151
	btrfs_put_delayed_ref(&head->node);
7152
	return ret;
7153
out:
7154
	spin_unlock(&head->lock);
7155 7156

out_delayed_unlock:
7157 7158 7159 7160
	spin_unlock(&delayed_refs->lock);
	return 0;
}

7161 7162 7163
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
7164
			   u64 parent, int last_ref)
7165
{
7166
	int pin = 1;
7167 7168 7169
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
A
Arne Jansen 已提交
7170 7171 7172 7173
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					buf->start, buf->len,
					parent, root->root_key.objectid,
					btrfs_header_level(buf),
7174
					BTRFS_DROP_DELAYED_REF, NULL);
7175
		BUG_ON(ret); /* -ENOMEM */
7176 7177 7178 7179 7180 7181
	}

	if (!last_ref)
		return;

	if (btrfs_header_generation(buf) == trans->transid) {
7182 7183
		struct btrfs_block_group_cache *cache;

7184 7185 7186
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
7187
				goto out;
7188 7189
		}

7190 7191
		cache = btrfs_lookup_block_group(root->fs_info, buf->start);

7192 7193
		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
7194
			btrfs_put_block_group(cache);
7195
			goto out;
7196 7197 7198 7199 7200
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
7201
		btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE, 0);
7202
		btrfs_put_block_group(cache);
J
Josef Bacik 已提交
7203
		trace_btrfs_reserved_extent_free(root, buf->start, buf->len);
7204
		pin = 0;
7205 7206
	}
out:
7207 7208 7209 7210 7211
	if (pin)
		add_pinned_bytes(root->fs_info, buf->len,
				 btrfs_header_level(buf),
				 root->root_key.objectid);

7212 7213 7214 7215 7216
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
7217 7218
}

7219
/* Can return -ENOMEM */
A
Arne Jansen 已提交
7220 7221
int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
7222
		      u64 owner, u64 offset)
7223 7224
{
	int ret;
A
Arne Jansen 已提交
7225
	struct btrfs_fs_info *fs_info = root->fs_info;
7226

7227
	if (btrfs_test_is_dummy_root(root))
7228
		return 0;
7229

7230 7231
	add_pinned_bytes(root->fs_info, num_bytes, owner, root_objectid);

7232 7233 7234 7235
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
7236 7237
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
7238
		/* unlocks the pinned mutex */
7239
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
7240
		ret = 0;
7241
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
7242 7243
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
7244
					parent, root_objectid, (int)owner,
7245
					BTRFS_DROP_DELAYED_REF, NULL);
7246
	} else {
A
Arne Jansen 已提交
7247 7248 7249
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
						num_bytes,
						parent, root_objectid, owner,
7250 7251
						offset, 0,
						BTRFS_DROP_DELAYED_REF, NULL);
7252
	}
7253 7254 7255
	return ret;
}

J
Josef Bacik 已提交
7256 7257 7258 7259 7260 7261 7262 7263 7264 7265
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
7266 7267 7268
 *
 * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using
 * any of the information in this block group.
J
Josef Bacik 已提交
7269
 */
7270
static noinline void
J
Josef Bacik 已提交
7271 7272 7273
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
7274
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
7275

7276 7277
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
7278
		return;
J
Josef Bacik 已提交
7279

7280
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
7281
		   (cache->free_space_ctl->free_space >= num_bytes));
7282 7283 7284 7285 7286 7287 7288 7289

	put_caching_control(caching_ctl);
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;
7290
	int ret = 0;
7291 7292 7293

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
7294
		return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0;
7295 7296

	wait_event(caching_ctl->wait, block_group_cache_done(cache));
7297 7298
	if (cache->cached == BTRFS_CACHE_ERROR)
		ret = -EIO;
7299
	put_caching_control(caching_ctl);
7300
	return ret;
J
Josef Bacik 已提交
7301 7302
}

7303
int __get_raid_index(u64 flags)
7304
{
7305
	if (flags & BTRFS_BLOCK_GROUP_RAID10)
7306
		return BTRFS_RAID_RAID10;
7307
	else if (flags & BTRFS_BLOCK_GROUP_RAID1)
7308
		return BTRFS_RAID_RAID1;
7309
	else if (flags & BTRFS_BLOCK_GROUP_DUP)
7310
		return BTRFS_RAID_DUP;
7311
	else if (flags & BTRFS_BLOCK_GROUP_RAID0)
7312
		return BTRFS_RAID_RAID0;
D
David Woodhouse 已提交
7313
	else if (flags & BTRFS_BLOCK_GROUP_RAID5)
7314
		return BTRFS_RAID_RAID5;
D
David Woodhouse 已提交
7315
	else if (flags & BTRFS_BLOCK_GROUP_RAID6)
7316
		return BTRFS_RAID_RAID6;
7317

7318
	return BTRFS_RAID_SINGLE; /* BTRFS_BLOCK_GROUP_SINGLE */
7319 7320
}

7321
int get_block_group_index(struct btrfs_block_group_cache *cache)
7322
{
7323
	return __get_raid_index(cache->flags);
7324 7325
}

7326 7327 7328 7329 7330 7331 7332 7333 7334 7335
static const char *btrfs_raid_type_names[BTRFS_NR_RAID_TYPES] = {
	[BTRFS_RAID_RAID10]	= "raid10",
	[BTRFS_RAID_RAID1]	= "raid1",
	[BTRFS_RAID_DUP]	= "dup",
	[BTRFS_RAID_RAID0]	= "raid0",
	[BTRFS_RAID_SINGLE]	= "single",
	[BTRFS_RAID_RAID5]	= "raid5",
	[BTRFS_RAID_RAID6]	= "raid6",
};

7336
static const char *get_raid_name(enum btrfs_raid_types type)
7337 7338 7339 7340 7341 7342 7343
{
	if (type >= BTRFS_NR_RAID_TYPES)
		return NULL;

	return btrfs_raid_type_names[type];
}

J
Josef Bacik 已提交
7344
enum btrfs_loop_type {
7345 7346 7347 7348
	LOOP_CACHING_NOWAIT = 0,
	LOOP_CACHING_WAIT = 1,
	LOOP_ALLOC_CHUNK = 2,
	LOOP_NO_EMPTY_SIZE = 3,
J
Josef Bacik 已提交
7349 7350
};

7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372
static inline void
btrfs_lock_block_group(struct btrfs_block_group_cache *cache,
		       int delalloc)
{
	if (delalloc)
		down_read(&cache->data_rwsem);
}

static inline void
btrfs_grab_block_group(struct btrfs_block_group_cache *cache,
		       int delalloc)
{
	btrfs_get_block_group(cache);
	if (delalloc)
		down_read(&cache->data_rwsem);
}

static struct btrfs_block_group_cache *
btrfs_lock_cluster(struct btrfs_block_group_cache *block_group,
		   struct btrfs_free_cluster *cluster,
		   int delalloc)
{
S
Sudip Mukherjee 已提交
7373
	struct btrfs_block_group_cache *used_bg = NULL;
7374

7375
	spin_lock(&cluster->refill_lock);
7376 7377 7378 7379 7380 7381
	while (1) {
		used_bg = cluster->block_group;
		if (!used_bg)
			return NULL;

		if (used_bg == block_group)
7382 7383
			return used_bg;

7384
		btrfs_get_block_group(used_bg);
7385

7386 7387
		if (!delalloc)
			return used_bg;
7388

7389 7390
		if (down_read_trylock(&used_bg->data_rwsem))
			return used_bg;
7391

7392
		spin_unlock(&cluster->refill_lock);
7393

7394
		down_read(&used_bg->data_rwsem);
7395

7396 7397 7398
		spin_lock(&cluster->refill_lock);
		if (used_bg == cluster->block_group)
			return used_bg;
7399

7400 7401 7402
		up_read(&used_bg->data_rwsem);
		btrfs_put_block_group(used_bg);
	}
7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413
}

static inline void
btrfs_release_block_group(struct btrfs_block_group_cache *cache,
			 int delalloc)
{
	if (delalloc)
		up_read(&cache->data_rwsem);
	btrfs_put_block_group(cache);
}

7414 7415 7416
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
7417
 * ins->objectid == start position
7418
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
7419
 * ins->offset == the size of the hole.
7420
 * Any available blocks before search_start are skipped.
7421 7422 7423
 *
 * If there is no suitable free space, we will record the max size of
 * the free space extent currently.
7424
 */
7425
static noinline int find_free_extent(struct btrfs_root *orig_root,
7426 7427
				     u64 num_bytes, u64 empty_size,
				     u64 hint_byte, struct btrfs_key *ins,
7428
				     u64 flags, int delalloc)
7429
{
7430
	int ret = 0;
C
Chris Mason 已提交
7431
	struct btrfs_root *root = orig_root->fs_info->extent_root;
7432
	struct btrfs_free_cluster *last_ptr = NULL;
7433
	struct btrfs_block_group_cache *block_group = NULL;
7434
	u64 search_start = 0;
7435
	u64 max_extent_size = 0;
7436
	u64 empty_cluster = 0;
7437
	struct btrfs_space_info *space_info;
7438
	int loop = 0;
7439 7440
	int index = __get_raid_index(flags);
	int alloc_type = (flags & BTRFS_BLOCK_GROUP_DATA) ?
7441
		RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
7442
	bool failed_cluster_refill = false;
7443
	bool failed_alloc = false;
7444
	bool use_cluster = true;
7445
	bool have_caching_bg = false;
7446
	bool orig_have_caching_bg = false;
7447
	bool full_search = false;
7448

7449
	WARN_ON(num_bytes < root->sectorsize);
7450
	ins->type = BTRFS_EXTENT_ITEM_KEY;
7451 7452
	ins->objectid = 0;
	ins->offset = 0;
7453

7454
	trace_find_free_extent(orig_root, num_bytes, empty_size, flags);
J
Josef Bacik 已提交
7455

7456
	space_info = __find_space_info(root->fs_info, flags);
7457
	if (!space_info) {
7458
		btrfs_err(root->fs_info, "No space info for %llu", flags);
7459 7460
		return -ENOSPC;
	}
J
Josef Bacik 已提交
7461

7462
	/*
7463 7464 7465 7466 7467 7468 7469 7470
	 * If our free space is heavily fragmented we may not be able to make
	 * big contiguous allocations, so instead of doing the expensive search
	 * for free space, simply return ENOSPC with our max_extent_size so we
	 * can go ahead and search for a more manageable chunk.
	 *
	 * If our max_extent_size is large enough for our allocation simply
	 * disable clustering since we will likely not be able to find enough
	 * space to create a cluster and induce latency trying.
7471
	 */
7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482
	if (unlikely(space_info->max_extent_size)) {
		spin_lock(&space_info->lock);
		if (space_info->max_extent_size &&
		    num_bytes > space_info->max_extent_size) {
			ins->offset = space_info->max_extent_size;
			spin_unlock(&space_info->lock);
			return -ENOSPC;
		} else if (space_info->max_extent_size) {
			use_cluster = false;
		}
		spin_unlock(&space_info->lock);
7483
	}
J
Josef Bacik 已提交
7484

7485
	last_ptr = fetch_cluster_info(orig_root, space_info, &empty_cluster);
7486
	if (last_ptr) {
7487 7488 7489
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
7490 7491 7492 7493 7494 7495 7496 7497 7498
		if (last_ptr->fragmented) {
			/*
			 * We still set window_start so we can keep track of the
			 * last place we found an allocation to try and save
			 * some time.
			 */
			hint_byte = last_ptr->window_start;
			use_cluster = false;
		}
7499
		spin_unlock(&last_ptr->lock);
7500
	}
7501

7502
	search_start = max(search_start, first_logical_byte(root, 0));
7503
	search_start = max(search_start, hint_byte);
J
Josef Bacik 已提交
7504 7505 7506
	if (search_start == hint_byte) {
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
J
Josef Bacik 已提交
7507 7508 7509
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
7510 7511 7512
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
7513
		 */
7514
		if (block_group && block_group_bits(block_group, flags) &&
7515
		    block_group->cached != BTRFS_CACHE_NO) {
J
Josef Bacik 已提交
7516
			down_read(&space_info->groups_sem);
7517 7518 7519 7520 7521 7522 7523 7524 7525 7526
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
7527
			} else {
7528
				index = get_block_group_index(block_group);
7529
				btrfs_lock_block_group(block_group, delalloc);
7530
				goto have_block_group;
7531
			}
J
Josef Bacik 已提交
7532
		} else if (block_group) {
7533
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
7534
		}
7535
	}
J
Josef Bacik 已提交
7536
search:
7537
	have_caching_bg = false;
7538 7539
	if (index == 0 || index == __get_raid_index(flags))
		full_search = true;
7540
	down_read(&space_info->groups_sem);
7541 7542
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
7543
		u64 offset;
J
Josef Bacik 已提交
7544
		int cached;
7545

7546
		btrfs_grab_block_group(block_group, delalloc);
J
Josef Bacik 已提交
7547
		search_start = block_group->key.objectid;
7548

7549 7550 7551 7552 7553
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
7554
		if (!block_group_bits(block_group, flags)) {
7555 7556
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
D
David Woodhouse 已提交
7557 7558
				BTRFS_BLOCK_GROUP_RAID5 |
				BTRFS_BLOCK_GROUP_RAID6 |
7559 7560 7561 7562 7563 7564 7565
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
7566
			if ((flags & extra) && !(block_group->flags & extra))
7567 7568 7569
				goto loop;
		}

J
Josef Bacik 已提交
7570
have_block_group:
7571 7572
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached)) {
7573
			have_caching_bg = true;
7574
			ret = cache_block_group(block_group, 0);
7575 7576
			BUG_ON(ret < 0);
			ret = 0;
J
Josef Bacik 已提交
7577 7578
		}

7579 7580
		if (unlikely(block_group->cached == BTRFS_CACHE_ERROR))
			goto loop;
7581
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
7582
			goto loop;
J
Josef Bacik 已提交
7583

7584
		/*
7585 7586
		 * Ok we want to try and use the cluster allocator, so
		 * lets look there
7587
		 */
7588
		if (last_ptr && use_cluster) {
7589
			struct btrfs_block_group_cache *used_block_group;
7590
			unsigned long aligned_cluster;
7591 7592 7593 7594
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
7595 7596 7597 7598
			used_block_group = btrfs_lock_cluster(block_group,
							      last_ptr,
							      delalloc);
			if (!used_block_group)
7599
				goto refill_cluster;
7600

7601 7602 7603 7604
			if (used_block_group != block_group &&
			    (used_block_group->ro ||
			     !block_group_bits(used_block_group, flags)))
				goto release_cluster;
7605

7606
			offset = btrfs_alloc_from_cluster(used_block_group,
7607 7608 7609 7610
						last_ptr,
						num_bytes,
						used_block_group->key.objectid,
						&max_extent_size);
7611 7612 7613
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
7614
				trace_btrfs_reserve_extent_cluster(root,
7615 7616
						used_block_group,
						search_start, num_bytes);
7617
				if (used_block_group != block_group) {
7618 7619
					btrfs_release_block_group(block_group,
								  delalloc);
7620 7621
					block_group = used_block_group;
				}
7622 7623 7624
				goto checks;
			}

7625
			WARN_ON(last_ptr->block_group != used_block_group);
7626
release_cluster:
7627 7628 7629 7630 7631 7632 7633 7634
			/* If we are on LOOP_NO_EMPTY_SIZE, we can't
			 * set up a new clusters, so lets just skip it
			 * and let the allocator find whatever block
			 * it can find.  If we reach this point, we
			 * will have tried the cluster allocator
			 * plenty of times and not have found
			 * anything, so we are likely way too
			 * fragmented for the clustering stuff to find
7635 7636 7637 7638 7639 7640 7641 7642
			 * anything.
			 *
			 * However, if the cluster is taken from the
			 * current block group, release the cluster
			 * first, so that we stand a better chance of
			 * succeeding in the unclustered
			 * allocation.  */
			if (loop >= LOOP_NO_EMPTY_SIZE &&
7643
			    used_block_group != block_group) {
7644
				spin_unlock(&last_ptr->refill_lock);
7645 7646
				btrfs_release_block_group(used_block_group,
							  delalloc);
7647 7648 7649
				goto unclustered_alloc;
			}

7650 7651 7652 7653 7654 7655
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

7656 7657 7658 7659
			if (used_block_group != block_group)
				btrfs_release_block_group(used_block_group,
							  delalloc);
refill_cluster:
7660 7661 7662 7663 7664
			if (loop >= LOOP_NO_EMPTY_SIZE) {
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

7665 7666 7667 7668
			aligned_cluster = max_t(unsigned long,
						empty_cluster + empty_size,
					      block_group->full_stripe_len);

7669
			/* allocate a cluster in this block group */
7670 7671 7672 7673
			ret = btrfs_find_space_cluster(root, block_group,
						       last_ptr, search_start,
						       num_bytes,
						       aligned_cluster);
7674 7675 7676 7677 7678 7679
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
7680 7681 7682 7683
							last_ptr,
							num_bytes,
							search_start,
							&max_extent_size);
7684 7685 7686
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
7687 7688 7689
					trace_btrfs_reserve_extent_cluster(root,
						block_group, search_start,
						num_bytes);
7690 7691
					goto checks;
				}
7692 7693
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
7694 7695
				spin_unlock(&last_ptr->refill_lock);

7696
				failed_cluster_refill = true;
J
Josef Bacik 已提交
7697 7698 7699
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
7700
			}
J
Josef Bacik 已提交
7701

7702 7703 7704 7705 7706 7707
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
7708
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
7709
			spin_unlock(&last_ptr->refill_lock);
7710
			goto loop;
7711 7712
		}

7713
unclustered_alloc:
7714 7715 7716 7717 7718 7719 7720 7721 7722 7723
		/*
		 * We are doing an unclustered alloc, set the fragmented flag so
		 * we don't bother trying to setup a cluster again until we get
		 * more space.
		 */
		if (unlikely(last_ptr)) {
			spin_lock(&last_ptr->lock);
			last_ptr->fragmented = 1;
			spin_unlock(&last_ptr->lock);
		}
7724 7725 7726 7727
		spin_lock(&block_group->free_space_ctl->tree_lock);
		if (cached &&
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_cluster + empty_size) {
7728 7729 7730 7731
			if (block_group->free_space_ctl->free_space >
			    max_extent_size)
				max_extent_size =
					block_group->free_space_ctl->free_space;
7732 7733 7734 7735 7736
			spin_unlock(&block_group->free_space_ctl->tree_lock);
			goto loop;
		}
		spin_unlock(&block_group->free_space_ctl->tree_lock);

7737
		offset = btrfs_find_space_for_alloc(block_group, search_start,
7738 7739
						    num_bytes, empty_size,
						    &max_extent_size);
7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
7751
			wait_block_group_cache_progress(block_group,
7752 7753
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
7754
			goto have_block_group;
7755 7756
		} else if (!offset) {
			goto loop;
J
Josef Bacik 已提交
7757
		}
7758
checks:
7759
		search_start = ALIGN(offset, root->stripesize);
7760

J
Josef Bacik 已提交
7761 7762
		/* move on to the next group */
		if (search_start + num_bytes >
7763 7764
		    block_group->key.objectid + block_group->key.offset) {
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
7765
			goto loop;
7766
		}
7767

7768
		if (offset < search_start)
7769
			btrfs_add_free_space(block_group, offset,
7770 7771
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
7772

7773
		ret = btrfs_update_reserved_bytes(block_group, num_bytes,
7774
						  alloc_type, delalloc);
7775
		if (ret == -EAGAIN) {
7776
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
7777
			goto loop;
J
Josef Bacik 已提交
7778
		}
7779
		btrfs_inc_block_group_reservations(block_group);
7780

7781
		/* we are all good, lets return */
J
Josef Bacik 已提交
7782 7783
		ins->objectid = search_start;
		ins->offset = num_bytes;
7784

J
Josef Bacik 已提交
7785 7786
		trace_btrfs_reserve_extent(orig_root, block_group,
					   search_start, num_bytes);
7787
		btrfs_release_block_group(block_group, delalloc);
J
Josef Bacik 已提交
7788 7789
		break;
loop:
7790
		failed_cluster_refill = false;
7791
		failed_alloc = false;
7792
		BUG_ON(index != get_block_group_index(block_group));
7793
		btrfs_release_block_group(block_group, delalloc);
J
Josef Bacik 已提交
7794 7795 7796
	}
	up_read(&space_info->groups_sem);

7797 7798 7799 7800
	if ((loop == LOOP_CACHING_NOWAIT) && have_caching_bg
		&& !orig_have_caching_bg)
		orig_have_caching_bg = true;

7801 7802 7803
	if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
		goto search;

7804 7805 7806
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

7807
	/*
7808 7809
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
7810 7811 7812 7813
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
7814
	 */
7815
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
7816
		index = 0;
7817 7818 7819
		if (loop == LOOP_CACHING_NOWAIT) {
			/*
			 * We want to skip the LOOP_CACHING_WAIT step if we
7820
			 * don't have any uncached bgs and we've already done a
7821 7822
			 * full search through.
			 */
7823
			if (orig_have_caching_bg || !full_search)
7824 7825 7826 7827 7828 7829 7830
				loop = LOOP_CACHING_WAIT;
			else
				loop = LOOP_ALLOC_CHUNK;
		} else {
			loop++;
		}

J
Josef Bacik 已提交
7831
		if (loop == LOOP_ALLOC_CHUNK) {
7832
			struct btrfs_trans_handle *trans;
7833 7834 7835 7836 7837 7838 7839
			int exist = 0;

			trans = current->journal_info;
			if (trans)
				exist = 1;
			else
				trans = btrfs_join_transaction(root);
7840 7841 7842 7843 7844 7845

			if (IS_ERR(trans)) {
				ret = PTR_ERR(trans);
				goto out;
			}

7846
			ret = do_chunk_alloc(trans, root, flags,
7847
					     CHUNK_ALLOC_FORCE);
7848 7849 7850 7851 7852 7853 7854 7855 7856

			/*
			 * If we can't allocate a new chunk we've already looped
			 * through at least once, move on to the NO_EMPTY_SIZE
			 * case.
			 */
			if (ret == -ENOSPC)
				loop = LOOP_NO_EMPTY_SIZE;

7857 7858 7859 7860
			/*
			 * Do not bail out on ENOSPC since we
			 * can do more things.
			 */
7861
			if (ret < 0 && ret != -ENOSPC)
7862 7863
				btrfs_abort_transaction(trans,
							root, ret);
7864 7865
			else
				ret = 0;
7866 7867
			if (!exist)
				btrfs_end_transaction(trans, root);
7868
			if (ret)
7869
				goto out;
J
Josef Bacik 已提交
7870 7871
		}

7872
		if (loop == LOOP_NO_EMPTY_SIZE) {
7873 7874 7875 7876 7877 7878 7879 7880 7881
			/*
			 * Don't loop again if we already have no empty_size and
			 * no empty_cluster.
			 */
			if (empty_size == 0 &&
			    empty_cluster == 0) {
				ret = -ENOSPC;
				goto out;
			}
7882 7883
			empty_size = 0;
			empty_cluster = 0;
7884
		}
7885 7886

		goto search;
J
Josef Bacik 已提交
7887 7888
	} else if (!ins->objectid) {
		ret = -ENOSPC;
7889
	} else if (ins->objectid) {
7890 7891 7892 7893 7894
		if (!use_cluster && last_ptr) {
			spin_lock(&last_ptr->lock);
			last_ptr->window_start = ins->objectid;
			spin_unlock(&last_ptr->lock);
		}
7895
		ret = 0;
C
Chris Mason 已提交
7896
	}
7897
out:
7898 7899 7900 7901
	if (ret == -ENOSPC) {
		spin_lock(&space_info->lock);
		space_info->max_extent_size = max_extent_size;
		spin_unlock(&space_info->lock);
7902
		ins->offset = max_extent_size;
7903
	}
C
Chris Mason 已提交
7904
	return ret;
7905
}
7906

J
Josef Bacik 已提交
7907 7908
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
7909 7910
{
	struct btrfs_block_group_cache *cache;
7911
	int index = 0;
J
Josef Bacik 已提交
7912

J
Josef Bacik 已提交
7913
	spin_lock(&info->lock);
7914
	printk(KERN_INFO "BTRFS: space_info %llu has %llu free, is %sfull\n",
7915 7916 7917
	       info->flags,
	       info->total_bytes - info->bytes_used - info->bytes_pinned -
	       info->bytes_reserved - info->bytes_readonly,
C
Chris Mason 已提交
7918
	       (info->full) ? "" : "not ");
7919
	printk(KERN_INFO "BTRFS: space_info total=%llu, used=%llu, pinned=%llu, "
7920
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
7921 7922 7923
	       info->total_bytes, info->bytes_used, info->bytes_pinned,
	       info->bytes_reserved, info->bytes_may_use,
	       info->bytes_readonly);
J
Josef Bacik 已提交
7924 7925 7926 7927
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
7928

7929
	down_read(&info->groups_sem);
7930 7931
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
7932
		spin_lock(&cache->lock);
7933 7934 7935
		printk(KERN_INFO "BTRFS: "
			   "block group %llu has %llu bytes, "
			   "%llu used %llu pinned %llu reserved %s\n",
7936 7937 7938
		       cache->key.objectid, cache->key.offset,
		       btrfs_block_group_used(&cache->item), cache->pinned,
		       cache->reserved, cache->ro ? "[readonly]" : "");
J
Josef Bacik 已提交
7939 7940 7941
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
7942 7943
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
7944
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
7945
}
7946

7947
int btrfs_reserve_extent(struct btrfs_root *root,
7948 7949
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
7950
			 struct btrfs_key *ins, int is_data, int delalloc)
7951
{
7952
	bool final_tried = num_bytes == min_alloc_size;
7953
	u64 flags;
7954
	int ret;
7955

7956
	flags = btrfs_get_alloc_profile(root, is_data);
7957
again:
7958
	WARN_ON(num_bytes < root->sectorsize);
7959
	ret = find_free_extent(root, num_bytes, empty_size, hint_byte, ins,
7960
			       flags, delalloc);
7961 7962 7963 7964
	if (!ret && !is_data) {
		btrfs_dec_block_group_reservations(root->fs_info,
						   ins->objectid);
	} else if (ret == -ENOSPC) {
7965 7966
		if (!final_tried && ins->offset) {
			num_bytes = min(num_bytes >> 1, ins->offset);
7967
			num_bytes = round_down(num_bytes, root->sectorsize);
7968 7969 7970 7971 7972 7973 7974
			num_bytes = max(num_bytes, min_alloc_size);
			if (num_bytes == min_alloc_size)
				final_tried = true;
			goto again;
		} else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
			struct btrfs_space_info *sinfo;

7975
			sinfo = __find_space_info(root->fs_info, flags);
7976
			btrfs_err(root->fs_info, "allocation failed flags %llu, wanted %llu",
7977
				flags, num_bytes);
7978 7979
			if (sinfo)
				dump_space_info(sinfo, num_bytes, 1);
7980
		}
7981
	}
J
Josef Bacik 已提交
7982 7983

	return ret;
7984 7985
}

7986
static int __btrfs_free_reserved_extent(struct btrfs_root *root,
7987 7988
					u64 start, u64 len,
					int pin, int delalloc)
7989
{
J
Josef Bacik 已提交
7990
	struct btrfs_block_group_cache *cache;
7991
	int ret = 0;
J
Josef Bacik 已提交
7992 7993 7994

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
7995
		btrfs_err(root->fs_info, "Unable to find block group for %llu",
7996
			start);
J
Josef Bacik 已提交
7997 7998
		return -ENOSPC;
	}
7999

8000 8001 8002
	if (pin)
		pin_down_extent(root, cache, start, len, 1);
	else {
8003 8004
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start, len, NULL);
8005
		btrfs_add_free_space(cache, start, len);
8006
		btrfs_update_reserved_bytes(cache, len, RESERVE_FREE, delalloc);
8007
	}
8008

8009
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
8010

8011 8012
	trace_btrfs_reserved_extent_free(root, start, len);

8013 8014 8015
	return ret;
}

8016
int btrfs_free_reserved_extent(struct btrfs_root *root,
8017
			       u64 start, u64 len, int delalloc)
8018
{
8019
	return __btrfs_free_reserved_extent(root, start, len, 0, delalloc);
8020 8021 8022 8023 8024
}

int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
				       u64 start, u64 len)
{
8025
	return __btrfs_free_reserved_extent(root, start, len, 1, 0);
8026 8027
}

8028 8029 8030 8031 8032
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
8033 8034
{
	int ret;
8035
	struct btrfs_fs_info *fs_info = root->fs_info;
8036
	struct btrfs_extent_item *extent_item;
8037
	struct btrfs_extent_inline_ref *iref;
8038
	struct btrfs_path *path;
8039 8040 8041
	struct extent_buffer *leaf;
	int type;
	u32 size;
8042

8043 8044 8045 8046
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
8047

8048
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
8049 8050

	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
8051 8052
	if (!path)
		return -ENOMEM;
8053

8054
	path->leave_spinning = 1;
8055 8056
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
8057 8058 8059 8060
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
J
Josef Bacik 已提交
8061

8062 8063
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
8064
				     struct btrfs_extent_item);
8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
8085 8086

	btrfs_mark_buffer_dirty(path->nodes[0]);
8087
	btrfs_free_path(path);
8088

8089 8090 8091 8092 8093
	ret = remove_from_free_space_tree(trans, fs_info, ins->objectid,
					  ins->offset);
	if (ret)
		return ret;

8094
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
8095
	if (ret) { /* -ENOENT, logic error */
8096
		btrfs_err(fs_info, "update block group failed for %llu %llu",
8097
			ins->objectid, ins->offset);
8098 8099
		BUG();
	}
J
Josef Bacik 已提交
8100
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);
8101 8102 8103
	return ret;
}

8104 8105 8106 8107
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
8108
				     int level, struct btrfs_key *ins)
8109 8110
{
	int ret;
8111 8112 8113 8114 8115 8116
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
8117
	u32 size = sizeof(*extent_item) + sizeof(*iref);
J
Josef Bacik 已提交
8118
	u64 num_bytes = ins->offset;
8119 8120 8121 8122 8123
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);

	if (!skinny_metadata)
		size += sizeof(*block_info);
8124

8125
	path = btrfs_alloc_path();
8126 8127
	if (!path) {
		btrfs_free_and_pin_reserved_extent(root, ins->objectid,
8128
						   root->nodesize);
8129
		return -ENOMEM;
8130
	}
8131

8132 8133 8134
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
8135
	if (ret) {
8136
		btrfs_free_path(path);
8137
		btrfs_free_and_pin_reserved_extent(root, ins->objectid,
8138
						   root->nodesize);
8139 8140
		return ret;
	}
8141 8142 8143 8144 8145 8146 8147 8148 8149

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);

8150 8151
	if (skinny_metadata) {
		iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
8152
		num_bytes = root->nodesize;
8153 8154 8155 8156 8157 8158
	} else {
		block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
		btrfs_set_tree_block_key(leaf, block_info, key);
		btrfs_set_tree_block_level(leaf, block_info, level);
		iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	}
8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173

	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

8174 8175 8176 8177 8178
	ret = remove_from_free_space_tree(trans, fs_info, ins->objectid,
					  num_bytes);
	if (ret)
		return ret;

8179 8180
	ret = update_block_group(trans, root, ins->objectid, root->nodesize,
				 1);
8181
	if (ret) { /* -ENOENT, logic error */
8182
		btrfs_err(fs_info, "update block group failed for %llu %llu",
8183
			ins->objectid, ins->offset);
8184 8185
		BUG();
	}
J
Josef Bacik 已提交
8186

8187
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, root->nodesize);
8188 8189 8190 8191 8192 8193
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
8194 8195
				     u64 offset, u64 ram_bytes,
				     struct btrfs_key *ins)
8196 8197 8198 8199 8200
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

A
Arne Jansen 已提交
8201 8202 8203
	ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
					 ins->offset, 0,
					 root_objectid, owner, offset,
8204 8205
					 ram_bytes, BTRFS_ADD_DELAYED_EXTENT,
					 NULL);
8206 8207
	return ret;
}
8208 8209 8210 8211 8212 8213

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
8214 8215 8216 8217
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
8218 8219 8220
{
	int ret;
	struct btrfs_block_group_cache *block_group;
8221

8222 8223
	/*
	 * Mixed block groups will exclude before processing the log so we only
8224
	 * need to do the exclude dance if this fs isn't mixed.
8225 8226 8227
	 */
	if (!btrfs_fs_incompat(root->fs_info, MIXED_GROUPS)) {
		ret = __exclude_logged_extent(root, ins->objectid, ins->offset);
8228
		if (ret)
8229
			return ret;
8230 8231
	}

8232 8233 8234 8235
	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
	if (!block_group)
		return -EINVAL;

8236
	ret = btrfs_update_reserved_bytes(block_group, ins->offset,
8237
					  RESERVE_ALLOC_NO_ACCOUNT, 0);
8238
	BUG_ON(ret); /* logic error */
8239 8240
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
8241
	btrfs_put_block_group(block_group);
8242 8243 8244
	return ret;
}

8245 8246
static struct extent_buffer *
btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root,
8247
		      u64 bytenr, int level)
8248 8249 8250
{
	struct extent_buffer *buf;

8251
	buf = btrfs_find_create_tree_block(root, bytenr);
8252 8253 8254
	if (IS_ERR(buf))
		return buf;

8255
	btrfs_set_header_generation(buf, trans->transid);
8256
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
8257
	btrfs_tree_lock(buf);
8258
	clean_tree_block(trans, root->fs_info, buf);
8259
	clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
8260 8261

	btrfs_set_lock_blocking(buf);
8262
	set_extent_buffer_uptodate(buf);
8263

8264
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
8265
		buf->log_index = root->log_transid % 2;
8266 8267 8268 8269
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
8270
		if (buf->log_index == 0)
8271 8272 8273 8274
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
8275
					buf->start + buf->len - 1);
8276
	} else {
8277
		buf->log_index = -1;
8278
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
8279
			 buf->start + buf->len - 1, GFP_NOFS);
8280
	}
8281
	trans->dirty = true;
8282
	/* this returns a buffer locked for blocking */
8283 8284 8285
	return buf;
}

8286 8287 8288 8289 8290
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
8291
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
8292
	int ret;
8293
	bool global_updated = false;
8294 8295 8296

	block_rsv = get_block_rsv(trans, root);

8297 8298
	if (unlikely(block_rsv->size == 0))
		goto try_reserve;
8299
again:
8300 8301 8302 8303
	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;

8304 8305 8306
	if (block_rsv->failfast)
		return ERR_PTR(ret);

8307 8308 8309 8310 8311 8312
	if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) {
		global_updated = true;
		update_global_block_rsv(root->fs_info);
		goto again;
	}

8313 8314 8315 8316 8317 8318
	if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
		static DEFINE_RATELIMIT_STATE(_rs,
				DEFAULT_RATELIMIT_INTERVAL * 10,
				/*DEFAULT_RATELIMIT_BURST*/ 1);
		if (__ratelimit(&_rs))
			WARN(1, KERN_DEBUG
8319
				"BTRFS: block rsv returned %d\n", ret);
8320 8321 8322 8323 8324 8325 8326 8327
	}
try_reserve:
	ret = reserve_metadata_bytes(root, block_rsv, blocksize,
				     BTRFS_RESERVE_NO_FLUSH);
	if (!ret)
		return block_rsv;
	/*
	 * If we couldn't reserve metadata bytes try and use some from
8328 8329
	 * the global reserve if its space type is the same as the global
	 * reservation.
8330
	 */
8331 8332
	if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL &&
	    block_rsv->space_info == global_rsv->space_info) {
8333 8334 8335 8336 8337
		ret = block_rsv_use_bytes(global_rsv, blocksize);
		if (!ret)
			return global_rsv;
	}
	return ERR_PTR(ret);
8338 8339
}

J
Josef Bacik 已提交
8340 8341
static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
			    struct btrfs_block_rsv *block_rsv, u32 blocksize)
8342 8343
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
J
Josef Bacik 已提交
8344
	block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
8345 8346
}

8347
/*
8348
 * finds a free extent and does all the dirty work required for allocation
8349
 * returns the tree buffer or an ERR_PTR on error.
8350
 */
8351 8352
struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
8353 8354
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
8355
					u64 hint, u64 empty_size)
8356
{
C
Chris Mason 已提交
8357
	struct btrfs_key ins;
8358
	struct btrfs_block_rsv *block_rsv;
8359
	struct extent_buffer *buf;
8360
	struct btrfs_delayed_extent_op *extent_op;
8361 8362
	u64 flags = 0;
	int ret;
8363
	u32 blocksize = root->nodesize;
8364 8365
	bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
						 SKINNY_METADATA);
8366

8367
	if (btrfs_test_is_dummy_root(root)) {
8368
		buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr,
8369
					    level);
8370 8371 8372 8373
		if (!IS_ERR(buf))
			root->alloc_bytenr += blocksize;
		return buf;
	}
8374

8375 8376 8377 8378
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

8379
	ret = btrfs_reserve_extent(root, blocksize, blocksize,
8380
				   empty_size, hint, &ins, 0, 0);
8381 8382
	if (ret)
		goto out_unuse;
8383

8384
	buf = btrfs_init_new_buffer(trans, root, ins.objectid, level);
8385 8386 8387 8388
	if (IS_ERR(buf)) {
		ret = PTR_ERR(buf);
		goto out_free_reserved;
	}
8389 8390 8391 8392 8393 8394 8395 8396 8397

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
8398
		extent_op = btrfs_alloc_delayed_extent_op();
8399 8400 8401 8402
		if (!extent_op) {
			ret = -ENOMEM;
			goto out_free_buf;
		}
8403 8404 8405 8406 8407
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
8408 8409 8410
		extent_op->update_key = skinny_metadata ? false : true;
		extent_op->update_flags = true;
		extent_op->is_data = false;
8411
		extent_op->level = level;
8412

A
Arne Jansen 已提交
8413
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
8414 8415 8416
						 ins.objectid, ins.offset,
						 parent, root_objectid, level,
						 BTRFS_ADD_DELAYED_EXTENT,
8417
						 extent_op);
8418 8419
		if (ret)
			goto out_free_delayed;
8420
	}
8421
	return buf;
8422 8423 8424 8425 8426 8427 8428 8429 8430 8431

out_free_delayed:
	btrfs_free_delayed_extent_op(extent_op);
out_free_buf:
	free_extent_buffer(buf);
out_free_reserved:
	btrfs_free_reserved_extent(root, ins.objectid, ins.offset, 0);
out_unuse:
	unuse_block_rsv(root->fs_info, block_rsv, blocksize);
	return ERR_PTR(ret);
8432
}
8433

8434 8435 8436 8437 8438 8439 8440 8441 8442
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
8443 8444
	int reada_slot;
	int reada_count;
A
Arne Jansen 已提交
8445
	int for_reloc;
8446 8447 8448 8449 8450
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
8451 8452 8453 8454
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
8455
{
Y
Yan, Zheng 已提交
8456 8457 8458
	u64 bytenr;
	u64 generation;
	u64 refs;
8459
	u64 flags;
8460
	u32 nritems;
Y
Yan, Zheng 已提交
8461 8462 8463
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
8464
	int ret;
Y
Yan, Zheng 已提交
8465 8466
	int slot;
	int nread = 0;
8467

Y
Yan, Zheng 已提交
8468 8469 8470 8471 8472 8473 8474 8475
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
8476

Y
Yan, Zheng 已提交
8477 8478
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
8479
	blocksize = root->nodesize;
8480

Y
Yan, Zheng 已提交
8481 8482 8483
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
8484

C
Chris Mason 已提交
8485
		cond_resched();
Y
Yan, Zheng 已提交
8486 8487
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
8488

Y
Yan, Zheng 已提交
8489 8490
		if (slot == path->slots[wc->level])
			goto reada;
8491

Y
Yan, Zheng 已提交
8492 8493
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
8494 8495
			continue;

8496
		/* We don't lock the tree block, it's OK to be racy here */
8497 8498 8499
		ret = btrfs_lookup_extent_info(trans, root, bytenr,
					       wc->level - 1, 1, &refs,
					       &flags);
8500 8501 8502
		/* We don't care about errors in readahead. */
		if (ret < 0)
			continue;
8503 8504
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
8505 8506 8507
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
8508

8509 8510 8511
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
8512 8513 8514 8515 8516 8517 8518 8519
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
8520 8521 8522 8523
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
8524
		}
Y
Yan, Zheng 已提交
8525
reada:
8526
		readahead_tree_block(root, bytenr);
Y
Yan, Zheng 已提交
8527
		nread++;
C
Chris Mason 已提交
8528
	}
Y
Yan, Zheng 已提交
8529
	wc->reada_slot = slot;
C
Chris Mason 已提交
8530
}
8531

8532
/*
8533 8534
 * These may not be seen by the usual inc/dec ref code so we have to
 * add them here.
8535
 */
8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559
static int record_one_subtree_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root, u64 bytenr,
				     u64 num_bytes)
{
	struct btrfs_qgroup_extent_record *qrecord;
	struct btrfs_delayed_ref_root *delayed_refs;

	qrecord = kmalloc(sizeof(*qrecord), GFP_NOFS);
	if (!qrecord)
		return -ENOMEM;

	qrecord->bytenr = bytenr;
	qrecord->num_bytes = num_bytes;
	qrecord->old_roots = NULL;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	if (btrfs_qgroup_insert_dirty_extent(delayed_refs, qrecord))
		kfree(qrecord);
	spin_unlock(&delayed_refs->lock);

	return 0;
}

8560 8561 8562 8563 8564
static int account_leaf_items(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
			      struct extent_buffer *eb)
{
	int nr = btrfs_header_nritems(eb);
8565
	int i, extent_type, ret;
8566 8567 8568 8569
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	u64 bytenr, num_bytes;

8570 8571 8572 8573
	/* We can be called directly from walk_up_proc() */
	if (!root->fs_info->quota_enabled)
		return 0;

8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591
	for (i = 0; i < nr; i++) {
		btrfs_item_key_to_cpu(eb, &key, i);

		if (key.type != BTRFS_EXTENT_DATA_KEY)
			continue;

		fi = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
		/* filter out non qgroup-accountable extents  */
		extent_type = btrfs_file_extent_type(eb, fi);

		if (extent_type == BTRFS_FILE_EXTENT_INLINE)
			continue;

		bytenr = btrfs_file_extent_disk_bytenr(eb, fi);
		if (!bytenr)
			continue;

		num_bytes = btrfs_file_extent_disk_num_bytes(eb, fi);
8592 8593 8594 8595

		ret = record_one_subtree_extent(trans, root, bytenr, num_bytes);
		if (ret)
			return ret;
8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619 8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714 8715 8716 8717 8718 8719 8720 8721 8722 8723 8724
	}
	return 0;
}

/*
 * Walk up the tree from the bottom, freeing leaves and any interior
 * nodes which have had all slots visited. If a node (leaf or
 * interior) is freed, the node above it will have it's slot
 * incremented. The root node will never be freed.
 *
 * At the end of this function, we should have a path which has all
 * slots incremented to the next position for a search. If we need to
 * read a new node it will be NULL and the node above it will have the
 * correct slot selected for a later read.
 *
 * If we increment the root nodes slot counter past the number of
 * elements, 1 is returned to signal completion of the search.
 */
static int adjust_slots_upwards(struct btrfs_root *root,
				struct btrfs_path *path, int root_level)
{
	int level = 0;
	int nr, slot;
	struct extent_buffer *eb;

	if (root_level == 0)
		return 1;

	while (level <= root_level) {
		eb = path->nodes[level];
		nr = btrfs_header_nritems(eb);
		path->slots[level]++;
		slot = path->slots[level];
		if (slot >= nr || level == 0) {
			/*
			 * Don't free the root -  we will detect this
			 * condition after our loop and return a
			 * positive value for caller to stop walking the tree.
			 */
			if (level != root_level) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
				path->locks[level] = 0;

				free_extent_buffer(eb);
				path->nodes[level] = NULL;
				path->slots[level] = 0;
			}
		} else {
			/*
			 * We have a valid slot to walk back down
			 * from. Stop here so caller can process these
			 * new nodes.
			 */
			break;
		}

		level++;
	}

	eb = path->nodes[root_level];
	if (path->slots[root_level] >= btrfs_header_nritems(eb))
		return 1;

	return 0;
}

/*
 * root_eb is the subtree root and is locked before this function is called.
 */
static int account_shared_subtree(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct extent_buffer *root_eb,
				  u64 root_gen,
				  int root_level)
{
	int ret = 0;
	int level;
	struct extent_buffer *eb = root_eb;
	struct btrfs_path *path = NULL;

	BUG_ON(root_level < 0 || root_level > BTRFS_MAX_LEVEL);
	BUG_ON(root_eb == NULL);

	if (!root->fs_info->quota_enabled)
		return 0;

	if (!extent_buffer_uptodate(root_eb)) {
		ret = btrfs_read_buffer(root_eb, root_gen);
		if (ret)
			goto out;
	}

	if (root_level == 0) {
		ret = account_leaf_items(trans, root, root_eb);
		goto out;
	}

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	/*
	 * Walk down the tree.  Missing extent blocks are filled in as
	 * we go. Metadata is accounted every time we read a new
	 * extent block.
	 *
	 * When we reach a leaf, we account for file extent items in it,
	 * walk back up the tree (adjusting slot pointers as we go)
	 * and restart the search process.
	 */
	extent_buffer_get(root_eb); /* For path */
	path->nodes[root_level] = root_eb;
	path->slots[root_level] = 0;
	path->locks[root_level] = 0; /* so release_path doesn't try to unlock */
walk_down:
	level = root_level;
	while (level >= 0) {
		if (path->nodes[level] == NULL) {
			int parent_slot;
			u64 child_gen;
			u64 child_bytenr;

			/* We need to get child blockptr/gen from
			 * parent before we can read it. */
			eb = path->nodes[level + 1];
			parent_slot = path->slots[level + 1];
			child_bytenr = btrfs_node_blockptr(eb, parent_slot);
			child_gen = btrfs_node_ptr_generation(eb, parent_slot);

8725
			eb = read_tree_block(root, child_bytenr, child_gen);
8726 8727 8728 8729
			if (IS_ERR(eb)) {
				ret = PTR_ERR(eb);
				goto out;
			} else if (!extent_buffer_uptodate(eb)) {
L
Liu Bo 已提交
8730
				free_extent_buffer(eb);
8731
				ret = -EIO;
8732 8733 8734 8735 8736 8737 8738 8739 8740
				goto out;
			}

			path->nodes[level] = eb;
			path->slots[level] = 0;

			btrfs_tree_read_lock(eb);
			btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK);
			path->locks[level] = BTRFS_READ_LOCK_BLOCKING;
8741 8742 8743 8744 8745

			ret = record_one_subtree_extent(trans, root, child_bytenr,
							root->nodesize);
			if (ret)
				goto out;
8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771
		}

		if (level == 0) {
			ret = account_leaf_items(trans, root, path->nodes[level]);
			if (ret)
				goto out;

			/* Nonzero return here means we completed our search */
			ret = adjust_slots_upwards(root, path, root_level);
			if (ret)
				break;

			/* Restart search with new slots */
			goto walk_down;
		}

		level--;
	}

	ret = 0;
out:
	btrfs_free_path(path);

	return ret;
}

Y
Yan Zheng 已提交
8772
/*
L
Liu Bo 已提交
8773
 * helper to process tree block while walking down the tree.
8774 8775 8776 8777 8778
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
8779
 */
8780
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
8781
				   struct btrfs_root *root,
8782
				   struct btrfs_path *path,
8783
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
8784
{
8785 8786 8787
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
8788 8789
	int ret;

8790 8791 8792
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
8793

8794 8795 8796 8797
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
8798 8799 8800
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
8801 8802
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
8803
					       eb->start, level, 1,
8804 8805
					       &wc->refs[level],
					       &wc->flags[level]);
8806 8807 8808
		BUG_ON(ret == -ENOMEM);
		if (ret)
			return ret;
8809 8810
		BUG_ON(wc->refs[level] == 0);
	}
8811

8812 8813 8814
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
8815

8816
		if (path->locks[level] && !wc->keep_locks) {
8817
			btrfs_tree_unlock_rw(eb, path->locks[level]);
8818 8819 8820 8821
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
8822

8823 8824 8825
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
8826
		ret = btrfs_inc_ref(trans, root, eb, 1);
8827
		BUG_ON(ret); /* -ENOMEM */
8828
		ret = btrfs_dec_ref(trans, root, eb, 0);
8829
		BUG_ON(ret); /* -ENOMEM */
8830
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
8831 8832
						  eb->len, flag,
						  btrfs_header_level(eb), 0);
8833
		BUG_ON(ret); /* -ENOMEM */
8834 8835 8836 8837 8838 8839 8840 8841
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
8842
		btrfs_tree_unlock_rw(eb, path->locks[level]);
8843 8844 8845 8846 8847
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
8848
/*
L
Liu Bo 已提交
8849
 * helper to process tree block pointer.
Y
Yan, Zheng 已提交
8850 8851 8852 8853 8854 8855 8856 8857 8858 8859 8860 8861 8862 8863
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
8864
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
8865 8866 8867 8868 8869 8870 8871 8872 8873 8874
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;
8875
	bool need_account = false;
Y
Yan, Zheng 已提交
8876 8877 8878 8879 8880 8881 8882 8883 8884

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
8885 8886
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
8887
		return 1;
8888
	}
Y
Yan, Zheng 已提交
8889 8890

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
8891
	blocksize = root->nodesize;
Y
Yan, Zheng 已提交
8892

8893
	next = btrfs_find_tree_block(root->fs_info, bytenr);
Y
Yan, Zheng 已提交
8894
	if (!next) {
8895
		next = btrfs_find_create_tree_block(root, bytenr);
8896 8897 8898
		if (IS_ERR(next))
			return PTR_ERR(next);

8899 8900
		btrfs_set_buffer_lockdep_class(root->root_key.objectid, next,
					       level - 1);
Y
Yan, Zheng 已提交
8901 8902 8903 8904 8905
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

8906
	ret = btrfs_lookup_extent_info(trans, root, bytenr, level - 1, 1,
8907 8908
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
8909 8910 8911 8912 8913
	if (ret < 0) {
		btrfs_tree_unlock(next);
		return ret;
	}

8914 8915 8916 8917
	if (unlikely(wc->refs[level - 1] == 0)) {
		btrfs_err(root->fs_info, "Missing references.");
		BUG();
	}
8918
	*lookup_info = 0;
Y
Yan, Zheng 已提交
8919

8920
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
8921
		if (wc->refs[level - 1] > 1) {
8922
			need_account = true;
8923 8924 8925 8926
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
8927 8928 8929 8930 8931 8932 8933 8934 8935 8936 8937 8938 8939
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
8940 8941 8942 8943
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
8944 8945
	}

8946
	if (!btrfs_buffer_uptodate(next, generation, 0)) {
Y
Yan, Zheng 已提交
8947 8948 8949
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
8950
		*lookup_info = 1;
Y
Yan, Zheng 已提交
8951 8952 8953 8954 8955
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
8956
		next = read_tree_block(root, bytenr, generation);
8957 8958 8959
		if (IS_ERR(next)) {
			return PTR_ERR(next);
		} else if (!extent_buffer_uptodate(next)) {
8960
			free_extent_buffer(next);
8961
			return -EIO;
8962
		}
Y
Yan, Zheng 已提交
8963 8964 8965 8966 8967 8968 8969 8970
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
8971
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Y
Yan, Zheng 已提交
8972 8973 8974 8975 8976 8977 8978
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
8979 8980 8981 8982 8983 8984 8985 8986
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
8987

8988 8989 8990 8991
		if (need_account) {
			ret = account_shared_subtree(trans, root, next,
						     generation, level - 1);
			if (ret) {
8992 8993
				btrfs_err_rl(root->fs_info,
					"Error "
8994
					"%d accounting shared subtree. Quota "
8995 8996
					"is out of sync, rescan required.",
					ret);
8997 8998
			}
		}
8999
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
9000
				root->root_key.objectid, level - 1, 0);
9001
		BUG_ON(ret); /* -ENOMEM */
Y
Yan, Zheng 已提交
9002 9003 9004
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
9005
	*lookup_info = 1;
Y
Yan, Zheng 已提交
9006 9007 9008
	return 1;
}

9009
/*
L
Liu Bo 已提交
9010
 * helper to process tree block while walking up the tree.
9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022 9023 9024 9025
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
9026
	int ret;
9027 9028 9029 9030 9031 9032 9033 9034 9035 9036 9037 9038 9039 9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051 9052
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
9053
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
9054 9055

			ret = btrfs_lookup_extent_info(trans, root,
9056
						       eb->start, level, 1,
9057 9058
						       &wc->refs[level],
						       &wc->flags[level]);
9059 9060
			if (ret < 0) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
L
Liu Bo 已提交
9061
				path->locks[level] = 0;
9062 9063
				return ret;
			}
9064 9065
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
9066
				btrfs_tree_unlock_rw(eb, path->locks[level]);
L
Liu Bo 已提交
9067
				path->locks[level] = 0;
9068 9069
				return 1;
			}
Y
Yan Zheng 已提交
9070
		}
9071
	}
Y
Yan Zheng 已提交
9072

9073 9074
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
9075

9076 9077 9078
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
9079
				ret = btrfs_dec_ref(trans, root, eb, 1);
9080
			else
9081
				ret = btrfs_dec_ref(trans, root, eb, 0);
9082
			BUG_ON(ret); /* -ENOMEM */
9083 9084
			ret = account_leaf_items(trans, root, eb);
			if (ret) {
9085 9086
				btrfs_err_rl(root->fs_info,
					"error "
9087
					"%d accounting leaf items. Quota "
9088 9089
					"is out of sync, rescan required.",
					ret);
9090
			}
9091 9092 9093 9094 9095 9096
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
9097
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
9098
		}
9099
		clean_tree_block(trans, root->fs_info, eb);
9100 9101 9102 9103 9104 9105 9106 9107 9108 9109 9110 9111 9112 9113
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
9114 9115
	}

9116
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
9117 9118 9119
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
9120
	return 0;
9121 9122 9123 9124 9125 9126 9127 9128
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
9129
	int lookup_info = 1;
9130 9131 9132
	int ret;

	while (level >= 0) {
9133
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
9134 9135 9136 9137 9138 9139
		if (ret > 0)
			break;

		if (level == 0)
			break;

9140 9141 9142 9143
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

9144
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
9145 9146 9147
		if (ret > 0) {
			path->slots[level]++;
			continue;
9148 9149
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
9150
		level = wc->level;
Y
Yan Zheng 已提交
9151 9152 9153 9154
	}
	return 0;
}

C
Chris Mason 已提交
9155
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
9156
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
9157
				 struct btrfs_path *path,
9158
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
9159
{
9160
	int level = wc->level;
C
Chris Mason 已提交
9161
	int ret;
9162

9163 9164 9165 9166 9167 9168
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
9169 9170
			return 0;
		} else {
9171 9172 9173
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
9174

9175
			if (path->locks[level]) {
9176 9177
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
9178
				path->locks[level] = 0;
Y
Yan Zheng 已提交
9179
			}
9180 9181 9182
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
9183 9184 9185 9186 9187
		}
	}
	return 1;
}

C
Chris Mason 已提交
9188
/*
9189 9190 9191 9192 9193 9194 9195 9196 9197
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
D
David Sterba 已提交
9198 9199
 *
 * If called with for_reloc == 0, may exit early with -EAGAIN
C
Chris Mason 已提交
9200
 */
9201
int btrfs_drop_snapshot(struct btrfs_root *root,
A
Arne Jansen 已提交
9202 9203
			 struct btrfs_block_rsv *block_rsv, int update_ref,
			 int for_reloc)
C
Chris Mason 已提交
9204
{
9205
	struct btrfs_path *path;
9206 9207
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
9208
	struct btrfs_root_item *root_item = &root->root_item;
9209 9210 9211 9212 9213
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
9214
	bool root_dropped = false;
C
Chris Mason 已提交
9215

9216 9217
	btrfs_debug(root->fs_info, "Drop subvolume %llu", root->objectid);

9218
	path = btrfs_alloc_path();
9219 9220 9221 9222
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
C
Chris Mason 已提交
9223

9224
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
9225 9226
	if (!wc) {
		btrfs_free_path(path);
9227 9228
		err = -ENOMEM;
		goto out;
9229
	}
9230

9231
	trans = btrfs_start_transaction(tree_root, 0);
9232 9233 9234 9235
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
		goto out_free;
	}
9236

9237 9238
	if (block_rsv)
		trans->block_rsv = block_rsv;
9239

9240
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
9241
		level = btrfs_header_level(root->node);
9242 9243
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
9244
		path->slots[level] = 0;
9245
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
9246 9247
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
9248 9249
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
9250 9251 9252
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

9253
		level = root_item->drop_level;
9254
		BUG_ON(level == 0);
9255
		path->lowest_level = level;
9256 9257 9258 9259
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
9260
			goto out_end_trans;
9261
		}
Y
Yan, Zheng 已提交
9262
		WARN_ON(ret > 0);
9263

9264 9265 9266 9267
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
9268
		btrfs_unlock_up_safe(path, 0);
9269 9270 9271 9272 9273

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);
9274
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
9275 9276 9277

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
9278
						level, 1, &wc->refs[level],
9279
						&wc->flags[level]);
9280 9281 9282 9283
			if (ret < 0) {
				err = ret;
				goto out_end_trans;
			}
9284 9285 9286 9287 9288 9289
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
9290
			path->locks[level] = 0;
9291 9292 9293
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
9294
	}
9295 9296 9297 9298 9299 9300

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
A
Arne Jansen 已提交
9301
	wc->for_reloc = for_reloc;
Y
Yan, Zheng 已提交
9302
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
9303

C
Chris Mason 已提交
9304
	while (1) {
D
David Sterba 已提交
9305

9306 9307 9308
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
9309
			break;
9310
		}
C
Chris Mason 已提交
9311

9312 9313 9314
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
9315
			break;
9316 9317 9318 9319
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
9320 9321
			break;
		}
9322 9323 9324 9325 9326 9327 9328 9329 9330 9331

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
9332 9333
		if (btrfs_should_end_transaction(trans, tree_root) ||
		    (!for_reloc && btrfs_need_cleaner_sleep(root))) {
9334 9335 9336
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
9337 9338 9339 9340 9341
			if (ret) {
				btrfs_abort_transaction(trans, tree_root, ret);
				err = ret;
				goto out_end_trans;
			}
9342

9343
			btrfs_end_transaction_throttle(trans, tree_root);
9344
			if (!for_reloc && btrfs_need_cleaner_sleep(root)) {
9345
				pr_debug("BTRFS: drop snapshot early exit\n");
9346 9347 9348 9349
				err = -EAGAIN;
				goto out_free;
			}

9350
			trans = btrfs_start_transaction(tree_root, 0);
9351 9352 9353 9354
			if (IS_ERR(trans)) {
				err = PTR_ERR(trans);
				goto out_free;
			}
9355 9356
			if (block_rsv)
				trans->block_rsv = block_rsv;
9357
		}
C
Chris Mason 已提交
9358
	}
9359
	btrfs_release_path(path);
9360 9361
	if (err)
		goto out_end_trans;
9362 9363

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
9364 9365 9366 9367
	if (ret) {
		btrfs_abort_transaction(trans, tree_root, ret);
		goto out_end_trans;
	}
9368

9369
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
9370 9371
		ret = btrfs_find_root(tree_root, &root->root_key, path,
				      NULL, NULL);
9372 9373 9374 9375 9376
		if (ret < 0) {
			btrfs_abort_transaction(trans, tree_root, ret);
			err = ret;
			goto out_end_trans;
		} else if (ret > 0) {
9377 9378 9379 9380 9381 9382 9383
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
9384 9385 9386
		}
	}

9387
	if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) {
9388
		btrfs_add_dropped_root(trans, root);
9389 9390 9391
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
9392
		btrfs_put_fs_root(root);
9393
	}
9394
	root_dropped = true;
9395
out_end_trans:
9396
	btrfs_end_transaction_throttle(trans, tree_root);
9397
out_free:
9398
	kfree(wc);
9399
	btrfs_free_path(path);
9400
out:
9401 9402 9403 9404 9405 9406 9407
	/*
	 * So if we need to stop dropping the snapshot for whatever reason we
	 * need to make sure to add it back to the dead root list so that we
	 * keep trying to do the work later.  This also cleans up roots if we
	 * don't have it in the radix (like when we recover after a power fail
	 * or unmount) so we don't leak memory.
	 */
9408
	if (!for_reloc && root_dropped == false)
9409
		btrfs_add_dead_root(root);
9410
	if (err && err != -EAGAIN)
9411
		btrfs_handle_fs_error(root->fs_info, err, NULL);
9412
	return err;
C
Chris Mason 已提交
9413
}
C
Chris Mason 已提交
9414

9415 9416 9417 9418
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
A
Arne Jansen 已提交
9419
 * only used by relocation code
9420
 */
Y
Yan Zheng 已提交
9421 9422 9423 9424 9425 9426
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
9427
	struct walk_control *wc;
Y
Yan Zheng 已提交
9428 9429 9430 9431 9432
	int level;
	int parent_level;
	int ret = 0;
	int wret;

9433 9434
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
9435
	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
9436 9437
	if (!path)
		return -ENOMEM;
Y
Yan Zheng 已提交
9438

9439
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
T
Tsutomu Itoh 已提交
9440 9441 9442 9443
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
9444

9445
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
9446 9447 9448 9449 9450
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

9451
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
9452 9453 9454
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
9455
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
9456 9457 9458 9459 9460 9461 9462 9463

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
A
Arne Jansen 已提交
9464
	wc->for_reloc = 1;
Y
Yan, Zheng 已提交
9465
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
9466 9467

	while (1) {
9468 9469
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
9470 9471
			ret = wret;
			break;
9472
		}
Y
Yan Zheng 已提交
9473

9474
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
9475 9476 9477 9478 9479 9480
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

9481
	kfree(wc);
Y
Yan Zheng 已提交
9482 9483 9484 9485
	btrfs_free_path(path);
	return ret;
}

9486 9487 9488
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
9489
	u64 stripped;
9490

9491 9492 9493 9494 9495 9496 9497
	/*
	 * if restripe for this chunk_type is on pick target profile and
	 * return, otherwise do the usual balance
	 */
	stripped = get_restripe_target(root->fs_info, flags);
	if (stripped)
		return extended_to_chunk(stripped);
9498

9499
	num_devices = root->fs_info->fs_devices->rw_devices;
9500

9501
	stripped = BTRFS_BLOCK_GROUP_RAID0 |
D
David Woodhouse 已提交
9502
		BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 |
9503 9504
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

9529
		/* this is drive concat, leave it alone */
9530
	}
9531

9532 9533 9534
	return flags;
}

9535
static int inc_block_group_ro(struct btrfs_block_group_cache *cache, int force)
C
Chris Mason 已提交
9536
{
9537 9538
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
9539
	u64 min_allocable_bytes;
9540
	int ret = -ENOSPC;
C
Chris Mason 已提交
9541

9542 9543 9544 9545 9546 9547 9548 9549
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
9550
		min_allocable_bytes = SZ_1M;
9551 9552 9553
	else
		min_allocable_bytes = 0;

9554 9555
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
9556 9557

	if (cache->ro) {
9558
		cache->ro++;
9559 9560 9561 9562
		ret = 0;
		goto out;
	}

9563 9564 9565 9566
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
9567 9568
	    sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
	    min_allocable_bytes <= sinfo->total_bytes) {
9569
		sinfo->bytes_readonly += num_bytes;
9570
		cache->ro++;
9571
		list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
9572 9573
		ret = 0;
	}
9574
out:
9575 9576 9577 9578
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
9579

9580
int btrfs_inc_block_group_ro(struct btrfs_root *root,
9581
			     struct btrfs_block_group_cache *cache)
9582

9583 9584 9585 9586
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
9587

9588
again:
C
Chris Mason 已提交
9589
	trans = btrfs_join_transaction(root);
9590 9591
	if (IS_ERR(trans))
		return PTR_ERR(trans);
9592

9593 9594 9595 9596 9597 9598
	/*
	 * we're not allowed to set block groups readonly after the dirty
	 * block groups cache has started writing.  If it already started,
	 * back off and let this transaction commit
	 */
	mutex_lock(&root->fs_info->ro_block_group_mutex);
9599
	if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) {
9600 9601 9602 9603 9604 9605 9606 9607 9608 9609 9610
		u64 transid = trans->transid;

		mutex_unlock(&root->fs_info->ro_block_group_mutex);
		btrfs_end_transaction(trans, root);

		ret = btrfs_wait_for_commit(root, transid);
		if (ret)
			return ret;
		goto again;
	}

9611 9612 9613 9614 9615 9616 9617 9618 9619 9620 9621 9622 9623 9624 9625 9626 9627 9628
	/*
	 * if we are changing raid levels, try to allocate a corresponding
	 * block group with the new raid level.
	 */
	alloc_flags = update_block_group_flags(root, cache->flags);
	if (alloc_flags != cache->flags) {
		ret = do_chunk_alloc(trans, root, alloc_flags,
				     CHUNK_ALLOC_FORCE);
		/*
		 * ENOSPC is allowed here, we may have enough space
		 * already allocated at the new raid level to
		 * carry on
		 */
		if (ret == -ENOSPC)
			ret = 0;
		if (ret < 0)
			goto out;
	}
9629

9630
	ret = inc_block_group_ro(cache, 0);
9631 9632 9633
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
9634
	ret = do_chunk_alloc(trans, root, alloc_flags,
9635
			     CHUNK_ALLOC_FORCE);
9636 9637
	if (ret < 0)
		goto out;
9638
	ret = inc_block_group_ro(cache, 0);
9639
out:
9640 9641
	if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) {
		alloc_flags = update_block_group_flags(root, cache->flags);
9642
		lock_chunks(root->fs_info->chunk_root);
9643
		check_system_chunk(trans, root, alloc_flags);
9644
		unlock_chunks(root->fs_info->chunk_root);
9645
	}
9646
	mutex_unlock(&root->fs_info->ro_block_group_mutex);
9647

9648 9649 9650
	btrfs_end_transaction(trans, root);
	return ret;
}
9651

9652 9653 9654 9655
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
9656
	return do_chunk_alloc(trans, root, alloc_flags,
9657
			      CHUNK_ALLOC_FORCE);
9658 9659
}

9660 9661
/*
 * helper to account the unused space of all the readonly block group in the
9662
 * space_info. takes mirrors into account.
9663
 */
9664
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
9665 9666 9667 9668 9669
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

9670
	/* It's df, we don't care if it's racy */
9671 9672 9673 9674 9675
	if (list_empty(&sinfo->ro_bgs))
		return 0;

	spin_lock(&sinfo->lock);
	list_for_each_entry(block_group, &sinfo->ro_bgs, ro_list) {
9676 9677 9678 9679 9680 9681 9682 9683 9684 9685 9686 9687 9688 9689 9690 9691 9692 9693 9694 9695 9696 9697 9698 9699 9700
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}
	spin_unlock(&sinfo->lock);

	return free_bytes;
}

9701
void btrfs_dec_block_group_ro(struct btrfs_root *root,
9702
			      struct btrfs_block_group_cache *cache)
9703
{
9704 9705 9706 9707 9708 9709 9710
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
9711 9712 9713 9714 9715 9716 9717
	if (!--cache->ro) {
		num_bytes = cache->key.offset - cache->reserved -
			    cache->pinned - cache->bytes_super -
			    btrfs_block_group_used(&cache->item);
		sinfo->bytes_readonly -= num_bytes;
		list_del_init(&cache->ro_list);
	}
9718 9719
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
9720 9721
}

9722 9723 9724 9725 9726 9727 9728
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
9729
{
9730 9731 9732 9733
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
9734
	struct btrfs_trans_handle *trans;
9735
	u64 min_free;
J
Josef Bacik 已提交
9736 9737
	u64 dev_min = 1;
	u64 dev_nr = 0;
9738
	u64 target;
9739
	int debug;
9740
	int index;
9741 9742
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
9743

9744 9745
	debug = btrfs_test_opt(root, ENOSPC_DEBUG);

9746
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
9747

9748
	/* odd, couldn't find the block group, leave it alone */
9749 9750 9751 9752 9753
	if (!block_group) {
		if (debug)
			btrfs_warn(root->fs_info,
				   "can't find block group for bytenr %llu",
				   bytenr);
9754
		return -1;
9755
	}
Z
Zheng Yan 已提交
9756

9757 9758
	min_free = btrfs_block_group_used(&block_group->item);

9759
	/* no bytes used, we're good */
9760
	if (!min_free)
Z
Zheng Yan 已提交
9761 9762
		goto out;

9763 9764
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
9765

9766
	full = space_info->full;
9767

9768 9769
	/*
	 * if this is the last block group we have in this space, we can't
9770 9771 9772 9773
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
9774
	 */
9775
	if ((space_info->total_bytes != block_group->key.offset) &&
9776 9777 9778
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
9779 9780
		spin_unlock(&space_info->lock);
		goto out;
9781
	}
9782
	spin_unlock(&space_info->lock);
9783

9784 9785 9786
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
9787 9788 9789
	 * alloc devices and guess if we have enough space.  if this block
	 * group is going to be restriped, run checks against the target
	 * profile instead of the current one.
9790 9791
	 */
	ret = -1;
9792

9793 9794 9795 9796 9797 9798 9799 9800
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
9801 9802
	target = get_restripe_target(root->fs_info, block_group->flags);
	if (target) {
9803
		index = __get_raid_index(extended_to_chunk(target));
9804 9805 9806 9807 9808
	} else {
		/*
		 * this is just a balance, so if we were marked as full
		 * we know there is no space for a new chunk
		 */
9809 9810 9811 9812 9813
		if (full) {
			if (debug)
				btrfs_warn(root->fs_info,
					"no space to alloc new chunk for block group %llu",
					block_group->key.objectid);
9814
			goto out;
9815
		}
9816 9817 9818 9819

		index = get_block_group_index(block_group);
	}

9820
	if (index == BTRFS_RAID_RAID10) {
9821
		dev_min = 4;
J
Josef Bacik 已提交
9822 9823
		/* Divide by 2 */
		min_free >>= 1;
9824
	} else if (index == BTRFS_RAID_RAID1) {
9825
		dev_min = 2;
9826
	} else if (index == BTRFS_RAID_DUP) {
J
Josef Bacik 已提交
9827 9828
		/* Multiply by 2 */
		min_free <<= 1;
9829
	} else if (index == BTRFS_RAID_RAID0) {
9830
		dev_min = fs_devices->rw_devices;
9831
		min_free = div64_u64(min_free, dev_min);
9832 9833
	}

9834 9835 9836 9837 9838 9839 9840
	/* We need to do this so that we can look at pending chunks */
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans)) {
		ret = PTR_ERR(trans);
		goto out;
	}

9841 9842
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
9843
		u64 dev_offset;
9844

9845 9846 9847 9848
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
9849 9850
		if (device->total_bytes > device->bytes_used + min_free &&
		    !device->is_tgtdev_for_dev_replace) {
9851
			ret = find_free_dev_extent(trans, device, min_free,
9852
						   &dev_offset, NULL);
9853
			if (!ret)
9854 9855 9856
				dev_nr++;

			if (dev_nr >= dev_min)
9857
				break;
9858

9859
			ret = -1;
9860
		}
9861
	}
9862 9863 9864 9865
	if (debug && ret == -1)
		btrfs_warn(root->fs_info,
			"no space to allocate a new chunk for block group %llu",
			block_group->key.objectid);
9866
	mutex_unlock(&root->fs_info->chunk_mutex);
9867
	btrfs_end_transaction(trans, root);
9868
out:
9869
	btrfs_put_block_group(block_group);
9870 9871 9872
	return ret;
}

9873 9874
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
9875
{
9876
	int ret = 0;
9877 9878 9879
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
9880

9881 9882
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
9883 9884
		goto out;

C
Chris Mason 已提交
9885
	while (1) {
9886
		slot = path->slots[0];
9887
		leaf = path->nodes[0];
9888 9889 9890 9891 9892
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
9893
				goto out;
9894
			break;
9895
		}
9896
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
9897

9898
		if (found_key.objectid >= key->objectid &&
9899 9900 9901 9902
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
9903
		path->slots[0]++;
9904
	}
9905
out:
9906
	return ret;
9907 9908
}

9909 9910 9911 9912 9913 9914 9915 9916 9917 9918 9919 9920 9921 9922 9923 9924 9925 9926 9927 9928 9929 9930 9931 9932 9933 9934 9935 9936 9937 9938 9939 9940 9941 9942
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
9943 9944 9945
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
9946
	struct btrfs_space_info *space_info;
9947
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
9948 9949
	struct rb_node *n;

9950
	down_write(&info->commit_root_sem);
9951 9952 9953 9954 9955 9956
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
9957
	up_write(&info->commit_root_sem);
9958

9959 9960 9961 9962 9963 9964 9965 9966 9967 9968
	spin_lock(&info->unused_bgs_lock);
	while (!list_empty(&info->unused_bgs)) {
		block_group = list_first_entry(&info->unused_bgs,
					       struct btrfs_block_group_cache,
					       bg_list);
		list_del_init(&block_group->bg_list);
		btrfs_put_block_group(block_group);
	}
	spin_unlock(&info->unused_bgs_lock);

Z
Zheng Yan 已提交
9969 9970 9971 9972 9973 9974
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
9975
		RB_CLEAR_NODE(&block_group->cache_node);
Y
Yan Zheng 已提交
9976 9977
		spin_unlock(&info->block_group_cache_lock);

9978
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
9979
		list_del(&block_group->list);
9980
		up_write(&block_group->space_info->groups_sem);
9981

J
Josef Bacik 已提交
9982
		if (block_group->cached == BTRFS_CACHE_STARTED)
9983
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
9984

9985 9986 9987 9988
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
9989 9990
		if (block_group->cached == BTRFS_CACHE_NO ||
		    block_group->cached == BTRFS_CACHE_ERROR)
9991 9992
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
9993
		btrfs_remove_free_space_cache(block_group);
9994
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
9995 9996

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
9997 9998
	}
	spin_unlock(&info->block_group_cache_lock);
9999 10000 10001 10002 10003 10004 10005 10006 10007

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

10008 10009
	release_global_block_rsv(info);

10010
	while (!list_empty(&info->space_info)) {
10011 10012
		int i;

10013 10014 10015
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
10016 10017 10018 10019 10020 10021

		/*
		 * Do not hide this behind enospc_debug, this is actually
		 * important and indicates a real bug if this happens.
		 */
		if (WARN_ON(space_info->bytes_pinned > 0 ||
10022
			    space_info->bytes_reserved > 0 ||
10023 10024
			    space_info->bytes_may_use > 0))
			dump_space_info(space_info, 0, 0);
10025
		list_del(&space_info->list);
10026 10027
		for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
			struct kobject *kobj;
10028 10029 10030
			kobj = space_info->block_group_kobjs[i];
			space_info->block_group_kobjs[i] = NULL;
			if (kobj) {
10031 10032 10033 10034 10035 10036
				kobject_del(kobj);
				kobject_put(kobj);
			}
		}
		kobject_del(&space_info->kobj);
		kobject_put(&space_info->kobj);
10037
	}
Z
Zheng Yan 已提交
10038 10039 10040
	return 0;
}

10041 10042 10043 10044
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);
10045
	bool first = false;
10046 10047

	down_write(&space_info->groups_sem);
10048 10049 10050 10051 10052 10053
	if (list_empty(&space_info->block_groups[index]))
		first = true;
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);

	if (first) {
10054
		struct raid_kobject *rkobj;
10055 10056
		int ret;

10057 10058 10059 10060 10061 10062 10063
		rkobj = kzalloc(sizeof(*rkobj), GFP_NOFS);
		if (!rkobj)
			goto out_err;
		rkobj->raid_type = index;
		kobject_init(&rkobj->kobj, &btrfs_raid_ktype);
		ret = kobject_add(&rkobj->kobj, &space_info->kobj,
				  "%s", get_raid_name(index));
10064
		if (ret) {
10065 10066
			kobject_put(&rkobj->kobj);
			goto out_err;
10067
		}
10068
		space_info->block_group_kobjs[index] = &rkobj->kobj;
10069
	}
10070 10071 10072 10073

	return;
out_err:
	pr_warn("BTRFS: failed to add kobject for block cache. ignoring.\n");
10074 10075
}

10076 10077 10078 10079 10080 10081 10082 10083 10084 10085 10086 10087 10088 10089 10090 10091 10092 10093 10094 10095 10096 10097 10098 10099 10100
static struct btrfs_block_group_cache *
btrfs_create_block_group_cache(struct btrfs_root *root, u64 start, u64 size)
{
	struct btrfs_block_group_cache *cache;

	cache = kzalloc(sizeof(*cache), GFP_NOFS);
	if (!cache)
		return NULL;

	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return NULL;
	}

	cache->key.objectid = start;
	cache->key.offset = size;
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;

	cache->sectorsize = root->sectorsize;
	cache->fs_info = root->fs_info;
	cache->full_stripe_len = btrfs_full_stripe_len(root,
					       &root->fs_info->mapping_tree,
					       start);
10101 10102
	set_free_space_tree_thresholds(cache);

10103 10104
	atomic_set(&cache->count, 1);
	spin_lock_init(&cache->lock);
10105
	init_rwsem(&cache->data_rwsem);
10106 10107
	INIT_LIST_HEAD(&cache->list);
	INIT_LIST_HEAD(&cache->cluster_list);
10108
	INIT_LIST_HEAD(&cache->bg_list);
10109
	INIT_LIST_HEAD(&cache->ro_list);
10110
	INIT_LIST_HEAD(&cache->dirty_list);
10111
	INIT_LIST_HEAD(&cache->io_list);
10112
	btrfs_init_free_space_ctl(cache);
10113
	atomic_set(&cache->trimming, 0);
10114
	mutex_init(&cache->free_space_lock);
10115 10116 10117 10118

	return cache;
}

C
Chris Mason 已提交
10119 10120 10121 10122 10123
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
10124
	struct btrfs_fs_info *info = root->fs_info;
10125
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
10126 10127
	struct btrfs_key key;
	struct btrfs_key found_key;
10128
	struct extent_buffer *leaf;
10129 10130
	int need_clear = 0;
	u64 cache_gen;
10131

C
Chris Mason 已提交
10132
	root = info->extent_root;
C
Chris Mason 已提交
10133
	key.objectid = 0;
10134
	key.offset = 0;
10135
	key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
C
Chris Mason 已提交
10136 10137 10138
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
10139
	path->reada = READA_FORWARD;
C
Chris Mason 已提交
10140

10141
	cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
10142
	if (btrfs_test_opt(root, SPACE_CACHE) &&
10143
	    btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
10144
		need_clear = 1;
10145 10146
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
10147

C
Chris Mason 已提交
10148
	while (1) {
10149
		ret = find_first_block_group(root, path, &key);
10150 10151
		if (ret > 0)
			break;
10152 10153
		if (ret != 0)
			goto error;
10154

10155 10156
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
10157 10158 10159

		cache = btrfs_create_block_group_cache(root, found_key.objectid,
						       found_key.offset);
C
Chris Mason 已提交
10160
		if (!cache) {
10161
			ret = -ENOMEM;
10162
			goto error;
C
Chris Mason 已提交
10163
		}
10164

10165 10166 10167 10168 10169 10170 10171 10172 10173 10174 10175 10176
		if (need_clear) {
			/*
			 * When we mount with old space cache, we need to
			 * set BTRFS_DC_CLEAR and set dirty flag.
			 *
			 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
			 *    truncate the old free space cache inode and
			 *    setup a new one.
			 * b) Setting 'dirty flag' makes sure that we flush
			 *    the new space cache info onto disk.
			 */
			if (btrfs_test_opt(root, SPACE_CACHE))
10177
				cache->disk_cache_state = BTRFS_DC_CLEAR;
10178
		}
10179

10180 10181 10182
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
10183
		cache->flags = btrfs_block_group_flags(&cache->item);
10184

C
Chris Mason 已提交
10185
		key.objectid = found_key.objectid + found_key.offset;
10186
		btrfs_release_path(path);
10187

10188 10189 10190 10191 10192
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
10193 10194 10195 10196 10197 10198 10199
		ret = exclude_super_stripes(root, cache);
		if (ret) {
			/*
			 * We may have excluded something, so call this just in
			 * case.
			 */
			free_excluded_extents(root, cache);
10200
			btrfs_put_block_group(cache);
10201 10202
			goto error;
		}
10203

J
Josef Bacik 已提交
10204 10205 10206 10207 10208 10209 10210 10211
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
10212
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
10213
			cache->cached = BTRFS_CACHE_FINISHED;
10214
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
10215
		} else if (btrfs_block_group_used(&cache->item) == 0) {
10216
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
10217 10218 10219 10220 10221
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
10222
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
10223
		}
10224

10225 10226 10227 10228 10229 10230 10231
		ret = btrfs_add_block_group_cache(root->fs_info, cache);
		if (ret) {
			btrfs_remove_free_space_cache(cache);
			btrfs_put_block_group(cache);
			goto error;
		}

10232
		trace_btrfs_add_block_group(root->fs_info, cache, 0);
10233 10234
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
10235
					cache->bytes_super, &space_info);
10236 10237 10238 10239 10240
		if (ret) {
			btrfs_remove_free_space_cache(cache);
			spin_lock(&info->block_group_cache_lock);
			rb_erase(&cache->cache_node,
				 &info->block_group_cache_tree);
10241
			RB_CLEAR_NODE(&cache->cache_node);
10242 10243 10244 10245 10246
			spin_unlock(&info->block_group_cache_lock);
			btrfs_put_block_group(cache);
			goto error;
		}

10247
		cache->space_info = space_info;
10248

10249
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
10250

10251
		set_avail_alloc_bits(root->fs_info, cache->flags);
10252
		if (btrfs_chunk_readonly(root, cache->key.objectid)) {
10253
			inc_block_group_ro(cache, 1);
10254 10255 10256 10257 10258 10259 10260 10261 10262 10263
		} else if (btrfs_block_group_used(&cache->item) == 0) {
			spin_lock(&info->unused_bgs_lock);
			/* Should always be true but just in case. */
			if (list_empty(&cache->bg_list)) {
				btrfs_get_block_group(cache);
				list_add_tail(&cache->bg_list,
					      &info->unused_bgs);
			}
			spin_unlock(&info->unused_bgs_lock);
		}
C
Chris Mason 已提交
10264
	}
10265 10266 10267 10268 10269

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
D
David Woodhouse 已提交
10270 10271
		       BTRFS_BLOCK_GROUP_RAID5 |
		       BTRFS_BLOCK_GROUP_RAID6 |
10272 10273 10274 10275 10276 10277
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
10278 10279 10280
		list_for_each_entry(cache,
				&space_info->block_groups[BTRFS_RAID_RAID0],
				list)
10281
			inc_block_group_ro(cache, 1);
10282 10283 10284
		list_for_each_entry(cache,
				&space_info->block_groups[BTRFS_RAID_SINGLE],
				list)
10285
			inc_block_group_ro(cache, 1);
C
Chris Mason 已提交
10286
	}
10287 10288

	init_global_block_rsv(info);
10289 10290
	ret = 0;
error:
C
Chris Mason 已提交
10291
	btrfs_free_path(path);
10292
	return ret;
C
Chris Mason 已提交
10293
}
10294

10295 10296 10297 10298 10299 10300 10301 10302
void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root)
{
	struct btrfs_block_group_cache *block_group, *tmp;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
	struct btrfs_block_group_item item;
	struct btrfs_key key;
	int ret = 0;
10303
	bool can_flush_pending_bgs = trans->can_flush_pending_bgs;
10304

10305
	trans->can_flush_pending_bgs = false;
10306
	list_for_each_entry_safe(block_group, tmp, &trans->new_bgs, bg_list) {
10307
		if (ret)
10308
			goto next;
10309 10310 10311 10312 10313 10314 10315 10316 10317 10318

		spin_lock(&block_group->lock);
		memcpy(&item, &block_group->item, sizeof(item));
		memcpy(&key, &block_group->key, sizeof(key));
		spin_unlock(&block_group->lock);

		ret = btrfs_insert_item(trans, extent_root, &key, &item,
					sizeof(item));
		if (ret)
			btrfs_abort_transaction(trans, extent_root, ret);
10319 10320 10321 10322
		ret = btrfs_finish_chunk_alloc(trans, extent_root,
					       key.objectid, key.offset);
		if (ret)
			btrfs_abort_transaction(trans, extent_root, ret);
10323 10324
		add_block_group_free_space(trans, root->fs_info, block_group);
		/* already aborted the transaction if it failed. */
10325 10326
next:
		list_del_init(&block_group->bg_list);
10327
	}
10328
	trans->can_flush_pending_bgs = can_flush_pending_bgs;
10329 10330
}

10331 10332
int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
10333
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
10334 10335 10336 10337 10338 10339 10340
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;
	extent_root = root->fs_info->extent_root;

10341
	btrfs_set_log_full_commit(root->fs_info, trans);
10342

10343
	cache = btrfs_create_block_group_cache(root, chunk_offset, size);
J
Josef Bacik 已提交
10344 10345
	if (!cache)
		return -ENOMEM;
10346

10347 10348 10349 10350
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	btrfs_set_block_group_flags(&cache->item, type);

10351
	cache->flags = type;
10352
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
10353
	cache->cached = BTRFS_CACHE_FINISHED;
10354
	cache->needs_free_space = 1;
10355 10356 10357 10358 10359 10360 10361
	ret = exclude_super_stripes(root, cache);
	if (ret) {
		/*
		 * We may have excluded something, so call this just in
		 * case.
		 */
		free_excluded_extents(root, cache);
10362
		btrfs_put_block_group(cache);
10363 10364
		return ret;
	}
10365

J
Josef Bacik 已提交
10366 10367 10368
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

10369 10370
	free_excluded_extents(root, cache);

10371 10372 10373 10374 10375 10376 10377 10378
#ifdef CONFIG_BTRFS_DEBUG
	if (btrfs_should_fragment_free_space(root, cache)) {
		u64 new_bytes_used = size - bytes_used;

		bytes_used += new_bytes_used >> 1;
		fragment_free_space(root, cache);
	}
#endif
10379 10380 10381 10382 10383
	/*
	 * Call to ensure the corresponding space_info object is created and
	 * assigned to our block group, but don't update its counters just yet.
	 * We want our bg to be added to the rbtree with its ->space_info set.
	 */
10384
	ret = update_space_info(root->fs_info, cache->flags, 0, 0, 0,
10385 10386 10387 10388 10389 10390 10391
				&cache->space_info);
	if (ret) {
		btrfs_remove_free_space_cache(cache);
		btrfs_put_block_group(cache);
		return ret;
	}

10392 10393 10394 10395 10396 10397 10398
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
	if (ret) {
		btrfs_remove_free_space_cache(cache);
		btrfs_put_block_group(cache);
		return ret;
	}

10399 10400 10401 10402
	/*
	 * Now that our block group has its ->space_info set and is inserted in
	 * the rbtree, update the space info's counters.
	 */
10403
	trace_btrfs_add_block_group(root->fs_info, cache, 1);
10404
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
10405
				cache->bytes_super, &cache->space_info);
10406 10407 10408 10409 10410
	if (ret) {
		btrfs_remove_free_space_cache(cache);
		spin_lock(&root->fs_info->block_group_cache_lock);
		rb_erase(&cache->cache_node,
			 &root->fs_info->block_group_cache_tree);
10411
		RB_CLEAR_NODE(&cache->cache_node);
10412 10413 10414 10415
		spin_unlock(&root->fs_info->block_group_cache_lock);
		btrfs_put_block_group(cache);
		return ret;
	}
10416
	update_global_block_rsv(root->fs_info);
10417

10418
	__link_block_group(cache->space_info, cache);
10419

10420
	list_add_tail(&cache->bg_list, &trans->new_bgs);
10421

C
Chris Mason 已提交
10422
	set_avail_alloc_bits(extent_root->fs_info, type);
10423 10424
	return 0;
}
Z
Zheng Yan 已提交
10425

10426 10427
static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
10428 10429
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
10430

10431
	write_seqlock(&fs_info->profiles_lock);
10432 10433 10434 10435 10436 10437
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits &= ~extra_flags;
10438
	write_sequnlock(&fs_info->profiles_lock);
10439 10440
}

Z
Zheng Yan 已提交
10441
int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
10442 10443
			     struct btrfs_root *root, u64 group_start,
			     struct extent_map *em)
Z
Zheng Yan 已提交
10444 10445 10446
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
10447
	struct btrfs_free_cluster *cluster;
10448
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
10449
	struct btrfs_key key;
10450
	struct inode *inode;
10451
	struct kobject *kobj = NULL;
Z
Zheng Yan 已提交
10452
	int ret;
10453
	int index;
J
Josef Bacik 已提交
10454
	int factor;
10455
	struct btrfs_caching_control *caching_ctl = NULL;
10456
	bool remove_em;
Z
Zheng Yan 已提交
10457 10458 10459 10460 10461

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
10462
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
10463

10464 10465 10466 10467 10468 10469
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

Z
Zheng Yan 已提交
10470
	memcpy(&key, &block_group->key, sizeof(key));
10471
	index = get_block_group_index(block_group);
J
Josef Bacik 已提交
10472 10473 10474 10475 10476 10477
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
10478

10479 10480 10481 10482 10483 10484 10485 10486 10487 10488 10489 10490 10491 10492 10493
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
10494
	path = btrfs_alloc_path();
10495 10496 10497 10498
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
Z
Zheng Yan 已提交
10499

10500 10501 10502 10503
	/*
	 * get the inode first so any iput calls done for the io_list
	 * aren't the final iput (no unlinks allowed now)
	 */
10504
	inode = lookup_free_space_inode(tree_root, block_group, path);
10505 10506 10507 10508 10509 10510 10511 10512 10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523 10524 10525 10526 10527 10528 10529 10530 10531

	mutex_lock(&trans->transaction->cache_write_mutex);
	/*
	 * make sure our free spache cache IO is done before remove the
	 * free space inode
	 */
	spin_lock(&trans->transaction->dirty_bgs_lock);
	if (!list_empty(&block_group->io_list)) {
		list_del_init(&block_group->io_list);

		WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode);

		spin_unlock(&trans->transaction->dirty_bgs_lock);
		btrfs_wait_cache_io(root, trans, block_group,
				    &block_group->io_ctl, path,
				    block_group->key.objectid);
		btrfs_put_block_group(block_group);
		spin_lock(&trans->transaction->dirty_bgs_lock);
	}

	if (!list_empty(&block_group->dirty_list)) {
		list_del_init(&block_group->dirty_list);
		btrfs_put_block_group(block_group);
	}
	spin_unlock(&trans->transaction->dirty_bgs_lock);
	mutex_unlock(&trans->transaction->cache_write_mutex);

10532
	if (!IS_ERR(inode)) {
10533
		ret = btrfs_orphan_add(trans, inode);
10534 10535 10536 10537
		if (ret) {
			btrfs_add_delayed_iput(inode);
			goto out;
		}
10538 10539 10540 10541 10542 10543 10544 10545 10546 10547 10548 10549
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
10550
		btrfs_add_delayed_iput(inode);
10551 10552 10553 10554 10555 10556 10557 10558 10559 10560
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
10561
		btrfs_release_path(path);
10562 10563 10564 10565
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
10566
		btrfs_release_path(path);
10567 10568
	}

10569
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
10570 10571
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
10572
	RB_CLEAR_NODE(&block_group->cache_node);
10573 10574 10575

	if (root->fs_info->first_logical_byte == block_group->key.objectid)
		root->fs_info->first_logical_byte = (u64)-1;
10576
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
10577

10578
	down_write(&block_group->space_info->groups_sem);
10579 10580 10581 10582 10583
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
10584
	if (list_empty(&block_group->space_info->block_groups[index])) {
10585 10586
		kobj = block_group->space_info->block_group_kobjs[index];
		block_group->space_info->block_group_kobjs[index] = NULL;
10587
		clear_avail_alloc_bits(root->fs_info, block_group->flags);
10588
	}
10589
	up_write(&block_group->space_info->groups_sem);
10590 10591 10592 10593
	if (kobj) {
		kobject_del(kobj);
		kobject_put(kobj);
	}
Z
Zheng Yan 已提交
10594

10595 10596
	if (block_group->has_caching_ctl)
		caching_ctl = get_caching_control(block_group);
J
Josef Bacik 已提交
10597
	if (block_group->cached == BTRFS_CACHE_STARTED)
10598
		wait_block_group_cache_done(block_group);
10599 10600 10601 10602 10603 10604 10605 10606 10607 10608 10609 10610 10611 10612 10613 10614 10615 10616 10617 10618 10619 10620
	if (block_group->has_caching_ctl) {
		down_write(&root->fs_info->commit_root_sem);
		if (!caching_ctl) {
			struct btrfs_caching_control *ctl;

			list_for_each_entry(ctl,
				    &root->fs_info->caching_block_groups, list)
				if (ctl->block_group == block_group) {
					caching_ctl = ctl;
					atomic_inc(&caching_ctl->count);
					break;
				}
		}
		if (caching_ctl)
			list_del_init(&caching_ctl->list);
		up_write(&root->fs_info->commit_root_sem);
		if (caching_ctl) {
			/* Once for the caching bgs list and once for us. */
			put_caching_control(caching_ctl);
			put_caching_control(caching_ctl);
		}
	}
J
Josef Bacik 已提交
10621

10622 10623
	spin_lock(&trans->transaction->dirty_bgs_lock);
	if (!list_empty(&block_group->dirty_list)) {
10624 10625 10626 10627
		WARN_ON(1);
	}
	if (!list_empty(&block_group->io_list)) {
		WARN_ON(1);
10628 10629
	}
	spin_unlock(&trans->transaction->dirty_bgs_lock);
J
Josef Bacik 已提交
10630 10631
	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
10632
	spin_lock(&block_group->space_info->lock);
10633
	list_del_init(&block_group->ro_list);
10634 10635 10636 10637 10638 10639 10640 10641 10642

	if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
		WARN_ON(block_group->space_info->total_bytes
			< block_group->key.offset);
		WARN_ON(block_group->space_info->bytes_readonly
			< block_group->key.offset);
		WARN_ON(block_group->space_info->disk_total
			< block_group->key.offset * factor);
	}
Y
Yan Zheng 已提交
10643 10644
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
10645
	block_group->space_info->disk_total -= block_group->key.offset * factor;
10646

Y
Yan Zheng 已提交
10647
	spin_unlock(&block_group->space_info->lock);
10648

10649 10650
	memcpy(&key, &block_group->key, sizeof(key));

10651
	lock_chunks(root);
10652 10653 10654 10655
	if (!list_empty(&em->list)) {
		/* We're in the transaction->pending_chunks list. */
		free_extent_map(em);
	}
10656 10657 10658 10659 10660 10661 10662 10663 10664 10665 10666 10667 10668 10669 10670 10671 10672 10673 10674
	spin_lock(&block_group->lock);
	block_group->removed = 1;
	/*
	 * At this point trimming can't start on this block group, because we
	 * removed the block group from the tree fs_info->block_group_cache_tree
	 * so no one can't find it anymore and even if someone already got this
	 * block group before we removed it from the rbtree, they have already
	 * incremented block_group->trimming - if they didn't, they won't find
	 * any free space entries because we already removed them all when we
	 * called btrfs_remove_free_space_cache().
	 *
	 * And we must not remove the extent map from the fs_info->mapping_tree
	 * to prevent the same logical address range and physical device space
	 * ranges from being reused for a new block group. This is because our
	 * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
	 * completely transactionless, so while it is trimming a range the
	 * currently running transaction might finish and a new one start,
	 * allowing for new block groups to be created that can reuse the same
	 * physical device locations unless we take this special care.
10675 10676 10677 10678 10679
	 *
	 * There may also be an implicit trim operation if the file system
	 * is mounted with -odiscard. The same protections must remain
	 * in place until the extents have been discarded completely when
	 * the transaction commit has completed.
10680 10681 10682 10683 10684 10685 10686 10687 10688 10689 10690 10691 10692 10693 10694 10695 10696 10697 10698 10699 10700 10701 10702 10703 10704 10705 10706 10707
	 */
	remove_em = (atomic_read(&block_group->trimming) == 0);
	/*
	 * Make sure a trimmer task always sees the em in the pinned_chunks list
	 * if it sees block_group->removed == 1 (needs to lock block_group->lock
	 * before checking block_group->removed).
	 */
	if (!remove_em) {
		/*
		 * Our em might be in trans->transaction->pending_chunks which
		 * is protected by fs_info->chunk_mutex ([lock|unlock]_chunks),
		 * and so is the fs_info->pinned_chunks list.
		 *
		 * So at this point we must be holding the chunk_mutex to avoid
		 * any races with chunk allocation (more specifically at
		 * volumes.c:contains_pending_extent()), to ensure it always
		 * sees the em, either in the pending_chunks list or in the
		 * pinned_chunks list.
		 */
		list_move_tail(&em->list, &root->fs_info->pinned_chunks);
	}
	spin_unlock(&block_group->lock);

	if (remove_em) {
		struct extent_map_tree *em_tree;

		em_tree = &root->fs_info->mapping_tree.map_tree;
		write_lock(&em_tree->lock);
10708 10709 10710 10711 10712
		/*
		 * The em might be in the pending_chunks list, so make sure the
		 * chunk mutex is locked, since remove_extent_mapping() will
		 * delete us from that list.
		 */
10713 10714 10715 10716 10717 10718
		remove_extent_mapping(em_tree, em);
		write_unlock(&em_tree->lock);
		/* once for the tree */
		free_extent_map(em);
	}

10719 10720
	unlock_chunks(root);

10721 10722 10723 10724
	ret = remove_block_group_free_space(trans, root->fs_info, block_group);
	if (ret)
		goto out;

10725 10726
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
10727 10728 10729 10730 10731 10732 10733 10734 10735 10736 10737 10738

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
10739

10740
struct btrfs_trans_handle *
10741 10742
btrfs_start_trans_remove_block_group(struct btrfs_fs_info *fs_info,
				     const u64 chunk_offset)
10743
{
10744 10745 10746 10747 10748 10749 10750 10751 10752 10753
	struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree;
	struct extent_map *em;
	struct map_lookup *map;
	unsigned int num_items;

	read_lock(&em_tree->lock);
	em = lookup_extent_mapping(em_tree, chunk_offset, 1);
	read_unlock(&em_tree->lock);
	ASSERT(em && em->start == chunk_offset);

10754
	/*
10755 10756 10757 10758
	 * We need to reserve 3 + N units from the metadata space info in order
	 * to remove a block group (done at btrfs_remove_chunk() and at
	 * btrfs_remove_block_group()), which are used for:
	 *
10759 10760
	 * 1 unit for adding the free space inode's orphan (located in the tree
	 * of tree roots).
10761 10762 10763 10764 10765 10766 10767 10768 10769 10770 10771
	 * 1 unit for deleting the block group item (located in the extent
	 * tree).
	 * 1 unit for deleting the free space item (located in tree of tree
	 * roots).
	 * N units for deleting N device extent items corresponding to each
	 * stripe (located in the device tree).
	 *
	 * In order to remove a block group we also need to reserve units in the
	 * system space info in order to update the chunk tree (update one or
	 * more device items and remove one chunk item), but this is done at
	 * btrfs_remove_chunk() through a call to check_system_chunk().
10772
	 */
10773
	map = em->map_lookup;
10774 10775 10776
	num_items = 3 + map->num_stripes;
	free_extent_map(em);

10777
	return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root,
10778
							   num_items, 1);
10779 10780
}

10781 10782 10783 10784 10785 10786 10787 10788 10789 10790 10791 10792 10793 10794 10795 10796 10797 10798
/*
 * Process the unused_bgs list and remove any that don't have any allocated
 * space inside of them.
 */
void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_root *root = fs_info->extent_root;
	struct btrfs_trans_handle *trans;
	int ret = 0;

	if (!fs_info->open)
		return;

	spin_lock(&fs_info->unused_bgs_lock);
	while (!list_empty(&fs_info->unused_bgs)) {
		u64 start, end;
10799
		int trimming;
10800 10801 10802 10803 10804

		block_group = list_first_entry(&fs_info->unused_bgs,
					       struct btrfs_block_group_cache,
					       bg_list);
		list_del_init(&block_group->bg_list);
10805 10806 10807

		space_info = block_group->space_info;

10808 10809 10810 10811 10812 10813
		if (ret || btrfs_mixed_space_info(space_info)) {
			btrfs_put_block_group(block_group);
			continue;
		}
		spin_unlock(&fs_info->unused_bgs_lock);

10814
		mutex_lock(&fs_info->delete_unused_bgs_mutex);
10815

10816 10817 10818 10819 10820
		/* Don't want to race with allocators so take the groups_sem */
		down_write(&space_info->groups_sem);
		spin_lock(&block_group->lock);
		if (block_group->reserved ||
		    btrfs_block_group_used(&block_group->item) ||
10821 10822
		    block_group->ro ||
		    list_is_singular(&block_group->list)) {
10823 10824 10825 10826 10827 10828 10829 10830 10831 10832 10833 10834 10835
			/*
			 * We want to bail if we made new allocations or have
			 * outstanding allocations in this block group.  We do
			 * the ro check in case balance is currently acting on
			 * this block group.
			 */
			spin_unlock(&block_group->lock);
			up_write(&space_info->groups_sem);
			goto next;
		}
		spin_unlock(&block_group->lock);

		/* We don't want to force the issue, only flip if it's ok. */
10836
		ret = inc_block_group_ro(block_group, 0);
10837 10838 10839 10840 10841 10842 10843 10844 10845 10846
		up_write(&space_info->groups_sem);
		if (ret < 0) {
			ret = 0;
			goto next;
		}

		/*
		 * Want to do this before we do anything else so we can recover
		 * properly if we fail to join the transaction.
		 */
10847 10848
		trans = btrfs_start_trans_remove_block_group(fs_info,
						     block_group->key.objectid);
10849
		if (IS_ERR(trans)) {
10850
			btrfs_dec_block_group_ro(root, block_group);
10851 10852 10853 10854 10855 10856 10857 10858 10859 10860
			ret = PTR_ERR(trans);
			goto next;
		}

		/*
		 * We could have pending pinned extents for this block group,
		 * just delete them, we don't care about them anymore.
		 */
		start = block_group->key.objectid;
		end = start + block_group->key.offset - 1;
10861 10862 10863 10864 10865 10866 10867 10868 10869 10870 10871 10872
		/*
		 * Hold the unused_bg_unpin_mutex lock to avoid racing with
		 * btrfs_finish_extent_commit(). If we are at transaction N,
		 * another task might be running finish_extent_commit() for the
		 * previous transaction N - 1, and have seen a range belonging
		 * to the block group in freed_extents[] before we were able to
		 * clear the whole block group range from freed_extents[]. This
		 * means that task can lookup for the block group after we
		 * unpinned it from freed_extents[] and removed it, leading to
		 * a BUG_ON() at btrfs_unpin_extent_range().
		 */
		mutex_lock(&fs_info->unused_bg_unpin_mutex);
10873
		ret = clear_extent_bits(&fs_info->freed_extents[0], start, end,
10874
				  EXTENT_DIRTY);
10875
		if (ret) {
10876
			mutex_unlock(&fs_info->unused_bg_unpin_mutex);
10877
			btrfs_dec_block_group_ro(root, block_group);
10878 10879 10880
			goto end_trans;
		}
		ret = clear_extent_bits(&fs_info->freed_extents[1], start, end,
10881
				  EXTENT_DIRTY);
10882
		if (ret) {
10883
			mutex_unlock(&fs_info->unused_bg_unpin_mutex);
10884
			btrfs_dec_block_group_ro(root, block_group);
10885 10886
			goto end_trans;
		}
10887
		mutex_unlock(&fs_info->unused_bg_unpin_mutex);
10888 10889

		/* Reset pinned so btrfs_put_block_group doesn't complain */
10890 10891 10892 10893 10894 10895 10896
		spin_lock(&space_info->lock);
		spin_lock(&block_group->lock);

		space_info->bytes_pinned -= block_group->pinned;
		space_info->bytes_readonly += block_group->pinned;
		percpu_counter_add(&space_info->total_bytes_pinned,
				   -block_group->pinned);
10897 10898
		block_group->pinned = 0;

10899 10900 10901
		spin_unlock(&block_group->lock);
		spin_unlock(&space_info->lock);

10902 10903 10904 10905 10906 10907 10908
		/* DISCARD can flip during remount */
		trimming = btrfs_test_opt(root, DISCARD);

		/* Implicit trim during transaction commit. */
		if (trimming)
			btrfs_get_block_group_trimming(block_group);

10909 10910 10911 10912 10913 10914
		/*
		 * Btrfs_remove_chunk will abort the transaction if things go
		 * horribly wrong.
		 */
		ret = btrfs_remove_chunk(trans, root,
					 block_group->key.objectid);
10915 10916 10917 10918 10919 10920 10921 10922 10923 10924 10925 10926 10927

		if (ret) {
			if (trimming)
				btrfs_put_block_group_trimming(block_group);
			goto end_trans;
		}

		/*
		 * If we're not mounted with -odiscard, we can just forget
		 * about this block group. Otherwise we'll need to wait
		 * until transaction commit to do the actual discard.
		 */
		if (trimming) {
10928 10929 10930 10931 10932 10933
			spin_lock(&fs_info->unused_bgs_lock);
			/*
			 * A concurrent scrub might have added us to the list
			 * fs_info->unused_bgs, so use a list_move operation
			 * to add the block group to the deleted_bgs list.
			 */
10934 10935
			list_move(&block_group->bg_list,
				  &trans->transaction->deleted_bgs);
10936
			spin_unlock(&fs_info->unused_bgs_lock);
10937 10938
			btrfs_get_block_group(block_group);
		}
10939
end_trans:
10940 10941
		btrfs_end_transaction(trans, root);
next:
10942
		mutex_unlock(&fs_info->delete_unused_bgs_mutex);
10943 10944 10945 10946 10947 10948
		btrfs_put_block_group(block_group);
		spin_lock(&fs_info->unused_bgs_lock);
	}
	spin_unlock(&fs_info->unused_bgs_lock);
}

10949 10950 10951
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
10952 10953 10954 10955
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
10956 10957
	int ret;

10958
	disk_super = fs_info->super_copy;
10959
	if (!btrfs_super_root(disk_super))
10960
		return -EINVAL;
10961

10962 10963 10964
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
10965

10966
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
10967
	ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info);
10968
	if (ret)
10969
		goto out;
10970

10971 10972
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
10973
		ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info);
10974 10975
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
10976
		ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info);
10977 10978 10979 10980
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
10981
		ret = update_space_info(fs_info, flags, 0, 0, 0, &space_info);
10982 10983
	}
out:
10984 10985 10986
	return ret;
}

L
liubo 已提交
10987 10988
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
10989
	return unpin_extent_range(root, start, end, false);
L
liubo 已提交
10990 10991
}

10992 10993 10994 10995 10996 10997 10998 10999 11000 11001 11002 11003 11004 11005 11006 11007 11008 11009 11010 11011 11012 11013 11014 11015 11016 11017 11018 11019 11020 11021 11022 11023 11024 11025 11026 11027 11028 11029 11030 11031 11032 11033 11034 11035 11036 11037 11038 11039 11040 11041 11042 11043 11044 11045 11046 11047 11048 11049 11050 11051 11052 11053 11054 11055 11056 11057 11058 11059 11060 11061 11062 11063 11064 11065 11066 11067 11068 11069 11070 11071 11072 11073 11074 11075 11076 11077 11078
/*
 * It used to be that old block groups would be left around forever.
 * Iterating over them would be enough to trim unused space.  Since we
 * now automatically remove them, we also need to iterate over unallocated
 * space.
 *
 * We don't want a transaction for this since the discard may take a
 * substantial amount of time.  We don't require that a transaction be
 * running, but we do need to take a running transaction into account
 * to ensure that we're not discarding chunks that were released in
 * the current transaction.
 *
 * Holding the chunks lock will prevent other threads from allocating
 * or releasing chunks, but it won't prevent a running transaction
 * from committing and releasing the memory that the pending chunks
 * list head uses.  For that, we need to take a reference to the
 * transaction.
 */
static int btrfs_trim_free_extents(struct btrfs_device *device,
				   u64 minlen, u64 *trimmed)
{
	u64 start = 0, len = 0;
	int ret;

	*trimmed = 0;

	/* Not writeable = nothing to do. */
	if (!device->writeable)
		return 0;

	/* No free space = nothing to do. */
	if (device->total_bytes <= device->bytes_used)
		return 0;

	ret = 0;

	while (1) {
		struct btrfs_fs_info *fs_info = device->dev_root->fs_info;
		struct btrfs_transaction *trans;
		u64 bytes;

		ret = mutex_lock_interruptible(&fs_info->chunk_mutex);
		if (ret)
			return ret;

		down_read(&fs_info->commit_root_sem);

		spin_lock(&fs_info->trans_lock);
		trans = fs_info->running_transaction;
		if (trans)
			atomic_inc(&trans->use_count);
		spin_unlock(&fs_info->trans_lock);

		ret = find_free_dev_extent_start(trans, device, minlen, start,
						 &start, &len);
		if (trans)
			btrfs_put_transaction(trans);

		if (ret) {
			up_read(&fs_info->commit_root_sem);
			mutex_unlock(&fs_info->chunk_mutex);
			if (ret == -ENOSPC)
				ret = 0;
			break;
		}

		ret = btrfs_issue_discard(device->bdev, start, len, &bytes);
		up_read(&fs_info->commit_root_sem);
		mutex_unlock(&fs_info->chunk_mutex);

		if (ret)
			break;

		start += len;
		*trimmed += bytes;

		if (fatal_signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}

		cond_resched();
	}

	return ret;
}

11079 11080 11081 11082
int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
11083 11084
	struct btrfs_device *device;
	struct list_head *devices;
11085 11086 11087 11088
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
11089
	u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
11090 11091
	int ret = 0;

11092 11093 11094 11095 11096 11097 11098
	/*
	 * try to trim all FS space, our block group may start from non-zero.
	 */
	if (range->len == total_bytes)
		cache = btrfs_lookup_first_block_group(fs_info, range->start);
	else
		cache = btrfs_lookup_block_group(fs_info, range->start);
11099 11100 11101 11102 11103 11104 11105 11106 11107 11108 11109 11110 11111

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
11112
				ret = cache_block_group(cache, 0);
11113 11114 11115 11116 11117 11118 11119 11120 11121
				if (ret) {
					btrfs_put_block_group(cache);
					break;
				}
				ret = wait_block_group_cache_done(cache);
				if (ret) {
					btrfs_put_block_group(cache);
					break;
				}
11122 11123 11124 11125 11126 11127 11128 11129 11130 11131 11132 11133 11134 11135 11136 11137 11138
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

11139 11140 11141 11142 11143 11144 11145 11146 11147 11148 11149 11150
	mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
	devices = &root->fs_info->fs_devices->alloc_list;
	list_for_each_entry(device, devices, dev_alloc_list) {
		ret = btrfs_trim_free_extents(device, range->minlen,
					      &group_trimmed);
		if (ret)
			break;

		trimmed += group_trimmed;
	}
	mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);

11151 11152 11153
	range->len = trimmed;
	return ret;
}
11154 11155

/*
11156 11157 11158 11159 11160 11161
 * btrfs_{start,end}_write_no_snapshoting() are similar to
 * mnt_{want,drop}_write(), they are used to prevent some tasks from writing
 * data into the page cache through nocow before the subvolume is snapshoted,
 * but flush the data into disk after the snapshot creation, or to prevent
 * operations while snapshoting is ongoing and that cause the snapshot to be
 * inconsistent (writes followed by expanding truncates for example).
11162
 */
11163
void btrfs_end_write_no_snapshoting(struct btrfs_root *root)
11164 11165 11166
{
	percpu_counter_dec(&root->subv_writers->counter);
	/*
11167
	 * Make sure counter is updated before we wake up waiters.
11168 11169 11170 11171 11172 11173
	 */
	smp_mb();
	if (waitqueue_active(&root->subv_writers->wait))
		wake_up(&root->subv_writers->wait);
}

11174
int btrfs_start_write_no_snapshoting(struct btrfs_root *root)
11175
{
11176
	if (atomic_read(&root->will_be_snapshoted))
11177 11178 11179 11180 11181 11182 11183
		return 0;

	percpu_counter_inc(&root->subv_writers->counter);
	/*
	 * Make sure counter is updated before we check for snapshot creation.
	 */
	smp_mb();
11184
	if (atomic_read(&root->will_be_snapshoted)) {
11185
		btrfs_end_write_no_snapshoting(root);
11186 11187 11188 11189
		return 0;
	}
	return 1;
}
11190 11191 11192 11193 11194 11195 11196 11197 11198 11199 11200 11201 11202 11203 11204 11205 11206 11207 11208 11209

static int wait_snapshoting_atomic_t(atomic_t *a)
{
	schedule();
	return 0;
}

void btrfs_wait_for_snapshot_creation(struct btrfs_root *root)
{
	while (true) {
		int ret;

		ret = btrfs_start_write_no_snapshoting(root);
		if (ret)
			break;
		wait_on_atomic_t(&root->will_be_snapshoted,
				 wait_snapshoting_atomic_t,
				 TASK_UNINTERRUPTIBLE);
	}
}