extent-tree.c 216.1 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
26
#include <linux/ratelimit.h>
C
Chris Mason 已提交
27
#include "compat.h"
28
#include "hash.h"
29 30 31
#include "ctree.h"
#include "disk-io.h"
#include "print-tree.h"
32
#include "transaction.h"
33
#include "volumes.h"
34
#include "locking.h"
35
#include "free-space-cache.h"
36

37 38
#undef SCRAMBLE_DELAYED_REFS

39 40
/*
 * control flags for do_chunk_alloc's force field
41 42 43 44 45 46 47 48 49
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
50 51
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
52 53 54
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
55 56
	CHUNK_ALLOC_LIMITED = 1,
	CHUNK_ALLOC_FORCE = 2,
57 58
};

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
/*
 * Control how reservations are dealt with.
 *
 * RESERVE_FREE - freeing a reservation.
 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
 *   ENOSPC accounting
 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
 *   bytes_may_use as the ENOSPC accounting is done elsewhere
 */
enum {
	RESERVE_FREE = 0,
	RESERVE_ALLOC = 1,
	RESERVE_ALLOC_NO_ACCOUNT = 2,
};

74 75
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
76
			      u64 bytenr, u64 num_bytes, int alloc);
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extra_op);
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins);
J
Josef Bacik 已提交
96 97 98
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
			  u64 flags, int force);
99 100
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
101 102
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
103 104
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve);
J
Josef Bacik 已提交
105

J
Josef Bacik 已提交
106 107 108 109 110 111 112
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
	return cache->cached == BTRFS_CACHE_FINISHED;
}

J
Josef Bacik 已提交
113 114 115 116 117
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

118
static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
119 120 121 122 123 124
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
125 126 127
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
128
		kfree(cache->free_space_ctl);
129
		kfree(cache);
130
	}
131 132
}

J
Josef Bacik 已提交
133 134 135 136
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
137
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
205
	if (ret)
206
		btrfs_get_block_group(ret);
J
Josef Bacik 已提交
207 208 209 210 211
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

212 213
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
214
{
215 216 217 218 219 220 221
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
J
Josef Bacik 已提交
222

223 224 225 226
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
227

228 229 230 231 232 233 234
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
J
Josef Bacik 已提交
235 236
}

237 238
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
239 240 241 242 243 244
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

245 246 247 248 249
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
250
		BUG_ON(ret); /* -ENOMEM */
251 252
	}

J
Josef Bacik 已提交
253 254 255 256 257
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
258
		BUG_ON(ret); /* -ENOMEM */
259

J
Josef Bacik 已提交
260
		while (nr--) {
261
			cache->bytes_super += stripe_len;
262 263
			ret = add_excluded_extent(root, logical[nr],
						  stripe_len);
264
			BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
265
		}
266

J
Josef Bacik 已提交
267 268 269 270 271
		kfree(logical);
	}
	return 0;
}

272 273 274 275 276 277 278 279 280 281 282
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_STARTED) {
		spin_unlock(&cache->lock);
		return NULL;
	}

283 284 285
	/* We're loading it the fast way, so we don't have a caching_ctl. */
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

J
Josef Bacik 已提交
301 302 303 304 305
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
J
Josef Bacik 已提交
306
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
J
Josef Bacik 已提交
307 308
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
J
Josef Bacik 已提交
309
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
310 311 312
	int ret;

	while (start < end) {
313
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
314
					    &extent_start, &extent_end,
315
					    EXTENT_DIRTY | EXTENT_UPTODATE);
J
Josef Bacik 已提交
316 317 318
		if (ret)
			break;

319
		if (extent_start <= start) {
J
Josef Bacik 已提交
320 321 322
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
323
			total_added += size;
324 325
			ret = btrfs_add_free_space(block_group, start,
						   size);
326
			BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
327 328 329 330 331 332 333 334
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
335
		total_added += size;
336
		ret = btrfs_add_free_space(block_group, start, size);
337
		BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
338 339
	}

J
Josef Bacik 已提交
340
	return total_added;
J
Josef Bacik 已提交
341 342
}

343
static noinline void caching_thread(struct btrfs_work *work)
344
{
345 346 347 348
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_root *extent_root;
349
	struct btrfs_path *path;
350
	struct extent_buffer *leaf;
351
	struct btrfs_key key;
J
Josef Bacik 已提交
352
	u64 total_found = 0;
353 354 355
	u64 last = 0;
	u32 nritems;
	int ret = 0;
356

357 358 359 360 361
	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

362 363
	path = btrfs_alloc_path();
	if (!path)
364
		goto out;
365

J
Josef Bacik 已提交
366
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
367

368
	/*
J
Josef Bacik 已提交
369 370 371 372
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
373 374
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
375
	path->search_commit_root = 1;
J
Josef Bacik 已提交
376
	path->reada = 1;
J
Josef Bacik 已提交
377

Y
Yan Zheng 已提交
378
	key.objectid = last;
379
	key.offset = 0;
380
	key.type = BTRFS_EXTENT_ITEM_KEY;
381
again:
382
	mutex_lock(&caching_ctl->mutex);
383 384 385
	/* need to make sure the commit_root doesn't disappear */
	down_read(&fs_info->extent_commit_sem);

386
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
387
	if (ret < 0)
388
		goto err;
Y
Yan Zheng 已提交
389

390 391 392
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
393
	while (1) {
394
		if (btrfs_fs_closing(fs_info) > 1) {
395
			last = (u64)-1;
J
Josef Bacik 已提交
396
			break;
397
		}
J
Josef Bacik 已提交
398

399 400 401 402 403
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
404
				break;
J
Josef Bacik 已提交
405

406 407 408
			if (need_resched() ||
			    btrfs_next_leaf(extent_root, path)) {
				caching_ctl->progress = last;
C
Chris Mason 已提交
409
				btrfs_release_path(path);
410 411
				up_read(&fs_info->extent_commit_sem);
				mutex_unlock(&caching_ctl->mutex);
412
				cond_resched();
413 414 415 416 417
				goto again;
			}
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
418
		}
J
Josef Bacik 已提交
419

420 421
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
422
			continue;
423
		}
J
Josef Bacik 已提交
424

425
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
426
		    block_group->key.offset)
427
			break;
428

429
		if (key.type == BTRFS_EXTENT_ITEM_KEY) {
J
Josef Bacik 已提交
430 431 432
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
433
			last = key.objectid + key.offset;
J
Josef Bacik 已提交
434

435 436 437 438
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
J
Josef Bacik 已提交
439
		}
440 441
		path->slots[0]++;
	}
J
Josef Bacik 已提交
442
	ret = 0;
443

J
Josef Bacik 已提交
444 445 446
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
447
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
448 449

	spin_lock(&block_group->lock);
450
	block_group->caching_ctl = NULL;
J
Josef Bacik 已提交
451 452
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
453

454
err:
455
	btrfs_free_path(path);
456
	up_read(&fs_info->extent_commit_sem);
J
Josef Bacik 已提交
457

458 459 460
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
461
out:
462 463 464
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
465
	btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
466 467
}

468 469
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     struct btrfs_trans_handle *trans,
470
			     struct btrfs_root *root,
471
			     int load_cache_only)
J
Josef Bacik 已提交
472
{
473
	DEFINE_WAIT(wait);
474 475
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
476 477
	int ret = 0;

478
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
479 480
	if (!caching_ctl)
		return -ENOMEM;
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	atomic_set(&caching_ctl->count, 1);
	caching_ctl->work.func = caching_thread;

	spin_lock(&cache->lock);
	/*
	 * This should be a rare occasion, but this could happen I think in the
	 * case where one thread starts to load the space cache info, and then
	 * some other thread starts a transaction commit which tries to do an
	 * allocation while the other thread is still loading the space cache
	 * info.  The previous loop should have kept us from choosing this block
	 * group, but if we've moved to the state where we will wait on caching
	 * block groups we need to first check if we're doing a fast load here,
	 * so we can wait for it to finish, otherwise we could end up allocating
	 * from a block group who's cache gets evicted for one reason or
	 * another.
	 */
	while (cache->cached == BTRFS_CACHE_FAST) {
		struct btrfs_caching_control *ctl;

		ctl = cache->caching_ctl;
		atomic_inc(&ctl->count);
		prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&cache->lock);

		schedule();

		finish_wait(&ctl->wait, &wait);
		put_caching_control(ctl);
		spin_lock(&cache->lock);
	}

	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
		kfree(caching_ctl);
521
		return 0;
522 523 524 525 526
	}
	WARN_ON(cache->caching_ctl);
	cache->caching_ctl = caching_ctl;
	cache->cached = BTRFS_CACHE_FAST;
	spin_unlock(&cache->lock);
527

528 529
	/*
	 * We can't do the read from on-disk cache during a commit since we need
530 531 532
	 * to have the normal tree locking.  Also if we are currently trying to
	 * allocate blocks for the tree root we can't do the fast caching since
	 * we likely hold important locks.
533
	 */
534
	if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
535 536 537 538
		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
539
			cache->caching_ctl = NULL;
540 541 542
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
		} else {
543 544 545 546 547 548
			if (load_cache_only) {
				cache->caching_ctl = NULL;
				cache->cached = BTRFS_CACHE_NO;
			} else {
				cache->cached = BTRFS_CACHE_STARTED;
			}
549 550
		}
		spin_unlock(&cache->lock);
551
		wake_up(&caching_ctl->wait);
552
		if (ret == 1) {
553
			put_caching_control(caching_ctl);
554
			free_excluded_extents(fs_info->extent_root, cache);
555
			return 0;
556
		}
557 558 559 560 561 562 563 564 565 566 567 568 569 570
	} else {
		/*
		 * We are not going to do the fast caching, set cached to the
		 * appropriate value and wakeup any waiters.
		 */
		spin_lock(&cache->lock);
		if (load_cache_only) {
			cache->caching_ctl = NULL;
			cache->cached = BTRFS_CACHE_NO;
		} else {
			cache->cached = BTRFS_CACHE_STARTED;
		}
		spin_unlock(&cache->lock);
		wake_up(&caching_ctl->wait);
571 572
	}

573 574
	if (load_cache_only) {
		put_caching_control(caching_ctl);
575
		return 0;
J
Josef Bacik 已提交
576 577
	}

578
	down_write(&fs_info->extent_commit_sem);
579
	atomic_inc(&caching_ctl->count);
580 581 582
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
	up_write(&fs_info->extent_commit_sem);

583
	btrfs_get_block_group(cache);
584

585
	btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
J
Josef Bacik 已提交
586

587
	return ret;
588 589
}

J
Josef Bacik 已提交
590 591 592
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
593 594
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
595
{
J
Josef Bacik 已提交
596
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
597

J
Josef Bacik 已提交
598
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
599

J
Josef Bacik 已提交
600
	return cache;
C
Chris Mason 已提交
601 602
}

J
Josef Bacik 已提交
603
/*
604
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
605
 */
C
Chris Mason 已提交
606 607 608
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
609
{
J
Josef Bacik 已提交
610
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
611

J
Josef Bacik 已提交
612
	cache = block_group_cache_tree_search(info, bytenr, 1);
613

J
Josef Bacik 已提交
614
	return cache;
C
Chris Mason 已提交
615
}
616

J
Josef Bacik 已提交
617 618
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
619
{
J
Josef Bacik 已提交
620 621
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
622

623
	flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
624

625 626
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
627
		if (found->flags & flags) {
628
			rcu_read_unlock();
J
Josef Bacik 已提交
629
			return found;
630
		}
J
Josef Bacik 已提交
631
	}
632
	rcu_read_unlock();
J
Josef Bacik 已提交
633
	return NULL;
634 635
}

636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

651 652 653 654 655 656 657 658 659
static u64 div_factor(u64 num, int factor)
{
	if (factor == 10)
		return num;
	num *= factor;
	do_div(num, 10);
	return num;
}

660 661 662 663 664 665 666 667 668
static u64 div_factor_fine(u64 num, int factor)
{
	if (factor == 100)
		return num;
	num *= factor;
	do_div(num, 100);
	return num;
}

669 670
u64 btrfs_find_block_group(struct btrfs_root *root,
			   u64 search_start, u64 search_hint, int owner)
C
Chris Mason 已提交
671
{
672
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
673
	u64 used;
674 675
	u64 last = max(search_hint, search_start);
	u64 group_start = 0;
676
	int full_search = 0;
677
	int factor = 9;
C
Chris Mason 已提交
678
	int wrapped = 0;
679
again:
680 681
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
J
Josef Bacik 已提交
682 683
		if (!cache)
			break;
684

685
		spin_lock(&cache->lock);
686 687 688
		last = cache->key.objectid + cache->key.offset;
		used = btrfs_block_group_used(&cache->item);

689 690
		if ((full_search || !cache->ro) &&
		    block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
691
			if (used + cache->pinned + cache->reserved <
692 693
			    div_factor(cache->key.offset, factor)) {
				group_start = cache->key.objectid;
694
				spin_unlock(&cache->lock);
695
				btrfs_put_block_group(cache);
696 697
				goto found;
			}
698
		}
699
		spin_unlock(&cache->lock);
700
		btrfs_put_block_group(cache);
701
		cond_resched();
C
Chris Mason 已提交
702
	}
C
Chris Mason 已提交
703 704 705 706 707 708
	if (!wrapped) {
		last = search_start;
		wrapped = 1;
		goto again;
	}
	if (!full_search && factor < 10) {
C
Chris Mason 已提交
709
		last = search_start;
710
		full_search = 1;
C
Chris Mason 已提交
711
		factor = 10;
712 713
		goto again;
	}
C
Chris Mason 已提交
714
found:
715
	return group_start;
716
}
J
Josef Bacik 已提交
717

718
/* simple helper to search for an existing extent at a given offset */
Z
Zheng Yan 已提交
719
int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
720 721 722
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
723
	struct btrfs_path *path;
724

Z
Zheng Yan 已提交
725
	path = btrfs_alloc_path();
726 727 728
	if (!path)
		return -ENOMEM;

729 730 731 732 733
	key.objectid = start;
	key.offset = len;
	btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
734
	btrfs_free_path(path);
735 736 737
	return ret;
}

738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
/*
 * helper function to lookup reference count and flags of extent.
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
			     u64 num_bytes, u64 *refs, u64 *flags)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = num_bytes;
	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
again:
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

818
			btrfs_release_path(path);
819

820 821 822 823
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
			goto again;
		}
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

849 850 851 852 853 854 855 856 857 858 859 860 861 862
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
908 909 910
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
911
 * - different files inside a single subvolume
912 913
 * - different offsets inside a file (bookend extents in file.c)
 *
914
 * The extent ref structure for the implicit back refs has fields for:
915 916 917
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
918 919
 * - original offset in the file
 * - how many bookend extents
920
 *
921 922
 * The key offset for the implicit back refs is hash of the first
 * three fields.
923
 *
924
 * The extent ref structure for the full back refs has field for:
925
 *
926
 * - number of pointers in the tree leaf
927
 *
928 929
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
930
 *
931 932
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
933
 *
934
 *     (root_key.objectid, inode objectid, offset in file, 1)
935
 *
936 937
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
938
 *
939
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
940
 *
941
 * Btree extents can be referenced by:
942
 *
943
 * - Different subvolumes
944
 *
945 946 947 948
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
949
 *
950 951 952
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
953
 */
Z
Zheng Yan 已提交
954

955 956 957 958 959
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
960
{
961 962 963 964 965
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
966
	struct btrfs_key key;
967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
986
				BUG_ON(ret > 0); /* Corruption */
987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
1002
	btrfs_release_path(path);
1003 1004 1005 1006 1007 1008 1009 1010 1011

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
1012
	BUG_ON(ret); /* Corruption */
1013

1014
	btrfs_extend_item(trans, root, path, new_size);
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
1044
	high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1045
	lenum = cpu_to_le64(owner);
1046
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1047
	lenum = cpu_to_le64(offset);
1048
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1081
	struct extent_buffer *leaf;
1082
	u32 nritems;
1083
	int ret;
1084 1085
	int recow;
	int err = -ENOENT;
1086

Z
Zheng Yan 已提交
1087
	key.objectid = bytenr;
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1103

1104 1105 1106 1107 1108
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1109
		btrfs_release_path(path);
1110 1111 1112 1113 1114 1115 1116 1117 1118
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1119 1120 1121
	}

	leaf = path->nodes[0];
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1147
				btrfs_release_path(path);
1148 1149 1150 1151 1152 1153
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1154
	}
1155 1156
fail:
	return err;
Z
Zheng Yan 已提交
1157 1158
}

1159 1160 1161 1162 1163 1164
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1165 1166 1167
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1168
	u32 size;
Z
Zheng Yan 已提交
1169 1170
	u32 num_refs;
	int ret;
1171 1172

	key.objectid = bytenr;
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1183

1184 1185 1186 1187 1188 1189 1190
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1191
		ref = btrfs_item_ptr(leaf, path->slots[0],
1192 1193 1194 1195 1196 1197 1198
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1199
		}
1200 1201 1202 1203 1204 1205 1206 1207
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1208
			btrfs_release_path(path);
1209 1210 1211 1212 1213
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1214

1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1229 1230
		}
	}
1231 1232 1233
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1234
	btrfs_release_path(path);
1235
	return ret;
1236 1237
}

1238 1239 1240 1241
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   int refs_to_drop)
Z
Zheng Yan 已提交
1242
{
1243 1244 1245
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1246
	struct extent_buffer *leaf;
1247
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1248 1249 1250
	int ret = 0;

	leaf = path->nodes[0];
1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1272 1273
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1274

Z
Zheng Yan 已提交
1275 1276 1277
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
	} else {
1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1290 1291 1292 1293 1294
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1295 1296 1297
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1298
{
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1330
#endif
1331 1332 1333 1334 1335
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1336

1337 1338 1339 1340 1341
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1342
{
1343
	struct btrfs_key key;
1344 1345
	int ret;

1346 1347 1348 1349 1350 1351 1352
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1353 1354
	}

1355 1356 1357 1358 1359
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1360
		btrfs_release_path(path);
1361 1362 1363 1364 1365
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1366
#endif
1367
	return ret;
1368 1369
}

1370 1371 1372 1373 1374
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1375
{
1376
	struct btrfs_key key;
Z
Zheng Yan 已提交
1377 1378
	int ret;

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1389
	btrfs_release_path(path);
Z
Zheng Yan 已提交
1390 1391 1392
	return ret;
}

1393
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1394
{
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1408
}
1409

1410 1411
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1412

C
Chris Mason 已提交
1413
{
1414
	for (; level < BTRFS_MAX_LEVEL; level++) {
1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1430

1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1466

1467
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1468
	key.type = BTRFS_EXTENT_ITEM_KEY;
1469
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1470

1471 1472 1473
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1474
		path->keep_locks = 1;
1475 1476 1477
	} else
		extra_size = -1;
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1478
	if (ret < 0) {
1479 1480 1481
		err = ret;
		goto out;
	}
1482 1483 1484 1485 1486
	if (ret && !insert) {
		err = -ENOENT;
		goto out;
	}
	BUG_ON(ret); /* Corruption */
1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	} else {
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1580 1581
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1582
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1583 1584 1585 1586 1587 1588
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1589
	if (insert) {
1590 1591 1592 1593 1594 1595 1596 1597 1598 1599
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
1600 1601 1602 1603 1604 1605 1606
void setup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

1624
	btrfs_extend_item(trans, root, path, size);
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1674
		return ret;
1675

1676
	btrfs_release_path(path);
1677 1678 1679 1680 1681 1682 1683 1684
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1685
	}
1686 1687
	return ret;
}
Z
Zheng Yan 已提交
1688

1689 1690 1691 1692
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
1693 1694 1695 1696 1697 1698
void update_inline_extent_backref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  struct btrfs_extent_inline_ref *iref,
				  int refs_to_mod,
				  struct btrfs_delayed_extent_op *extent_op)
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1731
	}
Z
Zheng Yan 已提交
1732

1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
1750
		btrfs_truncate_item(trans, root, path, item_size, 1);
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771
	}
	btrfs_mark_buffer_dirty(leaf);
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1772 1773
		update_inline_extent_backref(trans, root, path, iref,
					     refs_to_add, extent_op);
1774
	} else if (ret == -ENOENT) {
1775 1776 1777 1778
		setup_inline_extent_backref(trans, root, path, iref, parent,
					    root_objectid, owner, offset,
					    refs_to_add, extent_op);
		ret = 0;
1779
	}
1780 1781
	return ret;
}
Z
Zheng Yan 已提交
1782

1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1801

1802 1803 1804 1805 1806 1807
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_drop, int is_data)
{
1808
	int ret = 0;
1809

1810 1811
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
1812 1813
		update_inline_extent_backref(trans, root, path, iref,
					     -refs_to_drop, NULL);
1814 1815 1816 1817 1818 1819 1820 1821
	} else if (is_data) {
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
	} else {
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1822
static int btrfs_issue_discard(struct block_device *bdev,
1823 1824
				u64 start, u64 len)
{
1825
	return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1826 1827 1828
}

static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1829
				u64 num_bytes, u64 *actual_bytes)
1830 1831
{
	int ret;
1832
	u64 discarded_bytes = 0;
1833
	struct btrfs_bio *bbio = NULL;
1834

C
Christoph Hellwig 已提交
1835

1836
	/* Tell the block device(s) that the sectors can be discarded */
1837
	ret = btrfs_map_block(&root->fs_info->mapping_tree, REQ_DISCARD,
1838
			      bytenr, &num_bytes, &bbio, 0);
1839
	/* Error condition is -ENOMEM */
1840
	if (!ret) {
1841
		struct btrfs_bio_stripe *stripe = bbio->stripes;
1842 1843 1844
		int i;


1845
		for (i = 0; i < bbio->num_stripes; i++, stripe++) {
1846 1847 1848
			if (!stripe->dev->can_discard)
				continue;

1849 1850 1851 1852 1853 1854
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
						  stripe->length);
			if (!ret)
				discarded_bytes += stripe->length;
			else if (ret != -EOPNOTSUPP)
1855
				break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
1856 1857 1858 1859 1860 1861 1862

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
1863
		}
1864
		kfree(bbio);
1865
	}
1866 1867 1868 1869

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

1870 1871 1872 1873

	return ret;
}

1874
/* Can return -ENOMEM */
1875 1876 1877
int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
A
Arne Jansen 已提交
1878
			 u64 root_objectid, u64 owner, u64 offset, int for_cow)
1879 1880
{
	int ret;
A
Arne Jansen 已提交
1881 1882
	struct btrfs_fs_info *fs_info = root->fs_info;

1883 1884 1885 1886
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
1887 1888
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
1889
					parent, root_objectid, (int)owner,
A
Arne Jansen 已提交
1890
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1891
	} else {
A
Arne Jansen 已提交
1892 1893
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
					num_bytes,
1894
					parent, root_objectid, owner, offset,
A
Arne Jansen 已提交
1895
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
	u64 refs;
	int ret;
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
	ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
					   path, bytenr, num_bytes, parent,
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
	if (ret == 0)
		goto out;

	if (ret != -EAGAIN) {
		err = ret;
		goto out;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
1939

1940
	btrfs_mark_buffer_dirty(leaf);
1941
	btrfs_release_path(path);
1942 1943

	path->reada = 1;
1944 1945
	path->leave_spinning = 1;

1946 1947
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
1948 1949
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
1950 1951
	if (ret)
		btrfs_abort_transaction(trans, root, ret);
1952
out:
1953
	btrfs_free_path(path);
1954
	return err;
1955 1956
}

1957 1958 1959 1960 1961
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
1962
{
1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		if (extent_op) {
			BUG_ON(extent_op->update_key);
			flags |= extent_op->flags_to_set;
		}
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
					     extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
					  extent_op);
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
2035
	int ret;
2036 2037
	int err = 0;

2038 2039 2040
	if (trans->aborted)
		return 0;

2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = node->num_bytes;

	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
		err = -EIO;
		goto out;
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2079

2080 2081 2082 2083
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2084 2085
}

2086 2087 2088 2089 2090
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2091 2092
{
	int ret = 0;
2093 2094 2095 2096
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2097

2098 2099 2100
	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;
2101

2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127
	ref = btrfs_delayed_node_to_tree_ref(node);
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		BUG_ON(!extent_op || !extent_op->update_flags ||
		       !extent_op->update_key);
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
						ref->level, &ins);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
					     ref->level, 0, 1, extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
					  ref->level, 0, 1, extent_op);
	} else {
		BUG();
	}
2128 2129 2130 2131
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2132 2133 2134 2135 2136
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2137
{
2138 2139 2140 2141 2142
	int ret = 0;

	if (trans->aborted)
		return 0;

2143
	if (btrfs_delayed_ref_is_head(node)) {
2144 2145 2146 2147 2148 2149 2150
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2151 2152
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2153
		if (insert_reserved) {
2154 2155
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2156 2157 2158 2159 2160
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
			}
2161 2162
		}
		mutex_unlock(&head->mutex);
2163
		return ret;
2164 2165
	}

2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_node *ref;
	int action = BTRFS_ADD_DELAYED_REF;
again:
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
	node = rb_prev(&head->node.rb_node);
	while (1) {
		if (!node)
			break;
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
		if (ref->bytenr != head->node.bytenr)
			break;
2199
		if (ref->action == action)
2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
			return ref;
		node = rb_prev(node);
	}
	if (action == BTRFS_ADD_DELAYED_REF) {
		action = BTRFS_DROP_DELAYED_REF;
		goto again;
	}
	return NULL;
}

2210 2211 2212 2213
/*
 * Returns 0 on success or if called with an already aborted transaction.
 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
 */
2214 2215 2216
static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root,
				       struct list_head *cluster)
2217 2218 2219 2220
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2221
	struct btrfs_delayed_extent_op *extent_op;
2222
	struct btrfs_fs_info *fs_info = root->fs_info;
2223
	int ret;
2224
	int count = 0;
2225 2226 2227 2228 2229
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2230 2231
			/* pick a new head ref from the cluster list */
			if (list_empty(cluster))
2232 2233
				break;

2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250
			locked_ref = list_entry(cluster->next,
				     struct btrfs_delayed_ref_head, cluster);

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);

			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2251 2252
			}
		}
2253

2254 2255 2256 2257 2258 2259 2260
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);

		if (ref && ref->seq &&
2261
		    btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) {
2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275
			/*
			 * there are still refs with lower seq numbers in the
			 * process of being added. Don't run this ref yet.
			 */
			list_del_init(&locked_ref->cluster);
			mutex_unlock(&locked_ref->mutex);
			locked_ref = NULL;
			delayed_refs->num_heads_ready++;
			spin_unlock(&delayed_refs->lock);
			cond_resched();
			spin_lock(&delayed_refs->lock);
			continue;
		}

2276 2277 2278 2279 2280 2281
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2282

2283 2284 2285
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2286 2287 2288 2289 2290 2291
		if (!ref) {
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304

			if (extent_op && must_insert_reserved) {
				kfree(extent_op);
				extent_op = NULL;
			}

			if (extent_op) {
				spin_unlock(&delayed_refs->lock);

				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
				kfree(extent_op);

2305 2306
				if (ret) {
					printk(KERN_DEBUG "btrfs: run_delayed_extent_op returned %d\n", ret);
2307
					spin_lock(&delayed_refs->lock);
2308 2309 2310
					return ret;
				}

2311
				goto next;
2312 2313
			}

2314
			list_del_init(&locked_ref->cluster);
2315 2316
			locked_ref = NULL;
		}
C
Chris Mason 已提交
2317

2318 2319 2320
		ref->in_tree = 0;
		rb_erase(&ref->rb_node, &delayed_refs->root);
		delayed_refs->num_entries--;
2321 2322 2323 2324 2325 2326
		/*
		 * we modified num_entries, but as we're currently running
		 * delayed refs, skip
		 *     wake_up(&delayed_refs->seq_wait);
		 * here.
		 */
2327
		spin_unlock(&delayed_refs->lock);
2328

2329
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2330
					  must_insert_reserved);
2331

2332 2333
		btrfs_put_delayed_ref(ref);
		kfree(extent_op);
2334
		count++;
2335 2336 2337

		if (ret) {
			printk(KERN_DEBUG "btrfs: run_one_delayed_ref returned %d\n", ret);
2338
			spin_lock(&delayed_refs->lock);
2339 2340 2341
			return ret;
		}

2342
next:
2343
		do_chunk_alloc(trans, fs_info->extent_root,
2344 2345 2346
			       2 * 1024 * 1024,
			       btrfs_get_alloc_profile(root, 0),
			       CHUNK_ALLOC_NO_FORCE);
2347 2348 2349 2350 2351 2352
		cond_resched();
		spin_lock(&delayed_refs->lock);
	}
	return count;
}

2353 2354
static void wait_for_more_refs(struct btrfs_fs_info *fs_info,
			       struct btrfs_delayed_ref_root *delayed_refs,
2355 2356
			       unsigned long num_refs,
			       struct list_head *first_seq)
2357 2358 2359 2360
{
	spin_unlock(&delayed_refs->lock);
	pr_debug("waiting for more refs (num %ld, first %p)\n",
		 num_refs, first_seq);
2361
	wait_event(fs_info->tree_mod_seq_wait,
2362
		   num_refs != delayed_refs->num_entries ||
2363
		   fs_info->tree_mod_seq_list.next != first_seq);
2364
	pr_debug("done waiting for more refs (num %ld, first %p)\n",
2365
		 delayed_refs->num_entries, fs_info->tree_mod_seq_list.next);
2366 2367 2368
	spin_lock(&delayed_refs->lock);
}

2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411
#ifdef SCRAMBLE_DELAYED_REFS
/*
 * Normally delayed refs get processed in ascending bytenr order. This
 * correlates in most cases to the order added. To expose dependencies on this
 * order, we start to process the tree in the middle instead of the beginning
 */
static u64 find_middle(struct rb_root *root)
{
	struct rb_node *n = root->rb_node;
	struct btrfs_delayed_ref_node *entry;
	int alt = 1;
	u64 middle;
	u64 first = 0, last = 0;

	n = rb_first(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		first = entry->bytenr;
	}
	n = rb_last(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		last = entry->bytenr;
	}
	n = root->rb_node;

	while (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		WARN_ON(!entry->in_tree);

		middle = entry->bytenr;

		if (alt)
			n = n->rb_left;
		else
			n = n->rb_right;

		alt = 1 - alt;
	}
	return middle;
}
#endif

2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445
int btrfs_delayed_refs_qgroup_accounting(struct btrfs_trans_handle *trans,
					 struct btrfs_fs_info *fs_info)
{
	struct qgroup_update *qgroup_update;
	int ret = 0;

	if (list_empty(&trans->qgroup_ref_list) !=
	    !trans->delayed_ref_elem.seq) {
		/* list without seq or seq without list */
		printk(KERN_ERR "btrfs: qgroup accounting update error, list is%s empty, seq is %llu\n",
			list_empty(&trans->qgroup_ref_list) ? "" : " not",
			trans->delayed_ref_elem.seq);
		BUG();
	}

	if (!trans->delayed_ref_elem.seq)
		return 0;

	while (!list_empty(&trans->qgroup_ref_list)) {
		qgroup_update = list_first_entry(&trans->qgroup_ref_list,
						 struct qgroup_update, list);
		list_del(&qgroup_update->list);
		if (!ret)
			ret = btrfs_qgroup_account_ref(
					trans, fs_info, qgroup_update->node,
					qgroup_update->extent_op);
		kfree(qgroup_update);
	}

	btrfs_put_tree_mod_seq(fs_info, &trans->delayed_ref_elem);

	return ret;
}

2446 2447 2448 2449 2450 2451
/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
2452 2453 2454
 *
 * Returns 0 on success or if called with an aborted transaction
 * Returns <0 on error and aborts the transaction
2455 2456 2457 2458 2459 2460 2461 2462
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct list_head cluster;
2463
	struct list_head *first_seq = NULL;
2464
	int ret;
2465
	u64 delayed_start;
2466 2467
	int run_all = count == (unsigned long)-1;
	int run_most = 0;
2468 2469
	unsigned long num_refs = 0;
	int consider_waiting;
2470

2471 2472 2473 2474
	/* We'll clean this up in btrfs_cleanup_transaction */
	if (trans->aborted)
		return 0;

2475 2476 2477
	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

2478 2479 2480 2481
	do_chunk_alloc(trans, root->fs_info->extent_root,
		       2 * 1024 * 1024, btrfs_get_alloc_profile(root, 0),
		       CHUNK_ALLOC_NO_FORCE);

2482 2483
	btrfs_delayed_refs_qgroup_accounting(trans, root->fs_info);

2484 2485 2486
	delayed_refs = &trans->transaction->delayed_refs;
	INIT_LIST_HEAD(&cluster);
again:
2487
	consider_waiting = 0;
2488
	spin_lock(&delayed_refs->lock);
2489

2490 2491 2492 2493
#ifdef SCRAMBLE_DELAYED_REFS
	delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
#endif

2494 2495 2496 2497 2498 2499 2500 2501
	if (count == 0) {
		count = delayed_refs->num_entries * 2;
		run_most = 1;
	}
	while (1) {
		if (!(run_all || run_most) &&
		    delayed_refs->num_heads_ready < 64)
			break;
2502

2503
		/*
2504 2505 2506 2507
		 * go find something we can process in the rbtree.  We start at
		 * the beginning of the tree, and then build a cluster
		 * of refs to process starting at the first one we are able to
		 * lock
2508
		 */
2509
		delayed_start = delayed_refs->run_delayed_start;
2510 2511 2512
		ret = btrfs_find_ref_cluster(trans, &cluster,
					     delayed_refs->run_delayed_start);
		if (ret)
2513 2514
			break;

2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525
		if (delayed_start >= delayed_refs->run_delayed_start) {
			if (consider_waiting == 0) {
				/*
				 * btrfs_find_ref_cluster looped. let's do one
				 * more cycle. if we don't run any delayed ref
				 * during that cycle (because we can't because
				 * all of them are blocked) and if the number of
				 * refs doesn't change, we avoid busy waiting.
				 */
				consider_waiting = 1;
				num_refs = delayed_refs->num_entries;
2526
				first_seq = root->fs_info->tree_mod_seq_list.next;
2527
			} else {
2528
				wait_for_more_refs(root->fs_info, delayed_refs,
2529
						   num_refs, first_seq);
2530 2531 2532 2533 2534 2535 2536 2537 2538 2539
				/*
				 * after waiting, things have changed. we
				 * dropped the lock and someone else might have
				 * run some refs, built new clusters and so on.
				 * therefore, we restart staleness detection.
				 */
				consider_waiting = 0;
			}
		}

2540
		ret = run_clustered_refs(trans, root, &cluster);
2541 2542 2543 2544 2545
		if (ret < 0) {
			spin_unlock(&delayed_refs->lock);
			btrfs_abort_transaction(trans, root, ret);
			return ret;
		}
2546 2547 2548 2549 2550

		count -= min_t(unsigned long, ret, count);

		if (count == 0)
			break;
2551 2552 2553 2554 2555

		if (ret || delayed_refs->run_delayed_start == 0) {
			/* refs were run, let's reset staleness detection */
			consider_waiting = 0;
		}
2556
	}
2557

2558 2559
	if (run_all) {
		node = rb_first(&delayed_refs->root);
2560
		if (!node)
2561
			goto out;
2562
		count = (unsigned long)-1;
2563

2564 2565 2566 2567 2568
		while (node) {
			ref = rb_entry(node, struct btrfs_delayed_ref_node,
				       rb_node);
			if (btrfs_delayed_ref_is_head(ref)) {
				struct btrfs_delayed_ref_head *head;
2569

2570 2571 2572 2573
				head = btrfs_delayed_node_to_head(ref);
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2574 2575 2576 2577
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
2578 2579 2580 2581
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2582
				cond_resched();
2583 2584 2585 2586 2587 2588 2589
				goto again;
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
		schedule_timeout(1);
		goto again;
2590
	}
2591
out:
2592
	spin_unlock(&delayed_refs->lock);
2593
	assert_qgroups_uptodate(trans);
2594 2595 2596
	return 0;
}

2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
				int is_data)
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

	extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;

A
Arne Jansen 已提交
2614 2615
	ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
					  num_bytes, extent_op);
2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643
	if (ret)
		kfree(extent_op);
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	ret = -ENOENT;
	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

2644
		btrfs_release_path(path);
2645

2646 2647 2648 2649
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out_unlock;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	if (ref->bytenr != bytenr)
		goto out_unlock;

	ret = 1;
	if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
		goto out_unlock;

	data_ref = btrfs_delayed_node_to_data_ref(ref);

	node = rb_prev(node);
	if (node) {
2673 2674
		int seq = ref->seq;

2675
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2676
		if (ref->bytenr == bytenr && ref->seq == seq)
2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695
			goto out_unlock;
	}

	if (data_ref->root != root->root_key.objectid ||
	    data_ref->objectid != objectid || data_ref->offset != offset)
		goto out_unlock;

	ret = 0;
out_unlock:
	mutex_unlock(&head->mutex);
out:
	spin_unlock(&delayed_refs->lock);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2696 2697
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2698
	struct extent_buffer *leaf;
2699 2700 2701
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2702
	struct btrfs_key key;
2703
	u32 item_size;
2704
	int ret;
2705

2706
	key.objectid = bytenr;
Z
Zheng Yan 已提交
2707
	key.offset = (u64)-1;
2708
	key.type = BTRFS_EXTENT_ITEM_KEY;
2709 2710 2711 2712

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
2713
	BUG_ON(ret == 0); /* Corruption */
Y
Yan Zheng 已提交
2714 2715 2716

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
2717
		goto out;
2718

Z
Zheng Yan 已提交
2719
	path->slots[0]--;
2720
	leaf = path->nodes[0];
2721
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2722

2723
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2724
		goto out;
2725

2726 2727 2728 2729 2730 2731 2732 2733 2734
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2735

2736 2737 2738
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2739

2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
2779
			goto out;
Y
Yan Zheng 已提交
2780

2781 2782 2783 2784 2785 2786 2787
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
2788
	}
2789 2790 2791

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
2792
out:
Y
Yan Zheng 已提交
2793
	btrfs_free_path(path);
2794 2795
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
2796
	return ret;
2797
}
C
Chris Mason 已提交
2798

2799
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2800
			   struct btrfs_root *root,
2801
			   struct extent_buffer *buf,
A
Arne Jansen 已提交
2802
			   int full_backref, int inc, int for_cow)
Z
Zheng Yan 已提交
2803 2804
{
	u64 bytenr;
2805 2806
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
2807 2808 2809 2810 2811 2812 2813 2814
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
A
Arne Jansen 已提交
2815
			    u64, u64, u64, u64, u64, u64, int);
Z
Zheng Yan 已提交
2816 2817 2818 2819 2820

	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

2821 2822
	if (!root->ref_cows && level == 0)
		return 0;
Z
Zheng Yan 已提交
2823

2824 2825 2826 2827
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
2828

2829 2830 2831 2832 2833 2834
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
2835
		if (level == 0) {
2836
			btrfs_item_key_to_cpu(buf, &key, i);
Z
Zheng Yan 已提交
2837 2838
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
2839
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
2840 2841 2842 2843 2844 2845 2846
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
2847 2848 2849 2850 2851

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
A
Arne Jansen 已提交
2852
					   key.offset, for_cow);
Z
Zheng Yan 已提交
2853 2854 2855
			if (ret)
				goto fail;
		} else {
2856 2857 2858
			bytenr = btrfs_node_blockptr(buf, i);
			num_bytes = btrfs_level_size(root, level - 1);
			ret = process_func(trans, root, bytenr, num_bytes,
A
Arne Jansen 已提交
2859 2860
					   parent, ref_root, level - 1, 0,
					   for_cow);
Z
Zheng Yan 已提交
2861 2862 2863 2864 2865 2866
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
2867 2868 2869 2870
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
A
Arne Jansen 已提交
2871
		  struct extent_buffer *buf, int full_backref, int for_cow)
2872
{
A
Arne Jansen 已提交
2873
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1, for_cow);
2874 2875 2876
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
A
Arne Jansen 已提交
2877
		  struct extent_buffer *buf, int full_backref, int for_cow)
2878
{
A
Arne Jansen 已提交
2879
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0, for_cow);
Z
Zheng Yan 已提交
2880 2881
}

C
Chris Mason 已提交
2882 2883 2884 2885 2886 2887 2888
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2889 2890
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
2891 2892

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2893 2894
	if (ret < 0)
		goto fail;
2895
	BUG_ON(ret); /* Corruption */
2896 2897 2898 2899 2900

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
2901
	btrfs_release_path(path);
2902
fail:
2903 2904
	if (ret) {
		btrfs_abort_transaction(trans, root, ret);
C
Chris Mason 已提交
2905
		return ret;
2906
	}
C
Chris Mason 已提交
2907 2908 2909 2910
	return 0;

}

2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
	spin_lock(&root->fs_info->block_group_cache_lock);
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
2922
		btrfs_get_block_group(cache);
2923 2924 2925 2926 2927 2928
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

2929 2930 2931 2932 2933 2934 2935
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
2936
	int dcs = BTRFS_DC_ERROR;
2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955
	int num_pages = 0;
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
2956
		btrfs_release_path(path);
2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

2973 2974 2975 2976 2977 2978 2979
	/* We've already setup this transaction, go ahead and exit */
	if (block_group->cache_generation == trans->transid &&
	    i_size_read(inode)) {
		dcs = BTRFS_DC_SETUP;
		goto out_put;
	}

2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996
	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
		ret = btrfs_truncate_free_space_cache(root, trans, path,
						      inode);
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
2997 2998 2999 3000 3001 3002 3003
	if (block_group->cached != BTRFS_CACHE_FINISHED ||
	    !btrfs_test_opt(root, SPACE_CACHE)) {
		/*
		 * don't bother trying to write stuff out _if_
		 * a) we're not cached,
		 * b) we're with nospace_cache mount option.
		 */
3004
		dcs = BTRFS_DC_WRITTEN;
3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

	num_pages = (int)div64_u64(block_group->key.offset, 1024 * 1024 * 1024);
	if (!num_pages)
		num_pages = 1;

	/*
	 * Just to make absolutely sure we have enough space, we're going to
	 * preallocate 12 pages worth of space for each block group.  In
	 * practice we ought to use at most 8, but we need extra space so we can
	 * add our header and have a terminator between the extents and the
	 * bitmaps.
	 */
	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
3031 3032
	if (!ret)
		dcs = BTRFS_DC_SETUP;
3033
	btrfs_free_reserved_data_space(inode, num_pages);
3034

3035 3036 3037
out_put:
	iput(inode);
out_free:
3038
	btrfs_release_path(path);
3039 3040
out:
	spin_lock(&block_group->lock);
3041
	if (!ret && dcs == BTRFS_DC_SETUP)
3042
		block_group->cache_generation = trans->transid;
3043
	block_group->disk_cache_state = dcs;
3044 3045 3046 3047 3048
	spin_unlock(&block_group->lock);

	return ret;
}

3049 3050
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
C
Chris Mason 已提交
3051
{
3052
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
3053 3054
	int err = 0;
	struct btrfs_path *path;
3055
	u64 last = 0;
C
Chris Mason 已提交
3056 3057 3058 3059 3060

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079
again:
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			if (cache->disk_cache_state == BTRFS_DC_CLEAR)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
		err = cache_save_setup(cache, trans, path);
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
3080
	while (1) {
3081 3082 3083
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3084 3085
			if (err) /* File system offline */
				goto out;
J
Josef Bacik 已提交
3086
		}
3087

3088 3089
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
3090 3091 3092 3093 3094
			if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
				btrfs_put_block_group(cache);
				goto again;
			}

3095 3096 3097 3098 3099 3100 3101 3102 3103 3104
			if (cache->dirty)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
J
Josef Bacik 已提交
3105

J
Josef Bacik 已提交
3106 3107
		if (cache->disk_cache_state == BTRFS_DC_SETUP)
			cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
3108
		cache->dirty = 0;
3109
		last = cache->key.objectid + cache->key.offset;
J
Josef Bacik 已提交
3110

3111
		err = write_one_cache_group(trans, root, path, cache);
3112 3113 3114
		if (err) /* File system offline */
			goto out;

3115
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
3116
	}
3117

J
Josef Bacik 已提交
3118 3119 3120 3121 3122 3123 3124 3125 3126
	while (1) {
		/*
		 * I don't think this is needed since we're just marking our
		 * preallocated extent as written, but just in case it can't
		 * hurt.
		 */
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3127 3128
			if (err) /* File system offline */
				goto out;
J
Josef Bacik 已提交
3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152
		}

		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			/*
			 * Really this shouldn't happen, but it could if we
			 * couldn't write the entire preallocated extent and
			 * splitting the extent resulted in a new block.
			 */
			if (cache->dirty) {
				btrfs_put_block_group(cache);
				goto again;
			}
			if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

3153
		err = btrfs_write_out_cache(root, trans, cache, path);
J
Josef Bacik 已提交
3154 3155 3156 3157 3158

		/*
		 * If we didn't have an error then the cache state is still
		 * NEED_WRITE, so we can set it to WRITTEN.
		 */
3159
		if (!err && cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
J
Josef Bacik 已提交
3160 3161 3162 3163
			cache->disk_cache_state = BTRFS_DC_WRITTEN;
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}
3164
out:
J
Josef Bacik 已提交
3165

C
Chris Mason 已提交
3166
	btrfs_free_path(path);
3167
	return err;
C
Chris Mason 已提交
3168 3169
}

3170 3171 3172 3173 3174 3175 3176 3177 3178
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
3179
		btrfs_put_block_group(block_group);
3180 3181 3182
	return readonly;
}

3183 3184 3185 3186 3187
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
3188 3189 3190 3191 3192 3193 3194 3195
	int i;
	int factor;

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
3196 3197 3198

	found = __find_space_info(info, flags);
	if (found) {
3199
		spin_lock(&found->lock);
3200
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
3201
		found->disk_total += total_bytes * factor;
3202
		found->bytes_used += bytes_used;
3203
		found->disk_used += bytes_used * factor;
3204
		found->full = 0;
3205
		spin_unlock(&found->lock);
3206 3207 3208
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
3209
	found = kzalloc(sizeof(*found), GFP_NOFS);
3210 3211 3212
	if (!found)
		return -ENOMEM;

3213 3214
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		INIT_LIST_HEAD(&found->block_groups[i]);
3215
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
3216
	spin_lock_init(&found->lock);
3217
	found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3218
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
3219
	found->disk_total = total_bytes * factor;
3220
	found->bytes_used = bytes_used;
3221
	found->disk_used = bytes_used * factor;
3222
	found->bytes_pinned = 0;
3223
	found->bytes_reserved = 0;
Y
Yan Zheng 已提交
3224
	found->bytes_readonly = 0;
3225
	found->bytes_may_use = 0;
3226
	found->full = 0;
3227
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3228
	found->chunk_alloc = 0;
3229 3230
	found->flush = 0;
	init_waitqueue_head(&found->wait);
3231
	*space_info = found;
3232
	list_add_rcu(&found->list, &info->space_info);
3233 3234
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		info->data_sinfo = found;
3235 3236 3237
	return 0;
}

3238 3239
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
3240 3241
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
3242 3243 3244 3245 3246 3247 3248

	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits |= extra_flags;
3249
}
3250

3251 3252 3253
/*
 * returns target flags in extended format or 0 if restripe for this
 * chunk_type is not in progress
3254 3255
 *
 * should be called with either volume_mutex or balance_lock held
3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278
 */
static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
{
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
	u64 target = 0;

	if (!bctl)
		return 0;

	if (flags & BTRFS_BLOCK_GROUP_DATA &&
	    bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
	} else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
		   bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
	} else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
		   bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
	}

	return target;
}

3279 3280 3281
/*
 * @flags: available profiles in extended format (see ctree.h)
 *
3282 3283 3284
 * Returns reduced profile in chunk format.  If profile changing is in
 * progress (either running or paused) picks the target profile (if it's
 * already available), otherwise falls back to plain reducing.
3285
 */
Y
Yan Zheng 已提交
3286
u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3287
{
3288 3289 3290 3291 3292 3293 3294
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	u64 num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;
3295
	u64 target;
3296

3297 3298 3299 3300
	/*
	 * see if restripe for this chunk_type is in progress, if so
	 * try to reduce to the target profile
	 */
3301
	spin_lock(&root->fs_info->balance_lock);
3302 3303 3304 3305
	target = get_restripe_target(root->fs_info, flags);
	if (target) {
		/* pick target profile only if it's already available */
		if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
3306
			spin_unlock(&root->fs_info->balance_lock);
3307
			return extended_to_chunk(target);
3308 3309 3310 3311
		}
	}
	spin_unlock(&root->fs_info->balance_lock);

3312 3313 3314 3315 3316
	if (num_devices == 1)
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

3317 3318
	if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
	    (flags & (BTRFS_BLOCK_GROUP_RAID1 |
3319
		      BTRFS_BLOCK_GROUP_RAID10))) {
3320
		flags &= ~BTRFS_BLOCK_GROUP_DUP;
3321
	}
3322 3323

	if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
3324
	    (flags & BTRFS_BLOCK_GROUP_RAID10)) {
3325
		flags &= ~BTRFS_BLOCK_GROUP_RAID1;
3326
	}
3327 3328 3329 3330

	if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
	    ((flags & BTRFS_BLOCK_GROUP_RAID1) |
	     (flags & BTRFS_BLOCK_GROUP_RAID10) |
3331
	     (flags & BTRFS_BLOCK_GROUP_DUP))) {
3332
		flags &= ~BTRFS_BLOCK_GROUP_RAID0;
3333 3334
	}

3335
	return extended_to_chunk(flags);
3336 3337
}

3338
static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
J
Josef Bacik 已提交
3339
{
3340
	if (flags & BTRFS_BLOCK_GROUP_DATA)
3341
		flags |= root->fs_info->avail_data_alloc_bits;
3342
	else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3343
		flags |= root->fs_info->avail_system_alloc_bits;
3344
	else if (flags & BTRFS_BLOCK_GROUP_METADATA)
3345 3346
		flags |= root->fs_info->avail_metadata_alloc_bits;

3347
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
3348 3349
}

3350
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
3351
{
3352
	u64 flags;
J
Josef Bacik 已提交
3353

3354 3355 3356 3357
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
3358
	else
3359
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
3360

3361
	return get_alloc_profile(root, flags);
J
Josef Bacik 已提交
3362
}
J
Josef Bacik 已提交
3363

J
Josef Bacik 已提交
3364 3365 3366 3367
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3368
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
3369 3370
{
	struct btrfs_space_info *data_sinfo;
3371
	struct btrfs_root *root = BTRFS_I(inode)->root;
3372
	struct btrfs_fs_info *fs_info = root->fs_info;
3373
	u64 used;
3374
	int ret = 0, committed = 0, alloc_chunk = 1;
J
Josef Bacik 已提交
3375 3376 3377 3378

	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);

3379 3380
	if (root == root->fs_info->tree_root ||
	    BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3381 3382 3383 3384
		alloc_chunk = 0;
		committed = 1;
	}

3385
	data_sinfo = fs_info->data_sinfo;
C
Chris Mason 已提交
3386 3387
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
3388

J
Josef Bacik 已提交
3389 3390 3391
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3392 3393 3394
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3395 3396

	if (used + bytes > data_sinfo->total_bytes) {
3397
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3398

J
Josef Bacik 已提交
3399 3400 3401 3402
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3403
		if (!data_sinfo->full && alloc_chunk) {
J
Josef Bacik 已提交
3404
			u64 alloc_target;
J
Josef Bacik 已提交
3405

3406
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
J
Josef Bacik 已提交
3407
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
3408
alloc:
J
Josef Bacik 已提交
3409
			alloc_target = btrfs_get_alloc_profile(root, 1);
3410
			trans = btrfs_join_transaction(root);
3411 3412
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
3413

J
Josef Bacik 已提交
3414 3415
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
					     bytes + 2 * 1024 * 1024,
3416 3417
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
J
Josef Bacik 已提交
3418
			btrfs_end_transaction(trans, root);
3419 3420 3421 3422 3423 3424
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
J
Josef Bacik 已提交
3425

3426 3427 3428
			if (!data_sinfo)
				data_sinfo = fs_info->data_sinfo;

J
Josef Bacik 已提交
3429 3430
			goto again;
		}
3431 3432 3433 3434 3435 3436 3437

		/*
		 * If we have less pinned bytes than we want to allocate then
		 * don't bother committing the transaction, it won't help us.
		 */
		if (data_sinfo->bytes_pinned < bytes)
			committed = 1;
J
Josef Bacik 已提交
3438 3439
		spin_unlock(&data_sinfo->lock);

3440
		/* commit the current transaction and try again */
3441
commit_trans:
J
Josef Bacik 已提交
3442 3443
		if (!committed &&
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
3444
			committed = 1;
3445
			trans = btrfs_join_transaction(root);
3446 3447
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3448 3449 3450 3451 3452
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
J
Josef Bacik 已提交
3453

J
Josef Bacik 已提交
3454 3455 3456
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
J
Josef Bacik 已提交
3457
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3458
				      data_sinfo->flags, bytes, 1);
J
Josef Bacik 已提交
3459 3460
	spin_unlock(&data_sinfo->lock);

J
Josef Bacik 已提交
3461 3462
	return 0;
}
J
Josef Bacik 已提交
3463 3464

/*
3465
 * Called if we need to clear a data reservation for this inode.
J
Josef Bacik 已提交
3466
 */
3467
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3468
{
3469
	struct btrfs_root *root = BTRFS_I(inode)->root;
J
Josef Bacik 已提交
3470
	struct btrfs_space_info *data_sinfo;
3471

J
Josef Bacik 已提交
3472 3473
	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3474

3475
	data_sinfo = root->fs_info->data_sinfo;
J
Josef Bacik 已提交
3476 3477
	spin_lock(&data_sinfo->lock);
	data_sinfo->bytes_may_use -= bytes;
J
Josef Bacik 已提交
3478
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3479
				      data_sinfo->flags, bytes, 0);
J
Josef Bacik 已提交
3480
	spin_unlock(&data_sinfo->lock);
3481 3482
}

3483
static void force_metadata_allocation(struct btrfs_fs_info *info)
3484
{
3485 3486
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3487

3488 3489 3490
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3491
			found->force_alloc = CHUNK_ALLOC_FORCE;
3492
	}
3493
	rcu_read_unlock();
3494 3495
}

3496
static int should_alloc_chunk(struct btrfs_root *root,
3497 3498
			      struct btrfs_space_info *sinfo, u64 alloc_bytes,
			      int force)
3499
{
3500
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3501
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3502
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3503
	u64 thresh;
3504

3505 3506 3507
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

3508 3509 3510 3511 3512 3513 3514
	/*
	 * We need to take into account the global rsv because for all intents
	 * and purposes it's used space.  Don't worry about locking the
	 * global_rsv, it doesn't change except when the transaction commits.
	 */
	num_allocated += global_rsv->size;

3515 3516 3517 3518 3519
	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
3520
		thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3521 3522 3523 3524 3525 3526
		thresh = max_t(u64, 64 * 1024 * 1024,
			       div_factor_fine(thresh, 1));

		if (num_bytes - num_allocated < thresh)
			return 1;
	}
3527
	thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3528

3529 3530
	/* 256MB or 2% of the FS */
	thresh = max_t(u64, 256 * 1024 * 1024, div_factor_fine(thresh, 2));
C
Chris Mason 已提交
3531 3532 3533
	/* system chunks need a much small threshold */
	if (sinfo->flags & BTRFS_BLOCK_GROUP_SYSTEM)
		thresh = 32 * 1024 * 1024;
3534

3535
	if (num_bytes > thresh && sinfo->bytes_used < div_factor(num_bytes, 8))
3536
		return 0;
3537
	return 1;
3538 3539
}

3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583
static u64 get_system_chunk_thresh(struct btrfs_root *root, u64 type)
{
	u64 num_dev;

	if (type & BTRFS_BLOCK_GROUP_RAID10 ||
	    type & BTRFS_BLOCK_GROUP_RAID0)
		num_dev = root->fs_info->fs_devices->rw_devices;
	else if (type & BTRFS_BLOCK_GROUP_RAID1)
		num_dev = 2;
	else
		num_dev = 1;	/* DUP or single */

	/* metadata for updaing devices and chunk tree */
	return btrfs_calc_trans_metadata_size(root, num_dev + 1);
}

static void check_system_chunk(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root, u64 type)
{
	struct btrfs_space_info *info;
	u64 left;
	u64 thresh;

	info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	spin_lock(&info->lock);
	left = info->total_bytes - info->bytes_used - info->bytes_pinned -
		info->bytes_reserved - info->bytes_readonly;
	spin_unlock(&info->lock);

	thresh = get_system_chunk_thresh(root, type);
	if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
		printk(KERN_INFO "left=%llu, need=%llu, flags=%llu\n",
		       left, thresh, type);
		dump_space_info(info, 0, 0);
	}

	if (left < thresh) {
		u64 flags;

		flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
		btrfs_alloc_chunk(trans, root, flags);
	}
}

3584 3585
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
C
Chris Mason 已提交
3586
			  u64 flags, int force)
J
Josef Bacik 已提交
3587
{
3588
	struct btrfs_space_info *space_info;
3589
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
3590
	int wait_for_alloc = 0;
J
Josef Bacik 已提交
3591 3592
	int ret = 0;

3593
	space_info = __find_space_info(extent_root->fs_info, flags);
3594 3595 3596
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
3597
		BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
3598
	}
3599
	BUG_ON(!space_info); /* Logic error */
J
Josef Bacik 已提交
3600

3601
again:
3602
	spin_lock(&space_info->lock);
3603
	if (force < space_info->force_alloc)
3604
		force = space_info->force_alloc;
3605 3606
	if (space_info->full) {
		spin_unlock(&space_info->lock);
3607
		return 0;
J
Josef Bacik 已提交
3608 3609
	}

3610
	if (!should_alloc_chunk(extent_root, space_info, alloc_bytes, force)) {
3611
		spin_unlock(&space_info->lock);
3612 3613 3614 3615 3616
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
J
Josef Bacik 已提交
3617
	}
3618

3619
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3620

3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

3635 3636 3637 3638 3639 3640 3641
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3642 3643 3644 3645 3646
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
3647
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3648 3649 3650 3651
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
3652 3653
	}

3654 3655 3656 3657 3658 3659
	/*
	 * Check if we have enough space in SYSTEM chunk because we may need
	 * to update devices.
	 */
	check_system_chunk(trans, extent_root, flags);

Y
Yan Zheng 已提交
3660
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
3661 3662 3663
	if (ret < 0 && ret != -ENOSPC)
		goto out;

J
Josef Bacik 已提交
3664 3665
	spin_lock(&space_info->lock);
	if (ret)
3666
		space_info->full = 1;
3667 3668
	else
		ret = 1;
3669

3670
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3671
	space_info->chunk_alloc = 0;
J
Josef Bacik 已提交
3672
	spin_unlock(&space_info->lock);
3673
out:
3674
	mutex_unlock(&fs_info->chunk_mutex);
J
Josef Bacik 已提交
3675
	return ret;
3676
}
J
Josef Bacik 已提交
3677 3678

/*
3679
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
3680
 */
J
Josef Bacik 已提交
3681 3682
static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig,
			    bool wait_ordered)
3683
{
3684
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
3685
	struct btrfs_space_info *space_info;
3686
	struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3687
	u64 delalloc_bytes;
3688
	u64 max_reclaim;
3689
	long time_left;
3690
	unsigned long nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3691
	int loops = 0;
3692

3693
	trans = (struct btrfs_trans_handle *)current->journal_info;
3694
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
3695
	space_info = block_rsv->space_info;
3696 3697

	smp_mb();
J
Josef Bacik 已提交
3698 3699
	delalloc_bytes = root->fs_info->delalloc_bytes;
	if (delalloc_bytes == 0) {
3700
		if (trans)
J
Josef Bacik 已提交
3701
			return;
3702
		btrfs_wait_ordered_extents(root, 0, 0);
J
Josef Bacik 已提交
3703
		return;
3704 3705
	}

J
Josef Bacik 已提交
3706 3707 3708
	while (delalloc_bytes && loops < 3) {
		max_reclaim = min(delalloc_bytes, to_reclaim);
		nr_pages = max_reclaim >> PAGE_CACHE_SHIFT;
3709
		writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages,
J
Josef Bacik 已提交
3710
					       WB_REASON_FS_FREE_SPACE);
3711

J
Josef Bacik 已提交
3712
		spin_lock(&space_info->lock);
J
Josef Bacik 已提交
3713 3714 3715 3716 3717 3718 3719
		if (space_info->bytes_used + space_info->bytes_reserved +
		    space_info->bytes_pinned + space_info->bytes_readonly +
		    space_info->bytes_may_use + orig <=
		    space_info->total_bytes) {
			spin_unlock(&space_info->lock);
			break;
		}
J
Josef Bacik 已提交
3720
		spin_unlock(&space_info->lock);
3721

3722
		loops++;
3723 3724 3725
		if (wait_ordered && !trans) {
			btrfs_wait_ordered_extents(root, 0, 0);
		} else {
J
Josef Bacik 已提交
3726
			time_left = schedule_timeout_killable(1);
3727 3728 3729
			if (time_left)
				break;
		}
J
Josef Bacik 已提交
3730 3731
		smp_mb();
		delalloc_bytes = root->fs_info->delalloc_bytes;
3732 3733 3734
	}
}

3735 3736 3737 3738 3739
/**
 * maybe_commit_transaction - possibly commit the transaction if its ok to
 * @root - the root we're allocating for
 * @bytes - the number of bytes we want to reserve
 * @force - force the commit
3740
 *
3741 3742 3743
 * This will check to make sure that committing the transaction will actually
 * get us somewhere and then commit the transaction if it does.  Otherwise it
 * will return -ENOSPC.
3744
 */
3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773
static int may_commit_transaction(struct btrfs_root *root,
				  struct btrfs_space_info *space_info,
				  u64 bytes, int force)
{
	struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
	struct btrfs_trans_handle *trans;

	trans = (struct btrfs_trans_handle *)current->journal_info;
	if (trans)
		return -EAGAIN;

	if (force)
		goto commit;

	/* See if there is enough pinned space to make this reservation */
	spin_lock(&space_info->lock);
	if (space_info->bytes_pinned >= bytes) {
		spin_unlock(&space_info->lock);
		goto commit;
	}
	spin_unlock(&space_info->lock);

	/*
	 * See if there is some space in the delayed insertion reservation for
	 * this reservation.
	 */
	if (space_info != delayed_rsv->space_info)
		return -ENOSPC;

L
Liu Bo 已提交
3774
	spin_lock(&space_info->lock);
3775
	spin_lock(&delayed_rsv->lock);
L
Liu Bo 已提交
3776
	if (space_info->bytes_pinned + delayed_rsv->size < bytes) {
3777
		spin_unlock(&delayed_rsv->lock);
L
Liu Bo 已提交
3778
		spin_unlock(&space_info->lock);
3779 3780 3781
		return -ENOSPC;
	}
	spin_unlock(&delayed_rsv->lock);
L
Liu Bo 已提交
3782
	spin_unlock(&space_info->lock);
3783 3784 3785 3786 3787 3788 3789 3790 3791

commit:
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans))
		return -ENOSPC;

	return btrfs_commit_transaction(trans, root);
}

3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805
enum flush_state {
	FLUSH_DELALLOC		=	1,
	FLUSH_DELALLOC_WAIT	=	2,
	FLUSH_DELAYED_ITEMS_NR	=	3,
	FLUSH_DELAYED_ITEMS	=	4,
	COMMIT_TRANS		=	5,
};

static int flush_space(struct btrfs_root *root,
		       struct btrfs_space_info *space_info, u64 num_bytes,
		       u64 orig_bytes, int state)
{
	struct btrfs_trans_handle *trans;
	int nr;
J
Josef Bacik 已提交
3806
	int ret = 0;
3807 3808 3809 3810

	switch (state) {
	case FLUSH_DELALLOC:
	case FLUSH_DELALLOC_WAIT:
J
Josef Bacik 已提交
3811 3812
		shrink_delalloc(root, num_bytes, orig_bytes,
				state == FLUSH_DELALLOC_WAIT);
3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843
		break;
	case FLUSH_DELAYED_ITEMS_NR:
	case FLUSH_DELAYED_ITEMS:
		if (state == FLUSH_DELAYED_ITEMS_NR) {
			u64 bytes = btrfs_calc_trans_metadata_size(root, 1);

			nr = (int)div64_u64(num_bytes, bytes);
			if (!nr)
				nr = 1;
			nr *= 2;
		} else {
			nr = -1;
		}
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = btrfs_run_delayed_items_nr(trans, root, nr);
		btrfs_end_transaction(trans, root);
		break;
	case COMMIT_TRANS:
		ret = may_commit_transaction(root, space_info, orig_bytes, 0);
		break;
	default:
		ret = -ENOSPC;
		break;
	}

	return ret;
}
3844 3845 3846 3847 3848 3849
/**
 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
 * @root - the root we're allocating for
 * @block_rsv - the block_rsv we're allocating for
 * @orig_bytes - the number of bytes we want
 * @flush - wether or not we can flush to make our reservation
3850
 *
3851 3852 3853 3854 3855 3856
 * This will reserve orgi_bytes number of bytes from the space info associated
 * with the block_rsv.  If there is not enough space it will make an attempt to
 * flush out space to make room.  It will do this by flushing delalloc if
 * possible or committing the transaction.  If flush is 0 then no attempts to
 * regain reservations will be made and this will fail if there is not enough
 * space already.
3857
 */
3858
static int reserve_metadata_bytes(struct btrfs_root *root,
3859 3860
				  struct btrfs_block_rsv *block_rsv,
				  u64 orig_bytes, int flush)
J
Josef Bacik 已提交
3861
{
3862
	struct btrfs_space_info *space_info = block_rsv->space_info;
3863
	u64 used;
3864
	u64 num_bytes = orig_bytes;
3865
	int flush_state = FLUSH_DELALLOC;
3866
	int ret = 0;
3867
	bool flushing = false;
3868
	bool committed = false;
J
Josef Bacik 已提交
3869

3870
again:
3871
	ret = 0;
3872
	spin_lock(&space_info->lock);
3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884
	/*
	 * We only want to wait if somebody other than us is flushing and we are
	 * actually alloed to flush.
	 */
	while (flush && !flushing && space_info->flush) {
		spin_unlock(&space_info->lock);
		/*
		 * If we have a trans handle we can't wait because the flusher
		 * may have to commit the transaction, which would mean we would
		 * deadlock since we are waiting for the flusher to finish, but
		 * hold the current transaction open.
		 */
3885
		if (current->journal_info)
3886
			return -EAGAIN;
A
Arne Jansen 已提交
3887 3888 3889
		ret = wait_event_killable(space_info->wait, !space_info->flush);
		/* Must have been killed, return */
		if (ret)
3890 3891 3892 3893 3894 3895
			return -EINTR;

		spin_lock(&space_info->lock);
	}

	ret = -ENOSPC;
3896 3897 3898
	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;
J
Josef Bacik 已提交
3899

3900 3901 3902 3903 3904 3905 3906
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
3907 3908
	if (used <= space_info->total_bytes) {
		if (used + orig_bytes <= space_info->total_bytes) {
3909
			space_info->bytes_may_use += orig_bytes;
J
Josef Bacik 已提交
3910
			trace_btrfs_space_reservation(root->fs_info,
3911
				"space_info", space_info->flags, orig_bytes, 1);
3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
3927
		num_bytes = used - space_info->total_bytes +
3928
			(orig_bytes * 2);
3929
	}
J
Josef Bacik 已提交
3930

3931
	if (ret) {
3932 3933 3934
		u64 profile = btrfs_get_alloc_profile(root, 0);
		u64 avail;

3935 3936 3937 3938 3939 3940
		/*
		 * If we have a lot of space that's pinned, don't bother doing
		 * the overcommit dance yet and just commit the transaction.
		 */
		avail = (space_info->total_bytes - space_info->bytes_used) * 8;
		do_div(avail, 10);
3941
		if (space_info->bytes_pinned >= avail && flush && !committed) {
3942 3943 3944
			space_info->flush = 1;
			flushing = true;
			spin_unlock(&space_info->lock);
3945 3946 3947 3948 3949 3950
			ret = may_commit_transaction(root, space_info,
						     orig_bytes, 1);
			if (ret)
				goto out;
			committed = true;
			goto again;
3951 3952
		}

3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975
		spin_lock(&root->fs_info->free_chunk_lock);
		avail = root->fs_info->free_chunk_space;

		/*
		 * If we have dup, raid1 or raid10 then only half of the free
		 * space is actually useable.
		 */
		if (profile & (BTRFS_BLOCK_GROUP_DUP |
			       BTRFS_BLOCK_GROUP_RAID1 |
			       BTRFS_BLOCK_GROUP_RAID10))
			avail >>= 1;

		/*
		 * If we aren't flushing don't let us overcommit too much, say
		 * 1/8th of the space.  If we can flush, let it overcommit up to
		 * 1/2 of the space.
		 */
		if (flush)
			avail >>= 3;
		else
			avail >>= 1;
		 spin_unlock(&root->fs_info->free_chunk_lock);

3976
		if (used + num_bytes < space_info->total_bytes + avail) {
3977
			space_info->bytes_may_use += orig_bytes;
J
Josef Bacik 已提交
3978
			trace_btrfs_space_reservation(root->fs_info,
3979
				"space_info", space_info->flags, orig_bytes, 1);
3980 3981 3982 3983
			ret = 0;
		}
	}

3984 3985 3986 3987 3988
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
	 */
3989 3990 3991
	if (ret && flush) {
		flushing = true;
		space_info->flush = 1;
3992
	}
J
Josef Bacik 已提交
3993

3994
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3995

3996
	if (!ret || !flush)
3997
		goto out;
3998

3999 4000 4001 4002
	ret = flush_space(root, space_info, num_bytes, orig_bytes,
			  flush_state);
	flush_state++;
	if (!ret)
4003
		goto again;
4004
	else if (flush_state <= COMMIT_TRANS)
4005 4006 4007
		goto again;

out:
4008
	if (flushing) {
4009
		spin_lock(&space_info->lock);
4010 4011
		space_info->flush = 0;
		wake_up_all(&space_info->wait);
4012
		spin_unlock(&space_info->lock);
4013 4014 4015 4016
	}
	return ret;
}

4017 4018 4019
static struct btrfs_block_rsv *get_block_rsv(
					const struct btrfs_trans_handle *trans,
					const struct btrfs_root *root)
4020
{
4021 4022
	struct btrfs_block_rsv *block_rsv = NULL;

4023 4024 4025 4026
	if (root->ref_cows)
		block_rsv = trans->block_rsv;

	if (root == root->fs_info->csum_root && trans->adding_csums)
4027
		block_rsv = trans->block_rsv;
4028 4029

	if (!block_rsv)
4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

J
Josef Bacik 已提交
4065 4066
static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
				    struct btrfs_block_rsv *block_rsv,
4067
				    struct btrfs_block_rsv *dest, u64 num_bytes)
4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
4100
			spin_lock(&space_info->lock);
4101
			space_info->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4102
			trace_btrfs_space_reservation(fs_info, "space_info",
4103
					space_info->flags, num_bytes, 0);
4104
			space_info->reservation_progress++;
4105
			spin_unlock(&space_info->lock);
4106
		}
J
Josef Bacik 已提交
4107
	}
4108
}
4109

4110 4111 4112 4113
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
J
Josef Bacik 已提交
4114

4115 4116 4117
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
4118

4119
	block_rsv_add_bytes(dst, num_bytes, 1);
J
Josef Bacik 已提交
4120 4121 4122
	return 0;
}

4123
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv)
J
Josef Bacik 已提交
4124
{
4125 4126 4127 4128 4129 4130 4131 4132
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
}

struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root)
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
4133

4134 4135 4136
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
4137

4138 4139 4140 4141 4142
	btrfs_init_block_rsv(block_rsv);
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
4143

4144 4145 4146
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
4147 4148
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	kfree(rsv);
J
Josef Bacik 已提交
4149 4150
}

4151 4152 4153
static inline int __block_rsv_add(struct btrfs_root *root,
				  struct btrfs_block_rsv *block_rsv,
				  u64 num_bytes, int flush)
J
Josef Bacik 已提交
4154
{
4155
	int ret;
J
Josef Bacik 已提交
4156

4157 4158
	if (num_bytes == 0)
		return 0;
4159

4160
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4161 4162 4163 4164
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
4165

4166 4167
	return ret;
}
J
Josef Bacik 已提交
4168

4169 4170 4171 4172 4173 4174 4175
int btrfs_block_rsv_add(struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv,
			u64 num_bytes)
{
	return __block_rsv_add(root, block_rsv, num_bytes, 1);
}

4176 4177 4178
int btrfs_block_rsv_add_noflush(struct btrfs_root *root,
				struct btrfs_block_rsv *block_rsv,
				u64 num_bytes)
4179
{
4180
	return __block_rsv_add(root, block_rsv, num_bytes, 0);
4181
}
J
Josef Bacik 已提交
4182

4183
int btrfs_block_rsv_check(struct btrfs_root *root,
4184
			  struct btrfs_block_rsv *block_rsv, int min_factor)
4185 4186 4187
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
4188

4189 4190
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
4191

4192
	spin_lock(&block_rsv->lock);
4193 4194 4195 4196
	num_bytes = div_factor(block_rsv->size, min_factor);
	if (block_rsv->reserved >= num_bytes)
		ret = 0;
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
4197

4198 4199 4200
	return ret;
}

4201 4202 4203
static inline int __btrfs_block_rsv_refill(struct btrfs_root *root,
					   struct btrfs_block_rsv *block_rsv,
					   u64 min_reserved, int flush)
4204 4205 4206 4207 4208 4209 4210 4211 4212
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;

	if (!block_rsv)
		return 0;

	spin_lock(&block_rsv->lock);
	num_bytes = min_reserved;
4213
	if (block_rsv->reserved >= num_bytes)
4214
		ret = 0;
4215
	else
4216 4217
		num_bytes -= block_rsv->reserved;
	spin_unlock(&block_rsv->lock);
4218

4219 4220 4221
	if (!ret)
		return 0;

4222
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4223 4224
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 0);
4225
		return 0;
J
Josef Bacik 已提交
4226
	}
J
Josef Bacik 已提交
4227

4228
	return ret;
4229 4230
}

4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244
int btrfs_block_rsv_refill(struct btrfs_root *root,
			   struct btrfs_block_rsv *block_rsv,
			   u64 min_reserved)
{
	return __btrfs_block_rsv_refill(root, block_rsv, min_reserved, 1);
}

int btrfs_block_rsv_refill_noflush(struct btrfs_root *root,
				   struct btrfs_block_rsv *block_rsv,
				   u64 min_reserved)
{
	return __btrfs_block_rsv_refill(root, block_rsv, min_reserved, 0);
}

4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259
int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
	if (global_rsv->full || global_rsv == block_rsv ||
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
J
Josef Bacik 已提交
4260 4261
	block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
				num_bytes);
J
Josef Bacik 已提交
4262 4263 4264
}

/*
4265 4266 4267
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
4268
 */
4269
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4270
{
4271 4272 4273 4274
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
4275
	int csum_size = btrfs_super_csum_size(fs_info->super_copy);
J
Josef Bacik 已提交
4276

4277 4278 4279 4280
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
4281

4282 4283
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
4284 4285
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
4286 4287
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
4288

4289 4290 4291
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
	num_bytes += div64_u64(data_used + meta_used, 50);
4292

4293
	if (num_bytes * 3 > meta_used)
4294
		num_bytes = div64_u64(meta_used, 3);
4295

4296 4297
	return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
}
J
Josef Bacik 已提交
4298

4299 4300 4301 4302 4303
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
4304

4305
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
4306

4307
	spin_lock(&sinfo->lock);
4308
	spin_lock(&block_rsv->lock);
4309

4310
	block_rsv->size = num_bytes;
4311

4312
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
4313 4314
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
4315 4316 4317 4318

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
4319
		sinfo->bytes_may_use += num_bytes;
J
Josef Bacik 已提交
4320
		trace_btrfs_space_reservation(fs_info, "space_info",
4321
				      sinfo->flags, num_bytes, 1);
J
Josef Bacik 已提交
4322 4323
	}

4324 4325
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
4326
		sinfo->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4327
		trace_btrfs_space_reservation(fs_info, "space_info",
4328
				      sinfo->flags, num_bytes, 0);
4329
		sinfo->reservation_progress++;
4330 4331 4332
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
4333

4334
	spin_unlock(&block_rsv->lock);
4335
	spin_unlock(&sinfo->lock);
J
Josef Bacik 已提交
4336 4337
}

4338
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4339
{
4340
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
4341

4342 4343
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
J
Josef Bacik 已提交
4344

4345
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4346 4347
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->delalloc_block_rsv.space_info = space_info;
4348 4349
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
4350
	fs_info->delayed_block_rsv.space_info = space_info;
4351

4352 4353 4354 4355
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
4356
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
4357 4358

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
4359 4360
}

4361
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4362
{
J
Josef Bacik 已提交
4363 4364
	block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
				(u64)-1);
4365 4366 4367 4368 4369 4370
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
4371 4372
	WARN_ON(fs_info->delayed_block_rsv.size > 0);
	WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
4373 4374
}

4375 4376
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
4377
{
4378 4379 4380
	if (!trans->block_rsv)
		return;

4381 4382
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
4383

4384
	trace_btrfs_space_reservation(root->fs_info, "transaction",
4385
				      trans->transid, trans->bytes_reserved, 0);
4386
	btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
4387 4388
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
4389

4390
/* Can only return 0 or -ENOSPC */
4391 4392 4393 4394 4395 4396 4397 4398
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
4399 4400 4401
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
4402
	 */
C
Chris Mason 已提交
4403
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4404 4405
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 1);
4406
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
J
Josef Bacik 已提交
4407 4408
}

4409
void btrfs_orphan_release_metadata(struct inode *inode)
4410
{
4411
	struct btrfs_root *root = BTRFS_I(inode)->root;
C
Chris Mason 已提交
4412
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4413 4414
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 0);
4415 4416
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
4417

4418 4419 4420 4421 4422 4423 4424 4425 4426 4427
int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
				struct btrfs_pending_snapshot *pending)
{
	struct btrfs_root *root = pending->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
	/*
	 * two for root back/forward refs, two for directory entries
	 * and one for root of the snapshot.
	 */
4428
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 5);
4429 4430
	dst_rsv->space_info = src_rsv->space_info;
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4431 4432
}

4433 4434 4435 4436 4437 4438 4439 4440 4441
/**
 * drop_outstanding_extent - drop an outstanding extent
 * @inode: the inode we're dropping the extent for
 *
 * This is called when we are freeing up an outstanding extent, either called
 * after an error or after an extent is written.  This will return the number of
 * reserved extents that need to be freed.  This must be called with
 * BTRFS_I(inode)->lock held.
 */
4442 4443
static unsigned drop_outstanding_extent(struct inode *inode)
{
4444
	unsigned drop_inode_space = 0;
4445 4446 4447 4448 4449
	unsigned dropped_extents = 0;

	BUG_ON(!BTRFS_I(inode)->outstanding_extents);
	BTRFS_I(inode)->outstanding_extents--;

4450
	if (BTRFS_I(inode)->outstanding_extents == 0 &&
4451 4452
	    test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			       &BTRFS_I(inode)->runtime_flags))
4453 4454
		drop_inode_space = 1;

4455 4456 4457 4458 4459 4460
	/*
	 * If we have more or the same amount of outsanding extents than we have
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
4461
		return drop_inode_space;
4462 4463 4464 4465

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
4466
	return dropped_extents + drop_inode_space;
4467 4468
}

4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488
/**
 * calc_csum_metadata_size - return the amount of metada space that must be
 *	reserved/free'd for the given bytes.
 * @inode: the inode we're manipulating
 * @num_bytes: the number of bytes in question
 * @reserve: 1 if we are reserving space, 0 if we are freeing space
 *
 * This adjusts the number of csum_bytes in the inode and then returns the
 * correct amount of metadata that must either be reserved or freed.  We
 * calculate how many checksums we can fit into one leaf and then divide the
 * number of bytes that will need to be checksumed by this value to figure out
 * how many checksums will be required.  If we are adding bytes then the number
 * may go up and we will return the number of additional bytes that must be
 * reserved.  If it is going down we will return the number of bytes that must
 * be freed.
 *
 * This must be called with BTRFS_I(inode)->lock held.
 */
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
				   int reserve)
4489
{
4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 csum_size;
	int num_csums_per_leaf;
	int num_csums;
	int old_csums;

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
	    BTRFS_I(inode)->csum_bytes == 0)
		return 0;

	old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	if (reserve)
		BTRFS_I(inode)->csum_bytes += num_bytes;
	else
		BTRFS_I(inode)->csum_bytes -= num_bytes;
	csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
	num_csums_per_leaf = (int)div64_u64(csum_size,
					    sizeof(struct btrfs_csum_item) +
					    sizeof(struct btrfs_disk_key));
	num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	num_csums = num_csums + num_csums_per_leaf - 1;
	num_csums = num_csums / num_csums_per_leaf;

	old_csums = old_csums + num_csums_per_leaf - 1;
	old_csums = old_csums / num_csums_per_leaf;

	/* No change, no need to reserve more */
	if (old_csums == num_csums)
		return 0;

	if (reserve)
		return btrfs_calc_trans_metadata_size(root,
						      num_csums - old_csums);

	return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
4525
}
Y
Yan Zheng 已提交
4526

4527 4528 4529 4530
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4531
	u64 to_reserve = 0;
4532
	u64 csum_bytes;
4533
	unsigned nr_extents = 0;
4534
	int extra_reserve = 0;
4535
	int flush = 1;
4536
	int ret;
4537

4538
	/* Need to be holding the i_mutex here if we aren't free space cache */
4539
	if (btrfs_is_free_space_inode(inode))
4540 4541 4542
		flush = 0;

	if (flush && btrfs_transaction_in_commit(root->fs_info))
4543
		schedule_timeout(1);
4544

4545
	mutex_lock(&BTRFS_I(inode)->delalloc_mutex);
4546
	num_bytes = ALIGN(num_bytes, root->sectorsize);
4547

4548 4549 4550 4551
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;

	if (BTRFS_I(inode)->outstanding_extents >
4552
	    BTRFS_I(inode)->reserved_extents)
4553 4554
		nr_extents = BTRFS_I(inode)->outstanding_extents -
			BTRFS_I(inode)->reserved_extents;
4555

4556 4557 4558 4559
	/*
	 * Add an item to reserve for updating the inode when we complete the
	 * delalloc io.
	 */
4560 4561
	if (!test_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
		      &BTRFS_I(inode)->runtime_flags)) {
4562
		nr_extents++;
4563
		extra_reserve = 1;
4564
	}
4565 4566

	to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4567
	to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
4568
	csum_bytes = BTRFS_I(inode)->csum_bytes;
4569
	spin_unlock(&BTRFS_I(inode)->lock);
4570

4571 4572 4573 4574 4575 4576 4577
	if (root->fs_info->quota_enabled) {
		ret = btrfs_qgroup_reserve(root, num_bytes +
					   nr_extents * root->leafsize);
		if (ret)
			return ret;
	}

4578
	ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush);
4579
	if (ret) {
4580
		u64 to_free = 0;
4581
		unsigned dropped;
4582

4583
		spin_lock(&BTRFS_I(inode)->lock);
4584 4585
		dropped = drop_outstanding_extent(inode);
		/*
4586 4587 4588 4589 4590 4591
		 * If the inodes csum_bytes is the same as the original
		 * csum_bytes then we know we haven't raced with any free()ers
		 * so we can just reduce our inodes csum bytes and carry on.
		 * Otherwise we have to do the normal free thing to account for
		 * the case that the free side didn't free up its reserve
		 * because of this outstanding reservation.
4592
		 */
4593 4594 4595 4596 4597 4598 4599 4600
		if (BTRFS_I(inode)->csum_bytes == csum_bytes)
			calc_csum_metadata_size(inode, num_bytes, 0);
		else
			to_free = calc_csum_metadata_size(inode, num_bytes, 0);
		spin_unlock(&BTRFS_I(inode)->lock);
		if (dropped)
			to_free += btrfs_calc_trans_metadata_size(root, dropped);

J
Josef Bacik 已提交
4601
		if (to_free) {
4602
			btrfs_block_rsv_release(root, block_rsv, to_free);
J
Josef Bacik 已提交
4603 4604 4605 4606 4607
			trace_btrfs_space_reservation(root->fs_info,
						      "delalloc",
						      btrfs_ino(inode),
						      to_free, 0);
		}
4608
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4609
		return ret;
4610
	}
4611

4612 4613
	spin_lock(&BTRFS_I(inode)->lock);
	if (extra_reserve) {
4614 4615
		set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			&BTRFS_I(inode)->runtime_flags);
4616 4617 4618 4619
		nr_extents--;
	}
	BTRFS_I(inode)->reserved_extents += nr_extents;
	spin_unlock(&BTRFS_I(inode)->lock);
4620
	mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4621

J
Josef Bacik 已提交
4622 4623 4624
	if (to_reserve)
		trace_btrfs_space_reservation(root->fs_info,"delalloc",
					      btrfs_ino(inode), to_reserve, 1);
4625 4626 4627 4628 4629
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	return 0;
}

4630 4631 4632 4633 4634 4635 4636 4637 4638
/**
 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
 * @inode: the inode to release the reservation for
 * @num_bytes: the number of bytes we're releasing
 *
 * This will release the metadata reservation for an inode.  This can be called
 * once we complete IO for a given set of bytes to release their metadata
 * reservations.
 */
4639 4640 4641
void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
4642 4643
	u64 to_free = 0;
	unsigned dropped;
4644 4645

	num_bytes = ALIGN(num_bytes, root->sectorsize);
4646
	spin_lock(&BTRFS_I(inode)->lock);
4647
	dropped = drop_outstanding_extent(inode);
4648

4649 4650
	to_free = calc_csum_metadata_size(inode, num_bytes, 0);
	spin_unlock(&BTRFS_I(inode)->lock);
4651 4652
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
4653

J
Josef Bacik 已提交
4654 4655
	trace_btrfs_space_reservation(root->fs_info, "delalloc",
				      btrfs_ino(inode), to_free, 0);
4656 4657 4658 4659 4660
	if (root->fs_info->quota_enabled) {
		btrfs_qgroup_free(root, num_bytes +
					dropped * root->leafsize);
	}

4661 4662 4663 4664
	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679
/**
 * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
 * @inode: inode we're writing to
 * @num_bytes: the number of bytes we want to allocate
 *
 * This will do the following things
 *
 * o reserve space in the data space info for num_bytes
 * o reserve space in the metadata space info based on number of outstanding
 *   extents and how much csums will be needed
 * o add to the inodes ->delalloc_bytes
 * o add it to the fs_info's delalloc inodes list.
 *
 * This will return 0 for success and -ENOSPC if there is no space left.
 */
4680 4681 4682 4683 4684
int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
C
Chris Mason 已提交
4685
	if (ret)
4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709
/**
 * btrfs_delalloc_release_space - release data and metadata space for delalloc
 * @inode: inode we're releasing space for
 * @num_bytes: the number of bytes we want to free up
 *
 * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
 * called in the case that we don't need the metadata AND data reservations
 * anymore.  So if there is an error or we insert an inline extent.
 *
 * This function will release the metadata space that was not used and will
 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
 * list if there are no delalloc bytes left.
 */
4710 4711 4712 4713
void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
4714 4715
}

C
Chris Mason 已提交
4716 4717
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
4718
			      u64 bytenr, u64 num_bytes, int alloc)
C
Chris Mason 已提交
4719
{
4720
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
4721
	struct btrfs_fs_info *info = root->fs_info;
4722
	u64 total = num_bytes;
C
Chris Mason 已提交
4723
	u64 old_val;
4724
	u64 byte_in_group;
4725
	int factor;
C
Chris Mason 已提交
4726

4727 4728
	/* block accounting for super block */
	spin_lock(&info->delalloc_lock);
4729
	old_val = btrfs_super_bytes_used(info->super_copy);
4730 4731 4732 4733
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
4734
	btrfs_set_super_bytes_used(info->super_copy, old_val);
4735 4736
	spin_unlock(&info->delalloc_lock);

C
Chris Mason 已提交
4737
	while (total) {
4738
		cache = btrfs_lookup_block_group(info, bytenr);
4739
		if (!cache)
4740
			return -ENOENT;
4741 4742 4743 4744 4745 4746
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
4747 4748 4749 4750 4751 4752 4753
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
4754
			cache_block_group(cache, trans, NULL, 1);
4755

4756 4757
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
4758

4759
		spin_lock(&cache->space_info->lock);
4760
		spin_lock(&cache->lock);
4761

4762
		if (btrfs_test_opt(root, SPACE_CACHE) &&
4763 4764 4765
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

J
Josef Bacik 已提交
4766
		cache->dirty = 1;
C
Chris Mason 已提交
4767
		old_val = btrfs_block_group_used(&cache->item);
4768
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
4769
		if (alloc) {
4770
			old_val += num_bytes;
4771 4772 4773
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
4774 4775
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
4776
			spin_unlock(&cache->lock);
4777
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4778
		} else {
4779
			old_val -= num_bytes;
4780
			btrfs_set_block_group_used(&cache->item, old_val);
4781 4782
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
4783
			cache->space_info->bytes_used -= num_bytes;
4784
			cache->space_info->disk_used -= num_bytes * factor;
4785
			spin_unlock(&cache->lock);
4786
			spin_unlock(&cache->space_info->lock);
4787

4788 4789 4790
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
C
Chris Mason 已提交
4791
		}
4792
		btrfs_put_block_group(cache);
4793 4794
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
4795 4796 4797
	}
	return 0;
}
4798

4799 4800
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
4801
	struct btrfs_block_group_cache *cache;
4802
	u64 bytenr;
J
Josef Bacik 已提交
4803 4804 4805

	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
4806
		return 0;
J
Josef Bacik 已提交
4807

4808
	bytenr = cache->key.objectid;
4809
	btrfs_put_block_group(cache);
4810 4811

	return bytenr;
4812 4813
}

4814 4815 4816
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
4817
{
4818 4819 4820 4821 4822 4823 4824 4825 4826 4827
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
4828

4829 4830 4831 4832
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
	return 0;
}
J
Josef Bacik 已提交
4833

4834 4835 4836 4837 4838 4839 4840
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
4841

4842
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4843
	BUG_ON(!cache); /* Logic error */
4844 4845 4846 4847

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
4848 4849 4850
	return 0;
}

4851
/*
4852 4853 4854 4855 4856 4857 4858 4859 4860
 * this function must be called within transaction
 */
int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans,
				    struct btrfs_root *root,
				    u64 bytenr, u64 num_bytes)
{
	struct btrfs_block_group_cache *cache;

	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4861
	BUG_ON(!cache); /* Logic error */
4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878

	/*
	 * pull in the free space cache (if any) so that our pin
	 * removes the free space from the cache.  We have load_only set
	 * to one because the slow code to read in the free extents does check
	 * the pinned extents.
	 */
	cache_block_group(cache, trans, root, 1);

	pin_down_extent(root, cache, bytenr, num_bytes, 0);

	/* remove us from the free space cache (if we're there at all) */
	btrfs_remove_free_space(cache, bytenr, num_bytes);
	btrfs_put_block_group(cache);
	return 0;
}

4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899
/**
 * btrfs_update_reserved_bytes - update the block_group and space info counters
 * @cache:	The cache we are manipulating
 * @num_bytes:	The number of bytes in question
 * @reserve:	One of the reservation enums
 *
 * This is called by the allocator when it reserves space, or by somebody who is
 * freeing space that was never actually used on disk.  For example if you
 * reserve some space for a new leaf in transaction A and before transaction A
 * commits you free that leaf, you call this with reserve set to 0 in order to
 * clear the reservation.
 *
 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
 * ENOSPC accounting.  For data we handle the reservation through clearing the
 * delalloc bits in the io_tree.  We have to do this since we could end up
 * allocating less disk space for the amount of data we have reserved in the
 * case of compression.
 *
 * If this is a reservation and the block group has become read only we cannot
 * make the reservation and return -EAGAIN, otherwise this function always
 * succeeds.
4900
 */
4901 4902
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve)
4903
{
4904
	struct btrfs_space_info *space_info = cache->space_info;
4905
	int ret = 0;
4906

4907 4908 4909
	spin_lock(&space_info->lock);
	spin_lock(&cache->lock);
	if (reserve != RESERVE_FREE) {
4910 4911 4912
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
4913 4914 4915
			cache->reserved += num_bytes;
			space_info->bytes_reserved += num_bytes;
			if (reserve == RESERVE_ALLOC) {
J
Josef Bacik 已提交
4916
				trace_btrfs_space_reservation(cache->fs_info,
4917 4918
						"space_info", space_info->flags,
						num_bytes, 0);
4919 4920
				space_info->bytes_may_use -= num_bytes;
			}
4921
		}
4922 4923 4924 4925 4926 4927
	} else {
		if (cache->ro)
			space_info->bytes_readonly += num_bytes;
		cache->reserved -= num_bytes;
		space_info->bytes_reserved -= num_bytes;
		space_info->reservation_progress++;
4928
	}
4929 4930
	spin_unlock(&cache->lock);
	spin_unlock(&space_info->lock);
4931
	return ret;
4932
}
C
Chris Mason 已提交
4933

4934
void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
4935
				struct btrfs_root *root)
4936 4937
{
	struct btrfs_fs_info *fs_info = root->fs_info;
4938 4939 4940
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
4941

4942
	down_write(&fs_info->extent_commit_sem);
4943

4944 4945 4946 4947 4948 4949 4950
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
4951
		} else {
4952
			cache->last_byte_to_unpin = caching_ctl->progress;
4953 4954
		}
	}
4955 4956 4957 4958 4959 4960 4961

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

	up_write(&fs_info->extent_commit_sem);
4962 4963

	update_global_block_rsv(fs_info);
4964 4965
}

4966
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
C
Chris Mason 已提交
4967
{
4968 4969 4970
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 len;
C
Chris Mason 已提交
4971

4972 4973 4974 4975 4976 4977
	while (start <= end) {
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
4978
			BUG_ON(!cache); /* Logic error */
4979 4980 4981 4982 4983 4984 4985 4986 4987 4988
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
			btrfs_add_free_space(cache, start, len);
		}

4989 4990
		start += len;

4991 4992 4993 4994
		spin_lock(&cache->space_info->lock);
		spin_lock(&cache->lock);
		cache->pinned -= len;
		cache->space_info->bytes_pinned -= len;
4995
		if (cache->ro)
4996
			cache->space_info->bytes_readonly += len;
4997 4998
		spin_unlock(&cache->lock);
		spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4999
	}
5000 5001 5002

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
5003 5004 5005 5006
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
5007
			       struct btrfs_root *root)
5008
{
5009 5010
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
5011 5012
	u64 start;
	u64 end;
5013 5014
	int ret;

5015 5016 5017
	if (trans->aborted)
		return 0;

5018 5019 5020 5021 5022
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

C
Chris Mason 已提交
5023
	while (1) {
5024 5025 5026
		ret = find_first_extent_bit(unpin, 0, &start, &end,
					    EXTENT_DIRTY);
		if (ret)
5027
			break;
5028

5029 5030 5031
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
5032

5033
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
5034
		unpin_extent_range(root, start, end);
5035
		cond_resched();
5036
	}
J
Josef Bacik 已提交
5037

C
Chris Mason 已提交
5038 5039 5040
	return 0;
}

5041 5042 5043 5044 5045 5046
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extent_op)
5047
{
C
Chris Mason 已提交
5048
	struct btrfs_key key;
5049
	struct btrfs_path *path;
5050 5051
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
5052
	struct extent_buffer *leaf;
5053 5054
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
5055
	int ret;
5056
	int is_data;
5057 5058 5059
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
5060 5061
	u32 item_size;
	u64 refs;
C
Chris Mason 已提交
5062

5063
	path = btrfs_alloc_path();
5064 5065
	if (!path)
		return -ENOMEM;
5066

5067
	path->reada = 1;
5068
	path->leave_spinning = 1;
5069 5070 5071 5072 5073 5074 5075 5076

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
5077
	if (ret == 0) {
5078
		extent_slot = path->slots[0];
5079 5080
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
5081
					      extent_slot);
5082
			if (key.objectid != bytenr)
5083
				break;
5084 5085
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
5086 5087 5088 5089 5090
				found_extent = 1;
				break;
			}
			if (path->slots[0] - extent_slot > 5)
				break;
5091
			extent_slot--;
5092
		}
5093 5094 5095 5096 5097
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
5098
		if (!found_extent) {
5099
			BUG_ON(iref);
5100
			ret = remove_extent_backref(trans, extent_root, path,
5101 5102
						    NULL, refs_to_drop,
						    is_data);
5103 5104
			if (ret)
				goto abort;
5105
			btrfs_release_path(path);
5106
			path->leave_spinning = 1;
5107 5108 5109 5110 5111

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

Z
Zheng Yan 已提交
5112 5113
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
5114 5115
			if (ret) {
				printk(KERN_ERR "umm, got %d back from search"
C
Chris Mason 已提交
5116 5117
				       ", was looking for %llu\n", ret,
				       (unsigned long long)bytenr);
5118 5119 5120
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
5121
			}
5122 5123
			if (ret < 0)
				goto abort;
Z
Zheng Yan 已提交
5124 5125
			extent_slot = path->slots[0];
		}
5126
	} else if (ret == -ENOENT) {
5127 5128
		btrfs_print_leaf(extent_root, path->nodes[0]);
		WARN_ON(1);
C
Chris Mason 已提交
5129
		printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
5130
		       "parent %llu root %llu  owner %llu offset %llu\n",
C
Chris Mason 已提交
5131
		       (unsigned long long)bytenr,
5132
		       (unsigned long long)parent,
C
Chris Mason 已提交
5133
		       (unsigned long long)root_objectid,
5134 5135
		       (unsigned long long)owner_objectid,
		       (unsigned long long)owner_offset);
5136 5137
	} else {
		goto abort;
5138
	}
5139 5140

	leaf = path->nodes[0];
5141 5142 5143 5144 5145 5146
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
5147 5148
		if (ret < 0)
			goto abort;
5149

5150
		btrfs_release_path(path);
5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
			printk(KERN_ERR "umm, got %d back from search"
			       ", was looking for %llu\n", ret,
			       (unsigned long long)bytenr);
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
5165 5166
		if (ret < 0)
			goto abort;
5167 5168 5169 5170 5171 5172
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
5173
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
5174
			    struct btrfs_extent_item);
5175 5176 5177 5178 5179 5180
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
5181

5182
	refs = btrfs_extent_refs(leaf, ei);
5183 5184
	BUG_ON(refs < refs_to_drop);
	refs -= refs_to_drop;
5185

5186 5187 5188 5189 5190 5191
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
5192
		 */
5193 5194 5195 5196 5197 5198 5199 5200 5201 5202
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
						    is_data);
5203 5204
			if (ret)
				goto abort;
5205
		}
5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
5217
		}
5218

5219 5220
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
5221 5222
		if (ret)
			goto abort;
5223
		btrfs_release_path(path);
5224

5225
		if (is_data) {
5226
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
5227 5228
			if (ret)
				goto abort;
5229 5230
		}

5231
		ret = update_block_group(trans, root, bytenr, num_bytes, 0);
5232 5233
		if (ret)
			goto abort;
5234
	}
5235
out:
5236
	btrfs_free_path(path);
5237
	return ret;
5238 5239 5240 5241

abort:
	btrfs_abort_transaction(trans, extent_root, ret);
	goto out;
5242 5243
}

5244
/*
5245
 * when we free an block, it is possible (and likely) that we free the last
5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct rb_node *node;
5257
	int ret = 0;
5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	/* there are still entries for this ref, we can't drop it */
	if (ref->bytenr == bytenr)
		goto out;

5275 5276 5277 5278 5279 5280 5281
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
		kfree(head->extent_op);
		head->extent_op = NULL;
	}

5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
	rb_erase(&head->node.rb_node, &delayed_refs->root);
5295

5296
	delayed_refs->num_entries--;
5297
	smp_mb();
5298 5299
	if (waitqueue_active(&root->fs_info->tree_mod_seq_wait))
		wake_up(&root->fs_info->tree_mod_seq_wait);
5300 5301 5302 5303 5304

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
5305 5306 5307 5308 5309
	delayed_refs->num_heads--;
	if (list_empty(&head->cluster))
		delayed_refs->num_heads_ready--;

	list_del_init(&head->cluster);
5310 5311
	spin_unlock(&delayed_refs->lock);

5312 5313 5314 5315 5316
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
5317
	btrfs_put_delayed_ref(&head->node);
5318
	return ret;
5319 5320 5321 5322 5323
out:
	spin_unlock(&delayed_refs->lock);
	return 0;
}

5324 5325 5326
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
5327
			   u64 parent, int last_ref)
5328 5329 5330 5331 5332
{
	struct btrfs_block_group_cache *cache = NULL;
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
A
Arne Jansen 已提交
5333 5334 5335 5336
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					buf->start, buf->len,
					parent, root->root_key.objectid,
					btrfs_header_level(buf),
5337
					BTRFS_DROP_DELAYED_REF, NULL, 0);
5338
		BUG_ON(ret); /* -ENOMEM */
5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349
	}

	if (!last_ref)
		return;

	cache = btrfs_lookup_block_group(root->fs_info, buf->start);

	if (btrfs_header_generation(buf) == trans->transid) {
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
5350
				goto out;
5351 5352 5353 5354
		}

		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
5355
			goto out;
5356 5357 5358 5359 5360
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
5361
		btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
5362 5363
	}
out:
5364 5365 5366 5367 5368
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
5369 5370 5371
	btrfs_put_block_group(cache);
}

5372
/* Can return -ENOMEM */
A
Arne Jansen 已提交
5373 5374 5375
int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
		      u64 owner, u64 offset, int for_cow)
5376 5377
{
	int ret;
A
Arne Jansen 已提交
5378
	struct btrfs_fs_info *fs_info = root->fs_info;
5379

5380 5381 5382 5383
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
5384 5385
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
5386
		/* unlocks the pinned mutex */
5387
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
5388
		ret = 0;
5389
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
5390 5391
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
5392
					parent, root_objectid, (int)owner,
A
Arne Jansen 已提交
5393
					BTRFS_DROP_DELAYED_REF, NULL, for_cow);
5394
	} else {
A
Arne Jansen 已提交
5395 5396 5397 5398 5399
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
						num_bytes,
						parent, root_objectid, owner,
						offset, BTRFS_DROP_DELAYED_REF,
						NULL, for_cow);
5400
	}
5401 5402 5403
	return ret;
}

5404 5405 5406 5407 5408 5409 5410
static u64 stripe_align(struct btrfs_root *root, u64 val)
{
	u64 mask = ((u64)root->stripesize - 1);
	u64 ret = (val + mask) & ~mask;
	return ret;
}

J
Josef Bacik 已提交
5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
 */
static noinline int
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
5426
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
5427 5428
	DEFINE_WAIT(wait);

5429 5430
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
J
Josef Bacik 已提交
5431 5432
		return 0;

5433
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
5434
		   (cache->free_space_ctl->free_space >= num_bytes));
5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452

	put_caching_control(caching_ctl);
	return 0;
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;
	DEFINE_WAIT(wait);

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
		return 0;

	wait_event(caching_ctl->wait, block_group_cache_done(cache));

	put_caching_control(caching_ctl);
J
Josef Bacik 已提交
5453 5454 5455
	return 0;
}

5456
static int __get_block_group_index(u64 flags)
5457 5458
{
	int index;
5459 5460

	if (flags & BTRFS_BLOCK_GROUP_RAID10)
5461
		index = 0;
5462
	else if (flags & BTRFS_BLOCK_GROUP_RAID1)
5463
		index = 1;
5464
	else if (flags & BTRFS_BLOCK_GROUP_DUP)
5465
		index = 2;
5466
	else if (flags & BTRFS_BLOCK_GROUP_RAID0)
5467 5468 5469
		index = 3;
	else
		index = 4;
5470

5471 5472 5473
	return index;
}

5474 5475 5476 5477 5478
static int get_block_group_index(struct btrfs_block_group_cache *cache)
{
	return __get_block_group_index(cache->flags);
}

J
Josef Bacik 已提交
5479
enum btrfs_loop_type {
5480 5481 5482 5483
	LOOP_CACHING_NOWAIT = 0,
	LOOP_CACHING_WAIT = 1,
	LOOP_ALLOC_CHUNK = 2,
	LOOP_NO_EMPTY_SIZE = 3,
J
Josef Bacik 已提交
5484 5485
};

5486 5487 5488 5489
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
 * ins->objectid == block start
5490
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
5491 5492 5493
 * ins->offset == number of blocks
 * Any available blocks before search_start are skipped.
 */
C
Chris Mason 已提交
5494
static noinline int find_free_extent(struct btrfs_trans_handle *trans,
5495 5496 5497
				     struct btrfs_root *orig_root,
				     u64 num_bytes, u64 empty_size,
				     u64 hint_byte, struct btrfs_key *ins,
5498
				     u64 data)
5499
{
5500
	int ret = 0;
C
Chris Mason 已提交
5501
	struct btrfs_root *root = orig_root->fs_info->extent_root;
5502
	struct btrfs_free_cluster *last_ptr = NULL;
5503
	struct btrfs_block_group_cache *block_group = NULL;
5504
	struct btrfs_block_group_cache *used_block_group;
5505
	u64 search_start = 0;
5506
	int empty_cluster = 2 * 1024 * 1024;
C
Chris Mason 已提交
5507
	int allowed_chunk_alloc = 0;
5508
	int done_chunk_alloc = 0;
5509
	struct btrfs_space_info *space_info;
5510
	int loop = 0;
5511
	int index = 0;
5512 5513
	int alloc_type = (data & BTRFS_BLOCK_GROUP_DATA) ?
		RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
J
Josef Bacik 已提交
5514
	bool found_uncached_bg = false;
5515
	bool failed_cluster_refill = false;
5516
	bool failed_alloc = false;
5517
	bool use_cluster = true;
5518
	bool have_caching_bg = false;
5519

5520
	WARN_ON(num_bytes < root->sectorsize);
5521
	btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
5522 5523
	ins->objectid = 0;
	ins->offset = 0;
5524

J
Josef Bacik 已提交
5525 5526
	trace_find_free_extent(orig_root, num_bytes, empty_size, data);

J
Josef Bacik 已提交
5527
	space_info = __find_space_info(root->fs_info, data);
5528
	if (!space_info) {
5529
		printk(KERN_ERR "No space info for %llu\n", data);
5530 5531
		return -ENOSPC;
	}
J
Josef Bacik 已提交
5532

5533 5534 5535 5536 5537 5538 5539
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

C
Chris Mason 已提交
5540 5541 5542
	if (orig_root->ref_cows || empty_size)
		allowed_chunk_alloc = 1;

5543
	if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
5544
		last_ptr = &root->fs_info->meta_alloc_cluster;
5545 5546
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
5547 5548
	}

5549 5550
	if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
	    btrfs_test_opt(root, SSD)) {
5551 5552
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
J
Josef Bacik 已提交
5553

5554
	if (last_ptr) {
5555 5556 5557 5558
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
5559
	}
5560

5561
	search_start = max(search_start, first_logical_byte(root, 0));
5562
	search_start = max(search_start, hint_byte);
5563

J
Josef Bacik 已提交
5564
	if (!last_ptr)
5565 5566
		empty_cluster = 0;

J
Josef Bacik 已提交
5567 5568 5569
	if (search_start == hint_byte) {
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
5570
		used_block_group = block_group;
J
Josef Bacik 已提交
5571 5572 5573
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
5574 5575 5576
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
5577 5578
		 */
		if (block_group && block_group_bits(block_group, data) &&
5579
		    block_group->cached != BTRFS_CACHE_NO) {
J
Josef Bacik 已提交
5580
			down_read(&space_info->groups_sem);
5581 5582 5583 5584 5585 5586 5587 5588 5589 5590
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
5591
			} else {
5592
				index = get_block_group_index(block_group);
5593
				goto have_block_group;
5594
			}
J
Josef Bacik 已提交
5595
		} else if (block_group) {
5596
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5597
		}
5598
	}
J
Josef Bacik 已提交
5599
search:
5600
	have_caching_bg = false;
5601
	down_read(&space_info->groups_sem);
5602 5603
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
5604
		u64 offset;
J
Josef Bacik 已提交
5605
		int cached;
5606

5607
		used_block_group = block_group;
5608
		btrfs_get_block_group(block_group);
J
Josef Bacik 已提交
5609
		search_start = block_group->key.objectid;
5610

5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
		if (!block_group_bits(block_group, data)) {
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
			if ((data & extra) && !(block_group->flags & extra))
				goto loop;
		}

J
Josef Bacik 已提交
5630
have_block_group:
5631 5632 5633
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached)) {
			found_uncached_bg = true;
5634
			ret = cache_block_group(block_group, trans,
5635
						orig_root, 0);
5636 5637
			BUG_ON(ret < 0);
			ret = 0;
J
Josef Bacik 已提交
5638 5639
		}

5640
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
5641
			goto loop;
J
Josef Bacik 已提交
5642

5643
		/*
5644 5645
		 * Ok we want to try and use the cluster allocator, so
		 * lets look there
5646
		 */
5647
		if (last_ptr) {
5648 5649 5650 5651 5652
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
			spin_lock(&last_ptr->refill_lock);
5653 5654 5655 5656 5657 5658
			used_block_group = last_ptr->block_group;
			if (used_block_group != block_group &&
			    (!used_block_group ||
			     used_block_group->ro ||
			     !block_group_bits(used_block_group, data))) {
				used_block_group = block_group;
5659
				goto refill_cluster;
5660 5661 5662 5663
			}

			if (used_block_group != block_group)
				btrfs_get_block_group(used_block_group);
5664

5665 5666
			offset = btrfs_alloc_from_cluster(used_block_group,
			  last_ptr, num_bytes, used_block_group->key.objectid);
5667 5668 5669
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
5670 5671
				trace_btrfs_reserve_extent_cluster(root,
					block_group, search_start, num_bytes);
5672 5673 5674
				goto checks;
			}

5675 5676 5677 5678
			WARN_ON(last_ptr->block_group != used_block_group);
			if (used_block_group != block_group) {
				btrfs_put_block_group(used_block_group);
				used_block_group = block_group;
5679
			}
5680
refill_cluster:
5681
			BUG_ON(used_block_group != block_group);
5682 5683 5684 5685 5686 5687 5688 5689
			/* If we are on LOOP_NO_EMPTY_SIZE, we can't
			 * set up a new clusters, so lets just skip it
			 * and let the allocator find whatever block
			 * it can find.  If we reach this point, we
			 * will have tried the cluster allocator
			 * plenty of times and not have found
			 * anything, so we are likely way too
			 * fragmented for the clustering stuff to find
5690 5691 5692 5693 5694 5695 5696 5697 5698
			 * anything.
			 *
			 * However, if the cluster is taken from the
			 * current block group, release the cluster
			 * first, so that we stand a better chance of
			 * succeeding in the unclustered
			 * allocation.  */
			if (loop >= LOOP_NO_EMPTY_SIZE &&
			    last_ptr->block_group != block_group) {
5699 5700 5701 5702
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

5703 5704 5705 5706 5707 5708
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

5709 5710 5711 5712 5713
			if (loop >= LOOP_NO_EMPTY_SIZE) {
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

5714
			/* allocate a cluster in this block group */
5715
			ret = btrfs_find_space_cluster(trans, root,
5716
					       block_group, last_ptr,
5717
					       search_start, num_bytes,
5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729
					       empty_cluster + empty_size);
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
						  last_ptr, num_bytes,
						  search_start);
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
5730 5731 5732
					trace_btrfs_reserve_extent_cluster(root,
						block_group, search_start,
						num_bytes);
5733 5734
					goto checks;
				}
5735 5736
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
5737 5738
				spin_unlock(&last_ptr->refill_lock);

5739
				failed_cluster_refill = true;
J
Josef Bacik 已提交
5740 5741 5742
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
5743
			}
J
Josef Bacik 已提交
5744

5745 5746 5747 5748 5749 5750
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
5751
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
5752
			spin_unlock(&last_ptr->refill_lock);
5753
			goto loop;
5754 5755
		}

5756
unclustered_alloc:
5757 5758 5759 5760 5761 5762 5763 5764 5765
		spin_lock(&block_group->free_space_ctl->tree_lock);
		if (cached &&
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_cluster + empty_size) {
			spin_unlock(&block_group->free_space_ctl->tree_lock);
			goto loop;
		}
		spin_unlock(&block_group->free_space_ctl->tree_lock);

5766 5767
		offset = btrfs_find_space_for_alloc(block_group, search_start,
						    num_bytes, empty_size);
5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
5779
			wait_block_group_cache_progress(block_group,
5780 5781
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
5782
			goto have_block_group;
5783
		} else if (!offset) {
5784 5785
			if (!cached)
				have_caching_bg = true;
5786
			goto loop;
J
Josef Bacik 已提交
5787
		}
5788
checks:
5789
		search_start = stripe_align(root, offset);
5790

J
Josef Bacik 已提交
5791 5792
		/* move on to the next group */
		if (search_start + num_bytes >
5793 5794
		    used_block_group->key.objectid + used_block_group->key.offset) {
			btrfs_add_free_space(used_block_group, offset, num_bytes);
J
Josef Bacik 已提交
5795
			goto loop;
5796
		}
5797

5798
		if (offset < search_start)
5799
			btrfs_add_free_space(used_block_group, offset,
5800 5801
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
5802

5803
		ret = btrfs_update_reserved_bytes(used_block_group, num_bytes,
5804
						  alloc_type);
5805
		if (ret == -EAGAIN) {
5806
			btrfs_add_free_space(used_block_group, offset, num_bytes);
J
Josef Bacik 已提交
5807
			goto loop;
J
Josef Bacik 已提交
5808
		}
5809

5810
		/* we are all good, lets return */
J
Josef Bacik 已提交
5811 5812
		ins->objectid = search_start;
		ins->offset = num_bytes;
5813

J
Josef Bacik 已提交
5814 5815
		trace_btrfs_reserve_extent(orig_root, block_group,
					   search_start, num_bytes);
5816
		if (offset < search_start)
5817
			btrfs_add_free_space(used_block_group, offset,
5818 5819
					     search_start - offset);
		BUG_ON(offset > search_start);
5820 5821
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
5822
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5823 5824
		break;
loop:
5825
		failed_cluster_refill = false;
5826
		failed_alloc = false;
5827
		BUG_ON(index != get_block_group_index(block_group));
5828 5829
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
5830
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5831 5832 5833
	}
	up_read(&space_info->groups_sem);

5834 5835 5836
	if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
		goto search;

5837 5838 5839
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

5840
	/*
5841 5842
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
5843 5844 5845 5846
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
5847
	 */
5848
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
5849
		index = 0;
5850
		loop++;
J
Josef Bacik 已提交
5851
		if (loop == LOOP_ALLOC_CHUNK) {
5852 5853 5854 5855
		       if (allowed_chunk_alloc) {
				ret = do_chunk_alloc(trans, root, num_bytes +
						     2 * 1024 * 1024, data,
						     CHUNK_ALLOC_LIMITED);
5856 5857 5858 5859 5860
				/*
				 * Do not bail out on ENOSPC since we
				 * can do more things.
				 */
				if (ret < 0 && ret != -ENOSPC) {
5861 5862 5863 5864
					btrfs_abort_transaction(trans,
								root, ret);
					goto out;
				}
5865 5866 5867 5868 5869 5870 5871 5872
				allowed_chunk_alloc = 0;
				if (ret == 1)
					done_chunk_alloc = 1;
			} else if (!done_chunk_alloc &&
				   space_info->force_alloc ==
				   CHUNK_ALLOC_NO_FORCE) {
				space_info->force_alloc = CHUNK_ALLOC_LIMITED;
			}
J
Josef Bacik 已提交
5873

5874 5875 5876 5877 5878 5879
		       /*
			* We didn't allocate a chunk, go ahead and drop the
			* empty size and loop again.
			*/
		       if (!done_chunk_alloc)
			       loop = LOOP_NO_EMPTY_SIZE;
J
Josef Bacik 已提交
5880 5881
		}

5882 5883 5884
		if (loop == LOOP_NO_EMPTY_SIZE) {
			empty_size = 0;
			empty_cluster = 0;
5885
		}
5886 5887

		goto search;
J
Josef Bacik 已提交
5888 5889
	} else if (!ins->objectid) {
		ret = -ENOSPC;
5890
	} else if (ins->objectid) {
5891
		ret = 0;
C
Chris Mason 已提交
5892
	}
5893
out:
C
Chris Mason 已提交
5894

C
Chris Mason 已提交
5895
	return ret;
5896
}
5897

J
Josef Bacik 已提交
5898 5899
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
5900 5901
{
	struct btrfs_block_group_cache *cache;
5902
	int index = 0;
J
Josef Bacik 已提交
5903

J
Josef Bacik 已提交
5904
	spin_lock(&info->lock);
5905 5906
	printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
	       (unsigned long long)info->flags,
C
Chris Mason 已提交
5907
	       (unsigned long long)(info->total_bytes - info->bytes_used -
J
Josef Bacik 已提交
5908
				    info->bytes_pinned - info->bytes_reserved -
5909
				    info->bytes_readonly),
C
Chris Mason 已提交
5910
	       (info->full) ? "" : "not ");
5911 5912
	printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
5913
	       (unsigned long long)info->total_bytes,
5914
	       (unsigned long long)info->bytes_used,
5915
	       (unsigned long long)info->bytes_pinned,
5916
	       (unsigned long long)info->bytes_reserved,
5917
	       (unsigned long long)info->bytes_may_use,
5918
	       (unsigned long long)info->bytes_readonly);
J
Josef Bacik 已提交
5919 5920 5921 5922
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
5923

5924
	down_read(&info->groups_sem);
5925 5926
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
5927
		spin_lock(&cache->lock);
5928
		printk(KERN_INFO "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s\n",
C
Chris Mason 已提交
5929 5930 5931 5932
		       (unsigned long long)cache->key.objectid,
		       (unsigned long long)cache->key.offset,
		       (unsigned long long)btrfs_block_group_used(&cache->item),
		       (unsigned long long)cache->pinned,
5933 5934
		       (unsigned long long)cache->reserved,
		       cache->ro ? "[readonly]" : "");
J
Josef Bacik 已提交
5935 5936 5937
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
5938 5939
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
5940
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
5941
}
5942

5943 5944 5945 5946
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
5947
			 struct btrfs_key *ins, u64 data)
5948
{
5949
	bool final_tried = false;
5950
	int ret;
5951

J
Josef Bacik 已提交
5952
	data = btrfs_get_alloc_profile(root, data);
5953
again:
C
Chris Mason 已提交
5954 5955 5956 5957
	/*
	 * the only place that sets empty_size is btrfs_realloc_node, which
	 * is not called recursively on allocations
	 */
5958
	if (empty_size || root->ref_cows) {
5959
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
5960 5961
				     num_bytes + 2 * 1024 * 1024, data,
				     CHUNK_ALLOC_NO_FORCE);
5962 5963 5964 5965 5966
		if (ret < 0 && ret != -ENOSPC) {
			btrfs_abort_transaction(trans, root, ret);
			return ret;
		}
	}
5967

5968 5969
	WARN_ON(num_bytes < root->sectorsize);
	ret = find_free_extent(trans, root, num_bytes, empty_size,
5970
			       hint_byte, ins, data);
5971

5972 5973 5974 5975 5976
	if (ret == -ENOSPC) {
		if (!final_tried) {
			num_bytes = num_bytes >> 1;
			num_bytes = num_bytes & ~(root->sectorsize - 1);
			num_bytes = max(num_bytes, min_alloc_size);
5977
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
5978
				       num_bytes, data, CHUNK_ALLOC_FORCE);
5979 5980 5981 5982
			if (ret < 0 && ret != -ENOSPC) {
				btrfs_abort_transaction(trans, root, ret);
				return ret;
			}
5983 5984 5985 5986 5987 5988 5989 5990 5991 5992
			if (num_bytes == min_alloc_size)
				final_tried = true;
			goto again;
		} else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
			struct btrfs_space_info *sinfo;

			sinfo = __find_space_info(root->fs_info, data);
			printk(KERN_ERR "btrfs allocation failed flags %llu, "
			       "wanted %llu\n", (unsigned long long)data,
			       (unsigned long long)num_bytes);
5993 5994
			if (sinfo)
				dump_space_info(sinfo, num_bytes, 1);
5995
		}
5996
	}
J
Josef Bacik 已提交
5997

5998 5999
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);

J
Josef Bacik 已提交
6000
	return ret;
6001 6002
}

6003 6004
static int __btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len, int pin)
6005
{
J
Josef Bacik 已提交
6006
	struct btrfs_block_group_cache *cache;
6007
	int ret = 0;
J
Josef Bacik 已提交
6008 6009 6010

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
C
Chris Mason 已提交
6011 6012
		printk(KERN_ERR "Unable to find block group for %llu\n",
		       (unsigned long long)start);
J
Josef Bacik 已提交
6013 6014
		return -ENOSPC;
	}
6015

6016 6017
	if (btrfs_test_opt(root, DISCARD))
		ret = btrfs_discard_extent(root, start, len, NULL);
6018

6019 6020 6021 6022 6023 6024
	if (pin)
		pin_down_extent(root, cache, start, len, 1);
	else {
		btrfs_add_free_space(cache, start, len);
		btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
	}
6025
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
6026

6027 6028
	trace_btrfs_reserved_extent_free(root, start, len);

6029 6030 6031
	return ret;
}

6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043
int btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 0);
}

int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
				       u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 1);
}

6044 6045 6046 6047 6048
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
6049 6050
{
	int ret;
6051
	struct btrfs_fs_info *fs_info = root->fs_info;
6052
	struct btrfs_extent_item *extent_item;
6053
	struct btrfs_extent_inline_ref *iref;
6054
	struct btrfs_path *path;
6055 6056 6057
	struct extent_buffer *leaf;
	int type;
	u32 size;
6058

6059 6060 6061 6062
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
6063

6064
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
6065 6066

	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
6067 6068
	if (!path)
		return -ENOMEM;
6069

6070
	path->leave_spinning = 1;
6071 6072
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6073 6074 6075 6076
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
J
Josef Bacik 已提交
6077

6078 6079
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
6080
				     struct btrfs_extent_item);
6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
6101 6102

	btrfs_mark_buffer_dirty(path->nodes[0]);
6103
	btrfs_free_path(path);
6104

6105
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
6106
	if (ret) { /* -ENOENT, logic error */
C
Chris Mason 已提交
6107 6108 6109
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
6110 6111
		BUG();
	}
6112 6113 6114
	return ret;
}

6115 6116 6117 6118 6119
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins)
6120 6121
{
	int ret;
6122 6123 6124 6125 6126 6127 6128
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
6129

6130
	path = btrfs_alloc_path();
6131 6132
	if (!path)
		return -ENOMEM;
6133

6134 6135 6136
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6137 6138 6139 6140
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
	block_info = (struct btrfs_tree_block_info *)(extent_item + 1);

	btrfs_set_tree_block_key(leaf, block_info, key);
	btrfs_set_tree_block_level(leaf, block_info, level);

	iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

6169
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
6170
	if (ret) { /* -ENOENT, logic error */
6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
		BUG();
	}
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

A
Arne Jansen 已提交
6188 6189 6190 6191
	ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
					 ins->offset, 0,
					 root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL, 0);
6192 6193
	return ret;
}
6194 6195 6196 6197 6198 6199

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
6200 6201 6202 6203
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
6204 6205 6206
{
	int ret;
	struct btrfs_block_group_cache *block_group;
6207 6208 6209
	struct btrfs_caching_control *caching_ctl;
	u64 start = ins->objectid;
	u64 num_bytes = ins->offset;
6210 6211

	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
6212
	cache_block_group(block_group, trans, NULL, 0);
6213
	caching_ctl = get_caching_control(block_group);
6214

6215 6216 6217
	if (!caching_ctl) {
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
6218
		BUG_ON(ret); /* -ENOMEM */
6219 6220 6221 6222 6223
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
6224
			BUG_ON(ret); /* -ENOMEM */
6225 6226 6227
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
6228
			BUG_ON(ret); /* -ENOMEM */
6229 6230 6231 6232
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
6233
			BUG_ON(ret); /* -ENOMEM */
6234 6235 6236 6237 6238

			start = caching_ctl->progress;
			num_bytes = ins->objectid + ins->offset -
				    caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
6239
			BUG_ON(ret); /* -ENOMEM */
6240 6241 6242 6243 6244 6245
		}

		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}

6246 6247
	ret = btrfs_update_reserved_bytes(block_group, ins->offset,
					  RESERVE_ALLOC_NO_ACCOUNT);
6248
	BUG_ON(ret); /* logic error */
6249
	btrfs_put_block_group(block_group);
6250 6251
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
6252 6253 6254
	return ret;
}

6255 6256
struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
					    struct btrfs_root *root,
6257 6258
					    u64 bytenr, u32 blocksize,
					    int level)
6259 6260 6261 6262 6263 6264 6265
{
	struct extent_buffer *buf;

	buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
6266
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
6267 6268
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
6269
	clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
6270 6271

	btrfs_set_lock_blocking(buf);
6272
	btrfs_set_buffer_uptodate(buf);
6273

6274
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
6275 6276 6277 6278 6279 6280 6281 6282 6283 6284
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
		if (root->log_transid % 2 == 0)
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
6285 6286
	} else {
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
6287
			 buf->start + buf->len - 1, GFP_NOFS);
6288
	}
6289
	trans->blocks_used++;
6290
	/* this returns a buffer locked for blocking */
6291 6292 6293
	return buf;
}

6294 6295 6296 6297 6298
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
6299
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
6300 6301 6302 6303 6304
	int ret;

	block_rsv = get_block_rsv(trans, root);

	if (block_rsv->size == 0) {
6305
		ret = reserve_metadata_bytes(root, block_rsv, blocksize, 0);
6306 6307 6308 6309 6310 6311 6312 6313
		/*
		 * If we couldn't reserve metadata bytes try and use some from
		 * the global reserve.
		 */
		if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
6314
			return ERR_PTR(ret);
6315
		} else if (ret) {
6316
			return ERR_PTR(ret);
6317
		}
6318 6319 6320 6321 6322 6323
		return block_rsv;
	}

	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;
6324
	if (ret) {
6325 6326 6327 6328 6329 6330 6331
		static DEFINE_RATELIMIT_STATE(_rs,
				DEFAULT_RATELIMIT_INTERVAL,
				/*DEFAULT_RATELIMIT_BURST*/ 2);
		if (__ratelimit(&_rs)) {
			printk(KERN_DEBUG "btrfs: block rsv returned %d\n", ret);
			WARN_ON(1);
		}
6332
		ret = reserve_metadata_bytes(root, block_rsv, blocksize, 0);
6333 6334 6335 6336 6337 6338 6339 6340
		if (!ret) {
			return block_rsv;
		} else if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
		}
	}
6341 6342 6343 6344

	return ERR_PTR(-ENOSPC);
}

J
Josef Bacik 已提交
6345 6346
static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
			    struct btrfs_block_rsv *block_rsv, u32 blocksize)
6347 6348
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
J
Josef Bacik 已提交
6349
	block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
6350 6351
}

6352
/*
6353 6354 6355 6356
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
6357 6358
 * returns the tree buffer or NULL.
 */
6359
struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
6360 6361 6362
					struct btrfs_root *root, u32 blocksize,
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
6363
					u64 hint, u64 empty_size)
6364
{
C
Chris Mason 已提交
6365
	struct btrfs_key ins;
6366
	struct btrfs_block_rsv *block_rsv;
6367
	struct extent_buffer *buf;
6368 6369 6370
	u64 flags = 0;
	int ret;

6371

6372 6373 6374 6375 6376
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

	ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
6377
				   empty_size, hint, &ins, 0);
6378
	if (ret) {
J
Josef Bacik 已提交
6379
		unuse_block_rsv(root->fs_info, block_rsv, blocksize);
6380
		return ERR_PTR(ret);
6381
	}
6382

6383 6384
	buf = btrfs_init_new_buffer(trans, root, ins.objectid,
				    blocksize, level);
6385
	BUG_ON(IS_ERR(buf)); /* -ENOMEM */
6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
		extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
6397
		BUG_ON(!extent_op); /* -ENOMEM */
6398 6399 6400 6401 6402 6403 6404 6405 6406
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
		extent_op->update_key = 1;
		extent_op->update_flags = 1;
		extent_op->is_data = 0;

A
Arne Jansen 已提交
6407 6408
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					ins.objectid,
6409 6410
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
6411
					extent_op, 0);
6412
		BUG_ON(ret); /* -ENOMEM */
6413
	}
6414 6415
	return buf;
}
6416

6417 6418 6419 6420 6421 6422 6423 6424 6425
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
6426 6427
	int reada_slot;
	int reada_count;
A
Arne Jansen 已提交
6428
	int for_reloc;
6429 6430 6431 6432 6433
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
6434 6435 6436 6437
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
6438
{
Y
Yan, Zheng 已提交
6439 6440 6441
	u64 bytenr;
	u64 generation;
	u64 refs;
6442
	u64 flags;
6443
	u32 nritems;
Y
Yan, Zheng 已提交
6444 6445 6446
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
6447
	int ret;
Y
Yan, Zheng 已提交
6448 6449
	int slot;
	int nread = 0;
6450

Y
Yan, Zheng 已提交
6451 6452 6453 6454 6455 6456 6457 6458
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
6459

Y
Yan, Zheng 已提交
6460 6461 6462
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
	blocksize = btrfs_level_size(root, wc->level - 1);
6463

Y
Yan, Zheng 已提交
6464 6465 6466
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
6467

C
Chris Mason 已提交
6468
		cond_resched();
Y
Yan, Zheng 已提交
6469 6470
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
6471

Y
Yan, Zheng 已提交
6472 6473
		if (slot == path->slots[wc->level])
			goto reada;
6474

Y
Yan, Zheng 已提交
6475 6476
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
6477 6478
			continue;

6479 6480 6481
		/* We don't lock the tree block, it's OK to be racy here */
		ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
					       &refs, &flags);
6482 6483 6484
		/* We don't care about errors in readahead. */
		if (ret < 0)
			continue;
6485 6486
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
6487 6488 6489
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
6490

6491 6492 6493
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
6494 6495 6496 6497 6498 6499 6500 6501
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
6502 6503 6504 6505
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
6506
		}
Y
Yan, Zheng 已提交
6507 6508 6509 6510
reada:
		ret = readahead_tree_block(root, bytenr, blocksize,
					   generation);
		if (ret)
6511
			break;
Y
Yan, Zheng 已提交
6512
		nread++;
C
Chris Mason 已提交
6513
	}
Y
Yan, Zheng 已提交
6514
	wc->reada_slot = slot;
C
Chris Mason 已提交
6515
}
6516

Y
Yan Zheng 已提交
6517
/*
6518 6519 6520 6521 6522 6523
 * hepler to process tree block while walking down the tree.
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
6524
 */
6525
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
6526
				   struct btrfs_root *root,
6527
				   struct btrfs_path *path,
6528
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
6529
{
6530 6531 6532
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
6533 6534
	int ret;

6535 6536 6537
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
6538

6539 6540 6541 6542
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
6543 6544 6545
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
6546 6547 6548 6549 6550
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
					       eb->start, eb->len,
					       &wc->refs[level],
					       &wc->flags[level]);
6551 6552 6553
		BUG_ON(ret == -ENOMEM);
		if (ret)
			return ret;
6554 6555
		BUG_ON(wc->refs[level] == 0);
	}
6556

6557 6558 6559
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
6560

6561
		if (path->locks[level] && !wc->keep_locks) {
6562
			btrfs_tree_unlock_rw(eb, path->locks[level]);
6563 6564 6565 6566
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
6567

6568 6569 6570
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
A
Arne Jansen 已提交
6571
		ret = btrfs_inc_ref(trans, root, eb, 1, wc->for_reloc);
6572
		BUG_ON(ret); /* -ENOMEM */
A
Arne Jansen 已提交
6573
		ret = btrfs_dec_ref(trans, root, eb, 0, wc->for_reloc);
6574
		BUG_ON(ret); /* -ENOMEM */
6575 6576
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
						  eb->len, flag, 0);
6577
		BUG_ON(ret); /* -ENOMEM */
6578 6579 6580 6581 6582 6583 6584 6585
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
6586
		btrfs_tree_unlock_rw(eb, path->locks[level]);
6587 6588 6589 6590 6591
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607
/*
 * hepler to process tree block pointer.
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
6608
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
6628 6629
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6630
		return 1;
6631
	}
Y
Yan, Zheng 已提交
6632 6633 6634 6635 6636 6637 6638

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
	blocksize = btrfs_level_size(root, level - 1);

	next = btrfs_find_tree_block(root, bytenr, blocksize);
	if (!next) {
		next = btrfs_find_create_tree_block(root, bytenr, blocksize);
6639 6640
		if (!next)
			return -ENOMEM;
Y
Yan, Zheng 已提交
6641 6642 6643 6644 6645
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

6646 6647 6648
	ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
6649 6650 6651 6652 6653
	if (ret < 0) {
		btrfs_tree_unlock(next);
		return ret;
	}

6654 6655
	BUG_ON(wc->refs[level - 1] == 0);
	*lookup_info = 0;
Y
Yan, Zheng 已提交
6656

6657
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
6658
		if (wc->refs[level - 1] > 1) {
6659 6660 6661 6662
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
6676 6677 6678 6679
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
6680 6681
	}

6682
	if (!btrfs_buffer_uptodate(next, generation, 0)) {
Y
Yan, Zheng 已提交
6683 6684 6685
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
6686
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6687 6688 6689 6690 6691 6692
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
		next = read_tree_block(root, bytenr, blocksize, generation);
6693 6694
		if (!next)
			return -EIO;
Y
Yan, Zheng 已提交
6695 6696 6697 6698 6699 6700 6701 6702
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
6703
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Y
Yan, Zheng 已提交
6704 6705 6706 6707 6708 6709 6710
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
6711 6712 6713 6714 6715 6716 6717 6718
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
6719

6720
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
A
Arne Jansen 已提交
6721
				root->root_key.objectid, level - 1, 0, 0);
6722
		BUG_ON(ret); /* -ENOMEM */
Y
Yan, Zheng 已提交
6723 6724 6725
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
6726
	*lookup_info = 1;
Y
Yan, Zheng 已提交
6727 6728 6729
	return 1;
}

6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746
/*
 * hepler to process tree block while walking up the tree.
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
6747
	int ret;
6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6774
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6775 6776 6777 6778 6779

			ret = btrfs_lookup_extent_info(trans, root,
						       eb->start, eb->len,
						       &wc->refs[level],
						       &wc->flags[level]);
6780 6781 6782 6783
			if (ret < 0) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
				return ret;
			}
6784 6785
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
6786
				btrfs_tree_unlock_rw(eb, path->locks[level]);
6787 6788
				return 1;
			}
Y
Yan Zheng 已提交
6789
		}
6790
	}
Y
Yan Zheng 已提交
6791

6792 6793
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6794

6795 6796 6797
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
A
Arne Jansen 已提交
6798 6799
				ret = btrfs_dec_ref(trans, root, eb, 1,
						    wc->for_reloc);
6800
			else
A
Arne Jansen 已提交
6801 6802
				ret = btrfs_dec_ref(trans, root, eb, 0,
						    wc->for_reloc);
6803
			BUG_ON(ret); /* -ENOMEM */
6804 6805 6806 6807 6808 6809
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6810
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
6827 6828
	}

6829
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
6830 6831 6832
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
6833
	return 0;
6834 6835 6836 6837 6838 6839 6840 6841
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
6842
	int lookup_info = 1;
6843 6844 6845
	int ret;

	while (level >= 0) {
6846
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
6847 6848 6849 6850 6851 6852
		if (ret > 0)
			break;

		if (level == 0)
			break;

6853 6854 6855 6856
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

6857
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
6858 6859 6860
		if (ret > 0) {
			path->slots[level]++;
			continue;
6861 6862
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
6863
		level = wc->level;
Y
Yan Zheng 已提交
6864 6865 6866 6867
	}
	return 0;
}

C
Chris Mason 已提交
6868
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6869
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
6870
				 struct btrfs_path *path,
6871
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
6872
{
6873
	int level = wc->level;
C
Chris Mason 已提交
6874
	int ret;
6875

6876 6877 6878 6879 6880 6881
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
6882 6883
			return 0;
		} else {
6884 6885 6886
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
6887

6888
			if (path->locks[level]) {
6889 6890
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
6891
				path->locks[level] = 0;
Y
Yan Zheng 已提交
6892
			}
6893 6894 6895
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
6896 6897 6898 6899 6900
		}
	}
	return 1;
}

C
Chris Mason 已提交
6901
/*
6902 6903 6904 6905 6906 6907 6908 6909 6910
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
C
Chris Mason 已提交
6911
 */
6912
int btrfs_drop_snapshot(struct btrfs_root *root,
A
Arne Jansen 已提交
6913 6914
			 struct btrfs_block_rsv *block_rsv, int update_ref,
			 int for_reloc)
C
Chris Mason 已提交
6915
{
6916
	struct btrfs_path *path;
6917 6918
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
6919
	struct btrfs_root_item *root_item = &root->root_item;
6920 6921 6922 6923 6924
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
C
Chris Mason 已提交
6925

6926
	path = btrfs_alloc_path();
6927 6928 6929 6930
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
C
Chris Mason 已提交
6931

6932
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
6933 6934
	if (!wc) {
		btrfs_free_path(path);
6935 6936
		err = -ENOMEM;
		goto out;
6937
	}
6938

6939
	trans = btrfs_start_transaction(tree_root, 0);
6940 6941 6942 6943
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
		goto out_free;
	}
6944

6945 6946
	if (block_rsv)
		trans->block_rsv = block_rsv;
6947

6948
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
6949
		level = btrfs_header_level(root->node);
6950 6951
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
6952
		path->slots[level] = 0;
6953
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6954 6955
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
6956 6957
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
6958 6959 6960
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

6961
		level = root_item->drop_level;
6962
		BUG_ON(level == 0);
6963
		path->lowest_level = level;
6964 6965 6966 6967
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
6968
			goto out_end_trans;
6969
		}
Y
Yan, Zheng 已提交
6970
		WARN_ON(ret > 0);
6971

6972 6973 6974 6975
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
6976
		btrfs_unlock_up_safe(path, 0);
6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
						path->nodes[level]->len,
						&wc->refs[level],
						&wc->flags[level]);
6988 6989 6990 6991
			if (ret < 0) {
				err = ret;
				goto out_end_trans;
			}
6992 6993 6994 6995 6996 6997 6998 6999 7000
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
7001
	}
7002 7003 7004 7005 7006 7007

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
A
Arne Jansen 已提交
7008
	wc->for_reloc = for_reloc;
Y
Yan, Zheng 已提交
7009
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
7010

C
Chris Mason 已提交
7011
	while (1) {
7012 7013 7014
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
7015
			break;
7016
		}
C
Chris Mason 已提交
7017

7018 7019 7020
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
7021
			break;
7022 7023 7024 7025
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
7026 7027
			break;
		}
7028 7029 7030 7031 7032 7033 7034 7035 7036 7037

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
7038
		if (btrfs_should_end_transaction(trans, tree_root)) {
7039 7040 7041
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
7042 7043 7044 7045 7046
			if (ret) {
				btrfs_abort_transaction(trans, tree_root, ret);
				err = ret;
				goto out_end_trans;
			}
7047

7048
			btrfs_end_transaction_throttle(trans, tree_root);
7049
			trans = btrfs_start_transaction(tree_root, 0);
7050 7051 7052 7053
			if (IS_ERR(trans)) {
				err = PTR_ERR(trans);
				goto out_free;
			}
7054 7055
			if (block_rsv)
				trans->block_rsv = block_rsv;
7056
		}
C
Chris Mason 已提交
7057
	}
7058
	btrfs_release_path(path);
7059 7060
	if (err)
		goto out_end_trans;
7061 7062

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
7063 7064 7065 7066
	if (ret) {
		btrfs_abort_transaction(trans, tree_root, ret);
		goto out_end_trans;
	}
7067

7068 7069 7070
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
		ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
					   NULL, NULL);
7071 7072 7073 7074 7075
		if (ret < 0) {
			btrfs_abort_transaction(trans, tree_root, ret);
			err = ret;
			goto out_end_trans;
		} else if (ret > 0) {
7076 7077 7078 7079 7080 7081 7082
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
7083 7084 7085 7086 7087 7088 7089 7090 7091 7092
		}
	}

	if (root->in_radix) {
		btrfs_free_fs_root(tree_root->fs_info, root);
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
		kfree(root);
	}
7093
out_end_trans:
7094
	btrfs_end_transaction_throttle(trans, tree_root);
7095
out_free:
7096
	kfree(wc);
7097
	btrfs_free_path(path);
7098 7099 7100
out:
	if (err)
		btrfs_std_error(root->fs_info, err);
7101
	return err;
C
Chris Mason 已提交
7102
}
C
Chris Mason 已提交
7103

7104 7105 7106 7107
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
A
Arne Jansen 已提交
7108
 * only used by relocation code
7109
 */
Y
Yan Zheng 已提交
7110 7111 7112 7113 7114 7115
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
7116
	struct walk_control *wc;
Y
Yan Zheng 已提交
7117 7118 7119 7120 7121
	int level;
	int parent_level;
	int ret = 0;
	int wret;

7122 7123
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
7124
	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
7125 7126
	if (!path)
		return -ENOMEM;
Y
Yan Zheng 已提交
7127

7128
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
T
Tsutomu Itoh 已提交
7129 7130 7131 7132
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
7133

7134
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
7135 7136 7137 7138 7139
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

7140
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
7141 7142 7143
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
7144
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7145 7146 7147 7148 7149 7150 7151 7152

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
A
Arne Jansen 已提交
7153
	wc->for_reloc = 1;
Y
Yan, Zheng 已提交
7154
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
7155 7156

	while (1) {
7157 7158
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
7159 7160
			ret = wret;
			break;
7161
		}
Y
Yan Zheng 已提交
7162

7163
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
7164 7165 7166 7167 7168 7169
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

7170
	kfree(wc);
Y
Yan Zheng 已提交
7171 7172 7173 7174
	btrfs_free_path(path);
	return ret;
}

7175 7176 7177
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
7178
	u64 stripped;
7179

7180 7181 7182 7183 7184 7185 7186
	/*
	 * if restripe for this chunk_type is on pick target profile and
	 * return, otherwise do the usual balance
	 */
	stripped = get_restripe_target(root->fs_info, flags);
	if (stripped)
		return extended_to_chunk(stripped);
7187

7188 7189 7190 7191 7192 7193 7194 7195
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;

7196 7197 7198
	stripped = BTRFS_BLOCK_GROUP_RAID0 |
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

7223
		/* this is drive concat, leave it alone */
7224
	}
7225

7226 7227 7228
	return flags;
}

7229
static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
C
Chris Mason 已提交
7230
{
7231 7232
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
7233
	u64 min_allocable_bytes;
7234
	int ret = -ENOSPC;
C
Chris Mason 已提交
7235

7236

7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
		min_allocable_bytes = 1 * 1024 * 1024;
	else
		min_allocable_bytes = 0;

7249 7250
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
7251 7252 7253 7254 7255 7256

	if (cache->ro) {
		ret = 0;
		goto out;
	}

7257 7258 7259 7260
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
7261 7262
	    sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
	    min_allocable_bytes <= sinfo->total_bytes) {
7263 7264 7265 7266
		sinfo->bytes_readonly += num_bytes;
		cache->ro = 1;
		ret = 0;
	}
7267
out:
7268 7269 7270 7271
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
7272

7273 7274
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
7275

7276 7277 7278 7279
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
7280

7281
	BUG_ON(cache->ro);
C
Chris Mason 已提交
7282

C
Chris Mason 已提交
7283
	trans = btrfs_join_transaction(root);
7284 7285
	if (IS_ERR(trans))
		return PTR_ERR(trans);
7286

7287
	alloc_flags = update_block_group_flags(root, cache->flags);
7288 7289 7290 7291 7292 7293
	if (alloc_flags != cache->flags) {
		ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
				     CHUNK_ALLOC_FORCE);
		if (ret < 0)
			goto out;
	}
7294

7295
	ret = set_block_group_ro(cache, 0);
7296 7297 7298
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
7299 7300
	ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			     CHUNK_ALLOC_FORCE);
7301 7302
	if (ret < 0)
		goto out;
7303
	ret = set_block_group_ro(cache, 0);
7304 7305 7306 7307
out:
	btrfs_end_transaction(trans, root);
	return ret;
}
7308

7309 7310 7311 7312
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
7313 7314
	return do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			      CHUNK_ALLOC_FORCE);
7315 7316
}

7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372
/*
 * helper to account the unused space of all the readonly block group in the
 * list. takes mirrors into account.
 */
static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

	list_for_each_entry(block_group, groups_list, list) {
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}

	return free_bytes;
}

/*
 * helper to account the unused space of all the readonly block group in the
 * space_info. takes mirrors into account.
 */
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
{
	int i;
	u64 free_bytes = 0;

	spin_lock(&sinfo->lock);

	for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		if (!list_empty(&sinfo->block_groups[i]))
			free_bytes += __btrfs_get_ro_block_group_free_space(
						&sinfo->block_groups[i]);

	spin_unlock(&sinfo->lock);

	return free_bytes;
}

7373
void btrfs_set_block_group_rw(struct btrfs_root *root,
7374
			      struct btrfs_block_group_cache *cache)
7375
{
7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
7389 7390
}

7391 7392 7393 7394 7395 7396 7397
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
7398
{
7399 7400 7401 7402
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
7403
	u64 min_free;
J
Josef Bacik 已提交
7404 7405
	u64 dev_min = 1;
	u64 dev_nr = 0;
7406
	u64 target;
7407
	int index;
7408 7409
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
7410

7411
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
7412

7413 7414 7415
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
Z
Zheng Yan 已提交
7416

7417 7418
	min_free = btrfs_block_group_used(&block_group->item);

7419
	/* no bytes used, we're good */
7420
	if (!min_free)
Z
Zheng Yan 已提交
7421 7422
		goto out;

7423 7424
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
7425

7426
	full = space_info->full;
7427

7428 7429
	/*
	 * if this is the last block group we have in this space, we can't
7430 7431 7432 7433
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
7434
	 */
7435
	if ((space_info->total_bytes != block_group->key.offset) &&
7436 7437 7438
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
7439 7440
		spin_unlock(&space_info->lock);
		goto out;
7441
	}
7442
	spin_unlock(&space_info->lock);
7443

7444 7445 7446
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
7447 7448 7449
	 * alloc devices and guess if we have enough space.  if this block
	 * group is going to be restriped, run checks against the target
	 * profile instead of the current one.
7450 7451
	 */
	ret = -1;
7452

7453 7454 7455 7456 7457 7458 7459 7460
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474
	target = get_restripe_target(root->fs_info, block_group->flags);
	if (target) {
		index = __get_block_group_index(extended_to_chunk(target));
	} else {
		/*
		 * this is just a balance, so if we were marked as full
		 * we know there is no space for a new chunk
		 */
		if (full)
			goto out;

		index = get_block_group_index(block_group);
	}

7475 7476
	if (index == 0) {
		dev_min = 4;
J
Josef Bacik 已提交
7477 7478
		/* Divide by 2 */
		min_free >>= 1;
7479 7480 7481
	} else if (index == 1) {
		dev_min = 2;
	} else if (index == 2) {
J
Josef Bacik 已提交
7482 7483
		/* Multiply by 2 */
		min_free <<= 1;
7484 7485
	} else if (index == 3) {
		dev_min = fs_devices->rw_devices;
J
Josef Bacik 已提交
7486
		do_div(min_free, dev_min);
7487 7488
	}

7489 7490
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
7491
		u64 dev_offset;
7492

7493 7494 7495 7496 7497
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
		if (device->total_bytes > device->bytes_used + min_free) {
7498
			ret = find_free_dev_extent(device, min_free,
7499
						   &dev_offset, NULL);
7500
			if (!ret)
7501 7502 7503
				dev_nr++;

			if (dev_nr >= dev_min)
7504
				break;
7505

7506
			ret = -1;
7507
		}
7508
	}
7509
	mutex_unlock(&root->fs_info->chunk_mutex);
7510
out:
7511
	btrfs_put_block_group(block_group);
7512 7513 7514
	return ret;
}

7515 7516
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
7517
{
7518
	int ret = 0;
7519 7520 7521
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
7522

7523 7524
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
7525 7526
		goto out;

C
Chris Mason 已提交
7527
	while (1) {
7528
		slot = path->slots[0];
7529
		leaf = path->nodes[0];
7530 7531 7532 7533 7534
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
7535
				goto out;
7536
			break;
7537
		}
7538
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
7539

7540
		if (found_key.objectid >= key->objectid &&
7541 7542 7543 7544
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
7545
		path->slots[0]++;
7546
	}
7547
out:
7548
	return ret;
7549 7550
}

7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
7585 7586 7587
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
7588
	struct btrfs_space_info *space_info;
7589
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
7590 7591
	struct rb_node *n;

7592 7593 7594 7595 7596 7597 7598 7599 7600
	down_write(&info->extent_commit_sem);
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
	up_write(&info->extent_commit_sem);

Z
Zheng Yan 已提交
7601 7602 7603 7604 7605 7606
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
Y
Yan Zheng 已提交
7607 7608
		spin_unlock(&info->block_group_cache_lock);

7609
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
7610
		list_del(&block_group->list);
7611
		up_write(&block_group->space_info->groups_sem);
7612

J
Josef Bacik 已提交
7613
		if (block_group->cached == BTRFS_CACHE_STARTED)
7614
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
7615

7616 7617 7618 7619 7620 7621 7622
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
		if (block_group->cached == BTRFS_CACHE_NO)
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
7623
		btrfs_remove_free_space_cache(block_group);
7624
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
7625 7626

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
7627 7628
	}
	spin_unlock(&info->block_group_cache_lock);
7629 7630 7631 7632 7633 7634 7635 7636 7637

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

7638 7639
	release_global_block_rsv(info);

7640 7641 7642 7643
	while(!list_empty(&info->space_info)) {
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
7644
		if (space_info->bytes_pinned > 0 ||
7645 7646
		    space_info->bytes_reserved > 0 ||
		    space_info->bytes_may_use > 0) {
7647 7648 7649
			WARN_ON(1);
			dump_space_info(space_info, 0, 0);
		}
7650 7651 7652
		list_del(&space_info->list);
		kfree(space_info);
	}
Z
Zheng Yan 已提交
7653 7654 7655
	return 0;
}

7656 7657 7658 7659 7660 7661 7662 7663 7664 7665
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);

	down_write(&space_info->groups_sem);
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);
}

C
Chris Mason 已提交
7666 7667 7668 7669 7670
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
7671
	struct btrfs_fs_info *info = root->fs_info;
7672
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
7673 7674
	struct btrfs_key key;
	struct btrfs_key found_key;
7675
	struct extent_buffer *leaf;
7676 7677
	int need_clear = 0;
	u64 cache_gen;
7678

C
Chris Mason 已提交
7679
	root = info->extent_root;
C
Chris Mason 已提交
7680
	key.objectid = 0;
7681
	key.offset = 0;
C
Chris Mason 已提交
7682 7683 7684 7685
	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
J
Josef Bacik 已提交
7686
	path->reada = 1;
C
Chris Mason 已提交
7687

7688
	cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
7689
	if (btrfs_test_opt(root, SPACE_CACHE) &&
7690
	    btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
7691
		need_clear = 1;
7692 7693
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
7694

C
Chris Mason 已提交
7695
	while (1) {
7696
		ret = find_first_block_group(root, path, &key);
7697 7698
		if (ret > 0)
			break;
7699 7700
		if (ret != 0)
			goto error;
7701 7702
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7703
		cache = kzalloc(sizeof(*cache), GFP_NOFS);
C
Chris Mason 已提交
7704
		if (!cache) {
7705
			ret = -ENOMEM;
7706
			goto error;
C
Chris Mason 已提交
7707
		}
7708 7709 7710 7711 7712 7713 7714
		cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
						GFP_NOFS);
		if (!cache->free_space_ctl) {
			kfree(cache);
			ret = -ENOMEM;
			goto error;
		}
C
Chris Mason 已提交
7715

7716
		atomic_set(&cache->count, 1);
7717
		spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7718
		cache->fs_info = info;
J
Josef Bacik 已提交
7719
		INIT_LIST_HEAD(&cache->list);
7720
		INIT_LIST_HEAD(&cache->cluster_list);
7721

7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732
		if (need_clear) {
			/*
			 * When we mount with old space cache, we need to
			 * set BTRFS_DC_CLEAR and set dirty flag.
			 *
			 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
			 *    truncate the old free space cache inode and
			 *    setup a new one.
			 * b) Setting 'dirty flag' makes sure that we flush
			 *    the new space cache info onto disk.
			 */
7733
			cache->disk_cache_state = BTRFS_DC_CLEAR;
7734 7735 7736
			if (btrfs_test_opt(root, SPACE_CACHE))
				cache->dirty = 1;
		}
7737

7738 7739 7740
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
C
Chris Mason 已提交
7741
		memcpy(&cache->key, &found_key, sizeof(found_key));
7742

C
Chris Mason 已提交
7743
		key.objectid = found_key.objectid + found_key.offset;
7744
		btrfs_release_path(path);
7745
		cache->flags = btrfs_block_group_flags(&cache->item);
J
Josef Bacik 已提交
7746 7747
		cache->sectorsize = root->sectorsize;

7748 7749
		btrfs_init_free_space_ctl(cache);

7750 7751 7752 7753 7754 7755 7756
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
		exclude_super_stripes(root, cache);

J
Josef Bacik 已提交
7757 7758 7759 7760 7761 7762 7763 7764
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
7765
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7766
			cache->cached = BTRFS_CACHE_FINISHED;
7767
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7768
		} else if (btrfs_block_group_used(&cache->item) == 0) {
7769
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7770 7771 7772 7773 7774
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
7775
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7776
		}
7777

7778 7779 7780
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
7781
		BUG_ON(ret); /* -ENOMEM */
7782
		cache->space_info = space_info;
7783
		spin_lock(&cache->space_info->lock);
7784
		cache->space_info->bytes_readonly += cache->bytes_super;
7785 7786
		spin_unlock(&cache->space_info->lock);

7787
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
7788 7789

		ret = btrfs_add_block_group_cache(root->fs_info, cache);
7790
		BUG_ON(ret); /* Logic error */
7791 7792

		set_avail_alloc_bits(root->fs_info, cache->flags);
Y
Yan Zheng 已提交
7793
		if (btrfs_chunk_readonly(root, cache->key.objectid))
7794
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7795
	}
7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
		list_for_each_entry(cache, &space_info->block_groups[3], list)
7808
			set_block_group_ro(cache, 1);
7809
		list_for_each_entry(cache, &space_info->block_groups[4], list)
7810
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7811
	}
7812 7813

	init_global_block_rsv(info);
7814 7815
	ret = 0;
error:
C
Chris Mason 已提交
7816
	btrfs_free_path(path);
7817
	return ret;
C
Chris Mason 已提交
7818
}
7819 7820 7821

int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
7822
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
7823 7824 7825 7826 7827 7828 7829 7830
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

7831
	root->fs_info->last_trans_log_full_commit = trans->transid;
7832

7833
	cache = kzalloc(sizeof(*cache), GFP_NOFS);
J
Josef Bacik 已提交
7834 7835
	if (!cache)
		return -ENOMEM;
7836 7837 7838 7839 7840 7841
	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return -ENOMEM;
	}
J
Josef Bacik 已提交
7842

7843
	cache->key.objectid = chunk_offset;
7844
	cache->key.offset = size;
7845
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
7846
	cache->sectorsize = root->sectorsize;
7847
	cache->fs_info = root->fs_info;
7848

7849
	atomic_set(&cache->count, 1);
7850
	spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7851
	INIT_LIST_HEAD(&cache->list);
7852
	INIT_LIST_HEAD(&cache->cluster_list);
C
Chris Mason 已提交
7853

7854 7855
	btrfs_init_free_space_ctl(cache);

7856 7857 7858 7859 7860
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	cache->flags = type;
	btrfs_set_block_group_flags(&cache->item, type);

7861
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7862
	cache->cached = BTRFS_CACHE_FINISHED;
7863
	exclude_super_stripes(root, cache);
7864

J
Josef Bacik 已提交
7865 7866 7867
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

7868 7869
	free_excluded_extents(root, cache);

7870 7871
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
7872
	BUG_ON(ret); /* -ENOMEM */
7873
	update_global_block_rsv(root->fs_info);
7874 7875

	spin_lock(&cache->space_info->lock);
7876
	cache->space_info->bytes_readonly += cache->bytes_super;
7877 7878
	spin_unlock(&cache->space_info->lock);

7879
	__link_block_group(cache->space_info, cache);
7880

J
Josef Bacik 已提交
7881
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
7882
	BUG_ON(ret); /* Logic error */
7883

7884 7885
	ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
				sizeof(cache->item));
7886 7887 7888 7889
	if (ret) {
		btrfs_abort_transaction(trans, extent_root, ret);
		return ret;
	}
7890

C
Chris Mason 已提交
7891
	set_avail_alloc_bits(extent_root->fs_info, type);
7892

7893 7894
	return 0;
}
Z
Zheng Yan 已提交
7895

7896 7897
static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
7898 7899
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
7900 7901 7902 7903 7904 7905 7906 7907 7908

	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits &= ~extra_flags;
}

Z
Zheng Yan 已提交
7909 7910 7911 7912 7913
int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 group_start)
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
7914
	struct btrfs_free_cluster *cluster;
7915
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
7916
	struct btrfs_key key;
7917
	struct inode *inode;
Z
Zheng Yan 已提交
7918
	int ret;
7919
	int index;
J
Josef Bacik 已提交
7920
	int factor;
Z
Zheng Yan 已提交
7921 7922 7923 7924 7925

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
7926
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
7927

7928 7929 7930 7931 7932 7933
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

Z
Zheng Yan 已提交
7934
	memcpy(&key, &block_group->key, sizeof(key));
7935
	index = get_block_group_index(block_group);
J
Josef Bacik 已提交
7936 7937 7938 7939 7940 7941
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
7942

7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
7958
	path = btrfs_alloc_path();
7959 7960 7961 7962
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
Z
Zheng Yan 已提交
7963

7964
	inode = lookup_free_space_inode(tree_root, block_group, path);
7965
	if (!IS_ERR(inode)) {
7966
		ret = btrfs_orphan_add(trans, inode);
7967 7968 7969 7970
		if (ret) {
			btrfs_add_delayed_iput(inode);
			goto out;
		}
7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
7983
		btrfs_add_delayed_iput(inode);
7984 7985 7986 7987 7988 7989 7990 7991 7992 7993
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
7994
		btrfs_release_path(path);
7995 7996 7997 7998
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
7999
		btrfs_release_path(path);
8000 8001
	}

8002
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
8003 8004
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
8005
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
8006

8007
	down_write(&block_group->space_info->groups_sem);
8008 8009 8010 8011 8012
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
8013 8014
	if (list_empty(&block_group->space_info->block_groups[index]))
		clear_avail_alloc_bits(root->fs_info, block_group->flags);
8015
	up_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
8016

J
Josef Bacik 已提交
8017
	if (block_group->cached == BTRFS_CACHE_STARTED)
8018
		wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
8019 8020 8021

	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
8022 8023 8024
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
8025
	block_group->space_info->disk_total -= block_group->key.offset * factor;
Y
Yan Zheng 已提交
8026
	spin_unlock(&block_group->space_info->lock);
8027

8028 8029
	memcpy(&key, &block_group->key, sizeof(key));

8030
	btrfs_clear_space_info_full(root->fs_info);
Y
Yan Zheng 已提交
8031

8032 8033
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
8046

8047 8048 8049
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
8050 8051 8052 8053
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
8054 8055
	int ret;

8056
	disk_super = fs_info->super_copy;
8057 8058
	if (!btrfs_super_root(disk_super))
		return 1;
8059

8060 8061 8062
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
8063

8064 8065
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
	ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8066
	if (ret)
8067
		goto out;
8068

8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	}
out:
8082 8083 8084
	return ret;
}

L
liubo 已提交
8085 8086 8087 8088 8089 8090
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
	return unpin_extent_range(root, start, end);
}

int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
8091
			       u64 num_bytes, u64 *actual_bytes)
L
liubo 已提交
8092
{
8093
	return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
L
liubo 已提交
8094
}
8095 8096 8097 8098 8099 8100 8101 8102 8103

int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
8104
	u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
8105 8106
	int ret = 0;

8107 8108 8109 8110 8111 8112 8113
	/*
	 * try to trim all FS space, our block group may start from non-zero.
	 */
	if (range->len == total_bytes)
		cache = btrfs_lookup_first_block_group(fs_info, range->start);
	else
		cache = btrfs_lookup_block_group(fs_info, range->start);
8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146 8147 8148 8149

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
				ret = cache_block_group(cache, NULL, root, 0);
				if (!ret)
					wait_block_group_cache_done(cache);
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

	range->len = trimmed;
	return ret;
}