extent-tree.c 215.6 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
26
#include <linux/ratelimit.h>
C
Chris Mason 已提交
27
#include "compat.h"
28
#include "hash.h"
29 30 31
#include "ctree.h"
#include "disk-io.h"
#include "print-tree.h"
32
#include "transaction.h"
33
#include "volumes.h"
34
#include "locking.h"
35
#include "free-space-cache.h"
36
#include "math.h"
37

38 39
#undef SCRAMBLE_DELAYED_REFS

40 41
/*
 * control flags for do_chunk_alloc's force field
42 43 44 45 46 47 48 49 50
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
51 52
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
53 54 55
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
56 57
	CHUNK_ALLOC_LIMITED = 1,
	CHUNK_ALLOC_FORCE = 2,
58 59
};

60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
/*
 * Control how reservations are dealt with.
 *
 * RESERVE_FREE - freeing a reservation.
 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
 *   ENOSPC accounting
 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
 *   bytes_may_use as the ENOSPC accounting is done elsewhere
 */
enum {
	RESERVE_FREE = 0,
	RESERVE_ALLOC = 1,
	RESERVE_ALLOC_NO_ACCOUNT = 2,
};

75 76
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
77
			      u64 bytenr, u64 num_bytes, int alloc);
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extra_op);
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins);
J
Josef Bacik 已提交
97
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
98 99
			  struct btrfs_root *extent_root, u64 flags,
			  int force);
100 101
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
102 103
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
104 105
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve);
J
Josef Bacik 已提交
106

J
Josef Bacik 已提交
107 108 109 110 111 112 113
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
	return cache->cached == BTRFS_CACHE_FINISHED;
}

J
Josef Bacik 已提交
114 115 116 117 118
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

119
static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
120 121 122 123 124 125
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
126 127 128
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
129
		kfree(cache->free_space_ctl);
130
		kfree(cache);
131
	}
132 133
}

J
Josef Bacik 已提交
134 135 136 137
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
138
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
206
	if (ret)
207
		btrfs_get_block_group(ret);
J
Josef Bacik 已提交
208 209 210 211 212
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

213 214
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
215
{
216 217 218 219 220 221 222
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
J
Josef Bacik 已提交
223

224 225 226 227
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
228

229 230 231 232 233 234 235
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
J
Josef Bacik 已提交
236 237
}

238 239
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
240 241 242 243 244 245
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

246 247 248 249 250
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
251
		BUG_ON(ret); /* -ENOMEM */
252 253
	}

J
Josef Bacik 已提交
254 255 256 257 258
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
259
		BUG_ON(ret); /* -ENOMEM */
260

J
Josef Bacik 已提交
261
		while (nr--) {
262
			cache->bytes_super += stripe_len;
263 264
			ret = add_excluded_extent(root, logical[nr],
						  stripe_len);
265
			BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
266
		}
267

J
Josef Bacik 已提交
268 269 270 271 272
		kfree(logical);
	}
	return 0;
}

273 274 275 276 277 278 279 280 281 282 283
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_STARTED) {
		spin_unlock(&cache->lock);
		return NULL;
	}

284 285 286
	/* We're loading it the fast way, so we don't have a caching_ctl. */
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

J
Josef Bacik 已提交
302 303 304 305 306
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
J
Josef Bacik 已提交
307
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
J
Josef Bacik 已提交
308 309
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
J
Josef Bacik 已提交
310
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
311 312 313
	int ret;

	while (start < end) {
314
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
315
					    &extent_start, &extent_end,
316 317
					    EXTENT_DIRTY | EXTENT_UPTODATE,
					    NULL);
J
Josef Bacik 已提交
318 319 320
		if (ret)
			break;

321
		if (extent_start <= start) {
J
Josef Bacik 已提交
322 323 324
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
325
			total_added += size;
326 327
			ret = btrfs_add_free_space(block_group, start,
						   size);
328
			BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
329 330 331 332 333 334 335 336
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
337
		total_added += size;
338
		ret = btrfs_add_free_space(block_group, start, size);
339
		BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
340 341
	}

J
Josef Bacik 已提交
342
	return total_added;
J
Josef Bacik 已提交
343 344
}

345
static noinline void caching_thread(struct btrfs_work *work)
346
{
347 348 349 350
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_root *extent_root;
351
	struct btrfs_path *path;
352
	struct extent_buffer *leaf;
353
	struct btrfs_key key;
J
Josef Bacik 已提交
354
	u64 total_found = 0;
355 356 357
	u64 last = 0;
	u32 nritems;
	int ret = 0;
358

359 360 361 362 363
	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

364 365
	path = btrfs_alloc_path();
	if (!path)
366
		goto out;
367

J
Josef Bacik 已提交
368
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
369

370
	/*
J
Josef Bacik 已提交
371 372 373 374
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
375 376
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
377
	path->search_commit_root = 1;
J
Josef Bacik 已提交
378
	path->reada = 1;
J
Josef Bacik 已提交
379

Y
Yan Zheng 已提交
380
	key.objectid = last;
381
	key.offset = 0;
382
	key.type = BTRFS_EXTENT_ITEM_KEY;
383
again:
384
	mutex_lock(&caching_ctl->mutex);
385 386 387
	/* need to make sure the commit_root doesn't disappear */
	down_read(&fs_info->extent_commit_sem);

388
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
389
	if (ret < 0)
390
		goto err;
Y
Yan Zheng 已提交
391

392 393 394
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
395
	while (1) {
396
		if (btrfs_fs_closing(fs_info) > 1) {
397
			last = (u64)-1;
J
Josef Bacik 已提交
398
			break;
399
		}
J
Josef Bacik 已提交
400

401 402 403 404 405
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
406
				break;
J
Josef Bacik 已提交
407

408 409 410
			if (need_resched() ||
			    btrfs_next_leaf(extent_root, path)) {
				caching_ctl->progress = last;
C
Chris Mason 已提交
411
				btrfs_release_path(path);
412 413
				up_read(&fs_info->extent_commit_sem);
				mutex_unlock(&caching_ctl->mutex);
414
				cond_resched();
415 416 417 418 419
				goto again;
			}
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
420
		}
J
Josef Bacik 已提交
421

422 423
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
424
			continue;
425
		}
J
Josef Bacik 已提交
426

427
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
428
		    block_group->key.offset)
429
			break;
430

431
		if (key.type == BTRFS_EXTENT_ITEM_KEY) {
J
Josef Bacik 已提交
432 433 434
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
435
			last = key.objectid + key.offset;
J
Josef Bacik 已提交
436

437 438 439 440
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
J
Josef Bacik 已提交
441
		}
442 443
		path->slots[0]++;
	}
J
Josef Bacik 已提交
444
	ret = 0;
445

J
Josef Bacik 已提交
446 447 448
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
449
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
450 451

	spin_lock(&block_group->lock);
452
	block_group->caching_ctl = NULL;
J
Josef Bacik 已提交
453 454
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
455

456
err:
457
	btrfs_free_path(path);
458
	up_read(&fs_info->extent_commit_sem);
J
Josef Bacik 已提交
459

460 461 462
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
463
out:
464 465 466
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
467
	btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
468 469
}

470 471
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     struct btrfs_trans_handle *trans,
472
			     struct btrfs_root *root,
473
			     int load_cache_only)
J
Josef Bacik 已提交
474
{
475
	DEFINE_WAIT(wait);
476 477
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
478 479
	int ret = 0;

480
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
481 482
	if (!caching_ctl)
		return -ENOMEM;
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	atomic_set(&caching_ctl->count, 1);
	caching_ctl->work.func = caching_thread;

	spin_lock(&cache->lock);
	/*
	 * This should be a rare occasion, but this could happen I think in the
	 * case where one thread starts to load the space cache info, and then
	 * some other thread starts a transaction commit which tries to do an
	 * allocation while the other thread is still loading the space cache
	 * info.  The previous loop should have kept us from choosing this block
	 * group, but if we've moved to the state where we will wait on caching
	 * block groups we need to first check if we're doing a fast load here,
	 * so we can wait for it to finish, otherwise we could end up allocating
	 * from a block group who's cache gets evicted for one reason or
	 * another.
	 */
	while (cache->cached == BTRFS_CACHE_FAST) {
		struct btrfs_caching_control *ctl;

		ctl = cache->caching_ctl;
		atomic_inc(&ctl->count);
		prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&cache->lock);

		schedule();

		finish_wait(&ctl->wait, &wait);
		put_caching_control(ctl);
		spin_lock(&cache->lock);
	}

	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
		kfree(caching_ctl);
523
		return 0;
524 525 526 527 528
	}
	WARN_ON(cache->caching_ctl);
	cache->caching_ctl = caching_ctl;
	cache->cached = BTRFS_CACHE_FAST;
	spin_unlock(&cache->lock);
529

530 531
	/*
	 * We can't do the read from on-disk cache during a commit since we need
532 533 534
	 * to have the normal tree locking.  Also if we are currently trying to
	 * allocate blocks for the tree root we can't do the fast caching since
	 * we likely hold important locks.
535
	 */
536
	if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
537 538 539 540
		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
541
			cache->caching_ctl = NULL;
542 543 544
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
		} else {
545 546 547 548 549 550
			if (load_cache_only) {
				cache->caching_ctl = NULL;
				cache->cached = BTRFS_CACHE_NO;
			} else {
				cache->cached = BTRFS_CACHE_STARTED;
			}
551 552
		}
		spin_unlock(&cache->lock);
553
		wake_up(&caching_ctl->wait);
554
		if (ret == 1) {
555
			put_caching_control(caching_ctl);
556
			free_excluded_extents(fs_info->extent_root, cache);
557
			return 0;
558
		}
559 560 561 562 563 564 565 566 567 568 569 570 571 572
	} else {
		/*
		 * We are not going to do the fast caching, set cached to the
		 * appropriate value and wakeup any waiters.
		 */
		spin_lock(&cache->lock);
		if (load_cache_only) {
			cache->caching_ctl = NULL;
			cache->cached = BTRFS_CACHE_NO;
		} else {
			cache->cached = BTRFS_CACHE_STARTED;
		}
		spin_unlock(&cache->lock);
		wake_up(&caching_ctl->wait);
573 574
	}

575 576
	if (load_cache_only) {
		put_caching_control(caching_ctl);
577
		return 0;
J
Josef Bacik 已提交
578 579
	}

580
	down_write(&fs_info->extent_commit_sem);
581
	atomic_inc(&caching_ctl->count);
582 583 584
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
	up_write(&fs_info->extent_commit_sem);

585
	btrfs_get_block_group(cache);
586

587
	btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
J
Josef Bacik 已提交
588

589
	return ret;
590 591
}

J
Josef Bacik 已提交
592 593 594
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
595 596
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
597
{
J
Josef Bacik 已提交
598
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
599

J
Josef Bacik 已提交
600
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
601

J
Josef Bacik 已提交
602
	return cache;
C
Chris Mason 已提交
603 604
}

J
Josef Bacik 已提交
605
/*
606
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
607
 */
C
Chris Mason 已提交
608 609 610
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
611
{
J
Josef Bacik 已提交
612
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
613

J
Josef Bacik 已提交
614
	cache = block_group_cache_tree_search(info, bytenr, 1);
615

J
Josef Bacik 已提交
616
	return cache;
C
Chris Mason 已提交
617
}
618

J
Josef Bacik 已提交
619 620
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
621
{
J
Josef Bacik 已提交
622 623
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
624

625
	flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
626

627 628
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
629
		if (found->flags & flags) {
630
			rcu_read_unlock();
J
Josef Bacik 已提交
631
			return found;
632
		}
J
Josef Bacik 已提交
633
	}
634
	rcu_read_unlock();
J
Josef Bacik 已提交
635
	return NULL;
636 637
}

638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

653 654
u64 btrfs_find_block_group(struct btrfs_root *root,
			   u64 search_start, u64 search_hint, int owner)
C
Chris Mason 已提交
655
{
656
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
657
	u64 used;
658 659
	u64 last = max(search_hint, search_start);
	u64 group_start = 0;
660
	int full_search = 0;
661
	int factor = 9;
C
Chris Mason 已提交
662
	int wrapped = 0;
663
again:
664 665
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
J
Josef Bacik 已提交
666 667
		if (!cache)
			break;
668

669
		spin_lock(&cache->lock);
670 671 672
		last = cache->key.objectid + cache->key.offset;
		used = btrfs_block_group_used(&cache->item);

673 674
		if ((full_search || !cache->ro) &&
		    block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
675
			if (used + cache->pinned + cache->reserved <
676 677
			    div_factor(cache->key.offset, factor)) {
				group_start = cache->key.objectid;
678
				spin_unlock(&cache->lock);
679
				btrfs_put_block_group(cache);
680 681
				goto found;
			}
682
		}
683
		spin_unlock(&cache->lock);
684
		btrfs_put_block_group(cache);
685
		cond_resched();
C
Chris Mason 已提交
686
	}
C
Chris Mason 已提交
687 688 689 690 691 692
	if (!wrapped) {
		last = search_start;
		wrapped = 1;
		goto again;
	}
	if (!full_search && factor < 10) {
C
Chris Mason 已提交
693
		last = search_start;
694
		full_search = 1;
C
Chris Mason 已提交
695
		factor = 10;
696 697
		goto again;
	}
C
Chris Mason 已提交
698
found:
699
	return group_start;
700
}
J
Josef Bacik 已提交
701

702
/* simple helper to search for an existing extent at a given offset */
Z
Zheng Yan 已提交
703
int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
704 705 706
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
707
	struct btrfs_path *path;
708

Z
Zheng Yan 已提交
709
	path = btrfs_alloc_path();
710 711 712
	if (!path)
		return -ENOMEM;

713 714 715 716 717
	key.objectid = start;
	key.offset = len;
	btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
718
	btrfs_free_path(path);
719 720 721
	return ret;
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
/*
 * helper function to lookup reference count and flags of extent.
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
			     u64 num_bytes, u64 *refs, u64 *flags)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = num_bytes;
	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
again:
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

802
			btrfs_release_path(path);
803

804 805 806 807
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
			goto again;
		}
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

833 834 835 836 837 838 839 840 841 842 843 844 845 846
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
892 893 894
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
895
 * - different files inside a single subvolume
896 897
 * - different offsets inside a file (bookend extents in file.c)
 *
898
 * The extent ref structure for the implicit back refs has fields for:
899 900 901
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
902 903
 * - original offset in the file
 * - how many bookend extents
904
 *
905 906
 * The key offset for the implicit back refs is hash of the first
 * three fields.
907
 *
908
 * The extent ref structure for the full back refs has field for:
909
 *
910
 * - number of pointers in the tree leaf
911
 *
912 913
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
914
 *
915 916
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
917
 *
918
 *     (root_key.objectid, inode objectid, offset in file, 1)
919
 *
920 921
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
922
 *
923
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
924
 *
925
 * Btree extents can be referenced by:
926
 *
927
 * - Different subvolumes
928
 *
929 930 931 932
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
933
 *
934 935 936
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
937
 */
Z
Zheng Yan 已提交
938

939 940 941 942 943
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
944
{
945 946 947 948 949
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
950
	struct btrfs_key key;
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
970
				BUG_ON(ret > 0); /* Corruption */
971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
986
	btrfs_release_path(path);
987 988 989 990 991 992 993 994 995

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
996
	BUG_ON(ret); /* Corruption */
997

998
	btrfs_extend_item(trans, root, path, new_size);
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
1028
	high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1029
	lenum = cpu_to_le64(owner);
1030
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1031
	lenum = cpu_to_le64(offset);
1032
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1065
	struct extent_buffer *leaf;
1066
	u32 nritems;
1067
	int ret;
1068 1069
	int recow;
	int err = -ENOENT;
1070

Z
Zheng Yan 已提交
1071
	key.objectid = bytenr;
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1087

1088 1089 1090 1091 1092
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1093
		btrfs_release_path(path);
1094 1095 1096 1097 1098 1099 1100 1101 1102
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1103 1104 1105
	}

	leaf = path->nodes[0];
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1131
				btrfs_release_path(path);
1132 1133 1134 1135 1136 1137
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1138
	}
1139 1140
fail:
	return err;
Z
Zheng Yan 已提交
1141 1142
}

1143 1144 1145 1146 1147 1148
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1149 1150 1151
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1152
	u32 size;
Z
Zheng Yan 已提交
1153 1154
	u32 num_refs;
	int ret;
1155 1156

	key.objectid = bytenr;
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1167

1168 1169 1170 1171 1172 1173 1174
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1175
		ref = btrfs_item_ptr(leaf, path->slots[0],
1176 1177 1178 1179 1180 1181 1182
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1183
		}
1184 1185 1186 1187 1188 1189 1190 1191
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1192
			btrfs_release_path(path);
1193 1194 1195 1196 1197
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1198

1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1213 1214
		}
	}
1215 1216 1217
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1218
	btrfs_release_path(path);
1219
	return ret;
1220 1221
}

1222 1223 1224 1225
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   int refs_to_drop)
Z
Zheng Yan 已提交
1226
{
1227 1228 1229
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1230
	struct extent_buffer *leaf;
1231
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1232 1233 1234
	int ret = 0;

	leaf = path->nodes[0];
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1256 1257
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1258

Z
Zheng Yan 已提交
1259 1260 1261
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
	} else {
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1274 1275 1276 1277 1278
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1279 1280 1281
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1282
{
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1314
#endif
1315 1316 1317 1318 1319
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1320

1321 1322 1323 1324 1325
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1326
{
1327
	struct btrfs_key key;
1328 1329
	int ret;

1330 1331 1332 1333 1334 1335 1336
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1337 1338
	}

1339 1340 1341 1342 1343
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1344
		btrfs_release_path(path);
1345 1346 1347 1348 1349
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1350
#endif
1351
	return ret;
1352 1353
}

1354 1355 1356 1357 1358
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1359
{
1360
	struct btrfs_key key;
Z
Zheng Yan 已提交
1361 1362
	int ret;

1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1373
	btrfs_release_path(path);
Z
Zheng Yan 已提交
1374 1375 1376
	return ret;
}

1377
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1378
{
1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1392
}
1393

1394 1395
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1396

C
Chris Mason 已提交
1397
{
1398
	for (; level < BTRFS_MAX_LEVEL; level++) {
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1414

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1450

1451
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1452
	key.type = BTRFS_EXTENT_ITEM_KEY;
1453
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1454

1455 1456 1457
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1458
		path->keep_locks = 1;
1459 1460 1461
	} else
		extra_size = -1;
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1462
	if (ret < 0) {
1463 1464 1465
		err = ret;
		goto out;
	}
1466 1467 1468 1469 1470
	if (ret && !insert) {
		err = -ENOENT;
		goto out;
	}
	BUG_ON(ret); /* Corruption */
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	} else {
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1564 1565
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1566
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1567 1568 1569 1570 1571 1572
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1573
	if (insert) {
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
1584 1585 1586 1587 1588 1589 1590
void setup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

1608
	btrfs_extend_item(trans, root, path, size);
1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1658
		return ret;
1659

1660
	btrfs_release_path(path);
1661 1662 1663 1664 1665 1666 1667 1668
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1669
	}
1670 1671
	return ret;
}
Z
Zheng Yan 已提交
1672

1673 1674 1675 1676
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
1677 1678 1679 1680 1681 1682
void update_inline_extent_backref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  struct btrfs_extent_inline_ref *iref,
				  int refs_to_mod,
				  struct btrfs_delayed_extent_op *extent_op)
1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1715
	}
Z
Zheng Yan 已提交
1716

1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
1734
		btrfs_truncate_item(trans, root, path, item_size, 1);
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755
	}
	btrfs_mark_buffer_dirty(leaf);
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1756 1757
		update_inline_extent_backref(trans, root, path, iref,
					     refs_to_add, extent_op);
1758
	} else if (ret == -ENOENT) {
1759 1760 1761 1762
		setup_inline_extent_backref(trans, root, path, iref, parent,
					    root_objectid, owner, offset,
					    refs_to_add, extent_op);
		ret = 0;
1763
	}
1764 1765
	return ret;
}
Z
Zheng Yan 已提交
1766

1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1785

1786 1787 1788 1789 1790 1791
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_drop, int is_data)
{
1792
	int ret = 0;
1793

1794 1795
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
1796 1797
		update_inline_extent_backref(trans, root, path, iref,
					     -refs_to_drop, NULL);
1798 1799 1800 1801 1802 1803 1804 1805
	} else if (is_data) {
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
	} else {
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1806
static int btrfs_issue_discard(struct block_device *bdev,
1807 1808
				u64 start, u64 len)
{
1809
	return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1810 1811 1812
}

static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1813
				u64 num_bytes, u64 *actual_bytes)
1814 1815
{
	int ret;
1816
	u64 discarded_bytes = 0;
1817
	struct btrfs_bio *bbio = NULL;
1818

C
Christoph Hellwig 已提交
1819

1820
	/* Tell the block device(s) that the sectors can be discarded */
1821
	ret = btrfs_map_block(&root->fs_info->mapping_tree, REQ_DISCARD,
1822
			      bytenr, &num_bytes, &bbio, 0);
1823
	/* Error condition is -ENOMEM */
1824
	if (!ret) {
1825
		struct btrfs_bio_stripe *stripe = bbio->stripes;
1826 1827 1828
		int i;


1829
		for (i = 0; i < bbio->num_stripes; i++, stripe++) {
1830 1831 1832
			if (!stripe->dev->can_discard)
				continue;

1833 1834 1835 1836 1837 1838
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
						  stripe->length);
			if (!ret)
				discarded_bytes += stripe->length;
			else if (ret != -EOPNOTSUPP)
1839
				break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
1840 1841 1842 1843 1844 1845 1846

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
1847
		}
1848
		kfree(bbio);
1849
	}
1850 1851 1852 1853

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

1854 1855 1856 1857

	return ret;
}

1858
/* Can return -ENOMEM */
1859 1860 1861
int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
A
Arne Jansen 已提交
1862
			 u64 root_objectid, u64 owner, u64 offset, int for_cow)
1863 1864
{
	int ret;
A
Arne Jansen 已提交
1865 1866
	struct btrfs_fs_info *fs_info = root->fs_info;

1867 1868 1869 1870
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
1871 1872
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
1873
					parent, root_objectid, (int)owner,
A
Arne Jansen 已提交
1874
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1875
	} else {
A
Arne Jansen 已提交
1876 1877
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
					num_bytes,
1878
					parent, root_objectid, owner, offset,
A
Arne Jansen 已提交
1879
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
	u64 refs;
	int ret;
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
	ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
					   path, bytenr, num_bytes, parent,
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
	if (ret == 0)
		goto out;

	if (ret != -EAGAIN) {
		err = ret;
		goto out;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
1923

1924
	btrfs_mark_buffer_dirty(leaf);
1925
	btrfs_release_path(path);
1926 1927

	path->reada = 1;
1928 1929
	path->leave_spinning = 1;

1930 1931
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
1932 1933
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
1934 1935
	if (ret)
		btrfs_abort_transaction(trans, root, ret);
1936
out:
1937
	btrfs_free_path(path);
1938
	return err;
1939 1940
}

1941 1942 1943 1944 1945
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
1946
{
1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		if (extent_op) {
			BUG_ON(extent_op->update_key);
			flags |= extent_op->flags_to_set;
		}
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
					     extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
					  extent_op);
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
2019
	int ret;
2020 2021
	int err = 0;

2022 2023 2024
	if (trans->aborted)
		return 0;

2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = node->num_bytes;

	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
		err = -EIO;
		goto out;
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2063

2064 2065 2066 2067
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2068 2069
}

2070 2071 2072 2073 2074
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2075 2076
{
	int ret = 0;
2077 2078 2079 2080
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2081

2082 2083 2084
	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;
2085

2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111
	ref = btrfs_delayed_node_to_tree_ref(node);
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		BUG_ON(!extent_op || !extent_op->update_flags ||
		       !extent_op->update_key);
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
						ref->level, &ins);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
					     ref->level, 0, 1, extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
					  ref->level, 0, 1, extent_op);
	} else {
		BUG();
	}
2112 2113 2114 2115
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2116 2117 2118 2119 2120
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2121
{
2122 2123 2124 2125 2126
	int ret = 0;

	if (trans->aborted)
		return 0;

2127
	if (btrfs_delayed_ref_is_head(node)) {
2128 2129 2130 2131 2132 2133 2134
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2135 2136
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2137
		if (insert_reserved) {
2138 2139
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2140 2141 2142 2143 2144
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
			}
2145 2146
		}
		mutex_unlock(&head->mutex);
2147
		return ret;
2148 2149
	}

2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_node *ref;
	int action = BTRFS_ADD_DELAYED_REF;
again:
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
	node = rb_prev(&head->node.rb_node);
	while (1) {
		if (!node)
			break;
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
		if (ref->bytenr != head->node.bytenr)
			break;
2183
		if (ref->action == action)
2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
			return ref;
		node = rb_prev(node);
	}
	if (action == BTRFS_ADD_DELAYED_REF) {
		action = BTRFS_DROP_DELAYED_REF;
		goto again;
	}
	return NULL;
}

2194 2195 2196 2197
/*
 * Returns 0 on success or if called with an already aborted transaction.
 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
 */
2198 2199 2200
static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root,
				       struct list_head *cluster)
2201 2202 2203 2204
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2205
	struct btrfs_delayed_extent_op *extent_op;
2206
	struct btrfs_fs_info *fs_info = root->fs_info;
2207
	int ret;
2208
	int count = 0;
2209 2210 2211 2212 2213
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2214 2215
			/* pick a new head ref from the cluster list */
			if (list_empty(cluster))
2216 2217
				break;

2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234
			locked_ref = list_entry(cluster->next,
				     struct btrfs_delayed_ref_head, cluster);

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);

			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2235 2236
			}
		}
2237

2238 2239 2240 2241 2242 2243 2244 2245 2246 2247
		/*
		 * We need to try and merge add/drops of the same ref since we
		 * can run into issues with relocate dropping the implicit ref
		 * and then it being added back again before the drop can
		 * finish.  If we merged anything we need to re-loop so we can
		 * get a good ref.
		 */
		btrfs_merge_delayed_refs(trans, fs_info, delayed_refs,
					 locked_ref);

2248 2249 2250 2251 2252 2253 2254
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);

		if (ref && ref->seq &&
2255
		    btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) {
2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269
			/*
			 * there are still refs with lower seq numbers in the
			 * process of being added. Don't run this ref yet.
			 */
			list_del_init(&locked_ref->cluster);
			mutex_unlock(&locked_ref->mutex);
			locked_ref = NULL;
			delayed_refs->num_heads_ready++;
			spin_unlock(&delayed_refs->lock);
			cond_resched();
			spin_lock(&delayed_refs->lock);
			continue;
		}

2270 2271 2272 2273 2274 2275
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2276

2277 2278 2279
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2280 2281 2282 2283 2284 2285
		if (!ref) {
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298

			if (extent_op && must_insert_reserved) {
				kfree(extent_op);
				extent_op = NULL;
			}

			if (extent_op) {
				spin_unlock(&delayed_refs->lock);

				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
				kfree(extent_op);

2299 2300
				if (ret) {
					printk(KERN_DEBUG "btrfs: run_delayed_extent_op returned %d\n", ret);
2301
					spin_lock(&delayed_refs->lock);
2302 2303 2304
					return ret;
				}

2305
				goto next;
2306 2307
			}

2308
			list_del_init(&locked_ref->cluster);
2309 2310
			locked_ref = NULL;
		}
C
Chris Mason 已提交
2311

2312 2313 2314
		ref->in_tree = 0;
		rb_erase(&ref->rb_node, &delayed_refs->root);
		delayed_refs->num_entries--;
2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331
		if (locked_ref) {
			/*
			 * when we play the delayed ref, also correct the
			 * ref_mod on head
			 */
			switch (ref->action) {
			case BTRFS_ADD_DELAYED_REF:
			case BTRFS_ADD_DELAYED_EXTENT:
				locked_ref->node.ref_mod -= ref->ref_mod;
				break;
			case BTRFS_DROP_DELAYED_REF:
				locked_ref->node.ref_mod += ref->ref_mod;
				break;
			default:
				WARN_ON(1);
			}
		}
2332
		spin_unlock(&delayed_refs->lock);
2333

2334
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2335
					  must_insert_reserved);
2336

2337 2338
		btrfs_put_delayed_ref(ref);
		kfree(extent_op);
2339
		count++;
2340 2341 2342

		if (ret) {
			printk(KERN_DEBUG "btrfs: run_one_delayed_ref returned %d\n", ret);
2343
			spin_lock(&delayed_refs->lock);
2344 2345 2346
			return ret;
		}

2347
next:
2348 2349 2350 2351 2352 2353
		cond_resched();
		spin_lock(&delayed_refs->lock);
	}
	return count;
}

2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396
#ifdef SCRAMBLE_DELAYED_REFS
/*
 * Normally delayed refs get processed in ascending bytenr order. This
 * correlates in most cases to the order added. To expose dependencies on this
 * order, we start to process the tree in the middle instead of the beginning
 */
static u64 find_middle(struct rb_root *root)
{
	struct rb_node *n = root->rb_node;
	struct btrfs_delayed_ref_node *entry;
	int alt = 1;
	u64 middle;
	u64 first = 0, last = 0;

	n = rb_first(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		first = entry->bytenr;
	}
	n = rb_last(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		last = entry->bytenr;
	}
	n = root->rb_node;

	while (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		WARN_ON(!entry->in_tree);

		middle = entry->bytenr;

		if (alt)
			n = n->rb_left;
		else
			n = n->rb_right;

		alt = 1 - alt;
	}
	return middle;
}
#endif

2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430
int btrfs_delayed_refs_qgroup_accounting(struct btrfs_trans_handle *trans,
					 struct btrfs_fs_info *fs_info)
{
	struct qgroup_update *qgroup_update;
	int ret = 0;

	if (list_empty(&trans->qgroup_ref_list) !=
	    !trans->delayed_ref_elem.seq) {
		/* list without seq or seq without list */
		printk(KERN_ERR "btrfs: qgroup accounting update error, list is%s empty, seq is %llu\n",
			list_empty(&trans->qgroup_ref_list) ? "" : " not",
			trans->delayed_ref_elem.seq);
		BUG();
	}

	if (!trans->delayed_ref_elem.seq)
		return 0;

	while (!list_empty(&trans->qgroup_ref_list)) {
		qgroup_update = list_first_entry(&trans->qgroup_ref_list,
						 struct qgroup_update, list);
		list_del(&qgroup_update->list);
		if (!ret)
			ret = btrfs_qgroup_account_ref(
					trans, fs_info, qgroup_update->node,
					qgroup_update->extent_op);
		kfree(qgroup_update);
	}

	btrfs_put_tree_mod_seq(fs_info, &trans->delayed_ref_elem);

	return ret;
}

2431 2432 2433 2434 2435 2436
/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
2437 2438 2439
 *
 * Returns 0 on success or if called with an aborted transaction
 * Returns <0 on error and aborts the transaction
2440 2441 2442 2443 2444 2445 2446 2447 2448
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct list_head cluster;
	int ret;
2449
	u64 delayed_start;
2450 2451
	int run_all = count == (unsigned long)-1;
	int run_most = 0;
2452
	int loops;
2453

2454 2455 2456 2457
	/* We'll clean this up in btrfs_cleanup_transaction */
	if (trans->aborted)
		return 0;

2458 2459 2460
	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

2461 2462
	btrfs_delayed_refs_qgroup_accounting(trans, root->fs_info);

2463 2464 2465
	delayed_refs = &trans->transaction->delayed_refs;
	INIT_LIST_HEAD(&cluster);
again:
2466
	loops = 0;
2467
	spin_lock(&delayed_refs->lock);
2468

2469 2470 2471 2472
#ifdef SCRAMBLE_DELAYED_REFS
	delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
#endif

2473 2474 2475 2476 2477 2478 2479 2480
	if (count == 0) {
		count = delayed_refs->num_entries * 2;
		run_most = 1;
	}
	while (1) {
		if (!(run_all || run_most) &&
		    delayed_refs->num_heads_ready < 64)
			break;
2481

2482
		/*
2483 2484 2485 2486
		 * go find something we can process in the rbtree.  We start at
		 * the beginning of the tree, and then build a cluster
		 * of refs to process starting at the first one we are able to
		 * lock
2487
		 */
2488
		delayed_start = delayed_refs->run_delayed_start;
2489 2490 2491
		ret = btrfs_find_ref_cluster(trans, &cluster,
					     delayed_refs->run_delayed_start);
		if (ret)
2492 2493
			break;

2494
		ret = run_clustered_refs(trans, root, &cluster);
2495 2496 2497 2498 2499
		if (ret < 0) {
			spin_unlock(&delayed_refs->lock);
			btrfs_abort_transaction(trans, root, ret);
			return ret;
		}
2500 2501 2502 2503 2504

		count -= min_t(unsigned long, ret, count);

		if (count == 0)
			break;
2505

2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
		if (delayed_start >= delayed_refs->run_delayed_start) {
			if (loops == 0) {
				/*
				 * btrfs_find_ref_cluster looped. let's do one
				 * more cycle. if we don't run any delayed ref
				 * during that cycle (because we can't because
				 * all of them are blocked), bail out.
				 */
				loops = 1;
			} else {
				/*
				 * no runnable refs left, stop trying
				 */
				BUG_ON(run_all);
				break;
			}
		}
		if (ret) {
2524
			/* refs were run, let's reset staleness detection */
2525
			loops = 0;
2526
		}
2527
	}
2528

2529
	if (run_all) {
2530 2531 2532 2533 2534 2535
		if (!list_empty(&trans->new_bgs)) {
			spin_unlock(&delayed_refs->lock);
			btrfs_create_pending_block_groups(trans, root);
			spin_lock(&delayed_refs->lock);
		}

2536
		node = rb_first(&delayed_refs->root);
2537
		if (!node)
2538
			goto out;
2539
		count = (unsigned long)-1;
2540

2541 2542 2543 2544 2545
		while (node) {
			ref = rb_entry(node, struct btrfs_delayed_ref_node,
				       rb_node);
			if (btrfs_delayed_ref_is_head(ref)) {
				struct btrfs_delayed_ref_head *head;
2546

2547 2548 2549 2550
				head = btrfs_delayed_node_to_head(ref);
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2551 2552 2553 2554
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
2555 2556 2557 2558
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2559
				cond_resched();
2560 2561 2562 2563 2564 2565 2566
				goto again;
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
		schedule_timeout(1);
		goto again;
2567
	}
2568
out:
2569
	spin_unlock(&delayed_refs->lock);
2570
	assert_qgroups_uptodate(trans);
2571 2572 2573
	return 0;
}

2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
				int is_data)
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

	extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;

A
Arne Jansen 已提交
2591 2592
	ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
					  num_bytes, extent_op);
2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620
	if (ret)
		kfree(extent_op);
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	ret = -ENOENT;
	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

2621
		btrfs_release_path(path);
2622

2623 2624 2625 2626
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out_unlock;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	if (ref->bytenr != bytenr)
		goto out_unlock;

	ret = 1;
	if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
		goto out_unlock;

	data_ref = btrfs_delayed_node_to_data_ref(ref);

	node = rb_prev(node);
	if (node) {
2650 2651
		int seq = ref->seq;

2652
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2653
		if (ref->bytenr == bytenr && ref->seq == seq)
2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672
			goto out_unlock;
	}

	if (data_ref->root != root->root_key.objectid ||
	    data_ref->objectid != objectid || data_ref->offset != offset)
		goto out_unlock;

	ret = 0;
out_unlock:
	mutex_unlock(&head->mutex);
out:
	spin_unlock(&delayed_refs->lock);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2673 2674
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2675
	struct extent_buffer *leaf;
2676 2677 2678
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2679
	struct btrfs_key key;
2680
	u32 item_size;
2681
	int ret;
2682

2683
	key.objectid = bytenr;
Z
Zheng Yan 已提交
2684
	key.offset = (u64)-1;
2685
	key.type = BTRFS_EXTENT_ITEM_KEY;
2686 2687 2688 2689

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
2690
	BUG_ON(ret == 0); /* Corruption */
Y
Yan Zheng 已提交
2691 2692 2693

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
2694
		goto out;
2695

Z
Zheng Yan 已提交
2696
	path->slots[0]--;
2697
	leaf = path->nodes[0];
2698
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2699

2700
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2701
		goto out;
2702

2703 2704 2705 2706 2707 2708 2709 2710 2711
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2712

2713 2714 2715
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2716

2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
2756
			goto out;
Y
Yan Zheng 已提交
2757

2758 2759 2760 2761 2762 2763 2764
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
2765
	}
2766 2767 2768

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
2769
out:
Y
Yan Zheng 已提交
2770
	btrfs_free_path(path);
2771 2772
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
2773
	return ret;
2774
}
C
Chris Mason 已提交
2775

2776
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2777
			   struct btrfs_root *root,
2778
			   struct extent_buffer *buf,
A
Arne Jansen 已提交
2779
			   int full_backref, int inc, int for_cow)
Z
Zheng Yan 已提交
2780 2781
{
	u64 bytenr;
2782 2783
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
2784 2785 2786 2787 2788 2789 2790 2791
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
A
Arne Jansen 已提交
2792
			    u64, u64, u64, u64, u64, u64, int);
Z
Zheng Yan 已提交
2793 2794 2795 2796 2797

	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

2798 2799
	if (!root->ref_cows && level == 0)
		return 0;
Z
Zheng Yan 已提交
2800

2801 2802 2803 2804
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
2805

2806 2807 2808 2809 2810 2811
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
2812
		if (level == 0) {
2813
			btrfs_item_key_to_cpu(buf, &key, i);
Z
Zheng Yan 已提交
2814 2815
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
2816
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
2817 2818 2819 2820 2821 2822 2823
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
2824 2825 2826 2827 2828

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
A
Arne Jansen 已提交
2829
					   key.offset, for_cow);
Z
Zheng Yan 已提交
2830 2831 2832
			if (ret)
				goto fail;
		} else {
2833 2834 2835
			bytenr = btrfs_node_blockptr(buf, i);
			num_bytes = btrfs_level_size(root, level - 1);
			ret = process_func(trans, root, bytenr, num_bytes,
A
Arne Jansen 已提交
2836 2837
					   parent, ref_root, level - 1, 0,
					   for_cow);
Z
Zheng Yan 已提交
2838 2839 2840 2841 2842 2843
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
2844 2845 2846 2847
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
A
Arne Jansen 已提交
2848
		  struct extent_buffer *buf, int full_backref, int for_cow)
2849
{
A
Arne Jansen 已提交
2850
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1, for_cow);
2851 2852 2853
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
A
Arne Jansen 已提交
2854
		  struct extent_buffer *buf, int full_backref, int for_cow)
2855
{
A
Arne Jansen 已提交
2856
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0, for_cow);
Z
Zheng Yan 已提交
2857 2858
}

C
Chris Mason 已提交
2859 2860 2861 2862 2863 2864 2865
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2866 2867
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
2868 2869

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2870 2871
	if (ret < 0)
		goto fail;
2872
	BUG_ON(ret); /* Corruption */
2873 2874 2875 2876 2877

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
2878
	btrfs_release_path(path);
2879
fail:
2880 2881
	if (ret) {
		btrfs_abort_transaction(trans, root, ret);
C
Chris Mason 已提交
2882
		return ret;
2883
	}
C
Chris Mason 已提交
2884 2885 2886 2887
	return 0;

}

2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
	spin_lock(&root->fs_info->block_group_cache_lock);
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
2899
		btrfs_get_block_group(cache);
2900 2901 2902 2903 2904 2905
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

2906 2907 2908 2909 2910 2911 2912
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
2913
	int dcs = BTRFS_DC_ERROR;
2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932
	int num_pages = 0;
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
2933
		btrfs_release_path(path);
2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

2950 2951 2952 2953 2954 2955 2956
	/* We've already setup this transaction, go ahead and exit */
	if (block_group->cache_generation == trans->transid &&
	    i_size_read(inode)) {
		dcs = BTRFS_DC_SETUP;
		goto out_put;
	}

2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973
	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
		ret = btrfs_truncate_free_space_cache(root, trans, path,
						      inode);
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
2974 2975 2976 2977 2978 2979 2980
	if (block_group->cached != BTRFS_CACHE_FINISHED ||
	    !btrfs_test_opt(root, SPACE_CACHE)) {
		/*
		 * don't bother trying to write stuff out _if_
		 * a) we're not cached,
		 * b) we're with nospace_cache mount option.
		 */
2981
		dcs = BTRFS_DC_WRITTEN;
2982 2983 2984 2985 2986
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

2987 2988 2989 2990 2991 2992 2993
	/*
	 * Try to preallocate enough space based on how big the block group is.
	 * Keep in mind this has to include any pinned space which could end up
	 * taking up quite a bit since it's not folded into the other space
	 * cache.
	 */
	num_pages = (int)div64_u64(block_group->key.offset, 256 * 1024 * 1024);
2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006
	if (!num_pages)
		num_pages = 1;

	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
3007 3008
	if (!ret)
		dcs = BTRFS_DC_SETUP;
3009
	btrfs_free_reserved_data_space(inode, num_pages);
3010

3011 3012 3013
out_put:
	iput(inode);
out_free:
3014
	btrfs_release_path(path);
3015 3016
out:
	spin_lock(&block_group->lock);
3017
	if (!ret && dcs == BTRFS_DC_SETUP)
3018
		block_group->cache_generation = trans->transid;
3019
	block_group->disk_cache_state = dcs;
3020 3021 3022 3023 3024
	spin_unlock(&block_group->lock);

	return ret;
}

3025 3026
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
C
Chris Mason 已提交
3027
{
3028
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
3029 3030
	int err = 0;
	struct btrfs_path *path;
3031
	u64 last = 0;
C
Chris Mason 已提交
3032 3033 3034 3035 3036

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055
again:
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			if (cache->disk_cache_state == BTRFS_DC_CLEAR)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
		err = cache_save_setup(cache, trans, path);
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
3056
	while (1) {
3057 3058 3059
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3060 3061
			if (err) /* File system offline */
				goto out;
J
Josef Bacik 已提交
3062
		}
3063

3064 3065
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
3066 3067 3068 3069 3070
			if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
				btrfs_put_block_group(cache);
				goto again;
			}

3071 3072 3073 3074 3075 3076 3077 3078 3079 3080
			if (cache->dirty)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
J
Josef Bacik 已提交
3081

J
Josef Bacik 已提交
3082 3083
		if (cache->disk_cache_state == BTRFS_DC_SETUP)
			cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
3084
		cache->dirty = 0;
3085
		last = cache->key.objectid + cache->key.offset;
J
Josef Bacik 已提交
3086

3087
		err = write_one_cache_group(trans, root, path, cache);
3088 3089 3090
		if (err) /* File system offline */
			goto out;

3091
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
3092
	}
3093

J
Josef Bacik 已提交
3094 3095 3096 3097 3098 3099 3100 3101 3102
	while (1) {
		/*
		 * I don't think this is needed since we're just marking our
		 * preallocated extent as written, but just in case it can't
		 * hurt.
		 */
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3103 3104
			if (err) /* File system offline */
				goto out;
J
Josef Bacik 已提交
3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128
		}

		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			/*
			 * Really this shouldn't happen, but it could if we
			 * couldn't write the entire preallocated extent and
			 * splitting the extent resulted in a new block.
			 */
			if (cache->dirty) {
				btrfs_put_block_group(cache);
				goto again;
			}
			if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

3129
		err = btrfs_write_out_cache(root, trans, cache, path);
J
Josef Bacik 已提交
3130 3131 3132 3133 3134

		/*
		 * If we didn't have an error then the cache state is still
		 * NEED_WRITE, so we can set it to WRITTEN.
		 */
3135
		if (!err && cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
J
Josef Bacik 已提交
3136 3137 3138 3139
			cache->disk_cache_state = BTRFS_DC_WRITTEN;
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}
3140
out:
J
Josef Bacik 已提交
3141

C
Chris Mason 已提交
3142
	btrfs_free_path(path);
3143
	return err;
C
Chris Mason 已提交
3144 3145
}

3146 3147 3148 3149 3150 3151 3152 3153 3154
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
3155
		btrfs_put_block_group(block_group);
3156 3157 3158
	return readonly;
}

3159 3160 3161 3162 3163
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
3164 3165 3166 3167 3168 3169 3170 3171
	int i;
	int factor;

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
3172 3173 3174

	found = __find_space_info(info, flags);
	if (found) {
3175
		spin_lock(&found->lock);
3176
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
3177
		found->disk_total += total_bytes * factor;
3178
		found->bytes_used += bytes_used;
3179
		found->disk_used += bytes_used * factor;
3180
		found->full = 0;
3181
		spin_unlock(&found->lock);
3182 3183 3184
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
3185
	found = kzalloc(sizeof(*found), GFP_NOFS);
3186 3187 3188
	if (!found)
		return -ENOMEM;

3189 3190
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		INIT_LIST_HEAD(&found->block_groups[i]);
3191
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
3192
	spin_lock_init(&found->lock);
3193
	found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3194
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
3195
	found->disk_total = total_bytes * factor;
3196
	found->bytes_used = bytes_used;
3197
	found->disk_used = bytes_used * factor;
3198
	found->bytes_pinned = 0;
3199
	found->bytes_reserved = 0;
Y
Yan Zheng 已提交
3200
	found->bytes_readonly = 0;
3201
	found->bytes_may_use = 0;
3202
	found->full = 0;
3203
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3204
	found->chunk_alloc = 0;
3205 3206
	found->flush = 0;
	init_waitqueue_head(&found->wait);
3207
	*space_info = found;
3208
	list_add_rcu(&found->list, &info->space_info);
3209 3210
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		info->data_sinfo = found;
3211 3212 3213
	return 0;
}

3214 3215
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
3216 3217
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
3218 3219 3220 3221 3222 3223 3224

	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits |= extra_flags;
3225
}
3226

3227 3228 3229
/*
 * returns target flags in extended format or 0 if restripe for this
 * chunk_type is not in progress
3230 3231
 *
 * should be called with either volume_mutex or balance_lock held
3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254
 */
static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
{
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
	u64 target = 0;

	if (!bctl)
		return 0;

	if (flags & BTRFS_BLOCK_GROUP_DATA &&
	    bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
	} else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
		   bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
	} else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
		   bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
	}

	return target;
}

3255 3256 3257
/*
 * @flags: available profiles in extended format (see ctree.h)
 *
3258 3259 3260
 * Returns reduced profile in chunk format.  If profile changing is in
 * progress (either running or paused) picks the target profile (if it's
 * already available), otherwise falls back to plain reducing.
3261
 */
Y
Yan Zheng 已提交
3262
u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3263
{
3264 3265 3266 3267 3268 3269 3270
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	u64 num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;
3271
	u64 target;
3272

3273 3274 3275 3276
	/*
	 * see if restripe for this chunk_type is in progress, if so
	 * try to reduce to the target profile
	 */
3277
	spin_lock(&root->fs_info->balance_lock);
3278 3279 3280 3281
	target = get_restripe_target(root->fs_info, flags);
	if (target) {
		/* pick target profile only if it's already available */
		if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
3282
			spin_unlock(&root->fs_info->balance_lock);
3283
			return extended_to_chunk(target);
3284 3285 3286 3287
		}
	}
	spin_unlock(&root->fs_info->balance_lock);

3288 3289 3290 3291 3292
	if (num_devices == 1)
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

3293 3294
	if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
	    (flags & (BTRFS_BLOCK_GROUP_RAID1 |
3295
		      BTRFS_BLOCK_GROUP_RAID10))) {
3296
		flags &= ~BTRFS_BLOCK_GROUP_DUP;
3297
	}
3298 3299

	if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
3300
	    (flags & BTRFS_BLOCK_GROUP_RAID10)) {
3301
		flags &= ~BTRFS_BLOCK_GROUP_RAID1;
3302
	}
3303 3304 3305 3306

	if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
	    ((flags & BTRFS_BLOCK_GROUP_RAID1) |
	     (flags & BTRFS_BLOCK_GROUP_RAID10) |
3307
	     (flags & BTRFS_BLOCK_GROUP_DUP))) {
3308
		flags &= ~BTRFS_BLOCK_GROUP_RAID0;
3309 3310
	}

3311
	return extended_to_chunk(flags);
3312 3313
}

3314
static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
J
Josef Bacik 已提交
3315
{
3316
	if (flags & BTRFS_BLOCK_GROUP_DATA)
3317
		flags |= root->fs_info->avail_data_alloc_bits;
3318
	else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3319
		flags |= root->fs_info->avail_system_alloc_bits;
3320
	else if (flags & BTRFS_BLOCK_GROUP_METADATA)
3321 3322
		flags |= root->fs_info->avail_metadata_alloc_bits;

3323
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
3324 3325
}

3326
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
3327
{
3328
	u64 flags;
J
Josef Bacik 已提交
3329

3330 3331 3332 3333
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
3334
	else
3335
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
3336

3337
	return get_alloc_profile(root, flags);
J
Josef Bacik 已提交
3338
}
J
Josef Bacik 已提交
3339

J
Josef Bacik 已提交
3340 3341 3342 3343
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3344
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
3345 3346
{
	struct btrfs_space_info *data_sinfo;
3347
	struct btrfs_root *root = BTRFS_I(inode)->root;
3348
	struct btrfs_fs_info *fs_info = root->fs_info;
3349
	u64 used;
3350
	int ret = 0, committed = 0, alloc_chunk = 1;
J
Josef Bacik 已提交
3351 3352 3353 3354

	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);

3355 3356
	if (root == root->fs_info->tree_root ||
	    BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3357 3358 3359 3360
		alloc_chunk = 0;
		committed = 1;
	}

3361
	data_sinfo = fs_info->data_sinfo;
C
Chris Mason 已提交
3362 3363
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
3364

J
Josef Bacik 已提交
3365 3366 3367
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3368 3369 3370
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3371 3372

	if (used + bytes > data_sinfo->total_bytes) {
3373
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3374

J
Josef Bacik 已提交
3375 3376 3377 3378
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3379
		if (!data_sinfo->full && alloc_chunk) {
J
Josef Bacik 已提交
3380
			u64 alloc_target;
J
Josef Bacik 已提交
3381

3382
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
J
Josef Bacik 已提交
3383
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
3384
alloc:
J
Josef Bacik 已提交
3385
			alloc_target = btrfs_get_alloc_profile(root, 1);
3386
			trans = btrfs_join_transaction(root);
3387 3388
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
3389

J
Josef Bacik 已提交
3390
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
3391 3392
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
J
Josef Bacik 已提交
3393
			btrfs_end_transaction(trans, root);
3394 3395 3396 3397 3398 3399
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
J
Josef Bacik 已提交
3400

3401 3402 3403
			if (!data_sinfo)
				data_sinfo = fs_info->data_sinfo;

J
Josef Bacik 已提交
3404 3405
			goto again;
		}
3406 3407 3408 3409 3410 3411 3412

		/*
		 * If we have less pinned bytes than we want to allocate then
		 * don't bother committing the transaction, it won't help us.
		 */
		if (data_sinfo->bytes_pinned < bytes)
			committed = 1;
J
Josef Bacik 已提交
3413 3414
		spin_unlock(&data_sinfo->lock);

3415
		/* commit the current transaction and try again */
3416
commit_trans:
J
Josef Bacik 已提交
3417 3418
		if (!committed &&
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
3419
			committed = 1;
3420
			trans = btrfs_join_transaction(root);
3421 3422
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3423 3424 3425 3426 3427
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
J
Josef Bacik 已提交
3428

J
Josef Bacik 已提交
3429 3430 3431
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
J
Josef Bacik 已提交
3432
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3433
				      data_sinfo->flags, bytes, 1);
J
Josef Bacik 已提交
3434 3435
	spin_unlock(&data_sinfo->lock);

J
Josef Bacik 已提交
3436 3437
	return 0;
}
J
Josef Bacik 已提交
3438 3439

/*
3440
 * Called if we need to clear a data reservation for this inode.
J
Josef Bacik 已提交
3441
 */
3442
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3443
{
3444
	struct btrfs_root *root = BTRFS_I(inode)->root;
J
Josef Bacik 已提交
3445
	struct btrfs_space_info *data_sinfo;
3446

J
Josef Bacik 已提交
3447 3448
	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3449

3450
	data_sinfo = root->fs_info->data_sinfo;
J
Josef Bacik 已提交
3451 3452
	spin_lock(&data_sinfo->lock);
	data_sinfo->bytes_may_use -= bytes;
J
Josef Bacik 已提交
3453
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3454
				      data_sinfo->flags, bytes, 0);
J
Josef Bacik 已提交
3455
	spin_unlock(&data_sinfo->lock);
3456 3457
}

3458
static void force_metadata_allocation(struct btrfs_fs_info *info)
3459
{
3460 3461
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3462

3463 3464 3465
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3466
			found->force_alloc = CHUNK_ALLOC_FORCE;
3467
	}
3468
	rcu_read_unlock();
3469 3470
}

3471
static int should_alloc_chunk(struct btrfs_root *root,
3472
			      struct btrfs_space_info *sinfo, int force)
3473
{
3474
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3475
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3476
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3477
	u64 thresh;
3478

3479 3480 3481
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

3482 3483 3484 3485 3486
	/*
	 * We need to take into account the global rsv because for all intents
	 * and purposes it's used space.  Don't worry about locking the
	 * global_rsv, it doesn't change except when the transaction commits.
	 */
3487 3488
	if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)
		num_allocated += global_rsv->size;
3489

3490 3491 3492 3493 3494
	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
3495
		thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3496 3497 3498 3499 3500 3501 3502
		thresh = max_t(u64, 64 * 1024 * 1024,
			       div_factor_fine(thresh, 1));

		if (num_bytes - num_allocated < thresh)
			return 1;
	}

3503
	if (num_allocated + 2 * 1024 * 1024 < div_factor(num_bytes, 8))
3504
		return 0;
3505
	return 1;
3506 3507
}

3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
static u64 get_system_chunk_thresh(struct btrfs_root *root, u64 type)
{
	u64 num_dev;

	if (type & BTRFS_BLOCK_GROUP_RAID10 ||
	    type & BTRFS_BLOCK_GROUP_RAID0)
		num_dev = root->fs_info->fs_devices->rw_devices;
	else if (type & BTRFS_BLOCK_GROUP_RAID1)
		num_dev = 2;
	else
		num_dev = 1;	/* DUP or single */

	/* metadata for updaing devices and chunk tree */
	return btrfs_calc_trans_metadata_size(root, num_dev + 1);
}

static void check_system_chunk(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root, u64 type)
{
	struct btrfs_space_info *info;
	u64 left;
	u64 thresh;

	info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	spin_lock(&info->lock);
	left = info->total_bytes - info->bytes_used - info->bytes_pinned -
		info->bytes_reserved - info->bytes_readonly;
	spin_unlock(&info->lock);

	thresh = get_system_chunk_thresh(root, type);
	if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
		printk(KERN_INFO "left=%llu, need=%llu, flags=%llu\n",
		       left, thresh, type);
		dump_space_info(info, 0, 0);
	}

	if (left < thresh) {
		u64 flags;

		flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
		btrfs_alloc_chunk(trans, root, flags);
	}
}

3552
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
3553
			  struct btrfs_root *extent_root, u64 flags, int force)
J
Josef Bacik 已提交
3554
{
3555
	struct btrfs_space_info *space_info;
3556
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
3557
	int wait_for_alloc = 0;
J
Josef Bacik 已提交
3558 3559
	int ret = 0;

3560
	space_info = __find_space_info(extent_root->fs_info, flags);
3561 3562 3563
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
3564
		BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
3565
	}
3566
	BUG_ON(!space_info); /* Logic error */
J
Josef Bacik 已提交
3567

3568
again:
3569
	spin_lock(&space_info->lock);
3570
	if (force < space_info->force_alloc)
3571
		force = space_info->force_alloc;
3572 3573
	if (space_info->full) {
		spin_unlock(&space_info->lock);
3574
		return 0;
J
Josef Bacik 已提交
3575 3576
	}

3577
	if (!should_alloc_chunk(extent_root, space_info, force)) {
3578
		spin_unlock(&space_info->lock);
3579 3580 3581 3582 3583
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
J
Josef Bacik 已提交
3584
	}
3585

3586
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3587

3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

3602 3603 3604 3605 3606 3607 3608
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3609 3610 3611 3612 3613
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
3614
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3615 3616 3617 3618
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
3619 3620
	}

3621 3622 3623 3624 3625 3626
	/*
	 * Check if we have enough space in SYSTEM chunk because we may need
	 * to update devices.
	 */
	check_system_chunk(trans, extent_root, flags);

Y
Yan Zheng 已提交
3627
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
3628 3629 3630
	if (ret < 0 && ret != -ENOSPC)
		goto out;

J
Josef Bacik 已提交
3631 3632
	spin_lock(&space_info->lock);
	if (ret)
3633
		space_info->full = 1;
3634 3635
	else
		ret = 1;
3636

3637
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3638
	space_info->chunk_alloc = 0;
J
Josef Bacik 已提交
3639
	spin_unlock(&space_info->lock);
3640
out:
3641
	mutex_unlock(&fs_info->chunk_mutex);
J
Josef Bacik 已提交
3642
	return ret;
3643
}
J
Josef Bacik 已提交
3644

J
Josef Bacik 已提交
3645 3646
static int can_overcommit(struct btrfs_root *root,
			  struct btrfs_space_info *space_info, u64 bytes,
M
Miao Xie 已提交
3647
			  enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670
{
	u64 profile = btrfs_get_alloc_profile(root, 0);
	u64 avail;
	u64 used;

	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;

	spin_lock(&root->fs_info->free_chunk_lock);
	avail = root->fs_info->free_chunk_space;
	spin_unlock(&root->fs_info->free_chunk_lock);

	/*
	 * If we have dup, raid1 or raid10 then only half of the free
	 * space is actually useable.
	 */
	if (profile & (BTRFS_BLOCK_GROUP_DUP |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_RAID10))
		avail >>= 1;

	/*
3671 3672 3673
	 * If we aren't flushing all things, let us overcommit up to
	 * 1/2th of the space. If we can flush, don't let us overcommit
	 * too much, let it overcommit up to 1/8 of the space.
J
Josef Bacik 已提交
3674
	 */
M
Miao Xie 已提交
3675
	if (flush == BTRFS_RESERVE_FLUSH_ALL)
J
Josef Bacik 已提交
3676 3677 3678 3679 3680 3681 3682 3683 3684
		avail >>= 3;
	else
		avail >>= 1;

	if (used + bytes < space_info->total_bytes + avail)
		return 1;
	return 0;
}

J
Josef Bacik 已提交
3685
/*
3686
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
3687
 */
J
Josef Bacik 已提交
3688 3689
static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig,
			    bool wait_ordered)
3690
{
3691
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
3692
	struct btrfs_space_info *space_info;
3693
	struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3694
	u64 delalloc_bytes;
3695
	u64 max_reclaim;
3696
	long time_left;
3697
	unsigned long nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3698
	int loops = 0;
M
Miao Xie 已提交
3699
	enum btrfs_reserve_flush_enum flush;
3700

3701
	trans = (struct btrfs_trans_handle *)current->journal_info;
3702
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
3703
	space_info = block_rsv->space_info;
3704 3705

	smp_mb();
J
Josef Bacik 已提交
3706 3707
	delalloc_bytes = root->fs_info->delalloc_bytes;
	if (delalloc_bytes == 0) {
3708
		if (trans)
J
Josef Bacik 已提交
3709
			return;
3710
		btrfs_wait_ordered_extents(root, 0);
J
Josef Bacik 已提交
3711
		return;
3712 3713
	}

J
Josef Bacik 已提交
3714 3715 3716
	while (delalloc_bytes && loops < 3) {
		max_reclaim = min(delalloc_bytes, to_reclaim);
		nr_pages = max_reclaim >> PAGE_CACHE_SHIFT;
3717
		writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages,
J
Josef Bacik 已提交
3718
					       WB_REASON_FS_FREE_SPACE);
3719

3720 3721 3722 3723 3724 3725 3726
		/*
		 * We need to wait for the async pages to actually start before
		 * we do anything.
		 */
		wait_event(root->fs_info->async_submit_wait,
			   !atomic_read(&root->fs_info->async_delalloc_pages));

M
Miao Xie 已提交
3727 3728 3729 3730
		if (!trans)
			flush = BTRFS_RESERVE_FLUSH_ALL;
		else
			flush = BTRFS_RESERVE_NO_FLUSH;
J
Josef Bacik 已提交
3731
		spin_lock(&space_info->lock);
M
Miao Xie 已提交
3732
		if (can_overcommit(root, space_info, orig, flush)) {
J
Josef Bacik 已提交
3733 3734 3735
			spin_unlock(&space_info->lock);
			break;
		}
J
Josef Bacik 已提交
3736
		spin_unlock(&space_info->lock);
3737

3738
		loops++;
3739
		if (wait_ordered && !trans) {
3740
			btrfs_wait_ordered_extents(root, 0);
3741
		} else {
J
Josef Bacik 已提交
3742
			time_left = schedule_timeout_killable(1);
3743 3744 3745
			if (time_left)
				break;
		}
J
Josef Bacik 已提交
3746 3747
		smp_mb();
		delalloc_bytes = root->fs_info->delalloc_bytes;
3748 3749 3750
	}
}

3751 3752 3753 3754 3755
/**
 * maybe_commit_transaction - possibly commit the transaction if its ok to
 * @root - the root we're allocating for
 * @bytes - the number of bytes we want to reserve
 * @force - force the commit
3756
 *
3757 3758 3759
 * This will check to make sure that committing the transaction will actually
 * get us somewhere and then commit the transaction if it does.  Otherwise it
 * will return -ENOSPC.
3760
 */
3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789
static int may_commit_transaction(struct btrfs_root *root,
				  struct btrfs_space_info *space_info,
				  u64 bytes, int force)
{
	struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
	struct btrfs_trans_handle *trans;

	trans = (struct btrfs_trans_handle *)current->journal_info;
	if (trans)
		return -EAGAIN;

	if (force)
		goto commit;

	/* See if there is enough pinned space to make this reservation */
	spin_lock(&space_info->lock);
	if (space_info->bytes_pinned >= bytes) {
		spin_unlock(&space_info->lock);
		goto commit;
	}
	spin_unlock(&space_info->lock);

	/*
	 * See if there is some space in the delayed insertion reservation for
	 * this reservation.
	 */
	if (space_info != delayed_rsv->space_info)
		return -ENOSPC;

L
Liu Bo 已提交
3790
	spin_lock(&space_info->lock);
3791
	spin_lock(&delayed_rsv->lock);
L
Liu Bo 已提交
3792
	if (space_info->bytes_pinned + delayed_rsv->size < bytes) {
3793
		spin_unlock(&delayed_rsv->lock);
L
Liu Bo 已提交
3794
		spin_unlock(&space_info->lock);
3795 3796 3797
		return -ENOSPC;
	}
	spin_unlock(&delayed_rsv->lock);
L
Liu Bo 已提交
3798
	spin_unlock(&space_info->lock);
3799 3800 3801 3802 3803 3804 3805 3806 3807

commit:
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans))
		return -ENOSPC;

	return btrfs_commit_transaction(trans, root);
}

3808
enum flush_state {
3809 3810 3811 3812
	FLUSH_DELAYED_ITEMS_NR	=	1,
	FLUSH_DELAYED_ITEMS	=	2,
	FLUSH_DELALLOC		=	3,
	FLUSH_DELALLOC_WAIT	=	4,
3813 3814
	ALLOC_CHUNK		=	5,
	COMMIT_TRANS		=	6,
3815 3816 3817 3818 3819 3820 3821 3822
};

static int flush_space(struct btrfs_root *root,
		       struct btrfs_space_info *space_info, u64 num_bytes,
		       u64 orig_bytes, int state)
{
	struct btrfs_trans_handle *trans;
	int nr;
J
Josef Bacik 已提交
3823
	int ret = 0;
3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845

	switch (state) {
	case FLUSH_DELAYED_ITEMS_NR:
	case FLUSH_DELAYED_ITEMS:
		if (state == FLUSH_DELAYED_ITEMS_NR) {
			u64 bytes = btrfs_calc_trans_metadata_size(root, 1);

			nr = (int)div64_u64(num_bytes, bytes);
			if (!nr)
				nr = 1;
			nr *= 2;
		} else {
			nr = -1;
		}
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = btrfs_run_delayed_items_nr(trans, root, nr);
		btrfs_end_transaction(trans, root);
		break;
3846 3847 3848 3849 3850
	case FLUSH_DELALLOC:
	case FLUSH_DELALLOC_WAIT:
		shrink_delalloc(root, num_bytes, orig_bytes,
				state == FLUSH_DELALLOC_WAIT);
		break;
3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863
	case ALLOC_CHUNK:
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
				     btrfs_get_alloc_profile(root, 0),
				     CHUNK_ALLOC_NO_FORCE);
		btrfs_end_transaction(trans, root);
		if (ret == -ENOSPC)
			ret = 0;
		break;
3864 3865 3866 3867 3868 3869 3870 3871 3872 3873
	case COMMIT_TRANS:
		ret = may_commit_transaction(root, space_info, orig_bytes, 0);
		break;
	default:
		ret = -ENOSPC;
		break;
	}

	return ret;
}
3874 3875 3876 3877 3878 3879
/**
 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
 * @root - the root we're allocating for
 * @block_rsv - the block_rsv we're allocating for
 * @orig_bytes - the number of bytes we want
 * @flush - wether or not we can flush to make our reservation
3880
 *
3881 3882 3883 3884 3885 3886
 * This will reserve orgi_bytes number of bytes from the space info associated
 * with the block_rsv.  If there is not enough space it will make an attempt to
 * flush out space to make room.  It will do this by flushing delalloc if
 * possible or committing the transaction.  If flush is 0 then no attempts to
 * regain reservations will be made and this will fail if there is not enough
 * space already.
3887
 */
3888
static int reserve_metadata_bytes(struct btrfs_root *root,
3889
				  struct btrfs_block_rsv *block_rsv,
M
Miao Xie 已提交
3890 3891
				  u64 orig_bytes,
				  enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
3892
{
3893
	struct btrfs_space_info *space_info = block_rsv->space_info;
3894
	u64 used;
3895
	u64 num_bytes = orig_bytes;
3896
	int flush_state = FLUSH_DELAYED_ITEMS_NR;
3897
	int ret = 0;
3898
	bool flushing = false;
J
Josef Bacik 已提交
3899

3900
again:
3901
	ret = 0;
3902
	spin_lock(&space_info->lock);
3903
	/*
M
Miao Xie 已提交
3904 3905
	 * We only want to wait if somebody other than us is flushing and we
	 * are actually allowed to flush all things.
3906
	 */
M
Miao Xie 已提交
3907 3908
	while (flush == BTRFS_RESERVE_FLUSH_ALL && !flushing &&
	       space_info->flush) {
3909 3910 3911 3912 3913 3914 3915
		spin_unlock(&space_info->lock);
		/*
		 * If we have a trans handle we can't wait because the flusher
		 * may have to commit the transaction, which would mean we would
		 * deadlock since we are waiting for the flusher to finish, but
		 * hold the current transaction open.
		 */
3916
		if (current->journal_info)
3917
			return -EAGAIN;
A
Arne Jansen 已提交
3918 3919 3920
		ret = wait_event_killable(space_info->wait, !space_info->flush);
		/* Must have been killed, return */
		if (ret)
3921 3922 3923 3924 3925 3926
			return -EINTR;

		spin_lock(&space_info->lock);
	}

	ret = -ENOSPC;
3927 3928 3929
	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;
J
Josef Bacik 已提交
3930

3931 3932 3933 3934 3935 3936 3937
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
3938 3939
	if (used <= space_info->total_bytes) {
		if (used + orig_bytes <= space_info->total_bytes) {
3940
			space_info->bytes_may_use += orig_bytes;
J
Josef Bacik 已提交
3941
			trace_btrfs_space_reservation(root->fs_info,
3942
				"space_info", space_info->flags, orig_bytes, 1);
3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
3958
		num_bytes = used - space_info->total_bytes +
3959
			(orig_bytes * 2);
3960
	}
J
Josef Bacik 已提交
3961

3962 3963 3964 3965 3966 3967
	if (ret && can_overcommit(root, space_info, orig_bytes, flush)) {
		space_info->bytes_may_use += orig_bytes;
		trace_btrfs_space_reservation(root->fs_info, "space_info",
					      space_info->flags, orig_bytes,
					      1);
		ret = 0;
3968 3969
	}

3970 3971 3972 3973
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
M
Miao Xie 已提交
3974 3975 3976
	 *
	 * We make the other tasks wait for the flush only when we can flush
	 * all things.
3977
	 */
M
Miao Xie 已提交
3978
	if (ret && flush == BTRFS_RESERVE_FLUSH_ALL) {
3979 3980
		flushing = true;
		space_info->flush = 1;
3981
	}
J
Josef Bacik 已提交
3982

3983
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3984

M
Miao Xie 已提交
3985
	if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
3986
		goto out;
3987

3988 3989 3990
	ret = flush_space(root, space_info, num_bytes, orig_bytes,
			  flush_state);
	flush_state++;
M
Miao Xie 已提交
3991 3992 3993 3994 3995 3996 3997 3998 3999 4000

	/*
	 * If we are FLUSH_LIMIT, we can not flush delalloc, or the deadlock
	 * would happen. So skip delalloc flush.
	 */
	if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
	    (flush_state == FLUSH_DELALLOC ||
	     flush_state == FLUSH_DELALLOC_WAIT))
		flush_state = ALLOC_CHUNK;

4001
	if (!ret)
4002
		goto again;
M
Miao Xie 已提交
4003 4004 4005 4006 4007
	else if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
		 flush_state < COMMIT_TRANS)
		goto again;
	else if (flush == BTRFS_RESERVE_FLUSH_ALL &&
		 flush_state <= COMMIT_TRANS)
4008 4009 4010
		goto again;

out:
4011
	if (flushing) {
4012
		spin_lock(&space_info->lock);
4013 4014
		space_info->flush = 0;
		wake_up_all(&space_info->wait);
4015
		spin_unlock(&space_info->lock);
4016 4017 4018 4019
	}
	return ret;
}

4020 4021 4022
static struct btrfs_block_rsv *get_block_rsv(
					const struct btrfs_trans_handle *trans,
					const struct btrfs_root *root)
4023
{
4024 4025
	struct btrfs_block_rsv *block_rsv = NULL;

4026 4027 4028 4029
	if (root->ref_cows)
		block_rsv = trans->block_rsv;

	if (root == root->fs_info->csum_root && trans->adding_csums)
4030
		block_rsv = trans->block_rsv;
4031 4032

	if (!block_rsv)
4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

J
Josef Bacik 已提交
4068 4069
static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
				    struct btrfs_block_rsv *block_rsv,
4070
				    struct btrfs_block_rsv *dest, u64 num_bytes)
4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
4103
			spin_lock(&space_info->lock);
4104
			space_info->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4105
			trace_btrfs_space_reservation(fs_info, "space_info",
4106
					space_info->flags, num_bytes, 0);
4107
			space_info->reservation_progress++;
4108
			spin_unlock(&space_info->lock);
4109
		}
J
Josef Bacik 已提交
4110
	}
4111
}
4112

4113 4114 4115 4116
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
J
Josef Bacik 已提交
4117

4118 4119 4120
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
4121

4122
	block_rsv_add_bytes(dst, num_bytes, 1);
J
Josef Bacik 已提交
4123 4124 4125
	return 0;
}

4126
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
J
Josef Bacik 已提交
4127
{
4128 4129
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
4130
	rsv->type = type;
4131 4132
}

4133 4134
struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root,
					      unsigned short type)
4135 4136 4137
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
4138

4139 4140 4141
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
4142

4143
	btrfs_init_block_rsv(block_rsv, type);
4144 4145 4146 4147
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
4148

4149 4150 4151
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
J
Josef Bacik 已提交
4152 4153
	if (!rsv)
		return;
4154 4155
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	kfree(rsv);
J
Josef Bacik 已提交
4156 4157
}

M
Miao Xie 已提交
4158 4159 4160
int btrfs_block_rsv_add(struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv, u64 num_bytes,
			enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
4161
{
4162
	int ret;
J
Josef Bacik 已提交
4163

4164 4165
	if (num_bytes == 0)
		return 0;
4166

4167
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4168 4169 4170 4171
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
4172

4173 4174
	return ret;
}
J
Josef Bacik 已提交
4175

4176
int btrfs_block_rsv_check(struct btrfs_root *root,
4177
			  struct btrfs_block_rsv *block_rsv, int min_factor)
4178 4179 4180
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
4181

4182 4183
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
4184

4185
	spin_lock(&block_rsv->lock);
4186 4187 4188 4189
	num_bytes = div_factor(block_rsv->size, min_factor);
	if (block_rsv->reserved >= num_bytes)
		ret = 0;
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
4190

4191 4192 4193
	return ret;
}

M
Miao Xie 已提交
4194 4195 4196
int btrfs_block_rsv_refill(struct btrfs_root *root,
			   struct btrfs_block_rsv *block_rsv, u64 min_reserved,
			   enum btrfs_reserve_flush_enum flush)
4197 4198 4199 4200 4201 4202 4203 4204 4205
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;

	if (!block_rsv)
		return 0;

	spin_lock(&block_rsv->lock);
	num_bytes = min_reserved;
4206
	if (block_rsv->reserved >= num_bytes)
4207
		ret = 0;
4208
	else
4209 4210
		num_bytes -= block_rsv->reserved;
	spin_unlock(&block_rsv->lock);
4211

4212 4213 4214
	if (!ret)
		return 0;

4215
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4216 4217
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 0);
4218
		return 0;
J
Josef Bacik 已提交
4219
	}
J
Josef Bacik 已提交
4220

4221
	return ret;
4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238
}

int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
	if (global_rsv->full || global_rsv == block_rsv ||
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
J
Josef Bacik 已提交
4239 4240
	block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
				num_bytes);
J
Josef Bacik 已提交
4241 4242 4243
}

/*
4244 4245 4246
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
4247
 */
4248
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4249
{
4250 4251 4252 4253
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
4254
	int csum_size = btrfs_super_csum_size(fs_info->super_copy);
J
Josef Bacik 已提交
4255

4256 4257 4258 4259
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
4260

4261 4262
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
4263 4264
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
4265 4266
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
4267

4268 4269 4270
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
	num_bytes += div64_u64(data_used + meta_used, 50);
4271

4272
	if (num_bytes * 3 > meta_used)
4273
		num_bytes = div64_u64(meta_used, 3);
4274

4275 4276
	return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
}
J
Josef Bacik 已提交
4277

4278 4279 4280 4281 4282
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
4283

4284
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
4285

4286
	spin_lock(&sinfo->lock);
4287
	spin_lock(&block_rsv->lock);
4288

4289
	block_rsv->size = num_bytes;
4290

4291
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
4292 4293
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
4294 4295 4296 4297

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
4298
		sinfo->bytes_may_use += num_bytes;
J
Josef Bacik 已提交
4299
		trace_btrfs_space_reservation(fs_info, "space_info",
4300
				      sinfo->flags, num_bytes, 1);
J
Josef Bacik 已提交
4301 4302
	}

4303 4304
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
4305
		sinfo->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4306
		trace_btrfs_space_reservation(fs_info, "space_info",
4307
				      sinfo->flags, num_bytes, 0);
4308
		sinfo->reservation_progress++;
4309 4310 4311
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
4312

4313
	spin_unlock(&block_rsv->lock);
4314
	spin_unlock(&sinfo->lock);
J
Josef Bacik 已提交
4315 4316
}

4317
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4318
{
4319
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
4320

4321 4322
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
J
Josef Bacik 已提交
4323

4324
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4325 4326
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->delalloc_block_rsv.space_info = space_info;
4327 4328
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
4329
	fs_info->delayed_block_rsv.space_info = space_info;
4330

4331 4332 4333 4334
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
4335
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
4336 4337

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
4338 4339
}

4340
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4341
{
J
Josef Bacik 已提交
4342 4343
	block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
				(u64)-1);
4344 4345 4346 4347 4348 4349
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
4350 4351
	WARN_ON(fs_info->delayed_block_rsv.size > 0);
	WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
4352 4353
}

4354 4355
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
4356
{
4357 4358 4359
	if (!trans->block_rsv)
		return;

4360 4361
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
4362

4363
	trace_btrfs_space_reservation(root->fs_info, "transaction",
4364
				      trans->transid, trans->bytes_reserved, 0);
4365
	btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
4366 4367
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
4368

4369
/* Can only return 0 or -ENOSPC */
4370 4371 4372 4373 4374 4375 4376 4377
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
4378 4379 4380
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
4381
	 */
C
Chris Mason 已提交
4382
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4383 4384
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 1);
4385
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
J
Josef Bacik 已提交
4386 4387
}

4388
void btrfs_orphan_release_metadata(struct inode *inode)
4389
{
4390
	struct btrfs_root *root = BTRFS_I(inode)->root;
C
Chris Mason 已提交
4391
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4392 4393
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 0);
4394 4395
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
4396

4397 4398 4399 4400 4401 4402 4403
int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
				struct btrfs_pending_snapshot *pending)
{
	struct btrfs_root *root = pending->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
	/*
4404 4405
	 * two for root back/forward refs, two for directory entries,
	 * one for root of the snapshot and one for parent inode.
4406
	 */
4407
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 6);
4408 4409
	dst_rsv->space_info = src_rsv->space_info;
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4410 4411
}

4412 4413 4414 4415 4416 4417 4418 4419 4420
/**
 * drop_outstanding_extent - drop an outstanding extent
 * @inode: the inode we're dropping the extent for
 *
 * This is called when we are freeing up an outstanding extent, either called
 * after an error or after an extent is written.  This will return the number of
 * reserved extents that need to be freed.  This must be called with
 * BTRFS_I(inode)->lock held.
 */
4421 4422
static unsigned drop_outstanding_extent(struct inode *inode)
{
4423
	unsigned drop_inode_space = 0;
4424 4425 4426 4427 4428
	unsigned dropped_extents = 0;

	BUG_ON(!BTRFS_I(inode)->outstanding_extents);
	BTRFS_I(inode)->outstanding_extents--;

4429
	if (BTRFS_I(inode)->outstanding_extents == 0 &&
4430 4431
	    test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			       &BTRFS_I(inode)->runtime_flags))
4432 4433
		drop_inode_space = 1;

4434 4435 4436 4437 4438 4439
	/*
	 * If we have more or the same amount of outsanding extents than we have
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
4440
		return drop_inode_space;
4441 4442 4443 4444

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
4445
	return dropped_extents + drop_inode_space;
4446 4447
}

4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467
/**
 * calc_csum_metadata_size - return the amount of metada space that must be
 *	reserved/free'd for the given bytes.
 * @inode: the inode we're manipulating
 * @num_bytes: the number of bytes in question
 * @reserve: 1 if we are reserving space, 0 if we are freeing space
 *
 * This adjusts the number of csum_bytes in the inode and then returns the
 * correct amount of metadata that must either be reserved or freed.  We
 * calculate how many checksums we can fit into one leaf and then divide the
 * number of bytes that will need to be checksumed by this value to figure out
 * how many checksums will be required.  If we are adding bytes then the number
 * may go up and we will return the number of additional bytes that must be
 * reserved.  If it is going down we will return the number of bytes that must
 * be freed.
 *
 * This must be called with BTRFS_I(inode)->lock held.
 */
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
				   int reserve)
4468
{
4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 csum_size;
	int num_csums_per_leaf;
	int num_csums;
	int old_csums;

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
	    BTRFS_I(inode)->csum_bytes == 0)
		return 0;

	old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	if (reserve)
		BTRFS_I(inode)->csum_bytes += num_bytes;
	else
		BTRFS_I(inode)->csum_bytes -= num_bytes;
	csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
	num_csums_per_leaf = (int)div64_u64(csum_size,
					    sizeof(struct btrfs_csum_item) +
					    sizeof(struct btrfs_disk_key));
	num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	num_csums = num_csums + num_csums_per_leaf - 1;
	num_csums = num_csums / num_csums_per_leaf;

	old_csums = old_csums + num_csums_per_leaf - 1;
	old_csums = old_csums / num_csums_per_leaf;

	/* No change, no need to reserve more */
	if (old_csums == num_csums)
		return 0;

	if (reserve)
		return btrfs_calc_trans_metadata_size(root,
						      num_csums - old_csums);

	return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
4504
}
Y
Yan Zheng 已提交
4505

4506 4507 4508 4509
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4510
	u64 to_reserve = 0;
4511
	u64 csum_bytes;
4512
	unsigned nr_extents = 0;
4513
	int extra_reserve = 0;
M
Miao Xie 已提交
4514
	enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
4515
	int ret;
4516

4517
	/* Need to be holding the i_mutex here if we aren't free space cache */
4518
	if (btrfs_is_free_space_inode(inode))
M
Miao Xie 已提交
4519
		flush = BTRFS_RESERVE_NO_FLUSH;
4520

M
Miao Xie 已提交
4521 4522
	if (flush != BTRFS_RESERVE_NO_FLUSH &&
	    btrfs_transaction_in_commit(root->fs_info))
4523
		schedule_timeout(1);
4524

4525
	mutex_lock(&BTRFS_I(inode)->delalloc_mutex);
4526
	num_bytes = ALIGN(num_bytes, root->sectorsize);
4527

4528 4529 4530 4531
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;

	if (BTRFS_I(inode)->outstanding_extents >
4532
	    BTRFS_I(inode)->reserved_extents)
4533 4534
		nr_extents = BTRFS_I(inode)->outstanding_extents -
			BTRFS_I(inode)->reserved_extents;
4535

4536 4537 4538 4539
	/*
	 * Add an item to reserve for updating the inode when we complete the
	 * delalloc io.
	 */
4540 4541
	if (!test_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
		      &BTRFS_I(inode)->runtime_flags)) {
4542
		nr_extents++;
4543
		extra_reserve = 1;
4544
	}
4545 4546

	to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4547
	to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
4548
	csum_bytes = BTRFS_I(inode)->csum_bytes;
4549
	spin_unlock(&BTRFS_I(inode)->lock);
4550

4551 4552 4553
	if (root->fs_info->quota_enabled) {
		ret = btrfs_qgroup_reserve(root, num_bytes +
					   nr_extents * root->leafsize);
4554 4555
		if (ret) {
			mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4556
			return ret;
4557
		}
4558 4559
	}

4560
	ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush);
4561
	if (ret) {
4562
		u64 to_free = 0;
4563
		unsigned dropped;
4564

4565
		spin_lock(&BTRFS_I(inode)->lock);
4566 4567
		dropped = drop_outstanding_extent(inode);
		/*
4568 4569 4570 4571 4572 4573
		 * If the inodes csum_bytes is the same as the original
		 * csum_bytes then we know we haven't raced with any free()ers
		 * so we can just reduce our inodes csum bytes and carry on.
		 * Otherwise we have to do the normal free thing to account for
		 * the case that the free side didn't free up its reserve
		 * because of this outstanding reservation.
4574
		 */
4575 4576 4577 4578 4579 4580 4581 4582
		if (BTRFS_I(inode)->csum_bytes == csum_bytes)
			calc_csum_metadata_size(inode, num_bytes, 0);
		else
			to_free = calc_csum_metadata_size(inode, num_bytes, 0);
		spin_unlock(&BTRFS_I(inode)->lock);
		if (dropped)
			to_free += btrfs_calc_trans_metadata_size(root, dropped);

J
Josef Bacik 已提交
4583
		if (to_free) {
4584
			btrfs_block_rsv_release(root, block_rsv, to_free);
J
Josef Bacik 已提交
4585 4586 4587 4588 4589
			trace_btrfs_space_reservation(root->fs_info,
						      "delalloc",
						      btrfs_ino(inode),
						      to_free, 0);
		}
4590
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4591
		return ret;
4592
	}
4593

4594 4595
	spin_lock(&BTRFS_I(inode)->lock);
	if (extra_reserve) {
4596 4597
		set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			&BTRFS_I(inode)->runtime_flags);
4598 4599 4600 4601
		nr_extents--;
	}
	BTRFS_I(inode)->reserved_extents += nr_extents;
	spin_unlock(&BTRFS_I(inode)->lock);
4602
	mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4603

J
Josef Bacik 已提交
4604 4605 4606
	if (to_reserve)
		trace_btrfs_space_reservation(root->fs_info,"delalloc",
					      btrfs_ino(inode), to_reserve, 1);
4607 4608 4609 4610 4611
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	return 0;
}

4612 4613 4614 4615 4616 4617 4618 4619 4620
/**
 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
 * @inode: the inode to release the reservation for
 * @num_bytes: the number of bytes we're releasing
 *
 * This will release the metadata reservation for an inode.  This can be called
 * once we complete IO for a given set of bytes to release their metadata
 * reservations.
 */
4621 4622 4623
void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
4624 4625
	u64 to_free = 0;
	unsigned dropped;
4626 4627

	num_bytes = ALIGN(num_bytes, root->sectorsize);
4628
	spin_lock(&BTRFS_I(inode)->lock);
4629
	dropped = drop_outstanding_extent(inode);
4630

4631 4632
	to_free = calc_csum_metadata_size(inode, num_bytes, 0);
	spin_unlock(&BTRFS_I(inode)->lock);
4633 4634
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
4635

J
Josef Bacik 已提交
4636 4637
	trace_btrfs_space_reservation(root->fs_info, "delalloc",
				      btrfs_ino(inode), to_free, 0);
4638 4639 4640 4641 4642
	if (root->fs_info->quota_enabled) {
		btrfs_qgroup_free(root, num_bytes +
					dropped * root->leafsize);
	}

4643 4644 4645 4646
	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661
/**
 * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
 * @inode: inode we're writing to
 * @num_bytes: the number of bytes we want to allocate
 *
 * This will do the following things
 *
 * o reserve space in the data space info for num_bytes
 * o reserve space in the metadata space info based on number of outstanding
 *   extents and how much csums will be needed
 * o add to the inodes ->delalloc_bytes
 * o add it to the fs_info's delalloc inodes list.
 *
 * This will return 0 for success and -ENOSPC if there is no space left.
 */
4662 4663 4664 4665 4666
int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
C
Chris Mason 已提交
4667
	if (ret)
4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691
/**
 * btrfs_delalloc_release_space - release data and metadata space for delalloc
 * @inode: inode we're releasing space for
 * @num_bytes: the number of bytes we want to free up
 *
 * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
 * called in the case that we don't need the metadata AND data reservations
 * anymore.  So if there is an error or we insert an inline extent.
 *
 * This function will release the metadata space that was not used and will
 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
 * list if there are no delalloc bytes left.
 */
4692 4693 4694 4695
void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
4696 4697
}

C
Chris Mason 已提交
4698 4699
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
4700
			      u64 bytenr, u64 num_bytes, int alloc)
C
Chris Mason 已提交
4701
{
4702
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
4703
	struct btrfs_fs_info *info = root->fs_info;
4704
	u64 total = num_bytes;
C
Chris Mason 已提交
4705
	u64 old_val;
4706
	u64 byte_in_group;
4707
	int factor;
C
Chris Mason 已提交
4708

4709 4710
	/* block accounting for super block */
	spin_lock(&info->delalloc_lock);
4711
	old_val = btrfs_super_bytes_used(info->super_copy);
4712 4713 4714 4715
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
4716
	btrfs_set_super_bytes_used(info->super_copy, old_val);
4717 4718
	spin_unlock(&info->delalloc_lock);

C
Chris Mason 已提交
4719
	while (total) {
4720
		cache = btrfs_lookup_block_group(info, bytenr);
4721
		if (!cache)
4722
			return -ENOENT;
4723 4724 4725 4726 4727 4728
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
4729 4730 4731 4732 4733 4734 4735
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
4736
			cache_block_group(cache, trans, NULL, 1);
4737

4738 4739
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
4740

4741
		spin_lock(&cache->space_info->lock);
4742
		spin_lock(&cache->lock);
4743

4744
		if (btrfs_test_opt(root, SPACE_CACHE) &&
4745 4746 4747
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

J
Josef Bacik 已提交
4748
		cache->dirty = 1;
C
Chris Mason 已提交
4749
		old_val = btrfs_block_group_used(&cache->item);
4750
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
4751
		if (alloc) {
4752
			old_val += num_bytes;
4753 4754 4755
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
4756 4757
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
4758
			spin_unlock(&cache->lock);
4759
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4760
		} else {
4761
			old_val -= num_bytes;
4762
			btrfs_set_block_group_used(&cache->item, old_val);
4763 4764
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
4765
			cache->space_info->bytes_used -= num_bytes;
4766
			cache->space_info->disk_used -= num_bytes * factor;
4767
			spin_unlock(&cache->lock);
4768
			spin_unlock(&cache->space_info->lock);
4769

4770 4771 4772
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
C
Chris Mason 已提交
4773
		}
4774
		btrfs_put_block_group(cache);
4775 4776
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
4777 4778 4779
	}
	return 0;
}
4780

4781 4782
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
4783
	struct btrfs_block_group_cache *cache;
4784
	u64 bytenr;
J
Josef Bacik 已提交
4785 4786 4787

	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
4788
		return 0;
J
Josef Bacik 已提交
4789

4790
	bytenr = cache->key.objectid;
4791
	btrfs_put_block_group(cache);
4792 4793

	return bytenr;
4794 4795
}

4796 4797 4798
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
4799
{
4800 4801 4802 4803 4804 4805 4806 4807 4808 4809
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
4810

4811 4812 4813 4814
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
	return 0;
}
J
Josef Bacik 已提交
4815

4816 4817 4818 4819 4820 4821 4822
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
4823

4824
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4825
	BUG_ON(!cache); /* Logic error */
4826 4827 4828 4829

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
4830 4831 4832
	return 0;
}

4833
/*
4834 4835 4836 4837 4838 4839 4840 4841 4842
 * this function must be called within transaction
 */
int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans,
				    struct btrfs_root *root,
				    u64 bytenr, u64 num_bytes)
{
	struct btrfs_block_group_cache *cache;

	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4843
	BUG_ON(!cache); /* Logic error */
4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860

	/*
	 * pull in the free space cache (if any) so that our pin
	 * removes the free space from the cache.  We have load_only set
	 * to one because the slow code to read in the free extents does check
	 * the pinned extents.
	 */
	cache_block_group(cache, trans, root, 1);

	pin_down_extent(root, cache, bytenr, num_bytes, 0);

	/* remove us from the free space cache (if we're there at all) */
	btrfs_remove_free_space(cache, bytenr, num_bytes);
	btrfs_put_block_group(cache);
	return 0;
}

4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881
/**
 * btrfs_update_reserved_bytes - update the block_group and space info counters
 * @cache:	The cache we are manipulating
 * @num_bytes:	The number of bytes in question
 * @reserve:	One of the reservation enums
 *
 * This is called by the allocator when it reserves space, or by somebody who is
 * freeing space that was never actually used on disk.  For example if you
 * reserve some space for a new leaf in transaction A and before transaction A
 * commits you free that leaf, you call this with reserve set to 0 in order to
 * clear the reservation.
 *
 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
 * ENOSPC accounting.  For data we handle the reservation through clearing the
 * delalloc bits in the io_tree.  We have to do this since we could end up
 * allocating less disk space for the amount of data we have reserved in the
 * case of compression.
 *
 * If this is a reservation and the block group has become read only we cannot
 * make the reservation and return -EAGAIN, otherwise this function always
 * succeeds.
4882
 */
4883 4884
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve)
4885
{
4886
	struct btrfs_space_info *space_info = cache->space_info;
4887
	int ret = 0;
4888

4889 4890 4891
	spin_lock(&space_info->lock);
	spin_lock(&cache->lock);
	if (reserve != RESERVE_FREE) {
4892 4893 4894
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
4895 4896 4897
			cache->reserved += num_bytes;
			space_info->bytes_reserved += num_bytes;
			if (reserve == RESERVE_ALLOC) {
J
Josef Bacik 已提交
4898
				trace_btrfs_space_reservation(cache->fs_info,
4899 4900
						"space_info", space_info->flags,
						num_bytes, 0);
4901 4902
				space_info->bytes_may_use -= num_bytes;
			}
4903
		}
4904 4905 4906 4907 4908 4909
	} else {
		if (cache->ro)
			space_info->bytes_readonly += num_bytes;
		cache->reserved -= num_bytes;
		space_info->bytes_reserved -= num_bytes;
		space_info->reservation_progress++;
4910
	}
4911 4912
	spin_unlock(&cache->lock);
	spin_unlock(&space_info->lock);
4913
	return ret;
4914
}
C
Chris Mason 已提交
4915

4916
void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
4917
				struct btrfs_root *root)
4918 4919
{
	struct btrfs_fs_info *fs_info = root->fs_info;
4920 4921 4922
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
4923

4924
	down_write(&fs_info->extent_commit_sem);
4925

4926 4927 4928 4929 4930 4931 4932
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
4933
		} else {
4934
			cache->last_byte_to_unpin = caching_ctl->progress;
4935 4936
		}
	}
4937 4938 4939 4940 4941 4942 4943

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

	up_write(&fs_info->extent_commit_sem);
4944 4945

	update_global_block_rsv(fs_info);
4946 4947
}

4948
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
C
Chris Mason 已提交
4949
{
4950 4951
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
4952 4953
	struct btrfs_space_info *space_info;
	struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
4954
	u64 len;
4955
	bool readonly;
C
Chris Mason 已提交
4956

4957
	while (start <= end) {
4958
		readonly = false;
4959 4960 4961 4962 4963
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
4964
			BUG_ON(!cache); /* Logic error */
4965 4966 4967 4968 4969 4970 4971 4972 4973 4974
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
			btrfs_add_free_space(cache, start, len);
		}

4975
		start += len;
4976
		space_info = cache->space_info;
4977

4978
		spin_lock(&space_info->lock);
4979 4980
		spin_lock(&cache->lock);
		cache->pinned -= len;
4981 4982 4983 4984 4985
		space_info->bytes_pinned -= len;
		if (cache->ro) {
			space_info->bytes_readonly += len;
			readonly = true;
		}
4986
		spin_unlock(&cache->lock);
4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999
		if (!readonly && global_rsv->space_info == space_info) {
			spin_lock(&global_rsv->lock);
			if (!global_rsv->full) {
				len = min(len, global_rsv->size -
					  global_rsv->reserved);
				global_rsv->reserved += len;
				space_info->bytes_may_use += len;
				if (global_rsv->reserved >= global_rsv->size)
					global_rsv->full = 1;
			}
			spin_unlock(&global_rsv->lock);
		}
		spin_unlock(&space_info->lock);
C
Chris Mason 已提交
5000
	}
5001 5002 5003

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
5004 5005 5006 5007
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
5008
			       struct btrfs_root *root)
5009
{
5010 5011
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
5012 5013
	u64 start;
	u64 end;
5014 5015
	int ret;

5016 5017 5018
	if (trans->aborted)
		return 0;

5019 5020 5021 5022 5023
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

C
Chris Mason 已提交
5024
	while (1) {
5025
		ret = find_first_extent_bit(unpin, 0, &start, &end,
5026
					    EXTENT_DIRTY, NULL);
5027
		if (ret)
5028
			break;
5029

5030 5031 5032
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
5033

5034
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
5035
		unpin_extent_range(root, start, end);
5036
		cond_resched();
5037
	}
J
Josef Bacik 已提交
5038

C
Chris Mason 已提交
5039 5040 5041
	return 0;
}

5042 5043 5044 5045 5046 5047
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extent_op)
5048
{
C
Chris Mason 已提交
5049
	struct btrfs_key key;
5050
	struct btrfs_path *path;
5051 5052
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
5053
	struct extent_buffer *leaf;
5054 5055
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
5056
	int ret;
5057
	int is_data;
5058 5059 5060
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
5061 5062
	u32 item_size;
	u64 refs;
C
Chris Mason 已提交
5063

5064
	path = btrfs_alloc_path();
5065 5066
	if (!path)
		return -ENOMEM;
5067

5068
	path->reada = 1;
5069
	path->leave_spinning = 1;
5070 5071 5072 5073 5074 5075 5076 5077

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
5078
	if (ret == 0) {
5079
		extent_slot = path->slots[0];
5080 5081
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
5082
					      extent_slot);
5083
			if (key.objectid != bytenr)
5084
				break;
5085 5086
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
5087 5088 5089 5090 5091
				found_extent = 1;
				break;
			}
			if (path->slots[0] - extent_slot > 5)
				break;
5092
			extent_slot--;
5093
		}
5094 5095 5096 5097 5098
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
5099
		if (!found_extent) {
5100
			BUG_ON(iref);
5101
			ret = remove_extent_backref(trans, extent_root, path,
5102 5103
						    NULL, refs_to_drop,
						    is_data);
5104 5105 5106 5107
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5108
			btrfs_release_path(path);
5109
			path->leave_spinning = 1;
5110 5111 5112 5113 5114

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

Z
Zheng Yan 已提交
5115 5116
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
5117 5118
			if (ret) {
				printk(KERN_ERR "umm, got %d back from search"
C
Chris Mason 已提交
5119 5120
				       ", was looking for %llu\n", ret,
				       (unsigned long long)bytenr);
5121 5122 5123
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
5124
			}
5125 5126 5127 5128
			if (ret < 0) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
Z
Zheng Yan 已提交
5129 5130
			extent_slot = path->slots[0];
		}
5131
	} else if (ret == -ENOENT) {
5132 5133
		btrfs_print_leaf(extent_root, path->nodes[0]);
		WARN_ON(1);
C
Chris Mason 已提交
5134
		printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
5135
		       "parent %llu root %llu  owner %llu offset %llu\n",
C
Chris Mason 已提交
5136
		       (unsigned long long)bytenr,
5137
		       (unsigned long long)parent,
C
Chris Mason 已提交
5138
		       (unsigned long long)root_objectid,
5139 5140
		       (unsigned long long)owner_objectid,
		       (unsigned long long)owner_offset);
5141
	} else {
5142 5143
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
5144
	}
5145 5146

	leaf = path->nodes[0];
5147 5148 5149 5150 5151 5152
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
5153 5154 5155 5156
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5157

5158
		btrfs_release_path(path);
5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
			printk(KERN_ERR "umm, got %d back from search"
			       ", was looking for %llu\n", ret,
			       (unsigned long long)bytenr);
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
5173 5174 5175 5176 5177
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}

5178 5179 5180 5181 5182 5183
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
5184
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
5185
			    struct btrfs_extent_item);
5186 5187 5188 5189 5190 5191
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
5192

5193
	refs = btrfs_extent_refs(leaf, ei);
5194 5195
	BUG_ON(refs < refs_to_drop);
	refs -= refs_to_drop;
5196

5197 5198 5199 5200 5201 5202
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
5203
		 */
5204 5205 5206 5207 5208 5209 5210 5211 5212 5213
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
						    is_data);
5214 5215 5216 5217
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5218
		}
5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
5230
		}
5231

5232 5233
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
5234 5235 5236 5237
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5238
		btrfs_release_path(path);
5239

5240
		if (is_data) {
5241
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
5242 5243 5244 5245
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5246 5247
		}

5248
		ret = update_block_group(trans, root, bytenr, num_bytes, 0);
5249 5250 5251 5252
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5253
	}
5254
out:
5255
	btrfs_free_path(path);
5256 5257 5258
	return ret;
}

5259
/*
5260
 * when we free an block, it is possible (and likely) that we free the last
5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct rb_node *node;
5272
	int ret = 0;
5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	/* there are still entries for this ref, we can't drop it */
	if (ref->bytenr == bytenr)
		goto out;

5290 5291 5292 5293 5294 5295 5296
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
		kfree(head->extent_op);
		head->extent_op = NULL;
	}

5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
	rb_erase(&head->node.rb_node, &delayed_refs->root);
5310

5311 5312 5313 5314 5315 5316
	delayed_refs->num_entries--;

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
5317 5318 5319 5320 5321
	delayed_refs->num_heads--;
	if (list_empty(&head->cluster))
		delayed_refs->num_heads_ready--;

	list_del_init(&head->cluster);
5322 5323
	spin_unlock(&delayed_refs->lock);

5324 5325 5326 5327 5328
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
5329
	btrfs_put_delayed_ref(&head->node);
5330
	return ret;
5331 5332 5333 5334 5335
out:
	spin_unlock(&delayed_refs->lock);
	return 0;
}

5336 5337 5338
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
5339
			   u64 parent, int last_ref)
5340 5341 5342 5343 5344
{
	struct btrfs_block_group_cache *cache = NULL;
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
A
Arne Jansen 已提交
5345 5346 5347 5348
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					buf->start, buf->len,
					parent, root->root_key.objectid,
					btrfs_header_level(buf),
5349
					BTRFS_DROP_DELAYED_REF, NULL, 0);
5350
		BUG_ON(ret); /* -ENOMEM */
5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361
	}

	if (!last_ref)
		return;

	cache = btrfs_lookup_block_group(root->fs_info, buf->start);

	if (btrfs_header_generation(buf) == trans->transid) {
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
5362
				goto out;
5363 5364 5365 5366
		}

		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
5367
			goto out;
5368 5369 5370 5371 5372
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
5373
		btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
5374 5375
	}
out:
5376 5377 5378 5379 5380
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
5381 5382 5383
	btrfs_put_block_group(cache);
}

5384
/* Can return -ENOMEM */
A
Arne Jansen 已提交
5385 5386 5387
int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
		      u64 owner, u64 offset, int for_cow)
5388 5389
{
	int ret;
A
Arne Jansen 已提交
5390
	struct btrfs_fs_info *fs_info = root->fs_info;
5391

5392 5393 5394 5395
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
5396 5397
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
5398
		/* unlocks the pinned mutex */
5399
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
5400
		ret = 0;
5401
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
5402 5403
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
5404
					parent, root_objectid, (int)owner,
A
Arne Jansen 已提交
5405
					BTRFS_DROP_DELAYED_REF, NULL, for_cow);
5406
	} else {
A
Arne Jansen 已提交
5407 5408 5409 5410 5411
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
						num_bytes,
						parent, root_objectid, owner,
						offset, BTRFS_DROP_DELAYED_REF,
						NULL, for_cow);
5412
	}
5413 5414 5415
	return ret;
}

5416 5417 5418 5419 5420 5421 5422
static u64 stripe_align(struct btrfs_root *root, u64 val)
{
	u64 mask = ((u64)root->stripesize - 1);
	u64 ret = (val + mask) & ~mask;
	return ret;
}

J
Josef Bacik 已提交
5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
 */
static noinline int
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
5438
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
5439 5440
	DEFINE_WAIT(wait);

5441 5442
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
J
Josef Bacik 已提交
5443 5444
		return 0;

5445
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
5446
		   (cache->free_space_ctl->free_space >= num_bytes));
5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464

	put_caching_control(caching_ctl);
	return 0;
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;
	DEFINE_WAIT(wait);

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
		return 0;

	wait_event(caching_ctl->wait, block_group_cache_done(cache));

	put_caching_control(caching_ctl);
J
Josef Bacik 已提交
5465 5466 5467
	return 0;
}

5468
static int __get_block_group_index(u64 flags)
5469 5470
{
	int index;
5471 5472

	if (flags & BTRFS_BLOCK_GROUP_RAID10)
5473
		index = 0;
5474
	else if (flags & BTRFS_BLOCK_GROUP_RAID1)
5475
		index = 1;
5476
	else if (flags & BTRFS_BLOCK_GROUP_DUP)
5477
		index = 2;
5478
	else if (flags & BTRFS_BLOCK_GROUP_RAID0)
5479 5480 5481
		index = 3;
	else
		index = 4;
5482

5483 5484 5485
	return index;
}

5486 5487 5488 5489 5490
static int get_block_group_index(struct btrfs_block_group_cache *cache)
{
	return __get_block_group_index(cache->flags);
}

J
Josef Bacik 已提交
5491
enum btrfs_loop_type {
5492 5493 5494 5495
	LOOP_CACHING_NOWAIT = 0,
	LOOP_CACHING_WAIT = 1,
	LOOP_ALLOC_CHUNK = 2,
	LOOP_NO_EMPTY_SIZE = 3,
J
Josef Bacik 已提交
5496 5497
};

5498 5499 5500 5501
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
 * ins->objectid == block start
5502
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
5503 5504 5505
 * ins->offset == number of blocks
 * Any available blocks before search_start are skipped.
 */
C
Chris Mason 已提交
5506
static noinline int find_free_extent(struct btrfs_trans_handle *trans,
5507 5508 5509
				     struct btrfs_root *orig_root,
				     u64 num_bytes, u64 empty_size,
				     u64 hint_byte, struct btrfs_key *ins,
5510
				     u64 data)
5511
{
5512
	int ret = 0;
C
Chris Mason 已提交
5513
	struct btrfs_root *root = orig_root->fs_info->extent_root;
5514
	struct btrfs_free_cluster *last_ptr = NULL;
5515
	struct btrfs_block_group_cache *block_group = NULL;
5516
	struct btrfs_block_group_cache *used_block_group;
5517
	u64 search_start = 0;
5518
	int empty_cluster = 2 * 1024 * 1024;
5519
	struct btrfs_space_info *space_info;
5520
	int loop = 0;
5521
	int index = 0;
5522 5523
	int alloc_type = (data & BTRFS_BLOCK_GROUP_DATA) ?
		RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
J
Josef Bacik 已提交
5524
	bool found_uncached_bg = false;
5525
	bool failed_cluster_refill = false;
5526
	bool failed_alloc = false;
5527
	bool use_cluster = true;
5528
	bool have_caching_bg = false;
5529

5530
	WARN_ON(num_bytes < root->sectorsize);
5531
	btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
5532 5533
	ins->objectid = 0;
	ins->offset = 0;
5534

J
Josef Bacik 已提交
5535 5536
	trace_find_free_extent(orig_root, num_bytes, empty_size, data);

J
Josef Bacik 已提交
5537
	space_info = __find_space_info(root->fs_info, data);
5538
	if (!space_info) {
5539
		printk(KERN_ERR "No space info for %llu\n", data);
5540 5541
		return -ENOSPC;
	}
J
Josef Bacik 已提交
5542

5543 5544 5545 5546 5547 5548 5549 5550
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

	if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
5551
		last_ptr = &root->fs_info->meta_alloc_cluster;
5552 5553
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
5554 5555
	}

5556 5557
	if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
	    btrfs_test_opt(root, SSD)) {
5558 5559
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
J
Josef Bacik 已提交
5560

5561
	if (last_ptr) {
5562 5563 5564 5565
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
5566
	}
5567

5568
	search_start = max(search_start, first_logical_byte(root, 0));
5569
	search_start = max(search_start, hint_byte);
5570

J
Josef Bacik 已提交
5571
	if (!last_ptr)
5572 5573
		empty_cluster = 0;

J
Josef Bacik 已提交
5574 5575 5576
	if (search_start == hint_byte) {
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
5577
		used_block_group = block_group;
J
Josef Bacik 已提交
5578 5579 5580
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
5581 5582 5583
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
5584 5585
		 */
		if (block_group && block_group_bits(block_group, data) &&
5586
		    block_group->cached != BTRFS_CACHE_NO) {
J
Josef Bacik 已提交
5587
			down_read(&space_info->groups_sem);
5588 5589 5590 5591 5592 5593 5594 5595 5596 5597
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
5598
			} else {
5599
				index = get_block_group_index(block_group);
5600
				goto have_block_group;
5601
			}
J
Josef Bacik 已提交
5602
		} else if (block_group) {
5603
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5604
		}
5605
	}
J
Josef Bacik 已提交
5606
search:
5607
	have_caching_bg = false;
5608
	down_read(&space_info->groups_sem);
5609 5610
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
5611
		u64 offset;
J
Josef Bacik 已提交
5612
		int cached;
5613

5614
		used_block_group = block_group;
5615
		btrfs_get_block_group(block_group);
J
Josef Bacik 已提交
5616
		search_start = block_group->key.objectid;
5617

5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
		if (!block_group_bits(block_group, data)) {
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
			if ((data & extra) && !(block_group->flags & extra))
				goto loop;
		}

J
Josef Bacik 已提交
5637
have_block_group:
5638 5639 5640
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached)) {
			found_uncached_bg = true;
5641
			ret = cache_block_group(block_group, trans,
5642
						orig_root, 0);
5643 5644
			BUG_ON(ret < 0);
			ret = 0;
J
Josef Bacik 已提交
5645 5646
		}

5647
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
5648
			goto loop;
J
Josef Bacik 已提交
5649

5650
		/*
5651 5652
		 * Ok we want to try and use the cluster allocator, so
		 * lets look there
5653
		 */
5654
		if (last_ptr) {
5655 5656 5657 5658 5659
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
			spin_lock(&last_ptr->refill_lock);
5660 5661 5662 5663 5664 5665
			used_block_group = last_ptr->block_group;
			if (used_block_group != block_group &&
			    (!used_block_group ||
			     used_block_group->ro ||
			     !block_group_bits(used_block_group, data))) {
				used_block_group = block_group;
5666
				goto refill_cluster;
5667 5668 5669 5670
			}

			if (used_block_group != block_group)
				btrfs_get_block_group(used_block_group);
5671

5672 5673
			offset = btrfs_alloc_from_cluster(used_block_group,
			  last_ptr, num_bytes, used_block_group->key.objectid);
5674 5675 5676
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
5677 5678
				trace_btrfs_reserve_extent_cluster(root,
					block_group, search_start, num_bytes);
5679 5680 5681
				goto checks;
			}

5682 5683 5684 5685
			WARN_ON(last_ptr->block_group != used_block_group);
			if (used_block_group != block_group) {
				btrfs_put_block_group(used_block_group);
				used_block_group = block_group;
5686
			}
5687
refill_cluster:
5688
			BUG_ON(used_block_group != block_group);
5689 5690 5691 5692 5693 5694 5695 5696
			/* If we are on LOOP_NO_EMPTY_SIZE, we can't
			 * set up a new clusters, so lets just skip it
			 * and let the allocator find whatever block
			 * it can find.  If we reach this point, we
			 * will have tried the cluster allocator
			 * plenty of times and not have found
			 * anything, so we are likely way too
			 * fragmented for the clustering stuff to find
5697 5698 5699 5700 5701 5702 5703 5704 5705
			 * anything.
			 *
			 * However, if the cluster is taken from the
			 * current block group, release the cluster
			 * first, so that we stand a better chance of
			 * succeeding in the unclustered
			 * allocation.  */
			if (loop >= LOOP_NO_EMPTY_SIZE &&
			    last_ptr->block_group != block_group) {
5706 5707 5708 5709
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

5710 5711 5712 5713 5714 5715
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

5716 5717 5718 5719 5720
			if (loop >= LOOP_NO_EMPTY_SIZE) {
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

5721
			/* allocate a cluster in this block group */
5722
			ret = btrfs_find_space_cluster(trans, root,
5723
					       block_group, last_ptr,
5724
					       search_start, num_bytes,
5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736
					       empty_cluster + empty_size);
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
						  last_ptr, num_bytes,
						  search_start);
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
5737 5738 5739
					trace_btrfs_reserve_extent_cluster(root,
						block_group, search_start,
						num_bytes);
5740 5741
					goto checks;
				}
5742 5743
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
5744 5745
				spin_unlock(&last_ptr->refill_lock);

5746
				failed_cluster_refill = true;
J
Josef Bacik 已提交
5747 5748 5749
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
5750
			}
J
Josef Bacik 已提交
5751

5752 5753 5754 5755 5756 5757
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
5758
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
5759
			spin_unlock(&last_ptr->refill_lock);
5760
			goto loop;
5761 5762
		}

5763
unclustered_alloc:
5764 5765 5766 5767 5768 5769 5770 5771 5772
		spin_lock(&block_group->free_space_ctl->tree_lock);
		if (cached &&
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_cluster + empty_size) {
			spin_unlock(&block_group->free_space_ctl->tree_lock);
			goto loop;
		}
		spin_unlock(&block_group->free_space_ctl->tree_lock);

5773 5774
		offset = btrfs_find_space_for_alloc(block_group, search_start,
						    num_bytes, empty_size);
5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
5786
			wait_block_group_cache_progress(block_group,
5787 5788
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
5789
			goto have_block_group;
5790
		} else if (!offset) {
5791 5792
			if (!cached)
				have_caching_bg = true;
5793
			goto loop;
J
Josef Bacik 已提交
5794
		}
5795
checks:
5796
		search_start = stripe_align(root, offset);
5797

J
Josef Bacik 已提交
5798 5799
		/* move on to the next group */
		if (search_start + num_bytes >
5800 5801
		    used_block_group->key.objectid + used_block_group->key.offset) {
			btrfs_add_free_space(used_block_group, offset, num_bytes);
J
Josef Bacik 已提交
5802
			goto loop;
5803
		}
5804

5805
		if (offset < search_start)
5806
			btrfs_add_free_space(used_block_group, offset,
5807 5808
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
5809

5810
		ret = btrfs_update_reserved_bytes(used_block_group, num_bytes,
5811
						  alloc_type);
5812
		if (ret == -EAGAIN) {
5813
			btrfs_add_free_space(used_block_group, offset, num_bytes);
J
Josef Bacik 已提交
5814
			goto loop;
J
Josef Bacik 已提交
5815
		}
5816

5817
		/* we are all good, lets return */
J
Josef Bacik 已提交
5818 5819
		ins->objectid = search_start;
		ins->offset = num_bytes;
5820

J
Josef Bacik 已提交
5821 5822
		trace_btrfs_reserve_extent(orig_root, block_group,
					   search_start, num_bytes);
5823 5824
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
5825
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5826 5827
		break;
loop:
5828
		failed_cluster_refill = false;
5829
		failed_alloc = false;
5830
		BUG_ON(index != get_block_group_index(block_group));
5831 5832
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
5833
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5834 5835 5836
	}
	up_read(&space_info->groups_sem);

5837 5838 5839
	if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
		goto search;

5840 5841 5842
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

5843
	/*
5844 5845
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
5846 5847 5848 5849
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
5850
	 */
5851
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
5852
		index = 0;
5853
		loop++;
J
Josef Bacik 已提交
5854
		if (loop == LOOP_ALLOC_CHUNK) {
5855
			ret = do_chunk_alloc(trans, root, data,
5856 5857 5858 5859 5860 5861 5862 5863 5864
					     CHUNK_ALLOC_FORCE);
			/*
			 * Do not bail out on ENOSPC since we
			 * can do more things.
			 */
			if (ret < 0 && ret != -ENOSPC) {
				btrfs_abort_transaction(trans,
							root, ret);
				goto out;
5865
			}
J
Josef Bacik 已提交
5866 5867
		}

5868 5869 5870
		if (loop == LOOP_NO_EMPTY_SIZE) {
			empty_size = 0;
			empty_cluster = 0;
5871
		}
5872 5873

		goto search;
J
Josef Bacik 已提交
5874 5875
	} else if (!ins->objectid) {
		ret = -ENOSPC;
5876
	} else if (ins->objectid) {
5877
		ret = 0;
C
Chris Mason 已提交
5878
	}
5879
out:
C
Chris Mason 已提交
5880

C
Chris Mason 已提交
5881
	return ret;
5882
}
5883

J
Josef Bacik 已提交
5884 5885
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
5886 5887
{
	struct btrfs_block_group_cache *cache;
5888
	int index = 0;
J
Josef Bacik 已提交
5889

J
Josef Bacik 已提交
5890
	spin_lock(&info->lock);
5891 5892
	printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
	       (unsigned long long)info->flags,
C
Chris Mason 已提交
5893
	       (unsigned long long)(info->total_bytes - info->bytes_used -
J
Josef Bacik 已提交
5894
				    info->bytes_pinned - info->bytes_reserved -
5895
				    info->bytes_readonly),
C
Chris Mason 已提交
5896
	       (info->full) ? "" : "not ");
5897 5898
	printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
5899
	       (unsigned long long)info->total_bytes,
5900
	       (unsigned long long)info->bytes_used,
5901
	       (unsigned long long)info->bytes_pinned,
5902
	       (unsigned long long)info->bytes_reserved,
5903
	       (unsigned long long)info->bytes_may_use,
5904
	       (unsigned long long)info->bytes_readonly);
J
Josef Bacik 已提交
5905 5906 5907 5908
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
5909

5910
	down_read(&info->groups_sem);
5911 5912
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
5913
		spin_lock(&cache->lock);
5914
		printk(KERN_INFO "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s\n",
C
Chris Mason 已提交
5915 5916 5917 5918
		       (unsigned long long)cache->key.objectid,
		       (unsigned long long)cache->key.offset,
		       (unsigned long long)btrfs_block_group_used(&cache->item),
		       (unsigned long long)cache->pinned,
5919 5920
		       (unsigned long long)cache->reserved,
		       cache->ro ? "[readonly]" : "");
J
Josef Bacik 已提交
5921 5922 5923
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
5924 5925
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
5926
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
5927
}
5928

5929 5930 5931 5932
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
5933
			 struct btrfs_key *ins, u64 data)
5934
{
5935
	bool final_tried = false;
5936
	int ret;
5937

J
Josef Bacik 已提交
5938
	data = btrfs_get_alloc_profile(root, data);
5939
again:
5940 5941
	WARN_ON(num_bytes < root->sectorsize);
	ret = find_free_extent(trans, root, num_bytes, empty_size,
5942
			       hint_byte, ins, data);
5943

5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958
	if (ret == -ENOSPC) {
		if (!final_tried) {
			num_bytes = num_bytes >> 1;
			num_bytes = num_bytes & ~(root->sectorsize - 1);
			num_bytes = max(num_bytes, min_alloc_size);
			if (num_bytes == min_alloc_size)
				final_tried = true;
			goto again;
		} else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
			struct btrfs_space_info *sinfo;

			sinfo = __find_space_info(root->fs_info, data);
			printk(KERN_ERR "btrfs allocation failed flags %llu, "
			       "wanted %llu\n", (unsigned long long)data,
			       (unsigned long long)num_bytes);
5959 5960
			if (sinfo)
				dump_space_info(sinfo, num_bytes, 1);
5961
		}
5962
	}
J
Josef Bacik 已提交
5963

5964 5965
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);

J
Josef Bacik 已提交
5966
	return ret;
5967 5968
}

5969 5970
static int __btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len, int pin)
5971
{
J
Josef Bacik 已提交
5972
	struct btrfs_block_group_cache *cache;
5973
	int ret = 0;
J
Josef Bacik 已提交
5974 5975 5976

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
C
Chris Mason 已提交
5977 5978
		printk(KERN_ERR "Unable to find block group for %llu\n",
		       (unsigned long long)start);
J
Josef Bacik 已提交
5979 5980
		return -ENOSPC;
	}
5981

5982 5983
	if (btrfs_test_opt(root, DISCARD))
		ret = btrfs_discard_extent(root, start, len, NULL);
5984

5985 5986 5987 5988 5989 5990
	if (pin)
		pin_down_extent(root, cache, start, len, 1);
	else {
		btrfs_add_free_space(cache, start, len);
		btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
	}
5991
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
5992

5993 5994
	trace_btrfs_reserved_extent_free(root, start, len);

5995 5996 5997
	return ret;
}

5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009
int btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 0);
}

int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
				       u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 1);
}

6010 6011 6012 6013 6014
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
6015 6016
{
	int ret;
6017
	struct btrfs_fs_info *fs_info = root->fs_info;
6018
	struct btrfs_extent_item *extent_item;
6019
	struct btrfs_extent_inline_ref *iref;
6020
	struct btrfs_path *path;
6021 6022 6023
	struct extent_buffer *leaf;
	int type;
	u32 size;
6024

6025 6026 6027 6028
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
6029

6030
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
6031 6032

	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
6033 6034
	if (!path)
		return -ENOMEM;
6035

6036
	path->leave_spinning = 1;
6037 6038
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6039 6040 6041 6042
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
J
Josef Bacik 已提交
6043

6044 6045
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
6046
				     struct btrfs_extent_item);
6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
6067 6068

	btrfs_mark_buffer_dirty(path->nodes[0]);
6069
	btrfs_free_path(path);
6070

6071
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
6072
	if (ret) { /* -ENOENT, logic error */
C
Chris Mason 已提交
6073 6074 6075
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
6076 6077
		BUG();
	}
6078 6079 6080
	return ret;
}

6081 6082 6083 6084 6085
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins)
6086 6087
{
	int ret;
6088 6089 6090 6091 6092 6093 6094
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
6095

6096
	path = btrfs_alloc_path();
6097 6098
	if (!path)
		return -ENOMEM;
6099

6100 6101 6102
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6103 6104 6105 6106
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
	block_info = (struct btrfs_tree_block_info *)(extent_item + 1);

	btrfs_set_tree_block_key(leaf, block_info, key);
	btrfs_set_tree_block_level(leaf, block_info, level);

	iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

6135
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
6136
	if (ret) { /* -ENOENT, logic error */
6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
		BUG();
	}
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

A
Arne Jansen 已提交
6154 6155 6156 6157
	ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
					 ins->offset, 0,
					 root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL, 0);
6158 6159
	return ret;
}
6160 6161 6162 6163 6164 6165

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
6166 6167 6168 6169
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
6170 6171 6172
{
	int ret;
	struct btrfs_block_group_cache *block_group;
6173 6174 6175
	struct btrfs_caching_control *caching_ctl;
	u64 start = ins->objectid;
	u64 num_bytes = ins->offset;
6176 6177

	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
6178
	cache_block_group(block_group, trans, NULL, 0);
6179
	caching_ctl = get_caching_control(block_group);
6180

6181 6182 6183
	if (!caching_ctl) {
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
6184
		BUG_ON(ret); /* -ENOMEM */
6185 6186 6187 6188 6189
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
6190
			BUG_ON(ret); /* -ENOMEM */
6191 6192 6193
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
6194
			BUG_ON(ret); /* -ENOMEM */
6195 6196 6197 6198
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
6199
			BUG_ON(ret); /* -ENOMEM */
6200 6201 6202 6203 6204

			start = caching_ctl->progress;
			num_bytes = ins->objectid + ins->offset -
				    caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
6205
			BUG_ON(ret); /* -ENOMEM */
6206 6207 6208 6209 6210 6211
		}

		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}

6212 6213
	ret = btrfs_update_reserved_bytes(block_group, ins->offset,
					  RESERVE_ALLOC_NO_ACCOUNT);
6214
	BUG_ON(ret); /* logic error */
6215
	btrfs_put_block_group(block_group);
6216 6217
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
6218 6219 6220
	return ret;
}

6221 6222
struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
					    struct btrfs_root *root,
6223 6224
					    u64 bytenr, u32 blocksize,
					    int level)
6225 6226 6227 6228 6229 6230 6231
{
	struct extent_buffer *buf;

	buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
6232
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
6233 6234
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
6235
	clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
6236 6237

	btrfs_set_lock_blocking(buf);
6238
	btrfs_set_buffer_uptodate(buf);
6239

6240
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
6241 6242 6243 6244 6245 6246 6247 6248 6249 6250
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
		if (root->log_transid % 2 == 0)
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
6251 6252
	} else {
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
6253
			 buf->start + buf->len - 1, GFP_NOFS);
6254
	}
6255
	trans->blocks_used++;
6256
	/* this returns a buffer locked for blocking */
6257 6258 6259
	return buf;
}

6260 6261 6262 6263 6264
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
6265
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
6266 6267 6268 6269 6270
	int ret;

	block_rsv = get_block_rsv(trans, root);

	if (block_rsv->size == 0) {
M
Miao Xie 已提交
6271 6272
		ret = reserve_metadata_bytes(root, block_rsv, blocksize,
					     BTRFS_RESERVE_NO_FLUSH);
6273 6274 6275 6276 6277 6278 6279 6280
		/*
		 * If we couldn't reserve metadata bytes try and use some from
		 * the global reserve.
		 */
		if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
6281
			return ERR_PTR(ret);
6282
		} else if (ret) {
6283
			return ERR_PTR(ret);
6284
		}
6285 6286 6287 6288 6289 6290
		return block_rsv;
	}

	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;
6291
	if (ret && !block_rsv->failfast) {
6292 6293 6294 6295 6296 6297 6298
		static DEFINE_RATELIMIT_STATE(_rs,
				DEFAULT_RATELIMIT_INTERVAL,
				/*DEFAULT_RATELIMIT_BURST*/ 2);
		if (__ratelimit(&_rs)) {
			printk(KERN_DEBUG "btrfs: block rsv returned %d\n", ret);
			WARN_ON(1);
		}
M
Miao Xie 已提交
6299 6300
		ret = reserve_metadata_bytes(root, block_rsv, blocksize,
					     BTRFS_RESERVE_NO_FLUSH);
6301 6302 6303 6304 6305 6306 6307 6308
		if (!ret) {
			return block_rsv;
		} else if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
		}
	}
6309 6310 6311 6312

	return ERR_PTR(-ENOSPC);
}

J
Josef Bacik 已提交
6313 6314
static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
			    struct btrfs_block_rsv *block_rsv, u32 blocksize)
6315 6316
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
J
Josef Bacik 已提交
6317
	block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
6318 6319
}

6320
/*
6321 6322 6323 6324
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
6325 6326
 * returns the tree buffer or NULL.
 */
6327
struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
6328 6329 6330
					struct btrfs_root *root, u32 blocksize,
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
6331
					u64 hint, u64 empty_size)
6332
{
C
Chris Mason 已提交
6333
	struct btrfs_key ins;
6334
	struct btrfs_block_rsv *block_rsv;
6335
	struct extent_buffer *buf;
6336 6337 6338
	u64 flags = 0;
	int ret;

6339

6340 6341 6342 6343 6344
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

	ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
6345
				   empty_size, hint, &ins, 0);
6346
	if (ret) {
J
Josef Bacik 已提交
6347
		unuse_block_rsv(root->fs_info, block_rsv, blocksize);
6348
		return ERR_PTR(ret);
6349
	}
6350

6351 6352
	buf = btrfs_init_new_buffer(trans, root, ins.objectid,
				    blocksize, level);
6353
	BUG_ON(IS_ERR(buf)); /* -ENOMEM */
6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
		extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
6365
		BUG_ON(!extent_op); /* -ENOMEM */
6366 6367 6368 6369 6370 6371 6372 6373 6374
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
		extent_op->update_key = 1;
		extent_op->update_flags = 1;
		extent_op->is_data = 0;

A
Arne Jansen 已提交
6375 6376
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					ins.objectid,
6377 6378
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
6379
					extent_op, 0);
6380
		BUG_ON(ret); /* -ENOMEM */
6381
	}
6382 6383
	return buf;
}
6384

6385 6386 6387 6388 6389 6390 6391 6392 6393
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
6394 6395
	int reada_slot;
	int reada_count;
A
Arne Jansen 已提交
6396
	int for_reloc;
6397 6398 6399 6400 6401
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
6402 6403 6404 6405
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
6406
{
Y
Yan, Zheng 已提交
6407 6408 6409
	u64 bytenr;
	u64 generation;
	u64 refs;
6410
	u64 flags;
6411
	u32 nritems;
Y
Yan, Zheng 已提交
6412 6413 6414
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
6415
	int ret;
Y
Yan, Zheng 已提交
6416 6417
	int slot;
	int nread = 0;
6418

Y
Yan, Zheng 已提交
6419 6420 6421 6422 6423 6424 6425 6426
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
6427

Y
Yan, Zheng 已提交
6428 6429 6430
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
	blocksize = btrfs_level_size(root, wc->level - 1);
6431

Y
Yan, Zheng 已提交
6432 6433 6434
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
6435

C
Chris Mason 已提交
6436
		cond_resched();
Y
Yan, Zheng 已提交
6437 6438
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
6439

Y
Yan, Zheng 已提交
6440 6441
		if (slot == path->slots[wc->level])
			goto reada;
6442

Y
Yan, Zheng 已提交
6443 6444
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
6445 6446
			continue;

6447 6448 6449
		/* We don't lock the tree block, it's OK to be racy here */
		ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
					       &refs, &flags);
6450 6451 6452
		/* We don't care about errors in readahead. */
		if (ret < 0)
			continue;
6453 6454
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
6455 6456 6457
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
6458

6459 6460 6461
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
6462 6463 6464 6465 6466 6467 6468 6469
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
6470 6471 6472 6473
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
6474
		}
Y
Yan, Zheng 已提交
6475 6476 6477 6478
reada:
		ret = readahead_tree_block(root, bytenr, blocksize,
					   generation);
		if (ret)
6479
			break;
Y
Yan, Zheng 已提交
6480
		nread++;
C
Chris Mason 已提交
6481
	}
Y
Yan, Zheng 已提交
6482
	wc->reada_slot = slot;
C
Chris Mason 已提交
6483
}
6484

Y
Yan Zheng 已提交
6485
/*
6486 6487 6488 6489 6490 6491
 * hepler to process tree block while walking down the tree.
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
6492
 */
6493
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
6494
				   struct btrfs_root *root,
6495
				   struct btrfs_path *path,
6496
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
6497
{
6498 6499 6500
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
6501 6502
	int ret;

6503 6504 6505
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
6506

6507 6508 6509 6510
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
6511 6512 6513
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
6514 6515 6516 6517 6518
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
					       eb->start, eb->len,
					       &wc->refs[level],
					       &wc->flags[level]);
6519 6520 6521
		BUG_ON(ret == -ENOMEM);
		if (ret)
			return ret;
6522 6523
		BUG_ON(wc->refs[level] == 0);
	}
6524

6525 6526 6527
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
6528

6529
		if (path->locks[level] && !wc->keep_locks) {
6530
			btrfs_tree_unlock_rw(eb, path->locks[level]);
6531 6532 6533 6534
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
6535

6536 6537 6538
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
A
Arne Jansen 已提交
6539
		ret = btrfs_inc_ref(trans, root, eb, 1, wc->for_reloc);
6540
		BUG_ON(ret); /* -ENOMEM */
A
Arne Jansen 已提交
6541
		ret = btrfs_dec_ref(trans, root, eb, 0, wc->for_reloc);
6542
		BUG_ON(ret); /* -ENOMEM */
6543 6544
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
						  eb->len, flag, 0);
6545
		BUG_ON(ret); /* -ENOMEM */
6546 6547 6548 6549 6550 6551 6552 6553
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
6554
		btrfs_tree_unlock_rw(eb, path->locks[level]);
6555 6556 6557 6558 6559
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575
/*
 * hepler to process tree block pointer.
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
6576
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
6596 6597
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6598
		return 1;
6599
	}
Y
Yan, Zheng 已提交
6600 6601 6602 6603 6604 6605 6606

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
	blocksize = btrfs_level_size(root, level - 1);

	next = btrfs_find_tree_block(root, bytenr, blocksize);
	if (!next) {
		next = btrfs_find_create_tree_block(root, bytenr, blocksize);
6607 6608
		if (!next)
			return -ENOMEM;
Y
Yan, Zheng 已提交
6609 6610 6611 6612 6613
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

6614 6615 6616
	ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
6617 6618 6619 6620 6621
	if (ret < 0) {
		btrfs_tree_unlock(next);
		return ret;
	}

6622 6623
	BUG_ON(wc->refs[level - 1] == 0);
	*lookup_info = 0;
Y
Yan, Zheng 已提交
6624

6625
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
6626
		if (wc->refs[level - 1] > 1) {
6627 6628 6629 6630
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
6644 6645 6646 6647
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
6648 6649
	}

6650
	if (!btrfs_buffer_uptodate(next, generation, 0)) {
Y
Yan, Zheng 已提交
6651 6652 6653
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
6654
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6655 6656 6657 6658 6659 6660
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
		next = read_tree_block(root, bytenr, blocksize, generation);
6661 6662
		if (!next)
			return -EIO;
Y
Yan, Zheng 已提交
6663 6664 6665 6666 6667 6668 6669 6670
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
6671
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Y
Yan, Zheng 已提交
6672 6673 6674 6675 6676 6677 6678
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
6679 6680 6681 6682 6683 6684 6685 6686
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
6687

6688
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
A
Arne Jansen 已提交
6689
				root->root_key.objectid, level - 1, 0, 0);
6690
		BUG_ON(ret); /* -ENOMEM */
Y
Yan, Zheng 已提交
6691 6692 6693
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
6694
	*lookup_info = 1;
Y
Yan, Zheng 已提交
6695 6696 6697
	return 1;
}

6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714
/*
 * hepler to process tree block while walking up the tree.
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
6715
	int ret;
6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6742
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6743 6744 6745 6746 6747

			ret = btrfs_lookup_extent_info(trans, root,
						       eb->start, eb->len,
						       &wc->refs[level],
						       &wc->flags[level]);
6748 6749 6750 6751
			if (ret < 0) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
				return ret;
			}
6752 6753
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
6754
				btrfs_tree_unlock_rw(eb, path->locks[level]);
6755 6756
				return 1;
			}
Y
Yan Zheng 已提交
6757
		}
6758
	}
Y
Yan Zheng 已提交
6759

6760 6761
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6762

6763 6764 6765
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
A
Arne Jansen 已提交
6766 6767
				ret = btrfs_dec_ref(trans, root, eb, 1,
						    wc->for_reloc);
6768
			else
A
Arne Jansen 已提交
6769 6770
				ret = btrfs_dec_ref(trans, root, eb, 0,
						    wc->for_reloc);
6771
			BUG_ON(ret); /* -ENOMEM */
6772 6773 6774 6775 6776 6777
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6778
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
6795 6796
	}

6797
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
6798 6799 6800
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
6801
	return 0;
6802 6803 6804 6805 6806 6807 6808 6809
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
6810
	int lookup_info = 1;
6811 6812 6813
	int ret;

	while (level >= 0) {
6814
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
6815 6816 6817 6818 6819 6820
		if (ret > 0)
			break;

		if (level == 0)
			break;

6821 6822 6823 6824
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

6825
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
6826 6827 6828
		if (ret > 0) {
			path->slots[level]++;
			continue;
6829 6830
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
6831
		level = wc->level;
Y
Yan Zheng 已提交
6832 6833 6834 6835
	}
	return 0;
}

C
Chris Mason 已提交
6836
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6837
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
6838
				 struct btrfs_path *path,
6839
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
6840
{
6841
	int level = wc->level;
C
Chris Mason 已提交
6842
	int ret;
6843

6844 6845 6846 6847 6848 6849
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
6850 6851
			return 0;
		} else {
6852 6853 6854
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
6855

6856
			if (path->locks[level]) {
6857 6858
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
6859
				path->locks[level] = 0;
Y
Yan Zheng 已提交
6860
			}
6861 6862 6863
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
6864 6865 6866 6867 6868
		}
	}
	return 1;
}

C
Chris Mason 已提交
6869
/*
6870 6871 6872 6873 6874 6875 6876 6877 6878
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
C
Chris Mason 已提交
6879
 */
6880
int btrfs_drop_snapshot(struct btrfs_root *root,
A
Arne Jansen 已提交
6881 6882
			 struct btrfs_block_rsv *block_rsv, int update_ref,
			 int for_reloc)
C
Chris Mason 已提交
6883
{
6884
	struct btrfs_path *path;
6885 6886
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
6887
	struct btrfs_root_item *root_item = &root->root_item;
6888 6889 6890 6891 6892
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
C
Chris Mason 已提交
6893

6894
	path = btrfs_alloc_path();
6895 6896 6897 6898
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
C
Chris Mason 已提交
6899

6900
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
6901 6902
	if (!wc) {
		btrfs_free_path(path);
6903 6904
		err = -ENOMEM;
		goto out;
6905
	}
6906

6907
	trans = btrfs_start_transaction(tree_root, 0);
6908 6909 6910 6911
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
		goto out_free;
	}
6912

6913 6914
	if (block_rsv)
		trans->block_rsv = block_rsv;
6915

6916
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
6917
		level = btrfs_header_level(root->node);
6918 6919
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
6920
		path->slots[level] = 0;
6921
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6922 6923
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
6924 6925
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
6926 6927 6928
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

6929
		level = root_item->drop_level;
6930
		BUG_ON(level == 0);
6931
		path->lowest_level = level;
6932 6933 6934 6935
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
6936
			goto out_end_trans;
6937
		}
Y
Yan, Zheng 已提交
6938
		WARN_ON(ret > 0);
6939

6940 6941 6942 6943
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
6944
		btrfs_unlock_up_safe(path, 0);
6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
						path->nodes[level]->len,
						&wc->refs[level],
						&wc->flags[level]);
6956 6957 6958 6959
			if (ret < 0) {
				err = ret;
				goto out_end_trans;
			}
6960 6961 6962 6963 6964 6965 6966 6967 6968
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
6969
	}
6970 6971 6972 6973 6974 6975

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
A
Arne Jansen 已提交
6976
	wc->for_reloc = for_reloc;
Y
Yan, Zheng 已提交
6977
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
6978

C
Chris Mason 已提交
6979
	while (1) {
6980 6981 6982
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6983
			break;
6984
		}
C
Chris Mason 已提交
6985

6986 6987 6988
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6989
			break;
6990 6991 6992 6993
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
6994 6995
			break;
		}
6996 6997 6998 6999 7000 7001 7002 7003 7004 7005

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
7006
		if (btrfs_should_end_transaction(trans, tree_root)) {
7007 7008 7009
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
7010 7011 7012 7013 7014
			if (ret) {
				btrfs_abort_transaction(trans, tree_root, ret);
				err = ret;
				goto out_end_trans;
			}
7015

7016
			btrfs_end_transaction_throttle(trans, tree_root);
7017
			trans = btrfs_start_transaction(tree_root, 0);
7018 7019 7020 7021
			if (IS_ERR(trans)) {
				err = PTR_ERR(trans);
				goto out_free;
			}
7022 7023
			if (block_rsv)
				trans->block_rsv = block_rsv;
7024
		}
C
Chris Mason 已提交
7025
	}
7026
	btrfs_release_path(path);
7027 7028
	if (err)
		goto out_end_trans;
7029 7030

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
7031 7032 7033 7034
	if (ret) {
		btrfs_abort_transaction(trans, tree_root, ret);
		goto out_end_trans;
	}
7035

7036 7037 7038
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
		ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
					   NULL, NULL);
7039 7040 7041 7042 7043
		if (ret < 0) {
			btrfs_abort_transaction(trans, tree_root, ret);
			err = ret;
			goto out_end_trans;
		} else if (ret > 0) {
7044 7045 7046 7047 7048 7049 7050
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
7051 7052 7053 7054 7055 7056 7057 7058 7059 7060
		}
	}

	if (root->in_radix) {
		btrfs_free_fs_root(tree_root->fs_info, root);
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
		kfree(root);
	}
7061
out_end_trans:
7062
	btrfs_end_transaction_throttle(trans, tree_root);
7063
out_free:
7064
	kfree(wc);
7065
	btrfs_free_path(path);
7066 7067 7068
out:
	if (err)
		btrfs_std_error(root->fs_info, err);
7069
	return err;
C
Chris Mason 已提交
7070
}
C
Chris Mason 已提交
7071

7072 7073 7074 7075
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
A
Arne Jansen 已提交
7076
 * only used by relocation code
7077
 */
Y
Yan Zheng 已提交
7078 7079 7080 7081 7082 7083
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
7084
	struct walk_control *wc;
Y
Yan Zheng 已提交
7085 7086 7087 7088 7089
	int level;
	int parent_level;
	int ret = 0;
	int wret;

7090 7091
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
7092
	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
7093 7094
	if (!path)
		return -ENOMEM;
Y
Yan Zheng 已提交
7095

7096
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
T
Tsutomu Itoh 已提交
7097 7098 7099 7100
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
7101

7102
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
7103 7104 7105 7106 7107
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

7108
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
7109 7110 7111
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
7112
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7113 7114 7115 7116 7117 7118 7119 7120

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
A
Arne Jansen 已提交
7121
	wc->for_reloc = 1;
Y
Yan, Zheng 已提交
7122
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
7123 7124

	while (1) {
7125 7126
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
7127 7128
			ret = wret;
			break;
7129
		}
Y
Yan Zheng 已提交
7130

7131
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
7132 7133 7134 7135 7136 7137
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

7138
	kfree(wc);
Y
Yan Zheng 已提交
7139 7140 7141 7142
	btrfs_free_path(path);
	return ret;
}

7143 7144 7145
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
7146
	u64 stripped;
7147

7148 7149 7150 7151 7152 7153 7154
	/*
	 * if restripe for this chunk_type is on pick target profile and
	 * return, otherwise do the usual balance
	 */
	stripped = get_restripe_target(root->fs_info, flags);
	if (stripped)
		return extended_to_chunk(stripped);
7155

7156 7157 7158 7159 7160 7161 7162 7163
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;

7164 7165 7166
	stripped = BTRFS_BLOCK_GROUP_RAID0 |
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

7191
		/* this is drive concat, leave it alone */
7192
	}
7193

7194 7195 7196
	return flags;
}

7197
static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
C
Chris Mason 已提交
7198
{
7199 7200
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
7201
	u64 min_allocable_bytes;
7202
	int ret = -ENOSPC;
C
Chris Mason 已提交
7203

7204

7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
		min_allocable_bytes = 1 * 1024 * 1024;
	else
		min_allocable_bytes = 0;

7217 7218
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
7219 7220 7221 7222 7223 7224

	if (cache->ro) {
		ret = 0;
		goto out;
	}

7225 7226 7227 7228
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
7229 7230
	    sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
	    min_allocable_bytes <= sinfo->total_bytes) {
7231 7232 7233 7234
		sinfo->bytes_readonly += num_bytes;
		cache->ro = 1;
		ret = 0;
	}
7235
out:
7236 7237 7238 7239
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
7240

7241 7242
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
7243

7244 7245 7246 7247
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
7248

7249
	BUG_ON(cache->ro);
C
Chris Mason 已提交
7250

C
Chris Mason 已提交
7251
	trans = btrfs_join_transaction(root);
7252 7253
	if (IS_ERR(trans))
		return PTR_ERR(trans);
7254

7255
	alloc_flags = update_block_group_flags(root, cache->flags);
7256
	if (alloc_flags != cache->flags) {
7257
		ret = do_chunk_alloc(trans, root, alloc_flags,
7258 7259 7260 7261
				     CHUNK_ALLOC_FORCE);
		if (ret < 0)
			goto out;
	}
7262

7263
	ret = set_block_group_ro(cache, 0);
7264 7265 7266
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
7267
	ret = do_chunk_alloc(trans, root, alloc_flags,
7268
			     CHUNK_ALLOC_FORCE);
7269 7270
	if (ret < 0)
		goto out;
7271
	ret = set_block_group_ro(cache, 0);
7272 7273 7274 7275
out:
	btrfs_end_transaction(trans, root);
	return ret;
}
7276

7277 7278 7279 7280
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
7281
	return do_chunk_alloc(trans, root, alloc_flags,
7282
			      CHUNK_ALLOC_FORCE);
7283 7284
}

7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340
/*
 * helper to account the unused space of all the readonly block group in the
 * list. takes mirrors into account.
 */
static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

	list_for_each_entry(block_group, groups_list, list) {
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}

	return free_bytes;
}

/*
 * helper to account the unused space of all the readonly block group in the
 * space_info. takes mirrors into account.
 */
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
{
	int i;
	u64 free_bytes = 0;

	spin_lock(&sinfo->lock);

	for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		if (!list_empty(&sinfo->block_groups[i]))
			free_bytes += __btrfs_get_ro_block_group_free_space(
						&sinfo->block_groups[i]);

	spin_unlock(&sinfo->lock);

	return free_bytes;
}

7341
void btrfs_set_block_group_rw(struct btrfs_root *root,
7342
			      struct btrfs_block_group_cache *cache)
7343
{
7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
7357 7358
}

7359 7360 7361 7362 7363 7364 7365
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
7366
{
7367 7368 7369 7370
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
7371
	u64 min_free;
J
Josef Bacik 已提交
7372 7373
	u64 dev_min = 1;
	u64 dev_nr = 0;
7374
	u64 target;
7375
	int index;
7376 7377
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
7378

7379
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
7380

7381 7382 7383
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
Z
Zheng Yan 已提交
7384

7385 7386
	min_free = btrfs_block_group_used(&block_group->item);

7387
	/* no bytes used, we're good */
7388
	if (!min_free)
Z
Zheng Yan 已提交
7389 7390
		goto out;

7391 7392
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
7393

7394
	full = space_info->full;
7395

7396 7397
	/*
	 * if this is the last block group we have in this space, we can't
7398 7399 7400 7401
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
7402
	 */
7403
	if ((space_info->total_bytes != block_group->key.offset) &&
7404 7405 7406
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
7407 7408
		spin_unlock(&space_info->lock);
		goto out;
7409
	}
7410
	spin_unlock(&space_info->lock);
7411

7412 7413 7414
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
7415 7416 7417
	 * alloc devices and guess if we have enough space.  if this block
	 * group is going to be restriped, run checks against the target
	 * profile instead of the current one.
7418 7419
	 */
	ret = -1;
7420

7421 7422 7423 7424 7425 7426 7427 7428
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442
	target = get_restripe_target(root->fs_info, block_group->flags);
	if (target) {
		index = __get_block_group_index(extended_to_chunk(target));
	} else {
		/*
		 * this is just a balance, so if we were marked as full
		 * we know there is no space for a new chunk
		 */
		if (full)
			goto out;

		index = get_block_group_index(block_group);
	}

7443 7444
	if (index == 0) {
		dev_min = 4;
J
Josef Bacik 已提交
7445 7446
		/* Divide by 2 */
		min_free >>= 1;
7447 7448 7449
	} else if (index == 1) {
		dev_min = 2;
	} else if (index == 2) {
J
Josef Bacik 已提交
7450 7451
		/* Multiply by 2 */
		min_free <<= 1;
7452 7453
	} else if (index == 3) {
		dev_min = fs_devices->rw_devices;
J
Josef Bacik 已提交
7454
		do_div(min_free, dev_min);
7455 7456
	}

7457 7458
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
7459
		u64 dev_offset;
7460

7461 7462 7463 7464 7465
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
		if (device->total_bytes > device->bytes_used + min_free) {
7466
			ret = find_free_dev_extent(device, min_free,
7467
						   &dev_offset, NULL);
7468
			if (!ret)
7469 7470 7471
				dev_nr++;

			if (dev_nr >= dev_min)
7472
				break;
7473

7474
			ret = -1;
7475
		}
7476
	}
7477
	mutex_unlock(&root->fs_info->chunk_mutex);
7478
out:
7479
	btrfs_put_block_group(block_group);
7480 7481 7482
	return ret;
}

7483 7484
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
7485
{
7486
	int ret = 0;
7487 7488 7489
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
7490

7491 7492
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
7493 7494
		goto out;

C
Chris Mason 已提交
7495
	while (1) {
7496
		slot = path->slots[0];
7497
		leaf = path->nodes[0];
7498 7499 7500 7501 7502
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
7503
				goto out;
7504
			break;
7505
		}
7506
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
7507

7508
		if (found_key.objectid >= key->objectid &&
7509 7510 7511 7512
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
7513
		path->slots[0]++;
7514
	}
7515
out:
7516
	return ret;
7517 7518
}

7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
7553 7554 7555
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
7556
	struct btrfs_space_info *space_info;
7557
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
7558 7559
	struct rb_node *n;

7560 7561 7562 7563 7564 7565 7566 7567 7568
	down_write(&info->extent_commit_sem);
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
	up_write(&info->extent_commit_sem);

Z
Zheng Yan 已提交
7569 7570 7571 7572 7573 7574
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
Y
Yan Zheng 已提交
7575 7576
		spin_unlock(&info->block_group_cache_lock);

7577
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
7578
		list_del(&block_group->list);
7579
		up_write(&block_group->space_info->groups_sem);
7580

J
Josef Bacik 已提交
7581
		if (block_group->cached == BTRFS_CACHE_STARTED)
7582
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
7583

7584 7585 7586 7587 7588 7589 7590
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
		if (block_group->cached == BTRFS_CACHE_NO)
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
7591
		btrfs_remove_free_space_cache(block_group);
7592
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
7593 7594

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
7595 7596
	}
	spin_unlock(&info->block_group_cache_lock);
7597 7598 7599 7600 7601 7602 7603 7604 7605

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

7606 7607
	release_global_block_rsv(info);

7608 7609 7610 7611
	while(!list_empty(&info->space_info)) {
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
7612
		if (space_info->bytes_pinned > 0 ||
7613 7614
		    space_info->bytes_reserved > 0 ||
		    space_info->bytes_may_use > 0) {
7615 7616 7617
			WARN_ON(1);
			dump_space_info(space_info, 0, 0);
		}
7618 7619 7620
		list_del(&space_info->list);
		kfree(space_info);
	}
Z
Zheng Yan 已提交
7621 7622 7623
	return 0;
}

7624 7625 7626 7627 7628 7629 7630 7631 7632 7633
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);

	down_write(&space_info->groups_sem);
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);
}

C
Chris Mason 已提交
7634 7635 7636 7637 7638
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
7639
	struct btrfs_fs_info *info = root->fs_info;
7640
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
7641 7642
	struct btrfs_key key;
	struct btrfs_key found_key;
7643
	struct extent_buffer *leaf;
7644 7645
	int need_clear = 0;
	u64 cache_gen;
7646

C
Chris Mason 已提交
7647
	root = info->extent_root;
C
Chris Mason 已提交
7648
	key.objectid = 0;
7649
	key.offset = 0;
C
Chris Mason 已提交
7650 7651 7652 7653
	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
J
Josef Bacik 已提交
7654
	path->reada = 1;
C
Chris Mason 已提交
7655

7656
	cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
7657
	if (btrfs_test_opt(root, SPACE_CACHE) &&
7658
	    btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
7659
		need_clear = 1;
7660 7661
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
7662

C
Chris Mason 已提交
7663
	while (1) {
7664
		ret = find_first_block_group(root, path, &key);
7665 7666
		if (ret > 0)
			break;
7667 7668
		if (ret != 0)
			goto error;
7669 7670
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7671
		cache = kzalloc(sizeof(*cache), GFP_NOFS);
C
Chris Mason 已提交
7672
		if (!cache) {
7673
			ret = -ENOMEM;
7674
			goto error;
C
Chris Mason 已提交
7675
		}
7676 7677 7678 7679 7680 7681 7682
		cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
						GFP_NOFS);
		if (!cache->free_space_ctl) {
			kfree(cache);
			ret = -ENOMEM;
			goto error;
		}
C
Chris Mason 已提交
7683

7684
		atomic_set(&cache->count, 1);
7685
		spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7686
		cache->fs_info = info;
J
Josef Bacik 已提交
7687
		INIT_LIST_HEAD(&cache->list);
7688
		INIT_LIST_HEAD(&cache->cluster_list);
7689

7690 7691 7692 7693 7694 7695 7696 7697 7698 7699 7700
		if (need_clear) {
			/*
			 * When we mount with old space cache, we need to
			 * set BTRFS_DC_CLEAR and set dirty flag.
			 *
			 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
			 *    truncate the old free space cache inode and
			 *    setup a new one.
			 * b) Setting 'dirty flag' makes sure that we flush
			 *    the new space cache info onto disk.
			 */
7701
			cache->disk_cache_state = BTRFS_DC_CLEAR;
7702 7703 7704
			if (btrfs_test_opt(root, SPACE_CACHE))
				cache->dirty = 1;
		}
7705

7706 7707 7708
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
C
Chris Mason 已提交
7709
		memcpy(&cache->key, &found_key, sizeof(found_key));
7710

C
Chris Mason 已提交
7711
		key.objectid = found_key.objectid + found_key.offset;
7712
		btrfs_release_path(path);
7713
		cache->flags = btrfs_block_group_flags(&cache->item);
J
Josef Bacik 已提交
7714 7715
		cache->sectorsize = root->sectorsize;

7716 7717
		btrfs_init_free_space_ctl(cache);

7718 7719 7720 7721 7722 7723 7724
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
		exclude_super_stripes(root, cache);

J
Josef Bacik 已提交
7725 7726 7727 7728 7729 7730 7731 7732
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
7733
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7734
			cache->cached = BTRFS_CACHE_FINISHED;
7735
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7736
		} else if (btrfs_block_group_used(&cache->item) == 0) {
7737
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7738 7739 7740 7741 7742
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
7743
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7744
		}
7745

7746 7747 7748
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
7749
		BUG_ON(ret); /* -ENOMEM */
7750
		cache->space_info = space_info;
7751
		spin_lock(&cache->space_info->lock);
7752
		cache->space_info->bytes_readonly += cache->bytes_super;
7753 7754
		spin_unlock(&cache->space_info->lock);

7755
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
7756 7757

		ret = btrfs_add_block_group_cache(root->fs_info, cache);
7758
		BUG_ON(ret); /* Logic error */
7759 7760

		set_avail_alloc_bits(root->fs_info, cache->flags);
Y
Yan Zheng 已提交
7761
		if (btrfs_chunk_readonly(root, cache->key.objectid))
7762
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7763
	}
7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
		list_for_each_entry(cache, &space_info->block_groups[3], list)
7776
			set_block_group_ro(cache, 1);
7777
		list_for_each_entry(cache, &space_info->block_groups[4], list)
7778
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7779
	}
7780 7781

	init_global_block_rsv(info);
7782 7783
	ret = 0;
error:
C
Chris Mason 已提交
7784
	btrfs_free_path(path);
7785
	return ret;
C
Chris Mason 已提交
7786
}
7787

7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815
void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root)
{
	struct btrfs_block_group_cache *block_group, *tmp;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
	struct btrfs_block_group_item item;
	struct btrfs_key key;
	int ret = 0;

	list_for_each_entry_safe(block_group, tmp, &trans->new_bgs,
				 new_bg_list) {
		list_del_init(&block_group->new_bg_list);

		if (ret)
			continue;

		spin_lock(&block_group->lock);
		memcpy(&item, &block_group->item, sizeof(item));
		memcpy(&key, &block_group->key, sizeof(key));
		spin_unlock(&block_group->lock);

		ret = btrfs_insert_item(trans, extent_root, &key, &item,
					sizeof(item));
		if (ret)
			btrfs_abort_transaction(trans, extent_root, ret);
	}
}

7816 7817
int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
7818
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
7819 7820 7821 7822 7823 7824 7825 7826
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

7827
	root->fs_info->last_trans_log_full_commit = trans->transid;
7828

7829
	cache = kzalloc(sizeof(*cache), GFP_NOFS);
J
Josef Bacik 已提交
7830 7831
	if (!cache)
		return -ENOMEM;
7832 7833 7834 7835 7836 7837
	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return -ENOMEM;
	}
J
Josef Bacik 已提交
7838

7839
	cache->key.objectid = chunk_offset;
7840
	cache->key.offset = size;
7841
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
7842
	cache->sectorsize = root->sectorsize;
7843
	cache->fs_info = root->fs_info;
7844

7845
	atomic_set(&cache->count, 1);
7846
	spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7847
	INIT_LIST_HEAD(&cache->list);
7848
	INIT_LIST_HEAD(&cache->cluster_list);
7849
	INIT_LIST_HEAD(&cache->new_bg_list);
C
Chris Mason 已提交
7850

7851 7852
	btrfs_init_free_space_ctl(cache);

7853 7854 7855 7856 7857
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	cache->flags = type;
	btrfs_set_block_group_flags(&cache->item, type);

7858
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7859
	cache->cached = BTRFS_CACHE_FINISHED;
7860
	exclude_super_stripes(root, cache);
7861

J
Josef Bacik 已提交
7862 7863 7864
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

7865 7866
	free_excluded_extents(root, cache);

7867 7868
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
7869
	BUG_ON(ret); /* -ENOMEM */
7870
	update_global_block_rsv(root->fs_info);
7871 7872

	spin_lock(&cache->space_info->lock);
7873
	cache->space_info->bytes_readonly += cache->bytes_super;
7874 7875
	spin_unlock(&cache->space_info->lock);

7876
	__link_block_group(cache->space_info, cache);
7877

J
Josef Bacik 已提交
7878
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
7879
	BUG_ON(ret); /* Logic error */
7880

7881
	list_add_tail(&cache->new_bg_list, &trans->new_bgs);
7882

C
Chris Mason 已提交
7883
	set_avail_alloc_bits(extent_root->fs_info, type);
7884

7885 7886
	return 0;
}
Z
Zheng Yan 已提交
7887

7888 7889
static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
7890 7891
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
7892 7893 7894 7895 7896 7897 7898 7899 7900

	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits &= ~extra_flags;
}

Z
Zheng Yan 已提交
7901 7902 7903 7904 7905
int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 group_start)
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
7906
	struct btrfs_free_cluster *cluster;
7907
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
7908
	struct btrfs_key key;
7909
	struct inode *inode;
Z
Zheng Yan 已提交
7910
	int ret;
7911
	int index;
J
Josef Bacik 已提交
7912
	int factor;
Z
Zheng Yan 已提交
7913 7914 7915 7916 7917

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
7918
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
7919

7920 7921 7922 7923 7924 7925
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

Z
Zheng Yan 已提交
7926
	memcpy(&key, &block_group->key, sizeof(key));
7927
	index = get_block_group_index(block_group);
J
Josef Bacik 已提交
7928 7929 7930 7931 7932 7933
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
7934

7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
7950
	path = btrfs_alloc_path();
7951 7952 7953 7954
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
Z
Zheng Yan 已提交
7955

7956
	inode = lookup_free_space_inode(tree_root, block_group, path);
7957
	if (!IS_ERR(inode)) {
7958
		ret = btrfs_orphan_add(trans, inode);
7959 7960 7961 7962
		if (ret) {
			btrfs_add_delayed_iput(inode);
			goto out;
		}
7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
7975
		btrfs_add_delayed_iput(inode);
7976 7977 7978 7979 7980 7981 7982 7983 7984 7985
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
7986
		btrfs_release_path(path);
7987 7988 7989 7990
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
7991
		btrfs_release_path(path);
7992 7993
	}

7994
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
7995 7996
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
7997
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
7998

7999
	down_write(&block_group->space_info->groups_sem);
8000 8001 8002 8003 8004
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
8005 8006
	if (list_empty(&block_group->space_info->block_groups[index]))
		clear_avail_alloc_bits(root->fs_info, block_group->flags);
8007
	up_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
8008

J
Josef Bacik 已提交
8009
	if (block_group->cached == BTRFS_CACHE_STARTED)
8010
		wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
8011 8012 8013

	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
8014 8015 8016
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
8017
	block_group->space_info->disk_total -= block_group->key.offset * factor;
Y
Yan Zheng 已提交
8018
	spin_unlock(&block_group->space_info->lock);
8019

8020 8021
	memcpy(&key, &block_group->key, sizeof(key));

8022
	btrfs_clear_space_info_full(root->fs_info);
Y
Yan Zheng 已提交
8023

8024 8025
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
8038

8039 8040 8041
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
8042 8043 8044 8045
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
8046 8047
	int ret;

8048
	disk_super = fs_info->super_copy;
8049 8050
	if (!btrfs_super_root(disk_super))
		return 1;
8051

8052 8053 8054
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
8055

8056 8057
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
	ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8058
	if (ret)
8059
		goto out;
8060

8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	}
out:
8074 8075 8076
	return ret;
}

L
liubo 已提交
8077 8078 8079 8080 8081 8082
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
	return unpin_extent_range(root, start, end);
}

int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
8083
			       u64 num_bytes, u64 *actual_bytes)
L
liubo 已提交
8084
{
8085
	return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
L
liubo 已提交
8086
}
8087 8088 8089 8090 8091 8092 8093 8094 8095

int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
8096
	u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
8097 8098
	int ret = 0;

8099 8100 8101 8102 8103 8104 8105
	/*
	 * try to trim all FS space, our block group may start from non-zero.
	 */
	if (range->len == total_bytes)
		cache = btrfs_lookup_first_block_group(fs_info, range->start);
	else
		cache = btrfs_lookup_block_group(fs_info, range->start);
8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
				ret = cache_block_group(cache, NULL, root, 0);
				if (!ret)
					wait_block_group_cache_done(cache);
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

	range->len = trimmed;
	return ret;
}