extent-tree.c 219.0 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
26
#include <linux/ratelimit.h>
C
Chris Mason 已提交
27
#include "compat.h"
28
#include "hash.h"
29 30 31
#include "ctree.h"
#include "disk-io.h"
#include "print-tree.h"
32
#include "transaction.h"
33
#include "volumes.h"
34
#include "locking.h"
35
#include "free-space-cache.h"
36
#include "math.h"
37

38 39
#undef SCRAMBLE_DELAYED_REFS

40 41
/*
 * control flags for do_chunk_alloc's force field
42 43 44 45 46 47 48 49 50
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
51 52
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
53 54 55
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
56 57
	CHUNK_ALLOC_LIMITED = 1,
	CHUNK_ALLOC_FORCE = 2,
58 59
};

60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
/*
 * Control how reservations are dealt with.
 *
 * RESERVE_FREE - freeing a reservation.
 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
 *   ENOSPC accounting
 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
 *   bytes_may_use as the ENOSPC accounting is done elsewhere
 */
enum {
	RESERVE_FREE = 0,
	RESERVE_ALLOC = 1,
	RESERVE_ALLOC_NO_ACCOUNT = 2,
};

75
static int update_block_group(struct btrfs_root *root,
76
			      u64 bytenr, u64 num_bytes, int alloc);
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extra_op);
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins);
J
Josef Bacik 已提交
96
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
97 98
			  struct btrfs_root *extent_root, u64 flags,
			  int force);
99 100
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
101 102
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
103 104
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve);
J
Josef Bacik 已提交
105

J
Josef Bacik 已提交
106 107 108 109 110 111 112
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
	return cache->cached == BTRFS_CACHE_FINISHED;
}

J
Josef Bacik 已提交
113 114 115 116 117
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

118
static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
119 120 121 122 123 124
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
125 126 127
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
128
		kfree(cache->free_space_ctl);
129
		kfree(cache);
130
	}
131 132
}

J
Josef Bacik 已提交
133 134 135 136
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
137
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
164 165 166 167

	if (info->first_logical_byte > block_group->key.objectid)
		info->first_logical_byte = block_group->key.objectid;

J
Josef Bacik 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
209
	if (ret) {
210
		btrfs_get_block_group(ret);
211 212 213
		if (bytenr == 0 && info->first_logical_byte > ret->key.objectid)
			info->first_logical_byte = ret->key.objectid;
	}
J
Josef Bacik 已提交
214 215 216 217 218
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

219 220
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
221
{
222 223 224 225 226 227 228
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
J
Josef Bacik 已提交
229

230 231 232 233
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
234

235 236 237 238 239 240 241
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
J
Josef Bacik 已提交
242 243
}

244 245
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
246 247 248 249 250 251
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

252 253 254 255 256
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
257
		BUG_ON(ret); /* -ENOMEM */
258 259
	}

J
Josef Bacik 已提交
260 261 262 263 264
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
265
		BUG_ON(ret); /* -ENOMEM */
266

J
Josef Bacik 已提交
267
		while (nr--) {
268
			cache->bytes_super += stripe_len;
269 270
			ret = add_excluded_extent(root, logical[nr],
						  stripe_len);
271
			BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
272
		}
273

J
Josef Bacik 已提交
274 275 276 277 278
		kfree(logical);
	}
	return 0;
}

279 280 281 282 283 284 285 286 287 288 289
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_STARTED) {
		spin_unlock(&cache->lock);
		return NULL;
	}

290 291 292
	/* We're loading it the fast way, so we don't have a caching_ctl. */
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

J
Josef Bacik 已提交
308 309 310 311 312
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
J
Josef Bacik 已提交
313
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
J
Josef Bacik 已提交
314 315
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
J
Josef Bacik 已提交
316
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
317 318 319
	int ret;

	while (start < end) {
320
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
321
					    &extent_start, &extent_end,
322 323
					    EXTENT_DIRTY | EXTENT_UPTODATE,
					    NULL);
J
Josef Bacik 已提交
324 325 326
		if (ret)
			break;

327
		if (extent_start <= start) {
J
Josef Bacik 已提交
328 329 330
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
331
			total_added += size;
332 333
			ret = btrfs_add_free_space(block_group, start,
						   size);
334
			BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
335 336 337 338 339 340 341 342
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
343
		total_added += size;
344
		ret = btrfs_add_free_space(block_group, start, size);
345
		BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
346 347
	}

J
Josef Bacik 已提交
348
	return total_added;
J
Josef Bacik 已提交
349 350
}

351
static noinline void caching_thread(struct btrfs_work *work)
352
{
353 354 355 356
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_root *extent_root;
357
	struct btrfs_path *path;
358
	struct extent_buffer *leaf;
359
	struct btrfs_key key;
J
Josef Bacik 已提交
360
	u64 total_found = 0;
361 362 363
	u64 last = 0;
	u32 nritems;
	int ret = 0;
364

365 366 367 368 369
	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

370 371
	path = btrfs_alloc_path();
	if (!path)
372
		goto out;
373

J
Josef Bacik 已提交
374
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
375

376
	/*
J
Josef Bacik 已提交
377 378 379 380
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
381 382
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
383
	path->search_commit_root = 1;
J
Josef Bacik 已提交
384
	path->reada = 1;
J
Josef Bacik 已提交
385

Y
Yan Zheng 已提交
386
	key.objectid = last;
387
	key.offset = 0;
388
	key.type = BTRFS_EXTENT_ITEM_KEY;
389
again:
390
	mutex_lock(&caching_ctl->mutex);
391 392 393
	/* need to make sure the commit_root doesn't disappear */
	down_read(&fs_info->extent_commit_sem);

394
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
395
	if (ret < 0)
396
		goto err;
Y
Yan Zheng 已提交
397

398 399 400
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
401
	while (1) {
402
		if (btrfs_fs_closing(fs_info) > 1) {
403
			last = (u64)-1;
J
Josef Bacik 已提交
404
			break;
405
		}
J
Josef Bacik 已提交
406

407 408 409 410 411
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
412
				break;
J
Josef Bacik 已提交
413

414 415 416
			if (need_resched() ||
			    btrfs_next_leaf(extent_root, path)) {
				caching_ctl->progress = last;
C
Chris Mason 已提交
417
				btrfs_release_path(path);
418 419
				up_read(&fs_info->extent_commit_sem);
				mutex_unlock(&caching_ctl->mutex);
420
				cond_resched();
421 422 423 424 425
				goto again;
			}
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
426
		}
J
Josef Bacik 已提交
427

428 429
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
430
			continue;
431
		}
J
Josef Bacik 已提交
432

433
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
434
		    block_group->key.offset)
435
			break;
436

437
		if (key.type == BTRFS_EXTENT_ITEM_KEY) {
J
Josef Bacik 已提交
438 439 440
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
441
			last = key.objectid + key.offset;
J
Josef Bacik 已提交
442

443 444 445 446
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
J
Josef Bacik 已提交
447
		}
448 449
		path->slots[0]++;
	}
J
Josef Bacik 已提交
450
	ret = 0;
451

J
Josef Bacik 已提交
452 453 454
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
455
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
456 457

	spin_lock(&block_group->lock);
458
	block_group->caching_ctl = NULL;
J
Josef Bacik 已提交
459 460
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
461

462
err:
463
	btrfs_free_path(path);
464
	up_read(&fs_info->extent_commit_sem);
J
Josef Bacik 已提交
465

466 467 468
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
469
out:
470 471 472
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
473
	btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
474 475
}

476 477
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     int load_cache_only)
J
Josef Bacik 已提交
478
{
479
	DEFINE_WAIT(wait);
480 481
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
482 483
	int ret = 0;

484
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
485 486
	if (!caching_ctl)
		return -ENOMEM;
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	atomic_set(&caching_ctl->count, 1);
	caching_ctl->work.func = caching_thread;

	spin_lock(&cache->lock);
	/*
	 * This should be a rare occasion, but this could happen I think in the
	 * case where one thread starts to load the space cache info, and then
	 * some other thread starts a transaction commit which tries to do an
	 * allocation while the other thread is still loading the space cache
	 * info.  The previous loop should have kept us from choosing this block
	 * group, but if we've moved to the state where we will wait on caching
	 * block groups we need to first check if we're doing a fast load here,
	 * so we can wait for it to finish, otherwise we could end up allocating
	 * from a block group who's cache gets evicted for one reason or
	 * another.
	 */
	while (cache->cached == BTRFS_CACHE_FAST) {
		struct btrfs_caching_control *ctl;

		ctl = cache->caching_ctl;
		atomic_inc(&ctl->count);
		prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&cache->lock);

		schedule();

		finish_wait(&ctl->wait, &wait);
		put_caching_control(ctl);
		spin_lock(&cache->lock);
	}

	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
		kfree(caching_ctl);
527
		return 0;
528 529 530 531 532
	}
	WARN_ON(cache->caching_ctl);
	cache->caching_ctl = caching_ctl;
	cache->cached = BTRFS_CACHE_FAST;
	spin_unlock(&cache->lock);
533

534
	if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
535 536 537 538
		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
539
			cache->caching_ctl = NULL;
540 541 542
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
		} else {
543 544 545 546 547 548
			if (load_cache_only) {
				cache->caching_ctl = NULL;
				cache->cached = BTRFS_CACHE_NO;
			} else {
				cache->cached = BTRFS_CACHE_STARTED;
			}
549 550
		}
		spin_unlock(&cache->lock);
551
		wake_up(&caching_ctl->wait);
552
		if (ret == 1) {
553
			put_caching_control(caching_ctl);
554
			free_excluded_extents(fs_info->extent_root, cache);
555
			return 0;
556
		}
557 558 559 560 561 562 563 564 565 566 567 568 569 570
	} else {
		/*
		 * We are not going to do the fast caching, set cached to the
		 * appropriate value and wakeup any waiters.
		 */
		spin_lock(&cache->lock);
		if (load_cache_only) {
			cache->caching_ctl = NULL;
			cache->cached = BTRFS_CACHE_NO;
		} else {
			cache->cached = BTRFS_CACHE_STARTED;
		}
		spin_unlock(&cache->lock);
		wake_up(&caching_ctl->wait);
571 572
	}

573 574
	if (load_cache_only) {
		put_caching_control(caching_ctl);
575
		return 0;
J
Josef Bacik 已提交
576 577
	}

578
	down_write(&fs_info->extent_commit_sem);
579
	atomic_inc(&caching_ctl->count);
580 581 582
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
	up_write(&fs_info->extent_commit_sem);

583
	btrfs_get_block_group(cache);
584

585
	btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
J
Josef Bacik 已提交
586

587
	return ret;
588 589
}

J
Josef Bacik 已提交
590 591 592
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
593 594
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
595
{
J
Josef Bacik 已提交
596
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
597

J
Josef Bacik 已提交
598
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
599

J
Josef Bacik 已提交
600
	return cache;
C
Chris Mason 已提交
601 602
}

J
Josef Bacik 已提交
603
/*
604
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
605
 */
C
Chris Mason 已提交
606 607 608
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
609
{
J
Josef Bacik 已提交
610
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
611

J
Josef Bacik 已提交
612
	cache = block_group_cache_tree_search(info, bytenr, 1);
613

J
Josef Bacik 已提交
614
	return cache;
C
Chris Mason 已提交
615
}
616

J
Josef Bacik 已提交
617 618
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
619
{
J
Josef Bacik 已提交
620 621
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
622

623
	flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
624

625 626
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
627
		if (found->flags & flags) {
628
			rcu_read_unlock();
J
Josef Bacik 已提交
629
			return found;
630
		}
J
Josef Bacik 已提交
631
	}
632
	rcu_read_unlock();
J
Josef Bacik 已提交
633
	return NULL;
634 635
}

636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

651 652
u64 btrfs_find_block_group(struct btrfs_root *root,
			   u64 search_start, u64 search_hint, int owner)
C
Chris Mason 已提交
653
{
654
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
655
	u64 used;
656 657
	u64 last = max(search_hint, search_start);
	u64 group_start = 0;
658
	int full_search = 0;
659
	int factor = 9;
C
Chris Mason 已提交
660
	int wrapped = 0;
661
again:
662 663
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
J
Josef Bacik 已提交
664 665
		if (!cache)
			break;
666

667
		spin_lock(&cache->lock);
668 669 670
		last = cache->key.objectid + cache->key.offset;
		used = btrfs_block_group_used(&cache->item);

671 672
		if ((full_search || !cache->ro) &&
		    block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
673
			if (used + cache->pinned + cache->reserved <
674 675
			    div_factor(cache->key.offset, factor)) {
				group_start = cache->key.objectid;
676
				spin_unlock(&cache->lock);
677
				btrfs_put_block_group(cache);
678 679
				goto found;
			}
680
		}
681
		spin_unlock(&cache->lock);
682
		btrfs_put_block_group(cache);
683
		cond_resched();
C
Chris Mason 已提交
684
	}
C
Chris Mason 已提交
685 686 687 688 689 690
	if (!wrapped) {
		last = search_start;
		wrapped = 1;
		goto again;
	}
	if (!full_search && factor < 10) {
C
Chris Mason 已提交
691
		last = search_start;
692
		full_search = 1;
C
Chris Mason 已提交
693
		factor = 10;
694 695
		goto again;
	}
C
Chris Mason 已提交
696
found:
697
	return group_start;
698
}
J
Josef Bacik 已提交
699

700
/* simple helper to search for an existing extent at a given offset */
Z
Zheng Yan 已提交
701
int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
702 703 704
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
705
	struct btrfs_path *path;
706

Z
Zheng Yan 已提交
707
	path = btrfs_alloc_path();
708 709 710
	if (!path)
		return -ENOMEM;

711 712 713 714 715
	key.objectid = start;
	key.offset = len;
	btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
716
	btrfs_free_path(path);
717 718 719
	return ret;
}

720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
/*
 * helper function to lookup reference count and flags of extent.
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
			     u64 num_bytes, u64 *refs, u64 *flags)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = num_bytes;
	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
again:
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

800
			btrfs_release_path(path);
801

802 803 804 805
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
			goto again;
		}
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

831 832 833 834 835 836 837 838 839 840 841 842 843 844
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
890 891 892
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
893
 * - different files inside a single subvolume
894 895
 * - different offsets inside a file (bookend extents in file.c)
 *
896
 * The extent ref structure for the implicit back refs has fields for:
897 898 899
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
900 901
 * - original offset in the file
 * - how many bookend extents
902
 *
903 904
 * The key offset for the implicit back refs is hash of the first
 * three fields.
905
 *
906
 * The extent ref structure for the full back refs has field for:
907
 *
908
 * - number of pointers in the tree leaf
909
 *
910 911
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
912
 *
913 914
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
915
 *
916
 *     (root_key.objectid, inode objectid, offset in file, 1)
917
 *
918 919
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
920
 *
921
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
922
 *
923
 * Btree extents can be referenced by:
924
 *
925
 * - Different subvolumes
926
 *
927 928 929 930
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
931
 *
932 933 934
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
935
 */
Z
Zheng Yan 已提交
936

937 938 939 940 941
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
942
{
943 944 945 946 947
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
948
	struct btrfs_key key;
949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
968
				BUG_ON(ret > 0); /* Corruption */
969 970 971 972 973 974 975 976 977 978 979 980 981 982 983
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
984
	btrfs_release_path(path);
985 986 987 988 989 990 991 992 993

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
994
	BUG_ON(ret); /* Corruption */
995

996
	btrfs_extend_item(trans, root, path, new_size);
997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
1026
	high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1027
	lenum = cpu_to_le64(owner);
1028
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1029
	lenum = cpu_to_le64(offset);
1030
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1063
	struct extent_buffer *leaf;
1064
	u32 nritems;
1065
	int ret;
1066 1067
	int recow;
	int err = -ENOENT;
1068

Z
Zheng Yan 已提交
1069
	key.objectid = bytenr;
1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1085

1086 1087 1088 1089 1090
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1091
		btrfs_release_path(path);
1092 1093 1094 1095 1096 1097 1098 1099 1100
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1101 1102 1103
	}

	leaf = path->nodes[0];
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1129
				btrfs_release_path(path);
1130 1131 1132 1133 1134 1135
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1136
	}
1137 1138
fail:
	return err;
Z
Zheng Yan 已提交
1139 1140
}

1141 1142 1143 1144 1145 1146
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1147 1148 1149
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1150
	u32 size;
Z
Zheng Yan 已提交
1151 1152
	u32 num_refs;
	int ret;
1153 1154

	key.objectid = bytenr;
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1165

1166 1167 1168 1169 1170 1171 1172
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1173
		ref = btrfs_item_ptr(leaf, path->slots[0],
1174 1175 1176 1177 1178 1179 1180
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1181
		}
1182 1183 1184 1185 1186 1187 1188 1189
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1190
			btrfs_release_path(path);
1191 1192 1193 1194 1195
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1196

1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1211 1212
		}
	}
1213 1214 1215
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1216
	btrfs_release_path(path);
1217
	return ret;
1218 1219
}

1220 1221 1222 1223
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   int refs_to_drop)
Z
Zheng Yan 已提交
1224
{
1225 1226 1227
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1228
	struct extent_buffer *leaf;
1229
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1230 1231 1232
	int ret = 0;

	leaf = path->nodes[0];
1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1254 1255
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1256

Z
Zheng Yan 已提交
1257 1258 1259
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
	} else {
1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1272 1273 1274 1275 1276
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1277 1278 1279
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1280
{
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1312
#endif
1313 1314 1315 1316 1317
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1318

1319 1320 1321 1322 1323
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1324
{
1325
	struct btrfs_key key;
1326 1327
	int ret;

1328 1329 1330 1331 1332 1333 1334
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1335 1336
	}

1337 1338 1339 1340 1341
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1342
		btrfs_release_path(path);
1343 1344 1345 1346 1347
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1348
#endif
1349
	return ret;
1350 1351
}

1352 1353 1354 1355 1356
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1357
{
1358
	struct btrfs_key key;
Z
Zheng Yan 已提交
1359 1360
	int ret;

1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1371
	btrfs_release_path(path);
Z
Zheng Yan 已提交
1372 1373 1374
	return ret;
}

1375
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1376
{
1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1390
}
1391

1392 1393
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1394

C
Chris Mason 已提交
1395
{
1396
	for (; level < BTRFS_MAX_LEVEL; level++) {
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1412

1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1448

1449
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1450
	key.type = BTRFS_EXTENT_ITEM_KEY;
1451
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1452

1453 1454 1455
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1456
		path->keep_locks = 1;
1457 1458 1459
	} else
		extra_size = -1;
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1460
	if (ret < 0) {
1461 1462 1463
		err = ret;
		goto out;
	}
1464 1465 1466 1467 1468
	if (ret && !insert) {
		err = -ENOENT;
		goto out;
	}
	BUG_ON(ret); /* Corruption */
1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	} else {
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1562 1563
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1564
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1565 1566 1567 1568 1569 1570
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1571
	if (insert) {
1572 1573 1574 1575 1576 1577 1578 1579 1580 1581
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
1582 1583 1584 1585 1586 1587 1588
void setup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

1606
	btrfs_extend_item(trans, root, path, size);
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1656
		return ret;
1657

1658
	btrfs_release_path(path);
1659 1660 1661 1662 1663 1664 1665 1666
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1667
	}
1668 1669
	return ret;
}
Z
Zheng Yan 已提交
1670

1671 1672 1673 1674
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
1675 1676 1677 1678 1679 1680
void update_inline_extent_backref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  struct btrfs_extent_inline_ref *iref,
				  int refs_to_mod,
				  struct btrfs_delayed_extent_op *extent_op)
1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1713
	}
Z
Zheng Yan 已提交
1714

1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
1732
		btrfs_truncate_item(trans, root, path, item_size, 1);
1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753
	}
	btrfs_mark_buffer_dirty(leaf);
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1754 1755
		update_inline_extent_backref(trans, root, path, iref,
					     refs_to_add, extent_op);
1756
	} else if (ret == -ENOENT) {
1757 1758 1759 1760
		setup_inline_extent_backref(trans, root, path, iref, parent,
					    root_objectid, owner, offset,
					    refs_to_add, extent_op);
		ret = 0;
1761
	}
1762 1763
	return ret;
}
Z
Zheng Yan 已提交
1764

1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1783

1784 1785 1786 1787 1788 1789
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_drop, int is_data)
{
1790
	int ret = 0;
1791

1792 1793
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
1794 1795
		update_inline_extent_backref(trans, root, path, iref,
					     -refs_to_drop, NULL);
1796 1797 1798 1799 1800 1801 1802 1803
	} else if (is_data) {
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
	} else {
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1804
static int btrfs_issue_discard(struct block_device *bdev,
1805 1806
				u64 start, u64 len)
{
1807
	return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1808 1809 1810
}

static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1811
				u64 num_bytes, u64 *actual_bytes)
1812 1813
{
	int ret;
1814
	u64 discarded_bytes = 0;
1815
	struct btrfs_bio *bbio = NULL;
1816

C
Christoph Hellwig 已提交
1817

1818
	/* Tell the block device(s) that the sectors can be discarded */
1819
	ret = btrfs_map_block(root->fs_info, REQ_DISCARD,
1820
			      bytenr, &num_bytes, &bbio, 0);
1821
	/* Error condition is -ENOMEM */
1822
	if (!ret) {
1823
		struct btrfs_bio_stripe *stripe = bbio->stripes;
1824 1825 1826
		int i;


1827
		for (i = 0; i < bbio->num_stripes; i++, stripe++) {
1828 1829 1830
			if (!stripe->dev->can_discard)
				continue;

1831 1832 1833 1834 1835 1836
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
						  stripe->length);
			if (!ret)
				discarded_bytes += stripe->length;
			else if (ret != -EOPNOTSUPP)
1837
				break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
1838 1839 1840 1841 1842 1843 1844

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
1845
		}
1846
		kfree(bbio);
1847
	}
1848 1849 1850 1851

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

1852 1853 1854 1855

	return ret;
}

1856
/* Can return -ENOMEM */
1857 1858 1859
int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
A
Arne Jansen 已提交
1860
			 u64 root_objectid, u64 owner, u64 offset, int for_cow)
1861 1862
{
	int ret;
A
Arne Jansen 已提交
1863 1864
	struct btrfs_fs_info *fs_info = root->fs_info;

1865 1866 1867 1868
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
1869 1870
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
1871
					parent, root_objectid, (int)owner,
A
Arne Jansen 已提交
1872
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1873
	} else {
A
Arne Jansen 已提交
1874 1875
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
					num_bytes,
1876
					parent, root_objectid, owner, offset,
A
Arne Jansen 已提交
1877
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
	u64 refs;
	int ret;
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
	ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
					   path, bytenr, num_bytes, parent,
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
	if (ret == 0)
		goto out;

	if (ret != -EAGAIN) {
		err = ret;
		goto out;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
1921

1922
	btrfs_mark_buffer_dirty(leaf);
1923
	btrfs_release_path(path);
1924 1925

	path->reada = 1;
1926 1927
	path->leave_spinning = 1;

1928 1929
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
1930 1931
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
1932 1933
	if (ret)
		btrfs_abort_transaction(trans, root, ret);
1934
out:
1935
	btrfs_free_path(path);
1936
	return err;
1937 1938
}

1939 1940 1941 1942 1943
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
1944
{
1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		if (extent_op) {
			BUG_ON(extent_op->update_key);
			flags |= extent_op->flags_to_set;
		}
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
					     extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
					  extent_op);
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
2017
	int ret;
2018 2019
	int err = 0;

2020 2021 2022
	if (trans->aborted)
		return 0;

2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = node->num_bytes;

	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
		err = -EIO;
		goto out;
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2061

2062 2063 2064 2065
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2066 2067
}

2068 2069 2070 2071 2072
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2073 2074
{
	int ret = 0;
2075 2076 2077 2078
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2079

2080 2081 2082
	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;
2083

2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
	ref = btrfs_delayed_node_to_tree_ref(node);
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		BUG_ON(!extent_op || !extent_op->update_flags ||
		       !extent_op->update_key);
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
						ref->level, &ins);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
					     ref->level, 0, 1, extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
					  ref->level, 0, 1, extent_op);
	} else {
		BUG();
	}
2110 2111 2112 2113
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2114 2115 2116 2117 2118
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2119
{
2120 2121 2122 2123 2124
	int ret = 0;

	if (trans->aborted)
		return 0;

2125
	if (btrfs_delayed_ref_is_head(node)) {
2126 2127 2128 2129 2130 2131 2132
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2133 2134
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2135
		if (insert_reserved) {
2136 2137
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2138 2139 2140 2141 2142
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
			}
2143
		}
2144
		return ret;
2145 2146
	}

2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_node *ref;
	int action = BTRFS_ADD_DELAYED_REF;
again:
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
	node = rb_prev(&head->node.rb_node);
	while (1) {
		if (!node)
			break;
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
		if (ref->bytenr != head->node.bytenr)
			break;
2180
		if (ref->action == action)
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
			return ref;
		node = rb_prev(node);
	}
	if (action == BTRFS_ADD_DELAYED_REF) {
		action = BTRFS_DROP_DELAYED_REF;
		goto again;
	}
	return NULL;
}

2191 2192 2193 2194
/*
 * Returns 0 on success or if called with an already aborted transaction.
 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
 */
2195 2196 2197
static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root,
				       struct list_head *cluster)
2198 2199 2200 2201
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2202
	struct btrfs_delayed_extent_op *extent_op;
2203
	struct btrfs_fs_info *fs_info = root->fs_info;
2204
	int ret;
2205
	int count = 0;
2206 2207 2208 2209 2210
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2211 2212
			/* pick a new head ref from the cluster list */
			if (list_empty(cluster))
2213 2214
				break;

2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231
			locked_ref = list_entry(cluster->next,
				     struct btrfs_delayed_ref_head, cluster);

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);

			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2232 2233
			}
		}
2234

2235 2236 2237 2238 2239 2240 2241 2242 2243 2244
		/*
		 * We need to try and merge add/drops of the same ref since we
		 * can run into issues with relocate dropping the implicit ref
		 * and then it being added back again before the drop can
		 * finish.  If we merged anything we need to re-loop so we can
		 * get a good ref.
		 */
		btrfs_merge_delayed_refs(trans, fs_info, delayed_refs,
					 locked_ref);

2245 2246 2247 2248 2249 2250 2251
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);

		if (ref && ref->seq &&
2252
		    btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) {
2253 2254 2255 2256 2257
			/*
			 * there are still refs with lower seq numbers in the
			 * process of being added. Don't run this ref yet.
			 */
			list_del_init(&locked_ref->cluster);
2258
			btrfs_delayed_ref_unlock(locked_ref);
2259 2260 2261 2262 2263 2264 2265 2266
			locked_ref = NULL;
			delayed_refs->num_heads_ready++;
			spin_unlock(&delayed_refs->lock);
			cond_resched();
			spin_lock(&delayed_refs->lock);
			continue;
		}

2267 2268 2269 2270 2271 2272
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2273

2274 2275 2276
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2277 2278 2279 2280 2281 2282
		if (!ref) {
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2283 2284

			if (extent_op && must_insert_reserved) {
2285
				btrfs_free_delayed_extent_op(extent_op);
2286 2287 2288 2289 2290 2291 2292 2293
				extent_op = NULL;
			}

			if (extent_op) {
				spin_unlock(&delayed_refs->lock);

				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
2294
				btrfs_free_delayed_extent_op(extent_op);
2295

2296
				if (ret) {
2297 2298 2299
					printk(KERN_DEBUG
					       "btrfs: run_delayed_extent_op "
					       "returned %d\n", ret);
2300
					spin_lock(&delayed_refs->lock);
2301
					btrfs_delayed_ref_unlock(locked_ref);
2302 2303 2304
					return ret;
				}

2305
				goto next;
2306
			}
2307
		}
C
Chris Mason 已提交
2308

2309 2310 2311
		ref->in_tree = 0;
		rb_erase(&ref->rb_node, &delayed_refs->root);
		delayed_refs->num_entries--;
2312
		if (!btrfs_delayed_ref_is_head(ref)) {
2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328
			/*
			 * when we play the delayed ref, also correct the
			 * ref_mod on head
			 */
			switch (ref->action) {
			case BTRFS_ADD_DELAYED_REF:
			case BTRFS_ADD_DELAYED_EXTENT:
				locked_ref->node.ref_mod -= ref->ref_mod;
				break;
			case BTRFS_DROP_DELAYED_REF:
				locked_ref->node.ref_mod += ref->ref_mod;
				break;
			default:
				WARN_ON(1);
			}
		}
2329
		spin_unlock(&delayed_refs->lock);
2330

2331
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2332
					  must_insert_reserved);
2333

2334
		btrfs_free_delayed_extent_op(extent_op);
2335
		if (ret) {
2336 2337 2338 2339
			btrfs_delayed_ref_unlock(locked_ref);
			btrfs_put_delayed_ref(ref);
			printk(KERN_DEBUG
			       "btrfs: run_one_delayed_ref returned %d\n", ret);
2340
			spin_lock(&delayed_refs->lock);
2341 2342 2343
			return ret;
		}

2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356
		/*
		 * If this node is a head, that means all the refs in this head
		 * have been dealt with, and we will pick the next head to deal
		 * with, so we must unlock the head and drop it from the cluster
		 * list before we release it.
		 */
		if (btrfs_delayed_ref_is_head(ref)) {
			list_del_init(&locked_ref->cluster);
			btrfs_delayed_ref_unlock(locked_ref);
			locked_ref = NULL;
		}
		btrfs_put_delayed_ref(ref);
		count++;
2357
next:
2358 2359 2360 2361 2362 2363
		cond_resched();
		spin_lock(&delayed_refs->lock);
	}
	return count;
}

2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406
#ifdef SCRAMBLE_DELAYED_REFS
/*
 * Normally delayed refs get processed in ascending bytenr order. This
 * correlates in most cases to the order added. To expose dependencies on this
 * order, we start to process the tree in the middle instead of the beginning
 */
static u64 find_middle(struct rb_root *root)
{
	struct rb_node *n = root->rb_node;
	struct btrfs_delayed_ref_node *entry;
	int alt = 1;
	u64 middle;
	u64 first = 0, last = 0;

	n = rb_first(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		first = entry->bytenr;
	}
	n = rb_last(root);
	if (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		last = entry->bytenr;
	}
	n = root->rb_node;

	while (n) {
		entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
		WARN_ON(!entry->in_tree);

		middle = entry->bytenr;

		if (alt)
			n = n->rb_left;
		else
			n = n->rb_right;

		alt = 1 - alt;
	}
	return middle;
}
#endif

2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440
int btrfs_delayed_refs_qgroup_accounting(struct btrfs_trans_handle *trans,
					 struct btrfs_fs_info *fs_info)
{
	struct qgroup_update *qgroup_update;
	int ret = 0;

	if (list_empty(&trans->qgroup_ref_list) !=
	    !trans->delayed_ref_elem.seq) {
		/* list without seq or seq without list */
		printk(KERN_ERR "btrfs: qgroup accounting update error, list is%s empty, seq is %llu\n",
			list_empty(&trans->qgroup_ref_list) ? "" : " not",
			trans->delayed_ref_elem.seq);
		BUG();
	}

	if (!trans->delayed_ref_elem.seq)
		return 0;

	while (!list_empty(&trans->qgroup_ref_list)) {
		qgroup_update = list_first_entry(&trans->qgroup_ref_list,
						 struct qgroup_update, list);
		list_del(&qgroup_update->list);
		if (!ret)
			ret = btrfs_qgroup_account_ref(
					trans, fs_info, qgroup_update->node,
					qgroup_update->extent_op);
		kfree(qgroup_update);
	}

	btrfs_put_tree_mod_seq(fs_info, &trans->delayed_ref_elem);

	return ret;
}

2441 2442 2443 2444 2445 2446
/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
2447 2448 2449
 *
 * Returns 0 on success or if called with an aborted transaction
 * Returns <0 on error and aborts the transaction
2450 2451 2452 2453 2454 2455 2456 2457 2458
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct list_head cluster;
	int ret;
2459
	u64 delayed_start;
2460 2461
	int run_all = count == (unsigned long)-1;
	int run_most = 0;
2462
	int loops;
2463

2464 2465 2466 2467
	/* We'll clean this up in btrfs_cleanup_transaction */
	if (trans->aborted)
		return 0;

2468 2469 2470
	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

2471 2472
	btrfs_delayed_refs_qgroup_accounting(trans, root->fs_info);

2473 2474 2475
	delayed_refs = &trans->transaction->delayed_refs;
	INIT_LIST_HEAD(&cluster);
again:
2476
	loops = 0;
2477
	spin_lock(&delayed_refs->lock);
2478

2479 2480 2481 2482
#ifdef SCRAMBLE_DELAYED_REFS
	delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
#endif

2483 2484 2485 2486 2487 2488 2489 2490
	if (count == 0) {
		count = delayed_refs->num_entries * 2;
		run_most = 1;
	}
	while (1) {
		if (!(run_all || run_most) &&
		    delayed_refs->num_heads_ready < 64)
			break;
2491

2492
		/*
2493 2494 2495 2496
		 * go find something we can process in the rbtree.  We start at
		 * the beginning of the tree, and then build a cluster
		 * of refs to process starting at the first one we are able to
		 * lock
2497
		 */
2498
		delayed_start = delayed_refs->run_delayed_start;
2499 2500 2501
		ret = btrfs_find_ref_cluster(trans, &cluster,
					     delayed_refs->run_delayed_start);
		if (ret)
2502 2503
			break;

2504
		ret = run_clustered_refs(trans, root, &cluster);
2505
		if (ret < 0) {
2506
			btrfs_release_ref_cluster(&cluster);
2507 2508 2509 2510
			spin_unlock(&delayed_refs->lock);
			btrfs_abort_transaction(trans, root, ret);
			return ret;
		}
2511 2512 2513 2514 2515

		count -= min_t(unsigned long, ret, count);

		if (count == 0)
			break;
2516

2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534
		if (delayed_start >= delayed_refs->run_delayed_start) {
			if (loops == 0) {
				/*
				 * btrfs_find_ref_cluster looped. let's do one
				 * more cycle. if we don't run any delayed ref
				 * during that cycle (because we can't because
				 * all of them are blocked), bail out.
				 */
				loops = 1;
			} else {
				/*
				 * no runnable refs left, stop trying
				 */
				BUG_ON(run_all);
				break;
			}
		}
		if (ret) {
2535
			/* refs were run, let's reset staleness detection */
2536
			loops = 0;
2537
		}
2538
	}
2539

2540
	if (run_all) {
2541 2542 2543 2544 2545 2546
		if (!list_empty(&trans->new_bgs)) {
			spin_unlock(&delayed_refs->lock);
			btrfs_create_pending_block_groups(trans, root);
			spin_lock(&delayed_refs->lock);
		}

2547
		node = rb_first(&delayed_refs->root);
2548
		if (!node)
2549
			goto out;
2550
		count = (unsigned long)-1;
2551

2552 2553 2554 2555 2556
		while (node) {
			ref = rb_entry(node, struct btrfs_delayed_ref_node,
				       rb_node);
			if (btrfs_delayed_ref_is_head(ref)) {
				struct btrfs_delayed_ref_head *head;
2557

2558 2559 2560 2561
				head = btrfs_delayed_node_to_head(ref);
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2562 2563 2564 2565
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
2566 2567 2568 2569
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2570
				cond_resched();
2571 2572 2573 2574 2575 2576 2577
				goto again;
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
		schedule_timeout(1);
		goto again;
2578
	}
2579
out:
2580
	spin_unlock(&delayed_refs->lock);
2581
	assert_qgroups_uptodate(trans);
2582 2583 2584
	return 0;
}

2585 2586 2587 2588 2589 2590 2591 2592
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
				int is_data)
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

2593
	extent_op = btrfs_alloc_delayed_extent_op();
2594 2595 2596 2597 2598 2599 2600 2601
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;

A
Arne Jansen 已提交
2602 2603
	ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
					  num_bytes, extent_op);
2604
	if (ret)
2605
		btrfs_free_delayed_extent_op(extent_op);
2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	ret = -ENOENT;
	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

2632
		btrfs_release_path(path);
2633

2634 2635 2636 2637
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out_unlock;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	if (ref->bytenr != bytenr)
		goto out_unlock;

	ret = 1;
	if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
		goto out_unlock;

	data_ref = btrfs_delayed_node_to_data_ref(ref);

	node = rb_prev(node);
	if (node) {
2661 2662
		int seq = ref->seq;

2663
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2664
		if (ref->bytenr == bytenr && ref->seq == seq)
2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683
			goto out_unlock;
	}

	if (data_ref->root != root->root_key.objectid ||
	    data_ref->objectid != objectid || data_ref->offset != offset)
		goto out_unlock;

	ret = 0;
out_unlock:
	mutex_unlock(&head->mutex);
out:
	spin_unlock(&delayed_refs->lock);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2684 2685
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2686
	struct extent_buffer *leaf;
2687 2688 2689
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2690
	struct btrfs_key key;
2691
	u32 item_size;
2692
	int ret;
2693

2694
	key.objectid = bytenr;
Z
Zheng Yan 已提交
2695
	key.offset = (u64)-1;
2696
	key.type = BTRFS_EXTENT_ITEM_KEY;
2697 2698 2699 2700

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
2701
	BUG_ON(ret == 0); /* Corruption */
Y
Yan Zheng 已提交
2702 2703 2704

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
2705
		goto out;
2706

Z
Zheng Yan 已提交
2707
	path->slots[0]--;
2708
	leaf = path->nodes[0];
2709
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2710

2711
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2712
		goto out;
2713

2714 2715 2716 2717 2718 2719 2720 2721 2722
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2723

2724 2725 2726
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2727

2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
2767
			goto out;
Y
Yan Zheng 已提交
2768

2769 2770 2771 2772 2773 2774 2775
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
2776
	}
2777 2778 2779

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
2780
out:
Y
Yan Zheng 已提交
2781
	btrfs_free_path(path);
2782 2783
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
2784
	return ret;
2785
}
C
Chris Mason 已提交
2786

2787
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2788
			   struct btrfs_root *root,
2789
			   struct extent_buffer *buf,
A
Arne Jansen 已提交
2790
			   int full_backref, int inc, int for_cow)
Z
Zheng Yan 已提交
2791 2792
{
	u64 bytenr;
2793 2794
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
2795 2796 2797 2798 2799 2800 2801 2802
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
A
Arne Jansen 已提交
2803
			    u64, u64, u64, u64, u64, u64, int);
Z
Zheng Yan 已提交
2804 2805 2806 2807 2808

	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

2809 2810
	if (!root->ref_cows && level == 0)
		return 0;
Z
Zheng Yan 已提交
2811

2812 2813 2814 2815
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
2816

2817 2818 2819 2820 2821 2822
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
2823
		if (level == 0) {
2824
			btrfs_item_key_to_cpu(buf, &key, i);
Z
Zheng Yan 已提交
2825 2826
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
2827
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
2828 2829 2830 2831 2832 2833 2834
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
2835 2836 2837 2838 2839

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
A
Arne Jansen 已提交
2840
					   key.offset, for_cow);
Z
Zheng Yan 已提交
2841 2842 2843
			if (ret)
				goto fail;
		} else {
2844 2845 2846
			bytenr = btrfs_node_blockptr(buf, i);
			num_bytes = btrfs_level_size(root, level - 1);
			ret = process_func(trans, root, bytenr, num_bytes,
A
Arne Jansen 已提交
2847 2848
					   parent, ref_root, level - 1, 0,
					   for_cow);
Z
Zheng Yan 已提交
2849 2850 2851 2852 2853 2854
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
2855 2856 2857 2858
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
A
Arne Jansen 已提交
2859
		  struct extent_buffer *buf, int full_backref, int for_cow)
2860
{
A
Arne Jansen 已提交
2861
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1, for_cow);
2862 2863 2864
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
A
Arne Jansen 已提交
2865
		  struct extent_buffer *buf, int full_backref, int for_cow)
2866
{
A
Arne Jansen 已提交
2867
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0, for_cow);
Z
Zheng Yan 已提交
2868 2869
}

C
Chris Mason 已提交
2870 2871 2872 2873 2874 2875 2876
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2877 2878
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
2879 2880

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2881 2882
	if (ret < 0)
		goto fail;
2883
	BUG_ON(ret); /* Corruption */
2884 2885 2886 2887 2888

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
2889
	btrfs_release_path(path);
2890
fail:
2891 2892
	if (ret) {
		btrfs_abort_transaction(trans, root, ret);
C
Chris Mason 已提交
2893
		return ret;
2894
	}
C
Chris Mason 已提交
2895 2896 2897 2898
	return 0;

}

2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
	spin_lock(&root->fs_info->block_group_cache_lock);
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
2910
		btrfs_get_block_group(cache);
2911 2912 2913 2914 2915 2916
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

2917 2918 2919 2920 2921 2922 2923
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
2924
	int dcs = BTRFS_DC_ERROR;
2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943
	int num_pages = 0;
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
2944
		btrfs_release_path(path);
2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

2961 2962 2963 2964 2965 2966 2967
	/* We've already setup this transaction, go ahead and exit */
	if (block_group->cache_generation == trans->transid &&
	    i_size_read(inode)) {
		dcs = BTRFS_DC_SETUP;
		goto out_put;
	}

2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
		ret = btrfs_truncate_free_space_cache(root, trans, path,
						      inode);
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
2985 2986 2987 2988 2989 2990 2991
	if (block_group->cached != BTRFS_CACHE_FINISHED ||
	    !btrfs_test_opt(root, SPACE_CACHE)) {
		/*
		 * don't bother trying to write stuff out _if_
		 * a) we're not cached,
		 * b) we're with nospace_cache mount option.
		 */
2992
		dcs = BTRFS_DC_WRITTEN;
2993 2994 2995 2996 2997
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

2998 2999 3000 3001 3002 3003 3004
	/*
	 * Try to preallocate enough space based on how big the block group is.
	 * Keep in mind this has to include any pinned space which could end up
	 * taking up quite a bit since it's not folded into the other space
	 * cache.
	 */
	num_pages = (int)div64_u64(block_group->key.offset, 256 * 1024 * 1024);
3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017
	if (!num_pages)
		num_pages = 1;

	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
3018 3019
	if (!ret)
		dcs = BTRFS_DC_SETUP;
3020
	btrfs_free_reserved_data_space(inode, num_pages);
3021

3022 3023 3024
out_put:
	iput(inode);
out_free:
3025
	btrfs_release_path(path);
3026 3027
out:
	spin_lock(&block_group->lock);
3028
	if (!ret && dcs == BTRFS_DC_SETUP)
3029
		block_group->cache_generation = trans->transid;
3030
	block_group->disk_cache_state = dcs;
3031 3032 3033 3034 3035
	spin_unlock(&block_group->lock);

	return ret;
}

3036 3037
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
C
Chris Mason 已提交
3038
{
3039
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
3040 3041
	int err = 0;
	struct btrfs_path *path;
3042
	u64 last = 0;
C
Chris Mason 已提交
3043 3044 3045 3046 3047

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066
again:
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			if (cache->disk_cache_state == BTRFS_DC_CLEAR)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
		err = cache_save_setup(cache, trans, path);
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
3067
	while (1) {
3068 3069 3070
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3071 3072
			if (err) /* File system offline */
				goto out;
J
Josef Bacik 已提交
3073
		}
3074

3075 3076
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
3077 3078 3079 3080 3081
			if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
				btrfs_put_block_group(cache);
				goto again;
			}

3082 3083 3084 3085 3086 3087 3088 3089 3090 3091
			if (cache->dirty)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
J
Josef Bacik 已提交
3092

J
Josef Bacik 已提交
3093 3094
		if (cache->disk_cache_state == BTRFS_DC_SETUP)
			cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
3095
		cache->dirty = 0;
3096
		last = cache->key.objectid + cache->key.offset;
J
Josef Bacik 已提交
3097

3098
		err = write_one_cache_group(trans, root, path, cache);
3099 3100 3101
		if (err) /* File system offline */
			goto out;

3102
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
3103
	}
3104

J
Josef Bacik 已提交
3105 3106 3107 3108 3109 3110 3111 3112 3113
	while (1) {
		/*
		 * I don't think this is needed since we're just marking our
		 * preallocated extent as written, but just in case it can't
		 * hurt.
		 */
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3114 3115
			if (err) /* File system offline */
				goto out;
J
Josef Bacik 已提交
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139
		}

		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			/*
			 * Really this shouldn't happen, but it could if we
			 * couldn't write the entire preallocated extent and
			 * splitting the extent resulted in a new block.
			 */
			if (cache->dirty) {
				btrfs_put_block_group(cache);
				goto again;
			}
			if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

3140
		err = btrfs_write_out_cache(root, trans, cache, path);
J
Josef Bacik 已提交
3141 3142 3143 3144 3145

		/*
		 * If we didn't have an error then the cache state is still
		 * NEED_WRITE, so we can set it to WRITTEN.
		 */
3146
		if (!err && cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
J
Josef Bacik 已提交
3147 3148 3149 3150
			cache->disk_cache_state = BTRFS_DC_WRITTEN;
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}
3151
out:
J
Josef Bacik 已提交
3152

C
Chris Mason 已提交
3153
	btrfs_free_path(path);
3154
	return err;
C
Chris Mason 已提交
3155 3156
}

3157 3158 3159 3160 3161 3162 3163 3164 3165
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
3166
		btrfs_put_block_group(block_group);
3167 3168 3169
	return readonly;
}

3170 3171 3172 3173 3174
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
3175 3176 3177 3178 3179 3180 3181 3182
	int i;
	int factor;

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
3183 3184 3185

	found = __find_space_info(info, flags);
	if (found) {
3186
		spin_lock(&found->lock);
3187
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
3188
		found->disk_total += total_bytes * factor;
3189
		found->bytes_used += bytes_used;
3190
		found->disk_used += bytes_used * factor;
3191
		found->full = 0;
3192
		spin_unlock(&found->lock);
3193 3194 3195
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
3196
	found = kzalloc(sizeof(*found), GFP_NOFS);
3197 3198 3199
	if (!found)
		return -ENOMEM;

3200 3201
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		INIT_LIST_HEAD(&found->block_groups[i]);
3202
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
3203
	spin_lock_init(&found->lock);
3204
	found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3205
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
3206
	found->disk_total = total_bytes * factor;
3207
	found->bytes_used = bytes_used;
3208
	found->disk_used = bytes_used * factor;
3209
	found->bytes_pinned = 0;
3210
	found->bytes_reserved = 0;
Y
Yan Zheng 已提交
3211
	found->bytes_readonly = 0;
3212
	found->bytes_may_use = 0;
3213
	found->full = 0;
3214
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3215
	found->chunk_alloc = 0;
3216 3217
	found->flush = 0;
	init_waitqueue_head(&found->wait);
3218
	*space_info = found;
3219
	list_add_rcu(&found->list, &info->space_info);
3220 3221
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		info->data_sinfo = found;
3222 3223 3224
	return 0;
}

3225 3226
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
3227 3228
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
3229

3230
	write_seqlock(&fs_info->profiles_lock);
3231 3232 3233 3234 3235 3236
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits |= extra_flags;
3237
	write_sequnlock(&fs_info->profiles_lock);
3238
}
3239

3240 3241 3242
/*
 * returns target flags in extended format or 0 if restripe for this
 * chunk_type is not in progress
3243 3244
 *
 * should be called with either volume_mutex or balance_lock held
3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267
 */
static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
{
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
	u64 target = 0;

	if (!bctl)
		return 0;

	if (flags & BTRFS_BLOCK_GROUP_DATA &&
	    bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
	} else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
		   bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
	} else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
		   bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
	}

	return target;
}

3268 3269 3270
/*
 * @flags: available profiles in extended format (see ctree.h)
 *
3271 3272 3273
 * Returns reduced profile in chunk format.  If profile changing is in
 * progress (either running or paused) picks the target profile (if it's
 * already available), otherwise falls back to plain reducing.
3274
 */
Y
Yan Zheng 已提交
3275
u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3276
{
3277 3278 3279 3280 3281 3282 3283
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	u64 num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;
3284
	u64 target;
3285

3286 3287 3288 3289
	/*
	 * see if restripe for this chunk_type is in progress, if so
	 * try to reduce to the target profile
	 */
3290
	spin_lock(&root->fs_info->balance_lock);
3291 3292 3293 3294
	target = get_restripe_target(root->fs_info, flags);
	if (target) {
		/* pick target profile only if it's already available */
		if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
3295
			spin_unlock(&root->fs_info->balance_lock);
3296
			return extended_to_chunk(target);
3297 3298 3299 3300
		}
	}
	spin_unlock(&root->fs_info->balance_lock);

3301 3302 3303 3304 3305
	if (num_devices == 1)
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

3306 3307
	if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
	    (flags & (BTRFS_BLOCK_GROUP_RAID1 |
3308
		      BTRFS_BLOCK_GROUP_RAID10))) {
3309
		flags &= ~BTRFS_BLOCK_GROUP_DUP;
3310
	}
3311 3312

	if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
3313
	    (flags & BTRFS_BLOCK_GROUP_RAID10)) {
3314
		flags &= ~BTRFS_BLOCK_GROUP_RAID1;
3315
	}
3316 3317 3318 3319

	if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
	    ((flags & BTRFS_BLOCK_GROUP_RAID1) |
	     (flags & BTRFS_BLOCK_GROUP_RAID10) |
3320
	     (flags & BTRFS_BLOCK_GROUP_DUP))) {
3321
		flags &= ~BTRFS_BLOCK_GROUP_RAID0;
3322 3323
	}

3324
	return extended_to_chunk(flags);
3325 3326
}

3327
static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
J
Josef Bacik 已提交
3328
{
3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340
	unsigned seq;

	do {
		seq = read_seqbegin(&root->fs_info->profiles_lock);

		if (flags & BTRFS_BLOCK_GROUP_DATA)
			flags |= root->fs_info->avail_data_alloc_bits;
		else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
			flags |= root->fs_info->avail_system_alloc_bits;
		else if (flags & BTRFS_BLOCK_GROUP_METADATA)
			flags |= root->fs_info->avail_metadata_alloc_bits;
	} while (read_seqretry(&root->fs_info->profiles_lock, seq));
3341

3342
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
3343 3344
}

3345
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
3346
{
3347
	u64 flags;
J
Josef Bacik 已提交
3348

3349 3350 3351 3352
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
3353
	else
3354
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
3355

3356
	return get_alloc_profile(root, flags);
J
Josef Bacik 已提交
3357
}
J
Josef Bacik 已提交
3358

J
Josef Bacik 已提交
3359 3360 3361 3362
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3363
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
3364 3365
{
	struct btrfs_space_info *data_sinfo;
3366
	struct btrfs_root *root = BTRFS_I(inode)->root;
3367
	struct btrfs_fs_info *fs_info = root->fs_info;
3368
	u64 used;
3369
	int ret = 0, committed = 0, alloc_chunk = 1;
J
Josef Bacik 已提交
3370 3371 3372 3373

	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);

3374 3375
	if (root == root->fs_info->tree_root ||
	    BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3376 3377 3378 3379
		alloc_chunk = 0;
		committed = 1;
	}

3380
	data_sinfo = fs_info->data_sinfo;
C
Chris Mason 已提交
3381 3382
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
3383

J
Josef Bacik 已提交
3384 3385 3386
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3387 3388 3389
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3390 3391

	if (used + bytes > data_sinfo->total_bytes) {
3392
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3393

J
Josef Bacik 已提交
3394 3395 3396 3397
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3398
		if (!data_sinfo->full && alloc_chunk) {
J
Josef Bacik 已提交
3399
			u64 alloc_target;
J
Josef Bacik 已提交
3400

3401
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
J
Josef Bacik 已提交
3402
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
3403
alloc:
J
Josef Bacik 已提交
3404
			alloc_target = btrfs_get_alloc_profile(root, 1);
3405
			trans = btrfs_join_transaction(root);
3406 3407
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
3408

J
Josef Bacik 已提交
3409
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
3410 3411
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
J
Josef Bacik 已提交
3412
			btrfs_end_transaction(trans, root);
3413 3414 3415 3416 3417 3418
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
J
Josef Bacik 已提交
3419

3420 3421 3422
			if (!data_sinfo)
				data_sinfo = fs_info->data_sinfo;

J
Josef Bacik 已提交
3423 3424
			goto again;
		}
3425 3426 3427 3428 3429 3430 3431

		/*
		 * If we have less pinned bytes than we want to allocate then
		 * don't bother committing the transaction, it won't help us.
		 */
		if (data_sinfo->bytes_pinned < bytes)
			committed = 1;
J
Josef Bacik 已提交
3432 3433
		spin_unlock(&data_sinfo->lock);

3434
		/* commit the current transaction and try again */
3435
commit_trans:
J
Josef Bacik 已提交
3436 3437
		if (!committed &&
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
3438
			committed = 1;
3439
			trans = btrfs_join_transaction(root);
3440 3441
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3442 3443 3444 3445 3446
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
J
Josef Bacik 已提交
3447

J
Josef Bacik 已提交
3448 3449 3450
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
J
Josef Bacik 已提交
3451
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3452
				      data_sinfo->flags, bytes, 1);
J
Josef Bacik 已提交
3453 3454
	spin_unlock(&data_sinfo->lock);

J
Josef Bacik 已提交
3455 3456
	return 0;
}
J
Josef Bacik 已提交
3457 3458

/*
3459
 * Called if we need to clear a data reservation for this inode.
J
Josef Bacik 已提交
3460
 */
3461
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3462
{
3463
	struct btrfs_root *root = BTRFS_I(inode)->root;
J
Josef Bacik 已提交
3464
	struct btrfs_space_info *data_sinfo;
3465

J
Josef Bacik 已提交
3466 3467
	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3468

3469
	data_sinfo = root->fs_info->data_sinfo;
J
Josef Bacik 已提交
3470 3471
	spin_lock(&data_sinfo->lock);
	data_sinfo->bytes_may_use -= bytes;
J
Josef Bacik 已提交
3472
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3473
				      data_sinfo->flags, bytes, 0);
J
Josef Bacik 已提交
3474
	spin_unlock(&data_sinfo->lock);
3475 3476
}

3477
static void force_metadata_allocation(struct btrfs_fs_info *info)
3478
{
3479 3480
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3481

3482 3483 3484
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3485
			found->force_alloc = CHUNK_ALLOC_FORCE;
3486
	}
3487
	rcu_read_unlock();
3488 3489
}

3490
static int should_alloc_chunk(struct btrfs_root *root,
3491
			      struct btrfs_space_info *sinfo, int force)
3492
{
3493
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3494
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3495
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3496
	u64 thresh;
3497

3498 3499 3500
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

3501 3502 3503 3504 3505
	/*
	 * We need to take into account the global rsv because for all intents
	 * and purposes it's used space.  Don't worry about locking the
	 * global_rsv, it doesn't change except when the transaction commits.
	 */
3506 3507
	if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)
		num_allocated += global_rsv->size;
3508

3509 3510 3511 3512 3513
	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
3514
		thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3515 3516 3517 3518 3519 3520 3521
		thresh = max_t(u64, 64 * 1024 * 1024,
			       div_factor_fine(thresh, 1));

		if (num_bytes - num_allocated < thresh)
			return 1;
	}

3522
	if (num_allocated + 2 * 1024 * 1024 < div_factor(num_bytes, 8))
3523
		return 0;
3524
	return 1;
3525 3526
}

3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570
static u64 get_system_chunk_thresh(struct btrfs_root *root, u64 type)
{
	u64 num_dev;

	if (type & BTRFS_BLOCK_GROUP_RAID10 ||
	    type & BTRFS_BLOCK_GROUP_RAID0)
		num_dev = root->fs_info->fs_devices->rw_devices;
	else if (type & BTRFS_BLOCK_GROUP_RAID1)
		num_dev = 2;
	else
		num_dev = 1;	/* DUP or single */

	/* metadata for updaing devices and chunk tree */
	return btrfs_calc_trans_metadata_size(root, num_dev + 1);
}

static void check_system_chunk(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root, u64 type)
{
	struct btrfs_space_info *info;
	u64 left;
	u64 thresh;

	info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	spin_lock(&info->lock);
	left = info->total_bytes - info->bytes_used - info->bytes_pinned -
		info->bytes_reserved - info->bytes_readonly;
	spin_unlock(&info->lock);

	thresh = get_system_chunk_thresh(root, type);
	if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
		printk(KERN_INFO "left=%llu, need=%llu, flags=%llu\n",
		       left, thresh, type);
		dump_space_info(info, 0, 0);
	}

	if (left < thresh) {
		u64 flags;

		flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
		btrfs_alloc_chunk(trans, root, flags);
	}
}

3571
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
3572
			  struct btrfs_root *extent_root, u64 flags, int force)
J
Josef Bacik 已提交
3573
{
3574
	struct btrfs_space_info *space_info;
3575
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
3576
	int wait_for_alloc = 0;
J
Josef Bacik 已提交
3577 3578
	int ret = 0;

3579 3580 3581 3582
	/* Don't re-enter if we're already allocating a chunk */
	if (trans->allocating_chunk)
		return -ENOSPC;

3583
	space_info = __find_space_info(extent_root->fs_info, flags);
3584 3585 3586
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
3587
		BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
3588
	}
3589
	BUG_ON(!space_info); /* Logic error */
J
Josef Bacik 已提交
3590

3591
again:
3592
	spin_lock(&space_info->lock);
3593
	if (force < space_info->force_alloc)
3594
		force = space_info->force_alloc;
3595 3596
	if (space_info->full) {
		spin_unlock(&space_info->lock);
3597
		return 0;
J
Josef Bacik 已提交
3598 3599
	}

3600
	if (!should_alloc_chunk(extent_root, space_info, force)) {
3601
		spin_unlock(&space_info->lock);
3602 3603 3604 3605 3606
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
J
Josef Bacik 已提交
3607
	}
3608

3609
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3610

3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

3625 3626
	trans->allocating_chunk = true;

3627 3628 3629 3630 3631 3632 3633
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3634 3635 3636 3637 3638
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
3639
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3640 3641 3642 3643
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
3644 3645
	}

3646 3647 3648 3649 3650 3651
	/*
	 * Check if we have enough space in SYSTEM chunk because we may need
	 * to update devices.
	 */
	check_system_chunk(trans, extent_root, flags);

Y
Yan Zheng 已提交
3652
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
3653
	trans->allocating_chunk = false;
3654 3655 3656
	if (ret < 0 && ret != -ENOSPC)
		goto out;

J
Josef Bacik 已提交
3657 3658
	spin_lock(&space_info->lock);
	if (ret)
3659
		space_info->full = 1;
3660 3661
	else
		ret = 1;
3662

3663
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3664
	space_info->chunk_alloc = 0;
J
Josef Bacik 已提交
3665
	spin_unlock(&space_info->lock);
3666
out:
3667
	mutex_unlock(&fs_info->chunk_mutex);
J
Josef Bacik 已提交
3668
	return ret;
3669
}
J
Josef Bacik 已提交
3670

J
Josef Bacik 已提交
3671 3672
static int can_overcommit(struct btrfs_root *root,
			  struct btrfs_space_info *space_info, u64 bytes,
M
Miao Xie 已提交
3673
			  enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
3674
{
3675
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
J
Josef Bacik 已提交
3676
	u64 profile = btrfs_get_alloc_profile(root, 0);
3677
	u64 rsv_size = 0;
J
Josef Bacik 已提交
3678 3679 3680 3681
	u64 avail;
	u64 used;

	used = space_info->bytes_used + space_info->bytes_reserved +
3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698
		space_info->bytes_pinned + space_info->bytes_readonly;

	spin_lock(&global_rsv->lock);
	rsv_size = global_rsv->size;
	spin_unlock(&global_rsv->lock);

	/*
	 * We only want to allow over committing if we have lots of actual space
	 * free, but if we don't have enough space to handle the global reserve
	 * space then we could end up having a real enospc problem when trying
	 * to allocate a chunk or some other such important allocation.
	 */
	rsv_size <<= 1;
	if (used + rsv_size >= space_info->total_bytes)
		return 0;

	used += space_info->bytes_may_use;
J
Josef Bacik 已提交
3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713

	spin_lock(&root->fs_info->free_chunk_lock);
	avail = root->fs_info->free_chunk_space;
	spin_unlock(&root->fs_info->free_chunk_lock);

	/*
	 * If we have dup, raid1 or raid10 then only half of the free
	 * space is actually useable.
	 */
	if (profile & (BTRFS_BLOCK_GROUP_DUP |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_RAID10))
		avail >>= 1;

	/*
3714 3715 3716
	 * If we aren't flushing all things, let us overcommit up to
	 * 1/2th of the space. If we can flush, don't let us overcommit
	 * too much, let it overcommit up to 1/8 of the space.
J
Josef Bacik 已提交
3717
	 */
M
Miao Xie 已提交
3718
	if (flush == BTRFS_RESERVE_FLUSH_ALL)
J
Josef Bacik 已提交
3719 3720 3721 3722 3723 3724 3725 3726 3727
		avail >>= 3;
	else
		avail >>= 1;

	if (used + bytes < space_info->total_bytes + avail)
		return 1;
	return 0;
}

3728 3729 3730
static inline int writeback_inodes_sb_nr_if_idle_safe(struct super_block *sb,
						      unsigned long nr_pages,
						      enum wb_reason reason)
3731
{
3732 3733 3734 3735 3736
	/* the flusher is dealing with the dirty inodes now. */
	if (writeback_in_progress(sb->s_bdi))
		return 1;

	if (down_read_trylock(&sb->s_umount)) {
3737 3738 3739 3740 3741 3742 3743 3744
		writeback_inodes_sb_nr(sb, nr_pages, reason);
		up_read(&sb->s_umount);
		return 1;
	}

	return 0;
}

3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766
void btrfs_writeback_inodes_sb_nr(struct btrfs_root *root,
				  unsigned long nr_pages)
{
	struct super_block *sb = root->fs_info->sb;
	int started;

	/* If we can not start writeback, just sync all the delalloc file. */
	started = writeback_inodes_sb_nr_if_idle_safe(sb, nr_pages,
						      WB_REASON_FS_FREE_SPACE);
	if (!started) {
		/*
		 * We needn't worry the filesystem going from r/w to r/o though
		 * we don't acquire ->s_umount mutex, because the filesystem
		 * should guarantee the delalloc inodes list be empty after
		 * the filesystem is readonly(all dirty pages are written to
		 * the disk).
		 */
		btrfs_start_delalloc_inodes(root, 0);
		btrfs_wait_ordered_extents(root, 0);
	}
}

J
Josef Bacik 已提交
3767
/*
3768
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
3769
 */
J
Josef Bacik 已提交
3770 3771
static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig,
			    bool wait_ordered)
3772
{
3773
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
3774
	struct btrfs_space_info *space_info;
3775
	struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3776
	u64 delalloc_bytes;
3777
	u64 max_reclaim;
3778
	long time_left;
3779
	unsigned long nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3780
	int loops = 0;
M
Miao Xie 已提交
3781
	enum btrfs_reserve_flush_enum flush;
3782

3783
	trans = (struct btrfs_trans_handle *)current->journal_info;
3784
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
3785
	space_info = block_rsv->space_info;
3786 3787

	smp_mb();
3788 3789
	delalloc_bytes = percpu_counter_sum_positive(
						&root->fs_info->delalloc_bytes);
J
Josef Bacik 已提交
3790
	if (delalloc_bytes == 0) {
3791
		if (trans)
J
Josef Bacik 已提交
3792
			return;
3793
		btrfs_wait_ordered_extents(root, 0);
J
Josef Bacik 已提交
3794
		return;
3795 3796
	}

J
Josef Bacik 已提交
3797 3798 3799
	while (delalloc_bytes && loops < 3) {
		max_reclaim = min(delalloc_bytes, to_reclaim);
		nr_pages = max_reclaim >> PAGE_CACHE_SHIFT;
3800
		btrfs_writeback_inodes_sb_nr(root, nr_pages);
3801 3802 3803 3804 3805 3806 3807
		/*
		 * We need to wait for the async pages to actually start before
		 * we do anything.
		 */
		wait_event(root->fs_info->async_submit_wait,
			   !atomic_read(&root->fs_info->async_delalloc_pages));

M
Miao Xie 已提交
3808 3809 3810 3811
		if (!trans)
			flush = BTRFS_RESERVE_FLUSH_ALL;
		else
			flush = BTRFS_RESERVE_NO_FLUSH;
J
Josef Bacik 已提交
3812
		spin_lock(&space_info->lock);
M
Miao Xie 已提交
3813
		if (can_overcommit(root, space_info, orig, flush)) {
J
Josef Bacik 已提交
3814 3815 3816
			spin_unlock(&space_info->lock);
			break;
		}
J
Josef Bacik 已提交
3817
		spin_unlock(&space_info->lock);
3818

3819
		loops++;
3820
		if (wait_ordered && !trans) {
3821
			btrfs_wait_ordered_extents(root, 0);
3822
		} else {
J
Josef Bacik 已提交
3823
			time_left = schedule_timeout_killable(1);
3824 3825 3826
			if (time_left)
				break;
		}
J
Josef Bacik 已提交
3827
		smp_mb();
3828 3829
		delalloc_bytes = percpu_counter_sum_positive(
						&root->fs_info->delalloc_bytes);
3830 3831 3832
	}
}

3833 3834 3835 3836 3837
/**
 * maybe_commit_transaction - possibly commit the transaction if its ok to
 * @root - the root we're allocating for
 * @bytes - the number of bytes we want to reserve
 * @force - force the commit
3838
 *
3839 3840 3841
 * This will check to make sure that committing the transaction will actually
 * get us somewhere and then commit the transaction if it does.  Otherwise it
 * will return -ENOSPC.
3842
 */
3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871
static int may_commit_transaction(struct btrfs_root *root,
				  struct btrfs_space_info *space_info,
				  u64 bytes, int force)
{
	struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
	struct btrfs_trans_handle *trans;

	trans = (struct btrfs_trans_handle *)current->journal_info;
	if (trans)
		return -EAGAIN;

	if (force)
		goto commit;

	/* See if there is enough pinned space to make this reservation */
	spin_lock(&space_info->lock);
	if (space_info->bytes_pinned >= bytes) {
		spin_unlock(&space_info->lock);
		goto commit;
	}
	spin_unlock(&space_info->lock);

	/*
	 * See if there is some space in the delayed insertion reservation for
	 * this reservation.
	 */
	if (space_info != delayed_rsv->space_info)
		return -ENOSPC;

L
Liu Bo 已提交
3872
	spin_lock(&space_info->lock);
3873
	spin_lock(&delayed_rsv->lock);
L
Liu Bo 已提交
3874
	if (space_info->bytes_pinned + delayed_rsv->size < bytes) {
3875
		spin_unlock(&delayed_rsv->lock);
L
Liu Bo 已提交
3876
		spin_unlock(&space_info->lock);
3877 3878 3879
		return -ENOSPC;
	}
	spin_unlock(&delayed_rsv->lock);
L
Liu Bo 已提交
3880
	spin_unlock(&space_info->lock);
3881 3882 3883 3884 3885 3886 3887 3888 3889

commit:
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans))
		return -ENOSPC;

	return btrfs_commit_transaction(trans, root);
}

3890
enum flush_state {
3891 3892 3893 3894
	FLUSH_DELAYED_ITEMS_NR	=	1,
	FLUSH_DELAYED_ITEMS	=	2,
	FLUSH_DELALLOC		=	3,
	FLUSH_DELALLOC_WAIT	=	4,
3895 3896
	ALLOC_CHUNK		=	5,
	COMMIT_TRANS		=	6,
3897 3898 3899 3900 3901 3902 3903 3904
};

static int flush_space(struct btrfs_root *root,
		       struct btrfs_space_info *space_info, u64 num_bytes,
		       u64 orig_bytes, int state)
{
	struct btrfs_trans_handle *trans;
	int nr;
J
Josef Bacik 已提交
3905
	int ret = 0;
3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927

	switch (state) {
	case FLUSH_DELAYED_ITEMS_NR:
	case FLUSH_DELAYED_ITEMS:
		if (state == FLUSH_DELAYED_ITEMS_NR) {
			u64 bytes = btrfs_calc_trans_metadata_size(root, 1);

			nr = (int)div64_u64(num_bytes, bytes);
			if (!nr)
				nr = 1;
			nr *= 2;
		} else {
			nr = -1;
		}
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = btrfs_run_delayed_items_nr(trans, root, nr);
		btrfs_end_transaction(trans, root);
		break;
3928 3929 3930 3931 3932
	case FLUSH_DELALLOC:
	case FLUSH_DELALLOC_WAIT:
		shrink_delalloc(root, num_bytes, orig_bytes,
				state == FLUSH_DELALLOC_WAIT);
		break;
3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945
	case ALLOC_CHUNK:
		trans = btrfs_join_transaction(root);
		if (IS_ERR(trans)) {
			ret = PTR_ERR(trans);
			break;
		}
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
				     btrfs_get_alloc_profile(root, 0),
				     CHUNK_ALLOC_NO_FORCE);
		btrfs_end_transaction(trans, root);
		if (ret == -ENOSPC)
			ret = 0;
		break;
3946 3947 3948 3949 3950 3951 3952 3953 3954 3955
	case COMMIT_TRANS:
		ret = may_commit_transaction(root, space_info, orig_bytes, 0);
		break;
	default:
		ret = -ENOSPC;
		break;
	}

	return ret;
}
3956 3957 3958 3959 3960 3961
/**
 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
 * @root - the root we're allocating for
 * @block_rsv - the block_rsv we're allocating for
 * @orig_bytes - the number of bytes we want
 * @flush - wether or not we can flush to make our reservation
3962
 *
3963 3964 3965 3966 3967 3968
 * This will reserve orgi_bytes number of bytes from the space info associated
 * with the block_rsv.  If there is not enough space it will make an attempt to
 * flush out space to make room.  It will do this by flushing delalloc if
 * possible or committing the transaction.  If flush is 0 then no attempts to
 * regain reservations will be made and this will fail if there is not enough
 * space already.
3969
 */
3970
static int reserve_metadata_bytes(struct btrfs_root *root,
3971
				  struct btrfs_block_rsv *block_rsv,
M
Miao Xie 已提交
3972 3973
				  u64 orig_bytes,
				  enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
3974
{
3975
	struct btrfs_space_info *space_info = block_rsv->space_info;
3976
	u64 used;
3977
	u64 num_bytes = orig_bytes;
3978
	int flush_state = FLUSH_DELAYED_ITEMS_NR;
3979
	int ret = 0;
3980
	bool flushing = false;
J
Josef Bacik 已提交
3981

3982
again:
3983
	ret = 0;
3984
	spin_lock(&space_info->lock);
3985
	/*
M
Miao Xie 已提交
3986 3987
	 * We only want to wait if somebody other than us is flushing and we
	 * are actually allowed to flush all things.
3988
	 */
M
Miao Xie 已提交
3989 3990
	while (flush == BTRFS_RESERVE_FLUSH_ALL && !flushing &&
	       space_info->flush) {
3991 3992 3993 3994 3995 3996 3997
		spin_unlock(&space_info->lock);
		/*
		 * If we have a trans handle we can't wait because the flusher
		 * may have to commit the transaction, which would mean we would
		 * deadlock since we are waiting for the flusher to finish, but
		 * hold the current transaction open.
		 */
3998
		if (current->journal_info)
3999
			return -EAGAIN;
A
Arne Jansen 已提交
4000 4001 4002
		ret = wait_event_killable(space_info->wait, !space_info->flush);
		/* Must have been killed, return */
		if (ret)
4003 4004 4005 4006 4007 4008
			return -EINTR;

		spin_lock(&space_info->lock);
	}

	ret = -ENOSPC;
4009 4010 4011
	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;
J
Josef Bacik 已提交
4012

4013 4014 4015 4016 4017 4018 4019
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
4020 4021
	if (used <= space_info->total_bytes) {
		if (used + orig_bytes <= space_info->total_bytes) {
4022
			space_info->bytes_may_use += orig_bytes;
J
Josef Bacik 已提交
4023
			trace_btrfs_space_reservation(root->fs_info,
4024
				"space_info", space_info->flags, orig_bytes, 1);
4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
4040
		num_bytes = used - space_info->total_bytes +
4041
			(orig_bytes * 2);
4042
	}
J
Josef Bacik 已提交
4043

4044 4045 4046 4047 4048 4049
	if (ret && can_overcommit(root, space_info, orig_bytes, flush)) {
		space_info->bytes_may_use += orig_bytes;
		trace_btrfs_space_reservation(root->fs_info, "space_info",
					      space_info->flags, orig_bytes,
					      1);
		ret = 0;
4050 4051
	}

4052 4053 4054 4055
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
M
Miao Xie 已提交
4056 4057 4058
	 *
	 * We make the other tasks wait for the flush only when we can flush
	 * all things.
4059
	 */
4060
	if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
4061 4062
		flushing = true;
		space_info->flush = 1;
4063
	}
J
Josef Bacik 已提交
4064

4065
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
4066

M
Miao Xie 已提交
4067
	if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
4068
		goto out;
4069

4070 4071 4072
	ret = flush_space(root, space_info, num_bytes, orig_bytes,
			  flush_state);
	flush_state++;
M
Miao Xie 已提交
4073 4074 4075 4076 4077 4078 4079 4080 4081 4082

	/*
	 * If we are FLUSH_LIMIT, we can not flush delalloc, or the deadlock
	 * would happen. So skip delalloc flush.
	 */
	if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
	    (flush_state == FLUSH_DELALLOC ||
	     flush_state == FLUSH_DELALLOC_WAIT))
		flush_state = ALLOC_CHUNK;

4083
	if (!ret)
4084
		goto again;
M
Miao Xie 已提交
4085 4086 4087 4088 4089
	else if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
		 flush_state < COMMIT_TRANS)
		goto again;
	else if (flush == BTRFS_RESERVE_FLUSH_ALL &&
		 flush_state <= COMMIT_TRANS)
4090 4091 4092
		goto again;

out:
4093
	if (flushing) {
4094
		spin_lock(&space_info->lock);
4095 4096
		space_info->flush = 0;
		wake_up_all(&space_info->wait);
4097
		spin_unlock(&space_info->lock);
4098 4099 4100 4101
	}
	return ret;
}

4102 4103 4104
static struct btrfs_block_rsv *get_block_rsv(
					const struct btrfs_trans_handle *trans,
					const struct btrfs_root *root)
4105
{
4106 4107
	struct btrfs_block_rsv *block_rsv = NULL;

4108 4109 4110 4111
	if (root->ref_cows)
		block_rsv = trans->block_rsv;

	if (root == root->fs_info->csum_root && trans->adding_csums)
4112
		block_rsv = trans->block_rsv;
4113 4114

	if (!block_rsv)
4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

J
Josef Bacik 已提交
4150 4151
static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
				    struct btrfs_block_rsv *block_rsv,
4152
				    struct btrfs_block_rsv *dest, u64 num_bytes)
4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
4185
			spin_lock(&space_info->lock);
4186
			space_info->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4187
			trace_btrfs_space_reservation(fs_info, "space_info",
4188
					space_info->flags, num_bytes, 0);
4189
			space_info->reservation_progress++;
4190
			spin_unlock(&space_info->lock);
4191
		}
J
Josef Bacik 已提交
4192
	}
4193
}
4194

4195 4196 4197 4198
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
J
Josef Bacik 已提交
4199

4200 4201 4202
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
4203

4204
	block_rsv_add_bytes(dst, num_bytes, 1);
J
Josef Bacik 已提交
4205 4206 4207
	return 0;
}

4208
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
J
Josef Bacik 已提交
4209
{
4210 4211
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
4212
	rsv->type = type;
4213 4214
}

4215 4216
struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root,
					      unsigned short type)
4217 4218 4219
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
4220

4221 4222 4223
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
4224

4225
	btrfs_init_block_rsv(block_rsv, type);
4226 4227 4228 4229
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
4230

4231 4232 4233
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
J
Josef Bacik 已提交
4234 4235
	if (!rsv)
		return;
4236 4237
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	kfree(rsv);
J
Josef Bacik 已提交
4238 4239
}

M
Miao Xie 已提交
4240 4241 4242
int btrfs_block_rsv_add(struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv, u64 num_bytes,
			enum btrfs_reserve_flush_enum flush)
J
Josef Bacik 已提交
4243
{
4244
	int ret;
J
Josef Bacik 已提交
4245

4246 4247
	if (num_bytes == 0)
		return 0;
4248

4249
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4250 4251 4252 4253
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
4254

4255 4256
	return ret;
}
J
Josef Bacik 已提交
4257

4258
int btrfs_block_rsv_check(struct btrfs_root *root,
4259
			  struct btrfs_block_rsv *block_rsv, int min_factor)
4260 4261 4262
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
4263

4264 4265
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
4266

4267
	spin_lock(&block_rsv->lock);
4268 4269 4270 4271
	num_bytes = div_factor(block_rsv->size, min_factor);
	if (block_rsv->reserved >= num_bytes)
		ret = 0;
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
4272

4273 4274 4275
	return ret;
}

M
Miao Xie 已提交
4276 4277 4278
int btrfs_block_rsv_refill(struct btrfs_root *root,
			   struct btrfs_block_rsv *block_rsv, u64 min_reserved,
			   enum btrfs_reserve_flush_enum flush)
4279 4280 4281 4282 4283 4284 4285 4286 4287
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;

	if (!block_rsv)
		return 0;

	spin_lock(&block_rsv->lock);
	num_bytes = min_reserved;
4288
	if (block_rsv->reserved >= num_bytes)
4289
		ret = 0;
4290
	else
4291 4292
		num_bytes -= block_rsv->reserved;
	spin_unlock(&block_rsv->lock);
4293

4294 4295 4296
	if (!ret)
		return 0;

4297
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4298 4299
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 0);
4300
		return 0;
J
Josef Bacik 已提交
4301
	}
J
Josef Bacik 已提交
4302

4303
	return ret;
4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320
}

int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
	if (global_rsv->full || global_rsv == block_rsv ||
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
J
Josef Bacik 已提交
4321 4322
	block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
				num_bytes);
J
Josef Bacik 已提交
4323 4324 4325
}

/*
4326 4327 4328
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
4329
 */
4330
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4331
{
4332 4333 4334 4335
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
4336
	int csum_size = btrfs_super_csum_size(fs_info->super_copy);
J
Josef Bacik 已提交
4337

4338 4339 4340 4341
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
4342

4343 4344
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
4345 4346
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
4347 4348
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
4349

4350 4351 4352
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
	num_bytes += div64_u64(data_used + meta_used, 50);
4353

4354
	if (num_bytes * 3 > meta_used)
4355
		num_bytes = div64_u64(meta_used, 3);
4356

4357 4358
	return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
}
J
Josef Bacik 已提交
4359

4360 4361 4362 4363 4364
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
4365

4366
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
4367

4368
	spin_lock(&sinfo->lock);
4369
	spin_lock(&block_rsv->lock);
4370

4371
	block_rsv->size = num_bytes;
4372

4373
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
4374 4375
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
4376 4377 4378 4379

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
4380
		sinfo->bytes_may_use += num_bytes;
J
Josef Bacik 已提交
4381
		trace_btrfs_space_reservation(fs_info, "space_info",
4382
				      sinfo->flags, num_bytes, 1);
J
Josef Bacik 已提交
4383 4384
	}

4385 4386
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
4387
		sinfo->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4388
		trace_btrfs_space_reservation(fs_info, "space_info",
4389
				      sinfo->flags, num_bytes, 0);
4390
		sinfo->reservation_progress++;
4391 4392 4393
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
4394

4395
	spin_unlock(&block_rsv->lock);
4396
	spin_unlock(&sinfo->lock);
J
Josef Bacik 已提交
4397 4398
}

4399
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4400
{
4401
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
4402

4403 4404
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
J
Josef Bacik 已提交
4405

4406
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4407 4408
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->delalloc_block_rsv.space_info = space_info;
4409 4410
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
4411
	fs_info->delayed_block_rsv.space_info = space_info;
4412

4413 4414 4415 4416
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
4417
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
4418 4419

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
4420 4421
}

4422
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4423
{
J
Josef Bacik 已提交
4424 4425
	block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
				(u64)-1);
4426 4427 4428 4429 4430 4431
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
4432 4433
	WARN_ON(fs_info->delayed_block_rsv.size > 0);
	WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
4434 4435
}

4436 4437
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
4438
{
4439 4440 4441
	if (!trans->block_rsv)
		return;

4442 4443
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
4444

4445
	trace_btrfs_space_reservation(root->fs_info, "transaction",
4446
				      trans->transid, trans->bytes_reserved, 0);
4447
	btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
4448 4449
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
4450

4451
/* Can only return 0 or -ENOSPC */
4452 4453 4454 4455 4456 4457 4458 4459
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
4460 4461 4462
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
4463
	 */
C
Chris Mason 已提交
4464
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4465 4466
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 1);
4467
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
J
Josef Bacik 已提交
4468 4469
}

4470
void btrfs_orphan_release_metadata(struct inode *inode)
4471
{
4472
	struct btrfs_root *root = BTRFS_I(inode)->root;
C
Chris Mason 已提交
4473
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4474 4475
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 0);
4476 4477
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
4478

4479 4480 4481 4482 4483 4484 4485
int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
				struct btrfs_pending_snapshot *pending)
{
	struct btrfs_root *root = pending->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
	/*
4486 4487
	 * two for root back/forward refs, two for directory entries,
	 * one for root of the snapshot and one for parent inode.
4488
	 */
4489
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 6);
4490 4491
	dst_rsv->space_info = src_rsv->space_info;
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4492 4493
}

4494 4495 4496 4497 4498 4499 4500 4501 4502
/**
 * drop_outstanding_extent - drop an outstanding extent
 * @inode: the inode we're dropping the extent for
 *
 * This is called when we are freeing up an outstanding extent, either called
 * after an error or after an extent is written.  This will return the number of
 * reserved extents that need to be freed.  This must be called with
 * BTRFS_I(inode)->lock held.
 */
4503 4504
static unsigned drop_outstanding_extent(struct inode *inode)
{
4505
	unsigned drop_inode_space = 0;
4506 4507 4508 4509 4510
	unsigned dropped_extents = 0;

	BUG_ON(!BTRFS_I(inode)->outstanding_extents);
	BTRFS_I(inode)->outstanding_extents--;

4511
	if (BTRFS_I(inode)->outstanding_extents == 0 &&
4512 4513
	    test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			       &BTRFS_I(inode)->runtime_flags))
4514 4515
		drop_inode_space = 1;

4516 4517 4518 4519 4520 4521
	/*
	 * If we have more or the same amount of outsanding extents than we have
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
4522
		return drop_inode_space;
4523 4524 4525 4526

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
4527
	return dropped_extents + drop_inode_space;
4528 4529
}

4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549
/**
 * calc_csum_metadata_size - return the amount of metada space that must be
 *	reserved/free'd for the given bytes.
 * @inode: the inode we're manipulating
 * @num_bytes: the number of bytes in question
 * @reserve: 1 if we are reserving space, 0 if we are freeing space
 *
 * This adjusts the number of csum_bytes in the inode and then returns the
 * correct amount of metadata that must either be reserved or freed.  We
 * calculate how many checksums we can fit into one leaf and then divide the
 * number of bytes that will need to be checksumed by this value to figure out
 * how many checksums will be required.  If we are adding bytes then the number
 * may go up and we will return the number of additional bytes that must be
 * reserved.  If it is going down we will return the number of bytes that must
 * be freed.
 *
 * This must be called with BTRFS_I(inode)->lock held.
 */
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
				   int reserve)
4550
{
4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 csum_size;
	int num_csums_per_leaf;
	int num_csums;
	int old_csums;

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
	    BTRFS_I(inode)->csum_bytes == 0)
		return 0;

	old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	if (reserve)
		BTRFS_I(inode)->csum_bytes += num_bytes;
	else
		BTRFS_I(inode)->csum_bytes -= num_bytes;
	csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
	num_csums_per_leaf = (int)div64_u64(csum_size,
					    sizeof(struct btrfs_csum_item) +
					    sizeof(struct btrfs_disk_key));
	num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	num_csums = num_csums + num_csums_per_leaf - 1;
	num_csums = num_csums / num_csums_per_leaf;

	old_csums = old_csums + num_csums_per_leaf - 1;
	old_csums = old_csums / num_csums_per_leaf;

	/* No change, no need to reserve more */
	if (old_csums == num_csums)
		return 0;

	if (reserve)
		return btrfs_calc_trans_metadata_size(root,
						      num_csums - old_csums);

	return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
4586
}
Y
Yan Zheng 已提交
4587

4588 4589 4590 4591
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4592
	u64 to_reserve = 0;
4593
	u64 csum_bytes;
4594
	unsigned nr_extents = 0;
4595
	int extra_reserve = 0;
M
Miao Xie 已提交
4596
	enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
4597
	int ret = 0;
4598
	bool delalloc_lock = true;
4599

4600 4601 4602 4603 4604 4605
	/* If we are a free space inode we need to not flush since we will be in
	 * the middle of a transaction commit.  We also don't need the delalloc
	 * mutex since we won't race with anybody.  We need this mostly to make
	 * lockdep shut its filthy mouth.
	 */
	if (btrfs_is_free_space_inode(inode)) {
M
Miao Xie 已提交
4606
		flush = BTRFS_RESERVE_NO_FLUSH;
4607 4608
		delalloc_lock = false;
	}
4609

M
Miao Xie 已提交
4610 4611
	if (flush != BTRFS_RESERVE_NO_FLUSH &&
	    btrfs_transaction_in_commit(root->fs_info))
4612
		schedule_timeout(1);
4613

4614 4615 4616
	if (delalloc_lock)
		mutex_lock(&BTRFS_I(inode)->delalloc_mutex);

4617
	num_bytes = ALIGN(num_bytes, root->sectorsize);
4618

4619 4620 4621 4622
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;

	if (BTRFS_I(inode)->outstanding_extents >
4623
	    BTRFS_I(inode)->reserved_extents)
4624 4625
		nr_extents = BTRFS_I(inode)->outstanding_extents -
			BTRFS_I(inode)->reserved_extents;
4626

4627 4628 4629 4630
	/*
	 * Add an item to reserve for updating the inode when we complete the
	 * delalloc io.
	 */
4631 4632
	if (!test_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
		      &BTRFS_I(inode)->runtime_flags)) {
4633
		nr_extents++;
4634
		extra_reserve = 1;
4635
	}
4636 4637

	to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4638
	to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
4639
	csum_bytes = BTRFS_I(inode)->csum_bytes;
4640
	spin_unlock(&BTRFS_I(inode)->lock);
4641

4642
	if (root->fs_info->quota_enabled)
4643 4644 4645
		ret = btrfs_qgroup_reserve(root, num_bytes +
					   nr_extents * root->leafsize);

4646 4647 4648 4649 4650 4651 4652 4653
	/*
	 * ret != 0 here means the qgroup reservation failed, we go straight to
	 * the shared error handling then.
	 */
	if (ret == 0)
		ret = reserve_metadata_bytes(root, block_rsv,
					     to_reserve, flush);

4654
	if (ret) {
4655
		u64 to_free = 0;
4656
		unsigned dropped;
4657

4658
		spin_lock(&BTRFS_I(inode)->lock);
4659 4660
		dropped = drop_outstanding_extent(inode);
		/*
4661 4662 4663 4664 4665 4666
		 * If the inodes csum_bytes is the same as the original
		 * csum_bytes then we know we haven't raced with any free()ers
		 * so we can just reduce our inodes csum bytes and carry on.
		 * Otherwise we have to do the normal free thing to account for
		 * the case that the free side didn't free up its reserve
		 * because of this outstanding reservation.
4667
		 */
4668 4669 4670 4671 4672 4673 4674 4675
		if (BTRFS_I(inode)->csum_bytes == csum_bytes)
			calc_csum_metadata_size(inode, num_bytes, 0);
		else
			to_free = calc_csum_metadata_size(inode, num_bytes, 0);
		spin_unlock(&BTRFS_I(inode)->lock);
		if (dropped)
			to_free += btrfs_calc_trans_metadata_size(root, dropped);

J
Josef Bacik 已提交
4676
		if (to_free) {
4677
			btrfs_block_rsv_release(root, block_rsv, to_free);
J
Josef Bacik 已提交
4678 4679 4680 4681 4682
			trace_btrfs_space_reservation(root->fs_info,
						      "delalloc",
						      btrfs_ino(inode),
						      to_free, 0);
		}
4683 4684 4685 4686
		if (root->fs_info->quota_enabled) {
			btrfs_qgroup_free(root, num_bytes +
						nr_extents * root->leafsize);
		}
4687 4688
		if (delalloc_lock)
			mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4689
		return ret;
4690
	}
4691

4692 4693
	spin_lock(&BTRFS_I(inode)->lock);
	if (extra_reserve) {
4694 4695
		set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
			&BTRFS_I(inode)->runtime_flags);
4696 4697 4698 4699
		nr_extents--;
	}
	BTRFS_I(inode)->reserved_extents += nr_extents;
	spin_unlock(&BTRFS_I(inode)->lock);
4700 4701 4702

	if (delalloc_lock)
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4703

J
Josef Bacik 已提交
4704 4705 4706
	if (to_reserve)
		trace_btrfs_space_reservation(root->fs_info,"delalloc",
					      btrfs_ino(inode), to_reserve, 1);
4707 4708 4709 4710 4711
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	return 0;
}

4712 4713 4714 4715 4716 4717 4718 4719 4720
/**
 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
 * @inode: the inode to release the reservation for
 * @num_bytes: the number of bytes we're releasing
 *
 * This will release the metadata reservation for an inode.  This can be called
 * once we complete IO for a given set of bytes to release their metadata
 * reservations.
 */
4721 4722 4723
void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
4724 4725
	u64 to_free = 0;
	unsigned dropped;
4726 4727

	num_bytes = ALIGN(num_bytes, root->sectorsize);
4728
	spin_lock(&BTRFS_I(inode)->lock);
4729
	dropped = drop_outstanding_extent(inode);
4730

4731 4732
	to_free = calc_csum_metadata_size(inode, num_bytes, 0);
	spin_unlock(&BTRFS_I(inode)->lock);
4733 4734
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
4735

J
Josef Bacik 已提交
4736 4737
	trace_btrfs_space_reservation(root->fs_info, "delalloc",
				      btrfs_ino(inode), to_free, 0);
4738 4739 4740 4741 4742
	if (root->fs_info->quota_enabled) {
		btrfs_qgroup_free(root, num_bytes +
					dropped * root->leafsize);
	}

4743 4744 4745 4746
	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761
/**
 * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
 * @inode: inode we're writing to
 * @num_bytes: the number of bytes we want to allocate
 *
 * This will do the following things
 *
 * o reserve space in the data space info for num_bytes
 * o reserve space in the metadata space info based on number of outstanding
 *   extents and how much csums will be needed
 * o add to the inodes ->delalloc_bytes
 * o add it to the fs_info's delalloc inodes list.
 *
 * This will return 0 for success and -ENOSPC if there is no space left.
 */
4762 4763 4764 4765 4766
int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
C
Chris Mason 已提交
4767
	if (ret)
4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791
/**
 * btrfs_delalloc_release_space - release data and metadata space for delalloc
 * @inode: inode we're releasing space for
 * @num_bytes: the number of bytes we want to free up
 *
 * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
 * called in the case that we don't need the metadata AND data reservations
 * anymore.  So if there is an error or we insert an inline extent.
 *
 * This function will release the metadata space that was not used and will
 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
 * list if there are no delalloc bytes left.
 */
4792 4793 4794 4795
void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
4796 4797
}

4798
static int update_block_group(struct btrfs_root *root,
4799
			      u64 bytenr, u64 num_bytes, int alloc)
C
Chris Mason 已提交
4800
{
4801
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
4802
	struct btrfs_fs_info *info = root->fs_info;
4803
	u64 total = num_bytes;
C
Chris Mason 已提交
4804
	u64 old_val;
4805
	u64 byte_in_group;
4806
	int factor;
C
Chris Mason 已提交
4807

4808 4809
	/* block accounting for super block */
	spin_lock(&info->delalloc_lock);
4810
	old_val = btrfs_super_bytes_used(info->super_copy);
4811 4812 4813 4814
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
4815
	btrfs_set_super_bytes_used(info->super_copy, old_val);
4816 4817
	spin_unlock(&info->delalloc_lock);

C
Chris Mason 已提交
4818
	while (total) {
4819
		cache = btrfs_lookup_block_group(info, bytenr);
4820
		if (!cache)
4821
			return -ENOENT;
4822 4823 4824 4825 4826 4827
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
4828 4829 4830 4831 4832 4833 4834
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
4835
			cache_block_group(cache, 1);
4836

4837 4838
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
4839

4840
		spin_lock(&cache->space_info->lock);
4841
		spin_lock(&cache->lock);
4842

4843
		if (btrfs_test_opt(root, SPACE_CACHE) &&
4844 4845 4846
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

J
Josef Bacik 已提交
4847
		cache->dirty = 1;
C
Chris Mason 已提交
4848
		old_val = btrfs_block_group_used(&cache->item);
4849
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
4850
		if (alloc) {
4851
			old_val += num_bytes;
4852 4853 4854
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
4855 4856
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
4857
			spin_unlock(&cache->lock);
4858
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4859
		} else {
4860
			old_val -= num_bytes;
4861
			btrfs_set_block_group_used(&cache->item, old_val);
4862 4863
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
4864
			cache->space_info->bytes_used -= num_bytes;
4865
			cache->space_info->disk_used -= num_bytes * factor;
4866
			spin_unlock(&cache->lock);
4867
			spin_unlock(&cache->space_info->lock);
4868

4869 4870 4871
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
C
Chris Mason 已提交
4872
		}
4873
		btrfs_put_block_group(cache);
4874 4875
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
4876 4877 4878
	}
	return 0;
}
4879

4880 4881
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
4882
	struct btrfs_block_group_cache *cache;
4883
	u64 bytenr;
J
Josef Bacik 已提交
4884

4885 4886 4887 4888 4889 4890 4891
	spin_lock(&root->fs_info->block_group_cache_lock);
	bytenr = root->fs_info->first_logical_byte;
	spin_unlock(&root->fs_info->block_group_cache_lock);

	if (bytenr < (u64)-1)
		return bytenr;

J
Josef Bacik 已提交
4892 4893
	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
4894
		return 0;
J
Josef Bacik 已提交
4895

4896
	bytenr = cache->key.objectid;
4897
	btrfs_put_block_group(cache);
4898 4899

	return bytenr;
4900 4901
}

4902 4903 4904
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
4905
{
4906 4907 4908 4909 4910 4911 4912 4913 4914 4915
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
4916

4917 4918 4919 4920
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
	return 0;
}
J
Josef Bacik 已提交
4921

4922 4923 4924 4925 4926 4927 4928
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
4929

4930
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4931
	BUG_ON(!cache); /* Logic error */
4932 4933 4934 4935

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
4936 4937 4938
	return 0;
}

4939
/*
4940 4941
 * this function must be called within transaction
 */
4942
int btrfs_pin_extent_for_log_replay(struct btrfs_root *root,
4943 4944 4945 4946 4947
				    u64 bytenr, u64 num_bytes)
{
	struct btrfs_block_group_cache *cache;

	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4948
	BUG_ON(!cache); /* Logic error */
4949 4950 4951 4952 4953 4954 4955

	/*
	 * pull in the free space cache (if any) so that our pin
	 * removes the free space from the cache.  We have load_only set
	 * to one because the slow code to read in the free extents does check
	 * the pinned extents.
	 */
4956
	cache_block_group(cache, 1);
4957 4958 4959 4960 4961 4962 4963 4964 4965

	pin_down_extent(root, cache, bytenr, num_bytes, 0);

	/* remove us from the free space cache (if we're there at all) */
	btrfs_remove_free_space(cache, bytenr, num_bytes);
	btrfs_put_block_group(cache);
	return 0;
}

4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986
/**
 * btrfs_update_reserved_bytes - update the block_group and space info counters
 * @cache:	The cache we are manipulating
 * @num_bytes:	The number of bytes in question
 * @reserve:	One of the reservation enums
 *
 * This is called by the allocator when it reserves space, or by somebody who is
 * freeing space that was never actually used on disk.  For example if you
 * reserve some space for a new leaf in transaction A and before transaction A
 * commits you free that leaf, you call this with reserve set to 0 in order to
 * clear the reservation.
 *
 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
 * ENOSPC accounting.  For data we handle the reservation through clearing the
 * delalloc bits in the io_tree.  We have to do this since we could end up
 * allocating less disk space for the amount of data we have reserved in the
 * case of compression.
 *
 * If this is a reservation and the block group has become read only we cannot
 * make the reservation and return -EAGAIN, otherwise this function always
 * succeeds.
4987
 */
4988 4989
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve)
4990
{
4991
	struct btrfs_space_info *space_info = cache->space_info;
4992
	int ret = 0;
4993

4994 4995 4996
	spin_lock(&space_info->lock);
	spin_lock(&cache->lock);
	if (reserve != RESERVE_FREE) {
4997 4998 4999
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
5000 5001 5002
			cache->reserved += num_bytes;
			space_info->bytes_reserved += num_bytes;
			if (reserve == RESERVE_ALLOC) {
J
Josef Bacik 已提交
5003
				trace_btrfs_space_reservation(cache->fs_info,
5004 5005
						"space_info", space_info->flags,
						num_bytes, 0);
5006 5007
				space_info->bytes_may_use -= num_bytes;
			}
5008
		}
5009 5010 5011 5012 5013 5014
	} else {
		if (cache->ro)
			space_info->bytes_readonly += num_bytes;
		cache->reserved -= num_bytes;
		space_info->bytes_reserved -= num_bytes;
		space_info->reservation_progress++;
5015
	}
5016 5017
	spin_unlock(&cache->lock);
	spin_unlock(&space_info->lock);
5018
	return ret;
5019
}
C
Chris Mason 已提交
5020

5021
void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
5022
				struct btrfs_root *root)
5023 5024
{
	struct btrfs_fs_info *fs_info = root->fs_info;
5025 5026 5027
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
5028

5029
	down_write(&fs_info->extent_commit_sem);
5030

5031 5032 5033 5034 5035 5036 5037
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
5038
		} else {
5039
			cache->last_byte_to_unpin = caching_ctl->progress;
5040 5041
		}
	}
5042 5043 5044 5045 5046 5047 5048

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

	up_write(&fs_info->extent_commit_sem);
5049 5050

	update_global_block_rsv(fs_info);
5051 5052
}

5053
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
C
Chris Mason 已提交
5054
{
5055 5056
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
5057 5058
	struct btrfs_space_info *space_info;
	struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5059
	u64 len;
5060
	bool readonly;
C
Chris Mason 已提交
5061

5062
	while (start <= end) {
5063
		readonly = false;
5064 5065 5066 5067 5068
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
5069
			BUG_ON(!cache); /* Logic error */
5070 5071 5072 5073 5074 5075 5076 5077 5078 5079
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
			btrfs_add_free_space(cache, start, len);
		}

5080
		start += len;
5081
		space_info = cache->space_info;
5082

5083
		spin_lock(&space_info->lock);
5084 5085
		spin_lock(&cache->lock);
		cache->pinned -= len;
5086 5087 5088 5089 5090
		space_info->bytes_pinned -= len;
		if (cache->ro) {
			space_info->bytes_readonly += len;
			readonly = true;
		}
5091
		spin_unlock(&cache->lock);
5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104
		if (!readonly && global_rsv->space_info == space_info) {
			spin_lock(&global_rsv->lock);
			if (!global_rsv->full) {
				len = min(len, global_rsv->size -
					  global_rsv->reserved);
				global_rsv->reserved += len;
				space_info->bytes_may_use += len;
				if (global_rsv->reserved >= global_rsv->size)
					global_rsv->full = 1;
			}
			spin_unlock(&global_rsv->lock);
		}
		spin_unlock(&space_info->lock);
C
Chris Mason 已提交
5105
	}
5106 5107 5108

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
5109 5110 5111 5112
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
5113
			       struct btrfs_root *root)
5114
{
5115 5116
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
5117 5118
	u64 start;
	u64 end;
5119 5120
	int ret;

5121 5122 5123
	if (trans->aborted)
		return 0;

5124 5125 5126 5127 5128
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

C
Chris Mason 已提交
5129
	while (1) {
5130
		ret = find_first_extent_bit(unpin, 0, &start, &end,
5131
					    EXTENT_DIRTY, NULL);
5132
		if (ret)
5133
			break;
5134

5135 5136 5137
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
5138

5139
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
5140
		unpin_extent_range(root, start, end);
5141
		cond_resched();
5142
	}
J
Josef Bacik 已提交
5143

C
Chris Mason 已提交
5144 5145 5146
	return 0;
}

5147 5148 5149 5150 5151 5152
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extent_op)
5153
{
C
Chris Mason 已提交
5154
	struct btrfs_key key;
5155
	struct btrfs_path *path;
5156 5157
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
5158
	struct extent_buffer *leaf;
5159 5160
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
5161
	int ret;
5162
	int is_data;
5163 5164 5165
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
5166 5167
	u32 item_size;
	u64 refs;
C
Chris Mason 已提交
5168

5169
	path = btrfs_alloc_path();
5170 5171
	if (!path)
		return -ENOMEM;
5172

5173
	path->reada = 1;
5174
	path->leave_spinning = 1;
5175 5176 5177 5178 5179 5180 5181 5182

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
5183
	if (ret == 0) {
5184
		extent_slot = path->slots[0];
5185 5186
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
5187
					      extent_slot);
5188
			if (key.objectid != bytenr)
5189
				break;
5190 5191
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
5192 5193 5194 5195 5196
				found_extent = 1;
				break;
			}
			if (path->slots[0] - extent_slot > 5)
				break;
5197
			extent_slot--;
5198
		}
5199 5200 5201 5202 5203
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
5204
		if (!found_extent) {
5205
			BUG_ON(iref);
5206
			ret = remove_extent_backref(trans, extent_root, path,
5207 5208
						    NULL, refs_to_drop,
						    is_data);
5209 5210 5211 5212
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5213
			btrfs_release_path(path);
5214
			path->leave_spinning = 1;
5215 5216 5217 5218 5219

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

Z
Zheng Yan 已提交
5220 5221
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
5222 5223
			if (ret) {
				printk(KERN_ERR "umm, got %d back from search"
C
Chris Mason 已提交
5224 5225
				       ", was looking for %llu\n", ret,
				       (unsigned long long)bytenr);
5226 5227 5228
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
5229
			}
5230 5231 5232 5233
			if (ret < 0) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
Z
Zheng Yan 已提交
5234 5235
			extent_slot = path->slots[0];
		}
5236
	} else if (ret == -ENOENT) {
5237 5238
		btrfs_print_leaf(extent_root, path->nodes[0]);
		WARN_ON(1);
C
Chris Mason 已提交
5239
		printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
5240
		       "parent %llu root %llu  owner %llu offset %llu\n",
C
Chris Mason 已提交
5241
		       (unsigned long long)bytenr,
5242
		       (unsigned long long)parent,
C
Chris Mason 已提交
5243
		       (unsigned long long)root_objectid,
5244 5245
		       (unsigned long long)owner_objectid,
		       (unsigned long long)owner_offset);
5246
	} else {
5247 5248
		btrfs_abort_transaction(trans, extent_root, ret);
		goto out;
5249
	}
5250 5251

	leaf = path->nodes[0];
5252 5253 5254 5255 5256 5257
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
5258 5259 5260 5261
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5262

5263
		btrfs_release_path(path);
5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
			printk(KERN_ERR "umm, got %d back from search"
			       ", was looking for %llu\n", ret,
			       (unsigned long long)bytenr);
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
5278 5279 5280 5281 5282
		if (ret < 0) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}

5283 5284 5285 5286 5287 5288
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
5289
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
5290
			    struct btrfs_extent_item);
5291 5292 5293 5294 5295 5296
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
5297

5298
	refs = btrfs_extent_refs(leaf, ei);
5299 5300
	BUG_ON(refs < refs_to_drop);
	refs -= refs_to_drop;
5301

5302 5303 5304 5305 5306 5307
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
5308
		 */
5309 5310 5311 5312 5313 5314 5315 5316 5317 5318
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
						    is_data);
5319 5320 5321 5322
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5323
		}
5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
5335
		}
5336

5337 5338
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
5339 5340 5341 5342
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5343
		btrfs_release_path(path);
5344

5345
		if (is_data) {
5346
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
5347 5348 5349 5350
			if (ret) {
				btrfs_abort_transaction(trans, extent_root, ret);
				goto out;
			}
5351 5352
		}

5353
		ret = update_block_group(root, bytenr, num_bytes, 0);
5354 5355 5356 5357
		if (ret) {
			btrfs_abort_transaction(trans, extent_root, ret);
			goto out;
		}
5358
	}
5359
out:
5360
	btrfs_free_path(path);
5361 5362 5363
	return ret;
}

5364
/*
5365
 * when we free an block, it is possible (and likely) that we free the last
5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct rb_node *node;
5377
	int ret = 0;
5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	/* there are still entries for this ref, we can't drop it */
	if (ref->bytenr == bytenr)
		goto out;

5395 5396 5397
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
5398
		btrfs_free_delayed_extent_op(head->extent_op);
5399 5400 5401
		head->extent_op = NULL;
	}

5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
	rb_erase(&head->node.rb_node, &delayed_refs->root);
5415

5416 5417 5418 5419 5420 5421
	delayed_refs->num_entries--;

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
5422 5423 5424 5425 5426
	delayed_refs->num_heads--;
	if (list_empty(&head->cluster))
		delayed_refs->num_heads_ready--;

	list_del_init(&head->cluster);
5427 5428
	spin_unlock(&delayed_refs->lock);

5429 5430 5431 5432 5433
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
5434
	btrfs_put_delayed_ref(&head->node);
5435
	return ret;
5436 5437 5438 5439 5440
out:
	spin_unlock(&delayed_refs->lock);
	return 0;
}

5441 5442 5443
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
5444
			   u64 parent, int last_ref)
5445 5446 5447 5448 5449
{
	struct btrfs_block_group_cache *cache = NULL;
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
A
Arne Jansen 已提交
5450 5451 5452 5453
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					buf->start, buf->len,
					parent, root->root_key.objectid,
					btrfs_header_level(buf),
5454
					BTRFS_DROP_DELAYED_REF, NULL, 0);
5455
		BUG_ON(ret); /* -ENOMEM */
5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466
	}

	if (!last_ref)
		return;

	cache = btrfs_lookup_block_group(root->fs_info, buf->start);

	if (btrfs_header_generation(buf) == trans->transid) {
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
5467
				goto out;
5468 5469 5470 5471
		}

		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
5472
			goto out;
5473 5474 5475 5476 5477
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
5478
		btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
5479 5480
	}
out:
5481 5482 5483 5484 5485
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
5486 5487 5488
	btrfs_put_block_group(cache);
}

5489
/* Can return -ENOMEM */
A
Arne Jansen 已提交
5490 5491 5492
int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
		      u64 owner, u64 offset, int for_cow)
5493 5494
{
	int ret;
A
Arne Jansen 已提交
5495
	struct btrfs_fs_info *fs_info = root->fs_info;
5496

5497 5498 5499 5500
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
5501 5502
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
5503
		/* unlocks the pinned mutex */
5504
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
5505
		ret = 0;
5506
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
5507 5508
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
5509
					parent, root_objectid, (int)owner,
A
Arne Jansen 已提交
5510
					BTRFS_DROP_DELAYED_REF, NULL, for_cow);
5511
	} else {
A
Arne Jansen 已提交
5512 5513 5514 5515 5516
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
						num_bytes,
						parent, root_objectid, owner,
						offset, BTRFS_DROP_DELAYED_REF,
						NULL, for_cow);
5517
	}
5518 5519 5520
	return ret;
}

5521 5522 5523 5524 5525 5526 5527
static u64 stripe_align(struct btrfs_root *root, u64 val)
{
	u64 mask = ((u64)root->stripesize - 1);
	u64 ret = (val + mask) & ~mask;
	return ret;
}

J
Josef Bacik 已提交
5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
 */
static noinline int
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
5543
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
5544

5545 5546
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
J
Josef Bacik 已提交
5547 5548
		return 0;

5549
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
5550
		   (cache->free_space_ctl->free_space >= num_bytes));
5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567

	put_caching_control(caching_ctl);
	return 0;
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
		return 0;

	wait_event(caching_ctl->wait, block_group_cache_done(cache));

	put_caching_control(caching_ctl);
J
Josef Bacik 已提交
5568 5569 5570
	return 0;
}

5571
int __get_raid_index(u64 flags)
5572
{
5573
	if (flags & BTRFS_BLOCK_GROUP_RAID10)
5574
		return BTRFS_RAID_RAID10;
5575
	else if (flags & BTRFS_BLOCK_GROUP_RAID1)
5576
		return BTRFS_RAID_RAID1;
5577
	else if (flags & BTRFS_BLOCK_GROUP_DUP)
5578
		return BTRFS_RAID_DUP;
5579
	else if (flags & BTRFS_BLOCK_GROUP_RAID0)
5580
		return BTRFS_RAID_RAID0;
5581
	else
5582
		return BTRFS_RAID_SINGLE;
5583 5584
}

5585 5586
static int get_block_group_index(struct btrfs_block_group_cache *cache)
{
5587
	return __get_raid_index(cache->flags);
5588 5589
}

J
Josef Bacik 已提交
5590
enum btrfs_loop_type {
5591 5592 5593 5594
	LOOP_CACHING_NOWAIT = 0,
	LOOP_CACHING_WAIT = 1,
	LOOP_ALLOC_CHUNK = 2,
	LOOP_NO_EMPTY_SIZE = 3,
J
Josef Bacik 已提交
5595 5596
};

5597 5598 5599 5600
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
 * ins->objectid == block start
5601
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
5602 5603 5604
 * ins->offset == number of blocks
 * Any available blocks before search_start are skipped.
 */
C
Chris Mason 已提交
5605
static noinline int find_free_extent(struct btrfs_trans_handle *trans,
5606 5607 5608
				     struct btrfs_root *orig_root,
				     u64 num_bytes, u64 empty_size,
				     u64 hint_byte, struct btrfs_key *ins,
5609
				     u64 data)
5610
{
5611
	int ret = 0;
C
Chris Mason 已提交
5612
	struct btrfs_root *root = orig_root->fs_info->extent_root;
5613
	struct btrfs_free_cluster *last_ptr = NULL;
5614
	struct btrfs_block_group_cache *block_group = NULL;
5615
	struct btrfs_block_group_cache *used_block_group;
5616
	u64 search_start = 0;
5617
	int empty_cluster = 2 * 1024 * 1024;
5618
	struct btrfs_space_info *space_info;
5619
	int loop = 0;
5620
	int index = __get_raid_index(data);
5621 5622
	int alloc_type = (data & BTRFS_BLOCK_GROUP_DATA) ?
		RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
J
Josef Bacik 已提交
5623
	bool found_uncached_bg = false;
5624
	bool failed_cluster_refill = false;
5625
	bool failed_alloc = false;
5626
	bool use_cluster = true;
5627
	bool have_caching_bg = false;
5628

5629
	WARN_ON(num_bytes < root->sectorsize);
5630
	btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
5631 5632
	ins->objectid = 0;
	ins->offset = 0;
5633

J
Josef Bacik 已提交
5634 5635
	trace_find_free_extent(orig_root, num_bytes, empty_size, data);

J
Josef Bacik 已提交
5636
	space_info = __find_space_info(root->fs_info, data);
5637
	if (!space_info) {
5638
		printk(KERN_ERR "No space info for %llu\n", data);
5639 5640
		return -ENOSPC;
	}
J
Josef Bacik 已提交
5641

5642 5643 5644 5645 5646 5647 5648 5649
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

	if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
5650
		last_ptr = &root->fs_info->meta_alloc_cluster;
5651 5652
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
5653 5654
	}

5655 5656
	if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
	    btrfs_test_opt(root, SSD)) {
5657 5658
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
J
Josef Bacik 已提交
5659

5660
	if (last_ptr) {
5661 5662 5663 5664
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
5665
	}
5666

5667
	search_start = max(search_start, first_logical_byte(root, 0));
5668
	search_start = max(search_start, hint_byte);
5669

J
Josef Bacik 已提交
5670
	if (!last_ptr)
5671 5672
		empty_cluster = 0;

J
Josef Bacik 已提交
5673 5674 5675
	if (search_start == hint_byte) {
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
5676
		used_block_group = block_group;
J
Josef Bacik 已提交
5677 5678 5679
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
5680 5681 5682
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
5683 5684
		 */
		if (block_group && block_group_bits(block_group, data) &&
5685
		    block_group->cached != BTRFS_CACHE_NO) {
J
Josef Bacik 已提交
5686
			down_read(&space_info->groups_sem);
5687 5688 5689 5690 5691 5692 5693 5694 5695 5696
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
5697
			} else {
5698
				index = get_block_group_index(block_group);
5699
				goto have_block_group;
5700
			}
J
Josef Bacik 已提交
5701
		} else if (block_group) {
5702
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5703
		}
5704
	}
J
Josef Bacik 已提交
5705
search:
5706
	have_caching_bg = false;
5707
	down_read(&space_info->groups_sem);
5708 5709
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
5710
		u64 offset;
J
Josef Bacik 已提交
5711
		int cached;
5712

5713
		used_block_group = block_group;
5714
		btrfs_get_block_group(block_group);
J
Josef Bacik 已提交
5715
		search_start = block_group->key.objectid;
5716

5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
		if (!block_group_bits(block_group, data)) {
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
			if ((data & extra) && !(block_group->flags & extra))
				goto loop;
		}

J
Josef Bacik 已提交
5736
have_block_group:
5737 5738 5739
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached)) {
			found_uncached_bg = true;
5740
			ret = cache_block_group(block_group, 0);
5741 5742
			BUG_ON(ret < 0);
			ret = 0;
J
Josef Bacik 已提交
5743 5744
		}

5745
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
5746
			goto loop;
J
Josef Bacik 已提交
5747

5748
		/*
5749 5750
		 * Ok we want to try and use the cluster allocator, so
		 * lets look there
5751
		 */
5752
		if (last_ptr) {
5753 5754 5755 5756 5757
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
			spin_lock(&last_ptr->refill_lock);
5758 5759 5760 5761 5762 5763
			used_block_group = last_ptr->block_group;
			if (used_block_group != block_group &&
			    (!used_block_group ||
			     used_block_group->ro ||
			     !block_group_bits(used_block_group, data))) {
				used_block_group = block_group;
5764
				goto refill_cluster;
5765 5766 5767 5768
			}

			if (used_block_group != block_group)
				btrfs_get_block_group(used_block_group);
5769

5770 5771
			offset = btrfs_alloc_from_cluster(used_block_group,
			  last_ptr, num_bytes, used_block_group->key.objectid);
5772 5773 5774
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
5775 5776
				trace_btrfs_reserve_extent_cluster(root,
					block_group, search_start, num_bytes);
5777 5778 5779
				goto checks;
			}

5780 5781 5782 5783
			WARN_ON(last_ptr->block_group != used_block_group);
			if (used_block_group != block_group) {
				btrfs_put_block_group(used_block_group);
				used_block_group = block_group;
5784
			}
5785
refill_cluster:
5786
			BUG_ON(used_block_group != block_group);
5787 5788 5789 5790 5791 5792 5793 5794
			/* If we are on LOOP_NO_EMPTY_SIZE, we can't
			 * set up a new clusters, so lets just skip it
			 * and let the allocator find whatever block
			 * it can find.  If we reach this point, we
			 * will have tried the cluster allocator
			 * plenty of times and not have found
			 * anything, so we are likely way too
			 * fragmented for the clustering stuff to find
5795 5796 5797 5798 5799 5800 5801 5802 5803
			 * anything.
			 *
			 * However, if the cluster is taken from the
			 * current block group, release the cluster
			 * first, so that we stand a better chance of
			 * succeeding in the unclustered
			 * allocation.  */
			if (loop >= LOOP_NO_EMPTY_SIZE &&
			    last_ptr->block_group != block_group) {
5804 5805 5806 5807
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

5808 5809 5810 5811 5812 5813
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

5814 5815 5816 5817 5818
			if (loop >= LOOP_NO_EMPTY_SIZE) {
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

5819
			/* allocate a cluster in this block group */
5820
			ret = btrfs_find_space_cluster(trans, root,
5821
					       block_group, last_ptr,
5822
					       search_start, num_bytes,
5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834
					       empty_cluster + empty_size);
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
						  last_ptr, num_bytes,
						  search_start);
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
5835 5836 5837
					trace_btrfs_reserve_extent_cluster(root,
						block_group, search_start,
						num_bytes);
5838 5839
					goto checks;
				}
5840 5841
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
5842 5843
				spin_unlock(&last_ptr->refill_lock);

5844
				failed_cluster_refill = true;
J
Josef Bacik 已提交
5845 5846 5847
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
5848
			}
J
Josef Bacik 已提交
5849

5850 5851 5852 5853 5854 5855
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
5856
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
5857
			spin_unlock(&last_ptr->refill_lock);
5858
			goto loop;
5859 5860
		}

5861
unclustered_alloc:
5862 5863 5864 5865 5866 5867 5868 5869 5870
		spin_lock(&block_group->free_space_ctl->tree_lock);
		if (cached &&
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_cluster + empty_size) {
			spin_unlock(&block_group->free_space_ctl->tree_lock);
			goto loop;
		}
		spin_unlock(&block_group->free_space_ctl->tree_lock);

5871 5872
		offset = btrfs_find_space_for_alloc(block_group, search_start,
						    num_bytes, empty_size);
5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
5884
			wait_block_group_cache_progress(block_group,
5885 5886
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
5887
			goto have_block_group;
5888
		} else if (!offset) {
5889 5890
			if (!cached)
				have_caching_bg = true;
5891
			goto loop;
J
Josef Bacik 已提交
5892
		}
5893
checks:
5894
		search_start = stripe_align(root, offset);
5895

J
Josef Bacik 已提交
5896 5897
		/* move on to the next group */
		if (search_start + num_bytes >
5898 5899
		    used_block_group->key.objectid + used_block_group->key.offset) {
			btrfs_add_free_space(used_block_group, offset, num_bytes);
J
Josef Bacik 已提交
5900
			goto loop;
5901
		}
5902

5903
		if (offset < search_start)
5904
			btrfs_add_free_space(used_block_group, offset,
5905 5906
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
5907

5908
		ret = btrfs_update_reserved_bytes(used_block_group, num_bytes,
5909
						  alloc_type);
5910
		if (ret == -EAGAIN) {
5911
			btrfs_add_free_space(used_block_group, offset, num_bytes);
J
Josef Bacik 已提交
5912
			goto loop;
J
Josef Bacik 已提交
5913
		}
5914

5915
		/* we are all good, lets return */
J
Josef Bacik 已提交
5916 5917
		ins->objectid = search_start;
		ins->offset = num_bytes;
5918

J
Josef Bacik 已提交
5919 5920
		trace_btrfs_reserve_extent(orig_root, block_group,
					   search_start, num_bytes);
5921 5922
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
5923
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5924 5925
		break;
loop:
5926
		failed_cluster_refill = false;
5927
		failed_alloc = false;
5928
		BUG_ON(index != get_block_group_index(block_group));
5929 5930
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
5931
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5932 5933 5934
	}
	up_read(&space_info->groups_sem);

5935 5936 5937
	if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
		goto search;

5938 5939 5940
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

5941
	/*
5942 5943
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
5944 5945 5946 5947
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
5948
	 */
5949
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
5950
		index = 0;
5951
		loop++;
J
Josef Bacik 已提交
5952
		if (loop == LOOP_ALLOC_CHUNK) {
5953
			ret = do_chunk_alloc(trans, root, data,
5954 5955 5956 5957 5958 5959 5960 5961 5962
					     CHUNK_ALLOC_FORCE);
			/*
			 * Do not bail out on ENOSPC since we
			 * can do more things.
			 */
			if (ret < 0 && ret != -ENOSPC) {
				btrfs_abort_transaction(trans,
							root, ret);
				goto out;
5963
			}
J
Josef Bacik 已提交
5964 5965
		}

5966 5967 5968
		if (loop == LOOP_NO_EMPTY_SIZE) {
			empty_size = 0;
			empty_cluster = 0;
5969
		}
5970 5971

		goto search;
J
Josef Bacik 已提交
5972 5973
	} else if (!ins->objectid) {
		ret = -ENOSPC;
5974
	} else if (ins->objectid) {
5975
		ret = 0;
C
Chris Mason 已提交
5976
	}
5977
out:
C
Chris Mason 已提交
5978

C
Chris Mason 已提交
5979
	return ret;
5980
}
5981

J
Josef Bacik 已提交
5982 5983
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
5984 5985
{
	struct btrfs_block_group_cache *cache;
5986
	int index = 0;
J
Josef Bacik 已提交
5987

J
Josef Bacik 已提交
5988
	spin_lock(&info->lock);
5989 5990
	printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
	       (unsigned long long)info->flags,
C
Chris Mason 已提交
5991
	       (unsigned long long)(info->total_bytes - info->bytes_used -
J
Josef Bacik 已提交
5992
				    info->bytes_pinned - info->bytes_reserved -
5993
				    info->bytes_readonly),
C
Chris Mason 已提交
5994
	       (info->full) ? "" : "not ");
5995 5996
	printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
5997
	       (unsigned long long)info->total_bytes,
5998
	       (unsigned long long)info->bytes_used,
5999
	       (unsigned long long)info->bytes_pinned,
6000
	       (unsigned long long)info->bytes_reserved,
6001
	       (unsigned long long)info->bytes_may_use,
6002
	       (unsigned long long)info->bytes_readonly);
J
Josef Bacik 已提交
6003 6004 6005 6006
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
6007

6008
	down_read(&info->groups_sem);
6009 6010
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
6011
		spin_lock(&cache->lock);
6012
		printk(KERN_INFO "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s\n",
C
Chris Mason 已提交
6013 6014 6015 6016
		       (unsigned long long)cache->key.objectid,
		       (unsigned long long)cache->key.offset,
		       (unsigned long long)btrfs_block_group_used(&cache->item),
		       (unsigned long long)cache->pinned,
6017 6018
		       (unsigned long long)cache->reserved,
		       cache->ro ? "[readonly]" : "");
J
Josef Bacik 已提交
6019 6020 6021
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
6022 6023
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
6024
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
6025
}
6026

6027 6028 6029 6030
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
6031
			 struct btrfs_key *ins, u64 data)
6032
{
6033
	bool final_tried = false;
6034
	int ret;
6035

J
Josef Bacik 已提交
6036
	data = btrfs_get_alloc_profile(root, data);
6037
again:
6038 6039
	WARN_ON(num_bytes < root->sectorsize);
	ret = find_free_extent(trans, root, num_bytes, empty_size,
6040
			       hint_byte, ins, data);
6041

6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056
	if (ret == -ENOSPC) {
		if (!final_tried) {
			num_bytes = num_bytes >> 1;
			num_bytes = num_bytes & ~(root->sectorsize - 1);
			num_bytes = max(num_bytes, min_alloc_size);
			if (num_bytes == min_alloc_size)
				final_tried = true;
			goto again;
		} else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
			struct btrfs_space_info *sinfo;

			sinfo = __find_space_info(root->fs_info, data);
			printk(KERN_ERR "btrfs allocation failed flags %llu, "
			       "wanted %llu\n", (unsigned long long)data,
			       (unsigned long long)num_bytes);
6057 6058
			if (sinfo)
				dump_space_info(sinfo, num_bytes, 1);
6059
		}
6060
	}
J
Josef Bacik 已提交
6061

6062 6063
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);

J
Josef Bacik 已提交
6064
	return ret;
6065 6066
}

6067 6068
static int __btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len, int pin)
6069
{
J
Josef Bacik 已提交
6070
	struct btrfs_block_group_cache *cache;
6071
	int ret = 0;
J
Josef Bacik 已提交
6072 6073 6074

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
C
Chris Mason 已提交
6075 6076
		printk(KERN_ERR "Unable to find block group for %llu\n",
		       (unsigned long long)start);
J
Josef Bacik 已提交
6077 6078
		return -ENOSPC;
	}
6079

6080 6081
	if (btrfs_test_opt(root, DISCARD))
		ret = btrfs_discard_extent(root, start, len, NULL);
6082

6083 6084 6085 6086 6087 6088
	if (pin)
		pin_down_extent(root, cache, start, len, 1);
	else {
		btrfs_add_free_space(cache, start, len);
		btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
	}
6089
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
6090

6091 6092
	trace_btrfs_reserved_extent_free(root, start, len);

6093 6094 6095
	return ret;
}

6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107
int btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 0);
}

int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
				       u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 1);
}

6108 6109 6110 6111 6112
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
6113 6114
{
	int ret;
6115
	struct btrfs_fs_info *fs_info = root->fs_info;
6116
	struct btrfs_extent_item *extent_item;
6117
	struct btrfs_extent_inline_ref *iref;
6118
	struct btrfs_path *path;
6119 6120 6121
	struct extent_buffer *leaf;
	int type;
	u32 size;
6122

6123 6124 6125 6126
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
6127

6128
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
6129 6130

	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
6131 6132
	if (!path)
		return -ENOMEM;
6133

6134
	path->leave_spinning = 1;
6135 6136
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6137 6138 6139 6140
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
J
Josef Bacik 已提交
6141

6142 6143
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
6144
				     struct btrfs_extent_item);
6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
6165 6166

	btrfs_mark_buffer_dirty(path->nodes[0]);
6167
	btrfs_free_path(path);
6168

6169
	ret = update_block_group(root, ins->objectid, ins->offset, 1);
6170
	if (ret) { /* -ENOENT, logic error */
C
Chris Mason 已提交
6171 6172 6173
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
6174 6175
		BUG();
	}
6176 6177 6178
	return ret;
}

6179 6180 6181 6182 6183
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins)
6184 6185
{
	int ret;
6186 6187 6188 6189 6190 6191 6192
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
6193

6194
	path = btrfs_alloc_path();
6195 6196
	if (!path)
		return -ENOMEM;
6197

6198 6199 6200
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6201 6202 6203 6204
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
	block_info = (struct btrfs_tree_block_info *)(extent_item + 1);

	btrfs_set_tree_block_key(leaf, block_info, key);
	btrfs_set_tree_block_level(leaf, block_info, level);

	iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

6233
	ret = update_block_group(root, ins->objectid, ins->offset, 1);
6234
	if (ret) { /* -ENOENT, logic error */
6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
		BUG();
	}
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

A
Arne Jansen 已提交
6252 6253 6254 6255
	ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
					 ins->offset, 0,
					 root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL, 0);
6256 6257
	return ret;
}
6258 6259 6260 6261 6262 6263

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
6264 6265 6266 6267
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
6268 6269 6270
{
	int ret;
	struct btrfs_block_group_cache *block_group;
6271 6272 6273
	struct btrfs_caching_control *caching_ctl;
	u64 start = ins->objectid;
	u64 num_bytes = ins->offset;
6274 6275

	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
6276
	cache_block_group(block_group, 0);
6277
	caching_ctl = get_caching_control(block_group);
6278

6279 6280 6281
	if (!caching_ctl) {
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
6282
		BUG_ON(ret); /* -ENOMEM */
6283 6284 6285 6286 6287
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
6288
			BUG_ON(ret); /* -ENOMEM */
6289 6290 6291
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
6292
			BUG_ON(ret); /* -ENOMEM */
6293 6294 6295 6296
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
6297
			BUG_ON(ret); /* -ENOMEM */
6298 6299 6300 6301 6302

			start = caching_ctl->progress;
			num_bytes = ins->objectid + ins->offset -
				    caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
6303
			BUG_ON(ret); /* -ENOMEM */
6304 6305 6306 6307 6308 6309
		}

		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}

6310 6311
	ret = btrfs_update_reserved_bytes(block_group, ins->offset,
					  RESERVE_ALLOC_NO_ACCOUNT);
6312
	BUG_ON(ret); /* logic error */
6313
	btrfs_put_block_group(block_group);
6314 6315
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
6316 6317 6318
	return ret;
}

6319 6320
struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
					    struct btrfs_root *root,
6321 6322
					    u64 bytenr, u32 blocksize,
					    int level)
6323 6324 6325 6326 6327 6328 6329
{
	struct extent_buffer *buf;

	buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
6330
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
6331 6332
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
6333
	clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
6334 6335

	btrfs_set_lock_blocking(buf);
6336
	btrfs_set_buffer_uptodate(buf);
6337

6338
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
6339 6340 6341 6342 6343 6344 6345 6346 6347 6348
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
		if (root->log_transid % 2 == 0)
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
6349 6350
	} else {
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
6351
			 buf->start + buf->len - 1, GFP_NOFS);
6352
	}
6353
	trans->blocks_used++;
6354
	/* this returns a buffer locked for blocking */
6355 6356 6357
	return buf;
}

6358 6359 6360 6361 6362
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
6363
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
6364 6365 6366 6367 6368
	int ret;

	block_rsv = get_block_rsv(trans, root);

	if (block_rsv->size == 0) {
M
Miao Xie 已提交
6369 6370
		ret = reserve_metadata_bytes(root, block_rsv, blocksize,
					     BTRFS_RESERVE_NO_FLUSH);
6371 6372 6373 6374 6375 6376 6377 6378
		/*
		 * If we couldn't reserve metadata bytes try and use some from
		 * the global reserve.
		 */
		if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
6379
			return ERR_PTR(ret);
6380
		} else if (ret) {
6381
			return ERR_PTR(ret);
6382
		}
6383 6384 6385 6386 6387 6388
		return block_rsv;
	}

	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;
6389
	if (ret && !block_rsv->failfast) {
6390 6391 6392
		static DEFINE_RATELIMIT_STATE(_rs,
				DEFAULT_RATELIMIT_INTERVAL,
				/*DEFAULT_RATELIMIT_BURST*/ 2);
J
Julia Lawall 已提交
6393 6394 6395
		if (__ratelimit(&_rs))
			WARN(1, KERN_DEBUG "btrfs: block rsv returned %d\n",
			     ret);
M
Miao Xie 已提交
6396 6397
		ret = reserve_metadata_bytes(root, block_rsv, blocksize,
					     BTRFS_RESERVE_NO_FLUSH);
6398 6399 6400 6401 6402 6403 6404 6405
		if (!ret) {
			return block_rsv;
		} else if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
		}
	}
6406 6407 6408 6409

	return ERR_PTR(-ENOSPC);
}

J
Josef Bacik 已提交
6410 6411
static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
			    struct btrfs_block_rsv *block_rsv, u32 blocksize)
6412 6413
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
J
Josef Bacik 已提交
6414
	block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
6415 6416
}

6417
/*
6418 6419 6420 6421
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
6422 6423
 * returns the tree buffer or NULL.
 */
6424
struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
6425 6426 6427
					struct btrfs_root *root, u32 blocksize,
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
6428
					u64 hint, u64 empty_size)
6429
{
C
Chris Mason 已提交
6430
	struct btrfs_key ins;
6431
	struct btrfs_block_rsv *block_rsv;
6432
	struct extent_buffer *buf;
6433 6434 6435
	u64 flags = 0;
	int ret;

6436

6437 6438 6439 6440 6441
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

	ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
6442
				   empty_size, hint, &ins, 0);
6443
	if (ret) {
J
Josef Bacik 已提交
6444
		unuse_block_rsv(root->fs_info, block_rsv, blocksize);
6445
		return ERR_PTR(ret);
6446
	}
6447

6448 6449
	buf = btrfs_init_new_buffer(trans, root, ins.objectid,
				    blocksize, level);
6450
	BUG_ON(IS_ERR(buf)); /* -ENOMEM */
6451 6452 6453 6454 6455 6456 6457 6458 6459 6460

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
6461
		extent_op = btrfs_alloc_delayed_extent_op();
6462
		BUG_ON(!extent_op); /* -ENOMEM */
6463 6464 6465 6466 6467 6468 6469 6470 6471
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
		extent_op->update_key = 1;
		extent_op->update_flags = 1;
		extent_op->is_data = 0;

A
Arne Jansen 已提交
6472 6473
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					ins.objectid,
6474 6475
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
6476
					extent_op, 0);
6477
		BUG_ON(ret); /* -ENOMEM */
6478
	}
6479 6480
	return buf;
}
6481

6482 6483 6484 6485 6486 6487 6488 6489 6490
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
6491 6492
	int reada_slot;
	int reada_count;
A
Arne Jansen 已提交
6493
	int for_reloc;
6494 6495 6496 6497 6498
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
6499 6500 6501 6502
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
6503
{
Y
Yan, Zheng 已提交
6504 6505 6506
	u64 bytenr;
	u64 generation;
	u64 refs;
6507
	u64 flags;
6508
	u32 nritems;
Y
Yan, Zheng 已提交
6509 6510 6511
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
6512
	int ret;
Y
Yan, Zheng 已提交
6513 6514
	int slot;
	int nread = 0;
6515

Y
Yan, Zheng 已提交
6516 6517 6518 6519 6520 6521 6522 6523
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
6524

Y
Yan, Zheng 已提交
6525 6526 6527
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
	blocksize = btrfs_level_size(root, wc->level - 1);
6528

Y
Yan, Zheng 已提交
6529 6530 6531
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
6532

C
Chris Mason 已提交
6533
		cond_resched();
Y
Yan, Zheng 已提交
6534 6535
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
6536

Y
Yan, Zheng 已提交
6537 6538
		if (slot == path->slots[wc->level])
			goto reada;
6539

Y
Yan, Zheng 已提交
6540 6541
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
6542 6543
			continue;

6544 6545 6546
		/* We don't lock the tree block, it's OK to be racy here */
		ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
					       &refs, &flags);
6547 6548 6549
		/* We don't care about errors in readahead. */
		if (ret < 0)
			continue;
6550 6551
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
6552 6553 6554
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
6555

6556 6557 6558
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
6559 6560 6561 6562 6563 6564 6565 6566
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
6567 6568 6569 6570
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
6571
		}
Y
Yan, Zheng 已提交
6572 6573 6574 6575
reada:
		ret = readahead_tree_block(root, bytenr, blocksize,
					   generation);
		if (ret)
6576
			break;
Y
Yan, Zheng 已提交
6577
		nread++;
C
Chris Mason 已提交
6578
	}
Y
Yan, Zheng 已提交
6579
	wc->reada_slot = slot;
C
Chris Mason 已提交
6580
}
6581

Y
Yan Zheng 已提交
6582
/*
6583 6584 6585 6586 6587 6588
 * hepler to process tree block while walking down the tree.
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
6589
 */
6590
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
6591
				   struct btrfs_root *root,
6592
				   struct btrfs_path *path,
6593
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
6594
{
6595 6596 6597
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
6598 6599
	int ret;

6600 6601 6602
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
6603

6604 6605 6606 6607
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
6608 6609 6610
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
6611 6612 6613 6614 6615
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
					       eb->start, eb->len,
					       &wc->refs[level],
					       &wc->flags[level]);
6616 6617 6618
		BUG_ON(ret == -ENOMEM);
		if (ret)
			return ret;
6619 6620
		BUG_ON(wc->refs[level] == 0);
	}
6621

6622 6623 6624
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
6625

6626
		if (path->locks[level] && !wc->keep_locks) {
6627
			btrfs_tree_unlock_rw(eb, path->locks[level]);
6628 6629 6630 6631
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
6632

6633 6634 6635
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
A
Arne Jansen 已提交
6636
		ret = btrfs_inc_ref(trans, root, eb, 1, wc->for_reloc);
6637
		BUG_ON(ret); /* -ENOMEM */
A
Arne Jansen 已提交
6638
		ret = btrfs_dec_ref(trans, root, eb, 0, wc->for_reloc);
6639
		BUG_ON(ret); /* -ENOMEM */
6640 6641
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
						  eb->len, flag, 0);
6642
		BUG_ON(ret); /* -ENOMEM */
6643 6644 6645 6646 6647 6648 6649 6650
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
6651
		btrfs_tree_unlock_rw(eb, path->locks[level]);
6652 6653 6654 6655 6656
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672
/*
 * hepler to process tree block pointer.
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
6673
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
6693 6694
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6695
		return 1;
6696
	}
Y
Yan, Zheng 已提交
6697 6698 6699 6700 6701 6702 6703

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
	blocksize = btrfs_level_size(root, level - 1);

	next = btrfs_find_tree_block(root, bytenr, blocksize);
	if (!next) {
		next = btrfs_find_create_tree_block(root, bytenr, blocksize);
6704 6705
		if (!next)
			return -ENOMEM;
Y
Yan, Zheng 已提交
6706 6707 6708 6709 6710
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

6711 6712 6713
	ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
6714 6715 6716 6717 6718
	if (ret < 0) {
		btrfs_tree_unlock(next);
		return ret;
	}

6719 6720
	BUG_ON(wc->refs[level - 1] == 0);
	*lookup_info = 0;
Y
Yan, Zheng 已提交
6721

6722
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
6723
		if (wc->refs[level - 1] > 1) {
6724 6725 6726 6727
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
6741 6742 6743 6744
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
6745 6746
	}

6747
	if (!btrfs_buffer_uptodate(next, generation, 0)) {
Y
Yan, Zheng 已提交
6748 6749 6750
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
6751
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6752 6753 6754 6755 6756 6757
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
		next = read_tree_block(root, bytenr, blocksize, generation);
6758 6759
		if (!next)
			return -EIO;
Y
Yan, Zheng 已提交
6760 6761 6762 6763 6764 6765 6766 6767
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
6768
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Y
Yan, Zheng 已提交
6769 6770 6771 6772 6773 6774 6775
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
6776 6777 6778 6779 6780 6781 6782 6783
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
6784

6785
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
A
Arne Jansen 已提交
6786
				root->root_key.objectid, level - 1, 0, 0);
6787
		BUG_ON(ret); /* -ENOMEM */
Y
Yan, Zheng 已提交
6788 6789 6790
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
6791
	*lookup_info = 1;
Y
Yan, Zheng 已提交
6792 6793 6794
	return 1;
}

6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811
/*
 * hepler to process tree block while walking up the tree.
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
6812
	int ret;
6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6839
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6840 6841 6842 6843 6844

			ret = btrfs_lookup_extent_info(trans, root,
						       eb->start, eb->len,
						       &wc->refs[level],
						       &wc->flags[level]);
6845 6846
			if (ret < 0) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
L
Liu Bo 已提交
6847
				path->locks[level] = 0;
6848 6849
				return ret;
			}
6850 6851
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
6852
				btrfs_tree_unlock_rw(eb, path->locks[level]);
L
Liu Bo 已提交
6853
				path->locks[level] = 0;
6854 6855
				return 1;
			}
Y
Yan Zheng 已提交
6856
		}
6857
	}
Y
Yan Zheng 已提交
6858

6859 6860
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6861

6862 6863 6864
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
A
Arne Jansen 已提交
6865 6866
				ret = btrfs_dec_ref(trans, root, eb, 1,
						    wc->for_reloc);
6867
			else
A
Arne Jansen 已提交
6868 6869
				ret = btrfs_dec_ref(trans, root, eb, 0,
						    wc->for_reloc);
6870
			BUG_ON(ret); /* -ENOMEM */
6871 6872 6873 6874 6875 6876
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6877
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
6894 6895
	}

6896
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
6897 6898 6899
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
6900
	return 0;
6901 6902 6903 6904 6905 6906 6907 6908
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
6909
	int lookup_info = 1;
6910 6911 6912
	int ret;

	while (level >= 0) {
6913
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
6914 6915 6916 6917 6918 6919
		if (ret > 0)
			break;

		if (level == 0)
			break;

6920 6921 6922 6923
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

6924
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
6925 6926 6927
		if (ret > 0) {
			path->slots[level]++;
			continue;
6928 6929
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
6930
		level = wc->level;
Y
Yan Zheng 已提交
6931 6932 6933 6934
	}
	return 0;
}

C
Chris Mason 已提交
6935
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6936
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
6937
				 struct btrfs_path *path,
6938
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
6939
{
6940
	int level = wc->level;
C
Chris Mason 已提交
6941
	int ret;
6942

6943 6944 6945 6946 6947 6948
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
6949 6950
			return 0;
		} else {
6951 6952 6953
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
6954

6955
			if (path->locks[level]) {
6956 6957
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
6958
				path->locks[level] = 0;
Y
Yan Zheng 已提交
6959
			}
6960 6961 6962
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
6963 6964 6965 6966 6967
		}
	}
	return 1;
}

C
Chris Mason 已提交
6968
/*
6969 6970 6971 6972 6973 6974 6975 6976 6977
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
C
Chris Mason 已提交
6978
 */
6979
int btrfs_drop_snapshot(struct btrfs_root *root,
A
Arne Jansen 已提交
6980 6981
			 struct btrfs_block_rsv *block_rsv, int update_ref,
			 int for_reloc)
C
Chris Mason 已提交
6982
{
6983
	struct btrfs_path *path;
6984 6985
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
6986
	struct btrfs_root_item *root_item = &root->root_item;
6987 6988 6989 6990 6991
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
C
Chris Mason 已提交
6992

6993
	path = btrfs_alloc_path();
6994 6995 6996 6997
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
C
Chris Mason 已提交
6998

6999
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
7000 7001
	if (!wc) {
		btrfs_free_path(path);
7002 7003
		err = -ENOMEM;
		goto out;
7004
	}
7005

7006
	trans = btrfs_start_transaction(tree_root, 0);
7007 7008 7009 7010
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
		goto out_free;
	}
7011

7012 7013
	if (block_rsv)
		trans->block_rsv = block_rsv;
7014

7015
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
7016
		level = btrfs_header_level(root->node);
7017 7018
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
7019
		path->slots[level] = 0;
7020
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7021 7022
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
7023 7024
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
7025 7026 7027
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

7028
		level = root_item->drop_level;
7029
		BUG_ON(level == 0);
7030
		path->lowest_level = level;
7031 7032 7033 7034
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
7035
			goto out_end_trans;
7036
		}
Y
Yan, Zheng 已提交
7037
		WARN_ON(ret > 0);
7038

7039 7040 7041 7042
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
7043
		btrfs_unlock_up_safe(path, 0);
7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
						path->nodes[level]->len,
						&wc->refs[level],
						&wc->flags[level]);
7055 7056 7057 7058
			if (ret < 0) {
				err = ret;
				goto out_end_trans;
			}
7059 7060 7061 7062 7063 7064 7065 7066 7067
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
7068
	}
7069 7070 7071 7072 7073 7074

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
A
Arne Jansen 已提交
7075
	wc->for_reloc = for_reloc;
Y
Yan, Zheng 已提交
7076
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
7077

C
Chris Mason 已提交
7078
	while (1) {
7079 7080 7081
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
7082
			break;
7083
		}
C
Chris Mason 已提交
7084

7085 7086 7087
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
7088
			break;
7089 7090 7091 7092
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
7093 7094
			break;
		}
7095 7096 7097 7098 7099 7100 7101 7102 7103 7104

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
7105
		if (btrfs_should_end_transaction(trans, tree_root)) {
7106 7107 7108
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
7109 7110 7111 7112 7113
			if (ret) {
				btrfs_abort_transaction(trans, tree_root, ret);
				err = ret;
				goto out_end_trans;
			}
7114

7115
			btrfs_end_transaction_throttle(trans, tree_root);
7116
			trans = btrfs_start_transaction(tree_root, 0);
7117 7118 7119 7120
			if (IS_ERR(trans)) {
				err = PTR_ERR(trans);
				goto out_free;
			}
7121 7122
			if (block_rsv)
				trans->block_rsv = block_rsv;
7123
		}
C
Chris Mason 已提交
7124
	}
7125
	btrfs_release_path(path);
7126 7127
	if (err)
		goto out_end_trans;
7128 7129

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
7130 7131 7132 7133
	if (ret) {
		btrfs_abort_transaction(trans, tree_root, ret);
		goto out_end_trans;
	}
7134

7135 7136 7137
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
		ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
					   NULL, NULL);
7138 7139 7140 7141 7142
		if (ret < 0) {
			btrfs_abort_transaction(trans, tree_root, ret);
			err = ret;
			goto out_end_trans;
		} else if (ret > 0) {
7143 7144 7145 7146 7147 7148 7149
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
7150 7151 7152 7153 7154 7155 7156 7157 7158 7159
		}
	}

	if (root->in_radix) {
		btrfs_free_fs_root(tree_root->fs_info, root);
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
		kfree(root);
	}
7160
out_end_trans:
7161
	btrfs_end_transaction_throttle(trans, tree_root);
7162
out_free:
7163
	kfree(wc);
7164
	btrfs_free_path(path);
7165 7166 7167
out:
	if (err)
		btrfs_std_error(root->fs_info, err);
7168
	return err;
C
Chris Mason 已提交
7169
}
C
Chris Mason 已提交
7170

7171 7172 7173 7174
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
A
Arne Jansen 已提交
7175
 * only used by relocation code
7176
 */
Y
Yan Zheng 已提交
7177 7178 7179 7180 7181 7182
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
7183
	struct walk_control *wc;
Y
Yan Zheng 已提交
7184 7185 7186 7187 7188
	int level;
	int parent_level;
	int ret = 0;
	int wret;

7189 7190
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
7191
	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
7192 7193
	if (!path)
		return -ENOMEM;
Y
Yan Zheng 已提交
7194

7195
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
T
Tsutomu Itoh 已提交
7196 7197 7198 7199
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
7200

7201
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
7202 7203 7204 7205 7206
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

7207
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
7208 7209 7210
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
7211
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7212 7213 7214 7215 7216 7217 7218 7219

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
A
Arne Jansen 已提交
7220
	wc->for_reloc = 1;
Y
Yan, Zheng 已提交
7221
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
7222 7223

	while (1) {
7224 7225
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
7226 7227
			ret = wret;
			break;
7228
		}
Y
Yan Zheng 已提交
7229

7230
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
7231 7232 7233 7234 7235 7236
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

7237
	kfree(wc);
Y
Yan Zheng 已提交
7238 7239 7240 7241
	btrfs_free_path(path);
	return ret;
}

7242 7243 7244
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
7245
	u64 stripped;
7246

7247 7248 7249 7250 7251 7252 7253
	/*
	 * if restripe for this chunk_type is on pick target profile and
	 * return, otherwise do the usual balance
	 */
	stripped = get_restripe_target(root->fs_info, flags);
	if (stripped)
		return extended_to_chunk(stripped);
7254

7255 7256 7257 7258 7259 7260 7261 7262
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;

7263 7264 7265
	stripped = BTRFS_BLOCK_GROUP_RAID0 |
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

7290
		/* this is drive concat, leave it alone */
7291
	}
7292

7293 7294 7295
	return flags;
}

7296
static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
C
Chris Mason 已提交
7297
{
7298 7299
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
7300
	u64 min_allocable_bytes;
7301
	int ret = -ENOSPC;
C
Chris Mason 已提交
7302

7303

7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
		min_allocable_bytes = 1 * 1024 * 1024;
	else
		min_allocable_bytes = 0;

7316 7317
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
7318 7319 7320 7321 7322 7323

	if (cache->ro) {
		ret = 0;
		goto out;
	}

7324 7325 7326 7327
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
7328 7329
	    sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
	    min_allocable_bytes <= sinfo->total_bytes) {
7330 7331 7332 7333
		sinfo->bytes_readonly += num_bytes;
		cache->ro = 1;
		ret = 0;
	}
7334
out:
7335 7336 7337 7338
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
7339

7340 7341
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
7342

7343 7344 7345 7346
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
7347

7348
	BUG_ON(cache->ro);
C
Chris Mason 已提交
7349

C
Chris Mason 已提交
7350
	trans = btrfs_join_transaction(root);
7351 7352
	if (IS_ERR(trans))
		return PTR_ERR(trans);
7353

7354
	alloc_flags = update_block_group_flags(root, cache->flags);
7355
	if (alloc_flags != cache->flags) {
7356
		ret = do_chunk_alloc(trans, root, alloc_flags,
7357 7358 7359 7360
				     CHUNK_ALLOC_FORCE);
		if (ret < 0)
			goto out;
	}
7361

7362
	ret = set_block_group_ro(cache, 0);
7363 7364 7365
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
7366
	ret = do_chunk_alloc(trans, root, alloc_flags,
7367
			     CHUNK_ALLOC_FORCE);
7368 7369
	if (ret < 0)
		goto out;
7370
	ret = set_block_group_ro(cache, 0);
7371 7372 7373 7374
out:
	btrfs_end_transaction(trans, root);
	return ret;
}
7375

7376 7377 7378 7379
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
7380
	return do_chunk_alloc(trans, root, alloc_flags,
7381
			      CHUNK_ALLOC_FORCE);
7382 7383
}

7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439
/*
 * helper to account the unused space of all the readonly block group in the
 * list. takes mirrors into account.
 */
static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

	list_for_each_entry(block_group, groups_list, list) {
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}

	return free_bytes;
}

/*
 * helper to account the unused space of all the readonly block group in the
 * space_info. takes mirrors into account.
 */
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
{
	int i;
	u64 free_bytes = 0;

	spin_lock(&sinfo->lock);

	for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		if (!list_empty(&sinfo->block_groups[i]))
			free_bytes += __btrfs_get_ro_block_group_free_space(
						&sinfo->block_groups[i]);

	spin_unlock(&sinfo->lock);

	return free_bytes;
}

7440
void btrfs_set_block_group_rw(struct btrfs_root *root,
7441
			      struct btrfs_block_group_cache *cache)
7442
{
7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
7456 7457
}

7458 7459 7460 7461 7462 7463 7464
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
7465
{
7466 7467 7468 7469
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
7470
	u64 min_free;
J
Josef Bacik 已提交
7471 7472
	u64 dev_min = 1;
	u64 dev_nr = 0;
7473
	u64 target;
7474
	int index;
7475 7476
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
7477

7478
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
7479

7480 7481 7482
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
Z
Zheng Yan 已提交
7483

7484 7485
	min_free = btrfs_block_group_used(&block_group->item);

7486
	/* no bytes used, we're good */
7487
	if (!min_free)
Z
Zheng Yan 已提交
7488 7489
		goto out;

7490 7491
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
7492

7493
	full = space_info->full;
7494

7495 7496
	/*
	 * if this is the last block group we have in this space, we can't
7497 7498 7499 7500
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
7501
	 */
7502
	if ((space_info->total_bytes != block_group->key.offset) &&
7503 7504 7505
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
7506 7507
		spin_unlock(&space_info->lock);
		goto out;
7508
	}
7509
	spin_unlock(&space_info->lock);
7510

7511 7512 7513
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
7514 7515 7516
	 * alloc devices and guess if we have enough space.  if this block
	 * group is going to be restriped, run checks against the target
	 * profile instead of the current one.
7517 7518
	 */
	ret = -1;
7519

7520 7521 7522 7523 7524 7525 7526 7527
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
7528 7529
	target = get_restripe_target(root->fs_info, block_group->flags);
	if (target) {
7530
		index = __get_raid_index(extended_to_chunk(target));
7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541
	} else {
		/*
		 * this is just a balance, so if we were marked as full
		 * we know there is no space for a new chunk
		 */
		if (full)
			goto out;

		index = get_block_group_index(block_group);
	}

7542
	if (index == BTRFS_RAID_RAID10) {
7543
		dev_min = 4;
J
Josef Bacik 已提交
7544 7545
		/* Divide by 2 */
		min_free >>= 1;
7546
	} else if (index == BTRFS_RAID_RAID1) {
7547
		dev_min = 2;
7548
	} else if (index == BTRFS_RAID_DUP) {
J
Josef Bacik 已提交
7549 7550
		/* Multiply by 2 */
		min_free <<= 1;
7551
	} else if (index == BTRFS_RAID_RAID0) {
7552
		dev_min = fs_devices->rw_devices;
J
Josef Bacik 已提交
7553
		do_div(min_free, dev_min);
7554 7555
	}

7556 7557
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
7558
		u64 dev_offset;
7559

7560 7561 7562 7563
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
7564 7565
		if (device->total_bytes > device->bytes_used + min_free &&
		    !device->is_tgtdev_for_dev_replace) {
7566
			ret = find_free_dev_extent(device, min_free,
7567
						   &dev_offset, NULL);
7568
			if (!ret)
7569 7570 7571
				dev_nr++;

			if (dev_nr >= dev_min)
7572
				break;
7573

7574
			ret = -1;
7575
		}
7576
	}
7577
	mutex_unlock(&root->fs_info->chunk_mutex);
7578
out:
7579
	btrfs_put_block_group(block_group);
7580 7581 7582
	return ret;
}

7583 7584
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
7585
{
7586
	int ret = 0;
7587 7588 7589
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
7590

7591 7592
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
7593 7594
		goto out;

C
Chris Mason 已提交
7595
	while (1) {
7596
		slot = path->slots[0];
7597
		leaf = path->nodes[0];
7598 7599 7600 7601 7602
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
7603
				goto out;
7604
			break;
7605
		}
7606
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
7607

7608
		if (found_key.objectid >= key->objectid &&
7609 7610 7611 7612
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
7613
		path->slots[0]++;
7614
	}
7615
out:
7616
	return ret;
7617 7618
}

7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
7653 7654 7655
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
7656
	struct btrfs_space_info *space_info;
7657
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
7658 7659
	struct rb_node *n;

7660 7661 7662 7663 7664 7665 7666 7667 7668
	down_write(&info->extent_commit_sem);
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
	up_write(&info->extent_commit_sem);

Z
Zheng Yan 已提交
7669 7670 7671 7672 7673 7674
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
Y
Yan Zheng 已提交
7675 7676
		spin_unlock(&info->block_group_cache_lock);

7677
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
7678
		list_del(&block_group->list);
7679
		up_write(&block_group->space_info->groups_sem);
7680

J
Josef Bacik 已提交
7681
		if (block_group->cached == BTRFS_CACHE_STARTED)
7682
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
7683

7684 7685 7686 7687 7688 7689 7690
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
		if (block_group->cached == BTRFS_CACHE_NO)
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
7691
		btrfs_remove_free_space_cache(block_group);
7692
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
7693 7694

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
7695 7696
	}
	spin_unlock(&info->block_group_cache_lock);
7697 7698 7699 7700 7701 7702 7703 7704 7705

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

7706 7707
	release_global_block_rsv(info);

7708 7709 7710 7711
	while(!list_empty(&info->space_info)) {
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
7712
		if (space_info->bytes_pinned > 0 ||
7713 7714
		    space_info->bytes_reserved > 0 ||
		    space_info->bytes_may_use > 0) {
7715 7716 7717
			WARN_ON(1);
			dump_space_info(space_info, 0, 0);
		}
7718 7719 7720
		list_del(&space_info->list);
		kfree(space_info);
	}
Z
Zheng Yan 已提交
7721 7722 7723
	return 0;
}

7724 7725 7726 7727 7728 7729 7730 7731 7732 7733
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);

	down_write(&space_info->groups_sem);
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);
}

C
Chris Mason 已提交
7734 7735 7736 7737 7738
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
7739
	struct btrfs_fs_info *info = root->fs_info;
7740
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
7741 7742
	struct btrfs_key key;
	struct btrfs_key found_key;
7743
	struct extent_buffer *leaf;
7744 7745
	int need_clear = 0;
	u64 cache_gen;
7746

C
Chris Mason 已提交
7747
	root = info->extent_root;
C
Chris Mason 已提交
7748
	key.objectid = 0;
7749
	key.offset = 0;
C
Chris Mason 已提交
7750 7751 7752 7753
	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
J
Josef Bacik 已提交
7754
	path->reada = 1;
C
Chris Mason 已提交
7755

7756
	cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
7757
	if (btrfs_test_opt(root, SPACE_CACHE) &&
7758
	    btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
7759
		need_clear = 1;
7760 7761
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
7762

C
Chris Mason 已提交
7763
	while (1) {
7764
		ret = find_first_block_group(root, path, &key);
7765 7766
		if (ret > 0)
			break;
7767 7768
		if (ret != 0)
			goto error;
7769 7770
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7771
		cache = kzalloc(sizeof(*cache), GFP_NOFS);
C
Chris Mason 已提交
7772
		if (!cache) {
7773
			ret = -ENOMEM;
7774
			goto error;
C
Chris Mason 已提交
7775
		}
7776 7777 7778 7779 7780 7781 7782
		cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
						GFP_NOFS);
		if (!cache->free_space_ctl) {
			kfree(cache);
			ret = -ENOMEM;
			goto error;
		}
C
Chris Mason 已提交
7783

7784
		atomic_set(&cache->count, 1);
7785
		spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7786
		cache->fs_info = info;
J
Josef Bacik 已提交
7787
		INIT_LIST_HEAD(&cache->list);
7788
		INIT_LIST_HEAD(&cache->cluster_list);
7789

7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800
		if (need_clear) {
			/*
			 * When we mount with old space cache, we need to
			 * set BTRFS_DC_CLEAR and set dirty flag.
			 *
			 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
			 *    truncate the old free space cache inode and
			 *    setup a new one.
			 * b) Setting 'dirty flag' makes sure that we flush
			 *    the new space cache info onto disk.
			 */
7801
			cache->disk_cache_state = BTRFS_DC_CLEAR;
7802 7803 7804
			if (btrfs_test_opt(root, SPACE_CACHE))
				cache->dirty = 1;
		}
7805

7806 7807 7808
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
C
Chris Mason 已提交
7809
		memcpy(&cache->key, &found_key, sizeof(found_key));
7810

C
Chris Mason 已提交
7811
		key.objectid = found_key.objectid + found_key.offset;
7812
		btrfs_release_path(path);
7813
		cache->flags = btrfs_block_group_flags(&cache->item);
J
Josef Bacik 已提交
7814 7815
		cache->sectorsize = root->sectorsize;

7816 7817
		btrfs_init_free_space_ctl(cache);

7818 7819 7820 7821 7822 7823 7824
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
		exclude_super_stripes(root, cache);

J
Josef Bacik 已提交
7825 7826 7827 7828 7829 7830 7831 7832
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
7833
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7834
			cache->cached = BTRFS_CACHE_FINISHED;
7835
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7836
		} else if (btrfs_block_group_used(&cache->item) == 0) {
7837
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7838 7839 7840 7841 7842
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
7843
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7844
		}
7845

7846 7847 7848
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
7849
		BUG_ON(ret); /* -ENOMEM */
7850
		cache->space_info = space_info;
7851
		spin_lock(&cache->space_info->lock);
7852
		cache->space_info->bytes_readonly += cache->bytes_super;
7853 7854
		spin_unlock(&cache->space_info->lock);

7855
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
7856 7857

		ret = btrfs_add_block_group_cache(root->fs_info, cache);
7858
		BUG_ON(ret); /* Logic error */
7859 7860

		set_avail_alloc_bits(root->fs_info, cache->flags);
Y
Yan Zheng 已提交
7861
		if (btrfs_chunk_readonly(root, cache->key.objectid))
7862
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7863
	}
7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
		list_for_each_entry(cache, &space_info->block_groups[3], list)
7876
			set_block_group_ro(cache, 1);
7877
		list_for_each_entry(cache, &space_info->block_groups[4], list)
7878
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7879
	}
7880 7881

	init_global_block_rsv(info);
7882 7883
	ret = 0;
error:
C
Chris Mason 已提交
7884
	btrfs_free_path(path);
7885
	return ret;
C
Chris Mason 已提交
7886
}
7887

7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915
void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root)
{
	struct btrfs_block_group_cache *block_group, *tmp;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
	struct btrfs_block_group_item item;
	struct btrfs_key key;
	int ret = 0;

	list_for_each_entry_safe(block_group, tmp, &trans->new_bgs,
				 new_bg_list) {
		list_del_init(&block_group->new_bg_list);

		if (ret)
			continue;

		spin_lock(&block_group->lock);
		memcpy(&item, &block_group->item, sizeof(item));
		memcpy(&key, &block_group->key, sizeof(key));
		spin_unlock(&block_group->lock);

		ret = btrfs_insert_item(trans, extent_root, &key, &item,
					sizeof(item));
		if (ret)
			btrfs_abort_transaction(trans, extent_root, ret);
	}
}

7916 7917
int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
7918
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
7919 7920 7921 7922 7923 7924 7925 7926
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

7927
	root->fs_info->last_trans_log_full_commit = trans->transid;
7928

7929
	cache = kzalloc(sizeof(*cache), GFP_NOFS);
J
Josef Bacik 已提交
7930 7931
	if (!cache)
		return -ENOMEM;
7932 7933 7934 7935 7936 7937
	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return -ENOMEM;
	}
J
Josef Bacik 已提交
7938

7939
	cache->key.objectid = chunk_offset;
7940
	cache->key.offset = size;
7941
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
7942
	cache->sectorsize = root->sectorsize;
7943
	cache->fs_info = root->fs_info;
7944

7945
	atomic_set(&cache->count, 1);
7946
	spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7947
	INIT_LIST_HEAD(&cache->list);
7948
	INIT_LIST_HEAD(&cache->cluster_list);
7949
	INIT_LIST_HEAD(&cache->new_bg_list);
C
Chris Mason 已提交
7950

7951 7952
	btrfs_init_free_space_ctl(cache);

7953 7954 7955 7956 7957
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	cache->flags = type;
	btrfs_set_block_group_flags(&cache->item, type);

7958
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7959
	cache->cached = BTRFS_CACHE_FINISHED;
7960
	exclude_super_stripes(root, cache);
7961

J
Josef Bacik 已提交
7962 7963 7964
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

7965 7966
	free_excluded_extents(root, cache);

7967 7968
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
7969
	BUG_ON(ret); /* -ENOMEM */
7970
	update_global_block_rsv(root->fs_info);
7971 7972

	spin_lock(&cache->space_info->lock);
7973
	cache->space_info->bytes_readonly += cache->bytes_super;
7974 7975
	spin_unlock(&cache->space_info->lock);

7976
	__link_block_group(cache->space_info, cache);
7977

J
Josef Bacik 已提交
7978
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
7979
	BUG_ON(ret); /* Logic error */
7980

7981
	list_add_tail(&cache->new_bg_list, &trans->new_bgs);
7982

C
Chris Mason 已提交
7983
	set_avail_alloc_bits(extent_root->fs_info, type);
7984

7985 7986
	return 0;
}
Z
Zheng Yan 已提交
7987

7988 7989
static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
7990 7991
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
7992

7993
	write_seqlock(&fs_info->profiles_lock);
7994 7995 7996 7997 7998 7999
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits &= ~extra_flags;
8000
	write_sequnlock(&fs_info->profiles_lock);
8001 8002
}

Z
Zheng Yan 已提交
8003 8004 8005 8006 8007
int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 group_start)
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
8008
	struct btrfs_free_cluster *cluster;
8009
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
8010
	struct btrfs_key key;
8011
	struct inode *inode;
Z
Zheng Yan 已提交
8012
	int ret;
8013
	int index;
J
Josef Bacik 已提交
8014
	int factor;
Z
Zheng Yan 已提交
8015 8016 8017 8018 8019

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
8020
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
8021

8022 8023 8024 8025 8026 8027
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

Z
Zheng Yan 已提交
8028
	memcpy(&key, &block_group->key, sizeof(key));
8029
	index = get_block_group_index(block_group);
J
Josef Bacik 已提交
8030 8031 8032 8033 8034 8035
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
8036

8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049 8050 8051
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
8052
	path = btrfs_alloc_path();
8053 8054 8055 8056
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
Z
Zheng Yan 已提交
8057

8058
	inode = lookup_free_space_inode(tree_root, block_group, path);
8059
	if (!IS_ERR(inode)) {
8060
		ret = btrfs_orphan_add(trans, inode);
8061 8062 8063 8064
		if (ret) {
			btrfs_add_delayed_iput(inode);
			goto out;
		}
8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
8077
		btrfs_add_delayed_iput(inode);
8078 8079 8080 8081 8082 8083 8084 8085 8086 8087
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
8088
		btrfs_release_path(path);
8089 8090 8091 8092
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
8093
		btrfs_release_path(path);
8094 8095
	}

8096
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
8097 8098
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
8099 8100 8101

	if (root->fs_info->first_logical_byte == block_group->key.objectid)
		root->fs_info->first_logical_byte = (u64)-1;
8102
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
8103

8104
	down_write(&block_group->space_info->groups_sem);
8105 8106 8107 8108 8109
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
8110 8111
	if (list_empty(&block_group->space_info->block_groups[index]))
		clear_avail_alloc_bits(root->fs_info, block_group->flags);
8112
	up_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
8113

J
Josef Bacik 已提交
8114
	if (block_group->cached == BTRFS_CACHE_STARTED)
8115
		wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
8116 8117 8118

	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
8119 8120 8121
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
8122
	block_group->space_info->disk_total -= block_group->key.offset * factor;
Y
Yan Zheng 已提交
8123
	spin_unlock(&block_group->space_info->lock);
8124

8125 8126
	memcpy(&key, &block_group->key, sizeof(key));

8127
	btrfs_clear_space_info_full(root->fs_info);
Y
Yan Zheng 已提交
8128

8129 8130
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
8143

8144 8145 8146
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
8147 8148 8149 8150
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
8151 8152
	int ret;

8153
	disk_super = fs_info->super_copy;
8154 8155
	if (!btrfs_super_root(disk_super))
		return 1;
8156

8157 8158 8159
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
8160

8161 8162
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
	ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8163
	if (ret)
8164
		goto out;
8165

8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	}
out:
8179 8180 8181
	return ret;
}

L
liubo 已提交
8182 8183 8184 8185 8186 8187
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
	return unpin_extent_range(root, start, end);
}

int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
8188
			       u64 num_bytes, u64 *actual_bytes)
L
liubo 已提交
8189
{
8190
	return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
L
liubo 已提交
8191
}
8192 8193 8194 8195 8196 8197 8198 8199 8200

int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
8201
	u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
8202 8203
	int ret = 0;

8204 8205 8206 8207 8208 8209 8210
	/*
	 * try to trim all FS space, our block group may start from non-zero.
	 */
	if (range->len == total_bytes)
		cache = btrfs_lookup_first_block_group(fs_info, range->start);
	else
		cache = btrfs_lookup_block_group(fs_info, range->start);
8211 8212 8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
8224
				ret = cache_block_group(cache, 0);
8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246
				if (!ret)
					wait_block_group_cache_done(cache);
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

	range->len = trimmed;
	return ret;
}