extent-tree.c 193.8 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
C
Chris Mason 已提交
26
#include "compat.h"
27
#include "hash.h"
28 29 30
#include "ctree.h"
#include "disk-io.h"
#include "print-tree.h"
31
#include "transaction.h"
32
#include "volumes.h"
33
#include "locking.h"
34
#include "free-space-cache.h"
35

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
/* control flags for do_chunk_alloc's force field
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
	CHUNK_ALLOC_FORCE = 1,
	CHUNK_ALLOC_LIMITED = 2,
};

55 56
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
57
			      u64 bytenr, u64 num_bytes, int alloc);
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extra_op);
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins);
J
Josef Bacik 已提交
77 78 79
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
			  u64 flags, int force);
80 81
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
82 83
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
J
Josef Bacik 已提交
84

J
Josef Bacik 已提交
85 86 87 88 89 90 91
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
	return cache->cached == BTRFS_CACHE_FINISHED;
}

J
Josef Bacik 已提交
92 93 94 95 96
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

97
static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
98 99 100 101 102 103
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
104 105 106 107
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
		WARN_ON(cache->reserved_pinned > 0);
108
		kfree(cache->free_space_ctl);
109
		kfree(cache);
110
	}
111 112
}

J
Josef Bacik 已提交
113 114 115 116
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
117
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
185
	if (ret)
186
		btrfs_get_block_group(ret);
J
Josef Bacik 已提交
187 188 189 190 191
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

192 193
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
194
{
195 196 197 198 199 200 201
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
J
Josef Bacik 已提交
202

203 204 205 206
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
207

208 209 210 211 212 213 214
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
J
Josef Bacik 已提交
215 216
}

217 218
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
219 220 221 222 223 224
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

225 226 227 228 229 230 231 232
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
		BUG_ON(ret);
	}

J
Josef Bacik 已提交
233 234 235 236 237 238
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
		BUG_ON(ret);
239

J
Josef Bacik 已提交
240
		while (nr--) {
241
			cache->bytes_super += stripe_len;
242 243 244
			ret = add_excluded_extent(root, logical[nr],
						  stripe_len);
			BUG_ON(ret);
J
Josef Bacik 已提交
245
		}
246

J
Josef Bacik 已提交
247 248 249 250 251
		kfree(logical);
	}
	return 0;
}

252 253 254 255 256 257 258 259 260 261 262
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_STARTED) {
		spin_unlock(&cache->lock);
		return NULL;
	}

263 264 265
	/* We're loading it the fast way, so we don't have a caching_ctl. */
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

J
Josef Bacik 已提交
281 282 283 284 285
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
J
Josef Bacik 已提交
286
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
J
Josef Bacik 已提交
287 288
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
J
Josef Bacik 已提交
289
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
290 291 292
	int ret;

	while (start < end) {
293
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
294
					    &extent_start, &extent_end,
295
					    EXTENT_DIRTY | EXTENT_UPTODATE);
J
Josef Bacik 已提交
296 297 298
		if (ret)
			break;

299
		if (extent_start <= start) {
J
Josef Bacik 已提交
300 301 302
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
303
			total_added += size;
304 305
			ret = btrfs_add_free_space(block_group, start,
						   size);
J
Josef Bacik 已提交
306 307 308 309 310 311 312 313 314
			BUG_ON(ret);
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
315
		total_added += size;
316
		ret = btrfs_add_free_space(block_group, start, size);
J
Josef Bacik 已提交
317 318 319
		BUG_ON(ret);
	}

J
Josef Bacik 已提交
320
	return total_added;
J
Josef Bacik 已提交
321 322
}

323
static noinline void caching_thread(struct btrfs_work *work)
324
{
325 326 327 328
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_root *extent_root;
329
	struct btrfs_path *path;
330
	struct extent_buffer *leaf;
331
	struct btrfs_key key;
J
Josef Bacik 已提交
332
	u64 total_found = 0;
333 334 335
	u64 last = 0;
	u32 nritems;
	int ret = 0;
336

337 338 339 340 341
	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

342 343
	path = btrfs_alloc_path();
	if (!path)
344
		goto out;
345

J
Josef Bacik 已提交
346
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
347

348
	/*
J
Josef Bacik 已提交
349 350 351 352
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
353 354
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
355
	path->search_commit_root = 1;
J
Josef Bacik 已提交
356
	path->reada = 1;
J
Josef Bacik 已提交
357

Y
Yan Zheng 已提交
358
	key.objectid = last;
359
	key.offset = 0;
360
	key.type = BTRFS_EXTENT_ITEM_KEY;
361
again:
362
	mutex_lock(&caching_ctl->mutex);
363 364 365
	/* need to make sure the commit_root doesn't disappear */
	down_read(&fs_info->extent_commit_sem);

366
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
367
	if (ret < 0)
368
		goto err;
Y
Yan Zheng 已提交
369

370 371 372
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
373
	while (1) {
374
		if (btrfs_fs_closing(fs_info) > 1) {
375
			last = (u64)-1;
J
Josef Bacik 已提交
376
			break;
377
		}
J
Josef Bacik 已提交
378

379 380 381 382 383
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
384
				break;
J
Josef Bacik 已提交
385

386 387 388
			if (need_resched() ||
			    btrfs_next_leaf(extent_root, path)) {
				caching_ctl->progress = last;
C
Chris Mason 已提交
389
				btrfs_release_path(path);
390 391
				up_read(&fs_info->extent_commit_sem);
				mutex_unlock(&caching_ctl->mutex);
392
				cond_resched();
393 394 395 396 397
				goto again;
			}
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
398
		}
J
Josef Bacik 已提交
399

400 401
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
402
			continue;
403
		}
J
Josef Bacik 已提交
404

405
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
406
		    block_group->key.offset)
407
			break;
408

409
		if (key.type == BTRFS_EXTENT_ITEM_KEY) {
J
Josef Bacik 已提交
410 411 412
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
413
			last = key.objectid + key.offset;
J
Josef Bacik 已提交
414

415 416 417 418
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
J
Josef Bacik 已提交
419
		}
420 421
		path->slots[0]++;
	}
J
Josef Bacik 已提交
422
	ret = 0;
423

J
Josef Bacik 已提交
424 425 426
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
427
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
428 429

	spin_lock(&block_group->lock);
430
	block_group->caching_ctl = NULL;
J
Josef Bacik 已提交
431 432
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
433

434
err:
435
	btrfs_free_path(path);
436
	up_read(&fs_info->extent_commit_sem);
J
Josef Bacik 已提交
437

438 439 440
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
441
out:
442 443 444
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
445
	btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
446 447
}

448 449
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     struct btrfs_trans_handle *trans,
450
			     struct btrfs_root *root,
451
			     int load_cache_only)
J
Josef Bacik 已提交
452
{
453 454
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
455 456
	int ret = 0;

457 458 459 460
	smp_mb();
	if (cache->cached != BTRFS_CACHE_NO)
		return 0;

461 462
	/*
	 * We can't do the read from on-disk cache during a commit since we need
463 464 465
	 * to have the normal tree locking.  Also if we are currently trying to
	 * allocate blocks for the tree root we can't do the fast caching since
	 * we likely hold important locks.
466
	 */
467
	if (trans && (!trans->transaction->in_commit) &&
468
	    (root && root != root->fs_info->tree_root)) {
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
		spin_lock(&cache->lock);
		if (cache->cached != BTRFS_CACHE_NO) {
			spin_unlock(&cache->lock);
			return 0;
		}
		cache->cached = BTRFS_CACHE_STARTED;
		spin_unlock(&cache->lock);

		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
		} else {
			cache->cached = BTRFS_CACHE_NO;
		}
		spin_unlock(&cache->lock);
487 488
		if (ret == 1) {
			free_excluded_extents(fs_info->extent_root, cache);
489
			return 0;
490
		}
491 492 493 494 495
	}

	if (load_cache_only)
		return 0;

496
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
497 498 499 500 501 502 503 504 505
	BUG_ON(!caching_ctl);

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	/* one for caching kthread, one for caching block group list */
	atomic_set(&caching_ctl->count, 2);
506
	caching_ctl->work.func = caching_thread;
507

J
Josef Bacik 已提交
508 509 510
	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
511 512
		kfree(caching_ctl);
		return 0;
J
Josef Bacik 已提交
513
	}
514
	cache->caching_ctl = caching_ctl;
J
Josef Bacik 已提交
515 516 517
	cache->cached = BTRFS_CACHE_STARTED;
	spin_unlock(&cache->lock);

518 519 520 521
	down_write(&fs_info->extent_commit_sem);
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
	up_write(&fs_info->extent_commit_sem);

522
	btrfs_get_block_group(cache);
523

524
	btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
J
Josef Bacik 已提交
525

526
	return ret;
527 528
}

J
Josef Bacik 已提交
529 530 531
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
532 533
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
534
{
J
Josef Bacik 已提交
535
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
536

J
Josef Bacik 已提交
537
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
538

J
Josef Bacik 已提交
539
	return cache;
C
Chris Mason 已提交
540 541
}

J
Josef Bacik 已提交
542
/*
543
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
544
 */
C
Chris Mason 已提交
545 546 547
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
548
{
J
Josef Bacik 已提交
549
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
550

J
Josef Bacik 已提交
551
	cache = block_group_cache_tree_search(info, bytenr, 1);
552

J
Josef Bacik 已提交
553
	return cache;
C
Chris Mason 已提交
554
}
555

J
Josef Bacik 已提交
556 557
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
558
{
J
Josef Bacik 已提交
559 560
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
561

562 563 564
	flags &= BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_SYSTEM |
		 BTRFS_BLOCK_GROUP_METADATA;

565 566
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
567
		if (found->flags & flags) {
568
			rcu_read_unlock();
J
Josef Bacik 已提交
569
			return found;
570
		}
J
Josef Bacik 已提交
571
	}
572
	rcu_read_unlock();
J
Josef Bacik 已提交
573
	return NULL;
574 575
}

576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

591 592 593 594 595 596 597 598 599
static u64 div_factor(u64 num, int factor)
{
	if (factor == 10)
		return num;
	num *= factor;
	do_div(num, 10);
	return num;
}

600 601 602 603 604 605 606 607 608
static u64 div_factor_fine(u64 num, int factor)
{
	if (factor == 100)
		return num;
	num *= factor;
	do_div(num, 100);
	return num;
}

609 610
u64 btrfs_find_block_group(struct btrfs_root *root,
			   u64 search_start, u64 search_hint, int owner)
C
Chris Mason 已提交
611
{
612
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
613
	u64 used;
614 615
	u64 last = max(search_hint, search_start);
	u64 group_start = 0;
616
	int full_search = 0;
617
	int factor = 9;
C
Chris Mason 已提交
618
	int wrapped = 0;
619
again:
620 621
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
J
Josef Bacik 已提交
622 623
		if (!cache)
			break;
624

625
		spin_lock(&cache->lock);
626 627 628
		last = cache->key.objectid + cache->key.offset;
		used = btrfs_block_group_used(&cache->item);

629 630
		if ((full_search || !cache->ro) &&
		    block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
631
			if (used + cache->pinned + cache->reserved <
632 633
			    div_factor(cache->key.offset, factor)) {
				group_start = cache->key.objectid;
634
				spin_unlock(&cache->lock);
635
				btrfs_put_block_group(cache);
636 637
				goto found;
			}
638
		}
639
		spin_unlock(&cache->lock);
640
		btrfs_put_block_group(cache);
641
		cond_resched();
C
Chris Mason 已提交
642
	}
C
Chris Mason 已提交
643 644 645 646 647 648
	if (!wrapped) {
		last = search_start;
		wrapped = 1;
		goto again;
	}
	if (!full_search && factor < 10) {
C
Chris Mason 已提交
649
		last = search_start;
650
		full_search = 1;
C
Chris Mason 已提交
651
		factor = 10;
652 653
		goto again;
	}
C
Chris Mason 已提交
654
found:
655
	return group_start;
656
}
J
Josef Bacik 已提交
657

658
/* simple helper to search for an existing extent at a given offset */
Z
Zheng Yan 已提交
659
int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
660 661 662
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
663
	struct btrfs_path *path;
664

Z
Zheng Yan 已提交
665
	path = btrfs_alloc_path();
666 667 668
	if (!path)
		return -ENOMEM;

669 670 671 672 673
	key.objectid = start;
	key.offset = len;
	btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
674
	btrfs_free_path(path);
675 676 677
	return ret;
}

678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757
/*
 * helper function to lookup reference count and flags of extent.
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
			     u64 num_bytes, u64 *refs, u64 *flags)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = num_bytes;
	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
again:
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

758
			btrfs_release_path(path);
759

760 761 762 763
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
			goto again;
		}
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

789 790 791 792 793 794 795 796 797 798 799 800 801 802
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
848 849 850
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
851
 * - different files inside a single subvolume
852 853
 * - different offsets inside a file (bookend extents in file.c)
 *
854
 * The extent ref structure for the implicit back refs has fields for:
855 856 857
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
858 859
 * - original offset in the file
 * - how many bookend extents
860
 *
861 862
 * The key offset for the implicit back refs is hash of the first
 * three fields.
863
 *
864
 * The extent ref structure for the full back refs has field for:
865
 *
866
 * - number of pointers in the tree leaf
867
 *
868 869
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
870
 *
871 872
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
873
 *
874
 *     (root_key.objectid, inode objectid, offset in file, 1)
875
 *
876 877
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
878
 *
879
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
880
 *
881
 * Btree extents can be referenced by:
882
 *
883
 * - Different subvolumes
884
 *
885 886 887 888
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
889
 *
890 891 892
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
893
 */
Z
Zheng Yan 已提交
894

895 896 897 898 899
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
900
{
901 902 903 904 905
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
906
	struct btrfs_key key;
907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
				BUG_ON(ret > 0);
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
942
	btrfs_release_path(path);
943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
	BUG_ON(ret);

	ret = btrfs_extend_item(trans, root, path, new_size);

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
984
	high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
985
	lenum = cpu_to_le64(owner);
986
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
987
	lenum = cpu_to_le64(offset);
988
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1021
	struct extent_buffer *leaf;
1022
	u32 nritems;
1023
	int ret;
1024 1025
	int recow;
	int err = -ENOENT;
1026

Z
Zheng Yan 已提交
1027
	key.objectid = bytenr;
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1043

1044 1045 1046 1047 1048
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1049
		btrfs_release_path(path);
1050 1051 1052 1053 1054 1055 1056 1057 1058
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1059 1060 1061
	}

	leaf = path->nodes[0];
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1087
				btrfs_release_path(path);
1088 1089 1090 1091 1092 1093
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1094
	}
1095 1096
fail:
	return err;
Z
Zheng Yan 已提交
1097 1098
}

1099 1100 1101 1102 1103 1104
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1105 1106 1107
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1108
	u32 size;
Z
Zheng Yan 已提交
1109 1110
	u32 num_refs;
	int ret;
1111 1112

	key.objectid = bytenr;
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1123

1124 1125 1126 1127 1128 1129 1130
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1131
		ref = btrfs_item_ptr(leaf, path->slots[0],
1132 1133 1134 1135 1136 1137 1138
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1139
		}
1140 1141 1142 1143 1144 1145 1146 1147
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1148
			btrfs_release_path(path);
1149 1150 1151 1152 1153
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1154

1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1169 1170
		}
	}
1171 1172 1173
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1174
	btrfs_release_path(path);
1175
	return ret;
1176 1177
}

1178 1179 1180 1181
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   int refs_to_drop)
Z
Zheng Yan 已提交
1182
{
1183 1184 1185
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1186
	struct extent_buffer *leaf;
1187
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1188 1189 1190
	int ret = 0;

	leaf = path->nodes[0];
1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1212 1213
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1214

Z
Zheng Yan 已提交
1215 1216 1217
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
	} else {
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1230 1231 1232 1233 1234
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1235 1236 1237
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1238
{
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1270
#endif
1271 1272 1273 1274 1275
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1276

1277 1278 1279 1280 1281
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1282
{
1283
	struct btrfs_key key;
1284 1285
	int ret;

1286 1287 1288 1289 1290 1291 1292
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1293 1294
	}

1295 1296 1297 1298 1299
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1300
		btrfs_release_path(path);
1301 1302 1303 1304 1305
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1306
#endif
1307
	return ret;
1308 1309
}

1310 1311 1312 1313 1314
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1315
{
1316
	struct btrfs_key key;
Z
Zheng Yan 已提交
1317 1318
	int ret;

1319 1320 1321 1322 1323 1324 1325 1326 1327 1328
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1329
	btrfs_release_path(path);
Z
Zheng Yan 已提交
1330 1331 1332
	return ret;
}

1333
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1334
{
1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1348
}
1349

1350 1351
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1352

C
Chris Mason 已提交
1353
{
1354
	for (; level < BTRFS_MAX_LEVEL; level++) {
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1370

1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1406

1407
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1408
	key.type = BTRFS_EXTENT_ITEM_KEY;
1409
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1410

1411 1412 1413
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1414
		path->keep_locks = 1;
1415 1416 1417
	} else
		extra_size = -1;
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1418
	if (ret < 0) {
1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
		err = ret;
		goto out;
	}
	BUG_ON(ret);

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	} else {
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1516 1517
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1518
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1519 1520 1521 1522 1523 1524
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1525
	if (insert) {
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
int setup_inline_extent_backref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_path *path,
				struct btrfs_extent_inline_ref *iref,
				u64 parent, u64 root_objectid,
				u64 owner, u64 offset, int refs_to_add,
				struct btrfs_delayed_extent_op *extent_op)
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;
	int ret;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

	ret = btrfs_extend_item(trans, root, path, size);

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1612
		return ret;
1613

1614
	btrfs_release_path(path);
1615 1616 1617 1618 1619 1620 1621 1622
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1623
	}
1624 1625
	return ret;
}
Z
Zheng Yan 已提交
1626

1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
int update_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_mod,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	int ret;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1670
	}
Z
Zheng Yan 已提交
1671

1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
		ret = btrfs_truncate_item(trans, root, path, item_size, 1);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
		ret = update_inline_extent_backref(trans, root, path, iref,
						   refs_to_add, extent_op);
	} else if (ret == -ENOENT) {
		ret = setup_inline_extent_backref(trans, root, path, iref,
						  parent, root_objectid,
						  owner, offset, refs_to_add,
						  extent_op);
1719
	}
1720 1721
	return ret;
}
Z
Zheng Yan 已提交
1722

1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1741

1742 1743 1744 1745 1746 1747 1748
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_drop, int is_data)
{
	int ret;
1749

1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
		ret = update_inline_extent_backref(trans, root, path, iref,
						   -refs_to_drop, NULL);
	} else if (is_data) {
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
	} else {
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1762
static int btrfs_issue_discard(struct block_device *bdev,
1763 1764
				u64 start, u64 len)
{
1765
	return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1766 1767 1768
}

static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1769
				u64 num_bytes, u64 *actual_bytes)
1770 1771
{
	int ret;
1772
	u64 discarded_bytes = 0;
1773 1774
	struct btrfs_multi_bio *multi = NULL;

C
Christoph Hellwig 已提交
1775

1776
	/* Tell the block device(s) that the sectors can be discarded */
1777 1778
	ret = btrfs_map_block(&root->fs_info->mapping_tree, REQ_DISCARD,
			      bytenr, &num_bytes, &multi, 0);
1779 1780 1781 1782 1783 1784
	if (!ret) {
		struct btrfs_bio_stripe *stripe = multi->stripes;
		int i;


		for (i = 0; i < multi->num_stripes; i++, stripe++) {
1785 1786 1787
			if (!stripe->dev->can_discard)
				continue;

1788 1789 1790 1791 1792 1793 1794
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
						  stripe->length);
			if (!ret)
				discarded_bytes += stripe->length;
			else if (ret != -EOPNOTSUPP)
				break;
1795 1796 1797 1798 1799 1800 1801

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
1802 1803 1804
		}
		kfree(multi);
	}
1805 1806 1807 1808

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872

	return ret;
}

int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
			 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
					parent, root_objectid, (int)owner,
					BTRFS_ADD_DELAYED_REF, NULL);
	} else {
		ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
					parent, root_objectid, owner, offset,
					BTRFS_ADD_DELAYED_REF, NULL);
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
	u64 refs;
	int ret;
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
	ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
					   path, bytenr, num_bytes, parent,
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
	if (ret == 0)
		goto out;

	if (ret != -EAGAIN) {
		err = ret;
		goto out;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
1873

1874
	btrfs_mark_buffer_dirty(leaf);
1875
	btrfs_release_path(path);
1876 1877

	path->reada = 1;
1878 1879
	path->leave_spinning = 1;

1880 1881
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
1882 1883
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
1884
	BUG_ON(ret);
1885
out:
1886
	btrfs_free_path(path);
1887
	return err;
1888 1889
}

1890 1891 1892 1893 1894
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
1895
{
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		if (extent_op) {
			BUG_ON(extent_op->update_key);
			flags |= extent_op->flags_to_set;
		}
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
					     extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
					  extent_op);
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
1968
	int ret;
1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = node->num_bytes;

	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
		err = -EIO;
		goto out;
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2009

2010 2011 2012 2013
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2014 2015
}

2016 2017 2018 2019 2020
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2021 2022
{
	int ret = 0;
2023 2024 2025 2026
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2027

2028 2029 2030
	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;
2031

2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057
	ref = btrfs_delayed_node_to_tree_ref(node);
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		BUG_ON(!extent_op || !extent_op->update_flags ||
		       !extent_op->update_key);
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
						ref->level, &ins);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
					     ref->level, 0, 1, extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
					  ref->level, 0, 1, extent_op);
	} else {
		BUG();
	}
2058 2059 2060 2061
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2062 2063 2064 2065 2066
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2067 2068
{
	int ret;
2069
	if (btrfs_delayed_ref_is_head(node)) {
2070 2071 2072 2073 2074 2075 2076
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2077 2078
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2079
		if (insert_reserved) {
2080 2081
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2082 2083 2084 2085 2086 2087
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
				BUG_ON(ret);
			}
2088 2089 2090 2091 2092
		}
		mutex_unlock(&head->mutex);
		return 0;
	}

2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_node *ref;
	int action = BTRFS_ADD_DELAYED_REF;
again:
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
	node = rb_prev(&head->node.rb_node);
	while (1) {
		if (!node)
			break;
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
		if (ref->bytenr != head->node.bytenr)
			break;
2126
		if (ref->action == action)
2127 2128 2129 2130 2131 2132 2133 2134 2135 2136
			return ref;
		node = rb_prev(node);
	}
	if (action == BTRFS_ADD_DELAYED_REF) {
		action = BTRFS_DROP_DELAYED_REF;
		goto again;
	}
	return NULL;
}

2137 2138 2139
static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root,
				       struct list_head *cluster)
2140 2141 2142 2143
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2144
	struct btrfs_delayed_extent_op *extent_op;
2145
	int ret;
2146
	int count = 0;
2147 2148 2149 2150 2151
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2152 2153
			/* pick a new head ref from the cluster list */
			if (list_empty(cluster))
2154 2155
				break;

2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172
			locked_ref = list_entry(cluster->next,
				     struct btrfs_delayed_ref_head, cluster);

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);

			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2173 2174
			}
		}
2175

2176 2177 2178 2179 2180 2181
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2182

2183 2184 2185
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);
		if (!ref) {
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215

			if (extent_op && must_insert_reserved) {
				kfree(extent_op);
				extent_op = NULL;
			}

			if (extent_op) {
				spin_unlock(&delayed_refs->lock);

				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
				BUG_ON(ret);
				kfree(extent_op);

				cond_resched();
				spin_lock(&delayed_refs->lock);
				continue;
			}

2216
			list_del_init(&locked_ref->cluster);
2217 2218
			locked_ref = NULL;
		}
C
Chris Mason 已提交
2219

2220 2221 2222
		ref->in_tree = 0;
		rb_erase(&ref->rb_node, &delayed_refs->root);
		delayed_refs->num_entries--;
2223

2224
		spin_unlock(&delayed_refs->lock);
2225

2226
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2227 2228
					  must_insert_reserved);
		BUG_ON(ret);
2229

2230 2231
		btrfs_put_delayed_ref(ref);
		kfree(extent_op);
2232
		count++;
2233

2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272
		cond_resched();
		spin_lock(&delayed_refs->lock);
	}
	return count;
}

/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct list_head cluster;
	int ret;
	int run_all = count == (unsigned long)-1;
	int run_most = 0;

	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

	delayed_refs = &trans->transaction->delayed_refs;
	INIT_LIST_HEAD(&cluster);
again:
	spin_lock(&delayed_refs->lock);
	if (count == 0) {
		count = delayed_refs->num_entries * 2;
		run_most = 1;
	}
	while (1) {
		if (!(run_all || run_most) &&
		    delayed_refs->num_heads_ready < 64)
			break;
2273

2274
		/*
2275 2276 2277 2278
		 * go find something we can process in the rbtree.  We start at
		 * the beginning of the tree, and then build a cluster
		 * of refs to process starting at the first one we are able to
		 * lock
2279
		 */
2280 2281 2282
		ret = btrfs_find_ref_cluster(trans, &cluster,
					     delayed_refs->run_delayed_start);
		if (ret)
2283 2284
			break;

2285 2286 2287 2288 2289 2290 2291
		ret = run_clustered_refs(trans, root, &cluster);
		BUG_ON(ret < 0);

		count -= min_t(unsigned long, ret, count);

		if (count == 0)
			break;
2292
	}
2293

2294 2295
	if (run_all) {
		node = rb_first(&delayed_refs->root);
2296
		if (!node)
2297
			goto out;
2298
		count = (unsigned long)-1;
2299

2300 2301 2302 2303 2304
		while (node) {
			ref = rb_entry(node, struct btrfs_delayed_ref_node,
				       rb_node);
			if (btrfs_delayed_ref_is_head(ref)) {
				struct btrfs_delayed_ref_head *head;
2305

2306 2307 2308 2309
				head = btrfs_delayed_node_to_head(ref);
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2310 2311 2312 2313
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
2314 2315 2316 2317
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2318
				cond_resched();
2319 2320 2321 2322 2323 2324 2325
				goto again;
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
		schedule_timeout(1);
		goto again;
2326
	}
2327
out:
2328
	spin_unlock(&delayed_refs->lock);
2329 2330 2331
	return 0;
}

2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
				int is_data)
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

	extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;

	ret = btrfs_add_delayed_extent_op(trans, bytenr, num_bytes, extent_op);
	if (ret)
		kfree(extent_op);
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	ret = -ENOENT;
	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

2378
		btrfs_release_path(path);
2379

2380 2381 2382 2383
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out_unlock;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	if (ref->bytenr != bytenr)
		goto out_unlock;

	ret = 1;
	if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
		goto out_unlock;

	data_ref = btrfs_delayed_node_to_data_ref(ref);

	node = rb_prev(node);
	if (node) {
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
		if (ref->bytenr == bytenr)
			goto out_unlock;
	}

	if (data_ref->root != root->root_key.objectid ||
	    data_ref->objectid != objectid || data_ref->offset != offset)
		goto out_unlock;

	ret = 0;
out_unlock:
	mutex_unlock(&head->mutex);
out:
	spin_unlock(&delayed_refs->lock);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2428 2429
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2430
	struct extent_buffer *leaf;
2431 2432 2433
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2434
	struct btrfs_key key;
2435
	u32 item_size;
2436
	int ret;
2437

2438
	key.objectid = bytenr;
Z
Zheng Yan 已提交
2439
	key.offset = (u64)-1;
2440
	key.type = BTRFS_EXTENT_ITEM_KEY;
2441 2442 2443 2444 2445

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
	BUG_ON(ret == 0);
Y
Yan Zheng 已提交
2446 2447 2448

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
2449
		goto out;
2450

Z
Zheng Yan 已提交
2451
	path->slots[0]--;
2452
	leaf = path->nodes[0];
2453
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2454

2455
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2456
		goto out;
2457

2458 2459 2460 2461 2462 2463 2464 2465 2466
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2467

2468 2469 2470
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2471

2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
2511
			goto out;
Y
Yan Zheng 已提交
2512

2513 2514 2515 2516 2517 2518 2519
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
2520
	}
2521 2522 2523

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
2524
out:
Y
Yan Zheng 已提交
2525
	btrfs_free_path(path);
2526 2527
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
2528
	return ret;
2529
}
C
Chris Mason 已提交
2530

2531
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2532
			   struct btrfs_root *root,
2533 2534
			   struct extent_buffer *buf,
			   int full_backref, int inc)
Z
Zheng Yan 已提交
2535 2536
{
	u64 bytenr;
2537 2538
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
2539 2540 2541 2542 2543 2544 2545 2546
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
2547
			    u64, u64, u64, u64, u64, u64);
Z
Zheng Yan 已提交
2548 2549 2550 2551 2552

	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

2553 2554
	if (!root->ref_cows && level == 0)
		return 0;
Z
Zheng Yan 已提交
2555

2556 2557 2558 2559
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
2560

2561 2562 2563 2564 2565 2566
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
2567
		if (level == 0) {
2568
			btrfs_item_key_to_cpu(buf, &key, i);
Z
Zheng Yan 已提交
2569 2570
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
2571
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
2572 2573 2574 2575 2576 2577 2578
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
2579 2580 2581 2582 2583 2584

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
					   key.offset);
Z
Zheng Yan 已提交
2585 2586 2587
			if (ret)
				goto fail;
		} else {
2588 2589 2590 2591
			bytenr = btrfs_node_blockptr(buf, i);
			num_bytes = btrfs_level_size(root, level - 1);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, level - 1, 0);
Z
Zheng Yan 已提交
2592 2593 2594 2595 2596 2597
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611
	BUG();
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		  struct extent_buffer *buf, int full_backref)
{
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		  struct extent_buffer *buf, int full_backref)
{
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
Z
Zheng Yan 已提交
2612 2613
}

C
Chris Mason 已提交
2614 2615 2616 2617 2618 2619 2620
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2621 2622
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
2623 2624

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2625 2626
	if (ret < 0)
		goto fail;
C
Chris Mason 已提交
2627
	BUG_ON(ret);
2628 2629 2630 2631 2632

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
2633
	btrfs_release_path(path);
2634
fail:
C
Chris Mason 已提交
2635 2636 2637 2638 2639 2640
	if (ret)
		return ret;
	return 0;

}

2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
	spin_lock(&root->fs_info->block_group_cache_lock);
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
2652
		btrfs_get_block_group(cache);
2653 2654 2655 2656 2657 2658
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

2659 2660 2661 2662 2663 2664 2665
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
2666
	int dcs = BTRFS_DC_ERROR;
2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685
	int num_pages = 0;
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
2686
		btrfs_release_path(path);
2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
		ret = btrfs_truncate_free_space_cache(root, trans, path,
						      inode);
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
	if (block_group->cached != BTRFS_CACHE_FINISHED) {
2721 2722
		/* We're not cached, don't bother trying to write stuff out */
		dcs = BTRFS_DC_WRITTEN;
2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

	num_pages = (int)div64_u64(block_group->key.offset, 1024 * 1024 * 1024);
	if (!num_pages)
		num_pages = 1;

	/*
	 * Just to make absolutely sure we have enough space, we're going to
	 * preallocate 12 pages worth of space for each block group.  In
	 * practice we ought to use at most 8, but we need extra space so we can
	 * add our header and have a terminator between the extents and the
	 * bitmaps.
	 */
	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
2749 2750
	if (!ret)
		dcs = BTRFS_DC_SETUP;
2751 2752 2753 2754
	btrfs_free_reserved_data_space(inode, num_pages);
out_put:
	iput(inode);
out_free:
2755
	btrfs_release_path(path);
2756 2757
out:
	spin_lock(&block_group->lock);
2758
	block_group->disk_cache_state = dcs;
2759 2760 2761 2762 2763
	spin_unlock(&block_group->lock);

	return ret;
}

2764 2765
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
C
Chris Mason 已提交
2766
{
2767
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
2768 2769
	int err = 0;
	struct btrfs_path *path;
2770
	u64 last = 0;
C
Chris Mason 已提交
2771 2772 2773 2774 2775

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794
again:
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			if (cache->disk_cache_state == BTRFS_DC_CLEAR)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
		err = cache_save_setup(cache, trans, path);
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
2795
	while (1) {
2796 2797 2798 2799
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
			BUG_ON(err);
J
Josef Bacik 已提交
2800
		}
2801

2802 2803
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
2804 2805 2806 2807 2808
			if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
				btrfs_put_block_group(cache);
				goto again;
			}

2809 2810 2811 2812 2813 2814 2815 2816 2817 2818
			if (cache->dirty)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
J
Josef Bacik 已提交
2819

J
Josef Bacik 已提交
2820 2821
		if (cache->disk_cache_state == BTRFS_DC_SETUP)
			cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
2822
		cache->dirty = 0;
2823
		last = cache->key.objectid + cache->key.offset;
J
Josef Bacik 已提交
2824

2825 2826 2827
		err = write_one_cache_group(trans, root, path, cache);
		BUG_ON(err);
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
2828
	}
2829

J
Josef Bacik 已提交
2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875
	while (1) {
		/*
		 * I don't think this is needed since we're just marking our
		 * preallocated extent as written, but just in case it can't
		 * hurt.
		 */
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
			BUG_ON(err);
		}

		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			/*
			 * Really this shouldn't happen, but it could if we
			 * couldn't write the entire preallocated extent and
			 * splitting the extent resulted in a new block.
			 */
			if (cache->dirty) {
				btrfs_put_block_group(cache);
				goto again;
			}
			if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		btrfs_write_out_cache(root, trans, cache, path);

		/*
		 * If we didn't have an error then the cache state is still
		 * NEED_WRITE, so we can set it to WRITTEN.
		 */
		if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
			cache->disk_cache_state = BTRFS_DC_WRITTEN;
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
2876
	btrfs_free_path(path);
2877
	return 0;
C
Chris Mason 已提交
2878 2879
}

2880 2881 2882 2883 2884 2885 2886 2887 2888
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
2889
		btrfs_put_block_group(block_group);
2890 2891 2892
	return readonly;
}

2893 2894 2895 2896 2897
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
2898 2899 2900 2901 2902 2903 2904 2905
	int i;
	int factor;

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
2906 2907 2908

	found = __find_space_info(info, flags);
	if (found) {
2909
		spin_lock(&found->lock);
2910
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
2911
		found->disk_total += total_bytes * factor;
2912
		found->bytes_used += bytes_used;
2913
		found->disk_used += bytes_used * factor;
2914
		found->full = 0;
2915
		spin_unlock(&found->lock);
2916 2917 2918
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
2919
	found = kzalloc(sizeof(*found), GFP_NOFS);
2920 2921 2922
	if (!found)
		return -ENOMEM;

2923 2924
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		INIT_LIST_HEAD(&found->block_groups[i]);
2925
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
2926
	spin_lock_init(&found->lock);
2927 2928 2929
	found->flags = flags & (BTRFS_BLOCK_GROUP_DATA |
				BTRFS_BLOCK_GROUP_SYSTEM |
				BTRFS_BLOCK_GROUP_METADATA);
2930
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
2931
	found->disk_total = total_bytes * factor;
2932
	found->bytes_used = bytes_used;
2933
	found->disk_used = bytes_used * factor;
2934
	found->bytes_pinned = 0;
2935
	found->bytes_reserved = 0;
Y
Yan Zheng 已提交
2936
	found->bytes_readonly = 0;
2937
	found->bytes_may_use = 0;
2938
	found->full = 0;
2939
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
2940
	found->chunk_alloc = 0;
2941 2942
	found->flush = 0;
	init_waitqueue_head(&found->wait);
2943
	*space_info = found;
2944
	list_add_rcu(&found->list, &info->space_info);
2945 2946 2947
	return 0;
}

2948 2949 2950
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
	u64 extra_flags = flags & (BTRFS_BLOCK_GROUP_RAID0 |
2951
				   BTRFS_BLOCK_GROUP_RAID1 |
C
Chris Mason 已提交
2952
				   BTRFS_BLOCK_GROUP_RAID10 |
2953
				   BTRFS_BLOCK_GROUP_DUP);
2954 2955 2956 2957 2958 2959 2960 2961 2962
	if (extra_flags) {
		if (flags & BTRFS_BLOCK_GROUP_DATA)
			fs_info->avail_data_alloc_bits |= extra_flags;
		if (flags & BTRFS_BLOCK_GROUP_METADATA)
			fs_info->avail_metadata_alloc_bits |= extra_flags;
		if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
			fs_info->avail_system_alloc_bits |= extra_flags;
	}
}
2963

Y
Yan Zheng 已提交
2964
u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
2965
{
2966 2967 2968 2969 2970 2971 2972
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	u64 num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;
2973 2974 2975 2976 2977 2978

	if (num_devices == 1)
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

2979 2980
	if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
	    (flags & (BTRFS_BLOCK_GROUP_RAID1 |
2981
		      BTRFS_BLOCK_GROUP_RAID10))) {
2982
		flags &= ~BTRFS_BLOCK_GROUP_DUP;
2983
	}
2984 2985

	if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
2986
	    (flags & BTRFS_BLOCK_GROUP_RAID10)) {
2987
		flags &= ~BTRFS_BLOCK_GROUP_RAID1;
2988
	}
2989 2990 2991 2992 2993 2994 2995 2996 2997

	if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
	    ((flags & BTRFS_BLOCK_GROUP_RAID1) |
	     (flags & BTRFS_BLOCK_GROUP_RAID10) |
	     (flags & BTRFS_BLOCK_GROUP_DUP)))
		flags &= ~BTRFS_BLOCK_GROUP_RAID0;
	return flags;
}

2998
static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
J
Josef Bacik 已提交
2999
{
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		flags |= root->fs_info->avail_data_alloc_bits &
			 root->fs_info->data_alloc_profile;
	else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		flags |= root->fs_info->avail_system_alloc_bits &
			 root->fs_info->system_alloc_profile;
	else if (flags & BTRFS_BLOCK_GROUP_METADATA)
		flags |= root->fs_info->avail_metadata_alloc_bits &
			 root->fs_info->metadata_alloc_profile;
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
3010 3011
}

3012
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
3013
{
3014
	u64 flags;
J
Josef Bacik 已提交
3015

3016 3017 3018 3019
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
3020
	else
3021
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
3022

3023
	return get_alloc_profile(root, flags);
J
Josef Bacik 已提交
3024
}
J
Josef Bacik 已提交
3025

J
Josef Bacik 已提交
3026 3027 3028
void btrfs_set_inode_space_info(struct btrfs_root *root, struct inode *inode)
{
	BTRFS_I(inode)->space_info = __find_space_info(root->fs_info,
3029
						       BTRFS_BLOCK_GROUP_DATA);
J
Josef Bacik 已提交
3030 3031
}

J
Josef Bacik 已提交
3032 3033 3034 3035
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3036
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
3037 3038
{
	struct btrfs_space_info *data_sinfo;
3039
	struct btrfs_root *root = BTRFS_I(inode)->root;
3040
	u64 used;
3041
	int ret = 0, committed = 0, alloc_chunk = 1;
J
Josef Bacik 已提交
3042 3043 3044 3045

	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);

3046 3047
	if (root == root->fs_info->tree_root ||
	    BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3048 3049 3050 3051
		alloc_chunk = 0;
		committed = 1;
	}

J
Josef Bacik 已提交
3052
	data_sinfo = BTRFS_I(inode)->space_info;
C
Chris Mason 已提交
3053 3054
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
3055

J
Josef Bacik 已提交
3056 3057 3058
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3059 3060 3061
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3062 3063

	if (used + bytes > data_sinfo->total_bytes) {
3064
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3065

J
Josef Bacik 已提交
3066 3067 3068 3069
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3070
		if (!data_sinfo->full && alloc_chunk) {
J
Josef Bacik 已提交
3071
			u64 alloc_target;
J
Josef Bacik 已提交
3072

3073
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
J
Josef Bacik 已提交
3074
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
3075
alloc:
J
Josef Bacik 已提交
3076
			alloc_target = btrfs_get_alloc_profile(root, 1);
3077
			trans = btrfs_join_transaction(root);
3078 3079
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
3080

J
Josef Bacik 已提交
3081 3082
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
					     bytes + 2 * 1024 * 1024,
3083 3084
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
J
Josef Bacik 已提交
3085
			btrfs_end_transaction(trans, root);
3086 3087 3088 3089 3090 3091
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
J
Josef Bacik 已提交
3092

C
Chris Mason 已提交
3093 3094 3095 3096
			if (!data_sinfo) {
				btrfs_set_inode_space_info(root, inode);
				data_sinfo = BTRFS_I(inode)->space_info;
			}
J
Josef Bacik 已提交
3097 3098
			goto again;
		}
3099 3100 3101 3102 3103 3104 3105

		/*
		 * If we have less pinned bytes than we want to allocate then
		 * don't bother committing the transaction, it won't help us.
		 */
		if (data_sinfo->bytes_pinned < bytes)
			committed = 1;
J
Josef Bacik 已提交
3106 3107
		spin_unlock(&data_sinfo->lock);

3108
		/* commit the current transaction and try again */
3109
commit_trans:
J
Josef Bacik 已提交
3110 3111
		if (!committed &&
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
3112
			committed = 1;
3113
			trans = btrfs_join_transaction(root);
3114 3115
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3116 3117 3118 3119 3120
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
J
Josef Bacik 已提交
3121

J
Josef Bacik 已提交
3122 3123 3124 3125 3126 3127
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
	BTRFS_I(inode)->reserved_bytes += bytes;
	spin_unlock(&data_sinfo->lock);

J
Josef Bacik 已提交
3128 3129
	return 0;
}
J
Josef Bacik 已提交
3130 3131

/*
3132 3133 3134
 * called when we are clearing an delalloc extent from the
 * inode's io_tree or there was an error for whatever reason
 * after calling btrfs_check_data_free_space
J
Josef Bacik 已提交
3135
 */
3136
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3137
{
3138
	struct btrfs_root *root = BTRFS_I(inode)->root;
J
Josef Bacik 已提交
3139
	struct btrfs_space_info *data_sinfo;
3140

J
Josef Bacik 已提交
3141 3142
	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3143

J
Josef Bacik 已提交
3144 3145 3146 3147 3148
	data_sinfo = BTRFS_I(inode)->space_info;
	spin_lock(&data_sinfo->lock);
	data_sinfo->bytes_may_use -= bytes;
	BTRFS_I(inode)->reserved_bytes -= bytes;
	spin_unlock(&data_sinfo->lock);
3149 3150
}

3151
static void force_metadata_allocation(struct btrfs_fs_info *info)
3152
{
3153 3154
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3155

3156 3157 3158
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3159
			found->force_alloc = CHUNK_ALLOC_FORCE;
3160
	}
3161
	rcu_read_unlock();
3162 3163
}

3164
static int should_alloc_chunk(struct btrfs_root *root,
3165 3166
			      struct btrfs_space_info *sinfo, u64 alloc_bytes,
			      int force)
3167
{
3168
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3169
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3170
	u64 thresh;
3171

3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
		thresh = btrfs_super_total_bytes(&root->fs_info->super_copy);
		thresh = max_t(u64, 64 * 1024 * 1024,
			       div_factor_fine(thresh, 1));

		if (num_bytes - num_allocated < thresh)
			return 1;
	}

	/*
	 * we have two similar checks here, one based on percentage
	 * and once based on a hard number of 256MB.  The idea
	 * is that if we have a good amount of free
	 * room, don't allocate a chunk.  A good mount is
	 * less than 80% utilized of the chunks we have allocated,
	 * or more than 256MB free
	 */
	if (num_allocated + alloc_bytes + 256 * 1024 * 1024 < num_bytes)
3197
		return 0;
3198

3199
	if (num_allocated + alloc_bytes < div_factor(num_bytes, 8))
3200
		return 0;
3201

3202
	thresh = btrfs_super_total_bytes(&root->fs_info->super_copy);
3203 3204

	/* 256MB or 5% of the FS */
3205 3206 3207
	thresh = max_t(u64, 256 * 1024 * 1024, div_factor_fine(thresh, 5));

	if (num_bytes > thresh && sinfo->bytes_used < div_factor(num_bytes, 3))
3208
		return 0;
3209
	return 1;
3210 3211
}

3212 3213
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
C
Chris Mason 已提交
3214
			  u64 flags, int force)
J
Josef Bacik 已提交
3215
{
3216
	struct btrfs_space_info *space_info;
3217
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
3218
	int wait_for_alloc = 0;
J
Josef Bacik 已提交
3219 3220
	int ret = 0;

Y
Yan Zheng 已提交
3221
	flags = btrfs_reduce_alloc_profile(extent_root, flags);
3222

3223
	space_info = __find_space_info(extent_root->fs_info, flags);
3224 3225 3226 3227
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
		BUG_ON(ret);
J
Josef Bacik 已提交
3228
	}
3229
	BUG_ON(!space_info);
J
Josef Bacik 已提交
3230

3231
again:
3232
	spin_lock(&space_info->lock);
J
Josef Bacik 已提交
3233
	if (space_info->force_alloc)
3234
		force = space_info->force_alloc;
3235 3236
	if (space_info->full) {
		spin_unlock(&space_info->lock);
3237
		return 0;
J
Josef Bacik 已提交
3238 3239
	}

3240
	if (!should_alloc_chunk(extent_root, space_info, alloc_bytes, force)) {
3241
		spin_unlock(&space_info->lock);
3242 3243 3244 3245 3246
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
J
Josef Bacik 已提交
3247
	}
3248

3249
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3250

3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

3265 3266 3267 3268 3269 3270 3271
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3272 3273 3274 3275 3276
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
3277
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3278 3279 3280 3281
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
3282 3283
	}

Y
Yan Zheng 已提交
3284
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
3285 3286 3287
	if (ret < 0 && ret != -ENOSPC)
		goto out;

J
Josef Bacik 已提交
3288 3289
	spin_lock(&space_info->lock);
	if (ret)
3290
		space_info->full = 1;
3291 3292
	else
		ret = 1;
3293

3294
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3295
	space_info->chunk_alloc = 0;
J
Josef Bacik 已提交
3296
	spin_unlock(&space_info->lock);
3297
out:
Y
Yan Zheng 已提交
3298
	mutex_unlock(&extent_root->fs_info->chunk_mutex);
J
Josef Bacik 已提交
3299
	return ret;
3300
}
J
Josef Bacik 已提交
3301 3302

/*
3303
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
3304
 */
3305
static int shrink_delalloc(struct btrfs_trans_handle *trans,
J
Josef Bacik 已提交
3306
			   struct btrfs_root *root, u64 to_reclaim, int sync)
3307
{
3308
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
3309
	struct btrfs_space_info *space_info;
3310 3311 3312
	u64 reserved;
	u64 max_reclaim;
	u64 reclaimed = 0;
3313
	long time_left;
3314
	int nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3315
	int loops = 0;
3316
	unsigned long progress;
3317

3318
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
3319
	space_info = block_rsv->space_info;
3320 3321

	smp_mb();
J
Josef Bacik 已提交
3322
	reserved = space_info->bytes_reserved;
3323
	progress = space_info->reservation_progress;
3324 3325 3326

	if (reserved == 0)
		return 0;
3327

3328 3329 3330 3331 3332 3333 3334 3335
	smp_mb();
	if (root->fs_info->delalloc_bytes == 0) {
		if (trans)
			return 0;
		btrfs_wait_ordered_extents(root, 0, 0);
		return 0;
	}

3336 3337
	max_reclaim = min(reserved, to_reclaim);

3338
	while (loops < 1024) {
3339 3340 3341 3342 3343
		/* have the flusher threads jump in and do some IO */
		smp_mb();
		nr_pages = min_t(unsigned long, nr_pages,
		       root->fs_info->delalloc_bytes >> PAGE_CACHE_SHIFT);
		writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages);
3344

J
Josef Bacik 已提交
3345
		spin_lock(&space_info->lock);
3346
		if (reserved > space_info->bytes_reserved)
J
Josef Bacik 已提交
3347 3348 3349
			reclaimed += reserved - space_info->bytes_reserved;
		reserved = space_info->bytes_reserved;
		spin_unlock(&space_info->lock);
3350

3351 3352
		loops++;

3353 3354 3355 3356 3357
		if (reserved == 0 || reclaimed >= max_reclaim)
			break;

		if (trans && trans->transaction->blocked)
			return -EAGAIN;
3358

3359
		time_left = schedule_timeout_interruptible(1);
3360 3361 3362 3363 3364

		/* We were interrupted, exit */
		if (time_left)
			break;

3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375
		/* we've kicked the IO a few times, if anything has been freed,
		 * exit.  There is no sense in looping here for a long time
		 * when we really need to commit the transaction, or there are
		 * just too many writers without enough free space
		 */

		if (loops > 3) {
			smp_mb();
			if (progress != space_info->reservation_progress)
				break;
		}
3376

3377
	}
3378 3379
	if (reclaimed >= to_reclaim && !trans)
		btrfs_wait_ordered_extents(root, 0, 0);
3380 3381 3382
	return reclaimed >= to_reclaim;
}

3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397
/*
 * Retries tells us how many times we've called reserve_metadata_bytes.  The
 * idea is if this is the first call (retries == 0) then we will add to our
 * reserved count if we can't make the allocation in order to hold our place
 * while we go and try and free up space.  That way for retries > 1 we don't try
 * and add space, we just check to see if the amount of unused space is >= the
 * total space, meaning that our reservation is valid.
 *
 * However if we don't intend to retry this reservation, pass -1 as retries so
 * that it short circuits this logic.
 */
static int reserve_metadata_bytes(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_block_rsv *block_rsv,
				  u64 orig_bytes, int flush)
J
Josef Bacik 已提交
3398
{
3399
	struct btrfs_space_info *space_info = block_rsv->space_info;
3400 3401 3402 3403
	u64 unused;
	u64 num_bytes = orig_bytes;
	int retries = 0;
	int ret = 0;
3404
	bool committed = false;
3405
	bool flushing = false;
J
Josef Bacik 已提交
3406

3407
again:
3408
	ret = 0;
3409
	spin_lock(&space_info->lock);
3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433
	/*
	 * We only want to wait if somebody other than us is flushing and we are
	 * actually alloed to flush.
	 */
	while (flush && !flushing && space_info->flush) {
		spin_unlock(&space_info->lock);
		/*
		 * If we have a trans handle we can't wait because the flusher
		 * may have to commit the transaction, which would mean we would
		 * deadlock since we are waiting for the flusher to finish, but
		 * hold the current transaction open.
		 */
		if (trans)
			return -EAGAIN;
		ret = wait_event_interruptible(space_info->wait,
					       !space_info->flush);
		/* Must have been interrupted, return */
		if (ret)
			return -EINTR;

		spin_lock(&space_info->lock);
	}

	ret = -ENOSPC;
3434 3435 3436
	unused = space_info->bytes_used + space_info->bytes_reserved +
		 space_info->bytes_pinned + space_info->bytes_readonly +
		 space_info->bytes_may_use;
J
Josef Bacik 已提交
3437

3438 3439 3440 3441 3442 3443 3444 3445
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
	if (unused <= space_info->total_bytes) {
3446
		unused = space_info->total_bytes - unused;
3447
		if (unused >= num_bytes) {
3448
			space_info->bytes_reserved += orig_bytes;
3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
		num_bytes = unused - space_info->total_bytes +
			(orig_bytes * (retries + 1));
	}
J
Josef Bacik 已提交
3467

3468 3469 3470 3471 3472
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
	 */
3473 3474 3475
	if (ret && flush) {
		flushing = true;
		space_info->flush = 1;
3476
	}
J
Josef Bacik 已提交
3477

3478
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3479

3480
	if (!ret || !flush)
3481
		goto out;
3482

3483 3484 3485 3486 3487
	/*
	 * We do synchronous shrinking since we don't actually unreserve
	 * metadata until after the IO is completed.
	 */
	ret = shrink_delalloc(trans, root, num_bytes, 1);
3488
	if (ret < 0)
3489
		goto out;
3490

3491 3492
	ret = 0;

3493 3494 3495 3496 3497 3498 3499 3500 3501
	/*
	 * So if we were overcommitted it's possible that somebody else flushed
	 * out enough space and we simply didn't have enough space to reclaim,
	 * so go back around and try again.
	 */
	if (retries < 2) {
		retries++;
		goto again;
	}
3502

3503 3504 3505 3506
	/*
	 * Not enough space to be reclaimed, don't bother committing the
	 * transaction.
	 */
3507
	spin_lock(&space_info->lock);
3508 3509 3510 3511 3512
	if (space_info->bytes_pinned < orig_bytes)
		ret = -ENOSPC;
	spin_unlock(&space_info->lock);
	if (ret)
		goto out;
3513

3514
	ret = -EAGAIN;
3515
	if (trans)
3516
		goto out;
3517

3518
	ret = -ENOSPC;
3519 3520 3521
	if (committed)
		goto out;

3522
	trans = btrfs_join_transaction(root);
3523 3524 3525
	if (IS_ERR(trans))
		goto out;
	ret = btrfs_commit_transaction(trans, root);
3526 3527 3528
	if (!ret) {
		trans = NULL;
		committed = true;
3529
		goto again;
3530
	}
3531 3532

out:
3533
	if (flushing) {
3534
		spin_lock(&space_info->lock);
3535 3536
		space_info->flush = 0;
		wake_up_all(&space_info->wait);
3537
		spin_unlock(&space_info->lock);
3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583
	}
	return ret;
}

static struct btrfs_block_rsv *get_block_rsv(struct btrfs_trans_handle *trans,
					     struct btrfs_root *root)
{
	struct btrfs_block_rsv *block_rsv;
	if (root->ref_cows)
		block_rsv = trans->block_rsv;
	else
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

3584 3585
static void block_rsv_release_bytes(struct btrfs_block_rsv *block_rsv,
				    struct btrfs_block_rsv *dest, u64 num_bytes)
3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
3618 3619
			spin_lock(&space_info->lock);
			space_info->bytes_reserved -= num_bytes;
3620
			space_info->reservation_progress++;
3621
			spin_unlock(&space_info->lock);
3622
		}
J
Josef Bacik 已提交
3623
	}
3624
}
3625

3626 3627 3628 3629
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
J
Josef Bacik 已提交
3630

3631 3632 3633
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
3634

3635
	block_rsv_add_bytes(dst, num_bytes, 1);
J
Josef Bacik 已提交
3636 3637 3638
	return 0;
}

3639
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv)
J
Josef Bacik 已提交
3640
{
3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
	atomic_set(&rsv->usage, 1);
	rsv->priority = 6;
	INIT_LIST_HEAD(&rsv->list);
}

struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root)
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
3652

3653 3654 3655
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
3656

3657 3658 3659 3660 3661
	btrfs_init_block_rsv(block_rsv);
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
3662

3663 3664 3665 3666 3667 3668 3669 3670
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
	if (rsv && atomic_dec_and_test(&rsv->usage)) {
		btrfs_block_rsv_release(root, rsv, (u64)-1);
		if (!rsv->durable)
			kfree(rsv);
	}
J
Josef Bacik 已提交
3671 3672 3673
}

/*
3674 3675 3676
 * make the block_rsv struct be able to capture freed space.
 * the captured space will re-add to the the block_rsv struct
 * after transaction commit
J
Josef Bacik 已提交
3677
 */
3678 3679
void btrfs_add_durable_block_rsv(struct btrfs_fs_info *fs_info,
				 struct btrfs_block_rsv *block_rsv)
J
Josef Bacik 已提交
3680
{
3681 3682 3683 3684 3685
	block_rsv->durable = 1;
	mutex_lock(&fs_info->durable_block_rsv_mutex);
	list_add_tail(&block_rsv->list, &fs_info->durable_block_rsv_list);
	mutex_unlock(&fs_info->durable_block_rsv_mutex);
}
J
Josef Bacik 已提交
3686

3687 3688 3689
int btrfs_block_rsv_add(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv,
3690
			u64 num_bytes)
3691 3692
{
	int ret;
J
Josef Bacik 已提交
3693

3694 3695
	if (num_bytes == 0)
		return 0;
3696 3697

	ret = reserve_metadata_bytes(trans, root, block_rsv, num_bytes, 1);
3698 3699 3700 3701
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
3702

3703 3704
	return ret;
}
J
Josef Bacik 已提交
3705

3706 3707 3708 3709 3710 3711 3712 3713
int btrfs_block_rsv_check(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  struct btrfs_block_rsv *block_rsv,
			  u64 min_reserved, int min_factor)
{
	u64 num_bytes = 0;
	int commit_trans = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
3714

3715 3716
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
3717

3718 3719 3720 3721 3722
	spin_lock(&block_rsv->lock);
	if (min_factor > 0)
		num_bytes = div_factor(block_rsv->size, min_factor);
	if (min_reserved > num_bytes)
		num_bytes = min_reserved;
J
Josef Bacik 已提交
3723

3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736
	if (block_rsv->reserved >= num_bytes) {
		ret = 0;
	} else {
		num_bytes -= block_rsv->reserved;
		if (block_rsv->durable &&
		    block_rsv->freed[0] + block_rsv->freed[1] >= num_bytes)
			commit_trans = 1;
	}
	spin_unlock(&block_rsv->lock);
	if (!ret)
		return 0;

	if (block_rsv->refill_used) {
3737 3738
		ret = reserve_metadata_bytes(trans, root, block_rsv,
					     num_bytes, 0);
3739 3740 3741
		if (!ret) {
			block_rsv_add_bytes(block_rsv, num_bytes, 0);
			return 0;
3742
		}
3743
	}
J
Josef Bacik 已提交
3744

3745 3746 3747
	if (commit_trans) {
		if (trans)
			return -EAGAIN;
3748
		trans = btrfs_join_transaction(root);
3749 3750 3751
		BUG_ON(IS_ERR(trans));
		ret = btrfs_commit_transaction(trans, root);
		return 0;
J
Josef Bacik 已提交
3752
	}
J
Josef Bacik 已提交
3753

3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772
	return -ENOSPC;
}

int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
	if (global_rsv->full || global_rsv == block_rsv ||
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
	block_rsv_release_bytes(block_rsv, global_rsv, num_bytes);
J
Josef Bacik 已提交
3773 3774 3775
}

/*
3776 3777 3778
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
3779
 */
3780
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3781
{
3782 3783 3784 3785 3786
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
	int csum_size = btrfs_super_csum_size(&fs_info->super_copy);
J
Josef Bacik 已提交
3787

3788 3789 3790 3791
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
3792

3793 3794
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
3795 3796
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
3797 3798
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
3799

3800 3801 3802
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
	num_bytes += div64_u64(data_used + meta_used, 50);
3803

3804 3805
	if (num_bytes * 3 > meta_used)
		num_bytes = div64_u64(meta_used, 3);
3806

3807 3808
	return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
}
J
Josef Bacik 已提交
3809

3810 3811 3812 3813 3814
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
3815

3816
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
3817

3818 3819
	spin_lock(&block_rsv->lock);
	spin_lock(&sinfo->lock);
3820

3821
	block_rsv->size = num_bytes;
3822

3823
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
3824 3825
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
3826 3827 3828 3829 3830

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
		sinfo->bytes_reserved += num_bytes;
J
Josef Bacik 已提交
3831 3832
	}

3833 3834 3835
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		sinfo->bytes_reserved -= num_bytes;
3836
		sinfo->reservation_progress++;
3837 3838 3839
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
3840

3841 3842
	spin_unlock(&sinfo->lock);
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
3843 3844
}

3845
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3846
{
3847
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
3848

3849 3850 3851
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
	fs_info->chunk_block_rsv.priority = 10;
J
Josef Bacik 已提交
3852

3853
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
3854 3855 3856 3857
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->global_block_rsv.priority = 10;
	fs_info->global_block_rsv.refill_used = 1;
	fs_info->delalloc_block_rsv.space_info = space_info;
3858 3859 3860 3861
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.priority = 10;

3862 3863 3864 3865
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
3866
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
3867 3868 3869 3870 3871 3872

	btrfs_add_durable_block_rsv(fs_info, &fs_info->global_block_rsv);

	btrfs_add_durable_block_rsv(fs_info, &fs_info->delalloc_block_rsv);

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
3873 3874
}

3875
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3876
{
3877 3878 3879 3880 3881 3882 3883
	block_rsv_release_bytes(&fs_info->global_block_rsv, NULL, (u64)-1);
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
3884
}
J
Josef Bacik 已提交
3885

3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898
int btrfs_truncate_reserve_metadata(struct btrfs_trans_handle *trans,
				    struct btrfs_root *root,
				    struct btrfs_block_rsv *rsv)
{
	struct btrfs_block_rsv *trans_rsv = &root->fs_info->trans_block_rsv;
	u64 num_bytes;
	int ret;

	/*
	 * Truncate should be freeing data, but give us 2 items just in case it
	 * needs to use some space.  We may want to be smarter about this in the
	 * future.
	 */
C
Chris Mason 已提交
3899
	num_bytes = btrfs_calc_trans_metadata_size(root, 2);
3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916

	/* We already have enough bytes, just return */
	if (rsv->reserved >= num_bytes)
		return 0;

	num_bytes -= rsv->reserved;

	/*
	 * You should have reserved enough space before hand to do this, so this
	 * should not fail.
	 */
	ret = block_rsv_migrate_bytes(trans_rsv, rsv, num_bytes);
	BUG_ON(ret);

	return 0;
}

3917 3918
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
3919
{
3920 3921
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
3922

3923 3924 3925 3926 3927
	BUG_ON(trans->block_rsv != &root->fs_info->trans_block_rsv);
	btrfs_block_rsv_release(root, trans->block_rsv,
				trans->bytes_reserved);
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
3928

3929 3930 3931 3932 3933 3934 3935 3936
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
3937 3938 3939
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
3940
	 */
C
Chris Mason 已提交
3941
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
3942
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
J
Josef Bacik 已提交
3943 3944
}

3945
void btrfs_orphan_release_metadata(struct inode *inode)
3946
{
3947
	struct btrfs_root *root = BTRFS_I(inode)->root;
C
Chris Mason 已提交
3948
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
3949 3950
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
3951

3952 3953 3954 3955 3956 3957 3958 3959 3960 3961
int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
				struct btrfs_pending_snapshot *pending)
{
	struct btrfs_root *root = pending->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
	/*
	 * two for root back/forward refs, two for directory entries
	 * and one for root of the snapshot.
	 */
3962
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 5);
3963 3964
	dst_rsv->space_info = src_rsv->space_info;
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
3965 3966
}

3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990
static unsigned drop_outstanding_extent(struct inode *inode)
{
	unsigned dropped_extents = 0;

	spin_lock(&BTRFS_I(inode)->lock);
	BUG_ON(!BTRFS_I(inode)->outstanding_extents);
	BTRFS_I(inode)->outstanding_extents--;

	/*
	 * If we have more or the same amount of outsanding extents than we have
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
		goto out;

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
out:
	spin_unlock(&BTRFS_I(inode)->lock);
	return dropped_extents;
}

3991
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes)
3992
{
3993 3994
	return num_bytes >>= 3;
}
Y
Yan Zheng 已提交
3995

3996 3997 3998 3999
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4000 4001
	u64 to_reserve = 0;
	unsigned nr_extents = 0;
4002
	int ret;
4003

4004 4005
	if (btrfs_transaction_in_commit(root->fs_info))
		schedule_timeout(1);
4006

4007
	num_bytes = ALIGN(num_bytes, root->sectorsize);
4008

4009 4010 4011 4012 4013 4014 4015 4016
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;

	if (BTRFS_I(inode)->outstanding_extents >
	    BTRFS_I(inode)->reserved_extents) {
		nr_extents = BTRFS_I(inode)->outstanding_extents -
			BTRFS_I(inode)->reserved_extents;
		BTRFS_I(inode)->reserved_extents += nr_extents;
4017

4018
		to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4019
	}
4020
	spin_unlock(&BTRFS_I(inode)->lock);
4021

4022
	to_reserve += calc_csum_metadata_size(inode, num_bytes);
4023
	ret = reserve_metadata_bytes(NULL, root, block_rsv, to_reserve, 1);
4024 4025 4026 4027 4028 4029 4030 4031
	if (ret) {
		unsigned dropped;
		/*
		 * We don't need the return value since our reservation failed,
		 * we just need to clean up our counter.
		 */
		dropped = drop_outstanding_extent(inode);
		WARN_ON(dropped > 1);
4032
		return ret;
4033
	}
4034

4035 4036 4037 4038 4039 4040 4041 4042
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	return 0;
}

void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
4043 4044
	u64 to_free = 0;
	unsigned dropped;
4045 4046

	num_bytes = ALIGN(num_bytes, root->sectorsize);
4047
	dropped = drop_outstanding_extent(inode);
4048

4049
	to_free = calc_csum_metadata_size(inode, num_bytes);
4050 4051
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
4052 4053 4054 4055 4056 4057 4058 4059 4060 4061

	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
C
Chris Mason 已提交
4062
	if (ret)
4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
4078 4079
}

C
Chris Mason 已提交
4080 4081
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
4082
			      u64 bytenr, u64 num_bytes, int alloc)
C
Chris Mason 已提交
4083
{
4084
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
4085
	struct btrfs_fs_info *info = root->fs_info;
4086
	u64 total = num_bytes;
C
Chris Mason 已提交
4087
	u64 old_val;
4088
	u64 byte_in_group;
4089
	int factor;
C
Chris Mason 已提交
4090

4091 4092 4093 4094 4095 4096 4097 4098 4099 4100
	/* block accounting for super block */
	spin_lock(&info->delalloc_lock);
	old_val = btrfs_super_bytes_used(&info->super_copy);
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
	btrfs_set_super_bytes_used(&info->super_copy, old_val);
	spin_unlock(&info->delalloc_lock);

C
Chris Mason 已提交
4101
	while (total) {
4102
		cache = btrfs_lookup_block_group(info, bytenr);
4103
		if (!cache)
C
Chris Mason 已提交
4104
			return -1;
4105 4106 4107 4108 4109 4110
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
4111 4112 4113 4114 4115 4116 4117
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
4118
			cache_block_group(cache, trans, NULL, 1);
4119

4120 4121
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
4122

4123
		spin_lock(&cache->space_info->lock);
4124
		spin_lock(&cache->lock);
4125 4126 4127 4128 4129

		if (btrfs_super_cache_generation(&info->super_copy) != 0 &&
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

J
Josef Bacik 已提交
4130
		cache->dirty = 1;
C
Chris Mason 已提交
4131
		old_val = btrfs_block_group_used(&cache->item);
4132
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
4133
		if (alloc) {
4134
			old_val += num_bytes;
4135 4136 4137
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
4138
			cache->space_info->reservation_progress++;
4139 4140
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
4141
			spin_unlock(&cache->lock);
4142
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4143
		} else {
4144
			old_val -= num_bytes;
4145
			btrfs_set_block_group_used(&cache->item, old_val);
4146 4147
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
4148
			cache->space_info->bytes_used -= num_bytes;
4149
			cache->space_info->disk_used -= num_bytes * factor;
4150
			spin_unlock(&cache->lock);
4151
			spin_unlock(&cache->space_info->lock);
4152

4153 4154 4155
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
C
Chris Mason 已提交
4156
		}
4157
		btrfs_put_block_group(cache);
4158 4159
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
4160 4161 4162
	}
	return 0;
}
4163

4164 4165
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
4166
	struct btrfs_block_group_cache *cache;
4167
	u64 bytenr;
J
Josef Bacik 已提交
4168 4169 4170

	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
4171
		return 0;
J
Josef Bacik 已提交
4172

4173
	bytenr = cache->key.objectid;
4174
	btrfs_put_block_group(cache);
4175 4176

	return bytenr;
4177 4178
}

4179 4180 4181
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
4182
{
4183 4184 4185 4186 4187 4188 4189
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
4190
		cache->space_info->reservation_progress++;
4191 4192 4193
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
4194

4195 4196 4197 4198
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
	return 0;
}
J
Josef Bacik 已提交
4199

4200 4201 4202 4203 4204 4205 4206
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
4207

4208 4209 4210 4211 4212 4213
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
	BUG_ON(!cache);

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
4214 4215 4216
	return 0;
}

4217 4218 4219 4220
/*
 * update size of reserved extents. this function may return -EAGAIN
 * if 'reserve' is true or 'sinfo' is false.
 */
4221 4222
int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				u64 num_bytes, int reserve, int sinfo)
4223
{
4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240
	int ret = 0;
	if (sinfo) {
		struct btrfs_space_info *space_info = cache->space_info;
		spin_lock(&space_info->lock);
		spin_lock(&cache->lock);
		if (reserve) {
			if (cache->ro) {
				ret = -EAGAIN;
			} else {
				cache->reserved += num_bytes;
				space_info->bytes_reserved += num_bytes;
			}
		} else {
			if (cache->ro)
				space_info->bytes_readonly += num_bytes;
			cache->reserved -= num_bytes;
			space_info->bytes_reserved -= num_bytes;
4241
			space_info->reservation_progress++;
4242 4243 4244
		}
		spin_unlock(&cache->lock);
		spin_unlock(&space_info->lock);
4245
	} else {
4246 4247 4248 4249 4250 4251 4252 4253 4254 4255
		spin_lock(&cache->lock);
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
			if (reserve)
				cache->reserved += num_bytes;
			else
				cache->reserved -= num_bytes;
		}
		spin_unlock(&cache->lock);
4256
	}
4257
	return ret;
4258
}
C
Chris Mason 已提交
4259

4260 4261
int btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
				struct btrfs_root *root)
4262 4263
{
	struct btrfs_fs_info *fs_info = root->fs_info;
4264 4265 4266
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
4267

4268
	down_write(&fs_info->extent_commit_sem);
4269

4270 4271 4272 4273 4274 4275 4276
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
4277
		} else {
4278
			cache->last_byte_to_unpin = caching_ctl->progress;
4279 4280
		}
	}
4281 4282 4283 4284 4285 4286 4287

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

	up_write(&fs_info->extent_commit_sem);
4288 4289

	update_global_block_rsv(fs_info);
4290 4291 4292
	return 0;
}

4293
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
C
Chris Mason 已提交
4294
{
4295 4296 4297
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 len;
C
Chris Mason 已提交
4298

4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315
	while (start <= end) {
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
			BUG_ON(!cache);
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
			btrfs_add_free_space(cache, start, len);
		}

4316 4317
		start += len;

4318 4319 4320 4321
		spin_lock(&cache->space_info->lock);
		spin_lock(&cache->lock);
		cache->pinned -= len;
		cache->space_info->bytes_pinned -= len;
4322 4323 4324 4325 4326 4327 4328
		if (cache->ro) {
			cache->space_info->bytes_readonly += len;
		} else if (cache->reserved_pinned > 0) {
			len = min(len, cache->reserved_pinned);
			cache->reserved_pinned -= len;
			cache->space_info->bytes_reserved += len;
		}
4329 4330
		spin_unlock(&cache->lock);
		spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4331
	}
4332 4333 4334

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
4335 4336 4337 4338
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
4339
			       struct btrfs_root *root)
4340
{
4341 4342
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
4343 4344
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_block_rsv *next_rsv;
4345 4346
	u64 start;
	u64 end;
4347
	int idx;
4348 4349
	int ret;

4350 4351 4352 4353 4354
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

C
Chris Mason 已提交
4355
	while (1) {
4356 4357 4358
		ret = find_first_extent_bit(unpin, 0, &start, &end,
					    EXTENT_DIRTY);
		if (ret)
4359
			break;
4360

4361 4362 4363
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
4364

4365
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
4366
		unpin_extent_range(root, start, end);
4367
		cond_resched();
4368
	}
J
Josef Bacik 已提交
4369

4370 4371 4372
	mutex_lock(&fs_info->durable_block_rsv_mutex);
	list_for_each_entry_safe(block_rsv, next_rsv,
				 &fs_info->durable_block_rsv_list, list) {
4373

4374 4375 4376 4377 4378 4379 4380 4381
		idx = trans->transid & 0x1;
		if (block_rsv->freed[idx] > 0) {
			block_rsv_add_bytes(block_rsv,
					    block_rsv->freed[idx], 0);
			block_rsv->freed[idx] = 0;
		}
		if (atomic_read(&block_rsv->usage) == 0) {
			btrfs_block_rsv_release(root, block_rsv, (u64)-1);
Z
Zheng Yan 已提交
4382

4383 4384 4385 4386 4387 4388 4389
			if (block_rsv->freed[0] == 0 &&
			    block_rsv->freed[1] == 0) {
				list_del_init(&block_rsv->list);
				kfree(block_rsv);
			}
		} else {
			btrfs_block_rsv_release(root, block_rsv, 0);
C
Chris Mason 已提交
4390
		}
C
Chris Mason 已提交
4391
	}
4392
	mutex_unlock(&fs_info->durable_block_rsv_mutex);
Z
Zheng Yan 已提交
4393

C
Chris Mason 已提交
4394 4395 4396
	return 0;
}

4397 4398 4399 4400 4401 4402
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extent_op)
4403
{
C
Chris Mason 已提交
4404
	struct btrfs_key key;
4405
	struct btrfs_path *path;
4406 4407
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
4408
	struct extent_buffer *leaf;
4409 4410
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
4411
	int ret;
4412
	int is_data;
4413 4414 4415
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
4416 4417
	u32 item_size;
	u64 refs;
C
Chris Mason 已提交
4418

4419
	path = btrfs_alloc_path();
4420 4421
	if (!path)
		return -ENOMEM;
4422

4423
	path->reada = 1;
4424
	path->leave_spinning = 1;
4425 4426 4427 4428 4429 4430 4431 4432

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
4433
	if (ret == 0) {
4434
		extent_slot = path->slots[0];
4435 4436
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
4437
					      extent_slot);
4438
			if (key.objectid != bytenr)
4439
				break;
4440 4441
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
4442 4443 4444 4445 4446
				found_extent = 1;
				break;
			}
			if (path->slots[0] - extent_slot > 5)
				break;
4447
			extent_slot--;
4448
		}
4449 4450 4451 4452 4453
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
4454
		if (!found_extent) {
4455
			BUG_ON(iref);
4456
			ret = remove_extent_backref(trans, extent_root, path,
4457 4458
						    NULL, refs_to_drop,
						    is_data);
Z
Zheng Yan 已提交
4459
			BUG_ON(ret);
4460
			btrfs_release_path(path);
4461
			path->leave_spinning = 1;
4462 4463 4464 4465 4466

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

Z
Zheng Yan 已提交
4467 4468
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
4469 4470
			if (ret) {
				printk(KERN_ERR "umm, got %d back from search"
C
Chris Mason 已提交
4471 4472
				       ", was looking for %llu\n", ret,
				       (unsigned long long)bytenr);
4473 4474 4475
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
4476
			}
Z
Zheng Yan 已提交
4477 4478 4479
			BUG_ON(ret);
			extent_slot = path->slots[0];
		}
4480 4481 4482
	} else {
		btrfs_print_leaf(extent_root, path->nodes[0]);
		WARN_ON(1);
C
Chris Mason 已提交
4483
		printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
4484
		       "parent %llu root %llu  owner %llu offset %llu\n",
C
Chris Mason 已提交
4485
		       (unsigned long long)bytenr,
4486
		       (unsigned long long)parent,
C
Chris Mason 已提交
4487
		       (unsigned long long)root_objectid,
4488 4489
		       (unsigned long long)owner_objectid,
		       (unsigned long long)owner_offset);
4490
	}
4491 4492

	leaf = path->nodes[0];
4493 4494 4495 4496 4497 4498 4499 4500
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
		BUG_ON(ret < 0);

4501
		btrfs_release_path(path);
4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
			printk(KERN_ERR "umm, got %d back from search"
			       ", was looking for %llu\n", ret,
			       (unsigned long long)bytenr);
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
		BUG_ON(ret);
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
4523
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
4524
			    struct btrfs_extent_item);
4525 4526 4527 4528 4529 4530
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
4531

4532
	refs = btrfs_extent_refs(leaf, ei);
4533 4534
	BUG_ON(refs < refs_to_drop);
	refs -= refs_to_drop;
4535

4536 4537 4538 4539 4540 4541
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
4542
		 */
4543 4544 4545 4546 4547 4548 4549 4550 4551 4552
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
						    is_data);
4553 4554
			BUG_ON(ret);
		}
4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
4566
		}
4567

4568 4569
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
Z
Zheng Yan 已提交
4570
		BUG_ON(ret);
4571
		btrfs_release_path(path);
4572

4573
		if (is_data) {
4574 4575
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
			BUG_ON(ret);
4576 4577 4578 4579
		} else {
			invalidate_mapping_pages(info->btree_inode->i_mapping,
			     bytenr >> PAGE_CACHE_SHIFT,
			     (bytenr + num_bytes - 1) >> PAGE_CACHE_SHIFT);
4580 4581
		}

4582
		ret = update_block_group(trans, root, bytenr, num_bytes, 0);
4583
		BUG_ON(ret);
4584
	}
4585
	btrfs_free_path(path);
4586 4587 4588
	return ret;
}

4589
/*
4590
 * when we free an block, it is possible (and likely) that we free the last
4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct rb_node *node;
4602
	int ret = 0;
4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	/* there are still entries for this ref, we can't drop it */
	if (ref->bytenr == bytenr)
		goto out;

4620 4621 4622 4623 4624 4625 4626
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
		kfree(head->extent_op);
		head->extent_op = NULL;
	}

4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
	rb_erase(&head->node.rb_node, &delayed_refs->root);
4640

4641 4642 4643 4644 4645 4646
	delayed_refs->num_entries--;

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
4647 4648 4649 4650 4651
	delayed_refs->num_heads--;
	if (list_empty(&head->cluster))
		delayed_refs->num_heads_ready--;

	list_del_init(&head->cluster);
4652 4653
	spin_unlock(&delayed_refs->lock);

4654 4655 4656 4657 4658
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
4659
	btrfs_put_delayed_ref(&head->node);
4660
	return ret;
4661 4662 4663 4664 4665
out:
	spin_unlock(&delayed_refs->lock);
	return 0;
}

4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
			   u64 parent, int last_ref)
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_block_group_cache *cache = NULL;
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, buf->start, buf->len,
						parent, root->root_key.objectid,
						btrfs_header_level(buf),
						BTRFS_DROP_DELAYED_REF, NULL);
		BUG_ON(ret);
	}

	if (!last_ref)
		return;

	block_rsv = get_block_rsv(trans, root);
	cache = btrfs_lookup_block_group(root->fs_info, buf->start);
4688 4689
	if (block_rsv->space_info != cache->space_info)
		goto out;
4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705

	if (btrfs_header_generation(buf) == trans->transid) {
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
				goto pin;
		}

		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
			goto pin;
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
4706
		ret = btrfs_update_reserved_bytes(cache, buf->len, 0, 0);
4707 4708
		if (ret == -EAGAIN) {
			/* block group became read-only */
4709
			btrfs_update_reserved_bytes(cache, buf->len, 0, 1);
4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723
			goto out;
		}

		ret = 1;
		spin_lock(&block_rsv->lock);
		if (block_rsv->reserved < block_rsv->size) {
			block_rsv->reserved += buf->len;
			ret = 0;
		}
		spin_unlock(&block_rsv->lock);

		if (ret) {
			spin_lock(&cache->space_info->lock);
			cache->space_info->bytes_reserved -= buf->len;
4724
			cache->space_info->reservation_progress++;
4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745
			spin_unlock(&cache->space_info->lock);
		}
		goto out;
	}
pin:
	if (block_rsv->durable && !cache->ro) {
		ret = 0;
		spin_lock(&cache->lock);
		if (!cache->ro) {
			cache->reserved_pinned += buf->len;
			ret = 1;
		}
		spin_unlock(&cache->lock);

		if (ret) {
			spin_lock(&block_rsv->lock);
			block_rsv->freed[trans->transid & 0x1] += buf->len;
			spin_unlock(&block_rsv->lock);
		}
	}
out:
4746 4747 4748 4749 4750
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
4751 4752 4753
	btrfs_put_block_group(cache);
}

4754
int btrfs_free_extent(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
4755 4756
		      struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent,
4757
		      u64 root_objectid, u64 owner, u64 offset)
4758 4759 4760
{
	int ret;

4761 4762 4763 4764
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
4765 4766
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
4767
		/* unlocks the pinned mutex */
4768
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
4769
		ret = 0;
4770 4771 4772 4773
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
					parent, root_objectid, (int)owner,
					BTRFS_DROP_DELAYED_REF, NULL);
4774
		BUG_ON(ret);
4775 4776 4777 4778 4779
	} else {
		ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
					parent, root_objectid, owner,
					offset, BTRFS_DROP_DELAYED_REF, NULL);
		BUG_ON(ret);
4780
	}
4781 4782 4783
	return ret;
}

4784 4785 4786 4787 4788 4789 4790
static u64 stripe_align(struct btrfs_root *root, u64 val)
{
	u64 mask = ((u64)root->stripesize - 1);
	u64 ret = (val + mask) & ~mask;
	return ret;
}

J
Josef Bacik 已提交
4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
 */
static noinline int
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
4806
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
4807 4808
	DEFINE_WAIT(wait);

4809 4810
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
J
Josef Bacik 已提交
4811 4812
		return 0;

4813
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
4814
		   (cache->free_space_ctl->free_space >= num_bytes));
4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832

	put_caching_control(caching_ctl);
	return 0;
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;
	DEFINE_WAIT(wait);

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
		return 0;

	wait_event(caching_ctl->wait, block_group_cache_done(cache));

	put_caching_control(caching_ctl);
J
Josef Bacik 已提交
4833 4834 4835
	return 0;
}

4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851
static int get_block_group_index(struct btrfs_block_group_cache *cache)
{
	int index;
	if (cache->flags & BTRFS_BLOCK_GROUP_RAID10)
		index = 0;
	else if (cache->flags & BTRFS_BLOCK_GROUP_RAID1)
		index = 1;
	else if (cache->flags & BTRFS_BLOCK_GROUP_DUP)
		index = 2;
	else if (cache->flags & BTRFS_BLOCK_GROUP_RAID0)
		index = 3;
	else
		index = 4;
	return index;
}

J
Josef Bacik 已提交
4852
enum btrfs_loop_type {
4853
	LOOP_FIND_IDEAL = 0,
J
Josef Bacik 已提交
4854 4855 4856 4857 4858 4859
	LOOP_CACHING_NOWAIT = 1,
	LOOP_CACHING_WAIT = 2,
	LOOP_ALLOC_CHUNK = 3,
	LOOP_NO_EMPTY_SIZE = 4,
};

4860 4861 4862 4863
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
 * ins->objectid == block start
4864
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
4865 4866 4867
 * ins->offset == number of blocks
 * Any available blocks before search_start are skipped.
 */
C
Chris Mason 已提交
4868
static noinline int find_free_extent(struct btrfs_trans_handle *trans,
4869 4870 4871 4872
				     struct btrfs_root *orig_root,
				     u64 num_bytes, u64 empty_size,
				     u64 search_start, u64 search_end,
				     u64 hint_byte, struct btrfs_key *ins,
4873
				     u64 data)
4874
{
4875
	int ret = 0;
C
Chris Mason 已提交
4876
	struct btrfs_root *root = orig_root->fs_info->extent_root;
4877
	struct btrfs_free_cluster *last_ptr = NULL;
4878
	struct btrfs_block_group_cache *block_group = NULL;
4879
	int empty_cluster = 2 * 1024 * 1024;
C
Chris Mason 已提交
4880
	int allowed_chunk_alloc = 0;
4881
	int done_chunk_alloc = 0;
4882
	struct btrfs_space_info *space_info;
4883 4884
	int last_ptr_loop = 0;
	int loop = 0;
4885
	int index = 0;
J
Josef Bacik 已提交
4886
	bool found_uncached_bg = false;
4887
	bool failed_cluster_refill = false;
4888
	bool failed_alloc = false;
4889
	bool use_cluster = true;
4890 4891
	u64 ideal_cache_percent = 0;
	u64 ideal_cache_offset = 0;
4892

4893
	WARN_ON(num_bytes < root->sectorsize);
4894
	btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
4895 4896
	ins->objectid = 0;
	ins->offset = 0;
4897

J
Josef Bacik 已提交
4898
	space_info = __find_space_info(root->fs_info, data);
4899
	if (!space_info) {
4900
		printk(KERN_ERR "No space info for %llu\n", data);
4901 4902
		return -ENOSPC;
	}
J
Josef Bacik 已提交
4903

4904 4905 4906 4907 4908 4909 4910
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

C
Chris Mason 已提交
4911 4912 4913
	if (orig_root->ref_cows || empty_size)
		allowed_chunk_alloc = 1;

4914
	if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
4915
		last_ptr = &root->fs_info->meta_alloc_cluster;
4916 4917
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
4918 4919
	}

4920 4921
	if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
	    btrfs_test_opt(root, SSD)) {
4922 4923
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
J
Josef Bacik 已提交
4924

4925
	if (last_ptr) {
4926 4927 4928 4929
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
4930
	}
4931

4932
	search_start = max(search_start, first_logical_byte(root, 0));
4933
	search_start = max(search_start, hint_byte);
4934

J
Josef Bacik 已提交
4935
	if (!last_ptr)
4936 4937
		empty_cluster = 0;

J
Josef Bacik 已提交
4938
	if (search_start == hint_byte) {
4939
ideal_cache:
J
Josef Bacik 已提交
4940 4941
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
J
Josef Bacik 已提交
4942 4943 4944
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
4945 4946 4947
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
4948 4949
		 */
		if (block_group && block_group_bits(block_group, data) &&
4950 4951
		    (block_group->cached != BTRFS_CACHE_NO ||
		     search_start == ideal_cache_offset)) {
J
Josef Bacik 已提交
4952
			down_read(&space_info->groups_sem);
4953 4954 4955 4956 4957 4958 4959 4960 4961 4962
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
4963
			} else {
4964
				index = get_block_group_index(block_group);
4965
				goto have_block_group;
4966
			}
J
Josef Bacik 已提交
4967
		} else if (block_group) {
4968
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
4969
		}
4970
	}
J
Josef Bacik 已提交
4971
search:
4972
	down_read(&space_info->groups_sem);
4973 4974
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
4975
		u64 offset;
J
Josef Bacik 已提交
4976
		int cached;
4977

4978
		btrfs_get_block_group(block_group);
J
Josef Bacik 已提交
4979
		search_start = block_group->key.objectid;
4980

4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
		if (!block_group_bits(block_group, data)) {
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
			if ((data & extra) && !(block_group->flags & extra))
				goto loop;
		}

J
Josef Bacik 已提交
5000
have_block_group:
J
Josef Bacik 已提交
5001
		if (unlikely(block_group->cached == BTRFS_CACHE_NO)) {
5002 5003
			u64 free_percent;

5004 5005
			ret = cache_block_group(block_group, trans,
						orig_root, 1);
5006 5007 5008
			if (block_group->cached == BTRFS_CACHE_FINISHED)
				goto have_block_group;

5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019
			free_percent = btrfs_block_group_used(&block_group->item);
			free_percent *= 100;
			free_percent = div64_u64(free_percent,
						 block_group->key.offset);
			free_percent = 100 - free_percent;
			if (free_percent > ideal_cache_percent &&
			    likely(!block_group->ro)) {
				ideal_cache_offset = block_group->key.objectid;
				ideal_cache_percent = free_percent;
			}

J
Josef Bacik 已提交
5020
			/*
5021 5022
			 * The caching workers are limited to 2 threads, so we
			 * can queue as much work as we care to.
J
Josef Bacik 已提交
5023
			 */
5024
			if (loop > LOOP_FIND_IDEAL) {
5025 5026
				ret = cache_block_group(block_group, trans,
							orig_root, 0);
J
Josef Bacik 已提交
5027
				BUG_ON(ret);
J
Josef Bacik 已提交
5028
			}
J
Josef Bacik 已提交
5029 5030
			found_uncached_bg = true;

5031 5032 5033 5034 5035
			/*
			 * If loop is set for cached only, try the next block
			 * group.
			 */
			if (loop == LOOP_FIND_IDEAL)
J
Josef Bacik 已提交
5036 5037 5038
				goto loop;
		}

5039 5040 5041 5042
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached))
			found_uncached_bg = true;

5043
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
5044
			goto loop;
J
Josef Bacik 已提交
5045

C
Chris Mason 已提交
5046
		spin_lock(&block_group->free_space_ctl->tree_lock);
5047
		if (cached &&
C
Chris Mason 已提交
5048 5049 5050
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_size) {
			spin_unlock(&block_group->free_space_ctl->tree_lock);
5051 5052
			goto loop;
		}
C
Chris Mason 已提交
5053
		spin_unlock(&block_group->free_space_ctl->tree_lock);
5054

5055 5056 5057 5058 5059 5060 5061 5062 5063 5064
		/*
		 * Ok we want to try and use the cluster allocator, so lets look
		 * there, unless we are on LOOP_NO_EMPTY_SIZE, since we will
		 * have tried the cluster allocator plenty of times at this
		 * point and not have found anything, so we are likely way too
		 * fragmented for the clustering stuff to find anything, so lets
		 * just skip it and let the allocator find whatever block it can
		 * find
		 */
		if (last_ptr && loop < LOOP_NO_EMPTY_SIZE) {
5065 5066 5067 5068 5069
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
			spin_lock(&last_ptr->refill_lock);
5070 5071 5072 5073 5074 5075 5076
			if (last_ptr->block_group &&
			    (last_ptr->block_group->ro ||
			    !block_group_bits(last_ptr->block_group, data))) {
				offset = 0;
				goto refill_cluster;
			}

5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091
			offset = btrfs_alloc_from_cluster(block_group, last_ptr,
						 num_bytes, search_start);
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
				goto checks;
			}

			spin_lock(&last_ptr->lock);
			/*
			 * whoops, this cluster doesn't actually point to
			 * this block group.  Get a ref on the block
			 * group is does point to and try again
			 */
			if (!last_ptr_loop && last_ptr->block_group &&
5092 5093 5094
			    last_ptr->block_group != block_group &&
			    index <=
				 get_block_group_index(last_ptr->block_group)) {
5095 5096 5097

				btrfs_put_block_group(block_group);
				block_group = last_ptr->block_group;
5098
				btrfs_get_block_group(block_group);
5099 5100 5101 5102 5103
				spin_unlock(&last_ptr->lock);
				spin_unlock(&last_ptr->refill_lock);

				last_ptr_loop = 1;
				search_start = block_group->key.objectid;
5104 5105 5106 5107 5108 5109 5110
				/*
				 * we know this block group is properly
				 * in the list because
				 * btrfs_remove_block_group, drops the
				 * cluster before it removes the block
				 * group from the list
				 */
5111 5112 5113
				goto have_block_group;
			}
			spin_unlock(&last_ptr->lock);
5114
refill_cluster:
5115 5116 5117 5118 5119 5120 5121 5122 5123
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

			last_ptr_loop = 0;

			/* allocate a cluster in this block group */
5124
			ret = btrfs_find_space_cluster(trans, root,
5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140
					       block_group, last_ptr,
					       offset, num_bytes,
					       empty_cluster + empty_size);
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
						  last_ptr, num_bytes,
						  search_start);
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
					goto checks;
				}
5141 5142
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
5143 5144
				spin_unlock(&last_ptr->refill_lock);

5145
				failed_cluster_refill = true;
J
Josef Bacik 已提交
5146 5147 5148
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
5149
			}
J
Josef Bacik 已提交
5150

5151 5152 5153 5154 5155 5156
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
5157
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
5158
			spin_unlock(&last_ptr->refill_lock);
5159
			goto loop;
5160 5161
		}

5162 5163
		offset = btrfs_find_space_for_alloc(block_group, search_start,
						    num_bytes, empty_size);
5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
5175
			wait_block_group_cache_progress(block_group,
5176 5177
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
5178
			goto have_block_group;
5179 5180
		} else if (!offset) {
			goto loop;
J
Josef Bacik 已提交
5181
		}
5182
checks:
5183
		search_start = stripe_align(root, offset);
J
Josef Bacik 已提交
5184
		/* move on to the next group */
5185 5186
		if (search_start + num_bytes >= search_end) {
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5187
			goto loop;
5188
		}
5189

J
Josef Bacik 已提交
5190 5191
		/* move on to the next group */
		if (search_start + num_bytes >
5192 5193
		    block_group->key.objectid + block_group->key.offset) {
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5194
			goto loop;
5195
		}
5196

5197 5198
		ins->objectid = search_start;
		ins->offset = num_bytes;
J
Josef Bacik 已提交
5199

5200 5201 5202 5203
		if (offset < search_start)
			btrfs_add_free_space(block_group, offset,
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
5204

5205
		ret = btrfs_update_reserved_bytes(block_group, num_bytes, 1,
5206 5207
					    (data & BTRFS_BLOCK_GROUP_DATA));
		if (ret == -EAGAIN) {
5208
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5209
			goto loop;
J
Josef Bacik 已提交
5210
		}
5211

5212
		/* we are all good, lets return */
J
Josef Bacik 已提交
5213 5214
		ins->objectid = search_start;
		ins->offset = num_bytes;
5215

5216 5217 5218 5219
		if (offset < search_start)
			btrfs_add_free_space(block_group, offset,
					     search_start - offset);
		BUG_ON(offset > search_start);
5220
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5221 5222
		break;
loop:
5223
		failed_cluster_refill = false;
5224
		failed_alloc = false;
5225
		BUG_ON(index != get_block_group_index(block_group));
5226
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5227 5228 5229
	}
	up_read(&space_info->groups_sem);

5230 5231 5232
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

5233 5234 5235 5236 5237
	/* LOOP_FIND_IDEAL, only search caching/cached bg's, and don't wait for
	 *			for them to make caching progress.  Also
	 *			determine the best possible bg to cache
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
5238 5239 5240 5241
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
5242
	 */
5243
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
5244
		index = 0;
5245
		if (loop == LOOP_FIND_IDEAL && found_uncached_bg) {
J
Josef Bacik 已提交
5246
			found_uncached_bg = false;
5247
			loop++;
5248
			if (!ideal_cache_percent)
J
Josef Bacik 已提交
5249
				goto search;
5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281

			/*
			 * 1 of the following 2 things have happened so far
			 *
			 * 1) We found an ideal block group for caching that
			 * is mostly full and will cache quickly, so we might
			 * as well wait for it.
			 *
			 * 2) We searched for cached only and we didn't find
			 * anything, and we didn't start any caching kthreads
			 * either, so chances are we will loop through and
			 * start a couple caching kthreads, and then come back
			 * around and just wait for them.  This will be slower
			 * because we will have 2 caching kthreads reading at
			 * the same time when we could have just started one
			 * and waited for it to get far enough to give us an
			 * allocation, so go ahead and go to the wait caching
			 * loop.
			 */
			loop = LOOP_CACHING_WAIT;
			search_start = ideal_cache_offset;
			ideal_cache_percent = 0;
			goto ideal_cache;
		} else if (loop == LOOP_FIND_IDEAL) {
			/*
			 * Didn't find a uncached bg, wait on anything we find
			 * next.
			 */
			loop = LOOP_CACHING_WAIT;
			goto search;
		}

5282
		loop++;
J
Josef Bacik 已提交
5283 5284

		if (loop == LOOP_ALLOC_CHUNK) {
5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296
		       if (allowed_chunk_alloc) {
				ret = do_chunk_alloc(trans, root, num_bytes +
						     2 * 1024 * 1024, data,
						     CHUNK_ALLOC_LIMITED);
				allowed_chunk_alloc = 0;
				if (ret == 1)
					done_chunk_alloc = 1;
			} else if (!done_chunk_alloc &&
				   space_info->force_alloc ==
				   CHUNK_ALLOC_NO_FORCE) {
				space_info->force_alloc = CHUNK_ALLOC_LIMITED;
			}
J
Josef Bacik 已提交
5297

5298 5299 5300 5301 5302 5303
		       /*
			* We didn't allocate a chunk, go ahead and drop the
			* empty size and loop again.
			*/
		       if (!done_chunk_alloc)
			       loop = LOOP_NO_EMPTY_SIZE;
J
Josef Bacik 已提交
5304 5305
		}

5306 5307 5308
		if (loop == LOOP_NO_EMPTY_SIZE) {
			empty_size = 0;
			empty_cluster = 0;
5309
		}
5310 5311

		goto search;
J
Josef Bacik 已提交
5312 5313
	} else if (!ins->objectid) {
		ret = -ENOSPC;
5314
	} else if (ins->objectid) {
5315
		ret = 0;
C
Chris Mason 已提交
5316 5317
	}

C
Chris Mason 已提交
5318
	return ret;
5319
}
5320

J
Josef Bacik 已提交
5321 5322
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
5323 5324
{
	struct btrfs_block_group_cache *cache;
5325
	int index = 0;
J
Josef Bacik 已提交
5326

J
Josef Bacik 已提交
5327
	spin_lock(&info->lock);
C
Chris Mason 已提交
5328 5329
	printk(KERN_INFO "space_info has %llu free, is %sfull\n",
	       (unsigned long long)(info->total_bytes - info->bytes_used -
J
Josef Bacik 已提交
5330
				    info->bytes_pinned - info->bytes_reserved -
5331
				    info->bytes_readonly),
C
Chris Mason 已提交
5332
	       (info->full) ? "" : "not ");
5333 5334
	printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
5335
	       (unsigned long long)info->total_bytes,
5336
	       (unsigned long long)info->bytes_used,
5337
	       (unsigned long long)info->bytes_pinned,
5338
	       (unsigned long long)info->bytes_reserved,
5339
	       (unsigned long long)info->bytes_may_use,
5340
	       (unsigned long long)info->bytes_readonly);
J
Josef Bacik 已提交
5341 5342 5343 5344
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
5345

5346
	down_read(&info->groups_sem);
5347 5348
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
5349
		spin_lock(&cache->lock);
C
Chris Mason 已提交
5350 5351 5352 5353 5354 5355 5356
		printk(KERN_INFO "block group %llu has %llu bytes, %llu used "
		       "%llu pinned %llu reserved\n",
		       (unsigned long long)cache->key.objectid,
		       (unsigned long long)cache->key.offset,
		       (unsigned long long)btrfs_block_group_used(&cache->item),
		       (unsigned long long)cache->pinned,
		       (unsigned long long)cache->reserved);
J
Josef Bacik 已提交
5357 5358 5359
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
5360 5361
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
5362
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
5363
}
5364

5365 5366 5367 5368 5369 5370
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
			 u64 search_end, struct btrfs_key *ins,
			 u64 data)
5371 5372
{
	int ret;
5373
	u64 search_start = 0;
5374

J
Josef Bacik 已提交
5375
	data = btrfs_get_alloc_profile(root, data);
5376
again:
C
Chris Mason 已提交
5377 5378 5379 5380
	/*
	 * the only place that sets empty_size is btrfs_realloc_node, which
	 * is not called recursively on allocations
	 */
J
Josef Bacik 已提交
5381
	if (empty_size || root->ref_cows)
5382
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
5383 5384
				     num_bytes + 2 * 1024 * 1024, data,
				     CHUNK_ALLOC_NO_FORCE);
5385

5386 5387
	WARN_ON(num_bytes < root->sectorsize);
	ret = find_free_extent(trans, root, num_bytes, empty_size,
5388 5389
			       search_start, search_end, hint_byte,
			       ins, data);
5390

5391 5392
	if (ret == -ENOSPC && num_bytes > min_alloc_size) {
		num_bytes = num_bytes >> 1;
J
Josef Bacik 已提交
5393
		num_bytes = num_bytes & ~(root->sectorsize - 1);
5394
		num_bytes = max(num_bytes, min_alloc_size);
C
Chris Mason 已提交
5395
		do_chunk_alloc(trans, root->fs_info->extent_root,
5396
			       num_bytes, data, CHUNK_ALLOC_FORCE);
5397 5398
		goto again;
	}
5399
	if (ret == -ENOSPC && btrfs_test_opt(root, ENOSPC_DEBUG)) {
J
Josef Bacik 已提交
5400 5401 5402
		struct btrfs_space_info *sinfo;

		sinfo = __find_space_info(root->fs_info, data);
C
Chris Mason 已提交
5403 5404 5405
		printk(KERN_ERR "btrfs allocation failed flags %llu, "
		       "wanted %llu\n", (unsigned long long)data,
		       (unsigned long long)num_bytes);
J
Josef Bacik 已提交
5406
		dump_space_info(sinfo, num_bytes, 1);
5407
	}
J
Josef Bacik 已提交
5408

5409 5410
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);

J
Josef Bacik 已提交
5411
	return ret;
5412 5413
}

5414 5415
int btrfs_free_reserved_extent(struct btrfs_root *root, u64 start, u64 len)
{
J
Josef Bacik 已提交
5416
	struct btrfs_block_group_cache *cache;
5417
	int ret = 0;
J
Josef Bacik 已提交
5418 5419 5420

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
C
Chris Mason 已提交
5421 5422
		printk(KERN_ERR "Unable to find block group for %llu\n",
		       (unsigned long long)start);
J
Josef Bacik 已提交
5423 5424
		return -ENOSPC;
	}
5425

5426 5427
	if (btrfs_test_opt(root, DISCARD))
		ret = btrfs_discard_extent(root, start, len, NULL);
5428

J
Josef Bacik 已提交
5429
	btrfs_add_free_space(cache, start, len);
5430
	btrfs_update_reserved_bytes(cache, len, 0, 1);
5431
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
5432

5433 5434
	trace_btrfs_reserved_extent_free(root, start, len);

5435 5436 5437
	return ret;
}

5438 5439 5440 5441 5442
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
5443 5444
{
	int ret;
5445
	struct btrfs_fs_info *fs_info = root->fs_info;
5446
	struct btrfs_extent_item *extent_item;
5447
	struct btrfs_extent_inline_ref *iref;
5448
	struct btrfs_path *path;
5449 5450 5451
	struct extent_buffer *leaf;
	int type;
	u32 size;
5452

5453 5454 5455 5456
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
5457

5458
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
5459 5460

	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
5461 5462
	if (!path)
		return -ENOMEM;
5463

5464
	path->leave_spinning = 1;
5465 5466
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
C
Chris Mason 已提交
5467
	BUG_ON(ret);
J
Josef Bacik 已提交
5468

5469 5470
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
5471
				     struct btrfs_extent_item);
5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
5492 5493

	btrfs_mark_buffer_dirty(path->nodes[0]);
5494
	btrfs_free_path(path);
5495

5496
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5497
	if (ret) {
C
Chris Mason 已提交
5498 5499 5500
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
5501 5502
		BUG();
	}
5503 5504 5505
	return ret;
}

5506 5507 5508 5509 5510
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins)
5511 5512
{
	int ret;
5513 5514 5515 5516 5517 5518 5519
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
5520

5521
	path = btrfs_alloc_path();
5522 5523
	if (!path)
		return -ENOMEM;
5524

5525 5526 5527
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
5528
	BUG_ON(ret);
5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
	block_info = (struct btrfs_tree_block_info *)(extent_item + 1);

	btrfs_set_tree_block_key(leaf, block_info, key);
	btrfs_set_tree_block_level(leaf, block_info, level);

	iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

5557
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578
	if (ret) {
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
		BUG();
	}
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

	ret = btrfs_add_delayed_data_ref(trans, ins->objectid, ins->offset,
					 0, root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL);
5579 5580
	return ret;
}
5581 5582 5583 5584 5585 5586

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
5587 5588 5589 5590
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
5591 5592 5593
{
	int ret;
	struct btrfs_block_group_cache *block_group;
5594 5595 5596
	struct btrfs_caching_control *caching_ctl;
	u64 start = ins->objectid;
	u64 num_bytes = ins->offset;
5597 5598

	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
5599
	cache_block_group(block_group, trans, NULL, 0);
5600
	caching_ctl = get_caching_control(block_group);
5601

5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632
	if (!caching_ctl) {
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
		BUG_ON(ret);
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
			BUG_ON(ret);
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			BUG_ON(ret);
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			BUG_ON(ret);

			start = caching_ctl->progress;
			num_bytes = ins->objectid + ins->offset -
				    caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
			BUG_ON(ret);
		}

		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}

5633
	ret = btrfs_update_reserved_bytes(block_group, ins->offset, 1, 1);
5634
	BUG_ON(ret);
5635
	btrfs_put_block_group(block_group);
5636 5637
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
5638 5639 5640
	return ret;
}

5641 5642
struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
					    struct btrfs_root *root,
5643 5644
					    u64 bytenr, u32 blocksize,
					    int level)
5645 5646 5647 5648 5649 5650 5651
{
	struct extent_buffer *buf;

	buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
5652
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
5653 5654
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
5655 5656

	btrfs_set_lock_blocking(buf);
5657
	btrfs_set_buffer_uptodate(buf);
5658

5659
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
5660 5661 5662 5663 5664 5665 5666 5667 5668 5669
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
		if (root->log_transid % 2 == 0)
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
5670 5671
	} else {
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
5672
			 buf->start + buf->len - 1, GFP_NOFS);
5673
	}
5674
	trans->blocks_used++;
5675
	/* this returns a buffer locked for blocking */
5676 5677 5678
	return buf;
}

5679 5680 5681 5682 5683
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
5684
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
5685 5686 5687 5688 5689
	int ret;

	block_rsv = get_block_rsv(trans, root);

	if (block_rsv->size == 0) {
5690 5691
		ret = reserve_metadata_bytes(trans, root, block_rsv,
					     blocksize, 0);
5692 5693 5694 5695 5696 5697 5698 5699
		/*
		 * If we couldn't reserve metadata bytes try and use some from
		 * the global reserve.
		 */
		if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
5700
			return ERR_PTR(ret);
5701
		} else if (ret) {
5702
			return ERR_PTR(ret);
5703
		}
5704 5705 5706 5707 5708 5709
		return block_rsv;
	}

	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;
5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724
	if (ret) {
		WARN_ON(1);
		ret = reserve_metadata_bytes(trans, root, block_rsv, blocksize,
					     0);
		if (!ret) {
			spin_lock(&block_rsv->lock);
			block_rsv->size += blocksize;
			spin_unlock(&block_rsv->lock);
			return block_rsv;
		} else if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
		}
	}
5725 5726 5727 5728 5729 5730 5731 5732 5733 5734

	return ERR_PTR(-ENOSPC);
}

static void unuse_block_rsv(struct btrfs_block_rsv *block_rsv, u32 blocksize)
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
	block_rsv_release_bytes(block_rsv, NULL, 0);
}

5735
/*
5736 5737 5738 5739
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
5740 5741
 * returns the tree buffer or NULL.
 */
5742
struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
5743 5744 5745 5746
					struct btrfs_root *root, u32 blocksize,
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
					u64 hint, u64 empty_size)
5747
{
C
Chris Mason 已提交
5748
	struct btrfs_key ins;
5749
	struct btrfs_block_rsv *block_rsv;
5750
	struct extent_buffer *buf;
5751 5752 5753
	u64 flags = 0;
	int ret;

5754

5755 5756 5757 5758 5759 5760
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

	ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
				   empty_size, hint, (u64)-1, &ins, 0);
5761
	if (ret) {
5762
		unuse_block_rsv(block_rsv, blocksize);
5763
		return ERR_PTR(ret);
5764
	}
5765

5766 5767
	buf = btrfs_init_new_buffer(trans, root, ins.objectid,
				    blocksize, level);
5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795
	BUG_ON(IS_ERR(buf));

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
		extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
		BUG_ON(!extent_op);
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
		extent_op->update_key = 1;
		extent_op->update_flags = 1;
		extent_op->is_data = 0;

		ret = btrfs_add_delayed_tree_ref(trans, ins.objectid,
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
					extent_op);
		BUG_ON(ret);
	}
5796 5797
	return buf;
}
5798

5799 5800 5801 5802 5803 5804 5805 5806 5807
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
5808 5809
	int reada_slot;
	int reada_count;
5810 5811 5812 5813 5814
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
5815 5816 5817 5818
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
5819
{
Y
Yan, Zheng 已提交
5820 5821 5822
	u64 bytenr;
	u64 generation;
	u64 refs;
5823
	u64 flags;
5824
	u32 nritems;
Y
Yan, Zheng 已提交
5825 5826 5827
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
5828
	int ret;
Y
Yan, Zheng 已提交
5829 5830
	int slot;
	int nread = 0;
5831

Y
Yan, Zheng 已提交
5832 5833 5834 5835 5836 5837 5838 5839
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
5840

Y
Yan, Zheng 已提交
5841 5842 5843
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
	blocksize = btrfs_level_size(root, wc->level - 1);
5844

Y
Yan, Zheng 已提交
5845 5846 5847
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
5848

C
Chris Mason 已提交
5849
		cond_resched();
Y
Yan, Zheng 已提交
5850 5851
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
5852

Y
Yan, Zheng 已提交
5853 5854
		if (slot == path->slots[wc->level])
			goto reada;
5855

Y
Yan, Zheng 已提交
5856 5857
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
5858 5859
			continue;

5860 5861 5862 5863 5864 5865
		/* We don't lock the tree block, it's OK to be racy here */
		ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
					       &refs, &flags);
		BUG_ON(ret);
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
5866 5867 5868
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
5869

5870 5871 5872
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
5873 5874 5875 5876 5877 5878 5879 5880
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
5881 5882 5883 5884
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
5885
		}
Y
Yan, Zheng 已提交
5886 5887 5888 5889
reada:
		ret = readahead_tree_block(root, bytenr, blocksize,
					   generation);
		if (ret)
5890
			break;
Y
Yan, Zheng 已提交
5891
		nread++;
C
Chris Mason 已提交
5892
	}
Y
Yan, Zheng 已提交
5893
	wc->reada_slot = slot;
C
Chris Mason 已提交
5894
}
5895

Y
Yan Zheng 已提交
5896
/*
5897 5898 5899 5900 5901 5902
 * hepler to process tree block while walking down the tree.
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
5903
 */
5904
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
5905
				   struct btrfs_root *root,
5906
				   struct btrfs_path *path,
5907
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
5908
{
5909 5910 5911
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
5912 5913
	int ret;

5914 5915 5916
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
5917

5918 5919 5920 5921
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
5922 5923 5924
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
5925 5926 5927 5928 5929 5930 5931 5932
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
					       eb->start, eb->len,
					       &wc->refs[level],
					       &wc->flags[level]);
		BUG_ON(ret);
		BUG_ON(wc->refs[level] == 0);
	}
5933

5934 5935 5936
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
5937

5938
		if (path->locks[level] && !wc->keep_locks) {
5939
			btrfs_tree_unlock_rw(eb, path->locks[level]);
5940 5941 5942 5943
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
5944

5945 5946 5947 5948
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
		ret = btrfs_inc_ref(trans, root, eb, 1);
Y
Yan Zheng 已提交
5949
		BUG_ON(ret);
5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962
		ret = btrfs_dec_ref(trans, root, eb, 0);
		BUG_ON(ret);
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
						  eb->len, flag, 0);
		BUG_ON(ret);
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
5963
		btrfs_tree_unlock_rw(eb, path->locks[level]);
5964 5965 5966 5967 5968
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984
/*
 * hepler to process tree block pointer.
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
5985
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
6005 6006
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6007
		return 1;
6008
	}
Y
Yan, Zheng 已提交
6009 6010 6011 6012 6013 6014 6015

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
	blocksize = btrfs_level_size(root, level - 1);

	next = btrfs_find_tree_block(root, bytenr, blocksize);
	if (!next) {
		next = btrfs_find_create_tree_block(root, bytenr, blocksize);
6016 6017
		if (!next)
			return -ENOMEM;
Y
Yan, Zheng 已提交
6018 6019 6020 6021 6022
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

6023 6024 6025 6026 6027 6028
	ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
	BUG_ON(ret);
	BUG_ON(wc->refs[level - 1] == 0);
	*lookup_info = 0;
Y
Yan, Zheng 已提交
6029

6030
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
6031
		if (wc->refs[level - 1] > 1) {
6032 6033 6034 6035
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
6049 6050 6051 6052
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
6053 6054 6055 6056 6057 6058
	}

	if (!btrfs_buffer_uptodate(next, generation)) {
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
6059
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6060 6061 6062 6063 6064 6065
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
		next = read_tree_block(root, bytenr, blocksize, generation);
6066 6067
		if (!next)
			return -EIO;
Y
Yan, Zheng 已提交
6068 6069 6070 6071 6072 6073 6074 6075
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
6076
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Y
Yan, Zheng 已提交
6077 6078 6079 6080 6081 6082 6083
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
6084 6085 6086 6087 6088 6089 6090 6091
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
6092

6093 6094 6095
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
					root->root_key.objectid, level - 1, 0);
		BUG_ON(ret);
Y
Yan, Zheng 已提交
6096 6097 6098
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
6099
	*lookup_info = 1;
Y
Yan, Zheng 已提交
6100 6101 6102
	return 1;
}

6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119
/*
 * hepler to process tree block while walking up the tree.
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
6120
	int ret;
6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6147
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6148 6149 6150 6151 6152

			ret = btrfs_lookup_extent_info(trans, root,
						       eb->start, eb->len,
						       &wc->refs[level],
						       &wc->flags[level]);
Y
Yan Zheng 已提交
6153
			BUG_ON(ret);
6154 6155
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
6156
				btrfs_tree_unlock_rw(eb, path->locks[level]);
6157 6158
				return 1;
			}
Y
Yan Zheng 已提交
6159
		}
6160
	}
Y
Yan Zheng 已提交
6161

6162 6163
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6164

6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
				ret = btrfs_dec_ref(trans, root, eb, 1);
			else
				ret = btrfs_dec_ref(trans, root, eb, 0);
			BUG_ON(ret);
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6178
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
6195 6196
	}

6197
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
6198 6199 6200
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
6201
	return 0;
6202 6203 6204 6205 6206 6207 6208 6209
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
6210
	int lookup_info = 1;
6211 6212 6213
	int ret;

	while (level >= 0) {
6214
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
6215 6216 6217 6218 6219 6220
		if (ret > 0)
			break;

		if (level == 0)
			break;

6221 6222 6223 6224
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

6225
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
6226 6227 6228
		if (ret > 0) {
			path->slots[level]++;
			continue;
6229 6230
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
6231
		level = wc->level;
Y
Yan Zheng 已提交
6232 6233 6234 6235
	}
	return 0;
}

C
Chris Mason 已提交
6236
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6237
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
6238
				 struct btrfs_path *path,
6239
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
6240
{
6241
	int level = wc->level;
C
Chris Mason 已提交
6242
	int ret;
6243

6244 6245 6246 6247 6248 6249
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
6250 6251
			return 0;
		} else {
6252 6253 6254
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
6255

6256
			if (path->locks[level]) {
6257 6258
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
6259
				path->locks[level] = 0;
Y
Yan Zheng 已提交
6260
			}
6261 6262 6263
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
6264 6265 6266 6267 6268
		}
	}
	return 1;
}

C
Chris Mason 已提交
6269
/*
6270 6271 6272 6273 6274 6275 6276 6277 6278
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
C
Chris Mason 已提交
6279
 */
6280 6281
void btrfs_drop_snapshot(struct btrfs_root *root,
			 struct btrfs_block_rsv *block_rsv, int update_ref)
C
Chris Mason 已提交
6282
{
6283
	struct btrfs_path *path;
6284 6285
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
6286
	struct btrfs_root_item *root_item = &root->root_item;
6287 6288 6289 6290 6291
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
C
Chris Mason 已提交
6292

6293
	path = btrfs_alloc_path();
6294 6295 6296 6297
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
C
Chris Mason 已提交
6298

6299
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
6300 6301
	if (!wc) {
		btrfs_free_path(path);
6302 6303
		err = -ENOMEM;
		goto out;
6304
	}
6305

6306
	trans = btrfs_start_transaction(tree_root, 0);
6307 6308
	BUG_ON(IS_ERR(trans));

6309 6310
	if (block_rsv)
		trans->block_rsv = block_rsv;
6311

6312
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
6313
		level = btrfs_header_level(root->node);
6314 6315
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
6316
		path->slots[level] = 0;
6317
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6318 6319
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
6320 6321
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
6322 6323 6324
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

6325
		level = root_item->drop_level;
6326
		BUG_ON(level == 0);
6327
		path->lowest_level = level;
6328 6329 6330 6331
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
6332
			goto out_free;
6333
		}
Y
Yan, Zheng 已提交
6334
		WARN_ON(ret > 0);
6335

6336 6337 6338 6339
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
6340
		btrfs_unlock_up_safe(path, 0);
6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
						path->nodes[level]->len,
						&wc->refs[level],
						&wc->flags[level]);
			BUG_ON(ret);
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
6362
	}
6363 6364 6365 6366 6367 6368

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
Y
Yan, Zheng 已提交
6369
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
6370

C
Chris Mason 已提交
6371
	while (1) {
6372 6373 6374
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6375
			break;
6376
		}
C
Chris Mason 已提交
6377

6378 6379 6380
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6381
			break;
6382 6383 6384 6385
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
6386 6387
			break;
		}
6388 6389 6390 6391 6392 6393 6394 6395 6396 6397

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
6398
		if (btrfs_should_end_transaction(trans, tree_root)) {
6399 6400 6401 6402 6403
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
			BUG_ON(ret);

6404
			btrfs_end_transaction_throttle(trans, tree_root);
6405
			trans = btrfs_start_transaction(tree_root, 0);
6406
			BUG_ON(IS_ERR(trans));
6407 6408
			if (block_rsv)
				trans->block_rsv = block_rsv;
6409
		}
C
Chris Mason 已提交
6410
	}
6411
	btrfs_release_path(path);
6412 6413 6414 6415 6416
	BUG_ON(err);

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
	BUG_ON(ret);

6417 6418 6419 6420 6421
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
		ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
					   NULL, NULL);
		BUG_ON(ret < 0);
		if (ret > 0) {
6422 6423 6424 6425 6426 6427 6428
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
6429 6430 6431 6432 6433 6434 6435 6436 6437 6438
		}
	}

	if (root->in_radix) {
		btrfs_free_fs_root(tree_root->fs_info, root);
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
		kfree(root);
	}
6439
out_free:
6440
	btrfs_end_transaction_throttle(trans, tree_root);
6441
	kfree(wc);
6442
	btrfs_free_path(path);
6443 6444 6445 6446
out:
	if (err)
		btrfs_std_error(root->fs_info, err);
	return;
C
Chris Mason 已提交
6447
}
C
Chris Mason 已提交
6448

6449 6450 6451 6452 6453
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
 */
Y
Yan Zheng 已提交
6454 6455 6456 6457 6458 6459
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
6460
	struct walk_control *wc;
Y
Yan Zheng 已提交
6461 6462 6463 6464 6465
	int level;
	int parent_level;
	int ret = 0;
	int wret;

6466 6467
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
6468
	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
6469 6470
	if (!path)
		return -ENOMEM;
Y
Yan Zheng 已提交
6471

6472
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
T
Tsutomu Itoh 已提交
6473 6474 6475 6476
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
6477

6478
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
6479 6480 6481 6482 6483
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

6484
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
6485 6486 6487
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
6488
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6489 6490 6491 6492 6493 6494 6495 6496

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
Y
Yan, Zheng 已提交
6497
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
6498 6499

	while (1) {
6500 6501
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
6502 6503
			ret = wret;
			break;
6504
		}
Y
Yan Zheng 已提交
6505

6506
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
6507 6508 6509 6510 6511 6512
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

6513
	kfree(wc);
Y
Yan Zheng 已提交
6514 6515 6516 6517
	btrfs_free_path(path);
	return ret;
}

6518 6519 6520 6521 6522 6523
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
	u64 stripped = BTRFS_BLOCK_GROUP_RAID0 |
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

6524 6525 6526 6527 6528 6529 6530 6531
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;

6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
		return flags;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

		/* turn single device chunks into raid0 */
		return stripped | BTRFS_BLOCK_GROUP_RAID0;
	}
	return flags;
}

6563
static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
C
Chris Mason 已提交
6564
{
6565 6566
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
6567
	u64 min_allocable_bytes;
6568
	int ret = -ENOSPC;
C
Chris Mason 已提交
6569

6570

6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
		min_allocable_bytes = 1 * 1024 * 1024;
	else
		min_allocable_bytes = 0;

6583 6584
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
6585 6586 6587 6588 6589 6590

	if (cache->ro) {
		ret = 0;
		goto out;
	}

6591 6592 6593 6594 6595
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
	    sinfo->bytes_may_use + sinfo->bytes_readonly +
6596 6597
	    cache->reserved_pinned + num_bytes + min_allocable_bytes <=
	    sinfo->total_bytes) {
6598 6599 6600 6601 6602 6603
		sinfo->bytes_readonly += num_bytes;
		sinfo->bytes_reserved += cache->reserved_pinned;
		cache->reserved_pinned = 0;
		cache->ro = 1;
		ret = 0;
	}
6604
out:
6605 6606 6607 6608
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
6609

6610 6611
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
6612

6613 6614 6615 6616
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
6617

6618
	BUG_ON(cache->ro);
C
Chris Mason 已提交
6619

C
Chris Mason 已提交
6620
	trans = btrfs_join_transaction(root);
6621
	BUG_ON(IS_ERR(trans));
6622

6623 6624
	alloc_flags = update_block_group_flags(root, cache->flags);
	if (alloc_flags != cache->flags)
6625 6626
		do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			       CHUNK_ALLOC_FORCE);
6627

6628
	ret = set_block_group_ro(cache, 0);
6629 6630 6631
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
6632 6633
	ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			     CHUNK_ALLOC_FORCE);
6634 6635
	if (ret < 0)
		goto out;
6636
	ret = set_block_group_ro(cache, 0);
6637 6638 6639 6640
out:
	btrfs_end_transaction(trans, root);
	return ret;
}
6641

6642 6643 6644 6645
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
6646 6647
	return do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			      CHUNK_ALLOC_FORCE);
6648 6649
}

6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705
/*
 * helper to account the unused space of all the readonly block group in the
 * list. takes mirrors into account.
 */
static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

	list_for_each_entry(block_group, groups_list, list) {
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}

	return free_bytes;
}

/*
 * helper to account the unused space of all the readonly block group in the
 * space_info. takes mirrors into account.
 */
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
{
	int i;
	u64 free_bytes = 0;

	spin_lock(&sinfo->lock);

	for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		if (!list_empty(&sinfo->block_groups[i]))
			free_bytes += __btrfs_get_ro_block_group_free_space(
						&sinfo->block_groups[i]);

	spin_unlock(&sinfo->lock);

	return free_bytes;
}

6706 6707
int btrfs_set_block_group_rw(struct btrfs_root *root,
			      struct btrfs_block_group_cache *cache)
6708
{
6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
6722 6723 6724
	return 0;
}

6725 6726 6727 6728 6729 6730 6731
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
6732
{
6733 6734 6735 6736
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
6737 6738 6739 6740
	u64 min_free;
	int index;
	int dev_nr = 0;
	int dev_min = 1;
6741 6742
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
6743

6744
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
6745

6746 6747 6748
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
Z
Zheng Yan 已提交
6749

6750 6751
	min_free = btrfs_block_group_used(&block_group->item);

6752
	/* no bytes used, we're good */
6753
	if (!min_free)
Z
Zheng Yan 已提交
6754 6755
		goto out;

6756 6757
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
6758

6759
	full = space_info->full;
6760

6761 6762
	/*
	 * if this is the last block group we have in this space, we can't
6763 6764 6765 6766
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
6767
	 */
6768
	if ((space_info->total_bytes != block_group->key.offset) &&
6769 6770 6771
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
6772 6773
		spin_unlock(&space_info->lock);
		goto out;
6774
	}
6775
	spin_unlock(&space_info->lock);
6776

6777 6778 6779 6780 6781 6782 6783 6784 6785 6786
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
	 * alloc devices and guess if we have enough space.  However, if we
	 * were marked as full, then we know there aren't enough chunks, and we
	 * can just return.
	 */
	ret = -1;
	if (full)
		goto out;
6787

6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
	index = get_block_group_index(block_group);
	if (index == 0) {
		dev_min = 4;
		min_free /= 2;
	} else if (index == 1) {
		dev_min = 2;
	} else if (index == 2) {
		min_free *= 2;
	} else if (index == 3) {
		dev_min = fs_devices->rw_devices;
		min_free /= dev_min;
	}

6809 6810
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
6811
		u64 dev_offset;
6812

6813 6814 6815 6816 6817 6818
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
		if (device->total_bytes > device->bytes_used + min_free) {
			ret = find_free_dev_extent(NULL, device, min_free,
6819
						   &dev_offset, NULL);
6820
			if (!ret)
6821 6822 6823
				dev_nr++;

			if (dev_nr >= dev_min)
6824
				break;
6825

6826
			ret = -1;
6827
		}
6828
	}
6829
	mutex_unlock(&root->fs_info->chunk_mutex);
6830
out:
6831
	btrfs_put_block_group(block_group);
6832 6833 6834
	return ret;
}

6835 6836
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
6837
{
6838
	int ret = 0;
6839 6840 6841
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
6842

6843 6844
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
6845 6846
		goto out;

C
Chris Mason 已提交
6847
	while (1) {
6848
		slot = path->slots[0];
6849
		leaf = path->nodes[0];
6850 6851 6852 6853 6854
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
6855
				goto out;
6856
			break;
6857
		}
6858
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
6859

6860
		if (found_key.objectid >= key->objectid &&
6861 6862 6863 6864
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
6865
		path->slots[0]++;
6866
	}
6867
out:
6868
	return ret;
6869 6870
}

6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
6905 6906 6907
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
6908
	struct btrfs_space_info *space_info;
6909
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
6910 6911
	struct rb_node *n;

6912 6913 6914 6915 6916 6917 6918 6919 6920
	down_write(&info->extent_commit_sem);
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
	up_write(&info->extent_commit_sem);

Z
Zheng Yan 已提交
6921 6922 6923 6924 6925 6926
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
Y
Yan Zheng 已提交
6927 6928
		spin_unlock(&info->block_group_cache_lock);

6929
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
6930
		list_del(&block_group->list);
6931
		up_write(&block_group->space_info->groups_sem);
6932

J
Josef Bacik 已提交
6933
		if (block_group->cached == BTRFS_CACHE_STARTED)
6934
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
6935

6936 6937 6938 6939 6940 6941 6942
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
		if (block_group->cached == BTRFS_CACHE_NO)
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
6943
		btrfs_remove_free_space_cache(block_group);
6944
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
6945 6946

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
6947 6948
	}
	spin_unlock(&info->block_group_cache_lock);
6949 6950 6951 6952 6953 6954 6955 6956 6957

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

6958 6959
	release_global_block_rsv(info);

6960 6961 6962 6963
	while(!list_empty(&info->space_info)) {
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
6964 6965 6966 6967 6968
		if (space_info->bytes_pinned > 0 ||
		    space_info->bytes_reserved > 0) {
			WARN_ON(1);
			dump_space_info(space_info, 0, 0);
		}
6969 6970 6971
		list_del(&space_info->list);
		kfree(space_info);
	}
Z
Zheng Yan 已提交
6972 6973 6974
	return 0;
}

6975 6976 6977 6978 6979 6980 6981 6982 6983 6984
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);

	down_write(&space_info->groups_sem);
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);
}

C
Chris Mason 已提交
6985 6986 6987 6988 6989
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
6990
	struct btrfs_fs_info *info = root->fs_info;
6991
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
6992 6993
	struct btrfs_key key;
	struct btrfs_key found_key;
6994
	struct extent_buffer *leaf;
6995 6996
	int need_clear = 0;
	u64 cache_gen;
6997

C
Chris Mason 已提交
6998
	root = info->extent_root;
C
Chris Mason 已提交
6999
	key.objectid = 0;
7000
	key.offset = 0;
C
Chris Mason 已提交
7001 7002 7003 7004
	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
J
Josef Bacik 已提交
7005
	path->reada = 1;
C
Chris Mason 已提交
7006

7007 7008 7009 7010
	cache_gen = btrfs_super_cache_generation(&root->fs_info->super_copy);
	if (cache_gen != 0 &&
	    btrfs_super_generation(&root->fs_info->super_copy) != cache_gen)
		need_clear = 1;
7011 7012
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
7013 7014
	if (!btrfs_test_opt(root, SPACE_CACHE) && cache_gen)
		printk(KERN_INFO "btrfs: disk space caching is enabled\n");
7015

C
Chris Mason 已提交
7016
	while (1) {
7017
		ret = find_first_block_group(root, path, &key);
7018 7019
		if (ret > 0)
			break;
7020 7021
		if (ret != 0)
			goto error;
7022 7023
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7024
		cache = kzalloc(sizeof(*cache), GFP_NOFS);
C
Chris Mason 已提交
7025
		if (!cache) {
7026
			ret = -ENOMEM;
7027
			goto error;
C
Chris Mason 已提交
7028
		}
7029 7030 7031 7032 7033 7034 7035
		cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
						GFP_NOFS);
		if (!cache->free_space_ctl) {
			kfree(cache);
			ret = -ENOMEM;
			goto error;
		}
C
Chris Mason 已提交
7036

7037
		atomic_set(&cache->count, 1);
7038
		spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7039
		cache->fs_info = info;
J
Josef Bacik 已提交
7040
		INIT_LIST_HEAD(&cache->list);
7041
		INIT_LIST_HEAD(&cache->cluster_list);
7042

7043 7044 7045
		if (need_clear)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

7046 7047 7048
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
C
Chris Mason 已提交
7049
		memcpy(&cache->key, &found_key, sizeof(found_key));
7050

C
Chris Mason 已提交
7051
		key.objectid = found_key.objectid + found_key.offset;
7052
		btrfs_release_path(path);
7053
		cache->flags = btrfs_block_group_flags(&cache->item);
J
Josef Bacik 已提交
7054 7055
		cache->sectorsize = root->sectorsize;

7056 7057
		btrfs_init_free_space_ctl(cache);

7058 7059 7060 7061 7062 7063 7064
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
		exclude_super_stripes(root, cache);

J
Josef Bacik 已提交
7065 7066 7067 7068 7069 7070 7071 7072
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
7073
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7074
			cache->cached = BTRFS_CACHE_FINISHED;
7075
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7076
		} else if (btrfs_block_group_used(&cache->item) == 0) {
7077
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7078 7079 7080 7081 7082
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
7083
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7084
		}
7085

7086 7087 7088 7089 7090
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
		BUG_ON(ret);
		cache->space_info = space_info;
7091
		spin_lock(&cache->space_info->lock);
7092
		cache->space_info->bytes_readonly += cache->bytes_super;
7093 7094
		spin_unlock(&cache->space_info->lock);

7095
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
7096 7097 7098

		ret = btrfs_add_block_group_cache(root->fs_info, cache);
		BUG_ON(ret);
7099 7100

		set_avail_alloc_bits(root->fs_info, cache->flags);
Y
Yan Zheng 已提交
7101
		if (btrfs_chunk_readonly(root, cache->key.objectid))
7102
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7103
	}
7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
		list_for_each_entry(cache, &space_info->block_groups[3], list)
7116
			set_block_group_ro(cache, 1);
7117
		list_for_each_entry(cache, &space_info->block_groups[4], list)
7118
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7119
	}
7120 7121

	init_global_block_rsv(info);
7122 7123
	ret = 0;
error:
C
Chris Mason 已提交
7124
	btrfs_free_path(path);
7125
	return ret;
C
Chris Mason 已提交
7126
}
7127 7128 7129

int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
7130
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
7131 7132 7133 7134 7135 7136 7137 7138
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

7139
	root->fs_info->last_trans_log_full_commit = trans->transid;
7140

7141
	cache = kzalloc(sizeof(*cache), GFP_NOFS);
J
Josef Bacik 已提交
7142 7143
	if (!cache)
		return -ENOMEM;
7144 7145 7146 7147 7148 7149
	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return -ENOMEM;
	}
J
Josef Bacik 已提交
7150

7151
	cache->key.objectid = chunk_offset;
7152
	cache->key.offset = size;
7153
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
7154
	cache->sectorsize = root->sectorsize;
7155
	cache->fs_info = root->fs_info;
7156

7157
	atomic_set(&cache->count, 1);
7158
	spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7159
	INIT_LIST_HEAD(&cache->list);
7160
	INIT_LIST_HEAD(&cache->cluster_list);
C
Chris Mason 已提交
7161

7162 7163
	btrfs_init_free_space_ctl(cache);

7164 7165 7166 7167 7168
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	cache->flags = type;
	btrfs_set_block_group_flags(&cache->item, type);

7169
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7170
	cache->cached = BTRFS_CACHE_FINISHED;
7171
	exclude_super_stripes(root, cache);
7172

J
Josef Bacik 已提交
7173 7174 7175
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

7176 7177
	free_excluded_extents(root, cache);

7178 7179 7180
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
	BUG_ON(ret);
7181 7182

	spin_lock(&cache->space_info->lock);
7183
	cache->space_info->bytes_readonly += cache->bytes_super;
7184 7185
	spin_unlock(&cache->space_info->lock);

7186
	__link_block_group(cache->space_info, cache);
7187

J
Josef Bacik 已提交
7188 7189
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
	BUG_ON(ret);
7190

7191 7192 7193 7194
	ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
				sizeof(cache->item));
	BUG_ON(ret);

C
Chris Mason 已提交
7195
	set_avail_alloc_bits(extent_root->fs_info, type);
7196

7197 7198
	return 0;
}
Z
Zheng Yan 已提交
7199 7200 7201 7202 7203 7204

int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 group_start)
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
7205
	struct btrfs_free_cluster *cluster;
7206
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
7207
	struct btrfs_key key;
7208
	struct inode *inode;
Z
Zheng Yan 已提交
7209
	int ret;
J
Josef Bacik 已提交
7210
	int factor;
Z
Zheng Yan 已提交
7211 7212 7213 7214 7215

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
7216
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
7217

7218 7219 7220 7221 7222 7223
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

Z
Zheng Yan 已提交
7224
	memcpy(&key, &block_group->key, sizeof(key));
J
Josef Bacik 已提交
7225 7226 7227 7228 7229 7230
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
7231

7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
7247
	path = btrfs_alloc_path();
7248 7249 7250 7251
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
Z
Zheng Yan 已提交
7252

7253 7254
	inode = lookup_free_space_inode(root, block_group, path);
	if (!IS_ERR(inode)) {
7255 7256
		ret = btrfs_orphan_add(trans, inode);
		BUG_ON(ret);
7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
		iput(inode);
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
7280
		btrfs_release_path(path);
7281 7282 7283 7284
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
7285
		btrfs_release_path(path);
7286 7287
	}

7288
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
7289 7290
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
7291
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
7292

7293
	down_write(&block_group->space_info->groups_sem);
7294 7295 7296 7297 7298
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
7299
	up_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
7300

J
Josef Bacik 已提交
7301
	if (block_group->cached == BTRFS_CACHE_STARTED)
7302
		wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
7303 7304 7305

	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
7306 7307 7308
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
7309
	block_group->space_info->disk_total -= block_group->key.offset * factor;
Y
Yan Zheng 已提交
7310
	spin_unlock(&block_group->space_info->lock);
7311

7312 7313
	memcpy(&key, &block_group->key, sizeof(key));

7314
	btrfs_clear_space_info_full(root->fs_info);
Y
Yan Zheng 已提交
7315

7316 7317
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
7330

7331 7332 7333
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
7334 7335 7336 7337
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
7338 7339
	int ret;

7340 7341 7342
	disk_super = &fs_info->super_copy;
	if (!btrfs_super_root(disk_super))
		return 1;
7343

7344 7345 7346
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
7347

7348 7349
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
	ret = update_space_info(fs_info, flags, 0, 0, &space_info);
7350
	if (ret)
7351
		goto out;
7352

7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	}
out:
7366 7367 7368
	return ret;
}

L
liubo 已提交
7369 7370 7371 7372 7373 7374
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
	return unpin_extent_range(root, start, end);
}

int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
7375
			       u64 num_bytes, u64 *actual_bytes)
L
liubo 已提交
7376
{
7377
	return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
L
liubo 已提交
7378
}
7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426

int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
	int ret = 0;

	cache = btrfs_lookup_block_group(fs_info, range->start);

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
				ret = cache_block_group(cache, NULL, root, 0);
				if (!ret)
					wait_block_group_cache_done(cache);
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

	range->len = trimmed;
	return ret;
}