extent-tree.c 195.5 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
C
Chris Mason 已提交
26
#include "compat.h"
27
#include "hash.h"
28 29 30
#include "ctree.h"
#include "disk-io.h"
#include "print-tree.h"
31
#include "transaction.h"
32
#include "volumes.h"
33
#include "locking.h"
34
#include "free-space-cache.h"
35

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
/* control flags for do_chunk_alloc's force field
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
	CHUNK_ALLOC_FORCE = 1,
	CHUNK_ALLOC_LIMITED = 2,
};

55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
/*
 * Control how reservations are dealt with.
 *
 * RESERVE_FREE - freeing a reservation.
 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
 *   ENOSPC accounting
 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
 *   bytes_may_use as the ENOSPC accounting is done elsewhere
 */
enum {
	RESERVE_FREE = 0,
	RESERVE_ALLOC = 1,
	RESERVE_ALLOC_NO_ACCOUNT = 2,
};

70 71
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
72
			      u64 bytenr, u64 num_bytes, int alloc);
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extra_op);
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins);
J
Josef Bacik 已提交
92 93 94
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
			  u64 flags, int force);
95 96
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
97 98
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
99 100
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve);
J
Josef Bacik 已提交
101

J
Josef Bacik 已提交
102 103 104 105 106 107 108
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
	return cache->cached == BTRFS_CACHE_FINISHED;
}

J
Josef Bacik 已提交
109 110 111 112 113
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

114
static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
115 116 117 118 119 120
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
121 122 123
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
124
		kfree(cache->free_space_ctl);
125
		kfree(cache);
126
	}
127 128
}

J
Josef Bacik 已提交
129 130 131 132
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
133
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
201
	if (ret)
202
		btrfs_get_block_group(ret);
J
Josef Bacik 已提交
203 204 205 206 207
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

208 209
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
210
{
211 212 213 214 215 216 217
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
J
Josef Bacik 已提交
218

219 220 221 222
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
223

224 225 226 227 228 229 230
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
J
Josef Bacik 已提交
231 232
}

233 234
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
235 236 237 238 239 240
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

241 242 243 244 245 246 247 248
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
		BUG_ON(ret);
	}

J
Josef Bacik 已提交
249 250 251 252 253 254
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
		BUG_ON(ret);
255

J
Josef Bacik 已提交
256
		while (nr--) {
257
			cache->bytes_super += stripe_len;
258 259 260
			ret = add_excluded_extent(root, logical[nr],
						  stripe_len);
			BUG_ON(ret);
J
Josef Bacik 已提交
261
		}
262

J
Josef Bacik 已提交
263 264 265 266 267
		kfree(logical);
	}
	return 0;
}

268 269 270 271 272 273 274 275 276 277 278
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_STARTED) {
		spin_unlock(&cache->lock);
		return NULL;
	}

279 280 281
	/* We're loading it the fast way, so we don't have a caching_ctl. */
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

J
Josef Bacik 已提交
297 298 299 300 301
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
J
Josef Bacik 已提交
302
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
J
Josef Bacik 已提交
303 304
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
J
Josef Bacik 已提交
305
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
306 307 308
	int ret;

	while (start < end) {
309
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
310
					    &extent_start, &extent_end,
311
					    EXTENT_DIRTY | EXTENT_UPTODATE);
J
Josef Bacik 已提交
312 313 314
		if (ret)
			break;

315
		if (extent_start <= start) {
J
Josef Bacik 已提交
316 317 318
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
319
			total_added += size;
320 321
			ret = btrfs_add_free_space(block_group, start,
						   size);
J
Josef Bacik 已提交
322 323 324 325 326 327 328 329 330
			BUG_ON(ret);
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
331
		total_added += size;
332
		ret = btrfs_add_free_space(block_group, start, size);
J
Josef Bacik 已提交
333 334 335
		BUG_ON(ret);
	}

J
Josef Bacik 已提交
336
	return total_added;
J
Josef Bacik 已提交
337 338
}

339
static noinline void caching_thread(struct btrfs_work *work)
340
{
341 342 343 344
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_root *extent_root;
345
	struct btrfs_path *path;
346
	struct extent_buffer *leaf;
347
	struct btrfs_key key;
J
Josef Bacik 已提交
348
	u64 total_found = 0;
349 350 351
	u64 last = 0;
	u32 nritems;
	int ret = 0;
352

353 354 355 356 357
	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

358 359
	path = btrfs_alloc_path();
	if (!path)
360
		goto out;
361

J
Josef Bacik 已提交
362
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
363

364
	/*
J
Josef Bacik 已提交
365 366 367 368
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
369 370
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
371
	path->search_commit_root = 1;
J
Josef Bacik 已提交
372
	path->reada = 1;
J
Josef Bacik 已提交
373

Y
Yan Zheng 已提交
374
	key.objectid = last;
375
	key.offset = 0;
376
	key.type = BTRFS_EXTENT_ITEM_KEY;
377
again:
378
	mutex_lock(&caching_ctl->mutex);
379 380 381
	/* need to make sure the commit_root doesn't disappear */
	down_read(&fs_info->extent_commit_sem);

382
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
383
	if (ret < 0)
384
		goto err;
Y
Yan Zheng 已提交
385

386 387 388
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
389
	while (1) {
390
		if (btrfs_fs_closing(fs_info) > 1) {
391
			last = (u64)-1;
J
Josef Bacik 已提交
392
			break;
393
		}
J
Josef Bacik 已提交
394

395 396 397 398 399
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
400
				break;
J
Josef Bacik 已提交
401

402 403 404
			if (need_resched() ||
			    btrfs_next_leaf(extent_root, path)) {
				caching_ctl->progress = last;
C
Chris Mason 已提交
405
				btrfs_release_path(path);
406 407
				up_read(&fs_info->extent_commit_sem);
				mutex_unlock(&caching_ctl->mutex);
408
				cond_resched();
409 410 411 412 413
				goto again;
			}
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
414
		}
J
Josef Bacik 已提交
415

416 417
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
418
			continue;
419
		}
J
Josef Bacik 已提交
420

421
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
422
		    block_group->key.offset)
423
			break;
424

425
		if (key.type == BTRFS_EXTENT_ITEM_KEY) {
J
Josef Bacik 已提交
426 427 428
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
429
			last = key.objectid + key.offset;
J
Josef Bacik 已提交
430

431 432 433 434
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
J
Josef Bacik 已提交
435
		}
436 437
		path->slots[0]++;
	}
J
Josef Bacik 已提交
438
	ret = 0;
439

J
Josef Bacik 已提交
440 441 442
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
443
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
444 445

	spin_lock(&block_group->lock);
446
	block_group->caching_ctl = NULL;
J
Josef Bacik 已提交
447 448
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
449

450
err:
451
	btrfs_free_path(path);
452
	up_read(&fs_info->extent_commit_sem);
J
Josef Bacik 已提交
453

454 455 456
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
457
out:
458 459 460
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
461
	btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
462 463
}

464 465
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     struct btrfs_trans_handle *trans,
466
			     struct btrfs_root *root,
467
			     int load_cache_only)
J
Josef Bacik 已提交
468
{
469 470
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
471 472
	int ret = 0;

473 474 475 476
	smp_mb();
	if (cache->cached != BTRFS_CACHE_NO)
		return 0;

477 478
	/*
	 * We can't do the read from on-disk cache during a commit since we need
479 480 481
	 * to have the normal tree locking.  Also if we are currently trying to
	 * allocate blocks for the tree root we can't do the fast caching since
	 * we likely hold important locks.
482
	 */
483
	if (trans && (!trans->transaction->in_commit) &&
484
	    (root && root != root->fs_info->tree_root)) {
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
		spin_lock(&cache->lock);
		if (cache->cached != BTRFS_CACHE_NO) {
			spin_unlock(&cache->lock);
			return 0;
		}
		cache->cached = BTRFS_CACHE_STARTED;
		spin_unlock(&cache->lock);

		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
		} else {
			cache->cached = BTRFS_CACHE_NO;
		}
		spin_unlock(&cache->lock);
503 504
		if (ret == 1) {
			free_excluded_extents(fs_info->extent_root, cache);
505
			return 0;
506
		}
507 508 509 510 511
	}

	if (load_cache_only)
		return 0;

512
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
513 514 515 516 517 518 519 520 521
	BUG_ON(!caching_ctl);

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	/* one for caching kthread, one for caching block group list */
	atomic_set(&caching_ctl->count, 2);
522
	caching_ctl->work.func = caching_thread;
523

J
Josef Bacik 已提交
524 525 526
	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
527 528
		kfree(caching_ctl);
		return 0;
J
Josef Bacik 已提交
529
	}
530
	cache->caching_ctl = caching_ctl;
J
Josef Bacik 已提交
531 532 533
	cache->cached = BTRFS_CACHE_STARTED;
	spin_unlock(&cache->lock);

534 535 536 537
	down_write(&fs_info->extent_commit_sem);
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
	up_write(&fs_info->extent_commit_sem);

538
	btrfs_get_block_group(cache);
539

540
	btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
J
Josef Bacik 已提交
541

542
	return ret;
543 544
}

J
Josef Bacik 已提交
545 546 547
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
548 549
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
550
{
J
Josef Bacik 已提交
551
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
552

J
Josef Bacik 已提交
553
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
554

J
Josef Bacik 已提交
555
	return cache;
C
Chris Mason 已提交
556 557
}

J
Josef Bacik 已提交
558
/*
559
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
560
 */
C
Chris Mason 已提交
561 562 563
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
564
{
J
Josef Bacik 已提交
565
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
566

J
Josef Bacik 已提交
567
	cache = block_group_cache_tree_search(info, bytenr, 1);
568

J
Josef Bacik 已提交
569
	return cache;
C
Chris Mason 已提交
570
}
571

J
Josef Bacik 已提交
572 573
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
574
{
J
Josef Bacik 已提交
575 576
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
577

578 579 580
	flags &= BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_SYSTEM |
		 BTRFS_BLOCK_GROUP_METADATA;

581 582
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
583
		if (found->flags & flags) {
584
			rcu_read_unlock();
J
Josef Bacik 已提交
585
			return found;
586
		}
J
Josef Bacik 已提交
587
	}
588
	rcu_read_unlock();
J
Josef Bacik 已提交
589
	return NULL;
590 591
}

592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

607 608 609 610 611 612 613 614 615
static u64 div_factor(u64 num, int factor)
{
	if (factor == 10)
		return num;
	num *= factor;
	do_div(num, 10);
	return num;
}

616 617 618 619 620 621 622 623 624
static u64 div_factor_fine(u64 num, int factor)
{
	if (factor == 100)
		return num;
	num *= factor;
	do_div(num, 100);
	return num;
}

625 626
u64 btrfs_find_block_group(struct btrfs_root *root,
			   u64 search_start, u64 search_hint, int owner)
C
Chris Mason 已提交
627
{
628
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
629
	u64 used;
630 631
	u64 last = max(search_hint, search_start);
	u64 group_start = 0;
632
	int full_search = 0;
633
	int factor = 9;
C
Chris Mason 已提交
634
	int wrapped = 0;
635
again:
636 637
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
J
Josef Bacik 已提交
638 639
		if (!cache)
			break;
640

641
		spin_lock(&cache->lock);
642 643 644
		last = cache->key.objectid + cache->key.offset;
		used = btrfs_block_group_used(&cache->item);

645 646
		if ((full_search || !cache->ro) &&
		    block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
647
			if (used + cache->pinned + cache->reserved <
648 649
			    div_factor(cache->key.offset, factor)) {
				group_start = cache->key.objectid;
650
				spin_unlock(&cache->lock);
651
				btrfs_put_block_group(cache);
652 653
				goto found;
			}
654
		}
655
		spin_unlock(&cache->lock);
656
		btrfs_put_block_group(cache);
657
		cond_resched();
C
Chris Mason 已提交
658
	}
C
Chris Mason 已提交
659 660 661 662 663 664
	if (!wrapped) {
		last = search_start;
		wrapped = 1;
		goto again;
	}
	if (!full_search && factor < 10) {
C
Chris Mason 已提交
665
		last = search_start;
666
		full_search = 1;
C
Chris Mason 已提交
667
		factor = 10;
668 669
		goto again;
	}
C
Chris Mason 已提交
670
found:
671
	return group_start;
672
}
J
Josef Bacik 已提交
673

674
/* simple helper to search for an existing extent at a given offset */
Z
Zheng Yan 已提交
675
int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
676 677 678
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
679
	struct btrfs_path *path;
680

Z
Zheng Yan 已提交
681
	path = btrfs_alloc_path();
682 683 684
	if (!path)
		return -ENOMEM;

685 686 687 688 689
	key.objectid = start;
	key.offset = len;
	btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
690
	btrfs_free_path(path);
691 692 693
	return ret;
}

694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
/*
 * helper function to lookup reference count and flags of extent.
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
			     u64 num_bytes, u64 *refs, u64 *flags)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = num_bytes;
	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
again:
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

774
			btrfs_release_path(path);
775

776 777 778 779
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
			goto again;
		}
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

805 806 807 808 809 810 811 812 813 814 815 816 817 818
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
864 865 866
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
867
 * - different files inside a single subvolume
868 869
 * - different offsets inside a file (bookend extents in file.c)
 *
870
 * The extent ref structure for the implicit back refs has fields for:
871 872 873
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
874 875
 * - original offset in the file
 * - how many bookend extents
876
 *
877 878
 * The key offset for the implicit back refs is hash of the first
 * three fields.
879
 *
880
 * The extent ref structure for the full back refs has field for:
881
 *
882
 * - number of pointers in the tree leaf
883
 *
884 885
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
886
 *
887 888
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
889
 *
890
 *     (root_key.objectid, inode objectid, offset in file, 1)
891
 *
892 893
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
894
 *
895
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
896
 *
897
 * Btree extents can be referenced by:
898
 *
899
 * - Different subvolumes
900
 *
901 902 903 904
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
905
 *
906 907 908
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
909
 */
Z
Zheng Yan 已提交
910

911 912 913 914 915
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
916
{
917 918 919 920 921
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
922
	struct btrfs_key key;
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
				BUG_ON(ret > 0);
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
958
	btrfs_release_path(path);
959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
	BUG_ON(ret);

	ret = btrfs_extend_item(trans, root, path, new_size);

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
1000
	high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1001
	lenum = cpu_to_le64(owner);
1002
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1003
	lenum = cpu_to_le64(offset);
1004
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1037
	struct extent_buffer *leaf;
1038
	u32 nritems;
1039
	int ret;
1040 1041
	int recow;
	int err = -ENOENT;
1042

Z
Zheng Yan 已提交
1043
	key.objectid = bytenr;
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1059

1060 1061 1062 1063 1064
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1065
		btrfs_release_path(path);
1066 1067 1068 1069 1070 1071 1072 1073 1074
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1075 1076 1077
	}

	leaf = path->nodes[0];
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1103
				btrfs_release_path(path);
1104 1105 1106 1107 1108 1109
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1110
	}
1111 1112
fail:
	return err;
Z
Zheng Yan 已提交
1113 1114
}

1115 1116 1117 1118 1119 1120
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1121 1122 1123
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1124
	u32 size;
Z
Zheng Yan 已提交
1125 1126
	u32 num_refs;
	int ret;
1127 1128

	key.objectid = bytenr;
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1139

1140 1141 1142 1143 1144 1145 1146
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1147
		ref = btrfs_item_ptr(leaf, path->slots[0],
1148 1149 1150 1151 1152 1153 1154
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1155
		}
1156 1157 1158 1159 1160 1161 1162 1163
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1164
			btrfs_release_path(path);
1165 1166 1167 1168 1169
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1170

1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1185 1186
		}
	}
1187 1188 1189
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1190
	btrfs_release_path(path);
1191
	return ret;
1192 1193
}

1194 1195 1196 1197
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   int refs_to_drop)
Z
Zheng Yan 已提交
1198
{
1199 1200 1201
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1202
	struct extent_buffer *leaf;
1203
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1204 1205 1206
	int ret = 0;

	leaf = path->nodes[0];
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1228 1229
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1230

Z
Zheng Yan 已提交
1231 1232 1233
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
	} else {
1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1246 1247 1248 1249 1250
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1251 1252 1253
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1254
{
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1286
#endif
1287 1288 1289 1290 1291
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1292

1293 1294 1295 1296 1297
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1298
{
1299
	struct btrfs_key key;
1300 1301
	int ret;

1302 1303 1304 1305 1306 1307 1308
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1309 1310
	}

1311 1312 1313 1314 1315
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1316
		btrfs_release_path(path);
1317 1318 1319 1320 1321
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1322
#endif
1323
	return ret;
1324 1325
}

1326 1327 1328 1329 1330
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1331
{
1332
	struct btrfs_key key;
Z
Zheng Yan 已提交
1333 1334
	int ret;

1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1345
	btrfs_release_path(path);
Z
Zheng Yan 已提交
1346 1347 1348
	return ret;
}

1349
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1350
{
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1364
}
1365

1366 1367
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1368

C
Chris Mason 已提交
1369
{
1370
	for (; level < BTRFS_MAX_LEVEL; level++) {
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1386

1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1422

1423
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1424
	key.type = BTRFS_EXTENT_ITEM_KEY;
1425
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1426

1427 1428 1429
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1430
		path->keep_locks = 1;
1431 1432 1433
	} else
		extra_size = -1;
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1434
	if (ret < 0) {
1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531
		err = ret;
		goto out;
	}
	BUG_ON(ret);

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	} else {
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1532 1533
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1534
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1535 1536 1537 1538 1539 1540
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1541
	if (insert) {
1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
int setup_inline_extent_backref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_path *path,
				struct btrfs_extent_inline_ref *iref,
				u64 parent, u64 root_objectid,
				u64 owner, u64 offset, int refs_to_add,
				struct btrfs_delayed_extent_op *extent_op)
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;
	int ret;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

	ret = btrfs_extend_item(trans, root, path, size);

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1628
		return ret;
1629

1630
	btrfs_release_path(path);
1631 1632 1633 1634 1635 1636 1637 1638
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1639
	}
1640 1641
	return ret;
}
Z
Zheng Yan 已提交
1642

1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
int update_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_mod,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	int ret;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1686
	}
Z
Zheng Yan 已提交
1687

1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
		ret = btrfs_truncate_item(trans, root, path, item_size, 1);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
		ret = update_inline_extent_backref(trans, root, path, iref,
						   refs_to_add, extent_op);
	} else if (ret == -ENOENT) {
		ret = setup_inline_extent_backref(trans, root, path, iref,
						  parent, root_objectid,
						  owner, offset, refs_to_add,
						  extent_op);
1735
	}
1736 1737
	return ret;
}
Z
Zheng Yan 已提交
1738

1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1757

1758 1759 1760 1761 1762 1763 1764
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_drop, int is_data)
{
	int ret;
1765

1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
		ret = update_inline_extent_backref(trans, root, path, iref,
						   -refs_to_drop, NULL);
	} else if (is_data) {
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
	} else {
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1778
static int btrfs_issue_discard(struct block_device *bdev,
1779 1780
				u64 start, u64 len)
{
1781
	return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1782 1783 1784
}

static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1785
				u64 num_bytes, u64 *actual_bytes)
1786 1787
{
	int ret;
1788
	u64 discarded_bytes = 0;
1789 1790
	struct btrfs_multi_bio *multi = NULL;

C
Christoph Hellwig 已提交
1791

1792
	/* Tell the block device(s) that the sectors can be discarded */
1793 1794
	ret = btrfs_map_block(&root->fs_info->mapping_tree, REQ_DISCARD,
			      bytenr, &num_bytes, &multi, 0);
1795 1796 1797 1798 1799 1800
	if (!ret) {
		struct btrfs_bio_stripe *stripe = multi->stripes;
		int i;


		for (i = 0; i < multi->num_stripes; i++, stripe++) {
1801 1802 1803
			if (!stripe->dev->can_discard)
				continue;

1804 1805 1806 1807 1808 1809 1810
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
						  stripe->length);
			if (!ret)
				discarded_bytes += stripe->length;
			else if (ret != -EOPNOTSUPP)
				break;
1811 1812 1813 1814 1815 1816 1817

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
1818 1819 1820
		}
		kfree(multi);
	}
1821 1822 1823 1824

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888

	return ret;
}

int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
			 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
					parent, root_objectid, (int)owner,
					BTRFS_ADD_DELAYED_REF, NULL);
	} else {
		ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
					parent, root_objectid, owner, offset,
					BTRFS_ADD_DELAYED_REF, NULL);
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
	u64 refs;
	int ret;
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
	ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
					   path, bytenr, num_bytes, parent,
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
	if (ret == 0)
		goto out;

	if (ret != -EAGAIN) {
		err = ret;
		goto out;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
1889

1890
	btrfs_mark_buffer_dirty(leaf);
1891
	btrfs_release_path(path);
1892 1893

	path->reada = 1;
1894 1895
	path->leave_spinning = 1;

1896 1897
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
1898 1899
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
1900
	BUG_ON(ret);
1901
out:
1902
	btrfs_free_path(path);
1903
	return err;
1904 1905
}

1906 1907 1908 1909 1910
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
1911
{
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		if (extent_op) {
			BUG_ON(extent_op->update_key);
			flags |= extent_op->flags_to_set;
		}
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
					     extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
					  extent_op);
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
1984
	int ret;
1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = node->num_bytes;

	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
		err = -EIO;
		goto out;
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2025

2026 2027 2028 2029
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2030 2031
}

2032 2033 2034 2035 2036
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2037 2038
{
	int ret = 0;
2039 2040 2041 2042
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2043

2044 2045 2046
	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;
2047

2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073
	ref = btrfs_delayed_node_to_tree_ref(node);
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		BUG_ON(!extent_op || !extent_op->update_flags ||
		       !extent_op->update_key);
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
						ref->level, &ins);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
					     ref->level, 0, 1, extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
					  ref->level, 0, 1, extent_op);
	} else {
		BUG();
	}
2074 2075 2076 2077
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2078 2079 2080 2081 2082
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2083 2084
{
	int ret;
2085
	if (btrfs_delayed_ref_is_head(node)) {
2086 2087 2088 2089 2090 2091 2092
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2093 2094
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2095
		if (insert_reserved) {
2096 2097
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2098 2099 2100 2101 2102 2103
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
				BUG_ON(ret);
			}
2104 2105 2106 2107 2108
		}
		mutex_unlock(&head->mutex);
		return 0;
	}

2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_node *ref;
	int action = BTRFS_ADD_DELAYED_REF;
again:
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
	node = rb_prev(&head->node.rb_node);
	while (1) {
		if (!node)
			break;
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
		if (ref->bytenr != head->node.bytenr)
			break;
2142
		if (ref->action == action)
2143 2144 2145 2146 2147 2148 2149 2150 2151 2152
			return ref;
		node = rb_prev(node);
	}
	if (action == BTRFS_ADD_DELAYED_REF) {
		action = BTRFS_DROP_DELAYED_REF;
		goto again;
	}
	return NULL;
}

2153 2154 2155
static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root,
				       struct list_head *cluster)
2156 2157 2158 2159
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2160
	struct btrfs_delayed_extent_op *extent_op;
2161
	int ret;
2162
	int count = 0;
2163 2164 2165 2166 2167
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2168 2169
			/* pick a new head ref from the cluster list */
			if (list_empty(cluster))
2170 2171
				break;

2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188
			locked_ref = list_entry(cluster->next,
				     struct btrfs_delayed_ref_head, cluster);

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);

			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2189 2190
			}
		}
2191

2192 2193 2194 2195 2196 2197
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2198

2199 2200 2201
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);
		if (!ref) {
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231

			if (extent_op && must_insert_reserved) {
				kfree(extent_op);
				extent_op = NULL;
			}

			if (extent_op) {
				spin_unlock(&delayed_refs->lock);

				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
				BUG_ON(ret);
				kfree(extent_op);

				cond_resched();
				spin_lock(&delayed_refs->lock);
				continue;
			}

2232
			list_del_init(&locked_ref->cluster);
2233 2234
			locked_ref = NULL;
		}
C
Chris Mason 已提交
2235

2236 2237 2238
		ref->in_tree = 0;
		rb_erase(&ref->rb_node, &delayed_refs->root);
		delayed_refs->num_entries--;
2239

2240
		spin_unlock(&delayed_refs->lock);
2241

2242
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2243 2244
					  must_insert_reserved);
		BUG_ON(ret);
2245

2246 2247
		btrfs_put_delayed_ref(ref);
		kfree(extent_op);
2248
		count++;
2249

2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288
		cond_resched();
		spin_lock(&delayed_refs->lock);
	}
	return count;
}

/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct list_head cluster;
	int ret;
	int run_all = count == (unsigned long)-1;
	int run_most = 0;

	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

	delayed_refs = &trans->transaction->delayed_refs;
	INIT_LIST_HEAD(&cluster);
again:
	spin_lock(&delayed_refs->lock);
	if (count == 0) {
		count = delayed_refs->num_entries * 2;
		run_most = 1;
	}
	while (1) {
		if (!(run_all || run_most) &&
		    delayed_refs->num_heads_ready < 64)
			break;
2289

2290
		/*
2291 2292 2293 2294
		 * go find something we can process in the rbtree.  We start at
		 * the beginning of the tree, and then build a cluster
		 * of refs to process starting at the first one we are able to
		 * lock
2295
		 */
2296 2297 2298
		ret = btrfs_find_ref_cluster(trans, &cluster,
					     delayed_refs->run_delayed_start);
		if (ret)
2299 2300
			break;

2301 2302 2303 2304 2305 2306 2307
		ret = run_clustered_refs(trans, root, &cluster);
		BUG_ON(ret < 0);

		count -= min_t(unsigned long, ret, count);

		if (count == 0)
			break;
2308
	}
2309

2310 2311
	if (run_all) {
		node = rb_first(&delayed_refs->root);
2312
		if (!node)
2313
			goto out;
2314
		count = (unsigned long)-1;
2315

2316 2317 2318 2319 2320
		while (node) {
			ref = rb_entry(node, struct btrfs_delayed_ref_node,
				       rb_node);
			if (btrfs_delayed_ref_is_head(ref)) {
				struct btrfs_delayed_ref_head *head;
2321

2322 2323 2324 2325
				head = btrfs_delayed_node_to_head(ref);
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2326 2327 2328 2329
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
2330 2331 2332 2333
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2334
				cond_resched();
2335 2336 2337 2338 2339 2340 2341
				goto again;
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
		schedule_timeout(1);
		goto again;
2342
	}
2343
out:
2344
	spin_unlock(&delayed_refs->lock);
2345 2346 2347
	return 0;
}

2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
				int is_data)
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

	extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;

	ret = btrfs_add_delayed_extent_op(trans, bytenr, num_bytes, extent_op);
	if (ret)
		kfree(extent_op);
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	ret = -ENOENT;
	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

2394
		btrfs_release_path(path);
2395

2396 2397 2398 2399
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out_unlock;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	if (ref->bytenr != bytenr)
		goto out_unlock;

	ret = 1;
	if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
		goto out_unlock;

	data_ref = btrfs_delayed_node_to_data_ref(ref);

	node = rb_prev(node);
	if (node) {
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
		if (ref->bytenr == bytenr)
			goto out_unlock;
	}

	if (data_ref->root != root->root_key.objectid ||
	    data_ref->objectid != objectid || data_ref->offset != offset)
		goto out_unlock;

	ret = 0;
out_unlock:
	mutex_unlock(&head->mutex);
out:
	spin_unlock(&delayed_refs->lock);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2444 2445
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2446
	struct extent_buffer *leaf;
2447 2448 2449
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2450
	struct btrfs_key key;
2451
	u32 item_size;
2452
	int ret;
2453

2454
	key.objectid = bytenr;
Z
Zheng Yan 已提交
2455
	key.offset = (u64)-1;
2456
	key.type = BTRFS_EXTENT_ITEM_KEY;
2457 2458 2459 2460 2461

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
	BUG_ON(ret == 0);
Y
Yan Zheng 已提交
2462 2463 2464

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
2465
		goto out;
2466

Z
Zheng Yan 已提交
2467
	path->slots[0]--;
2468
	leaf = path->nodes[0];
2469
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2470

2471
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2472
		goto out;
2473

2474 2475 2476 2477 2478 2479 2480 2481 2482
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2483

2484 2485 2486
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2487

2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
2527
			goto out;
Y
Yan Zheng 已提交
2528

2529 2530 2531 2532 2533 2534 2535
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
2536
	}
2537 2538 2539

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
2540
out:
Y
Yan Zheng 已提交
2541
	btrfs_free_path(path);
2542 2543
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
2544
	return ret;
2545
}
C
Chris Mason 已提交
2546

2547
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2548
			   struct btrfs_root *root,
2549 2550
			   struct extent_buffer *buf,
			   int full_backref, int inc)
Z
Zheng Yan 已提交
2551 2552
{
	u64 bytenr;
2553 2554
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
2555 2556 2557 2558 2559 2560 2561 2562
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
2563
			    u64, u64, u64, u64, u64, u64);
Z
Zheng Yan 已提交
2564 2565 2566 2567 2568

	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

2569 2570
	if (!root->ref_cows && level == 0)
		return 0;
Z
Zheng Yan 已提交
2571

2572 2573 2574 2575
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
2576

2577 2578 2579 2580 2581 2582
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
2583
		if (level == 0) {
2584
			btrfs_item_key_to_cpu(buf, &key, i);
Z
Zheng Yan 已提交
2585 2586
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
2587
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
2588 2589 2590 2591 2592 2593 2594
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
2595 2596 2597 2598 2599 2600

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
					   key.offset);
Z
Zheng Yan 已提交
2601 2602 2603
			if (ret)
				goto fail;
		} else {
2604 2605 2606 2607
			bytenr = btrfs_node_blockptr(buf, i);
			num_bytes = btrfs_level_size(root, level - 1);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, level - 1, 0);
Z
Zheng Yan 已提交
2608 2609 2610 2611 2612 2613
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627
	BUG();
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		  struct extent_buffer *buf, int full_backref)
{
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		  struct extent_buffer *buf, int full_backref)
{
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
Z
Zheng Yan 已提交
2628 2629
}

C
Chris Mason 已提交
2630 2631 2632 2633 2634 2635 2636
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2637 2638
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
2639 2640

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2641 2642
	if (ret < 0)
		goto fail;
C
Chris Mason 已提交
2643
	BUG_ON(ret);
2644 2645 2646 2647 2648

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
2649
	btrfs_release_path(path);
2650
fail:
C
Chris Mason 已提交
2651 2652 2653 2654 2655 2656
	if (ret)
		return ret;
	return 0;

}

2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
	spin_lock(&root->fs_info->block_group_cache_lock);
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
2668
		btrfs_get_block_group(cache);
2669 2670 2671 2672 2673 2674
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

2675 2676 2677 2678 2679 2680 2681
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
2682
	int dcs = BTRFS_DC_ERROR;
2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701
	int num_pages = 0;
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
2702
		btrfs_release_path(path);
2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
		ret = btrfs_truncate_free_space_cache(root, trans, path,
						      inode);
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
	if (block_group->cached != BTRFS_CACHE_FINISHED) {
2737 2738
		/* We're not cached, don't bother trying to write stuff out */
		dcs = BTRFS_DC_WRITTEN;
2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

	num_pages = (int)div64_u64(block_group->key.offset, 1024 * 1024 * 1024);
	if (!num_pages)
		num_pages = 1;

	/*
	 * Just to make absolutely sure we have enough space, we're going to
	 * preallocate 12 pages worth of space for each block group.  In
	 * practice we ought to use at most 8, but we need extra space so we can
	 * add our header and have a terminator between the extents and the
	 * bitmaps.
	 */
	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
2765 2766
	if (!ret)
		dcs = BTRFS_DC_SETUP;
2767 2768 2769 2770
	btrfs_free_reserved_data_space(inode, num_pages);
out_put:
	iput(inode);
out_free:
2771
	btrfs_release_path(path);
2772 2773
out:
	spin_lock(&block_group->lock);
2774
	block_group->disk_cache_state = dcs;
2775 2776 2777 2778 2779
	spin_unlock(&block_group->lock);

	return ret;
}

2780 2781
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
C
Chris Mason 已提交
2782
{
2783
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
2784 2785
	int err = 0;
	struct btrfs_path *path;
2786
	u64 last = 0;
C
Chris Mason 已提交
2787 2788 2789 2790 2791

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810
again:
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			if (cache->disk_cache_state == BTRFS_DC_CLEAR)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
		err = cache_save_setup(cache, trans, path);
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
2811
	while (1) {
2812 2813 2814 2815
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
			BUG_ON(err);
J
Josef Bacik 已提交
2816
		}
2817

2818 2819
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
2820 2821 2822 2823 2824
			if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
				btrfs_put_block_group(cache);
				goto again;
			}

2825 2826 2827 2828 2829 2830 2831 2832 2833 2834
			if (cache->dirty)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
J
Josef Bacik 已提交
2835

J
Josef Bacik 已提交
2836 2837
		if (cache->disk_cache_state == BTRFS_DC_SETUP)
			cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
2838
		cache->dirty = 0;
2839
		last = cache->key.objectid + cache->key.offset;
J
Josef Bacik 已提交
2840

2841 2842 2843
		err = write_one_cache_group(trans, root, path, cache);
		BUG_ON(err);
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
2844
	}
2845

J
Josef Bacik 已提交
2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891
	while (1) {
		/*
		 * I don't think this is needed since we're just marking our
		 * preallocated extent as written, but just in case it can't
		 * hurt.
		 */
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
			BUG_ON(err);
		}

		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			/*
			 * Really this shouldn't happen, but it could if we
			 * couldn't write the entire preallocated extent and
			 * splitting the extent resulted in a new block.
			 */
			if (cache->dirty) {
				btrfs_put_block_group(cache);
				goto again;
			}
			if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		btrfs_write_out_cache(root, trans, cache, path);

		/*
		 * If we didn't have an error then the cache state is still
		 * NEED_WRITE, so we can set it to WRITTEN.
		 */
		if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
			cache->disk_cache_state = BTRFS_DC_WRITTEN;
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
2892
	btrfs_free_path(path);
2893
	return 0;
C
Chris Mason 已提交
2894 2895
}

2896 2897 2898 2899 2900 2901 2902 2903 2904
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
2905
		btrfs_put_block_group(block_group);
2906 2907 2908
	return readonly;
}

2909 2910 2911 2912 2913
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
2914 2915 2916 2917 2918 2919 2920 2921
	int i;
	int factor;

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
2922 2923 2924

	found = __find_space_info(info, flags);
	if (found) {
2925
		spin_lock(&found->lock);
2926
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
2927
		found->disk_total += total_bytes * factor;
2928
		found->bytes_used += bytes_used;
2929
		found->disk_used += bytes_used * factor;
2930
		found->full = 0;
2931
		spin_unlock(&found->lock);
2932 2933 2934
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
2935
	found = kzalloc(sizeof(*found), GFP_NOFS);
2936 2937 2938
	if (!found)
		return -ENOMEM;

2939 2940
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		INIT_LIST_HEAD(&found->block_groups[i]);
2941
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
2942
	spin_lock_init(&found->lock);
2943 2944 2945
	found->flags = flags & (BTRFS_BLOCK_GROUP_DATA |
				BTRFS_BLOCK_GROUP_SYSTEM |
				BTRFS_BLOCK_GROUP_METADATA);
2946
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
2947
	found->disk_total = total_bytes * factor;
2948
	found->bytes_used = bytes_used;
2949
	found->disk_used = bytes_used * factor;
2950
	found->bytes_pinned = 0;
2951
	found->bytes_reserved = 0;
Y
Yan Zheng 已提交
2952
	found->bytes_readonly = 0;
2953
	found->bytes_may_use = 0;
2954
	found->full = 0;
2955
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
2956
	found->chunk_alloc = 0;
2957 2958
	found->flush = 0;
	init_waitqueue_head(&found->wait);
2959
	*space_info = found;
2960
	list_add_rcu(&found->list, &info->space_info);
2961 2962 2963
	return 0;
}

2964 2965 2966
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
	u64 extra_flags = flags & (BTRFS_BLOCK_GROUP_RAID0 |
2967
				   BTRFS_BLOCK_GROUP_RAID1 |
C
Chris Mason 已提交
2968
				   BTRFS_BLOCK_GROUP_RAID10 |
2969
				   BTRFS_BLOCK_GROUP_DUP);
2970 2971 2972 2973 2974 2975 2976 2977 2978
	if (extra_flags) {
		if (flags & BTRFS_BLOCK_GROUP_DATA)
			fs_info->avail_data_alloc_bits |= extra_flags;
		if (flags & BTRFS_BLOCK_GROUP_METADATA)
			fs_info->avail_metadata_alloc_bits |= extra_flags;
		if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
			fs_info->avail_system_alloc_bits |= extra_flags;
	}
}
2979

Y
Yan Zheng 已提交
2980
u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
2981
{
2982 2983 2984 2985 2986 2987 2988
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	u64 num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;
2989 2990 2991 2992 2993 2994

	if (num_devices == 1)
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

2995 2996
	if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
	    (flags & (BTRFS_BLOCK_GROUP_RAID1 |
2997
		      BTRFS_BLOCK_GROUP_RAID10))) {
2998
		flags &= ~BTRFS_BLOCK_GROUP_DUP;
2999
	}
3000 3001

	if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
3002
	    (flags & BTRFS_BLOCK_GROUP_RAID10)) {
3003
		flags &= ~BTRFS_BLOCK_GROUP_RAID1;
3004
	}
3005 3006 3007 3008 3009 3010 3011 3012 3013

	if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
	    ((flags & BTRFS_BLOCK_GROUP_RAID1) |
	     (flags & BTRFS_BLOCK_GROUP_RAID10) |
	     (flags & BTRFS_BLOCK_GROUP_DUP)))
		flags &= ~BTRFS_BLOCK_GROUP_RAID0;
	return flags;
}

3014
static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
J
Josef Bacik 已提交
3015
{
3016 3017 3018 3019 3020 3021 3022 3023 3024 3025
	if (flags & BTRFS_BLOCK_GROUP_DATA)
		flags |= root->fs_info->avail_data_alloc_bits &
			 root->fs_info->data_alloc_profile;
	else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		flags |= root->fs_info->avail_system_alloc_bits &
			 root->fs_info->system_alloc_profile;
	else if (flags & BTRFS_BLOCK_GROUP_METADATA)
		flags |= root->fs_info->avail_metadata_alloc_bits &
			 root->fs_info->metadata_alloc_profile;
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
3026 3027
}

3028
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
3029
{
3030
	u64 flags;
J
Josef Bacik 已提交
3031

3032 3033 3034 3035
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
3036
	else
3037
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
3038

3039
	return get_alloc_profile(root, flags);
J
Josef Bacik 已提交
3040
}
J
Josef Bacik 已提交
3041

J
Josef Bacik 已提交
3042 3043 3044
void btrfs_set_inode_space_info(struct btrfs_root *root, struct inode *inode)
{
	BTRFS_I(inode)->space_info = __find_space_info(root->fs_info,
3045
						       BTRFS_BLOCK_GROUP_DATA);
J
Josef Bacik 已提交
3046 3047
}

J
Josef Bacik 已提交
3048 3049 3050 3051
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3052
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
3053 3054
{
	struct btrfs_space_info *data_sinfo;
3055
	struct btrfs_root *root = BTRFS_I(inode)->root;
3056
	u64 used;
3057
	int ret = 0, committed = 0, alloc_chunk = 1;
J
Josef Bacik 已提交
3058 3059 3060 3061

	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);

3062 3063
	if (root == root->fs_info->tree_root ||
	    BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3064 3065 3066 3067
		alloc_chunk = 0;
		committed = 1;
	}

J
Josef Bacik 已提交
3068
	data_sinfo = BTRFS_I(inode)->space_info;
C
Chris Mason 已提交
3069 3070
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
3071

J
Josef Bacik 已提交
3072 3073 3074
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3075 3076 3077
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3078 3079

	if (used + bytes > data_sinfo->total_bytes) {
3080
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3081

J
Josef Bacik 已提交
3082 3083 3084 3085
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3086
		if (!data_sinfo->full && alloc_chunk) {
J
Josef Bacik 已提交
3087
			u64 alloc_target;
J
Josef Bacik 已提交
3088

3089
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
J
Josef Bacik 已提交
3090
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
3091
alloc:
J
Josef Bacik 已提交
3092
			alloc_target = btrfs_get_alloc_profile(root, 1);
3093
			trans = btrfs_join_transaction(root);
3094 3095
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
3096

J
Josef Bacik 已提交
3097 3098
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
					     bytes + 2 * 1024 * 1024,
3099 3100
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
J
Josef Bacik 已提交
3101
			btrfs_end_transaction(trans, root);
3102 3103 3104 3105 3106 3107
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
J
Josef Bacik 已提交
3108

C
Chris Mason 已提交
3109 3110 3111 3112
			if (!data_sinfo) {
				btrfs_set_inode_space_info(root, inode);
				data_sinfo = BTRFS_I(inode)->space_info;
			}
J
Josef Bacik 已提交
3113 3114
			goto again;
		}
3115 3116 3117 3118 3119 3120 3121

		/*
		 * If we have less pinned bytes than we want to allocate then
		 * don't bother committing the transaction, it won't help us.
		 */
		if (data_sinfo->bytes_pinned < bytes)
			committed = 1;
J
Josef Bacik 已提交
3122 3123
		spin_unlock(&data_sinfo->lock);

3124
		/* commit the current transaction and try again */
3125
commit_trans:
J
Josef Bacik 已提交
3126 3127
		if (!committed &&
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
3128
			committed = 1;
3129
			trans = btrfs_join_transaction(root);
3130 3131
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3132 3133 3134 3135 3136
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
J
Josef Bacik 已提交
3137

J
Josef Bacik 已提交
3138 3139 3140 3141 3142
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
	spin_unlock(&data_sinfo->lock);

J
Josef Bacik 已提交
3143 3144
	return 0;
}
J
Josef Bacik 已提交
3145 3146

/*
3147
 * Called if we need to clear a data reservation for this inode.
J
Josef Bacik 已提交
3148
 */
3149
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3150
{
3151
	struct btrfs_root *root = BTRFS_I(inode)->root;
J
Josef Bacik 已提交
3152
	struct btrfs_space_info *data_sinfo;
3153

J
Josef Bacik 已提交
3154 3155
	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3156

J
Josef Bacik 已提交
3157 3158 3159 3160
	data_sinfo = BTRFS_I(inode)->space_info;
	spin_lock(&data_sinfo->lock);
	data_sinfo->bytes_may_use -= bytes;
	spin_unlock(&data_sinfo->lock);
3161 3162
}

3163
static void force_metadata_allocation(struct btrfs_fs_info *info)
3164
{
3165 3166
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3167

3168 3169 3170
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3171
			found->force_alloc = CHUNK_ALLOC_FORCE;
3172
	}
3173
	rcu_read_unlock();
3174 3175
}

3176
static int should_alloc_chunk(struct btrfs_root *root,
3177 3178
			      struct btrfs_space_info *sinfo, u64 alloc_bytes,
			      int force)
3179
{
3180
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3181
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3182
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3183
	u64 thresh;
3184

3185 3186 3187
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

3188 3189 3190 3191 3192 3193 3194
	/*
	 * We need to take into account the global rsv because for all intents
	 * and purposes it's used space.  Don't worry about locking the
	 * global_rsv, it doesn't change except when the transaction commits.
	 */
	num_allocated += global_rsv->size;

3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216
	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
		thresh = btrfs_super_total_bytes(&root->fs_info->super_copy);
		thresh = max_t(u64, 64 * 1024 * 1024,
			       div_factor_fine(thresh, 1));

		if (num_bytes - num_allocated < thresh)
			return 1;
	}

	/*
	 * we have two similar checks here, one based on percentage
	 * and once based on a hard number of 256MB.  The idea
	 * is that if we have a good amount of free
	 * room, don't allocate a chunk.  A good mount is
	 * less than 80% utilized of the chunks we have allocated,
	 * or more than 256MB free
	 */
	if (num_allocated + alloc_bytes + 256 * 1024 * 1024 < num_bytes)
3217
		return 0;
3218

3219
	if (num_allocated + alloc_bytes < div_factor(num_bytes, 8))
3220
		return 0;
3221

3222
	thresh = btrfs_super_total_bytes(&root->fs_info->super_copy);
3223 3224

	/* 256MB or 5% of the FS */
3225 3226 3227
	thresh = max_t(u64, 256 * 1024 * 1024, div_factor_fine(thresh, 5));

	if (num_bytes > thresh && sinfo->bytes_used < div_factor(num_bytes, 3))
3228
		return 0;
3229
	return 1;
3230 3231
}

3232 3233
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
C
Chris Mason 已提交
3234
			  u64 flags, int force)
J
Josef Bacik 已提交
3235
{
3236
	struct btrfs_space_info *space_info;
3237
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
3238
	int wait_for_alloc = 0;
J
Josef Bacik 已提交
3239 3240
	int ret = 0;

Y
Yan Zheng 已提交
3241
	flags = btrfs_reduce_alloc_profile(extent_root, flags);
3242

3243
	space_info = __find_space_info(extent_root->fs_info, flags);
3244 3245 3246 3247
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
		BUG_ON(ret);
J
Josef Bacik 已提交
3248
	}
3249
	BUG_ON(!space_info);
J
Josef Bacik 已提交
3250

3251
again:
3252
	spin_lock(&space_info->lock);
J
Josef Bacik 已提交
3253
	if (space_info->force_alloc)
3254
		force = space_info->force_alloc;
3255 3256
	if (space_info->full) {
		spin_unlock(&space_info->lock);
3257
		return 0;
J
Josef Bacik 已提交
3258 3259
	}

3260
	if (!should_alloc_chunk(extent_root, space_info, alloc_bytes, force)) {
3261
		spin_unlock(&space_info->lock);
3262 3263 3264 3265 3266
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
J
Josef Bacik 已提交
3267
	}
3268

3269
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3270

3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

3285 3286 3287 3288 3289 3290 3291
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3292 3293 3294 3295 3296
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
3297
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3298 3299 3300 3301
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
3302 3303
	}

Y
Yan Zheng 已提交
3304
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
3305 3306 3307
	if (ret < 0 && ret != -ENOSPC)
		goto out;

J
Josef Bacik 已提交
3308 3309
	spin_lock(&space_info->lock);
	if (ret)
3310
		space_info->full = 1;
3311 3312
	else
		ret = 1;
3313

3314
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3315
	space_info->chunk_alloc = 0;
J
Josef Bacik 已提交
3316
	spin_unlock(&space_info->lock);
3317
out:
Y
Yan Zheng 已提交
3318
	mutex_unlock(&extent_root->fs_info->chunk_mutex);
J
Josef Bacik 已提交
3319
	return ret;
3320
}
J
Josef Bacik 已提交
3321 3322

/*
3323
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
3324
 */
3325
static int shrink_delalloc(struct btrfs_trans_handle *trans,
J
Josef Bacik 已提交
3326
			   struct btrfs_root *root, u64 to_reclaim, int sync)
3327
{
3328
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
3329
	struct btrfs_space_info *space_info;
3330 3331 3332
	u64 reserved;
	u64 max_reclaim;
	u64 reclaimed = 0;
3333
	long time_left;
3334
	int nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3335
	int loops = 0;
3336
	unsigned long progress;
3337

3338
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
3339
	space_info = block_rsv->space_info;
3340 3341

	smp_mb();
3342
	reserved = space_info->bytes_may_use;
3343
	progress = space_info->reservation_progress;
3344 3345 3346

	if (reserved == 0)
		return 0;
3347

3348 3349 3350 3351 3352 3353 3354 3355
	smp_mb();
	if (root->fs_info->delalloc_bytes == 0) {
		if (trans)
			return 0;
		btrfs_wait_ordered_extents(root, 0, 0);
		return 0;
	}

3356 3357
	max_reclaim = min(reserved, to_reclaim);

3358
	while (loops < 1024) {
3359 3360 3361 3362 3363
		/* have the flusher threads jump in and do some IO */
		smp_mb();
		nr_pages = min_t(unsigned long, nr_pages,
		       root->fs_info->delalloc_bytes >> PAGE_CACHE_SHIFT);
		writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages);
3364

J
Josef Bacik 已提交
3365
		spin_lock(&space_info->lock);
3366 3367 3368
		if (reserved > space_info->bytes_may_use)
			reclaimed += reserved - space_info->bytes_may_use;
		reserved = space_info->bytes_may_use;
J
Josef Bacik 已提交
3369
		spin_unlock(&space_info->lock);
3370

3371 3372
		loops++;

3373 3374 3375 3376 3377
		if (reserved == 0 || reclaimed >= max_reclaim)
			break;

		if (trans && trans->transaction->blocked)
			return -EAGAIN;
3378

3379
		time_left = schedule_timeout_interruptible(1);
3380 3381 3382 3383 3384

		/* We were interrupted, exit */
		if (time_left)
			break;

3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395
		/* we've kicked the IO a few times, if anything has been freed,
		 * exit.  There is no sense in looping here for a long time
		 * when we really need to commit the transaction, or there are
		 * just too many writers without enough free space
		 */

		if (loops > 3) {
			smp_mb();
			if (progress != space_info->reservation_progress)
				break;
		}
3396

3397
	}
3398 3399
	if (reclaimed >= to_reclaim && !trans)
		btrfs_wait_ordered_extents(root, 0, 0);
3400 3401 3402
	return reclaimed >= to_reclaim;
}

3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417
/*
 * Retries tells us how many times we've called reserve_metadata_bytes.  The
 * idea is if this is the first call (retries == 0) then we will add to our
 * reserved count if we can't make the allocation in order to hold our place
 * while we go and try and free up space.  That way for retries > 1 we don't try
 * and add space, we just check to see if the amount of unused space is >= the
 * total space, meaning that our reservation is valid.
 *
 * However if we don't intend to retry this reservation, pass -1 as retries so
 * that it short circuits this logic.
 */
static int reserve_metadata_bytes(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_block_rsv *block_rsv,
				  u64 orig_bytes, int flush)
J
Josef Bacik 已提交
3418
{
3419
	struct btrfs_space_info *space_info = block_rsv->space_info;
3420 3421 3422 3423
	u64 unused;
	u64 num_bytes = orig_bytes;
	int retries = 0;
	int ret = 0;
3424
	bool committed = false;
3425
	bool flushing = false;
3426
again:
3427
	ret = 0;
3428
	spin_lock(&space_info->lock);
3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452
	/*
	 * We only want to wait if somebody other than us is flushing and we are
	 * actually alloed to flush.
	 */
	while (flush && !flushing && space_info->flush) {
		spin_unlock(&space_info->lock);
		/*
		 * If we have a trans handle we can't wait because the flusher
		 * may have to commit the transaction, which would mean we would
		 * deadlock since we are waiting for the flusher to finish, but
		 * hold the current transaction open.
		 */
		if (trans)
			return -EAGAIN;
		ret = wait_event_interruptible(space_info->wait,
					       !space_info->flush);
		/* Must have been interrupted, return */
		if (ret)
			return -EINTR;

		spin_lock(&space_info->lock);
	}

	ret = -ENOSPC;
3453 3454 3455
	unused = space_info->bytes_used + space_info->bytes_reserved +
		 space_info->bytes_pinned + space_info->bytes_readonly +
		 space_info->bytes_may_use;
J
Josef Bacik 已提交
3456

3457 3458 3459 3460 3461 3462 3463 3464
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
	if (unused <= space_info->total_bytes) {
3465
		unused = space_info->total_bytes - unused;
3466
		if (unused >= num_bytes) {
3467
			space_info->bytes_may_use += orig_bytes;
3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
		num_bytes = unused - space_info->total_bytes +
			(orig_bytes * (retries + 1));
	}
J
Josef Bacik 已提交
3486

3487 3488 3489 3490 3491
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
	 */
3492 3493 3494
	if (ret && flush) {
		flushing = true;
		space_info->flush = 1;
3495
	}
J
Josef Bacik 已提交
3496

3497
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3498

3499
	if (!ret || !flush)
3500
		goto out;
3501

3502 3503 3504 3505 3506
	/*
	 * We do synchronous shrinking since we don't actually unreserve
	 * metadata until after the IO is completed.
	 */
	ret = shrink_delalloc(trans, root, num_bytes, 1);
3507
	if (ret < 0)
3508
		goto out;
3509

3510 3511
	ret = 0;

3512 3513 3514 3515 3516 3517 3518 3519 3520
	/*
	 * So if we were overcommitted it's possible that somebody else flushed
	 * out enough space and we simply didn't have enough space to reclaim,
	 * so go back around and try again.
	 */
	if (retries < 2) {
		retries++;
		goto again;
	}
3521

3522 3523 3524 3525
	/*
	 * Not enough space to be reclaimed, don't bother committing the
	 * transaction.
	 */
3526
	spin_lock(&space_info->lock);
3527 3528 3529 3530 3531
	if (space_info->bytes_pinned < orig_bytes)
		ret = -ENOSPC;
	spin_unlock(&space_info->lock);
	if (ret)
		goto out;
3532

3533
	ret = -EAGAIN;
3534
	if (trans)
3535
		goto out;
3536

3537
	ret = -ENOSPC;
3538 3539 3540
	if (committed)
		goto out;

3541
	trans = btrfs_join_transaction(root);
3542 3543 3544
	if (IS_ERR(trans))
		goto out;
	ret = btrfs_commit_transaction(trans, root);
3545 3546 3547
	if (!ret) {
		trans = NULL;
		committed = true;
3548
		goto again;
3549
	}
3550 3551

out:
3552
	if (flushing) {
3553
		spin_lock(&space_info->lock);
3554 3555
		space_info->flush = 0;
		wake_up_all(&space_info->wait);
3556
		spin_unlock(&space_info->lock);
3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602
	}
	return ret;
}

static struct btrfs_block_rsv *get_block_rsv(struct btrfs_trans_handle *trans,
					     struct btrfs_root *root)
{
	struct btrfs_block_rsv *block_rsv;
	if (root->ref_cows)
		block_rsv = trans->block_rsv;
	else
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

3603 3604
static void block_rsv_release_bytes(struct btrfs_block_rsv *block_rsv,
				    struct btrfs_block_rsv *dest, u64 num_bytes)
3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
3637
			spin_lock(&space_info->lock);
3638
			space_info->bytes_may_use -= num_bytes;
3639
			space_info->reservation_progress++;
3640
			spin_unlock(&space_info->lock);
3641
		}
J
Josef Bacik 已提交
3642
	}
3643
}
3644

3645 3646 3647 3648
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
J
Josef Bacik 已提交
3649

3650 3651 3652
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
3653

3654
	block_rsv_add_bytes(dst, num_bytes, 1);
J
Josef Bacik 已提交
3655 3656 3657
	return 0;
}

3658
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv)
J
Josef Bacik 已提交
3659
{
3660 3661 3662 3663 3664 3665 3666 3667
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
}

struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root)
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
3668

3669 3670 3671
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
3672

3673 3674 3675 3676 3677
	btrfs_init_block_rsv(block_rsv);
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
3678

3679 3680 3681
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
3682 3683
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	kfree(rsv);
J
Josef Bacik 已提交
3684 3685
}

3686 3687 3688
int btrfs_block_rsv_add(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv,
3689
			u64 num_bytes)
3690 3691
{
	int ret;
J
Josef Bacik 已提交
3692

3693 3694
	if (num_bytes == 0)
		return 0;
3695 3696

	ret = reserve_metadata_bytes(trans, root, block_rsv, num_bytes, 1);
3697 3698 3699 3700
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
3701

3702 3703
	return ret;
}
J
Josef Bacik 已提交
3704

3705 3706 3707
int btrfs_block_rsv_check(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  struct btrfs_block_rsv *block_rsv,
3708
			  u64 min_reserved, int min_factor, int flush)
3709 3710 3711
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
3712

3713 3714
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
3715

3716 3717 3718 3719 3720
	spin_lock(&block_rsv->lock);
	if (min_factor > 0)
		num_bytes = div_factor(block_rsv->size, min_factor);
	if (min_reserved > num_bytes)
		num_bytes = min_reserved;
J
Josef Bacik 已提交
3721

3722
	if (block_rsv->reserved >= num_bytes)
3723
		ret = 0;
3724
	else
3725 3726
		num_bytes -= block_rsv->reserved;
	spin_unlock(&block_rsv->lock);
3727

3728 3729 3730
	if (!ret)
		return 0;

3731
	ret = reserve_metadata_bytes(trans, root, block_rsv, num_bytes, flush);
3732 3733 3734
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 0);
		return 0;
3735
	}
J
Josef Bacik 已提交
3736

3737
	return ret;
3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755
}

int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
	if (global_rsv->full || global_rsv == block_rsv ||
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
	block_rsv_release_bytes(block_rsv, global_rsv, num_bytes);
J
Josef Bacik 已提交
3756 3757 3758
}

/*
3759 3760 3761
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
3762
 */
3763
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3764
{
3765 3766 3767 3768 3769
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
	int csum_size = btrfs_super_csum_size(&fs_info->super_copy);
J
Josef Bacik 已提交
3770

3771 3772 3773 3774
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
3775

3776 3777
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
3778 3779
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
3780 3781
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
3782

3783 3784 3785
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
	num_bytes += div64_u64(data_used + meta_used, 50);
3786

3787 3788
	if (num_bytes * 3 > meta_used)
		num_bytes = div64_u64(meta_used, 3);
3789

3790 3791
	return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
}
J
Josef Bacik 已提交
3792

3793 3794 3795 3796 3797
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
3798

3799
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
3800

3801 3802
	spin_lock(&block_rsv->lock);
	spin_lock(&sinfo->lock);
3803

3804
	block_rsv->size = num_bytes;
3805

3806
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
3807 3808
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
3809 3810 3811 3812

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
3813
		sinfo->bytes_may_use += num_bytes;
J
Josef Bacik 已提交
3814 3815
	}

3816 3817
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
3818
		sinfo->bytes_may_use -= num_bytes;
3819
		sinfo->reservation_progress++;
3820 3821 3822
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
3823

3824 3825
	spin_unlock(&sinfo->lock);
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
3826 3827
}

3828
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3829
{
3830
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
3831

3832 3833
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
J
Josef Bacik 已提交
3834

3835
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
3836 3837
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->delalloc_block_rsv.space_info = space_info;
3838 3839 3840
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;

3841 3842 3843 3844
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
3845
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
3846 3847

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
3848 3849
}

3850
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
3851
{
3852 3853 3854 3855 3856 3857 3858
	block_rsv_release_bytes(&fs_info->global_block_rsv, NULL, (u64)-1);
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
3859
}
J
Josef Bacik 已提交
3860

3861 3862
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
3863
{
3864 3865
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
3866

3867 3868 3869 3870 3871
	BUG_ON(trans->block_rsv != &root->fs_info->trans_block_rsv);
	btrfs_block_rsv_release(root, trans->block_rsv,
				trans->bytes_reserved);
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
3872

3873 3874 3875 3876 3877 3878 3879 3880
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
3881 3882 3883
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
3884
	 */
C
Chris Mason 已提交
3885
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
3886
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
J
Josef Bacik 已提交
3887 3888
}

3889
void btrfs_orphan_release_metadata(struct inode *inode)
3890
{
3891
	struct btrfs_root *root = BTRFS_I(inode)->root;
C
Chris Mason 已提交
3892
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
3893 3894
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
3895

3896 3897 3898 3899 3900 3901 3902 3903 3904 3905
int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
				struct btrfs_pending_snapshot *pending)
{
	struct btrfs_root *root = pending->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
	/*
	 * two for root back/forward refs, two for directory entries
	 * and one for root of the snapshot.
	 */
3906
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 5);
3907 3908
	dst_rsv->space_info = src_rsv->space_info;
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
3909 3910
}

3911 3912 3913 3914 3915 3916 3917 3918 3919
/**
 * drop_outstanding_extent - drop an outstanding extent
 * @inode: the inode we're dropping the extent for
 *
 * This is called when we are freeing up an outstanding extent, either called
 * after an error or after an extent is written.  This will return the number of
 * reserved extents that need to be freed.  This must be called with
 * BTRFS_I(inode)->lock held.
 */
3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932
static unsigned drop_outstanding_extent(struct inode *inode)
{
	unsigned dropped_extents = 0;

	BUG_ON(!BTRFS_I(inode)->outstanding_extents);
	BTRFS_I(inode)->outstanding_extents--;

	/*
	 * If we have more or the same amount of outsanding extents than we have
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
3933
		return 0;
3934 3935 3936 3937 3938 3939 3940

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
	return dropped_extents;
}

3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960
/**
 * calc_csum_metadata_size - return the amount of metada space that must be
 *	reserved/free'd for the given bytes.
 * @inode: the inode we're manipulating
 * @num_bytes: the number of bytes in question
 * @reserve: 1 if we are reserving space, 0 if we are freeing space
 *
 * This adjusts the number of csum_bytes in the inode and then returns the
 * correct amount of metadata that must either be reserved or freed.  We
 * calculate how many checksums we can fit into one leaf and then divide the
 * number of bytes that will need to be checksumed by this value to figure out
 * how many checksums will be required.  If we are adding bytes then the number
 * may go up and we will return the number of additional bytes that must be
 * reserved.  If it is going down we will return the number of bytes that must
 * be freed.
 *
 * This must be called with BTRFS_I(inode)->lock held.
 */
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
				   int reserve)
3961
{
3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 csum_size;
	int num_csums_per_leaf;
	int num_csums;
	int old_csums;

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
	    BTRFS_I(inode)->csum_bytes == 0)
		return 0;

	old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	if (reserve)
		BTRFS_I(inode)->csum_bytes += num_bytes;
	else
		BTRFS_I(inode)->csum_bytes -= num_bytes;
	csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
	num_csums_per_leaf = (int)div64_u64(csum_size,
					    sizeof(struct btrfs_csum_item) +
					    sizeof(struct btrfs_disk_key));
	num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	num_csums = num_csums + num_csums_per_leaf - 1;
	num_csums = num_csums / num_csums_per_leaf;

	old_csums = old_csums + num_csums_per_leaf - 1;
	old_csums = old_csums / num_csums_per_leaf;

	/* No change, no need to reserve more */
	if (old_csums == num_csums)
		return 0;

	if (reserve)
		return btrfs_calc_trans_metadata_size(root,
						      num_csums - old_csums);

	return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
3997
}
Y
Yan Zheng 已提交
3998

3999 4000 4001 4002
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4003 4004
	u64 to_reserve = 0;
	unsigned nr_extents = 0;
4005
	int ret;
4006

4007 4008
	if (btrfs_transaction_in_commit(root->fs_info))
		schedule_timeout(1);
4009

4010
	num_bytes = ALIGN(num_bytes, root->sectorsize);
4011

4012 4013 4014 4015 4016 4017 4018 4019
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;

	if (BTRFS_I(inode)->outstanding_extents >
	    BTRFS_I(inode)->reserved_extents) {
		nr_extents = BTRFS_I(inode)->outstanding_extents -
			BTRFS_I(inode)->reserved_extents;
		BTRFS_I(inode)->reserved_extents += nr_extents;
4020

4021
		to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4022
	}
4023
	to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
4024
	spin_unlock(&BTRFS_I(inode)->lock);
4025

4026
	ret = reserve_metadata_bytes(NULL, root, block_rsv, to_reserve, 1);
4027 4028 4029 4030 4031 4032
	if (ret) {
		unsigned dropped;
		/*
		 * We don't need the return value since our reservation failed,
		 * we just need to clean up our counter.
		 */
4033
		spin_lock(&BTRFS_I(inode)->lock);
4034 4035
		dropped = drop_outstanding_extent(inode);
		WARN_ON(dropped > 1);
4036 4037
		BTRFS_I(inode)->csum_bytes -= num_bytes;
		spin_unlock(&BTRFS_I(inode)->lock);
4038
		return ret;
4039
	}
4040

4041 4042 4043 4044 4045
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	return 0;
}

4046 4047 4048 4049 4050 4051 4052 4053 4054
/**
 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
 * @inode: the inode to release the reservation for
 * @num_bytes: the number of bytes we're releasing
 *
 * This will release the metadata reservation for an inode.  This can be called
 * once we complete IO for a given set of bytes to release their metadata
 * reservations.
 */
4055 4056 4057
void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
4058 4059
	u64 to_free = 0;
	unsigned dropped;
4060 4061

	num_bytes = ALIGN(num_bytes, root->sectorsize);
4062
	spin_lock(&BTRFS_I(inode)->lock);
4063
	dropped = drop_outstanding_extent(inode);
4064

4065 4066
	to_free = calc_csum_metadata_size(inode, num_bytes, 0);
	spin_unlock(&BTRFS_I(inode)->lock);
4067 4068
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
4069 4070 4071 4072 4073

	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088
/**
 * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
 * @inode: inode we're writing to
 * @num_bytes: the number of bytes we want to allocate
 *
 * This will do the following things
 *
 * o reserve space in the data space info for num_bytes
 * o reserve space in the metadata space info based on number of outstanding
 *   extents and how much csums will be needed
 * o add to the inodes ->delalloc_bytes
 * o add it to the fs_info's delalloc inodes list.
 *
 * This will return 0 for success and -ENOSPC if there is no space left.
 */
4089 4090 4091 4092 4093
int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
C
Chris Mason 已提交
4094
	if (ret)
4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118
/**
 * btrfs_delalloc_release_space - release data and metadata space for delalloc
 * @inode: inode we're releasing space for
 * @num_bytes: the number of bytes we want to free up
 *
 * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
 * called in the case that we don't need the metadata AND data reservations
 * anymore.  So if there is an error or we insert an inline extent.
 *
 * This function will release the metadata space that was not used and will
 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
 * list if there are no delalloc bytes left.
 */
4119 4120 4121 4122
void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
4123 4124
}

C
Chris Mason 已提交
4125 4126
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
4127
			      u64 bytenr, u64 num_bytes, int alloc)
C
Chris Mason 已提交
4128
{
4129
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
4130
	struct btrfs_fs_info *info = root->fs_info;
4131
	u64 total = num_bytes;
C
Chris Mason 已提交
4132
	u64 old_val;
4133
	u64 byte_in_group;
4134
	int factor;
C
Chris Mason 已提交
4135

4136 4137 4138 4139 4140 4141 4142 4143 4144 4145
	/* block accounting for super block */
	spin_lock(&info->delalloc_lock);
	old_val = btrfs_super_bytes_used(&info->super_copy);
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
	btrfs_set_super_bytes_used(&info->super_copy, old_val);
	spin_unlock(&info->delalloc_lock);

C
Chris Mason 已提交
4146
	while (total) {
4147
		cache = btrfs_lookup_block_group(info, bytenr);
4148
		if (!cache)
C
Chris Mason 已提交
4149
			return -1;
4150 4151 4152 4153 4154 4155
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
4156 4157 4158 4159 4160 4161 4162
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
4163
			cache_block_group(cache, trans, NULL, 1);
4164

4165 4166
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
4167

4168
		spin_lock(&cache->space_info->lock);
4169
		spin_lock(&cache->lock);
4170 4171 4172 4173 4174

		if (btrfs_super_cache_generation(&info->super_copy) != 0 &&
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

J
Josef Bacik 已提交
4175
		cache->dirty = 1;
C
Chris Mason 已提交
4176
		old_val = btrfs_block_group_used(&cache->item);
4177
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
4178
		if (alloc) {
4179
			old_val += num_bytes;
4180 4181 4182
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
4183 4184
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
4185
			spin_unlock(&cache->lock);
4186
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4187
		} else {
4188
			old_val -= num_bytes;
4189
			btrfs_set_block_group_used(&cache->item, old_val);
4190 4191
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
4192
			cache->space_info->bytes_used -= num_bytes;
4193
			cache->space_info->disk_used -= num_bytes * factor;
4194
			spin_unlock(&cache->lock);
4195
			spin_unlock(&cache->space_info->lock);
4196

4197 4198 4199
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
C
Chris Mason 已提交
4200
		}
4201
		btrfs_put_block_group(cache);
4202 4203
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
4204 4205 4206
	}
	return 0;
}
4207

4208 4209
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
4210
	struct btrfs_block_group_cache *cache;
4211
	u64 bytenr;
J
Josef Bacik 已提交
4212 4213 4214

	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
4215
		return 0;
J
Josef Bacik 已提交
4216

4217
	bytenr = cache->key.objectid;
4218
	btrfs_put_block_group(cache);
4219 4220

	return bytenr;
4221 4222
}

4223 4224 4225
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
4226
{
4227 4228 4229 4230 4231 4232 4233 4234 4235 4236
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
4237

4238 4239 4240 4241
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
	return 0;
}
J
Josef Bacik 已提交
4242

4243 4244 4245 4246 4247 4248 4249
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
4250

4251 4252 4253 4254 4255 4256
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
	BUG_ON(!cache);

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
4257 4258 4259
	return 0;
}

4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280
/**
 * btrfs_update_reserved_bytes - update the block_group and space info counters
 * @cache:	The cache we are manipulating
 * @num_bytes:	The number of bytes in question
 * @reserve:	One of the reservation enums
 *
 * This is called by the allocator when it reserves space, or by somebody who is
 * freeing space that was never actually used on disk.  For example if you
 * reserve some space for a new leaf in transaction A and before transaction A
 * commits you free that leaf, you call this with reserve set to 0 in order to
 * clear the reservation.
 *
 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
 * ENOSPC accounting.  For data we handle the reservation through clearing the
 * delalloc bits in the io_tree.  We have to do this since we could end up
 * allocating less disk space for the amount of data we have reserved in the
 * case of compression.
 *
 * If this is a reservation and the block group has become read only we cannot
 * make the reservation and return -EAGAIN, otherwise this function always
 * succeeds.
4281
 */
4282 4283
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve)
4284
{
4285
	struct btrfs_space_info *space_info = cache->space_info;
4286
	int ret = 0;
4287 4288 4289
	spin_lock(&space_info->lock);
	spin_lock(&cache->lock);
	if (reserve != RESERVE_FREE) {
4290 4291 4292
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
4293 4294 4295 4296 4297 4298
			cache->reserved += num_bytes;
			space_info->bytes_reserved += num_bytes;
			if (reserve == RESERVE_ALLOC) {
				BUG_ON(space_info->bytes_may_use < num_bytes);
				space_info->bytes_may_use -= num_bytes;
			}
4299
		}
4300 4301 4302 4303 4304 4305
	} else {
		if (cache->ro)
			space_info->bytes_readonly += num_bytes;
		cache->reserved -= num_bytes;
		space_info->bytes_reserved -= num_bytes;
		space_info->reservation_progress++;
4306
	}
4307 4308
	spin_unlock(&cache->lock);
	spin_unlock(&space_info->lock);
4309
	return ret;
4310
}
C
Chris Mason 已提交
4311

4312 4313
int btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
				struct btrfs_root *root)
4314 4315
{
	struct btrfs_fs_info *fs_info = root->fs_info;
4316 4317 4318
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
4319

4320
	down_write(&fs_info->extent_commit_sem);
4321

4322 4323 4324 4325 4326 4327 4328
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
4329
		} else {
4330
			cache->last_byte_to_unpin = caching_ctl->progress;
4331 4332
		}
	}
4333 4334 4335 4336 4337 4338 4339

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

	up_write(&fs_info->extent_commit_sem);
4340 4341

	update_global_block_rsv(fs_info);
4342 4343 4344
	return 0;
}

4345
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
C
Chris Mason 已提交
4346
{
4347 4348 4349
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 len;
C
Chris Mason 已提交
4350

4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367
	while (start <= end) {
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
			BUG_ON(!cache);
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
			btrfs_add_free_space(cache, start, len);
		}

4368 4369
		start += len;

4370 4371 4372 4373
		spin_lock(&cache->space_info->lock);
		spin_lock(&cache->lock);
		cache->pinned -= len;
		cache->space_info->bytes_pinned -= len;
4374
		if (cache->ro)
4375
			cache->space_info->bytes_readonly += len;
4376 4377
		spin_unlock(&cache->lock);
		spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4378
	}
4379 4380 4381

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
4382 4383 4384 4385
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
4386
			       struct btrfs_root *root)
4387
{
4388 4389
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
4390 4391
	u64 start;
	u64 end;
4392 4393
	int ret;

4394 4395 4396 4397 4398
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

C
Chris Mason 已提交
4399
	while (1) {
4400 4401 4402
		ret = find_first_extent_bit(unpin, 0, &start, &end,
					    EXTENT_DIRTY);
		if (ret)
4403
			break;
4404

4405 4406 4407
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
4408

4409
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
4410
		unpin_extent_range(root, start, end);
4411
		cond_resched();
4412
	}
J
Josef Bacik 已提交
4413

C
Chris Mason 已提交
4414 4415 4416
	return 0;
}

4417 4418 4419 4420 4421 4422
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extent_op)
4423
{
C
Chris Mason 已提交
4424
	struct btrfs_key key;
4425
	struct btrfs_path *path;
4426 4427
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
4428
	struct extent_buffer *leaf;
4429 4430
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
4431
	int ret;
4432
	int is_data;
4433 4434 4435
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
4436 4437
	u32 item_size;
	u64 refs;
C
Chris Mason 已提交
4438

4439
	path = btrfs_alloc_path();
4440 4441
	if (!path)
		return -ENOMEM;
4442

4443
	path->reada = 1;
4444
	path->leave_spinning = 1;
4445 4446 4447 4448 4449 4450 4451 4452

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
4453
	if (ret == 0) {
4454
		extent_slot = path->slots[0];
4455 4456
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
4457
					      extent_slot);
4458
			if (key.objectid != bytenr)
4459
				break;
4460 4461
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
4462 4463 4464 4465 4466
				found_extent = 1;
				break;
			}
			if (path->slots[0] - extent_slot > 5)
				break;
4467
			extent_slot--;
4468
		}
4469 4470 4471 4472 4473
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
4474
		if (!found_extent) {
4475
			BUG_ON(iref);
4476
			ret = remove_extent_backref(trans, extent_root, path,
4477 4478
						    NULL, refs_to_drop,
						    is_data);
Z
Zheng Yan 已提交
4479
			BUG_ON(ret);
4480
			btrfs_release_path(path);
4481
			path->leave_spinning = 1;
4482 4483 4484 4485 4486

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

Z
Zheng Yan 已提交
4487 4488
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
4489 4490
			if (ret) {
				printk(KERN_ERR "umm, got %d back from search"
C
Chris Mason 已提交
4491 4492
				       ", was looking for %llu\n", ret,
				       (unsigned long long)bytenr);
4493 4494 4495
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
4496
			}
Z
Zheng Yan 已提交
4497 4498 4499
			BUG_ON(ret);
			extent_slot = path->slots[0];
		}
4500 4501 4502
	} else {
		btrfs_print_leaf(extent_root, path->nodes[0]);
		WARN_ON(1);
C
Chris Mason 已提交
4503
		printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
4504
		       "parent %llu root %llu  owner %llu offset %llu\n",
C
Chris Mason 已提交
4505
		       (unsigned long long)bytenr,
4506
		       (unsigned long long)parent,
C
Chris Mason 已提交
4507
		       (unsigned long long)root_objectid,
4508 4509
		       (unsigned long long)owner_objectid,
		       (unsigned long long)owner_offset);
4510
	}
4511 4512

	leaf = path->nodes[0];
4513 4514 4515 4516 4517 4518 4519 4520
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
		BUG_ON(ret < 0);

4521
		btrfs_release_path(path);
4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
			printk(KERN_ERR "umm, got %d back from search"
			       ", was looking for %llu\n", ret,
			       (unsigned long long)bytenr);
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
		BUG_ON(ret);
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
4543
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
4544
			    struct btrfs_extent_item);
4545 4546 4547 4548 4549 4550
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
4551

4552
	refs = btrfs_extent_refs(leaf, ei);
4553 4554
	BUG_ON(refs < refs_to_drop);
	refs -= refs_to_drop;
4555

4556 4557 4558 4559 4560 4561
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
4562
		 */
4563 4564 4565 4566 4567 4568 4569 4570 4571 4572
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
						    is_data);
4573 4574
			BUG_ON(ret);
		}
4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
4586
		}
4587

4588 4589
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
Z
Zheng Yan 已提交
4590
		BUG_ON(ret);
4591
		btrfs_release_path(path);
4592

4593
		if (is_data) {
4594 4595
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
			BUG_ON(ret);
4596 4597 4598 4599
		} else {
			invalidate_mapping_pages(info->btree_inode->i_mapping,
			     bytenr >> PAGE_CACHE_SHIFT,
			     (bytenr + num_bytes - 1) >> PAGE_CACHE_SHIFT);
4600 4601
		}

4602
		ret = update_block_group(trans, root, bytenr, num_bytes, 0);
4603
		BUG_ON(ret);
4604
	}
4605
	btrfs_free_path(path);
4606 4607 4608
	return ret;
}

4609
/*
4610
 * when we free an block, it is possible (and likely) that we free the last
4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct rb_node *node;
4622
	int ret = 0;
4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	/* there are still entries for this ref, we can't drop it */
	if (ref->bytenr == bytenr)
		goto out;

4640 4641 4642 4643 4644 4645 4646
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
		kfree(head->extent_op);
		head->extent_op = NULL;
	}

4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
	rb_erase(&head->node.rb_node, &delayed_refs->root);
4660

4661 4662 4663 4664 4665 4666
	delayed_refs->num_entries--;

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
4667 4668 4669 4670 4671
	delayed_refs->num_heads--;
	if (list_empty(&head->cluster))
		delayed_refs->num_heads_ready--;

	list_del_init(&head->cluster);
4672 4673
	spin_unlock(&delayed_refs->lock);

4674 4675 4676 4677 4678
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
4679
	btrfs_put_delayed_ref(&head->node);
4680
	return ret;
4681 4682 4683 4684 4685
out:
	spin_unlock(&delayed_refs->lock);
	return 0;
}

4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
			   u64 parent, int last_ref)
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_block_group_cache *cache = NULL;
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, buf->start, buf->len,
						parent, root->root_key.objectid,
						btrfs_header_level(buf),
						BTRFS_DROP_DELAYED_REF, NULL);
		BUG_ON(ret);
	}

	if (!last_ref)
		return;

	block_rsv = get_block_rsv(trans, root);
	cache = btrfs_lookup_block_group(root->fs_info, buf->start);
4708 4709
	if (block_rsv->space_info != cache->space_info)
		goto out;
4710 4711 4712 4713 4714

	if (btrfs_header_generation(buf) == trans->transid) {
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
4715
				goto out;
4716 4717 4718 4719
		}

		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
4720
			goto out;
4721 4722 4723 4724 4725
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
4726
		btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
4727 4728
	}
out:
4729 4730 4731 4732 4733
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
4734 4735 4736
	btrfs_put_block_group(cache);
}

4737
int btrfs_free_extent(struct btrfs_trans_handle *trans,
Z
Zheng Yan 已提交
4738 4739
		      struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent,
4740
		      u64 root_objectid, u64 owner, u64 offset)
4741 4742 4743
{
	int ret;

4744 4745 4746 4747
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
4748 4749
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
4750
		/* unlocks the pinned mutex */
4751
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
4752
		ret = 0;
4753 4754 4755 4756
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = btrfs_add_delayed_tree_ref(trans, bytenr, num_bytes,
					parent, root_objectid, (int)owner,
					BTRFS_DROP_DELAYED_REF, NULL);
4757
		BUG_ON(ret);
4758 4759 4760 4761 4762
	} else {
		ret = btrfs_add_delayed_data_ref(trans, bytenr, num_bytes,
					parent, root_objectid, owner,
					offset, BTRFS_DROP_DELAYED_REF, NULL);
		BUG_ON(ret);
4763
	}
4764 4765 4766
	return ret;
}

4767 4768 4769 4770 4771 4772 4773
static u64 stripe_align(struct btrfs_root *root, u64 val)
{
	u64 mask = ((u64)root->stripesize - 1);
	u64 ret = (val + mask) & ~mask;
	return ret;
}

J
Josef Bacik 已提交
4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
 */
static noinline int
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
4789
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
4790 4791
	DEFINE_WAIT(wait);

4792 4793
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
J
Josef Bacik 已提交
4794 4795
		return 0;

4796
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
4797
		   (cache->free_space_ctl->free_space >= num_bytes));
4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815

	put_caching_control(caching_ctl);
	return 0;
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;
	DEFINE_WAIT(wait);

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
		return 0;

	wait_event(caching_ctl->wait, block_group_cache_done(cache));

	put_caching_control(caching_ctl);
J
Josef Bacik 已提交
4816 4817 4818
	return 0;
}

4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834
static int get_block_group_index(struct btrfs_block_group_cache *cache)
{
	int index;
	if (cache->flags & BTRFS_BLOCK_GROUP_RAID10)
		index = 0;
	else if (cache->flags & BTRFS_BLOCK_GROUP_RAID1)
		index = 1;
	else if (cache->flags & BTRFS_BLOCK_GROUP_DUP)
		index = 2;
	else if (cache->flags & BTRFS_BLOCK_GROUP_RAID0)
		index = 3;
	else
		index = 4;
	return index;
}

J
Josef Bacik 已提交
4835
enum btrfs_loop_type {
4836
	LOOP_FIND_IDEAL = 0,
J
Josef Bacik 已提交
4837 4838 4839 4840 4841 4842
	LOOP_CACHING_NOWAIT = 1,
	LOOP_CACHING_WAIT = 2,
	LOOP_ALLOC_CHUNK = 3,
	LOOP_NO_EMPTY_SIZE = 4,
};

4843 4844 4845 4846
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
 * ins->objectid == block start
4847
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
4848 4849 4850
 * ins->offset == number of blocks
 * Any available blocks before search_start are skipped.
 */
C
Chris Mason 已提交
4851
static noinline int find_free_extent(struct btrfs_trans_handle *trans,
4852 4853 4854 4855
				     struct btrfs_root *orig_root,
				     u64 num_bytes, u64 empty_size,
				     u64 search_start, u64 search_end,
				     u64 hint_byte, struct btrfs_key *ins,
4856
				     u64 data)
4857
{
4858
	int ret = 0;
C
Chris Mason 已提交
4859
	struct btrfs_root *root = orig_root->fs_info->extent_root;
4860
	struct btrfs_free_cluster *last_ptr = NULL;
4861
	struct btrfs_block_group_cache *block_group = NULL;
4862
	int empty_cluster = 2 * 1024 * 1024;
C
Chris Mason 已提交
4863
	int allowed_chunk_alloc = 0;
4864
	int done_chunk_alloc = 0;
4865
	struct btrfs_space_info *space_info;
4866 4867
	int last_ptr_loop = 0;
	int loop = 0;
4868
	int index = 0;
4869 4870
	int alloc_type = (data & BTRFS_BLOCK_GROUP_DATA) ?
		RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
J
Josef Bacik 已提交
4871
	bool found_uncached_bg = false;
4872
	bool failed_cluster_refill = false;
4873
	bool failed_alloc = false;
4874
	bool use_cluster = true;
4875 4876
	u64 ideal_cache_percent = 0;
	u64 ideal_cache_offset = 0;
4877

4878
	WARN_ON(num_bytes < root->sectorsize);
4879
	btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
4880 4881
	ins->objectid = 0;
	ins->offset = 0;
4882

J
Josef Bacik 已提交
4883
	space_info = __find_space_info(root->fs_info, data);
4884
	if (!space_info) {
4885
		printk(KERN_ERR "No space info for %llu\n", data);
4886 4887
		return -ENOSPC;
	}
J
Josef Bacik 已提交
4888

4889 4890 4891 4892 4893 4894 4895
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

C
Chris Mason 已提交
4896 4897 4898
	if (orig_root->ref_cows || empty_size)
		allowed_chunk_alloc = 1;

4899
	if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
4900
		last_ptr = &root->fs_info->meta_alloc_cluster;
4901 4902
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
4903 4904
	}

4905 4906
	if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
	    btrfs_test_opt(root, SSD)) {
4907 4908
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
J
Josef Bacik 已提交
4909

4910
	if (last_ptr) {
4911 4912 4913 4914
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
4915
	}
4916

4917
	search_start = max(search_start, first_logical_byte(root, 0));
4918
	search_start = max(search_start, hint_byte);
4919

J
Josef Bacik 已提交
4920
	if (!last_ptr)
4921 4922
		empty_cluster = 0;

J
Josef Bacik 已提交
4923
	if (search_start == hint_byte) {
4924
ideal_cache:
J
Josef Bacik 已提交
4925 4926
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
J
Josef Bacik 已提交
4927 4928 4929
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
4930 4931 4932
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
4933 4934
		 */
		if (block_group && block_group_bits(block_group, data) &&
4935 4936
		    (block_group->cached != BTRFS_CACHE_NO ||
		     search_start == ideal_cache_offset)) {
J
Josef Bacik 已提交
4937
			down_read(&space_info->groups_sem);
4938 4939 4940 4941 4942 4943 4944 4945 4946 4947
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
4948
			} else {
4949
				index = get_block_group_index(block_group);
4950
				goto have_block_group;
4951
			}
J
Josef Bacik 已提交
4952
		} else if (block_group) {
4953
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
4954
		}
4955
	}
J
Josef Bacik 已提交
4956
search:
4957
	down_read(&space_info->groups_sem);
4958 4959
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
4960
		u64 offset;
J
Josef Bacik 已提交
4961
		int cached;
4962

4963
		btrfs_get_block_group(block_group);
J
Josef Bacik 已提交
4964
		search_start = block_group->key.objectid;
4965

4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
		if (!block_group_bits(block_group, data)) {
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
			if ((data & extra) && !(block_group->flags & extra))
				goto loop;
		}

J
Josef Bacik 已提交
4985
have_block_group:
J
Josef Bacik 已提交
4986
		if (unlikely(block_group->cached == BTRFS_CACHE_NO)) {
4987 4988
			u64 free_percent;

4989 4990
			ret = cache_block_group(block_group, trans,
						orig_root, 1);
4991 4992 4993
			if (block_group->cached == BTRFS_CACHE_FINISHED)
				goto have_block_group;

4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004
			free_percent = btrfs_block_group_used(&block_group->item);
			free_percent *= 100;
			free_percent = div64_u64(free_percent,
						 block_group->key.offset);
			free_percent = 100 - free_percent;
			if (free_percent > ideal_cache_percent &&
			    likely(!block_group->ro)) {
				ideal_cache_offset = block_group->key.objectid;
				ideal_cache_percent = free_percent;
			}

J
Josef Bacik 已提交
5005
			/*
5006 5007
			 * The caching workers are limited to 2 threads, so we
			 * can queue as much work as we care to.
J
Josef Bacik 已提交
5008
			 */
5009
			if (loop > LOOP_FIND_IDEAL) {
5010 5011
				ret = cache_block_group(block_group, trans,
							orig_root, 0);
J
Josef Bacik 已提交
5012
				BUG_ON(ret);
J
Josef Bacik 已提交
5013
			}
J
Josef Bacik 已提交
5014 5015
			found_uncached_bg = true;

5016 5017 5018 5019 5020
			/*
			 * If loop is set for cached only, try the next block
			 * group.
			 */
			if (loop == LOOP_FIND_IDEAL)
J
Josef Bacik 已提交
5021 5022 5023
				goto loop;
		}

5024 5025 5026 5027
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached))
			found_uncached_bg = true;

5028
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
5029
			goto loop;
J
Josef Bacik 已提交
5030

C
Chris Mason 已提交
5031
		spin_lock(&block_group->free_space_ctl->tree_lock);
5032
		if (cached &&
C
Chris Mason 已提交
5033 5034 5035
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_size) {
			spin_unlock(&block_group->free_space_ctl->tree_lock);
5036 5037
			goto loop;
		}
C
Chris Mason 已提交
5038
		spin_unlock(&block_group->free_space_ctl->tree_lock);
5039

5040 5041 5042 5043 5044 5045 5046 5047 5048 5049
		/*
		 * Ok we want to try and use the cluster allocator, so lets look
		 * there, unless we are on LOOP_NO_EMPTY_SIZE, since we will
		 * have tried the cluster allocator plenty of times at this
		 * point and not have found anything, so we are likely way too
		 * fragmented for the clustering stuff to find anything, so lets
		 * just skip it and let the allocator find whatever block it can
		 * find
		 */
		if (last_ptr && loop < LOOP_NO_EMPTY_SIZE) {
5050 5051 5052 5053 5054
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
			spin_lock(&last_ptr->refill_lock);
5055 5056 5057 5058 5059 5060 5061
			if (last_ptr->block_group &&
			    (last_ptr->block_group->ro ||
			    !block_group_bits(last_ptr->block_group, data))) {
				offset = 0;
				goto refill_cluster;
			}

5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076
			offset = btrfs_alloc_from_cluster(block_group, last_ptr,
						 num_bytes, search_start);
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
				goto checks;
			}

			spin_lock(&last_ptr->lock);
			/*
			 * whoops, this cluster doesn't actually point to
			 * this block group.  Get a ref on the block
			 * group is does point to and try again
			 */
			if (!last_ptr_loop && last_ptr->block_group &&
5077 5078 5079
			    last_ptr->block_group != block_group &&
			    index <=
				 get_block_group_index(last_ptr->block_group)) {
5080 5081 5082

				btrfs_put_block_group(block_group);
				block_group = last_ptr->block_group;
5083
				btrfs_get_block_group(block_group);
5084 5085 5086 5087 5088
				spin_unlock(&last_ptr->lock);
				spin_unlock(&last_ptr->refill_lock);

				last_ptr_loop = 1;
				search_start = block_group->key.objectid;
5089 5090 5091 5092 5093 5094 5095
				/*
				 * we know this block group is properly
				 * in the list because
				 * btrfs_remove_block_group, drops the
				 * cluster before it removes the block
				 * group from the list
				 */
5096 5097 5098
				goto have_block_group;
			}
			spin_unlock(&last_ptr->lock);
5099
refill_cluster:
5100 5101 5102 5103 5104 5105 5106 5107 5108
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

			last_ptr_loop = 0;

			/* allocate a cluster in this block group */
5109
			ret = btrfs_find_space_cluster(trans, root,
5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125
					       block_group, last_ptr,
					       offset, num_bytes,
					       empty_cluster + empty_size);
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
						  last_ptr, num_bytes,
						  search_start);
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
					goto checks;
				}
5126 5127
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
5128 5129
				spin_unlock(&last_ptr->refill_lock);

5130
				failed_cluster_refill = true;
J
Josef Bacik 已提交
5131 5132 5133
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
5134
			}
J
Josef Bacik 已提交
5135

5136 5137 5138 5139 5140 5141
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
5142
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
5143
			spin_unlock(&last_ptr->refill_lock);
5144
			goto loop;
5145 5146
		}

5147 5148
		offset = btrfs_find_space_for_alloc(block_group, search_start,
						    num_bytes, empty_size);
5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
5160
			wait_block_group_cache_progress(block_group,
5161 5162
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
5163
			goto have_block_group;
5164 5165
		} else if (!offset) {
			goto loop;
J
Josef Bacik 已提交
5166
		}
5167
checks:
5168
		search_start = stripe_align(root, offset);
J
Josef Bacik 已提交
5169
		/* move on to the next group */
5170 5171
		if (search_start + num_bytes >= search_end) {
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5172
			goto loop;
5173
		}
5174

J
Josef Bacik 已提交
5175 5176
		/* move on to the next group */
		if (search_start + num_bytes >
5177 5178
		    block_group->key.objectid + block_group->key.offset) {
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5179
			goto loop;
5180
		}
5181

5182 5183
		ins->objectid = search_start;
		ins->offset = num_bytes;
J
Josef Bacik 已提交
5184

5185 5186 5187 5188
		if (offset < search_start)
			btrfs_add_free_space(block_group, offset,
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
5189

5190 5191
		ret = btrfs_update_reserved_bytes(block_group, num_bytes,
						  alloc_type);
5192
		if (ret == -EAGAIN) {
5193
			btrfs_add_free_space(block_group, offset, num_bytes);
J
Josef Bacik 已提交
5194
			goto loop;
J
Josef Bacik 已提交
5195
		}
5196

5197
		/* we are all good, lets return */
J
Josef Bacik 已提交
5198 5199
		ins->objectid = search_start;
		ins->offset = num_bytes;
5200

5201 5202 5203 5204
		if (offset < search_start)
			btrfs_add_free_space(block_group, offset,
					     search_start - offset);
		BUG_ON(offset > search_start);
5205
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5206 5207
		break;
loop:
5208
		failed_cluster_refill = false;
5209
		failed_alloc = false;
5210
		BUG_ON(index != get_block_group_index(block_group));
5211
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5212 5213 5214
	}
	up_read(&space_info->groups_sem);

5215 5216 5217
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

5218 5219 5220 5221 5222
	/* LOOP_FIND_IDEAL, only search caching/cached bg's, and don't wait for
	 *			for them to make caching progress.  Also
	 *			determine the best possible bg to cache
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
5223 5224 5225 5226
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
5227
	 */
5228
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
5229
		index = 0;
5230
		if (loop == LOOP_FIND_IDEAL && found_uncached_bg) {
J
Josef Bacik 已提交
5231
			found_uncached_bg = false;
5232
			loop++;
5233
			if (!ideal_cache_percent)
J
Josef Bacik 已提交
5234
				goto search;
5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266

			/*
			 * 1 of the following 2 things have happened so far
			 *
			 * 1) We found an ideal block group for caching that
			 * is mostly full and will cache quickly, so we might
			 * as well wait for it.
			 *
			 * 2) We searched for cached only and we didn't find
			 * anything, and we didn't start any caching kthreads
			 * either, so chances are we will loop through and
			 * start a couple caching kthreads, and then come back
			 * around and just wait for them.  This will be slower
			 * because we will have 2 caching kthreads reading at
			 * the same time when we could have just started one
			 * and waited for it to get far enough to give us an
			 * allocation, so go ahead and go to the wait caching
			 * loop.
			 */
			loop = LOOP_CACHING_WAIT;
			search_start = ideal_cache_offset;
			ideal_cache_percent = 0;
			goto ideal_cache;
		} else if (loop == LOOP_FIND_IDEAL) {
			/*
			 * Didn't find a uncached bg, wait on anything we find
			 * next.
			 */
			loop = LOOP_CACHING_WAIT;
			goto search;
		}

5267
		loop++;
J
Josef Bacik 已提交
5268 5269

		if (loop == LOOP_ALLOC_CHUNK) {
5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281
		       if (allowed_chunk_alloc) {
				ret = do_chunk_alloc(trans, root, num_bytes +
						     2 * 1024 * 1024, data,
						     CHUNK_ALLOC_LIMITED);
				allowed_chunk_alloc = 0;
				if (ret == 1)
					done_chunk_alloc = 1;
			} else if (!done_chunk_alloc &&
				   space_info->force_alloc ==
				   CHUNK_ALLOC_NO_FORCE) {
				space_info->force_alloc = CHUNK_ALLOC_LIMITED;
			}
J
Josef Bacik 已提交
5282

5283 5284 5285 5286 5287 5288
		       /*
			* We didn't allocate a chunk, go ahead and drop the
			* empty size and loop again.
			*/
		       if (!done_chunk_alloc)
			       loop = LOOP_NO_EMPTY_SIZE;
J
Josef Bacik 已提交
5289 5290
		}

5291 5292 5293
		if (loop == LOOP_NO_EMPTY_SIZE) {
			empty_size = 0;
			empty_cluster = 0;
5294
		}
5295 5296

		goto search;
J
Josef Bacik 已提交
5297 5298
	} else if (!ins->objectid) {
		ret = -ENOSPC;
5299
	} else if (ins->objectid) {
5300
		ret = 0;
C
Chris Mason 已提交
5301 5302
	}

C
Chris Mason 已提交
5303
	return ret;
5304
}
5305

J
Josef Bacik 已提交
5306 5307
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
5308 5309
{
	struct btrfs_block_group_cache *cache;
5310
	int index = 0;
J
Josef Bacik 已提交
5311

J
Josef Bacik 已提交
5312
	spin_lock(&info->lock);
5313 5314
	printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
	       (unsigned long long)info->flags,
C
Chris Mason 已提交
5315
	       (unsigned long long)(info->total_bytes - info->bytes_used -
J
Josef Bacik 已提交
5316
				    info->bytes_pinned - info->bytes_reserved -
5317
				    info->bytes_readonly),
C
Chris Mason 已提交
5318
	       (info->full) ? "" : "not ");
5319 5320
	printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
5321
	       (unsigned long long)info->total_bytes,
5322
	       (unsigned long long)info->bytes_used,
5323
	       (unsigned long long)info->bytes_pinned,
5324
	       (unsigned long long)info->bytes_reserved,
5325
	       (unsigned long long)info->bytes_may_use,
5326
	       (unsigned long long)info->bytes_readonly);
J
Josef Bacik 已提交
5327 5328 5329 5330
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
5331

5332
	down_read(&info->groups_sem);
5333 5334
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
5335
		spin_lock(&cache->lock);
C
Chris Mason 已提交
5336 5337 5338 5339 5340 5341 5342
		printk(KERN_INFO "block group %llu has %llu bytes, %llu used "
		       "%llu pinned %llu reserved\n",
		       (unsigned long long)cache->key.objectid,
		       (unsigned long long)cache->key.offset,
		       (unsigned long long)btrfs_block_group_used(&cache->item),
		       (unsigned long long)cache->pinned,
		       (unsigned long long)cache->reserved);
J
Josef Bacik 已提交
5343 5344 5345
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
5346 5347
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
5348
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
5349
}
5350

5351 5352 5353 5354 5355 5356
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
			 u64 search_end, struct btrfs_key *ins,
			 u64 data)
5357 5358
{
	int ret;
5359
	u64 search_start = 0;
5360

J
Josef Bacik 已提交
5361
	data = btrfs_get_alloc_profile(root, data);
5362
again:
C
Chris Mason 已提交
5363 5364 5365 5366
	/*
	 * the only place that sets empty_size is btrfs_realloc_node, which
	 * is not called recursively on allocations
	 */
J
Josef Bacik 已提交
5367
	if (empty_size || root->ref_cows)
5368
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
5369 5370
				     num_bytes + 2 * 1024 * 1024, data,
				     CHUNK_ALLOC_NO_FORCE);
5371

5372 5373
	WARN_ON(num_bytes < root->sectorsize);
	ret = find_free_extent(trans, root, num_bytes, empty_size,
5374 5375
			       search_start, search_end, hint_byte,
			       ins, data);
5376

5377 5378
	if (ret == -ENOSPC && num_bytes > min_alloc_size) {
		num_bytes = num_bytes >> 1;
J
Josef Bacik 已提交
5379
		num_bytes = num_bytes & ~(root->sectorsize - 1);
5380
		num_bytes = max(num_bytes, min_alloc_size);
C
Chris Mason 已提交
5381
		do_chunk_alloc(trans, root->fs_info->extent_root,
5382
			       num_bytes, data, CHUNK_ALLOC_FORCE);
5383 5384
		goto again;
	}
5385
	if (ret == -ENOSPC && btrfs_test_opt(root, ENOSPC_DEBUG)) {
J
Josef Bacik 已提交
5386 5387 5388
		struct btrfs_space_info *sinfo;

		sinfo = __find_space_info(root->fs_info, data);
C
Chris Mason 已提交
5389 5390 5391
		printk(KERN_ERR "btrfs allocation failed flags %llu, "
		       "wanted %llu\n", (unsigned long long)data,
		       (unsigned long long)num_bytes);
J
Josef Bacik 已提交
5392
		dump_space_info(sinfo, num_bytes, 1);
5393
	}
J
Josef Bacik 已提交
5394

5395 5396
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);

J
Josef Bacik 已提交
5397
	return ret;
5398 5399
}

5400 5401
int btrfs_free_reserved_extent(struct btrfs_root *root, u64 start, u64 len)
{
J
Josef Bacik 已提交
5402
	struct btrfs_block_group_cache *cache;
5403
	int ret = 0;
J
Josef Bacik 已提交
5404 5405 5406

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
C
Chris Mason 已提交
5407 5408
		printk(KERN_ERR "Unable to find block group for %llu\n",
		       (unsigned long long)start);
J
Josef Bacik 已提交
5409 5410
		return -ENOSPC;
	}
5411

5412 5413
	if (btrfs_test_opt(root, DISCARD))
		ret = btrfs_discard_extent(root, start, len, NULL);
5414

J
Josef Bacik 已提交
5415
	btrfs_add_free_space(cache, start, len);
5416
	btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
5417
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
5418

5419 5420
	trace_btrfs_reserved_extent_free(root, start, len);

5421 5422 5423
	return ret;
}

5424 5425 5426 5427 5428
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
5429 5430
{
	int ret;
5431
	struct btrfs_fs_info *fs_info = root->fs_info;
5432
	struct btrfs_extent_item *extent_item;
5433
	struct btrfs_extent_inline_ref *iref;
5434
	struct btrfs_path *path;
5435 5436 5437
	struct extent_buffer *leaf;
	int type;
	u32 size;
5438

5439 5440 5441 5442
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
5443

5444
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
5445 5446

	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
5447 5448
	if (!path)
		return -ENOMEM;
5449

5450
	path->leave_spinning = 1;
5451 5452
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
C
Chris Mason 已提交
5453
	BUG_ON(ret);
J
Josef Bacik 已提交
5454

5455 5456
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
5457
				     struct btrfs_extent_item);
5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
5478 5479

	btrfs_mark_buffer_dirty(path->nodes[0]);
5480
	btrfs_free_path(path);
5481

5482
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5483
	if (ret) {
C
Chris Mason 已提交
5484 5485 5486
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
5487 5488
		BUG();
	}
5489 5490 5491
	return ret;
}

5492 5493 5494 5495 5496
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins)
5497 5498
{
	int ret;
5499 5500 5501 5502 5503 5504 5505
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
5506

5507
	path = btrfs_alloc_path();
5508 5509
	if (!path)
		return -ENOMEM;
5510

5511 5512 5513
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
5514
	BUG_ON(ret);
5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
	block_info = (struct btrfs_tree_block_info *)(extent_item + 1);

	btrfs_set_tree_block_key(leaf, block_info, key);
	btrfs_set_tree_block_level(leaf, block_info, level);

	iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

5543
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564
	if (ret) {
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
		BUG();
	}
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

	ret = btrfs_add_delayed_data_ref(trans, ins->objectid, ins->offset,
					 0, root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL);
5565 5566
	return ret;
}
5567 5568 5569 5570 5571 5572

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
5573 5574 5575 5576
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
5577 5578 5579
{
	int ret;
	struct btrfs_block_group_cache *block_group;
5580 5581 5582
	struct btrfs_caching_control *caching_ctl;
	u64 start = ins->objectid;
	u64 num_bytes = ins->offset;
5583 5584

	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
5585
	cache_block_group(block_group, trans, NULL, 0);
5586
	caching_ctl = get_caching_control(block_group);
5587

5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618
	if (!caching_ctl) {
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
		BUG_ON(ret);
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
			BUG_ON(ret);
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			BUG_ON(ret);
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
			BUG_ON(ret);

			start = caching_ctl->progress;
			num_bytes = ins->objectid + ins->offset -
				    caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
			BUG_ON(ret);
		}

		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}

5619 5620
	ret = btrfs_update_reserved_bytes(block_group, ins->offset,
					  RESERVE_ALLOC_NO_ACCOUNT);
5621
	BUG_ON(ret);
5622
	btrfs_put_block_group(block_group);
5623 5624
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
5625 5626 5627
	return ret;
}

5628 5629
struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
					    struct btrfs_root *root,
5630 5631
					    u64 bytenr, u32 blocksize,
					    int level)
5632 5633 5634 5635 5636 5637 5638
{
	struct extent_buffer *buf;

	buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
5639
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
5640 5641
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
5642 5643

	btrfs_set_lock_blocking(buf);
5644
	btrfs_set_buffer_uptodate(buf);
5645

5646
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
5647 5648 5649 5650 5651 5652 5653 5654 5655 5656
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
		if (root->log_transid % 2 == 0)
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
5657 5658
	} else {
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
5659
			 buf->start + buf->len - 1, GFP_NOFS);
5660
	}
5661
	trans->blocks_used++;
5662
	/* this returns a buffer locked for blocking */
5663 5664 5665
	return buf;
}

5666 5667 5668 5669 5670
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
5671
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
5672 5673 5674 5675 5676
	int ret;

	block_rsv = get_block_rsv(trans, root);

	if (block_rsv->size == 0) {
5677 5678
		ret = reserve_metadata_bytes(trans, root, block_rsv,
					     blocksize, 0);
5679 5680 5681 5682 5683 5684 5685 5686
		/*
		 * If we couldn't reserve metadata bytes try and use some from
		 * the global reserve.
		 */
		if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
5687
			return ERR_PTR(ret);
5688
		} else if (ret) {
5689
			return ERR_PTR(ret);
5690
		}
5691 5692 5693 5694 5695 5696
		return block_rsv;
	}

	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;
5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711
	if (ret) {
		WARN_ON(1);
		ret = reserve_metadata_bytes(trans, root, block_rsv, blocksize,
					     0);
		if (!ret) {
			spin_lock(&block_rsv->lock);
			block_rsv->size += blocksize;
			spin_unlock(&block_rsv->lock);
			return block_rsv;
		} else if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
		}
	}
5712 5713 5714 5715 5716 5717 5718 5719 5720 5721

	return ERR_PTR(-ENOSPC);
}

static void unuse_block_rsv(struct btrfs_block_rsv *block_rsv, u32 blocksize)
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
	block_rsv_release_bytes(block_rsv, NULL, 0);
}

5722
/*
5723 5724 5725 5726
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
5727 5728
 * returns the tree buffer or NULL.
 */
5729
struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
5730 5731 5732 5733
					struct btrfs_root *root, u32 blocksize,
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
					u64 hint, u64 empty_size)
5734
{
C
Chris Mason 已提交
5735
	struct btrfs_key ins;
5736
	struct btrfs_block_rsv *block_rsv;
5737
	struct extent_buffer *buf;
5738 5739 5740
	u64 flags = 0;
	int ret;

5741

5742 5743 5744 5745 5746 5747
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

	ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
				   empty_size, hint, (u64)-1, &ins, 0);
5748
	if (ret) {
5749
		unuse_block_rsv(block_rsv, blocksize);
5750
		return ERR_PTR(ret);
5751
	}
5752

5753 5754
	buf = btrfs_init_new_buffer(trans, root, ins.objectid,
				    blocksize, level);
5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782
	BUG_ON(IS_ERR(buf));

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
		extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
		BUG_ON(!extent_op);
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
		extent_op->update_key = 1;
		extent_op->update_flags = 1;
		extent_op->is_data = 0;

		ret = btrfs_add_delayed_tree_ref(trans, ins.objectid,
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
					extent_op);
		BUG_ON(ret);
	}
5783 5784
	return buf;
}
5785

5786 5787 5788 5789 5790 5791 5792 5793 5794
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
5795 5796
	int reada_slot;
	int reada_count;
5797 5798 5799 5800 5801
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
5802 5803 5804 5805
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
5806
{
Y
Yan, Zheng 已提交
5807 5808 5809
	u64 bytenr;
	u64 generation;
	u64 refs;
5810
	u64 flags;
5811
	u32 nritems;
Y
Yan, Zheng 已提交
5812 5813 5814
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
5815
	int ret;
Y
Yan, Zheng 已提交
5816 5817
	int slot;
	int nread = 0;
5818

Y
Yan, Zheng 已提交
5819 5820 5821 5822 5823 5824 5825 5826
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
5827

Y
Yan, Zheng 已提交
5828 5829 5830
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
	blocksize = btrfs_level_size(root, wc->level - 1);
5831

Y
Yan, Zheng 已提交
5832 5833 5834
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
5835

C
Chris Mason 已提交
5836
		cond_resched();
Y
Yan, Zheng 已提交
5837 5838
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
5839

Y
Yan, Zheng 已提交
5840 5841
		if (slot == path->slots[wc->level])
			goto reada;
5842

Y
Yan, Zheng 已提交
5843 5844
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
5845 5846
			continue;

5847 5848 5849 5850 5851 5852
		/* We don't lock the tree block, it's OK to be racy here */
		ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
					       &refs, &flags);
		BUG_ON(ret);
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
5853 5854 5855
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
5856

5857 5858 5859
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
5860 5861 5862 5863 5864 5865 5866 5867
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
5868 5869 5870 5871
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
5872
		}
Y
Yan, Zheng 已提交
5873 5874 5875 5876
reada:
		ret = readahead_tree_block(root, bytenr, blocksize,
					   generation);
		if (ret)
5877
			break;
Y
Yan, Zheng 已提交
5878
		nread++;
C
Chris Mason 已提交
5879
	}
Y
Yan, Zheng 已提交
5880
	wc->reada_slot = slot;
C
Chris Mason 已提交
5881
}
5882

Y
Yan Zheng 已提交
5883
/*
5884 5885 5886 5887 5888 5889
 * hepler to process tree block while walking down the tree.
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
5890
 */
5891
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
5892
				   struct btrfs_root *root,
5893
				   struct btrfs_path *path,
5894
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
5895
{
5896 5897 5898
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
5899 5900
	int ret;

5901 5902 5903
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
5904

5905 5906 5907 5908
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
5909 5910 5911
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
5912 5913 5914 5915 5916 5917 5918 5919
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
					       eb->start, eb->len,
					       &wc->refs[level],
					       &wc->flags[level]);
		BUG_ON(ret);
		BUG_ON(wc->refs[level] == 0);
	}
5920

5921 5922 5923
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
5924

5925
		if (path->locks[level] && !wc->keep_locks) {
5926
			btrfs_tree_unlock_rw(eb, path->locks[level]);
5927 5928 5929 5930
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
5931

5932 5933 5934 5935
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
		ret = btrfs_inc_ref(trans, root, eb, 1);
Y
Yan Zheng 已提交
5936
		BUG_ON(ret);
5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949
		ret = btrfs_dec_ref(trans, root, eb, 0);
		BUG_ON(ret);
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
						  eb->len, flag, 0);
		BUG_ON(ret);
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
5950
		btrfs_tree_unlock_rw(eb, path->locks[level]);
5951 5952 5953 5954 5955
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971
/*
 * hepler to process tree block pointer.
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
5972
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
5992 5993
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
5994
		return 1;
5995
	}
Y
Yan, Zheng 已提交
5996 5997 5998 5999 6000 6001 6002

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
	blocksize = btrfs_level_size(root, level - 1);

	next = btrfs_find_tree_block(root, bytenr, blocksize);
	if (!next) {
		next = btrfs_find_create_tree_block(root, bytenr, blocksize);
6003 6004
		if (!next)
			return -ENOMEM;
Y
Yan, Zheng 已提交
6005 6006 6007 6008 6009
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

6010 6011 6012 6013 6014 6015
	ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
	BUG_ON(ret);
	BUG_ON(wc->refs[level - 1] == 0);
	*lookup_info = 0;
Y
Yan, Zheng 已提交
6016

6017
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
6018
		if (wc->refs[level - 1] > 1) {
6019 6020 6021 6022
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
6036 6037 6038 6039
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
6040 6041 6042 6043 6044 6045
	}

	if (!btrfs_buffer_uptodate(next, generation)) {
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
6046
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6047 6048 6049 6050 6051 6052
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
		next = read_tree_block(root, bytenr, blocksize, generation);
6053 6054
		if (!next)
			return -EIO;
Y
Yan, Zheng 已提交
6055 6056 6057 6058 6059 6060 6061 6062
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
6063
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Y
Yan, Zheng 已提交
6064 6065 6066 6067 6068 6069 6070
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
6071 6072 6073 6074 6075 6076 6077 6078
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
6079

6080 6081 6082
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
					root->root_key.objectid, level - 1, 0);
		BUG_ON(ret);
Y
Yan, Zheng 已提交
6083 6084 6085
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
6086
	*lookup_info = 1;
Y
Yan, Zheng 已提交
6087 6088 6089
	return 1;
}

6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106
/*
 * hepler to process tree block while walking up the tree.
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
6107
	int ret;
6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6134
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6135 6136 6137 6138 6139

			ret = btrfs_lookup_extent_info(trans, root,
						       eb->start, eb->len,
						       &wc->refs[level],
						       &wc->flags[level]);
Y
Yan Zheng 已提交
6140
			BUG_ON(ret);
6141 6142
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
6143
				btrfs_tree_unlock_rw(eb, path->locks[level]);
6144 6145
				return 1;
			}
Y
Yan Zheng 已提交
6146
		}
6147
	}
Y
Yan Zheng 已提交
6148

6149 6150
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6151

6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
				ret = btrfs_dec_ref(trans, root, eb, 1);
			else
				ret = btrfs_dec_ref(trans, root, eb, 0);
			BUG_ON(ret);
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6165
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
6182 6183
	}

6184
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
6185 6186 6187
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
6188
	return 0;
6189 6190 6191 6192 6193 6194 6195 6196
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
6197
	int lookup_info = 1;
6198 6199 6200
	int ret;

	while (level >= 0) {
6201
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
6202 6203 6204 6205 6206 6207
		if (ret > 0)
			break;

		if (level == 0)
			break;

6208 6209 6210 6211
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

6212
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
6213 6214 6215
		if (ret > 0) {
			path->slots[level]++;
			continue;
6216 6217
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
6218
		level = wc->level;
Y
Yan Zheng 已提交
6219 6220 6221 6222
	}
	return 0;
}

C
Chris Mason 已提交
6223
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6224
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
6225
				 struct btrfs_path *path,
6226
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
6227
{
6228
	int level = wc->level;
C
Chris Mason 已提交
6229
	int ret;
6230

6231 6232 6233 6234 6235 6236
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
6237 6238
			return 0;
		} else {
6239 6240 6241
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
6242

6243
			if (path->locks[level]) {
6244 6245
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
6246
				path->locks[level] = 0;
Y
Yan Zheng 已提交
6247
			}
6248 6249 6250
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
6251 6252 6253 6254 6255
		}
	}
	return 1;
}

C
Chris Mason 已提交
6256
/*
6257 6258 6259 6260 6261 6262 6263 6264 6265
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
C
Chris Mason 已提交
6266
 */
6267 6268
void btrfs_drop_snapshot(struct btrfs_root *root,
			 struct btrfs_block_rsv *block_rsv, int update_ref)
C
Chris Mason 已提交
6269
{
6270
	struct btrfs_path *path;
6271 6272
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
6273
	struct btrfs_root_item *root_item = &root->root_item;
6274 6275 6276 6277 6278
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
C
Chris Mason 已提交
6279

6280
	path = btrfs_alloc_path();
6281 6282 6283 6284
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
C
Chris Mason 已提交
6285

6286
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
6287 6288
	if (!wc) {
		btrfs_free_path(path);
6289 6290
		err = -ENOMEM;
		goto out;
6291
	}
6292

6293
	trans = btrfs_start_transaction(tree_root, 0);
6294 6295
	BUG_ON(IS_ERR(trans));

6296 6297
	if (block_rsv)
		trans->block_rsv = block_rsv;
6298

6299
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
6300
		level = btrfs_header_level(root->node);
6301 6302
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
6303
		path->slots[level] = 0;
6304
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6305 6306
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
6307 6308
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
6309 6310 6311
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

6312
		level = root_item->drop_level;
6313
		BUG_ON(level == 0);
6314
		path->lowest_level = level;
6315 6316 6317 6318
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
6319
			goto out_free;
6320
		}
Y
Yan, Zheng 已提交
6321
		WARN_ON(ret > 0);
6322

6323 6324 6325 6326
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
6327
		btrfs_unlock_up_safe(path, 0);
6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
						path->nodes[level]->len,
						&wc->refs[level],
						&wc->flags[level]);
			BUG_ON(ret);
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
6349
	}
6350 6351 6352 6353 6354 6355

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
Y
Yan, Zheng 已提交
6356
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
6357

C
Chris Mason 已提交
6358
	while (1) {
6359 6360 6361
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6362
			break;
6363
		}
C
Chris Mason 已提交
6364

6365 6366 6367
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6368
			break;
6369 6370 6371 6372
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
6373 6374
			break;
		}
6375 6376 6377 6378 6379 6380 6381 6382 6383 6384

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
6385
		if (btrfs_should_end_transaction(trans, tree_root)) {
6386 6387 6388 6389 6390
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
			BUG_ON(ret);

6391
			btrfs_end_transaction_throttle(trans, tree_root);
6392
			trans = btrfs_start_transaction(tree_root, 0);
6393
			BUG_ON(IS_ERR(trans));
6394 6395
			if (block_rsv)
				trans->block_rsv = block_rsv;
6396
		}
C
Chris Mason 已提交
6397
	}
6398
	btrfs_release_path(path);
6399 6400 6401 6402 6403
	BUG_ON(err);

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
	BUG_ON(ret);

6404 6405 6406 6407 6408
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
		ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
					   NULL, NULL);
		BUG_ON(ret < 0);
		if (ret > 0) {
6409 6410 6411 6412 6413 6414 6415
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
6416 6417 6418 6419 6420 6421 6422 6423 6424 6425
		}
	}

	if (root->in_radix) {
		btrfs_free_fs_root(tree_root->fs_info, root);
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
		kfree(root);
	}
6426
out_free:
6427
	btrfs_end_transaction_throttle(trans, tree_root);
6428
	kfree(wc);
6429
	btrfs_free_path(path);
6430 6431 6432 6433
out:
	if (err)
		btrfs_std_error(root->fs_info, err);
	return;
C
Chris Mason 已提交
6434
}
C
Chris Mason 已提交
6435

6436 6437 6438 6439 6440
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
 */
Y
Yan Zheng 已提交
6441 6442 6443 6444 6445 6446
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
6447
	struct walk_control *wc;
Y
Yan Zheng 已提交
6448 6449 6450 6451 6452
	int level;
	int parent_level;
	int ret = 0;
	int wret;

6453 6454
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
6455
	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
6456 6457
	if (!path)
		return -ENOMEM;
Y
Yan Zheng 已提交
6458

6459
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
T
Tsutomu Itoh 已提交
6460 6461 6462 6463
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
6464

6465
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
6466 6467 6468 6469 6470
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

6471
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
6472 6473 6474
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
6475
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6476 6477 6478 6479 6480 6481 6482 6483

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
Y
Yan, Zheng 已提交
6484
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
6485 6486

	while (1) {
6487 6488
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
6489 6490
			ret = wret;
			break;
6491
		}
Y
Yan Zheng 已提交
6492

6493
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
6494 6495 6496 6497 6498 6499
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

6500
	kfree(wc);
Y
Yan Zheng 已提交
6501 6502 6503 6504
	btrfs_free_path(path);
	return ret;
}

6505 6506 6507 6508 6509 6510
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
	u64 stripped = BTRFS_BLOCK_GROUP_RAID0 |
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

6511 6512 6513 6514 6515 6516 6517 6518
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;

6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
		return flags;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

		/* turn single device chunks into raid0 */
		return stripped | BTRFS_BLOCK_GROUP_RAID0;
	}
	return flags;
}

6550
static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
C
Chris Mason 已提交
6551
{
6552 6553
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
6554
	u64 min_allocable_bytes;
6555
	int ret = -ENOSPC;
C
Chris Mason 已提交
6556

6557

6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
		min_allocable_bytes = 1 * 1024 * 1024;
	else
		min_allocable_bytes = 0;

6570 6571
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
6572 6573 6574 6575 6576 6577

	if (cache->ro) {
		ret = 0;
		goto out;
	}

6578 6579 6580 6581
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
6582 6583
	    sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
	    min_allocable_bytes <= sinfo->total_bytes) {
6584 6585 6586 6587
		sinfo->bytes_readonly += num_bytes;
		cache->ro = 1;
		ret = 0;
	}
6588
out:
6589 6590 6591 6592
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
6593

6594 6595
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
6596

6597 6598 6599 6600
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
6601

6602
	BUG_ON(cache->ro);
C
Chris Mason 已提交
6603

C
Chris Mason 已提交
6604
	trans = btrfs_join_transaction(root);
6605
	BUG_ON(IS_ERR(trans));
6606

6607 6608
	alloc_flags = update_block_group_flags(root, cache->flags);
	if (alloc_flags != cache->flags)
6609 6610
		do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			       CHUNK_ALLOC_FORCE);
6611

6612
	ret = set_block_group_ro(cache, 0);
6613 6614 6615
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
6616 6617
	ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			     CHUNK_ALLOC_FORCE);
6618 6619
	if (ret < 0)
		goto out;
6620
	ret = set_block_group_ro(cache, 0);
6621 6622 6623 6624
out:
	btrfs_end_transaction(trans, root);
	return ret;
}
6625

6626 6627 6628 6629
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
6630 6631
	return do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			      CHUNK_ALLOC_FORCE);
6632 6633
}

6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689
/*
 * helper to account the unused space of all the readonly block group in the
 * list. takes mirrors into account.
 */
static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

	list_for_each_entry(block_group, groups_list, list) {
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}

	return free_bytes;
}

/*
 * helper to account the unused space of all the readonly block group in the
 * space_info. takes mirrors into account.
 */
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
{
	int i;
	u64 free_bytes = 0;

	spin_lock(&sinfo->lock);

	for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		if (!list_empty(&sinfo->block_groups[i]))
			free_bytes += __btrfs_get_ro_block_group_free_space(
						&sinfo->block_groups[i]);

	spin_unlock(&sinfo->lock);

	return free_bytes;
}

6690 6691
int btrfs_set_block_group_rw(struct btrfs_root *root,
			      struct btrfs_block_group_cache *cache)
6692
{
6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
6706 6707 6708
	return 0;
}

6709 6710 6711 6712 6713 6714 6715
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
6716
{
6717 6718 6719 6720
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
6721
	u64 min_free;
J
Josef Bacik 已提交
6722 6723
	u64 dev_min = 1;
	u64 dev_nr = 0;
6724
	int index;
6725 6726
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
6727

6728
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
6729

6730 6731 6732
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
Z
Zheng Yan 已提交
6733

6734 6735
	min_free = btrfs_block_group_used(&block_group->item);

6736
	/* no bytes used, we're good */
6737
	if (!min_free)
Z
Zheng Yan 已提交
6738 6739
		goto out;

6740 6741
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
6742

6743
	full = space_info->full;
6744

6745 6746
	/*
	 * if this is the last block group we have in this space, we can't
6747 6748 6749 6750
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
6751
	 */
6752
	if ((space_info->total_bytes != block_group->key.offset) &&
6753 6754 6755
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
6756 6757
		spin_unlock(&space_info->lock);
		goto out;
6758
	}
6759
	spin_unlock(&space_info->lock);
6760

6761 6762 6763 6764 6765 6766 6767 6768 6769 6770
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
	 * alloc devices and guess if we have enough space.  However, if we
	 * were marked as full, then we know there aren't enough chunks, and we
	 * can just return.
	 */
	ret = -1;
	if (full)
		goto out;
6771

6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
	index = get_block_group_index(block_group);
	if (index == 0) {
		dev_min = 4;
J
Josef Bacik 已提交
6783 6784
		/* Divide by 2 */
		min_free >>= 1;
6785 6786 6787
	} else if (index == 1) {
		dev_min = 2;
	} else if (index == 2) {
J
Josef Bacik 已提交
6788 6789
		/* Multiply by 2 */
		min_free <<= 1;
6790 6791
	} else if (index == 3) {
		dev_min = fs_devices->rw_devices;
J
Josef Bacik 已提交
6792
		do_div(min_free, dev_min);
6793 6794
	}

6795 6796
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
6797
		u64 dev_offset;
6798

6799 6800 6801 6802 6803 6804
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
		if (device->total_bytes > device->bytes_used + min_free) {
			ret = find_free_dev_extent(NULL, device, min_free,
6805
						   &dev_offset, NULL);
6806
			if (!ret)
6807 6808 6809
				dev_nr++;

			if (dev_nr >= dev_min)
6810
				break;
6811

6812
			ret = -1;
6813
		}
6814
	}
6815
	mutex_unlock(&root->fs_info->chunk_mutex);
6816
out:
6817
	btrfs_put_block_group(block_group);
6818 6819 6820
	return ret;
}

6821 6822
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
6823
{
6824
	int ret = 0;
6825 6826 6827
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
6828

6829 6830
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
6831 6832
		goto out;

C
Chris Mason 已提交
6833
	while (1) {
6834
		slot = path->slots[0];
6835
		leaf = path->nodes[0];
6836 6837 6838 6839 6840
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
6841
				goto out;
6842
			break;
6843
		}
6844
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
6845

6846
		if (found_key.objectid >= key->objectid &&
6847 6848 6849 6850
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
6851
		path->slots[0]++;
6852
	}
6853
out:
6854
	return ret;
6855 6856
}

6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
6891 6892 6893
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
6894
	struct btrfs_space_info *space_info;
6895
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
6896 6897
	struct rb_node *n;

6898 6899 6900 6901 6902 6903 6904 6905 6906
	down_write(&info->extent_commit_sem);
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
	up_write(&info->extent_commit_sem);

Z
Zheng Yan 已提交
6907 6908 6909 6910 6911 6912
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
Y
Yan Zheng 已提交
6913 6914
		spin_unlock(&info->block_group_cache_lock);

6915
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
6916
		list_del(&block_group->list);
6917
		up_write(&block_group->space_info->groups_sem);
6918

J
Josef Bacik 已提交
6919
		if (block_group->cached == BTRFS_CACHE_STARTED)
6920
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
6921

6922 6923 6924 6925 6926 6927 6928
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
		if (block_group->cached == BTRFS_CACHE_NO)
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
6929
		btrfs_remove_free_space_cache(block_group);
6930
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
6931 6932

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
6933 6934
	}
	spin_unlock(&info->block_group_cache_lock);
6935 6936 6937 6938 6939 6940 6941 6942 6943

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

6944 6945
	release_global_block_rsv(info);

6946 6947 6948 6949
	while(!list_empty(&info->space_info)) {
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
6950
		if (space_info->bytes_pinned > 0 ||
6951 6952
		    space_info->bytes_reserved > 0 ||
		    space_info->bytes_may_use > 0) {
6953 6954 6955
			WARN_ON(1);
			dump_space_info(space_info, 0, 0);
		}
6956 6957 6958
		list_del(&space_info->list);
		kfree(space_info);
	}
Z
Zheng Yan 已提交
6959 6960 6961
	return 0;
}

6962 6963 6964 6965 6966 6967 6968 6969 6970 6971
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);

	down_write(&space_info->groups_sem);
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);
}

C
Chris Mason 已提交
6972 6973 6974 6975 6976
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
6977
	struct btrfs_fs_info *info = root->fs_info;
6978
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
6979 6980
	struct btrfs_key key;
	struct btrfs_key found_key;
6981
	struct extent_buffer *leaf;
6982 6983
	int need_clear = 0;
	u64 cache_gen;
6984

C
Chris Mason 已提交
6985
	root = info->extent_root;
C
Chris Mason 已提交
6986
	key.objectid = 0;
6987
	key.offset = 0;
C
Chris Mason 已提交
6988 6989 6990 6991
	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
J
Josef Bacik 已提交
6992
	path->reada = 1;
C
Chris Mason 已提交
6993

6994 6995 6996 6997
	cache_gen = btrfs_super_cache_generation(&root->fs_info->super_copy);
	if (cache_gen != 0 &&
	    btrfs_super_generation(&root->fs_info->super_copy) != cache_gen)
		need_clear = 1;
6998 6999
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
7000 7001
	if (!btrfs_test_opt(root, SPACE_CACHE) && cache_gen)
		printk(KERN_INFO "btrfs: disk space caching is enabled\n");
7002

C
Chris Mason 已提交
7003
	while (1) {
7004
		ret = find_first_block_group(root, path, &key);
7005 7006
		if (ret > 0)
			break;
7007 7008
		if (ret != 0)
			goto error;
7009 7010
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7011
		cache = kzalloc(sizeof(*cache), GFP_NOFS);
C
Chris Mason 已提交
7012
		if (!cache) {
7013
			ret = -ENOMEM;
7014
			goto error;
C
Chris Mason 已提交
7015
		}
7016 7017 7018 7019 7020 7021 7022
		cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
						GFP_NOFS);
		if (!cache->free_space_ctl) {
			kfree(cache);
			ret = -ENOMEM;
			goto error;
		}
C
Chris Mason 已提交
7023

7024
		atomic_set(&cache->count, 1);
7025
		spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7026
		cache->fs_info = info;
J
Josef Bacik 已提交
7027
		INIT_LIST_HEAD(&cache->list);
7028
		INIT_LIST_HEAD(&cache->cluster_list);
7029

7030 7031 7032
		if (need_clear)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

7033 7034 7035
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
C
Chris Mason 已提交
7036
		memcpy(&cache->key, &found_key, sizeof(found_key));
7037

C
Chris Mason 已提交
7038
		key.objectid = found_key.objectid + found_key.offset;
7039
		btrfs_release_path(path);
7040
		cache->flags = btrfs_block_group_flags(&cache->item);
J
Josef Bacik 已提交
7041 7042
		cache->sectorsize = root->sectorsize;

7043 7044
		btrfs_init_free_space_ctl(cache);

7045 7046 7047 7048 7049 7050 7051
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
		exclude_super_stripes(root, cache);

J
Josef Bacik 已提交
7052 7053 7054 7055 7056 7057 7058 7059
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
7060
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7061
			cache->cached = BTRFS_CACHE_FINISHED;
7062
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7063
		} else if (btrfs_block_group_used(&cache->item) == 0) {
7064
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7065 7066 7067 7068 7069
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
7070
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7071
		}
7072

7073 7074 7075 7076 7077
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
		BUG_ON(ret);
		cache->space_info = space_info;
7078
		spin_lock(&cache->space_info->lock);
7079
		cache->space_info->bytes_readonly += cache->bytes_super;
7080 7081
		spin_unlock(&cache->space_info->lock);

7082
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
7083 7084 7085

		ret = btrfs_add_block_group_cache(root->fs_info, cache);
		BUG_ON(ret);
7086 7087

		set_avail_alloc_bits(root->fs_info, cache->flags);
Y
Yan Zheng 已提交
7088
		if (btrfs_chunk_readonly(root, cache->key.objectid))
7089
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7090
	}
7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
		list_for_each_entry(cache, &space_info->block_groups[3], list)
7103
			set_block_group_ro(cache, 1);
7104
		list_for_each_entry(cache, &space_info->block_groups[4], list)
7105
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7106
	}
7107 7108

	init_global_block_rsv(info);
7109 7110
	ret = 0;
error:
C
Chris Mason 已提交
7111
	btrfs_free_path(path);
7112
	return ret;
C
Chris Mason 已提交
7113
}
7114 7115 7116

int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
7117
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
7118 7119 7120 7121 7122 7123 7124 7125
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

7126
	root->fs_info->last_trans_log_full_commit = trans->transid;
7127

7128
	cache = kzalloc(sizeof(*cache), GFP_NOFS);
J
Josef Bacik 已提交
7129 7130
	if (!cache)
		return -ENOMEM;
7131 7132 7133 7134 7135 7136
	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return -ENOMEM;
	}
J
Josef Bacik 已提交
7137

7138
	cache->key.objectid = chunk_offset;
7139
	cache->key.offset = size;
7140
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
7141
	cache->sectorsize = root->sectorsize;
7142
	cache->fs_info = root->fs_info;
7143

7144
	atomic_set(&cache->count, 1);
7145
	spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7146
	INIT_LIST_HEAD(&cache->list);
7147
	INIT_LIST_HEAD(&cache->cluster_list);
C
Chris Mason 已提交
7148

7149 7150
	btrfs_init_free_space_ctl(cache);

7151 7152 7153 7154 7155
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	cache->flags = type;
	btrfs_set_block_group_flags(&cache->item, type);

7156
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7157
	cache->cached = BTRFS_CACHE_FINISHED;
7158
	exclude_super_stripes(root, cache);
7159

J
Josef Bacik 已提交
7160 7161 7162
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

7163 7164
	free_excluded_extents(root, cache);

7165 7166 7167
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
	BUG_ON(ret);
7168 7169

	spin_lock(&cache->space_info->lock);
7170
	cache->space_info->bytes_readonly += cache->bytes_super;
7171 7172
	spin_unlock(&cache->space_info->lock);

7173
	__link_block_group(cache->space_info, cache);
7174

J
Josef Bacik 已提交
7175 7176
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
	BUG_ON(ret);
7177

7178 7179 7180 7181
	ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
				sizeof(cache->item));
	BUG_ON(ret);

C
Chris Mason 已提交
7182
	set_avail_alloc_bits(extent_root->fs_info, type);
7183

7184 7185
	return 0;
}
Z
Zheng Yan 已提交
7186 7187 7188 7189 7190 7191

int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 group_start)
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
7192
	struct btrfs_free_cluster *cluster;
7193
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
7194
	struct btrfs_key key;
7195
	struct inode *inode;
Z
Zheng Yan 已提交
7196
	int ret;
J
Josef Bacik 已提交
7197
	int factor;
Z
Zheng Yan 已提交
7198 7199 7200 7201 7202

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
7203
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
7204

7205 7206 7207 7208 7209 7210
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

Z
Zheng Yan 已提交
7211
	memcpy(&key, &block_group->key, sizeof(key));
J
Josef Bacik 已提交
7212 7213 7214 7215 7216 7217
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
7218

7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
7234
	path = btrfs_alloc_path();
7235 7236 7237 7238
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
Z
Zheng Yan 已提交
7239

7240 7241
	inode = lookup_free_space_inode(root, block_group, path);
	if (!IS_ERR(inode)) {
7242 7243
		ret = btrfs_orphan_add(trans, inode);
		BUG_ON(ret);
7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
		iput(inode);
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
7267
		btrfs_release_path(path);
7268 7269 7270 7271
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
7272
		btrfs_release_path(path);
7273 7274
	}

7275
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
7276 7277
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
7278
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
7279

7280
	down_write(&block_group->space_info->groups_sem);
7281 7282 7283 7284 7285
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
7286
	up_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
7287

J
Josef Bacik 已提交
7288
	if (block_group->cached == BTRFS_CACHE_STARTED)
7289
		wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
7290 7291 7292

	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
7293 7294 7295
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
7296
	block_group->space_info->disk_total -= block_group->key.offset * factor;
Y
Yan Zheng 已提交
7297
	spin_unlock(&block_group->space_info->lock);
7298

7299 7300
	memcpy(&key, &block_group->key, sizeof(key));

7301
	btrfs_clear_space_info_full(root->fs_info);
Y
Yan Zheng 已提交
7302

7303 7304
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
7317

7318 7319 7320
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
7321 7322 7323 7324
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
7325 7326
	int ret;

7327 7328 7329
	disk_super = &fs_info->super_copy;
	if (!btrfs_super_root(disk_super))
		return 1;
7330

7331 7332 7333
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
7334

7335 7336
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
	ret = update_space_info(fs_info, flags, 0, 0, &space_info);
7337
	if (ret)
7338
		goto out;
7339

7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	}
out:
7353 7354 7355
	return ret;
}

L
liubo 已提交
7356 7357 7358 7359 7360 7361
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
	return unpin_extent_range(root, start, end);
}

int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
7362
			       u64 num_bytes, u64 *actual_bytes)
L
liubo 已提交
7363
{
7364
	return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
L
liubo 已提交
7365
}
7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413

int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
	int ret = 0;

	cache = btrfs_lookup_block_group(fs_info, range->start);

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
				ret = cache_block_group(cache, NULL, root, 0);
				if (!ret)
					wait_block_group_cache_done(cache);
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

	range->len = trimmed;
	return ret;
}