extent-tree.c 213.0 KB
Newer Older
C
Chris Mason 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (C) 2007 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
Z
Zach Brown 已提交
18
#include <linux/sched.h>
19
#include <linux/pagemap.h>
20
#include <linux/writeback.h>
21
#include <linux/blkdev.h>
22
#include <linux/sort.h>
23
#include <linux/rcupdate.h>
J
Josef Bacik 已提交
24
#include <linux/kthread.h>
25
#include <linux/slab.h>
26
#include <linux/ratelimit.h>
C
Chris Mason 已提交
27
#include "compat.h"
28
#include "hash.h"
29 30 31
#include "ctree.h"
#include "disk-io.h"
#include "print-tree.h"
32
#include "transaction.h"
33
#include "volumes.h"
34
#include "locking.h"
35
#include "free-space-cache.h"
36

37 38
/*
 * control flags for do_chunk_alloc's force field
39 40 41 42 43 44 45 46 47
 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
 * if we really need one.
 *
 * CHUNK_ALLOC_LIMITED means to only try and allocate one
 * if we have very few chunks already allocated.  This is
 * used as part of the clustering code to help make sure
 * we have a good pool of storage to cluster in, without
 * filling the FS with empty chunks
 *
48 49
 * CHUNK_ALLOC_FORCE means it must try to allocate one
 *
50 51 52
 */
enum {
	CHUNK_ALLOC_NO_FORCE = 0,
53 54
	CHUNK_ALLOC_LIMITED = 1,
	CHUNK_ALLOC_FORCE = 2,
55 56
};

57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
/*
 * Control how reservations are dealt with.
 *
 * RESERVE_FREE - freeing a reservation.
 * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
 *   ENOSPC accounting
 * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
 *   bytes_may_use as the ENOSPC accounting is done elsewhere
 */
enum {
	RESERVE_FREE = 0,
	RESERVE_ALLOC = 1,
	RESERVE_ALLOC_NO_ACCOUNT = 2,
};

72 73
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
74
			      u64 bytenr, u64 num_bytes, int alloc);
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extra_op);
static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei);
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod);
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins);
J
Josef Bacik 已提交
94 95 96
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
			  u64 flags, int force);
97 98
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key);
J
Josef Bacik 已提交
99 100
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups);
101 102
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve);
J
Josef Bacik 已提交
103

J
Josef Bacik 已提交
104 105 106 107 108 109 110
static noinline int
block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	smp_mb();
	return cache->cached == BTRFS_CACHE_FINISHED;
}

J
Josef Bacik 已提交
111 112 113 114 115
static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
{
	return (cache->flags & bits) == bits;
}

116
static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
117 118 119 120 121 122
{
	atomic_inc(&cache->count);
}

void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
{
123 124 125
	if (atomic_dec_and_test(&cache->count)) {
		WARN_ON(cache->pinned > 0);
		WARN_ON(cache->reserved > 0);
126
		kfree(cache->free_space_ctl);
127
		kfree(cache);
128
	}
129 130
}

J
Josef Bacik 已提交
131 132 133 134
/*
 * this adds the block group to the fs_info rb tree for the block group
 * cache
 */
135
static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
J
Josef Bacik 已提交
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
				struct btrfs_block_group_cache *block_group)
{
	struct rb_node **p;
	struct rb_node *parent = NULL;
	struct btrfs_block_group_cache *cache;

	spin_lock(&info->block_group_cache_lock);
	p = &info->block_group_cache_tree.rb_node;

	while (*p) {
		parent = *p;
		cache = rb_entry(parent, struct btrfs_block_group_cache,
				 cache_node);
		if (block_group->key.objectid < cache->key.objectid) {
			p = &(*p)->rb_left;
		} else if (block_group->key.objectid > cache->key.objectid) {
			p = &(*p)->rb_right;
		} else {
			spin_unlock(&info->block_group_cache_lock);
			return -EEXIST;
		}
	}

	rb_link_node(&block_group->cache_node, parent, p);
	rb_insert_color(&block_group->cache_node,
			&info->block_group_cache_tree);
	spin_unlock(&info->block_group_cache_lock);

	return 0;
}

/*
 * This will return the block group at or after bytenr if contains is 0, else
 * it will return the block group that contains the bytenr
 */
static struct btrfs_block_group_cache *
block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
			      int contains)
{
	struct btrfs_block_group_cache *cache, *ret = NULL;
	struct rb_node *n;
	u64 end, start;

	spin_lock(&info->block_group_cache_lock);
	n = info->block_group_cache_tree.rb_node;

	while (n) {
		cache = rb_entry(n, struct btrfs_block_group_cache,
				 cache_node);
		end = cache->key.objectid + cache->key.offset - 1;
		start = cache->key.objectid;

		if (bytenr < start) {
			if (!contains && (!ret || start < ret->key.objectid))
				ret = cache;
			n = n->rb_left;
		} else if (bytenr > start) {
			if (contains && bytenr <= end) {
				ret = cache;
				break;
			}
			n = n->rb_right;
		} else {
			ret = cache;
			break;
		}
	}
203
	if (ret)
204
		btrfs_get_block_group(ret);
J
Josef Bacik 已提交
205 206 207 208 209
	spin_unlock(&info->block_group_cache_lock);

	return ret;
}

210 211
static int add_excluded_extent(struct btrfs_root *root,
			       u64 start, u64 num_bytes)
J
Josef Bacik 已提交
212
{
213 214 215 216 217 218 219
	u64 end = start + num_bytes - 1;
	set_extent_bits(&root->fs_info->freed_extents[0],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	set_extent_bits(&root->fs_info->freed_extents[1],
			start, end, EXTENT_UPTODATE, GFP_NOFS);
	return 0;
}
J
Josef Bacik 已提交
220

221 222 223 224
static void free_excluded_extents(struct btrfs_root *root,
				  struct btrfs_block_group_cache *cache)
{
	u64 start, end;
J
Josef Bacik 已提交
225

226 227 228 229 230 231 232
	start = cache->key.objectid;
	end = start + cache->key.offset - 1;

	clear_extent_bits(&root->fs_info->freed_extents[0],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
	clear_extent_bits(&root->fs_info->freed_extents[1],
			  start, end, EXTENT_UPTODATE, GFP_NOFS);
J
Josef Bacik 已提交
233 234
}

235 236
static int exclude_super_stripes(struct btrfs_root *root,
				 struct btrfs_block_group_cache *cache)
J
Josef Bacik 已提交
237 238 239 240 241 242
{
	u64 bytenr;
	u64 *logical;
	int stripe_len;
	int i, nr, ret;

243 244 245 246 247
	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
		cache->bytes_super += stripe_len;
		ret = add_excluded_extent(root, cache->key.objectid,
					  stripe_len);
248
		BUG_ON(ret); /* -ENOMEM */
249 250
	}

J
Josef Bacik 已提交
251 252 253 254 255
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		bytenr = btrfs_sb_offset(i);
		ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
				       cache->key.objectid, bytenr,
				       0, &logical, &nr, &stripe_len);
256
		BUG_ON(ret); /* -ENOMEM */
257

J
Josef Bacik 已提交
258
		while (nr--) {
259
			cache->bytes_super += stripe_len;
260 261
			ret = add_excluded_extent(root, logical[nr],
						  stripe_len);
262
			BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
263
		}
264

J
Josef Bacik 已提交
265 266 267 268 269
		kfree(logical);
	}
	return 0;
}

270 271 272 273 274 275 276 277 278 279 280
static struct btrfs_caching_control *
get_caching_control(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *ctl;

	spin_lock(&cache->lock);
	if (cache->cached != BTRFS_CACHE_STARTED) {
		spin_unlock(&cache->lock);
		return NULL;
	}

281 282 283
	/* We're loading it the fast way, so we don't have a caching_ctl. */
	if (!cache->caching_ctl) {
		spin_unlock(&cache->lock);
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
		return NULL;
	}

	ctl = cache->caching_ctl;
	atomic_inc(&ctl->count);
	spin_unlock(&cache->lock);
	return ctl;
}

static void put_caching_control(struct btrfs_caching_control *ctl)
{
	if (atomic_dec_and_test(&ctl->count))
		kfree(ctl);
}

J
Josef Bacik 已提交
299 300 301 302 303
/*
 * this is only called by cache_block_group, since we could have freed extents
 * we need to check the pinned_extents for any extents that can't be used yet
 * since their free space will be released as soon as the transaction commits.
 */
J
Josef Bacik 已提交
304
static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
J
Josef Bacik 已提交
305 306
			      struct btrfs_fs_info *info, u64 start, u64 end)
{
J
Josef Bacik 已提交
307
	u64 extent_start, extent_end, size, total_added = 0;
J
Josef Bacik 已提交
308 309 310
	int ret;

	while (start < end) {
311
		ret = find_first_extent_bit(info->pinned_extents, start,
J
Josef Bacik 已提交
312
					    &extent_start, &extent_end,
313
					    EXTENT_DIRTY | EXTENT_UPTODATE);
J
Josef Bacik 已提交
314 315 316
		if (ret)
			break;

317
		if (extent_start <= start) {
J
Josef Bacik 已提交
318 319 320
			start = extent_end + 1;
		} else if (extent_start > start && extent_start < end) {
			size = extent_start - start;
J
Josef Bacik 已提交
321
			total_added += size;
322 323
			ret = btrfs_add_free_space(block_group, start,
						   size);
324
			BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
325 326 327 328 329 330 331 332
			start = extent_end + 1;
		} else {
			break;
		}
	}

	if (start < end) {
		size = end - start;
J
Josef Bacik 已提交
333
		total_added += size;
334
		ret = btrfs_add_free_space(block_group, start, size);
335
		BUG_ON(ret); /* -ENOMEM or logic error */
J
Josef Bacik 已提交
336 337
	}

J
Josef Bacik 已提交
338
	return total_added;
J
Josef Bacik 已提交
339 340
}

341
static noinline void caching_thread(struct btrfs_work *work)
342
{
343 344 345 346
	struct btrfs_block_group_cache *block_group;
	struct btrfs_fs_info *fs_info;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_root *extent_root;
347
	struct btrfs_path *path;
348
	struct extent_buffer *leaf;
349
	struct btrfs_key key;
J
Josef Bacik 已提交
350
	u64 total_found = 0;
351 352 353
	u64 last = 0;
	u32 nritems;
	int ret = 0;
354

355 356 357 358 359
	caching_ctl = container_of(work, struct btrfs_caching_control, work);
	block_group = caching_ctl->block_group;
	fs_info = block_group->fs_info;
	extent_root = fs_info->extent_root;

360 361
	path = btrfs_alloc_path();
	if (!path)
362
		goto out;
363

J
Josef Bacik 已提交
364
	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
365

366
	/*
J
Josef Bacik 已提交
367 368 369 370
	 * We don't want to deadlock with somebody trying to allocate a new
	 * extent for the extent root while also trying to search the extent
	 * root to add free space.  So we skip locking and search the commit
	 * root, since its read-only
371 372
	 */
	path->skip_locking = 1;
J
Josef Bacik 已提交
373
	path->search_commit_root = 1;
J
Josef Bacik 已提交
374
	path->reada = 1;
J
Josef Bacik 已提交
375

Y
Yan Zheng 已提交
376
	key.objectid = last;
377
	key.offset = 0;
378
	key.type = BTRFS_EXTENT_ITEM_KEY;
379
again:
380
	mutex_lock(&caching_ctl->mutex);
381 382 383
	/* need to make sure the commit_root doesn't disappear */
	down_read(&fs_info->extent_commit_sem);

384
	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
385
	if (ret < 0)
386
		goto err;
Y
Yan Zheng 已提交
387

388 389 390
	leaf = path->nodes[0];
	nritems = btrfs_header_nritems(leaf);

C
Chris Mason 已提交
391
	while (1) {
392
		if (btrfs_fs_closing(fs_info) > 1) {
393
			last = (u64)-1;
J
Josef Bacik 已提交
394
			break;
395
		}
J
Josef Bacik 已提交
396

397 398 399 400 401
		if (path->slots[0] < nritems) {
			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		} else {
			ret = find_next_key(path, 0, &key);
			if (ret)
402
				break;
J
Josef Bacik 已提交
403

404 405 406
			if (need_resched() ||
			    btrfs_next_leaf(extent_root, path)) {
				caching_ctl->progress = last;
C
Chris Mason 已提交
407
				btrfs_release_path(path);
408 409
				up_read(&fs_info->extent_commit_sem);
				mutex_unlock(&caching_ctl->mutex);
410
				cond_resched();
411 412 413 414 415
				goto again;
			}
			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			continue;
416
		}
J
Josef Bacik 已提交
417

418 419
		if (key.objectid < block_group->key.objectid) {
			path->slots[0]++;
J
Josef Bacik 已提交
420
			continue;
421
		}
J
Josef Bacik 已提交
422

423
		if (key.objectid >= block_group->key.objectid +
J
Josef Bacik 已提交
424
		    block_group->key.offset)
425
			break;
426

427
		if (key.type == BTRFS_EXTENT_ITEM_KEY) {
J
Josef Bacik 已提交
428 429 430
			total_found += add_new_free_space(block_group,
							  fs_info, last,
							  key.objectid);
431
			last = key.objectid + key.offset;
J
Josef Bacik 已提交
432

433 434 435 436
			if (total_found > (1024 * 1024 * 2)) {
				total_found = 0;
				wake_up(&caching_ctl->wait);
			}
J
Josef Bacik 已提交
437
		}
438 439
		path->slots[0]++;
	}
J
Josef Bacik 已提交
440
	ret = 0;
441

J
Josef Bacik 已提交
442 443 444
	total_found += add_new_free_space(block_group, fs_info, last,
					  block_group->key.objectid +
					  block_group->key.offset);
445
	caching_ctl->progress = (u64)-1;
J
Josef Bacik 已提交
446 447

	spin_lock(&block_group->lock);
448
	block_group->caching_ctl = NULL;
J
Josef Bacik 已提交
449 450
	block_group->cached = BTRFS_CACHE_FINISHED;
	spin_unlock(&block_group->lock);
J
Josef Bacik 已提交
451

452
err:
453
	btrfs_free_path(path);
454
	up_read(&fs_info->extent_commit_sem);
J
Josef Bacik 已提交
455

456 457 458
	free_excluded_extents(extent_root, block_group);

	mutex_unlock(&caching_ctl->mutex);
459
out:
460 461 462
	wake_up(&caching_ctl->wait);

	put_caching_control(caching_ctl);
463
	btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
464 465
}

466 467
static int cache_block_group(struct btrfs_block_group_cache *cache,
			     struct btrfs_trans_handle *trans,
468
			     struct btrfs_root *root,
469
			     int load_cache_only)
J
Josef Bacik 已提交
470
{
471
	DEFINE_WAIT(wait);
472 473
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
474 475
	int ret = 0;

476
	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
477 478
	if (!caching_ctl)
		return -ENOMEM;
479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518

	INIT_LIST_HEAD(&caching_ctl->list);
	mutex_init(&caching_ctl->mutex);
	init_waitqueue_head(&caching_ctl->wait);
	caching_ctl->block_group = cache;
	caching_ctl->progress = cache->key.objectid;
	atomic_set(&caching_ctl->count, 1);
	caching_ctl->work.func = caching_thread;

	spin_lock(&cache->lock);
	/*
	 * This should be a rare occasion, but this could happen I think in the
	 * case where one thread starts to load the space cache info, and then
	 * some other thread starts a transaction commit which tries to do an
	 * allocation while the other thread is still loading the space cache
	 * info.  The previous loop should have kept us from choosing this block
	 * group, but if we've moved to the state where we will wait on caching
	 * block groups we need to first check if we're doing a fast load here,
	 * so we can wait for it to finish, otherwise we could end up allocating
	 * from a block group who's cache gets evicted for one reason or
	 * another.
	 */
	while (cache->cached == BTRFS_CACHE_FAST) {
		struct btrfs_caching_control *ctl;

		ctl = cache->caching_ctl;
		atomic_inc(&ctl->count);
		prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
		spin_unlock(&cache->lock);

		schedule();

		finish_wait(&ctl->wait, &wait);
		put_caching_control(ctl);
		spin_lock(&cache->lock);
	}

	if (cache->cached != BTRFS_CACHE_NO) {
		spin_unlock(&cache->lock);
		kfree(caching_ctl);
519
		return 0;
520 521 522 523 524
	}
	WARN_ON(cache->caching_ctl);
	cache->caching_ctl = caching_ctl;
	cache->cached = BTRFS_CACHE_FAST;
	spin_unlock(&cache->lock);
525

526 527
	/*
	 * We can't do the read from on-disk cache during a commit since we need
528 529 530
	 * to have the normal tree locking.  Also if we are currently trying to
	 * allocate blocks for the tree root we can't do the fast caching since
	 * we likely hold important locks.
531
	 */
532
	if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
533 534 535 536
		ret = load_free_space_cache(fs_info, cache);

		spin_lock(&cache->lock);
		if (ret == 1) {
537
			cache->caching_ctl = NULL;
538 539 540
			cache->cached = BTRFS_CACHE_FINISHED;
			cache->last_byte_to_unpin = (u64)-1;
		} else {
541 542 543 544 545 546
			if (load_cache_only) {
				cache->caching_ctl = NULL;
				cache->cached = BTRFS_CACHE_NO;
			} else {
				cache->cached = BTRFS_CACHE_STARTED;
			}
547 548
		}
		spin_unlock(&cache->lock);
549
		wake_up(&caching_ctl->wait);
550
		if (ret == 1) {
551
			put_caching_control(caching_ctl);
552
			free_excluded_extents(fs_info->extent_root, cache);
553
			return 0;
554
		}
555 556 557 558 559 560 561 562 563 564 565 566 567 568
	} else {
		/*
		 * We are not going to do the fast caching, set cached to the
		 * appropriate value and wakeup any waiters.
		 */
		spin_lock(&cache->lock);
		if (load_cache_only) {
			cache->caching_ctl = NULL;
			cache->cached = BTRFS_CACHE_NO;
		} else {
			cache->cached = BTRFS_CACHE_STARTED;
		}
		spin_unlock(&cache->lock);
		wake_up(&caching_ctl->wait);
569 570
	}

571 572
	if (load_cache_only) {
		put_caching_control(caching_ctl);
573
		return 0;
J
Josef Bacik 已提交
574 575
	}

576
	down_write(&fs_info->extent_commit_sem);
577
	atomic_inc(&caching_ctl->count);
578 579 580
	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
	up_write(&fs_info->extent_commit_sem);

581
	btrfs_get_block_group(cache);
582

583
	btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
J
Josef Bacik 已提交
584

585
	return ret;
586 587
}

J
Josef Bacik 已提交
588 589 590
/*
 * return the block group that starts at or after bytenr
 */
C
Chris Mason 已提交
591 592
static struct btrfs_block_group_cache *
btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
C
Chris Mason 已提交
593
{
J
Josef Bacik 已提交
594
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
595

J
Josef Bacik 已提交
596
	cache = block_group_cache_tree_search(info, bytenr, 0);
C
Chris Mason 已提交
597

J
Josef Bacik 已提交
598
	return cache;
C
Chris Mason 已提交
599 600
}

J
Josef Bacik 已提交
601
/*
602
 * return the block group that contains the given bytenr
J
Josef Bacik 已提交
603
 */
C
Chris Mason 已提交
604 605 606
struct btrfs_block_group_cache *btrfs_lookup_block_group(
						 struct btrfs_fs_info *info,
						 u64 bytenr)
C
Chris Mason 已提交
607
{
J
Josef Bacik 已提交
608
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
609

J
Josef Bacik 已提交
610
	cache = block_group_cache_tree_search(info, bytenr, 1);
611

J
Josef Bacik 已提交
612
	return cache;
C
Chris Mason 已提交
613
}
614

J
Josef Bacik 已提交
615 616
static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
						  u64 flags)
617
{
J
Josef Bacik 已提交
618 619
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
620

621
	flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
622

623 624
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
625
		if (found->flags & flags) {
626
			rcu_read_unlock();
J
Josef Bacik 已提交
627
			return found;
628
		}
J
Josef Bacik 已提交
629
	}
630
	rcu_read_unlock();
J
Josef Bacik 已提交
631
	return NULL;
632 633
}

634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
/*
 * after adding space to the filesystem, we need to clear the full flags
 * on all the space infos.
 */
void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
{
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;

	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list)
		found->full = 0;
	rcu_read_unlock();
}

649 650 651 652 653 654 655 656 657
static u64 div_factor(u64 num, int factor)
{
	if (factor == 10)
		return num;
	num *= factor;
	do_div(num, 10);
	return num;
}

658 659 660 661 662 663 664 665 666
static u64 div_factor_fine(u64 num, int factor)
{
	if (factor == 100)
		return num;
	num *= factor;
	do_div(num, 100);
	return num;
}

667 668
u64 btrfs_find_block_group(struct btrfs_root *root,
			   u64 search_start, u64 search_hint, int owner)
C
Chris Mason 已提交
669
{
670
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
671
	u64 used;
672 673
	u64 last = max(search_hint, search_start);
	u64 group_start = 0;
674
	int full_search = 0;
675
	int factor = 9;
C
Chris Mason 已提交
676
	int wrapped = 0;
677
again:
678 679
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
J
Josef Bacik 已提交
680 681
		if (!cache)
			break;
682

683
		spin_lock(&cache->lock);
684 685 686
		last = cache->key.objectid + cache->key.offset;
		used = btrfs_block_group_used(&cache->item);

687 688
		if ((full_search || !cache->ro) &&
		    block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
689
			if (used + cache->pinned + cache->reserved <
690 691
			    div_factor(cache->key.offset, factor)) {
				group_start = cache->key.objectid;
692
				spin_unlock(&cache->lock);
693
				btrfs_put_block_group(cache);
694 695
				goto found;
			}
696
		}
697
		spin_unlock(&cache->lock);
698
		btrfs_put_block_group(cache);
699
		cond_resched();
C
Chris Mason 已提交
700
	}
C
Chris Mason 已提交
701 702 703 704 705 706
	if (!wrapped) {
		last = search_start;
		wrapped = 1;
		goto again;
	}
	if (!full_search && factor < 10) {
C
Chris Mason 已提交
707
		last = search_start;
708
		full_search = 1;
C
Chris Mason 已提交
709
		factor = 10;
710 711
		goto again;
	}
C
Chris Mason 已提交
712
found:
713
	return group_start;
714
}
J
Josef Bacik 已提交
715

716
/* simple helper to search for an existing extent at a given offset */
Z
Zheng Yan 已提交
717
int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
718 719 720
{
	int ret;
	struct btrfs_key key;
Z
Zheng Yan 已提交
721
	struct btrfs_path *path;
722

Z
Zheng Yan 已提交
723
	path = btrfs_alloc_path();
724 725 726
	if (!path)
		return -ENOMEM;

727 728 729 730 731
	key.objectid = start;
	key.offset = len;
	btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
	ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
				0, 0);
Z
Zheng Yan 已提交
732
	btrfs_free_path(path);
733 734 735
	return ret;
}

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
/*
 * helper function to lookup reference count and flags of extent.
 *
 * the head node for delayed ref is used to store the sum of all the
 * reference count modifications queued up in the rbtree. the head
 * node may also store the extent flags to set. This way you can check
 * to see what the reference count and extent flags would be if all of
 * the delayed refs are not processed.
 */
int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 bytenr,
			     u64 num_bytes, u64 *refs, u64 *flags)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	struct btrfs_key key;
	u32 item_size;
	u64 num_refs;
	u64 extent_flags;
	int ret;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = num_bytes;
	if (!trans) {
		path->skip_locking = 1;
		path->search_commit_root = 1;
	}
again:
	ret = btrfs_search_slot(trans, root->fs_info->extent_root,
				&key, path, 0, 0);
	if (ret < 0)
		goto out_free;

	if (ret == 0) {
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		if (item_size >= sizeof(*ei)) {
			ei = btrfs_item_ptr(leaf, path->slots[0],
					    struct btrfs_extent_item);
			num_refs = btrfs_extent_refs(leaf, ei);
			extent_flags = btrfs_extent_flags(leaf, ei);
		} else {
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
			struct btrfs_extent_item_v0 *ei0;
			BUG_ON(item_size != sizeof(*ei0));
			ei0 = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_item_v0);
			num_refs = btrfs_extent_refs_v0(leaf, ei0);
			/* FIXME: this isn't correct for data */
			extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
#else
			BUG();
#endif
		}
		BUG_ON(num_refs == 0);
	} else {
		num_refs = 0;
		extent_flags = 0;
		ret = 0;
	}

	if (!trans)
		goto out;

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (head) {
		if (!mutex_trylock(&head->mutex)) {
			atomic_inc(&head->node.refs);
			spin_unlock(&delayed_refs->lock);

816
			btrfs_release_path(path);
817

818 819 820 821
			/*
			 * Mutex was contended, block until it's released and try
			 * again
			 */
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
			mutex_lock(&head->mutex);
			mutex_unlock(&head->mutex);
			btrfs_put_delayed_ref(&head->node);
			goto again;
		}
		if (head->extent_op && head->extent_op->update_flags)
			extent_flags |= head->extent_op->flags_to_set;
		else
			BUG_ON(num_refs == 0);

		num_refs += head->node.ref_mod;
		mutex_unlock(&head->mutex);
	}
	spin_unlock(&delayed_refs->lock);
out:
	WARN_ON(num_refs == 0);
	if (refs)
		*refs = num_refs;
	if (flags)
		*flags = extent_flags;
out_free:
	btrfs_free_path(path);
	return ret;
}

847 848 849 850 851 852 853 854 855 856 857 858 859 860
/*
 * Back reference rules.  Back refs have three main goals:
 *
 * 1) differentiate between all holders of references to an extent so that
 *    when a reference is dropped we can make sure it was a valid reference
 *    before freeing the extent.
 *
 * 2) Provide enough information to quickly find the holders of an extent
 *    if we notice a given block is corrupted or bad.
 *
 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 *    maintenance.  This is actually the same as #2, but with a slightly
 *    different use case.
 *
861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905
 * There are two kinds of back refs. The implicit back refs is optimized
 * for pointers in non-shared tree blocks. For a given pointer in a block,
 * back refs of this kind provide information about the block's owner tree
 * and the pointer's key. These information allow us to find the block by
 * b-tree searching. The full back refs is for pointers in tree blocks not
 * referenced by their owner trees. The location of tree block is recorded
 * in the back refs. Actually the full back refs is generic, and can be
 * used in all cases the implicit back refs is used. The major shortcoming
 * of the full back refs is its overhead. Every time a tree block gets
 * COWed, we have to update back refs entry for all pointers in it.
 *
 * For a newly allocated tree block, we use implicit back refs for
 * pointers in it. This means most tree related operations only involve
 * implicit back refs. For a tree block created in old transaction, the
 * only way to drop a reference to it is COW it. So we can detect the
 * event that tree block loses its owner tree's reference and do the
 * back refs conversion.
 *
 * When a tree block is COW'd through a tree, there are four cases:
 *
 * The reference count of the block is one and the tree is the block's
 * owner tree. Nothing to do in this case.
 *
 * The reference count of the block is one and the tree is not the
 * block's owner tree. In this case, full back refs is used for pointers
 * in the block. Remove these full back refs, add implicit back refs for
 * every pointers in the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * the block's owner tree. In this case, implicit back refs is used for
 * pointers in the block. Add full back refs for every pointers in the
 * block, increase lower level extents' reference counts. The original
 * implicit back refs are entailed to the new block.
 *
 * The reference count of the block is greater than one and the tree is
 * not the block's owner tree. Add implicit back refs for every pointer in
 * the new block, increase lower level extents' reference count.
 *
 * Back Reference Key composing:
 *
 * The key objectid corresponds to the first byte in the extent,
 * The key type is used to differentiate between types of back refs.
 * There are different meanings of the key offset for different types
 * of back refs.
 *
906 907 908
 * File extents can be referenced by:
 *
 * - multiple snapshots, subvolumes, or different generations in one subvol
Z
Zheng Yan 已提交
909
 * - different files inside a single subvolume
910 911
 * - different offsets inside a file (bookend extents in file.c)
 *
912
 * The extent ref structure for the implicit back refs has fields for:
913 914 915
 *
 * - Objectid of the subvolume root
 * - objectid of the file holding the reference
916 917
 * - original offset in the file
 * - how many bookend extents
918
 *
919 920
 * The key offset for the implicit back refs is hash of the first
 * three fields.
921
 *
922
 * The extent ref structure for the full back refs has field for:
923
 *
924
 * - number of pointers in the tree leaf
925
 *
926 927
 * The key offset for the implicit back refs is the first byte of
 * the tree leaf
928
 *
929 930
 * When a file extent is allocated, The implicit back refs is used.
 * the fields are filled in:
931
 *
932
 *     (root_key.objectid, inode objectid, offset in file, 1)
933
 *
934 935
 * When a file extent is removed file truncation, we find the
 * corresponding implicit back refs and check the following fields:
936
 *
937
 *     (btrfs_header_owner(leaf), inode objectid, offset in file)
938
 *
939
 * Btree extents can be referenced by:
940
 *
941
 * - Different subvolumes
942
 *
943 944 945 946
 * Both the implicit back refs and the full back refs for tree blocks
 * only consist of key. The key offset for the implicit back refs is
 * objectid of block's owner tree. The key offset for the full back refs
 * is the first byte of parent block.
947
 *
948 949 950
 * When implicit back refs is used, information about the lowest key and
 * level of the tree block are required. These information are stored in
 * tree block info structure.
951
 */
Z
Zheng Yan 已提交
952

953 954 955 956 957
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  u64 owner, u32 extra_size)
958
{
959 960 961 962 963
	struct btrfs_extent_item *item;
	struct btrfs_extent_item_v0 *ei0;
	struct btrfs_extent_ref_v0 *ref0;
	struct btrfs_tree_block_info *bi;
	struct extent_buffer *leaf;
964
	struct btrfs_key key;
965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983
	struct btrfs_key found_key;
	u32 new_size = sizeof(*item);
	u64 refs;
	int ret;

	leaf = path->nodes[0];
	BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));

	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	ei0 = btrfs_item_ptr(leaf, path->slots[0],
			     struct btrfs_extent_item_v0);
	refs = btrfs_extent_refs_v0(leaf, ei0);

	if (owner == (u64)-1) {
		while (1) {
			if (path->slots[0] >= btrfs_header_nritems(leaf)) {
				ret = btrfs_next_leaf(root, path);
				if (ret < 0)
					return ret;
984
				BUG_ON(ret > 0); /* Corruption */
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999
				leaf = path->nodes[0];
			}
			btrfs_item_key_to_cpu(leaf, &found_key,
					      path->slots[0]);
			BUG_ON(key.objectid != found_key.objectid);
			if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
				path->slots[0]++;
				continue;
			}
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					      struct btrfs_extent_ref_v0);
			owner = btrfs_ref_objectid_v0(leaf, ref0);
			break;
		}
	}
1000
	btrfs_release_path(path);
1001 1002 1003 1004 1005 1006 1007 1008 1009

	if (owner < BTRFS_FIRST_FREE_OBJECTID)
		new_size += sizeof(*bi);

	new_size -= sizeof(*ei0);
	ret = btrfs_search_slot(trans, root, &key, path,
				new_size + extra_size, 1);
	if (ret < 0)
		return ret;
1010
	BUG_ON(ret); /* Corruption */
1011

1012
	btrfs_extend_item(trans, root, path, new_size);
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, item, refs);
	/* FIXME: get real generation */
	btrfs_set_extent_generation(leaf, item, 0);
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		btrfs_set_extent_flags(leaf, item,
				       BTRFS_EXTENT_FLAG_TREE_BLOCK |
				       BTRFS_BLOCK_FLAG_FULL_BACKREF);
		bi = (struct btrfs_tree_block_info *)(item + 1);
		/* FIXME: get first key of the block */
		memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
		btrfs_set_tree_block_level(leaf, bi, (int)owner);
	} else {
		btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
	}
	btrfs_mark_buffer_dirty(leaf);
	return 0;
}
#endif

static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
{
	u32 high_crc = ~(u32)0;
	u32 low_crc = ~(u32)0;
	__le64 lenum;

	lenum = cpu_to_le64(root_objectid);
1042
	high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1043
	lenum = cpu_to_le64(owner);
1044
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1045
	lenum = cpu_to_le64(offset);
1046
	low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078

	return ((u64)high_crc << 31) ^ (u64)low_crc;
}

static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
				     struct btrfs_extent_data_ref *ref)
{
	return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
				    btrfs_extent_data_ref_objectid(leaf, ref),
				    btrfs_extent_data_ref_offset(leaf, ref));
}

static int match_extent_data_ref(struct extent_buffer *leaf,
				 struct btrfs_extent_data_ref *ref,
				 u64 root_objectid, u64 owner, u64 offset)
{
	if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		return 0;
	return 1;
}

static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid,
					   u64 owner, u64 offset)
{
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref;
Z
Zheng Yan 已提交
1079
	struct extent_buffer *leaf;
1080
	u32 nritems;
1081
	int ret;
1082 1083
	int recow;
	int err = -ENOENT;
1084

Z
Zheng Yan 已提交
1085
	key.objectid = bytenr;
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
	}
again:
	recow = 0;
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret < 0) {
		err = ret;
		goto fail;
	}
Z
Zheng Yan 已提交
1101

1102 1103 1104 1105 1106
	if (parent) {
		if (!ret)
			return 0;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		key.type = BTRFS_EXTENT_REF_V0_KEY;
1107
		btrfs_release_path(path);
1108 1109 1110 1111 1112 1113 1114 1115 1116
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret < 0) {
			err = ret;
			goto fail;
		}
		if (!ret)
			return 0;
#endif
		goto fail;
Z
Zheng Yan 已提交
1117 1118 1119
	}

	leaf = path->nodes[0];
1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
	nritems = btrfs_header_nritems(leaf);
	while (1) {
		if (path->slots[0] >= nritems) {
			ret = btrfs_next_leaf(root, path);
			if (ret < 0)
				err = ret;
			if (ret)
				goto fail;

			leaf = path->nodes[0];
			nritems = btrfs_header_nritems(leaf);
			recow = 1;
		}

		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
		if (key.objectid != bytenr ||
		    key.type != BTRFS_EXTENT_DATA_REF_KEY)
			goto fail;

		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);

		if (match_extent_data_ref(leaf, ref, root_objectid,
					  owner, offset)) {
			if (recow) {
1145
				btrfs_release_path(path);
1146 1147 1148 1149 1150 1151
				goto again;
			}
			err = 0;
			break;
		}
		path->slots[0]++;
Z
Zheng Yan 已提交
1152
	}
1153 1154
fail:
	return err;
Z
Zheng Yan 已提交
1155 1156
}

1157 1158 1159 1160 1161 1162
static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   u64 bytenr, u64 parent,
					   u64 root_objectid, u64 owner,
					   u64 offset, int refs_to_add)
Z
Zheng Yan 已提交
1163 1164 1165
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
1166
	u32 size;
Z
Zheng Yan 已提交
1167 1168
	u32 num_refs;
	int ret;
1169 1170

	key.objectid = bytenr;
1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	if (parent) {
		key.type = BTRFS_SHARED_DATA_REF_KEY;
		key.offset = parent;
		size = sizeof(struct btrfs_shared_data_ref);
	} else {
		key.type = BTRFS_EXTENT_DATA_REF_KEY;
		key.offset = hash_extent_data_ref(root_objectid,
						  owner, offset);
		size = sizeof(struct btrfs_extent_data_ref);
	}
1181

1182 1183 1184 1185 1186 1187 1188
	ret = btrfs_insert_empty_item(trans, root, path, &key, size);
	if (ret && ret != -EEXIST)
		goto fail;

	leaf = path->nodes[0];
	if (parent) {
		struct btrfs_shared_data_ref *ref;
Z
Zheng Yan 已提交
1189
		ref = btrfs_item_ptr(leaf, path->slots[0],
1190 1191 1192 1193 1194 1195 1196
				     struct btrfs_shared_data_ref);
		if (ret == 0) {
			btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_shared_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1197
		}
1198 1199 1200 1201 1202 1203 1204 1205
	} else {
		struct btrfs_extent_data_ref *ref;
		while (ret == -EEXIST) {
			ref = btrfs_item_ptr(leaf, path->slots[0],
					     struct btrfs_extent_data_ref);
			if (match_extent_data_ref(leaf, ref, root_objectid,
						  owner, offset))
				break;
1206
			btrfs_release_path(path);
1207 1208 1209 1210 1211
			key.offset++;
			ret = btrfs_insert_empty_item(trans, root, path, &key,
						      size);
			if (ret && ret != -EEXIST)
				goto fail;
Z
Zheng Yan 已提交
1212

1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
			leaf = path->nodes[0];
		}
		ref = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_data_ref);
		if (ret == 0) {
			btrfs_set_extent_data_ref_root(leaf, ref,
						       root_objectid);
			btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
			btrfs_set_extent_data_ref_offset(leaf, ref, offset);
			btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
		} else {
			num_refs = btrfs_extent_data_ref_count(leaf, ref);
			num_refs += refs_to_add;
			btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
Z
Zheng Yan 已提交
1227 1228
		}
	}
1229 1230 1231
	btrfs_mark_buffer_dirty(leaf);
	ret = 0;
fail:
1232
	btrfs_release_path(path);
1233
	return ret;
1234 1235
}

1236 1237 1238 1239
static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
					   struct btrfs_root *root,
					   struct btrfs_path *path,
					   int refs_to_drop)
Z
Zheng Yan 已提交
1240
{
1241 1242 1243
	struct btrfs_key key;
	struct btrfs_extent_data_ref *ref1 = NULL;
	struct btrfs_shared_data_ref *ref2 = NULL;
Z
Zheng Yan 已提交
1244
	struct extent_buffer *leaf;
1245
	u32 num_refs = 0;
Z
Zheng Yan 已提交
1246 1247 1248
	int ret = 0;

	leaf = path->nodes[0];
1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

	if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
#endif
	} else {
		BUG();
	}

1270 1271
	BUG_ON(num_refs < refs_to_drop);
	num_refs -= refs_to_drop;
1272

Z
Zheng Yan 已提交
1273 1274 1275
	if (num_refs == 0) {
		ret = btrfs_del_item(trans, root, path);
	} else {
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
		if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
		else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
			btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		else {
			struct btrfs_extent_ref_v0 *ref0;
			ref0 = btrfs_item_ptr(leaf, path->slots[0],
					struct btrfs_extent_ref_v0);
			btrfs_set_ref_count_v0(leaf, ref0, num_refs);
		}
#endif
Z
Zheng Yan 已提交
1288 1289 1290 1291 1292
		btrfs_mark_buffer_dirty(leaf);
	}
	return ret;
}

1293 1294 1295
static noinline u32 extent_data_ref_count(struct btrfs_root *root,
					  struct btrfs_path *path,
					  struct btrfs_extent_inline_ref *iref)
1296
{
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_data_ref *ref1;
	struct btrfs_shared_data_ref *ref2;
	u32 num_refs = 0;

	leaf = path->nodes[0];
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
	if (iref) {
		if (btrfs_extent_inline_ref_type(leaf, iref) ==
		    BTRFS_EXTENT_DATA_REF_KEY) {
			ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
			num_refs = btrfs_extent_data_ref_count(leaf, ref1);
		} else {
			ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
			num_refs = btrfs_shared_data_ref_count(leaf, ref2);
		}
	} else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
		ref1 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_data_ref);
		num_refs = btrfs_extent_data_ref_count(leaf, ref1);
	} else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
		ref2 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_shared_data_ref);
		num_refs = btrfs_shared_data_ref_count(leaf, ref2);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	} else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
		struct btrfs_extent_ref_v0 *ref0;
		ref0 = btrfs_item_ptr(leaf, path->slots[0],
				      struct btrfs_extent_ref_v0);
		num_refs = btrfs_ref_count_v0(leaf, ref0);
C
Chris Mason 已提交
1328
#endif
1329 1330 1331 1332 1333
	} else {
		WARN_ON(1);
	}
	return num_refs;
}
1334

1335 1336 1337 1338 1339
static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
1340
{
1341
	struct btrfs_key key;
1342 1343
	int ret;

1344 1345 1346 1347 1348 1349 1350
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
1351 1352
	}

1353 1354 1355 1356 1357
	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -ENOENT;
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (ret == -ENOENT && parent) {
1358
		btrfs_release_path(path);
1359 1360 1361 1362 1363
		key.type = BTRFS_EXTENT_REF_V0_KEY;
		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
		if (ret > 0)
			ret = -ENOENT;
	}
1364
#endif
1365
	return ret;
1366 1367
}

1368 1369 1370 1371 1372
static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
					  struct btrfs_root *root,
					  struct btrfs_path *path,
					  u64 bytenr, u64 parent,
					  u64 root_objectid)
Z
Zheng Yan 已提交
1373
{
1374
	struct btrfs_key key;
Z
Zheng Yan 已提交
1375 1376
	int ret;

1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
	key.objectid = bytenr;
	if (parent) {
		key.type = BTRFS_SHARED_BLOCK_REF_KEY;
		key.offset = parent;
	} else {
		key.type = BTRFS_TREE_BLOCK_REF_KEY;
		key.offset = root_objectid;
	}

	ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1387
	btrfs_release_path(path);
Z
Zheng Yan 已提交
1388 1389 1390
	return ret;
}

1391
static inline int extent_ref_type(u64 parent, u64 owner)
Z
Zheng Yan 已提交
1392
{
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405
	int type;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		if (parent > 0)
			type = BTRFS_SHARED_BLOCK_REF_KEY;
		else
			type = BTRFS_TREE_BLOCK_REF_KEY;
	} else {
		if (parent > 0)
			type = BTRFS_SHARED_DATA_REF_KEY;
		else
			type = BTRFS_EXTENT_DATA_REF_KEY;
	}
	return type;
Z
Zheng Yan 已提交
1406
}
1407

1408 1409
static int find_next_key(struct btrfs_path *path, int level,
			 struct btrfs_key *key)
1410

C
Chris Mason 已提交
1411
{
1412
	for (; level < BTRFS_MAX_LEVEL; level++) {
1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
		if (!path->nodes[level])
			break;
		if (path->slots[level] + 1 >=
		    btrfs_header_nritems(path->nodes[level]))
			continue;
		if (level == 0)
			btrfs_item_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		else
			btrfs_node_key_to_cpu(path->nodes[level], key,
					      path->slots[level] + 1);
		return 0;
	}
	return 1;
}
C
Chris Mason 已提交
1428

1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463
/*
 * look for inline back ref. if back ref is found, *ref_ret is set
 * to the address of inline back ref, and 0 is returned.
 *
 * if back ref isn't found, *ref_ret is set to the address where it
 * should be inserted, and -ENOENT is returned.
 *
 * if insert is true and there are too many inline back refs, the path
 * points to the extent item, and -EAGAIN is returned.
 *
 * NOTE: inline back refs are ordered in the same way that back ref
 *	 items in the tree are ordered.
 */
static noinline_for_stack
int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int insert)
{
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
	u64 flags;
	u64 item_size;
	unsigned long ptr;
	unsigned long end;
	int extra_size;
	int type;
	int want;
	int ret;
	int err = 0;
1464

1465
	key.objectid = bytenr;
Z
Zheng Yan 已提交
1466
	key.type = BTRFS_EXTENT_ITEM_KEY;
1467
	key.offset = num_bytes;
Z
Zheng Yan 已提交
1468

1469 1470 1471
	want = extent_ref_type(parent, owner);
	if (insert) {
		extra_size = btrfs_extent_inline_ref_size(want);
1472
		path->keep_locks = 1;
1473 1474 1475
	} else
		extra_size = -1;
	ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1476
	if (ret < 0) {
1477 1478 1479
		err = ret;
		goto out;
	}
1480 1481 1482 1483 1484
	if (ret && !insert) {
		err = -ENOENT;
		goto out;
	}
	BUG_ON(ret); /* Corruption */
1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		if (!insert) {
			err = -ENOENT;
			goto out;
		}
		ret = convert_extent_item_v0(trans, root, path, owner,
					     extra_size);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	flags = btrfs_extent_flags(leaf, ei);

	ptr = (unsigned long)(ei + 1);
	end = (unsigned long)ei + item_size;

	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
		ptr += sizeof(struct btrfs_tree_block_info);
		BUG_ON(ptr > end);
	} else {
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
	}

	err = -ENOENT;
	while (1) {
		if (ptr >= end) {
			WARN_ON(ptr > end);
			break;
		}
		iref = (struct btrfs_extent_inline_ref *)ptr;
		type = btrfs_extent_inline_ref_type(leaf, iref);
		if (want < type)
			break;
		if (want > type) {
			ptr += btrfs_extent_inline_ref_size(type);
			continue;
		}

		if (type == BTRFS_EXTENT_DATA_REF_KEY) {
			struct btrfs_extent_data_ref *dref;
			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
			if (match_extent_data_ref(leaf, dref, root_objectid,
						  owner, offset)) {
				err = 0;
				break;
			}
			if (hash_extent_data_ref_item(leaf, dref) <
			    hash_extent_data_ref(root_objectid, owner, offset))
				break;
		} else {
			u64 ref_offset;
			ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
			if (parent > 0) {
				if (parent == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < parent)
					break;
			} else {
				if (root_objectid == ref_offset) {
					err = 0;
					break;
				}
				if (ref_offset < root_objectid)
					break;
			}
		}
		ptr += btrfs_extent_inline_ref_size(type);
	}
	if (err == -ENOENT && insert) {
		if (item_size + extra_size >=
		    BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
			err = -EAGAIN;
			goto out;
		}
		/*
		 * To add new inline back ref, we have to make sure
		 * there is no corresponding back ref item.
		 * For simplicity, we just do not add new inline back
		 * ref if there is any kind of item for this block
		 */
1578 1579
		if (find_next_key(path, 0, &key) == 0 &&
		    key.objectid == bytenr &&
1580
		    key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1581 1582 1583 1584 1585 1586
			err = -EAGAIN;
			goto out;
		}
	}
	*ref_ret = (struct btrfs_extent_inline_ref *)ptr;
out:
1587
	if (insert) {
1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
		path->keep_locks = 0;
		btrfs_unlock_up_safe(path, 1);
	}
	return err;
}

/*
 * helper to add new inline back ref
 */
static noinline_for_stack
1598 1599 1600 1601 1602 1603 1604
void setup_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	unsigned long ptr;
	unsigned long end;
	unsigned long item_offset;
	u64 refs;
	int size;
	int type;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	item_offset = (unsigned long)iref - (unsigned long)ei;

	type = extent_ref_type(parent, owner);
	size = btrfs_extent_inline_ref_size(type);

1622
	btrfs_extend_item(trans, root, path, size);
1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671

	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	refs += refs_to_add;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	ptr = (unsigned long)ei + item_offset;
	end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
	if (ptr < end - size)
		memmove_extent_buffer(leaf, ptr + size, ptr,
				      end - size - ptr);

	iref = (struct btrfs_extent_inline_ref *)ptr;
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		struct btrfs_extent_data_ref *dref;
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
		btrfs_set_extent_data_ref_offset(leaf, dref, offset);
		btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		struct btrfs_shared_data_ref *sref;
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}
	btrfs_mark_buffer_dirty(leaf);
}

static int lookup_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref **ref_ret,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner, u64 offset)
{
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 0);
	if (ret != -ENOENT)
1672
		return ret;
1673

1674
	btrfs_release_path(path);
1675 1676 1677 1678 1679 1680 1681 1682
	*ref_ret = NULL;

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
					    root_objectid);
	} else {
		ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
					     root_objectid, owner, offset);
1683
	}
1684 1685
	return ret;
}
Z
Zheng Yan 已提交
1686

1687 1688 1689 1690
/*
 * helper to update/remove inline back ref
 */
static noinline_for_stack
1691 1692 1693 1694 1695 1696
void update_inline_extent_backref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  struct btrfs_path *path,
				  struct btrfs_extent_inline_ref *iref,
				  int refs_to_mod,
				  struct btrfs_delayed_extent_op *extent_op)
1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728
{
	struct extent_buffer *leaf;
	struct btrfs_extent_item *ei;
	struct btrfs_extent_data_ref *dref = NULL;
	struct btrfs_shared_data_ref *sref = NULL;
	unsigned long ptr;
	unsigned long end;
	u32 item_size;
	int size;
	int type;
	u64 refs;

	leaf = path->nodes[0];
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, ei);
	WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
	refs += refs_to_mod;
	btrfs_set_extent_refs(leaf, ei, refs);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, ei);

	type = btrfs_extent_inline_ref_type(leaf, iref);

	if (type == BTRFS_EXTENT_DATA_REF_KEY) {
		dref = (struct btrfs_extent_data_ref *)(&iref->offset);
		refs = btrfs_extent_data_ref_count(leaf, dref);
	} else if (type == BTRFS_SHARED_DATA_REF_KEY) {
		sref = (struct btrfs_shared_data_ref *)(iref + 1);
		refs = btrfs_shared_data_ref_count(leaf, sref);
	} else {
		refs = 1;
		BUG_ON(refs_to_mod != -1);
1729
	}
Z
Zheng Yan 已提交
1730

1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
	BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
	refs += refs_to_mod;

	if (refs > 0) {
		if (type == BTRFS_EXTENT_DATA_REF_KEY)
			btrfs_set_extent_data_ref_count(leaf, dref, refs);
		else
			btrfs_set_shared_data_ref_count(leaf, sref, refs);
	} else {
		size =  btrfs_extent_inline_ref_size(type);
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
		ptr = (unsigned long)iref;
		end = (unsigned long)ei + item_size;
		if (ptr + size < end)
			memmove_extent_buffer(leaf, ptr, ptr + size,
					      end - ptr - size);
		item_size -= size;
1748
		btrfs_truncate_item(trans, root, path, item_size, 1);
1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769
	}
	btrfs_mark_buffer_dirty(leaf);
}

static noinline_for_stack
int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 num_bytes, u64 parent,
				 u64 root_objectid, u64 owner,
				 u64 offset, int refs_to_add,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_extent_inline_ref *iref;
	int ret;

	ret = lookup_inline_extent_backref(trans, root, path, &iref,
					   bytenr, num_bytes, parent,
					   root_objectid, owner, offset, 1);
	if (ret == 0) {
		BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1770 1771
		update_inline_extent_backref(trans, root, path, iref,
					     refs_to_add, extent_op);
1772
	} else if (ret == -ENOENT) {
1773 1774 1775 1776
		setup_inline_extent_backref(trans, root, path, iref, parent,
					    root_objectid, owner, offset,
					    refs_to_add, extent_op);
		ret = 0;
1777
	}
1778 1779
	return ret;
}
Z
Zheng Yan 已提交
1780

1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798
static int insert_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 u64 bytenr, u64 parent, u64 root_objectid,
				 u64 owner, u64 offset, int refs_to_add)
{
	int ret;
	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
		BUG_ON(refs_to_add != 1);
		ret = insert_tree_block_ref(trans, root, path, bytenr,
					    parent, root_objectid);
	} else {
		ret = insert_extent_data_ref(trans, root, path, bytenr,
					     parent, root_objectid,
					     owner, offset, refs_to_add);
	}
	return ret;
}
1799

1800 1801 1802 1803 1804 1805
static int remove_extent_backref(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_extent_inline_ref *iref,
				 int refs_to_drop, int is_data)
{
1806
	int ret = 0;
1807

1808 1809
	BUG_ON(!is_data && refs_to_drop != 1);
	if (iref) {
1810 1811
		update_inline_extent_backref(trans, root, path, iref,
					     -refs_to_drop, NULL);
1812 1813 1814 1815 1816 1817 1818 1819
	} else if (is_data) {
		ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
	} else {
		ret = btrfs_del_item(trans, root, path);
	}
	return ret;
}

1820
static int btrfs_issue_discard(struct block_device *bdev,
1821 1822
				u64 start, u64 len)
{
1823
	return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1824 1825 1826
}

static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1827
				u64 num_bytes, u64 *actual_bytes)
1828 1829
{
	int ret;
1830
	u64 discarded_bytes = 0;
1831
	struct btrfs_bio *bbio = NULL;
1832

C
Christoph Hellwig 已提交
1833

1834
	/* Tell the block device(s) that the sectors can be discarded */
1835
	ret = btrfs_map_block(&root->fs_info->mapping_tree, REQ_DISCARD,
1836
			      bytenr, &num_bytes, &bbio, 0);
1837
	/* Error condition is -ENOMEM */
1838
	if (!ret) {
1839
		struct btrfs_bio_stripe *stripe = bbio->stripes;
1840 1841 1842
		int i;


1843
		for (i = 0; i < bbio->num_stripes; i++, stripe++) {
1844 1845 1846
			if (!stripe->dev->can_discard)
				continue;

1847 1848 1849 1850 1851 1852
			ret = btrfs_issue_discard(stripe->dev->bdev,
						  stripe->physical,
						  stripe->length);
			if (!ret)
				discarded_bytes += stripe->length;
			else if (ret != -EOPNOTSUPP)
1853
				break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
1854 1855 1856 1857 1858 1859 1860

			/*
			 * Just in case we get back EOPNOTSUPP for some reason,
			 * just ignore the return value so we don't screw up
			 * people calling discard_extent.
			 */
			ret = 0;
1861
		}
1862
		kfree(bbio);
1863
	}
1864 1865 1866 1867

	if (actual_bytes)
		*actual_bytes = discarded_bytes;

1868 1869 1870 1871

	return ret;
}

1872
/* Can return -ENOMEM */
1873 1874 1875
int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 bytenr, u64 num_bytes, u64 parent,
A
Arne Jansen 已提交
1876
			 u64 root_objectid, u64 owner, u64 offset, int for_cow)
1877 1878
{
	int ret;
A
Arne Jansen 已提交
1879 1880
	struct btrfs_fs_info *fs_info = root->fs_info;

1881 1882 1883 1884
	BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
	       root_objectid == BTRFS_TREE_LOG_OBJECTID);

	if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
1885 1886
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
1887
					parent, root_objectid, (int)owner,
A
Arne Jansen 已提交
1888
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1889
	} else {
A
Arne Jansen 已提交
1890 1891
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
					num_bytes,
1892
					parent, root_objectid, owner, offset,
A
Arne Jansen 已提交
1893
					BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936
	}
	return ret;
}

static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root,
				  u64 bytenr, u64 num_bytes,
				  u64 parent, u64 root_objectid,
				  u64 owner, u64 offset, int refs_to_add,
				  struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	struct btrfs_extent_item *item;
	u64 refs;
	int ret;
	int err = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	path->reada = 1;
	path->leave_spinning = 1;
	/* this will setup the path even if it fails to insert the back ref */
	ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
					   path, bytenr, num_bytes, parent,
					   root_objectid, owner, offset,
					   refs_to_add, extent_op);
	if (ret == 0)
		goto out;

	if (ret != -EAGAIN) {
		err = ret;
		goto out;
	}

	leaf = path->nodes[0];
	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	refs = btrfs_extent_refs(leaf, item);
	btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
	if (extent_op)
		__run_delayed_extent_op(extent_op, leaf, item);
1937

1938
	btrfs_mark_buffer_dirty(leaf);
1939
	btrfs_release_path(path);
1940 1941

	path->reada = 1;
1942 1943
	path->leave_spinning = 1;

1944 1945
	/* now insert the actual backref */
	ret = insert_extent_backref(trans, root->fs_info->extent_root,
1946 1947
				    path, bytenr, parent, root_objectid,
				    owner, offset, refs_to_add);
1948 1949
	if (ret)
		btrfs_abort_transaction(trans, root, ret);
1950
out:
1951
	btrfs_free_path(path);
1952
	return err;
1953 1954
}

1955 1956 1957 1958 1959
static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
1960
{
1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032
	int ret = 0;
	struct btrfs_delayed_data_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
	u64 flags = 0;

	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;

	ref = btrfs_delayed_node_to_data_ref(node);
	if (node->type == BTRFS_SHARED_DATA_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		if (extent_op) {
			BUG_ON(extent_op->update_key);
			flags |= extent_op->flags_to_set;
		}
		ret = alloc_reserved_file_extent(trans, root,
						 parent, ref_root, flags,
						 ref->objectid, ref->offset,
						 &ins, node->ref_mod);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent,
					     ref_root, ref->objectid,
					     ref->offset, node->ref_mod,
					     extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent,
					  ref_root, ref->objectid,
					  ref->offset, node->ref_mod,
					  extent_op);
	} else {
		BUG();
	}
	return ret;
}

static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
				    struct extent_buffer *leaf,
				    struct btrfs_extent_item *ei)
{
	u64 flags = btrfs_extent_flags(leaf, ei);
	if (extent_op->update_flags) {
		flags |= extent_op->flags_to_set;
		btrfs_set_extent_flags(leaf, ei, flags);
	}

	if (extent_op->update_key) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
	}
}

static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_delayed_ref_node *node,
				 struct btrfs_delayed_extent_op *extent_op)
{
	struct btrfs_key key;
	struct btrfs_path *path;
	struct btrfs_extent_item *ei;
	struct extent_buffer *leaf;
	u32 item_size;
2033
	int ret;
2034 2035
	int err = 0;

2036 2037 2038
	if (trans->aborted)
		return 0;

2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = node->bytenr;
	key.type = BTRFS_EXTENT_ITEM_KEY;
	key.offset = node->num_bytes;

	path->reada = 1;
	path->leave_spinning = 1;
	ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
				path, 0, 1);
	if (ret < 0) {
		err = ret;
		goto out;
	}
	if (ret > 0) {
		err = -EIO;
		goto out;
	}

	leaf = path->nodes[0];
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
					     path, (u64)-1, 0);
		if (ret < 0) {
			err = ret;
			goto out;
		}
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, path->slots[0]);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
	__run_delayed_extent_op(extent_op, leaf, ei);
2077

2078 2079 2080 2081
	btrfs_mark_buffer_dirty(leaf);
out:
	btrfs_free_path(path);
	return err;
2082 2083
}

2084 2085 2086 2087 2088
static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				struct btrfs_delayed_ref_node *node,
				struct btrfs_delayed_extent_op *extent_op,
				int insert_reserved)
2089 2090
{
	int ret = 0;
2091 2092 2093 2094
	struct btrfs_delayed_tree_ref *ref;
	struct btrfs_key ins;
	u64 parent = 0;
	u64 ref_root = 0;
2095

2096 2097 2098
	ins.objectid = node->bytenr;
	ins.offset = node->num_bytes;
	ins.type = BTRFS_EXTENT_ITEM_KEY;
2099

2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125
	ref = btrfs_delayed_node_to_tree_ref(node);
	if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		parent = ref->parent;
	else
		ref_root = ref->root;

	BUG_ON(node->ref_mod != 1);
	if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
		BUG_ON(!extent_op || !extent_op->update_flags ||
		       !extent_op->update_key);
		ret = alloc_reserved_tree_block(trans, root,
						parent, ref_root,
						extent_op->flags_to_set,
						&extent_op->key,
						ref->level, &ins);
	} else if (node->action == BTRFS_ADD_DELAYED_REF) {
		ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
					     node->num_bytes, parent, ref_root,
					     ref->level, 0, 1, extent_op);
	} else if (node->action == BTRFS_DROP_DELAYED_REF) {
		ret = __btrfs_free_extent(trans, root, node->bytenr,
					  node->num_bytes, parent, ref_root,
					  ref->level, 0, 1, extent_op);
	} else {
		BUG();
	}
2126 2127 2128 2129
	return ret;
}

/* helper function to actually process a single delayed ref entry */
2130 2131 2132 2133 2134
static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root,
			       struct btrfs_delayed_ref_node *node,
			       struct btrfs_delayed_extent_op *extent_op,
			       int insert_reserved)
2135
{
2136 2137 2138 2139 2140
	int ret = 0;

	if (trans->aborted)
		return 0;

2141
	if (btrfs_delayed_ref_is_head(node)) {
2142 2143 2144 2145 2146 2147 2148
		struct btrfs_delayed_ref_head *head;
		/*
		 * we've hit the end of the chain and we were supposed
		 * to insert this extent into the tree.  But, it got
		 * deleted before we ever needed to insert it, so all
		 * we have to do is clean up the accounting
		 */
2149 2150
		BUG_ON(extent_op);
		head = btrfs_delayed_node_to_head(node);
2151
		if (insert_reserved) {
2152 2153
			btrfs_pin_extent(root, node->bytenr,
					 node->num_bytes, 1);
2154 2155 2156 2157 2158
			if (head->is_data) {
				ret = btrfs_del_csums(trans, root,
						      node->bytenr,
						      node->num_bytes);
			}
2159 2160
		}
		mutex_unlock(&head->mutex);
2161
		return ret;
2162 2163
	}

2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174
	if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
	    node->type == BTRFS_SHARED_BLOCK_REF_KEY)
		ret = run_delayed_tree_ref(trans, root, node, extent_op,
					   insert_reserved);
	else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
		 node->type == BTRFS_SHARED_DATA_REF_KEY)
		ret = run_delayed_data_ref(trans, root, node, extent_op,
					   insert_reserved);
	else
		BUG();
	return ret;
2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196
}

static noinline struct btrfs_delayed_ref_node *
select_delayed_ref(struct btrfs_delayed_ref_head *head)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_node *ref;
	int action = BTRFS_ADD_DELAYED_REF;
again:
	/*
	 * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
	 * this prevents ref count from going down to zero when
	 * there still are pending delayed ref.
	 */
	node = rb_prev(&head->node.rb_node);
	while (1) {
		if (!node)
			break;
		ref = rb_entry(node, struct btrfs_delayed_ref_node,
				rb_node);
		if (ref->bytenr != head->node.bytenr)
			break;
2197
		if (ref->action == action)
2198 2199 2200 2201 2202 2203 2204 2205 2206 2207
			return ref;
		node = rb_prev(node);
	}
	if (action == BTRFS_ADD_DELAYED_REF) {
		action = BTRFS_DROP_DELAYED_REF;
		goto again;
	}
	return NULL;
}

2208 2209 2210 2211
/*
 * Returns 0 on success or if called with an already aborted transaction.
 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
 */
2212 2213 2214
static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
				       struct btrfs_root *root,
				       struct list_head *cluster)
2215 2216 2217 2218
{
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_ref_head *locked_ref = NULL;
2219
	struct btrfs_delayed_extent_op *extent_op;
2220
	int ret;
2221
	int count = 0;
2222 2223 2224 2225 2226
	int must_insert_reserved = 0;

	delayed_refs = &trans->transaction->delayed_refs;
	while (1) {
		if (!locked_ref) {
2227 2228
			/* pick a new head ref from the cluster list */
			if (list_empty(cluster))
2229 2230
				break;

2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247
			locked_ref = list_entry(cluster->next,
				     struct btrfs_delayed_ref_head, cluster);

			/* grab the lock that says we are going to process
			 * all the refs for this head */
			ret = btrfs_delayed_ref_lock(trans, locked_ref);

			/*
			 * we may have dropped the spin lock to get the head
			 * mutex lock, and that might have given someone else
			 * time to free the head.  If that's true, it has been
			 * removed from our list and we can move on.
			 */
			if (ret == -EAGAIN) {
				locked_ref = NULL;
				count++;
				continue;
2248 2249
			}
		}
2250

2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272
		/*
		 * locked_ref is the head node, so we have to go one
		 * node back for any delayed ref updates
		 */
		ref = select_delayed_ref(locked_ref);

		if (ref && ref->seq &&
		    btrfs_check_delayed_seq(delayed_refs, ref->seq)) {
			/*
			 * there are still refs with lower seq numbers in the
			 * process of being added. Don't run this ref yet.
			 */
			list_del_init(&locked_ref->cluster);
			mutex_unlock(&locked_ref->mutex);
			locked_ref = NULL;
			delayed_refs->num_heads_ready++;
			spin_unlock(&delayed_refs->lock);
			cond_resched();
			spin_lock(&delayed_refs->lock);
			continue;
		}

2273 2274 2275 2276 2277 2278
		/*
		 * record the must insert reserved flag before we
		 * drop the spin lock.
		 */
		must_insert_reserved = locked_ref->must_insert_reserved;
		locked_ref->must_insert_reserved = 0;
2279

2280 2281 2282
		extent_op = locked_ref->extent_op;
		locked_ref->extent_op = NULL;

2283 2284 2285 2286 2287 2288
		if (!ref) {
			/* All delayed refs have been processed, Go ahead
			 * and send the head node to run_one_delayed_ref,
			 * so that any accounting fixes can happen
			 */
			ref = &locked_ref->node;
2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301

			if (extent_op && must_insert_reserved) {
				kfree(extent_op);
				extent_op = NULL;
			}

			if (extent_op) {
				spin_unlock(&delayed_refs->lock);

				ret = run_delayed_extent_op(trans, root,
							    ref, extent_op);
				kfree(extent_op);

2302 2303
				if (ret) {
					printk(KERN_DEBUG "btrfs: run_delayed_extent_op returned %d\n", ret);
2304
					spin_lock(&delayed_refs->lock);
2305 2306 2307
					return ret;
				}

2308
				goto next;
2309 2310
			}

2311
			list_del_init(&locked_ref->cluster);
2312 2313
			locked_ref = NULL;
		}
C
Chris Mason 已提交
2314

2315 2316 2317
		ref->in_tree = 0;
		rb_erase(&ref->rb_node, &delayed_refs->root);
		delayed_refs->num_entries--;
2318 2319 2320 2321 2322 2323
		/*
		 * we modified num_entries, but as we're currently running
		 * delayed refs, skip
		 *     wake_up(&delayed_refs->seq_wait);
		 * here.
		 */
2324
		spin_unlock(&delayed_refs->lock);
2325

2326
		ret = run_one_delayed_ref(trans, root, ref, extent_op,
2327
					  must_insert_reserved);
2328

2329 2330
		btrfs_put_delayed_ref(ref);
		kfree(extent_op);
2331
		count++;
2332 2333 2334

		if (ret) {
			printk(KERN_DEBUG "btrfs: run_one_delayed_ref returned %d\n", ret);
2335
			spin_lock(&delayed_refs->lock);
2336 2337 2338
			return ret;
		}

2339 2340 2341 2342 2343
next:
		do_chunk_alloc(trans, root->fs_info->extent_root,
			       2 * 1024 * 1024,
			       btrfs_get_alloc_profile(root, 0),
			       CHUNK_ALLOC_NO_FORCE);
2344 2345 2346 2347 2348 2349
		cond_resched();
		spin_lock(&delayed_refs->lock);
	}
	return count;
}

2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366

static void wait_for_more_refs(struct btrfs_delayed_ref_root *delayed_refs,
			unsigned long num_refs)
{
	struct list_head *first_seq = delayed_refs->seq_head.next;

	spin_unlock(&delayed_refs->lock);
	pr_debug("waiting for more refs (num %ld, first %p)\n",
		 num_refs, first_seq);
	wait_event(delayed_refs->seq_wait,
		   num_refs != delayed_refs->num_entries ||
		   delayed_refs->seq_head.next != first_seq);
	pr_debug("done waiting for more refs (num %ld, first %p)\n",
		 delayed_refs->num_entries, delayed_refs->seq_head.next);
	spin_lock(&delayed_refs->lock);
}

2367 2368 2369 2370 2371 2372
/*
 * this starts processing the delayed reference count updates and
 * extent insertions we have queued up so far.  count can be
 * 0, which means to process everything in the tree at the start
 * of the run (but not newly added entries), or it can be some target
 * number you'd like to process.
2373 2374 2375
 *
 * Returns 0 on success or if called with an aborted transaction
 * Returns <0 on error and aborts the transaction
2376 2377 2378 2379 2380 2381 2382 2383 2384
 */
int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, unsigned long count)
{
	struct rb_node *node;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct list_head cluster;
	int ret;
2385
	u64 delayed_start;
2386 2387
	int run_all = count == (unsigned long)-1;
	int run_most = 0;
2388 2389
	unsigned long num_refs = 0;
	int consider_waiting;
2390

2391 2392 2393 2394
	/* We'll clean this up in btrfs_cleanup_transaction */
	if (trans->aborted)
		return 0;

2395 2396 2397
	if (root == root->fs_info->extent_root)
		root = root->fs_info->tree_root;

2398 2399 2400 2401
	do_chunk_alloc(trans, root->fs_info->extent_root,
		       2 * 1024 * 1024, btrfs_get_alloc_profile(root, 0),
		       CHUNK_ALLOC_NO_FORCE);

2402 2403 2404
	delayed_refs = &trans->transaction->delayed_refs;
	INIT_LIST_HEAD(&cluster);
again:
2405
	consider_waiting = 0;
2406 2407 2408 2409 2410 2411 2412 2413 2414
	spin_lock(&delayed_refs->lock);
	if (count == 0) {
		count = delayed_refs->num_entries * 2;
		run_most = 1;
	}
	while (1) {
		if (!(run_all || run_most) &&
		    delayed_refs->num_heads_ready < 64)
			break;
2415

2416
		/*
2417 2418 2419 2420
		 * go find something we can process in the rbtree.  We start at
		 * the beginning of the tree, and then build a cluster
		 * of refs to process starting at the first one we are able to
		 * lock
2421
		 */
2422
		delayed_start = delayed_refs->run_delayed_start;
2423 2424 2425
		ret = btrfs_find_ref_cluster(trans, &cluster,
					     delayed_refs->run_delayed_start);
		if (ret)
2426 2427
			break;

2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450
		if (delayed_start >= delayed_refs->run_delayed_start) {
			if (consider_waiting == 0) {
				/*
				 * btrfs_find_ref_cluster looped. let's do one
				 * more cycle. if we don't run any delayed ref
				 * during that cycle (because we can't because
				 * all of them are blocked) and if the number of
				 * refs doesn't change, we avoid busy waiting.
				 */
				consider_waiting = 1;
				num_refs = delayed_refs->num_entries;
			} else {
				wait_for_more_refs(delayed_refs, num_refs);
				/*
				 * after waiting, things have changed. we
				 * dropped the lock and someone else might have
				 * run some refs, built new clusters and so on.
				 * therefore, we restart staleness detection.
				 */
				consider_waiting = 0;
			}
		}

2451
		ret = run_clustered_refs(trans, root, &cluster);
2452 2453 2454 2455 2456
		if (ret < 0) {
			spin_unlock(&delayed_refs->lock);
			btrfs_abort_transaction(trans, root, ret);
			return ret;
		}
2457 2458 2459 2460 2461

		count -= min_t(unsigned long, ret, count);

		if (count == 0)
			break;
2462 2463 2464 2465 2466

		if (ret || delayed_refs->run_delayed_start == 0) {
			/* refs were run, let's reset staleness detection */
			consider_waiting = 0;
		}
2467
	}
2468

2469 2470
	if (run_all) {
		node = rb_first(&delayed_refs->root);
2471
		if (!node)
2472
			goto out;
2473
		count = (unsigned long)-1;
2474

2475 2476 2477 2478 2479
		while (node) {
			ref = rb_entry(node, struct btrfs_delayed_ref_node,
				       rb_node);
			if (btrfs_delayed_ref_is_head(ref)) {
				struct btrfs_delayed_ref_head *head;
2480

2481 2482 2483 2484
				head = btrfs_delayed_node_to_head(ref);
				atomic_inc(&ref->refs);

				spin_unlock(&delayed_refs->lock);
2485 2486 2487 2488
				/*
				 * Mutex was contended, block until it's
				 * released and try again
				 */
2489 2490 2491 2492
				mutex_lock(&head->mutex);
				mutex_unlock(&head->mutex);

				btrfs_put_delayed_ref(ref);
2493
				cond_resched();
2494 2495 2496 2497 2498 2499 2500
				goto again;
			}
			node = rb_next(node);
		}
		spin_unlock(&delayed_refs->lock);
		schedule_timeout(1);
		goto again;
2501
	}
2502
out:
2503
	spin_unlock(&delayed_refs->lock);
2504 2505 2506
	return 0;
}

2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 flags,
				int is_data)
{
	struct btrfs_delayed_extent_op *extent_op;
	int ret;

	extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
	if (!extent_op)
		return -ENOMEM;

	extent_op->flags_to_set = flags;
	extent_op->update_flags = 1;
	extent_op->update_key = 0;
	extent_op->is_data = is_data ? 1 : 0;

A
Arne Jansen 已提交
2524 2525
	ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
					  num_bytes, extent_op);
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553
	if (ret)
		kfree(extent_op);
	return ret;
}

static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      struct btrfs_path *path,
				      u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_node *ref;
	struct btrfs_delayed_data_ref *data_ref;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct rb_node *node;
	int ret = 0;

	ret = -ENOENT;
	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	if (!mutex_trylock(&head->mutex)) {
		atomic_inc(&head->node.refs);
		spin_unlock(&delayed_refs->lock);

2554
		btrfs_release_path(path);
2555

2556 2557 2558 2559
		/*
		 * Mutex was contended, block until it's released and let
		 * caller try again
		 */
2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603
		mutex_lock(&head->mutex);
		mutex_unlock(&head->mutex);
		btrfs_put_delayed_ref(&head->node);
		return -EAGAIN;
	}

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out_unlock;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	if (ref->bytenr != bytenr)
		goto out_unlock;

	ret = 1;
	if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
		goto out_unlock;

	data_ref = btrfs_delayed_node_to_data_ref(ref);

	node = rb_prev(node);
	if (node) {
		ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
		if (ref->bytenr == bytenr)
			goto out_unlock;
	}

	if (data_ref->root != root->root_key.objectid ||
	    data_ref->objectid != objectid || data_ref->offset != offset)
		goto out_unlock;

	ret = 0;
out_unlock:
	mutex_unlock(&head->mutex);
out:
	spin_unlock(&delayed_refs->lock);
	return ret;
}

static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
					struct btrfs_root *root,
					struct btrfs_path *path,
					u64 objectid, u64 offset, u64 bytenr)
2604 2605
{
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2606
	struct extent_buffer *leaf;
2607 2608 2609
	struct btrfs_extent_data_ref *ref;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_extent_item *ei;
2610
	struct btrfs_key key;
2611
	u32 item_size;
2612
	int ret;
2613

2614
	key.objectid = bytenr;
Z
Zheng Yan 已提交
2615
	key.offset = (u64)-1;
2616
	key.type = BTRFS_EXTENT_ITEM_KEY;
2617 2618 2619 2620

	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
	if (ret < 0)
		goto out;
2621
	BUG_ON(ret == 0); /* Corruption */
Y
Yan Zheng 已提交
2622 2623 2624

	ret = -ENOENT;
	if (path->slots[0] == 0)
Z
Zheng Yan 已提交
2625
		goto out;
2626

Z
Zheng Yan 已提交
2627
	path->slots[0]--;
2628
	leaf = path->nodes[0];
2629
	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2630

2631
	if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2632
		goto out;
2633

2634 2635 2636 2637 2638 2639 2640 2641 2642
	ret = 1;
	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
		goto out;
	}
#endif
	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2643

2644 2645 2646
	if (item_size != sizeof(*ei) +
	    btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
		goto out;
2647

2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686
	if (btrfs_extent_generation(leaf, ei) <=
	    btrfs_root_last_snapshot(&root->root_item))
		goto out;

	iref = (struct btrfs_extent_inline_ref *)(ei + 1);
	if (btrfs_extent_inline_ref_type(leaf, iref) !=
	    BTRFS_EXTENT_DATA_REF_KEY)
		goto out;

	ref = (struct btrfs_extent_data_ref *)(&iref->offset);
	if (btrfs_extent_refs(leaf, ei) !=
	    btrfs_extent_data_ref_count(leaf, ref) ||
	    btrfs_extent_data_ref_root(leaf, ref) !=
	    root->root_key.objectid ||
	    btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
	    btrfs_extent_data_ref_offset(leaf, ref) != offset)
		goto out;

	ret = 0;
out:
	return ret;
}

int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
			  struct btrfs_root *root,
			  u64 objectid, u64 offset, u64 bytenr)
{
	struct btrfs_path *path;
	int ret;
	int ret2;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOENT;

	do {
		ret = check_committed_ref(trans, root, path, objectid,
					  offset, bytenr);
		if (ret && ret != -ENOENT)
2687
			goto out;
Y
Yan Zheng 已提交
2688

2689 2690 2691 2692 2693 2694 2695
		ret2 = check_delayed_ref(trans, root, path, objectid,
					 offset, bytenr);
	} while (ret2 == -EAGAIN);

	if (ret2 && ret2 != -ENOENT) {
		ret = ret2;
		goto out;
2696
	}
2697 2698 2699

	if (ret != -ENOENT || ret2 != -ENOENT)
		ret = 0;
2700
out:
Y
Yan Zheng 已提交
2701
	btrfs_free_path(path);
2702 2703
	if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
		WARN_ON(ret > 0);
2704
	return ret;
2705
}
C
Chris Mason 已提交
2706

2707
static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2708
			   struct btrfs_root *root,
2709
			   struct extent_buffer *buf,
A
Arne Jansen 已提交
2710
			   int full_backref, int inc, int for_cow)
Z
Zheng Yan 已提交
2711 2712
{
	u64 bytenr;
2713 2714
	u64 num_bytes;
	u64 parent;
Z
Zheng Yan 已提交
2715 2716 2717 2718 2719 2720 2721 2722
	u64 ref_root;
	u32 nritems;
	struct btrfs_key key;
	struct btrfs_file_extent_item *fi;
	int i;
	int level;
	int ret = 0;
	int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
A
Arne Jansen 已提交
2723
			    u64, u64, u64, u64, u64, u64, int);
Z
Zheng Yan 已提交
2724 2725 2726 2727 2728

	ref_root = btrfs_header_owner(buf);
	nritems = btrfs_header_nritems(buf);
	level = btrfs_header_level(buf);

2729 2730
	if (!root->ref_cows && level == 0)
		return 0;
Z
Zheng Yan 已提交
2731

2732 2733 2734 2735
	if (inc)
		process_func = btrfs_inc_extent_ref;
	else
		process_func = btrfs_free_extent;
Z
Zheng Yan 已提交
2736

2737 2738 2739 2740 2741 2742
	if (full_backref)
		parent = buf->start;
	else
		parent = 0;

	for (i = 0; i < nritems; i++) {
Z
Zheng Yan 已提交
2743
		if (level == 0) {
2744
			btrfs_item_key_to_cpu(buf, &key, i);
Z
Zheng Yan 已提交
2745 2746
			if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
				continue;
2747
			fi = btrfs_item_ptr(buf, i,
Z
Zheng Yan 已提交
2748 2749 2750 2751 2752 2753 2754
					    struct btrfs_file_extent_item);
			if (btrfs_file_extent_type(buf, fi) ==
			    BTRFS_FILE_EXTENT_INLINE)
				continue;
			bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
			if (bytenr == 0)
				continue;
2755 2756 2757 2758 2759

			num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
			key.offset -= btrfs_file_extent_offset(buf, fi);
			ret = process_func(trans, root, bytenr, num_bytes,
					   parent, ref_root, key.objectid,
A
Arne Jansen 已提交
2760
					   key.offset, for_cow);
Z
Zheng Yan 已提交
2761 2762 2763
			if (ret)
				goto fail;
		} else {
2764 2765 2766
			bytenr = btrfs_node_blockptr(buf, i);
			num_bytes = btrfs_level_size(root, level - 1);
			ret = process_func(trans, root, bytenr, num_bytes,
A
Arne Jansen 已提交
2767 2768
					   parent, ref_root, level - 1, 0,
					   for_cow);
Z
Zheng Yan 已提交
2769 2770 2771 2772 2773 2774
			if (ret)
				goto fail;
		}
	}
	return 0;
fail:
2775 2776 2777 2778
	return ret;
}

int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
A
Arne Jansen 已提交
2779
		  struct extent_buffer *buf, int full_backref, int for_cow)
2780
{
A
Arne Jansen 已提交
2781
	return __btrfs_mod_ref(trans, root, buf, full_backref, 1, for_cow);
2782 2783 2784
}

int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
A
Arne Jansen 已提交
2785
		  struct extent_buffer *buf, int full_backref, int for_cow)
2786
{
A
Arne Jansen 已提交
2787
	return __btrfs_mod_ref(trans, root, buf, full_backref, 0, for_cow);
Z
Zheng Yan 已提交
2788 2789
}

C
Chris Mason 已提交
2790 2791 2792 2793 2794 2795 2796
static int write_one_cache_group(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct btrfs_block_group_cache *cache)
{
	int ret;
	struct btrfs_root *extent_root = root->fs_info->extent_root;
2797 2798
	unsigned long bi;
	struct extent_buffer *leaf;
C
Chris Mason 已提交
2799 2800

	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2801 2802
	if (ret < 0)
		goto fail;
2803
	BUG_ON(ret); /* Corruption */
2804 2805 2806 2807 2808

	leaf = path->nodes[0];
	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
	btrfs_mark_buffer_dirty(leaf);
2809
	btrfs_release_path(path);
2810
fail:
2811 2812
	if (ret) {
		btrfs_abort_transaction(trans, root, ret);
C
Chris Mason 已提交
2813
		return ret;
2814
	}
C
Chris Mason 已提交
2815 2816 2817 2818
	return 0;

}

2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829
static struct btrfs_block_group_cache *
next_block_group(struct btrfs_root *root,
		 struct btrfs_block_group_cache *cache)
{
	struct rb_node *node;
	spin_lock(&root->fs_info->block_group_cache_lock);
	node = rb_next(&cache->cache_node);
	btrfs_put_block_group(cache);
	if (node) {
		cache = rb_entry(node, struct btrfs_block_group_cache,
				 cache_node);
2830
		btrfs_get_block_group(cache);
2831 2832 2833 2834 2835 2836
	} else
		cache = NULL;
	spin_unlock(&root->fs_info->block_group_cache_lock);
	return cache;
}

2837 2838 2839 2840 2841 2842 2843
static int cache_save_setup(struct btrfs_block_group_cache *block_group,
			    struct btrfs_trans_handle *trans,
			    struct btrfs_path *path)
{
	struct btrfs_root *root = block_group->fs_info->tree_root;
	struct inode *inode = NULL;
	u64 alloc_hint = 0;
2844
	int dcs = BTRFS_DC_ERROR;
2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863
	int num_pages = 0;
	int retries = 0;
	int ret = 0;

	/*
	 * If this block group is smaller than 100 megs don't bother caching the
	 * block group.
	 */
	if (block_group->key.offset < (100 * 1024 * 1024)) {
		spin_lock(&block_group->lock);
		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
		spin_unlock(&block_group->lock);
		return 0;
	}

again:
	inode = lookup_free_space_inode(root, block_group, path);
	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
		ret = PTR_ERR(inode);
2864
		btrfs_release_path(path);
2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880
		goto out;
	}

	if (IS_ERR(inode)) {
		BUG_ON(retries);
		retries++;

		if (block_group->ro)
			goto out_free;

		ret = create_free_space_inode(root, trans, block_group, path);
		if (ret)
			goto out_free;
		goto again;
	}

2881 2882 2883 2884 2885 2886 2887
	/* We've already setup this transaction, go ahead and exit */
	if (block_group->cache_generation == trans->transid &&
	    i_size_read(inode)) {
		dcs = BTRFS_DC_SETUP;
		goto out_put;
	}

2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905
	/*
	 * We want to set the generation to 0, that way if anything goes wrong
	 * from here on out we know not to trust this cache when we load up next
	 * time.
	 */
	BTRFS_I(inode)->generation = 0;
	ret = btrfs_update_inode(trans, root, inode);
	WARN_ON(ret);

	if (i_size_read(inode) > 0) {
		ret = btrfs_truncate_free_space_cache(root, trans, path,
						      inode);
		if (ret)
			goto out_put;
	}

	spin_lock(&block_group->lock);
	if (block_group->cached != BTRFS_CACHE_FINISHED) {
2906 2907
		/* We're not cached, don't bother trying to write stuff out */
		dcs = BTRFS_DC_WRITTEN;
2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933
		spin_unlock(&block_group->lock);
		goto out_put;
	}
	spin_unlock(&block_group->lock);

	num_pages = (int)div64_u64(block_group->key.offset, 1024 * 1024 * 1024);
	if (!num_pages)
		num_pages = 1;

	/*
	 * Just to make absolutely sure we have enough space, we're going to
	 * preallocate 12 pages worth of space for each block group.  In
	 * practice we ought to use at most 8, but we need extra space so we can
	 * add our header and have a terminator between the extents and the
	 * bitmaps.
	 */
	num_pages *= 16;
	num_pages *= PAGE_CACHE_SIZE;

	ret = btrfs_check_data_free_space(inode, num_pages);
	if (ret)
		goto out_put;

	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
					      num_pages, num_pages,
					      &alloc_hint);
2934 2935
	if (!ret)
		dcs = BTRFS_DC_SETUP;
2936
	btrfs_free_reserved_data_space(inode, num_pages);
2937

2938 2939 2940
out_put:
	iput(inode);
out_free:
2941
	btrfs_release_path(path);
2942 2943
out:
	spin_lock(&block_group->lock);
2944
	if (!ret && dcs == BTRFS_DC_SETUP)
2945
		block_group->cache_generation = trans->transid;
2946
	block_group->disk_cache_state = dcs;
2947 2948 2949 2950 2951
	spin_unlock(&block_group->lock);

	return ret;
}

2952 2953
int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root)
C
Chris Mason 已提交
2954
{
2955
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
2956 2957
	int err = 0;
	struct btrfs_path *path;
2958
	u64 last = 0;
C
Chris Mason 已提交
2959 2960 2961 2962 2963

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982
again:
	while (1) {
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			if (cache->disk_cache_state == BTRFS_DC_CLEAR)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
		err = cache_save_setup(cache, trans, path);
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}

C
Chris Mason 已提交
2983
	while (1) {
2984 2985 2986
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
2987 2988
			if (err) /* File system offline */
				goto out;
J
Josef Bacik 已提交
2989
		}
2990

2991 2992
		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
2993 2994 2995 2996 2997
			if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
				btrfs_put_block_group(cache);
				goto again;
			}

2998 2999 3000 3001 3002 3003 3004 3005 3006 3007
			if (cache->dirty)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}
J
Josef Bacik 已提交
3008

J
Josef Bacik 已提交
3009 3010
		if (cache->disk_cache_state == BTRFS_DC_SETUP)
			cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
3011
		cache->dirty = 0;
3012
		last = cache->key.objectid + cache->key.offset;
J
Josef Bacik 已提交
3013

3014
		err = write_one_cache_group(trans, root, path, cache);
3015 3016 3017
		if (err) /* File system offline */
			goto out;

3018
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
3019
	}
3020

J
Josef Bacik 已提交
3021 3022 3023 3024 3025 3026 3027 3028 3029
	while (1) {
		/*
		 * I don't think this is needed since we're just marking our
		 * preallocated extent as written, but just in case it can't
		 * hurt.
		 */
		if (last == 0) {
			err = btrfs_run_delayed_refs(trans, root,
						     (unsigned long)-1);
3030 3031
			if (err) /* File system offline */
				goto out;
J
Josef Bacik 已提交
3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055
		}

		cache = btrfs_lookup_first_block_group(root->fs_info, last);
		while (cache) {
			/*
			 * Really this shouldn't happen, but it could if we
			 * couldn't write the entire preallocated extent and
			 * splitting the extent resulted in a new block.
			 */
			if (cache->dirty) {
				btrfs_put_block_group(cache);
				goto again;
			}
			if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
				break;
			cache = next_block_group(root, cache);
		}
		if (!cache) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

3056
		err = btrfs_write_out_cache(root, trans, cache, path);
J
Josef Bacik 已提交
3057 3058 3059 3060 3061

		/*
		 * If we didn't have an error then the cache state is still
		 * NEED_WRITE, so we can set it to WRITTEN.
		 */
3062
		if (!err && cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
J
Josef Bacik 已提交
3063 3064 3065 3066
			cache->disk_cache_state = BTRFS_DC_WRITTEN;
		last = cache->key.objectid + cache->key.offset;
		btrfs_put_block_group(cache);
	}
3067
out:
J
Josef Bacik 已提交
3068

C
Chris Mason 已提交
3069
	btrfs_free_path(path);
3070
	return err;
C
Chris Mason 已提交
3071 3072
}

3073 3074 3075 3076 3077 3078 3079 3080 3081
int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_block_group_cache *block_group;
	int readonly = 0;

	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
	if (!block_group || block_group->ro)
		readonly = 1;
	if (block_group)
3082
		btrfs_put_block_group(block_group);
3083 3084 3085
	return readonly;
}

3086 3087 3088 3089 3090
static int update_space_info(struct btrfs_fs_info *info, u64 flags,
			     u64 total_bytes, u64 bytes_used,
			     struct btrfs_space_info **space_info)
{
	struct btrfs_space_info *found;
3091 3092 3093 3094 3095 3096 3097 3098
	int i;
	int factor;

	if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
		     BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
3099 3100 3101

	found = __find_space_info(info, flags);
	if (found) {
3102
		spin_lock(&found->lock);
3103
		found->total_bytes += total_bytes;
J
Josef Bacik 已提交
3104
		found->disk_total += total_bytes * factor;
3105
		found->bytes_used += bytes_used;
3106
		found->disk_used += bytes_used * factor;
3107
		found->full = 0;
3108
		spin_unlock(&found->lock);
3109 3110 3111
		*space_info = found;
		return 0;
	}
Y
Yan Zheng 已提交
3112
	found = kzalloc(sizeof(*found), GFP_NOFS);
3113 3114 3115
	if (!found)
		return -ENOMEM;

3116 3117
	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		INIT_LIST_HEAD(&found->block_groups[i]);
3118
	init_rwsem(&found->groups_sem);
J
Josef Bacik 已提交
3119
	spin_lock_init(&found->lock);
3120
	found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3121
	found->total_bytes = total_bytes;
J
Josef Bacik 已提交
3122
	found->disk_total = total_bytes * factor;
3123
	found->bytes_used = bytes_used;
3124
	found->disk_used = bytes_used * factor;
3125
	found->bytes_pinned = 0;
3126
	found->bytes_reserved = 0;
Y
Yan Zheng 已提交
3127
	found->bytes_readonly = 0;
3128
	found->bytes_may_use = 0;
3129
	found->full = 0;
3130
	found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3131
	found->chunk_alloc = 0;
3132 3133
	found->flush = 0;
	init_waitqueue_head(&found->wait);
3134
	*space_info = found;
3135
	list_add_rcu(&found->list, &info->space_info);
3136 3137 3138
	return 0;
}

3139 3140
static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
3141 3142
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
3143 3144 3145 3146 3147 3148 3149

	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits |= extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits |= extra_flags;
3150
}
3151

3152 3153 3154
/*
 * returns target flags in extended format or 0 if restripe for this
 * chunk_type is not in progress
3155 3156
 *
 * should be called with either volume_mutex or balance_lock held
3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179
 */
static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
{
	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
	u64 target = 0;

	if (!bctl)
		return 0;

	if (flags & BTRFS_BLOCK_GROUP_DATA &&
	    bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
	} else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
		   bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
	} else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
		   bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
		target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
	}

	return target;
}

3180 3181 3182
/*
 * @flags: available profiles in extended format (see ctree.h)
 *
3183 3184 3185
 * Returns reduced profile in chunk format.  If profile changing is in
 * progress (either running or paused) picks the target profile (if it's
 * already available), otherwise falls back to plain reducing.
3186
 */
Y
Yan Zheng 已提交
3187
u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3188
{
3189 3190 3191 3192 3193 3194 3195
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	u64 num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;
3196
	u64 target;
3197

3198 3199 3200 3201
	/*
	 * see if restripe for this chunk_type is in progress, if so
	 * try to reduce to the target profile
	 */
3202
	spin_lock(&root->fs_info->balance_lock);
3203 3204 3205 3206
	target = get_restripe_target(root->fs_info, flags);
	if (target) {
		/* pick target profile only if it's already available */
		if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
3207
			spin_unlock(&root->fs_info->balance_lock);
3208
			return extended_to_chunk(target);
3209 3210 3211 3212
		}
	}
	spin_unlock(&root->fs_info->balance_lock);

3213 3214 3215 3216 3217
	if (num_devices == 1)
		flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0);
	if (num_devices < 4)
		flags &= ~BTRFS_BLOCK_GROUP_RAID10;

3218 3219
	if ((flags & BTRFS_BLOCK_GROUP_DUP) &&
	    (flags & (BTRFS_BLOCK_GROUP_RAID1 |
3220
		      BTRFS_BLOCK_GROUP_RAID10))) {
3221
		flags &= ~BTRFS_BLOCK_GROUP_DUP;
3222
	}
3223 3224

	if ((flags & BTRFS_BLOCK_GROUP_RAID1) &&
3225
	    (flags & BTRFS_BLOCK_GROUP_RAID10)) {
3226
		flags &= ~BTRFS_BLOCK_GROUP_RAID1;
3227
	}
3228 3229 3230 3231

	if ((flags & BTRFS_BLOCK_GROUP_RAID0) &&
	    ((flags & BTRFS_BLOCK_GROUP_RAID1) |
	     (flags & BTRFS_BLOCK_GROUP_RAID10) |
3232
	     (flags & BTRFS_BLOCK_GROUP_DUP))) {
3233
		flags &= ~BTRFS_BLOCK_GROUP_RAID0;
3234 3235
	}

3236
	return extended_to_chunk(flags);
3237 3238
}

3239
static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
J
Josef Bacik 已提交
3240
{
3241
	if (flags & BTRFS_BLOCK_GROUP_DATA)
3242
		flags |= root->fs_info->avail_data_alloc_bits;
3243
	else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3244
		flags |= root->fs_info->avail_system_alloc_bits;
3245
	else if (flags & BTRFS_BLOCK_GROUP_METADATA)
3246 3247
		flags |= root->fs_info->avail_metadata_alloc_bits;

3248
	return btrfs_reduce_alloc_profile(root, flags);
J
Josef Bacik 已提交
3249 3250
}

3251
u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
J
Josef Bacik 已提交
3252
{
3253
	u64 flags;
J
Josef Bacik 已提交
3254

3255 3256 3257 3258
	if (data)
		flags = BTRFS_BLOCK_GROUP_DATA;
	else if (root == root->fs_info->chunk_root)
		flags = BTRFS_BLOCK_GROUP_SYSTEM;
J
Josef Bacik 已提交
3259
	else
3260
		flags = BTRFS_BLOCK_GROUP_METADATA;
J
Josef Bacik 已提交
3261

3262
	return get_alloc_profile(root, flags);
J
Josef Bacik 已提交
3263
}
J
Josef Bacik 已提交
3264

J
Josef Bacik 已提交
3265 3266 3267
void btrfs_set_inode_space_info(struct btrfs_root *root, struct inode *inode)
{
	BTRFS_I(inode)->space_info = __find_space_info(root->fs_info,
3268
						       BTRFS_BLOCK_GROUP_DATA);
J
Josef Bacik 已提交
3269 3270
}

J
Josef Bacik 已提交
3271 3272 3273 3274
/*
 * This will check the space that the inode allocates from to make sure we have
 * enough space for bytes.
 */
3275
int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
J
Josef Bacik 已提交
3276 3277
{
	struct btrfs_space_info *data_sinfo;
3278
	struct btrfs_root *root = BTRFS_I(inode)->root;
3279
	u64 used;
3280
	int ret = 0, committed = 0, alloc_chunk = 1;
J
Josef Bacik 已提交
3281 3282 3283 3284

	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);

3285 3286
	if (root == root->fs_info->tree_root ||
	    BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3287 3288 3289 3290
		alloc_chunk = 0;
		committed = 1;
	}

J
Josef Bacik 已提交
3291
	data_sinfo = BTRFS_I(inode)->space_info;
C
Chris Mason 已提交
3292 3293
	if (!data_sinfo)
		goto alloc;
J
Josef Bacik 已提交
3294

J
Josef Bacik 已提交
3295 3296 3297
again:
	/* make sure we have enough space to handle the data first */
	spin_lock(&data_sinfo->lock);
3298 3299 3300
	used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
		data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
		data_sinfo->bytes_may_use;
3301 3302

	if (used + bytes > data_sinfo->total_bytes) {
3303
		struct btrfs_trans_handle *trans;
J
Josef Bacik 已提交
3304

J
Josef Bacik 已提交
3305 3306 3307 3308
		/*
		 * if we don't have enough free bytes in this space then we need
		 * to alloc a new chunk.
		 */
3309
		if (!data_sinfo->full && alloc_chunk) {
J
Josef Bacik 已提交
3310
			u64 alloc_target;
J
Josef Bacik 已提交
3311

3312
			data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
J
Josef Bacik 已提交
3313
			spin_unlock(&data_sinfo->lock);
C
Chris Mason 已提交
3314
alloc:
J
Josef Bacik 已提交
3315
			alloc_target = btrfs_get_alloc_profile(root, 1);
3316
			trans = btrfs_join_transaction(root);
3317 3318
			if (IS_ERR(trans))
				return PTR_ERR(trans);
J
Josef Bacik 已提交
3319

J
Josef Bacik 已提交
3320 3321
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
					     bytes + 2 * 1024 * 1024,
3322 3323
					     alloc_target,
					     CHUNK_ALLOC_NO_FORCE);
J
Josef Bacik 已提交
3324
			btrfs_end_transaction(trans, root);
3325 3326 3327 3328 3329 3330
			if (ret < 0) {
				if (ret != -ENOSPC)
					return ret;
				else
					goto commit_trans;
			}
J
Josef Bacik 已提交
3331

C
Chris Mason 已提交
3332 3333 3334 3335
			if (!data_sinfo) {
				btrfs_set_inode_space_info(root, inode);
				data_sinfo = BTRFS_I(inode)->space_info;
			}
J
Josef Bacik 已提交
3336 3337
			goto again;
		}
3338 3339 3340 3341 3342 3343 3344

		/*
		 * If we have less pinned bytes than we want to allocate then
		 * don't bother committing the transaction, it won't help us.
		 */
		if (data_sinfo->bytes_pinned < bytes)
			committed = 1;
J
Josef Bacik 已提交
3345 3346
		spin_unlock(&data_sinfo->lock);

3347
		/* commit the current transaction and try again */
3348
commit_trans:
J
Josef Bacik 已提交
3349 3350
		if (!committed &&
		    !atomic_read(&root->fs_info->open_ioctl_trans)) {
3351
			committed = 1;
3352
			trans = btrfs_join_transaction(root);
3353 3354
			if (IS_ERR(trans))
				return PTR_ERR(trans);
3355 3356 3357 3358 3359
			ret = btrfs_commit_transaction(trans, root);
			if (ret)
				return ret;
			goto again;
		}
J
Josef Bacik 已提交
3360

J
Josef Bacik 已提交
3361 3362 3363
		return -ENOSPC;
	}
	data_sinfo->bytes_may_use += bytes;
J
Josef Bacik 已提交
3364
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3365
				      data_sinfo->flags, bytes, 1);
J
Josef Bacik 已提交
3366 3367
	spin_unlock(&data_sinfo->lock);

J
Josef Bacik 已提交
3368 3369
	return 0;
}
J
Josef Bacik 已提交
3370 3371

/*
3372
 * Called if we need to clear a data reservation for this inode.
J
Josef Bacik 已提交
3373
 */
3374
void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3375
{
3376
	struct btrfs_root *root = BTRFS_I(inode)->root;
J
Josef Bacik 已提交
3377
	struct btrfs_space_info *data_sinfo;
3378

J
Josef Bacik 已提交
3379 3380
	/* make sure bytes are sectorsize aligned */
	bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3381

J
Josef Bacik 已提交
3382 3383 3384
	data_sinfo = BTRFS_I(inode)->space_info;
	spin_lock(&data_sinfo->lock);
	data_sinfo->bytes_may_use -= bytes;
J
Josef Bacik 已提交
3385
	trace_btrfs_space_reservation(root->fs_info, "space_info",
3386
				      data_sinfo->flags, bytes, 0);
J
Josef Bacik 已提交
3387
	spin_unlock(&data_sinfo->lock);
3388 3389
}

3390
static void force_metadata_allocation(struct btrfs_fs_info *info)
3391
{
3392 3393
	struct list_head *head = &info->space_info;
	struct btrfs_space_info *found;
3394

3395 3396 3397
	rcu_read_lock();
	list_for_each_entry_rcu(found, head, list) {
		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3398
			found->force_alloc = CHUNK_ALLOC_FORCE;
3399
	}
3400
	rcu_read_unlock();
3401 3402
}

3403
static int should_alloc_chunk(struct btrfs_root *root,
3404 3405
			      struct btrfs_space_info *sinfo, u64 alloc_bytes,
			      int force)
3406
{
3407
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3408
	u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3409
	u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3410
	u64 thresh;
3411

3412 3413 3414
	if (force == CHUNK_ALLOC_FORCE)
		return 1;

3415 3416 3417 3418 3419 3420 3421
	/*
	 * We need to take into account the global rsv because for all intents
	 * and purposes it's used space.  Don't worry about locking the
	 * global_rsv, it doesn't change except when the transaction commits.
	 */
	num_allocated += global_rsv->size;

3422 3423 3424 3425 3426
	/*
	 * in limited mode, we want to have some free space up to
	 * about 1% of the FS size.
	 */
	if (force == CHUNK_ALLOC_LIMITED) {
3427
		thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3428 3429 3430 3431 3432 3433
		thresh = max_t(u64, 64 * 1024 * 1024,
			       div_factor_fine(thresh, 1));

		if (num_bytes - num_allocated < thresh)
			return 1;
	}
3434
	thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3435

3436 3437
	/* 256MB or 2% of the FS */
	thresh = max_t(u64, 256 * 1024 * 1024, div_factor_fine(thresh, 2));
C
Chris Mason 已提交
3438 3439 3440
	/* system chunks need a much small threshold */
	if (sinfo->flags & BTRFS_BLOCK_GROUP_SYSTEM)
		thresh = 32 * 1024 * 1024;
3441

3442
	if (num_bytes > thresh && sinfo->bytes_used < div_factor(num_bytes, 8))
3443
		return 0;
3444
	return 1;
3445 3446
}

3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490
static u64 get_system_chunk_thresh(struct btrfs_root *root, u64 type)
{
	u64 num_dev;

	if (type & BTRFS_BLOCK_GROUP_RAID10 ||
	    type & BTRFS_BLOCK_GROUP_RAID0)
		num_dev = root->fs_info->fs_devices->rw_devices;
	else if (type & BTRFS_BLOCK_GROUP_RAID1)
		num_dev = 2;
	else
		num_dev = 1;	/* DUP or single */

	/* metadata for updaing devices and chunk tree */
	return btrfs_calc_trans_metadata_size(root, num_dev + 1);
}

static void check_system_chunk(struct btrfs_trans_handle *trans,
			       struct btrfs_root *root, u64 type)
{
	struct btrfs_space_info *info;
	u64 left;
	u64 thresh;

	info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	spin_lock(&info->lock);
	left = info->total_bytes - info->bytes_used - info->bytes_pinned -
		info->bytes_reserved - info->bytes_readonly;
	spin_unlock(&info->lock);

	thresh = get_system_chunk_thresh(root, type);
	if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
		printk(KERN_INFO "left=%llu, need=%llu, flags=%llu\n",
		       left, thresh, type);
		dump_space_info(info, 0, 0);
	}

	if (left < thresh) {
		u64 flags;

		flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
		btrfs_alloc_chunk(trans, root, flags);
	}
}

3491 3492
static int do_chunk_alloc(struct btrfs_trans_handle *trans,
			  struct btrfs_root *extent_root, u64 alloc_bytes,
C
Chris Mason 已提交
3493
			  u64 flags, int force)
J
Josef Bacik 已提交
3494
{
3495
	struct btrfs_space_info *space_info;
3496
	struct btrfs_fs_info *fs_info = extent_root->fs_info;
3497
	int wait_for_alloc = 0;
J
Josef Bacik 已提交
3498 3499
	int ret = 0;

3500
	space_info = __find_space_info(extent_root->fs_info, flags);
3501 3502 3503
	if (!space_info) {
		ret = update_space_info(extent_root->fs_info, flags,
					0, 0, &space_info);
3504
		BUG_ON(ret); /* -ENOMEM */
J
Josef Bacik 已提交
3505
	}
3506
	BUG_ON(!space_info); /* Logic error */
J
Josef Bacik 已提交
3507

3508
again:
3509
	spin_lock(&space_info->lock);
3510
	if (force < space_info->force_alloc)
3511
		force = space_info->force_alloc;
3512 3513
	if (space_info->full) {
		spin_unlock(&space_info->lock);
3514
		return 0;
J
Josef Bacik 已提交
3515 3516
	}

3517
	if (!should_alloc_chunk(extent_root, space_info, alloc_bytes, force)) {
3518
		spin_unlock(&space_info->lock);
3519 3520 3521 3522 3523
		return 0;
	} else if (space_info->chunk_alloc) {
		wait_for_alloc = 1;
	} else {
		space_info->chunk_alloc = 1;
J
Josef Bacik 已提交
3524
	}
3525

3526
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3527

3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541
	mutex_lock(&fs_info->chunk_mutex);

	/*
	 * The chunk_mutex is held throughout the entirety of a chunk
	 * allocation, so once we've acquired the chunk_mutex we know that the
	 * other guy is done and we need to recheck and see if we should
	 * allocate.
	 */
	if (wait_for_alloc) {
		mutex_unlock(&fs_info->chunk_mutex);
		wait_for_alloc = 0;
		goto again;
	}

3542 3543 3544 3545 3546 3547 3548
	/*
	 * If we have mixed data/metadata chunks we want to make sure we keep
	 * allocating mixed chunks instead of individual chunks.
	 */
	if (btrfs_mixed_space_info(space_info))
		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);

3549 3550 3551 3552 3553
	/*
	 * if we're doing a data chunk, go ahead and make sure that
	 * we keep a reasonable number of metadata chunks allocated in the
	 * FS as well.
	 */
J
Josef Bacik 已提交
3554
	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3555 3556 3557 3558
		fs_info->data_chunk_allocations++;
		if (!(fs_info->data_chunk_allocations %
		      fs_info->metadata_ratio))
			force_metadata_allocation(fs_info);
J
Josef Bacik 已提交
3559 3560
	}

3561 3562 3563 3564 3565 3566
	/*
	 * Check if we have enough space in SYSTEM chunk because we may need
	 * to update devices.
	 */
	check_system_chunk(trans, extent_root, flags);

Y
Yan Zheng 已提交
3567
	ret = btrfs_alloc_chunk(trans, extent_root, flags);
3568 3569 3570
	if (ret < 0 && ret != -ENOSPC)
		goto out;

J
Josef Bacik 已提交
3571 3572
	spin_lock(&space_info->lock);
	if (ret)
3573
		space_info->full = 1;
3574 3575
	else
		ret = 1;
3576

3577
	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3578
	space_info->chunk_alloc = 0;
J
Josef Bacik 已提交
3579
	spin_unlock(&space_info->lock);
3580
out:
Y
Yan Zheng 已提交
3581
	mutex_unlock(&extent_root->fs_info->chunk_mutex);
J
Josef Bacik 已提交
3582
	return ret;
3583
}
J
Josef Bacik 已提交
3584 3585

/*
3586
 * shrink metadata reservation for delalloc
J
Josef Bacik 已提交
3587
 */
3588
static int shrink_delalloc(struct btrfs_root *root, u64 to_reclaim,
3589
			   bool wait_ordered)
3590
{
3591
	struct btrfs_block_rsv *block_rsv;
J
Josef Bacik 已提交
3592
	struct btrfs_space_info *space_info;
3593
	struct btrfs_trans_handle *trans;
3594 3595 3596
	u64 reserved;
	u64 max_reclaim;
	u64 reclaimed = 0;
3597
	long time_left;
3598
	unsigned long nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3599
	int loops = 0;
3600
	unsigned long progress;
3601

3602
	trans = (struct btrfs_trans_handle *)current->journal_info;
3603
	block_rsv = &root->fs_info->delalloc_block_rsv;
J
Josef Bacik 已提交
3604
	space_info = block_rsv->space_info;
3605 3606

	smp_mb();
3607
	reserved = space_info->bytes_may_use;
3608
	progress = space_info->reservation_progress;
3609 3610 3611

	if (reserved == 0)
		return 0;
3612

3613 3614 3615 3616 3617 3618 3619 3620
	smp_mb();
	if (root->fs_info->delalloc_bytes == 0) {
		if (trans)
			return 0;
		btrfs_wait_ordered_extents(root, 0, 0);
		return 0;
	}

3621
	max_reclaim = min(reserved, to_reclaim);
3622 3623
	nr_pages = max_t(unsigned long, nr_pages,
			 max_reclaim >> PAGE_CACHE_SHIFT);
3624
	while (loops < 1024) {
3625 3626 3627 3628
		/* have the flusher threads jump in and do some IO */
		smp_mb();
		nr_pages = min_t(unsigned long, nr_pages,
		       root->fs_info->delalloc_bytes >> PAGE_CACHE_SHIFT);
3629 3630
		writeback_inodes_sb_nr_if_idle(root->fs_info->sb, nr_pages,
						WB_REASON_FS_FREE_SPACE);
3631

J
Josef Bacik 已提交
3632
		spin_lock(&space_info->lock);
3633 3634 3635
		if (reserved > space_info->bytes_may_use)
			reclaimed += reserved - space_info->bytes_may_use;
		reserved = space_info->bytes_may_use;
J
Josef Bacik 已提交
3636
		spin_unlock(&space_info->lock);
3637

3638 3639
		loops++;

3640 3641 3642 3643 3644
		if (reserved == 0 || reclaimed >= max_reclaim)
			break;

		if (trans && trans->transaction->blocked)
			return -EAGAIN;
3645

3646 3647 3648 3649
		if (wait_ordered && !trans) {
			btrfs_wait_ordered_extents(root, 0, 0);
		} else {
			time_left = schedule_timeout_interruptible(1);
3650

3651 3652 3653 3654
			/* We were interrupted, exit */
			if (time_left)
				break;
		}
3655

3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666
		/* we've kicked the IO a few times, if anything has been freed,
		 * exit.  There is no sense in looping here for a long time
		 * when we really need to commit the transaction, or there are
		 * just too many writers without enough free space
		 */

		if (loops > 3) {
			smp_mb();
			if (progress != space_info->reservation_progress)
				break;
		}
3667

3668
	}
3669

3670 3671 3672
	return reclaimed >= to_reclaim;
}

3673 3674 3675 3676 3677
/**
 * maybe_commit_transaction - possibly commit the transaction if its ok to
 * @root - the root we're allocating for
 * @bytes - the number of bytes we want to reserve
 * @force - force the commit
3678
 *
3679 3680 3681
 * This will check to make sure that committing the transaction will actually
 * get us somewhere and then commit the transaction if it does.  Otherwise it
 * will return -ENOSPC.
3682
 */
3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711
static int may_commit_transaction(struct btrfs_root *root,
				  struct btrfs_space_info *space_info,
				  u64 bytes, int force)
{
	struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
	struct btrfs_trans_handle *trans;

	trans = (struct btrfs_trans_handle *)current->journal_info;
	if (trans)
		return -EAGAIN;

	if (force)
		goto commit;

	/* See if there is enough pinned space to make this reservation */
	spin_lock(&space_info->lock);
	if (space_info->bytes_pinned >= bytes) {
		spin_unlock(&space_info->lock);
		goto commit;
	}
	spin_unlock(&space_info->lock);

	/*
	 * See if there is some space in the delayed insertion reservation for
	 * this reservation.
	 */
	if (space_info != delayed_rsv->space_info)
		return -ENOSPC;

L
Liu Bo 已提交
3712
	spin_lock(&space_info->lock);
3713
	spin_lock(&delayed_rsv->lock);
L
Liu Bo 已提交
3714
	if (space_info->bytes_pinned + delayed_rsv->size < bytes) {
3715
		spin_unlock(&delayed_rsv->lock);
L
Liu Bo 已提交
3716
		spin_unlock(&space_info->lock);
3717 3718 3719
		return -ENOSPC;
	}
	spin_unlock(&delayed_rsv->lock);
L
Liu Bo 已提交
3720
	spin_unlock(&space_info->lock);
3721 3722 3723 3724 3725 3726 3727 3728 3729

commit:
	trans = btrfs_join_transaction(root);
	if (IS_ERR(trans))
		return -ENOSPC;

	return btrfs_commit_transaction(trans, root);
}

3730 3731 3732 3733 3734 3735
/**
 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
 * @root - the root we're allocating for
 * @block_rsv - the block_rsv we're allocating for
 * @orig_bytes - the number of bytes we want
 * @flush - wether or not we can flush to make our reservation
3736
 *
3737 3738 3739 3740 3741 3742
 * This will reserve orgi_bytes number of bytes from the space info associated
 * with the block_rsv.  If there is not enough space it will make an attempt to
 * flush out space to make room.  It will do this by flushing delalloc if
 * possible or committing the transaction.  If flush is 0 then no attempts to
 * regain reservations will be made and this will fail if there is not enough
 * space already.
3743
 */
3744
static int reserve_metadata_bytes(struct btrfs_root *root,
3745 3746
				  struct btrfs_block_rsv *block_rsv,
				  u64 orig_bytes, int flush)
J
Josef Bacik 已提交
3747
{
3748
	struct btrfs_space_info *space_info = block_rsv->space_info;
3749
	u64 used;
3750 3751 3752
	u64 num_bytes = orig_bytes;
	int retries = 0;
	int ret = 0;
3753
	bool committed = false;
3754
	bool flushing = false;
3755
	bool wait_ordered = false;
J
Josef Bacik 已提交
3756

3757
again:
3758
	ret = 0;
3759
	spin_lock(&space_info->lock);
3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771
	/*
	 * We only want to wait if somebody other than us is flushing and we are
	 * actually alloed to flush.
	 */
	while (flush && !flushing && space_info->flush) {
		spin_unlock(&space_info->lock);
		/*
		 * If we have a trans handle we can't wait because the flusher
		 * may have to commit the transaction, which would mean we would
		 * deadlock since we are waiting for the flusher to finish, but
		 * hold the current transaction open.
		 */
3772
		if (current->journal_info)
3773
			return -EAGAIN;
A
Arne Jansen 已提交
3774 3775 3776
		ret = wait_event_killable(space_info->wait, !space_info->flush);
		/* Must have been killed, return */
		if (ret)
3777 3778 3779 3780 3781 3782
			return -EINTR;

		spin_lock(&space_info->lock);
	}

	ret = -ENOSPC;
3783 3784 3785
	used = space_info->bytes_used + space_info->bytes_reserved +
		space_info->bytes_pinned + space_info->bytes_readonly +
		space_info->bytes_may_use;
J
Josef Bacik 已提交
3786

3787 3788 3789 3790 3791 3792 3793
	/*
	 * The idea here is that we've not already over-reserved the block group
	 * then we can go ahead and save our reservation first and then start
	 * flushing if we need to.  Otherwise if we've already overcommitted
	 * lets start flushing stuff first and then come back and try to make
	 * our reservation.
	 */
3794 3795
	if (used <= space_info->total_bytes) {
		if (used + orig_bytes <= space_info->total_bytes) {
3796
			space_info->bytes_may_use += orig_bytes;
J
Josef Bacik 已提交
3797
			trace_btrfs_space_reservation(root->fs_info,
3798
				"space_info", space_info->flags, orig_bytes, 1);
3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813
			ret = 0;
		} else {
			/*
			 * Ok set num_bytes to orig_bytes since we aren't
			 * overocmmitted, this way we only try and reclaim what
			 * we need.
			 */
			num_bytes = orig_bytes;
		}
	} else {
		/*
		 * Ok we're over committed, set num_bytes to the overcommitted
		 * amount plus the amount of bytes that we need for this
		 * reservation.
		 */
3814
		wait_ordered = true;
3815
		num_bytes = used - space_info->total_bytes +
3816 3817
			(orig_bytes * (retries + 1));
	}
J
Josef Bacik 已提交
3818

3819
	if (ret) {
3820 3821 3822
		u64 profile = btrfs_get_alloc_profile(root, 0);
		u64 avail;

3823 3824 3825 3826 3827 3828
		/*
		 * If we have a lot of space that's pinned, don't bother doing
		 * the overcommit dance yet and just commit the transaction.
		 */
		avail = (space_info->total_bytes - space_info->bytes_used) * 8;
		do_div(avail, 10);
3829
		if (space_info->bytes_pinned >= avail && flush && !committed) {
3830 3831 3832
			space_info->flush = 1;
			flushing = true;
			spin_unlock(&space_info->lock);
3833 3834 3835 3836 3837 3838
			ret = may_commit_transaction(root, space_info,
						     orig_bytes, 1);
			if (ret)
				goto out;
			committed = true;
			goto again;
3839 3840
		}

3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863
		spin_lock(&root->fs_info->free_chunk_lock);
		avail = root->fs_info->free_chunk_space;

		/*
		 * If we have dup, raid1 or raid10 then only half of the free
		 * space is actually useable.
		 */
		if (profile & (BTRFS_BLOCK_GROUP_DUP |
			       BTRFS_BLOCK_GROUP_RAID1 |
			       BTRFS_BLOCK_GROUP_RAID10))
			avail >>= 1;

		/*
		 * If we aren't flushing don't let us overcommit too much, say
		 * 1/8th of the space.  If we can flush, let it overcommit up to
		 * 1/2 of the space.
		 */
		if (flush)
			avail >>= 3;
		else
			avail >>= 1;
		 spin_unlock(&root->fs_info->free_chunk_lock);

3864
		if (used + num_bytes < space_info->total_bytes + avail) {
3865
			space_info->bytes_may_use += orig_bytes;
J
Josef Bacik 已提交
3866
			trace_btrfs_space_reservation(root->fs_info,
3867
				"space_info", space_info->flags, orig_bytes, 1);
3868
			ret = 0;
3869 3870
		} else {
			wait_ordered = true;
3871 3872 3873
		}
	}

3874 3875 3876 3877 3878
	/*
	 * Couldn't make our reservation, save our place so while we're trying
	 * to reclaim space we can actually use it instead of somebody else
	 * stealing it from us.
	 */
3879 3880 3881
	if (ret && flush) {
		flushing = true;
		space_info->flush = 1;
3882
	}
J
Josef Bacik 已提交
3883

3884
	spin_unlock(&space_info->lock);
J
Josef Bacik 已提交
3885

3886
	if (!ret || !flush)
3887
		goto out;
3888

3889 3890 3891 3892
	/*
	 * We do synchronous shrinking since we don't actually unreserve
	 * metadata until after the IO is completed.
	 */
3893
	ret = shrink_delalloc(root, num_bytes, wait_ordered);
3894
	if (ret < 0)
3895
		goto out;
3896

3897 3898
	ret = 0;

3899 3900 3901 3902 3903 3904
	/*
	 * So if we were overcommitted it's possible that somebody else flushed
	 * out enough space and we simply didn't have enough space to reclaim,
	 * so go back around and try again.
	 */
	if (retries < 2) {
3905
		wait_ordered = true;
3906 3907 3908
		retries++;
		goto again;
	}
3909

3910
	ret = -ENOSPC;
3911 3912 3913
	if (committed)
		goto out;

3914
	ret = may_commit_transaction(root, space_info, orig_bytes, 0);
3915 3916
	if (!ret) {
		committed = true;
3917
		goto again;
3918
	}
3919 3920

out:
3921
	if (flushing) {
3922
		spin_lock(&space_info->lock);
3923 3924
		space_info->flush = 0;
		wake_up_all(&space_info->wait);
3925
		spin_unlock(&space_info->lock);
3926 3927 3928 3929
	}
	return ret;
}

3930 3931 3932
static struct btrfs_block_rsv *get_block_rsv(
					const struct btrfs_trans_handle *trans,
					const struct btrfs_root *root)
3933
{
3934 3935 3936
	struct btrfs_block_rsv *block_rsv = NULL;

	if (root->ref_cows || root == root->fs_info->csum_root)
3937
		block_rsv = trans->block_rsv;
3938 3939

	if (!block_rsv)
3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974
		block_rsv = root->block_rsv;

	if (!block_rsv)
		block_rsv = &root->fs_info->empty_block_rsv;

	return block_rsv;
}

static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
			       u64 num_bytes)
{
	int ret = -ENOSPC;
	spin_lock(&block_rsv->lock);
	if (block_rsv->reserved >= num_bytes) {
		block_rsv->reserved -= num_bytes;
		if (block_rsv->reserved < block_rsv->size)
			block_rsv->full = 0;
		ret = 0;
	}
	spin_unlock(&block_rsv->lock);
	return ret;
}

static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
				u64 num_bytes, int update_size)
{
	spin_lock(&block_rsv->lock);
	block_rsv->reserved += num_bytes;
	if (update_size)
		block_rsv->size += num_bytes;
	else if (block_rsv->reserved >= block_rsv->size)
		block_rsv->full = 1;
	spin_unlock(&block_rsv->lock);
}

J
Josef Bacik 已提交
3975 3976
static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
				    struct btrfs_block_rsv *block_rsv,
3977
				    struct btrfs_block_rsv *dest, u64 num_bytes)
3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995
{
	struct btrfs_space_info *space_info = block_rsv->space_info;

	spin_lock(&block_rsv->lock);
	if (num_bytes == (u64)-1)
		num_bytes = block_rsv->size;
	block_rsv->size -= num_bytes;
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	} else {
		num_bytes = 0;
	}
	spin_unlock(&block_rsv->lock);

	if (num_bytes > 0) {
		if (dest) {
3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009
			spin_lock(&dest->lock);
			if (!dest->full) {
				u64 bytes_to_add;

				bytes_to_add = dest->size - dest->reserved;
				bytes_to_add = min(num_bytes, bytes_to_add);
				dest->reserved += bytes_to_add;
				if (dest->reserved >= dest->size)
					dest->full = 1;
				num_bytes -= bytes_to_add;
			}
			spin_unlock(&dest->lock);
		}
		if (num_bytes) {
4010
			spin_lock(&space_info->lock);
4011
			space_info->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4012
			trace_btrfs_space_reservation(fs_info, "space_info",
4013
					space_info->flags, num_bytes, 0);
4014
			space_info->reservation_progress++;
4015
			spin_unlock(&space_info->lock);
4016
		}
J
Josef Bacik 已提交
4017
	}
4018
}
4019

4020 4021 4022 4023
static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
				   struct btrfs_block_rsv *dst, u64 num_bytes)
{
	int ret;
J
Josef Bacik 已提交
4024

4025 4026 4027
	ret = block_rsv_use_bytes(src, num_bytes);
	if (ret)
		return ret;
J
Josef Bacik 已提交
4028

4029
	block_rsv_add_bytes(dst, num_bytes, 1);
J
Josef Bacik 已提交
4030 4031 4032
	return 0;
}

4033
void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv)
J
Josef Bacik 已提交
4034
{
4035 4036 4037 4038 4039 4040 4041 4042
	memset(rsv, 0, sizeof(*rsv));
	spin_lock_init(&rsv->lock);
}

struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root)
{
	struct btrfs_block_rsv *block_rsv;
	struct btrfs_fs_info *fs_info = root->fs_info;
J
Josef Bacik 已提交
4043

4044 4045 4046
	block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
	if (!block_rsv)
		return NULL;
J
Josef Bacik 已提交
4047

4048 4049 4050 4051 4052
	btrfs_init_block_rsv(block_rsv);
	block_rsv->space_info = __find_space_info(fs_info,
						  BTRFS_BLOCK_GROUP_METADATA);
	return block_rsv;
}
J
Josef Bacik 已提交
4053

4054 4055 4056
void btrfs_free_block_rsv(struct btrfs_root *root,
			  struct btrfs_block_rsv *rsv)
{
4057 4058
	btrfs_block_rsv_release(root, rsv, (u64)-1);
	kfree(rsv);
J
Josef Bacik 已提交
4059 4060
}

4061 4062 4063
static inline int __block_rsv_add(struct btrfs_root *root,
				  struct btrfs_block_rsv *block_rsv,
				  u64 num_bytes, int flush)
J
Josef Bacik 已提交
4064
{
4065
	int ret;
J
Josef Bacik 已提交
4066

4067 4068
	if (num_bytes == 0)
		return 0;
4069

4070
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4071 4072 4073 4074
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 1);
		return 0;
	}
J
Josef Bacik 已提交
4075

4076 4077
	return ret;
}
J
Josef Bacik 已提交
4078

4079 4080 4081 4082 4083 4084 4085
int btrfs_block_rsv_add(struct btrfs_root *root,
			struct btrfs_block_rsv *block_rsv,
			u64 num_bytes)
{
	return __block_rsv_add(root, block_rsv, num_bytes, 1);
}

4086 4087 4088
int btrfs_block_rsv_add_noflush(struct btrfs_root *root,
				struct btrfs_block_rsv *block_rsv,
				u64 num_bytes)
4089
{
4090
	return __block_rsv_add(root, block_rsv, num_bytes, 0);
4091
}
J
Josef Bacik 已提交
4092

4093
int btrfs_block_rsv_check(struct btrfs_root *root,
4094
			  struct btrfs_block_rsv *block_rsv, int min_factor)
4095 4096 4097
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;
J
Josef Bacik 已提交
4098

4099 4100
	if (!block_rsv)
		return 0;
J
Josef Bacik 已提交
4101

4102
	spin_lock(&block_rsv->lock);
4103 4104 4105 4106
	num_bytes = div_factor(block_rsv->size, min_factor);
	if (block_rsv->reserved >= num_bytes)
		ret = 0;
	spin_unlock(&block_rsv->lock);
J
Josef Bacik 已提交
4107

4108 4109 4110
	return ret;
}

4111 4112 4113
static inline int __btrfs_block_rsv_refill(struct btrfs_root *root,
					   struct btrfs_block_rsv *block_rsv,
					   u64 min_reserved, int flush)
4114 4115 4116 4117 4118 4119 4120 4121 4122
{
	u64 num_bytes = 0;
	int ret = -ENOSPC;

	if (!block_rsv)
		return 0;

	spin_lock(&block_rsv->lock);
	num_bytes = min_reserved;
4123
	if (block_rsv->reserved >= num_bytes)
4124
		ret = 0;
4125
	else
4126 4127
		num_bytes -= block_rsv->reserved;
	spin_unlock(&block_rsv->lock);
4128

4129 4130 4131
	if (!ret)
		return 0;

4132
	ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4133 4134
	if (!ret) {
		block_rsv_add_bytes(block_rsv, num_bytes, 0);
4135
		return 0;
J
Josef Bacik 已提交
4136
	}
J
Josef Bacik 已提交
4137

4138
	return ret;
4139 4140
}

4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154
int btrfs_block_rsv_refill(struct btrfs_root *root,
			   struct btrfs_block_rsv *block_rsv,
			   u64 min_reserved)
{
	return __btrfs_block_rsv_refill(root, block_rsv, min_reserved, 1);
}

int btrfs_block_rsv_refill_noflush(struct btrfs_root *root,
				   struct btrfs_block_rsv *block_rsv,
				   u64 min_reserved)
{
	return __btrfs_block_rsv_refill(root, block_rsv, min_reserved, 0);
}

4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169
int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
			    struct btrfs_block_rsv *dst_rsv,
			    u64 num_bytes)
{
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
}

void btrfs_block_rsv_release(struct btrfs_root *root,
			     struct btrfs_block_rsv *block_rsv,
			     u64 num_bytes)
{
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
	if (global_rsv->full || global_rsv == block_rsv ||
	    block_rsv->space_info != global_rsv->space_info)
		global_rsv = NULL;
J
Josef Bacik 已提交
4170 4171
	block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
				num_bytes);
J
Josef Bacik 已提交
4172 4173 4174
}

/*
4175 4176 4177
 * helper to calculate size of global block reservation.
 * the desired value is sum of space used by extent tree,
 * checksum tree and root tree
J
Josef Bacik 已提交
4178
 */
4179
static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4180
{
4181 4182 4183 4184
	struct btrfs_space_info *sinfo;
	u64 num_bytes;
	u64 meta_used;
	u64 data_used;
4185
	int csum_size = btrfs_super_csum_size(fs_info->super_copy);
J
Josef Bacik 已提交
4186

4187 4188 4189 4190
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
	spin_lock(&sinfo->lock);
	data_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
C
Chris Mason 已提交
4191

4192 4193
	sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
	spin_lock(&sinfo->lock);
4194 4195
	if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
		data_used = 0;
4196 4197
	meta_used = sinfo->bytes_used;
	spin_unlock(&sinfo->lock);
4198

4199 4200 4201
	num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
		    csum_size * 2;
	num_bytes += div64_u64(data_used + meta_used, 50);
4202

4203
	if (num_bytes * 3 > meta_used)
4204
		num_bytes = div64_u64(meta_used, 3);
4205

4206 4207
	return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
}
J
Josef Bacik 已提交
4208

4209 4210 4211 4212 4213
static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
	struct btrfs_space_info *sinfo = block_rsv->space_info;
	u64 num_bytes;
J
Josef Bacik 已提交
4214

4215
	num_bytes = calc_global_metadata_size(fs_info);
C
Chris Mason 已提交
4216

4217
	spin_lock(&sinfo->lock);
4218
	spin_lock(&block_rsv->lock);
4219

4220
	block_rsv->size = num_bytes;
4221

4222
	num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
4223 4224
		    sinfo->bytes_reserved + sinfo->bytes_readonly +
		    sinfo->bytes_may_use;
4225 4226 4227 4228

	if (sinfo->total_bytes > num_bytes) {
		num_bytes = sinfo->total_bytes - num_bytes;
		block_rsv->reserved += num_bytes;
4229
		sinfo->bytes_may_use += num_bytes;
J
Josef Bacik 已提交
4230
		trace_btrfs_space_reservation(fs_info, "space_info",
4231
				      sinfo->flags, num_bytes, 1);
J
Josef Bacik 已提交
4232 4233
	}

4234 4235
	if (block_rsv->reserved >= block_rsv->size) {
		num_bytes = block_rsv->reserved - block_rsv->size;
4236
		sinfo->bytes_may_use -= num_bytes;
J
Josef Bacik 已提交
4237
		trace_btrfs_space_reservation(fs_info, "space_info",
4238
				      sinfo->flags, num_bytes, 0);
4239
		sinfo->reservation_progress++;
4240 4241 4242
		block_rsv->reserved = block_rsv->size;
		block_rsv->full = 1;
	}
4243

4244
	spin_unlock(&block_rsv->lock);
4245
	spin_unlock(&sinfo->lock);
J
Josef Bacik 已提交
4246 4247
}

4248
static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4249
{
4250
	struct btrfs_space_info *space_info;
J
Josef Bacik 已提交
4251

4252 4253
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
	fs_info->chunk_block_rsv.space_info = space_info;
J
Josef Bacik 已提交
4254

4255
	space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4256 4257
	fs_info->global_block_rsv.space_info = space_info;
	fs_info->delalloc_block_rsv.space_info = space_info;
4258 4259
	fs_info->trans_block_rsv.space_info = space_info;
	fs_info->empty_block_rsv.space_info = space_info;
4260
	fs_info->delayed_block_rsv.space_info = space_info;
4261

4262 4263 4264 4265
	fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
	fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
4266
	fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
4267 4268

	update_global_block_rsv(fs_info);
J
Josef Bacik 已提交
4269 4270
}

4271
static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
J
Josef Bacik 已提交
4272
{
J
Josef Bacik 已提交
4273 4274
	block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
				(u64)-1);
4275 4276 4277 4278 4279 4280
	WARN_ON(fs_info->delalloc_block_rsv.size > 0);
	WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
	WARN_ON(fs_info->trans_block_rsv.size > 0);
	WARN_ON(fs_info->trans_block_rsv.reserved > 0);
	WARN_ON(fs_info->chunk_block_rsv.size > 0);
	WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
4281 4282
	WARN_ON(fs_info->delayed_block_rsv.size > 0);
	WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
4283 4284
}

4285 4286
void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
				  struct btrfs_root *root)
J
Josef Bacik 已提交
4287
{
4288 4289
	if (!trans->bytes_reserved)
		return;
J
Josef Bacik 已提交
4290

4291
	trace_btrfs_space_reservation(root->fs_info, "transaction",
4292
				      trans->transid, trans->bytes_reserved, 0);
4293
	btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
4294 4295
	trans->bytes_reserved = 0;
}
J
Josef Bacik 已提交
4296

4297
/* Can only return 0 or -ENOSPC */
4298 4299 4300 4301 4302 4303 4304 4305
int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
				  struct inode *inode)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;

	/*
4306 4307 4308
	 * We need to hold space in order to delete our orphan item once we've
	 * added it, so this takes the reservation so we can release it later
	 * when we are truly done with the orphan item.
4309
	 */
C
Chris Mason 已提交
4310
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4311 4312
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 1);
4313
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
J
Josef Bacik 已提交
4314 4315
}

4316
void btrfs_orphan_release_metadata(struct inode *inode)
4317
{
4318
	struct btrfs_root *root = BTRFS_I(inode)->root;
C
Chris Mason 已提交
4319
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
J
Josef Bacik 已提交
4320 4321
	trace_btrfs_space_reservation(root->fs_info, "orphan",
				      btrfs_ino(inode), num_bytes, 0);
4322 4323
	btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
}
4324

4325 4326 4327 4328 4329 4330 4331 4332 4333 4334
int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
				struct btrfs_pending_snapshot *pending)
{
	struct btrfs_root *root = pending->root;
	struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
	struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
	/*
	 * two for root back/forward refs, two for directory entries
	 * and one for root of the snapshot.
	 */
4335
	u64 num_bytes = btrfs_calc_trans_metadata_size(root, 5);
4336 4337
	dst_rsv->space_info = src_rsv->space_info;
	return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4338 4339
}

4340 4341 4342 4343 4344 4345 4346 4347 4348
/**
 * drop_outstanding_extent - drop an outstanding extent
 * @inode: the inode we're dropping the extent for
 *
 * This is called when we are freeing up an outstanding extent, either called
 * after an error or after an extent is written.  This will return the number of
 * reserved extents that need to be freed.  This must be called with
 * BTRFS_I(inode)->lock held.
 */
4349 4350
static unsigned drop_outstanding_extent(struct inode *inode)
{
4351
	unsigned drop_inode_space = 0;
4352 4353 4354 4355 4356
	unsigned dropped_extents = 0;

	BUG_ON(!BTRFS_I(inode)->outstanding_extents);
	BTRFS_I(inode)->outstanding_extents--;

4357 4358 4359 4360 4361 4362
	if (BTRFS_I(inode)->outstanding_extents == 0 &&
	    BTRFS_I(inode)->delalloc_meta_reserved) {
		drop_inode_space = 1;
		BTRFS_I(inode)->delalloc_meta_reserved = 0;
	}

4363 4364 4365 4366 4367 4368
	/*
	 * If we have more or the same amount of outsanding extents than we have
	 * reserved then we need to leave the reserved extents count alone.
	 */
	if (BTRFS_I(inode)->outstanding_extents >=
	    BTRFS_I(inode)->reserved_extents)
4369
		return drop_inode_space;
4370 4371 4372 4373

	dropped_extents = BTRFS_I(inode)->reserved_extents -
		BTRFS_I(inode)->outstanding_extents;
	BTRFS_I(inode)->reserved_extents -= dropped_extents;
4374
	return dropped_extents + drop_inode_space;
4375 4376
}

4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396
/**
 * calc_csum_metadata_size - return the amount of metada space that must be
 *	reserved/free'd for the given bytes.
 * @inode: the inode we're manipulating
 * @num_bytes: the number of bytes in question
 * @reserve: 1 if we are reserving space, 0 if we are freeing space
 *
 * This adjusts the number of csum_bytes in the inode and then returns the
 * correct amount of metadata that must either be reserved or freed.  We
 * calculate how many checksums we can fit into one leaf and then divide the
 * number of bytes that will need to be checksumed by this value to figure out
 * how many checksums will be required.  If we are adding bytes then the number
 * may go up and we will return the number of additional bytes that must be
 * reserved.  If it is going down we will return the number of bytes that must
 * be freed.
 *
 * This must be called with BTRFS_I(inode)->lock held.
 */
static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
				   int reserve)
4397
{
4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432
	struct btrfs_root *root = BTRFS_I(inode)->root;
	u64 csum_size;
	int num_csums_per_leaf;
	int num_csums;
	int old_csums;

	if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
	    BTRFS_I(inode)->csum_bytes == 0)
		return 0;

	old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	if (reserve)
		BTRFS_I(inode)->csum_bytes += num_bytes;
	else
		BTRFS_I(inode)->csum_bytes -= num_bytes;
	csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
	num_csums_per_leaf = (int)div64_u64(csum_size,
					    sizeof(struct btrfs_csum_item) +
					    sizeof(struct btrfs_disk_key));
	num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
	num_csums = num_csums + num_csums_per_leaf - 1;
	num_csums = num_csums / num_csums_per_leaf;

	old_csums = old_csums + num_csums_per_leaf - 1;
	old_csums = old_csums / num_csums_per_leaf;

	/* No change, no need to reserve more */
	if (old_csums == num_csums)
		return 0;

	if (reserve)
		return btrfs_calc_trans_metadata_size(root,
						      num_csums - old_csums);

	return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
4433
}
Y
Yan Zheng 已提交
4434

4435 4436 4437 4438
int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
	struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4439
	u64 to_reserve = 0;
4440
	u64 csum_bytes;
4441
	unsigned nr_extents = 0;
4442
	int extra_reserve = 0;
4443
	int flush = 1;
4444
	int ret;
4445

4446
	/* Need to be holding the i_mutex here if we aren't free space cache */
4447 4448 4449 4450
	if (btrfs_is_free_space_inode(root, inode))
		flush = 0;

	if (flush && btrfs_transaction_in_commit(root->fs_info))
4451
		schedule_timeout(1);
4452

4453
	mutex_lock(&BTRFS_I(inode)->delalloc_mutex);
4454
	num_bytes = ALIGN(num_bytes, root->sectorsize);
4455

4456 4457 4458 4459
	spin_lock(&BTRFS_I(inode)->lock);
	BTRFS_I(inode)->outstanding_extents++;

	if (BTRFS_I(inode)->outstanding_extents >
4460
	    BTRFS_I(inode)->reserved_extents)
4461 4462
		nr_extents = BTRFS_I(inode)->outstanding_extents -
			BTRFS_I(inode)->reserved_extents;
4463

4464 4465 4466 4467 4468 4469
	/*
	 * Add an item to reserve for updating the inode when we complete the
	 * delalloc io.
	 */
	if (!BTRFS_I(inode)->delalloc_meta_reserved) {
		nr_extents++;
4470
		extra_reserve = 1;
4471
	}
4472 4473

	to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4474
	to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
4475
	csum_bytes = BTRFS_I(inode)->csum_bytes;
4476
	spin_unlock(&BTRFS_I(inode)->lock);
4477

4478
	ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush);
4479
	if (ret) {
4480
		u64 to_free = 0;
4481
		unsigned dropped;
4482

4483
		spin_lock(&BTRFS_I(inode)->lock);
4484 4485
		dropped = drop_outstanding_extent(inode);
		/*
4486 4487 4488 4489 4490 4491
		 * If the inodes csum_bytes is the same as the original
		 * csum_bytes then we know we haven't raced with any free()ers
		 * so we can just reduce our inodes csum bytes and carry on.
		 * Otherwise we have to do the normal free thing to account for
		 * the case that the free side didn't free up its reserve
		 * because of this outstanding reservation.
4492
		 */
4493 4494 4495 4496 4497 4498 4499 4500
		if (BTRFS_I(inode)->csum_bytes == csum_bytes)
			calc_csum_metadata_size(inode, num_bytes, 0);
		else
			to_free = calc_csum_metadata_size(inode, num_bytes, 0);
		spin_unlock(&BTRFS_I(inode)->lock);
		if (dropped)
			to_free += btrfs_calc_trans_metadata_size(root, dropped);

J
Josef Bacik 已提交
4501
		if (to_free) {
4502
			btrfs_block_rsv_release(root, block_rsv, to_free);
J
Josef Bacik 已提交
4503 4504 4505 4506 4507
			trace_btrfs_space_reservation(root->fs_info,
						      "delalloc",
						      btrfs_ino(inode),
						      to_free, 0);
		}
4508
		mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4509
		return ret;
4510
	}
4511

4512 4513 4514 4515 4516 4517 4518
	spin_lock(&BTRFS_I(inode)->lock);
	if (extra_reserve) {
		BTRFS_I(inode)->delalloc_meta_reserved = 1;
		nr_extents--;
	}
	BTRFS_I(inode)->reserved_extents += nr_extents;
	spin_unlock(&BTRFS_I(inode)->lock);
4519
	mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4520

J
Josef Bacik 已提交
4521 4522 4523
	if (to_reserve)
		trace_btrfs_space_reservation(root->fs_info,"delalloc",
					      btrfs_ino(inode), to_reserve, 1);
4524 4525 4526 4527 4528
	block_rsv_add_bytes(block_rsv, to_reserve, 1);

	return 0;
}

4529 4530 4531 4532 4533 4534 4535 4536 4537
/**
 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
 * @inode: the inode to release the reservation for
 * @num_bytes: the number of bytes we're releasing
 *
 * This will release the metadata reservation for an inode.  This can be called
 * once we complete IO for a given set of bytes to release their metadata
 * reservations.
 */
4538 4539 4540
void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
{
	struct btrfs_root *root = BTRFS_I(inode)->root;
4541 4542
	u64 to_free = 0;
	unsigned dropped;
4543 4544

	num_bytes = ALIGN(num_bytes, root->sectorsize);
4545
	spin_lock(&BTRFS_I(inode)->lock);
4546
	dropped = drop_outstanding_extent(inode);
4547

4548 4549
	to_free = calc_csum_metadata_size(inode, num_bytes, 0);
	spin_unlock(&BTRFS_I(inode)->lock);
4550 4551
	if (dropped > 0)
		to_free += btrfs_calc_trans_metadata_size(root, dropped);
4552

J
Josef Bacik 已提交
4553 4554
	trace_btrfs_space_reservation(root->fs_info, "delalloc",
				      btrfs_ino(inode), to_free, 0);
4555 4556 4557 4558
	btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
				to_free);
}

4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573
/**
 * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
 * @inode: inode we're writing to
 * @num_bytes: the number of bytes we want to allocate
 *
 * This will do the following things
 *
 * o reserve space in the data space info for num_bytes
 * o reserve space in the metadata space info based on number of outstanding
 *   extents and how much csums will be needed
 * o add to the inodes ->delalloc_bytes
 * o add it to the fs_info's delalloc inodes list.
 *
 * This will return 0 for success and -ENOSPC if there is no space left.
 */
4574 4575 4576 4577 4578
int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
{
	int ret;

	ret = btrfs_check_data_free_space(inode, num_bytes);
C
Chris Mason 已提交
4579
	if (ret)
4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590
		return ret;

	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
	if (ret) {
		btrfs_free_reserved_data_space(inode, num_bytes);
		return ret;
	}

	return 0;
}

4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603
/**
 * btrfs_delalloc_release_space - release data and metadata space for delalloc
 * @inode: inode we're releasing space for
 * @num_bytes: the number of bytes we want to free up
 *
 * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
 * called in the case that we don't need the metadata AND data reservations
 * anymore.  So if there is an error or we insert an inline extent.
 *
 * This function will release the metadata space that was not used and will
 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
 * list if there are no delalloc bytes left.
 */
4604 4605 4606 4607
void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
{
	btrfs_delalloc_release_metadata(inode, num_bytes);
	btrfs_free_reserved_data_space(inode, num_bytes);
4608 4609
}

C
Chris Mason 已提交
4610 4611
static int update_block_group(struct btrfs_trans_handle *trans,
			      struct btrfs_root *root,
4612
			      u64 bytenr, u64 num_bytes, int alloc)
C
Chris Mason 已提交
4613
{
4614
	struct btrfs_block_group_cache *cache = NULL;
C
Chris Mason 已提交
4615
	struct btrfs_fs_info *info = root->fs_info;
4616
	u64 total = num_bytes;
C
Chris Mason 已提交
4617
	u64 old_val;
4618
	u64 byte_in_group;
4619
	int factor;
C
Chris Mason 已提交
4620

4621 4622
	/* block accounting for super block */
	spin_lock(&info->delalloc_lock);
4623
	old_val = btrfs_super_bytes_used(info->super_copy);
4624 4625 4626 4627
	if (alloc)
		old_val += num_bytes;
	else
		old_val -= num_bytes;
4628
	btrfs_set_super_bytes_used(info->super_copy, old_val);
4629 4630
	spin_unlock(&info->delalloc_lock);

C
Chris Mason 已提交
4631
	while (total) {
4632
		cache = btrfs_lookup_block_group(info, bytenr);
4633
		if (!cache)
4634
			return -ENOENT;
4635 4636 4637 4638 4639 4640
		if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
				    BTRFS_BLOCK_GROUP_RAID1 |
				    BTRFS_BLOCK_GROUP_RAID10))
			factor = 2;
		else
			factor = 1;
4641 4642 4643 4644 4645 4646 4647
		/*
		 * If this block group has free space cache written out, we
		 * need to make sure to load it if we are removing space.  This
		 * is because we need the unpinning stage to actually add the
		 * space back to the block group, otherwise we will leak space.
		 */
		if (!alloc && cache->cached == BTRFS_CACHE_NO)
4648
			cache_block_group(cache, trans, NULL, 1);
4649

4650 4651
		byte_in_group = bytenr - cache->key.objectid;
		WARN_ON(byte_in_group > cache->key.offset);
C
Chris Mason 已提交
4652

4653
		spin_lock(&cache->space_info->lock);
4654
		spin_lock(&cache->lock);
4655

4656
		if (btrfs_test_opt(root, SPACE_CACHE) &&
4657 4658 4659
		    cache->disk_cache_state < BTRFS_DC_CLEAR)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

J
Josef Bacik 已提交
4660
		cache->dirty = 1;
C
Chris Mason 已提交
4661
		old_val = btrfs_block_group_used(&cache->item);
4662
		num_bytes = min(total, cache->key.offset - byte_in_group);
C
Chris Mason 已提交
4663
		if (alloc) {
4664
			old_val += num_bytes;
4665 4666 4667
			btrfs_set_block_group_used(&cache->item, old_val);
			cache->reserved -= num_bytes;
			cache->space_info->bytes_reserved -= num_bytes;
4668 4669
			cache->space_info->bytes_used += num_bytes;
			cache->space_info->disk_used += num_bytes * factor;
4670
			spin_unlock(&cache->lock);
4671
			spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4672
		} else {
4673
			old_val -= num_bytes;
4674
			btrfs_set_block_group_used(&cache->item, old_val);
4675 4676
			cache->pinned += num_bytes;
			cache->space_info->bytes_pinned += num_bytes;
4677
			cache->space_info->bytes_used -= num_bytes;
4678
			cache->space_info->disk_used -= num_bytes * factor;
4679
			spin_unlock(&cache->lock);
4680
			spin_unlock(&cache->space_info->lock);
4681

4682 4683 4684
			set_extent_dirty(info->pinned_extents,
					 bytenr, bytenr + num_bytes - 1,
					 GFP_NOFS | __GFP_NOFAIL);
C
Chris Mason 已提交
4685
		}
4686
		btrfs_put_block_group(cache);
4687 4688
		total -= num_bytes;
		bytenr += num_bytes;
C
Chris Mason 已提交
4689 4690 4691
	}
	return 0;
}
4692

4693 4694
static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
{
J
Josef Bacik 已提交
4695
	struct btrfs_block_group_cache *cache;
4696
	u64 bytenr;
J
Josef Bacik 已提交
4697 4698 4699

	cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
	if (!cache)
4700
		return 0;
J
Josef Bacik 已提交
4701

4702
	bytenr = cache->key.objectid;
4703
	btrfs_put_block_group(cache);
4704 4705

	return bytenr;
4706 4707
}

4708 4709 4710
static int pin_down_extent(struct btrfs_root *root,
			   struct btrfs_block_group_cache *cache,
			   u64 bytenr, u64 num_bytes, int reserved)
4711
{
4712 4713 4714 4715 4716 4717 4718 4719 4720 4721
	spin_lock(&cache->space_info->lock);
	spin_lock(&cache->lock);
	cache->pinned += num_bytes;
	cache->space_info->bytes_pinned += num_bytes;
	if (reserved) {
		cache->reserved -= num_bytes;
		cache->space_info->bytes_reserved -= num_bytes;
	}
	spin_unlock(&cache->lock);
	spin_unlock(&cache->space_info->lock);
J
Josef Bacik 已提交
4722

4723 4724 4725 4726
	set_extent_dirty(root->fs_info->pinned_extents, bytenr,
			 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
	return 0;
}
J
Josef Bacik 已提交
4727

4728 4729 4730 4731 4732 4733 4734
/*
 * this function must be called within transaction
 */
int btrfs_pin_extent(struct btrfs_root *root,
		     u64 bytenr, u64 num_bytes, int reserved)
{
	struct btrfs_block_group_cache *cache;
J
Josef Bacik 已提交
4735

4736
	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4737
	BUG_ON(!cache); /* Logic error */
4738 4739 4740 4741

	pin_down_extent(root, cache, bytenr, num_bytes, reserved);

	btrfs_put_block_group(cache);
4742 4743 4744
	return 0;
}

4745
/*
4746 4747 4748 4749 4750 4751 4752 4753 4754
 * this function must be called within transaction
 */
int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans,
				    struct btrfs_root *root,
				    u64 bytenr, u64 num_bytes)
{
	struct btrfs_block_group_cache *cache;

	cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4755
	BUG_ON(!cache); /* Logic error */
4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772

	/*
	 * pull in the free space cache (if any) so that our pin
	 * removes the free space from the cache.  We have load_only set
	 * to one because the slow code to read in the free extents does check
	 * the pinned extents.
	 */
	cache_block_group(cache, trans, root, 1);

	pin_down_extent(root, cache, bytenr, num_bytes, 0);

	/* remove us from the free space cache (if we're there at all) */
	btrfs_remove_free_space(cache, bytenr, num_bytes);
	btrfs_put_block_group(cache);
	return 0;
}

4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793
/**
 * btrfs_update_reserved_bytes - update the block_group and space info counters
 * @cache:	The cache we are manipulating
 * @num_bytes:	The number of bytes in question
 * @reserve:	One of the reservation enums
 *
 * This is called by the allocator when it reserves space, or by somebody who is
 * freeing space that was never actually used on disk.  For example if you
 * reserve some space for a new leaf in transaction A and before transaction A
 * commits you free that leaf, you call this with reserve set to 0 in order to
 * clear the reservation.
 *
 * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
 * ENOSPC accounting.  For data we handle the reservation through clearing the
 * delalloc bits in the io_tree.  We have to do this since we could end up
 * allocating less disk space for the amount of data we have reserved in the
 * case of compression.
 *
 * If this is a reservation and the block group has become read only we cannot
 * make the reservation and return -EAGAIN, otherwise this function always
 * succeeds.
4794
 */
4795 4796
static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
				       u64 num_bytes, int reserve)
4797
{
4798
	struct btrfs_space_info *space_info = cache->space_info;
4799
	int ret = 0;
4800

4801 4802 4803
	spin_lock(&space_info->lock);
	spin_lock(&cache->lock);
	if (reserve != RESERVE_FREE) {
4804 4805 4806
		if (cache->ro) {
			ret = -EAGAIN;
		} else {
4807 4808 4809
			cache->reserved += num_bytes;
			space_info->bytes_reserved += num_bytes;
			if (reserve == RESERVE_ALLOC) {
J
Josef Bacik 已提交
4810
				trace_btrfs_space_reservation(cache->fs_info,
4811 4812
						"space_info", space_info->flags,
						num_bytes, 0);
4813 4814
				space_info->bytes_may_use -= num_bytes;
			}
4815
		}
4816 4817 4818 4819 4820 4821
	} else {
		if (cache->ro)
			space_info->bytes_readonly += num_bytes;
		cache->reserved -= num_bytes;
		space_info->bytes_reserved -= num_bytes;
		space_info->reservation_progress++;
4822
	}
4823 4824
	spin_unlock(&cache->lock);
	spin_unlock(&space_info->lock);
4825
	return ret;
4826
}
C
Chris Mason 已提交
4827

4828
void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
4829
				struct btrfs_root *root)
4830 4831
{
	struct btrfs_fs_info *fs_info = root->fs_info;
4832 4833 4834
	struct btrfs_caching_control *next;
	struct btrfs_caching_control *caching_ctl;
	struct btrfs_block_group_cache *cache;
4835

4836
	down_write(&fs_info->extent_commit_sem);
4837

4838 4839 4840 4841 4842 4843 4844
	list_for_each_entry_safe(caching_ctl, next,
				 &fs_info->caching_block_groups, list) {
		cache = caching_ctl->block_group;
		if (block_group_cache_done(cache)) {
			cache->last_byte_to_unpin = (u64)-1;
			list_del_init(&caching_ctl->list);
			put_caching_control(caching_ctl);
4845
		} else {
4846
			cache->last_byte_to_unpin = caching_ctl->progress;
4847 4848
		}
	}
4849 4850 4851 4852 4853 4854 4855

	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		fs_info->pinned_extents = &fs_info->freed_extents[1];
	else
		fs_info->pinned_extents = &fs_info->freed_extents[0];

	up_write(&fs_info->extent_commit_sem);
4856 4857

	update_global_block_rsv(fs_info);
4858 4859
}

4860
static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
C
Chris Mason 已提交
4861
{
4862 4863 4864
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 len;
C
Chris Mason 已提交
4865

4866 4867 4868 4869 4870 4871
	while (start <= end) {
		if (!cache ||
		    start >= cache->key.objectid + cache->key.offset) {
			if (cache)
				btrfs_put_block_group(cache);
			cache = btrfs_lookup_block_group(fs_info, start);
4872
			BUG_ON(!cache); /* Logic error */
4873 4874 4875 4876 4877 4878 4879 4880 4881 4882
		}

		len = cache->key.objectid + cache->key.offset - start;
		len = min(len, end + 1 - start);

		if (start < cache->last_byte_to_unpin) {
			len = min(len, cache->last_byte_to_unpin - start);
			btrfs_add_free_space(cache, start, len);
		}

4883 4884
		start += len;

4885 4886 4887 4888
		spin_lock(&cache->space_info->lock);
		spin_lock(&cache->lock);
		cache->pinned -= len;
		cache->space_info->bytes_pinned -= len;
4889
		if (cache->ro)
4890
			cache->space_info->bytes_readonly += len;
4891 4892
		spin_unlock(&cache->lock);
		spin_unlock(&cache->space_info->lock);
C
Chris Mason 已提交
4893
	}
4894 4895 4896

	if (cache)
		btrfs_put_block_group(cache);
C
Chris Mason 已提交
4897 4898 4899 4900
	return 0;
}

int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
4901
			       struct btrfs_root *root)
4902
{
4903 4904
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct extent_io_tree *unpin;
4905 4906
	u64 start;
	u64 end;
4907 4908
	int ret;

4909 4910 4911
	if (trans->aborted)
		return 0;

4912 4913 4914 4915 4916
	if (fs_info->pinned_extents == &fs_info->freed_extents[0])
		unpin = &fs_info->freed_extents[1];
	else
		unpin = &fs_info->freed_extents[0];

C
Chris Mason 已提交
4917
	while (1) {
4918 4919 4920
		ret = find_first_extent_bit(unpin, 0, &start, &end,
					    EXTENT_DIRTY);
		if (ret)
4921
			break;
4922

4923 4924 4925
		if (btrfs_test_opt(root, DISCARD))
			ret = btrfs_discard_extent(root, start,
						   end + 1 - start, NULL);
4926

4927
		clear_extent_dirty(unpin, start, end, GFP_NOFS);
4928
		unpin_extent_range(root, start, end);
4929
		cond_resched();
4930
	}
J
Josef Bacik 已提交
4931

C
Chris Mason 已提交
4932 4933 4934
	return 0;
}

4935 4936 4937 4938 4939 4940
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
				struct btrfs_root *root,
				u64 bytenr, u64 num_bytes, u64 parent,
				u64 root_objectid, u64 owner_objectid,
				u64 owner_offset, int refs_to_drop,
				struct btrfs_delayed_extent_op *extent_op)
4941
{
C
Chris Mason 已提交
4942
	struct btrfs_key key;
4943
	struct btrfs_path *path;
4944 4945
	struct btrfs_fs_info *info = root->fs_info;
	struct btrfs_root *extent_root = info->extent_root;
4946
	struct extent_buffer *leaf;
4947 4948
	struct btrfs_extent_item *ei;
	struct btrfs_extent_inline_ref *iref;
4949
	int ret;
4950
	int is_data;
4951 4952 4953
	int extent_slot = 0;
	int found_extent = 0;
	int num_to_del = 1;
4954 4955
	u32 item_size;
	u64 refs;
C
Chris Mason 已提交
4956

4957
	path = btrfs_alloc_path();
4958 4959
	if (!path)
		return -ENOMEM;
4960

4961
	path->reada = 1;
4962
	path->leave_spinning = 1;
4963 4964 4965 4966 4967 4968 4969 4970

	is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
	BUG_ON(!is_data && refs_to_drop != 1);

	ret = lookup_extent_backref(trans, extent_root, path, &iref,
				    bytenr, num_bytes, parent,
				    root_objectid, owner_objectid,
				    owner_offset);
4971
	if (ret == 0) {
4972
		extent_slot = path->slots[0];
4973 4974
		while (extent_slot >= 0) {
			btrfs_item_key_to_cpu(path->nodes[0], &key,
4975
					      extent_slot);
4976
			if (key.objectid != bytenr)
4977
				break;
4978 4979
			if (key.type == BTRFS_EXTENT_ITEM_KEY &&
			    key.offset == num_bytes) {
4980 4981 4982 4983 4984
				found_extent = 1;
				break;
			}
			if (path->slots[0] - extent_slot > 5)
				break;
4985
			extent_slot--;
4986
		}
4987 4988 4989 4990 4991
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
		item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
		if (found_extent && item_size < sizeof(*ei))
			found_extent = 0;
#endif
Z
Zheng Yan 已提交
4992
		if (!found_extent) {
4993
			BUG_ON(iref);
4994
			ret = remove_extent_backref(trans, extent_root, path,
4995 4996
						    NULL, refs_to_drop,
						    is_data);
4997 4998
			if (ret)
				goto abort;
4999
			btrfs_release_path(path);
5000
			path->leave_spinning = 1;
5001 5002 5003 5004 5005

			key.objectid = bytenr;
			key.type = BTRFS_EXTENT_ITEM_KEY;
			key.offset = num_bytes;

Z
Zheng Yan 已提交
5006 5007
			ret = btrfs_search_slot(trans, extent_root,
						&key, path, -1, 1);
5008 5009
			if (ret) {
				printk(KERN_ERR "umm, got %d back from search"
C
Chris Mason 已提交
5010 5011
				       ", was looking for %llu\n", ret,
				       (unsigned long long)bytenr);
5012 5013 5014
				if (ret > 0)
					btrfs_print_leaf(extent_root,
							 path->nodes[0]);
5015
			}
5016 5017
			if (ret < 0)
				goto abort;
Z
Zheng Yan 已提交
5018 5019
			extent_slot = path->slots[0];
		}
5020
	} else if (ret == -ENOENT) {
5021 5022
		btrfs_print_leaf(extent_root, path->nodes[0]);
		WARN_ON(1);
C
Chris Mason 已提交
5023
		printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
5024
		       "parent %llu root %llu  owner %llu offset %llu\n",
C
Chris Mason 已提交
5025
		       (unsigned long long)bytenr,
5026
		       (unsigned long long)parent,
C
Chris Mason 已提交
5027
		       (unsigned long long)root_objectid,
5028 5029
		       (unsigned long long)owner_objectid,
		       (unsigned long long)owner_offset);
5030 5031
	} else {
		goto abort;
5032
	}
5033 5034

	leaf = path->nodes[0];
5035 5036 5037 5038 5039 5040
	item_size = btrfs_item_size_nr(leaf, extent_slot);
#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
	if (item_size < sizeof(*ei)) {
		BUG_ON(found_extent || extent_slot != path->slots[0]);
		ret = convert_extent_item_v0(trans, extent_root, path,
					     owner_objectid, 0);
5041 5042
		if (ret < 0)
			goto abort;
5043

5044
		btrfs_release_path(path);
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058
		path->leave_spinning = 1;

		key.objectid = bytenr;
		key.type = BTRFS_EXTENT_ITEM_KEY;
		key.offset = num_bytes;

		ret = btrfs_search_slot(trans, extent_root, &key, path,
					-1, 1);
		if (ret) {
			printk(KERN_ERR "umm, got %d back from search"
			       ", was looking for %llu\n", ret,
			       (unsigned long long)bytenr);
			btrfs_print_leaf(extent_root, path->nodes[0]);
		}
5059 5060
		if (ret < 0)
			goto abort;
5061 5062 5063 5064 5065 5066
		extent_slot = path->slots[0];
		leaf = path->nodes[0];
		item_size = btrfs_item_size_nr(leaf, extent_slot);
	}
#endif
	BUG_ON(item_size < sizeof(*ei));
5067
	ei = btrfs_item_ptr(leaf, extent_slot,
C
Chris Mason 已提交
5068
			    struct btrfs_extent_item);
5069 5070 5071 5072 5073 5074
	if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
		struct btrfs_tree_block_info *bi;
		BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
		bi = (struct btrfs_tree_block_info *)(ei + 1);
		WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
	}
5075

5076
	refs = btrfs_extent_refs(leaf, ei);
5077 5078
	BUG_ON(refs < refs_to_drop);
	refs -= refs_to_drop;
5079

5080 5081 5082 5083 5084 5085
	if (refs > 0) {
		if (extent_op)
			__run_delayed_extent_op(extent_op, leaf, ei);
		/*
		 * In the case of inline back ref, reference count will
		 * be updated by remove_extent_backref
5086
		 */
5087 5088 5089 5090 5091 5092 5093 5094 5095 5096
		if (iref) {
			BUG_ON(!found_extent);
		} else {
			btrfs_set_extent_refs(leaf, ei, refs);
			btrfs_mark_buffer_dirty(leaf);
		}
		if (found_extent) {
			ret = remove_extent_backref(trans, extent_root, path,
						    iref, refs_to_drop,
						    is_data);
5097 5098
			if (ret)
				goto abort;
5099
		}
5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110
	} else {
		if (found_extent) {
			BUG_ON(is_data && refs_to_drop !=
			       extent_data_ref_count(root, path, iref));
			if (iref) {
				BUG_ON(path->slots[0] != extent_slot);
			} else {
				BUG_ON(path->slots[0] != extent_slot + 1);
				path->slots[0] = extent_slot;
				num_to_del = 2;
			}
C
Chris Mason 已提交
5111
		}
5112

5113 5114
		ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
				      num_to_del);
5115 5116
		if (ret)
			goto abort;
5117
		btrfs_release_path(path);
5118

5119
		if (is_data) {
5120
			ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
5121 5122
			if (ret)
				goto abort;
5123 5124
		}

5125
		ret = update_block_group(trans, root, bytenr, num_bytes, 0);
5126 5127
		if (ret)
			goto abort;
5128
	}
5129
out:
5130
	btrfs_free_path(path);
5131
	return ret;
5132 5133 5134 5135

abort:
	btrfs_abort_transaction(trans, extent_root, ret);
	goto out;
5136 5137
}

5138
/*
5139
 * when we free an block, it is possible (and likely) that we free the last
5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150
 * delayed ref for that extent as well.  This searches the delayed ref tree for
 * a given extent, and if there are no other delayed refs to be processed, it
 * removes it from the tree.
 */
static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root, u64 bytenr)
{
	struct btrfs_delayed_ref_head *head;
	struct btrfs_delayed_ref_root *delayed_refs;
	struct btrfs_delayed_ref_node *ref;
	struct rb_node *node;
5151
	int ret = 0;
5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168

	delayed_refs = &trans->transaction->delayed_refs;
	spin_lock(&delayed_refs->lock);
	head = btrfs_find_delayed_ref_head(trans, bytenr);
	if (!head)
		goto out;

	node = rb_prev(&head->node.rb_node);
	if (!node)
		goto out;

	ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);

	/* there are still entries for this ref, we can't drop it */
	if (ref->bytenr == bytenr)
		goto out;

5169 5170 5171 5172 5173 5174 5175
	if (head->extent_op) {
		if (!head->must_insert_reserved)
			goto out;
		kfree(head->extent_op);
		head->extent_op = NULL;
	}

5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188
	/*
	 * waiting for the lock here would deadlock.  If someone else has it
	 * locked they are already in the process of dropping it anyway
	 */
	if (!mutex_trylock(&head->mutex))
		goto out;

	/*
	 * at this point we have a head with no other entries.  Go
	 * ahead and process it.
	 */
	head->node.in_tree = 0;
	rb_erase(&head->node.rb_node, &delayed_refs->root);
5189

5190
	delayed_refs->num_entries--;
5191 5192
	if (waitqueue_active(&delayed_refs->seq_wait))
		wake_up(&delayed_refs->seq_wait);
5193 5194 5195 5196 5197

	/*
	 * we don't take a ref on the node because we're removing it from the
	 * tree, so we just steal the ref the tree was holding.
	 */
5198 5199 5200 5201 5202
	delayed_refs->num_heads--;
	if (list_empty(&head->cluster))
		delayed_refs->num_heads_ready--;

	list_del_init(&head->cluster);
5203 5204
	spin_unlock(&delayed_refs->lock);

5205 5206 5207 5208 5209
	BUG_ON(head->extent_op);
	if (head->must_insert_reserved)
		ret = 1;

	mutex_unlock(&head->mutex);
5210
	btrfs_put_delayed_ref(&head->node);
5211
	return ret;
5212 5213 5214 5215 5216
out:
	spin_unlock(&delayed_refs->lock);
	return 0;
}

5217 5218 5219
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root,
			   struct extent_buffer *buf,
A
Arne Jansen 已提交
5220
			   u64 parent, int last_ref, int for_cow)
5221 5222 5223 5224 5225
{
	struct btrfs_block_group_cache *cache = NULL;
	int ret;

	if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
A
Arne Jansen 已提交
5226 5227 5228 5229 5230
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					buf->start, buf->len,
					parent, root->root_key.objectid,
					btrfs_header_level(buf),
					BTRFS_DROP_DELAYED_REF, NULL, for_cow);
5231
		BUG_ON(ret); /* -ENOMEM */
5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242
	}

	if (!last_ref)
		return;

	cache = btrfs_lookup_block_group(root->fs_info, buf->start);

	if (btrfs_header_generation(buf) == trans->transid) {
		if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
			ret = check_ref_cleanup(trans, root, buf->start);
			if (!ret)
5243
				goto out;
5244 5245 5246 5247
		}

		if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
			pin_down_extent(root, cache, buf->start, buf->len, 1);
5248
			goto out;
5249 5250 5251 5252 5253
		}

		WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));

		btrfs_add_free_space(cache, buf->start, buf->len);
5254
		btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
5255 5256
	}
out:
5257 5258 5259 5260 5261
	/*
	 * Deleting the buffer, clear the corrupt flag since it doesn't matter
	 * anymore.
	 */
	clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
5262 5263 5264
	btrfs_put_block_group(cache);
}

5265
/* Can return -ENOMEM */
A
Arne Jansen 已提交
5266 5267 5268
int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
		      u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
		      u64 owner, u64 offset, int for_cow)
5269 5270
{
	int ret;
A
Arne Jansen 已提交
5271
	struct btrfs_fs_info *fs_info = root->fs_info;
5272

5273 5274 5275 5276
	/*
	 * tree log blocks never actually go into the extent allocation
	 * tree, just update pinning info and exit early.
	 */
5277 5278
	if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
		WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
5279
		/* unlocks the pinned mutex */
5280
		btrfs_pin_extent(root, bytenr, num_bytes, 1);
5281
		ret = 0;
5282
	} else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
A
Arne Jansen 已提交
5283 5284
		ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
					num_bytes,
5285
					parent, root_objectid, (int)owner,
A
Arne Jansen 已提交
5286
					BTRFS_DROP_DELAYED_REF, NULL, for_cow);
5287
	} else {
A
Arne Jansen 已提交
5288 5289 5290 5291 5292
		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
						num_bytes,
						parent, root_objectid, owner,
						offset, BTRFS_DROP_DELAYED_REF,
						NULL, for_cow);
5293
	}
5294 5295 5296
	return ret;
}

5297 5298 5299 5300 5301 5302 5303
static u64 stripe_align(struct btrfs_root *root, u64 val)
{
	u64 mask = ((u64)root->stripesize - 1);
	u64 ret = (val + mask) & ~mask;
	return ret;
}

J
Josef Bacik 已提交
5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318
/*
 * when we wait for progress in the block group caching, its because
 * our allocation attempt failed at least once.  So, we must sleep
 * and let some progress happen before we try again.
 *
 * This function will sleep at least once waiting for new free space to
 * show up, and then it will check the block group free space numbers
 * for our min num_bytes.  Another option is to have it go ahead
 * and look in the rbtree for a free extent of a given size, but this
 * is a good start.
 */
static noinline int
wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
				u64 num_bytes)
{
5319
	struct btrfs_caching_control *caching_ctl;
J
Josef Bacik 已提交
5320 5321
	DEFINE_WAIT(wait);

5322 5323
	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
J
Josef Bacik 已提交
5324 5325
		return 0;

5326
	wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
5327
		   (cache->free_space_ctl->free_space >= num_bytes));
5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345

	put_caching_control(caching_ctl);
	return 0;
}

static noinline int
wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
{
	struct btrfs_caching_control *caching_ctl;
	DEFINE_WAIT(wait);

	caching_ctl = get_caching_control(cache);
	if (!caching_ctl)
		return 0;

	wait_event(caching_ctl->wait, block_group_cache_done(cache));

	put_caching_control(caching_ctl);
J
Josef Bacik 已提交
5346 5347 5348
	return 0;
}

5349
static int __get_block_group_index(u64 flags)
5350 5351
{
	int index;
5352 5353

	if (flags & BTRFS_BLOCK_GROUP_RAID10)
5354
		index = 0;
5355
	else if (flags & BTRFS_BLOCK_GROUP_RAID1)
5356
		index = 1;
5357
	else if (flags & BTRFS_BLOCK_GROUP_DUP)
5358
		index = 2;
5359
	else if (flags & BTRFS_BLOCK_GROUP_RAID0)
5360 5361 5362
		index = 3;
	else
		index = 4;
5363

5364 5365 5366
	return index;
}

5367 5368 5369 5370 5371
static int get_block_group_index(struct btrfs_block_group_cache *cache)
{
	return __get_block_group_index(cache->flags);
}

J
Josef Bacik 已提交
5372
enum btrfs_loop_type {
5373 5374 5375 5376
	LOOP_CACHING_NOWAIT = 0,
	LOOP_CACHING_WAIT = 1,
	LOOP_ALLOC_CHUNK = 2,
	LOOP_NO_EMPTY_SIZE = 3,
J
Josef Bacik 已提交
5377 5378
};

5379 5380 5381 5382
/*
 * walks the btree of allocated extents and find a hole of a given size.
 * The key ins is changed to record the hole:
 * ins->objectid == block start
5383
 * ins->flags = BTRFS_EXTENT_ITEM_KEY
5384 5385 5386
 * ins->offset == number of blocks
 * Any available blocks before search_start are skipped.
 */
C
Chris Mason 已提交
5387
static noinline int find_free_extent(struct btrfs_trans_handle *trans,
5388 5389 5390
				     struct btrfs_root *orig_root,
				     u64 num_bytes, u64 empty_size,
				     u64 hint_byte, struct btrfs_key *ins,
5391
				     u64 data)
5392
{
5393
	int ret = 0;
C
Chris Mason 已提交
5394
	struct btrfs_root *root = orig_root->fs_info->extent_root;
5395
	struct btrfs_free_cluster *last_ptr = NULL;
5396
	struct btrfs_block_group_cache *block_group = NULL;
5397
	struct btrfs_block_group_cache *used_block_group;
5398
	u64 search_start = 0;
5399
	int empty_cluster = 2 * 1024 * 1024;
C
Chris Mason 已提交
5400
	int allowed_chunk_alloc = 0;
5401
	int done_chunk_alloc = 0;
5402
	struct btrfs_space_info *space_info;
5403
	int loop = 0;
5404
	int index = 0;
5405 5406
	int alloc_type = (data & BTRFS_BLOCK_GROUP_DATA) ?
		RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
J
Josef Bacik 已提交
5407
	bool found_uncached_bg = false;
5408
	bool failed_cluster_refill = false;
5409
	bool failed_alloc = false;
5410
	bool use_cluster = true;
5411
	bool have_caching_bg = false;
5412

5413
	WARN_ON(num_bytes < root->sectorsize);
5414
	btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
5415 5416
	ins->objectid = 0;
	ins->offset = 0;
5417

J
Josef Bacik 已提交
5418 5419
	trace_find_free_extent(orig_root, num_bytes, empty_size, data);

J
Josef Bacik 已提交
5420
	space_info = __find_space_info(root->fs_info, data);
5421
	if (!space_info) {
5422
		printk(KERN_ERR "No space info for %llu\n", data);
5423 5424
		return -ENOSPC;
	}
J
Josef Bacik 已提交
5425

5426 5427 5428 5429 5430 5431 5432
	/*
	 * If the space info is for both data and metadata it means we have a
	 * small filesystem and we can't use the clustering stuff.
	 */
	if (btrfs_mixed_space_info(space_info))
		use_cluster = false;

C
Chris Mason 已提交
5433 5434 5435
	if (orig_root->ref_cows || empty_size)
		allowed_chunk_alloc = 1;

5436
	if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
5437
		last_ptr = &root->fs_info->meta_alloc_cluster;
5438 5439
		if (!btrfs_test_opt(root, SSD))
			empty_cluster = 64 * 1024;
5440 5441
	}

5442 5443
	if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
	    btrfs_test_opt(root, SSD)) {
5444 5445
		last_ptr = &root->fs_info->data_alloc_cluster;
	}
J
Josef Bacik 已提交
5446

5447
	if (last_ptr) {
5448 5449 5450 5451
		spin_lock(&last_ptr->lock);
		if (last_ptr->block_group)
			hint_byte = last_ptr->window_start;
		spin_unlock(&last_ptr->lock);
5452
	}
5453

5454
	search_start = max(search_start, first_logical_byte(root, 0));
5455
	search_start = max(search_start, hint_byte);
5456

J
Josef Bacik 已提交
5457
	if (!last_ptr)
5458 5459
		empty_cluster = 0;

J
Josef Bacik 已提交
5460 5461 5462
	if (search_start == hint_byte) {
		block_group = btrfs_lookup_block_group(root->fs_info,
						       search_start);
5463
		used_block_group = block_group;
J
Josef Bacik 已提交
5464 5465 5466
		/*
		 * we don't want to use the block group if it doesn't match our
		 * allocation bits, or if its not cached.
5467 5468 5469
		 *
		 * However if we are re-searching with an ideal block group
		 * picked out then we don't care that the block group is cached.
J
Josef Bacik 已提交
5470 5471
		 */
		if (block_group && block_group_bits(block_group, data) &&
5472
		    block_group->cached != BTRFS_CACHE_NO) {
J
Josef Bacik 已提交
5473
			down_read(&space_info->groups_sem);
5474 5475 5476 5477 5478 5479 5480 5481 5482 5483
			if (list_empty(&block_group->list) ||
			    block_group->ro) {
				/*
				 * someone is removing this block group,
				 * we can't jump into the have_block_group
				 * target because our list pointers are not
				 * valid
				 */
				btrfs_put_block_group(block_group);
				up_read(&space_info->groups_sem);
5484
			} else {
5485
				index = get_block_group_index(block_group);
5486
				goto have_block_group;
5487
			}
J
Josef Bacik 已提交
5488
		} else if (block_group) {
5489
			btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5490
		}
5491
	}
J
Josef Bacik 已提交
5492
search:
5493
	have_caching_bg = false;
5494
	down_read(&space_info->groups_sem);
5495 5496
	list_for_each_entry(block_group, &space_info->block_groups[index],
			    list) {
5497
		u64 offset;
J
Josef Bacik 已提交
5498
		int cached;
5499

5500
		used_block_group = block_group;
5501
		btrfs_get_block_group(block_group);
J
Josef Bacik 已提交
5502
		search_start = block_group->key.objectid;
5503

5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522
		/*
		 * this can happen if we end up cycling through all the
		 * raid types, but we want to make sure we only allocate
		 * for the proper type.
		 */
		if (!block_group_bits(block_group, data)) {
		    u64 extra = BTRFS_BLOCK_GROUP_DUP |
				BTRFS_BLOCK_GROUP_RAID1 |
				BTRFS_BLOCK_GROUP_RAID10;

			/*
			 * if they asked for extra copies and this block group
			 * doesn't provide them, bail.  This does allow us to
			 * fill raid0 from raid1.
			 */
			if ((data & extra) && !(block_group->flags & extra))
				goto loop;
		}

J
Josef Bacik 已提交
5523
have_block_group:
5524 5525 5526
		cached = block_group_cache_done(block_group);
		if (unlikely(!cached)) {
			found_uncached_bg = true;
5527
			ret = cache_block_group(block_group, trans,
5528
						orig_root, 0);
5529 5530
			BUG_ON(ret < 0);
			ret = 0;
J
Josef Bacik 已提交
5531 5532
		}

5533
		if (unlikely(block_group->ro))
J
Josef Bacik 已提交
5534
			goto loop;
J
Josef Bacik 已提交
5535

5536
		/*
5537 5538
		 * Ok we want to try and use the cluster allocator, so
		 * lets look there
5539
		 */
5540
		if (last_ptr) {
5541 5542 5543 5544 5545
			/*
			 * the refill lock keeps out other
			 * people trying to start a new cluster
			 */
			spin_lock(&last_ptr->refill_lock);
5546 5547 5548 5549 5550 5551
			used_block_group = last_ptr->block_group;
			if (used_block_group != block_group &&
			    (!used_block_group ||
			     used_block_group->ro ||
			     !block_group_bits(used_block_group, data))) {
				used_block_group = block_group;
5552
				goto refill_cluster;
5553 5554 5555 5556
			}

			if (used_block_group != block_group)
				btrfs_get_block_group(used_block_group);
5557

5558 5559
			offset = btrfs_alloc_from_cluster(used_block_group,
			  last_ptr, num_bytes, used_block_group->key.objectid);
5560 5561 5562
			if (offset) {
				/* we have a block, we're done */
				spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
5563 5564
				trace_btrfs_reserve_extent_cluster(root,
					block_group, search_start, num_bytes);
5565 5566 5567
				goto checks;
			}

5568 5569 5570 5571
			WARN_ON(last_ptr->block_group != used_block_group);
			if (used_block_group != block_group) {
				btrfs_put_block_group(used_block_group);
				used_block_group = block_group;
5572
			}
5573
refill_cluster:
5574
			BUG_ON(used_block_group != block_group);
5575 5576 5577 5578 5579 5580 5581 5582
			/* If we are on LOOP_NO_EMPTY_SIZE, we can't
			 * set up a new clusters, so lets just skip it
			 * and let the allocator find whatever block
			 * it can find.  If we reach this point, we
			 * will have tried the cluster allocator
			 * plenty of times and not have found
			 * anything, so we are likely way too
			 * fragmented for the clustering stuff to find
5583 5584 5585 5586 5587 5588 5589 5590 5591
			 * anything.
			 *
			 * However, if the cluster is taken from the
			 * current block group, release the cluster
			 * first, so that we stand a better chance of
			 * succeeding in the unclustered
			 * allocation.  */
			if (loop >= LOOP_NO_EMPTY_SIZE &&
			    last_ptr->block_group != block_group) {
5592 5593 5594 5595
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

5596 5597 5598 5599 5600 5601
			/*
			 * this cluster didn't work out, free it and
			 * start over
			 */
			btrfs_return_cluster_to_free_space(NULL, last_ptr);

5602 5603 5604 5605 5606
			if (loop >= LOOP_NO_EMPTY_SIZE) {
				spin_unlock(&last_ptr->refill_lock);
				goto unclustered_alloc;
			}

5607
			/* allocate a cluster in this block group */
5608
			ret = btrfs_find_space_cluster(trans, root,
5609
					       block_group, last_ptr,
5610
					       search_start, num_bytes,
5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622
					       empty_cluster + empty_size);
			if (ret == 0) {
				/*
				 * now pull our allocation out of this
				 * cluster
				 */
				offset = btrfs_alloc_from_cluster(block_group,
						  last_ptr, num_bytes,
						  search_start);
				if (offset) {
					/* we found one, proceed */
					spin_unlock(&last_ptr->refill_lock);
J
Josef Bacik 已提交
5623 5624 5625
					trace_btrfs_reserve_extent_cluster(root,
						block_group, search_start,
						num_bytes);
5626 5627
					goto checks;
				}
5628 5629
			} else if (!cached && loop > LOOP_CACHING_NOWAIT
				   && !failed_cluster_refill) {
J
Josef Bacik 已提交
5630 5631
				spin_unlock(&last_ptr->refill_lock);

5632
				failed_cluster_refill = true;
J
Josef Bacik 已提交
5633 5634 5635
				wait_block_group_cache_progress(block_group,
				       num_bytes + empty_cluster + empty_size);
				goto have_block_group;
5636
			}
J
Josef Bacik 已提交
5637

5638 5639 5640 5641 5642 5643
			/*
			 * at this point we either didn't find a cluster
			 * or we weren't able to allocate a block from our
			 * cluster.  Free the cluster we've been trying
			 * to use, and go to the next block group
			 */
5644
			btrfs_return_cluster_to_free_space(NULL, last_ptr);
5645
			spin_unlock(&last_ptr->refill_lock);
5646
			goto loop;
5647 5648
		}

5649
unclustered_alloc:
5650 5651 5652 5653 5654 5655 5656 5657 5658
		spin_lock(&block_group->free_space_ctl->tree_lock);
		if (cached &&
		    block_group->free_space_ctl->free_space <
		    num_bytes + empty_cluster + empty_size) {
			spin_unlock(&block_group->free_space_ctl->tree_lock);
			goto loop;
		}
		spin_unlock(&block_group->free_space_ctl->tree_lock);

5659 5660
		offset = btrfs_find_space_for_alloc(block_group, search_start,
						    num_bytes, empty_size);
5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671
		/*
		 * If we didn't find a chunk, and we haven't failed on this
		 * block group before, and this block group is in the middle of
		 * caching and we are ok with waiting, then go ahead and wait
		 * for progress to be made, and set failed_alloc to true.
		 *
		 * If failed_alloc is true then we've already waited on this
		 * block group once and should move on to the next block group.
		 */
		if (!offset && !failed_alloc && !cached &&
		    loop > LOOP_CACHING_NOWAIT) {
J
Josef Bacik 已提交
5672
			wait_block_group_cache_progress(block_group,
5673 5674
						num_bytes + empty_size);
			failed_alloc = true;
J
Josef Bacik 已提交
5675
			goto have_block_group;
5676
		} else if (!offset) {
5677 5678
			if (!cached)
				have_caching_bg = true;
5679
			goto loop;
J
Josef Bacik 已提交
5680
		}
5681
checks:
5682
		search_start = stripe_align(root, offset);
5683

J
Josef Bacik 已提交
5684 5685
		/* move on to the next group */
		if (search_start + num_bytes >
5686 5687
		    used_block_group->key.objectid + used_block_group->key.offset) {
			btrfs_add_free_space(used_block_group, offset, num_bytes);
J
Josef Bacik 已提交
5688
			goto loop;
5689
		}
5690

5691
		if (offset < search_start)
5692
			btrfs_add_free_space(used_block_group, offset,
5693 5694
					     search_start - offset);
		BUG_ON(offset > search_start);
J
Josef Bacik 已提交
5695

5696
		ret = btrfs_update_reserved_bytes(used_block_group, num_bytes,
5697
						  alloc_type);
5698
		if (ret == -EAGAIN) {
5699
			btrfs_add_free_space(used_block_group, offset, num_bytes);
J
Josef Bacik 已提交
5700
			goto loop;
J
Josef Bacik 已提交
5701
		}
5702

5703
		/* we are all good, lets return */
J
Josef Bacik 已提交
5704 5705
		ins->objectid = search_start;
		ins->offset = num_bytes;
5706

J
Josef Bacik 已提交
5707 5708
		trace_btrfs_reserve_extent(orig_root, block_group,
					   search_start, num_bytes);
5709
		if (offset < search_start)
5710
			btrfs_add_free_space(used_block_group, offset,
5711 5712
					     search_start - offset);
		BUG_ON(offset > search_start);
5713 5714
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
5715
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5716 5717
		break;
loop:
5718
		failed_cluster_refill = false;
5719
		failed_alloc = false;
5720
		BUG_ON(index != get_block_group_index(block_group));
5721 5722
		if (used_block_group != block_group)
			btrfs_put_block_group(used_block_group);
5723
		btrfs_put_block_group(block_group);
J
Josef Bacik 已提交
5724 5725 5726
	}
	up_read(&space_info->groups_sem);

5727 5728 5729
	if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
		goto search;

5730 5731 5732
	if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
		goto search;

5733
	/*
5734 5735
	 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
	 *			caching kthreads as we move along
J
Josef Bacik 已提交
5736 5737 5738 5739
	 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
	 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
	 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
	 *			again
5740
	 */
5741
	if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
5742
		index = 0;
5743
		loop++;
J
Josef Bacik 已提交
5744
		if (loop == LOOP_ALLOC_CHUNK) {
5745 5746 5747 5748
		       if (allowed_chunk_alloc) {
				ret = do_chunk_alloc(trans, root, num_bytes +
						     2 * 1024 * 1024, data,
						     CHUNK_ALLOC_LIMITED);
5749 5750 5751 5752 5753
				if (ret < 0) {
					btrfs_abort_transaction(trans,
								root, ret);
					goto out;
				}
5754 5755 5756 5757 5758 5759 5760 5761
				allowed_chunk_alloc = 0;
				if (ret == 1)
					done_chunk_alloc = 1;
			} else if (!done_chunk_alloc &&
				   space_info->force_alloc ==
				   CHUNK_ALLOC_NO_FORCE) {
				space_info->force_alloc = CHUNK_ALLOC_LIMITED;
			}
J
Josef Bacik 已提交
5762

5763 5764 5765 5766 5767 5768
		       /*
			* We didn't allocate a chunk, go ahead and drop the
			* empty size and loop again.
			*/
		       if (!done_chunk_alloc)
			       loop = LOOP_NO_EMPTY_SIZE;
J
Josef Bacik 已提交
5769 5770
		}

5771 5772 5773
		if (loop == LOOP_NO_EMPTY_SIZE) {
			empty_size = 0;
			empty_cluster = 0;
5774
		}
5775 5776

		goto search;
J
Josef Bacik 已提交
5777 5778
	} else if (!ins->objectid) {
		ret = -ENOSPC;
5779
	} else if (ins->objectid) {
5780
		ret = 0;
C
Chris Mason 已提交
5781
	}
5782
out:
C
Chris Mason 已提交
5783

C
Chris Mason 已提交
5784
	return ret;
5785
}
5786

J
Josef Bacik 已提交
5787 5788
static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
			    int dump_block_groups)
J
Josef Bacik 已提交
5789 5790
{
	struct btrfs_block_group_cache *cache;
5791
	int index = 0;
J
Josef Bacik 已提交
5792

J
Josef Bacik 已提交
5793
	spin_lock(&info->lock);
5794 5795
	printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
	       (unsigned long long)info->flags,
C
Chris Mason 已提交
5796
	       (unsigned long long)(info->total_bytes - info->bytes_used -
J
Josef Bacik 已提交
5797
				    info->bytes_pinned - info->bytes_reserved -
5798
				    info->bytes_readonly),
C
Chris Mason 已提交
5799
	       (info->full) ? "" : "not ");
5800 5801
	printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
	       "reserved=%llu, may_use=%llu, readonly=%llu\n",
5802
	       (unsigned long long)info->total_bytes,
5803
	       (unsigned long long)info->bytes_used,
5804
	       (unsigned long long)info->bytes_pinned,
5805
	       (unsigned long long)info->bytes_reserved,
5806
	       (unsigned long long)info->bytes_may_use,
5807
	       (unsigned long long)info->bytes_readonly);
J
Josef Bacik 已提交
5808 5809 5810 5811
	spin_unlock(&info->lock);

	if (!dump_block_groups)
		return;
J
Josef Bacik 已提交
5812

5813
	down_read(&info->groups_sem);
5814 5815
again:
	list_for_each_entry(cache, &info->block_groups[index], list) {
J
Josef Bacik 已提交
5816
		spin_lock(&cache->lock);
C
Chris Mason 已提交
5817 5818 5819 5820 5821 5822 5823
		printk(KERN_INFO "block group %llu has %llu bytes, %llu used "
		       "%llu pinned %llu reserved\n",
		       (unsigned long long)cache->key.objectid,
		       (unsigned long long)cache->key.offset,
		       (unsigned long long)btrfs_block_group_used(&cache->item),
		       (unsigned long long)cache->pinned,
		       (unsigned long long)cache->reserved);
J
Josef Bacik 已提交
5824 5825 5826
		btrfs_dump_free_space(cache, bytes);
		spin_unlock(&cache->lock);
	}
5827 5828
	if (++index < BTRFS_NR_RAID_TYPES)
		goto again;
5829
	up_read(&info->groups_sem);
J
Josef Bacik 已提交
5830
}
5831

5832 5833 5834 5835
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
			 struct btrfs_root *root,
			 u64 num_bytes, u64 min_alloc_size,
			 u64 empty_size, u64 hint_byte,
5836
			 struct btrfs_key *ins, u64 data)
5837
{
5838
	bool final_tried = false;
5839
	int ret;
5840

J
Josef Bacik 已提交
5841
	data = btrfs_get_alloc_profile(root, data);
5842
again:
C
Chris Mason 已提交
5843 5844 5845 5846
	/*
	 * the only place that sets empty_size is btrfs_realloc_node, which
	 * is not called recursively on allocations
	 */
5847
	if (empty_size || root->ref_cows) {
5848
		ret = do_chunk_alloc(trans, root->fs_info->extent_root,
5849 5850
				     num_bytes + 2 * 1024 * 1024, data,
				     CHUNK_ALLOC_NO_FORCE);
5851 5852 5853 5854 5855
		if (ret < 0 && ret != -ENOSPC) {
			btrfs_abort_transaction(trans, root, ret);
			return ret;
		}
	}
5856

5857 5858
	WARN_ON(num_bytes < root->sectorsize);
	ret = find_free_extent(trans, root, num_bytes, empty_size,
5859
			       hint_byte, ins, data);
5860

5861 5862 5863 5864 5865
	if (ret == -ENOSPC) {
		if (!final_tried) {
			num_bytes = num_bytes >> 1;
			num_bytes = num_bytes & ~(root->sectorsize - 1);
			num_bytes = max(num_bytes, min_alloc_size);
5866
			ret = do_chunk_alloc(trans, root->fs_info->extent_root,
5867
				       num_bytes, data, CHUNK_ALLOC_FORCE);
5868 5869 5870 5871
			if (ret < 0 && ret != -ENOSPC) {
				btrfs_abort_transaction(trans, root, ret);
				return ret;
			}
5872 5873 5874 5875 5876 5877 5878 5879 5880 5881
			if (num_bytes == min_alloc_size)
				final_tried = true;
			goto again;
		} else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
			struct btrfs_space_info *sinfo;

			sinfo = __find_space_info(root->fs_info, data);
			printk(KERN_ERR "btrfs allocation failed flags %llu, "
			       "wanted %llu\n", (unsigned long long)data,
			       (unsigned long long)num_bytes);
5882 5883
			if (sinfo)
				dump_space_info(sinfo, num_bytes, 1);
5884
		}
5885
	}
J
Josef Bacik 已提交
5886

5887 5888
	trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);

J
Josef Bacik 已提交
5889
	return ret;
5890 5891
}

5892 5893
static int __btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len, int pin)
5894
{
J
Josef Bacik 已提交
5895
	struct btrfs_block_group_cache *cache;
5896
	int ret = 0;
J
Josef Bacik 已提交
5897 5898 5899

	cache = btrfs_lookup_block_group(root->fs_info, start);
	if (!cache) {
C
Chris Mason 已提交
5900 5901
		printk(KERN_ERR "Unable to find block group for %llu\n",
		       (unsigned long long)start);
J
Josef Bacik 已提交
5902 5903
		return -ENOSPC;
	}
5904

5905 5906
	if (btrfs_test_opt(root, DISCARD))
		ret = btrfs_discard_extent(root, start, len, NULL);
5907

5908 5909 5910 5911 5912 5913
	if (pin)
		pin_down_extent(root, cache, start, len, 1);
	else {
		btrfs_add_free_space(cache, start, len);
		btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
	}
5914
	btrfs_put_block_group(cache);
J
Josef Bacik 已提交
5915

5916 5917
	trace_btrfs_reserved_extent_free(root, start, len);

5918 5919 5920
	return ret;
}

5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932
int btrfs_free_reserved_extent(struct btrfs_root *root,
					u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 0);
}

int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
				       u64 start, u64 len)
{
	return __btrfs_free_reserved_extent(root, start, len, 1);
}

5933 5934 5935 5936 5937
static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				      struct btrfs_root *root,
				      u64 parent, u64 root_objectid,
				      u64 flags, u64 owner, u64 offset,
				      struct btrfs_key *ins, int ref_mod)
5938 5939
{
	int ret;
5940
	struct btrfs_fs_info *fs_info = root->fs_info;
5941
	struct btrfs_extent_item *extent_item;
5942
	struct btrfs_extent_inline_ref *iref;
5943
	struct btrfs_path *path;
5944 5945 5946
	struct extent_buffer *leaf;
	int type;
	u32 size;
5947

5948 5949 5950 5951
	if (parent > 0)
		type = BTRFS_SHARED_DATA_REF_KEY;
	else
		type = BTRFS_EXTENT_DATA_REF_KEY;
5952

5953
	size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
5954 5955

	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
5956 5957
	if (!path)
		return -ENOMEM;
5958

5959
	path->leave_spinning = 1;
5960 5961
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
5962 5963 5964 5965
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
J
Josef Bacik 已提交
5966

5967 5968
	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
5969
				     struct btrfs_extent_item);
5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989
	btrfs_set_extent_refs(leaf, extent_item, ref_mod);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_DATA);

	iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
	btrfs_set_extent_inline_ref_type(leaf, iref, type);
	if (parent > 0) {
		struct btrfs_shared_data_ref *ref;
		ref = (struct btrfs_shared_data_ref *)(iref + 1);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
		btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
	} else {
		struct btrfs_extent_data_ref *ref;
		ref = (struct btrfs_extent_data_ref *)(&iref->offset);
		btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
		btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
		btrfs_set_extent_data_ref_offset(leaf, ref, offset);
		btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
	}
5990 5991

	btrfs_mark_buffer_dirty(path->nodes[0]);
5992
	btrfs_free_path(path);
5993

5994
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
5995
	if (ret) { /* -ENOENT, logic error */
C
Chris Mason 已提交
5996 5997 5998
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
5999 6000
		BUG();
	}
6001 6002 6003
	return ret;
}

6004 6005 6006 6007 6008
static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 parent, u64 root_objectid,
				     u64 flags, struct btrfs_disk_key *key,
				     int level, struct btrfs_key *ins)
6009 6010
{
	int ret;
6011 6012 6013 6014 6015 6016 6017
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_extent_item *extent_item;
	struct btrfs_tree_block_info *block_info;
	struct btrfs_extent_inline_ref *iref;
	struct btrfs_path *path;
	struct extent_buffer *leaf;
	u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
6018

6019
	path = btrfs_alloc_path();
6020 6021
	if (!path)
		return -ENOMEM;
6022

6023 6024 6025
	path->leave_spinning = 1;
	ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
				      ins, size);
6026 6027 6028 6029
	if (ret) {
		btrfs_free_path(path);
		return ret;
	}
6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057

	leaf = path->nodes[0];
	extent_item = btrfs_item_ptr(leaf, path->slots[0],
				     struct btrfs_extent_item);
	btrfs_set_extent_refs(leaf, extent_item, 1);
	btrfs_set_extent_generation(leaf, extent_item, trans->transid);
	btrfs_set_extent_flags(leaf, extent_item,
			       flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
	block_info = (struct btrfs_tree_block_info *)(extent_item + 1);

	btrfs_set_tree_block_key(leaf, block_info, key);
	btrfs_set_tree_block_level(leaf, block_info, level);

	iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
	if (parent > 0) {
		BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_SHARED_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
	} else {
		btrfs_set_extent_inline_ref_type(leaf, iref,
						 BTRFS_TREE_BLOCK_REF_KEY);
		btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
	}

	btrfs_mark_buffer_dirty(leaf);
	btrfs_free_path(path);

6058
	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
6059
	if (ret) { /* -ENOENT, logic error */
6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076
		printk(KERN_ERR "btrfs update block group failed for %llu "
		       "%llu\n", (unsigned long long)ins->objectid,
		       (unsigned long long)ins->offset);
		BUG();
	}
	return ret;
}

int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     u64 root_objectid, u64 owner,
				     u64 offset, struct btrfs_key *ins)
{
	int ret;

	BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);

A
Arne Jansen 已提交
6077 6078 6079 6080
	ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
					 ins->offset, 0,
					 root_objectid, owner, offset,
					 BTRFS_ADD_DELAYED_EXTENT, NULL, 0);
6081 6082
	return ret;
}
6083 6084 6085 6086 6087 6088

/*
 * this is used by the tree logging recovery code.  It records that
 * an extent has been allocated and makes sure to clear the free
 * space cache bits as well
 */
6089 6090 6091 6092
int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   u64 root_objectid, u64 owner, u64 offset,
				   struct btrfs_key *ins)
6093 6094 6095
{
	int ret;
	struct btrfs_block_group_cache *block_group;
6096 6097 6098
	struct btrfs_caching_control *caching_ctl;
	u64 start = ins->objectid;
	u64 num_bytes = ins->offset;
6099 6100

	block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
6101
	cache_block_group(block_group, trans, NULL, 0);
6102
	caching_ctl = get_caching_control(block_group);
6103

6104 6105 6106
	if (!caching_ctl) {
		BUG_ON(!block_group_cache_done(block_group));
		ret = btrfs_remove_free_space(block_group, start, num_bytes);
6107
		BUG_ON(ret); /* -ENOMEM */
6108 6109 6110 6111 6112
	} else {
		mutex_lock(&caching_ctl->mutex);

		if (start >= caching_ctl->progress) {
			ret = add_excluded_extent(root, start, num_bytes);
6113
			BUG_ON(ret); /* -ENOMEM */
6114 6115 6116
		} else if (start + num_bytes <= caching_ctl->progress) {
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
6117
			BUG_ON(ret); /* -ENOMEM */
6118 6119 6120 6121
		} else {
			num_bytes = caching_ctl->progress - start;
			ret = btrfs_remove_free_space(block_group,
						      start, num_bytes);
6122
			BUG_ON(ret); /* -ENOMEM */
6123 6124 6125 6126 6127

			start = caching_ctl->progress;
			num_bytes = ins->objectid + ins->offset -
				    caching_ctl->progress;
			ret = add_excluded_extent(root, start, num_bytes);
6128
			BUG_ON(ret); /* -ENOMEM */
6129 6130 6131 6132 6133 6134
		}

		mutex_unlock(&caching_ctl->mutex);
		put_caching_control(caching_ctl);
	}

6135 6136
	ret = btrfs_update_reserved_bytes(block_group, ins->offset,
					  RESERVE_ALLOC_NO_ACCOUNT);
6137
	BUG_ON(ret); /* logic error */
6138
	btrfs_put_block_group(block_group);
6139 6140
	ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
					 0, owner, offset, ins, 1);
6141 6142 6143
	return ret;
}

6144 6145
struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
					    struct btrfs_root *root,
6146 6147
					    u64 bytenr, u32 blocksize,
					    int level)
6148 6149 6150 6151 6152 6153 6154
{
	struct extent_buffer *buf;

	buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
	if (!buf)
		return ERR_PTR(-ENOMEM);
	btrfs_set_header_generation(buf, trans->transid);
6155
	btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
6156 6157
	btrfs_tree_lock(buf);
	clean_tree_block(trans, root, buf);
6158
	clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
6159 6160

	btrfs_set_lock_blocking(buf);
6161
	btrfs_set_buffer_uptodate(buf);
6162

6163
	if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
6164 6165 6166 6167 6168 6169 6170 6171 6172 6173
		/*
		 * we allow two log transactions at a time, use different
		 * EXENT bit to differentiate dirty pages.
		 */
		if (root->log_transid % 2 == 0)
			set_extent_dirty(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
		else
			set_extent_new(&root->dirty_log_pages, buf->start,
					buf->start + buf->len - 1, GFP_NOFS);
6174 6175
	} else {
		set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
6176
			 buf->start + buf->len - 1, GFP_NOFS);
6177
	}
6178
	trans->blocks_used++;
6179
	/* this returns a buffer locked for blocking */
6180 6181 6182
	return buf;
}

6183 6184 6185 6186 6187
static struct btrfs_block_rsv *
use_block_rsv(struct btrfs_trans_handle *trans,
	      struct btrfs_root *root, u32 blocksize)
{
	struct btrfs_block_rsv *block_rsv;
6188
	struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
6189 6190 6191 6192 6193
	int ret;

	block_rsv = get_block_rsv(trans, root);

	if (block_rsv->size == 0) {
6194
		ret = reserve_metadata_bytes(root, block_rsv, blocksize, 0);
6195 6196 6197 6198 6199 6200 6201 6202
		/*
		 * If we couldn't reserve metadata bytes try and use some from
		 * the global reserve.
		 */
		if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
6203
			return ERR_PTR(ret);
6204
		} else if (ret) {
6205
			return ERR_PTR(ret);
6206
		}
6207 6208 6209 6210 6211 6212
		return block_rsv;
	}

	ret = block_rsv_use_bytes(block_rsv, blocksize);
	if (!ret)
		return block_rsv;
6213
	if (ret) {
6214 6215 6216 6217 6218 6219 6220
		static DEFINE_RATELIMIT_STATE(_rs,
				DEFAULT_RATELIMIT_INTERVAL,
				/*DEFAULT_RATELIMIT_BURST*/ 2);
		if (__ratelimit(&_rs)) {
			printk(KERN_DEBUG "btrfs: block rsv returned %d\n", ret);
			WARN_ON(1);
		}
6221
		ret = reserve_metadata_bytes(root, block_rsv, blocksize, 0);
6222 6223 6224 6225 6226 6227 6228 6229
		if (!ret) {
			return block_rsv;
		} else if (ret && block_rsv != global_rsv) {
			ret = block_rsv_use_bytes(global_rsv, blocksize);
			if (!ret)
				return global_rsv;
		}
	}
6230 6231 6232 6233

	return ERR_PTR(-ENOSPC);
}

J
Josef Bacik 已提交
6234 6235
static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
			    struct btrfs_block_rsv *block_rsv, u32 blocksize)
6236 6237
{
	block_rsv_add_bytes(block_rsv, blocksize, 0);
J
Josef Bacik 已提交
6238
	block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
6239 6240
}

6241
/*
6242 6243 6244 6245
 * finds a free extent and does all the dirty work required for allocation
 * returns the key for the extent through ins, and a tree buffer for
 * the first block of the extent through buf.
 *
6246 6247
 * returns the tree buffer or NULL.
 */
6248
struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
6249 6250 6251
					struct btrfs_root *root, u32 blocksize,
					u64 parent, u64 root_objectid,
					struct btrfs_disk_key *key, int level,
A
Arne Jansen 已提交
6252
					u64 hint, u64 empty_size, int for_cow)
6253
{
C
Chris Mason 已提交
6254
	struct btrfs_key ins;
6255
	struct btrfs_block_rsv *block_rsv;
6256
	struct extent_buffer *buf;
6257 6258 6259
	u64 flags = 0;
	int ret;

6260

6261 6262 6263 6264 6265
	block_rsv = use_block_rsv(trans, root, blocksize);
	if (IS_ERR(block_rsv))
		return ERR_CAST(block_rsv);

	ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
6266
				   empty_size, hint, &ins, 0);
6267
	if (ret) {
J
Josef Bacik 已提交
6268
		unuse_block_rsv(root->fs_info, block_rsv, blocksize);
6269
		return ERR_PTR(ret);
6270
	}
6271

6272 6273
	buf = btrfs_init_new_buffer(trans, root, ins.objectid,
				    blocksize, level);
6274
	BUG_ON(IS_ERR(buf)); /* -ENOMEM */
6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285

	if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
		if (parent == 0)
			parent = ins.objectid;
		flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
	} else
		BUG_ON(parent > 0);

	if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
		struct btrfs_delayed_extent_op *extent_op;
		extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
6286
		BUG_ON(!extent_op); /* -ENOMEM */
6287 6288 6289 6290 6291 6292 6293 6294 6295
		if (key)
			memcpy(&extent_op->key, key, sizeof(extent_op->key));
		else
			memset(&extent_op->key, 0, sizeof(extent_op->key));
		extent_op->flags_to_set = flags;
		extent_op->update_key = 1;
		extent_op->update_flags = 1;
		extent_op->is_data = 0;

A
Arne Jansen 已提交
6296 6297
		ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
					ins.objectid,
6298 6299
					ins.offset, parent, root_objectid,
					level, BTRFS_ADD_DELAYED_EXTENT,
A
Arne Jansen 已提交
6300
					extent_op, for_cow);
6301
		BUG_ON(ret); /* -ENOMEM */
6302
	}
6303 6304
	return buf;
}
6305

6306 6307 6308 6309 6310 6311 6312 6313 6314
struct walk_control {
	u64 refs[BTRFS_MAX_LEVEL];
	u64 flags[BTRFS_MAX_LEVEL];
	struct btrfs_key update_progress;
	int stage;
	int level;
	int shared_level;
	int update_ref;
	int keep_locks;
Y
Yan, Zheng 已提交
6315 6316
	int reada_slot;
	int reada_count;
A
Arne Jansen 已提交
6317
	int for_reloc;
6318 6319 6320 6321 6322
};

#define DROP_REFERENCE	1
#define UPDATE_BACKREF	2

Y
Yan, Zheng 已提交
6323 6324 6325 6326
static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
				     struct btrfs_root *root,
				     struct walk_control *wc,
				     struct btrfs_path *path)
6327
{
Y
Yan, Zheng 已提交
6328 6329 6330
	u64 bytenr;
	u64 generation;
	u64 refs;
6331
	u64 flags;
6332
	u32 nritems;
Y
Yan, Zheng 已提交
6333 6334 6335
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *eb;
6336
	int ret;
Y
Yan, Zheng 已提交
6337 6338
	int slot;
	int nread = 0;
6339

Y
Yan, Zheng 已提交
6340 6341 6342 6343 6344 6345 6346 6347
	if (path->slots[wc->level] < wc->reada_slot) {
		wc->reada_count = wc->reada_count * 2 / 3;
		wc->reada_count = max(wc->reada_count, 2);
	} else {
		wc->reada_count = wc->reada_count * 3 / 2;
		wc->reada_count = min_t(int, wc->reada_count,
					BTRFS_NODEPTRS_PER_BLOCK(root));
	}
6348

Y
Yan, Zheng 已提交
6349 6350 6351
	eb = path->nodes[wc->level];
	nritems = btrfs_header_nritems(eb);
	blocksize = btrfs_level_size(root, wc->level - 1);
6352

Y
Yan, Zheng 已提交
6353 6354 6355
	for (slot = path->slots[wc->level]; slot < nritems; slot++) {
		if (nread >= wc->reada_count)
			break;
6356

C
Chris Mason 已提交
6357
		cond_resched();
Y
Yan, Zheng 已提交
6358 6359
		bytenr = btrfs_node_blockptr(eb, slot);
		generation = btrfs_node_ptr_generation(eb, slot);
C
Chris Mason 已提交
6360

Y
Yan, Zheng 已提交
6361 6362
		if (slot == path->slots[wc->level])
			goto reada;
6363

Y
Yan, Zheng 已提交
6364 6365
		if (wc->stage == UPDATE_BACKREF &&
		    generation <= root->root_key.offset)
6366 6367
			continue;

6368 6369 6370
		/* We don't lock the tree block, it's OK to be racy here */
		ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
					       &refs, &flags);
6371 6372 6373
		/* We don't care about errors in readahead. */
		if (ret < 0)
			continue;
6374 6375
		BUG_ON(refs == 0);

Y
Yan, Zheng 已提交
6376 6377 6378
		if (wc->stage == DROP_REFERENCE) {
			if (refs == 1)
				goto reada;
6379

6380 6381 6382
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
Y
Yan, Zheng 已提交
6383 6384 6385 6386 6387 6388 6389 6390
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				continue;
			btrfs_node_key_to_cpu(eb, &key, slot);
			ret = btrfs_comp_cpu_keys(&key,
						  &wc->update_progress);
			if (ret < 0)
				continue;
6391 6392 6393 6394
		} else {
			if (wc->level == 1 &&
			    (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				continue;
6395
		}
Y
Yan, Zheng 已提交
6396 6397 6398 6399
reada:
		ret = readahead_tree_block(root, bytenr, blocksize,
					   generation);
		if (ret)
6400
			break;
Y
Yan, Zheng 已提交
6401
		nread++;
C
Chris Mason 已提交
6402
	}
Y
Yan, Zheng 已提交
6403
	wc->reada_slot = slot;
C
Chris Mason 已提交
6404
}
6405

Y
Yan Zheng 已提交
6406
/*
6407 6408 6409 6410 6411 6412
 * hepler to process tree block while walking down the tree.
 *
 * when wc->stage == UPDATE_BACKREF, this function updates
 * back refs for pointers in the block.
 *
 * NOTE: return value 1 means we should stop walking down.
Y
Yan Zheng 已提交
6413
 */
6414
static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
6415
				   struct btrfs_root *root,
6416
				   struct btrfs_path *path,
6417
				   struct walk_control *wc, int lookup_info)
Y
Yan Zheng 已提交
6418
{
6419 6420 6421
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
Y
Yan Zheng 已提交
6422 6423
	int ret;

6424 6425 6426
	if (wc->stage == UPDATE_BACKREF &&
	    btrfs_header_owner(eb) != root->root_key.objectid)
		return 1;
Y
Yan Zheng 已提交
6427

6428 6429 6430 6431
	/*
	 * when reference count of tree block is 1, it won't increase
	 * again. once full backref flag is set, we never clear it.
	 */
6432 6433 6434
	if (lookup_info &&
	    ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
	     (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
6435 6436 6437 6438 6439
		BUG_ON(!path->locks[level]);
		ret = btrfs_lookup_extent_info(trans, root,
					       eb->start, eb->len,
					       &wc->refs[level],
					       &wc->flags[level]);
6440 6441 6442
		BUG_ON(ret == -ENOMEM);
		if (ret)
			return ret;
6443 6444
		BUG_ON(wc->refs[level] == 0);
	}
6445

6446 6447 6448
	if (wc->stage == DROP_REFERENCE) {
		if (wc->refs[level] > 1)
			return 1;
Y
Yan Zheng 已提交
6449

6450
		if (path->locks[level] && !wc->keep_locks) {
6451
			btrfs_tree_unlock_rw(eb, path->locks[level]);
6452 6453 6454 6455
			path->locks[level] = 0;
		}
		return 0;
	}
Y
Yan Zheng 已提交
6456

6457 6458 6459
	/* wc->stage == UPDATE_BACKREF */
	if (!(wc->flags[level] & flag)) {
		BUG_ON(!path->locks[level]);
A
Arne Jansen 已提交
6460
		ret = btrfs_inc_ref(trans, root, eb, 1, wc->for_reloc);
6461
		BUG_ON(ret); /* -ENOMEM */
A
Arne Jansen 已提交
6462
		ret = btrfs_dec_ref(trans, root, eb, 0, wc->for_reloc);
6463
		BUG_ON(ret); /* -ENOMEM */
6464 6465
		ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
						  eb->len, flag, 0);
6466
		BUG_ON(ret); /* -ENOMEM */
6467 6468 6469 6470 6471 6472 6473 6474
		wc->flags[level] |= flag;
	}

	/*
	 * the block is shared by multiple trees, so it's not good to
	 * keep the tree lock
	 */
	if (path->locks[level] && level > 0) {
6475
		btrfs_tree_unlock_rw(eb, path->locks[level]);
6476 6477 6478 6479 6480
		path->locks[level] = 0;
	}
	return 0;
}

Y
Yan, Zheng 已提交
6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496
/*
 * hepler to process tree block pointer.
 *
 * when wc->stage == DROP_REFERENCE, this function checks
 * reference count of the block pointed to. if the block
 * is shared and we need update back refs for the subtree
 * rooted at the block, this function changes wc->stage to
 * UPDATE_BACKREF. if the block is shared and there is no
 * need to update back, this function drops the reference
 * to the block.
 *
 * NOTE: return value 1 means we should stop walking down.
 */
static noinline int do_walk_down(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
6497
				 struct walk_control *wc, int *lookup_info)
Y
Yan, Zheng 已提交
6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516
{
	u64 bytenr;
	u64 generation;
	u64 parent;
	u32 blocksize;
	struct btrfs_key key;
	struct extent_buffer *next;
	int level = wc->level;
	int reada = 0;
	int ret = 0;

	generation = btrfs_node_ptr_generation(path->nodes[level],
					       path->slots[level]);
	/*
	 * if the lower level block was created before the snapshot
	 * was created, we know there is no need to update back refs
	 * for the subtree
	 */
	if (wc->stage == UPDATE_BACKREF &&
6517 6518
	    generation <= root->root_key.offset) {
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6519
		return 1;
6520
	}
Y
Yan, Zheng 已提交
6521 6522 6523 6524 6525 6526 6527

	bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
	blocksize = btrfs_level_size(root, level - 1);

	next = btrfs_find_tree_block(root, bytenr, blocksize);
	if (!next) {
		next = btrfs_find_create_tree_block(root, bytenr, blocksize);
6528 6529
		if (!next)
			return -ENOMEM;
Y
Yan, Zheng 已提交
6530 6531 6532 6533 6534
		reada = 1;
	}
	btrfs_tree_lock(next);
	btrfs_set_lock_blocking(next);

6535 6536 6537
	ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
				       &wc->refs[level - 1],
				       &wc->flags[level - 1]);
6538 6539 6540 6541 6542
	if (ret < 0) {
		btrfs_tree_unlock(next);
		return ret;
	}

6543 6544
	BUG_ON(wc->refs[level - 1] == 0);
	*lookup_info = 0;
Y
Yan, Zheng 已提交
6545

6546
	if (wc->stage == DROP_REFERENCE) {
Y
Yan, Zheng 已提交
6547
		if (wc->refs[level - 1] > 1) {
6548 6549 6550 6551
			if (level == 1 &&
			    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
				goto skip;

Y
Yan, Zheng 已提交
6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564
			if (!wc->update_ref ||
			    generation <= root->root_key.offset)
				goto skip;

			btrfs_node_key_to_cpu(path->nodes[level], &key,
					      path->slots[level]);
			ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
			if (ret < 0)
				goto skip;

			wc->stage = UPDATE_BACKREF;
			wc->shared_level = level - 1;
		}
6565 6566 6567 6568
	} else {
		if (level == 1 &&
		    (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
			goto skip;
Y
Yan, Zheng 已提交
6569 6570
	}

6571
	if (!btrfs_buffer_uptodate(next, generation, 0)) {
Y
Yan, Zheng 已提交
6572 6573 6574
		btrfs_tree_unlock(next);
		free_extent_buffer(next);
		next = NULL;
6575
		*lookup_info = 1;
Y
Yan, Zheng 已提交
6576 6577 6578 6579 6580 6581
	}

	if (!next) {
		if (reada && level == 1)
			reada_walk_down(trans, root, wc, path);
		next = read_tree_block(root, bytenr, blocksize, generation);
6582 6583
		if (!next)
			return -EIO;
Y
Yan, Zheng 已提交
6584 6585 6586 6587 6588 6589 6590 6591
		btrfs_tree_lock(next);
		btrfs_set_lock_blocking(next);
	}

	level--;
	BUG_ON(level != btrfs_header_level(next));
	path->nodes[level] = next;
	path->slots[level] = 0;
6592
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Y
Yan, Zheng 已提交
6593 6594 6595 6596 6597 6598 6599
	wc->level = level;
	if (wc->level == 1)
		wc->reada_slot = 0;
	return 0;
skip:
	wc->refs[level - 1] = 0;
	wc->flags[level - 1] = 0;
6600 6601 6602 6603 6604 6605 6606 6607
	if (wc->stage == DROP_REFERENCE) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
			parent = path->nodes[level]->start;
		} else {
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level]));
			parent = 0;
		}
Y
Yan, Zheng 已提交
6608

6609
		ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
A
Arne Jansen 已提交
6610
				root->root_key.objectid, level - 1, 0, 0);
6611
		BUG_ON(ret); /* -ENOMEM */
Y
Yan, Zheng 已提交
6612 6613 6614
	}
	btrfs_tree_unlock(next);
	free_extent_buffer(next);
6615
	*lookup_info = 1;
Y
Yan, Zheng 已提交
6616 6617 6618
	return 1;
}

6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635
/*
 * hepler to process tree block while walking up the tree.
 *
 * when wc->stage == DROP_REFERENCE, this function drops
 * reference count on the block.
 *
 * when wc->stage == UPDATE_BACKREF, this function changes
 * wc->stage back to DROP_REFERENCE if we changed wc->stage
 * to UPDATE_BACKREF previously while processing the block.
 *
 * NOTE: return value 1 means we should stop walking up.
 */
static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
				 struct btrfs_root *root,
				 struct btrfs_path *path,
				 struct walk_control *wc)
{
6636
	int ret;
6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662
	int level = wc->level;
	struct extent_buffer *eb = path->nodes[level];
	u64 parent = 0;

	if (wc->stage == UPDATE_BACKREF) {
		BUG_ON(wc->shared_level < level);
		if (level < wc->shared_level)
			goto out;

		ret = find_next_key(path, level + 1, &wc->update_progress);
		if (ret > 0)
			wc->update_ref = 0;

		wc->stage = DROP_REFERENCE;
		wc->shared_level = -1;
		path->slots[level] = 0;

		/*
		 * check reference count again if the block isn't locked.
		 * we should start walking down the tree again if reference
		 * count is one.
		 */
		if (!path->locks[level]) {
			BUG_ON(level == 0);
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6663
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6664 6665 6666 6667 6668

			ret = btrfs_lookup_extent_info(trans, root,
						       eb->start, eb->len,
						       &wc->refs[level],
						       &wc->flags[level]);
6669 6670 6671 6672
			if (ret < 0) {
				btrfs_tree_unlock_rw(eb, path->locks[level]);
				return ret;
			}
6673 6674
			BUG_ON(wc->refs[level] == 0);
			if (wc->refs[level] == 1) {
6675
				btrfs_tree_unlock_rw(eb, path->locks[level]);
6676 6677
				return 1;
			}
Y
Yan Zheng 已提交
6678
		}
6679
	}
Y
Yan Zheng 已提交
6680

6681 6682
	/* wc->stage == DROP_REFERENCE */
	BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6683

6684 6685 6686
	if (wc->refs[level] == 1) {
		if (level == 0) {
			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
A
Arne Jansen 已提交
6687 6688
				ret = btrfs_dec_ref(trans, root, eb, 1,
						    wc->for_reloc);
6689
			else
A
Arne Jansen 已提交
6690 6691
				ret = btrfs_dec_ref(trans, root, eb, 0,
						    wc->for_reloc);
6692
			BUG_ON(ret); /* -ENOMEM */
6693 6694 6695 6696 6697 6698
		}
		/* make block locked assertion in clean_tree_block happy */
		if (!path->locks[level] &&
		    btrfs_header_generation(eb) == trans->transid) {
			btrfs_tree_lock(eb);
			btrfs_set_lock_blocking(eb);
6699
			path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715
		}
		clean_tree_block(trans, root, eb);
	}

	if (eb == root->node) {
		if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = eb->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(eb));
	} else {
		if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
			parent = path->nodes[level + 1]->start;
		else
			BUG_ON(root->root_key.objectid !=
			       btrfs_header_owner(path->nodes[level + 1]));
Y
Yan Zheng 已提交
6716 6717
	}

A
Arne Jansen 已提交
6718
	btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1, 0);
6719 6720 6721
out:
	wc->refs[level] = 0;
	wc->flags[level] = 0;
6722
	return 0;
6723 6724 6725 6726 6727 6728 6729 6730
}

static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
				   struct btrfs_root *root,
				   struct btrfs_path *path,
				   struct walk_control *wc)
{
	int level = wc->level;
6731
	int lookup_info = 1;
6732 6733 6734
	int ret;

	while (level >= 0) {
6735
		ret = walk_down_proc(trans, root, path, wc, lookup_info);
6736 6737 6738 6739 6740 6741
		if (ret > 0)
			break;

		if (level == 0)
			break;

6742 6743 6744 6745
		if (path->slots[level] >=
		    btrfs_header_nritems(path->nodes[level]))
			break;

6746
		ret = do_walk_down(trans, root, path, wc, &lookup_info);
Y
Yan, Zheng 已提交
6747 6748 6749
		if (ret > 0) {
			path->slots[level]++;
			continue;
6750 6751
		} else if (ret < 0)
			return ret;
Y
Yan, Zheng 已提交
6752
		level = wc->level;
Y
Yan Zheng 已提交
6753 6754 6755 6756
	}
	return 0;
}

C
Chris Mason 已提交
6757
static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6758
				 struct btrfs_root *root,
Y
Yan Zheng 已提交
6759
				 struct btrfs_path *path,
6760
				 struct walk_control *wc, int max_level)
C
Chris Mason 已提交
6761
{
6762
	int level = wc->level;
C
Chris Mason 已提交
6763
	int ret;
6764

6765 6766 6767 6768 6769 6770
	path->slots[level] = btrfs_header_nritems(path->nodes[level]);
	while (level < max_level && path->nodes[level]) {
		wc->level = level;
		if (path->slots[level] + 1 <
		    btrfs_header_nritems(path->nodes[level])) {
			path->slots[level]++;
C
Chris Mason 已提交
6771 6772
			return 0;
		} else {
6773 6774 6775
			ret = walk_up_proc(trans, root, path, wc);
			if (ret > 0)
				return 0;
6776

6777
			if (path->locks[level]) {
6778 6779
				btrfs_tree_unlock_rw(path->nodes[level],
						     path->locks[level]);
6780
				path->locks[level] = 0;
Y
Yan Zheng 已提交
6781
			}
6782 6783 6784
			free_extent_buffer(path->nodes[level]);
			path->nodes[level] = NULL;
			level++;
C
Chris Mason 已提交
6785 6786 6787 6788 6789
		}
	}
	return 1;
}

C
Chris Mason 已提交
6790
/*
6791 6792 6793 6794 6795 6796 6797 6798 6799
 * drop a subvolume tree.
 *
 * this function traverses the tree freeing any blocks that only
 * referenced by the tree.
 *
 * when a shared tree block is found. this function decreases its
 * reference count by one. if update_ref is true, this function
 * also make sure backrefs for the shared block and all lower level
 * blocks are properly updated.
C
Chris Mason 已提交
6800
 */
6801
int btrfs_drop_snapshot(struct btrfs_root *root,
A
Arne Jansen 已提交
6802 6803
			 struct btrfs_block_rsv *block_rsv, int update_ref,
			 int for_reloc)
C
Chris Mason 已提交
6804
{
6805
	struct btrfs_path *path;
6806 6807
	struct btrfs_trans_handle *trans;
	struct btrfs_root *tree_root = root->fs_info->tree_root;
6808
	struct btrfs_root_item *root_item = &root->root_item;
6809 6810 6811 6812 6813
	struct walk_control *wc;
	struct btrfs_key key;
	int err = 0;
	int ret;
	int level;
C
Chris Mason 已提交
6814

6815
	path = btrfs_alloc_path();
6816 6817 6818 6819
	if (!path) {
		err = -ENOMEM;
		goto out;
	}
C
Chris Mason 已提交
6820

6821
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
6822 6823
	if (!wc) {
		btrfs_free_path(path);
6824 6825
		err = -ENOMEM;
		goto out;
6826
	}
6827

6828
	trans = btrfs_start_transaction(tree_root, 0);
6829 6830 6831 6832
	if (IS_ERR(trans)) {
		err = PTR_ERR(trans);
		goto out_free;
	}
6833

6834 6835
	if (block_rsv)
		trans->block_rsv = block_rsv;
6836

6837
	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
6838
		level = btrfs_header_level(root->node);
6839 6840
		path->nodes[level] = btrfs_lock_root_node(root);
		btrfs_set_lock_blocking(path->nodes[level]);
6841
		path->slots[level] = 0;
6842
		path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6843 6844
		memset(&wc->update_progress, 0,
		       sizeof(wc->update_progress));
6845 6846
	} else {
		btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
6847 6848 6849
		memcpy(&wc->update_progress, &key,
		       sizeof(wc->update_progress));

6850
		level = root_item->drop_level;
6851
		BUG_ON(level == 0);
6852
		path->lowest_level = level;
6853 6854 6855 6856
		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
		path->lowest_level = 0;
		if (ret < 0) {
			err = ret;
6857
			goto out_end_trans;
6858
		}
Y
Yan, Zheng 已提交
6859
		WARN_ON(ret > 0);
6860

6861 6862 6863 6864
		/*
		 * unlock our path, this is safe because only this
		 * function is allowed to delete this snapshot
		 */
6865
		btrfs_unlock_up_safe(path, 0);
6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876

		level = btrfs_header_level(root->node);
		while (1) {
			btrfs_tree_lock(path->nodes[level]);
			btrfs_set_lock_blocking(path->nodes[level]);

			ret = btrfs_lookup_extent_info(trans, root,
						path->nodes[level]->start,
						path->nodes[level]->len,
						&wc->refs[level],
						&wc->flags[level]);
6877 6878 6879 6880
			if (ret < 0) {
				err = ret;
				goto out_end_trans;
			}
6881 6882 6883 6884 6885 6886 6887 6888 6889
			BUG_ON(wc->refs[level] == 0);

			if (level == root_item->drop_level)
				break;

			btrfs_tree_unlock(path->nodes[level]);
			WARN_ON(wc->refs[level] != 1);
			level--;
		}
6890
	}
6891 6892 6893 6894 6895 6896

	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = update_ref;
	wc->keep_locks = 0;
A
Arne Jansen 已提交
6897
	wc->for_reloc = for_reloc;
Y
Yan, Zheng 已提交
6898
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
6899

C
Chris Mason 已提交
6900
	while (1) {
6901 6902 6903
		ret = walk_down_tree(trans, root, path, wc);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6904
			break;
6905
		}
C
Chris Mason 已提交
6906

6907 6908 6909
		ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
		if (ret < 0) {
			err = ret;
C
Chris Mason 已提交
6910
			break;
6911 6912 6913 6914
		}

		if (ret > 0) {
			BUG_ON(wc->stage != DROP_REFERENCE);
6915 6916
			break;
		}
6917 6918 6919 6920 6921 6922 6923 6924 6925 6926

		if (wc->stage == DROP_REFERENCE) {
			level = wc->level;
			btrfs_node_key(path->nodes[level],
				       &root_item->drop_progress,
				       path->slots[level]);
			root_item->drop_level = level;
		}

		BUG_ON(wc->level == 0);
6927
		if (btrfs_should_end_transaction(trans, tree_root)) {
6928 6929 6930
			ret = btrfs_update_root(trans, tree_root,
						&root->root_key,
						root_item);
6931 6932 6933 6934 6935
			if (ret) {
				btrfs_abort_transaction(trans, tree_root, ret);
				err = ret;
				goto out_end_trans;
			}
6936

6937
			btrfs_end_transaction_throttle(trans, tree_root);
6938
			trans = btrfs_start_transaction(tree_root, 0);
6939 6940 6941 6942
			if (IS_ERR(trans)) {
				err = PTR_ERR(trans);
				goto out_free;
			}
6943 6944
			if (block_rsv)
				trans->block_rsv = block_rsv;
6945
		}
C
Chris Mason 已提交
6946
	}
6947
	btrfs_release_path(path);
6948 6949
	if (err)
		goto out_end_trans;
6950 6951

	ret = btrfs_del_root(trans, tree_root, &root->root_key);
6952 6953 6954 6955
	if (ret) {
		btrfs_abort_transaction(trans, tree_root, ret);
		goto out_end_trans;
	}
6956

6957 6958 6959
	if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
		ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
					   NULL, NULL);
6960 6961 6962 6963 6964
		if (ret < 0) {
			btrfs_abort_transaction(trans, tree_root, ret);
			err = ret;
			goto out_end_trans;
		} else if (ret > 0) {
6965 6966 6967 6968 6969 6970 6971
			/* if we fail to delete the orphan item this time
			 * around, it'll get picked up the next time.
			 *
			 * The most common failure here is just -ENOENT.
			 */
			btrfs_del_orphan_item(trans, tree_root,
					      root->root_key.objectid);
6972 6973 6974 6975 6976 6977 6978 6979 6980 6981
		}
	}

	if (root->in_radix) {
		btrfs_free_fs_root(tree_root->fs_info, root);
	} else {
		free_extent_buffer(root->node);
		free_extent_buffer(root->commit_root);
		kfree(root);
	}
6982
out_end_trans:
6983
	btrfs_end_transaction_throttle(trans, tree_root);
6984
out_free:
6985
	kfree(wc);
6986
	btrfs_free_path(path);
6987 6988 6989
out:
	if (err)
		btrfs_std_error(root->fs_info, err);
6990
	return err;
C
Chris Mason 已提交
6991
}
C
Chris Mason 已提交
6992

6993 6994 6995 6996
/*
 * drop subtree rooted at tree block 'node'.
 *
 * NOTE: this function will unlock and release tree block 'node'
A
Arne Jansen 已提交
6997
 * only used by relocation code
6998
 */
Y
Yan Zheng 已提交
6999 7000 7001 7002 7003 7004
int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
			struct btrfs_root *root,
			struct extent_buffer *node,
			struct extent_buffer *parent)
{
	struct btrfs_path *path;
7005
	struct walk_control *wc;
Y
Yan Zheng 已提交
7006 7007 7008 7009 7010
	int level;
	int parent_level;
	int ret = 0;
	int wret;

7011 7012
	BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);

Y
Yan Zheng 已提交
7013
	path = btrfs_alloc_path();
T
Tsutomu Itoh 已提交
7014 7015
	if (!path)
		return -ENOMEM;
Y
Yan Zheng 已提交
7016

7017
	wc = kzalloc(sizeof(*wc), GFP_NOFS);
T
Tsutomu Itoh 已提交
7018 7019 7020 7021
	if (!wc) {
		btrfs_free_path(path);
		return -ENOMEM;
	}
7022

7023
	btrfs_assert_tree_locked(parent);
Y
Yan Zheng 已提交
7024 7025 7026 7027 7028
	parent_level = btrfs_header_level(parent);
	extent_buffer_get(parent);
	path->nodes[parent_level] = parent;
	path->slots[parent_level] = btrfs_header_nritems(parent);

7029
	btrfs_assert_tree_locked(node);
Y
Yan Zheng 已提交
7030 7031 7032
	level = btrfs_header_level(node);
	path->nodes[level] = node;
	path->slots[level] = 0;
7033
	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7034 7035 7036 7037 7038 7039 7040 7041

	wc->refs[parent_level] = 1;
	wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
	wc->level = level;
	wc->shared_level = -1;
	wc->stage = DROP_REFERENCE;
	wc->update_ref = 0;
	wc->keep_locks = 1;
A
Arne Jansen 已提交
7042
	wc->for_reloc = 1;
Y
Yan, Zheng 已提交
7043
	wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
Y
Yan Zheng 已提交
7044 7045

	while (1) {
7046 7047
		wret = walk_down_tree(trans, root, path, wc);
		if (wret < 0) {
Y
Yan Zheng 已提交
7048 7049
			ret = wret;
			break;
7050
		}
Y
Yan Zheng 已提交
7051

7052
		wret = walk_up_tree(trans, root, path, wc, parent_level);
Y
Yan Zheng 已提交
7053 7054 7055 7056 7057 7058
		if (wret < 0)
			ret = wret;
		if (wret != 0)
			break;
	}

7059
	kfree(wc);
Y
Yan Zheng 已提交
7060 7061 7062 7063
	btrfs_free_path(path);
	return ret;
}

7064 7065 7066
static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
{
	u64 num_devices;
7067
	u64 stripped;
7068

7069 7070 7071 7072 7073 7074 7075
	/*
	 * if restripe for this chunk_type is on pick target profile and
	 * return, otherwise do the usual balance
	 */
	stripped = get_restripe_target(root->fs_info, flags);
	if (stripped)
		return extended_to_chunk(stripped);
7076

7077 7078 7079 7080 7081 7082 7083 7084
	/*
	 * we add in the count of missing devices because we want
	 * to make sure that any RAID levels on a degraded FS
	 * continue to be honored.
	 */
	num_devices = root->fs_info->fs_devices->rw_devices +
		root->fs_info->fs_devices->missing_devices;

7085 7086 7087
	stripped = BTRFS_BLOCK_GROUP_RAID0 |
		BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;

7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111
	if (num_devices == 1) {
		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* turn raid0 into single device chunks */
		if (flags & BTRFS_BLOCK_GROUP_RAID0)
			return stripped;

		/* turn mirroring into duplication */
		if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
			     BTRFS_BLOCK_GROUP_RAID10))
			return stripped | BTRFS_BLOCK_GROUP_DUP;
	} else {
		/* they already had raid on here, just return */
		if (flags & stripped)
			return flags;

		stripped |= BTRFS_BLOCK_GROUP_DUP;
		stripped = flags & ~stripped;

		/* switch duplicated blocks with raid1 */
		if (flags & BTRFS_BLOCK_GROUP_DUP)
			return stripped | BTRFS_BLOCK_GROUP_RAID1;

7112
		/* this is drive concat, leave it alone */
7113
	}
7114

7115 7116 7117
	return flags;
}

7118
static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
C
Chris Mason 已提交
7119
{
7120 7121
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;
7122
	u64 min_allocable_bytes;
7123
	int ret = -ENOSPC;
C
Chris Mason 已提交
7124

7125

7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137
	/*
	 * We need some metadata space and system metadata space for
	 * allocating chunks in some corner cases until we force to set
	 * it to be readonly.
	 */
	if ((sinfo->flags &
	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
	    !force)
		min_allocable_bytes = 1 * 1024 * 1024;
	else
		min_allocable_bytes = 0;

7138 7139
	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
7140 7141 7142 7143 7144 7145

	if (cache->ro) {
		ret = 0;
		goto out;
	}

7146 7147 7148 7149
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);

	if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
7150 7151
	    sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
	    min_allocable_bytes <= sinfo->total_bytes) {
7152 7153 7154 7155
		sinfo->bytes_readonly += num_bytes;
		cache->ro = 1;
		ret = 0;
	}
7156
out:
7157 7158 7159 7160
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
	return ret;
}
7161

7162 7163
int btrfs_set_block_group_ro(struct btrfs_root *root,
			     struct btrfs_block_group_cache *cache)
7164

7165 7166 7167 7168
{
	struct btrfs_trans_handle *trans;
	u64 alloc_flags;
	int ret;
7169

7170
	BUG_ON(cache->ro);
C
Chris Mason 已提交
7171

C
Chris Mason 已提交
7172
	trans = btrfs_join_transaction(root);
7173 7174
	if (IS_ERR(trans))
		return PTR_ERR(trans);
7175

7176
	alloc_flags = update_block_group_flags(root, cache->flags);
7177 7178 7179 7180 7181 7182
	if (alloc_flags != cache->flags) {
		ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
				     CHUNK_ALLOC_FORCE);
		if (ret < 0)
			goto out;
	}
7183

7184
	ret = set_block_group_ro(cache, 0);
7185 7186 7187
	if (!ret)
		goto out;
	alloc_flags = get_alloc_profile(root, cache->space_info->flags);
7188 7189
	ret = do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			     CHUNK_ALLOC_FORCE);
7190 7191
	if (ret < 0)
		goto out;
7192
	ret = set_block_group_ro(cache, 0);
7193 7194 7195 7196
out:
	btrfs_end_transaction(trans, root);
	return ret;
}
7197

7198 7199 7200 7201
int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
			    struct btrfs_root *root, u64 type)
{
	u64 alloc_flags = get_alloc_profile(root, type);
7202 7203
	return do_chunk_alloc(trans, root, 2 * 1024 * 1024, alloc_flags,
			      CHUNK_ALLOC_FORCE);
7204 7205
}

7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261
/*
 * helper to account the unused space of all the readonly block group in the
 * list. takes mirrors into account.
 */
static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
{
	struct btrfs_block_group_cache *block_group;
	u64 free_bytes = 0;
	int factor;

	list_for_each_entry(block_group, groups_list, list) {
		spin_lock(&block_group->lock);

		if (!block_group->ro) {
			spin_unlock(&block_group->lock);
			continue;
		}

		if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
					  BTRFS_BLOCK_GROUP_RAID10 |
					  BTRFS_BLOCK_GROUP_DUP))
			factor = 2;
		else
			factor = 1;

		free_bytes += (block_group->key.offset -
			       btrfs_block_group_used(&block_group->item)) *
			       factor;

		spin_unlock(&block_group->lock);
	}

	return free_bytes;
}

/*
 * helper to account the unused space of all the readonly block group in the
 * space_info. takes mirrors into account.
 */
u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
{
	int i;
	u64 free_bytes = 0;

	spin_lock(&sinfo->lock);

	for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
		if (!list_empty(&sinfo->block_groups[i]))
			free_bytes += __btrfs_get_ro_block_group_free_space(
						&sinfo->block_groups[i]);

	spin_unlock(&sinfo->lock);

	return free_bytes;
}

7262
void btrfs_set_block_group_rw(struct btrfs_root *root,
7263
			      struct btrfs_block_group_cache *cache)
7264
{
7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277
	struct btrfs_space_info *sinfo = cache->space_info;
	u64 num_bytes;

	BUG_ON(!cache->ro);

	spin_lock(&sinfo->lock);
	spin_lock(&cache->lock);
	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
		    cache->bytes_super - btrfs_block_group_used(&cache->item);
	sinfo->bytes_readonly -= num_bytes;
	cache->ro = 0;
	spin_unlock(&cache->lock);
	spin_unlock(&sinfo->lock);
7278 7279
}

7280 7281 7282 7283 7284 7285 7286
/*
 * checks to see if its even possible to relocate this block group.
 *
 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
 * ok to go ahead and try.
 */
int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
Z
Zheng Yan 已提交
7287
{
7288 7289 7290 7291
	struct btrfs_block_group_cache *block_group;
	struct btrfs_space_info *space_info;
	struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
	struct btrfs_device *device;
7292
	u64 min_free;
J
Josef Bacik 已提交
7293 7294
	u64 dev_min = 1;
	u64 dev_nr = 0;
7295
	u64 target;
7296
	int index;
7297 7298
	int full = 0;
	int ret = 0;
Z
Zheng Yan 已提交
7299

7300
	block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
Z
Zheng Yan 已提交
7301

7302 7303 7304
	/* odd, couldn't find the block group, leave it alone */
	if (!block_group)
		return -1;
Z
Zheng Yan 已提交
7305

7306 7307
	min_free = btrfs_block_group_used(&block_group->item);

7308
	/* no bytes used, we're good */
7309
	if (!min_free)
Z
Zheng Yan 已提交
7310 7311
		goto out;

7312 7313
	space_info = block_group->space_info;
	spin_lock(&space_info->lock);
7314

7315
	full = space_info->full;
7316

7317 7318
	/*
	 * if this is the last block group we have in this space, we can't
7319 7320 7321 7322
	 * relocate it unless we're able to allocate a new chunk below.
	 *
	 * Otherwise, we need to make sure we have room in the space to handle
	 * all of the extents from this block group.  If we can, we're good
7323
	 */
7324
	if ((space_info->total_bytes != block_group->key.offset) &&
7325 7326 7327
	    (space_info->bytes_used + space_info->bytes_reserved +
	     space_info->bytes_pinned + space_info->bytes_readonly +
	     min_free < space_info->total_bytes)) {
7328 7329
		spin_unlock(&space_info->lock);
		goto out;
7330
	}
7331
	spin_unlock(&space_info->lock);
7332

7333 7334 7335
	/*
	 * ok we don't have enough space, but maybe we have free space on our
	 * devices to allocate new chunks for relocation, so loop through our
7336 7337 7338
	 * alloc devices and guess if we have enough space.  if this block
	 * group is going to be restriped, run checks against the target
	 * profile instead of the current one.
7339 7340
	 */
	ret = -1;
7341

7342 7343 7344 7345 7346 7347 7348 7349
	/*
	 * index:
	 *      0: raid10
	 *      1: raid1
	 *      2: dup
	 *      3: raid0
	 *      4: single
	 */
7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363
	target = get_restripe_target(root->fs_info, block_group->flags);
	if (target) {
		index = __get_block_group_index(extended_to_chunk(target));
	} else {
		/*
		 * this is just a balance, so if we were marked as full
		 * we know there is no space for a new chunk
		 */
		if (full)
			goto out;

		index = get_block_group_index(block_group);
	}

7364 7365
	if (index == 0) {
		dev_min = 4;
J
Josef Bacik 已提交
7366 7367
		/* Divide by 2 */
		min_free >>= 1;
7368 7369 7370
	} else if (index == 1) {
		dev_min = 2;
	} else if (index == 2) {
J
Josef Bacik 已提交
7371 7372
		/* Multiply by 2 */
		min_free <<= 1;
7373 7374
	} else if (index == 3) {
		dev_min = fs_devices->rw_devices;
J
Josef Bacik 已提交
7375
		do_div(min_free, dev_min);
7376 7377
	}

7378 7379
	mutex_lock(&root->fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
7380
		u64 dev_offset;
7381

7382 7383 7384 7385 7386
		/*
		 * check to make sure we can actually find a chunk with enough
		 * space to fit our block group in.
		 */
		if (device->total_bytes > device->bytes_used + min_free) {
7387
			ret = find_free_dev_extent(device, min_free,
7388
						   &dev_offset, NULL);
7389
			if (!ret)
7390 7391 7392
				dev_nr++;

			if (dev_nr >= dev_min)
7393
				break;
7394

7395
			ret = -1;
7396
		}
7397
	}
7398
	mutex_unlock(&root->fs_info->chunk_mutex);
7399
out:
7400
	btrfs_put_block_group(block_group);
7401 7402 7403
	return ret;
}

7404 7405
static int find_first_block_group(struct btrfs_root *root,
		struct btrfs_path *path, struct btrfs_key *key)
7406
{
7407
	int ret = 0;
7408 7409 7410
	struct btrfs_key found_key;
	struct extent_buffer *leaf;
	int slot;
7411

7412 7413
	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
	if (ret < 0)
7414 7415
		goto out;

C
Chris Mason 已提交
7416
	while (1) {
7417
		slot = path->slots[0];
7418
		leaf = path->nodes[0];
7419 7420 7421 7422 7423
		if (slot >= btrfs_header_nritems(leaf)) {
			ret = btrfs_next_leaf(root, path);
			if (ret == 0)
				continue;
			if (ret < 0)
7424
				goto out;
7425
			break;
7426
		}
7427
		btrfs_item_key_to_cpu(leaf, &found_key, slot);
7428

7429
		if (found_key.objectid >= key->objectid &&
7430 7431 7432 7433
		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
			ret = 0;
			goto out;
		}
7434
		path->slots[0]++;
7435
	}
7436
out:
7437
	return ret;
7438 7439
}

7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473
void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
	u64 last = 0;

	while (1) {
		struct inode *inode;

		block_group = btrfs_lookup_first_block_group(info, last);
		while (block_group) {
			spin_lock(&block_group->lock);
			if (block_group->iref)
				break;
			spin_unlock(&block_group->lock);
			block_group = next_block_group(info->tree_root,
						       block_group);
		}
		if (!block_group) {
			if (last == 0)
				break;
			last = 0;
			continue;
		}

		inode = block_group->inode;
		block_group->iref = 0;
		block_group->inode = NULL;
		spin_unlock(&block_group->lock);
		iput(inode);
		last = block_group->key.objectid + block_group->key.offset;
		btrfs_put_block_group(block_group);
	}
}

Z
Zheng Yan 已提交
7474 7475 7476
int btrfs_free_block_groups(struct btrfs_fs_info *info)
{
	struct btrfs_block_group_cache *block_group;
7477
	struct btrfs_space_info *space_info;
7478
	struct btrfs_caching_control *caching_ctl;
Z
Zheng Yan 已提交
7479 7480
	struct rb_node *n;

7481 7482 7483 7484 7485 7486 7487 7488 7489
	down_write(&info->extent_commit_sem);
	while (!list_empty(&info->caching_block_groups)) {
		caching_ctl = list_entry(info->caching_block_groups.next,
					 struct btrfs_caching_control, list);
		list_del(&caching_ctl->list);
		put_caching_control(caching_ctl);
	}
	up_write(&info->extent_commit_sem);

Z
Zheng Yan 已提交
7490 7491 7492 7493 7494 7495
	spin_lock(&info->block_group_cache_lock);
	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
		block_group = rb_entry(n, struct btrfs_block_group_cache,
				       cache_node);
		rb_erase(&block_group->cache_node,
			 &info->block_group_cache_tree);
Y
Yan Zheng 已提交
7496 7497
		spin_unlock(&info->block_group_cache_lock);

7498
		down_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
7499
		list_del(&block_group->list);
7500
		up_write(&block_group->space_info->groups_sem);
7501

J
Josef Bacik 已提交
7502
		if (block_group->cached == BTRFS_CACHE_STARTED)
7503
			wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
7504

7505 7506 7507 7508 7509 7510 7511
		/*
		 * We haven't cached this block group, which means we could
		 * possibly have excluded extents on this block group.
		 */
		if (block_group->cached == BTRFS_CACHE_NO)
			free_excluded_extents(info->extent_root, block_group);

J
Josef Bacik 已提交
7512
		btrfs_remove_free_space_cache(block_group);
7513
		btrfs_put_block_group(block_group);
Y
Yan Zheng 已提交
7514 7515

		spin_lock(&info->block_group_cache_lock);
Z
Zheng Yan 已提交
7516 7517
	}
	spin_unlock(&info->block_group_cache_lock);
7518 7519 7520 7521 7522 7523 7524 7525 7526

	/* now that all the block groups are freed, go through and
	 * free all the space_info structs.  This is only called during
	 * the final stages of unmount, and so we know nobody is
	 * using them.  We call synchronize_rcu() once before we start,
	 * just to be on the safe side.
	 */
	synchronize_rcu();

7527 7528
	release_global_block_rsv(info);

7529 7530 7531 7532
	while(!list_empty(&info->space_info)) {
		space_info = list_entry(info->space_info.next,
					struct btrfs_space_info,
					list);
7533
		if (space_info->bytes_pinned > 0 ||
7534 7535
		    space_info->bytes_reserved > 0 ||
		    space_info->bytes_may_use > 0) {
7536 7537 7538
			WARN_ON(1);
			dump_space_info(space_info, 0, 0);
		}
7539 7540 7541
		list_del(&space_info->list);
		kfree(space_info);
	}
Z
Zheng Yan 已提交
7542 7543 7544
	return 0;
}

7545 7546 7547 7548 7549 7550 7551 7552 7553 7554
static void __link_block_group(struct btrfs_space_info *space_info,
			       struct btrfs_block_group_cache *cache)
{
	int index = get_block_group_index(cache);

	down_write(&space_info->groups_sem);
	list_add_tail(&cache->list, &space_info->block_groups[index]);
	up_write(&space_info->groups_sem);
}

C
Chris Mason 已提交
7555 7556 7557 7558 7559
int btrfs_read_block_groups(struct btrfs_root *root)
{
	struct btrfs_path *path;
	int ret;
	struct btrfs_block_group_cache *cache;
C
Chris Mason 已提交
7560
	struct btrfs_fs_info *info = root->fs_info;
7561
	struct btrfs_space_info *space_info;
C
Chris Mason 已提交
7562 7563
	struct btrfs_key key;
	struct btrfs_key found_key;
7564
	struct extent_buffer *leaf;
7565 7566
	int need_clear = 0;
	u64 cache_gen;
7567

C
Chris Mason 已提交
7568
	root = info->extent_root;
C
Chris Mason 已提交
7569
	key.objectid = 0;
7570
	key.offset = 0;
C
Chris Mason 已提交
7571 7572 7573 7574
	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;
J
Josef Bacik 已提交
7575
	path->reada = 1;
C
Chris Mason 已提交
7576

7577
	cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
7578
	if (btrfs_test_opt(root, SPACE_CACHE) &&
7579
	    btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
7580
		need_clear = 1;
7581 7582
	if (btrfs_test_opt(root, CLEAR_CACHE))
		need_clear = 1;
7583

C
Chris Mason 已提交
7584
	while (1) {
7585
		ret = find_first_block_group(root, path, &key);
7586 7587
		if (ret > 0)
			break;
7588 7589
		if (ret != 0)
			goto error;
7590 7591
		leaf = path->nodes[0];
		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7592
		cache = kzalloc(sizeof(*cache), GFP_NOFS);
C
Chris Mason 已提交
7593
		if (!cache) {
7594
			ret = -ENOMEM;
7595
			goto error;
C
Chris Mason 已提交
7596
		}
7597 7598 7599 7600 7601 7602 7603
		cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
						GFP_NOFS);
		if (!cache->free_space_ctl) {
			kfree(cache);
			ret = -ENOMEM;
			goto error;
		}
C
Chris Mason 已提交
7604

7605
		atomic_set(&cache->count, 1);
7606
		spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7607
		cache->fs_info = info;
J
Josef Bacik 已提交
7608
		INIT_LIST_HEAD(&cache->list);
7609
		INIT_LIST_HEAD(&cache->cluster_list);
7610

7611 7612 7613
		if (need_clear)
			cache->disk_cache_state = BTRFS_DC_CLEAR;

7614 7615 7616
		read_extent_buffer(leaf, &cache->item,
				   btrfs_item_ptr_offset(leaf, path->slots[0]),
				   sizeof(cache->item));
C
Chris Mason 已提交
7617
		memcpy(&cache->key, &found_key, sizeof(found_key));
7618

C
Chris Mason 已提交
7619
		key.objectid = found_key.objectid + found_key.offset;
7620
		btrfs_release_path(path);
7621
		cache->flags = btrfs_block_group_flags(&cache->item);
J
Josef Bacik 已提交
7622 7623
		cache->sectorsize = root->sectorsize;

7624 7625
		btrfs_init_free_space_ctl(cache);

7626 7627 7628 7629 7630 7631 7632
		/*
		 * We need to exclude the super stripes now so that the space
		 * info has super bytes accounted for, otherwise we'll think
		 * we have more space than we actually do.
		 */
		exclude_super_stripes(root, cache);

J
Josef Bacik 已提交
7633 7634 7635 7636 7637 7638 7639 7640
		/*
		 * check for two cases, either we are full, and therefore
		 * don't need to bother with the caching work since we won't
		 * find any space, or we are empty, and we can just add all
		 * the space in and be done with it.  This saves us _alot_ of
		 * time, particularly in the full case.
		 */
		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
7641
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7642
			cache->cached = BTRFS_CACHE_FINISHED;
7643
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7644
		} else if (btrfs_block_group_used(&cache->item) == 0) {
7645
			cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7646 7647 7648 7649 7650
			cache->cached = BTRFS_CACHE_FINISHED;
			add_new_free_space(cache, root->fs_info,
					   found_key.objectid,
					   found_key.objectid +
					   found_key.offset);
7651
			free_excluded_extents(root, cache);
J
Josef Bacik 已提交
7652
		}
7653

7654 7655 7656
		ret = update_space_info(info, cache->flags, found_key.offset,
					btrfs_block_group_used(&cache->item),
					&space_info);
7657
		BUG_ON(ret); /* -ENOMEM */
7658
		cache->space_info = space_info;
7659
		spin_lock(&cache->space_info->lock);
7660
		cache->space_info->bytes_readonly += cache->bytes_super;
7661 7662
		spin_unlock(&cache->space_info->lock);

7663
		__link_block_group(space_info, cache);
J
Josef Bacik 已提交
7664 7665

		ret = btrfs_add_block_group_cache(root->fs_info, cache);
7666
		BUG_ON(ret); /* Logic error */
7667 7668

		set_avail_alloc_bits(root->fs_info, cache->flags);
Y
Yan Zheng 已提交
7669
		if (btrfs_chunk_readonly(root, cache->key.objectid))
7670
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7671
	}
7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683

	list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
		if (!(get_alloc_profile(root, space_info->flags) &
		      (BTRFS_BLOCK_GROUP_RAID10 |
		       BTRFS_BLOCK_GROUP_RAID1 |
		       BTRFS_BLOCK_GROUP_DUP)))
			continue;
		/*
		 * avoid allocating from un-mirrored block group if there are
		 * mirrored block groups.
		 */
		list_for_each_entry(cache, &space_info->block_groups[3], list)
7684
			set_block_group_ro(cache, 1);
7685
		list_for_each_entry(cache, &space_info->block_groups[4], list)
7686
			set_block_group_ro(cache, 1);
C
Chris Mason 已提交
7687
	}
7688 7689

	init_global_block_rsv(info);
7690 7691
	ret = 0;
error:
C
Chris Mason 已提交
7692
	btrfs_free_path(path);
7693
	return ret;
C
Chris Mason 已提交
7694
}
7695 7696 7697

int btrfs_make_block_group(struct btrfs_trans_handle *trans,
			   struct btrfs_root *root, u64 bytes_used,
7698
			   u64 type, u64 chunk_objectid, u64 chunk_offset,
7699 7700 7701 7702 7703 7704 7705 7706
			   u64 size)
{
	int ret;
	struct btrfs_root *extent_root;
	struct btrfs_block_group_cache *cache;

	extent_root = root->fs_info->extent_root;

7707
	root->fs_info->last_trans_log_full_commit = trans->transid;
7708

7709
	cache = kzalloc(sizeof(*cache), GFP_NOFS);
J
Josef Bacik 已提交
7710 7711
	if (!cache)
		return -ENOMEM;
7712 7713 7714 7715 7716 7717
	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
					GFP_NOFS);
	if (!cache->free_space_ctl) {
		kfree(cache);
		return -ENOMEM;
	}
J
Josef Bacik 已提交
7718

7719
	cache->key.objectid = chunk_offset;
7720
	cache->key.offset = size;
7721
	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
7722
	cache->sectorsize = root->sectorsize;
7723
	cache->fs_info = root->fs_info;
7724

7725
	atomic_set(&cache->count, 1);
7726
	spin_lock_init(&cache->lock);
J
Josef Bacik 已提交
7727
	INIT_LIST_HEAD(&cache->list);
7728
	INIT_LIST_HEAD(&cache->cluster_list);
C
Chris Mason 已提交
7729

7730 7731
	btrfs_init_free_space_ctl(cache);

7732 7733 7734 7735 7736
	btrfs_set_block_group_used(&cache->item, bytes_used);
	btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
	cache->flags = type;
	btrfs_set_block_group_flags(&cache->item, type);

7737
	cache->last_byte_to_unpin = (u64)-1;
J
Josef Bacik 已提交
7738
	cache->cached = BTRFS_CACHE_FINISHED;
7739
	exclude_super_stripes(root, cache);
7740

J
Josef Bacik 已提交
7741 7742 7743
	add_new_free_space(cache, root->fs_info, chunk_offset,
			   chunk_offset + size);

7744 7745
	free_excluded_extents(root, cache);

7746 7747
	ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
				&cache->space_info);
7748
	BUG_ON(ret); /* -ENOMEM */
7749
	update_global_block_rsv(root->fs_info);
7750 7751

	spin_lock(&cache->space_info->lock);
7752
	cache->space_info->bytes_readonly += cache->bytes_super;
7753 7754
	spin_unlock(&cache->space_info->lock);

7755
	__link_block_group(cache->space_info, cache);
7756

J
Josef Bacik 已提交
7757
	ret = btrfs_add_block_group_cache(root->fs_info, cache);
7758
	BUG_ON(ret); /* Logic error */
7759

7760 7761
	ret = btrfs_insert_item(trans, extent_root, &cache->key, &cache->item,
				sizeof(cache->item));
7762 7763 7764 7765
	if (ret) {
		btrfs_abort_transaction(trans, extent_root, ret);
		return ret;
	}
7766

C
Chris Mason 已提交
7767
	set_avail_alloc_bits(extent_root->fs_info, type);
7768

7769 7770
	return 0;
}
Z
Zheng Yan 已提交
7771

7772 7773
static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
{
7774 7775
	u64 extra_flags = chunk_to_extended(flags) &
				BTRFS_EXTENDED_PROFILE_MASK;
7776 7777 7778 7779 7780 7781 7782 7783 7784

	if (flags & BTRFS_BLOCK_GROUP_DATA)
		fs_info->avail_data_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_METADATA)
		fs_info->avail_metadata_alloc_bits &= ~extra_flags;
	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
		fs_info->avail_system_alloc_bits &= ~extra_flags;
}

Z
Zheng Yan 已提交
7785 7786 7787 7788 7789
int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
			     struct btrfs_root *root, u64 group_start)
{
	struct btrfs_path *path;
	struct btrfs_block_group_cache *block_group;
7790
	struct btrfs_free_cluster *cluster;
7791
	struct btrfs_root *tree_root = root->fs_info->tree_root;
Z
Zheng Yan 已提交
7792
	struct btrfs_key key;
7793
	struct inode *inode;
Z
Zheng Yan 已提交
7794
	int ret;
7795
	int index;
J
Josef Bacik 已提交
7796
	int factor;
Z
Zheng Yan 已提交
7797 7798 7799 7800 7801

	root = root->fs_info->extent_root;

	block_group = btrfs_lookup_block_group(root->fs_info, group_start);
	BUG_ON(!block_group);
Y
Yan Zheng 已提交
7802
	BUG_ON(!block_group->ro);
Z
Zheng Yan 已提交
7803

7804 7805 7806 7807 7808 7809
	/*
	 * Free the reserved super bytes from this block group before
	 * remove it.
	 */
	free_excluded_extents(root, block_group);

Z
Zheng Yan 已提交
7810
	memcpy(&key, &block_group->key, sizeof(key));
7811
	index = get_block_group_index(block_group);
J
Josef Bacik 已提交
7812 7813 7814 7815 7816 7817
	if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
				  BTRFS_BLOCK_GROUP_RAID1 |
				  BTRFS_BLOCK_GROUP_RAID10))
		factor = 2;
	else
		factor = 1;
Z
Zheng Yan 已提交
7818

7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833
	/* make sure this block group isn't part of an allocation cluster */
	cluster = &root->fs_info->data_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

	/*
	 * make sure this block group isn't part of a metadata
	 * allocation cluster
	 */
	cluster = &root->fs_info->meta_alloc_cluster;
	spin_lock(&cluster->refill_lock);
	btrfs_return_cluster_to_free_space(block_group, cluster);
	spin_unlock(&cluster->refill_lock);

Z
Zheng Yan 已提交
7834
	path = btrfs_alloc_path();
7835 7836 7837 7838
	if (!path) {
		ret = -ENOMEM;
		goto out;
	}
Z
Zheng Yan 已提交
7839

7840
	inode = lookup_free_space_inode(tree_root, block_group, path);
7841
	if (!IS_ERR(inode)) {
7842
		ret = btrfs_orphan_add(trans, inode);
7843 7844 7845 7846
		if (ret) {
			btrfs_add_delayed_iput(inode);
			goto out;
		}
7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858
		clear_nlink(inode);
		/* One for the block groups ref */
		spin_lock(&block_group->lock);
		if (block_group->iref) {
			block_group->iref = 0;
			block_group->inode = NULL;
			spin_unlock(&block_group->lock);
			iput(inode);
		} else {
			spin_unlock(&block_group->lock);
		}
		/* One for our lookup ref */
7859
		btrfs_add_delayed_iput(inode);
7860 7861 7862 7863 7864 7865 7866 7867 7868 7869
	}

	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
	key.offset = block_group->key.objectid;
	key.type = 0;

	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
	if (ret < 0)
		goto out;
	if (ret > 0)
7870
		btrfs_release_path(path);
7871 7872 7873 7874
	if (ret == 0) {
		ret = btrfs_del_item(trans, tree_root, path);
		if (ret)
			goto out;
7875
		btrfs_release_path(path);
7876 7877
	}

7878
	spin_lock(&root->fs_info->block_group_cache_lock);
Z
Zheng Yan 已提交
7879 7880
	rb_erase(&block_group->cache_node,
		 &root->fs_info->block_group_cache_tree);
7881
	spin_unlock(&root->fs_info->block_group_cache_lock);
J
Josef Bacik 已提交
7882

7883
	down_write(&block_group->space_info->groups_sem);
7884 7885 7886 7887 7888
	/*
	 * we must use list_del_init so people can check to see if they
	 * are still on the list after taking the semaphore
	 */
	list_del_init(&block_group->list);
7889 7890
	if (list_empty(&block_group->space_info->block_groups[index]))
		clear_avail_alloc_bits(root->fs_info, block_group->flags);
7891
	up_write(&block_group->space_info->groups_sem);
Z
Zheng Yan 已提交
7892

J
Josef Bacik 已提交
7893
	if (block_group->cached == BTRFS_CACHE_STARTED)
7894
		wait_block_group_cache_done(block_group);
J
Josef Bacik 已提交
7895 7896 7897

	btrfs_remove_free_space_cache(block_group);

Y
Yan Zheng 已提交
7898 7899 7900
	spin_lock(&block_group->space_info->lock);
	block_group->space_info->total_bytes -= block_group->key.offset;
	block_group->space_info->bytes_readonly -= block_group->key.offset;
J
Josef Bacik 已提交
7901
	block_group->space_info->disk_total -= block_group->key.offset * factor;
Y
Yan Zheng 已提交
7902
	spin_unlock(&block_group->space_info->lock);
7903

7904 7905
	memcpy(&key, &block_group->key, sizeof(key));

7906
	btrfs_clear_space_info_full(root->fs_info);
Y
Yan Zheng 已提交
7907

7908 7909
	btrfs_put_block_group(block_group);
	btrfs_put_block_group(block_group);
Z
Zheng Yan 已提交
7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921

	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
	if (ret > 0)
		ret = -EIO;
	if (ret < 0)
		goto out;

	ret = btrfs_del_item(trans, root, path);
out:
	btrfs_free_path(path);
	return ret;
}
L
liubo 已提交
7922

7923 7924 7925
int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
{
	struct btrfs_space_info *space_info;
7926 7927 7928 7929
	struct btrfs_super_block *disk_super;
	u64 features;
	u64 flags;
	int mixed = 0;
7930 7931
	int ret;

7932
	disk_super = fs_info->super_copy;
7933 7934
	if (!btrfs_super_root(disk_super))
		return 1;
7935

7936 7937 7938
	features = btrfs_super_incompat_flags(disk_super);
	if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
		mixed = 1;
7939

7940 7941
	flags = BTRFS_BLOCK_GROUP_SYSTEM;
	ret = update_space_info(fs_info, flags, 0, 0, &space_info);
7942
	if (ret)
7943
		goto out;
7944

7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957
	if (mixed) {
		flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	} else {
		flags = BTRFS_BLOCK_GROUP_METADATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
		if (ret)
			goto out;

		flags = BTRFS_BLOCK_GROUP_DATA;
		ret = update_space_info(fs_info, flags, 0, 0, &space_info);
	}
out:
7958 7959 7960
	return ret;
}

L
liubo 已提交
7961 7962 7963 7964 7965 7966
int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
{
	return unpin_extent_range(root, start, end);
}

int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
7967
			       u64 num_bytes, u64 *actual_bytes)
L
liubo 已提交
7968
{
7969
	return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
L
liubo 已提交
7970
}
7971 7972 7973 7974 7975 7976 7977 7978 7979

int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
{
	struct btrfs_fs_info *fs_info = root->fs_info;
	struct btrfs_block_group_cache *cache = NULL;
	u64 group_trimmed;
	u64 start;
	u64 end;
	u64 trimmed = 0;
7980
	u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
7981 7982
	int ret = 0;

7983 7984 7985 7986 7987 7988 7989
	/*
	 * try to trim all FS space, our block group may start from non-zero.
	 */
	if (range->len == total_bytes)
		cache = btrfs_lookup_first_block_group(fs_info, range->start);
	else
		cache = btrfs_lookup_block_group(fs_info, range->start);
7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025

	while (cache) {
		if (cache->key.objectid >= (range->start + range->len)) {
			btrfs_put_block_group(cache);
			break;
		}

		start = max(range->start, cache->key.objectid);
		end = min(range->start + range->len,
				cache->key.objectid + cache->key.offset);

		if (end - start >= range->minlen) {
			if (!block_group_cache_done(cache)) {
				ret = cache_block_group(cache, NULL, root, 0);
				if (!ret)
					wait_block_group_cache_done(cache);
			}
			ret = btrfs_trim_block_group(cache,
						     &group_trimmed,
						     start,
						     end,
						     range->minlen);

			trimmed += group_trimmed;
			if (ret) {
				btrfs_put_block_group(cache);
				break;
			}
		}

		cache = next_block_group(fs_info->tree_root, cache);
	}

	range->len = trimmed;
	return ret;
}