diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index c4792062dd53cb826d1267f12c6d90274b80a166..7114faafa9d48753887d81551b3c8d3761ace501 100644
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -181,7 +181,8 @@ int __btrfs_cow_block(struct btrfs_trans_handle *trans,
struct extent_buffer *buf,
struct extent_buffer *parent, int parent_slot,
struct extent_buffer **cow_ret,
- u64 search_start, u64 empty_size)
+ u64 search_start, u64 empty_size,
+ u64 prealloc_dest)
{
u64 root_gen;
struct extent_buffer *cow;
@@ -216,10 +217,27 @@ int __btrfs_cow_block(struct btrfs_trans_handle *trans,
} else {
first_key.objectid = 0;
}
- cow = btrfs_alloc_free_block(trans, root, buf->len,
- root->root_key.objectid,
- root_gen, first_key.objectid, level,
- search_start, empty_size);
+ if (prealloc_dest) {
+ struct btrfs_key ins;
+
+ ins.objectid = prealloc_dest;
+ ins.offset = buf->len;
+ ins.type = BTRFS_EXTENT_ITEM_KEY;
+
+ ret = btrfs_alloc_reserved_extent(trans, root,
+ root->root_key.objectid,
+ root_gen, level,
+ first_key.objectid,
+ &ins);
+ BUG_ON(ret);
+ cow = btrfs_init_new_buffer(trans, root, prealloc_dest,
+ buf->len);
+ } else {
+ cow = btrfs_alloc_free_block(trans, root, buf->len,
+ root->root_key.objectid,
+ root_gen, first_key.objectid,
+ level, search_start, empty_size);
+ }
if (IS_ERR(cow))
return PTR_ERR(cow);
@@ -279,7 +297,7 @@ int __btrfs_cow_block(struct btrfs_trans_handle *trans,
int btrfs_cow_block(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct extent_buffer *buf,
struct extent_buffer *parent, int parent_slot,
- struct extent_buffer **cow_ret)
+ struct extent_buffer **cow_ret, u64 prealloc_dest)
{
u64 search_start;
u64 header_trans;
@@ -302,12 +320,14 @@ int btrfs_cow_block(struct btrfs_trans_handle *trans,
!btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
*cow_ret = buf;
spin_unlock(&root->fs_info->hash_lock);
+ WARN_ON(prealloc_dest);
return 0;
}
spin_unlock(&root->fs_info->hash_lock);
search_start = buf->start & ~((u64)(1024 * 1024 * 1024) - 1);
ret = __btrfs_cow_block(trans, root, buf, parent,
- parent_slot, cow_ret, search_start, 0);
+ parent_slot, cow_ret, search_start, 0,
+ prealloc_dest);
return ret;
}
@@ -451,7 +471,7 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans,
err = __btrfs_cow_block(trans, root, cur, parent, i,
&cur, search_start,
min(16 * blocksize,
- (end_slot - i) * blocksize));
+ (end_slot - i) * blocksize), 0);
if (err) {
btrfs_tree_unlock(cur);
free_extent_buffer(cur);
@@ -803,7 +823,7 @@ static int balance_level(struct btrfs_trans_handle *trans,
child = read_node_slot(root, mid, 0);
btrfs_tree_lock(child);
BUG_ON(!child);
- ret = btrfs_cow_block(trans, root, child, mid, 0, &child);
+ ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 0);
BUG_ON(ret);
spin_lock(&root->node_lock);
@@ -836,7 +856,7 @@ static int balance_level(struct btrfs_trans_handle *trans,
if (left) {
btrfs_tree_lock(left);
wret = btrfs_cow_block(trans, root, left,
- parent, pslot - 1, &left);
+ parent, pslot - 1, &left, 0);
if (wret) {
ret = wret;
goto enospc;
@@ -846,7 +866,7 @@ static int balance_level(struct btrfs_trans_handle *trans,
if (right) {
btrfs_tree_lock(right);
wret = btrfs_cow_block(trans, root, right,
- parent, pslot + 1, &right);
+ parent, pslot + 1, &right, 0);
if (wret) {
ret = wret;
goto enospc;
@@ -1021,7 +1041,7 @@ static int noinline push_nodes_for_insert(struct btrfs_trans_handle *trans,
wret = 1;
} else {
ret = btrfs_cow_block(trans, root, left, parent,
- pslot - 1, &left);
+ pslot - 1, &left, 0);
if (ret)
wret = 1;
else {
@@ -1069,7 +1089,7 @@ static int noinline push_nodes_for_insert(struct btrfs_trans_handle *trans,
} else {
ret = btrfs_cow_block(trans, root, right,
parent, pslot + 1,
- &right);
+ &right, 0);
if (ret)
wret = 1;
else {
@@ -1245,6 +1265,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
u8 lowest_level = 0;
u64 blocknr;
u64 gen;
+ struct btrfs_key prealloc_block;
lowest_level = p->lowest_level;
WARN_ON(lowest_level && ins_len);
@@ -1253,6 +1274,9 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
!mutex_is_locked(&root->fs_info->alloc_mutex));
if (ins_len < 0)
lowest_unlock = 2;
+
+ prealloc_block.objectid = 0;
+
again:
if (p->skip_locking)
b = btrfs_root_node(root);
@@ -1261,27 +1285,82 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
while (b) {
level = btrfs_header_level(b);
+
+ /*
+ * setup the path here so we can release it under lock
+ * contention with the cow code
+ */
+ p->nodes[level] = b;
+ if (!p->skip_locking)
+ p->locks[level] = 1;
+
if (cow) {
int wret;
+
+ /* is a cow on this block not required */
+ spin_lock(&root->fs_info->hash_lock);
+ if (btrfs_header_generation(b) == trans->transid &&
+ !btrfs_header_flag(b, BTRFS_HEADER_FLAG_WRITTEN)) {
+ spin_unlock(&root->fs_info->hash_lock);
+ goto cow_done;
+ }
+ spin_unlock(&root->fs_info->hash_lock);
+
+ /* ok, we have to cow, is our old prealloc the right
+ * size?
+ */
+ if (prealloc_block.objectid &&
+ prealloc_block.offset != b->len) {
+ btrfs_free_reserved_extent(root,
+ prealloc_block.objectid,
+ prealloc_block.offset);
+ prealloc_block.objectid = 0;
+ }
+
+ /*
+ * for higher level blocks, try not to allocate blocks
+ * with the block and the parent locks held.
+ */
+ if (level > 1 && !prealloc_block.objectid &&
+ btrfs_path_lock_waiting(p, level)) {
+ u32 size = b->len;
+ u64 hint = b->start;
+
+ btrfs_release_path(root, p);
+ ret = btrfs_reserve_extent(trans, root,
+ size, size, 0,
+ hint, (u64)-1,
+ &prealloc_block, 0);
+ BUG_ON(ret);
+ goto again;
+ }
+
wret = btrfs_cow_block(trans, root, b,
p->nodes[level + 1],
p->slots[level + 1],
- &b);
+ &b, prealloc_block.objectid);
+ prealloc_block.objectid = 0;
if (wret) {
free_extent_buffer(b);
- return wret;
+ ret = wret;
+ goto done;
}
}
+cow_done:
BUG_ON(!cow && ins_len);
if (level != btrfs_header_level(b))
WARN_ON(1);
level = btrfs_header_level(b);
+
p->nodes[level] = b;
if (!p->skip_locking)
p->locks[level] = 1;
+
ret = check_block(root, p, level);
- if (ret)
- return -1;
+ if (ret) {
+ ret = -1;
+ goto done;
+ }
ret = bin_search(b, key, level, &slot);
if (level != 0) {
@@ -1292,15 +1371,19 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
BTRFS_NODEPTRS_PER_BLOCK(root) - 3) {
int sret = split_node(trans, root, p, level);
BUG_ON(sret > 0);
- if (sret)
- return sret;
+ if (sret) {
+ ret = sret;
+ goto done;
+ }
b = p->nodes[level];
slot = p->slots[level];
} else if (ins_len < 0) {
int sret = balance_level(trans, root, p,
level);
- if (sret)
- return sret;
+ if (sret) {
+ ret = sret;
+ goto done;
+ }
b = p->nodes[level];
if (!b) {
btrfs_release_path(NULL, p);
@@ -1362,14 +1445,24 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
int sret = split_leaf(trans, root, key,
p, ins_len, ret == 0);
BUG_ON(sret > 0);
- if (sret)
- return sret;
+ if (sret) {
+ ret = sret;
+ goto done;
+ }
}
unlock_up(p, level, lowest_unlock);
- return ret;
+ goto done;
}
}
- return 1;
+ ret = 1;
+done:
+ if (prealloc_block.objectid) {
+ btrfs_free_reserved_extent(root,
+ prealloc_block.objectid,
+ prealloc_block.offset);
+ }
+
+ return ret;
}
/*
@@ -1840,7 +1933,7 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
/* cow and double check */
ret = btrfs_cow_block(trans, root, right, upper,
- slot + 1, &right);
+ slot + 1, &right, 0);
if (ret)
goto out_unlock;
@@ -2021,7 +2114,7 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
/* cow and double check */
ret = btrfs_cow_block(trans, root, left,
- path->nodes[1], slot - 1, &left);
+ path->nodes[1], slot - 1, &left, 0);
if (ret) {
/* we hit -ENOSPC, but it isn't fatal here */
ret = 1;
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index d788ab0dcd96b72eae4f0c399dca03ef79385992..9b025960bbde106b950ce1e10f8cb55141373d1b 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -1421,6 +1421,9 @@ struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
int level,
u64 hint,
u64 empty_size);
+struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
+ struct btrfs_root *root,
+ u64 bytenr, u32 blocksize);
int btrfs_shrink_extent_tree(struct btrfs_root *root, u64 new_size);
int btrfs_insert_extent_backref(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
@@ -1451,6 +1454,7 @@ int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root
*root, u64 bytenr, u64 num_bytes,
u64 root_objectid, u64 ref_generation,
u64 owner_objectid, u64 owner_offset, int pin);
+int btrfs_free_reserved_extent(struct btrfs_root *root, u64 start, u64 len);
int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct extent_io_tree *unpin);
@@ -1484,7 +1488,7 @@ int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
int btrfs_cow_block(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct extent_buffer *buf,
struct extent_buffer *parent, int parent_slot,
- struct extent_buffer **cow_ret);
+ struct extent_buffer **cow_ret, u64 prealloc_dest);
int btrfs_copy_root(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct extent_buffer *buf,
diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c
index 74bcd48a9c43a6d0b25da1ee3915d1c6aade3dfa..98a1c0faedae1c7c5e28cca62a7fcaab30a73878 100644
--- a/fs/btrfs/extent-tree.c
+++ b/fs/btrfs/extent-tree.c
@@ -2118,6 +2118,15 @@ static int __btrfs_reserve_extent(struct btrfs_trans_handle *trans,
return 0;
}
+int btrfs_free_reserved_extent(struct btrfs_root *root, u64 start, u64 len)
+{
+ maybe_lock_mutex(root);
+ set_extent_dirty(&root->fs_info->free_space_cache,
+ start, start + len - 1, GFP_NOFS);
+ maybe_unlock_mutex(root);
+ return 0;
+}
+
int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
u64 num_bytes, u64 min_alloc_size,
@@ -2267,6 +2276,26 @@ int btrfs_alloc_extent(struct btrfs_trans_handle *trans,
maybe_unlock_mutex(root);
return ret;
}
+
+struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
+ struct btrfs_root *root,
+ u64 bytenr, u32 blocksize)
+{
+ struct extent_buffer *buf;
+
+ buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
+ if (!buf)
+ return ERR_PTR(-ENOMEM);
+ btrfs_set_header_generation(buf, trans->transid);
+ btrfs_tree_lock(buf);
+ clean_tree_block(trans, root, buf);
+ btrfs_set_buffer_uptodate(buf);
+ set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
+ buf->start + buf->len - 1, GFP_NOFS);
+ trans->blocks_used++;
+ return buf;
+}
+
/*
* helper function to allocate a block for a given tree
* returns the tree buffer or NULL.
@@ -2293,26 +2322,8 @@ struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
BUG_ON(ret > 0);
return ERR_PTR(ret);
}
- buf = btrfs_find_create_tree_block(root, ins.objectid, blocksize);
- if (!buf) {
- btrfs_free_extent(trans, root, ins.objectid, blocksize,
- root->root_key.objectid, ref_generation,
- 0, 0, 0);
- return ERR_PTR(-ENOMEM);
- }
- btrfs_set_header_generation(buf, trans->transid);
- btrfs_tree_lock(buf);
- clean_tree_block(trans, root, buf);
- btrfs_set_buffer_uptodate(buf);
-
- if (PageDirty(buf->first_page)) {
- printk("page %lu dirty\n", buf->first_page->index);
- WARN_ON(1);
- }
- set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
- buf->start + buf->len - 1, GFP_NOFS);
- trans->blocks_used++;
+ buf = btrfs_init_new_buffer(trans, root, ins.objectid, blocksize);
return buf;
}
diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c
index d43e14c7471add6ba29b203972b19cdafd0e24de..0cc314c10d66da148ee02d30ff1697b75f5c41ba 100644
--- a/fs/btrfs/locking.c
+++ b/fs/btrfs/locking.c
@@ -56,3 +56,19 @@ int btrfs_tree_locked(struct extent_buffer *eb)
{
return mutex_is_locked(&eb->mutex);
}
+
+int btrfs_path_lock_waiting(struct btrfs_path *path, int level)
+{
+ int i;
+ struct extent_buffer *eb;
+ for (i = level; i <= level + 1 && i < BTRFS_MAX_LEVEL; i++) {
+ eb = path->nodes[i];
+ if (!eb)
+ break;
+ smp_mb();
+ if (!list_empty(&eb->mutex.wait_list))
+ return 1;
+ }
+ return 0;
+}
+
diff --git a/fs/btrfs/locking.h b/fs/btrfs/locking.h
index 2dab96d8280e0686811742d7e3336183d06041e7..bc1faef12519451294e361becacbd9f404ac0afd 100644
--- a/fs/btrfs/locking.h
+++ b/fs/btrfs/locking.h
@@ -23,4 +23,5 @@ int btrfs_tree_lock(struct extent_buffer *eb);
int btrfs_tree_unlock(struct extent_buffer *eb);
int btrfs_tree_locked(struct extent_buffer *eb);
int btrfs_try_tree_lock(struct extent_buffer *eb);
+int btrfs_path_lock_waiting(struct btrfs_path *path, int level);
#endif
diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c
index a68779499302d2cd82ac1ff65d61bfa4462c34d4..9d84daf100086d113b6db4cd90a7c2b0022db42c 100644
--- a/fs/btrfs/transaction.c
+++ b/fs/btrfs/transaction.c
@@ -622,7 +622,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
btrfs_set_key_type(&key, BTRFS_ROOT_ITEM_KEY);
old = btrfs_lock_root_node(root);
- btrfs_cow_block(trans, root, old, NULL, 0, &old);
+ btrfs_cow_block(trans, root, old, NULL, 0, &old, 0);
btrfs_copy_root(trans, root, old, &tmp, objectid);
btrfs_tree_unlock(old);