diff --git a/fs/dcache.c b/fs/dcache.c
index 10988f7e5a23ce223a609a0cd30a3ef4ad2b1aca..ea2de7c19b088c623aed3dc7fc2c8e1e1ee16e5e 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -111,6 +111,17 @@ static inline struct hlist_bl_head *d_hash(const struct dentry *parent,
return dentry_hashtable + hash_32(hash, d_hash_shift);
}
+#define IN_LOOKUP_SHIFT 10
+static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
+
+static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
+ unsigned int hash)
+{
+ hash += (unsigned long) parent / L1_CACHE_BYTES;
+ return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
+}
+
+
/* Statistics gathering. */
struct dentry_stat_t dentry_stat = {
.age_limit = 45,
@@ -2380,9 +2391,102 @@ static inline void end_dir_add(struct inode *dir, unsigned n)
smp_store_release(&dir->i_dir_seq, n + 2);
}
+struct dentry *d_alloc_parallel(struct dentry *parent,
+ const struct qstr *name)
+{
+ unsigned int len = name->len;
+ unsigned int hash = name->hash;
+ const unsigned char *str = name->name;
+ struct hlist_bl_head *b = in_lookup_hash(parent, hash);
+ struct hlist_bl_node *node;
+ struct dentry *new = d_alloc(parent, name);
+ struct dentry *dentry;
+ unsigned seq, r_seq, d_seq;
+
+ if (unlikely(!new))
+ return ERR_PTR(-ENOMEM);
+
+retry:
+ rcu_read_lock();
+ seq = smp_load_acquire(&parent->d_inode->i_dir_seq) & ~1;
+ r_seq = read_seqbegin(&rename_lock);
+ dentry = __d_lookup_rcu(parent, name, &d_seq);
+ if (unlikely(dentry)) {
+ if (!lockref_get_not_dead(&dentry->d_lockref)) {
+ rcu_read_unlock();
+ goto retry;
+ }
+ if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
+ rcu_read_unlock();
+ dput(dentry);
+ goto retry;
+ }
+ rcu_read_unlock();
+ dput(new);
+ return dentry;
+ }
+ if (unlikely(read_seqretry(&rename_lock, r_seq))) {
+ rcu_read_unlock();
+ goto retry;
+ }
+ hlist_bl_lock(b);
+ if (unlikely(parent->d_inode->i_dir_seq != seq)) {
+ hlist_bl_unlock(b);
+ rcu_read_unlock();
+ goto retry;
+ }
+ rcu_read_unlock();
+ /*
+ * No changes for the parent since the beginning of d_lookup().
+ * Since all removals from the chain happen with hlist_bl_lock(),
+ * any potential in-lookup matches are going to stay here until
+ * we unlock the chain. All fields are stable in everything
+ * we encounter.
+ */
+ hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
+ if (dentry->d_name.hash != hash)
+ continue;
+ if (dentry->d_parent != parent)
+ continue;
+ if (d_unhashed(dentry))
+ continue;
+ if (parent->d_flags & DCACHE_OP_COMPARE) {
+ int tlen = dentry->d_name.len;
+ const char *tname = dentry->d_name.name;
+ if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
+ continue;
+ } else {
+ if (dentry->d_name.len != len)
+ continue;
+ if (dentry_cmp(dentry, str, len))
+ continue;
+ }
+ dget(dentry);
+ hlist_bl_unlock(b);
+ /* impossible until we actually enable parallel lookups */
+ BUG();
+ /* and this will be "wait for it to stop being in-lookup" */
+ /* this one will be handled in the next commit */
+ dput(new);
+ return dentry;
+ }
+ /* we can't take ->d_lock here; it's OK, though. */
+ new->d_flags |= DCACHE_PAR_LOOKUP;
+ hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b);
+ hlist_bl_unlock(b);
+ return new;
+}
+EXPORT_SYMBOL(d_alloc_parallel);
+
void __d_lookup_done(struct dentry *dentry)
{
+ struct hlist_bl_head *b = in_lookup_hash(dentry->d_parent,
+ dentry->d_name.hash);
+ hlist_bl_lock(b);
dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
+ __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
+ hlist_bl_unlock(b);
+ INIT_HLIST_NODE(&dentry->d_u.d_alias);
/* more stuff will land here */
}
EXPORT_SYMBOL(__d_lookup_done);
diff --git a/fs/namei.c b/fs/namei.c
index 26e5f84e0c366ad5fd40c9e031b481a3571d647e..aa04320e1f3756e7bf4261b0c44fd3d31cbfd8e2 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -1603,46 +1603,40 @@ static struct dentry *lookup_slow(const struct qstr *name,
struct dentry *dir,
unsigned int flags)
{
- struct dentry *dentry, *old;
+ struct dentry *dentry = ERR_PTR(-ENOENT), *old;
struct inode *inode = dir->d_inode;
inode_lock(inode);
/* Don't go there if it's already dead */
- if (unlikely(IS_DEADDIR(inode))) {
- inode_unlock(inode);
- return ERR_PTR(-ENOENT);
- }
- dentry = d_lookup(dir, name);
- if (unlikely(dentry)) {
+ if (unlikely(IS_DEADDIR(inode)))
+ goto out;
+again:
+ dentry = d_alloc_parallel(dir, name);
+ if (IS_ERR(dentry))
+ goto out;
+ if (unlikely(!d_in_lookup(dentry))) {
if ((dentry->d_flags & DCACHE_OP_REVALIDATE) &&
!(flags & LOOKUP_NO_REVAL)) {
int error = d_revalidate(dentry, flags);
if (unlikely(error <= 0)) {
- if (!error)
+ if (!error) {
d_invalidate(dentry);
+ dput(dentry);
+ goto again;
+ }
dput(dentry);
dentry = ERR_PTR(error);
}
}
- if (dentry) {
- inode_unlock(inode);
- return dentry;
+ } else {
+ old = inode->i_op->lookup(inode, dentry, flags);
+ d_lookup_done(dentry);
+ if (unlikely(old)) {
+ dput(dentry);
+ dentry = old;
}
}
- dentry = d_alloc(dir, name);
- if (unlikely(!dentry)) {
- inode_unlock(inode);
- return ERR_PTR(-ENOMEM);
- }
- spin_lock(&dentry->d_lock);
- dentry->d_flags |= DCACHE_PAR_LOOKUP;
- spin_unlock(&dentry->d_lock);
- old = inode->i_op->lookup(inode, dentry, flags);
- d_lookup_done(dentry);
- if (unlikely(old)) {
- dput(dentry);
- dentry = old;
- }
+out:
inode_unlock(inode);
return dentry;
}
diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index 9a7aa890b642b90ebe8d61252278459bd03a3acc..3eea562f5f2705ecf49db41d95741a103a48c184 100644
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -131,6 +131,7 @@ struct dentry {
*/
union {
struct hlist_node d_alias; /* inode alias list */
+ struct hlist_bl_node d_in_lookup_hash; /* only for in-lookup ones */
struct rcu_head d_rcu;
} d_u;
};
@@ -250,6 +251,7 @@ extern void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op
/* allocate/de-allocate */
extern struct dentry * d_alloc(struct dentry *, const struct qstr *);
extern struct dentry * d_alloc_pseudo(struct super_block *, const struct qstr *);
+extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *);
extern struct dentry * d_splice_alias(struct inode *, struct dentry *);
extern struct dentry * d_add_ci(struct dentry *, struct inode *, struct qstr *);
extern struct dentry * d_exact_alias(struct dentry *, struct inode *);