dcache.c 87.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
/*
 * fs/dcache.c
 *
 * Complete reimplementation
 * (C) 1997 Thomas Schoebel-Theuer,
 * with heavy changes by Linus Torvalds
 */

/*
 * Notes on the allocation strategy:
 *
 * The dcache is a master of the icache - whenever a dcache entry
 * exists, the inode will always exist. "iput()" is done either when
 * the dcache entry is deleted or garbage collected.
 */

#include <linux/syscalls.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/fs.h>
21
#include <linux/fsnotify.h>
L
Linus Torvalds 已提交
22 23 24 25
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/hash.h>
#include <linux/cache.h>
26
#include <linux/export.h>
L
Linus Torvalds 已提交
27 28 29 30 31 32 33
#include <linux/mount.h>
#include <linux/file.h>
#include <asm/uaccess.h>
#include <linux/security.h>
#include <linux/seqlock.h>
#include <linux/swap.h>
#include <linux/bootmem.h>
34
#include <linux/fs_struct.h>
35
#include <linux/hardirq.h>
36 37
#include <linux/bit_spinlock.h>
#include <linux/rculist_bl.h>
38
#include <linux/prefetch.h>
39
#include <linux/ratelimit.h>
40
#include <linux/list_lru.h>
41
#include "internal.h"
42
#include "mount.h"
L
Linus Torvalds 已提交
43

N
Nick Piggin 已提交
44 45
/*
 * Usage:
46 47
 * dcache->d_inode->i_lock protects:
 *   - i_dentry, d_alias, d_inode of aliases
48 49 50 51
 * dcache_hash_bucket lock protects:
 *   - the dcache hash table
 * s_anon bl list spinlock protects:
 *   - the s_anon list (see __d_drop)
D
Dave Chinner 已提交
52
 * dentry->d_sb->s_dentry_lru_lock protects:
N
Nick Piggin 已提交
53 54 55 56 57
 *   - the dcache lru lists and counters
 * d_lock protects:
 *   - d_flags
 *   - d_name
 *   - d_lru
N
Nick Piggin 已提交
58
 *   - d_count
N
Nick Piggin 已提交
59
 *   - d_unhashed()
N
Nick Piggin 已提交
60 61
 *   - d_parent and d_subdirs
 *   - childrens' d_child and d_parent
N
Nick Piggin 已提交
62
 *   - d_alias, d_inode
N
Nick Piggin 已提交
63 64
 *
 * Ordering:
65
 * dentry->d_inode->i_lock
N
Nick Piggin 已提交
66
 *   dentry->d_lock
D
Dave Chinner 已提交
67
 *     dentry->d_sb->s_dentry_lru_lock
68 69
 *     dcache_hash_bucket lock
 *     s_anon lock
N
Nick Piggin 已提交
70
 *
N
Nick Piggin 已提交
71 72 73 74 75 76 77
 * If there is an ancestor relationship:
 * dentry->d_parent->...->d_parent->d_lock
 *   ...
 *     dentry->d_parent->d_lock
 *       dentry->d_lock
 *
 * If no ancestor relationship:
N
Nick Piggin 已提交
78 79 80 81
 * if (dentry1 < dentry2)
 *   dentry1->d_lock
 *     dentry2->d_lock
 */
82
int sysctl_vfs_cache_pressure __read_mostly = 100;
L
Linus Torvalds 已提交
83 84
EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);

A
Al Viro 已提交
85
__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
L
Linus Torvalds 已提交
86

87
EXPORT_SYMBOL(rename_lock);
L
Linus Torvalds 已提交
88

89
static struct kmem_cache *dentry_cache __read_mostly;
L
Linus Torvalds 已提交
90 91 92 93 94 95 96 97 98 99

/*
 * This is the single most critical data structure when it comes
 * to the dcache: the hashtable for lookups. Somebody should try
 * to make this good - I've just made it work.
 *
 * This hash-function tries to avoid losing too many bits of hash
 * information, yet avoid using a prime hash-size or similar.
 */

100 101
static unsigned int d_hash_mask __read_mostly;
static unsigned int d_hash_shift __read_mostly;
102

103
static struct hlist_bl_head *dentry_hashtable __read_mostly;
104

105
static inline struct hlist_bl_head *d_hash(const struct dentry *parent,
106
					unsigned int hash)
107
{
108
	hash += (unsigned long) parent / L1_CACHE_BYTES;
A
Al Viro 已提交
109 110
	hash = hash + (hash >> d_hash_shift);
	return dentry_hashtable + (hash & d_hash_mask);
111 112
}

L
Linus Torvalds 已提交
113 114 115 116 117
/* Statistics gathering. */
struct dentry_stat_t dentry_stat = {
	.age_limit = 45,
};

118
static DEFINE_PER_CPU(long, nr_dentry);
119
static DEFINE_PER_CPU(long, nr_dentry_unused);
120 121

#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
122 123 124 125 126 127 128 129 130 131 132 133 134

/*
 * Here we resort to our own counters instead of using generic per-cpu counters
 * for consistency with what the vfs inode code does. We are expected to harvest
 * better code and performance by having our own specialized counters.
 *
 * Please note that the loop is done over all possible CPUs, not over all online
 * CPUs. The reason for this is that we don't want to play games with CPUs going
 * on and off. If one of them goes off, we will just keep their counters.
 *
 * glommer: See cffbc8a for details, and if you ever intend to change this,
 * please update all vfs counters to match.
 */
135
static long get_nr_dentry(void)
136 137
{
	int i;
138
	long sum = 0;
139 140 141 142 143
	for_each_possible_cpu(i)
		sum += per_cpu(nr_dentry, i);
	return sum < 0 ? 0 : sum;
}

144 145 146 147 148 149 150 151 152
static long get_nr_dentry_unused(void)
{
	int i;
	long sum = 0;
	for_each_possible_cpu(i)
		sum += per_cpu(nr_dentry_unused, i);
	return sum < 0 ? 0 : sum;
}

153 154 155
int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
		   size_t *lenp, loff_t *ppos)
{
156
	dentry_stat.nr_dentry = get_nr_dentry();
157
	dentry_stat.nr_unused = get_nr_dentry_unused();
158
	return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
159 160 161
}
#endif

162 163 164 165
/*
 * Compare 2 name strings, return 0 if they match, otherwise non-zero.
 * The strings are both count bytes long, and count is non-zero.
 */
166 167 168 169 170 171 172 173 174 175 176 177
#ifdef CONFIG_DCACHE_WORD_ACCESS

#include <asm/word-at-a-time.h>
/*
 * NOTE! 'cs' and 'scount' come from a dentry, so it has a
 * aligned allocation for this particular component. We don't
 * strictly need the load_unaligned_zeropad() safety, but it
 * doesn't hurt either.
 *
 * In contrast, 'ct' and 'tcount' can be from a pathname, and do
 * need the careful unaligned handling.
 */
178
static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
179
{
180 181 182
	unsigned long a,b,mask;

	for (;;) {
183
		a = *(unsigned long *)cs;
184
		b = load_unaligned_zeropad(ct);
185 186 187 188 189 190 191 192 193 194 195 196
		if (tcount < sizeof(unsigned long))
			break;
		if (unlikely(a != b))
			return 1;
		cs += sizeof(unsigned long);
		ct += sizeof(unsigned long);
		tcount -= sizeof(unsigned long);
		if (!tcount)
			return 0;
	}
	mask = ~(~0ul << tcount*8);
	return unlikely(!!((a ^ b) & mask));
197 198
}

199
#else
200

201
static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
202
{
203 204 205 206 207 208 209 210 211 212
	do {
		if (*cs != *ct)
			return 1;
		cs++;
		ct++;
		tcount--;
	} while (tcount);
	return 0;
}

213 214
#endif

215 216
static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
{
217
	const unsigned char *cs;
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
	/*
	 * Be careful about RCU walk racing with rename:
	 * use ACCESS_ONCE to fetch the name pointer.
	 *
	 * NOTE! Even if a rename will mean that the length
	 * was not loaded atomically, we don't care. The
	 * RCU walk will check the sequence count eventually,
	 * and catch it. And we won't overrun the buffer,
	 * because we're reading the name pointer atomically,
	 * and a dentry name is guaranteed to be properly
	 * terminated with a NUL byte.
	 *
	 * End result: even if 'len' is wrong, we'll exit
	 * early because the data cannot match (there can
	 * be no NUL in the ct/tcount data)
	 */
234 235 236
	cs = ACCESS_ONCE(dentry->d_name.name);
	smp_read_barrier_depends();
	return dentry_string_cmp(cs, ct, tcount);
237 238
}

C
Christoph Hellwig 已提交
239
static void __d_free(struct rcu_head *head)
L
Linus Torvalds 已提交
240
{
C
Christoph Hellwig 已提交
241 242
	struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);

A
Al Viro 已提交
243
	WARN_ON(!hlist_unhashed(&dentry->d_alias));
L
Linus Torvalds 已提交
244 245 246 247 248 249
	if (dname_external(dentry))
		kfree(dentry->d_name.name);
	kmem_cache_free(dentry_cache, dentry); 
}

/*
N
Nick Piggin 已提交
250
 * no locks, please.
L
Linus Torvalds 已提交
251 252 253
 */
static void d_free(struct dentry *dentry)
{
254
	BUG_ON((int)dentry->d_lockref.count > 0);
255
	this_cpu_dec(nr_dentry);
L
Linus Torvalds 已提交
256 257
	if (dentry->d_op && dentry->d_op->d_release)
		dentry->d_op->d_release(dentry);
258

259 260
	/* if dentry was never visible to RCU, immediate free is OK */
	if (!(dentry->d_flags & DCACHE_RCUACCESS))
C
Christoph Hellwig 已提交
261
		__d_free(&dentry->d_u.d_rcu);
262
	else
C
Christoph Hellwig 已提交
263
		call_rcu(&dentry->d_u.d_rcu, __d_free);
L
Linus Torvalds 已提交
264 265
}

N
Nick Piggin 已提交
266 267
/**
 * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups
268
 * @dentry: the target dentry
N
Nick Piggin 已提交
269 270 271 272 273 274 275 276 277 278 279
 * After this call, in-progress rcu-walk path lookup will fail. This
 * should be called after unhashing, and after changing d_inode (if
 * the dentry has not already been unhashed).
 */
static inline void dentry_rcuwalk_barrier(struct dentry *dentry)
{
	assert_spin_locked(&dentry->d_lock);
	/* Go through a barrier */
	write_seqcount_barrier(&dentry->d_seq);
}

L
Linus Torvalds 已提交
280 281
/*
 * Release the dentry's inode, using the filesystem
N
Nick Piggin 已提交
282 283
 * d_iput() operation if defined. Dentry has no refcount
 * and is unhashed.
L
Linus Torvalds 已提交
284
 */
285
static void dentry_iput(struct dentry * dentry)
286
	__releases(dentry->d_lock)
287
	__releases(dentry->d_inode->i_lock)
L
Linus Torvalds 已提交
288 289 290 291
{
	struct inode *inode = dentry->d_inode;
	if (inode) {
		dentry->d_inode = NULL;
A
Al Viro 已提交
292
		hlist_del_init(&dentry->d_alias);
L
Linus Torvalds 已提交
293
		spin_unlock(&dentry->d_lock);
294
		spin_unlock(&inode->i_lock);
295 296
		if (!inode->i_nlink)
			fsnotify_inoderemove(inode);
L
Linus Torvalds 已提交
297 298 299 300 301 302 303 304 305
		if (dentry->d_op && dentry->d_op->d_iput)
			dentry->d_op->d_iput(dentry, inode);
		else
			iput(inode);
	} else {
		spin_unlock(&dentry->d_lock);
	}
}

N
Nick Piggin 已提交
306 307 308 309 310 311
/*
 * Release the dentry's inode, using the filesystem
 * d_iput() operation if defined. dentry remains in-use.
 */
static void dentry_unlink_inode(struct dentry * dentry)
	__releases(dentry->d_lock)
312
	__releases(dentry->d_inode->i_lock)
N
Nick Piggin 已提交
313 314
{
	struct inode *inode = dentry->d_inode;
315
	__d_clear_type(dentry);
N
Nick Piggin 已提交
316
	dentry->d_inode = NULL;
A
Al Viro 已提交
317
	hlist_del_init(&dentry->d_alias);
N
Nick Piggin 已提交
318 319
	dentry_rcuwalk_barrier(dentry);
	spin_unlock(&dentry->d_lock);
320
	spin_unlock(&inode->i_lock);
N
Nick Piggin 已提交
321 322 323 324 325 326 327 328
	if (!inode->i_nlink)
		fsnotify_inoderemove(inode);
	if (dentry->d_op && dentry->d_op->d_iput)
		dentry->d_op->d_iput(dentry, inode);
	else
		iput(inode);
}

329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
/*
 * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
 * is in use - which includes both the "real" per-superblock
 * LRU list _and_ the DCACHE_SHRINK_LIST use.
 *
 * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
 * on the shrink list (ie not on the superblock LRU list).
 *
 * The per-cpu "nr_dentry_unused" counters are updated with
 * the DCACHE_LRU_LIST bit.
 *
 * These helper functions make sure we always follow the
 * rules. d_lock must be held by the caller.
 */
#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
static void d_lru_add(struct dentry *dentry)
{
	D_FLAG_VERIFY(dentry, 0);
	dentry->d_flags |= DCACHE_LRU_LIST;
	this_cpu_inc(nr_dentry_unused);
	WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
}

static void d_lru_del(struct dentry *dentry)
{
	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
	dentry->d_flags &= ~DCACHE_LRU_LIST;
	this_cpu_dec(nr_dentry_unused);
	WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
}

static void d_shrink_del(struct dentry *dentry)
{
	D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
	list_del_init(&dentry->d_lru);
	dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
	this_cpu_dec(nr_dentry_unused);
}

static void d_shrink_add(struct dentry *dentry, struct list_head *list)
{
	D_FLAG_VERIFY(dentry, 0);
	list_add(&dentry->d_lru, list);
	dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
	this_cpu_inc(nr_dentry_unused);
}

/*
 * These can only be called under the global LRU lock, ie during the
 * callback for freeing the LRU list. "isolate" removes it from the
 * LRU lists entirely, while shrink_move moves it to the indicated
 * private list.
 */
static void d_lru_isolate(struct dentry *dentry)
{
	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
	dentry->d_flags &= ~DCACHE_LRU_LIST;
	this_cpu_dec(nr_dentry_unused);
	list_del_init(&dentry->d_lru);
}

static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list)
{
	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
	dentry->d_flags |= DCACHE_SHRINK_LIST;
	list_move_tail(&dentry->d_lru, list);
}

397
/*
398
 * dentry_lru_(add|del)_list) must be called with d_lock held.
399 400 401
 */
static void dentry_lru_add(struct dentry *dentry)
{
402 403
	if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
		d_lru_add(dentry);
404 405
}

S
Sage Weil 已提交
406 407
/*
 * Remove a dentry with references from the LRU.
408 409 410 411
 *
 * If we are on the shrink list, then we can get to try_prune_one_dentry() and
 * lose our last reference through the parent walk. In this case, we need to
 * remove ourselves from the shrink list, not the LRU.
S
Sage Weil 已提交
412
 */
413 414
static void dentry_lru_del(struct dentry *dentry)
{
415 416 417 418
	if (dentry->d_flags & DCACHE_LRU_LIST) {
		if (dentry->d_flags & DCACHE_SHRINK_LIST)
			return d_shrink_del(dentry);
		d_lru_del(dentry);
419 420 421
	}
}

422 423 424
/**
 * d_kill - kill dentry and return parent
 * @dentry: dentry to kill
425
 * @parent: parent dentry
426
 *
427
 * The dentry must already be unhashed and removed from the LRU.
428 429
 *
 * If this is the root of the dentry tree, return NULL.
N
Nick Piggin 已提交
430
 *
N
Nick Piggin 已提交
431 432
 * dentry->d_lock and parent->d_lock must be held by caller, and are dropped by
 * d_kill.
433
 */
N
Nick Piggin 已提交
434
static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
435
	__releases(dentry->d_lock)
N
Nick Piggin 已提交
436
	__releases(parent->d_lock)
437
	__releases(dentry->d_inode->i_lock)
438 439
{
	list_del(&dentry->d_u.d_child);
440
	/*
441
	 * Inform d_walk() that we are no longer attached to the
442 443
	 * dentry tree
	 */
444
	dentry->d_flags |= DCACHE_DENTRY_KILLED;
N
Nick Piggin 已提交
445 446
	if (parent)
		spin_unlock(&parent->d_lock);
447
	dentry_iput(dentry);
N
Nick Piggin 已提交
448 449 450 451
	/*
	 * dentry_iput drops the locks, at which point nobody (except
	 * transient RCU lookups) can reach this dentry.
	 */
452
	d_free(dentry);
453
	return parent;
454 455
}

N
Nick Piggin 已提交
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
/**
 * d_drop - drop a dentry
 * @dentry: dentry to drop
 *
 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
 * be found through a VFS lookup any more. Note that this is different from
 * deleting the dentry - d_delete will try to mark the dentry negative if
 * possible, giving a successful _negative_ lookup, while d_drop will
 * just make the cache lookup fail.
 *
 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
 * reason (NFS timeouts or autofs deletes).
 *
 * __d_drop requires dentry->d_lock.
 */
void __d_drop(struct dentry *dentry)
{
473
	if (!d_unhashed(dentry)) {
474
		struct hlist_bl_head *b;
475 476 477 478 479 480
		/*
		 * Hashed dentries are normally on the dentry hashtable,
		 * with the exception of those newly allocated by
		 * d_obtain_alias, which are always IS_ROOT:
		 */
		if (unlikely(IS_ROOT(dentry)))
481 482 483 484 485 486 487 488
			b = &dentry->d_sb->s_anon;
		else
			b = d_hash(dentry->d_parent, dentry->d_name.hash);

		hlist_bl_lock(b);
		__hlist_bl_del(&dentry->d_hash);
		dentry->d_hash.pprev = NULL;
		hlist_bl_unlock(b);
489
		dentry_rcuwalk_barrier(dentry);
N
Nick Piggin 已提交
490 491 492 493 494 495 496 497 498 499 500 501
	}
}
EXPORT_SYMBOL(__d_drop);

void d_drop(struct dentry *dentry)
{
	spin_lock(&dentry->d_lock);
	__d_drop(dentry);
	spin_unlock(&dentry->d_lock);
}
EXPORT_SYMBOL(d_drop);

502 503 504 505 506 507
/*
 * Finish off a dentry we've decided to kill.
 * dentry->d_lock must be held, returns with it unlocked.
 * If ref is non-zero, then decrement the refcount too.
 * Returns dentry requiring refcount drop, or NULL if we're done.
 */
508
static struct dentry *
509
dentry_kill(struct dentry *dentry, int unlock_on_failure)
510 511
	__releases(dentry->d_lock)
{
512
	struct inode *inode;
513 514
	struct dentry *parent;

515 516
	inode = dentry->d_inode;
	if (inode && !spin_trylock(&inode->i_lock)) {
517
relock:
518 519 520 521
		if (unlock_on_failure) {
			spin_unlock(&dentry->d_lock);
			cpu_relax();
		}
522 523 524 525 526 527 528
		return dentry; /* try again with same dentry */
	}
	if (IS_ROOT(dentry))
		parent = NULL;
	else
		parent = dentry->d_parent;
	if (parent && !spin_trylock(&parent->d_lock)) {
529 530
		if (inode)
			spin_unlock(&inode->i_lock);
531 532
		goto relock;
	}
N
Nick Piggin 已提交
533

534 535 536 537 538
	/*
	 * The dentry is now unrecoverably dead to the world.
	 */
	lockref_mark_dead(&dentry->d_lockref);

S
Sage Weil 已提交
539 540 541 542
	/*
	 * inform the fs via d_prune that this dentry is about to be
	 * unhashed and destroyed.
	 */
543
	if ((dentry->d_flags & DCACHE_OP_PRUNE) && !d_unhashed(dentry))
Y
Yan, Zheng 已提交
544 545 546
		dentry->d_op->d_prune(dentry);

	dentry_lru_del(dentry);
547 548 549 550 551
	/* if it was on the hash then remove it */
	__d_drop(dentry);
	return d_kill(dentry, parent);
}

L
Linus Torvalds 已提交
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
/* 
 * This is dput
 *
 * This is complicated by the fact that we do not want to put
 * dentries that are no longer on any hash chain on the unused
 * list: we'd much rather just get rid of them immediately.
 *
 * However, that implies that we have to traverse the dentry
 * tree upwards to the parents which might _also_ now be
 * scheduled for deletion (it may have been only waiting for
 * its last child to go away).
 *
 * This tail recursion is done by hand as we don't want to depend
 * on the compiler to always get this right (gcc generally doesn't).
 * Real recursion would eat up our stack space.
 */

/*
 * dput - release a dentry
 * @dentry: dentry to release 
 *
 * Release a dentry. This will drop the usage count and if appropriate
 * call the dentry unlink method as well as removing it from the queues and
 * releasing its resources. If the parent dentries were scheduled for release
 * they too may now get deleted.
 */
void dput(struct dentry *dentry)
{
580
	if (unlikely(!dentry))
L
Linus Torvalds 已提交
581 582 583
		return;

repeat:
584
	if (lockref_put_or_lock(&dentry->d_lockref))
L
Linus Torvalds 已提交
585 586
		return;

587 588 589 590 591
	/* Unreachable? Get rid of it */
	if (unlikely(d_unhashed(dentry)))
		goto kill_it;

	if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
L
Linus Torvalds 已提交
592
		if (dentry->d_op->d_delete(dentry))
N
Nick Piggin 已提交
593
			goto kill_it;
L
Linus Torvalds 已提交
594
	}
N
Nick Piggin 已提交
595

596 597
	if (!(dentry->d_flags & DCACHE_REFERENCED))
		dentry->d_flags |= DCACHE_REFERENCED;
598
	dentry_lru_add(dentry);
N
Nick Piggin 已提交
599

600
	dentry->d_lockref.count--;
N
Nick Piggin 已提交
601
	spin_unlock(&dentry->d_lock);
L
Linus Torvalds 已提交
602 603
	return;

604
kill_it:
605
	dentry = dentry_kill(dentry, 1);
606 607
	if (dentry)
		goto repeat;
L
Linus Torvalds 已提交
608
}
609
EXPORT_SYMBOL(dput);
L
Linus Torvalds 已提交
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627

/**
 * d_invalidate - invalidate a dentry
 * @dentry: dentry to invalidate
 *
 * Try to invalidate the dentry if it turns out to be
 * possible. If there are other dentries that can be
 * reached through this one we can't delete it and we
 * return -EBUSY. On success we return 0.
 *
 * no dcache lock.
 */
 
int d_invalidate(struct dentry * dentry)
{
	/*
	 * If it's already been dropped, return OK.
	 */
N
Nick Piggin 已提交
628
	spin_lock(&dentry->d_lock);
L
Linus Torvalds 已提交
629
	if (d_unhashed(dentry)) {
N
Nick Piggin 已提交
630
		spin_unlock(&dentry->d_lock);
L
Linus Torvalds 已提交
631 632 633 634 635 636 637
		return 0;
	}
	/*
	 * Check whether to do a partial shrink_dcache
	 * to get rid of unused child entries.
	 */
	if (!list_empty(&dentry->d_subdirs)) {
N
Nick Piggin 已提交
638
		spin_unlock(&dentry->d_lock);
L
Linus Torvalds 已提交
639
		shrink_dcache_parent(dentry);
N
Nick Piggin 已提交
640
		spin_lock(&dentry->d_lock);
L
Linus Torvalds 已提交
641 642 643 644 645 646 647 648 649 650 651
	}

	/*
	 * Somebody else still using it?
	 *
	 * If it's a directory, we can't drop it
	 * for fear of somebody re-populating it
	 * with children (even though dropping it
	 * would make it unreachable from the root,
	 * we might still populate it if it was a
	 * working directory or similar).
652 653
	 * We also need to leave mountpoints alone,
	 * directory or not.
L
Linus Torvalds 已提交
654
	 */
655
	if (dentry->d_lockref.count > 1 && dentry->d_inode) {
656
		if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
L
Linus Torvalds 已提交
657 658 659 660 661 662 663 664 665
			spin_unlock(&dentry->d_lock);
			return -EBUSY;
		}
	}

	__d_drop(dentry);
	spin_unlock(&dentry->d_lock);
	return 0;
}
666
EXPORT_SYMBOL(d_invalidate);
L
Linus Torvalds 已提交
667

N
Nick Piggin 已提交
668
/* This must be called with d_lock held */
669
static inline void __dget_dlock(struct dentry *dentry)
N
Nick Piggin 已提交
670
{
671
	dentry->d_lockref.count++;
N
Nick Piggin 已提交
672 673
}

674
static inline void __dget(struct dentry *dentry)
L
Linus Torvalds 已提交
675
{
676
	lockref_get(&dentry->d_lockref);
L
Linus Torvalds 已提交
677 678
}

N
Nick Piggin 已提交
679 680
struct dentry *dget_parent(struct dentry *dentry)
{
681
	int gotref;
N
Nick Piggin 已提交
682 683
	struct dentry *ret;

684 685 686 687 688 689 690 691 692 693 694 695 696 697
	/*
	 * Do optimistic parent lookup without any
	 * locking.
	 */
	rcu_read_lock();
	ret = ACCESS_ONCE(dentry->d_parent);
	gotref = lockref_get_not_zero(&ret->d_lockref);
	rcu_read_unlock();
	if (likely(gotref)) {
		if (likely(ret == ACCESS_ONCE(dentry->d_parent)))
			return ret;
		dput(ret);
	}

N
Nick Piggin 已提交
698
repeat:
N
Nick Piggin 已提交
699 700 701 702 703
	/*
	 * Don't need rcu_dereference because we re-check it was correct under
	 * the lock.
	 */
	rcu_read_lock();
N
Nick Piggin 已提交
704
	ret = dentry->d_parent;
N
Nick Piggin 已提交
705 706 707 708
	spin_lock(&ret->d_lock);
	if (unlikely(ret != dentry->d_parent)) {
		spin_unlock(&ret->d_lock);
		rcu_read_unlock();
N
Nick Piggin 已提交
709 710
		goto repeat;
	}
N
Nick Piggin 已提交
711
	rcu_read_unlock();
712 713
	BUG_ON(!ret->d_lockref.count);
	ret->d_lockref.count++;
N
Nick Piggin 已提交
714 715 716 717 718
	spin_unlock(&ret->d_lock);
	return ret;
}
EXPORT_SYMBOL(dget_parent);

L
Linus Torvalds 已提交
719 720 721
/**
 * d_find_alias - grab a hashed alias of inode
 * @inode: inode in question
722 723
 * @want_discon:  flag, used by d_splice_alias, to request
 *          that only a DISCONNECTED alias be returned.
L
Linus Torvalds 已提交
724 725 726 727 728 729 730
 *
 * If inode has a hashed alias, or is a directory and has any alias,
 * acquire the reference to alias and return it. Otherwise return NULL.
 * Notice that if inode is a directory there can be only one alias and
 * it can be unhashed only if it has no children, or if it is the root
 * of a filesystem.
 *
731
 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
732 733
 * any other hashed alias over that one unless @want_discon is set,
 * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias.
L
Linus Torvalds 已提交
734
 */
735
static struct dentry *__d_find_alias(struct inode *inode, int want_discon)
L
Linus Torvalds 已提交
736
{
N
Nick Piggin 已提交
737
	struct dentry *alias, *discon_alias;
L
Linus Torvalds 已提交
738

N
Nick Piggin 已提交
739 740
again:
	discon_alias = NULL;
741
	hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
N
Nick Piggin 已提交
742
		spin_lock(&alias->d_lock);
L
Linus Torvalds 已提交
743
 		if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
744
			if (IS_ROOT(alias) &&
N
Nick Piggin 已提交
745
			    (alias->d_flags & DCACHE_DISCONNECTED)) {
L
Linus Torvalds 已提交
746
				discon_alias = alias;
747
			} else if (!want_discon) {
748
				__dget_dlock(alias);
N
Nick Piggin 已提交
749 750 751 752 753 754 755 756 757 758 759 760
				spin_unlock(&alias->d_lock);
				return alias;
			}
		}
		spin_unlock(&alias->d_lock);
	}
	if (discon_alias) {
		alias = discon_alias;
		spin_lock(&alias->d_lock);
		if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
			if (IS_ROOT(alias) &&
			    (alias->d_flags & DCACHE_DISCONNECTED)) {
761
				__dget_dlock(alias);
N
Nick Piggin 已提交
762
				spin_unlock(&alias->d_lock);
L
Linus Torvalds 已提交
763 764 765
				return alias;
			}
		}
N
Nick Piggin 已提交
766 767
		spin_unlock(&alias->d_lock);
		goto again;
L
Linus Torvalds 已提交
768
	}
N
Nick Piggin 已提交
769
	return NULL;
L
Linus Torvalds 已提交
770 771
}

N
Nick Piggin 已提交
772
struct dentry *d_find_alias(struct inode *inode)
L
Linus Torvalds 已提交
773
{
D
David Howells 已提交
774 775
	struct dentry *de = NULL;

A
Al Viro 已提交
776
	if (!hlist_empty(&inode->i_dentry)) {
777
		spin_lock(&inode->i_lock);
778
		de = __d_find_alias(inode, 0);
779
		spin_unlock(&inode->i_lock);
D
David Howells 已提交
780
	}
L
Linus Torvalds 已提交
781 782
	return de;
}
783
EXPORT_SYMBOL(d_find_alias);
L
Linus Torvalds 已提交
784 785 786 787 788 789 790

/*
 *	Try to kill dentries associated with this inode.
 * WARNING: you must own a reference to inode.
 */
void d_prune_aliases(struct inode *inode)
{
791
	struct dentry *dentry;
L
Linus Torvalds 已提交
792
restart:
793
	spin_lock(&inode->i_lock);
794
	hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
L
Linus Torvalds 已提交
795
		spin_lock(&dentry->d_lock);
796
		if (!dentry->d_lockref.count) {
797 798 799 800 801 802 803 804
			/*
			 * inform the fs via d_prune that this dentry
			 * is about to be unhashed and destroyed.
			 */
			if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
			    !d_unhashed(dentry))
				dentry->d_op->d_prune(dentry);

805
			__dget_dlock(dentry);
L
Linus Torvalds 已提交
806 807
			__d_drop(dentry);
			spin_unlock(&dentry->d_lock);
808
			spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
809 810 811 812 813
			dput(dentry);
			goto restart;
		}
		spin_unlock(&dentry->d_lock);
	}
814
	spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
815
}
816
EXPORT_SYMBOL(d_prune_aliases);
L
Linus Torvalds 已提交
817 818

/*
819 820 821
 * Try to throw away a dentry - free the inode, dput the parent.
 * Requires dentry->d_lock is held, and dentry->d_count == 0.
 * Releases dentry->d_lock.
822
 *
823
 * This may fail if locks cannot be acquired no problem, just try again.
L
Linus Torvalds 已提交
824
 */
825
static struct dentry * try_prune_one_dentry(struct dentry *dentry)
826
	__releases(dentry->d_lock)
L
Linus Torvalds 已提交
827
{
828
	struct dentry *parent;
829

830
	parent = dentry_kill(dentry, 0);
831
	/*
832 833 834 835 836 837 838 839
	 * If dentry_kill returns NULL, we have nothing more to do.
	 * if it returns the same dentry, trylocks failed. In either
	 * case, just loop again.
	 *
	 * Otherwise, we need to prune ancestors too. This is necessary
	 * to prevent quadratic behavior of shrink_dcache_parent(), but
	 * is also expected to be beneficial in reducing dentry cache
	 * fragmentation.
840
	 */
841
	if (!parent)
842
		return NULL;
843
	if (parent == dentry)
844
		return dentry;
845 846 847

	/* Prune ancestors. */
	dentry = parent;
848
	while (dentry) {
849
		if (lockref_put_or_lock(&dentry->d_lockref))
850 851
			return NULL;
		dentry = dentry_kill(dentry, 1);
852
	}
853
	return NULL;
L
Linus Torvalds 已提交
854 855
}

C
Christoph Hellwig 已提交
856
static void shrink_dentry_list(struct list_head *list)
L
Linus Torvalds 已提交
857
{
858 859
	struct dentry *dentry;

860 861 862 863 864
	rcu_read_lock();
	for (;;) {
		dentry = list_entry_rcu(list->prev, struct dentry, d_lru);
		if (&dentry->d_lru == list)
			break; /* empty */
865 866 867 868 869 870

		/*
		 * Get the dentry lock, and re-verify that the dentry is
		 * this on the shrinking list. If it is, we know that
		 * DCACHE_SHRINK_LIST and DCACHE_LRU_LIST are set.
		 */
871 872 873
		spin_lock(&dentry->d_lock);
		if (dentry != list_entry(list->prev, struct dentry, d_lru)) {
			spin_unlock(&dentry->d_lock);
N
Nick Piggin 已提交
874 875 876
			continue;
		}

877 878 879 880 881
		/*
		 * The dispose list is isolated and dentries are not accounted
		 * to the LRU here, so we can simply remove it from the list
		 * here regardless of whether it is referenced or not.
		 */
882
		d_shrink_del(dentry);
883

L
Linus Torvalds 已提交
884 885
		/*
		 * We found an inuse dentry which was not removed from
886
		 * the LRU because of laziness during lookup. Do not free it.
L
Linus Torvalds 已提交
887
		 */
888
		if (dentry->d_lockref.count) {
889
			spin_unlock(&dentry->d_lock);
L
Linus Torvalds 已提交
890 891
			continue;
		}
892
		rcu_read_unlock();
893

894 895 896 897 898 899 900 901 902
		/*
		 * If 'try_to_prune()' returns a dentry, it will
		 * be the same one we passed in, and d_lock will
		 * have been held the whole time, so it will not
		 * have been added to any other lists. We failed
		 * to get the inode lock.
		 *
		 * We just add it back to the shrink list.
		 */
903
		dentry = try_prune_one_dentry(dentry);
904

905
		rcu_read_lock();
906
		if (dentry) {
907
			d_shrink_add(dentry, list);
908 909
			spin_unlock(&dentry->d_lock);
		}
910
	}
911
	rcu_read_unlock();
C
Christoph Hellwig 已提交
912 913
}

914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
static enum lru_status
dentry_lru_isolate(struct list_head *item, spinlock_t *lru_lock, void *arg)
{
	struct list_head *freeable = arg;
	struct dentry	*dentry = container_of(item, struct dentry, d_lru);


	/*
	 * we are inverting the lru lock/dentry->d_lock here,
	 * so use a trylock. If we fail to get the lock, just skip
	 * it
	 */
	if (!spin_trylock(&dentry->d_lock))
		return LRU_SKIP;

	/*
	 * Referenced dentries are still in use. If they have active
	 * counts, just remove them from the LRU. Otherwise give them
	 * another pass through the LRU.
	 */
	if (dentry->d_lockref.count) {
935
		d_lru_isolate(dentry);
936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
		spin_unlock(&dentry->d_lock);
		return LRU_REMOVED;
	}

	if (dentry->d_flags & DCACHE_REFERENCED) {
		dentry->d_flags &= ~DCACHE_REFERENCED;
		spin_unlock(&dentry->d_lock);

		/*
		 * The list move itself will be made by the common LRU code. At
		 * this point, we've dropped the dentry->d_lock but keep the
		 * lru lock. This is safe to do, since every list movement is
		 * protected by the lru lock even if both locks are held.
		 *
		 * This is guaranteed by the fact that all LRU management
		 * functions are intermediated by the LRU API calls like
		 * list_lru_add and list_lru_del. List movement in this file
		 * only ever occur through this functions or through callbacks
		 * like this one, that are called from the LRU API.
		 *
		 * The only exceptions to this are functions like
		 * shrink_dentry_list, and code that first checks for the
		 * DCACHE_SHRINK_LIST flag.  Those are guaranteed to be
		 * operating only with stack provided lists after they are
		 * properly isolated from the main list.  It is thus, always a
		 * local access.
		 */
		return LRU_ROTATE;
	}

966
	d_lru_shrink_move(dentry, freeable);
967 968 969 970 971
	spin_unlock(&dentry->d_lock);

	return LRU_REMOVED;
}

C
Christoph Hellwig 已提交
972
/**
973 974
 * prune_dcache_sb - shrink the dcache
 * @sb: superblock
975
 * @nr_to_scan : number of entries to try to free
976
 * @nid: which node to scan for freeable entities
977
 *
978
 * Attempt to shrink the superblock dcache LRU by @nr_to_scan entries. This is
979 980
 * done when we need more memory an called from the superblock shrinker
 * function.
C
Christoph Hellwig 已提交
981
 *
982 983
 * This function may fail to free any resources if all the dentries are in
 * use.
C
Christoph Hellwig 已提交
984
 */
985 986
long prune_dcache_sb(struct super_block *sb, unsigned long nr_to_scan,
		     int nid)
C
Christoph Hellwig 已提交
987
{
988 989
	LIST_HEAD(dispose);
	long freed;
C
Christoph Hellwig 已提交
990

991 992
	freed = list_lru_walk_node(&sb->s_dentry_lru, nid, dentry_lru_isolate,
				       &dispose, &nr_to_scan);
993
	shrink_dentry_list(&dispose);
994
	return freed;
995
}
N
Nick Piggin 已提交
996

997 998
static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
						spinlock_t *lru_lock, void *arg)
999
{
1000 1001
	struct list_head *freeable = arg;
	struct dentry	*dentry = container_of(item, struct dentry, d_lru);
1002

1003 1004 1005 1006 1007 1008 1009 1010
	/*
	 * we are inverting the lru lock/dentry->d_lock here,
	 * so use a trylock. If we fail to get the lock, just skip
	 * it
	 */
	if (!spin_trylock(&dentry->d_lock))
		return LRU_SKIP;

1011
	d_lru_shrink_move(dentry, freeable);
1012
	spin_unlock(&dentry->d_lock);
1013

1014
	return LRU_REMOVED;
1015 1016
}

1017

L
Linus Torvalds 已提交
1018 1019 1020 1021
/**
 * shrink_dcache_sb - shrink dcache for a superblock
 * @sb: superblock
 *
C
Christoph Hellwig 已提交
1022 1023
 * Shrink the dcache for the specified super block. This is used to free
 * the dcache before unmounting a file system.
L
Linus Torvalds 已提交
1024
 */
C
Christoph Hellwig 已提交
1025
void shrink_dcache_sb(struct super_block *sb)
L
Linus Torvalds 已提交
1026
{
1027 1028 1029 1030 1031 1032 1033
	long freed;

	do {
		LIST_HEAD(dispose);

		freed = list_lru_walk(&sb->s_dentry_lru,
			dentry_lru_isolate_shrink, &dispose, UINT_MAX);
C
Christoph Hellwig 已提交
1034

1035 1036 1037
		this_cpu_sub(nr_dentry_unused, freed);
		shrink_dentry_list(&dispose);
	} while (freed > 0);
L
Linus Torvalds 已提交
1038
}
1039
EXPORT_SYMBOL(shrink_dcache_sb);
L
Linus Torvalds 已提交
1040

M
Miklos Szeredi 已提交
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
/**
 * enum d_walk_ret - action to talke during tree walk
 * @D_WALK_CONTINUE:	contrinue walk
 * @D_WALK_QUIT:	quit walk
 * @D_WALK_NORETRY:	quit when retry is needed
 * @D_WALK_SKIP:	skip this dentry and its children
 */
enum d_walk_ret {
	D_WALK_CONTINUE,
	D_WALK_QUIT,
	D_WALK_NORETRY,
	D_WALK_SKIP,
};
1054

L
Linus Torvalds 已提交
1055
/**
M
Miklos Szeredi 已提交
1056 1057 1058 1059 1060
 * d_walk - walk the dentry tree
 * @parent:	start of walk
 * @data:	data passed to @enter() and @finish()
 * @enter:	callback when first entering the dentry
 * @finish:	callback when successfully finished the walk
L
Linus Torvalds 已提交
1061
 *
M
Miklos Szeredi 已提交
1062
 * The @enter() and @finish() callbacks are called with d_lock held.
L
Linus Torvalds 已提交
1063
 */
M
Miklos Szeredi 已提交
1064 1065 1066
static void d_walk(struct dentry *parent, void *data,
		   enum d_walk_ret (*enter)(void *, struct dentry *),
		   void (*finish)(void *))
L
Linus Torvalds 已提交
1067
{
1068
	struct dentry *this_parent;
L
Linus Torvalds 已提交
1069
	struct list_head *next;
1070
	unsigned seq = 0;
M
Miklos Szeredi 已提交
1071 1072
	enum d_walk_ret ret;
	bool retry = true;
1073

1074
again:
1075
	read_seqbegin_or_lock(&rename_lock, &seq);
1076
	this_parent = parent;
N
Nick Piggin 已提交
1077
	spin_lock(&this_parent->d_lock);
M
Miklos Szeredi 已提交
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089

	ret = enter(data, this_parent);
	switch (ret) {
	case D_WALK_CONTINUE:
		break;
	case D_WALK_QUIT:
	case D_WALK_SKIP:
		goto out_unlock;
	case D_WALK_NORETRY:
		retry = false;
		break;
	}
L
Linus Torvalds 已提交
1090 1091 1092 1093 1094
repeat:
	next = this_parent->d_subdirs.next;
resume:
	while (next != &this_parent->d_subdirs) {
		struct list_head *tmp = next;
E
Eric Dumazet 已提交
1095
		struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
L
Linus Torvalds 已提交
1096
		next = tmp->next;
N
Nick Piggin 已提交
1097 1098

		spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
M
Miklos Szeredi 已提交
1099 1100 1101 1102 1103 1104

		ret = enter(data, dentry);
		switch (ret) {
		case D_WALK_CONTINUE:
			break;
		case D_WALK_QUIT:
N
Nick Piggin 已提交
1105
			spin_unlock(&dentry->d_lock);
M
Miklos Szeredi 已提交
1106 1107 1108 1109 1110 1111 1112
			goto out_unlock;
		case D_WALK_NORETRY:
			retry = false;
			break;
		case D_WALK_SKIP:
			spin_unlock(&dentry->d_lock);
			continue;
N
Nick Piggin 已提交
1113
		}
M
Miklos Szeredi 已提交
1114

L
Linus Torvalds 已提交
1115
		if (!list_empty(&dentry->d_subdirs)) {
N
Nick Piggin 已提交
1116 1117
			spin_unlock(&this_parent->d_lock);
			spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
L
Linus Torvalds 已提交
1118
			this_parent = dentry;
N
Nick Piggin 已提交
1119
			spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
L
Linus Torvalds 已提交
1120 1121
			goto repeat;
		}
N
Nick Piggin 已提交
1122
		spin_unlock(&dentry->d_lock);
L
Linus Torvalds 已提交
1123 1124 1125 1126 1127
	}
	/*
	 * All done at this level ... ascend and resume the search.
	 */
	if (this_parent != parent) {
1128
		struct dentry *child = this_parent;
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
		this_parent = child->d_parent;

		rcu_read_lock();
		spin_unlock(&child->d_lock);
		spin_lock(&this_parent->d_lock);

		/*
		 * might go back up the wrong parent if we have had a rename
		 * or deletion
		 */
		if (this_parent != child->d_parent ||
			 (child->d_flags & DCACHE_DENTRY_KILLED) ||
			 need_seqretry(&rename_lock, seq)) {
			spin_unlock(&this_parent->d_lock);
			rcu_read_unlock();
1144
			goto rename_retry;
1145 1146
		}
		rcu_read_unlock();
1147
		next = child->d_u.d_child.next;
L
Linus Torvalds 已提交
1148 1149
		goto resume;
	}
1150
	if (need_seqretry(&rename_lock, seq)) {
M
Miklos Szeredi 已提交
1151
		spin_unlock(&this_parent->d_lock);
1152
		goto rename_retry;
M
Miklos Szeredi 已提交
1153 1154 1155 1156 1157 1158
	}
	if (finish)
		finish(data);

out_unlock:
	spin_unlock(&this_parent->d_lock);
1159
	done_seqretry(&rename_lock, seq);
M
Miklos Szeredi 已提交
1160
	return;
1161 1162

rename_retry:
M
Miklos Szeredi 已提交
1163 1164
	if (!retry)
		return;
1165
	seq = 1;
1166
	goto again;
L
Linus Torvalds 已提交
1167
}
M
Miklos Szeredi 已提交
1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184

/*
 * Search for at least 1 mount point in the dentry's subdirs.
 * We descend to the next level whenever the d_subdirs
 * list is non-empty and continue searching.
 */

static enum d_walk_ret check_mount(void *data, struct dentry *dentry)
{
	int *ret = data;
	if (d_mountpoint(dentry)) {
		*ret = 1;
		return D_WALK_QUIT;
	}
	return D_WALK_CONTINUE;
}

R
Randy Dunlap 已提交
1185 1186 1187 1188 1189 1190 1191
/**
 * have_submounts - check for mounts over a dentry
 * @parent: dentry to check.
 *
 * Return true if the parent or its subdirectories contain
 * a mount point
 */
M
Miklos Szeredi 已提交
1192 1193 1194 1195 1196 1197 1198 1199
int have_submounts(struct dentry *parent)
{
	int ret = 0;

	d_walk(parent, &ret, check_mount, NULL);

	return ret;
}
1200
EXPORT_SYMBOL(have_submounts);
L
Linus Torvalds 已提交
1201

1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
/*
 * Called by mount code to set a mountpoint and check if the mountpoint is
 * reachable (e.g. NFS can unhash a directory dentry and then the complete
 * subtree can become unreachable).
 *
 * Only one of check_submounts_and_drop() and d_set_mounted() must succeed.  For
 * this reason take rename_lock and d_lock on dentry and ancestors.
 */
int d_set_mounted(struct dentry *dentry)
{
	struct dentry *p;
	int ret = -ENOENT;
	write_seqlock(&rename_lock);
	for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
		/* Need exclusion wrt. check_submounts_and_drop() */
		spin_lock(&p->d_lock);
		if (unlikely(d_unhashed(p))) {
			spin_unlock(&p->d_lock);
			goto out;
		}
		spin_unlock(&p->d_lock);
	}
	spin_lock(&dentry->d_lock);
	if (!d_unlinked(dentry)) {
		dentry->d_flags |= DCACHE_MOUNTED;
		ret = 0;
	}
 	spin_unlock(&dentry->d_lock);
out:
	write_sequnlock(&rename_lock);
	return ret;
}

L
Linus Torvalds 已提交
1235
/*
1236
 * Search the dentry child list of the specified parent,
L
Linus Torvalds 已提交
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249
 * and move any unused dentries to the end of the unused
 * list for prune_dcache(). We descend to the next level
 * whenever the d_subdirs list is non-empty and continue
 * searching.
 *
 * It returns zero iff there are no unused children,
 * otherwise  it returns the number of children moved to
 * the end of the unused list. This may not be the total
 * number of unused children, because select_parent can
 * drop the lock and return early due to latency
 * constraints.
 */

M
Miklos Szeredi 已提交
1250 1251 1252 1253 1254
struct select_data {
	struct dentry *start;
	struct list_head dispose;
	int found;
};
N
Nick Piggin 已提交
1255

M
Miklos Szeredi 已提交
1256 1257 1258 1259
static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
{
	struct select_data *data = _data;
	enum d_walk_ret ret = D_WALK_CONTINUE;
L
Linus Torvalds 已提交
1260

M
Miklos Szeredi 已提交
1261 1262
	if (data->start == dentry)
		goto out;
N
Nick Piggin 已提交
1263

L
Linus Torvalds 已提交
1264
	/*
M
Miklos Szeredi 已提交
1265 1266 1267 1268 1269 1270
	 * move only zero ref count dentries to the dispose list.
	 *
	 * Those which are presently on the shrink list, being processed
	 * by shrink_dentry_list(), shouldn't be moved.  Otherwise the
	 * loop in shrink_dcache_parent() might not make any progress
	 * and loop forever.
L
Linus Torvalds 已提交
1271
	 */
M
Miklos Szeredi 已提交
1272 1273 1274
	if (dentry->d_lockref.count) {
		dentry_lru_del(dentry);
	} else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
1275 1276 1277
		/*
		 * We can't use d_lru_shrink_move() because we
		 * need to get the global LRU lock and do the
1278
		 * LRU accounting.
1279 1280 1281
		 */
		d_lru_del(dentry);
		d_shrink_add(dentry, &data->dispose);
M
Miklos Szeredi 已提交
1282 1283
		data->found++;
		ret = D_WALK_NORETRY;
L
Linus Torvalds 已提交
1284
	}
M
Miklos Szeredi 已提交
1285 1286 1287 1288 1289 1290 1291
	/*
	 * We can return to the caller if we have found some (this
	 * ensures forward progress). We'll be coming back to find
	 * the rest.
	 */
	if (data->found && need_resched())
		ret = D_WALK_QUIT;
L
Linus Torvalds 已提交
1292
out:
M
Miklos Szeredi 已提交
1293
	return ret;
L
Linus Torvalds 已提交
1294 1295 1296 1297 1298 1299 1300 1301
}

/**
 * shrink_dcache_parent - prune dcache
 * @parent: parent of entries to prune
 *
 * Prune the dcache to remove unused children of the parent dentry.
 */
M
Miklos Szeredi 已提交
1302
void shrink_dcache_parent(struct dentry *parent)
L
Linus Torvalds 已提交
1303
{
M
Miklos Szeredi 已提交
1304 1305
	for (;;) {
		struct select_data data;
L
Linus Torvalds 已提交
1306

M
Miklos Szeredi 已提交
1307 1308 1309 1310 1311 1312 1313 1314 1315
		INIT_LIST_HEAD(&data.dispose);
		data.start = parent;
		data.found = 0;

		d_walk(parent, &data, select_collect, NULL);
		if (!data.found)
			break;

		shrink_dentry_list(&data.dispose);
1316 1317
		cond_resched();
	}
L
Linus Torvalds 已提交
1318
}
1319
EXPORT_SYMBOL(shrink_dcache_parent);
L
Linus Torvalds 已提交
1320

1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405
static enum d_walk_ret umount_collect(void *_data, struct dentry *dentry)
{
	struct select_data *data = _data;
	enum d_walk_ret ret = D_WALK_CONTINUE;

	if (dentry->d_lockref.count) {
		dentry_lru_del(dentry);
		if (likely(!list_empty(&dentry->d_subdirs)))
			goto out;
		if (dentry == data->start && dentry->d_lockref.count == 1)
			goto out;
		printk(KERN_ERR
		       "BUG: Dentry %p{i=%lx,n=%s}"
		       " still in use (%d)"
		       " [unmount of %s %s]\n",
		       dentry,
		       dentry->d_inode ?
		       dentry->d_inode->i_ino : 0UL,
		       dentry->d_name.name,
		       dentry->d_lockref.count,
		       dentry->d_sb->s_type->name,
		       dentry->d_sb->s_id);
		BUG();
	} else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
		/*
		 * We can't use d_lru_shrink_move() because we
		 * need to get the global LRU lock and do the
		 * LRU accounting.
		 */
		if (dentry->d_flags & DCACHE_LRU_LIST)
			d_lru_del(dentry);
		d_shrink_add(dentry, &data->dispose);
		data->found++;
		ret = D_WALK_NORETRY;
	}
out:
	if (data->found && need_resched())
		ret = D_WALK_QUIT;
	return ret;
}

/*
 * destroy the dentries attached to a superblock on unmounting
 */
void shrink_dcache_for_umount(struct super_block *sb)
{
	struct dentry *dentry;

	if (down_read_trylock(&sb->s_umount))
		BUG();

	dentry = sb->s_root;
	sb->s_root = NULL;
	for (;;) {
		struct select_data data;

		INIT_LIST_HEAD(&data.dispose);
		data.start = dentry;
		data.found = 0;

		d_walk(dentry, &data, umount_collect, NULL);
		if (!data.found)
			break;

		shrink_dentry_list(&data.dispose);
		cond_resched();
	}
	d_drop(dentry);
	dput(dentry);

	while (!hlist_bl_empty(&sb->s_anon)) {
		struct select_data data;
		dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);

		INIT_LIST_HEAD(&data.dispose);
		data.start = NULL;
		data.found = 0;

		d_walk(dentry, &data, umount_collect, NULL);
		if (data.found)
			shrink_dentry_list(&data.dispose);
		cond_resched();
	}
}

1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry)
{
	struct select_data *data = _data;

	if (d_mountpoint(dentry)) {
		data->found = -EBUSY;
		return D_WALK_QUIT;
	}

	return select_collect(_data, dentry);
}

static void check_and_drop(void *_data)
{
	struct select_data *data = _data;

	if (d_mountpoint(data->start))
		data->found = -EBUSY;
	if (!data->found)
		__d_drop(data->start);
}

/**
 * check_submounts_and_drop - prune dcache, check for submounts and drop
 *
 * All done as a single atomic operation relative to has_unlinked_ancestor().
 * Returns 0 if successfully unhashed @parent.  If there were submounts then
 * return -EBUSY.
 *
 * @dentry: dentry to prune and drop
 */
int check_submounts_and_drop(struct dentry *dentry)
{
	int ret = 0;

	/* Negative dentries can be dropped without further checks */
	if (!dentry->d_inode) {
		d_drop(dentry);
		goto out;
	}

	for (;;) {
		struct select_data data;

		INIT_LIST_HEAD(&data.dispose);
		data.start = dentry;
		data.found = 0;

		d_walk(dentry, &data, check_and_collect, check_and_drop);
		ret = data.found;

		if (!list_empty(&data.dispose))
			shrink_dentry_list(&data.dispose);

		if (ret <= 0)
			break;

		cond_resched();
	}

out:
	return ret;
}
EXPORT_SYMBOL(check_submounts_and_drop);

L
Linus Torvalds 已提交
1471
/**
1472 1473
 * __d_alloc	-	allocate a dcache entry
 * @sb: filesystem it will belong to
L
Linus Torvalds 已提交
1474 1475 1476 1477 1478 1479 1480
 * @name: qstr of the name
 *
 * Allocates a dentry. It returns %NULL if there is insufficient memory
 * available. On a success the dentry is returned. The name passed in is
 * copied and the copy passed in may be reused after this call.
 */
 
1481
struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
L
Linus Torvalds 已提交
1482 1483 1484 1485
{
	struct dentry *dentry;
	char *dname;

1486
	dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
L
Linus Torvalds 已提交
1487 1488 1489
	if (!dentry)
		return NULL;

1490 1491 1492 1493 1494 1495 1496
	/*
	 * We guarantee that the inline name is always NUL-terminated.
	 * This way the memcpy() done by the name switching in rename
	 * will still always have a NUL at the end, even if we might
	 * be overwriting an internal NUL character
	 */
	dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
L
Linus Torvalds 已提交
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511
	if (name->len > DNAME_INLINE_LEN-1) {
		dname = kmalloc(name->len + 1, GFP_KERNEL);
		if (!dname) {
			kmem_cache_free(dentry_cache, dentry); 
			return NULL;
		}
	} else  {
		dname = dentry->d_iname;
	}	

	dentry->d_name.len = name->len;
	dentry->d_name.hash = name->hash;
	memcpy(dname, name->name, name->len);
	dname[name->len] = 0;

1512 1513 1514 1515
	/* Make sure we always see the terminating NUL character */
	smp_wmb();
	dentry->d_name.name = dname;

1516
	dentry->d_lockref.count = 1;
1517
	dentry->d_flags = 0;
L
Linus Torvalds 已提交
1518
	spin_lock_init(&dentry->d_lock);
N
Nick Piggin 已提交
1519
	seqcount_init(&dentry->d_seq);
L
Linus Torvalds 已提交
1520
	dentry->d_inode = NULL;
1521 1522
	dentry->d_parent = dentry;
	dentry->d_sb = sb;
L
Linus Torvalds 已提交
1523 1524
	dentry->d_op = NULL;
	dentry->d_fsdata = NULL;
1525
	INIT_HLIST_BL_NODE(&dentry->d_hash);
L
Linus Torvalds 已提交
1526 1527
	INIT_LIST_HEAD(&dentry->d_lru);
	INIT_LIST_HEAD(&dentry->d_subdirs);
A
Al Viro 已提交
1528
	INIT_HLIST_NODE(&dentry->d_alias);
N
Nick Piggin 已提交
1529
	INIT_LIST_HEAD(&dentry->d_u.d_child);
1530
	d_set_d_op(dentry, dentry->d_sb->s_d_op);
L
Linus Torvalds 已提交
1531

1532
	this_cpu_inc(nr_dentry);
1533

L
Linus Torvalds 已提交
1534 1535
	return dentry;
}
1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563

/**
 * d_alloc	-	allocate a dcache entry
 * @parent: parent of entry to allocate
 * @name: qstr of the name
 *
 * Allocates a dentry. It returns %NULL if there is insufficient memory
 * available. On a success the dentry is returned. The name passed in is
 * copied and the copy passed in may be reused after this call.
 */
struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
{
	struct dentry *dentry = __d_alloc(parent->d_sb, name);
	if (!dentry)
		return NULL;

	spin_lock(&parent->d_lock);
	/*
	 * don't need child lock because it is not subject
	 * to concurrency here
	 */
	__dget_dlock(parent);
	dentry->d_parent = parent;
	list_add(&dentry->d_u.d_child, &parent->d_subdirs);
	spin_unlock(&parent->d_lock);

	return dentry;
}
1564
EXPORT_SYMBOL(d_alloc);
L
Linus Torvalds 已提交
1565

1566 1567 1568 1569 1570 1571 1572 1573
/**
 * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
 * @sb: the superblock
 * @name: qstr of the name
 *
 * For a filesystem that just pins its dentries in memory and never
 * performs lookups at all, return an unhashed IS_ROOT dentry.
 */
1574 1575
struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
{
1576
	return __d_alloc(sb, name);
1577 1578 1579
}
EXPORT_SYMBOL(d_alloc_pseudo);

L
Linus Torvalds 已提交
1580 1581 1582 1583 1584 1585 1586 1587 1588
struct dentry *d_alloc_name(struct dentry *parent, const char *name)
{
	struct qstr q;

	q.name = name;
	q.len = strlen(name);
	q.hash = full_name_hash(q.name, q.len);
	return d_alloc(parent, &q);
}
1589
EXPORT_SYMBOL(d_alloc_name);
L
Linus Torvalds 已提交
1590

1591 1592
void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
{
1593 1594
	WARN_ON_ONCE(dentry->d_op);
	WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH	|
1595 1596
				DCACHE_OP_COMPARE	|
				DCACHE_OP_REVALIDATE	|
1597
				DCACHE_OP_WEAK_REVALIDATE	|
1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
				DCACHE_OP_DELETE ));
	dentry->d_op = op;
	if (!op)
		return;
	if (op->d_hash)
		dentry->d_flags |= DCACHE_OP_HASH;
	if (op->d_compare)
		dentry->d_flags |= DCACHE_OP_COMPARE;
	if (op->d_revalidate)
		dentry->d_flags |= DCACHE_OP_REVALIDATE;
1608 1609
	if (op->d_weak_revalidate)
		dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
1610 1611
	if (op->d_delete)
		dentry->d_flags |= DCACHE_OP_DELETE;
S
Sage Weil 已提交
1612 1613
	if (op->d_prune)
		dentry->d_flags |= DCACHE_OP_PRUNE;
1614 1615 1616 1617

}
EXPORT_SYMBOL(d_set_d_op);

1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644
static unsigned d_flags_for_inode(struct inode *inode)
{
	unsigned add_flags = DCACHE_FILE_TYPE;

	if (!inode)
		return DCACHE_MISS_TYPE;

	if (S_ISDIR(inode->i_mode)) {
		add_flags = DCACHE_DIRECTORY_TYPE;
		if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
			if (unlikely(!inode->i_op->lookup))
				add_flags = DCACHE_AUTODIR_TYPE;
			else
				inode->i_opflags |= IOP_LOOKUP;
		}
	} else if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
		if (unlikely(inode->i_op->follow_link))
			add_flags = DCACHE_SYMLINK_TYPE;
		else
			inode->i_opflags |= IOP_NOFOLLOW;
	}

	if (unlikely(IS_AUTOMOUNT(inode)))
		add_flags |= DCACHE_NEED_AUTOMOUNT;
	return add_flags;
}

1645 1646
static void __d_instantiate(struct dentry *dentry, struct inode *inode)
{
1647 1648
	unsigned add_flags = d_flags_for_inode(inode);

N
Nick Piggin 已提交
1649
	spin_lock(&dentry->d_lock);
1650 1651 1652
	dentry->d_flags &= ~DCACHE_ENTRY_TYPE;
	dentry->d_flags |= add_flags;
	if (inode)
A
Al Viro 已提交
1653
		hlist_add_head(&dentry->d_alias, &inode->i_dentry);
1654
	dentry->d_inode = inode;
N
Nick Piggin 已提交
1655
	dentry_rcuwalk_barrier(dentry);
N
Nick Piggin 已提交
1656
	spin_unlock(&dentry->d_lock);
1657 1658 1659
	fsnotify_d_instantiate(dentry, inode);
}

L
Linus Torvalds 已提交
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676
/**
 * d_instantiate - fill in inode information for a dentry
 * @entry: dentry to complete
 * @inode: inode to attach to this dentry
 *
 * Fill in inode information in the entry.
 *
 * This turns negative dentries into productive full members
 * of society.
 *
 * NOTE! This assumes that the inode count has been incremented
 * (or otherwise set) by the caller to indicate that it is now
 * in use by the dcache.
 */
 
void d_instantiate(struct dentry *entry, struct inode * inode)
{
A
Al Viro 已提交
1677
	BUG_ON(!hlist_unhashed(&entry->d_alias));
1678 1679
	if (inode)
		spin_lock(&inode->i_lock);
1680
	__d_instantiate(entry, inode);
1681 1682
	if (inode)
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
1683 1684
	security_d_instantiate(entry, inode);
}
1685
EXPORT_SYMBOL(d_instantiate);
L
Linus Torvalds 已提交
1686 1687 1688 1689 1690 1691 1692 1693

/**
 * d_instantiate_unique - instantiate a non-aliased dentry
 * @entry: dentry to instantiate
 * @inode: inode to attach to this dentry
 *
 * Fill in inode information in the entry. On success, it returns NULL.
 * If an unhashed alias of "entry" already exists, then we return the
1694
 * aliased dentry instead and drop one reference to inode.
L
Linus Torvalds 已提交
1695 1696 1697
 *
 * Note that in order to avoid conflicts with rename() etc, the caller
 * had better be holding the parent directory semaphore.
1698 1699 1700 1701
 *
 * This also assumes that the inode count has been incremented
 * (or otherwise set) by the caller to indicate that it is now
 * in use by the dcache.
L
Linus Torvalds 已提交
1702
 */
1703 1704
static struct dentry *__d_instantiate_unique(struct dentry *entry,
					     struct inode *inode)
L
Linus Torvalds 已提交
1705 1706 1707 1708 1709 1710
{
	struct dentry *alias;
	int len = entry->d_name.len;
	const char *name = entry->d_name.name;
	unsigned int hash = entry->d_name.hash;

1711
	if (!inode) {
1712
		__d_instantiate(entry, NULL);
1713 1714 1715
		return NULL;
	}

1716
	hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
N
Nick Piggin 已提交
1717 1718 1719 1720 1721
		/*
		 * Don't need alias->d_lock here, because aliases with
		 * d_parent == entry->d_parent are not subject to name or
		 * parent changes, because the parent inode i_mutex is held.
		 */
1722
		if (alias->d_name.hash != hash)
L
Linus Torvalds 已提交
1723 1724 1725
			continue;
		if (alias->d_parent != entry->d_parent)
			continue;
1726 1727
		if (alias->d_name.len != len)
			continue;
1728
		if (dentry_cmp(alias, name, len))
L
Linus Torvalds 已提交
1729
			continue;
1730
		__dget(alias);
L
Linus Torvalds 已提交
1731 1732
		return alias;
	}
1733

1734
	__d_instantiate(entry, inode);
L
Linus Torvalds 已提交
1735 1736
	return NULL;
}
1737 1738 1739 1740 1741

struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
{
	struct dentry *result;

A
Al Viro 已提交
1742
	BUG_ON(!hlist_unhashed(&entry->d_alias));
1743

1744 1745
	if (inode)
		spin_lock(&inode->i_lock);
1746
	result = __d_instantiate_unique(entry, inode);
1747 1748
	if (inode)
		spin_unlock(&inode->i_lock);
1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759

	if (!result) {
		security_d_instantiate(entry, inode);
		return NULL;
	}

	BUG_ON(!d_unhashed(result));
	iput(inode);
	return result;
}

L
Linus Torvalds 已提交
1760 1761
EXPORT_SYMBOL(d_instantiate_unique);

1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788
/**
 * d_instantiate_no_diralias - instantiate a non-aliased dentry
 * @entry: dentry to complete
 * @inode: inode to attach to this dentry
 *
 * Fill in inode information in the entry.  If a directory alias is found, then
 * return an error (and drop inode).  Together with d_materialise_unique() this
 * guarantees that a directory inode may never have more than one alias.
 */
int d_instantiate_no_diralias(struct dentry *entry, struct inode *inode)
{
	BUG_ON(!hlist_unhashed(&entry->d_alias));

	spin_lock(&inode->i_lock);
	if (S_ISDIR(inode->i_mode) && !hlist_empty(&inode->i_dentry)) {
		spin_unlock(&inode->i_lock);
		iput(inode);
		return -EBUSY;
	}
	__d_instantiate(entry, inode);
	spin_unlock(&inode->i_lock);
	security_d_instantiate(entry, inode);

	return 0;
}
EXPORT_SYMBOL(d_instantiate_no_diralias);

A
Al Viro 已提交
1789 1790 1791 1792 1793
struct dentry *d_make_root(struct inode *root_inode)
{
	struct dentry *res = NULL;

	if (root_inode) {
1794
		static const struct qstr name = QSTR_INIT("/", 1);
A
Al Viro 已提交
1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805

		res = __d_alloc(root_inode->i_sb, &name);
		if (res)
			d_instantiate(res, root_inode);
		else
			iput(root_inode);
	}
	return res;
}
EXPORT_SYMBOL(d_make_root);

1806 1807 1808 1809
static struct dentry * __d_find_any_alias(struct inode *inode)
{
	struct dentry *alias;

A
Al Viro 已提交
1810
	if (hlist_empty(&inode->i_dentry))
1811
		return NULL;
A
Al Viro 已提交
1812
	alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
1813 1814 1815 1816
	__dget(alias);
	return alias;
}

1817 1818 1819 1820 1821 1822 1823 1824
/**
 * d_find_any_alias - find any alias for a given inode
 * @inode: inode to find an alias for
 *
 * If any aliases exist for the given inode, take and return a
 * reference for one of them.  If no aliases exist, return %NULL.
 */
struct dentry *d_find_any_alias(struct inode *inode)
1825 1826 1827 1828 1829 1830 1831 1832
{
	struct dentry *de;

	spin_lock(&inode->i_lock);
	de = __d_find_any_alias(inode);
	spin_unlock(&inode->i_lock);
	return de;
}
1833
EXPORT_SYMBOL(d_find_any_alias);
1834

1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847
/**
 * d_obtain_alias - find or allocate a dentry for a given inode
 * @inode: inode to allocate the dentry for
 *
 * Obtain a dentry for an inode resulting from NFS filehandle conversion or
 * similar open by handle operations.  The returned dentry may be anonymous,
 * or may have a full name (if the inode was already in the cache).
 *
 * When called on a directory inode, we must ensure that the inode only ever
 * has one dentry.  If a dentry is found, that is returned instead of
 * allocating a new one.
 *
 * On successful return, the reference to the inode has been transferred
1848 1849 1850 1851
 * to the dentry.  In case of an error the reference on the inode is released.
 * To make it easier to use in export operations a %NULL or IS_ERR inode may
 * be passed in and will be the error will be propagate to the return value,
 * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
1852 1853 1854
 */
struct dentry *d_obtain_alias(struct inode *inode)
{
1855
	static const struct qstr anonstring = QSTR_INIT("/", 1);
C
Christoph Hellwig 已提交
1856 1857
	struct dentry *tmp;
	struct dentry *res;
1858
	unsigned add_flags;
1859 1860

	if (!inode)
1861
		return ERR_PTR(-ESTALE);
1862 1863 1864
	if (IS_ERR(inode))
		return ERR_CAST(inode);

1865
	res = d_find_any_alias(inode);
C
Christoph Hellwig 已提交
1866 1867 1868
	if (res)
		goto out_iput;

1869
	tmp = __d_alloc(inode->i_sb, &anonstring);
C
Christoph Hellwig 已提交
1870 1871 1872
	if (!tmp) {
		res = ERR_PTR(-ENOMEM);
		goto out_iput;
1873
	}
N
Nick Piggin 已提交
1874

1875
	spin_lock(&inode->i_lock);
1876
	res = __d_find_any_alias(inode);
C
Christoph Hellwig 已提交
1877
	if (res) {
1878
		spin_unlock(&inode->i_lock);
C
Christoph Hellwig 已提交
1879 1880 1881 1882 1883
		dput(tmp);
		goto out_iput;
	}

	/* attach a disconnected dentry */
1884 1885
	add_flags = d_flags_for_inode(inode) | DCACHE_DISCONNECTED;

C
Christoph Hellwig 已提交
1886 1887
	spin_lock(&tmp->d_lock);
	tmp->d_inode = inode;
1888
	tmp->d_flags |= add_flags;
A
Al Viro 已提交
1889
	hlist_add_head(&tmp->d_alias, &inode->i_dentry);
1890
	hlist_bl_lock(&tmp->d_sb->s_anon);
1891
	hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
1892
	hlist_bl_unlock(&tmp->d_sb->s_anon);
C
Christoph Hellwig 已提交
1893
	spin_unlock(&tmp->d_lock);
1894
	spin_unlock(&inode->i_lock);
1895
	security_d_instantiate(tmp, inode);
C
Christoph Hellwig 已提交
1896 1897 1898 1899

	return tmp;

 out_iput:
1900 1901
	if (res && !IS_ERR(res))
		security_d_instantiate(res, inode);
C
Christoph Hellwig 已提交
1902 1903
	iput(inode);
	return res;
1904
}
1905
EXPORT_SYMBOL(d_obtain_alias);
L
Linus Torvalds 已提交
1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921

/**
 * d_splice_alias - splice a disconnected dentry into the tree if one exists
 * @inode:  the inode which may have a disconnected dentry
 * @dentry: a negative dentry which we want to point to the inode.
 *
 * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and
 * DCACHE_DISCONNECTED), then d_move that in place of the given dentry
 * and return it, else simply d_add the inode to the dentry and return NULL.
 *
 * This is needed in the lookup routine of any filesystem that is exportable
 * (via knfsd) so that we can build dcache paths to directories effectively.
 *
 * If a dentry was found and moved, then it is returned.  Otherwise NULL
 * is returned.  This matches the expected return value of ->lookup.
 *
1922 1923 1924 1925
 * Cluster filesystems may call this function with a negative, hashed dentry.
 * In that case, we know that the inode will be a regular file, and also this
 * will only occur during atomic_open. So we need to check for the dentry
 * being already hashed only in the final case.
L
Linus Torvalds 已提交
1926 1927 1928 1929 1930
 */
struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
{
	struct dentry *new = NULL;

1931 1932 1933
	if (IS_ERR(inode))
		return ERR_CAST(inode);

1934
	if (inode && S_ISDIR(inode->i_mode)) {
1935
		spin_lock(&inode->i_lock);
1936
		new = __d_find_alias(inode, 1);
L
Linus Torvalds 已提交
1937
		if (new) {
1938
			BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
1939
			spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
1940 1941 1942 1943
			security_d_instantiate(new, inode);
			d_move(new, dentry);
			iput(inode);
		} else {
1944
			/* already taking inode->i_lock, so d_add() by hand */
1945
			__d_instantiate(dentry, inode);
1946
			spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
1947 1948 1949
			security_d_instantiate(dentry, inode);
			d_rehash(dentry);
		}
1950 1951 1952 1953 1954
	} else {
		d_instantiate(dentry, inode);
		if (d_unhashed(dentry))
			d_rehash(dentry);
	}
L
Linus Torvalds 已提交
1955 1956
	return new;
}
1957
EXPORT_SYMBOL(d_splice_alias);
L
Linus Torvalds 已提交
1958

1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
/**
 * d_add_ci - lookup or allocate new dentry with case-exact name
 * @inode:  the inode case-insensitive lookup has found
 * @dentry: the negative dentry that was passed to the parent's lookup func
 * @name:   the case-exact name to be associated with the returned dentry
 *
 * This is to avoid filling the dcache with case-insensitive names to the
 * same inode, only the actual correct case is stored in the dcache for
 * case-insensitive filesystems.
 *
 * For a case-insensitive lookup match and if the the case-exact dentry
 * already exists in in the dcache, use it and return it.
 *
 * If no entry exists with the exact case name, allocate new dentry with
 * the exact case, and return the spliced entry.
 */
1975
struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
1976 1977 1978 1979 1980
			struct qstr *name)
{
	struct dentry *found;
	struct dentry *new;

C
Christoph Hellwig 已提交
1981 1982 1983 1984
	/*
	 * First check if a dentry matching the name already exists,
	 * if not go ahead and create it now.
	 */
1985
	found = d_hash_and_lookup(dentry->d_parent, name);
1986 1987
	if (unlikely(IS_ERR(found)))
		goto err_out;
1988 1989 1990
	if (!found) {
		new = d_alloc(dentry->d_parent, name);
		if (!new) {
1991
			found = ERR_PTR(-ENOMEM);
1992 1993
			goto err_out;
		}
C
Christoph Hellwig 已提交
1994

1995 1996 1997 1998 1999 2000 2001
		found = d_splice_alias(inode, new);
		if (found) {
			dput(new);
			return found;
		}
		return new;
	}
C
Christoph Hellwig 已提交
2002 2003 2004 2005 2006 2007 2008

	/*
	 * If a matching dentry exists, and it's not negative use it.
	 *
	 * Decrement the reference count to balance the iget() done
	 * earlier on.
	 */
2009 2010 2011 2012 2013 2014 2015 2016 2017
	if (found->d_inode) {
		if (unlikely(found->d_inode != inode)) {
			/* This can't happen because bad inodes are unhashed. */
			BUG_ON(!is_bad_inode(inode));
			BUG_ON(!is_bad_inode(found->d_inode));
		}
		iput(inode);
		return found;
	}
C
Christoph Hellwig 已提交
2018

2019 2020
	/*
	 * Negative dentry: instantiate it unless the inode is a directory and
C
Christoph Hellwig 已提交
2021
	 * already has a dentry.
2022
	 */
2023 2024 2025 2026
	new = d_splice_alias(inode, found);
	if (new) {
		dput(found);
		found = new;
2027
	}
2028
	return found;
2029 2030 2031

err_out:
	iput(inode);
2032
	return found;
2033
}
2034
EXPORT_SYMBOL(d_add_ci);
L
Linus Torvalds 已提交
2035

2036 2037 2038 2039
/*
 * Do the slow-case of the dentry name compare.
 *
 * Unlike the dentry_cmp() function, we need to atomically
2040
 * load the name and length information, so that the
2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
 * filesystem can rely on them, and can use the 'name' and
 * 'len' information without worrying about walking off the
 * end of memory etc.
 *
 * Thus the read_seqcount_retry() and the "duplicate" info
 * in arguments (the low-level filesystem should not look
 * at the dentry inode or name contents directly, since
 * rename can change them while we're in RCU mode).
 */
enum slow_d_compare {
	D_COMP_OK,
	D_COMP_NOMATCH,
	D_COMP_SEQRETRY,
};

static noinline enum slow_d_compare slow_dentry_cmp(
		const struct dentry *parent,
		struct dentry *dentry,
		unsigned int seq,
		const struct qstr *name)
{
	int tlen = dentry->d_name.len;
	const char *tname = dentry->d_name.name;

	if (read_seqcount_retry(&dentry->d_seq, seq)) {
		cpu_relax();
		return D_COMP_SEQRETRY;
	}
2069
	if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
2070 2071 2072 2073
		return D_COMP_NOMATCH;
	return D_COMP_OK;
}

N
Nick Piggin 已提交
2074 2075 2076 2077
/**
 * __d_lookup_rcu - search for a dentry (racy, store-free)
 * @parent: parent dentry
 * @name: qstr of name we wish to find
2078
 * @seqp: returns d_seq value at the point where the dentry was found
N
Nick Piggin 已提交
2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091
 * Returns: dentry, or NULL
 *
 * __d_lookup_rcu is the dcache lookup function for rcu-walk name
 * resolution (store-free path walking) design described in
 * Documentation/filesystems/path-lookup.txt.
 *
 * This is not to be used outside core vfs.
 *
 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
 * held, and rcu_read_lock held. The returned dentry must not be stored into
 * without taking d_lock and checking d_seq sequence count against @seq
 * returned here.
 *
2092
 * A refcount may be taken on the found dentry with the d_rcu_to_refcount
N
Nick Piggin 已提交
2093 2094 2095 2096 2097 2098
 * function.
 *
 * Alternatively, __d_lookup_rcu may be called again to look up the child of
 * the returned dentry, so long as its parent's seqlock is checked after the
 * child is looked up. Thus, an interlocking stepping of sequence lock checks
 * is formed, giving integrity down the path walk.
2099 2100 2101
 *
 * NOTE! The caller *has* to check the resulting dentry against the sequence
 * number we've returned before using any of the resulting dentry state!
N
Nick Piggin 已提交
2102
 */
2103 2104
struct dentry *__d_lookup_rcu(const struct dentry *parent,
				const struct qstr *name,
2105
				unsigned *seqp)
N
Nick Piggin 已提交
2106
{
2107
	u64 hashlen = name->hash_len;
N
Nick Piggin 已提交
2108
	const unsigned char *str = name->name;
2109
	struct hlist_bl_head *b = d_hash(parent, hashlen_hash(hashlen));
2110
	struct hlist_bl_node *node;
N
Nick Piggin 已提交
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
	struct dentry *dentry;

	/*
	 * Note: There is significant duplication with __d_lookup_rcu which is
	 * required to prevent single threaded performance regressions
	 * especially on architectures where smp_rmb (in seqcounts) are costly.
	 * Keep the two functions in sync.
	 */

	/*
	 * The hash list is protected using RCU.
	 *
	 * Carefully use d_seq when comparing a candidate dentry, to avoid
	 * races with d_move().
	 *
	 * It is possible that concurrent renames can mess up our list
	 * walk here and result in missing our dentry, resulting in the
	 * false-negative result. d_lookup() protects against concurrent
	 * renames using rename_lock seqlock.
	 *
2131
	 * See Documentation/filesystems/path-lookup.txt for more details.
N
Nick Piggin 已提交
2132
	 */
2133
	hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2134
		unsigned seq;
N
Nick Piggin 已提交
2135 2136

seqretry:
2137 2138
		/*
		 * The dentry sequence count protects us from concurrent
2139
		 * renames, and thus protects parent and name fields.
2140 2141
		 *
		 * The caller must perform a seqcount check in order
2142
		 * to do anything useful with the returned dentry.
2143 2144 2145 2146 2147 2148 2149 2150 2151
		 *
		 * NOTE! We do a "raw" seqcount_begin here. That means that
		 * we don't wait for the sequence count to stabilize if it
		 * is in the middle of a sequence change. If we do the slow
		 * dentry compare, we will do seqretries until it is stable,
		 * and if we end up with a successful lookup, we actually
		 * want to exit RCU lookup anyway.
		 */
		seq = raw_seqcount_begin(&dentry->d_seq);
N
Nick Piggin 已提交
2152 2153
		if (dentry->d_parent != parent)
			continue;
2154 2155
		if (d_unhashed(dentry))
			continue;
2156

2157
		if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
2158 2159
			if (dentry->d_name.hash != hashlen_hash(hashlen))
				continue;
2160 2161
			*seqp = seq;
			switch (slow_dentry_cmp(parent, dentry, seq, name)) {
2162 2163 2164
			case D_COMP_OK:
				return dentry;
			case D_COMP_NOMATCH:
N
Nick Piggin 已提交
2165
				continue;
2166 2167 2168
			default:
				goto seqretry;
			}
N
Nick Piggin 已提交
2169
		}
2170

2171
		if (dentry->d_name.hash_len != hashlen)
2172
			continue;
2173
		*seqp = seq;
2174
		if (!dentry_cmp(dentry, str, hashlen_len(hashlen)))
2175
			return dentry;
N
Nick Piggin 已提交
2176 2177 2178 2179
	}
	return NULL;
}

L
Linus Torvalds 已提交
2180 2181 2182 2183
/**
 * d_lookup - search for a dentry
 * @parent: parent dentry
 * @name: qstr of name we wish to find
2184
 * Returns: dentry, or NULL
L
Linus Torvalds 已提交
2185
 *
2186 2187 2188 2189
 * d_lookup searches the children of the parent dentry for the name in
 * question. If the dentry is found its reference count is incremented and the
 * dentry is returned. The caller must use dput to free the entry when it has
 * finished using it. %NULL is returned if the dentry does not exist.
L
Linus Torvalds 已提交
2190
 */
A
Al Viro 已提交
2191
struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
L
Linus Torvalds 已提交
2192
{
N
Nick Piggin 已提交
2193
	struct dentry *dentry;
2194
	unsigned seq;
L
Linus Torvalds 已提交
2195 2196 2197 2198 2199 2200 2201 2202 2203

        do {
                seq = read_seqbegin(&rename_lock);
                dentry = __d_lookup(parent, name);
                if (dentry)
			break;
	} while (read_seqretry(&rename_lock, seq));
	return dentry;
}
2204
EXPORT_SYMBOL(d_lookup);
L
Linus Torvalds 已提交
2205

N
Nick Piggin 已提交
2206
/**
2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220
 * __d_lookup - search for a dentry (racy)
 * @parent: parent dentry
 * @name: qstr of name we wish to find
 * Returns: dentry, or NULL
 *
 * __d_lookup is like d_lookup, however it may (rarely) return a
 * false-negative result due to unrelated rename activity.
 *
 * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
 * however it must be used carefully, eg. with a following d_lookup in
 * the case of failure.
 *
 * __d_lookup callers must be commented.
 */
A
Al Viro 已提交
2221
struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
L
Linus Torvalds 已提交
2222 2223 2224 2225
{
	unsigned int len = name->len;
	unsigned int hash = name->hash;
	const unsigned char *str = name->name;
2226
	struct hlist_bl_head *b = d_hash(parent, hash);
2227
	struct hlist_bl_node *node;
N
Nick Piggin 已提交
2228
	struct dentry *found = NULL;
2229
	struct dentry *dentry;
L
Linus Torvalds 已提交
2230

N
Nick Piggin 已提交
2231 2232 2233 2234 2235 2236 2237
	/*
	 * Note: There is significant duplication with __d_lookup_rcu which is
	 * required to prevent single threaded performance regressions
	 * especially on architectures where smp_rmb (in seqcounts) are costly.
	 * Keep the two functions in sync.
	 */

2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248
	/*
	 * The hash list is protected using RCU.
	 *
	 * Take d_lock when comparing a candidate dentry, to avoid races
	 * with d_move().
	 *
	 * It is possible that concurrent renames can mess up our list
	 * walk here and result in missing our dentry, resulting in the
	 * false-negative result. d_lookup() protects against concurrent
	 * renames using rename_lock seqlock.
	 *
2249
	 * See Documentation/filesystems/path-lookup.txt for more details.
2250
	 */
L
Linus Torvalds 已提交
2251 2252
	rcu_read_lock();
	
2253
	hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
L
Linus Torvalds 已提交
2254 2255 2256 2257 2258 2259 2260

		if (dentry->d_name.hash != hash)
			continue;

		spin_lock(&dentry->d_lock);
		if (dentry->d_parent != parent)
			goto next;
2261 2262 2263
		if (d_unhashed(dentry))
			goto next;

L
Linus Torvalds 已提交
2264 2265 2266 2267
		/*
		 * It is safe to compare names since d_move() cannot
		 * change the qstr (protected by d_lock).
		 */
2268
		if (parent->d_flags & DCACHE_OP_COMPARE) {
2269 2270
			int tlen = dentry->d_name.len;
			const char *tname = dentry->d_name.name;
2271
			if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
L
Linus Torvalds 已提交
2272 2273
				goto next;
		} else {
2274 2275
			if (dentry->d_name.len != len)
				goto next;
2276
			if (dentry_cmp(dentry, str, len))
L
Linus Torvalds 已提交
2277 2278 2279
				goto next;
		}

2280
		dentry->d_lockref.count++;
2281
		found = dentry;
L
Linus Torvalds 已提交
2282 2283 2284 2285 2286 2287 2288 2289 2290 2291
		spin_unlock(&dentry->d_lock);
		break;
next:
		spin_unlock(&dentry->d_lock);
 	}
 	rcu_read_unlock();

 	return found;
}

2292 2293 2294 2295 2296
/**
 * d_hash_and_lookup - hash the qstr then search for a dentry
 * @dir: Directory to search in
 * @name: qstr of name we wish to find
 *
2297
 * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
2298 2299 2300 2301 2302 2303 2304 2305 2306
 */
struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
{
	/*
	 * Check for a fs-specific hash function. Note that we must
	 * calculate the standard hash first, as the d_op->d_hash()
	 * routine may choose to leave the hash value unchanged.
	 */
	name->hash = full_name_hash(name->name, name->len);
2307
	if (dir->d_flags & DCACHE_OP_HASH) {
2308
		int err = dir->d_op->d_hash(dir, name);
2309 2310
		if (unlikely(err < 0))
			return ERR_PTR(err);
2311
	}
2312
	return d_lookup(dir, name);
2313
}
2314
EXPORT_SYMBOL(d_hash_and_lookup);
2315

L
Linus Torvalds 已提交
2316
/**
N
Nick Piggin 已提交
2317
 * d_validate - verify dentry provided from insecure source (deprecated)
L
Linus Torvalds 已提交
2318
 * @dentry: The dentry alleged to be valid child of @dparent
2319
 * @dparent: The parent dentry (known to be valid)
L
Linus Torvalds 已提交
2320 2321 2322 2323
 *
 * An insecure source has sent us a dentry, here we verify it and dget() it.
 * This is used by ncpfs in its readdir implementation.
 * Zero is returned in the dentry is invalid.
N
Nick Piggin 已提交
2324 2325
 *
 * This function is slow for big directories, and deprecated, do not use it.
L
Linus Torvalds 已提交
2326
 */
2327
int d_validate(struct dentry *dentry, struct dentry *dparent)
L
Linus Torvalds 已提交
2328
{
N
Nick Piggin 已提交
2329
	struct dentry *child;
2330

N
Nick Piggin 已提交
2331
	spin_lock(&dparent->d_lock);
N
Nick Piggin 已提交
2332 2333
	list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) {
		if (dentry == child) {
N
Nick Piggin 已提交
2334
			spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
2335
			__dget_dlock(dentry);
N
Nick Piggin 已提交
2336 2337
			spin_unlock(&dentry->d_lock);
			spin_unlock(&dparent->d_lock);
L
Linus Torvalds 已提交
2338 2339 2340
			return 1;
		}
	}
N
Nick Piggin 已提交
2341
	spin_unlock(&dparent->d_lock);
N
Nick Piggin 已提交
2342

L
Linus Torvalds 已提交
2343 2344
	return 0;
}
2345
EXPORT_SYMBOL(d_validate);
L
Linus Torvalds 已提交
2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369

/*
 * When a file is deleted, we have two options:
 * - turn this dentry into a negative dentry
 * - unhash this dentry and free it.
 *
 * Usually, we want to just turn this into
 * a negative dentry, but if anybody else is
 * currently using the dentry or the inode
 * we can't do that and we fall back on removing
 * it from the hash queues and waiting for
 * it to be deleted later when it has no users
 */
 
/**
 * d_delete - delete a dentry
 * @dentry: The dentry to delete
 *
 * Turn the dentry into a negative dentry if possible, otherwise
 * remove it from the hash queues so it can be deleted later
 */
 
void d_delete(struct dentry * dentry)
{
2370
	struct inode *inode;
2371
	int isdir = 0;
L
Linus Torvalds 已提交
2372 2373 2374
	/*
	 * Are we the only user?
	 */
2375
again:
L
Linus Torvalds 已提交
2376
	spin_lock(&dentry->d_lock);
2377 2378
	inode = dentry->d_inode;
	isdir = S_ISDIR(inode->i_mode);
2379
	if (dentry->d_lockref.count == 1) {
A
Alan Cox 已提交
2380
		if (!spin_trylock(&inode->i_lock)) {
2381 2382 2383 2384
			spin_unlock(&dentry->d_lock);
			cpu_relax();
			goto again;
		}
A
Al Viro 已提交
2385
		dentry->d_flags &= ~DCACHE_CANT_MOUNT;
N
Nick Piggin 已提交
2386
		dentry_unlink_inode(dentry);
2387
		fsnotify_nameremove(dentry, isdir);
L
Linus Torvalds 已提交
2388 2389 2390 2391 2392 2393 2394
		return;
	}

	if (!d_unhashed(dentry))
		__d_drop(dentry);

	spin_unlock(&dentry->d_lock);
2395 2396

	fsnotify_nameremove(dentry, isdir);
L
Linus Torvalds 已提交
2397
}
2398
EXPORT_SYMBOL(d_delete);
L
Linus Torvalds 已提交
2399

2400
static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b)
L
Linus Torvalds 已提交
2401
{
2402
	BUG_ON(!d_unhashed(entry));
2403
	hlist_bl_lock(b);
2404
	entry->d_flags |= DCACHE_RCUACCESS;
2405
	hlist_bl_add_head_rcu(&entry->d_hash, b);
2406
	hlist_bl_unlock(b);
L
Linus Torvalds 已提交
2407 2408
}

2409 2410 2411 2412 2413
static void _d_rehash(struct dentry * entry)
{
	__d_rehash(entry, d_hash(entry->d_parent, entry->d_name.hash));
}

L
Linus Torvalds 已提交
2414 2415 2416 2417 2418 2419 2420 2421 2422 2423
/**
 * d_rehash	- add an entry back to the hash
 * @entry: dentry to add to the hash
 *
 * Adds a dentry to the hash according to its name.
 */
 
void d_rehash(struct dentry * entry)
{
	spin_lock(&entry->d_lock);
2424
	_d_rehash(entry);
L
Linus Torvalds 已提交
2425 2426
	spin_unlock(&entry->d_lock);
}
2427
EXPORT_SYMBOL(d_rehash);
L
Linus Torvalds 已提交
2428

N
Nick Piggin 已提交
2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444
/**
 * dentry_update_name_case - update case insensitive dentry with a new name
 * @dentry: dentry to be updated
 * @name: new name
 *
 * Update a case insensitive dentry with new case of name.
 *
 * dentry must have been returned by d_lookup with name @name. Old and new
 * name lengths must match (ie. no d_compare which allows mismatched name
 * lengths).
 *
 * Parent inode i_mutex must be held over d_lookup and into this call (to
 * keep renames and concurrent inserts, and readdir(2) away).
 */
void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
{
2445
	BUG_ON(!mutex_is_locked(&dentry->d_parent->d_inode->i_mutex));
N
Nick Piggin 已提交
2446 2447 2448
	BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */

	spin_lock(&dentry->d_lock);
N
Nick Piggin 已提交
2449
	write_seqcount_begin(&dentry->d_seq);
N
Nick Piggin 已提交
2450
	memcpy((unsigned char *)dentry->d_name.name, name->name, name->len);
N
Nick Piggin 已提交
2451
	write_seqcount_end(&dentry->d_seq);
N
Nick Piggin 已提交
2452 2453 2454 2455
	spin_unlock(&dentry->d_lock);
}
EXPORT_SYMBOL(dentry_update_name_case);

L
Linus Torvalds 已提交
2456 2457 2458 2459 2460 2461 2462
static void switch_names(struct dentry *dentry, struct dentry *target)
{
	if (dname_external(target)) {
		if (dname_external(dentry)) {
			/*
			 * Both external: swap the pointers
			 */
2463
			swap(target->d_name.name, dentry->d_name.name);
L
Linus Torvalds 已提交
2464 2465 2466 2467 2468
		} else {
			/*
			 * dentry:internal, target:external.  Steal target's
			 * storage and make target internal.
			 */
2469 2470
			memcpy(target->d_iname, dentry->d_name.name,
					dentry->d_name.len + 1);
L
Linus Torvalds 已提交
2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489
			dentry->d_name.name = target->d_name.name;
			target->d_name.name = target->d_iname;
		}
	} else {
		if (dname_external(dentry)) {
			/*
			 * dentry:external, target:internal.  Give dentry's
			 * storage to target and make dentry internal
			 */
			memcpy(dentry->d_iname, target->d_name.name,
					target->d_name.len + 1);
			target->d_name.name = dentry->d_name.name;
			dentry->d_name.name = dentry->d_iname;
		} else {
			/*
			 * Both are internal.  Just copy target to dentry
			 */
			memcpy(dentry->d_iname, target->d_name.name,
					target->d_name.len + 1);
2490 2491
			dentry->d_name.len = target->d_name.len;
			return;
L
Linus Torvalds 已提交
2492 2493
		}
	}
2494
	swap(dentry->d_name.len, target->d_name.len);
L
Linus Torvalds 已提交
2495 2496
}

N
Nick Piggin 已提交
2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
{
	/*
	 * XXXX: do we really need to take target->d_lock?
	 */
	if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent)
		spin_lock(&target->d_parent->d_lock);
	else {
		if (d_ancestor(dentry->d_parent, target->d_parent)) {
			spin_lock(&dentry->d_parent->d_lock);
			spin_lock_nested(&target->d_parent->d_lock,
						DENTRY_D_LOCK_NESTED);
		} else {
			spin_lock(&target->d_parent->d_lock);
			spin_lock_nested(&dentry->d_parent->d_lock,
						DENTRY_D_LOCK_NESTED);
		}
	}
	if (target < dentry) {
		spin_lock_nested(&target->d_lock, 2);
		spin_lock_nested(&dentry->d_lock, 3);
	} else {
		spin_lock_nested(&dentry->d_lock, 2);
		spin_lock_nested(&target->d_lock, 3);
	}
}

static void dentry_unlock_parents_for_move(struct dentry *dentry,
					struct dentry *target)
{
	if (target->d_parent != dentry->d_parent)
		spin_unlock(&dentry->d_parent->d_lock);
	if (target->d_parent != target)
		spin_unlock(&target->d_parent->d_lock);
}

L
Linus Torvalds 已提交
2533
/*
N
Nick Piggin 已提交
2534 2535 2536 2537 2538 2539 2540 2541 2542
 * When switching names, the actual string doesn't strictly have to
 * be preserved in the target - because we're dropping the target
 * anyway. As such, we can just do a simple memcpy() to copy over
 * the new name before we switch.
 *
 * Note that we have to be a lot more careful about getting the hash
 * switched - we have to switch the hash value properly even if it
 * then no longer matches the actual (corrupted) string of the target.
 * The hash value has to match the hash queue that the dentry is on..
L
Linus Torvalds 已提交
2543
 */
2544
/*
2545
 * __d_move - move a dentry
L
Linus Torvalds 已提交
2546 2547 2548 2549
 * @dentry: entry to move
 * @target: new dentry
 *
 * Update the dcache to reflect the move of a file name. Negative
2550 2551 2552
 * dcache entries should not be moved in this way. Caller must hold
 * rename_lock, the i_mutex of the source and target directories,
 * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
L
Linus Torvalds 已提交
2553
 */
2554
static void __d_move(struct dentry * dentry, struct dentry * target)
L
Linus Torvalds 已提交
2555 2556 2557 2558
{
	if (!dentry->d_inode)
		printk(KERN_WARNING "VFS: moving negative dcache entry\n");

N
Nick Piggin 已提交
2559 2560 2561 2562
	BUG_ON(d_ancestor(dentry, target));
	BUG_ON(d_ancestor(target, dentry));

	dentry_lock_for_move(dentry, target);
L
Linus Torvalds 已提交
2563

N
Nick Piggin 已提交
2564
	write_seqcount_begin(&dentry->d_seq);
2565
	write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
N
Nick Piggin 已提交
2566

2567 2568 2569 2570 2571 2572 2573
	/* __d_drop does write_seqcount_barrier, but they're OK to nest. */

	/*
	 * Move the dentry to the target hash queue. Don't bother checking
	 * for the same hash queue because of how unlikely it is.
	 */
	__d_drop(dentry);
N
Nick Piggin 已提交
2574
	__d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
L
Linus Torvalds 已提交
2575 2576 2577 2578

	/* Unhash the target: dput() will then get rid of it */
	__d_drop(target);

E
Eric Dumazet 已提交
2579 2580
	list_del(&dentry->d_u.d_child);
	list_del(&target->d_u.d_child);
L
Linus Torvalds 已提交
2581 2582 2583

	/* Switch the names.. */
	switch_names(dentry, target);
2584
	swap(dentry->d_name.hash, target->d_name.hash);
L
Linus Torvalds 已提交
2585 2586 2587 2588 2589

	/* ... and switch the parents */
	if (IS_ROOT(dentry)) {
		dentry->d_parent = target->d_parent;
		target->d_parent = target;
E
Eric Dumazet 已提交
2590
		INIT_LIST_HEAD(&target->d_u.d_child);
L
Linus Torvalds 已提交
2591
	} else {
2592
		swap(dentry->d_parent, target->d_parent);
L
Linus Torvalds 已提交
2593 2594

		/* And add them back to the (new) parent lists */
E
Eric Dumazet 已提交
2595
		list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
L
Linus Torvalds 已提交
2596 2597
	}

E
Eric Dumazet 已提交
2598
	list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
N
Nick Piggin 已提交
2599

N
Nick Piggin 已提交
2600 2601 2602
	write_seqcount_end(&target->d_seq);
	write_seqcount_end(&dentry->d_seq);

N
Nick Piggin 已提交
2603
	dentry_unlock_parents_for_move(dentry, target);
L
Linus Torvalds 已提交
2604
	spin_unlock(&target->d_lock);
2605
	fsnotify_d_move(dentry);
L
Linus Torvalds 已提交
2606
	spin_unlock(&dentry->d_lock);
2607 2608 2609 2610 2611 2612 2613 2614
}

/*
 * d_move - move a dentry
 * @dentry: entry to move
 * @target: new dentry
 *
 * Update the dcache to reflect the move of a file name. Negative
2615 2616
 * dcache entries should not be moved in this way. See the locking
 * requirements for __d_move.
2617 2618 2619 2620 2621
 */
void d_move(struct dentry *dentry, struct dentry *target)
{
	write_seqlock(&rename_lock);
	__d_move(dentry, target);
L
Linus Torvalds 已提交
2622
	write_sequnlock(&rename_lock);
2623
}
2624
EXPORT_SYMBOL(d_move);
L
Linus Torvalds 已提交
2625

2626 2627 2628 2629 2630 2631 2632
/**
 * d_ancestor - search for an ancestor
 * @p1: ancestor dentry
 * @p2: child dentry
 *
 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
 * an ancestor of p2, else NULL.
2633
 */
2634
struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
2635 2636 2637
{
	struct dentry *p;

2638
	for (p = p2; !IS_ROOT(p); p = p->d_parent) {
2639
		if (p->d_parent == p1)
2640
			return p;
2641
	}
2642
	return NULL;
2643 2644 2645 2646 2647 2648
}

/*
 * This helper attempts to cope with remotely renamed directories
 *
 * It assumes that the caller is already holding
2649
 * dentry->d_parent->d_inode->i_mutex, inode->i_lock and rename_lock
2650 2651 2652 2653
 *
 * Note: If ever the locking in lock_rename() changes, then please
 * remember to update this too...
 */
2654 2655
static struct dentry *__d_unalias(struct inode *inode,
		struct dentry *dentry, struct dentry *alias)
2656 2657
{
	struct mutex *m1 = NULL, *m2 = NULL;
2658
	struct dentry *ret = ERR_PTR(-EBUSY);
2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671

	/* If alias and dentry share a parent, then no extra locks required */
	if (alias->d_parent == dentry->d_parent)
		goto out_unalias;

	/* See lock_rename() */
	if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
		goto out_err;
	m1 = &dentry->d_sb->s_vfs_rename_mutex;
	if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex))
		goto out_err;
	m2 = &alias->d_parent->d_inode->i_mutex;
out_unalias:
2672 2673 2674 2675
	if (likely(!d_mountpoint(alias))) {
		__d_move(alias, dentry);
		ret = alias;
	}
2676
out_err:
2677
	spin_unlock(&inode->i_lock);
2678 2679 2680 2681 2682 2683 2684
	if (m2)
		mutex_unlock(m2);
	if (m1)
		mutex_unlock(m1);
	return ret;
}

2685 2686 2687
/*
 * Prepare an anonymous dentry for life in the superblock's dentry tree as a
 * named dentry in place of the dentry to be replaced.
N
Nick Piggin 已提交
2688
 * returns with anon->d_lock held!
2689 2690 2691
 */
static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
{
2692
	struct dentry *dparent;
2693

N
Nick Piggin 已提交
2694
	dentry_lock_for_move(anon, dentry);
2695

N
Nick Piggin 已提交
2696
	write_seqcount_begin(&dentry->d_seq);
2697
	write_seqcount_begin_nested(&anon->d_seq, DENTRY_D_LOCK_NESTED);
N
Nick Piggin 已提交
2698

2699 2700
	dparent = dentry->d_parent;

N
Nick Piggin 已提交
2701 2702 2703
	switch_names(dentry, anon);
	swap(dentry->d_name.hash, anon->d_name.hash);

2704 2705 2706
	dentry->d_parent = dentry;
	list_del_init(&dentry->d_u.d_child);
	anon->d_parent = dparent;
2707
	list_move(&anon->d_u.d_child, &dparent->d_subdirs);
2708

N
Nick Piggin 已提交
2709 2710 2711
	write_seqcount_end(&dentry->d_seq);
	write_seqcount_end(&anon->d_seq);

N
Nick Piggin 已提交
2712 2713 2714 2715
	dentry_unlock_parents_for_move(anon, dentry);
	spin_unlock(&dentry->d_lock);

	/* anon->d_lock still locked, returns locked */
2716 2717 2718 2719 2720 2721 2722 2723
}

/**
 * d_materialise_unique - introduce an inode into the tree
 * @dentry: candidate dentry
 * @inode: inode to bind to the dentry, to which aliases may be attached
 *
 * Introduces an dentry into the tree, substituting an extant disconnected
2724 2725
 * root directory alias in its place if there is one. Caller must hold the
 * i_mutex of the parent directory.
2726 2727 2728
 */
struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
{
2729
	struct dentry *actual;
2730 2731 2732 2733 2734

	BUG_ON(!d_unhashed(dentry));

	if (!inode) {
		actual = dentry;
2735
		__d_instantiate(dentry, NULL);
2736 2737
		d_rehash(actual);
		goto out_nolock;
2738 2739
	}

2740
	spin_lock(&inode->i_lock);
2741

2742 2743 2744 2745
	if (S_ISDIR(inode->i_mode)) {
		struct dentry *alias;

		/* Does an aliased dentry already exist? */
2746
		alias = __d_find_alias(inode, 0);
2747 2748
		if (alias) {
			actual = alias;
2749 2750 2751 2752 2753
			write_seqlock(&rename_lock);

			if (d_ancestor(alias, dentry)) {
				/* Check for loops */
				actual = ERR_PTR(-ELOOP);
2754
				spin_unlock(&inode->i_lock);
2755 2756 2757
			} else if (IS_ROOT(alias)) {
				/* Is this an anonymous mountpoint that we
				 * could splice into our tree? */
2758
				__d_materialise_dentry(dentry, alias);
2759
				write_sequnlock(&rename_lock);
2760 2761
				__d_drop(alias);
				goto found;
2762 2763
			} else {
				/* Nope, but we must(!) avoid directory
2764
				 * aliasing. This drops inode->i_lock */
2765
				actual = __d_unalias(inode, dentry, alias);
2766
			}
2767
			write_sequnlock(&rename_lock);
2768 2769 2770 2771 2772 2773 2774 2775
			if (IS_ERR(actual)) {
				if (PTR_ERR(actual) == -ELOOP)
					pr_warn_ratelimited(
						"VFS: Lookup of '%s' in %s %s"
						" would have caused loop\n",
						dentry->d_name.name,
						inode->i_sb->s_type->name,
						inode->i_sb->s_id);
2776
				dput(alias);
2777
			}
2778 2779
			goto out_nolock;
		}
2780 2781 2782 2783 2784 2785
	}

	/* Add a unique reference */
	actual = __d_instantiate_unique(dentry, inode);
	if (!actual)
		actual = dentry;
2786 2787
	else
		BUG_ON(!d_unhashed(actual));
2788 2789 2790 2791 2792

	spin_lock(&actual->d_lock);
found:
	_d_rehash(actual);
	spin_unlock(&actual->d_lock);
2793
	spin_unlock(&inode->i_lock);
2794
out_nolock:
2795 2796 2797 2798 2799 2800 2801 2802
	if (actual == dentry) {
		security_d_instantiate(dentry, inode);
		return NULL;
	}

	iput(inode);
	return actual;
}
2803
EXPORT_SYMBOL_GPL(d_materialise_unique);
2804

2805
static int prepend(char **buffer, int *buflen, const char *str, int namelen)
2806 2807 2808 2809 2810 2811 2812 2813 2814
{
	*buflen -= namelen;
	if (*buflen < 0)
		return -ENAMETOOLONG;
	*buffer -= namelen;
	memcpy(*buffer, str, namelen);
	return 0;
}

2815 2816
/**
 * prepend_name - prepend a pathname in front of current buffer pointer
2817 2818 2819
 * @buffer: buffer pointer
 * @buflen: allocated length of the buffer
 * @name:   name string and length qstr structure
2820 2821 2822 2823 2824 2825 2826 2827 2828 2829
 *
 * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to
 * make sure that either the old or the new name pointer and length are
 * fetched. However, there may be mismatch between length and pointer.
 * The length cannot be trusted, we need to copy it byte-by-byte until
 * the length is reached or a null byte is found. It also prepends "/" at
 * the beginning of the name. The sequence number check at the caller will
 * retry it again when a d_move() does happen. So any garbage in the buffer
 * due to mismatched pointer and length will be discarded.
 */
2830 2831
static int prepend_name(char **buffer, int *buflen, struct qstr *name)
{
2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847
	const char *dname = ACCESS_ONCE(name->name);
	u32 dlen = ACCESS_ONCE(name->len);
	char *p;

	if (*buflen < dlen + 1)
		return -ENAMETOOLONG;
	*buflen -= dlen + 1;
	p = *buffer -= dlen + 1;
	*p++ = '/';
	while (dlen--) {
		char c = *dname++;
		if (!c)
			break;
		*p++ = c;
	}
	return 0;
2848 2849
}

L
Linus Torvalds 已提交
2850
/**
2851
 * prepend_path - Prepend path string to a buffer
2852
 * @path: the dentry/vfsmount to report
2853
 * @root: root vfsmnt/dentry
M
Miklos Szeredi 已提交
2854 2855
 * @buffer: pointer to the end of the buffer
 * @buflen: pointer to buffer length
2856
 *
2857 2858 2859 2860 2861 2862 2863 2864 2865
 * The function will first try to write out the pathname without taking any
 * lock other than the RCU read lock to make sure that dentries won't go away.
 * It only checks the sequence number of the global rename_lock as any change
 * in the dentry's d_seq will be preceded by changes in the rename_lock
 * sequence number. If the sequence number had been changed, it will restart
 * the whole pathname back-tracing sequence again by taking the rename_lock.
 * In this case, there is no need to take the RCU read lock as the recursive
 * parent pointer references will keep the dentry chain alive as long as no
 * rename operation is performed.
L
Linus Torvalds 已提交
2866
 */
2867 2868
static int prepend_path(const struct path *path,
			const struct path *root,
M
Miklos Szeredi 已提交
2869
			char **buffer, int *buflen)
L
Linus Torvalds 已提交
2870
{
2871 2872 2873
	struct dentry *dentry;
	struct vfsmount *vfsmnt;
	struct mount *mnt;
M
Miklos Szeredi 已提交
2874
	int error = 0;
A
Al Viro 已提交
2875
	unsigned seq, m_seq = 0;
2876 2877
	char *bptr;
	int blen;
2878

2879
	rcu_read_lock();
A
Al Viro 已提交
2880 2881 2882
restart_mnt:
	read_seqbegin_or_lock(&mount_lock, &m_seq);
	seq = 0;
L
Li Zhong 已提交
2883
	rcu_read_lock();
2884 2885 2886
restart:
	bptr = *buffer;
	blen = *buflen;
A
Al Viro 已提交
2887
	error = 0;
2888 2889 2890
	dentry = path->dentry;
	vfsmnt = path->mnt;
	mnt = real_mount(vfsmnt);
2891
	read_seqbegin_or_lock(&rename_lock, &seq);
M
Miklos Szeredi 已提交
2892
	while (dentry != root->dentry || vfsmnt != root->mnt) {
L
Linus Torvalds 已提交
2893 2894 2895
		struct dentry * parent;

		if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
A
Al Viro 已提交
2896
			struct mount *parent = ACCESS_ONCE(mnt->mnt_parent);
2897
			/* Global root? */
A
Al Viro 已提交
2898 2899 2900
			if (mnt != parent) {
				dentry = ACCESS_ONCE(mnt->mnt_mountpoint);
				mnt = parent;
2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917
				vfsmnt = &mnt->mnt;
				continue;
			}
			/*
			 * Filesystems needing to implement special "root names"
			 * should do so with ->d_dname()
			 */
			if (IS_ROOT(dentry) &&
			   (dentry->d_name.len != 1 ||
			    dentry->d_name.name[0] != '/')) {
				WARN(1, "Root dentry has weird name <%.*s>\n",
				     (int) dentry->d_name.len,
				     dentry->d_name.name);
			}
			if (!error)
				error = is_mounted(vfsmnt) ? 1 : 2;
			break;
L
Linus Torvalds 已提交
2918 2919 2920
		}
		parent = dentry->d_parent;
		prefetch(parent);
2921
		error = prepend_name(&bptr, &blen, &dentry->d_name);
M
Miklos Szeredi 已提交
2922 2923 2924
		if (error)
			break;

L
Linus Torvalds 已提交
2925 2926
		dentry = parent;
	}
2927 2928 2929 2930
	if (!(seq & 1))
		rcu_read_unlock();
	if (need_seqretry(&rename_lock, seq)) {
		seq = 1;
2931
		goto restart;
2932 2933
	}
	done_seqretry(&rename_lock, seq);
L
Li Zhong 已提交
2934 2935 2936

	if (!(m_seq & 1))
		rcu_read_unlock();
A
Al Viro 已提交
2937 2938 2939 2940 2941
	if (need_seqretry(&mount_lock, m_seq)) {
		m_seq = 1;
		goto restart_mnt;
	}
	done_seqretry(&mount_lock, m_seq);
L
Linus Torvalds 已提交
2942

2943 2944 2945 2946 2947 2948 2949 2950
	if (error >= 0 && bptr == *buffer) {
		if (--blen < 0)
			error = -ENAMETOOLONG;
		else
			*--bptr = '/';
	}
	*buffer = bptr;
	*buflen = blen;
A
Al Viro 已提交
2951
	return error;
M
Miklos Szeredi 已提交
2952
}
2953

M
Miklos Szeredi 已提交
2954 2955 2956
/**
 * __d_path - return the path of a dentry
 * @path: the dentry/vfsmount to report
2957
 * @root: root vfsmnt/dentry
2958
 * @buf: buffer to return value in
M
Miklos Szeredi 已提交
2959 2960
 * @buflen: buffer length
 *
2961
 * Convert a dentry into an ASCII path name.
M
Miklos Szeredi 已提交
2962 2963 2964 2965
 *
 * Returns a pointer into the buffer or an error code if the
 * path was too long.
 *
2966
 * "buflen" should be positive.
M
Miklos Szeredi 已提交
2967
 *
2968
 * If the path is not reachable from the supplied root, return %NULL.
M
Miklos Szeredi 已提交
2969
 */
2970 2971
char *__d_path(const struct path *path,
	       const struct path *root,
M
Miklos Szeredi 已提交
2972 2973 2974 2975 2976 2977 2978
	       char *buf, int buflen)
{
	char *res = buf + buflen;
	int error;

	prepend(&res, &buflen, "\0", 1);
	error = prepend_path(path, root, &res, &buflen);
2979

2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999
	if (error < 0)
		return ERR_PTR(error);
	if (error > 0)
		return NULL;
	return res;
}

char *d_absolute_path(const struct path *path,
	       char *buf, int buflen)
{
	struct path root = {};
	char *res = buf + buflen;
	int error;

	prepend(&res, &buflen, "\0", 1);
	error = prepend_path(path, &root, &res, &buflen);

	if (error > 1)
		error = -EINVAL;
	if (error < 0)
M
Miklos Szeredi 已提交
3000 3001
		return ERR_PTR(error);
	return res;
L
Linus Torvalds 已提交
3002 3003
}

3004 3005 3006
/*
 * same as __d_path but appends "(deleted)" for unlinked files.
 */
3007 3008 3009
static int path_with_deleted(const struct path *path,
			     const struct path *root,
			     char **buf, int *buflen)
3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020
{
	prepend(buf, buflen, "\0", 1);
	if (d_unlinked(path->dentry)) {
		int error = prepend(buf, buflen, " (deleted)", 10);
		if (error)
			return error;
	}

	return prepend_path(path, root, buf, buflen);
}

3021 3022 3023 3024 3025
static int prepend_unreachable(char **buffer, int *buflen)
{
	return prepend(buffer, buflen, "(unreachable)", 13);
}

3026 3027 3028 3029 3030 3031 3032 3033 3034 3035
static void get_fs_root_rcu(struct fs_struct *fs, struct path *root)
{
	unsigned seq;

	do {
		seq = read_seqcount_begin(&fs->seq);
		*root = fs->root;
	} while (read_seqcount_retry(&fs->seq, seq));
}

J
Jan Blunck 已提交
3036 3037
/**
 * d_path - return the path of a dentry
3038
 * @path: path to report
J
Jan Blunck 已提交
3039 3040 3041 3042 3043 3044
 * @buf: buffer to return value in
 * @buflen: buffer length
 *
 * Convert a dentry into an ASCII path name. If the entry has been deleted
 * the string " (deleted)" is appended. Note that this is ambiguous.
 *
3045 3046 3047 3048
 * Returns a pointer into the buffer or an error code if the path was
 * too long. Note: Callers should use the returned pointer, not the passed
 * in buffer, to use the name! The implementation often starts at an offset
 * into the buffer, and may leave 0 bytes at the start.
J
Jan Blunck 已提交
3049
 *
3050
 * "buflen" should be positive.
J
Jan Blunck 已提交
3051
 */
3052
char *d_path(const struct path *path, char *buf, int buflen)
L
Linus Torvalds 已提交
3053
{
3054
	char *res = buf + buflen;
J
Jan Blunck 已提交
3055
	struct path root;
3056
	int error;
L
Linus Torvalds 已提交
3057

3058 3059 3060 3061 3062 3063 3064
	/*
	 * We have various synthetic filesystems that never get mounted.  On
	 * these filesystems dentries are never used for lookup purposes, and
	 * thus don't need to be hashed.  They also don't need a name until a
	 * user wants to identify the object in /proc/pid/fd/.  The little hack
	 * below allows us to generate a name for these objects on demand:
	 */
3065 3066
	if (path->dentry->d_op && path->dentry->d_op->d_dname)
		return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
3067

3068 3069
	rcu_read_lock();
	get_fs_root_rcu(current->fs, &root);
3070
	error = path_with_deleted(path, &root, &res, &buflen);
3071 3072
	rcu_read_unlock();

3073
	if (error < 0)
3074
		res = ERR_PTR(error);
L
Linus Torvalds 已提交
3075 3076
	return res;
}
3077
EXPORT_SYMBOL(d_path);
L
Linus Torvalds 已提交
3078

3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099
/*
 * Helper function for dentry_operations.d_dname() members
 */
char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
			const char *fmt, ...)
{
	va_list args;
	char temp[64];
	int sz;

	va_start(args, fmt);
	sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1;
	va_end(args);

	if (sz > sizeof(temp) || sz > buflen)
		return ERR_PTR(-ENAMETOOLONG);

	buffer += buflen - sz;
	return memcpy(buffer, temp, sz);
}

3100 3101 3102 3103 3104
char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
{
	char *end = buffer + buflen;
	/* these dentries are never renamed, so d_lock is not needed */
	if (prepend(&end, &buflen, " (deleted)", 11) ||
3105
	    prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) ||
3106 3107
	    prepend(&end, &buflen, "/", 1))  
		end = ERR_PTR(-ENAMETOOLONG);
3108
	return end;
3109 3110
}

3111 3112 3113
/*
 * Write full pathname from the root of the filesystem into the buffer.
 */
N
Nick Piggin 已提交
3114
static char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
3115
{
3116 3117 3118
	char *end, *retval;
	int len, seq = 0;
	int error = 0;
3119

3120
	rcu_read_lock();
3121 3122 3123 3124
restart:
	end = buf + buflen;
	len = buflen;
	prepend(&end, &len, "\0", 1);
3125 3126 3127 3128 3129
	if (buflen < 1)
		goto Elong;
	/* Get '/' right */
	retval = end-1;
	*retval = '/';
3130
	read_seqbegin_or_lock(&rename_lock, &seq);
3131 3132
	while (!IS_ROOT(dentry)) {
		struct dentry *parent = dentry->d_parent;
N
Nick Piggin 已提交
3133
		int error;
3134 3135

		prefetch(parent);
3136 3137 3138
		error = prepend_name(&end, &len, &dentry->d_name);
		if (error)
			break;
3139 3140 3141 3142

		retval = end;
		dentry = parent;
	}
3143 3144 3145 3146
	if (!(seq & 1))
		rcu_read_unlock();
	if (need_seqretry(&rename_lock, seq)) {
		seq = 1;
3147
		goto restart;
3148 3149
	}
	done_seqretry(&rename_lock, seq);
3150 3151
	if (error)
		goto Elong;
A
Al Viro 已提交
3152 3153 3154 3155
	return retval;
Elong:
	return ERR_PTR(-ENAMETOOLONG);
}
N
Nick Piggin 已提交
3156 3157 3158

char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
{
3159
	return __dentry_path(dentry, buf, buflen);
N
Nick Piggin 已提交
3160 3161
}
EXPORT_SYMBOL(dentry_path_raw);
A
Al Viro 已提交
3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176

char *dentry_path(struct dentry *dentry, char *buf, int buflen)
{
	char *p = NULL;
	char *retval;

	if (d_unlinked(dentry)) {
		p = buf + buflen;
		if (prepend(&p, &buflen, "//deleted", 10) != 0)
			goto Elong;
		buflen++;
	}
	retval = __dentry_path(dentry, buf, buflen);
	if (!IS_ERR(retval) && p)
		*p = '/';	/* restore '/' overriden with '\0' */
3177 3178 3179 3180 3181
	return retval;
Elong:
	return ERR_PTR(-ENAMETOOLONG);
}

3182 3183
static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root,
				    struct path *pwd)
3184
{
3185 3186 3187 3188 3189 3190 3191
	unsigned seq;

	do {
		seq = read_seqcount_begin(&fs->seq);
		*root = fs->root;
		*pwd = fs->pwd;
	} while (read_seqcount_retry(&fs->seq, seq));
3192 3193
}

L
Linus Torvalds 已提交
3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211
/*
 * NOTE! The user-level library version returns a
 * character pointer. The kernel system call just
 * returns the length of the buffer filled (which
 * includes the ending '\0' character), or a negative
 * error value. So libc would do something like
 *
 *	char *getcwd(char * buf, size_t size)
 *	{
 *		int retval;
 *
 *		retval = sys_getcwd(buf, size);
 *		if (retval >= 0)
 *			return buf;
 *		errno = -retval;
 *		return NULL;
 *	}
 */
3212
SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
L
Linus Torvalds 已提交
3213
{
3214
	int error;
J
Jan Blunck 已提交
3215
	struct path pwd, root;
3216
	char *page = __getname();
L
Linus Torvalds 已提交
3217 3218 3219 3220

	if (!page)
		return -ENOMEM;

3221 3222
	rcu_read_lock();
	get_fs_root_and_pwd_rcu(current->fs, &root, &pwd);
L
Linus Torvalds 已提交
3223

3224
	error = -ENOENT;
3225
	if (!d_unlinked(pwd.dentry)) {
3226
		unsigned long len;
3227 3228
		char *cwd = page + PATH_MAX;
		int buflen = PATH_MAX;
L
Linus Torvalds 已提交
3229

3230
		prepend(&cwd, &buflen, "\0", 1);
3231
		error = prepend_path(&pwd, &root, &cwd, &buflen);
3232
		rcu_read_unlock();
3233

3234
		if (error < 0)
3235 3236
			goto out;

3237
		/* Unreachable from current root */
3238
		if (error > 0) {
3239 3240 3241 3242 3243
			error = prepend_unreachable(&cwd, &buflen);
			if (error)
				goto out;
		}

3244
		error = -ERANGE;
3245
		len = PATH_MAX + page - cwd;
3246 3247 3248 3249 3250
		if (len <= size) {
			error = len;
			if (copy_to_user(buf, cwd, len))
				error = -EFAULT;
		}
3251
	} else {
3252
		rcu_read_unlock();
3253
	}
L
Linus Torvalds 已提交
3254 3255

out:
3256
	__putname(page);
L
Linus Torvalds 已提交
3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275
	return error;
}

/*
 * Test whether new_dentry is a subdirectory of old_dentry.
 *
 * Trivially implemented using the dcache structure
 */

/**
 * is_subdir - is new dentry a subdirectory of old_dentry
 * @new_dentry: new dentry
 * @old_dentry: old dentry
 *
 * Returns 1 if new_dentry is a subdirectory of the parent (at any depth).
 * Returns 0 otherwise.
 * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
 */
  
3276
int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
L
Linus Torvalds 已提交
3277 3278
{
	int result;
3279
	unsigned seq;
L
Linus Torvalds 已提交
3280

3281 3282 3283 3284
	if (new_dentry == old_dentry)
		return 1;

	do {
L
Linus Torvalds 已提交
3285 3286
		/* for restarting inner loop in case of seq retry */
		seq = read_seqbegin(&rename_lock);
3287 3288 3289 3290 3291
		/*
		 * Need rcu_readlock to protect against the d_parent trashing
		 * due to d_move
		 */
		rcu_read_lock();
3292
		if (d_ancestor(old_dentry, new_dentry))
L
Linus Torvalds 已提交
3293
			result = 1;
3294 3295
		else
			result = 0;
3296
		rcu_read_unlock();
L
Linus Torvalds 已提交
3297 3298 3299 3300 3301
	} while (read_seqretry(&rename_lock, seq));

	return result;
}

M
Miklos Szeredi 已提交
3302
static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
L
Linus Torvalds 已提交
3303
{
M
Miklos Szeredi 已提交
3304 3305 3306 3307
	struct dentry *root = data;
	if (dentry != root) {
		if (d_unhashed(dentry) || !dentry->d_inode)
			return D_WALK_SKIP;
L
Linus Torvalds 已提交
3308

M
Miklos Szeredi 已提交
3309 3310 3311 3312
		if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
			dentry->d_flags |= DCACHE_GENOCIDE;
			dentry->d_lockref.count--;
		}
L
Linus Torvalds 已提交
3313
	}
M
Miklos Szeredi 已提交
3314 3315
	return D_WALK_CONTINUE;
}
3316

M
Miklos Szeredi 已提交
3317 3318 3319
void d_genocide(struct dentry *parent)
{
	d_walk(parent, parent, d_genocide_kill, NULL);
L
Linus Torvalds 已提交
3320 3321
}

3322
void d_tmpfile(struct dentry *dentry, struct inode *inode)
L
Linus Torvalds 已提交
3323
{
3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334
	inode_dec_link_count(inode);
	BUG_ON(dentry->d_name.name != dentry->d_iname ||
		!hlist_unhashed(&dentry->d_alias) ||
		!d_unlinked(dentry));
	spin_lock(&dentry->d_parent->d_lock);
	spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
	dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
				(unsigned long long)inode->i_ino);
	spin_unlock(&dentry->d_lock);
	spin_unlock(&dentry->d_parent->d_lock);
	d_instantiate(dentry, inode);
L
Linus Torvalds 已提交
3335
}
3336
EXPORT_SYMBOL(d_tmpfile);
L
Linus Torvalds 已提交
3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349

static __initdata unsigned long dhash_entries;
static int __init set_dhash_entries(char *str)
{
	if (!str)
		return 0;
	dhash_entries = simple_strtoul(str, &str, 0);
	return 1;
}
__setup("dhash_entries=", set_dhash_entries);

static void __init dcache_init_early(void)
{
3350
	unsigned int loop;
L
Linus Torvalds 已提交
3351 3352 3353 3354 3355 3356 3357 3358 3359

	/* If hashes are distributed across NUMA nodes, defer
	 * hash allocation until vmalloc space is available.
	 */
	if (hashdist)
		return;

	dentry_hashtable =
		alloc_large_system_hash("Dentry cache",
3360
					sizeof(struct hlist_bl_head),
L
Linus Torvalds 已提交
3361 3362 3363 3364 3365
					dhash_entries,
					13,
					HASH_EARLY,
					&d_hash_shift,
					&d_hash_mask,
3366
					0,
L
Linus Torvalds 已提交
3367 3368
					0);

3369
	for (loop = 0; loop < (1U << d_hash_shift); loop++)
3370
		INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
L
Linus Torvalds 已提交
3371 3372
}

3373
static void __init dcache_init(void)
L
Linus Torvalds 已提交
3374
{
3375
	unsigned int loop;
L
Linus Torvalds 已提交
3376 3377 3378 3379 3380 3381

	/* 
	 * A constructor could be added for stable state like the lists,
	 * but it is probably not worth it because of the cache nature
	 * of the dcache. 
	 */
3382 3383
	dentry_cache = KMEM_CACHE(dentry,
		SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD);
L
Linus Torvalds 已提交
3384 3385 3386 3387 3388 3389 3390

	/* Hash may have been set up in dcache_init_early */
	if (!hashdist)
		return;

	dentry_hashtable =
		alloc_large_system_hash("Dentry cache",
3391
					sizeof(struct hlist_bl_head),
L
Linus Torvalds 已提交
3392 3393 3394 3395 3396
					dhash_entries,
					13,
					0,
					&d_hash_shift,
					&d_hash_mask,
3397
					0,
L
Linus Torvalds 已提交
3398 3399
					0);

3400
	for (loop = 0; loop < (1U << d_hash_shift); loop++)
3401
		INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
L
Linus Torvalds 已提交
3402 3403 3404
}

/* SLAB cache for __getname() consumers */
3405
struct kmem_cache *names_cachep __read_mostly;
3406
EXPORT_SYMBOL(names_cachep);
L
Linus Torvalds 已提交
3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426

EXPORT_SYMBOL(d_genocide);

void __init vfs_caches_init_early(void)
{
	dcache_init_early();
	inode_init_early();
}

void __init vfs_caches_init(unsigned long mempages)
{
	unsigned long reserve;

	/* Base hash sizes on available memory, with a reserve equal to
           150% of current kernel size */

	reserve = min((mempages - nr_free_pages()) * 3/2, mempages - 1);
	mempages -= reserve;

	names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0,
3427
			SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3428

3429 3430
	dcache_init();
	inode_init();
L
Linus Torvalds 已提交
3431
	files_init(mempages);
3432
	mnt_init();
L
Linus Torvalds 已提交
3433 3434 3435
	bdev_cache_init();
	chrdev_init();
}