namei.c 121.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 *  linux/fs/namei.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 * Some corrections by tytso.
 */

/* [Feb 1997 T. Schoebel-Theuer] Complete rewrite of the pathname
 * lookup logic.
 */
/* [Feb-Apr 2000, AV] Rewrite to the new namespace architecture.
 */

#include <linux/init.h>
18
#include <linux/export.h>
19
#include <linux/kernel.h>
L
Linus Torvalds 已提交
20 21 22 23
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/namei.h>
#include <linux/pagemap.h>
R
Robert Love 已提交
24
#include <linux/fsnotify.h>
L
Linus Torvalds 已提交
25 26
#include <linux/personality.h>
#include <linux/security.h>
M
Mimi Zohar 已提交
27
#include <linux/ima.h>
L
Linus Torvalds 已提交
28 29 30
#include <linux/syscalls.h>
#include <linux/mount.h>
#include <linux/audit.h>
31
#include <linux/capability.h>
32
#include <linux/file.h>
33
#include <linux/fcntl.h>
34
#include <linux/device_cgroup.h>
35
#include <linux/fs_struct.h>
36
#include <linux/posix_acl.h>
37
#include <linux/hash.h>
38
#include <linux/bitops.h>
39
#include <linux/init_task.h>
40
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
41

42
#include "internal.h"
43
#include "mount.h"
44

L
Linus Torvalds 已提交
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
/* [Feb-1997 T. Schoebel-Theuer]
 * Fundamental changes in the pathname lookup mechanisms (namei)
 * were necessary because of omirr.  The reason is that omirr needs
 * to know the _real_ pathname, not the user-supplied one, in case
 * of symlinks (and also when transname replacements occur).
 *
 * The new code replaces the old recursive symlink resolution with
 * an iterative one (in case of non-nested symlink chains).  It does
 * this with calls to <fs>_follow_link().
 * As a side effect, dir_namei(), _namei() and follow_link() are now 
 * replaced with a single function lookup_dentry() that can handle all 
 * the special cases of the former code.
 *
 * With the new dcache, the pathname is stored at each inode, at least as
 * long as the refcount of the inode is positive.  As a side effect, the
 * size of the dcache depends on the inode cache and thus is dynamic.
 *
 * [29-Apr-1998 C. Scott Ananian] Updated above description of symlink
 * resolution to correspond with current state of the code.
 *
 * Note that the symlink resolution is not *completely* iterative.
 * There is still a significant amount of tail- and mid- recursion in
 * the algorithm.  Also, note that <fs>_readlink() is not used in
 * lookup_dentry(): lookup_dentry() on the result of <fs>_readlink()
 * may return different results than <fs>_follow_link().  Many virtual
 * filesystems (including /proc) exhibit this behavior.
 */

/* [24-Feb-97 T. Schoebel-Theuer] Side effects caused by new implementation:
 * New symlink semantics: when open() is called with flags O_CREAT | O_EXCL
 * and the name already exists in form of a symlink, try to create the new
 * name indicated by the symlink. The old code always complained that the
 * name already exists, due to not following the symlink even if its target
 * is nonexistent.  The new semantics affects also mknod() and link() when
L
Lucas De Marchi 已提交
79
 * the name is a symlink pointing to a non-existent name.
L
Linus Torvalds 已提交
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
 *
 * I don't know which semantics is the right one, since I have no access
 * to standards. But I found by trial that HP-UX 9.0 has the full "new"
 * semantics implemented, while SunOS 4.1.1 and Solaris (SunOS 5.4) have the
 * "old" one. Personally, I think the new semantics is much more logical.
 * Note that "ln old new" where "new" is a symlink pointing to a non-existing
 * file does succeed in both HP-UX and SunOs, but not in Solaris
 * and in the old Linux semantics.
 */

/* [16-Dec-97 Kevin Buhr] For security reasons, we change some symlink
 * semantics.  See the comments in "open_namei" and "do_link" below.
 *
 * [10-Sep-98 Alan Modra] Another symlink change.
 */

/* [Feb-Apr 2000 AV] Complete rewrite. Rules for symlinks:
 *	inside the path - always follow.
 *	in the last component in creation/removal/renaming - never follow.
 *	if LOOKUP_FOLLOW passed - follow.
 *	if the pathname has trailing slashes - follow.
 *	otherwise - don't follow.
 * (applied in that order).
 *
 * [Jun 2000 AV] Inconsistent behaviour of open() in case if flags==O_CREAT
 * restored for 2.4. This is the last surviving part of old 4.2BSD bug.
 * During the 2.4 we need to fix the userland stuff depending on it -
 * hopefully we will be able to get rid of that wart in 2.5. So far only
 * XEmacs seems to be relying on it...
 */
/*
 * [Sep 2001 AV] Single-semaphore locking scheme (kudos to David Holland)
112
 * implemented.  Let's see if raised priority of ->s_vfs_rename_mutex gives
L
Linus Torvalds 已提交
113 114 115 116 117 118 119 120 121 122
 * any extra contention...
 */

/* In order to reduce some races, while at the same time doing additional
 * checking and hopefully speeding things up, we copy filenames to the
 * kernel data space before using them..
 *
 * POSIX.1 2.4: an empty pathname is invalid (ENOENT).
 * PATH_MAX includes the nul terminator --RR.
 */
123

A
Al Viro 已提交
124
#define EMBEDDED_NAME_MAX	(PATH_MAX - offsetof(struct filename, iname))
125

126
struct filename *
127 128
getname_flags(const char __user *filename, int flags, int *empty)
{
A
Al Viro 已提交
129
	struct filename *result;
130
	char *kname;
A
Al Viro 已提交
131
	int len;
132

133 134 135 136
	result = audit_reusename(filename);
	if (result)
		return result;

137
	result = __getname();
138
	if (unlikely(!result))
139 140
		return ERR_PTR(-ENOMEM);

141 142 143 144
	/*
	 * First, try to embed the struct filename inside the names_cache
	 * allocation
	 */
A
Al Viro 已提交
145
	kname = (char *)result->iname;
146
	result->name = kname;
147

A
Al Viro 已提交
148
	len = strncpy_from_user(kname, filename, EMBEDDED_NAME_MAX);
149
	if (unlikely(len < 0)) {
A
Al Viro 已提交
150 151
		__putname(result);
		return ERR_PTR(len);
152
	}
153

154 155 156 157 158 159
	/*
	 * Uh-oh. We have a name that's approaching PATH_MAX. Allocate a
	 * separate struct filename so we can dedicate the entire
	 * names_cache allocation for the pathname, and re-do the copy from
	 * userland.
	 */
A
Al Viro 已提交
160
	if (unlikely(len == EMBEDDED_NAME_MAX)) {
A
Al Viro 已提交
161
		const size_t size = offsetof(struct filename, iname[1]);
162 163
		kname = (char *)result;

A
Al Viro 已提交
164 165 166 167 168 169
		/*
		 * size is chosen that way we to guarantee that
		 * result->iname[0] is within the same object and that
		 * kname can't be equal to result->iname, no matter what.
		 */
		result = kzalloc(size, GFP_KERNEL);
A
Al Viro 已提交
170 171 172
		if (unlikely(!result)) {
			__putname(kname);
			return ERR_PTR(-ENOMEM);
173 174
		}
		result->name = kname;
A
Al Viro 已提交
175 176 177 178 179 180 181 182 183 184 185
		len = strncpy_from_user(kname, filename, PATH_MAX);
		if (unlikely(len < 0)) {
			__putname(kname);
			kfree(result);
			return ERR_PTR(len);
		}
		if (unlikely(len == PATH_MAX)) {
			__putname(kname);
			kfree(result);
			return ERR_PTR(-ENAMETOOLONG);
		}
186 187
	}

A
Al Viro 已提交
188
	result->refcnt = 1;
189 190 191
	/* The empty path is special. */
	if (unlikely(!len)) {
		if (empty)
192
			*empty = 1;
A
Al Viro 已提交
193 194 195 196
		if (!(flags & LOOKUP_EMPTY)) {
			putname(result);
			return ERR_PTR(-ENOENT);
		}
L
Linus Torvalds 已提交
197
	}
198

199
	result->uptr = filename;
200
	result->aname = NULL;
201 202
	audit_getname(result);
	return result;
L
Linus Torvalds 已提交
203 204
}

205 206
struct filename *
getname(const char __user * filename)
A
Al Viro 已提交
207
{
208
	return getname_flags(filename, 0, NULL);
A
Al Viro 已提交
209 210
}

211 212 213 214
struct filename *
getname_kernel(const char * filename)
{
	struct filename *result;
215
	int len = strlen(filename) + 1;
216 217 218 219 220

	result = __getname();
	if (unlikely(!result))
		return ERR_PTR(-ENOMEM);

221
	if (len <= EMBEDDED_NAME_MAX) {
A
Al Viro 已提交
222
		result->name = (char *)result->iname;
223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
	} else if (len <= PATH_MAX) {
		struct filename *tmp;

		tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
		if (unlikely(!tmp)) {
			__putname(result);
			return ERR_PTR(-ENOMEM);
		}
		tmp->name = (char *)result;
		result = tmp;
	} else {
		__putname(result);
		return ERR_PTR(-ENAMETOOLONG);
	}
	memcpy((char *)result->name, filename, len);
238 239
	result->uptr = NULL;
	result->aname = NULL;
240
	result->refcnt = 1;
241
	audit_getname(result);
242 243 244 245

	return result;
}

246
void putname(struct filename *name)
L
Linus Torvalds 已提交
247
{
248 249 250 251 252
	BUG_ON(name->refcnt <= 0);

	if (--name->refcnt > 0)
		return;

A
Al Viro 已提交
253
	if (name->name != name->iname) {
254 255 256 257
		__putname(name->name);
		kfree(name);
	} else
		__putname(name);
L
Linus Torvalds 已提交
258 259
}

260 261
static int check_acl(struct inode *inode, int mask)
{
262
#ifdef CONFIG_FS_POSIX_ACL
263 264 265
	struct posix_acl *acl;

	if (mask & MAY_NOT_BLOCK) {
266 267
		acl = get_cached_acl_rcu(inode, ACL_TYPE_ACCESS);
	        if (!acl)
268
	                return -EAGAIN;
269
		/* no ->get_acl() calls in RCU mode... */
270
		if (is_uncached_acl(acl))
271
			return -ECHILD;
A
Ari Savolainen 已提交
272
	        return posix_acl_permission(inode, acl, mask & ~MAY_NOT_BLOCK);
273 274
	}

C
Christoph Hellwig 已提交
275 276 277
	acl = get_acl(inode, ACL_TYPE_ACCESS);
	if (IS_ERR(acl))
		return PTR_ERR(acl);
278 279 280 281 282
	if (acl) {
	        int error = posix_acl_permission(inode, acl, mask);
	        posix_acl_release(acl);
	        return error;
	}
283
#endif
284 285 286 287

	return -EAGAIN;
}

288
/*
289
 * This does the basic permission checking
L
Linus Torvalds 已提交
290
 */
291
static int acl_permission_check(struct inode *inode, int mask)
L
Linus Torvalds 已提交
292
{
293
	unsigned int mode = inode->i_mode;
L
Linus Torvalds 已提交
294

295
	if (likely(uid_eq(current_fsuid(), inode->i_uid)))
L
Linus Torvalds 已提交
296 297
		mode >>= 6;
	else {
298
		if (IS_POSIXACL(inode) && (mode & S_IRWXG)) {
299
			int error = check_acl(inode, mask);
300 301
			if (error != -EAGAIN)
				return error;
L
Linus Torvalds 已提交
302 303 304 305 306 307 308 309 310
		}

		if (in_group_p(inode->i_gid))
			mode >>= 3;
	}

	/*
	 * If the DACs are ok we don't need any capability check.
	 */
311
	if ((mask & ~mode & (MAY_READ | MAY_WRITE | MAY_EXEC)) == 0)
L
Linus Torvalds 已提交
312
		return 0;
313 314 315 316
	return -EACCES;
}

/**
317
 * generic_permission -  check for access rights on a Posix-like filesystem
318
 * @inode:	inode to check access rights for
319
 * @mask:	right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC, ...)
320 321 322 323
 *
 * Used to check for read/write/execute permissions on a file.
 * We use "fsuid" for this, letting us set arbitrary permissions
 * for filesystem access without changing the "normal" uids which
324 325 326 327 328
 * are used for other things.
 *
 * generic_permission is rcu-walk aware. It returns -ECHILD in case an rcu-walk
 * request cannot be satisfied (eg. requires blocking or too much complexity).
 * It would then be called again in ref-walk mode.
329
 */
330
int generic_permission(struct inode *inode, int mask)
331 332 333 334
{
	int ret;

	/*
335
	 * Do the basic permission checks.
336
	 */
337
	ret = acl_permission_check(inode, mask);
338 339
	if (ret != -EACCES)
		return ret;
L
Linus Torvalds 已提交
340

341 342 343
	if (S_ISDIR(inode->i_mode)) {
		/* DACs are overridable for directories */
		if (!(mask & MAY_WRITE))
344 345
			if (capable_wrt_inode_uidgid(inode,
						     CAP_DAC_READ_SEARCH))
346
				return 0;
347
		if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
L
Linus Torvalds 已提交
348
			return 0;
349 350
		return -EACCES;
	}
L
Linus Torvalds 已提交
351 352 353 354

	/*
	 * Searching includes executable on directories, else just read.
	 */
355
	mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
356
	if (mask == MAY_READ)
357
		if (capable_wrt_inode_uidgid(inode, CAP_DAC_READ_SEARCH))
L
Linus Torvalds 已提交
358
			return 0;
359 360 361 362 363 364 365 366
	/*
	 * Read/write DACs are always overridable.
	 * Executable DACs are overridable when there is
	 * at least one exec bit set.
	 */
	if (!(mask & MAY_EXEC) || (inode->i_mode & S_IXUGO))
		if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
			return 0;
L
Linus Torvalds 已提交
367 368 369

	return -EACCES;
}
370
EXPORT_SYMBOL(generic_permission);
L
Linus Torvalds 已提交
371

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
/*
 * We _really_ want to just do "generic_permission()" without
 * even looking at the inode->i_op values. So we keep a cache
 * flag in inode->i_opflags, that says "this has not special
 * permission function, use the fast case".
 */
static inline int do_inode_permission(struct inode *inode, int mask)
{
	if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) {
		if (likely(inode->i_op->permission))
			return inode->i_op->permission(inode, mask);

		/* This gets set once for the inode lifetime */
		spin_lock(&inode->i_lock);
		inode->i_opflags |= IOP_FASTPERM;
		spin_unlock(&inode->i_lock);
	}
	return generic_permission(inode, mask);
}

C
Christoph Hellwig 已提交
392
/**
D
David Howells 已提交
393 394 395
 * __inode_permission - Check for access rights to a given inode
 * @inode: Inode to check permission on
 * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
C
Christoph Hellwig 已提交
396
 *
D
David Howells 已提交
397
 * Check for read/write/execute permissions on an inode.
398 399
 *
 * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask.
D
David Howells 已提交
400 401 402
 *
 * This does not check for a read-only file system.  You probably want
 * inode_permission().
C
Christoph Hellwig 已提交
403
 */
D
David Howells 已提交
404
int __inode_permission(struct inode *inode, int mask)
L
Linus Torvalds 已提交
405
{
406
	int retval;
L
Linus Torvalds 已提交
407

408
	if (unlikely(mask & MAY_WRITE)) {
L
Linus Torvalds 已提交
409 410 411 412
		/*
		 * Nobody gets write access to an immutable file.
		 */
		if (IS_IMMUTABLE(inode))
E
Eryu Guan 已提交
413
			return -EPERM;
414 415 416 417 418 419 420 421

		/*
		 * Updating mtime will likely cause i_uid and i_gid to be
		 * written back improperly if their true value is unknown
		 * to the vfs.
		 */
		if (HAS_UNMAPPED_ID(inode))
			return -EACCES;
L
Linus Torvalds 已提交
422 423
	}

424
	retval = do_inode_permission(inode, mask);
L
Linus Torvalds 已提交
425 426 427
	if (retval)
		return retval;

428 429 430 431
	retval = devcgroup_inode_permission(inode, mask);
	if (retval)
		return retval;

432
	return security_inode_permission(inode, mask);
L
Linus Torvalds 已提交
433
}
434
EXPORT_SYMBOL(__inode_permission);
L
Linus Torvalds 已提交
435

D
David Howells 已提交
436 437 438
/**
 * sb_permission - Check superblock-level permissions
 * @sb: Superblock of inode to check permission on
439
 * @inode: Inode to check permission on
D
David Howells 已提交
440 441 442 443 444 445 446 447 448 449
 * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
 *
 * Separate out file-system wide checks from inode-specific permission checks.
 */
static int sb_permission(struct super_block *sb, struct inode *inode, int mask)
{
	if (unlikely(mask & MAY_WRITE)) {
		umode_t mode = inode->i_mode;

		/* Nobody gets write access to a read-only fs. */
450
		if (sb_rdonly(sb) && (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)))
D
David Howells 已提交
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
			return -EROFS;
	}
	return 0;
}

/**
 * inode_permission - Check for access rights to a given inode
 * @inode: Inode to check permission on
 * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
 *
 * Check for read/write/execute permissions on an inode.  We use fs[ug]id for
 * this, letting us set arbitrary permissions for filesystem access without
 * changing the "normal" UIDs which are used for other things.
 *
 * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask.
 */
int inode_permission(struct inode *inode, int mask)
{
	int retval;

	retval = sb_permission(inode->i_sb, inode, mask);
	if (retval)
		return retval;
	return __inode_permission(inode, mask);
}
476
EXPORT_SYMBOL(inode_permission);
D
David Howells 已提交
477

J
Jan Blunck 已提交
478 479 480 481 482 483
/**
 * path_get - get a reference to a path
 * @path: path to get the reference to
 *
 * Given a path increment the reference count to the dentry and the vfsmount.
 */
484
void path_get(const struct path *path)
J
Jan Blunck 已提交
485 486 487 488 489 490
{
	mntget(path->mnt);
	dget(path->dentry);
}
EXPORT_SYMBOL(path_get);

J
Jan Blunck 已提交
491 492 493 494 495 496
/**
 * path_put - put a reference to a path
 * @path: path to put the reference to
 *
 * Given a path decrement the reference count to the dentry and the vfsmount.
 */
497
void path_put(const struct path *path)
L
Linus Torvalds 已提交
498
{
J
Jan Blunck 已提交
499 500
	dput(path->dentry);
	mntput(path->mnt);
L
Linus Torvalds 已提交
501
}
J
Jan Blunck 已提交
502
EXPORT_SYMBOL(path_put);
L
Linus Torvalds 已提交
503

504
#define EMBEDDED_LEVELS 2
505 506
struct nameidata {
	struct path	path;
A
Al Viro 已提交
507
	struct qstr	last;
508 509 510
	struct path	root;
	struct inode	*inode; /* path.dentry.d_inode */
	unsigned int	flags;
511
	unsigned	seq, m_seq;
512 513
	int		last_type;
	unsigned	depth;
514
	int		total_link_count;
515 516
	struct saved {
		struct path link;
517
		struct delayed_call done;
518
		const char *name;
519
		unsigned seq;
520
	} *stack, internal[EMBEDDED_LEVELS];
521 522
	struct filename	*name;
	struct nameidata *saved;
523
	struct inode	*link_inode;
524 525
	unsigned	root_seq;
	int		dfd;
526
} __randomize_layout;
527

528
static void set_nameidata(struct nameidata *p, int dfd, struct filename *name)
529
{
530 531
	struct nameidata *old = current->nameidata;
	p->stack = p->internal;
532 533
	p->dfd = dfd;
	p->name = name;
534
	p->total_link_count = old ? old->total_link_count : 0;
535
	p->saved = old;
536
	current->nameidata = p;
537 538
}

539
static void restore_nameidata(void)
540
{
541
	struct nameidata *now = current->nameidata, *old = now->saved;
542 543 544 545

	current->nameidata = old;
	if (old)
		old->total_link_count = now->total_link_count;
546
	if (now->stack != now->internal)
547
		kfree(now->stack);
548 549 550 551
}

static int __nd_alloc_stack(struct nameidata *nd)
{
A
Al Viro 已提交
552 553 554 555 556 557 558 559 560
	struct saved *p;

	if (nd->flags & LOOKUP_RCU) {
		p= kmalloc(MAXSYMLINKS * sizeof(struct saved),
				  GFP_ATOMIC);
		if (unlikely(!p))
			return -ECHILD;
	} else {
		p= kmalloc(MAXSYMLINKS * sizeof(struct saved),
561
				  GFP_KERNEL);
A
Al Viro 已提交
562 563 564
		if (unlikely(!p))
			return -ENOMEM;
	}
565 566 567 568 569
	memcpy(p, nd->internal, sizeof(nd->internal));
	nd->stack = p;
	return 0;
}

570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
/**
 * path_connected - Verify that a path->dentry is below path->mnt.mnt_root
 * @path: nameidate to verify
 *
 * Rename can sometimes move a file or directory outside of a bind
 * mount, path_connected allows those cases to be detected.
 */
static bool path_connected(const struct path *path)
{
	struct vfsmount *mnt = path->mnt;

	/* Only bind mounts can have disconnected paths */
	if (mnt->mnt_root == mnt->mnt_sb->s_root)
		return true;

	return is_subdir(path->dentry, mnt->mnt_root);
}

588 589
static inline int nd_alloc_stack(struct nameidata *nd)
{
590
	if (likely(nd->depth != EMBEDDED_LEVELS))
591 592 593 594 595 596
		return 0;
	if (likely(nd->stack != nd->internal))
		return 0;
	return __nd_alloc_stack(nd);
}

597 598 599 600 601
static void drop_links(struct nameidata *nd)
{
	int i = nd->depth;
	while (i--) {
		struct saved *last = nd->stack + i;
602 603
		do_delayed_call(&last->done);
		clear_delayed_call(&last->done);
604 605 606 607 608 609 610 611 612 613 614
	}
}

static void terminate_walk(struct nameidata *nd)
{
	drop_links(nd);
	if (!(nd->flags & LOOKUP_RCU)) {
		int i;
		path_put(&nd->path);
		for (i = 0; i < nd->depth; i++)
			path_put(&nd->stack[i].link);
615 616 617 618
		if (nd->root.mnt && !(nd->flags & LOOKUP_ROOT)) {
			path_put(&nd->root);
			nd->root.mnt = NULL;
		}
619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659
	} else {
		nd->flags &= ~LOOKUP_RCU;
		if (!(nd->flags & LOOKUP_ROOT))
			nd->root.mnt = NULL;
		rcu_read_unlock();
	}
	nd->depth = 0;
}

/* path_put is needed afterwards regardless of success or failure */
static bool legitimize_path(struct nameidata *nd,
			    struct path *path, unsigned seq)
{
	int res = __legitimize_mnt(path->mnt, nd->m_seq);
	if (unlikely(res)) {
		if (res > 0)
			path->mnt = NULL;
		path->dentry = NULL;
		return false;
	}
	if (unlikely(!lockref_get_not_dead(&path->dentry->d_lockref))) {
		path->dentry = NULL;
		return false;
	}
	return !read_seqcount_retry(&path->dentry->d_seq, seq);
}

static bool legitimize_links(struct nameidata *nd)
{
	int i;
	for (i = 0; i < nd->depth; i++) {
		struct saved *last = nd->stack + i;
		if (unlikely(!legitimize_path(nd, &last->link, last->seq))) {
			drop_links(nd);
			nd->depth = i + 1;
			return false;
		}
	}
	return true;
}

A
Al Viro 已提交
660
/*
N
Nick Piggin 已提交
661
 * Path walking has 2 modes, rcu-walk and ref-walk (see
A
Al Viro 已提交
662 663
 * Documentation/filesystems/path-lookup.txt).  In situations when we can't
 * continue in RCU mode, we attempt to drop out of rcu-walk mode and grab
M
Mike Marshall 已提交
664
 * normal reference counts on dentries and vfsmounts to transition to ref-walk
A
Al Viro 已提交
665 666 667 668
 * mode.  Refcounts are grabbed at the last known good point before rcu-walk
 * got stuck, so ref-walk may continue from there. If this is not successful
 * (eg. a seqcount has changed), then failure is returned and it's up to caller
 * to restart the path walk from the beginning in ref-walk mode.
N
Nick Piggin 已提交
669 670 671
 */

/**
A
Al Viro 已提交
672 673
 * unlazy_walk - try to switch to ref-walk mode.
 * @nd: nameidata pathwalk data
674
 * Returns: 0 on success, -ECHILD on failure
N
Nick Piggin 已提交
675
 *
A
Al Viro 已提交
676 677 678
 * unlazy_walk attempts to legitimize the current nd->path and nd->root
 * for ref-walk mode.
 * Must be called from rcu-walk context.
679 680
 * Nothing should touch nameidata between unlazy_walk() failure and
 * terminate_walk().
N
Nick Piggin 已提交
681
 */
A
Al Viro 已提交
682
static int unlazy_walk(struct nameidata *nd)
N
Nick Piggin 已提交
683 684 685 686
{
	struct dentry *parent = nd->path.dentry;

	BUG_ON(!(nd->flags & LOOKUP_RCU));
687

A
Al Viro 已提交
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
	nd->flags &= ~LOOKUP_RCU;
	if (unlikely(!legitimize_links(nd)))
		goto out2;
	if (unlikely(!legitimize_path(nd, &nd->path, nd->seq)))
		goto out1;
	if (nd->root.mnt && !(nd->flags & LOOKUP_ROOT)) {
		if (unlikely(!legitimize_path(nd, &nd->root, nd->root_seq)))
			goto out;
	}
	rcu_read_unlock();
	BUG_ON(nd->inode != parent->d_inode);
	return 0;

out2:
	nd->path.mnt = NULL;
	nd->path.dentry = NULL;
out1:
	if (!(nd->flags & LOOKUP_ROOT))
		nd->root.mnt = NULL;
out:
	rcu_read_unlock();
	return -ECHILD;
}

/**
 * unlazy_child - try to switch to ref-walk mode.
 * @nd: nameidata pathwalk data
 * @dentry: child of nd->path.dentry
 * @seq: seq number to check dentry against
 * Returns: 0 on success, -ECHILD on failure
 *
 * unlazy_child attempts to legitimize the current nd->path, nd->root and dentry
 * for ref-walk mode.  @dentry must be a path found by a do_lookup call on
 * @nd.  Must be called from rcu-walk context.
 * Nothing should touch nameidata between unlazy_child() failure and
 * terminate_walk().
 */
static int unlazy_child(struct nameidata *nd, struct dentry *dentry, unsigned seq)
{
	BUG_ON(!(nd->flags & LOOKUP_RCU));

729
	nd->flags &= ~LOOKUP_RCU;
730 731 732 733
	if (unlikely(!legitimize_links(nd)))
		goto out2;
	if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq)))
		goto out2;
A
Al Viro 已提交
734
	if (unlikely(!lockref_get_not_dead(&nd->path.dentry->d_lockref)))
735
		goto out1;
A
Al Viro 已提交
736

737
	/*
A
Al Viro 已提交
738 739 740 741 742
	 * We need to move both the parent and the dentry from the RCU domain
	 * to be properly refcounted. And the sequence number in the dentry
	 * validates *both* dentry counters, since we checked the sequence
	 * number of the parent after we got the child sequence number. So we
	 * know the parent must still be valid if the child sequence number is
743
	 */
A
Al Viro 已提交
744 745 746 747 748 749
	if (unlikely(!lockref_get_not_dead(&dentry->d_lockref)))
		goto out;
	if (unlikely(read_seqcount_retry(&dentry->d_seq, seq))) {
		rcu_read_unlock();
		dput(dentry);
		goto drop_root_mnt;
A
Al Viro 已提交
750
	}
751 752 753 754 755
	/*
	 * Sequence counts matched. Now make sure that the root is
	 * still valid and get it if required.
	 */
	if (nd->root.mnt && !(nd->flags & LOOKUP_ROOT)) {
756 757 758 759
		if (unlikely(!legitimize_path(nd, &nd->root, nd->root_seq))) {
			rcu_read_unlock();
			dput(dentry);
			return -ECHILD;
760
		}
N
Nick Piggin 已提交
761 762
	}

A
Al Viro 已提交
763
	rcu_read_unlock();
N
Nick Piggin 已提交
764
	return 0;
A
Al Viro 已提交
765

766 767 768 769
out2:
	nd->path.mnt = NULL;
out1:
	nd->path.dentry = NULL;
770
out:
A
Al Viro 已提交
771
	rcu_read_unlock();
772 773 774
drop_root_mnt:
	if (!(nd->flags & LOOKUP_ROOT))
		nd->root.mnt = NULL;
N
Nick Piggin 已提交
775 776 777
	return -ECHILD;
}

778
static inline int d_revalidate(struct dentry *dentry, unsigned int flags)
779
{
780 781 782 783
	if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE))
		return dentry->d_op->d_revalidate(dentry, flags);
	else
		return 1;
784 785
}

786 787 788
/**
 * complete_walk - successful completion of path walk
 * @nd:  pointer nameidata
789
 *
790 791 792 793 794
 * If we had been in RCU mode, drop out of it and legitimize nd->path.
 * Revalidate the final result, unless we'd already done that during
 * the path walk or the filesystem doesn't ask for it.  Return 0 on
 * success, -error on failure.  In case of failure caller does not
 * need to drop nd->path.
795
 */
796
static int complete_walk(struct nameidata *nd)
797
{
A
Al Viro 已提交
798
	struct dentry *dentry = nd->path.dentry;
799 800
	int status;

801 802 803
	if (nd->flags & LOOKUP_RCU) {
		if (!(nd->flags & LOOKUP_ROOT))
			nd->root.mnt = NULL;
A
Al Viro 已提交
804
		if (unlikely(unlazy_walk(nd)))
805 806 807
			return -ECHILD;
	}

A
Al Viro 已提交
808 809 810
	if (likely(!(nd->flags & LOOKUP_JUMPED)))
		return 0;

811
	if (likely(!(dentry->d_flags & DCACHE_OP_WEAK_REVALIDATE)))
812 813
		return 0;

814
	status = dentry->d_op->d_weak_revalidate(dentry, nd->flags);
815 816 817
	if (status > 0)
		return 0;

A
Al Viro 已提交
818
	if (!status)
819
		status = -ESTALE;
A
Al Viro 已提交
820

821 822 823
	return status;
}

A
Al Viro 已提交
824
static void set_root(struct nameidata *nd)
N
Nick Piggin 已提交
825
{
826
	struct fs_struct *fs = current->fs;
N
Nick Piggin 已提交
827

828 829 830 831 832 833 834 835 836 837 838
	if (nd->flags & LOOKUP_RCU) {
		unsigned seq;

		do {
			seq = read_seqcount_begin(&fs->seq);
			nd->root = fs->root;
			nd->root_seq = __read_seqcount_begin(&nd->root.dentry->d_seq);
		} while (read_seqcount_retry(&fs->seq, seq));
	} else {
		get_fs_root(fs, &nd->root);
	}
N
Nick Piggin 已提交
839 840
}

J
Jan Blunck 已提交
841
static void path_put_conditional(struct path *path, struct nameidata *nd)
842 843
{
	dput(path->dentry);
844
	if (path->mnt != nd->path.mnt)
845 846 847
		mntput(path->mnt);
}

848 849
static inline void path_to_nameidata(const struct path *path,
					struct nameidata *nd)
850
{
N
Nick Piggin 已提交
851 852 853 854
	if (!(nd->flags & LOOKUP_RCU)) {
		dput(nd->path.dentry);
		if (nd->path.mnt != path->mnt)
			mntput(nd->path.mnt);
855
	}
N
Nick Piggin 已提交
856
	nd->path.mnt = path->mnt;
857
	nd->path.dentry = path->dentry;
858 859
}

860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879
static int nd_jump_root(struct nameidata *nd)
{
	if (nd->flags & LOOKUP_RCU) {
		struct dentry *d;
		nd->path = nd->root;
		d = nd->path.dentry;
		nd->inode = d->d_inode;
		nd->seq = nd->root_seq;
		if (unlikely(read_seqcount_retry(&d->d_seq, nd->seq)))
			return -ECHILD;
	} else {
		path_put(&nd->path);
		nd->path = nd->root;
		path_get(&nd->path);
		nd->inode = nd->path.dentry->d_inode;
	}
	nd->flags |= LOOKUP_JUMPED;
	return 0;
}

C
Christoph Hellwig 已提交
880
/*
881
 * Helper to directly jump to a known parsed path from ->get_link,
C
Christoph Hellwig 已提交
882 883
 * caller must have taken a reference to path beforehand.
 */
884
void nd_jump_link(struct path *path)
C
Christoph Hellwig 已提交
885
{
886
	struct nameidata *nd = current->nameidata;
C
Christoph Hellwig 已提交
887 888 889 890 891 892 893
	path_put(&nd->path);

	nd->path = *path;
	nd->inode = nd->path.dentry->d_inode;
	nd->flags |= LOOKUP_JUMPED;
}

894
static inline void put_link(struct nameidata *nd)
895
{
A
Al Viro 已提交
896
	struct saved *last = nd->stack + --nd->depth;
897
	do_delayed_call(&last->done);
A
Al Viro 已提交
898 899
	if (!(nd->flags & LOOKUP_RCU))
		path_put(&last->link);
900 901
}

902 903
int sysctl_protected_symlinks __read_mostly = 0;
int sysctl_protected_hardlinks __read_mostly = 0;
K
Kees Cook 已提交
904 905 906

/**
 * may_follow_link - Check symlink following for unsafe situations
907
 * @nd: nameidata pathwalk data
K
Kees Cook 已提交
908 909 910 911 912 913 914 915 916 917 918 919
 *
 * In the case of the sysctl_protected_symlinks sysctl being enabled,
 * CAP_DAC_OVERRIDE needs to be specifically ignored if the symlink is
 * in a sticky world-writable directory. This is to protect privileged
 * processes from failing races against path names that may change out
 * from under them by way of other users creating malicious symlinks.
 * It will permit symlinks to be followed only when outside a sticky
 * world-writable directory, or when the uid of the symlink and follower
 * match, or when the directory owner matches the symlink's owner.
 *
 * Returns 0 if following the symlink is allowed, -ve on error.
 */
A
Al Viro 已提交
920
static inline int may_follow_link(struct nameidata *nd)
K
Kees Cook 已提交
921 922 923
{
	const struct inode *inode;
	const struct inode *parent;
924
	kuid_t puid;
K
Kees Cook 已提交
925 926 927 928 929

	if (!sysctl_protected_symlinks)
		return 0;

	/* Allowed if owner and follower match. */
930
	inode = nd->link_inode;
931
	if (uid_eq(current_cred()->fsuid, inode->i_uid))
K
Kees Cook 已提交
932 933 934
		return 0;

	/* Allowed if parent directory not sticky and world-writable. */
A
Al Viro 已提交
935
	parent = nd->inode;
K
Kees Cook 已提交
936 937 938 939
	if ((parent->i_mode & (S_ISVTX|S_IWOTH)) != (S_ISVTX|S_IWOTH))
		return 0;

	/* Allowed if parent directory and link owner match. */
940 941
	puid = parent->i_uid;
	if (uid_valid(puid) && uid_eq(puid, inode->i_uid))
K
Kees Cook 已提交
942 943
		return 0;

944 945 946
	if (nd->flags & LOOKUP_RCU)
		return -ECHILD;

A
Al Viro 已提交
947
	audit_log_link_denied("follow_link", &nd->stack[0].link);
K
Kees Cook 已提交
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993
	return -EACCES;
}

/**
 * safe_hardlink_source - Check for safe hardlink conditions
 * @inode: the source inode to hardlink from
 *
 * Return false if at least one of the following conditions:
 *    - inode is not a regular file
 *    - inode is setuid
 *    - inode is setgid and group-exec
 *    - access failure for read and write
 *
 * Otherwise returns true.
 */
static bool safe_hardlink_source(struct inode *inode)
{
	umode_t mode = inode->i_mode;

	/* Special files should not get pinned to the filesystem. */
	if (!S_ISREG(mode))
		return false;

	/* Setuid files should not get pinned to the filesystem. */
	if (mode & S_ISUID)
		return false;

	/* Executable setgid files should not get pinned to the filesystem. */
	if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP))
		return false;

	/* Hardlinking to unreadable or unwritable sources is dangerous. */
	if (inode_permission(inode, MAY_READ | MAY_WRITE))
		return false;

	return true;
}

/**
 * may_linkat - Check permissions for creating a hardlink
 * @link: the source to hardlink from
 *
 * Block hardlink when all of:
 *  - sysctl_protected_hardlinks enabled
 *  - fsuid does not match inode
 *  - hardlink source is unsafe (see safe_hardlink_source() above)
994
 *  - not CAP_FOWNER in a namespace with the inode owner uid mapped
K
Kees Cook 已提交
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
 *
 * Returns 0 if successful, -ve on error.
 */
static int may_linkat(struct path *link)
{
	struct inode *inode;

	if (!sysctl_protected_hardlinks)
		return 0;

	inode = link->dentry->d_inode;

	/* Source inode owner (or CAP_FOWNER) can hardlink all they like,
	 * otherwise, it must be a safe source.
	 */
1010
	if (safe_hardlink_source(inode) || inode_owner_or_capable(inode))
K
Kees Cook 已提交
1011 1012
		return 0;

1013
	audit_log_link_denied("linkat", link);
K
Kees Cook 已提交
1014 1015 1016
	return -EPERM;
}

1017 1018
static __always_inline
const char *get_link(struct nameidata *nd)
L
Linus Torvalds 已提交
1019
{
1020
	struct saved *last = nd->stack + nd->depth - 1;
A
Al Viro 已提交
1021
	struct dentry *dentry = last->link.dentry;
1022
	struct inode *inode = nd->link_inode;
1023
	int error;
1024
	const char *res;
L
Linus Torvalds 已提交
1025

1026 1027 1028
	if (!(nd->flags & LOOKUP_RCU)) {
		touch_atime(&last->link);
		cond_resched();
1029
	} else if (atime_needs_update_rcu(&last->link, inode)) {
A
Al Viro 已提交
1030
		if (unlikely(unlazy_walk(nd)))
A
Al Viro 已提交
1031
			return ERR_PTR(-ECHILD);
1032
		touch_atime(&last->link);
A
Al Viro 已提交
1033
	}
1034

1035 1036 1037
	error = security_inode_follow_link(dentry, inode,
					   nd->flags & LOOKUP_RCU);
	if (unlikely(error))
1038
		return ERR_PTR(error);
1039

1040
	nd->last_type = LAST_BIND;
1041 1042
	res = inode->i_link;
	if (!res) {
1043 1044 1045
		const char * (*get)(struct dentry *, struct inode *,
				struct delayed_call *);
		get = inode->i_op->get_link;
1046
		if (nd->flags & LOOKUP_RCU) {
1047
			res = get(NULL, inode, &last->done);
1048
			if (res == ERR_PTR(-ECHILD)) {
A
Al Viro 已提交
1049
				if (unlikely(unlazy_walk(nd)))
1050
					return ERR_PTR(-ECHILD);
1051
				res = get(dentry, inode, &last->done);
1052 1053
			}
		} else {
1054
			res = get(dentry, inode, &last->done);
1055
		}
1056
		if (IS_ERR_OR_NULL(res))
1057 1058 1059
			return res;
	}
	if (*res == '/') {
1060 1061
		if (!nd->root.mnt)
			set_root(nd);
1062 1063
		if (unlikely(nd_jump_root(nd)))
			return ERR_PTR(-ECHILD);
1064 1065
		while (unlikely(*++res == '/'))
			;
L
Linus Torvalds 已提交
1066
	}
1067 1068
	if (!*res)
		res = NULL;
1069 1070
	return res;
}
1071

1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
/*
 * follow_up - Find the mountpoint of path's vfsmount
 *
 * Given a path, find the mountpoint of its source file system.
 * Replace @path with the path of the mountpoint in the parent mount.
 * Up is towards /.
 *
 * Return 1 if we went up a level and 0 if we were already at the
 * root.
 */
A
Al Viro 已提交
1082
int follow_up(struct path *path)
L
Linus Torvalds 已提交
1083
{
1084 1085
	struct mount *mnt = real_mount(path->mnt);
	struct mount *parent;
L
Linus Torvalds 已提交
1086
	struct dentry *mountpoint;
N
Nick Piggin 已提交
1087

A
Al Viro 已提交
1088
	read_seqlock_excl(&mount_lock);
1089
	parent = mnt->mnt_parent;
A
Al Viro 已提交
1090
	if (parent == mnt) {
A
Al Viro 已提交
1091
		read_sequnlock_excl(&mount_lock);
L
Linus Torvalds 已提交
1092 1093
		return 0;
	}
1094
	mntget(&parent->mnt);
1095
	mountpoint = dget(mnt->mnt_mountpoint);
A
Al Viro 已提交
1096
	read_sequnlock_excl(&mount_lock);
A
Al Viro 已提交
1097 1098 1099
	dput(path->dentry);
	path->dentry = mountpoint;
	mntput(path->mnt);
1100
	path->mnt = &parent->mnt;
L
Linus Torvalds 已提交
1101 1102
	return 1;
}
1103
EXPORT_SYMBOL(follow_up);
L
Linus Torvalds 已提交
1104

N
Nick Piggin 已提交
1105
/*
1106 1107 1108
 * Perform an automount
 * - return -EISDIR to tell follow_managed() to stop and return the path we
 *   were called with.
L
Linus Torvalds 已提交
1109
 */
1110
static int follow_automount(struct path *path, struct nameidata *nd,
1111
			    bool *need_mntput)
N
Nick Piggin 已提交
1112
{
1113
	struct vfsmount *mnt;
1114
	int err;
1115 1116 1117 1118

	if (!path->dentry->d_op || !path->dentry->d_op->d_automount)
		return -EREMOTE;

1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
	/* We don't want to mount if someone's just doing a stat -
	 * unless they're stat'ing a directory and appended a '/' to
	 * the name.
	 *
	 * We do, however, want to mount if someone wants to open or
	 * create a file of any type under the mountpoint, wants to
	 * traverse through the mountpoint or wants to open the
	 * mounted directory.  Also, autofs may mark negative dentries
	 * as being automount points.  These will need the attentions
	 * of the daemon to instantiate them before they can be used.
1129
	 */
1130
	if (!(nd->flags & (LOOKUP_PARENT | LOOKUP_DIRECTORY |
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
			   LOOKUP_OPEN | LOOKUP_CREATE |
			   LOOKUP_AUTOMOUNT))) {
		/* Positive dentry that isn't meant to trigger an
		 * automount, EISDIR will allow it to be used,
		 * otherwise there's no mount here "now" so return
		 * ENOENT.
		 */
		if (path->dentry->d_inode)
			return -EISDIR;
		else
			return -ENOENT;
	}
1143

1144 1145 1146
	if (path->dentry->d_sb->s_user_ns != &init_user_ns)
		return -EACCES;

1147 1148
	nd->total_link_count++;
	if (nd->total_link_count >= 40)
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
		return -ELOOP;

	mnt = path->dentry->d_op->d_automount(path);
	if (IS_ERR(mnt)) {
		/*
		 * The filesystem is allowed to return -EISDIR here to indicate
		 * it doesn't want to automount.  For instance, autofs would do
		 * this so that its userspace daemon can mount on this dentry.
		 *
		 * However, we can only permit this if it's a terminal point in
		 * the path being looked up; if it wasn't then the remainder of
		 * the path is inaccessible and we should say so.
		 */
1162
		if (PTR_ERR(mnt) == -EISDIR && (nd->flags & LOOKUP_PARENT))
1163 1164
			return -EREMOTE;
		return PTR_ERR(mnt);
N
Nick Piggin 已提交
1165
	}
1166

1167 1168
	if (!mnt) /* mount collision */
		return 0;
N
Nick Piggin 已提交
1169

1170 1171 1172 1173 1174
	if (!*need_mntput) {
		/* lock_mount() may release path->mnt on error */
		mntget(path->mnt);
		*need_mntput = true;
	}
1175
	err = finish_automount(mnt, path);
1176

1177 1178 1179
	switch (err) {
	case -EBUSY:
		/* Someone else made a mount here whilst we were busy */
1180
		return 0;
1181
	case 0:
1182
		path_put(path);
1183 1184 1185
		path->mnt = mnt;
		path->dentry = dget(mnt->mnt_root);
		return 0;
1186 1187
	default:
		return err;
1188
	}
1189

A
Al Viro 已提交
1190 1191
}

1192 1193
/*
 * Handle a dentry that is managed in some way.
1194
 * - Flagged for transit management (autofs)
1195 1196 1197 1198 1199 1200 1201
 * - Flagged as mountpoint
 * - Flagged as automount point
 *
 * This may only be called in refwalk mode.
 *
 * Serialization is taken care of in namespace.c
 */
1202
static int follow_managed(struct path *path, struct nameidata *nd)
L
Linus Torvalds 已提交
1203
{
1204
	struct vfsmount *mnt = path->mnt; /* held by caller, must be left alone */
1205 1206
	unsigned managed;
	bool need_mntput = false;
1207
	int ret = 0;
1208 1209 1210 1211 1212 1213 1214

	/* Given that we're not holding a lock here, we retain the value in a
	 * local variable for each dentry as we look at it so that we don't see
	 * the components of that value change under us */
	while (managed = ACCESS_ONCE(path->dentry->d_flags),
	       managed &= DCACHE_MANAGED_DENTRY,
	       unlikely(managed != 0)) {
1215 1216 1217 1218 1219
		/* Allow the filesystem to manage the transit without i_mutex
		 * being held. */
		if (managed & DCACHE_MANAGE_TRANSIT) {
			BUG_ON(!path->dentry->d_op);
			BUG_ON(!path->dentry->d_op->d_manage);
1220
			ret = path->dentry->d_op->d_manage(path, false);
1221
			if (ret < 0)
1222
				break;
1223 1224
		}

1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
		/* Transit to a mounted filesystem. */
		if (managed & DCACHE_MOUNTED) {
			struct vfsmount *mounted = lookup_mnt(path);
			if (mounted) {
				dput(path->dentry);
				if (need_mntput)
					mntput(path->mnt);
				path->mnt = mounted;
				path->dentry = dget(mounted->mnt_root);
				need_mntput = true;
				continue;
			}

			/* Something is mounted on this dentry in another
			 * namespace and/or whatever was mounted there in this
A
Al Viro 已提交
1240 1241
			 * namespace got unmounted before lookup_mnt() could
			 * get it */
1242 1243 1244 1245
		}

		/* Handle an automount point */
		if (managed & DCACHE_NEED_AUTOMOUNT) {
1246
			ret = follow_automount(path, nd, &need_mntput);
1247
			if (ret < 0)
1248
				break;
1249 1250 1251 1252 1253
			continue;
		}

		/* We didn't change the current path point */
		break;
L
Linus Torvalds 已提交
1254
	}
1255 1256 1257

	if (need_mntput && path->mnt == mnt)
		mntput(path->mnt);
1258 1259
	if (ret == -EISDIR || !ret)
		ret = 1;
1260 1261 1262 1263 1264
	if (need_mntput)
		nd->flags |= LOOKUP_JUMPED;
	if (unlikely(ret < 0))
		path_put_conditional(path, nd);
	return ret;
L
Linus Torvalds 已提交
1265 1266
}

1267
int follow_down_one(struct path *path)
L
Linus Torvalds 已提交
1268 1269 1270
{
	struct vfsmount *mounted;

A
Al Viro 已提交
1271
	mounted = lookup_mnt(path);
L
Linus Torvalds 已提交
1272
	if (mounted) {
A
Al Viro 已提交
1273 1274 1275 1276
		dput(path->dentry);
		mntput(path->mnt);
		path->mnt = mounted;
		path->dentry = dget(mounted->mnt_root);
L
Linus Torvalds 已提交
1277 1278 1279 1280
		return 1;
	}
	return 0;
}
1281
EXPORT_SYMBOL(follow_down_one);
L
Linus Torvalds 已提交
1282

1283
static inline int managed_dentry_rcu(const struct path *path)
1284
{
1285 1286
	return (path->dentry->d_flags & DCACHE_MANAGE_TRANSIT) ?
		path->dentry->d_op->d_manage(path, true) : 0;
1287 1288
}

1289
/*
1290 1291
 * Try to skip to top of mountpoint pile in rcuwalk mode.  Fail if
 * we meet a managed dentry that would need blocking.
1292 1293
 */
static bool __follow_mount_rcu(struct nameidata *nd, struct path *path,
1294
			       struct inode **inode, unsigned *seqp)
1295
{
1296
	for (;;) {
1297
		struct mount *mounted;
1298 1299 1300 1301
		/*
		 * Don't forget we might have a non-mountpoint managed dentry
		 * that wants to block transit.
		 */
1302
		switch (managed_dentry_rcu(path)) {
1303 1304
		case -ECHILD:
		default:
1305
			return false;
1306 1307 1308 1309 1310
		case -EISDIR:
			return true;
		case 0:
			break;
		}
1311 1312

		if (!d_mountpoint(path->dentry))
1313
			return !(path->dentry->d_flags & DCACHE_NEED_AUTOMOUNT);
1314

A
Al Viro 已提交
1315
		mounted = __lookup_mnt(path->mnt, path->dentry);
1316 1317
		if (!mounted)
			break;
1318 1319
		path->mnt = &mounted->mnt;
		path->dentry = mounted->mnt.mnt_root;
1320
		nd->flags |= LOOKUP_JUMPED;
1321
		*seqp = read_seqcount_begin(&path->dentry->d_seq);
1322 1323 1324 1325 1326 1327
		/*
		 * Update the inode too. We don't need to re-check the
		 * dentry sequence number here after this d_inode read,
		 * because a mount-point is always pinned.
		 */
		*inode = path->dentry->d_inode;
1328
	}
1329
	return !read_seqretry(&mount_lock, nd->m_seq) &&
1330
		!(path->dentry->d_flags & DCACHE_NEED_AUTOMOUNT);
1331 1332
}

N
Nick Piggin 已提交
1333 1334
static int follow_dotdot_rcu(struct nameidata *nd)
{
1335
	struct inode *inode = nd->inode;
N
Nick Piggin 已提交
1336

1337
	while (1) {
1338
		if (path_equal(&nd->path, &nd->root))
N
Nick Piggin 已提交
1339 1340 1341 1342 1343 1344
			break;
		if (nd->path.dentry != nd->path.mnt->mnt_root) {
			struct dentry *old = nd->path.dentry;
			struct dentry *parent = old->d_parent;
			unsigned seq;

1345
			inode = parent->d_inode;
N
Nick Piggin 已提交
1346
			seq = read_seqcount_begin(&parent->d_seq);
1347 1348
			if (unlikely(read_seqcount_retry(&old->d_seq, nd->seq)))
				return -ECHILD;
N
Nick Piggin 已提交
1349 1350
			nd->path.dentry = parent;
			nd->seq = seq;
1351 1352
			if (unlikely(!path_connected(&nd->path)))
				return -ENOENT;
N
Nick Piggin 已提交
1353
			break;
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
		} else {
			struct mount *mnt = real_mount(nd->path.mnt);
			struct mount *mparent = mnt->mnt_parent;
			struct dentry *mountpoint = mnt->mnt_mountpoint;
			struct inode *inode2 = mountpoint->d_inode;
			unsigned seq = read_seqcount_begin(&mountpoint->d_seq);
			if (unlikely(read_seqretry(&mount_lock, nd->m_seq)))
				return -ECHILD;
			if (&mparent->mnt == nd->path.mnt)
				break;
			/* we know that mountpoint was pinned */
			nd->path.dentry = mountpoint;
			nd->path.mnt = &mparent->mnt;
			inode = inode2;
			nd->seq = seq;
N
Nick Piggin 已提交
1369 1370
		}
	}
1371
	while (unlikely(d_mountpoint(nd->path.dentry))) {
1372 1373
		struct mount *mounted;
		mounted = __lookup_mnt(nd->path.mnt, nd->path.dentry);
1374 1375
		if (unlikely(read_seqretry(&mount_lock, nd->m_seq)))
			return -ECHILD;
1376 1377 1378 1379
		if (!mounted)
			break;
		nd->path.mnt = &mounted->mnt;
		nd->path.dentry = mounted->mnt.mnt_root;
1380
		inode = nd->path.dentry->d_inode;
1381 1382
		nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
	}
1383
	nd->inode = inode;
N
Nick Piggin 已提交
1384 1385 1386
	return 0;
}

1387 1388 1389 1390 1391
/*
 * Follow down to the covering mount currently visible to userspace.  At each
 * point, the filesystem owning that dentry may be queried as to whether the
 * caller is permitted to proceed or not.
 */
1392
int follow_down(struct path *path)
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
{
	unsigned managed;
	int ret;

	while (managed = ACCESS_ONCE(path->dentry->d_flags),
	       unlikely(managed & DCACHE_MANAGED_DENTRY)) {
		/* Allow the filesystem to manage the transit without i_mutex
		 * being held.
		 *
		 * We indicate to the filesystem if someone is trying to mount
		 * something here.  This gives autofs the chance to deny anyone
		 * other than its daemon the right to mount on its
		 * superstructure.
		 *
		 * The filesystem may sleep at this point.
		 */
		if (managed & DCACHE_MANAGE_TRANSIT) {
			BUG_ON(!path->dentry->d_op);
			BUG_ON(!path->dentry->d_op->d_manage);
1412
			ret = path->dentry->d_op->d_manage(path, false);
1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
			if (ret < 0)
				return ret == -EISDIR ? 0 : ret;
		}

		/* Transit to a mounted filesystem. */
		if (managed & DCACHE_MOUNTED) {
			struct vfsmount *mounted = lookup_mnt(path);
			if (!mounted)
				break;
			dput(path->dentry);
			mntput(path->mnt);
			path->mnt = mounted;
			path->dentry = dget(mounted->mnt_root);
			continue;
		}

		/* Don't handle automount points here */
		break;
	}
	return 0;
}
1434
EXPORT_SYMBOL(follow_down);
1435

1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
/*
 * Skip to top of mountpoint pile in refwalk mode for follow_dotdot()
 */
static void follow_mount(struct path *path)
{
	while (d_mountpoint(path->dentry)) {
		struct vfsmount *mounted = lookup_mnt(path);
		if (!mounted)
			break;
		dput(path->dentry);
		mntput(path->mnt);
		path->mnt = mounted;
		path->dentry = dget(mounted->mnt_root);
	}
}

1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
static int path_parent_directory(struct path *path)
{
	struct dentry *old = path->dentry;
	/* rare case of legitimate dget_parent()... */
	path->dentry = dget_parent(path->dentry);
	dput(old);
	if (unlikely(!path_connected(path)))
		return -ENOENT;
	return 0;
}

1463
static int follow_dotdot(struct nameidata *nd)
L
Linus Torvalds 已提交
1464 1465
{
	while(1) {
A
Al Viro 已提交
1466 1467
		if (nd->path.dentry == nd->root.dentry &&
		    nd->path.mnt == nd->root.mnt) {
L
Linus Torvalds 已提交
1468 1469
			break;
		}
1470
		if (nd->path.dentry != nd->path.mnt->mnt_root) {
1471 1472 1473
			int ret = path_parent_directory(&nd->path);
			if (ret)
				return ret;
L
Linus Torvalds 已提交
1474 1475
			break;
		}
A
Al Viro 已提交
1476
		if (!follow_up(&nd->path))
L
Linus Torvalds 已提交
1477 1478
			break;
	}
A
Al Viro 已提交
1479
	follow_mount(&nd->path);
N
Nick Piggin 已提交
1480
	nd->inode = nd->path.dentry->d_inode;
1481
	return 0;
L
Linus Torvalds 已提交
1482 1483
}

1484
/*
1485 1486
 * This looks up the name in dcache and possibly revalidates the found dentry.
 * NULL is returned if the dentry does not exist in the cache.
1487
 */
1488 1489
static struct dentry *lookup_dcache(const struct qstr *name,
				    struct dentry *dir,
1490
				    unsigned int flags)
1491
{
1492
	struct dentry *dentry = d_lookup(dir, name);
M
Miklos Szeredi 已提交
1493
	if (dentry) {
1494 1495 1496 1497 1498 1499
		int error = d_revalidate(dentry, flags);
		if (unlikely(error <= 0)) {
			if (!error)
				d_invalidate(dentry);
			dput(dentry);
			return ERR_PTR(error);
M
Miklos Szeredi 已提交
1500 1501
		}
	}
1502 1503 1504
	return dentry;
}

1505
/*
1506 1507
 * Call i_op->lookup on the dentry.  The dentry must be negative and
 * unhashed.
M
Miklos Szeredi 已提交
1508 1509
 *
 * dir->d_inode->i_mutex must be held
1510
 */
M
Miklos Szeredi 已提交
1511
static struct dentry *lookup_real(struct inode *dir, struct dentry *dentry,
1512
				  unsigned int flags)
1513 1514 1515 1516
{
	struct dentry *old;

	/* Don't create child dentry for a dead directory. */
M
Miklos Szeredi 已提交
1517
	if (unlikely(IS_DEADDIR(dir))) {
1518
		dput(dentry);
1519
		return ERR_PTR(-ENOENT);
1520
	}
1521

1522
	old = dir->i_op->lookup(dir, dentry, flags);
1523 1524 1525 1526 1527 1528 1529
	if (unlikely(old)) {
		dput(dentry);
		dentry = old;
	}
	return dentry;
}

1530
static struct dentry *__lookup_hash(const struct qstr *name,
1531
		struct dentry *base, unsigned int flags)
1532
{
1533
	struct dentry *dentry = lookup_dcache(name, base, flags);
1534

1535
	if (dentry)
M
Miklos Szeredi 已提交
1536
		return dentry;
1537

1538 1539 1540 1541
	dentry = d_alloc(base, name);
	if (unlikely(!dentry))
		return ERR_PTR(-ENOMEM);

1542
	return lookup_real(base->d_inode, dentry, flags);
1543 1544
}

A
Al Viro 已提交
1545
static int lookup_fast(struct nameidata *nd,
1546 1547
		       struct path *path, struct inode **inode,
		       unsigned *seqp)
L
Linus Torvalds 已提交
1548
{
1549
	struct vfsmount *mnt = nd->path.mnt;
N
Nick Piggin 已提交
1550
	struct dentry *dentry, *parent = nd->path.dentry;
A
Al Viro 已提交
1551
	int status = 1;
1552 1553
	int err;

1554 1555
	/*
	 * Rename seqlock is not required here because in the off chance
A
Al Viro 已提交
1556 1557
	 * of a false negative due to a concurrent rename, the caller is
	 * going to fall back to non-racy lookup.
1558
	 */
N
Nick Piggin 已提交
1559 1560
	if (nd->flags & LOOKUP_RCU) {
		unsigned seq;
1561
		bool negative;
1562
		dentry = __d_lookup_rcu(parent, &nd->last, &seq);
A
Al Viro 已提交
1563
		if (unlikely(!dentry)) {
A
Al Viro 已提交
1564
			if (unlazy_walk(nd))
A
Al Viro 已提交
1565
				return -ECHILD;
1566
			return 0;
A
Al Viro 已提交
1567
		}
A
Al Viro 已提交
1568

1569 1570 1571 1572
		/*
		 * This sequence count validates that the inode matches
		 * the dentry name information from lookup.
		 */
1573
		*inode = d_backing_inode(dentry);
1574
		negative = d_is_negative(dentry);
A
Al Viro 已提交
1575
		if (unlikely(read_seqcount_retry(&dentry->d_seq, seq)))
1576 1577 1578 1579 1580 1581 1582 1583 1584
			return -ECHILD;

		/*
		 * This sequence count validates that the parent had no
		 * changes while we did the lookup of the dentry above.
		 *
		 * The memory barrier in read_seqcount_begin of child is
		 *  enough, we can use __read_seqcount_retry here.
		 */
A
Al Viro 已提交
1585
		if (unlikely(__read_seqcount_retry(&parent->d_seq, nd->seq)))
N
Nick Piggin 已提交
1586
			return -ECHILD;
A
Al Viro 已提交
1587

1588
		*seqp = seq;
1589
		status = d_revalidate(dentry, nd->flags);
1590
		if (likely(status > 0)) {
A
Al Viro 已提交
1591 1592 1593 1594 1595 1596 1597 1598 1599
			/*
			 * Note: do negative dentry check after revalidation in
			 * case that drops it.
			 */
			if (unlikely(negative))
				return -ENOENT;
			path->mnt = mnt;
			path->dentry = dentry;
			if (likely(__follow_mount_rcu(nd, path, inode, seqp)))
1600
				return 1;
1601
		}
A
Al Viro 已提交
1602
		if (unlazy_child(nd, dentry, seq))
1603 1604 1605 1606
			return -ECHILD;
		if (unlikely(status == -ECHILD))
			/* we'd been told to redo it in non-rcu mode */
			status = d_revalidate(dentry, nd->flags);
A
Al Viro 已提交
1607
	} else {
A
Al Viro 已提交
1608
		dentry = __d_lookup(parent, &nd->last);
A
Al Viro 已提交
1609
		if (unlikely(!dentry))
1610
			return 0;
1611
		status = d_revalidate(dentry, nd->flags);
1612
	}
A
Al Viro 已提交
1613
	if (unlikely(status <= 0)) {
1614
		if (!status)
A
Al Viro 已提交
1615
			d_invalidate(dentry);
1616
		dput(dentry);
A
Al Viro 已提交
1617
		return status;
1618
	}
1619 1620 1621 1622
	if (unlikely(d_is_negative(dentry))) {
		dput(dentry);
		return -ENOENT;
	}
A
Al Viro 已提交
1623

1624 1625
	path->mnt = mnt;
	path->dentry = dentry;
1626
	err = follow_managed(path, nd);
1627
	if (likely(err > 0))
1628
		*inode = d_backing_inode(path->dentry);
1629
	return err;
M
Miklos Szeredi 已提交
1630 1631 1632
}

/* Fast lookup failed, do it the slow way */
1633 1634 1635
static struct dentry *lookup_slow(const struct qstr *name,
				  struct dentry *dir,
				  unsigned int flags)
M
Miklos Szeredi 已提交
1636
{
A
Al Viro 已提交
1637
	struct dentry *dentry = ERR_PTR(-ENOENT), *old;
1638
	struct inode *inode = dir->d_inode;
1639
	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
1640

1641
	inode_lock_shared(inode);
1642
	/* Don't go there if it's already dead */
A
Al Viro 已提交
1643 1644 1645
	if (unlikely(IS_DEADDIR(inode)))
		goto out;
again:
1646
	dentry = d_alloc_parallel(dir, name, &wq);
A
Al Viro 已提交
1647 1648 1649
	if (IS_ERR(dentry))
		goto out;
	if (unlikely(!d_in_lookup(dentry))) {
1650
		if (!(flags & LOOKUP_NO_REVAL)) {
1651 1652
			int error = d_revalidate(dentry, flags);
			if (unlikely(error <= 0)) {
A
Al Viro 已提交
1653
				if (!error) {
1654
					d_invalidate(dentry);
A
Al Viro 已提交
1655 1656 1657
					dput(dentry);
					goto again;
				}
1658 1659 1660 1661
				dput(dentry);
				dentry = ERR_PTR(error);
			}
		}
A
Al Viro 已提交
1662 1663 1664 1665 1666 1667
	} else {
		old = inode->i_op->lookup(inode, dentry, flags);
		d_lookup_done(dentry);
		if (unlikely(old)) {
			dput(dentry);
			dentry = old;
1668 1669
		}
	}
A
Al Viro 已提交
1670
out:
1671
	inode_unlock_shared(inode);
1672
	return dentry;
L
Linus Torvalds 已提交
1673 1674
}

1675 1676 1677
static inline int may_lookup(struct nameidata *nd)
{
	if (nd->flags & LOOKUP_RCU) {
1678
		int err = inode_permission(nd->inode, MAY_EXEC|MAY_NOT_BLOCK);
1679 1680
		if (err != -ECHILD)
			return err;
A
Al Viro 已提交
1681
		if (unlazy_walk(nd))
1682 1683
			return -ECHILD;
	}
1684
	return inode_permission(nd->inode, MAY_EXEC);
1685 1686
}

1687 1688 1689
static inline int handle_dots(struct nameidata *nd, int type)
{
	if (type == LAST_DOTDOT) {
1690 1691
		if (!nd->root.mnt)
			set_root(nd);
1692
		if (nd->flags & LOOKUP_RCU) {
1693
			return follow_dotdot_rcu(nd);
1694
		} else
1695
			return follow_dotdot(nd);
1696 1697 1698 1699
	}
	return 0;
}

1700 1701
static int pick_link(struct nameidata *nd, struct path *link,
		     struct inode *inode, unsigned seq)
1702
{
1703
	int error;
A
Al Viro 已提交
1704
	struct saved *last;
1705
	if (unlikely(nd->total_link_count++ >= MAXSYMLINKS)) {
1706 1707 1708
		path_to_nameidata(link, nd);
		return -ELOOP;
	}
A
Al Viro 已提交
1709
	if (!(nd->flags & LOOKUP_RCU)) {
1710 1711
		if (link->mnt == nd->path.mnt)
			mntget(link->mnt);
1712
	}
1713 1714
	error = nd_alloc_stack(nd);
	if (unlikely(error)) {
A
Al Viro 已提交
1715
		if (error == -ECHILD) {
1716 1717 1718 1719 1720 1721 1722 1723 1724
			if (unlikely(!legitimize_path(nd, link, seq))) {
				drop_links(nd);
				nd->depth = 0;
				nd->flags &= ~LOOKUP_RCU;
				nd->path.mnt = NULL;
				nd->path.dentry = NULL;
				if (!(nd->flags & LOOKUP_ROOT))
					nd->root.mnt = NULL;
				rcu_read_unlock();
A
Al Viro 已提交
1725
			} else if (likely(unlazy_walk(nd)) == 0)
1726
				error = nd_alloc_stack(nd);
A
Al Viro 已提交
1727 1728 1729 1730 1731
		}
		if (error) {
			path_put(link);
			return error;
		}
1732 1733
	}

1734
	last = nd->stack + nd->depth++;
A
Al Viro 已提交
1735
	last->link = *link;
1736 1737
	clear_delayed_call(&last->done);
	nd->link_inode = inode;
1738
	last->seq = seq;
1739 1740 1741
	return 1;
}

1742
enum {WALK_FOLLOW = 1, WALK_MORE = 2};
1743

1744 1745 1746 1747 1748 1749
/*
 * Do we need to follow links? We _really_ want to be able
 * to do this check without having to look at inode->i_op,
 * so we keep a cache of "no, this doesn't need follow_link"
 * for the common case.
 */
1750 1751
static inline int step_into(struct nameidata *nd, struct path *path,
			    int flags, struct inode *inode, unsigned seq)
1752
{
1753 1754
	if (!(flags & WALK_MORE) && nd->depth)
		put_link(nd);
1755 1756 1757 1758 1759 1760
	if (likely(!d_is_symlink(path->dentry)) ||
	   !(flags & WALK_FOLLOW || nd->flags & LOOKUP_FOLLOW)) {
		/* not a symlink or should not follow */
		path_to_nameidata(path, nd);
		nd->inode = inode;
		nd->seq = seq;
1761
		return 0;
1762
	}
1763 1764
	/* make sure that d_is_symlink above matches inode */
	if (nd->flags & LOOKUP_RCU) {
1765
		if (read_seqcount_retry(&path->dentry->d_seq, seq))
1766 1767
			return -ECHILD;
	}
1768
	return pick_link(nd, path, inode, seq);
1769 1770
}

1771
static int walk_component(struct nameidata *nd, int flags)
1772
{
A
Al Viro 已提交
1773
	struct path path;
1774
	struct inode *inode;
1775
	unsigned seq;
1776 1777 1778 1779 1780 1781
	int err;
	/*
	 * "." and ".." are special - ".." especially so because it has
	 * to be able to know about the current root directory and
	 * parent relationships.
	 */
1782 1783
	if (unlikely(nd->last_type != LAST_NORM)) {
		err = handle_dots(nd, nd->last_type);
A
Al Viro 已提交
1784
		if (!(flags & WALK_MORE) && nd->depth)
1785 1786 1787
			put_link(nd);
		return err;
	}
1788
	err = lookup_fast(nd, &path, &inode, &seq);
1789
	if (unlikely(err <= 0)) {
M
Miklos Szeredi 已提交
1790
		if (err < 0)
1791
			return err;
1792 1793 1794 1795
		path.dentry = lookup_slow(&nd->last, nd->path.dentry,
					  nd->flags);
		if (IS_ERR(path.dentry))
			return PTR_ERR(path.dentry);
1796

1797 1798 1799
		path.mnt = nd->path.mnt;
		err = follow_managed(&path, nd);
		if (unlikely(err < 0))
1800
			return err;
M
Miklos Szeredi 已提交
1801

1802 1803 1804 1805 1806
		if (unlikely(d_is_negative(path.dentry))) {
			path_to_nameidata(&path, nd);
			return -ENOENT;
		}

1807
		seq = 0;	/* we are already out of RCU mode */
1808
		inode = d_backing_inode(path.dentry);
1809
	}
M
Miklos Szeredi 已提交
1810

1811
	return step_into(nd, &path, flags, inode, seq);
1812 1813
}

1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
/*
 * We can do the critical dentry name comparison and hashing
 * operations one word at a time, but we are limited to:
 *
 * - Architectures with fast unaligned word accesses. We could
 *   do a "get_unaligned()" if this helps and is sufficiently
 *   fast.
 *
 * - non-CONFIG_DEBUG_PAGEALLOC configurations (so that we
 *   do not trap on the (extremely unlikely) case of a page
 *   crossing operation.
 *
 * - Furthermore, we need an efficient 64-bit compile for the
 *   64-bit case in order to generate the "number of bytes in
 *   the final mask". Again, that could be replaced with a
 *   efficient population count instruction or similar.
 */
#ifdef CONFIG_DCACHE_WORD_ACCESS

1833
#include <asm/word-at-a-time.h>
1834

1835
#ifdef HASH_MIX
1836

1837
/* Architecture provides HASH_MIX and fold_hash() in <asm/hash.h> */
1838

1839
#elif defined(CONFIG_64BIT)
1840
/*
1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867
 * Register pressure in the mixing function is an issue, particularly
 * on 32-bit x86, but almost any function requires one state value and
 * one temporary.  Instead, use a function designed for two state values
 * and no temporaries.
 *
 * This function cannot create a collision in only two iterations, so
 * we have two iterations to achieve avalanche.  In those two iterations,
 * we have six layers of mixing, which is enough to spread one bit's
 * influence out to 2^6 = 64 state bits.
 *
 * Rotate constants are scored by considering either 64 one-bit input
 * deltas or 64*63/2 = 2016 two-bit input deltas, and finding the
 * probability of that delta causing a change to each of the 128 output
 * bits, using a sample of random initial states.
 *
 * The Shannon entropy of the computed probabilities is then summed
 * to produce a score.  Ideally, any input change has a 50% chance of
 * toggling any given output bit.
 *
 * Mixing scores (in bits) for (12,45):
 * Input delta: 1-bit      2-bit
 * 1 round:     713.3    42542.6
 * 2 rounds:   2753.7   140389.8
 * 3 rounds:   5954.1   233458.2
 * 4 rounds:   7862.6   256672.2
 * Perfect:    8192     258048
 *            (64*128) (64*63/2 * 128)
1868
 */
1869 1870 1871 1872 1873
#define HASH_MIX(x, y, a)	\
	(	x ^= (a),	\
	y ^= x,	x = rol64(x,12),\
	x += y,	y = rol64(y,45),\
	y *= 9			)
1874

1875
/*
1876 1877 1878
 * Fold two longs into one 32-bit hash value.  This must be fast, but
 * latency isn't quite as critical, as there is a fair bit of additional
 * work done before the hash value is used.
1879
 */
1880
static inline unsigned int fold_hash(unsigned long x, unsigned long y)
1881
{
1882 1883 1884
	y ^= x * GOLDEN_RATIO_64;
	y *= GOLDEN_RATIO_64;
	return y >> 32;
1885 1886
}

1887 1888
#else	/* 32-bit case */

1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903
/*
 * Mixing scores (in bits) for (7,20):
 * Input delta: 1-bit      2-bit
 * 1 round:     330.3     9201.6
 * 2 rounds:   1246.4    25475.4
 * 3 rounds:   1907.1    31295.1
 * 4 rounds:   2042.3    31718.6
 * Perfect:    2048      31744
 *            (32*64)   (32*31/2 * 64)
 */
#define HASH_MIX(x, y, a)	\
	(	x ^= (a),	\
	y ^= x,	x = rol32(x, 7),\
	x += y,	y = rol32(y,20),\
	y *= 9			)
1904

1905
static inline unsigned int fold_hash(unsigned long x, unsigned long y)
1906
{
1907 1908
	/* Use arch-optimized multiply if one exists */
	return __hash_32(y ^ __hash_32(x));
1909 1910
}

1911 1912
#endif

1913 1914 1915 1916 1917 1918 1919
/*
 * Return the hash of a string of known length.  This is carfully
 * designed to match hash_name(), which is the more critical function.
 * In particular, we must end by hashing a final word containing 0..7
 * payload bytes, to match the way that hash_name() iterates until it
 * finds the delimiter after the name.
 */
1920
unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
1921
{
1922
	unsigned long a, x = 0, y = (unsigned long)salt;
1923 1924

	for (;;) {
1925 1926
		if (!len)
			goto done;
1927
		a = load_unaligned_zeropad(name);
1928 1929
		if (len < sizeof(unsigned long))
			break;
1930
		HASH_MIX(x, y, a);
1931 1932 1933
		name += sizeof(unsigned long);
		len -= sizeof(unsigned long);
	}
1934
	x ^= a & bytemask_from_count(len);
1935
done:
1936
	return fold_hash(x, y);
1937 1938 1939
}
EXPORT_SYMBOL(full_name_hash);

1940
/* Return the "hash_len" (hash and length) of a null-terminated string */
1941
u64 hashlen_string(const void *salt, const char *name)
1942
{
1943 1944
	unsigned long a = 0, x = 0, y = (unsigned long)salt;
	unsigned long adata, mask, len;
1945 1946
	const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;

1947 1948 1949
	len = 0;
	goto inside;

1950
	do {
1951
		HASH_MIX(x, y, a);
1952
		len += sizeof(unsigned long);
1953
inside:
1954 1955 1956 1957 1958
		a = load_unaligned_zeropad(name+len);
	} while (!has_zero(a, &adata, &constants));

	adata = prep_zero_mask(a, adata, &constants);
	mask = create_zero_mask(adata);
1959
	x ^= a & zero_bytemask(mask);
1960

1961
	return hashlen_create(fold_hash(x, y), len + find_zero(mask));
1962 1963 1964
}
EXPORT_SYMBOL(hashlen_string);

1965 1966
/*
 * Calculate the length and hash of the path component, and
1967
 * return the "hash_len" as the result.
1968
 */
1969
static inline u64 hash_name(const void *salt, const char *name)
1970
{
1971 1972
	unsigned long a = 0, b, x = 0, y = (unsigned long)salt;
	unsigned long adata, bdata, mask, len;
1973
	const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
1974

1975 1976 1977
	len = 0;
	goto inside;

1978
	do {
1979
		HASH_MIX(x, y, a);
1980
		len += sizeof(unsigned long);
1981
inside:
1982
		a = load_unaligned_zeropad(name+len);
1983 1984 1985 1986 1987 1988
		b = a ^ REPEAT_BYTE('/');
	} while (!(has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants)));

	adata = prep_zero_mask(a, adata, &constants);
	bdata = prep_zero_mask(b, bdata, &constants);
	mask = create_zero_mask(adata | bdata);
1989
	x ^= a & zero_bytemask(mask);
1990

1991
	return hashlen_create(fold_hash(x, y), len + find_zero(mask));
1992 1993
}

1994
#else	/* !CONFIG_DCACHE_WORD_ACCESS: Slow, byte-at-a-time version */
1995

1996
/* Return the hash of a string of known length */
1997
unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
L
Linus Torvalds 已提交
1998
{
1999
	unsigned long hash = init_name_hash(salt);
L
Linus Torvalds 已提交
2000
	while (len--)
2001
		hash = partial_name_hash((unsigned char)*name++, hash);
L
Linus Torvalds 已提交
2002 2003
	return end_name_hash(hash);
}
2004
EXPORT_SYMBOL(full_name_hash);
L
Linus Torvalds 已提交
2005

2006
/* Return the "hash_len" (hash and length) of a null-terminated string */
2007
u64 hashlen_string(const void *salt, const char *name)
2008
{
2009
	unsigned long hash = init_name_hash(salt);
2010 2011 2012
	unsigned long len = 0, c;

	c = (unsigned char)*name;
2013
	while (c) {
2014 2015 2016
		len++;
		hash = partial_name_hash(c, hash);
		c = (unsigned char)name[len];
2017
	}
2018 2019
	return hashlen_create(end_name_hash(hash), len);
}
2020
EXPORT_SYMBOL(hashlen_string);
2021

2022 2023 2024 2025
/*
 * We know there's a real path component here of at least
 * one character.
 */
2026
static inline u64 hash_name(const void *salt, const char *name)
2027
{
2028
	unsigned long hash = init_name_hash(salt);
2029 2030 2031 2032 2033 2034 2035 2036
	unsigned long len = 0, c;

	c = (unsigned char)*name;
	do {
		len++;
		hash = partial_name_hash(c, hash);
		c = (unsigned char)name[len];
	} while (c && c != '/');
2037
	return hashlen_create(end_name_hash(hash), len);
2038 2039
}

2040 2041
#endif

L
Linus Torvalds 已提交
2042 2043
/*
 * Name resolution.
2044 2045
 * This is the basic name resolution function, turning a pathname into
 * the final dentry. We expect 'base' to be positive and a directory.
L
Linus Torvalds 已提交
2046
 *
2047 2048
 * Returns 0 and nd will have valid dentry and mnt on success.
 * Returns error and drops reference to input namei data on failure.
L
Linus Torvalds 已提交
2049
 */
2050
static int link_path_walk(const char *name, struct nameidata *nd)
L
Linus Torvalds 已提交
2051 2052
{
	int err;
A
Al Viro 已提交
2053

L
Linus Torvalds 已提交
2054 2055 2056
	while (*name=='/')
		name++;
	if (!*name)
2057
		return 0;
L
Linus Torvalds 已提交
2058 2059 2060

	/* At this point we know we have a real path component. */
	for(;;) {
2061
		u64 hash_len;
A
Al Viro 已提交
2062
		int type;
L
Linus Torvalds 已提交
2063

2064
		err = may_lookup(nd);
2065
		if (err)
2066
			return err;
L
Linus Torvalds 已提交
2067

2068
		hash_len = hash_name(nd->path.dentry, name);
L
Linus Torvalds 已提交
2069

A
Al Viro 已提交
2070
		type = LAST_NORM;
2071
		if (name[0] == '.') switch (hashlen_len(hash_len)) {
A
Al Viro 已提交
2072
			case 2:
2073
				if (name[1] == '.') {
A
Al Viro 已提交
2074
					type = LAST_DOTDOT;
A
Al Viro 已提交
2075 2076
					nd->flags |= LOOKUP_JUMPED;
				}
A
Al Viro 已提交
2077 2078 2079 2080
				break;
			case 1:
				type = LAST_DOT;
		}
2081 2082
		if (likely(type == LAST_NORM)) {
			struct dentry *parent = nd->path.dentry;
A
Al Viro 已提交
2083
			nd->flags &= ~LOOKUP_JUMPED;
2084
			if (unlikely(parent->d_flags & DCACHE_OP_HASH)) {
2085
				struct qstr this = { { .hash_len = hash_len }, .name = name };
2086
				err = parent->d_op->d_hash(parent, &this);
2087
				if (err < 0)
2088
					return err;
2089 2090
				hash_len = this.hash_len;
				name = this.name;
2091 2092
			}
		}
A
Al Viro 已提交
2093

2094 2095
		nd->last.hash_len = hash_len;
		nd->last.name = name;
2096 2097
		nd->last_type = type;

2098 2099
		name += hashlen_len(hash_len);
		if (!*name)
2100
			goto OK;
2101 2102 2103 2104 2105
		/*
		 * If it wasn't NUL, we know it was '/'. Skip that
		 * slash, and continue until no more slashes.
		 */
		do {
2106 2107
			name++;
		} while (unlikely(*name == '/'));
2108 2109
		if (unlikely(!*name)) {
OK:
2110
			/* pathname body, done */
2111 2112 2113
			if (!nd->depth)
				return 0;
			name = nd->stack[nd->depth - 1].name;
2114
			/* trailing symlink, done */
2115 2116 2117
			if (!name)
				return 0;
			/* last component of nested symlink */
2118
			err = walk_component(nd, WALK_FOLLOW);
A
Al Viro 已提交
2119 2120
		} else {
			/* not the last component */
2121
			err = walk_component(nd, WALK_FOLLOW | WALK_MORE);
2122
		}
2123
		if (err < 0)
2124
			return err;
L
Linus Torvalds 已提交
2125

2126
		if (err) {
2127
			const char *s = get_link(nd);
2128

2129
			if (IS_ERR(s))
2130
				return PTR_ERR(s);
2131 2132 2133
			err = 0;
			if (unlikely(!s)) {
				/* jumped */
2134
				put_link(nd);
2135
			} else {
2136 2137 2138
				nd->stack[nd->depth - 1].name = name;
				name = s;
				continue;
2139
			}
N
Nick Piggin 已提交
2140
		}
2141 2142
		if (unlikely(!d_can_lookup(nd->path.dentry))) {
			if (nd->flags & LOOKUP_RCU) {
A
Al Viro 已提交
2143
				if (unlazy_walk(nd))
2144 2145
					return -ECHILD;
			}
2146
			return -ENOTDIR;
2147
		}
L
Linus Torvalds 已提交
2148 2149 2150
	}
}

2151
static const char *path_init(struct nameidata *nd, unsigned flags)
N
Nick Piggin 已提交
2152
{
2153
	const char *s = nd->name->name;
N
Nick Piggin 已提交
2154

2155 2156 2157
	if (!*s)
		flags &= ~LOOKUP_RCU;

N
Nick Piggin 已提交
2158
	nd->last_type = LAST_ROOT; /* if there are only slashes... */
2159
	nd->flags = flags | LOOKUP_JUMPED | LOOKUP_PARENT;
N
Nick Piggin 已提交
2160
	nd->depth = 0;
2161
	if (flags & LOOKUP_ROOT) {
2162 2163
		struct dentry *root = nd->root.dentry;
		struct inode *inode = root->d_inode;
2164 2165
		if (*s && unlikely(!d_can_lookup(root)))
			return ERR_PTR(-ENOTDIR);
2166 2167 2168
		nd->path = nd->root;
		nd->inode = inode;
		if (flags & LOOKUP_RCU) {
A
Al Viro 已提交
2169
			rcu_read_lock();
2170
			nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
2171
			nd->root_seq = nd->seq;
A
Al Viro 已提交
2172
			nd->m_seq = read_seqbegin(&mount_lock);
2173 2174 2175
		} else {
			path_get(&nd->path);
		}
2176
		return s;
2177 2178
	}

N
Nick Piggin 已提交
2179
	nd->root.mnt = NULL;
2180 2181
	nd->path.mnt = NULL;
	nd->path.dentry = NULL;
N
Nick Piggin 已提交
2182

A
Al Viro 已提交
2183
	nd->m_seq = read_seqbegin(&mount_lock);
A
Al Viro 已提交
2184
	if (*s == '/') {
2185
		if (flags & LOOKUP_RCU)
A
Al Viro 已提交
2186
			rcu_read_lock();
2187
		set_root(nd);
2188
		if (likely(!nd_jump_root(nd)))
2189
			return s;
2190
		nd->root.mnt = NULL;
2191 2192
		rcu_read_unlock();
		return ERR_PTR(-ECHILD);
2193
	} else if (nd->dfd == AT_FDCWD) {
A
Al Viro 已提交
2194 2195 2196
		if (flags & LOOKUP_RCU) {
			struct fs_struct *fs = current->fs;
			unsigned seq;
N
Nick Piggin 已提交
2197

A
Al Viro 已提交
2198
			rcu_read_lock();
N
Nick Piggin 已提交
2199

A
Al Viro 已提交
2200 2201 2202
			do {
				seq = read_seqcount_begin(&fs->seq);
				nd->path = fs->pwd;
2203
				nd->inode = nd->path.dentry->d_inode;
A
Al Viro 已提交
2204 2205 2206 2207
				nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
			} while (read_seqcount_retry(&fs->seq, seq));
		} else {
			get_fs_pwd(current->fs, &nd->path);
2208
			nd->inode = nd->path.dentry->d_inode;
A
Al Viro 已提交
2209
		}
2210
		return s;
N
Nick Piggin 已提交
2211
	} else {
2212
		/* Caller must check execute permissions on the starting path component */
2213
		struct fd f = fdget_raw(nd->dfd);
N
Nick Piggin 已提交
2214 2215
		struct dentry *dentry;

2216
		if (!f.file)
2217
			return ERR_PTR(-EBADF);
N
Nick Piggin 已提交
2218

2219
		dentry = f.file->f_path.dentry;
N
Nick Piggin 已提交
2220

A
Al Viro 已提交
2221
		if (*s) {
M
Miklos Szeredi 已提交
2222
			if (!d_can_lookup(dentry)) {
2223
				fdput(f);
2224
				return ERR_PTR(-ENOTDIR);
2225
			}
A
Al Viro 已提交
2226
		}
N
Nick Piggin 已提交
2227

2228
		nd->path = f.file->f_path;
A
Al Viro 已提交
2229
		if (flags & LOOKUP_RCU) {
A
Al Viro 已提交
2230
			rcu_read_lock();
A
Al Viro 已提交
2231 2232
			nd->inode = nd->path.dentry->d_inode;
			nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
A
Al Viro 已提交
2233
		} else {
2234
			path_get(&nd->path);
A
Al Viro 已提交
2235
			nd->inode = nd->path.dentry->d_inode;
A
Al Viro 已提交
2236
		}
A
Al Viro 已提交
2237
		fdput(f);
2238
		return s;
N
Nick Piggin 已提交
2239
	}
2240 2241
}

2242
static const char *trailing_symlink(struct nameidata *nd)
2243 2244
{
	const char *s;
A
Al Viro 已提交
2245
	int error = may_follow_link(nd);
2246
	if (unlikely(error))
2247
		return ERR_PTR(error);
2248
	nd->flags |= LOOKUP_PARENT;
2249
	nd->stack[0].name = NULL;
2250
	s = get_link(nd);
2251
	return s ? s : "";
2252 2253
}

A
Al Viro 已提交
2254
static inline int lookup_last(struct nameidata *nd)
2255 2256 2257 2258 2259
{
	if (nd->last_type == LAST_NORM && nd->last.name[nd->last.len])
		nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;

	nd->flags &= ~LOOKUP_PARENT;
A
Al Viro 已提交
2260
	return walk_component(nd, 0);
2261 2262
}

2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291
static int handle_lookup_down(struct nameidata *nd)
{
	struct path path = nd->path;
	struct inode *inode = nd->inode;
	unsigned seq = nd->seq;
	int err;

	if (nd->flags & LOOKUP_RCU) {
		/*
		 * don't bother with unlazy_walk on failure - we are
		 * at the very beginning of walk, so we lose nothing
		 * if we simply redo everything in non-RCU mode
		 */
		if (unlikely(!__follow_mount_rcu(nd, &path, &inode, &seq)))
			return -ECHILD;
	} else {
		dget(path.dentry);
		err = follow_managed(&path, nd);
		if (unlikely(err < 0))
			return err;
		inode = d_backing_inode(path.dentry);
		seq = 0;
	}
	path_to_nameidata(&path, nd);
	nd->inode = inode;
	nd->seq = seq;
	return 0;
}

2292
/* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
2293
static int path_lookupat(struct nameidata *nd, unsigned flags, struct path *path)
2294
{
2295
	const char *s = path_init(nd, flags);
2296
	int err;
N
Nick Piggin 已提交
2297

2298 2299
	if (IS_ERR(s))
		return PTR_ERR(s);
2300 2301 2302 2303 2304 2305 2306 2307 2308

	if (unlikely(flags & LOOKUP_DOWN)) {
		err = handle_lookup_down(nd);
		if (unlikely(err < 0)) {
			terminate_walk(nd);
			return err;
		}
	}

2309 2310 2311 2312 2313 2314
	while (!(err = link_path_walk(s, nd))
		&& ((err = lookup_last(nd)) > 0)) {
		s = trailing_symlink(nd);
		if (IS_ERR(s)) {
			err = PTR_ERR(s);
			break;
2315 2316
		}
	}
2317 2318
	if (!err)
		err = complete_walk(nd);
2319

2320 2321
	if (!err && nd->flags & LOOKUP_DIRECTORY)
		if (!d_can_lookup(nd->path.dentry))
A
Al Viro 已提交
2322
			err = -ENOTDIR;
2323 2324 2325 2326 2327 2328
	if (!err) {
		*path = nd->path;
		nd->path.mnt = NULL;
		nd->path.dentry = NULL;
	}
	terminate_walk(nd);
2329
	return err;
A
Al Viro 已提交
2330
}
N
Nick Piggin 已提交
2331

2332
static int filename_lookup(int dfd, struct filename *name, unsigned flags,
2333
			   struct path *path, struct path *root)
A
Al Viro 已提交
2334
{
2335
	int retval;
2336
	struct nameidata nd;
2337 2338
	if (IS_ERR(name))
		return PTR_ERR(name);
2339 2340 2341 2342
	if (unlikely(root)) {
		nd.root = *root;
		flags |= LOOKUP_ROOT;
	}
2343
	set_nameidata(&nd, dfd, name);
2344
	retval = path_lookupat(&nd, flags | LOOKUP_RCU, path);
A
Al Viro 已提交
2345
	if (unlikely(retval == -ECHILD))
2346
		retval = path_lookupat(&nd, flags, path);
A
Al Viro 已提交
2347
	if (unlikely(retval == -ESTALE))
2348
		retval = path_lookupat(&nd, flags | LOOKUP_REVAL, path);
N
Nick Piggin 已提交
2349

2350
	if (likely(!retval))
2351
		audit_inode(name, path->dentry, flags & LOOKUP_PARENT);
2352
	restore_nameidata();
2353
	putname(name);
2354
	return retval;
L
Linus Torvalds 已提交
2355 2356
}

2357
/* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
2358
static int path_parentat(struct nameidata *nd, unsigned flags,
2359
				struct path *parent)
2360
{
2361
	const char *s = path_init(nd, flags);
2362 2363 2364 2365
	int err;
	if (IS_ERR(s))
		return PTR_ERR(s);
	err = link_path_walk(s, nd);
2366 2367
	if (!err)
		err = complete_walk(nd);
2368 2369 2370 2371 2372 2373
	if (!err) {
		*parent = nd->path;
		nd->path.mnt = NULL;
		nd->path.dentry = NULL;
	}
	terminate_walk(nd);
2374 2375 2376
	return err;
}

2377
static struct filename *filename_parentat(int dfd, struct filename *name,
2378 2379
				unsigned int flags, struct path *parent,
				struct qstr *last, int *type)
2380 2381
{
	int retval;
2382
	struct nameidata nd;
2383

2384 2385
	if (IS_ERR(name))
		return name;
2386
	set_nameidata(&nd, dfd, name);
2387
	retval = path_parentat(&nd, flags | LOOKUP_RCU, parent);
2388
	if (unlikely(retval == -ECHILD))
2389
		retval = path_parentat(&nd, flags, parent);
2390
	if (unlikely(retval == -ESTALE))
2391
		retval = path_parentat(&nd, flags | LOOKUP_REVAL, parent);
2392 2393 2394 2395
	if (likely(!retval)) {
		*last = nd.last;
		*type = nd.last_type;
		audit_inode(name, parent->dentry, LOOKUP_PARENT);
2396 2397 2398
	} else {
		putname(name);
		name = ERR_PTR(retval);
2399
	}
2400
	restore_nameidata();
2401
	return name;
2402 2403
}

A
Al Viro 已提交
2404 2405
/* does lookup, returns the object with parent locked */
struct dentry *kern_path_locked(const char *name, struct path *path)
2406
{
2407 2408
	struct filename *filename;
	struct dentry *d;
2409 2410
	struct qstr last;
	int type;
2411

2412 2413
	filename = filename_parentat(AT_FDCWD, getname_kernel(name), 0, path,
				    &last, &type);
2414 2415
	if (IS_ERR(filename))
		return ERR_CAST(filename);
2416
	if (unlikely(type != LAST_NORM)) {
2417
		path_put(path);
2418 2419
		putname(filename);
		return ERR_PTR(-EINVAL);
A
Al Viro 已提交
2420
	}
A
Al Viro 已提交
2421
	inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
2422
	d = __lookup_hash(&last, path->dentry, 0);
A
Al Viro 已提交
2423
	if (IS_ERR(d)) {
A
Al Viro 已提交
2424
		inode_unlock(path->dentry->d_inode);
2425
		path_put(path);
A
Al Viro 已提交
2426
	}
2427
	putname(filename);
A
Al Viro 已提交
2428
	return d;
2429 2430
}

A
Al Viro 已提交
2431 2432
int kern_path(const char *name, unsigned int flags, struct path *path)
{
2433 2434
	return filename_lookup(AT_FDCWD, getname_kernel(name),
			       flags, path, NULL);
A
Al Viro 已提交
2435
}
2436
EXPORT_SYMBOL(kern_path);
A
Al Viro 已提交
2437

2438 2439 2440 2441 2442 2443
/**
 * vfs_path_lookup - lookup a file path relative to a dentry-vfsmount pair
 * @dentry:  pointer to dentry of the base directory
 * @mnt: pointer to vfs mount of the base directory
 * @name: pointer to file name
 * @flags: lookup flags
2444
 * @path: pointer to struct path to fill
2445 2446 2447
 */
int vfs_path_lookup(struct dentry *dentry, struct vfsmount *mnt,
		    const char *name, unsigned int flags,
2448
		    struct path *path)
2449
{
2450 2451
	struct path root = {.mnt = mnt, .dentry = dentry};
	/* the first argument of filename_lookup() is ignored with root */
2452 2453
	return filename_lookup(AT_FDCWD, getname_kernel(name),
			       flags , path, &root);
2454
}
2455
EXPORT_SYMBOL(vfs_path_lookup);
2456

2457
/**
2458
 * lookup_one_len - filesystem helper to lookup single pathname component
2459 2460 2461 2462
 * @name:	pathname component to lookup
 * @base:	base directory to lookup from
 * @len:	maximum length @len should be interpreted to
 *
2463
 * Note that this routine is purely a helper for filesystem usage and should
2464
 * not be called by generic code.
2465 2466
 *
 * The caller must hold base->i_mutex.
2467
 */
2468 2469 2470
struct dentry *lookup_one_len(const char *name, struct dentry *base, int len)
{
	struct qstr this;
A
Al Viro 已提交
2471
	unsigned int c;
2472
	int err;
2473

A
Al Viro 已提交
2474
	WARN_ON_ONCE(!inode_is_locked(base->d_inode));
2475

A
Al Viro 已提交
2476 2477
	this.name = name;
	this.len = len;
2478
	this.hash = full_name_hash(base, name, len);
A
Al Viro 已提交
2479 2480 2481
	if (!len)
		return ERR_PTR(-EACCES);

A
Al Viro 已提交
2482 2483 2484 2485 2486
	if (unlikely(name[0] == '.')) {
		if (len < 2 || (len == 2 && name[1] == '.'))
			return ERR_PTR(-EACCES);
	}

A
Al Viro 已提交
2487 2488 2489 2490 2491
	while (len--) {
		c = *(const unsigned char *)name++;
		if (c == '/' || c == '\0')
			return ERR_PTR(-EACCES);
	}
2492 2493 2494 2495 2496
	/*
	 * See if the low-level filesystem might want
	 * to use its own hash..
	 */
	if (base->d_flags & DCACHE_OP_HASH) {
2497
		int err = base->d_op->d_hash(base, &this);
2498 2499 2500
		if (err < 0)
			return ERR_PTR(err);
	}
2501

2502 2503 2504 2505
	err = inode_permission(base->d_inode, MAY_EXEC);
	if (err)
		return ERR_PTR(err);

2506
	return __lookup_hash(&this, base, 0);
2507
}
2508
EXPORT_SYMBOL(lookup_one_len);
2509

2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527
/**
 * lookup_one_len_unlocked - filesystem helper to lookup single pathname component
 * @name:	pathname component to lookup
 * @base:	base directory to lookup from
 * @len:	maximum length @len should be interpreted to
 *
 * Note that this routine is purely a helper for filesystem usage and should
 * not be called by generic code.
 *
 * Unlike lookup_one_len, it should be called without the parent
 * i_mutex held, and will take the i_mutex itself if necessary.
 */
struct dentry *lookup_one_len_unlocked(const char *name,
				       struct dentry *base, int len)
{
	struct qstr this;
	unsigned int c;
	int err;
2528
	struct dentry *ret;
2529 2530 2531

	this.name = name;
	this.len = len;
2532
	this.hash = full_name_hash(base, name, len);
2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559
	if (!len)
		return ERR_PTR(-EACCES);

	if (unlikely(name[0] == '.')) {
		if (len < 2 || (len == 2 && name[1] == '.'))
			return ERR_PTR(-EACCES);
	}

	while (len--) {
		c = *(const unsigned char *)name++;
		if (c == '/' || c == '\0')
			return ERR_PTR(-EACCES);
	}
	/*
	 * See if the low-level filesystem might want
	 * to use its own hash..
	 */
	if (base->d_flags & DCACHE_OP_HASH) {
		int err = base->d_op->d_hash(base, &this);
		if (err < 0)
			return ERR_PTR(err);
	}

	err = inode_permission(base->d_inode, MAY_EXEC);
	if (err)
		return ERR_PTR(err);

2560 2561 2562 2563
	ret = lookup_dcache(&this, base, 0);
	if (!ret)
		ret = lookup_slow(&this, base, 0);
	return ret;
2564 2565 2566
}
EXPORT_SYMBOL(lookup_one_len_unlocked);

2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594
#ifdef CONFIG_UNIX98_PTYS
int path_pts(struct path *path)
{
	/* Find something mounted on "pts" in the same directory as
	 * the input path.
	 */
	struct dentry *child, *parent;
	struct qstr this;
	int ret;

	ret = path_parent_directory(path);
	if (ret)
		return ret;

	parent = path->dentry;
	this.name = "pts";
	this.len = 3;
	child = d_hash_and_lookup(parent, &this);
	if (!child)
		return -ENOENT;

	path->dentry = child;
	dput(parent);
	follow_mount(path);
	return 0;
}
#endif

2595 2596
int user_path_at_empty(int dfd, const char __user *name, unsigned flags,
		 struct path *path, int *empty)
L
Linus Torvalds 已提交
2597
{
2598 2599
	return filename_lookup(dfd, getname_flags(name, flags, empty),
			       flags, path, NULL);
L
Linus Torvalds 已提交
2600
}
2601
EXPORT_SYMBOL(user_path_at_empty);
2602

2603
/**
2604
 * mountpoint_last - look up last component for umount
2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
 * @nd:   pathwalk nameidata - currently pointing at parent directory of "last"
 *
 * This is a special lookup_last function just for umount. In this case, we
 * need to resolve the path without doing any revalidation.
 *
 * The nameidata should be the result of doing a LOOKUP_PARENT pathwalk. Since
 * mountpoints are always pinned in the dcache, their ancestors are too. Thus,
 * in almost all cases, this lookup will be served out of the dcache. The only
 * cases where it won't are if nd->last refers to a symlink or the path is
 * bogus and it doesn't exist.
 *
 * Returns:
 * -error: if there was an error during lookup. This includes -ENOENT if the
2618
 *         lookup found a negative dentry.
2619
 *
2620 2621
 * 0:      if we successfully resolved nd->last and found it to not to be a
 *         symlink that needs to be followed.
2622 2623
 *
 * 1:      if we successfully resolved nd->last and found it to be a symlink
2624
 *         that needs to be followed.
2625 2626
 */
static int
2627
mountpoint_last(struct nameidata *nd)
2628 2629 2630
{
	int error = 0;
	struct dentry *dir = nd->path.dentry;
2631
	struct path path;
2632

2633 2634
	/* If we're in rcuwalk, drop out of it to handle last component */
	if (nd->flags & LOOKUP_RCU) {
A
Al Viro 已提交
2635
		if (unlazy_walk(nd))
2636
			return -ECHILD;
2637 2638 2639 2640 2641 2642
	}

	nd->flags &= ~LOOKUP_PARENT;

	if (unlikely(nd->last_type != LAST_NORM)) {
		error = handle_dots(nd, nd->last_type);
2643
		if (error)
2644
			return error;
2645
		path.dentry = dget(nd->path.dentry);
2646
	} else {
2647 2648
		path.dentry = d_lookup(dir, &nd->last);
		if (!path.dentry) {
2649 2650 2651 2652 2653 2654
			/*
			 * No cached dentry. Mounted dentries are pinned in the
			 * cache, so that means that this dentry is probably
			 * a symlink or the path doesn't actually point
			 * to a mounted dentry.
			 */
2655
			path.dentry = lookup_slow(&nd->last, dir,
2656
					     nd->flags | LOOKUP_NO_REVAL);
2657 2658
			if (IS_ERR(path.dentry))
				return PTR_ERR(path.dentry);
2659
		}
2660
	}
2661 2662
	if (d_is_negative(path.dentry)) {
		dput(path.dentry);
2663
		return -ENOENT;
2664
	}
2665
	path.mnt = nd->path.mnt;
2666
	return step_into(nd, &path, 0, d_backing_inode(path.dentry), 0);
2667 2668 2669
}

/**
2670
 * path_mountpoint - look up a path to be umounted
2671
 * @nd:		lookup context
2672
 * @flags:	lookup flags
2673
 * @path:	pointer to container for result
2674 2675
 *
 * Look up the given name, but don't attempt to revalidate the last component.
2676
 * Returns 0 and "path" will be valid on success; Returns error otherwise.
2677 2678
 */
static int
2679
path_mountpoint(struct nameidata *nd, unsigned flags, struct path *path)
2680
{
2681
	const char *s = path_init(nd, flags);
2682 2683 2684
	int err;
	if (IS_ERR(s))
		return PTR_ERR(s);
2685
	while (!(err = link_path_walk(s, nd)) &&
2686
		(err = mountpoint_last(nd)) > 0) {
2687 2688 2689
		s = trailing_symlink(nd);
		if (IS_ERR(s)) {
			err = PTR_ERR(s);
2690
			break;
2691
		}
2692
	}
2693 2694 2695 2696 2697 2698
	if (!err) {
		*path = nd->path;
		nd->path.mnt = NULL;
		nd->path.dentry = NULL;
		follow_mount(path);
	}
2699
	terminate_walk(nd);
2700 2701 2702
	return err;
}

A
Al Viro 已提交
2703
static int
2704
filename_mountpoint(int dfd, struct filename *name, struct path *path,
A
Al Viro 已提交
2705 2706
			unsigned int flags)
{
2707
	struct nameidata nd;
2708
	int error;
2709 2710
	if (IS_ERR(name))
		return PTR_ERR(name);
2711
	set_nameidata(&nd, dfd, name);
2712
	error = path_mountpoint(&nd, flags | LOOKUP_RCU, path);
A
Al Viro 已提交
2713
	if (unlikely(error == -ECHILD))
2714
		error = path_mountpoint(&nd, flags, path);
A
Al Viro 已提交
2715
	if (unlikely(error == -ESTALE))
2716
		error = path_mountpoint(&nd, flags | LOOKUP_REVAL, path);
A
Al Viro 已提交
2717
	if (likely(!error))
2718
		audit_inode(name, path->dentry, 0);
2719
	restore_nameidata();
2720
	putname(name);
A
Al Viro 已提交
2721 2722 2723
	return error;
}

2724
/**
2725
 * user_path_mountpoint_at - lookup a path from userland in order to umount it
2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738
 * @dfd:	directory file descriptor
 * @name:	pathname from userland
 * @flags:	lookup flags
 * @path:	pointer to container to hold result
 *
 * A umount is a special case for path walking. We're not actually interested
 * in the inode in this situation, and ESTALE errors can be a problem. We
 * simply want track down the dentry and vfsmount attached at the mountpoint
 * and avoid revalidating the last component.
 *
 * Returns 0 and populates "path" on success.
 */
int
2739
user_path_mountpoint_at(int dfd, const char __user *name, unsigned int flags,
2740 2741
			struct path *path)
{
2742
	return filename_mountpoint(dfd, getname(name), path, flags);
2743 2744
}

A
Al Viro 已提交
2745 2746 2747 2748
int
kern_path_mountpoint(int dfd, const char *name, struct path *path,
			unsigned int flags)
{
2749
	return filename_mountpoint(dfd, getname_kernel(name), path, flags);
A
Al Viro 已提交
2750 2751 2752
}
EXPORT_SYMBOL(kern_path_mountpoint);

M
Miklos Szeredi 已提交
2753
int __check_sticky(struct inode *dir, struct inode *inode)
L
Linus Torvalds 已提交
2754
{
2755
	kuid_t fsuid = current_fsuid();
2756

2757
	if (uid_eq(inode->i_uid, fsuid))
L
Linus Torvalds 已提交
2758
		return 0;
2759
	if (uid_eq(dir->i_uid, fsuid))
L
Linus Torvalds 已提交
2760
		return 0;
2761
	return !capable_wrt_inode_uidgid(inode, CAP_FOWNER);
L
Linus Torvalds 已提交
2762
}
M
Miklos Szeredi 已提交
2763
EXPORT_SYMBOL(__check_sticky);
L
Linus Torvalds 已提交
2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777

/*
 *	Check whether we can remove a link victim from directory dir, check
 *  whether the type of victim is right.
 *  1. We can't do it if dir is read-only (done in permission())
 *  2. We should have write and exec permissions on dir
 *  3. We can't remove anything from append-only dir
 *  4. We can't do anything with immutable dir (done in permission())
 *  5. If the sticky bit on dir is set we should either
 *	a. be owner of dir, or
 *	b. be owner of victim, or
 *	c. have CAP_FOWNER capability
 *  6. If the victim is append-only or immutable we can't do antyhing with
 *     links pointing to it.
2778 2779 2780 2781 2782
 *  7. If the victim has an unknown uid or gid we can't change the inode.
 *  8. If we were asked to remove a directory and victim isn't one - ENOTDIR.
 *  9. If we were asked to remove a non-directory and victim isn't one - EISDIR.
 * 10. We can't remove a root or mountpoint.
 * 11. We don't allow removal of NFS sillyrenamed files; it's handled by
L
Linus Torvalds 已提交
2783 2784
 *     nfs_async_unlink().
 */
2785
static int may_delete(struct inode *dir, struct dentry *victim, bool isdir)
L
Linus Torvalds 已提交
2786
{
2787
	struct inode *inode = d_backing_inode(victim);
L
Linus Torvalds 已提交
2788 2789
	int error;

2790
	if (d_is_negative(victim))
L
Linus Torvalds 已提交
2791
		return -ENOENT;
2792
	BUG_ON(!inode);
L
Linus Torvalds 已提交
2793 2794

	BUG_ON(victim->d_parent->d_inode != dir);
2795
	audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE);
L
Linus Torvalds 已提交
2796

2797
	error = inode_permission(dir, MAY_WRITE | MAY_EXEC);
L
Linus Torvalds 已提交
2798 2799 2800 2801
	if (error)
		return error;
	if (IS_APPEND(dir))
		return -EPERM;
2802 2803

	if (check_sticky(dir, inode) || IS_APPEND(inode) ||
2804
	    IS_IMMUTABLE(inode) || IS_SWAPFILE(inode) || HAS_UNMAPPED_ID(inode))
L
Linus Torvalds 已提交
2805 2806
		return -EPERM;
	if (isdir) {
M
Miklos Szeredi 已提交
2807
		if (!d_is_dir(victim))
L
Linus Torvalds 已提交
2808 2809 2810
			return -ENOTDIR;
		if (IS_ROOT(victim))
			return -EBUSY;
M
Miklos Szeredi 已提交
2811
	} else if (d_is_dir(victim))
L
Linus Torvalds 已提交
2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824
		return -EISDIR;
	if (IS_DEADDIR(dir))
		return -ENOENT;
	if (victim->d_flags & DCACHE_NFSFS_RENAMED)
		return -EBUSY;
	return 0;
}

/*	Check whether we can create an object with dentry child in directory
 *  dir.
 *  1. We can't do it if child already exists (open has special treatment for
 *     this case, but since we are inlined it's OK)
 *  2. We can't do it if dir is read-only (done in permission())
2825 2826 2827
 *  3. We can't do it if the fs can't represent the fsuid or fsgid.
 *  4. We should have write and exec permissions on dir
 *  5. We can't do it if dir is immutable (done in permission())
L
Linus Torvalds 已提交
2828
 */
2829
static inline int may_create(struct inode *dir, struct dentry *child)
L
Linus Torvalds 已提交
2830
{
2831
	struct user_namespace *s_user_ns;
2832
	audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE);
L
Linus Torvalds 已提交
2833 2834 2835 2836
	if (child->d_inode)
		return -EEXIST;
	if (IS_DEADDIR(dir))
		return -ENOENT;
2837 2838 2839 2840
	s_user_ns = dir->i_sb->s_user_ns;
	if (!kuid_has_mapping(s_user_ns, current_fsuid()) ||
	    !kgid_has_mapping(s_user_ns, current_fsgid()))
		return -EOVERFLOW;
2841
	return inode_permission(dir, MAY_WRITE | MAY_EXEC);
L
Linus Torvalds 已提交
2842 2843 2844 2845 2846 2847 2848 2849 2850 2851
}

/*
 * p1 and p2 should be directories on the same fs.
 */
struct dentry *lock_rename(struct dentry *p1, struct dentry *p2)
{
	struct dentry *p;

	if (p1 == p2) {
A
Al Viro 已提交
2852
		inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
L
Linus Torvalds 已提交
2853 2854 2855
		return NULL;
	}

2856
	mutex_lock(&p1->d_sb->s_vfs_rename_mutex);
L
Linus Torvalds 已提交
2857

2858 2859
	p = d_ancestor(p2, p1);
	if (p) {
A
Al Viro 已提交
2860 2861
		inode_lock_nested(p2->d_inode, I_MUTEX_PARENT);
		inode_lock_nested(p1->d_inode, I_MUTEX_CHILD);
2862
		return p;
L
Linus Torvalds 已提交
2863 2864
	}

2865 2866
	p = d_ancestor(p1, p2);
	if (p) {
A
Al Viro 已提交
2867 2868
		inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
		inode_lock_nested(p2->d_inode, I_MUTEX_CHILD);
2869
		return p;
L
Linus Torvalds 已提交
2870 2871
	}

A
Al Viro 已提交
2872 2873
	inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
	inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2);
L
Linus Torvalds 已提交
2874 2875
	return NULL;
}
2876
EXPORT_SYMBOL(lock_rename);
L
Linus Torvalds 已提交
2877 2878 2879

void unlock_rename(struct dentry *p1, struct dentry *p2)
{
A
Al Viro 已提交
2880
	inode_unlock(p1->d_inode);
L
Linus Torvalds 已提交
2881
	if (p1 != p2) {
A
Al Viro 已提交
2882
		inode_unlock(p2->d_inode);
2883
		mutex_unlock(&p1->d_sb->s_vfs_rename_mutex);
L
Linus Torvalds 已提交
2884 2885
	}
}
2886
EXPORT_SYMBOL(unlock_rename);
L
Linus Torvalds 已提交
2887

A
Al Viro 已提交
2888
int vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
A
Al Viro 已提交
2889
		bool want_excl)
L
Linus Torvalds 已提交
2890
{
2891
	int error = may_create(dir, dentry);
L
Linus Torvalds 已提交
2892 2893 2894
	if (error)
		return error;

A
Al Viro 已提交
2895
	if (!dir->i_op->create)
L
Linus Torvalds 已提交
2896 2897 2898 2899 2900 2901
		return -EACCES;	/* shouldn't it be ENOSYS? */
	mode &= S_IALLUGO;
	mode |= S_IFREG;
	error = security_inode_create(dir, dentry, mode);
	if (error)
		return error;
A
Al Viro 已提交
2902
	error = dir->i_op->create(dir, dentry, mode, want_excl);
2903
	if (!error)
2904
		fsnotify_create(dir, dentry);
L
Linus Torvalds 已提交
2905 2906
	return error;
}
2907
EXPORT_SYMBOL(vfs_create);
L
Linus Torvalds 已提交
2908

2909 2910 2911 2912 2913 2914
bool may_open_dev(const struct path *path)
{
	return !(path->mnt->mnt_flags & MNT_NODEV) &&
		!(path->mnt->mnt_sb->s_iflags & SB_I_NODEV);
}

A
Al Viro 已提交
2915
static int may_open(const struct path *path, int acc_mode, int flag)
L
Linus Torvalds 已提交
2916
{
2917
	struct dentry *dentry = path->dentry;
L
Linus Torvalds 已提交
2918 2919 2920 2921 2922 2923
	struct inode *inode = dentry->d_inode;
	int error;

	if (!inode)
		return -ENOENT;

C
Christoph Hellwig 已提交
2924 2925
	switch (inode->i_mode & S_IFMT) {
	case S_IFLNK:
L
Linus Torvalds 已提交
2926
		return -ELOOP;
C
Christoph Hellwig 已提交
2927 2928 2929 2930 2931 2932
	case S_IFDIR:
		if (acc_mode & MAY_WRITE)
			return -EISDIR;
		break;
	case S_IFBLK:
	case S_IFCHR:
2933
		if (!may_open_dev(path))
L
Linus Torvalds 已提交
2934
			return -EACCES;
C
Christoph Hellwig 已提交
2935 2936 2937
		/*FALLTHRU*/
	case S_IFIFO:
	case S_IFSOCK:
L
Linus Torvalds 已提交
2938
		flag &= ~O_TRUNC;
C
Christoph Hellwig 已提交
2939
		break;
2940
	}
2941

A
Al Viro 已提交
2942
	error = inode_permission(inode, MAY_OPEN | acc_mode);
2943 2944
	if (error)
		return error;
M
Mimi Zohar 已提交
2945

L
Linus Torvalds 已提交
2946 2947 2948 2949
	/*
	 * An append-only file must be opened in append mode for writing.
	 */
	if (IS_APPEND(inode)) {
2950
		if  ((flag & O_ACCMODE) != O_RDONLY && !(flag & O_APPEND))
2951
			return -EPERM;
L
Linus Torvalds 已提交
2952
		if (flag & O_TRUNC)
2953
			return -EPERM;
L
Linus Torvalds 已提交
2954 2955 2956
	}

	/* O_NOATIME can only be set by the owner or superuser */
2957
	if (flag & O_NOATIME && !inode_owner_or_capable(inode))
2958
		return -EPERM;
L
Linus Torvalds 已提交
2959

2960
	return 0;
2961
}
L
Linus Torvalds 已提交
2962

2963
static int handle_truncate(struct file *filp)
2964
{
A
Al Viro 已提交
2965
	const struct path *path = &filp->f_path;
2966 2967 2968 2969 2970 2971 2972
	struct inode *inode = path->dentry->d_inode;
	int error = get_write_access(inode);
	if (error)
		return error;
	/*
	 * Refuse to truncate files with mandatory locks held on them.
	 */
2973
	error = locks_verify_locked(filp);
2974
	if (!error)
2975
		error = security_path_truncate(path);
2976 2977 2978
	if (!error) {
		error = do_truncate(path->dentry, 0,
				    ATTR_MTIME|ATTR_CTIME|ATTR_OPEN,
2979
				    filp);
2980 2981
	}
	put_write_access(inode);
M
Mimi Zohar 已提交
2982
	return error;
L
Linus Torvalds 已提交
2983 2984
}

2985 2986
static inline int open_to_namei_flags(int flag)
{
2987 2988
	if ((flag & O_ACCMODE) == 3)
		flag--;
2989 2990 2991
	return flag;
}

2992
static int may_o_create(const struct path *dir, struct dentry *dentry, umode_t mode)
M
Miklos Szeredi 已提交
2993
{
2994
	struct user_namespace *s_user_ns;
M
Miklos Szeredi 已提交
2995 2996 2997 2998
	int error = security_path_mknod(dir, dentry, mode, 0);
	if (error)
		return error;

2999 3000 3001 3002 3003
	s_user_ns = dir->dentry->d_sb->s_user_ns;
	if (!kuid_has_mapping(s_user_ns, current_fsuid()) ||
	    !kgid_has_mapping(s_user_ns, current_fsgid()))
		return -EOVERFLOW;

M
Miklos Szeredi 已提交
3004 3005 3006 3007 3008 3009 3010
	error = inode_permission(dir->dentry->d_inode, MAY_WRITE | MAY_EXEC);
	if (error)
		return error;

	return security_inode_create(dir->dentry->d_inode, dentry, mode);
}

3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023
/*
 * Attempt to atomically look up, create and open a file from a negative
 * dentry.
 *
 * Returns 0 if successful.  The file will have been created and attached to
 * @file by the filesystem calling finish_open().
 *
 * Returns 1 if the file was looked up only or didn't need creating.  The
 * caller will need to perform the open themselves.  @path will have been
 * updated to point to the new dentry.  This may be negative.
 *
 * Returns an error code otherwise.
 */
3024 3025 3026
static int atomic_open(struct nameidata *nd, struct dentry *dentry,
			struct path *path, struct file *file,
			const struct open_flags *op,
3027
			int open_flag, umode_t mode,
3028
			int *opened)
M
Miklos Szeredi 已提交
3029
{
3030
	struct dentry *const DENTRY_NOT_SET = (void *) -1UL;
M
Miklos Szeredi 已提交
3031 3032 3033
	struct inode *dir =  nd->path.dentry->d_inode;
	int error;

3034
	if (!(~open_flag & (O_EXCL | O_CREAT)))	/* both O_EXCL and O_CREAT */
M
Miklos Szeredi 已提交
3035 3036 3037 3038 3039
		open_flag &= ~O_TRUNC;

	if (nd->flags & LOOKUP_DIRECTORY)
		open_flag |= O_DIRECTORY;

A
Al Viro 已提交
3040 3041
	file->f_path.dentry = DENTRY_NOT_SET;
	file->f_path.mnt = nd->path.mnt;
3042 3043 3044
	error = dir->i_op->atomic_open(dir, dentry, file,
				       open_to_namei_flags(open_flag),
				       mode, opened);
3045
	d_lookup_done(dentry);
3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060
	if (!error) {
		/*
		 * We didn't have the inode before the open, so check open
		 * permission here.
		 */
		int acc_mode = op->acc_mode;
		if (*opened & FILE_CREATED) {
			WARN_ON(!(open_flag & O_CREAT));
			fsnotify_create(dir, dentry);
			acc_mode = 0;
		}
		error = may_open(&file->f_path, acc_mode, open_flag);
		if (WARN_ON(error > 0))
			error = -EINVAL;
	} else if (error > 0) {
A
Al Viro 已提交
3061
		if (WARN_ON(file->f_path.dentry == DENTRY_NOT_SET)) {
3062
			error = -EIO;
3063
		} else {
3064 3065 3066
			if (file->f_path.dentry) {
				dput(dentry);
				dentry = file->f_path.dentry;
3067
			}
3068 3069
			if (*opened & FILE_CREATED)
				fsnotify_create(dir, dentry);
A
Al Viro 已提交
3070 3071 3072 3073 3074 3075 3076
			if (unlikely(d_is_negative(dentry))) {
				error = -ENOENT;
			} else {
				path->dentry = dentry;
				path->mnt = nd->path.mnt;
				return 1;
			}
3077
		}
M
Miklos Szeredi 已提交
3078 3079
	}
	dput(dentry);
3080
	return error;
M
Miklos Szeredi 已提交
3081 3082
}

M
Miklos Szeredi 已提交
3083
/*
3084
 * Look up and maybe create and open the last component.
M
Miklos Szeredi 已提交
3085 3086 3087
 *
 * Must be called with i_mutex held on parent.
 *
3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099
 * Returns 0 if the file was successfully atomically created (if necessary) and
 * opened.  In this case the file will be returned attached to @file.
 *
 * Returns 1 if the file was not completely opened at this time, though lookups
 * and creations will have been performed and the dentry returned in @path will
 * be positive upon return if O_CREAT was specified.  If O_CREAT wasn't
 * specified then a negative dentry may be returned.
 *
 * An error code is returned otherwise.
 *
 * FILE_CREATE will be set in @*opened if the dentry was created and will be
 * cleared otherwise prior to returning.
M
Miklos Szeredi 已提交
3100
 */
3101 3102 3103
static int lookup_open(struct nameidata *nd, struct path *path,
			struct file *file,
			const struct open_flags *op,
3104
			bool got_write, int *opened)
M
Miklos Szeredi 已提交
3105 3106
{
	struct dentry *dir = nd->path.dentry;
3107
	struct inode *dir_inode = dir->d_inode;
3108
	int open_flag = op->open_flag;
M
Miklos Szeredi 已提交
3109
	struct dentry *dentry;
3110 3111
	int error, create_error = 0;
	umode_t mode = op->mode;
3112
	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
M
Miklos Szeredi 已提交
3113

3114 3115
	if (unlikely(IS_DEADDIR(dir_inode)))
		return -ENOENT;
M
Miklos Szeredi 已提交
3116

3117
	*opened &= ~FILE_CREATED;
3118 3119 3120 3121 3122 3123 3124 3125 3126
	dentry = d_lookup(dir, &nd->last);
	for (;;) {
		if (!dentry) {
			dentry = d_alloc_parallel(dir, &nd->last, &wq);
			if (IS_ERR(dentry))
				return PTR_ERR(dentry);
		}
		if (d_in_lookup(dentry))
			break;
M
Miklos Szeredi 已提交
3127

3128 3129 3130 3131 3132 3133 3134 3135 3136 3137
		error = d_revalidate(dentry, nd->flags);
		if (likely(error > 0))
			break;
		if (error)
			goto out_dput;
		d_invalidate(dentry);
		dput(dentry);
		dentry = NULL;
	}
	if (dentry->d_inode) {
3138
		/* Cached positive dentry: will open in f_op->open */
M
Miklos Szeredi 已提交
3139
		goto out_no_open;
3140
	}
M
Miklos Szeredi 已提交
3141

3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174
	/*
	 * Checking write permission is tricky, bacuse we don't know if we are
	 * going to actually need it: O_CREAT opens should work as long as the
	 * file exists.  But checking existence breaks atomicity.  The trick is
	 * to check access and if not granted clear O_CREAT from the flags.
	 *
	 * Another problem is returing the "right" error value (e.g. for an
	 * O_EXCL open we want to return EEXIST not EROFS).
	 */
	if (open_flag & O_CREAT) {
		if (!IS_POSIXACL(dir->d_inode))
			mode &= ~current_umask();
		if (unlikely(!got_write)) {
			create_error = -EROFS;
			open_flag &= ~O_CREAT;
			if (open_flag & (O_EXCL | O_TRUNC))
				goto no_open;
			/* No side effects, safe to clear O_CREAT */
		} else {
			create_error = may_o_create(&nd->path, dentry, mode);
			if (create_error) {
				open_flag &= ~O_CREAT;
				if (open_flag & O_EXCL)
					goto no_open;
			}
		}
	} else if ((open_flag & (O_TRUNC|O_WRONLY|O_RDWR)) &&
		   unlikely(!got_write)) {
		/*
		 * No O_CREATE -> atomicity not a requirement -> fall
		 * back to lookup + open
		 */
		goto no_open;
M
Miklos Szeredi 已提交
3175 3176
	}

3177
	if (dir_inode->i_op->atomic_open) {
3178 3179 3180 3181 3182
		error = atomic_open(nd, dentry, path, file, op, open_flag,
				    mode, opened);
		if (unlikely(error == -ENOENT) && create_error)
			error = create_error;
		return error;
M
Miklos Szeredi 已提交
3183
	}
3184

3185
no_open:
3186
	if (d_in_lookup(dentry)) {
3187 3188
		struct dentry *res = dir_inode->i_op->lookup(dir_inode, dentry,
							     nd->flags);
3189
		d_lookup_done(dentry);
3190 3191 3192 3193 3194 3195 3196 3197
		if (unlikely(res)) {
			if (IS_ERR(res)) {
				error = PTR_ERR(res);
				goto out_dput;
			}
			dput(dentry);
			dentry = res;
		}
3198 3199
	}

M
Miklos Szeredi 已提交
3200
	/* Negative dentry, just create the file */
3201
	if (!dentry->d_inode && (open_flag & O_CREAT)) {
3202
		*opened |= FILE_CREATED;
3203 3204 3205
		audit_inode_child(dir_inode, dentry, AUDIT_TYPE_CHILD_CREATE);
		if (!dir_inode->i_op->create) {
			error = -EACCES;
M
Miklos Szeredi 已提交
3206
			goto out_dput;
3207 3208
		}
		error = dir_inode->i_op->create(dir_inode, dentry, mode,
3209
						open_flag & O_EXCL);
M
Miklos Szeredi 已提交
3210 3211
		if (error)
			goto out_dput;
3212
		fsnotify_create(dir_inode, dentry);
M
Miklos Szeredi 已提交
3213
	}
3214 3215 3216
	if (unlikely(create_error) && !dentry->d_inode) {
		error = create_error;
		goto out_dput;
M
Miklos Szeredi 已提交
3217
	}
M
Miklos Szeredi 已提交
3218
out_no_open:
M
Miklos Szeredi 已提交
3219 3220
	path->dentry = dentry;
	path->mnt = nd->path.mnt;
3221
	return 1;
M
Miklos Szeredi 已提交
3222 3223 3224

out_dput:
	dput(dentry);
3225
	return error;
M
Miklos Szeredi 已提交
3226 3227
}

N
Nick Piggin 已提交
3228
/*
3229
 * Handle the last step of open()
N
Nick Piggin 已提交
3230
 */
3231
static int do_last(struct nameidata *nd,
3232
		   struct file *file, const struct open_flags *op,
A
Al Viro 已提交
3233
		   int *opened)
3234
{
3235
	struct dentry *dir = nd->path.dentry;
3236
	int open_flag = op->open_flag;
M
Miklos Szeredi 已提交
3237
	bool will_truncate = (open_flag & O_TRUNC) != 0;
3238
	bool got_write = false;
A
Al Viro 已提交
3239
	int acc_mode = op->acc_mode;
3240
	unsigned seq;
3241
	struct inode *inode;
3242
	struct path path;
A
Al Viro 已提交
3243
	int error;
3244

3245 3246 3247
	nd->flags &= ~LOOKUP_PARENT;
	nd->flags |= op->intent;

3248
	if (nd->last_type != LAST_NORM) {
3249
		error = handle_dots(nd, nd->last_type);
3250
		if (unlikely(error))
3251
			return error;
M
Miklos Szeredi 已提交
3252
		goto finish_open;
3253
	}
3254

3255
	if (!(open_flag & O_CREAT)) {
3256 3257 3258
		if (nd->last.name[nd->last.len])
			nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
		/* we _can_ be in RCU mode here */
3259
		error = lookup_fast(nd, &path, &inode, &seq);
3260
		if (likely(error > 0))
3261 3262 3263
			goto finish_lookup;

		if (error < 0)
3264
			return error;
3265 3266

		BUG_ON(nd->inode != dir->d_inode);
A
Al Viro 已提交
3267
		BUG_ON(nd->flags & LOOKUP_RCU);
3268 3269 3270 3271 3272 3273 3274 3275
	} else {
		/* create side of things */
		/*
		 * This will *only* deal with leaving RCU mode - LOOKUP_JUMPED
		 * has been cleared when we got to the last component we are
		 * about to look up
		 */
		error = complete_walk(nd);
3276
		if (error)
3277
			return error;
3278

A
Al Viro 已提交
3279
		audit_inode(nd->name, dir, LOOKUP_PARENT);
3280
		/* trailing slashes? */
3281 3282
		if (unlikely(nd->last.name[nd->last.len]))
			return -EISDIR;
3283
	}
A
Al Viro 已提交
3284

3285
	if (open_flag & (O_CREAT | O_TRUNC | O_WRONLY | O_RDWR)) {
3286 3287 3288 3289 3290 3291 3292 3293 3294
		error = mnt_want_write(nd->path.mnt);
		if (!error)
			got_write = true;
		/*
		 * do _not_ fail yet - we might not need that or fail with
		 * a different error; let lookup_open() decide; we'll be
		 * dropping this one anyway.
		 */
	}
3295 3296 3297 3298
	if (open_flag & O_CREAT)
		inode_lock(dir->d_inode);
	else
		inode_lock_shared(dir->d_inode);
3299
	error = lookup_open(nd, &path, file, op, got_write, opened);
3300 3301 3302 3303
	if (open_flag & O_CREAT)
		inode_unlock(dir->d_inode);
	else
		inode_unlock_shared(dir->d_inode);
3304

3305 3306
	if (error <= 0) {
		if (error)
M
Miklos Szeredi 已提交
3307 3308
			goto out;

3309
		if ((*opened & FILE_CREATED) ||
A
Al Viro 已提交
3310
		    !S_ISREG(file_inode(file)->i_mode))
M
Miklos Szeredi 已提交
3311
			will_truncate = false;
M
Miklos Szeredi 已提交
3312

A
Al Viro 已提交
3313
		audit_inode(nd->name, file->f_path.dentry, 0);
M
Miklos Szeredi 已提交
3314 3315
		goto opened;
	}
3316

3317
	if (*opened & FILE_CREATED) {
3318
		/* Don't check for write permission, don't truncate */
3319
		open_flag &= ~O_TRUNC;
M
Miklos Szeredi 已提交
3320
		will_truncate = false;
A
Al Viro 已提交
3321
		acc_mode = 0;
3322
		path_to_nameidata(&path, nd);
M
Miklos Szeredi 已提交
3323
		goto finish_open_created;
3324 3325
	}

M
Miklos Szeredi 已提交
3326 3327 3328 3329 3330
	/*
	 * If atomic_open() acquired write access it is dropped now due to
	 * possible mount and symlink following (this might be optimized away if
	 * necessary...)
	 */
3331
	if (got_write) {
M
Miklos Szeredi 已提交
3332
		mnt_drop_write(nd->path.mnt);
3333
		got_write = false;
M
Miklos Szeredi 已提交
3334 3335
	}

A
Al Viro 已提交
3336 3337 3338 3339
	error = follow_managed(&path, nd);
	if (unlikely(error < 0))
		return error;

A
Al Viro 已提交
3340 3341 3342 3343 3344 3345 3346 3347 3348 3349
	if (unlikely(d_is_negative(path.dentry))) {
		path_to_nameidata(&path, nd);
		return -ENOENT;
	}

	/*
	 * create/update audit record if it already exists.
	 */
	audit_inode(nd->name, path.dentry, 0);

3350 3351 3352 3353
	if (unlikely((open_flag & (O_EXCL | O_CREAT)) == (O_EXCL | O_CREAT))) {
		path_to_nameidata(&path, nd);
		return -EEXIST;
	}
3354

3355
	seq = 0;	/* out of RCU mode, so the value doesn't matter */
3356
	inode = d_backing_inode(path.dentry);
3357
finish_lookup:
3358
	error = step_into(nd, &path, 0, inode, seq);
3359
	if (unlikely(error))
3360
		return error;
3361
finish_open:
3362
	/* Why this, you ask?  _Now_ we might have grown LOOKUP_JUMPED... */
3363
	error = complete_walk(nd);
A
Al Viro 已提交
3364
	if (error)
3365
		return error;
A
Al Viro 已提交
3366
	audit_inode(nd->name, nd->path.dentry, 0);
3367
	error = -EISDIR;
M
Miklos Szeredi 已提交
3368
	if ((open_flag & O_CREAT) && d_is_dir(nd->path.dentry))
3369
		goto out;
3370
	error = -ENOTDIR;
M
Miklos Szeredi 已提交
3371
	if ((nd->flags & LOOKUP_DIRECTORY) && !d_can_lookup(nd->path.dentry))
3372
		goto out;
3373
	if (!d_is_reg(nd->path.dentry))
M
Miklos Szeredi 已提交
3374
		will_truncate = false;
3375

3376 3377 3378
	if (will_truncate) {
		error = mnt_want_write(nd->path.mnt);
		if (error)
3379
			goto out;
3380
		got_write = true;
3381
	}
M
Miklos Szeredi 已提交
3382
finish_open_created:
3383 3384 3385
	error = may_open(&nd->path, acc_mode, open_flag);
	if (error)
		goto out;
M
Miklos Szeredi 已提交
3386 3387
	BUG_ON(*opened & FILE_OPENED); /* once it's opened, it's opened */
	error = vfs_open(&nd->path, file, current_cred());
A
Al Viro 已提交
3388
	if (error)
3389
		goto out;
A
Al Viro 已提交
3390
	*opened |= FILE_OPENED;
3391
opened:
3392
	error = open_check_o_direct(file);
3393 3394 3395
	if (!error)
		error = ima_file_check(file, op->acc_mode, *opened);
	if (!error && will_truncate)
3396
		error = handle_truncate(file);
3397
out:
3398 3399
	if (unlikely(error) && (*opened & FILE_OPENED))
		fput(file);
3400 3401 3402 3403
	if (unlikely(error > 0)) {
		WARN_ON(1);
		error = -EINVAL;
	}
3404
	if (got_write)
3405
		mnt_drop_write(nd->path.mnt);
3406
	return error;
3407 3408
}

3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423
struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode, int open_flag)
{
	struct dentry *child = NULL;
	struct inode *dir = dentry->d_inode;
	struct inode *inode;
	int error;

	/* we want directory to be writable */
	error = inode_permission(dir, MAY_WRITE | MAY_EXEC);
	if (error)
		goto out_err;
	error = -EOPNOTSUPP;
	if (!dir->i_op->tmpfile)
		goto out_err;
	error = -ENOMEM;
D
David Howells 已提交
3424
	child = d_alloc(dentry, &slash_name);
3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446
	if (unlikely(!child))
		goto out_err;
	error = dir->i_op->tmpfile(dir, child, mode);
	if (error)
		goto out_err;
	error = -ENOENT;
	inode = child->d_inode;
	if (unlikely(!inode))
		goto out_err;
	if (!(open_flag & O_EXCL)) {
		spin_lock(&inode->i_lock);
		inode->i_state |= I_LINKABLE;
		spin_unlock(&inode->i_lock);
	}
	return child;

out_err:
	dput(child);
	return ERR_PTR(error);
}
EXPORT_SYMBOL(vfs_tmpfile);

3447
static int do_tmpfile(struct nameidata *nd, unsigned flags,
3448 3449 3450
		const struct open_flags *op,
		struct file *file, int *opened)
{
3451 3452
	struct dentry *child;
	struct path path;
3453
	int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path);
3454 3455
	if (unlikely(error))
		return error;
3456
	error = mnt_want_write(path.mnt);
3457 3458
	if (unlikely(error))
		goto out;
3459 3460 3461
	child = vfs_tmpfile(path.dentry, op->mode, op->open_flag);
	error = PTR_ERR(child);
	if (unlikely(IS_ERR(child)))
3462
		goto out2;
3463 3464
	dput(path.dentry);
	path.dentry = child;
3465
	audit_inode(nd->name, child, 0);
3466
	/* Don't check for other permissions, the inode was just created */
A
Al Viro 已提交
3467
	error = may_open(&path, 0, op->open_flag);
3468 3469
	if (error)
		goto out2;
3470 3471
	file->f_path.mnt = path.mnt;
	error = finish_open(file, child, NULL, opened);
3472 3473 3474
	if (error)
		goto out2;
	error = open_check_o_direct(file);
3475
	if (error)
3476 3477
		fput(file);
out2:
3478
	mnt_drop_write(path.mnt);
3479
out:
3480
	path_put(&path);
3481 3482 3483
	return error;
}

3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495
static int do_o_path(struct nameidata *nd, unsigned flags, struct file *file)
{
	struct path path;
	int error = path_lookupat(nd, flags, &path);
	if (!error) {
		audit_inode(nd->name, path.dentry, 0);
		error = vfs_open(&path, file, current_cred());
		path_put(&path);
	}
	return error;
}

3496 3497
static struct file *path_openat(struct nameidata *nd,
			const struct open_flags *op, unsigned flags)
L
Linus Torvalds 已提交
3498
{
3499
	const char *s;
A
Al Viro 已提交
3500
	struct file *file;
3501
	int opened = 0;
3502
	int error;
N
Nick Piggin 已提交
3503

A
Al Viro 已提交
3504
	file = get_empty_filp();
3505 3506
	if (IS_ERR(file))
		return file;
N
Nick Piggin 已提交
3507

A
Al Viro 已提交
3508
	file->f_flags = op->open_flag;
N
Nick Piggin 已提交
3509

A
Al Viro 已提交
3510
	if (unlikely(file->f_flags & __O_TMPFILE)) {
3511
		error = do_tmpfile(nd, flags, op, file, &opened);
A
Al Viro 已提交
3512
		goto out2;
3513 3514
	}

3515 3516 3517 3518 3519 3520 3521
	if (unlikely(file->f_flags & O_PATH)) {
		error = do_o_path(nd, flags, file);
		if (!error)
			opened |= FILE_OPENED;
		goto out2;
	}

3522
	s = path_init(nd, flags);
3523 3524 3525 3526
	if (IS_ERR(s)) {
		put_filp(file);
		return ERR_CAST(s);
	}
3527
	while (!(error = link_path_walk(s, nd)) &&
A
Al Viro 已提交
3528
		(error = do_last(nd, file, op, &opened)) > 0) {
A
Al Viro 已提交
3529
		nd->flags &= ~(LOOKUP_OPEN|LOOKUP_CREATE|LOOKUP_EXCL);
3530 3531 3532
		s = trailing_symlink(nd);
		if (IS_ERR(s)) {
			error = PTR_ERR(s);
3533
			break;
3534
		}
3535
	}
3536
	terminate_walk(nd);
A
Al Viro 已提交
3537
out2:
3538 3539
	if (!(opened & FILE_OPENED)) {
		BUG_ON(!error);
A
Al Viro 已提交
3540
		put_filp(file);
3541
	}
3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
	if (unlikely(error)) {
		if (error == -EOPENSTALE) {
			if (flags & LOOKUP_RCU)
				error = -ECHILD;
			else
				error = -ESTALE;
		}
		file = ERR_PTR(error);
	}
	return file;
L
Linus Torvalds 已提交
3552 3553
}

3554
struct file *do_filp_open(int dfd, struct filename *pathname,
3555
		const struct open_flags *op)
3556
{
3557
	struct nameidata nd;
3558
	int flags = op->lookup_flags;
3559 3560
	struct file *filp;

3561
	set_nameidata(&nd, dfd, pathname);
3562
	filp = path_openat(&nd, op, flags | LOOKUP_RCU);
3563
	if (unlikely(filp == ERR_PTR(-ECHILD)))
3564
		filp = path_openat(&nd, op, flags);
3565
	if (unlikely(filp == ERR_PTR(-ESTALE)))
3566
		filp = path_openat(&nd, op, flags | LOOKUP_REVAL);
3567
	restore_nameidata();
3568 3569 3570
	return filp;
}

A
Al Viro 已提交
3571
struct file *do_file_open_root(struct dentry *dentry, struct vfsmount *mnt,
3572
		const char *name, const struct open_flags *op)
A
Al Viro 已提交
3573
{
3574
	struct nameidata nd;
A
Al Viro 已提交
3575
	struct file *file;
3576
	struct filename *filename;
3577
	int flags = op->lookup_flags | LOOKUP_ROOT;
A
Al Viro 已提交
3578 3579 3580 3581

	nd.root.mnt = mnt;
	nd.root.dentry = dentry;

3582
	if (d_is_symlink(dentry) && op->intent & LOOKUP_OPEN)
A
Al Viro 已提交
3583 3584
		return ERR_PTR(-ELOOP);

3585
	filename = getname_kernel(name);
3586
	if (IS_ERR(filename))
3587 3588
		return ERR_CAST(filename);

3589
	set_nameidata(&nd, -1, filename);
3590
	file = path_openat(&nd, op, flags | LOOKUP_RCU);
A
Al Viro 已提交
3591
	if (unlikely(file == ERR_PTR(-ECHILD)))
3592
		file = path_openat(&nd, op, flags);
A
Al Viro 已提交
3593
	if (unlikely(file == ERR_PTR(-ESTALE)))
3594
		file = path_openat(&nd, op, flags | LOOKUP_REVAL);
3595
	restore_nameidata();
3596
	putname(filename);
A
Al Viro 已提交
3597 3598 3599
	return file;
}

3600
static struct dentry *filename_create(int dfd, struct filename *name,
3601
				struct path *path, unsigned int lookup_flags)
L
Linus Torvalds 已提交
3602
{
3603
	struct dentry *dentry = ERR_PTR(-EEXIST);
3604 3605
	struct qstr last;
	int type;
3606
	int err2;
3607 3608 3609 3610 3611 3612 3613 3614 3615
	int error;
	bool is_dir = (lookup_flags & LOOKUP_DIRECTORY);

	/*
	 * Note that only LOOKUP_REVAL and LOOKUP_DIRECTORY matter here. Any
	 * other flags passed in are ignored!
	 */
	lookup_flags &= LOOKUP_REVAL;

3616 3617 3618
	name = filename_parentat(dfd, name, lookup_flags, path, &last, &type);
	if (IS_ERR(name))
		return ERR_CAST(name);
L
Linus Torvalds 已提交
3619

3620 3621 3622 3623
	/*
	 * Yucky last component or no last component at all?
	 * (foo/., foo/.., /////)
	 */
3624
	if (unlikely(type != LAST_NORM))
A
Al Viro 已提交
3625
		goto out;
3626

3627
	/* don't fail immediately if it's r/o, at least try to report other errors */
3628
	err2 = mnt_want_write(path->mnt);
3629 3630 3631
	/*
	 * Do the final lookup.
	 */
3632
	lookup_flags |= LOOKUP_CREATE | LOOKUP_EXCL;
A
Al Viro 已提交
3633
	inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
3634
	dentry = __lookup_hash(&last, path->dentry, lookup_flags);
L
Linus Torvalds 已提交
3635
	if (IS_ERR(dentry))
3636
		goto unlock;
3637

3638
	error = -EEXIST;
3639
	if (d_is_positive(dentry))
3640
		goto fail;
3641

3642 3643 3644 3645 3646 3647
	/*
	 * Special case - lookup gave negative, but... we had foo/bar/
	 * From the vfs_mknod() POV we just have a negative dentry -
	 * all is fine. Let's be bastards - you had / on the end, you've
	 * been asking for (non-existent) directory. -ENOENT for you.
	 */
3648
	if (unlikely(!is_dir && last.name[last.len])) {
3649
		error = -ENOENT;
A
Al Viro 已提交
3650
		goto fail;
3651
	}
3652 3653
	if (unlikely(err2)) {
		error = err2;
3654
		goto fail;
3655
	}
3656
	putname(name);
L
Linus Torvalds 已提交
3657 3658
	return dentry;
fail:
3659 3660 3661
	dput(dentry);
	dentry = ERR_PTR(error);
unlock:
A
Al Viro 已提交
3662
	inode_unlock(path->dentry->d_inode);
3663
	if (!err2)
3664
		mnt_drop_write(path->mnt);
A
Al Viro 已提交
3665
out:
3666
	path_put(path);
3667
	putname(name);
L
Linus Torvalds 已提交
3668 3669
	return dentry;
}
3670 3671 3672 3673

struct dentry *kern_path_create(int dfd, const char *pathname,
				struct path *path, unsigned int lookup_flags)
{
3674 3675
	return filename_create(dfd, getname_kernel(pathname),
				path, lookup_flags);
3676
}
3677 3678
EXPORT_SYMBOL(kern_path_create);

A
Al Viro 已提交
3679 3680 3681
void done_path_create(struct path *path, struct dentry *dentry)
{
	dput(dentry);
A
Al Viro 已提交
3682
	inode_unlock(path->dentry->d_inode);
3683
	mnt_drop_write(path->mnt);
A
Al Viro 已提交
3684 3685 3686 3687
	path_put(path);
}
EXPORT_SYMBOL(done_path_create);

A
Al Viro 已提交
3688
inline struct dentry *user_path_create(int dfd, const char __user *pathname,
3689
				struct path *path, unsigned int lookup_flags)
3690
{
3691
	return filename_create(dfd, getname(pathname), path, lookup_flags);
3692 3693 3694
}
EXPORT_SYMBOL(user_path_create);

A
Al Viro 已提交
3695
int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
L
Linus Torvalds 已提交
3696
{
3697
	int error = may_create(dir, dentry);
L
Linus Torvalds 已提交
3698 3699 3700 3701

	if (error)
		return error;

3702
	if ((S_ISCHR(mode) || S_ISBLK(mode)) && !capable(CAP_MKNOD))
L
Linus Torvalds 已提交
3703 3704
		return -EPERM;

A
Al Viro 已提交
3705
	if (!dir->i_op->mknod)
L
Linus Torvalds 已提交
3706 3707
		return -EPERM;

3708 3709 3710 3711
	error = devcgroup_inode_mknod(mode, dev);
	if (error)
		return error;

L
Linus Torvalds 已提交
3712 3713 3714 3715 3716
	error = security_inode_mknod(dir, dentry, mode, dev);
	if (error)
		return error;

	error = dir->i_op->mknod(dir, dentry, mode, dev);
3717
	if (!error)
3718
		fsnotify_create(dir, dentry);
L
Linus Torvalds 已提交
3719 3720
	return error;
}
3721
EXPORT_SYMBOL(vfs_mknod);
L
Linus Torvalds 已提交
3722

A
Al Viro 已提交
3723
static int may_mknod(umode_t mode)
3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739
{
	switch (mode & S_IFMT) {
	case S_IFREG:
	case S_IFCHR:
	case S_IFBLK:
	case S_IFIFO:
	case S_IFSOCK:
	case 0: /* zero mode translates to S_IFREG */
		return 0;
	case S_IFDIR:
		return -EPERM;
	default:
		return -EINVAL;
	}
}

A
Al Viro 已提交
3740
SYSCALL_DEFINE4(mknodat, int, dfd, const char __user *, filename, umode_t, mode,
3741
		unsigned, dev)
L
Linus Torvalds 已提交
3742
{
3743
	struct dentry *dentry;
3744 3745
	struct path path;
	int error;
3746
	unsigned int lookup_flags = 0;
L
Linus Torvalds 已提交
3747

3748 3749 3750
	error = may_mknod(mode);
	if (error)
		return error;
3751 3752
retry:
	dentry = user_path_create(dfd, filename, &path, lookup_flags);
3753 3754
	if (IS_ERR(dentry))
		return PTR_ERR(dentry);
3755

3756
	if (!IS_POSIXACL(path.dentry->d_inode))
A
Al Viro 已提交
3757
		mode &= ~current_umask();
3758
	error = security_path_mknod(&path, dentry, mode, dev);
3759
	if (error)
3760
		goto out;
3761
	switch (mode & S_IFMT) {
L
Linus Torvalds 已提交
3762
		case 0: case S_IFREG:
A
Al Viro 已提交
3763
			error = vfs_create(path.dentry->d_inode,dentry,mode,true);
3764 3765
			if (!error)
				ima_post_path_mknod(dentry);
L
Linus Torvalds 已提交
3766 3767
			break;
		case S_IFCHR: case S_IFBLK:
3768
			error = vfs_mknod(path.dentry->d_inode,dentry,mode,
L
Linus Torvalds 已提交
3769 3770 3771
					new_decode_dev(dev));
			break;
		case S_IFIFO: case S_IFSOCK:
3772
			error = vfs_mknod(path.dentry->d_inode,dentry,mode,0);
L
Linus Torvalds 已提交
3773 3774
			break;
	}
3775
out:
A
Al Viro 已提交
3776
	done_path_create(&path, dentry);
3777 3778 3779 3780
	if (retry_estale(error, lookup_flags)) {
		lookup_flags |= LOOKUP_REVAL;
		goto retry;
	}
L
Linus Torvalds 已提交
3781 3782 3783
	return error;
}

A
Al Viro 已提交
3784
SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, dev)
3785 3786 3787 3788
{
	return sys_mknodat(AT_FDCWD, filename, mode, dev);
}

3789
int vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
L
Linus Torvalds 已提交
3790
{
3791
	int error = may_create(dir, dentry);
3792
	unsigned max_links = dir->i_sb->s_max_links;
L
Linus Torvalds 已提交
3793 3794 3795 3796

	if (error)
		return error;

A
Al Viro 已提交
3797
	if (!dir->i_op->mkdir)
L
Linus Torvalds 已提交
3798 3799 3800 3801 3802 3803 3804
		return -EPERM;

	mode &= (S_IRWXUGO|S_ISVTX);
	error = security_inode_mkdir(dir, dentry, mode);
	if (error)
		return error;

3805 3806 3807
	if (max_links && dir->i_nlink >= max_links)
		return -EMLINK;

L
Linus Torvalds 已提交
3808
	error = dir->i_op->mkdir(dir, dentry, mode);
3809
	if (!error)
3810
		fsnotify_mkdir(dir, dentry);
L
Linus Torvalds 已提交
3811 3812
	return error;
}
3813
EXPORT_SYMBOL(vfs_mkdir);
L
Linus Torvalds 已提交
3814

3815
SYSCALL_DEFINE3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode)
L
Linus Torvalds 已提交
3816
{
3817
	struct dentry *dentry;
3818 3819
	struct path path;
	int error;
3820
	unsigned int lookup_flags = LOOKUP_DIRECTORY;
L
Linus Torvalds 已提交
3821

3822 3823
retry:
	dentry = user_path_create(dfd, pathname, &path, lookup_flags);
3824
	if (IS_ERR(dentry))
3825
		return PTR_ERR(dentry);
L
Linus Torvalds 已提交
3826

3827
	if (!IS_POSIXACL(path.dentry->d_inode))
A
Al Viro 已提交
3828
		mode &= ~current_umask();
3829
	error = security_path_mkdir(&path, dentry, mode);
3830 3831
	if (!error)
		error = vfs_mkdir(path.dentry->d_inode, dentry, mode);
A
Al Viro 已提交
3832
	done_path_create(&path, dentry);
3833 3834 3835 3836
	if (retry_estale(error, lookup_flags)) {
		lookup_flags |= LOOKUP_REVAL;
		goto retry;
	}
L
Linus Torvalds 已提交
3837 3838 3839
	return error;
}

3840
SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode)
3841 3842 3843 3844
{
	return sys_mkdirat(AT_FDCWD, pathname, mode);
}

L
Linus Torvalds 已提交
3845 3846 3847 3848 3849 3850 3851
int vfs_rmdir(struct inode *dir, struct dentry *dentry)
{
	int error = may_delete(dir, dentry, 1);

	if (error)
		return error;

A
Al Viro 已提交
3852
	if (!dir->i_op->rmdir)
L
Linus Torvalds 已提交
3853 3854
		return -EPERM;

3855
	dget(dentry);
A
Al Viro 已提交
3856
	inode_lock(dentry->d_inode);
S
Sage Weil 已提交
3857 3858

	error = -EBUSY;
3859
	if (is_local_mountpoint(dentry))
S
Sage Weil 已提交
3860 3861 3862 3863 3864 3865
		goto out;

	error = security_inode_rmdir(dir, dentry);
	if (error)
		goto out;

3866
	shrink_dcache_parent(dentry);
S
Sage Weil 已提交
3867 3868 3869 3870 3871 3872
	error = dir->i_op->rmdir(dir, dentry);
	if (error)
		goto out;

	dentry->d_inode->i_flags |= S_DEAD;
	dont_mount(dentry);
3873
	detach_mounts(dentry);
S
Sage Weil 已提交
3874 3875

out:
A
Al Viro 已提交
3876
	inode_unlock(dentry->d_inode);
3877
	dput(dentry);
S
Sage Weil 已提交
3878
	if (!error)
L
Linus Torvalds 已提交
3879 3880 3881
		d_delete(dentry);
	return error;
}
3882
EXPORT_SYMBOL(vfs_rmdir);
L
Linus Torvalds 已提交
3883

3884
static long do_rmdir(int dfd, const char __user *pathname)
L
Linus Torvalds 已提交
3885 3886
{
	int error = 0;
3887
	struct filename *name;
L
Linus Torvalds 已提交
3888
	struct dentry *dentry;
3889 3890 3891
	struct path path;
	struct qstr last;
	int type;
3892 3893
	unsigned int lookup_flags = 0;
retry:
A
Al Viro 已提交
3894 3895
	name = filename_parentat(dfd, getname(pathname), lookup_flags,
				&path, &last, &type);
3896 3897
	if (IS_ERR(name))
		return PTR_ERR(name);
L
Linus Torvalds 已提交
3898

3899
	switch (type) {
3900 3901 3902 3903 3904 3905 3906 3907 3908
	case LAST_DOTDOT:
		error = -ENOTEMPTY;
		goto exit1;
	case LAST_DOT:
		error = -EINVAL;
		goto exit1;
	case LAST_ROOT:
		error = -EBUSY;
		goto exit1;
L
Linus Torvalds 已提交
3909
	}
3910

3911
	error = mnt_want_write(path.mnt);
3912 3913
	if (error)
		goto exit1;
3914

A
Al Viro 已提交
3915
	inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
3916
	dentry = __lookup_hash(&last, path.dentry, lookup_flags);
L
Linus Torvalds 已提交
3917
	error = PTR_ERR(dentry);
3918 3919
	if (IS_ERR(dentry))
		goto exit2;
3920 3921 3922 3923
	if (!dentry->d_inode) {
		error = -ENOENT;
		goto exit3;
	}
3924
	error = security_path_rmdir(&path, dentry);
3925
	if (error)
3926
		goto exit3;
3927
	error = vfs_rmdir(path.dentry->d_inode, dentry);
3928
exit3:
3929 3930
	dput(dentry);
exit2:
A
Al Viro 已提交
3931
	inode_unlock(path.dentry->d_inode);
3932
	mnt_drop_write(path.mnt);
L
Linus Torvalds 已提交
3933
exit1:
3934
	path_put(&path);
L
Linus Torvalds 已提交
3935
	putname(name);
3936 3937 3938 3939
	if (retry_estale(error, lookup_flags)) {
		lookup_flags |= LOOKUP_REVAL;
		goto retry;
	}
L
Linus Torvalds 已提交
3940 3941 3942
	return error;
}

3943
SYSCALL_DEFINE1(rmdir, const char __user *, pathname)
3944 3945 3946 3947
{
	return do_rmdir(AT_FDCWD, pathname);
}

3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966
/**
 * vfs_unlink - unlink a filesystem object
 * @dir:	parent directory
 * @dentry:	victim
 * @delegated_inode: returns victim inode, if the inode is delegated.
 *
 * The caller must hold dir->i_mutex.
 *
 * If vfs_unlink discovers a delegation, it will return -EWOULDBLOCK and
 * return a reference to the inode in delegated_inode.  The caller
 * should then break the delegation on that inode and retry.  Because
 * breaking a delegation may take a long time, the caller should drop
 * dir->i_mutex before doing so.
 *
 * Alternatively, a caller may pass NULL for delegated_inode.  This may
 * be appropriate for callers that expect the underlying filesystem not
 * to be NFS exported.
 */
int vfs_unlink(struct inode *dir, struct dentry *dentry, struct inode **delegated_inode)
L
Linus Torvalds 已提交
3967
{
J
J. Bruce Fields 已提交
3968
	struct inode *target = dentry->d_inode;
L
Linus Torvalds 已提交
3969 3970 3971 3972 3973
	int error = may_delete(dir, dentry, 0);

	if (error)
		return error;

A
Al Viro 已提交
3974
	if (!dir->i_op->unlink)
L
Linus Torvalds 已提交
3975 3976
		return -EPERM;

A
Al Viro 已提交
3977
	inode_lock(target);
3978
	if (is_local_mountpoint(dentry))
L
Linus Torvalds 已提交
3979 3980 3981
		error = -EBUSY;
	else {
		error = security_inode_unlink(dir, dentry);
3982
		if (!error) {
3983 3984
			error = try_break_deleg(target, delegated_inode);
			if (error)
3985
				goto out;
L
Linus Torvalds 已提交
3986
			error = dir->i_op->unlink(dir, dentry);
3987
			if (!error) {
3988
				dont_mount(dentry);
3989 3990
				detach_mounts(dentry);
			}
3991
		}
L
Linus Torvalds 已提交
3992
	}
3993
out:
A
Al Viro 已提交
3994
	inode_unlock(target);
L
Linus Torvalds 已提交
3995 3996 3997

	/* We don't d_delete() NFS sillyrenamed files--they still exist. */
	if (!error && !(dentry->d_flags & DCACHE_NFSFS_RENAMED)) {
J
J. Bruce Fields 已提交
3998
		fsnotify_link_count(target);
J
John McCutchan 已提交
3999
		d_delete(dentry);
L
Linus Torvalds 已提交
4000
	}
R
Robert Love 已提交
4001

L
Linus Torvalds 已提交
4002 4003
	return error;
}
4004
EXPORT_SYMBOL(vfs_unlink);
L
Linus Torvalds 已提交
4005 4006 4007

/*
 * Make sure that the actual truncation of the file will occur outside its
4008
 * directory's i_mutex.  Truncate can take a long time if there is a lot of
L
Linus Torvalds 已提交
4009 4010 4011
 * writeout happening, and we don't want to prevent access to the directory
 * while waiting on the I/O.
 */
4012
static long do_unlinkat(int dfd, const char __user *pathname)
L
Linus Torvalds 已提交
4013
{
4014
	int error;
4015
	struct filename *name;
L
Linus Torvalds 已提交
4016
	struct dentry *dentry;
4017 4018 4019
	struct path path;
	struct qstr last;
	int type;
L
Linus Torvalds 已提交
4020
	struct inode *inode = NULL;
4021
	struct inode *delegated_inode = NULL;
4022 4023
	unsigned int lookup_flags = 0;
retry:
A
Al Viro 已提交
4024 4025
	name = filename_parentat(dfd, getname(pathname), lookup_flags,
				&path, &last, &type);
4026 4027
	if (IS_ERR(name))
		return PTR_ERR(name);
4028

L
Linus Torvalds 已提交
4029
	error = -EISDIR;
4030
	if (type != LAST_NORM)
L
Linus Torvalds 已提交
4031
		goto exit1;
4032

4033
	error = mnt_want_write(path.mnt);
4034 4035
	if (error)
		goto exit1;
4036
retry_deleg:
A
Al Viro 已提交
4037
	inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
4038
	dentry = __lookup_hash(&last, path.dentry, lookup_flags);
L
Linus Torvalds 已提交
4039 4040 4041
	error = PTR_ERR(dentry);
	if (!IS_ERR(dentry)) {
		/* Why not before? Because we want correct error value */
4042
		if (last.name[last.len])
4043
			goto slashes;
L
Linus Torvalds 已提交
4044
		inode = dentry->d_inode;
4045
		if (d_is_negative(dentry))
4046 4047
			goto slashes;
		ihold(inode);
4048
		error = security_path_unlink(&path, dentry);
4049
		if (error)
4050
			goto exit2;
4051
		error = vfs_unlink(path.dentry->d_inode, dentry, &delegated_inode);
4052
exit2:
L
Linus Torvalds 已提交
4053 4054
		dput(dentry);
	}
A
Al Viro 已提交
4055
	inode_unlock(path.dentry->d_inode);
L
Linus Torvalds 已提交
4056 4057
	if (inode)
		iput(inode);	/* truncate the inode here */
4058 4059
	inode = NULL;
	if (delegated_inode) {
4060
		error = break_deleg_wait(&delegated_inode);
4061 4062 4063
		if (!error)
			goto retry_deleg;
	}
4064
	mnt_drop_write(path.mnt);
L
Linus Torvalds 已提交
4065
exit1:
4066
	path_put(&path);
L
Linus Torvalds 已提交
4067
	putname(name);
4068 4069 4070 4071 4072
	if (retry_estale(error, lookup_flags)) {
		lookup_flags |= LOOKUP_REVAL;
		inode = NULL;
		goto retry;
	}
L
Linus Torvalds 已提交
4073 4074 4075
	return error;

slashes:
4076 4077
	if (d_is_negative(dentry))
		error = -ENOENT;
M
Miklos Szeredi 已提交
4078
	else if (d_is_dir(dentry))
4079 4080 4081
		error = -EISDIR;
	else
		error = -ENOTDIR;
L
Linus Torvalds 已提交
4082 4083 4084
	goto exit2;
}

4085
SYSCALL_DEFINE3(unlinkat, int, dfd, const char __user *, pathname, int, flag)
4086 4087 4088 4089 4090 4091 4092 4093 4094 4095
{
	if ((flag & ~AT_REMOVEDIR) != 0)
		return -EINVAL;

	if (flag & AT_REMOVEDIR)
		return do_rmdir(dfd, pathname);

	return do_unlinkat(dfd, pathname);
}

4096
SYSCALL_DEFINE1(unlink, const char __user *, pathname)
4097 4098 4099 4100
{
	return do_unlinkat(AT_FDCWD, pathname);
}

4101
int vfs_symlink(struct inode *dir, struct dentry *dentry, const char *oldname)
L
Linus Torvalds 已提交
4102
{
4103
	int error = may_create(dir, dentry);
L
Linus Torvalds 已提交
4104 4105 4106 4107

	if (error)
		return error;

A
Al Viro 已提交
4108
	if (!dir->i_op->symlink)
L
Linus Torvalds 已提交
4109 4110 4111 4112 4113 4114 4115
		return -EPERM;

	error = security_inode_symlink(dir, dentry, oldname);
	if (error)
		return error;

	error = dir->i_op->symlink(dir, dentry, oldname);
4116
	if (!error)
4117
		fsnotify_create(dir, dentry);
L
Linus Torvalds 已提交
4118 4119
	return error;
}
4120
EXPORT_SYMBOL(vfs_symlink);
L
Linus Torvalds 已提交
4121

4122 4123
SYSCALL_DEFINE3(symlinkat, const char __user *, oldname,
		int, newdfd, const char __user *, newname)
L
Linus Torvalds 已提交
4124
{
4125
	int error;
4126
	struct filename *from;
4127
	struct dentry *dentry;
4128
	struct path path;
4129
	unsigned int lookup_flags = 0;
L
Linus Torvalds 已提交
4130 4131

	from = getname(oldname);
4132
	if (IS_ERR(from))
L
Linus Torvalds 已提交
4133
		return PTR_ERR(from);
4134 4135
retry:
	dentry = user_path_create(newdfd, newname, &path, lookup_flags);
4136 4137
	error = PTR_ERR(dentry);
	if (IS_ERR(dentry))
4138
		goto out_putname;
4139

4140
	error = security_path_symlink(&path, dentry, from->name);
4141
	if (!error)
4142
		error = vfs_symlink(path.dentry->d_inode, dentry, from->name);
A
Al Viro 已提交
4143
	done_path_create(&path, dentry);
4144 4145 4146 4147
	if (retry_estale(error, lookup_flags)) {
		lookup_flags |= LOOKUP_REVAL;
		goto retry;
	}
4148
out_putname:
L
Linus Torvalds 已提交
4149 4150 4151 4152
	putname(from);
	return error;
}

4153
SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newname)
4154 4155 4156 4157
{
	return sys_symlinkat(oldname, AT_FDCWD, newname);
}

J
J. Bruce Fields 已提交
4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177
/**
 * vfs_link - create a new link
 * @old_dentry:	object to be linked
 * @dir:	new parent
 * @new_dentry:	where to create the new link
 * @delegated_inode: returns inode needing a delegation break
 *
 * The caller must hold dir->i_mutex
 *
 * If vfs_link discovers a delegation on the to-be-linked file in need
 * of breaking, it will return -EWOULDBLOCK and return a reference to the
 * inode in delegated_inode.  The caller should then break the delegation
 * and retry.  Because breaking a delegation may take a long time, the
 * caller should drop the i_mutex before doing so.
 *
 * Alternatively, a caller may pass NULL for delegated_inode.  This may
 * be appropriate for callers that expect the underlying filesystem not
 * to be NFS exported.
 */
int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry, struct inode **delegated_inode)
L
Linus Torvalds 已提交
4178 4179
{
	struct inode *inode = old_dentry->d_inode;
4180
	unsigned max_links = dir->i_sb->s_max_links;
L
Linus Torvalds 已提交
4181 4182 4183 4184 4185
	int error;

	if (!inode)
		return -ENOENT;

4186
	error = may_create(dir, new_dentry);
L
Linus Torvalds 已提交
4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197
	if (error)
		return error;

	if (dir->i_sb != inode->i_sb)
		return -EXDEV;

	/*
	 * A link to an append-only or immutable file cannot be created.
	 */
	if (IS_APPEND(inode) || IS_IMMUTABLE(inode))
		return -EPERM;
4198 4199 4200 4201 4202 4203 4204
	/*
	 * Updating the link count will likely cause i_uid and i_gid to
	 * be writen back improperly if their true value is unknown to
	 * the vfs.
	 */
	if (HAS_UNMAPPED_ID(inode))
		return -EPERM;
A
Al Viro 已提交
4205
	if (!dir->i_op->link)
L
Linus Torvalds 已提交
4206
		return -EPERM;
4207
	if (S_ISDIR(inode->i_mode))
L
Linus Torvalds 已提交
4208 4209 4210 4211 4212 4213
		return -EPERM;

	error = security_inode_link(old_dentry, dir, new_dentry);
	if (error)
		return error;

A
Al Viro 已提交
4214
	inode_lock(inode);
4215
	/* Make sure we don't allow creating hardlink to an unlinked file */
4216
	if (inode->i_nlink == 0 && !(inode->i_state & I_LINKABLE))
4217
		error =  -ENOENT;
4218 4219
	else if (max_links && inode->i_nlink >= max_links)
		error = -EMLINK;
J
J. Bruce Fields 已提交
4220 4221 4222 4223 4224
	else {
		error = try_break_deleg(inode, delegated_inode);
		if (!error)
			error = dir->i_op->link(old_dentry, dir, new_dentry);
	}
4225 4226 4227 4228 4229 4230

	if (!error && (inode->i_state & I_LINKABLE)) {
		spin_lock(&inode->i_lock);
		inode->i_state &= ~I_LINKABLE;
		spin_unlock(&inode->i_lock);
	}
A
Al Viro 已提交
4231
	inode_unlock(inode);
4232
	if (!error)
4233
		fsnotify_link(dir, inode, new_dentry);
L
Linus Torvalds 已提交
4234 4235
	return error;
}
4236
EXPORT_SYMBOL(vfs_link);
L
Linus Torvalds 已提交
4237 4238 4239 4240 4241 4242 4243 4244 4245 4246

/*
 * Hardlinks are often used in delicate situations.  We avoid
 * security-related surprises by not following symlinks on the
 * newname.  --KAB
 *
 * We don't follow them on the oldname either to be compatible
 * with linux 2.0, and to avoid hard-linking to directories
 * and other special files.  --ADM
 */
4247 4248
SYSCALL_DEFINE5(linkat, int, olddfd, const char __user *, oldname,
		int, newdfd, const char __user *, newname, int, flags)
L
Linus Torvalds 已提交
4249 4250
{
	struct dentry *new_dentry;
4251
	struct path old_path, new_path;
J
J. Bruce Fields 已提交
4252
	struct inode *delegated_inode = NULL;
4253
	int how = 0;
L
Linus Torvalds 已提交
4254 4255
	int error;

4256
	if ((flags & ~(AT_SYMLINK_FOLLOW | AT_EMPTY_PATH)) != 0)
4257
		return -EINVAL;
4258
	/*
4259 4260 4261
	 * To use null names we require CAP_DAC_READ_SEARCH
	 * This ensures that not everyone will be able to create
	 * handlink using the passed filedescriptor.
4262
	 */
4263 4264 4265
	if (flags & AT_EMPTY_PATH) {
		if (!capable(CAP_DAC_READ_SEARCH))
			return -ENOENT;
4266
		how = LOOKUP_EMPTY;
4267
	}
4268 4269 4270

	if (flags & AT_SYMLINK_FOLLOW)
		how |= LOOKUP_FOLLOW;
4271
retry:
4272
	error = user_path_at(olddfd, oldname, how, &old_path);
L
Linus Torvalds 已提交
4273
	if (error)
4274 4275
		return error;

4276 4277
	new_dentry = user_path_create(newdfd, newname, &new_path,
					(how & LOOKUP_REVAL));
L
Linus Torvalds 已提交
4278
	error = PTR_ERR(new_dentry);
4279
	if (IS_ERR(new_dentry))
4280 4281 4282 4283 4284
		goto out;

	error = -EXDEV;
	if (old_path.mnt != new_path.mnt)
		goto out_dput;
K
Kees Cook 已提交
4285 4286 4287
	error = may_linkat(&old_path);
	if (unlikely(error))
		goto out_dput;
4288
	error = security_path_link(old_path.dentry, &new_path, new_dentry);
4289
	if (error)
4290
		goto out_dput;
J
J. Bruce Fields 已提交
4291
	error = vfs_link(old_path.dentry, new_path.dentry->d_inode, new_dentry, &delegated_inode);
4292
out_dput:
A
Al Viro 已提交
4293
	done_path_create(&new_path, new_dentry);
J
J. Bruce Fields 已提交
4294 4295
	if (delegated_inode) {
		error = break_deleg_wait(&delegated_inode);
4296 4297
		if (!error) {
			path_put(&old_path);
J
J. Bruce Fields 已提交
4298
			goto retry;
4299
		}
J
J. Bruce Fields 已提交
4300
	}
4301
	if (retry_estale(error, how)) {
4302
		path_put(&old_path);
4303 4304 4305
		how |= LOOKUP_REVAL;
		goto retry;
	}
L
Linus Torvalds 已提交
4306
out:
4307
	path_put(&old_path);
L
Linus Torvalds 已提交
4308 4309 4310 4311

	return error;
}

4312
SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname)
4313
{
4314
	return sys_linkat(AT_FDCWD, oldname, AT_FDCWD, newname, 0);
4315 4316
}

4317 4318 4319 4320 4321 4322 4323
/**
 * vfs_rename - rename a filesystem object
 * @old_dir:	parent of source
 * @old_dentry:	source
 * @new_dir:	parent of destination
 * @new_dentry:	destination
 * @delegated_inode: returns an inode needing a delegation break
M
Miklos Szeredi 已提交
4324
 * @flags:	rename flags
4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338
 *
 * The caller must hold multiple mutexes--see lock_rename()).
 *
 * If vfs_rename discovers a delegation in need of breaking at either
 * the source or destination, it will return -EWOULDBLOCK and return a
 * reference to the inode in delegated_inode.  The caller should then
 * break the delegation and retry.  Because breaking a delegation may
 * take a long time, the caller should drop all locks before doing
 * so.
 *
 * Alternatively, a caller may pass NULL for delegated_inode.  This may
 * be appropriate for callers that expect the underlying filesystem not
 * to be NFS exported.
 *
L
Linus Torvalds 已提交
4339 4340 4341
 * The worst of all namespace operations - renaming directory. "Perverted"
 * doesn't even start to describe it. Somebody in UCB had a heck of a trip...
 * Problems:
4342
 *
4343
 *	a) we can get into loop creation.
L
Linus Torvalds 已提交
4344 4345
 *	b) race potential - two innocent renames can create a loop together.
 *	   That's where 4.4 screws up. Current fix: serialization on
4346
 *	   sb->s_vfs_rename_mutex. We might be more accurate, but that's another
L
Linus Torvalds 已提交
4347
 *	   story.
4348 4349
 *	c) we have to lock _four_ objects - parents and victim (if it exists),
 *	   and source (if it is not a directory).
4350
 *	   And that - after we got ->i_mutex on parents (until then we don't know
L
Linus Torvalds 已提交
4351 4352
 *	   whether the target exists).  Solution: try to be smart with locking
 *	   order for inodes.  We rely on the fact that tree topology may change
4353
 *	   only under ->s_vfs_rename_mutex _and_ that parent of the object we
L
Linus Torvalds 已提交
4354 4355 4356
 *	   move will be locked.  Thus we can rank directories by the tree
 *	   (ancestors first) and rank all non-directories after them.
 *	   That works since everybody except rename does "lock parent, lookup,
4357
 *	   lock child" and rename is under ->s_vfs_rename_mutex.
L
Linus Torvalds 已提交
4358 4359 4360
 *	   HOWEVER, it relies on the assumption that any object with ->lookup()
 *	   has no more than 1 dentry.  If "hybrid" objects will ever appear,
 *	   we'd better make sure that there's no link(2) for them.
4361
 *	d) conversion from fhandle to dentry may come in the wrong moment - when
4362
 *	   we are removing the target. Solution: we will have to grab ->i_mutex
L
Linus Torvalds 已提交
4363
 *	   in the fhandle_to_dentry code. [FIXME - current nfsfh.c relies on
4364
 *	   ->i_mutex on parents, which works but leads to some truly excessive
L
Linus Torvalds 已提交
4365 4366
 *	   locking].
 */
4367 4368
int vfs_rename(struct inode *old_dir, struct dentry *old_dentry,
	       struct inode *new_dir, struct dentry *new_dentry,
M
Miklos Szeredi 已提交
4369
	       struct inode **delegated_inode, unsigned int flags)
L
Linus Torvalds 已提交
4370
{
4371 4372 4373
	int error;
	bool is_dir = d_is_dir(old_dentry);
	struct inode *source = old_dentry->d_inode;
S
Sage Weil 已提交
4374
	struct inode *target = new_dentry->d_inode;
M
Miklos Szeredi 已提交
4375 4376
	bool new_is_dir = false;
	unsigned max_links = new_dir->i_sb->s_max_links;
A
Al Viro 已提交
4377
	struct name_snapshot old_name;
4378

4379
	if (source == target)
4380 4381 4382 4383 4384 4385
		return 0;

	error = may_delete(old_dir, old_dentry, is_dir);
	if (error)
		return error;

M
Miklos Szeredi 已提交
4386
	if (!target) {
4387
		error = may_create(new_dir, new_dentry);
M
Miklos Szeredi 已提交
4388 4389 4390 4391 4392 4393 4394 4395
	} else {
		new_is_dir = d_is_dir(new_dentry);

		if (!(flags & RENAME_EXCHANGE))
			error = may_delete(new_dir, new_dentry, is_dir);
		else
			error = may_delete(new_dir, new_dentry, new_is_dir);
	}
4396 4397 4398
	if (error)
		return error;

4399
	if (!old_dir->i_op->rename)
4400
		return -EPERM;
L
Linus Torvalds 已提交
4401 4402 4403 4404 4405

	/*
	 * If we are going to change the parent - check write permissions,
	 * we'll need to flip '..'.
	 */
M
Miklos Szeredi 已提交
4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416
	if (new_dir != old_dir) {
		if (is_dir) {
			error = inode_permission(source, MAY_WRITE);
			if (error)
				return error;
		}
		if ((flags & RENAME_EXCHANGE) && new_is_dir) {
			error = inode_permission(target, MAY_WRITE);
			if (error)
				return error;
		}
L
Linus Torvalds 已提交
4417 4418
	}

4419 4420
	error = security_inode_rename(old_dir, old_dentry, new_dir, new_dentry,
				      flags);
L
Linus Torvalds 已提交
4421 4422 4423
	if (error)
		return error;

A
Al Viro 已提交
4424
	take_dentry_name_snapshot(&old_name, old_dentry);
4425
	dget(new_dentry);
M
Miklos Szeredi 已提交
4426
	if (!is_dir || (flags & RENAME_EXCHANGE))
4427 4428
		lock_two_nondirectories(source, target);
	else if (target)
A
Al Viro 已提交
4429
		inode_lock(target);
S
Sage Weil 已提交
4430 4431

	error = -EBUSY;
4432
	if (is_local_mountpoint(old_dentry) || is_local_mountpoint(new_dentry))
S
Sage Weil 已提交
4433 4434
		goto out;

M
Miklos Szeredi 已提交
4435
	if (max_links && new_dir != old_dir) {
4436
		error = -EMLINK;
M
Miklos Szeredi 已提交
4437
		if (is_dir && !new_is_dir && new_dir->i_nlink >= max_links)
4438
			goto out;
M
Miklos Szeredi 已提交
4439 4440 4441 4442 4443 4444 4445
		if ((flags & RENAME_EXCHANGE) && !is_dir && new_is_dir &&
		    old_dir->i_nlink >= max_links)
			goto out;
	}
	if (is_dir && !(flags & RENAME_EXCHANGE) && target)
		shrink_dcache_parent(new_dentry);
	if (!is_dir) {
4446
		error = try_break_deleg(source, delegated_inode);
4447 4448
		if (error)
			goto out;
M
Miklos Szeredi 已提交
4449 4450 4451 4452 4453
	}
	if (target && !new_is_dir) {
		error = try_break_deleg(target, delegated_inode);
		if (error)
			goto out;
4454
	}
4455
	error = old_dir->i_op->rename(old_dir, old_dentry,
M
Miklos Szeredi 已提交
4456
				       new_dir, new_dentry, flags);
S
Sage Weil 已提交
4457 4458 4459
	if (error)
		goto out;

M
Miklos Szeredi 已提交
4460
	if (!(flags & RENAME_EXCHANGE) && target) {
4461 4462
		if (is_dir)
			target->i_flags |= S_DEAD;
S
Sage Weil 已提交
4463
		dont_mount(new_dentry);
4464
		detach_mounts(new_dentry);
4465
	}
M
Miklos Szeredi 已提交
4466 4467 4468 4469 4470 4471
	if (!(old_dir->i_sb->s_type->fs_flags & FS_RENAME_DOES_D_MOVE)) {
		if (!(flags & RENAME_EXCHANGE))
			d_move(old_dentry, new_dentry);
		else
			d_exchange(old_dentry, new_dentry);
	}
S
Sage Weil 已提交
4472
out:
M
Miklos Szeredi 已提交
4473
	if (!is_dir || (flags & RENAME_EXCHANGE))
4474 4475
		unlock_two_nondirectories(source, target);
	else if (target)
A
Al Viro 已提交
4476
		inode_unlock(target);
L
Linus Torvalds 已提交
4477
	dput(new_dentry);
M
Miklos Szeredi 已提交
4478
	if (!error) {
A
Al Viro 已提交
4479
		fsnotify_move(old_dir, new_dir, old_name.name, is_dir,
M
Miklos Szeredi 已提交
4480 4481 4482 4483 4484 4485
			      !(flags & RENAME_EXCHANGE) ? target : NULL, old_dentry);
		if (flags & RENAME_EXCHANGE) {
			fsnotify_move(new_dir, old_dir, old_dentry->d_name.name,
				      new_is_dir, NULL, new_dentry);
		}
	}
A
Al Viro 已提交
4486
	release_dentry_name_snapshot(&old_name);
R
Robert Love 已提交
4487

L
Linus Torvalds 已提交
4488 4489
	return error;
}
4490
EXPORT_SYMBOL(vfs_rename);
L
Linus Torvalds 已提交
4491

M
Miklos Szeredi 已提交
4492 4493
SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname,
		int, newdfd, const char __user *, newname, unsigned int, flags)
L
Linus Torvalds 已提交
4494
{
4495 4496
	struct dentry *old_dentry, *new_dentry;
	struct dentry *trap;
4497 4498 4499
	struct path old_path, new_path;
	struct qstr old_last, new_last;
	int old_type, new_type;
4500
	struct inode *delegated_inode = NULL;
4501 4502
	struct filename *from;
	struct filename *to;
4503
	unsigned int lookup_flags = 0, target_flags = LOOKUP_RENAME_TARGET;
4504
	bool should_retry = false;
4505
	int error;
M
Miklos Szeredi 已提交
4506

M
Miklos Szeredi 已提交
4507
	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
M
Miklos Szeredi 已提交
4508 4509
		return -EINVAL;

M
Miklos Szeredi 已提交
4510 4511
	if ((flags & (RENAME_NOREPLACE | RENAME_WHITEOUT)) &&
	    (flags & RENAME_EXCHANGE))
M
Miklos Szeredi 已提交
4512 4513
		return -EINVAL;

M
Miklos Szeredi 已提交
4514 4515 4516
	if ((flags & RENAME_WHITEOUT) && !capable(CAP_MKNOD))
		return -EPERM;

4517 4518 4519
	if (flags & RENAME_EXCHANGE)
		target_flags = 0;

4520
retry:
A
Al Viro 已提交
4521 4522
	from = filename_parentat(olddfd, getname(oldname), lookup_flags,
				&old_path, &old_last, &old_type);
4523 4524
	if (IS_ERR(from)) {
		error = PTR_ERR(from);
L
Linus Torvalds 已提交
4525
		goto exit;
4526
	}
L
Linus Torvalds 已提交
4527

A
Al Viro 已提交
4528 4529
	to = filename_parentat(newdfd, getname(newname), lookup_flags,
				&new_path, &new_last, &new_type);
4530 4531
	if (IS_ERR(to)) {
		error = PTR_ERR(to);
L
Linus Torvalds 已提交
4532
		goto exit1;
4533
	}
L
Linus Torvalds 已提交
4534 4535

	error = -EXDEV;
4536
	if (old_path.mnt != new_path.mnt)
L
Linus Torvalds 已提交
4537 4538 4539
		goto exit2;

	error = -EBUSY;
4540
	if (old_type != LAST_NORM)
L
Linus Torvalds 已提交
4541 4542
		goto exit2;

M
Miklos Szeredi 已提交
4543 4544
	if (flags & RENAME_NOREPLACE)
		error = -EEXIST;
4545
	if (new_type != LAST_NORM)
L
Linus Torvalds 已提交
4546 4547
		goto exit2;

4548
	error = mnt_want_write(old_path.mnt);
4549 4550 4551
	if (error)
		goto exit2;

4552
retry_deleg:
4553
	trap = lock_rename(new_path.dentry, old_path.dentry);
L
Linus Torvalds 已提交
4554

4555
	old_dentry = __lookup_hash(&old_last, old_path.dentry, lookup_flags);
L
Linus Torvalds 已提交
4556 4557 4558 4559 4560
	error = PTR_ERR(old_dentry);
	if (IS_ERR(old_dentry))
		goto exit3;
	/* source must exist */
	error = -ENOENT;
4561
	if (d_is_negative(old_dentry))
L
Linus Torvalds 已提交
4562
		goto exit4;
4563
	new_dentry = __lookup_hash(&new_last, new_path.dentry, lookup_flags | target_flags);
M
Miklos Szeredi 已提交
4564 4565 4566 4567 4568 4569
	error = PTR_ERR(new_dentry);
	if (IS_ERR(new_dentry))
		goto exit4;
	error = -EEXIST;
	if ((flags & RENAME_NOREPLACE) && d_is_positive(new_dentry))
		goto exit5;
M
Miklos Szeredi 已提交
4570 4571 4572 4573 4574 4575 4576
	if (flags & RENAME_EXCHANGE) {
		error = -ENOENT;
		if (d_is_negative(new_dentry))
			goto exit5;

		if (!d_is_dir(new_dentry)) {
			error = -ENOTDIR;
4577
			if (new_last.name[new_last.len])
M
Miklos Szeredi 已提交
4578 4579 4580
				goto exit5;
		}
	}
L
Linus Torvalds 已提交
4581
	/* unless the source is a directory trailing slashes give -ENOTDIR */
M
Miklos Szeredi 已提交
4582
	if (!d_is_dir(old_dentry)) {
L
Linus Torvalds 已提交
4583
		error = -ENOTDIR;
4584
		if (old_last.name[old_last.len])
M
Miklos Szeredi 已提交
4585
			goto exit5;
4586
		if (!(flags & RENAME_EXCHANGE) && new_last.name[new_last.len])
M
Miklos Szeredi 已提交
4587
			goto exit5;
L
Linus Torvalds 已提交
4588 4589 4590 4591
	}
	/* source should not be ancestor of target */
	error = -EINVAL;
	if (old_dentry == trap)
M
Miklos Szeredi 已提交
4592
		goto exit5;
L
Linus Torvalds 已提交
4593
	/* target should not be an ancestor of source */
M
Miklos Szeredi 已提交
4594 4595
	if (!(flags & RENAME_EXCHANGE))
		error = -ENOTEMPTY;
L
Linus Torvalds 已提交
4596 4597 4598
	if (new_dentry == trap)
		goto exit5;

4599 4600
	error = security_path_rename(&old_path, old_dentry,
				     &new_path, new_dentry, flags);
4601
	if (error)
4602
		goto exit5;
4603 4604
	error = vfs_rename(old_path.dentry->d_inode, old_dentry,
			   new_path.dentry->d_inode, new_dentry,
M
Miklos Szeredi 已提交
4605
			   &delegated_inode, flags);
L
Linus Torvalds 已提交
4606 4607 4608 4609 4610
exit5:
	dput(new_dentry);
exit4:
	dput(old_dentry);
exit3:
4611
	unlock_rename(new_path.dentry, old_path.dentry);
4612 4613 4614 4615 4616
	if (delegated_inode) {
		error = break_deleg_wait(&delegated_inode);
		if (!error)
			goto retry_deleg;
	}
4617
	mnt_drop_write(old_path.mnt);
L
Linus Torvalds 已提交
4618
exit2:
4619 4620
	if (retry_estale(error, lookup_flags))
		should_retry = true;
4621
	path_put(&new_path);
4622
	putname(to);
L
Linus Torvalds 已提交
4623
exit1:
4624
	path_put(&old_path);
L
Linus Torvalds 已提交
4625
	putname(from);
4626 4627 4628 4629 4630
	if (should_retry) {
		should_retry = false;
		lookup_flags |= LOOKUP_REVAL;
		goto retry;
	}
4631
exit:
L
Linus Torvalds 已提交
4632 4633 4634
	return error;
}

M
Miklos Szeredi 已提交
4635 4636 4637 4638 4639 4640
SYSCALL_DEFINE4(renameat, int, olddfd, const char __user *, oldname,
		int, newdfd, const char __user *, newname)
{
	return sys_renameat2(olddfd, oldname, newdfd, newname, 0);
}

4641
SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newname)
4642
{
M
Miklos Szeredi 已提交
4643
	return sys_renameat2(AT_FDCWD, oldname, AT_FDCWD, newname, 0);
4644 4645
}

M
Miklos Szeredi 已提交
4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659
int vfs_whiteout(struct inode *dir, struct dentry *dentry)
{
	int error = may_create(dir, dentry);
	if (error)
		return error;

	if (!dir->i_op->mknod)
		return -EPERM;

	return dir->i_op->mknod(dir, dentry,
				S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
}
EXPORT_SYMBOL(vfs_whiteout);

A
Al Viro 已提交
4660
int readlink_copy(char __user *buffer, int buflen, const char *link)
L
Linus Torvalds 已提交
4661
{
A
Al Viro 已提交
4662
	int len = PTR_ERR(link);
L
Linus Torvalds 已提交
4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676
	if (IS_ERR(link))
		goto out;

	len = strlen(link);
	if (len > (unsigned) buflen)
		len = buflen;
	if (copy_to_user(buffer, link, len))
		len = -EFAULT;
out:
	return len;
}

/*
 * A helper for ->readlink().  This should be used *ONLY* for symlinks that
4677 4678
 * have ->get_link() not calling nd_jump_link().  Using (or not using) it
 * for any given inode is up to filesystem.
L
Linus Torvalds 已提交
4679
 */
4680 4681
static int generic_readlink(struct dentry *dentry, char __user *buffer,
			    int buflen)
L
Linus Torvalds 已提交
4682
{
4683
	DEFINE_DELAYED_CALL(done);
4684 4685
	struct inode *inode = d_inode(dentry);
	const char *link = inode->i_link;
4686
	int res;
4687

4688
	if (!link) {
4689
		link = inode->i_op->get_link(dentry, inode, &done);
4690 4691 4692
		if (IS_ERR(link))
			return PTR_ERR(link);
	}
4693
	res = readlink_copy(buffer, buflen, link);
4694
	do_delayed_call(&done);
4695
	return res;
L
Linus Torvalds 已提交
4696 4697
}

4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711
/**
 * vfs_readlink - copy symlink body into userspace buffer
 * @dentry: dentry on which to get symbolic link
 * @buffer: user memory pointer
 * @buflen: size of buffer
 *
 * Does not touch atime.  That's up to the caller if necessary
 *
 * Does not call security hook.
 */
int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
{
	struct inode *inode = d_inode(dentry);

4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722
	if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
		if (unlikely(inode->i_op->readlink))
			return inode->i_op->readlink(dentry, buffer, buflen);

		if (!d_is_symlink(dentry))
			return -EINVAL;

		spin_lock(&inode->i_lock);
		inode->i_opflags |= IOP_DEFAULT_READLINK;
		spin_unlock(&inode->i_lock);
	}
4723

4724
	return generic_readlink(dentry, buffer, buflen);
4725 4726
}
EXPORT_SYMBOL(vfs_readlink);
L
Linus Torvalds 已提交
4727

M
Miklos Szeredi 已提交
4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752
/**
 * vfs_get_link - get symlink body
 * @dentry: dentry on which to get symbolic link
 * @done: caller needs to free returned data with this
 *
 * Calls security hook and i_op->get_link() on the supplied inode.
 *
 * It does not touch atime.  That's up to the caller if necessary.
 *
 * Does not work on "special" symlinks like /proc/$$/fd/N
 */
const char *vfs_get_link(struct dentry *dentry, struct delayed_call *done)
{
	const char *res = ERR_PTR(-EINVAL);
	struct inode *inode = d_inode(dentry);

	if (d_is_symlink(dentry)) {
		res = ERR_PTR(security_inode_readlink(dentry));
		if (!res)
			res = inode->i_op->get_link(dentry, inode, done);
	}
	return res;
}
EXPORT_SYMBOL(vfs_get_link);

L
Linus Torvalds 已提交
4753
/* get the link contents into pagecache */
4754
const char *page_get_link(struct dentry *dentry, struct inode *inode,
4755
			  struct delayed_call *callback)
L
Linus Torvalds 已提交
4756
{
4757 4758
	char *kaddr;
	struct page *page;
4759 4760
	struct address_space *mapping = inode->i_mapping;

4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773
	if (!dentry) {
		page = find_get_page(mapping, 0);
		if (!page)
			return ERR_PTR(-ECHILD);
		if (!PageUptodate(page)) {
			put_page(page);
			return ERR_PTR(-ECHILD);
		}
	} else {
		page = read_mapping_page(mapping, 0, NULL);
		if (IS_ERR(page))
			return (char*)page;
	}
4774
	set_delayed_call(callback, page_put_link, page);
4775 4776
	BUG_ON(mapping_gfp_mask(mapping) & __GFP_HIGHMEM);
	kaddr = page_address(page);
4777
	nd_terminate_link(kaddr, inode->i_size, PAGE_SIZE - 1);
4778
	return kaddr;
L
Linus Torvalds 已提交
4779 4780
}

4781
EXPORT_SYMBOL(page_get_link);
L
Linus Torvalds 已提交
4782

4783
void page_put_link(void *arg)
L
Linus Torvalds 已提交
4784
{
4785
	put_page(arg);
L
Linus Torvalds 已提交
4786
}
4787
EXPORT_SYMBOL(page_put_link);
L
Linus Torvalds 已提交
4788

4789 4790
int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
{
4791
	DEFINE_DELAYED_CALL(done);
4792 4793
	int res = readlink_copy(buffer, buflen,
				page_get_link(dentry, d_inode(dentry),
4794 4795
					      &done));
	do_delayed_call(&done);
4796 4797 4798 4799
	return res;
}
EXPORT_SYMBOL(page_readlink);

4800 4801 4802 4803
/*
 * The nofs argument instructs pagecache_write_begin to pass AOP_FLAG_NOFS
 */
int __page_symlink(struct inode *inode, const char *symname, int len, int nofs)
L
Linus Torvalds 已提交
4804 4805
{
	struct address_space *mapping = inode->i_mapping;
4806
	struct page *page;
4807
	void *fsdata;
4808
	int err;
4809
	unsigned int flags = 0;
4810 4811
	if (nofs)
		flags |= AOP_FLAG_NOFS;
L
Linus Torvalds 已提交
4812

4813
retry:
4814
	err = pagecache_write_begin(NULL, mapping, 0, len-1,
4815
				flags, &page, &fsdata);
L
Linus Torvalds 已提交
4816
	if (err)
4817 4818
		goto fail;

4819
	memcpy(page_address(page), symname, len-1);
4820 4821 4822

	err = pagecache_write_end(NULL, mapping, 0, len-1, len-1,
							page, fsdata);
L
Linus Torvalds 已提交
4823 4824
	if (err < 0)
		goto fail;
4825 4826 4827
	if (err < len-1)
		goto retry;

L
Linus Torvalds 已提交
4828 4829 4830 4831 4832
	mark_inode_dirty(inode);
	return 0;
fail:
	return err;
}
4833
EXPORT_SYMBOL(__page_symlink);
L
Linus Torvalds 已提交
4834

4835 4836 4837
int page_symlink(struct inode *inode, const char *symname, int len)
{
	return __page_symlink(inode, symname, len,
4838
			!mapping_gfp_constraint(inode->i_mapping, __GFP_FS));
4839
}
4840
EXPORT_SYMBOL(page_symlink);
4841

4842
const struct inode_operations page_symlink_inode_operations = {
4843
	.get_link	= page_get_link,
L
Linus Torvalds 已提交
4844 4845
};
EXPORT_SYMBOL(page_symlink_inode_operations);