shmem.c 63.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8
/*
 * Resizable virtual memory filesystem for Linux.
 *
 * Copyright (C) 2000 Linus Torvalds.
 *		 2000 Transmeta Corp.
 *		 2000-2001 Christoph Rohland
 *		 2000-2001 SAP AG
 *		 2002 Red Hat Inc.
9 10
 * Copyright (C) 2002-2011 Hugh Dickins.
 * Copyright (C) 2011 Google Inc.
11
 * Copyright (C) 2002-2005 VERITAS Software Corporation.
L
Linus Torvalds 已提交
12 13 14 15 16 17
 * Copyright (C) 2004 Andi Kleen, SuSE Labs
 *
 * Extended attribute support for tmpfs:
 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
 *
18 19 20
 * tiny-shmem:
 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
 *
L
Linus Torvalds 已提交
21 22 23
 * This file is released under the GPL.
 */

24 25 26 27
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/vfs.h>
#include <linux/mount.h>
H
Hugh Dickins 已提交
28
#include <linux/pagemap.h>
29 30 31 32 33 34 35 36
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/swap.h>

static struct vfsmount *shm_mnt;

#ifdef CONFIG_SHMEM
L
Linus Torvalds 已提交
37 38 39 40 41 42
/*
 * This virtual memory filesystem is heavily based on the ramfs. It
 * extends ramfs by the ability to use swap and honor resource limits
 * which makes it a completely usable filesystem.
 */

43
#include <linux/xattr.h>
44
#include <linux/exportfs.h>
45
#include <linux/posix_acl.h>
46
#include <linux/generic_acl.h>
L
Linus Torvalds 已提交
47 48 49 50 51 52 53
#include <linux/mman.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/backing-dev.h>
#include <linux/shmem_fs.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
54
#include <linux/pagevec.h>
55
#include <linux/percpu_counter.h>
56
#include <linux/splice.h>
L
Linus Torvalds 已提交
57 58 59 60
#include <linux/security.h>
#include <linux/swapops.h>
#include <linux/mempolicy.h>
#include <linux/namei.h>
61
#include <linux/ctype.h>
62
#include <linux/migrate.h>
63
#include <linux/highmem.h>
64
#include <linux/seq_file.h>
M
Mimi Zohar 已提交
65
#include <linux/magic.h>
66

L
Linus Torvalds 已提交
67 68 69
#include <asm/uaccess.h>
#include <asm/pgtable.h>

H
Hugh Dickins 已提交
70
#define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
L
Linus Torvalds 已提交
71 72 73 74 75
#define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)

/* Pretend that each entry is of this size in directory's i_size */
#define BOGO_DIRENT_SIZE 20

76 77 78
/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
#define SHORT_SYMLINK_LEN 128

79 80 81 82 83 84 85
struct shmem_xattr {
	struct list_head list;	/* anchored by shmem_inode_info->xattr_list */
	char *name;		/* xattr name */
	size_t size;
	char value[0];
};

86
/* Flag allocation requirements to shmem_getpage */
L
Linus Torvalds 已提交
87 88 89
enum sgp_type {
	SGP_READ,	/* don't exceed i_size, don't allocate page */
	SGP_CACHE,	/* don't exceed i_size, may allocate page */
90
	SGP_DIRTY,	/* like SGP_CACHE, but set new page dirty */
L
Linus Torvalds 已提交
91 92 93
	SGP_WRITE,	/* may exceed i_size, may allocate page */
};

A
Andrew Morton 已提交
94
#ifdef CONFIG_TMPFS
95 96 97 98 99 100 101 102 103
static unsigned long shmem_default_max_blocks(void)
{
	return totalram_pages / 2;
}

static unsigned long shmem_default_max_inodes(void)
{
	return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
}
A
Andrew Morton 已提交
104
#endif
105

106 107 108 109 110 111 112 113 114
static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
	struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);

static inline int shmem_getpage(struct inode *inode, pgoff_t index,
	struct page **pagep, enum sgp_type sgp, int *fault_type)
{
	return shmem_getpage_gfp(inode, index, pagep, sgp,
			mapping_gfp_mask(inode->i_mapping), fault_type);
}
L
Linus Torvalds 已提交
115 116 117 118 119 120 121 122 123 124 125 126 127 128

static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
{
	return sb->s_fs_info;
}

/*
 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
 * for shared memory and for shared anonymous (/dev/zero) mappings
 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
 * consistent with the pre-accounting of private mappings ...
 */
static inline int shmem_acct_size(unsigned long flags, loff_t size)
{
129 130
	return (flags & VM_NORESERVE) ?
		0 : security_vm_enough_memory_kern(VM_ACCT(size));
L
Linus Torvalds 已提交
131 132 133 134
}

static inline void shmem_unacct_size(unsigned long flags, loff_t size)
{
135
	if (!(flags & VM_NORESERVE))
L
Linus Torvalds 已提交
136 137 138 139 140 141 142 143 144 145 146
		vm_unacct_memory(VM_ACCT(size));
}

/*
 * ... whereas tmpfs objects are accounted incrementally as
 * pages are allocated, in order to allow huge sparse files.
 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
 */
static inline int shmem_acct_block(unsigned long flags)
{
147 148
	return (flags & VM_NORESERVE) ?
		security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
L
Linus Torvalds 已提交
149 150 151 152
}

static inline void shmem_unacct_blocks(unsigned long flags, long pages)
{
153
	if (flags & VM_NORESERVE)
L
Linus Torvalds 已提交
154 155 156
		vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
}

157
static const struct super_operations shmem_ops;
158
static const struct address_space_operations shmem_aops;
159
static const struct file_operations shmem_file_operations;
160 161 162
static const struct inode_operations shmem_inode_operations;
static const struct inode_operations shmem_dir_inode_operations;
static const struct inode_operations shmem_special_inode_operations;
163
static const struct vm_operations_struct shmem_vm_ops;
L
Linus Torvalds 已提交
164

165
static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
L
Linus Torvalds 已提交
166
	.ra_pages	= 0,	/* No readahead */
167
	.capabilities	= BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
L
Linus Torvalds 已提交
168 169 170
};

static LIST_HEAD(shmem_swaplist);
171
static DEFINE_MUTEX(shmem_swaplist_mutex);
L
Linus Torvalds 已提交
172

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
static int shmem_reserve_inode(struct super_block *sb)
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
	if (sbinfo->max_inodes) {
		spin_lock(&sbinfo->stat_lock);
		if (!sbinfo->free_inodes) {
			spin_unlock(&sbinfo->stat_lock);
			return -ENOSPC;
		}
		sbinfo->free_inodes--;
		spin_unlock(&sbinfo->stat_lock);
	}
	return 0;
}

static void shmem_free_inode(struct super_block *sb)
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
	if (sbinfo->max_inodes) {
		spin_lock(&sbinfo->stat_lock);
		sbinfo->free_inodes++;
		spin_unlock(&sbinfo->stat_lock);
	}
}

198
/**
199
 * shmem_recalc_inode - recalculate the block usage of an inode
L
Linus Torvalds 已提交
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
 * @inode: inode to recalc
 *
 * We have to calculate the free blocks since the mm can drop
 * undirtied hole pages behind our back.
 *
 * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
 *
 * It has to be called with the spinlock held.
 */
static void shmem_recalc_inode(struct inode *inode)
{
	struct shmem_inode_info *info = SHMEM_I(inode);
	long freed;

	freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
	if (freed > 0) {
217 218 219
		struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
		if (sbinfo->max_blocks)
			percpu_counter_add(&sbinfo->used_blocks, -freed);
L
Linus Torvalds 已提交
220
		info->alloced -= freed;
221
		inode->i_blocks -= freed * BLOCKS_PER_PAGE;
L
Linus Torvalds 已提交
222 223 224 225
		shmem_unacct_blocks(info->flags, freed);
	}
}

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
/*
 * Replace item expected in radix tree by a new item, while holding tree lock.
 */
static int shmem_radix_tree_replace(struct address_space *mapping,
			pgoff_t index, void *expected, void *replacement)
{
	void **pslot;
	void *item = NULL;

	VM_BUG_ON(!expected);
	pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
	if (pslot)
		item = radix_tree_deref_slot_protected(pslot,
							&mapping->tree_lock);
	if (item != expected)
		return -ENOENT;
	if (replacement)
		radix_tree_replace_slot(pslot, replacement);
	else
		radix_tree_delete(&mapping->page_tree, index);
	return 0;
}

249 250 251 252 253 254 255
/*
 * Like add_to_page_cache_locked, but error if expected item has gone.
 */
static int shmem_add_to_page_cache(struct page *page,
				   struct address_space *mapping,
				   pgoff_t index, gfp_t gfp, void *expected)
{
256
	int error = 0;
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292

	VM_BUG_ON(!PageLocked(page));
	VM_BUG_ON(!PageSwapBacked(page));

	if (!expected)
		error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
	if (!error) {
		page_cache_get(page);
		page->mapping = mapping;
		page->index = index;

		spin_lock_irq(&mapping->tree_lock);
		if (!expected)
			error = radix_tree_insert(&mapping->page_tree,
							index, page);
		else
			error = shmem_radix_tree_replace(mapping, index,
							expected, page);
		if (!error) {
			mapping->nrpages++;
			__inc_zone_page_state(page, NR_FILE_PAGES);
			__inc_zone_page_state(page, NR_SHMEM);
			spin_unlock_irq(&mapping->tree_lock);
		} else {
			page->mapping = NULL;
			spin_unlock_irq(&mapping->tree_lock);
			page_cache_release(page);
		}
		if (!expected)
			radix_tree_preload_end();
	}
	if (error)
		mem_cgroup_uncharge_cache_page(page);
	return error;
}

293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
/*
 * Like delete_from_page_cache, but substitutes swap for page.
 */
static void shmem_delete_from_page_cache(struct page *page, void *radswap)
{
	struct address_space *mapping = page->mapping;
	int error;

	spin_lock_irq(&mapping->tree_lock);
	error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
	page->mapping = NULL;
	mapping->nrpages--;
	__dec_zone_page_state(page, NR_FILE_PAGES);
	__dec_zone_page_state(page, NR_SHMEM);
	spin_unlock_irq(&mapping->tree_lock);
	page_cache_release(page);
	BUG_ON(error);
}

312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
/*
 * Like find_get_pages, but collecting swap entries as well as pages.
 */
static unsigned shmem_find_get_pages_and_swap(struct address_space *mapping,
					pgoff_t start, unsigned int nr_pages,
					struct page **pages, pgoff_t *indices)
{
	unsigned int i;
	unsigned int ret;
	unsigned int nr_found;

	rcu_read_lock();
restart:
	nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
				(void ***)pages, indices, start, nr_pages);
	ret = 0;
	for (i = 0; i < nr_found; i++) {
		struct page *page;
repeat:
		page = radix_tree_deref_slot((void **)pages[i]);
		if (unlikely(!page))
			continue;
		if (radix_tree_exception(page)) {
335 336 337 338 339 340 341 342
			if (radix_tree_deref_retry(page))
				goto restart;
			/*
			 * Otherwise, we must be storing a swap entry
			 * here as an exceptional entry: so return it
			 * without attempting to raise page count.
			 */
			goto export;
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
		}
		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
		if (unlikely(page != *((void **)pages[i]))) {
			page_cache_release(page);
			goto repeat;
		}
export:
		indices[ret] = indices[i];
		pages[ret] = page;
		ret++;
	}
	if (unlikely(!ret && nr_found))
		goto restart;
	rcu_read_unlock();
	return ret;
}

/*
 * Remove swap entry from radix tree, free the swap and its page cache.
 */
static int shmem_free_swap(struct address_space *mapping,
			   pgoff_t index, void *radswap)
{
	int error;

	spin_lock_irq(&mapping->tree_lock);
	error = shmem_radix_tree_replace(mapping, index, radswap, NULL);
	spin_unlock_irq(&mapping->tree_lock);
	if (!error)
		free_swap_and_cache(radix_to_swp_entry(radswap));
	return error;
}

/*
 * Pagevec may contain swap entries, so shuffle up pages before releasing.
 */
static void shmem_pagevec_release(struct pagevec *pvec)
{
	int i, j;

	for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
		struct page *page = pvec->pages[i];
		if (!radix_tree_exceptional_entry(page))
			pvec->pages[j++] = page;
	}
	pvec->nr = j;
	pagevec_release(pvec);
}

/*
 * Remove range of pages and swap entries from radix tree, and free them.
 */
398
void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
L
Linus Torvalds 已提交
399
{
400
	struct address_space *mapping = inode->i_mapping;
L
Linus Torvalds 已提交
401
	struct shmem_inode_info *info = SHMEM_I(inode);
402
	pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
403
	unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
404
	pgoff_t end = (lend >> PAGE_CACHE_SHIFT);
405
	struct pagevec pvec;
406 407
	pgoff_t indices[PAGEVEC_SIZE];
	long nr_swaps_freed = 0;
408
	pgoff_t index;
409 410 411 412 413 414
	int i;

	BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));

	pagevec_init(&pvec, 0);
	index = start;
415 416 417 418 419 420
	while (index <= end) {
		pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
			min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
							pvec.pages, indices);
		if (!pvec.nr)
			break;
421 422 423 424
		mem_cgroup_uncharge_start();
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];

425
			index = indices[i];
426 427 428
			if (index > end)
				break;

429 430 431
			if (radix_tree_exceptional_entry(page)) {
				nr_swaps_freed += !shmem_free_swap(mapping,
								index, page);
432
				continue;
433 434 435
			}

			if (!trylock_page(page))
436
				continue;
437 438 439
			if (page->mapping == mapping) {
				VM_BUG_ON(PageWriteback(page));
				truncate_inode_page(mapping, page);
440 441 442
			}
			unlock_page(page);
		}
443
		shmem_pagevec_release(&pvec);
444 445 446 447
		mem_cgroup_uncharge_end();
		cond_resched();
		index++;
	}
L
Linus Torvalds 已提交
448

449 450 451 452 453 454 455 456 457 458 459 460 461 462
	if (partial) {
		struct page *page = NULL;
		shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
		if (page) {
			zero_user_segment(page, partial, PAGE_CACHE_SIZE);
			set_page_dirty(page);
			unlock_page(page);
			page_cache_release(page);
		}
	}

	index = start;
	for ( ; ; ) {
		cond_resched();
463 464 465 466
		pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
			min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
							pvec.pages, indices);
		if (!pvec.nr) {
467 468 469 470 471
			if (index == start)
				break;
			index = start;
			continue;
		}
472 473
		if (index == start && indices[0] > end) {
			shmem_pagevec_release(&pvec);
474 475 476 477 478 479
			break;
		}
		mem_cgroup_uncharge_start();
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];

480
			index = indices[i];
481 482 483
			if (index > end)
				break;

484 485 486 487 488 489
			if (radix_tree_exceptional_entry(page)) {
				nr_swaps_freed += !shmem_free_swap(mapping,
								index, page);
				continue;
			}

490
			lock_page(page);
491 492 493 494
			if (page->mapping == mapping) {
				VM_BUG_ON(PageWriteback(page));
				truncate_inode_page(mapping, page);
			}
495 496
			unlock_page(page);
		}
497
		shmem_pagevec_release(&pvec);
498 499 500
		mem_cgroup_uncharge_end();
		index++;
	}
501

L
Linus Torvalds 已提交
502
	spin_lock(&info->lock);
503
	info->swapped -= nr_swaps_freed;
L
Linus Torvalds 已提交
504 505 506
	shmem_recalc_inode(inode);
	spin_unlock(&info->lock);

507
	inode->i_ctime = inode->i_mtime = CURRENT_TIME;
L
Linus Torvalds 已提交
508
}
509
EXPORT_SYMBOL_GPL(shmem_truncate_range);
L
Linus Torvalds 已提交
510

511
static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
L
Linus Torvalds 已提交
512 513 514 515
{
	struct inode *inode = dentry->d_inode;
	int error;

516 517 518 519
	error = inode_change_ok(inode, attr);
	if (error)
		return error;

520 521 522
	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
		loff_t oldsize = inode->i_size;
		loff_t newsize = attr->ia_size;
523

524 525 526 527 528 529 530 531 532 533 534
		if (newsize != oldsize) {
			i_size_write(inode, newsize);
			inode->i_ctime = inode->i_mtime = CURRENT_TIME;
		}
		if (newsize < oldsize) {
			loff_t holebegin = round_up(newsize, PAGE_SIZE);
			unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
			shmem_truncate_range(inode, newsize, (loff_t)-1);
			/* unmap again to remove racily COWed private pages */
			unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
		}
L
Linus Torvalds 已提交
535 536
	}

537
	setattr_copy(inode, attr);
538
#ifdef CONFIG_TMPFS_POSIX_ACL
539
	if (attr->ia_valid & ATTR_MODE)
540
		error = generic_acl_chmod(inode);
541
#endif
L
Linus Torvalds 已提交
542 543 544
	return error;
}

A
Al Viro 已提交
545
static void shmem_evict_inode(struct inode *inode)
L
Linus Torvalds 已提交
546 547
{
	struct shmem_inode_info *info = SHMEM_I(inode);
548
	struct shmem_xattr *xattr, *nxattr;
L
Linus Torvalds 已提交
549

550
	if (inode->i_mapping->a_ops == &shmem_aops) {
L
Linus Torvalds 已提交
551 552
		shmem_unacct_size(info->flags, inode->i_size);
		inode->i_size = 0;
553
		shmem_truncate_range(inode, 0, (loff_t)-1);
L
Linus Torvalds 已提交
554
		if (!list_empty(&info->swaplist)) {
555
			mutex_lock(&shmem_swaplist_mutex);
L
Linus Torvalds 已提交
556
			list_del_init(&info->swaplist);
557
			mutex_unlock(&shmem_swaplist_mutex);
L
Linus Torvalds 已提交
558
		}
559 560
	} else
		kfree(info->symlink);
561 562 563 564 565

	list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) {
		kfree(xattr->name);
		kfree(xattr);
	}
566
	BUG_ON(inode->i_blocks);
567
	shmem_free_inode(inode->i_sb);
A
Al Viro 已提交
568
	end_writeback(inode);
L
Linus Torvalds 已提交
569 570
}

571 572 573
/*
 * If swap found in inode, free it and move page from swapcache to filecache.
 */
574 575
static int shmem_unuse_inode(struct shmem_inode_info *info,
			     swp_entry_t swap, struct page *page)
L
Linus Torvalds 已提交
576
{
577
	struct address_space *mapping = info->vfs_inode.i_mapping;
578
	void *radswap;
579
	pgoff_t index;
580
	int error;
L
Linus Torvalds 已提交
581

582
	radswap = swp_to_radix_entry(swap);
583
	index = radix_tree_locate_item(&mapping->page_tree, radswap);
584
	if (index == -1)
585
		return 0;
586

H
Hugh Dickins 已提交
587 588
	/*
	 * Move _head_ to start search for next from here.
A
Al Viro 已提交
589
	 * But be careful: shmem_evict_inode checks list_empty without taking
H
Hugh Dickins 已提交
590
	 * mutex, and there's an instant in list_move_tail when info->swaplist
591
	 * would appear empty, if it were the only one on shmem_swaplist.
H
Hugh Dickins 已提交
592 593 594
	 */
	if (shmem_swaplist.next != &info->swaplist)
		list_move_tail(&shmem_swaplist, &info->swaplist);
595

K
KAMEZAWA Hiroyuki 已提交
596
	/*
597 598 599
	 * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
	 * but also to hold up shmem_evict_inode(): so inode cannot be freed
	 * beneath us (pagelock doesn't help until the page is in pagecache).
K
KAMEZAWA Hiroyuki 已提交
600
	 */
601 602
	error = shmem_add_to_page_cache(page, mapping, index,
						GFP_NOWAIT, radswap);
603
	/* which does mem_cgroup_uncharge_cache_page on error */
604

605
	if (error != -ENOMEM) {
606 607 608 609
		/*
		 * Truncation and eviction use free_swap_and_cache(), which
		 * only does trylock page: if we raced, best clean up here.
		 */
610 611
		delete_from_swap_cache(page);
		set_page_dirty(page);
612 613 614 615 616 617
		if (!error) {
			spin_lock(&info->lock);
			info->swapped--;
			spin_unlock(&info->lock);
			swap_free(swap);
		}
618
		error = 1;	/* not an error, but entry was found */
L
Linus Torvalds 已提交
619
	}
620
	return error;
L
Linus Torvalds 已提交
621 622 623
}

/*
624
 * Search through swapped inodes to find and replace swap by page.
L
Linus Torvalds 已提交
625
 */
626
int shmem_unuse(swp_entry_t swap, struct page *page)
L
Linus Torvalds 已提交
627
{
628
	struct list_head *this, *next;
L
Linus Torvalds 已提交
629 630
	struct shmem_inode_info *info;
	int found = 0;
631 632 633 634 635 636 637 638 639 640
	int error;

	/*
	 * Charge page using GFP_KERNEL while we can wait, before taking
	 * the shmem_swaplist_mutex which might hold up shmem_writepage().
	 * Charged back to the user (not to caller) when swap account is used.
	 */
	error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
	if (error)
		goto out;
641
	/* No radix_tree_preload: swap entry keeps a place for page in tree */
L
Linus Torvalds 已提交
642

643
	mutex_lock(&shmem_swaplist_mutex);
644 645
	list_for_each_safe(this, next, &shmem_swaplist) {
		info = list_entry(this, struct shmem_inode_info, swaplist);
646
		if (info->swapped)
647
			found = shmem_unuse_inode(info, swap, page);
648 649
		else
			list_del_init(&info->swaplist);
650
		cond_resched();
651
		if (found)
652
			break;
L
Linus Torvalds 已提交
653
	}
654
	mutex_unlock(&shmem_swaplist_mutex);
655 656 657 658 659 660

	if (!found)
		mem_cgroup_uncharge_cache_page(page);
	if (found < 0)
		error = found;
out:
H
Hugh Dickins 已提交
661 662
	unlock_page(page);
	page_cache_release(page);
663
	return error;
L
Linus Torvalds 已提交
664 665 666 667 668 669 670 671 672 673
}

/*
 * Move the page from the page cache to the swap cache.
 */
static int shmem_writepage(struct page *page, struct writeback_control *wbc)
{
	struct shmem_inode_info *info;
	struct address_space *mapping;
	struct inode *inode;
674 675
	swp_entry_t swap;
	pgoff_t index;
L
Linus Torvalds 已提交
676 677 678 679 680 681 682 683

	BUG_ON(!PageLocked(page));
	mapping = page->mapping;
	index = page->index;
	inode = mapping->host;
	info = SHMEM_I(inode);
	if (info->flags & VM_LOCKED)
		goto redirty;
684
	if (!total_swap_pages)
L
Linus Torvalds 已提交
685 686
		goto redirty;

687 688 689
	/*
	 * shmem_backing_dev_info's capabilities prevent regular writeback or
	 * sync from ever calling shmem_writepage; but a stacking filesystem
690
	 * might use ->writepage of its underlying filesystem, in which case
691
	 * tmpfs should write out to swap only in response to memory pressure,
692
	 * and not for the writeback threads or sync.
693
	 */
694 695 696 697 698 699 700
	if (!wbc->for_reclaim) {
		WARN_ON_ONCE(1);	/* Still happens? Tell us about it! */
		goto redirty;
	}
	swap = get_swap_page();
	if (!swap.val)
		goto redirty;
701

702 703
	/*
	 * Add inode to shmem_unuse()'s list of swapped-out inodes,
704 705
	 * if it's not already there.  Do it now before the page is
	 * moved to swap cache, when its pagelock no longer protects
706
	 * the inode from eviction.  But don't unlock the mutex until
707 708
	 * we've incremented swapped, because shmem_unuse_inode() will
	 * prune a !swapped inode from the swaplist under this mutex.
709
	 */
710 711 712
	mutex_lock(&shmem_swaplist_mutex);
	if (list_empty(&info->swaplist))
		list_add_tail(&info->swaplist, &shmem_swaplist);
713

714
	if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
H
Hugh Dickins 已提交
715
		swap_shmem_alloc(swap);
716 717 718 719 720
		shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));

		spin_lock(&info->lock);
		info->swapped++;
		shmem_recalc_inode(inode);
721
		spin_unlock(&info->lock);
722 723

		mutex_unlock(&shmem_swaplist_mutex);
724
		BUG_ON(page_mapped(page));
H
Hugh Dickins 已提交
725
		swap_writepage(page, wbc);
L
Linus Torvalds 已提交
726 727 728
		return 0;
	}

729
	mutex_unlock(&shmem_swaplist_mutex);
730
	swapcache_free(swap, NULL);
L
Linus Torvalds 已提交
731 732
redirty:
	set_page_dirty(page);
733 734 735 736
	if (wbc->for_reclaim)
		return AOP_WRITEPAGE_ACTIVATE;	/* Return with page locked */
	unlock_page(page);
	return 0;
L
Linus Torvalds 已提交
737 738 739
}

#ifdef CONFIG_NUMA
740
#ifdef CONFIG_TMPFS
741
static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
742
{
743
	char buffer[64];
744

745
	if (!mpol || mpol->mode == MPOL_DEFAULT)
746
		return;		/* show nothing */
747

748
	mpol_to_str(buffer, sizeof(buffer), mpol, 1);
749 750

	seq_printf(seq, ",mpol=%s", buffer);
751
}
752 753 754 755 756 757 758 759 760 761 762 763

static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
{
	struct mempolicy *mpol = NULL;
	if (sbinfo->mpol) {
		spin_lock(&sbinfo->stat_lock);	/* prevent replace/use races */
		mpol = sbinfo->mpol;
		mpol_get(mpol);
		spin_unlock(&sbinfo->stat_lock);
	}
	return mpol;
}
764 765
#endif /* CONFIG_TMPFS */

766 767
static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
			struct shmem_inode_info *info, pgoff_t index)
L
Linus Torvalds 已提交
768
{
769
	struct mempolicy mpol, *spol;
L
Linus Torvalds 已提交
770 771
	struct vm_area_struct pvma;

772
	spol = mpol_cond_copy(&mpol,
773
			mpol_shared_policy_lookup(&info->policy, index));
774

L
Linus Torvalds 已提交
775
	/* Create a pseudo vma that just contains the policy */
776
	pvma.vm_start = 0;
777
	pvma.vm_pgoff = index;
778
	pvma.vm_ops = NULL;
779
	pvma.vm_policy = spol;
780
	return swapin_readahead(swap, gfp, &pvma, 0);
L
Linus Torvalds 已提交
781 782
}

783
static struct page *shmem_alloc_page(gfp_t gfp,
784
			struct shmem_inode_info *info, pgoff_t index)
L
Linus Torvalds 已提交
785 786 787
{
	struct vm_area_struct pvma;

788 789
	/* Create a pseudo vma that just contains the policy */
	pvma.vm_start = 0;
790
	pvma.vm_pgoff = index;
791
	pvma.vm_ops = NULL;
792
	pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
793 794 795 796 797

	/*
	 * alloc_page_vma() will drop the shared policy reference
	 */
	return alloc_page_vma(gfp, &pvma, 0);
L
Linus Torvalds 已提交
798
}
799 800
#else /* !CONFIG_NUMA */
#ifdef CONFIG_TMPFS
801
static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
802 803 804 805
{
}
#endif /* CONFIG_TMPFS */

806 807
static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
			struct shmem_inode_info *info, pgoff_t index)
L
Linus Torvalds 已提交
808
{
809
	return swapin_readahead(swap, gfp, NULL, 0);
L
Linus Torvalds 已提交
810 811
}

812
static inline struct page *shmem_alloc_page(gfp_t gfp,
813
			struct shmem_inode_info *info, pgoff_t index)
L
Linus Torvalds 已提交
814
{
815
	return alloc_page(gfp);
L
Linus Torvalds 已提交
816
}
817
#endif /* CONFIG_NUMA */
L
Linus Torvalds 已提交
818

819 820 821 822 823 824 825
#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
{
	return NULL;
}
#endif

L
Linus Torvalds 已提交
826
/*
827
 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
L
Linus Torvalds 已提交
828 829 830 831 832
 *
 * If we allocate a new one we do not mark it dirty. That's up to the
 * vm. If we swap it in we mark it dirty since we also free the swap
 * entry since a page cannot live in both the swap and page cache
 */
833
static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
834
	struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)
L
Linus Torvalds 已提交
835 836
{
	struct address_space *mapping = inode->i_mapping;
837
	struct shmem_inode_info *info;
L
Linus Torvalds 已提交
838
	struct shmem_sb_info *sbinfo;
H
Hugh Dickins 已提交
839
	struct page *page;
L
Linus Torvalds 已提交
840 841
	swp_entry_t swap;
	int error;
842
	int once = 0;
L
Linus Torvalds 已提交
843

844
	if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
L
Linus Torvalds 已提交
845 846
		return -EFBIG;
repeat:
847
	swap.val = 0;
848
	page = find_lock_page(mapping, index);
849 850 851 852 853 854 855 856 857 858 859 860
	if (radix_tree_exceptional_entry(page)) {
		swap = radix_to_swp_entry(page);
		page = NULL;
	}

	if (sgp != SGP_WRITE &&
	    ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
		error = -EINVAL;
		goto failed;
	}

	if (page || (sgp == SGP_READ && !swap.val)) {
H
Hugh Dickins 已提交
861
		/*
H
Hugh Dickins 已提交
862 863 864
		 * Once we can get the page lock, it must be uptodate:
		 * if there were an error in reading back from swap,
		 * the page would not be inserted into the filecache.
H
Hugh Dickins 已提交
865
		 */
866 867 868
		BUG_ON(page && !PageUptodate(page));
		*pagep = page;
		return 0;
H
Hugh Dickins 已提交
869 870 871
	}

	/*
872 873
	 * Fast cache lookup did not find it:
	 * bring it back from swap or allocate.
H
Hugh Dickins 已提交
874
	 */
875 876
	info = SHMEM_I(inode);
	sbinfo = SHMEM_SB(inode->i_sb);
L
Linus Torvalds 已提交
877 878 879

	if (swap.val) {
		/* Look it up and read it in.. */
H
Hugh Dickins 已提交
880 881
		page = lookup_swap_cache(swap);
		if (!page) {
L
Linus Torvalds 已提交
882
			/* here we actually do the io */
883 884
			if (fault_type)
				*fault_type |= VM_FAULT_MAJOR;
885
			page = shmem_swapin(swap, gfp, info, index);
H
Hugh Dickins 已提交
886
			if (!page) {
887 888
				error = -ENOMEM;
				goto failed;
L
Linus Torvalds 已提交
889 890 891 892
			}
		}

		/* We have to do this with page locked to prevent races */
893
		lock_page(page);
H
Hugh Dickins 已提交
894
		if (!PageUptodate(page)) {
L
Linus Torvalds 已提交
895
			error = -EIO;
896
			goto failed;
L
Linus Torvalds 已提交
897
		}
898 899 900 901 902 903 904 905 906
		wait_on_page_writeback(page);

		/* Someone may have already done it for us */
		if (page->mapping) {
			if (page->mapping == mapping &&
			    page->index == index)
				goto done;
			error = -EEXIST;
			goto failed;
L
Linus Torvalds 已提交
907
		}
H
Hugh Dickins 已提交
908

909 910 911 912 913
		error = mem_cgroup_cache_charge(page, current->mm,
						gfp & GFP_RECLAIM_MASK);
		if (!error)
			error = shmem_add_to_page_cache(page, mapping, index,
						gfp, swp_to_radix_entry(swap));
914 915 916 917
		if (error)
			goto failed;

		spin_lock(&info->lock);
918
		info->swapped--;
919
		shmem_recalc_inode(inode);
H
Hugh Dickins 已提交
920
		spin_unlock(&info->lock);
921 922

		delete_from_swap_cache(page);
H
Hugh Dickins 已提交
923 924 925
		set_page_dirty(page);
		swap_free(swap);

926 927 928 929
	} else {
		if (shmem_acct_block(info->flags)) {
			error = -ENOSPC;
			goto failed;
L
Linus Torvalds 已提交
930
		}
931
		if (sbinfo->max_blocks) {
932
			if (percpu_counter_compare(&sbinfo->used_blocks,
933 934 935 936
						sbinfo->max_blocks) >= 0) {
				error = -ENOSPC;
				goto unacct;
			}
937
			percpu_counter_inc(&sbinfo->used_blocks);
938
		}
L
Linus Torvalds 已提交
939

940 941 942 943
		page = shmem_alloc_page(gfp, info, index);
		if (!page) {
			error = -ENOMEM;
			goto decused;
L
Linus Torvalds 已提交
944 945
		}

946 947
		SetPageSwapBacked(page);
		__set_page_locked(page);
948 949 950 951 952
		error = mem_cgroup_cache_charge(page, current->mm,
						gfp & GFP_RECLAIM_MASK);
		if (!error)
			error = shmem_add_to_page_cache(page, mapping, index,
						gfp, NULL);
953 954 955 956 957
		if (error)
			goto decused;
		lru_cache_add_anon(page);

		spin_lock(&info->lock);
L
Linus Torvalds 已提交
958
		info->alloced++;
959 960
		inode->i_blocks += BLOCKS_PER_PAGE;
		shmem_recalc_inode(inode);
L
Linus Torvalds 已提交
961
		spin_unlock(&info->lock);
962

H
Hugh Dickins 已提交
963 964 965
		clear_highpage(page);
		flush_dcache_page(page);
		SetPageUptodate(page);
966
		if (sgp == SGP_DIRTY)
H
Hugh Dickins 已提交
967
			set_page_dirty(page);
L
Linus Torvalds 已提交
968 969
	}
done:
970 971 972 973 974
	/* Perhaps the file has been truncated since we checked */
	if (sgp != SGP_WRITE &&
	    ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
		error = -EINVAL;
		goto trunc;
H
Hugh Dickins 已提交
975
	}
976 977
	*pagep = page;
	return 0;
L
Linus Torvalds 已提交
978

979
	/*
980
	 * Error recovery.
981
	 */
982 983 984 985 986 987
trunc:
	ClearPageDirty(page);
	delete_from_page_cache(page);
	spin_lock(&info->lock);
	info->alloced--;
	inode->i_blocks -= BLOCKS_PER_PAGE;
988
	spin_unlock(&info->lock);
989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
decused:
	if (sbinfo->max_blocks)
		percpu_counter_add(&sbinfo->used_blocks, -1);
unacct:
	shmem_unacct_blocks(info->flags, 1);
failed:
	if (swap.val && error != -EINVAL) {
		struct page *test = find_get_page(mapping, index);
		if (test && !radix_tree_exceptional_entry(test))
			page_cache_release(test);
		/* Have another try if the entry has changed */
		if (test != swp_to_radix_entry(swap))
			error = -EEXIST;
	}
H
Hugh Dickins 已提交
1003
	if (page) {
1004
		unlock_page(page);
H
Hugh Dickins 已提交
1005
		page_cache_release(page);
1006 1007 1008 1009 1010 1011
	}
	if (error == -ENOSPC && !once++) {
		info = SHMEM_I(inode);
		spin_lock(&info->lock);
		shmem_recalc_inode(inode);
		spin_unlock(&info->lock);
H
Hugh Dickins 已提交
1012
		goto repeat;
1013
	}
1014 1015 1016
	if (error == -EEXIST)
		goto repeat;
	return error;
L
Linus Torvalds 已提交
1017 1018
}

N
Nick Piggin 已提交
1019
static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
L
Linus Torvalds 已提交
1020
{
1021
	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
L
Linus Torvalds 已提交
1022
	int error;
1023
	int ret = VM_FAULT_LOCKED;
L
Linus Torvalds 已提交
1024

1025
	error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
N
Nick Piggin 已提交
1026 1027
	if (error)
		return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1028

1029 1030 1031 1032
	if (ret & VM_FAULT_MAJOR) {
		count_vm_event(PGMAJFAULT);
		mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
	}
1033
	return ret;
L
Linus Torvalds 已提交
1034 1035 1036
}

#ifdef CONFIG_NUMA
1037
static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
L
Linus Torvalds 已提交
1038
{
1039 1040
	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
	return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
L
Linus Torvalds 已提交
1041 1042
}

A
Adrian Bunk 已提交
1043 1044
static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
					  unsigned long addr)
L
Linus Torvalds 已提交
1045
{
1046 1047
	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
	pgoff_t index;
L
Linus Torvalds 已提交
1048

1049 1050
	index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
	return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
L
Linus Torvalds 已提交
1051 1052 1053 1054 1055
}
#endif

int shmem_lock(struct file *file, int lock, struct user_struct *user)
{
1056
	struct inode *inode = file->f_path.dentry->d_inode;
L
Linus Torvalds 已提交
1057 1058 1059 1060 1061 1062 1063 1064
	struct shmem_inode_info *info = SHMEM_I(inode);
	int retval = -ENOMEM;

	spin_lock(&info->lock);
	if (lock && !(info->flags & VM_LOCKED)) {
		if (!user_shm_lock(inode->i_size, user))
			goto out_nomem;
		info->flags |= VM_LOCKED;
1065
		mapping_set_unevictable(file->f_mapping);
L
Linus Torvalds 已提交
1066 1067 1068 1069
	}
	if (!lock && (info->flags & VM_LOCKED) && user) {
		user_shm_unlock(inode->i_size, user);
		info->flags &= ~VM_LOCKED;
1070 1071
		mapping_clear_unevictable(file->f_mapping);
		scan_mapping_unevictable_pages(file->f_mapping);
L
Linus Torvalds 已提交
1072 1073
	}
	retval = 0;
1074

L
Linus Torvalds 已提交
1075 1076 1077 1078 1079
out_nomem:
	spin_unlock(&info->lock);
	return retval;
}

1080
static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
L
Linus Torvalds 已提交
1081 1082 1083
{
	file_accessed(file);
	vma->vm_ops = &shmem_vm_ops;
N
Nick Piggin 已提交
1084
	vma->vm_flags |= VM_CAN_NONLINEAR;
L
Linus Torvalds 已提交
1085 1086 1087
	return 0;
}

1088 1089
static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
				     int mode, dev_t dev, unsigned long flags)
L
Linus Torvalds 已提交
1090 1091 1092 1093 1094
{
	struct inode *inode;
	struct shmem_inode_info *info;
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);

1095 1096
	if (shmem_reserve_inode(sb))
		return NULL;
L
Linus Torvalds 已提交
1097 1098 1099

	inode = new_inode(sb);
	if (inode) {
1100
		inode->i_ino = get_next_ino();
1101
		inode_init_owner(inode, dir, mode);
L
Linus Torvalds 已提交
1102 1103 1104
		inode->i_blocks = 0;
		inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1105
		inode->i_generation = get_seconds();
L
Linus Torvalds 已提交
1106 1107 1108
		info = SHMEM_I(inode);
		memset(info, 0, (char *)inode - (char *)info);
		spin_lock_init(&info->lock);
1109
		info->flags = flags & VM_NORESERVE;
L
Linus Torvalds 已提交
1110
		INIT_LIST_HEAD(&info->swaplist);
1111
		INIT_LIST_HEAD(&info->xattr_list);
1112
		cache_no_acl(inode);
L
Linus Torvalds 已提交
1113 1114 1115

		switch (mode & S_IFMT) {
		default:
1116
			inode->i_op = &shmem_special_inode_operations;
L
Linus Torvalds 已提交
1117 1118 1119
			init_special_inode(inode, mode, dev);
			break;
		case S_IFREG:
1120
			inode->i_mapping->a_ops = &shmem_aops;
L
Linus Torvalds 已提交
1121 1122
			inode->i_op = &shmem_inode_operations;
			inode->i_fop = &shmem_file_operations;
1123 1124
			mpol_shared_policy_init(&info->policy,
						 shmem_get_sbmpol(sbinfo));
L
Linus Torvalds 已提交
1125 1126
			break;
		case S_IFDIR:
1127
			inc_nlink(inode);
L
Linus Torvalds 已提交
1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
			/* Some things misbehave if size == 0 on a directory */
			inode->i_size = 2 * BOGO_DIRENT_SIZE;
			inode->i_op = &shmem_dir_inode_operations;
			inode->i_fop = &simple_dir_operations;
			break;
		case S_IFLNK:
			/*
			 * Must not load anything in the rbtree,
			 * mpol_free_shared_policy will not be called.
			 */
1138
			mpol_shared_policy_init(&info->policy, NULL);
L
Linus Torvalds 已提交
1139 1140
			break;
		}
1141 1142
	} else
		shmem_free_inode(sb);
L
Linus Torvalds 已提交
1143 1144 1145 1146
	return inode;
}

#ifdef CONFIG_TMPFS
1147
static const struct inode_operations shmem_symlink_inode_operations;
1148
static const struct inode_operations shmem_short_symlink_operations;
L
Linus Torvalds 已提交
1149 1150

static int
N
Nick Piggin 已提交
1151 1152 1153
shmem_write_begin(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned flags,
			struct page **pagep, void **fsdata)
L
Linus Torvalds 已提交
1154
{
N
Nick Piggin 已提交
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
	struct inode *inode = mapping->host;
	pgoff_t index = pos >> PAGE_CACHE_SHIFT;
	return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
}

static int
shmem_write_end(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned copied,
			struct page *page, void *fsdata)
{
	struct inode *inode = mapping->host;

H
Hugh Dickins 已提交
1167 1168 1169
	if (pos + copied > inode->i_size)
		i_size_write(inode, pos + copied);

N
Nick Piggin 已提交
1170
	set_page_dirty(page);
1171
	unlock_page(page);
N
Nick Piggin 已提交
1172 1173 1174
	page_cache_release(page);

	return copied;
L
Linus Torvalds 已提交
1175 1176 1177 1178
}

static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
{
1179
	struct inode *inode = filp->f_path.dentry->d_inode;
L
Linus Torvalds 已提交
1180
	struct address_space *mapping = inode->i_mapping;
1181 1182
	pgoff_t index;
	unsigned long offset;
1183 1184 1185 1186 1187 1188 1189 1190 1191
	enum sgp_type sgp = SGP_READ;

	/*
	 * Might this read be for a stacking filesystem?  Then when reading
	 * holes of a sparse file, we actually need to allocate those pages,
	 * and even mark them dirty, so it cannot exceed the max_blocks limit.
	 */
	if (segment_eq(get_fs(), KERNEL_DS))
		sgp = SGP_DIRTY;
L
Linus Torvalds 已提交
1192 1193 1194 1195 1196 1197

	index = *ppos >> PAGE_CACHE_SHIFT;
	offset = *ppos & ~PAGE_CACHE_MASK;

	for (;;) {
		struct page *page = NULL;
1198 1199
		pgoff_t end_index;
		unsigned long nr, ret;
L
Linus Torvalds 已提交
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210
		loff_t i_size = i_size_read(inode);

		end_index = i_size >> PAGE_CACHE_SHIFT;
		if (index > end_index)
			break;
		if (index == end_index) {
			nr = i_size & ~PAGE_CACHE_MASK;
			if (nr <= offset)
				break;
		}

1211
		desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
L
Linus Torvalds 已提交
1212 1213 1214 1215 1216
		if (desc->error) {
			if (desc->error == -EINVAL)
				desc->error = 0;
			break;
		}
H
Hugh Dickins 已提交
1217 1218
		if (page)
			unlock_page(page);
L
Linus Torvalds 已提交
1219 1220 1221

		/*
		 * We must evaluate after, since reads (unlike writes)
1222
		 * are called without i_mutex protection against truncate
L
Linus Torvalds 已提交
1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249
		 */
		nr = PAGE_CACHE_SIZE;
		i_size = i_size_read(inode);
		end_index = i_size >> PAGE_CACHE_SHIFT;
		if (index == end_index) {
			nr = i_size & ~PAGE_CACHE_MASK;
			if (nr <= offset) {
				if (page)
					page_cache_release(page);
				break;
			}
		}
		nr -= offset;

		if (page) {
			/*
			 * If users can be writing to this page using arbitrary
			 * virtual addresses, take care about potential aliasing
			 * before reading the page on the kernel side.
			 */
			if (mapping_writably_mapped(mapping))
				flush_dcache_page(page);
			/*
			 * Mark the page accessed if we read the beginning.
			 */
			if (!offset)
				mark_page_accessed(page);
N
Nick Piggin 已提交
1250
		} else {
L
Linus Torvalds 已提交
1251
			page = ZERO_PAGE(0);
N
Nick Piggin 已提交
1252 1253
			page_cache_get(page);
		}
L
Linus Torvalds 已提交
1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280

		/*
		 * Ok, we have the page, and it's up-to-date, so
		 * now we can copy it to user space...
		 *
		 * The actor routine returns how many bytes were actually used..
		 * NOTE! This may not be the same as how much of a user buffer
		 * we filled up (we may be padding etc), so we can only update
		 * "pos" here (the actor routine has to update the user buffer
		 * pointers and the remaining count).
		 */
		ret = actor(desc, page, offset, nr);
		offset += ret;
		index += offset >> PAGE_CACHE_SHIFT;
		offset &= ~PAGE_CACHE_MASK;

		page_cache_release(page);
		if (ret != nr || !desc->count)
			break;

		cond_resched();
	}

	*ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
	file_accessed(filp);
}

H
Hugh Dickins 已提交
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312
static ssize_t shmem_file_aio_read(struct kiocb *iocb,
		const struct iovec *iov, unsigned long nr_segs, loff_t pos)
{
	struct file *filp = iocb->ki_filp;
	ssize_t retval;
	unsigned long seg;
	size_t count;
	loff_t *ppos = &iocb->ki_pos;

	retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
	if (retval)
		return retval;

	for (seg = 0; seg < nr_segs; seg++) {
		read_descriptor_t desc;

		desc.written = 0;
		desc.arg.buf = iov[seg].iov_base;
		desc.count = iov[seg].iov_len;
		if (desc.count == 0)
			continue;
		desc.error = 0;
		do_shmem_file_read(filp, ppos, &desc, file_read_actor);
		retval += desc.written;
		if (desc.error) {
			retval = retval ?: desc.error;
			break;
		}
		if (desc.count > 0)
			break;
	}
	return retval;
L
Linus Torvalds 已提交
1313 1314
}

1315 1316 1317 1318 1319
static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
				struct pipe_inode_info *pipe, size_t len,
				unsigned int flags)
{
	struct address_space *mapping = in->f_mapping;
1320
	struct inode *inode = mapping->host;
1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
	unsigned int loff, nr_pages, req_pages;
	struct page *pages[PIPE_DEF_BUFFERS];
	struct partial_page partial[PIPE_DEF_BUFFERS];
	struct page *page;
	pgoff_t index, end_index;
	loff_t isize, left;
	int error, page_nr;
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
		.flags = flags,
		.ops = &page_cache_pipe_buf_ops,
		.spd_release = spd_release_page,
	};

1336
	isize = i_size_read(inode);
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
	if (unlikely(*ppos >= isize))
		return 0;

	left = isize - *ppos;
	if (unlikely(left < len))
		len = left;

	if (splice_grow_spd(pipe, &spd))
		return -ENOMEM;

	index = *ppos >> PAGE_CACHE_SHIFT;
	loff = *ppos & ~PAGE_CACHE_MASK;
	req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
	nr_pages = min(req_pages, pipe->buffers);

	spd.nr_pages = find_get_pages_contig(mapping, index,
						nr_pages, spd.pages);
	index += spd.nr_pages;
	error = 0;

1357 1358 1359 1360 1361
	while (spd.nr_pages < nr_pages) {
		error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
		if (error)
			break;
		unlock_page(page);
1362 1363 1364 1365 1366 1367 1368
		spd.pages[spd.nr_pages++] = page;
		index++;
	}

	index = *ppos >> PAGE_CACHE_SHIFT;
	nr_pages = spd.nr_pages;
	spd.nr_pages = 0;
1369

1370 1371 1372 1373 1374 1375 1376 1377 1378
	for (page_nr = 0; page_nr < nr_pages; page_nr++) {
		unsigned int this_len;

		if (!len)
			break;

		this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
		page = spd.pages[page_nr];

1379 1380 1381 1382
		if (!PageUptodate(page) || page->mapping != mapping) {
			error = shmem_getpage(inode, index, &page,
							SGP_CACHE, NULL);
			if (error)
1383
				break;
1384 1385 1386
			unlock_page(page);
			page_cache_release(spd.pages[page_nr]);
			spd.pages[page_nr] = page;
1387
		}
1388 1389

		isize = i_size_read(inode);
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
		end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
		if (unlikely(!isize || index > end_index))
			break;

		if (end_index == index) {
			unsigned int plen;

			plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
			if (plen <= loff)
				break;

			this_len = min(this_len, plen - loff);
			len = this_len;
		}

		spd.partial[page_nr].offset = loff;
		spd.partial[page_nr].len = this_len;
		len -= this_len;
		loff = 0;
		spd.nr_pages++;
		index++;
	}

	while (page_nr < nr_pages)
		page_cache_release(spd.pages[page_nr++]);

	if (spd.nr_pages)
		error = splice_to_pipe(pipe, &spd);

	splice_shrink_spd(pipe, &spd);

	if (error > 0) {
		*ppos += error;
		file_accessed(in);
	}
	return error;
}

1428
static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
L
Linus Torvalds 已提交
1429
{
1430
	struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
L
Linus Torvalds 已提交
1431 1432 1433 1434

	buf->f_type = TMPFS_MAGIC;
	buf->f_bsize = PAGE_CACHE_SIZE;
	buf->f_namelen = NAME_MAX;
1435
	if (sbinfo->max_blocks) {
L
Linus Torvalds 已提交
1436
		buf->f_blocks = sbinfo->max_blocks;
1437 1438 1439
		buf->f_bavail =
		buf->f_bfree  = sbinfo->max_blocks -
				percpu_counter_sum(&sbinfo->used_blocks);
1440 1441
	}
	if (sbinfo->max_inodes) {
L
Linus Torvalds 已提交
1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
		buf->f_files = sbinfo->max_inodes;
		buf->f_ffree = sbinfo->free_inodes;
	}
	/* else leave those fields 0 like simple_statfs */
	return 0;
}

/*
 * File creation. Allocate an inode, and we're done..
 */
static int
shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
{
1455
	struct inode *inode;
L
Linus Torvalds 已提交
1456 1457
	int error = -ENOSPC;

1458
	inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
L
Linus Torvalds 已提交
1459
	if (inode) {
1460
		error = security_inode_init_security(inode, dir,
1461
						     &dentry->d_name,
1462
						     NULL, NULL);
1463 1464 1465 1466 1467
		if (error) {
			if (error != -EOPNOTSUPP) {
				iput(inode);
				return error;
			}
1468
		}
1469 1470
#ifdef CONFIG_TMPFS_POSIX_ACL
		error = generic_acl_init(inode, dir);
1471 1472 1473
		if (error) {
			iput(inode);
			return error;
1474
		}
A
Al Viro 已提交
1475 1476
#else
		error = 0;
1477
#endif
L
Linus Torvalds 已提交
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
		dir->i_size += BOGO_DIRENT_SIZE;
		dir->i_ctime = dir->i_mtime = CURRENT_TIME;
		d_instantiate(dentry, inode);
		dget(dentry); /* Extra count - pin the dentry in core */
	}
	return error;
}

static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
{
	int error;

	if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
		return error;
1492
	inc_nlink(dir);
L
Linus Torvalds 已提交
1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507
	return 0;
}

static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
		struct nameidata *nd)
{
	return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
}

/*
 * Link a file..
 */
static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
{
	struct inode *inode = old_dentry->d_inode;
1508
	int ret;
L
Linus Torvalds 已提交
1509 1510 1511 1512 1513 1514

	/*
	 * No ordinary (disk based) filesystem counts links as inodes;
	 * but each new link needs a new dentry, pinning lowmem, and
	 * tmpfs dentries cannot be pruned until they are unlinked.
	 */
1515 1516 1517
	ret = shmem_reserve_inode(inode->i_sb);
	if (ret)
		goto out;
L
Linus Torvalds 已提交
1518 1519 1520

	dir->i_size += BOGO_DIRENT_SIZE;
	inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1521
	inc_nlink(inode);
A
Al Viro 已提交
1522
	ihold(inode);	/* New dentry reference */
L
Linus Torvalds 已提交
1523 1524
	dget(dentry);		/* Extra pinning count for the created dentry */
	d_instantiate(dentry, inode);
1525 1526
out:
	return ret;
L
Linus Torvalds 已提交
1527 1528 1529 1530 1531 1532
}

static int shmem_unlink(struct inode *dir, struct dentry *dentry)
{
	struct inode *inode = dentry->d_inode;

1533 1534
	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
		shmem_free_inode(inode->i_sb);
L
Linus Torvalds 已提交
1535 1536 1537

	dir->i_size -= BOGO_DIRENT_SIZE;
	inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1538
	drop_nlink(inode);
L
Linus Torvalds 已提交
1539 1540 1541 1542 1543 1544 1545 1546 1547
	dput(dentry);	/* Undo the count from "create" - this does all the work */
	return 0;
}

static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
{
	if (!simple_empty(dentry))
		return -ENOTEMPTY;

1548 1549
	drop_nlink(dentry->d_inode);
	drop_nlink(dir);
L
Linus Torvalds 已提交
1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
	return shmem_unlink(dir, dentry);
}

/*
 * The VFS layer already does all the dentry stuff for rename,
 * we just have to decrement the usage count for the target if
 * it exists so that the VFS layer correctly free's it when it
 * gets overwritten.
 */
static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
{
	struct inode *inode = old_dentry->d_inode;
	int they_are_dirs = S_ISDIR(inode->i_mode);

	if (!simple_empty(new_dentry))
		return -ENOTEMPTY;

	if (new_dentry->d_inode) {
		(void) shmem_unlink(new_dir, new_dentry);
		if (they_are_dirs)
1570
			drop_nlink(old_dir);
L
Linus Torvalds 已提交
1571
	} else if (they_are_dirs) {
1572
		drop_nlink(old_dir);
1573
		inc_nlink(new_dir);
L
Linus Torvalds 已提交
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588
	}

	old_dir->i_size -= BOGO_DIRENT_SIZE;
	new_dir->i_size += BOGO_DIRENT_SIZE;
	old_dir->i_ctime = old_dir->i_mtime =
	new_dir->i_ctime = new_dir->i_mtime =
	inode->i_ctime = CURRENT_TIME;
	return 0;
}

static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
{
	int error;
	int len;
	struct inode *inode;
H
Hugh Dickins 已提交
1589
	struct page *page;
L
Linus Torvalds 已提交
1590 1591 1592 1593 1594 1595 1596
	char *kaddr;
	struct shmem_inode_info *info;

	len = strlen(symname) + 1;
	if (len > PAGE_CACHE_SIZE)
		return -ENAMETOOLONG;

1597
	inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
L
Linus Torvalds 已提交
1598 1599 1600
	if (!inode)
		return -ENOSPC;

1601
	error = security_inode_init_security(inode, dir, &dentry->d_name,
1602
					     NULL, NULL);
1603 1604 1605 1606 1607 1608 1609 1610
	if (error) {
		if (error != -EOPNOTSUPP) {
			iput(inode);
			return error;
		}
		error = 0;
	}

L
Linus Torvalds 已提交
1611 1612
	info = SHMEM_I(inode);
	inode->i_size = len-1;
1613 1614 1615 1616 1617 1618 1619
	if (len <= SHORT_SYMLINK_LEN) {
		info->symlink = kmemdup(symname, len, GFP_KERNEL);
		if (!info->symlink) {
			iput(inode);
			return -ENOMEM;
		}
		inode->i_op = &shmem_short_symlink_operations;
L
Linus Torvalds 已提交
1620 1621 1622 1623 1624 1625
	} else {
		error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
		if (error) {
			iput(inode);
			return error;
		}
1626
		inode->i_mapping->a_ops = &shmem_aops;
L
Linus Torvalds 已提交
1627 1628 1629 1630 1631
		inode->i_op = &shmem_symlink_inode_operations;
		kaddr = kmap_atomic(page, KM_USER0);
		memcpy(kaddr, symname, len);
		kunmap_atomic(kaddr, KM_USER0);
		set_page_dirty(page);
1632
		unlock_page(page);
L
Linus Torvalds 已提交
1633 1634 1635 1636 1637 1638 1639 1640 1641
		page_cache_release(page);
	}
	dir->i_size += BOGO_DIRENT_SIZE;
	dir->i_ctime = dir->i_mtime = CURRENT_TIME;
	d_instantiate(dentry, inode);
	dget(dentry);
	return 0;
}

1642
static void *shmem_follow_short_symlink(struct dentry *dentry, struct nameidata *nd)
L
Linus Torvalds 已提交
1643
{
1644
	nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink);
1645
	return NULL;
L
Linus Torvalds 已提交
1646 1647
}

1648
static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
L
Linus Torvalds 已提交
1649 1650
{
	struct page *page = NULL;
1651 1652
	int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
	nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));
H
Hugh Dickins 已提交
1653 1654
	if (page)
		unlock_page(page);
1655
	return page;
L
Linus Torvalds 已提交
1656 1657
}

1658
static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
L
Linus Torvalds 已提交
1659 1660
{
	if (!IS_ERR(nd_get_link(nd))) {
1661
		struct page *page = cookie;
L
Linus Torvalds 已提交
1662 1663 1664 1665 1666 1667
		kunmap(page);
		mark_page_accessed(page);
		page_cache_release(page);
	}
}

1668
#ifdef CONFIG_TMPFS_XATTR
1669
/*
1670 1671
 * Superblocks without xattr inode operations may get some security.* xattr
 * support from the LSM "for free". As soon as we have any other xattrs
1672 1673 1674 1675
 * like ACLs, we also need to implement the security.* handlers at
 * filesystem level, though.
 */

1676 1677
static int shmem_xattr_get(struct dentry *dentry, const char *name,
			   void *buffer, size_t size)
1678
{
1679 1680 1681
	struct shmem_inode_info *info;
	struct shmem_xattr *xattr;
	int ret = -ENODATA;
1682

1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
	info = SHMEM_I(dentry->d_inode);

	spin_lock(&info->lock);
	list_for_each_entry(xattr, &info->xattr_list, list) {
		if (strcmp(name, xattr->name))
			continue;

		ret = xattr->size;
		if (buffer) {
			if (size < xattr->size)
				ret = -ERANGE;
			else
				memcpy(buffer, xattr->value, xattr->size);
		}
		break;
	}
	spin_unlock(&info->lock);
	return ret;
1701 1702
}

1703 1704
static int shmem_xattr_set(struct dentry *dentry, const char *name,
			   const void *value, size_t size, int flags)
1705
{
1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
	struct inode *inode = dentry->d_inode;
	struct shmem_inode_info *info = SHMEM_I(inode);
	struct shmem_xattr *xattr;
	struct shmem_xattr *new_xattr = NULL;
	size_t len;
	int err = 0;

	/* value == NULL means remove */
	if (value) {
		/* wrap around? */
		len = sizeof(*new_xattr) + size;
		if (len <= sizeof(*new_xattr))
			return -ENOMEM;

		new_xattr = kmalloc(len, GFP_KERNEL);
		if (!new_xattr)
			return -ENOMEM;

		new_xattr->name = kstrdup(name, GFP_KERNEL);
		if (!new_xattr->name) {
			kfree(new_xattr);
			return -ENOMEM;
		}

		new_xattr->size = size;
		memcpy(new_xattr->value, value, size);
	}

	spin_lock(&info->lock);
	list_for_each_entry(xattr, &info->xattr_list, list) {
		if (!strcmp(name, xattr->name)) {
			if (flags & XATTR_CREATE) {
				xattr = new_xattr;
				err = -EEXIST;
			} else if (new_xattr) {
				list_replace(&xattr->list, &new_xattr->list);
			} else {
				list_del(&xattr->list);
			}
			goto out;
		}
	}
	if (flags & XATTR_REPLACE) {
		xattr = new_xattr;
		err = -ENODATA;
	} else {
		list_add(&new_xattr->list, &info->xattr_list);
		xattr = NULL;
	}
out:
	spin_unlock(&info->lock);
	if (xattr)
		kfree(xattr->name);
	kfree(xattr);
	return err;
1761 1762
}

1763
static const struct xattr_handler *shmem_xattr_handlers[] = {
1764
#ifdef CONFIG_TMPFS_POSIX_ACL
1765 1766
	&generic_acl_access_handler,
	&generic_acl_default_handler,
1767
#endif
1768 1769
	NULL
};
1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891

static int shmem_xattr_validate(const char *name)
{
	struct { const char *prefix; size_t len; } arr[] = {
		{ XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
		{ XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
	};
	int i;

	for (i = 0; i < ARRAY_SIZE(arr); i++) {
		size_t preflen = arr[i].len;
		if (strncmp(name, arr[i].prefix, preflen) == 0) {
			if (!name[preflen])
				return -EINVAL;
			return 0;
		}
	}
	return -EOPNOTSUPP;
}

static ssize_t shmem_getxattr(struct dentry *dentry, const char *name,
			      void *buffer, size_t size)
{
	int err;

	/*
	 * If this is a request for a synthetic attribute in the system.*
	 * namespace use the generic infrastructure to resolve a handler
	 * for it via sb->s_xattr.
	 */
	if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
		return generic_getxattr(dentry, name, buffer, size);

	err = shmem_xattr_validate(name);
	if (err)
		return err;

	return shmem_xattr_get(dentry, name, buffer, size);
}

static int shmem_setxattr(struct dentry *dentry, const char *name,
			  const void *value, size_t size, int flags)
{
	int err;

	/*
	 * If this is a request for a synthetic attribute in the system.*
	 * namespace use the generic infrastructure to resolve a handler
	 * for it via sb->s_xattr.
	 */
	if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
		return generic_setxattr(dentry, name, value, size, flags);

	err = shmem_xattr_validate(name);
	if (err)
		return err;

	if (size == 0)
		value = "";  /* empty EA, do not remove */

	return shmem_xattr_set(dentry, name, value, size, flags);

}

static int shmem_removexattr(struct dentry *dentry, const char *name)
{
	int err;

	/*
	 * If this is a request for a synthetic attribute in the system.*
	 * namespace use the generic infrastructure to resolve a handler
	 * for it via sb->s_xattr.
	 */
	if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
		return generic_removexattr(dentry, name);

	err = shmem_xattr_validate(name);
	if (err)
		return err;

	return shmem_xattr_set(dentry, name, NULL, 0, XATTR_REPLACE);
}

static bool xattr_is_trusted(const char *name)
{
	return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN);
}

static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
{
	bool trusted = capable(CAP_SYS_ADMIN);
	struct shmem_xattr *xattr;
	struct shmem_inode_info *info;
	size_t used = 0;

	info = SHMEM_I(dentry->d_inode);

	spin_lock(&info->lock);
	list_for_each_entry(xattr, &info->xattr_list, list) {
		size_t len;

		/* skip "trusted." attributes for unprivileged callers */
		if (!trusted && xattr_is_trusted(xattr->name))
			continue;

		len = strlen(xattr->name) + 1;
		used += len;
		if (buffer) {
			if (size < used) {
				used = -ERANGE;
				break;
			}
			memcpy(buffer, xattr->name, len);
			buffer += len;
		}
	}
	spin_unlock(&info->lock);

	return used;
}
#endif /* CONFIG_TMPFS_XATTR */

1892
static const struct inode_operations shmem_short_symlink_operations = {
1893
	.readlink	= generic_readlink,
1894
	.follow_link	= shmem_follow_short_symlink,
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911
#ifdef CONFIG_TMPFS_XATTR
	.setxattr	= shmem_setxattr,
	.getxattr	= shmem_getxattr,
	.listxattr	= shmem_listxattr,
	.removexattr	= shmem_removexattr,
#endif
};

static const struct inode_operations shmem_symlink_inode_operations = {
	.readlink	= generic_readlink,
	.follow_link	= shmem_follow_link,
	.put_link	= shmem_put_link,
#ifdef CONFIG_TMPFS_XATTR
	.setxattr	= shmem_setxattr,
	.getxattr	= shmem_getxattr,
	.listxattr	= shmem_listxattr,
	.removexattr	= shmem_removexattr,
1912
#endif
1913
};
1914

1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927
static struct dentry *shmem_get_parent(struct dentry *child)
{
	return ERR_PTR(-ESTALE);
}

static int shmem_match(struct inode *ino, void *vfh)
{
	__u32 *fh = vfh;
	__u64 inum = fh[2];
	inum = (inum << 32) | fh[1];
	return ino->i_ino == inum && fh[0] == ino->i_generation;
}

C
Christoph Hellwig 已提交
1928 1929
static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
		struct fid *fid, int fh_len, int fh_type)
1930 1931
{
	struct inode *inode;
C
Christoph Hellwig 已提交
1932 1933 1934 1935 1936 1937
	struct dentry *dentry = NULL;
	u64 inum = fid->raw[2];
	inum = (inum << 32) | fid->raw[1];

	if (fh_len < 3)
		return NULL;
1938

C
Christoph Hellwig 已提交
1939 1940
	inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
			shmem_match, fid->raw);
1941
	if (inode) {
C
Christoph Hellwig 已提交
1942
		dentry = d_find_alias(inode);
1943 1944 1945
		iput(inode);
	}

C
Christoph Hellwig 已提交
1946
	return dentry;
1947 1948 1949 1950 1951 1952 1953
}

static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
				int connectable)
{
	struct inode *inode = dentry->d_inode;

1954 1955
	if (*len < 3) {
		*len = 3;
1956
		return 255;
1957
	}
1958

A
Al Viro 已提交
1959
	if (inode_unhashed(inode)) {
1960 1961 1962 1963 1964 1965 1966
		/* Unfortunately insert_inode_hash is not idempotent,
		 * so as we hash inodes here rather than at creation
		 * time, we need a lock to ensure we only try
		 * to do it once
		 */
		static DEFINE_SPINLOCK(lock);
		spin_lock(&lock);
A
Al Viro 已提交
1967
		if (inode_unhashed(inode))
1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980
			__insert_inode_hash(inode,
					    inode->i_ino + inode->i_generation);
		spin_unlock(&lock);
	}

	fh[0] = inode->i_generation;
	fh[1] = inode->i_ino;
	fh[2] = ((__u64)inode->i_ino) >> 32;

	*len = 3;
	return 1;
}

1981
static const struct export_operations shmem_export_ops = {
1982 1983
	.get_parent     = shmem_get_parent,
	.encode_fh      = shmem_encode_fh,
C
Christoph Hellwig 已提交
1984
	.fh_to_dentry	= shmem_fh_to_dentry,
1985 1986
};

1987 1988
static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
			       bool remount)
L
Linus Torvalds 已提交
1989 1990 1991
{
	char *this_char, *value, *rest;

1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008
	while (options != NULL) {
		this_char = options;
		for (;;) {
			/*
			 * NUL-terminate this option: unfortunately,
			 * mount options form a comma-separated list,
			 * but mpol's nodelist may also contain commas.
			 */
			options = strchr(options, ',');
			if (options == NULL)
				break;
			options++;
			if (!isdigit(*options)) {
				options[-1] = '\0';
				break;
			}
		}
L
Linus Torvalds 已提交
2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030
		if (!*this_char)
			continue;
		if ((value = strchr(this_char,'=')) != NULL) {
			*value++ = 0;
		} else {
			printk(KERN_ERR
			    "tmpfs: No value for mount option '%s'\n",
			    this_char);
			return 1;
		}

		if (!strcmp(this_char,"size")) {
			unsigned long long size;
			size = memparse(value,&rest);
			if (*rest == '%') {
				size <<= PAGE_SHIFT;
				size *= totalram_pages;
				do_div(size, 100);
				rest++;
			}
			if (*rest)
				goto bad_val;
2031 2032
			sbinfo->max_blocks =
				DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
L
Linus Torvalds 已提交
2033
		} else if (!strcmp(this_char,"nr_blocks")) {
2034
			sbinfo->max_blocks = memparse(value, &rest);
L
Linus Torvalds 已提交
2035 2036 2037
			if (*rest)
				goto bad_val;
		} else if (!strcmp(this_char,"nr_inodes")) {
2038
			sbinfo->max_inodes = memparse(value, &rest);
L
Linus Torvalds 已提交
2039 2040 2041
			if (*rest)
				goto bad_val;
		} else if (!strcmp(this_char,"mode")) {
2042
			if (remount)
L
Linus Torvalds 已提交
2043
				continue;
2044
			sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
L
Linus Torvalds 已提交
2045 2046 2047
			if (*rest)
				goto bad_val;
		} else if (!strcmp(this_char,"uid")) {
2048
			if (remount)
L
Linus Torvalds 已提交
2049
				continue;
2050
			sbinfo->uid = simple_strtoul(value, &rest, 0);
L
Linus Torvalds 已提交
2051 2052 2053
			if (*rest)
				goto bad_val;
		} else if (!strcmp(this_char,"gid")) {
2054
			if (remount)
L
Linus Torvalds 已提交
2055
				continue;
2056
			sbinfo->gid = simple_strtoul(value, &rest, 0);
L
Linus Torvalds 已提交
2057 2058
			if (*rest)
				goto bad_val;
2059
		} else if (!strcmp(this_char,"mpol")) {
2060
			if (mpol_parse_str(value, &sbinfo->mpol, 1))
2061
				goto bad_val;
L
Linus Torvalds 已提交
2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079
		} else {
			printk(KERN_ERR "tmpfs: Bad mount option %s\n",
			       this_char);
			return 1;
		}
	}
	return 0;

bad_val:
	printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
	       value, this_char);
	return 1;

}

static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2080
	struct shmem_sb_info config = *sbinfo;
2081 2082 2083
	unsigned long inodes;
	int error = -EINVAL;

2084
	if (shmem_parse_options(data, &config, true))
2085
		return error;
L
Linus Torvalds 已提交
2086

2087 2088
	spin_lock(&sbinfo->stat_lock);
	inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2089
	if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2090
		goto out;
2091
	if (config.max_inodes < inodes)
2092 2093
		goto out;
	/*
2094
	 * Those tests disallow limited->unlimited while any are in use;
2095 2096 2097
	 * but we must separately disallow unlimited->limited, because
	 * in that case we have no record of how much is already in use.
	 */
2098
	if (config.max_blocks && !sbinfo->max_blocks)
2099
		goto out;
2100
	if (config.max_inodes && !sbinfo->max_inodes)
2101 2102 2103
		goto out;

	error = 0;
2104 2105 2106
	sbinfo->max_blocks  = config.max_blocks;
	sbinfo->max_inodes  = config.max_inodes;
	sbinfo->free_inodes = config.max_inodes - inodes;
2107 2108 2109

	mpol_put(sbinfo->mpol);
	sbinfo->mpol        = config.mpol;	/* transfers initial ref */
2110 2111 2112
out:
	spin_unlock(&sbinfo->stat_lock);
	return error;
L
Linus Torvalds 已提交
2113
}
2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129

static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);

	if (sbinfo->max_blocks != shmem_default_max_blocks())
		seq_printf(seq, ",size=%luk",
			sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
	if (sbinfo->max_inodes != shmem_default_max_inodes())
		seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
	if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
		seq_printf(seq, ",mode=%03o", sbinfo->mode);
	if (sbinfo->uid != 0)
		seq_printf(seq, ",uid=%u", sbinfo->uid);
	if (sbinfo->gid != 0)
		seq_printf(seq, ",gid=%u", sbinfo->gid);
2130
	shmem_show_mpol(seq, sbinfo->mpol);
2131 2132 2133
	return 0;
}
#endif /* CONFIG_TMPFS */
L
Linus Torvalds 已提交
2134 2135 2136

static void shmem_put_super(struct super_block *sb)
{
2137 2138 2139 2140
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);

	percpu_counter_destroy(&sbinfo->used_blocks);
	kfree(sbinfo);
L
Linus Torvalds 已提交
2141 2142 2143
	sb->s_fs_info = NULL;
}

2144
int shmem_fill_super(struct super_block *sb, void *data, int silent)
L
Linus Torvalds 已提交
2145 2146 2147
{
	struct inode *inode;
	struct dentry *root;
2148
	struct shmem_sb_info *sbinfo;
2149 2150 2151
	int err = -ENOMEM;

	/* Round up to L1_CACHE_BYTES to resist false sharing */
2152
	sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2153 2154 2155 2156 2157
				L1_CACHE_BYTES), GFP_KERNEL);
	if (!sbinfo)
		return -ENOMEM;

	sbinfo->mode = S_IRWXUGO | S_ISVTX;
2158 2159
	sbinfo->uid = current_fsuid();
	sbinfo->gid = current_fsgid();
2160
	sb->s_fs_info = sbinfo;
L
Linus Torvalds 已提交
2161

2162
#ifdef CONFIG_TMPFS
L
Linus Torvalds 已提交
2163 2164 2165 2166 2167 2168
	/*
	 * Per default we only allow half of the physical ram per
	 * tmpfs instance, limiting inodes to one per page of lowmem;
	 * but the internal instance is left unlimited.
	 */
	if (!(sb->s_flags & MS_NOUSER)) {
2169 2170 2171 2172 2173 2174
		sbinfo->max_blocks = shmem_default_max_blocks();
		sbinfo->max_inodes = shmem_default_max_inodes();
		if (shmem_parse_options(data, sbinfo, false)) {
			err = -EINVAL;
			goto failed;
		}
L
Linus Torvalds 已提交
2175
	}
2176
	sb->s_export_op = &shmem_export_ops;
L
Linus Torvalds 已提交
2177 2178 2179 2180
#else
	sb->s_flags |= MS_NOUSER;
#endif

2181
	spin_lock_init(&sbinfo->stat_lock);
2182 2183
	if (percpu_counter_init(&sbinfo->used_blocks, 0))
		goto failed;
2184
	sbinfo->free_inodes = sbinfo->max_inodes;
2185

2186
	sb->s_maxbytes = MAX_LFS_FILESIZE;
L
Linus Torvalds 已提交
2187 2188 2189 2190
	sb->s_blocksize = PAGE_CACHE_SIZE;
	sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
	sb->s_magic = TMPFS_MAGIC;
	sb->s_op = &shmem_ops;
2191
	sb->s_time_gran = 1;
2192
#ifdef CONFIG_TMPFS_XATTR
2193
	sb->s_xattr = shmem_xattr_handlers;
2194 2195
#endif
#ifdef CONFIG_TMPFS_POSIX_ACL
2196 2197
	sb->s_flags |= MS_POSIXACL;
#endif
2198

2199
	inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
L
Linus Torvalds 已提交
2200 2201
	if (!inode)
		goto failed;
2202 2203
	inode->i_uid = sbinfo->uid;
	inode->i_gid = sbinfo->gid;
L
Linus Torvalds 已提交
2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216
	root = d_alloc_root(inode);
	if (!root)
		goto failed_iput;
	sb->s_root = root;
	return 0;

failed_iput:
	iput(inode);
failed:
	shmem_put_super(sb);
	return err;
}

2217
static struct kmem_cache *shmem_inode_cachep;
L
Linus Torvalds 已提交
2218 2219 2220

static struct inode *shmem_alloc_inode(struct super_block *sb)
{
2221 2222 2223
	struct shmem_inode_info *info;
	info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
	if (!info)
L
Linus Torvalds 已提交
2224
		return NULL;
2225
	return &info->vfs_inode;
L
Linus Torvalds 已提交
2226 2227
}

2228
static void shmem_destroy_callback(struct rcu_head *head)
N
Nick Piggin 已提交
2229 2230 2231 2232 2233 2234
{
	struct inode *inode = container_of(head, struct inode, i_rcu);
	INIT_LIST_HEAD(&inode->i_dentry);
	kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
}

L
Linus Torvalds 已提交
2235 2236
static void shmem_destroy_inode(struct inode *inode)
{
2237
	if ((inode->i_mode & S_IFMT) == S_IFREG)
L
Linus Torvalds 已提交
2238
		mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2239
	call_rcu(&inode->i_rcu, shmem_destroy_callback);
L
Linus Torvalds 已提交
2240 2241
}

2242
static void shmem_init_inode(void *foo)
L
Linus Torvalds 已提交
2243
{
2244 2245
	struct shmem_inode_info *info = foo;
	inode_init_once(&info->vfs_inode);
L
Linus Torvalds 已提交
2246 2247
}

2248
static int shmem_init_inodecache(void)
L
Linus Torvalds 已提交
2249 2250 2251
{
	shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
				sizeof(struct shmem_inode_info),
2252
				0, SLAB_PANIC, shmem_init_inode);
L
Linus Torvalds 已提交
2253 2254 2255
	return 0;
}

2256
static void shmem_destroy_inodecache(void)
L
Linus Torvalds 已提交
2257
{
2258
	kmem_cache_destroy(shmem_inode_cachep);
L
Linus Torvalds 已提交
2259 2260
}

2261
static const struct address_space_operations shmem_aops = {
L
Linus Torvalds 已提交
2262
	.writepage	= shmem_writepage,
2263
	.set_page_dirty	= __set_page_dirty_no_writeback,
L
Linus Torvalds 已提交
2264
#ifdef CONFIG_TMPFS
N
Nick Piggin 已提交
2265 2266
	.write_begin	= shmem_write_begin,
	.write_end	= shmem_write_end,
L
Linus Torvalds 已提交
2267
#endif
2268
	.migratepage	= migrate_page,
2269
	.error_remove_page = generic_error_remove_page,
L
Linus Torvalds 已提交
2270 2271
};

2272
static const struct file_operations shmem_file_operations = {
L
Linus Torvalds 已提交
2273 2274 2275
	.mmap		= shmem_mmap,
#ifdef CONFIG_TMPFS
	.llseek		= generic_file_llseek,
H
Hugh Dickins 已提交
2276
	.read		= do_sync_read,
H
Hugh Dickins 已提交
2277
	.write		= do_sync_write,
H
Hugh Dickins 已提交
2278
	.aio_read	= shmem_file_aio_read,
H
Hugh Dickins 已提交
2279
	.aio_write	= generic_file_aio_write,
2280
	.fsync		= noop_fsync,
2281
	.splice_read	= shmem_file_splice_read,
2282
	.splice_write	= generic_file_splice_write,
L
Linus Torvalds 已提交
2283 2284 2285
#endif
};

2286
static const struct inode_operations shmem_inode_operations = {
2287
	.setattr	= shmem_setattr,
2288
	.truncate_range	= shmem_truncate_range,
2289 2290 2291 2292 2293 2294
#ifdef CONFIG_TMPFS_XATTR
	.setxattr	= shmem_setxattr,
	.getxattr	= shmem_getxattr,
	.listxattr	= shmem_listxattr,
	.removexattr	= shmem_removexattr,
#endif
L
Linus Torvalds 已提交
2295 2296
};

2297
static const struct inode_operations shmem_dir_inode_operations = {
L
Linus Torvalds 已提交
2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308
#ifdef CONFIG_TMPFS
	.create		= shmem_create,
	.lookup		= simple_lookup,
	.link		= shmem_link,
	.unlink		= shmem_unlink,
	.symlink	= shmem_symlink,
	.mkdir		= shmem_mkdir,
	.rmdir		= shmem_rmdir,
	.mknod		= shmem_mknod,
	.rename		= shmem_rename,
#endif
2309 2310 2311 2312 2313 2314
#ifdef CONFIG_TMPFS_XATTR
	.setxattr	= shmem_setxattr,
	.getxattr	= shmem_getxattr,
	.listxattr	= shmem_listxattr,
	.removexattr	= shmem_removexattr,
#endif
2315
#ifdef CONFIG_TMPFS_POSIX_ACL
2316
	.setattr	= shmem_setattr,
2317 2318 2319
#endif
};

2320
static const struct inode_operations shmem_special_inode_operations = {
2321 2322 2323 2324 2325 2326
#ifdef CONFIG_TMPFS_XATTR
	.setxattr	= shmem_setxattr,
	.getxattr	= shmem_getxattr,
	.listxattr	= shmem_listxattr,
	.removexattr	= shmem_removexattr,
#endif
2327
#ifdef CONFIG_TMPFS_POSIX_ACL
2328
	.setattr	= shmem_setattr,
2329
#endif
L
Linus Torvalds 已提交
2330 2331
};

2332
static const struct super_operations shmem_ops = {
L
Linus Torvalds 已提交
2333 2334 2335 2336 2337
	.alloc_inode	= shmem_alloc_inode,
	.destroy_inode	= shmem_destroy_inode,
#ifdef CONFIG_TMPFS
	.statfs		= shmem_statfs,
	.remount_fs	= shmem_remount_fs,
2338
	.show_options	= shmem_show_options,
L
Linus Torvalds 已提交
2339
#endif
A
Al Viro 已提交
2340
	.evict_inode	= shmem_evict_inode,
L
Linus Torvalds 已提交
2341 2342 2343 2344
	.drop_inode	= generic_delete_inode,
	.put_super	= shmem_put_super,
};

2345
static const struct vm_operations_struct shmem_vm_ops = {
2346
	.fault		= shmem_fault,
L
Linus Torvalds 已提交
2347 2348 2349 2350 2351 2352
#ifdef CONFIG_NUMA
	.set_policy     = shmem_set_policy,
	.get_policy     = shmem_get_policy,
#endif
};

A
Al Viro 已提交
2353 2354
static struct dentry *shmem_mount(struct file_system_type *fs_type,
	int flags, const char *dev_name, void *data)
L
Linus Torvalds 已提交
2355
{
A
Al Viro 已提交
2356
	return mount_nodev(fs_type, flags, data, shmem_fill_super);
L
Linus Torvalds 已提交
2357 2358
}

2359
static struct file_system_type shmem_fs_type = {
L
Linus Torvalds 已提交
2360 2361
	.owner		= THIS_MODULE,
	.name		= "tmpfs",
A
Al Viro 已提交
2362
	.mount		= shmem_mount,
L
Linus Torvalds 已提交
2363 2364 2365
	.kill_sb	= kill_litter_super,
};

2366
int __init shmem_init(void)
L
Linus Torvalds 已提交
2367 2368 2369
{
	int error;

P
Peter Zijlstra 已提交
2370 2371 2372 2373
	error = bdi_init(&shmem_backing_dev_info);
	if (error)
		goto out4;

2374
	error = shmem_init_inodecache();
L
Linus Torvalds 已提交
2375 2376 2377
	if (error)
		goto out3;

2378
	error = register_filesystem(&shmem_fs_type);
L
Linus Torvalds 已提交
2379 2380 2381 2382
	if (error) {
		printk(KERN_ERR "Could not register tmpfs\n");
		goto out2;
	}
2383

2384 2385
	shm_mnt = vfs_kern_mount(&shmem_fs_type, MS_NOUSER,
				 shmem_fs_type.name, NULL);
L
Linus Torvalds 已提交
2386 2387 2388 2389 2390 2391 2392 2393
	if (IS_ERR(shm_mnt)) {
		error = PTR_ERR(shm_mnt);
		printk(KERN_ERR "Could not kern_mount tmpfs\n");
		goto out1;
	}
	return 0;

out1:
2394
	unregister_filesystem(&shmem_fs_type);
L
Linus Torvalds 已提交
2395
out2:
2396
	shmem_destroy_inodecache();
L
Linus Torvalds 已提交
2397
out3:
P
Peter Zijlstra 已提交
2398 2399
	bdi_destroy(&shmem_backing_dev_info);
out4:
L
Linus Torvalds 已提交
2400 2401 2402
	shm_mnt = ERR_PTR(error);
	return error;
}
2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416

#else /* !CONFIG_SHMEM */

/*
 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
 *
 * This is intended for small system where the benefits of the full
 * shmem code (swap-backed and resource-limited) are outweighed by
 * their complexity. On systems without swap this code should be
 * effectively equivalent, but much lighter weight.
 */

#include <linux/ramfs.h>

2417
static struct file_system_type shmem_fs_type = {
2418
	.name		= "tmpfs",
A
Al Viro 已提交
2419
	.mount		= ramfs_mount,
2420 2421 2422
	.kill_sb	= kill_litter_super,
};

2423
int __init shmem_init(void)
2424
{
2425
	BUG_ON(register_filesystem(&shmem_fs_type) != 0);
2426

2427
	shm_mnt = kern_mount(&shmem_fs_type);
2428 2429 2430 2431 2432
	BUG_ON(IS_ERR(shm_mnt));

	return 0;
}

2433
int shmem_unuse(swp_entry_t swap, struct page *page)
2434 2435 2436 2437
{
	return 0;
}

H
Hugh Dickins 已提交
2438 2439 2440 2441 2442
int shmem_lock(struct file *file, int lock, struct user_struct *user)
{
	return 0;
}

2443
void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
2444
{
2445
	truncate_inode_pages_range(inode->i_mapping, lstart, lend);
2446 2447 2448
}
EXPORT_SYMBOL_GPL(shmem_truncate_range);

2449 2450
#define shmem_vm_ops				generic_file_vm_ops
#define shmem_file_operations			ramfs_file_operations
2451
#define shmem_get_inode(sb, dir, mode, dev, flags)	ramfs_get_inode(sb, dir, mode, dev)
2452 2453
#define shmem_acct_size(flags, size)		0
#define shmem_unacct_size(flags, size)		do {} while (0)
2454 2455 2456 2457

#endif /* CONFIG_SHMEM */

/* common code */
L
Linus Torvalds 已提交
2458

2459
/**
L
Linus Torvalds 已提交
2460 2461 2462
 * shmem_file_setup - get an unlinked file living in tmpfs
 * @name: name for dentry (to be seen in /proc/<pid>/maps
 * @size: size to be set for the file
2463
 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
L
Linus Torvalds 已提交
2464
 */
2465
struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
L
Linus Torvalds 已提交
2466 2467 2468 2469
{
	int error;
	struct file *file;
	struct inode *inode;
2470 2471
	struct path path;
	struct dentry *root;
L
Linus Torvalds 已提交
2472 2473 2474 2475 2476
	struct qstr this;

	if (IS_ERR(shm_mnt))
		return (void *)shm_mnt;

2477
	if (size < 0 || size > MAX_LFS_FILESIZE)
L
Linus Torvalds 已提交
2478 2479 2480 2481 2482 2483 2484 2485 2486 2487
		return ERR_PTR(-EINVAL);

	if (shmem_acct_size(flags, size))
		return ERR_PTR(-ENOMEM);

	error = -ENOMEM;
	this.name = name;
	this.len = strlen(name);
	this.hash = 0; /* will go */
	root = shm_mnt->mnt_root;
2488 2489
	path.dentry = d_alloc(root, &this);
	if (!path.dentry)
L
Linus Torvalds 已提交
2490
		goto put_memory;
2491
	path.mnt = mntget(shm_mnt);
L
Linus Torvalds 已提交
2492 2493

	error = -ENOSPC;
2494
	inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
L
Linus Torvalds 已提交
2495
	if (!inode)
2496
		goto put_dentry;
L
Linus Torvalds 已提交
2497

2498
	d_instantiate(path.dentry, inode);
L
Linus Torvalds 已提交
2499 2500
	inode->i_size = size;
	inode->i_nlink = 0;	/* It is unlinked */
2501 2502 2503
#ifndef CONFIG_MMU
	error = ramfs_nommu_expand_for_mapping(inode, size);
	if (error)
2504
		goto put_dentry;
2505
#endif
2506 2507

	error = -ENFILE;
2508
	file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
2509 2510 2511 2512
		  &shmem_file_operations);
	if (!file)
		goto put_dentry;

L
Linus Torvalds 已提交
2513 2514 2515
	return file;

put_dentry:
2516
	path_put(&path);
L
Linus Torvalds 已提交
2517 2518 2519 2520
put_memory:
	shmem_unacct_size(flags, size);
	return ERR_PTR(error);
}
2521
EXPORT_SYMBOL_GPL(shmem_file_setup);
L
Linus Torvalds 已提交
2522

2523
/**
L
Linus Torvalds 已提交
2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539
 * shmem_zero_setup - setup a shared anonymous mapping
 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
 */
int shmem_zero_setup(struct vm_area_struct *vma)
{
	struct file *file;
	loff_t size = vma->vm_end - vma->vm_start;

	file = shmem_file_setup("dev/zero", size, vma->vm_flags);
	if (IS_ERR(file))
		return PTR_ERR(file);

	if (vma->vm_file)
		fput(vma->vm_file);
	vma->vm_file = file;
	vma->vm_ops = &shmem_vm_ops;
2540
	vma->vm_flags |= VM_CAN_NONLINEAR;
L
Linus Torvalds 已提交
2541 2542
	return 0;
}
2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555

/**
 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
 * @mapping:	the page's address_space
 * @index:	the page index
 * @gfp:	the page allocator flags to use if allocating
 *
 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
 * with any new page allocations done using the specified allocation flags.
 * But read_cache_page_gfp() uses the ->readpage() method: which does not
 * suit tmpfs, since it may have pages in swapcache, and needs to find those
 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
 *
2556 2557
 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
2558 2559 2560 2561
 */
struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
					 pgoff_t index, gfp_t gfp)
{
2562 2563
#ifdef CONFIG_SHMEM
	struct inode *inode = mapping->host;
H
Hugh Dickins 已提交
2564
	struct page *page;
2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577
	int error;

	BUG_ON(mapping->a_ops != &shmem_aops);
	error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
	if (error)
		page = ERR_PTR(error);
	else
		unlock_page(page);
	return page;
#else
	/*
	 * The tiny !SHMEM case uses ramfs without swap
	 */
2578
	return read_cache_page_gfp(mapping, index, gfp);
2579
#endif
2580 2581
}
EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);