shmem.c 110.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8
/*
 * Resizable virtual memory filesystem for Linux.
 *
 * Copyright (C) 2000 Linus Torvalds.
 *		 2000 Transmeta Corp.
 *		 2000-2001 Christoph Rohland
 *		 2000-2001 SAP AG
 *		 2002 Red Hat Inc.
9 10
 * Copyright (C) 2002-2011 Hugh Dickins.
 * Copyright (C) 2011 Google Inc.
11
 * Copyright (C) 2002-2005 VERITAS Software Corporation.
L
Linus Torvalds 已提交
12 13 14 15 16 17
 * Copyright (C) 2004 Andi Kleen, SuSE Labs
 *
 * Extended attribute support for tmpfs:
 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
 *
18 19 20
 * tiny-shmem:
 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
 *
L
Linus Torvalds 已提交
21 22 23
 * This file is released under the GPL.
 */

24 25 26 27
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/vfs.h>
#include <linux/mount.h>
A
Andrew Morton 已提交
28
#include <linux/ramfs.h>
H
Hugh Dickins 已提交
29
#include <linux/pagemap.h>
30 31
#include <linux/file.h>
#include <linux/mm.h>
32
#include <linux/random.h>
33
#include <linux/sched/signal.h>
34
#include <linux/export.h>
35
#include <linux/swap.h>
36
#include <linux/uio.h>
37
#include <linux/khugepaged.h>
38
#include <linux/hugetlb.h>
39
#include <linux/frontswap.h>
40
#include <linux/fs_parser.h>
41

42 43
#include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */

44 45 46
static struct vfsmount *shm_mnt;

#ifdef CONFIG_SHMEM
L
Linus Torvalds 已提交
47 48 49 50 51 52
/*
 * This virtual memory filesystem is heavily based on the ramfs. It
 * extends ramfs by the ability to use swap and honor resource limits
 * which makes it a completely usable filesystem.
 */

53
#include <linux/xattr.h>
54
#include <linux/exportfs.h>
55
#include <linux/posix_acl.h>
C
Christoph Hellwig 已提交
56
#include <linux/posix_acl_xattr.h>
L
Linus Torvalds 已提交
57 58 59 60 61 62 63
#include <linux/mman.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/backing-dev.h>
#include <linux/shmem_fs.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
64
#include <linux/pagevec.h>
65
#include <linux/percpu_counter.h>
66
#include <linux/falloc.h>
67
#include <linux/splice.h>
L
Linus Torvalds 已提交
68 69 70 71
#include <linux/security.h>
#include <linux/swapops.h>
#include <linux/mempolicy.h>
#include <linux/namei.h>
72
#include <linux/ctype.h>
73
#include <linux/migrate.h>
74
#include <linux/highmem.h>
75
#include <linux/seq_file.h>
M
Mimi Zohar 已提交
76
#include <linux/magic.h>
D
David Herrmann 已提交
77
#include <linux/syscalls.h>
D
David Herrmann 已提交
78
#include <linux/fcntl.h>
D
David Herrmann 已提交
79
#include <uapi/linux/memfd.h>
80
#include <linux/userfaultfd_k.h>
81
#include <linux/rmap.h>
82
#include <linux/uuid.h>
83

84
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
85

86 87
#include "internal.h"

88 89
#define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
#define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
L
Linus Torvalds 已提交
90 91 92 93

/* Pretend that each entry is of this size in directory's i_size */
#define BOGO_DIRENT_SIZE 20

94 95 96
/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
#define SHORT_SYMLINK_LEN 128

97
/*
98 99 100
 * shmem_fallocate communicates with shmem_fault or shmem_writepage via
 * inode->i_private (with i_mutex making sure that it has only one user at
 * a time): we would prefer not to enlarge the shmem inode just for that.
101 102
 */
struct shmem_falloc {
103
	wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
104 105 106 107 108 109
	pgoff_t start;		/* start of range currently being fallocated */
	pgoff_t next;		/* the next page offset to be fallocated */
	pgoff_t nr_falloced;	/* how many new pages have been fallocated */
	pgoff_t nr_unswapped;	/* how often writepage refused to swap out */
};

110 111 112 113 114 115 116
struct shmem_options {
	unsigned long long blocks;
	unsigned long long inodes;
	struct mempolicy *mpol;
	kuid_t uid;
	kgid_t gid;
	umode_t mode;
C
Chris Down 已提交
117
	bool full_inums;
118 119 120 121 122
	int huge;
	int seen;
#define SHMEM_SEEN_BLOCKS 1
#define SHMEM_SEEN_INODES 2
#define SHMEM_SEEN_HUGE 4
C
Chris Down 已提交
123
#define SHMEM_SEEN_INUMS 8
124 125
};

A
Andrew Morton 已提交
126
#ifdef CONFIG_TMPFS
127 128
static unsigned long shmem_default_max_blocks(void)
{
129
	return totalram_pages() / 2;
130 131 132 133
}

static unsigned long shmem_default_max_inodes(void)
{
134 135 136
	unsigned long nr_pages = totalram_pages();

	return min(nr_pages - totalhigh_pages(), nr_pages / 2);
137
}
A
Andrew Morton 已提交
138
#endif
139

140 141 142
static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
static int shmem_replace_page(struct page **pagep, gfp_t gfp,
				struct shmem_inode_info *info, pgoff_t index);
143 144 145 146
static int shmem_swapin_page(struct inode *inode, pgoff_t index,
			     struct page **pagep, enum sgp_type sgp,
			     gfp_t gfp, struct vm_area_struct *vma,
			     vm_fault_t *fault_type);
147
static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
148
		struct page **pagep, enum sgp_type sgp,
149
		gfp_t gfp, struct vm_area_struct *vma,
150
		struct vm_fault *vmf, vm_fault_t *fault_type);
151

152
int shmem_getpage(struct inode *inode, pgoff_t index,
153
		struct page **pagep, enum sgp_type sgp)
154 155
{
	return shmem_getpage_gfp(inode, index, pagep, sgp,
156
		mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
157
}
L
Linus Torvalds 已提交
158 159 160 161 162 163 164 165 166 167 168 169 170 171

static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
{
	return sb->s_fs_info;
}

/*
 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
 * for shared memory and for shared anonymous (/dev/zero) mappings
 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
 * consistent with the pre-accounting of private mappings ...
 */
static inline int shmem_acct_size(unsigned long flags, loff_t size)
{
172
	return (flags & VM_NORESERVE) ?
173
		0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
L
Linus Torvalds 已提交
174 175 176 177
}

static inline void shmem_unacct_size(unsigned long flags, loff_t size)
{
178
	if (!(flags & VM_NORESERVE))
L
Linus Torvalds 已提交
179 180 181
		vm_unacct_memory(VM_ACCT(size));
}

182 183 184 185 186 187 188 189 190 191 192 193 194
static inline int shmem_reacct_size(unsigned long flags,
		loff_t oldsize, loff_t newsize)
{
	if (!(flags & VM_NORESERVE)) {
		if (VM_ACCT(newsize) > VM_ACCT(oldsize))
			return security_vm_enough_memory_mm(current->mm,
					VM_ACCT(newsize) - VM_ACCT(oldsize));
		else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
			vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
	}
	return 0;
}

L
Linus Torvalds 已提交
195 196
/*
 * ... whereas tmpfs objects are accounted incrementally as
H
Hugh Dickins 已提交
197
 * pages are allocated, in order to allow large sparse files.
L
Linus Torvalds 已提交
198 199 200
 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
 */
201
static inline int shmem_acct_block(unsigned long flags, long pages)
L
Linus Torvalds 已提交
202
{
203 204 205 206 207
	if (!(flags & VM_NORESERVE))
		return 0;

	return security_vm_enough_memory_mm(current->mm,
			pages * VM_ACCT(PAGE_SIZE));
L
Linus Torvalds 已提交
208 209 210 211
}

static inline void shmem_unacct_blocks(unsigned long flags, long pages)
{
212
	if (flags & VM_NORESERVE)
213
		vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
L
Linus Torvalds 已提交
214 215
}

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
{
	struct shmem_inode_info *info = SHMEM_I(inode);
	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);

	if (shmem_acct_block(info->flags, pages))
		return false;

	if (sbinfo->max_blocks) {
		if (percpu_counter_compare(&sbinfo->used_blocks,
					   sbinfo->max_blocks - pages) > 0)
			goto unacct;
		percpu_counter_add(&sbinfo->used_blocks, pages);
	}

	return true;

unacct:
	shmem_unacct_blocks(info->flags, pages);
	return false;
}

static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
{
	struct shmem_inode_info *info = SHMEM_I(inode);
	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);

	if (sbinfo->max_blocks)
		percpu_counter_sub(&sbinfo->used_blocks, pages);
	shmem_unacct_blocks(info->flags, pages);
}

248
static const struct super_operations shmem_ops;
249
const struct address_space_operations shmem_aops;
250
static const struct file_operations shmem_file_operations;
251 252 253
static const struct inode_operations shmem_inode_operations;
static const struct inode_operations shmem_dir_inode_operations;
static const struct inode_operations shmem_special_inode_operations;
254
static const struct vm_operations_struct shmem_vm_ops;
255
static struct file_system_type shmem_fs_type;
L
Linus Torvalds 已提交
256

257 258 259 260 261
bool vma_is_shmem(struct vm_area_struct *vma)
{
	return vma->vm_ops == &shmem_vm_ops;
}

L
Linus Torvalds 已提交
262
static LIST_HEAD(shmem_swaplist);
263
static DEFINE_MUTEX(shmem_swaplist_mutex);
L
Linus Torvalds 已提交
264

C
Chris Down 已提交
265 266 267 268 269 270 271 272 273 274 275
/*
 * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
 * produces a novel ino for the newly allocated inode.
 *
 * It may also be called when making a hard link to permit the space needed by
 * each dentry. However, in that case, no new inode number is needed since that
 * internally draws from another pool of inode numbers (currently global
 * get_next_ino()). This case is indicated by passing NULL as inop.
 */
#define SHMEM_INO_BATCH 1024
static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
276 277
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
C
Chris Down 已提交
278 279 280
	ino_t ino;

	if (!(sb->s_flags & SB_KERNMOUNT)) {
281
		spin_lock(&sbinfo->stat_lock);
282 283 284 285 286 287
		if (sbinfo->max_inodes) {
			if (!sbinfo->free_inodes) {
				spin_unlock(&sbinfo->stat_lock);
				return -ENOSPC;
			}
			sbinfo->free_inodes--;
288
		}
C
Chris Down 已提交
289 290 291 292
		if (inop) {
			ino = sbinfo->next_ino++;
			if (unlikely(is_zero_ino(ino)))
				ino = sbinfo->next_ino++;
C
Chris Down 已提交
293 294
			if (unlikely(!sbinfo->full_inums &&
				     ino > UINT_MAX)) {
C
Chris Down 已提交
295 296 297 298
				/*
				 * Emulate get_next_ino uint wraparound for
				 * compatibility
				 */
C
Chris Down 已提交
299 300 301 302 303
				if (IS_ENABLED(CONFIG_64BIT))
					pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
						__func__, MINOR(sb->s_dev));
				sbinfo->next_ino = 1;
				ino = sbinfo->next_ino++;
C
Chris Down 已提交
304 305 306
			}
			*inop = ino;
		}
307
		spin_unlock(&sbinfo->stat_lock);
C
Chris Down 已提交
308 309 310 311 312 313 314 315
	} else if (inop) {
		/*
		 * __shmem_file_setup, one of our callers, is lock-free: it
		 * doesn't hold stat_lock in shmem_reserve_inode since
		 * max_inodes is always 0, and is called from potentially
		 * unknown contexts. As such, use a per-cpu batched allocator
		 * which doesn't require the per-sb stat_lock unless we are at
		 * the batch boundary.
C
Chris Down 已提交
316 317 318 319
		 *
		 * We don't need to worry about inode{32,64} since SB_KERNMOUNT
		 * shmem mounts are not exposed to userspace, so we don't need
		 * to worry about things like glibc compatibility.
C
Chris Down 已提交
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
		 */
		ino_t *next_ino;
		next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
		ino = *next_ino;
		if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
			spin_lock(&sbinfo->stat_lock);
			ino = sbinfo->next_ino;
			sbinfo->next_ino += SHMEM_INO_BATCH;
			spin_unlock(&sbinfo->stat_lock);
			if (unlikely(is_zero_ino(ino)))
				ino++;
		}
		*inop = ino;
		*next_ino = ++ino;
		put_cpu();
335
	}
C
Chris Down 已提交
336

337 338 339 340 341 342 343 344 345 346 347 348 349
	return 0;
}

static void shmem_free_inode(struct super_block *sb)
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
	if (sbinfo->max_inodes) {
		spin_lock(&sbinfo->stat_lock);
		sbinfo->free_inodes++;
		spin_unlock(&sbinfo->stat_lock);
	}
}

350
/**
351
 * shmem_recalc_inode - recalculate the block usage of an inode
L
Linus Torvalds 已提交
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
 * @inode: inode to recalc
 *
 * We have to calculate the free blocks since the mm can drop
 * undirtied hole pages behind our back.
 *
 * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
 *
 * It has to be called with the spinlock held.
 */
static void shmem_recalc_inode(struct inode *inode)
{
	struct shmem_inode_info *info = SHMEM_I(inode);
	long freed;

	freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
	if (freed > 0) {
		info->alloced -= freed;
370
		inode->i_blocks -= freed * BLOCKS_PER_PAGE;
371
		shmem_inode_unacct_blocks(inode, freed);
L
Linus Torvalds 已提交
372 373 374
	}
}

375 376 377
bool shmem_charge(struct inode *inode, long pages)
{
	struct shmem_inode_info *info = SHMEM_I(inode);
378
	unsigned long flags;
379

380
	if (!shmem_inode_acct_block(inode, pages))
381
		return false;
382

383 384 385
	/* nrpages adjustment first, then shmem_recalc_inode() when balanced */
	inode->i_mapping->nrpages += pages;

386
	spin_lock_irqsave(&info->lock, flags);
387 388 389
	info->alloced += pages;
	inode->i_blocks += pages * BLOCKS_PER_PAGE;
	shmem_recalc_inode(inode);
390
	spin_unlock_irqrestore(&info->lock, flags);
391 392 393 394 395 396 397

	return true;
}

void shmem_uncharge(struct inode *inode, long pages)
{
	struct shmem_inode_info *info = SHMEM_I(inode);
398
	unsigned long flags;
399

400 401
	/* nrpages adjustment done by __delete_from_page_cache() or caller */

402
	spin_lock_irqsave(&info->lock, flags);
403 404 405
	info->alloced -= pages;
	inode->i_blocks -= pages * BLOCKS_PER_PAGE;
	shmem_recalc_inode(inode);
406
	spin_unlock_irqrestore(&info->lock, flags);
407

408
	shmem_inode_unacct_blocks(inode, pages);
409 410
}

411
/*
412
 * Replace item expected in xarray by a new item, while holding xa_lock.
413
 */
414
static int shmem_replace_entry(struct address_space *mapping,
415 416
			pgoff_t index, void *expected, void *replacement)
{
417
	XA_STATE(xas, &mapping->i_pages, index);
418
	void *item;
419 420

	VM_BUG_ON(!expected);
421
	VM_BUG_ON(!replacement);
422
	item = xas_load(&xas);
423 424
	if (item != expected)
		return -ENOENT;
425
	xas_store(&xas, replacement);
426 427 428
	return 0;
}

429 430 431 432 433 434 435 436 437 438
/*
 * Sometimes, before we decide whether to proceed or to fail, we must check
 * that an entry was not already brought back from swap by a racing thread.
 *
 * Checking page is not enough: by the time a SwapCache page is locked, it
 * might be reused, and again be SwapCache, using the same swap as before.
 */
static bool shmem_confirm_swap(struct address_space *mapping,
			       pgoff_t index, swp_entry_t swap)
{
439
	return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
440 441
}

442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
/*
 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
 *
 * SHMEM_HUGE_NEVER:
 *	disables huge pages for the mount;
 * SHMEM_HUGE_ALWAYS:
 *	enables huge pages for the mount;
 * SHMEM_HUGE_WITHIN_SIZE:
 *	only allocate huge pages if the page will be fully within i_size,
 *	also respect fadvise()/madvise() hints;
 * SHMEM_HUGE_ADVISE:
 *	only allocate huge pages if requested with fadvise()/madvise();
 */

#define SHMEM_HUGE_NEVER	0
#define SHMEM_HUGE_ALWAYS	1
#define SHMEM_HUGE_WITHIN_SIZE	2
#define SHMEM_HUGE_ADVISE	3

/*
 * Special values.
 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
 *
 * SHMEM_HUGE_DENY:
 *	disables huge on shm_mnt and all mounts, for emergency use;
 * SHMEM_HUGE_FORCE:
 *	enables huge on shm_mnt and all mounts, w/o needing option, for testing;
 *
 */
#define SHMEM_HUGE_DENY		(-1)
#define SHMEM_HUGE_FORCE	(-2)

474
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
475 476
/* ifdef here to avoid bloating shmem.o when not necessary */

477
static int shmem_huge __read_mostly;
478

479
#if defined(CONFIG_SYSFS)
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
static int shmem_parse_huge(const char *str)
{
	if (!strcmp(str, "never"))
		return SHMEM_HUGE_NEVER;
	if (!strcmp(str, "always"))
		return SHMEM_HUGE_ALWAYS;
	if (!strcmp(str, "within_size"))
		return SHMEM_HUGE_WITHIN_SIZE;
	if (!strcmp(str, "advise"))
		return SHMEM_HUGE_ADVISE;
	if (!strcmp(str, "deny"))
		return SHMEM_HUGE_DENY;
	if (!strcmp(str, "force"))
		return SHMEM_HUGE_FORCE;
	return -EINVAL;
}
496
#endif
497

498
#if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
static const char *shmem_format_huge(int huge)
{
	switch (huge) {
	case SHMEM_HUGE_NEVER:
		return "never";
	case SHMEM_HUGE_ALWAYS:
		return "always";
	case SHMEM_HUGE_WITHIN_SIZE:
		return "within_size";
	case SHMEM_HUGE_ADVISE:
		return "advise";
	case SHMEM_HUGE_DENY:
		return "deny";
	case SHMEM_HUGE_FORCE:
		return "force";
	default:
		VM_BUG_ON(1);
		return "bad_val";
	}
}
519
#endif
520

521 522 523 524
static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
		struct shrink_control *sc, unsigned long nr_to_split)
{
	LIST_HEAD(list), *pos, *next;
525
	LIST_HEAD(to_remove);
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
	struct inode *inode;
	struct shmem_inode_info *info;
	struct page *page;
	unsigned long batch = sc ? sc->nr_to_scan : 128;
	int removed = 0, split = 0;

	if (list_empty(&sbinfo->shrinklist))
		return SHRINK_STOP;

	spin_lock(&sbinfo->shrinklist_lock);
	list_for_each_safe(pos, next, &sbinfo->shrinklist) {
		info = list_entry(pos, struct shmem_inode_info, shrinklist);

		/* pin the inode */
		inode = igrab(&info->vfs_inode);

		/* inode is about to be evicted */
		if (!inode) {
			list_del_init(&info->shrinklist);
			removed++;
			goto next;
		}

		/* Check if there's anything to gain */
		if (round_up(inode->i_size, PAGE_SIZE) ==
				round_up(inode->i_size, HPAGE_PMD_SIZE)) {
552
			list_move(&info->shrinklist, &to_remove);
553 554 555 556 557 558 559 560 561 562 563
			removed++;
			goto next;
		}

		list_move(&info->shrinklist, &list);
next:
		if (!--batch)
			break;
	}
	spin_unlock(&sbinfo->shrinklist_lock);

564 565 566 567 568 569 570
	list_for_each_safe(pos, next, &to_remove) {
		info = list_entry(pos, struct shmem_inode_info, shrinklist);
		inode = &info->vfs_inode;
		list_del_init(&info->shrinklist);
		iput(inode);
	}

571 572 573 574 575 576
	list_for_each_safe(pos, next, &list) {
		int ret;

		info = list_entry(pos, struct shmem_inode_info, shrinklist);
		inode = &info->vfs_inode;

577 578
		if (nr_to_split && split >= nr_to_split)
			goto leave;
579

580
		page = find_get_page(inode->i_mapping,
581 582 583 584
				(inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
		if (!page)
			goto drop;

585
		/* No huge page at the end of the file: nothing to split */
586 587 588 589 590
		if (!PageTransHuge(page)) {
			put_page(page);
			goto drop;
		}

591 592 593 594 595 596 597 598 599 600 601 602
		/*
		 * Leave the inode on the list if we failed to lock
		 * the page at this time.
		 *
		 * Waiting for the lock may lead to deadlock in the
		 * reclaim path.
		 */
		if (!trylock_page(page)) {
			put_page(page);
			goto leave;
		}

603 604 605 606
		ret = split_huge_page(page);
		unlock_page(page);
		put_page(page);

607 608 609
		/* If split failed leave the inode on the list */
		if (ret)
			goto leave;
610 611 612 613 614

		split++;
drop:
		list_del_init(&info->shrinklist);
		removed++;
615
leave:
616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
		iput(inode);
	}

	spin_lock(&sbinfo->shrinklist_lock);
	list_splice_tail(&list, &sbinfo->shrinklist);
	sbinfo->shrinklist_len -= removed;
	spin_unlock(&sbinfo->shrinklist_lock);

	return split;
}

static long shmem_unused_huge_scan(struct super_block *sb,
		struct shrink_control *sc)
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);

	if (!READ_ONCE(sbinfo->shrinklist_len))
		return SHRINK_STOP;

	return shmem_unused_huge_shrink(sbinfo, sc, 0);
}

static long shmem_unused_huge_count(struct super_block *sb,
		struct shrink_control *sc)
{
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
	return READ_ONCE(sbinfo->shrinklist_len);
}
644
#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
645 646 647

#define shmem_huge SHMEM_HUGE_DENY

648 649 650 651 652
static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
		struct shrink_control *sc, unsigned long nr_to_split)
{
	return 0;
}
653
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
654

655 656
static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo)
{
657
	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
658 659 660 661 662 663
	    (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) &&
	    shmem_huge != SHMEM_HUGE_DENY)
		return true;
	return false;
}

664 665 666 667 668
/*
 * Like add_to_page_cache_locked, but error if expected item has gone.
 */
static int shmem_add_to_page_cache(struct page *page,
				   struct address_space *mapping,
669 670
				   pgoff_t index, void *expected, gfp_t gfp,
				   struct mm_struct *charge_mm)
671
{
672 673
	XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
	unsigned long i = 0;
674
	unsigned long nr = compound_nr(page);
675
	int error;
676

677 678
	VM_BUG_ON_PAGE(PageTail(page), page);
	VM_BUG_ON_PAGE(index != round_down(index, nr), page);
679 680
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
681
	VM_BUG_ON(expected && PageTransHuge(page));
682

683
	page_ref_add(page, nr);
684 685 686
	page->mapping = mapping;
	page->index = index;

687
	if (!PageSwapCache(page)) {
688
		error = mem_cgroup_charge(page, charge_mm, gfp);
689 690 691 692 693 694
		if (error) {
			if (PageTransHuge(page)) {
				count_vm_event(THP_FILE_FALLBACK);
				count_vm_event(THP_FILE_FALLBACK_CHARGE);
			}
			goto error;
695 696 697 698
		}
	}
	cgroup_throttle_swaprate(page, gfp);

699 700 701 702 703 704 705 706 707 708
	do {
		void *entry;
		xas_lock_irq(&xas);
		entry = xas_find_conflict(&xas);
		if (entry != expected)
			xas_set_err(&xas, -EEXIST);
		xas_create_range(&xas);
		if (xas_error(&xas))
			goto unlock;
next:
709
		xas_store(&xas, page);
710 711 712
		if (++i < nr) {
			xas_next(&xas);
			goto next;
713
		}
714
		if (PageTransHuge(page)) {
715
			count_vm_event(THP_FILE_ALLOC);
716
			__inc_node_page_state(page, NR_SHMEM_THPS);
717 718
		}
		mapping->nrpages += nr;
719 720
		__mod_lruvec_page_state(page, NR_FILE_PAGES, nr);
		__mod_lruvec_page_state(page, NR_SHMEM, nr);
721 722 723 724 725
unlock:
		xas_unlock_irq(&xas);
	} while (xas_nomem(&xas, gfp));

	if (xas_error(&xas)) {
726 727
		error = xas_error(&xas);
		goto error;
728
	}
729 730

	return 0;
731 732 733 734
error:
	page->mapping = NULL;
	page_ref_sub(page, nr);
	return error;
735 736
}

737 738 739 740 741 742 743 744
/*
 * Like delete_from_page_cache, but substitutes swap for page.
 */
static void shmem_delete_from_page_cache(struct page *page, void *radswap)
{
	struct address_space *mapping = page->mapping;
	int error;

745 746
	VM_BUG_ON_PAGE(PageCompound(page), page);

M
Matthew Wilcox 已提交
747
	xa_lock_irq(&mapping->i_pages);
748
	error = shmem_replace_entry(mapping, page->index, page, radswap);
749 750
	page->mapping = NULL;
	mapping->nrpages--;
751 752
	__dec_lruvec_page_state(page, NR_FILE_PAGES);
	__dec_lruvec_page_state(page, NR_SHMEM);
M
Matthew Wilcox 已提交
753
	xa_unlock_irq(&mapping->i_pages);
754
	put_page(page);
755 756 757
	BUG_ON(error);
}

758
/*
759
 * Remove swap entry from page cache, free the swap and its page cache.
760 761 762 763
 */
static int shmem_free_swap(struct address_space *mapping,
			   pgoff_t index, void *radswap)
{
764
	void *old;
765

766
	old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
767 768 769 770
	if (old != radswap)
		return -ENOENT;
	free_swap_and_cache(radix_to_swp_entry(radswap));
	return 0;
771 772
}

773 774
/*
 * Determine (in bytes) how many of the shmem object's pages mapped by the
775
 * given offsets are swapped out.
776
 *
M
Matthew Wilcox 已提交
777
 * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
778 779
 * as long as the inode doesn't go away and racy results are not a problem.
 */
780 781
unsigned long shmem_partial_swap_usage(struct address_space *mapping,
						pgoff_t start, pgoff_t end)
782
{
783
	XA_STATE(xas, &mapping->i_pages, start);
784
	struct page *page;
785
	unsigned long swapped = 0;
786 787

	rcu_read_lock();
788 789
	xas_for_each(&xas, page, end - 1) {
		if (xas_retry(&xas, page))
M
Matthew Wilcox 已提交
790
			continue;
791
		if (xa_is_value(page))
792 793 794
			swapped++;

		if (need_resched()) {
795
			xas_pause(&xas);
796 797 798 799 800 801 802 803 804
			cond_resched_rcu();
		}
	}

	rcu_read_unlock();

	return swapped << PAGE_SHIFT;
}

805 806 807 808
/*
 * Determine (in bytes) how many of the shmem object's pages mapped by the
 * given vma is swapped out.
 *
M
Matthew Wilcox 已提交
809
 * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838
 * as long as the inode doesn't go away and racy results are not a problem.
 */
unsigned long shmem_swap_usage(struct vm_area_struct *vma)
{
	struct inode *inode = file_inode(vma->vm_file);
	struct shmem_inode_info *info = SHMEM_I(inode);
	struct address_space *mapping = inode->i_mapping;
	unsigned long swapped;

	/* Be careful as we don't hold info->lock */
	swapped = READ_ONCE(info->swapped);

	/*
	 * The easier cases are when the shmem object has nothing in swap, or
	 * the vma maps it whole. Then we can simply use the stats that we
	 * already track.
	 */
	if (!swapped)
		return 0;

	if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
		return swapped << PAGE_SHIFT;

	/* Here comes the more involved part */
	return shmem_partial_swap_usage(mapping,
			linear_page_index(vma, vma->vm_start),
			linear_page_index(vma, vma->vm_end));
}

839 840 841 842 843 844 845 846 847
/*
 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
 */
void shmem_unlock_mapping(struct address_space *mapping)
{
	struct pagevec pvec;
	pgoff_t indices[PAGEVEC_SIZE];
	pgoff_t index = 0;

848
	pagevec_init(&pvec);
849 850 851 852 853 854 855 856
	/*
	 * Minor point, but we might as well stop if someone else SHM_LOCKs it.
	 */
	while (!mapping_unevictable(mapping)) {
		/*
		 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
		 * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
		 */
857 858
		pvec.nr = find_get_entries(mapping, index,
					   PAGEVEC_SIZE, pvec.pages, indices);
859 860 861
		if (!pvec.nr)
			break;
		index = indices[pvec.nr - 1] + 1;
862
		pagevec_remove_exceptionals(&pvec);
863
		check_move_unevictable_pages(&pvec);
864 865 866
		pagevec_release(&pvec);
		cond_resched();
	}
867 868
}

869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894
/*
 * Check whether a hole-punch or truncation needs to split a huge page,
 * returning true if no split was required, or the split has been successful.
 *
 * Eviction (or truncation to 0 size) should never need to split a huge page;
 * but in rare cases might do so, if shmem_undo_range() failed to trylock on
 * head, and then succeeded to trylock on tail.
 *
 * A split can only succeed when there are no additional references on the
 * huge page: so the split below relies upon find_get_entries() having stopped
 * when it found a subpage of the huge page, without getting further references.
 */
static bool shmem_punch_compound(struct page *page, pgoff_t start, pgoff_t end)
{
	if (!PageTransCompound(page))
		return true;

	/* Just proceed to delete a huge page wholly within the range punched */
	if (PageHead(page) &&
	    page->index >= start && page->index + HPAGE_PMD_NR <= end)
		return true;

	/* Try to split huge page, so we can truly punch the hole or truncate */
	return split_huge_page(page) >= 0;
}

895
/*
M
Matthew Wilcox 已提交
896
 * Remove range of pages and swap entries from page cache, and free them.
897
 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
898
 */
899 900
static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
								 bool unfalloc)
L
Linus Torvalds 已提交
901
{
902
	struct address_space *mapping = inode->i_mapping;
L
Linus Torvalds 已提交
903
	struct shmem_inode_info *info = SHMEM_I(inode);
904 905 906 907
	pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
	pgoff_t end = (lend + 1) >> PAGE_SHIFT;
	unsigned int partial_start = lstart & (PAGE_SIZE - 1);
	unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
908
	struct pagevec pvec;
909 910
	pgoff_t indices[PAGEVEC_SIZE];
	long nr_swaps_freed = 0;
911
	pgoff_t index;
912 913
	int i;

914 915
	if (lend == -1)
		end = -1;	/* unsigned, so actually very big */
916

917
	pagevec_init(&pvec);
918
	index = start;
919
	while (index < end) {
920 921 922
		pvec.nr = find_get_entries(mapping, index,
			min(end - index, (pgoff_t)PAGEVEC_SIZE),
			pvec.pages, indices);
923 924
		if (!pvec.nr)
			break;
925 926 927
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];

928
			index = indices[i];
929
			if (index >= end)
930 931
				break;

932
			if (xa_is_value(page)) {
933 934
				if (unfalloc)
					continue;
935 936
				nr_swaps_freed += !shmem_free_swap(mapping,
								index, page);
937
				continue;
938 939
			}

940 941
			VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);

942
			if (!trylock_page(page))
943
				continue;
944

945 946 947 948
			if ((!unfalloc || !PageUptodate(page)) &&
			    page_mapping(page) == mapping) {
				VM_BUG_ON_PAGE(PageWriteback(page), page);
				if (shmem_punch_compound(page, start, end))
949
					truncate_inode_page(mapping, page);
950 951 952
			}
			unlock_page(page);
		}
953
		pagevec_remove_exceptionals(&pvec);
954
		pagevec_release(&pvec);
955 956 957
		cond_resched();
		index++;
	}
L
Linus Torvalds 已提交
958

959
	if (partial_start) {
960
		struct page *page = NULL;
961
		shmem_getpage(inode, start - 1, &page, SGP_READ);
962
		if (page) {
963
			unsigned int top = PAGE_SIZE;
964 965 966 967 968 969 970
			if (start > end) {
				top = partial_end;
				partial_end = 0;
			}
			zero_user_segment(page, partial_start, top);
			set_page_dirty(page);
			unlock_page(page);
971
			put_page(page);
972 973 974 975
		}
	}
	if (partial_end) {
		struct page *page = NULL;
976
		shmem_getpage(inode, end, &page, SGP_READ);
977 978
		if (page) {
			zero_user_segment(page, 0, partial_end);
979 980
			set_page_dirty(page);
			unlock_page(page);
981
			put_page(page);
982 983
		}
	}
984 985
	if (start >= end)
		return;
986 987

	index = start;
988
	while (index < end) {
989
		cond_resched();
990 991

		pvec.nr = find_get_entries(mapping, index,
992
				min(end - index, (pgoff_t)PAGEVEC_SIZE),
993
				pvec.pages, indices);
994
		if (!pvec.nr) {
995 996
			/* If all gone or hole-punch or unfalloc, we're done */
			if (index == start || end != -1)
997
				break;
998
			/* But if truncating, restart to make sure all gone */
999 1000 1001 1002 1003 1004
			index = start;
			continue;
		}
		for (i = 0; i < pagevec_count(&pvec); i++) {
			struct page *page = pvec.pages[i];

1005
			index = indices[i];
1006
			if (index >= end)
1007 1008
				break;

1009
			if (xa_is_value(page)) {
1010 1011
				if (unfalloc)
					continue;
1012 1013 1014 1015 1016 1017
				if (shmem_free_swap(mapping, index, page)) {
					/* Swap was replaced by page: retry */
					index--;
					break;
				}
				nr_swaps_freed++;
1018 1019 1020
				continue;
			}

1021
			lock_page(page);
1022

1023
			if (!unfalloc || !PageUptodate(page)) {
1024
				if (page_mapping(page) != mapping) {
1025 1026 1027 1028
					/* Page was replaced by swap: retry */
					unlock_page(page);
					index--;
					break;
1029
				}
1030 1031 1032
				VM_BUG_ON_PAGE(PageWriteback(page), page);
				if (shmem_punch_compound(page, start, end))
					truncate_inode_page(mapping, page);
H
Hugh Dickins 已提交
1033
				else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
1034 1035 1036 1037 1038 1039 1040 1041
					/* Wipe the page and don't get stuck */
					clear_highpage(page);
					flush_dcache_page(page);
					set_page_dirty(page);
					if (index <
					    round_up(start, HPAGE_PMD_NR))
						start = index + 1;
				}
1042
			}
1043 1044
			unlock_page(page);
		}
1045
		pagevec_remove_exceptionals(&pvec);
1046
		pagevec_release(&pvec);
1047 1048
		index++;
	}
1049

1050
	spin_lock_irq(&info->lock);
1051
	info->swapped -= nr_swaps_freed;
L
Linus Torvalds 已提交
1052
	shmem_recalc_inode(inode);
1053
	spin_unlock_irq(&info->lock);
1054
}
L
Linus Torvalds 已提交
1055

1056 1057 1058
void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
{
	shmem_undo_range(inode, lstart, lend, false);
1059
	inode->i_ctime = inode->i_mtime = current_time(inode);
L
Linus Torvalds 已提交
1060
}
1061
EXPORT_SYMBOL_GPL(shmem_truncate_range);
L
Linus Torvalds 已提交
1062

1063 1064
static int shmem_getattr(const struct path *path, struct kstat *stat,
			 u32 request_mask, unsigned int query_flags)
1065
{
1066
	struct inode *inode = path->dentry->d_inode;
1067
	struct shmem_inode_info *info = SHMEM_I(inode);
1068
	struct shmem_sb_info *sb_info = SHMEM_SB(inode->i_sb);
1069

1070
	if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1071
		spin_lock_irq(&info->lock);
1072
		shmem_recalc_inode(inode);
1073
		spin_unlock_irq(&info->lock);
1074
	}
1075
	generic_fillattr(inode, stat);
1076 1077 1078 1079

	if (is_huge_enabled(sb_info))
		stat->blksize = HPAGE_PMD_SIZE;

1080 1081 1082
	return 0;
}

1083
static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
L
Linus Torvalds 已提交
1084
{
1085
	struct inode *inode = d_inode(dentry);
D
David Herrmann 已提交
1086
	struct shmem_inode_info *info = SHMEM_I(inode);
1087
	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
L
Linus Torvalds 已提交
1088 1089
	int error;

1090
	error = setattr_prepare(dentry, attr);
1091 1092 1093
	if (error)
		return error;

1094 1095 1096
	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
		loff_t oldsize = inode->i_size;
		loff_t newsize = attr->ia_size;
1097

D
David Herrmann 已提交
1098 1099 1100 1101 1102
		/* protected by i_mutex */
		if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
		    (newsize > oldsize && (info->seals & F_SEAL_GROW)))
			return -EPERM;

1103
		if (newsize != oldsize) {
1104 1105 1106 1107
			error = shmem_reacct_size(SHMEM_I(inode)->flags,
					oldsize, newsize);
			if (error)
				return error;
1108
			i_size_write(inode, newsize);
1109
			inode->i_ctime = inode->i_mtime = current_time(inode);
1110
		}
1111
		if (newsize <= oldsize) {
1112
			loff_t holebegin = round_up(newsize, PAGE_SIZE);
1113 1114 1115 1116 1117 1118
			if (oldsize > holebegin)
				unmap_mapping_range(inode->i_mapping,
							holebegin, 0, 1);
			if (info->alloced)
				shmem_truncate_range(inode,
							newsize, (loff_t)-1);
1119
			/* unmap again to remove racily COWed private pages */
1120 1121 1122
			if (oldsize > holebegin)
				unmap_mapping_range(inode->i_mapping,
							holebegin, 0, 1);
1123 1124 1125 1126 1127

			/*
			 * Part of the huge page can be beyond i_size: subject
			 * to shrink under memory pressure.
			 */
1128
			if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
1129
				spin_lock(&sbinfo->shrinklist_lock);
1130 1131 1132 1133 1134
				/*
				 * _careful to defend against unlocked access to
				 * ->shrink_list in shmem_unused_huge_shrink()
				 */
				if (list_empty_careful(&info->shrinklist)) {
1135 1136 1137 1138 1139 1140
					list_add_tail(&info->shrinklist,
							&sbinfo->shrinklist);
					sbinfo->shrinklist_len++;
				}
				spin_unlock(&sbinfo->shrinklist_lock);
			}
1141
		}
L
Linus Torvalds 已提交
1142 1143
	}

1144 1145
	setattr_copy(inode, attr);
	if (attr->ia_valid & ATTR_MODE)
C
Christoph Hellwig 已提交
1146
		error = posix_acl_chmod(inode, inode->i_mode);
L
Linus Torvalds 已提交
1147 1148 1149
	return error;
}

A
Al Viro 已提交
1150
static void shmem_evict_inode(struct inode *inode)
L
Linus Torvalds 已提交
1151 1152
{
	struct shmem_inode_info *info = SHMEM_I(inode);
1153
	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
L
Linus Torvalds 已提交
1154

1155
	if (shmem_mapping(inode->i_mapping)) {
L
Linus Torvalds 已提交
1156 1157
		shmem_unacct_size(info->flags, inode->i_size);
		inode->i_size = 0;
1158
		shmem_truncate_range(inode, 0, (loff_t)-1);
1159 1160 1161 1162 1163 1164 1165 1166
		if (!list_empty(&info->shrinklist)) {
			spin_lock(&sbinfo->shrinklist_lock);
			if (!list_empty(&info->shrinklist)) {
				list_del_init(&info->shrinklist);
				sbinfo->shrinklist_len--;
			}
			spin_unlock(&sbinfo->shrinklist_lock);
		}
1167 1168 1169 1170
		while (!list_empty(&info->swaplist)) {
			/* Wait while shmem_unuse() is scanning this inode... */
			wait_var_event(&info->stop_eviction,
				       !atomic_read(&info->stop_eviction));
1171
			mutex_lock(&shmem_swaplist_mutex);
1172 1173 1174
			/* ...but beware of the race if we peeked too early */
			if (!atomic_read(&info->stop_eviction))
				list_del_init(&info->swaplist);
1175
			mutex_unlock(&shmem_swaplist_mutex);
L
Linus Torvalds 已提交
1176
		}
1177
	}
1178

1179
	simple_xattrs_free(&info->xattrs);
1180
	WARN_ON(inode->i_blocks);
1181
	shmem_free_inode(inode->i_sb);
1182
	clear_inode(inode);
L
Linus Torvalds 已提交
1183 1184
}

1185 1186 1187 1188 1189
extern struct swap_info_struct *swap_info[];

static int shmem_find_swap_entries(struct address_space *mapping,
				   pgoff_t start, unsigned int nr_entries,
				   struct page **entries, pgoff_t *indices,
1190
				   unsigned int type, bool frontswap)
1191
{
1192 1193
	XA_STATE(xas, &mapping->i_pages, start);
	struct page *page;
1194
	swp_entry_t entry;
1195 1196 1197 1198
	unsigned int ret = 0;

	if (!nr_entries)
		return 0;
1199 1200

	rcu_read_lock();
1201 1202
	xas_for_each(&xas, page, ULONG_MAX) {
		if (xas_retry(&xas, page))
1203
			continue;
1204 1205

		if (!xa_is_value(page))
1206
			continue;
1207

1208 1209 1210 1211 1212 1213
		entry = radix_to_swp_entry(page);
		if (swp_type(entry) != type)
			continue;
		if (frontswap &&
		    !frontswap_test(swap_info[type], swp_offset(entry)))
			continue;
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223

		indices[ret] = xas.xa_index;
		entries[ret] = page;

		if (need_resched()) {
			xas_pause(&xas);
			cond_resched_rcu();
		}
		if (++ret == nr_entries)
			break;
1224 1225
	}
	rcu_read_unlock();
1226

1227
	return ret;
1228 1229
}

1230
/*
1231 1232
 * Move the swapped pages for an inode to page cache. Returns the count
 * of pages swapped in, or the error in case of failure.
1233
 */
1234 1235
static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec,
				    pgoff_t *indices)
L
Linus Torvalds 已提交
1236
{
1237 1238
	int i = 0;
	int ret = 0;
1239
	int error = 0;
1240
	struct address_space *mapping = inode->i_mapping;
L
Linus Torvalds 已提交
1241

1242 1243
	for (i = 0; i < pvec.nr; i++) {
		struct page *page = pvec.pages[i];
1244

1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
		if (!xa_is_value(page))
			continue;
		error = shmem_swapin_page(inode, indices[i],
					  &page, SGP_CACHE,
					  mapping_gfp_mask(mapping),
					  NULL, NULL);
		if (error == 0) {
			unlock_page(page);
			put_page(page);
			ret++;
		}
		if (error == -ENOMEM)
			break;
		error = 0;
1259
	}
1260 1261
	return error ? error : ret;
}
1262

1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
/*
 * If swap found in inode, free it and move page from swapcache to filecache.
 */
static int shmem_unuse_inode(struct inode *inode, unsigned int type,
			     bool frontswap, unsigned long *fs_pages_to_unuse)
{
	struct address_space *mapping = inode->i_mapping;
	pgoff_t start = 0;
	struct pagevec pvec;
	pgoff_t indices[PAGEVEC_SIZE];
	bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0);
	int ret = 0;

	pagevec_init(&pvec);
	do {
		unsigned int nr_entries = PAGEVEC_SIZE;

		if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE)
			nr_entries = *fs_pages_to_unuse;

		pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries,
						  pvec.pages, indices,
1285
						  type, frontswap);
1286 1287 1288
		if (pvec.nr == 0) {
			ret = 0;
			break;
1289
		}
1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306

		ret = shmem_unuse_swap_entries(inode, pvec, indices);
		if (ret < 0)
			break;

		if (frontswap_partial) {
			*fs_pages_to_unuse -= ret;
			if (*fs_pages_to_unuse == 0) {
				ret = FRONTSWAP_PAGES_UNUSED;
				break;
			}
		}

		start = indices[pvec.nr - 1];
	} while (true);

	return ret;
L
Linus Torvalds 已提交
1307 1308 1309
}

/*
1310 1311 1312
 * Read all the shared memory data that resides in the swap
 * device 'type' back into memory, so the swap device can be
 * unused.
L
Linus Torvalds 已提交
1313
 */
1314 1315
int shmem_unuse(unsigned int type, bool frontswap,
		unsigned long *fs_pages_to_unuse)
L
Linus Torvalds 已提交
1316
{
1317
	struct shmem_inode_info *info, *next;
1318 1319
	int error = 0;

1320 1321 1322 1323 1324 1325
	if (list_empty(&shmem_swaplist))
		return 0;

	mutex_lock(&shmem_swaplist_mutex);
	list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
		if (!info->swapped) {
1326
			list_del_init(&info->swaplist);
1327 1328
			continue;
		}
1329 1330 1331 1332 1333 1334 1335
		/*
		 * Drop the swaplist mutex while searching the inode for swap;
		 * but before doing so, make sure shmem_evict_inode() will not
		 * remove placeholder inode from swaplist, nor let it be freed
		 * (igrab() would protect from unlink, but not from unmount).
		 */
		atomic_inc(&info->stop_eviction);
1336 1337
		mutex_unlock(&shmem_swaplist_mutex);

1338
		error = shmem_unuse_inode(&info->vfs_inode, type, frontswap,
1339
					  fs_pages_to_unuse);
1340
		cond_resched();
1341 1342 1343 1344 1345

		mutex_lock(&shmem_swaplist_mutex);
		next = list_next_entry(info, swaplist);
		if (!info->swapped)
			list_del_init(&info->swaplist);
1346 1347
		if (atomic_dec_and_test(&info->stop_eviction))
			wake_up_var(&info->stop_eviction);
1348
		if (error)
1349
			break;
L
Linus Torvalds 已提交
1350
	}
1351
	mutex_unlock(&shmem_swaplist_mutex);
1352 1353

	return error;
L
Linus Torvalds 已提交
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
}

/*
 * Move the page from the page cache to the swap cache.
 */
static int shmem_writepage(struct page *page, struct writeback_control *wbc)
{
	struct shmem_inode_info *info;
	struct address_space *mapping;
	struct inode *inode;
1364 1365
	swp_entry_t swap;
	pgoff_t index;
L
Linus Torvalds 已提交
1366

1367
	VM_BUG_ON_PAGE(PageCompound(page), page);
L
Linus Torvalds 已提交
1368 1369 1370 1371 1372 1373 1374
	BUG_ON(!PageLocked(page));
	mapping = page->mapping;
	index = page->index;
	inode = mapping->host;
	info = SHMEM_I(inode);
	if (info->flags & VM_LOCKED)
		goto redirty;
1375
	if (!total_swap_pages)
L
Linus Torvalds 已提交
1376 1377
		goto redirty;

1378
	/*
C
Christoph Hellwig 已提交
1379 1380 1381 1382 1383
	 * Our capabilities prevent regular writeback or sync from ever calling
	 * shmem_writepage; but a stacking filesystem might use ->writepage of
	 * its underlying filesystem, in which case tmpfs should write out to
	 * swap only in response to memory pressure, and not for the writeback
	 * threads or sync.
1384
	 */
1385 1386 1387 1388
	if (!wbc->for_reclaim) {
		WARN_ON_ONCE(1);	/* Still happens? Tell us about it! */
		goto redirty;
	}
1389 1390 1391 1392 1393

	/*
	 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
	 * value into swapfile.c, the only way we can correctly account for a
	 * fallocated page arriving here is now to initialize it and write it.
1394 1395 1396 1397 1398 1399
	 *
	 * That's okay for a page already fallocated earlier, but if we have
	 * not yet completed the fallocation, then (a) we want to keep track
	 * of this page in case we have to undo it, and (b) it may not be a
	 * good idea to continue anyway, once we're pushing into swap.  So
	 * reactivate the page, and let shmem_fallocate() quit when too many.
1400 1401
	 */
	if (!PageUptodate(page)) {
1402 1403 1404 1405 1406
		if (inode->i_private) {
			struct shmem_falloc *shmem_falloc;
			spin_lock(&inode->i_lock);
			shmem_falloc = inode->i_private;
			if (shmem_falloc &&
1407
			    !shmem_falloc->waitq &&
1408 1409 1410 1411 1412 1413 1414 1415 1416
			    index >= shmem_falloc->start &&
			    index < shmem_falloc->next)
				shmem_falloc->nr_unswapped++;
			else
				shmem_falloc = NULL;
			spin_unlock(&inode->i_lock);
			if (shmem_falloc)
				goto redirty;
		}
1417 1418 1419 1420 1421
		clear_highpage(page);
		flush_dcache_page(page);
		SetPageUptodate(page);
	}

1422
	swap = get_swap_page(page);
1423 1424
	if (!swap.val)
		goto redirty;
1425

1426 1427
	/*
	 * Add inode to shmem_unuse()'s list of swapped-out inodes,
1428 1429
	 * if it's not already there.  Do it now before the page is
	 * moved to swap cache, when its pagelock no longer protects
1430
	 * the inode from eviction.  But don't unlock the mutex until
1431 1432
	 * we've incremented swapped, because shmem_unuse_inode() will
	 * prune a !swapped inode from the swaplist under this mutex.
1433
	 */
1434 1435
	mutex_lock(&shmem_swaplist_mutex);
	if (list_empty(&info->swaplist))
1436
		list_add(&info->swaplist, &shmem_swaplist);
1437

1438
	if (add_to_swap_cache(page, swap,
1439 1440
			__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
			NULL) == 0) {
1441
		spin_lock_irq(&info->lock);
1442
		shmem_recalc_inode(inode);
1443
		info->swapped++;
1444
		spin_unlock_irq(&info->lock);
1445

1446 1447 1448
		swap_shmem_alloc(swap);
		shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));

1449
		mutex_unlock(&shmem_swaplist_mutex);
1450
		BUG_ON(page_mapped(page));
H
Hugh Dickins 已提交
1451
		swap_writepage(page, wbc);
L
Linus Torvalds 已提交
1452 1453 1454
		return 0;
	}

1455
	mutex_unlock(&shmem_swaplist_mutex);
1456
	put_swap_page(page, swap);
L
Linus Torvalds 已提交
1457 1458
redirty:
	set_page_dirty(page);
1459 1460 1461 1462
	if (wbc->for_reclaim)
		return AOP_WRITEPAGE_ACTIVATE;	/* Return with page locked */
	unlock_page(page);
	return 0;
L
Linus Torvalds 已提交
1463 1464
}

H
Hugh Dickins 已提交
1465
#if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1466
static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1467
{
1468
	char buffer[64];
1469

1470
	if (!mpol || mpol->mode == MPOL_DEFAULT)
1471
		return;		/* show nothing */
1472

1473
	mpol_to_str(buffer, sizeof(buffer), mpol);
1474 1475

	seq_printf(seq, ",mpol=%s", buffer);
1476
}
1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488

static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
{
	struct mempolicy *mpol = NULL;
	if (sbinfo->mpol) {
		spin_lock(&sbinfo->stat_lock);	/* prevent replace/use races */
		mpol = sbinfo->mpol;
		mpol_get(mpol);
		spin_unlock(&sbinfo->stat_lock);
	}
	return mpol;
}
H
Hugh Dickins 已提交
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
#else /* !CONFIG_NUMA || !CONFIG_TMPFS */
static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
{
}
static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
{
	return NULL;
}
#endif /* CONFIG_NUMA && CONFIG_TMPFS */
#ifndef CONFIG_NUMA
#define vm_policy vm_private_data
#endif
1501

1502 1503 1504 1505
static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
		struct shmem_inode_info *info, pgoff_t index)
{
	/* Create a pseudo vma that just contains the policy */
1506
	vma_init(vma, NULL);
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517
	/* Bias interleave by inode number to distribute better across nodes */
	vma->vm_pgoff = index + info->vfs_inode.i_ino;
	vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
}

static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
{
	/* Drop reference taken by mpol_shared_policy_lookup() */
	mpol_cond_put(vma->vm_policy);
}

1518 1519
static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
			struct shmem_inode_info *info, pgoff_t index)
L
Linus Torvalds 已提交
1520 1521
{
	struct vm_area_struct pvma;
M
Mel Gorman 已提交
1522
	struct page *page;
1523
	struct vm_fault vmf;
1524

1525
	shmem_pseudo_vma_init(&pvma, info, index);
1526 1527 1528
	vmf.vma = &pvma;
	vmf.address = 0;
	page = swap_cluster_readahead(swap, gfp, &vmf);
1529
	shmem_pseudo_vma_destroy(&pvma);
M
Mel Gorman 已提交
1530

1531 1532 1533 1534 1535 1536 1537
	return page;
}

static struct page *shmem_alloc_hugepage(gfp_t gfp,
		struct shmem_inode_info *info, pgoff_t index)
{
	struct vm_area_struct pvma;
1538 1539
	struct address_space *mapping = info->vfs_inode.i_mapping;
	pgoff_t hindex;
1540 1541
	struct page *page;

1542
	hindex = round_down(index, HPAGE_PMD_NR);
1543 1544
	if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
								XA_PRESENT))
1545
		return NULL;
M
Mel Gorman 已提交
1546

1547 1548
	shmem_pseudo_vma_init(&pvma, info, hindex);
	page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1549
			HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1550 1551 1552
	shmem_pseudo_vma_destroy(&pvma);
	if (page)
		prep_transhuge_page(page);
1553 1554
	else
		count_vm_event(THP_FILE_FALLBACK);
M
Mel Gorman 已提交
1555
	return page;
L
Linus Torvalds 已提交
1556 1557
}

1558
static struct page *shmem_alloc_page(gfp_t gfp,
1559
			struct shmem_inode_info *info, pgoff_t index)
L
Linus Torvalds 已提交
1560 1561
{
	struct vm_area_struct pvma;
M
Mel Gorman 已提交
1562
	struct page *page;
L
Linus Torvalds 已提交
1563

1564 1565 1566 1567 1568 1569 1570 1571
	shmem_pseudo_vma_init(&pvma, info, index);
	page = alloc_page_vma(gfp, &pvma, 0);
	shmem_pseudo_vma_destroy(&pvma);

	return page;
}

static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1572
		struct inode *inode,
1573 1574
		pgoff_t index, bool huge)
{
1575
	struct shmem_inode_info *info = SHMEM_I(inode);
1576 1577 1578
	struct page *page;
	int nr;
	int err = -ENOSPC;
1579

1580
	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1581 1582 1583
		huge = false;
	nr = huge ? HPAGE_PMD_NR : 1;

1584
	if (!shmem_inode_acct_block(inode, nr))
1585 1586 1587 1588 1589 1590
		goto failed;

	if (huge)
		page = shmem_alloc_hugepage(gfp, info, index);
	else
		page = shmem_alloc_page(gfp, info, index);
H
Hugh Dickins 已提交
1591 1592 1593
	if (page) {
		__SetPageLocked(page);
		__SetPageSwapBacked(page);
1594
		return page;
H
Hugh Dickins 已提交
1595
	}
M
Mel Gorman 已提交
1596

1597
	err = -ENOMEM;
1598
	shmem_inode_unacct_blocks(inode, nr);
1599 1600
failed:
	return ERR_PTR(err);
L
Linus Torvalds 已提交
1601
}
1602

1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
/*
 * When a page is moved from swapcache to shmem filecache (either by the
 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
 * shmem_unuse_inode()), it may have been read in earlier from swap, in
 * ignorance of the mapping it belongs to.  If that mapping has special
 * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
 * we may need to copy to a suitable page before moving to filecache.
 *
 * In a future release, this may well be extended to respect cpuset and
 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
 * but for now it is a simple matter of zone.
 */
static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
{
	return page_zonenum(page) > gfp_zone(gfp);
}

static int shmem_replace_page(struct page **pagep, gfp_t gfp,
				struct shmem_inode_info *info, pgoff_t index)
{
	struct page *oldpage, *newpage;
	struct address_space *swap_mapping;
1625
	swp_entry_t entry;
1626 1627 1628 1629
	pgoff_t swap_index;
	int error;

	oldpage = *pagep;
1630 1631
	entry.val = page_private(oldpage);
	swap_index = swp_offset(entry);
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642
	swap_mapping = page_mapping(oldpage);

	/*
	 * We have arrived here because our zones are constrained, so don't
	 * limit chance of success by further cpuset and node constraints.
	 */
	gfp &= ~GFP_CONSTRAINT_MASK;
	newpage = shmem_alloc_page(gfp, info, index);
	if (!newpage)
		return -ENOMEM;

1643
	get_page(newpage);
1644
	copy_highpage(newpage, oldpage);
1645
	flush_dcache_page(newpage);
1646

1647 1648
	__SetPageLocked(newpage);
	__SetPageSwapBacked(newpage);
1649
	SetPageUptodate(newpage);
1650
	set_page_private(newpage, entry.val);
1651 1652 1653 1654 1655 1656
	SetPageSwapCache(newpage);

	/*
	 * Our caller will very soon move newpage out of swapcache, but it's
	 * a nice clean interface for us to replace oldpage by newpage there.
	 */
M
Matthew Wilcox 已提交
1657
	xa_lock_irq(&swap_mapping->i_pages);
1658
	error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
1659
	if (!error) {
1660 1661 1662
		mem_cgroup_migrate(oldpage, newpage);
		__inc_lruvec_page_state(newpage, NR_FILE_PAGES);
		__dec_lruvec_page_state(oldpage, NR_FILE_PAGES);
1663
	}
M
Matthew Wilcox 已提交
1664
	xa_unlock_irq(&swap_mapping->i_pages);
1665

1666 1667 1668 1669 1670 1671 1672 1673
	if (unlikely(error)) {
		/*
		 * Is this possible?  I think not, now that our callers check
		 * both PageSwapCache and page_private after getting page lock;
		 * but be defensive.  Reverse old to newpage for clear and free.
		 */
		oldpage = newpage;
	} else {
1674
		lru_cache_add(newpage);
1675 1676
		*pagep = newpage;
	}
1677 1678 1679 1680 1681

	ClearPageSwapCache(oldpage);
	set_page_private(oldpage, 0);

	unlock_page(oldpage);
1682 1683
	put_page(oldpage);
	put_page(oldpage);
1684
	return error;
1685 1686
}

1687 1688 1689 1690
/*
 * Swap in the page pointed to by *pagep.
 * Caller has to make sure that *pagep contains a valid swapped page.
 * Returns 0 and the page in pagep if success. On failure, returns the
1691
 * error code and NULL in *pagep.
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
 */
static int shmem_swapin_page(struct inode *inode, pgoff_t index,
			     struct page **pagep, enum sgp_type sgp,
			     gfp_t gfp, struct vm_area_struct *vma,
			     vm_fault_t *fault_type)
{
	struct address_space *mapping = inode->i_mapping;
	struct shmem_inode_info *info = SHMEM_I(inode);
	struct mm_struct *charge_mm = vma ? vma->vm_mm : current->mm;
	struct page *page;
	swp_entry_t swap;
	int error;

	VM_BUG_ON(!*pagep || !xa_is_value(*pagep));
	swap = radix_to_swp_entry(*pagep);
	*pagep = NULL;

	/* Look it up and read it in.. */
	page = lookup_swap_cache(swap, NULL, 0);
	if (!page) {
		/* Or update major stats only when swapin succeeds?? */
		if (fault_type) {
			*fault_type |= VM_FAULT_MAJOR;
			count_vm_event(PGMAJFAULT);
			count_memcg_event_mm(charge_mm, PGMAJFAULT);
		}
		/* Here we actually start the io */
		page = shmem_swapin(swap, gfp, info, index);
		if (!page) {
			error = -ENOMEM;
			goto failed;
		}
	}

	/* We have to do this with page locked to prevent races */
	lock_page(page);
	if (!PageSwapCache(page) || page_private(page) != swap.val ||
	    !shmem_confirm_swap(mapping, index, swap)) {
		error = -EEXIST;
		goto unlock;
	}
	if (!PageUptodate(page)) {
		error = -EIO;
		goto failed;
	}
	wait_on_page_writeback(page);

1739 1740 1741 1742 1743 1744
	/*
	 * Some architectures may have to restore extra metadata to the
	 * physical page after reading from swap.
	 */
	arch_swap_restore(swap, page);

1745 1746 1747 1748 1749 1750
	if (shmem_should_replace_page(page, gfp)) {
		error = shmem_replace_page(&page, gfp, info, index);
		if (error)
			goto failed;
	}

1751
	error = shmem_add_to_page_cache(page, mapping, index,
1752 1753 1754
					swp_to_radix_entry(swap), gfp,
					charge_mm);
	if (error)
1755
		goto failed;
1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782

	spin_lock_irq(&info->lock);
	info->swapped--;
	shmem_recalc_inode(inode);
	spin_unlock_irq(&info->lock);

	if (sgp == SGP_WRITE)
		mark_page_accessed(page);

	delete_from_swap_cache(page);
	set_page_dirty(page);
	swap_free(swap);

	*pagep = page;
	return 0;
failed:
	if (!shmem_confirm_swap(mapping, index, swap))
		error = -EEXIST;
unlock:
	if (page) {
		unlock_page(page);
		put_page(page);
	}

	return error;
}

L
Linus Torvalds 已提交
1783
/*
1784
 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
L
Linus Torvalds 已提交
1785 1786 1787
 *
 * If we allocate a new one we do not mark it dirty. That's up to the
 * vm. If we swap it in we mark it dirty since we also free the swap
1788 1789
 * entry since a page cannot live in both the swap and page cache.
 *
1790
 * vmf and fault_type are only supplied by shmem_fault:
1791
 * otherwise they are NULL.
L
Linus Torvalds 已提交
1792
 */
1793
static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1794
	struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1795 1796
	struct vm_area_struct *vma, struct vm_fault *vmf,
			vm_fault_t *fault_type)
L
Linus Torvalds 已提交
1797 1798
{
	struct address_space *mapping = inode->i_mapping;
1799
	struct shmem_inode_info *info = SHMEM_I(inode);
L
Linus Torvalds 已提交
1800
	struct shmem_sb_info *sbinfo;
1801
	struct mm_struct *charge_mm;
H
Hugh Dickins 已提交
1802
	struct page *page;
1803
	enum sgp_type sgp_huge = sgp;
1804
	pgoff_t hindex = index;
L
Linus Torvalds 已提交
1805
	int error;
1806
	int once = 0;
1807
	int alloced = 0;
L
Linus Torvalds 已提交
1808

1809
	if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
L
Linus Torvalds 已提交
1810
		return -EFBIG;
1811 1812
	if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
		sgp = SGP_CACHE;
L
Linus Torvalds 已提交
1813
repeat:
1814 1815 1816 1817 1818 1819 1820 1821
	if (sgp <= SGP_CACHE &&
	    ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
		return -EINVAL;
	}

	sbinfo = SHMEM_SB(inode->i_sb);
	charge_mm = vma ? vma->vm_mm : current->mm;

1822
	page = find_lock_entry(mapping, index);
1823
	if (xa_is_value(page)) {
1824 1825 1826 1827
		error = shmem_swapin_page(inode, index, &page,
					  sgp, gfp, vma, fault_type);
		if (error == -EEXIST)
			goto repeat;
1828

1829 1830
		*pagep = page;
		return error;
1831 1832
	}

1833 1834
	if (page)
		hindex = page->index;
1835 1836 1837
	if (page && sgp == SGP_WRITE)
		mark_page_accessed(page);

1838 1839 1840 1841 1842
	/* fallocated page? */
	if (page && !PageUptodate(page)) {
		if (sgp != SGP_READ)
			goto clear;
		unlock_page(page);
1843
		put_page(page);
1844
		page = NULL;
1845
		hindex = index;
1846
	}
1847 1848
	if (page || sgp == SGP_READ)
		goto out;
H
Hugh Dickins 已提交
1849 1850

	/*
1851 1852
	 * Fast cache lookup did not find it:
	 * bring it back from swap or allocate.
H
Hugh Dickins 已提交
1853
	 */
1854

1855 1856 1857 1858
	if (vma && userfaultfd_missing(vma)) {
		*fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
		return 0;
	}
1859

1860
	/* shmem_symlink() */
1861
	if (!shmem_mapping(mapping))
1862 1863 1864 1865 1866 1867 1868 1869
		goto alloc_nohuge;
	if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
		goto alloc_nohuge;
	if (shmem_huge == SHMEM_HUGE_FORCE)
		goto alloc_huge;
	switch (sbinfo->huge) {
	case SHMEM_HUGE_NEVER:
		goto alloc_nohuge;
1870 1871 1872 1873
	case SHMEM_HUGE_WITHIN_SIZE: {
		loff_t i_size;
		pgoff_t off;

1874 1875 1876 1877
		off = round_up(index, HPAGE_PMD_NR);
		i_size = round_up(i_size_read(inode), PAGE_SIZE);
		if (i_size >= HPAGE_PMD_SIZE &&
		    i_size >> PAGE_SHIFT >= off)
1878
			goto alloc_huge;
1879 1880 1881

		fallthrough;
	}
1882 1883 1884 1885 1886 1887
	case SHMEM_HUGE_ADVISE:
		if (sgp_huge == SGP_HUGE)
			goto alloc_huge;
		/* TODO: implement fadvise() hints */
		goto alloc_nohuge;
	}
L
Linus Torvalds 已提交
1888

1889
alloc_huge:
1890 1891 1892 1893 1894 1895 1896 1897
	page = shmem_alloc_and_acct_page(gfp, inode, index, true);
	if (IS_ERR(page)) {
alloc_nohuge:
		page = shmem_alloc_and_acct_page(gfp, inode,
						 index, false);
	}
	if (IS_ERR(page)) {
		int retry = 5;
1898

1899 1900 1901 1902 1903 1904 1905 1906 1907 1908
		error = PTR_ERR(page);
		page = NULL;
		if (error != -ENOSPC)
			goto unlock;
		/*
		 * Try to reclaim some space by splitting a huge page
		 * beyond i_size on the filesystem.
		 */
		while (retry--) {
			int ret;
1909

1910 1911 1912 1913 1914
			ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
			if (ret == SHRINK_STOP)
				break;
			if (ret)
				goto alloc_nohuge;
1915
		}
1916 1917
		goto unlock;
	}
1918

1919 1920 1921 1922
	if (PageTransHuge(page))
		hindex = round_down(index, HPAGE_PMD_NR);
	else
		hindex = index;
1923

1924 1925 1926 1927
	if (sgp == SGP_WRITE)
		__SetPageReferenced(page);

	error = shmem_add_to_page_cache(page, mapping, hindex,
1928 1929 1930
					NULL, gfp & GFP_RECLAIM_MASK,
					charge_mm);
	if (error)
1931
		goto unacct;
1932
	lru_cache_add(page);
1933

1934
	spin_lock_irq(&info->lock);
1935
	info->alloced += compound_nr(page);
1936 1937 1938 1939 1940 1941 1942 1943
	inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
	shmem_recalc_inode(inode);
	spin_unlock_irq(&info->lock);
	alloced = true;

	if (PageTransHuge(page) &&
	    DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
			hindex + HPAGE_PMD_NR - 1) {
1944
		/*
1945 1946
		 * Part of the huge page is beyond i_size: subject
		 * to shrink under memory pressure.
1947
		 */
1948
		spin_lock(&sbinfo->shrinklist_lock);
1949
		/*
1950 1951
		 * _careful to defend against unlocked access to
		 * ->shrink_list in shmem_unused_huge_shrink()
1952
		 */
1953 1954 1955 1956 1957 1958 1959
		if (list_empty_careful(&info->shrinklist)) {
			list_add_tail(&info->shrinklist,
				      &sbinfo->shrinklist);
			sbinfo->shrinklist_len++;
		}
		spin_unlock(&sbinfo->shrinklist_lock);
	}
1960

1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
	/*
	 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
	 */
	if (sgp == SGP_FALLOC)
		sgp = SGP_WRITE;
clear:
	/*
	 * Let SGP_WRITE caller clear ends if write does not fill page;
	 * but SGP_FALLOC on a page fallocated earlier must initialize
	 * it now, lest undo on failure cancel our earlier guarantee.
	 */
	if (sgp != SGP_WRITE && !PageUptodate(page)) {
		int i;

1975 1976 1977
		for (i = 0; i < compound_nr(page); i++) {
			clear_highpage(page + i);
			flush_dcache_page(page + i);
1978
		}
1979
		SetPageUptodate(page);
L
Linus Torvalds 已提交
1980
	}
1981

1982
	/* Perhaps the file has been truncated since we checked */
H
Hugh Dickins 已提交
1983
	if (sgp <= SGP_CACHE &&
1984
	    ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1985 1986 1987
		if (alloced) {
			ClearPageDirty(page);
			delete_from_page_cache(page);
1988
			spin_lock_irq(&info->lock);
1989
			shmem_recalc_inode(inode);
1990
			spin_unlock_irq(&info->lock);
1991
		}
1992
		error = -EINVAL;
1993
		goto unlock;
H
Hugh Dickins 已提交
1994
	}
1995
out:
1996
	*pagep = page + index - hindex;
1997
	return 0;
L
Linus Torvalds 已提交
1998

1999
	/*
2000
	 * Error recovery.
2001
	 */
2002
unacct:
2003
	shmem_inode_unacct_blocks(inode, compound_nr(page));
2004 2005 2006 2007 2008 2009

	if (PageTransHuge(page)) {
		unlock_page(page);
		put_page(page);
		goto alloc_nohuge;
	}
2010
unlock:
H
Hugh Dickins 已提交
2011
	if (page) {
2012
		unlock_page(page);
2013
		put_page(page);
2014 2015
	}
	if (error == -ENOSPC && !once++) {
2016
		spin_lock_irq(&info->lock);
2017
		shmem_recalc_inode(inode);
2018
		spin_unlock_irq(&info->lock);
H
Hugh Dickins 已提交
2019
		goto repeat;
2020
	}
M
Matthew Wilcox 已提交
2021
	if (error == -EEXIST)
2022 2023
		goto repeat;
	return error;
L
Linus Torvalds 已提交
2024 2025
}

2026 2027 2028 2029 2030
/*
 * This is like autoremove_wake_function, but it removes the wait queue
 * entry unconditionally - even if something else had already woken the
 * target.
 */
2031
static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2032 2033
{
	int ret = default_wake_function(wait, mode, sync, key);
2034
	list_del_init(&wait->entry);
2035 2036 2037
	return ret;
}

2038
static vm_fault_t shmem_fault(struct vm_fault *vmf)
L
Linus Torvalds 已提交
2039
{
2040
	struct vm_area_struct *vma = vmf->vma;
A
Al Viro 已提交
2041
	struct inode *inode = file_inode(vma->vm_file);
2042
	gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2043
	enum sgp_type sgp;
2044 2045
	int err;
	vm_fault_t ret = VM_FAULT_LOCKED;
L
Linus Torvalds 已提交
2046

2047 2048 2049 2050
	/*
	 * Trinity finds that probing a hole which tmpfs is punching can
	 * prevent the hole-punch from ever completing: which in turn
	 * locks writers out with its hold on i_mutex.  So refrain from
2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062
	 * faulting pages into the hole while it's being punched.  Although
	 * shmem_undo_range() does remove the additions, it may be unable to
	 * keep up, as each new page needs its own unmap_mapping_range() call,
	 * and the i_mmap tree grows ever slower to scan if new vmas are added.
	 *
	 * It does not matter if we sometimes reach this check just before the
	 * hole-punch begins, so that one fault then races with the punch:
	 * we just need to make racing faults a rare case.
	 *
	 * The implementation below would be much simpler if we just used a
	 * standard mutex or completion: but we cannot take i_mutex in fault,
	 * and bloating every shmem inode for this unlikely case would be sad.
2063 2064 2065 2066 2067 2068
	 */
	if (unlikely(inode->i_private)) {
		struct shmem_falloc *shmem_falloc;

		spin_lock(&inode->i_lock);
		shmem_falloc = inode->i_private;
2069 2070 2071 2072
		if (shmem_falloc &&
		    shmem_falloc->waitq &&
		    vmf->pgoff >= shmem_falloc->start &&
		    vmf->pgoff < shmem_falloc->next) {
2073
			struct file *fpin;
2074
			wait_queue_head_t *shmem_falloc_waitq;
2075
			DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2076 2077

			ret = VM_FAULT_NOPAGE;
2078 2079
			fpin = maybe_unlock_mmap_for_io(vmf, NULL);
			if (fpin)
2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
				ret = VM_FAULT_RETRY;

			shmem_falloc_waitq = shmem_falloc->waitq;
			prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
					TASK_UNINTERRUPTIBLE);
			spin_unlock(&inode->i_lock);
			schedule();

			/*
			 * shmem_falloc_waitq points into the shmem_fallocate()
			 * stack of the hole-punching task: shmem_falloc_waitq
			 * is usually invalid by the time we reach here, but
			 * finish_wait() does not dereference it in that case;
			 * though i_lock needed lest racing with wake_up_all().
			 */
			spin_lock(&inode->i_lock);
			finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
			spin_unlock(&inode->i_lock);
2098 2099 2100

			if (fpin)
				fput(fpin);
2101
			return ret;
2102
		}
2103
		spin_unlock(&inode->i_lock);
2104 2105
	}

2106
	sgp = SGP_CACHE;
2107 2108 2109

	if ((vma->vm_flags & VM_NOHUGEPAGE) ||
	    test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
2110
		sgp = SGP_NOHUGE;
2111 2112
	else if (vma->vm_flags & VM_HUGEPAGE)
		sgp = SGP_HUGE;
2113

2114
	err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
2115
				  gfp, vma, vmf, &ret);
2116 2117
	if (err)
		return vmf_error(err);
2118
	return ret;
L
Linus Torvalds 已提交
2119 2120
}

2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138
unsigned long shmem_get_unmapped_area(struct file *file,
				      unsigned long uaddr, unsigned long len,
				      unsigned long pgoff, unsigned long flags)
{
	unsigned long (*get_area)(struct file *,
		unsigned long, unsigned long, unsigned long, unsigned long);
	unsigned long addr;
	unsigned long offset;
	unsigned long inflated_len;
	unsigned long inflated_addr;
	unsigned long inflated_offset;

	if (len > TASK_SIZE)
		return -ENOMEM;

	get_area = current->mm->get_unmapped_area;
	addr = get_area(file, uaddr, len, pgoff, flags);

2139
	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156
		return addr;
	if (IS_ERR_VALUE(addr))
		return addr;
	if (addr & ~PAGE_MASK)
		return addr;
	if (addr > TASK_SIZE - len)
		return addr;

	if (shmem_huge == SHMEM_HUGE_DENY)
		return addr;
	if (len < HPAGE_PMD_SIZE)
		return addr;
	if (flags & MAP_FIXED)
		return addr;
	/*
	 * Our priority is to support MAP_SHARED mapped hugely;
	 * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2157 2158
	 * But if caller specified an address hint and we allocated area there
	 * successfully, respect that as before.
2159
	 */
2160
	if (uaddr == addr)
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
		return addr;

	if (shmem_huge != SHMEM_HUGE_FORCE) {
		struct super_block *sb;

		if (file) {
			VM_BUG_ON(file->f_op != &shmem_file_operations);
			sb = file_inode(file)->i_sb;
		} else {
			/*
			 * Called directly from mm/mmap.c, or drivers/char/mem.c
			 * for "/dev/zero", to create a shared anonymous object.
			 */
			if (IS_ERR(shm_mnt))
				return addr;
			sb = shm_mnt->mnt_sb;
		}
2178
		if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
			return addr;
	}

	offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
	if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
		return addr;
	if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
		return addr;

	inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
	if (inflated_len > TASK_SIZE)
		return addr;
	if (inflated_len < len)
		return addr;

2194
	inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
	if (IS_ERR_VALUE(inflated_addr))
		return addr;
	if (inflated_addr & ~PAGE_MASK)
		return addr;

	inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
	inflated_addr += offset - inflated_offset;
	if (inflated_offset > offset)
		inflated_addr += HPAGE_PMD_SIZE;

	if (inflated_addr > TASK_SIZE - len)
		return addr;
	return inflated_addr;
}

L
Linus Torvalds 已提交
2210
#ifdef CONFIG_NUMA
2211
static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
L
Linus Torvalds 已提交
2212
{
A
Al Viro 已提交
2213
	struct inode *inode = file_inode(vma->vm_file);
2214
	return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
L
Linus Torvalds 已提交
2215 2216
}

A
Adrian Bunk 已提交
2217 2218
static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
					  unsigned long addr)
L
Linus Torvalds 已提交
2219
{
A
Al Viro 已提交
2220
	struct inode *inode = file_inode(vma->vm_file);
2221
	pgoff_t index;
L
Linus Torvalds 已提交
2222

2223 2224
	index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
	return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
L
Linus Torvalds 已提交
2225 2226 2227 2228 2229
}
#endif

int shmem_lock(struct file *file, int lock, struct user_struct *user)
{
A
Al Viro 已提交
2230
	struct inode *inode = file_inode(file);
L
Linus Torvalds 已提交
2231 2232 2233
	struct shmem_inode_info *info = SHMEM_I(inode);
	int retval = -ENOMEM;

2234 2235 2236 2237 2238
	/*
	 * What serializes the accesses to info->flags?
	 * ipc_lock_object() when called from shmctl_do_lock(),
	 * no serialization needed when called from shm_destroy().
	 */
L
Linus Torvalds 已提交
2239 2240 2241 2242
	if (lock && !(info->flags & VM_LOCKED)) {
		if (!user_shm_lock(inode->i_size, user))
			goto out_nomem;
		info->flags |= VM_LOCKED;
2243
		mapping_set_unevictable(file->f_mapping);
L
Linus Torvalds 已提交
2244 2245 2246 2247
	}
	if (!lock && (info->flags & VM_LOCKED) && user) {
		user_shm_unlock(inode->i_size, user);
		info->flags &= ~VM_LOCKED;
2248
		mapping_clear_unevictable(file->f_mapping);
L
Linus Torvalds 已提交
2249 2250
	}
	retval = 0;
2251

L
Linus Torvalds 已提交
2252 2253 2254 2255
out_nomem:
	return retval;
}

2256
static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
L
Linus Torvalds 已提交
2257
{
2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268
	struct shmem_inode_info *info = SHMEM_I(file_inode(file));

	if (info->seals & F_SEAL_FUTURE_WRITE) {
		/*
		 * New PROT_WRITE and MAP_SHARED mmaps are not allowed when
		 * "future write" seal active.
		 */
		if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
			return -EPERM;

		/*
2269 2270 2271 2272 2273
		 * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as
		 * MAP_SHARED and read-only, take care to not allow mprotect to
		 * revert protections on such mappings. Do this only for shared
		 * mappings. For private mappings, don't need to mask
		 * VM_MAYWRITE as we still want them to be COW-writable.
2274
		 */
2275 2276
		if (vma->vm_flags & VM_SHARED)
			vma->vm_flags &= ~(VM_MAYWRITE);
2277 2278
	}

2279 2280 2281
	/* arm64 - allow memory tagging on RAM-based files */
	vma->vm_flags |= VM_MTE_ALLOWED;

L
Linus Torvalds 已提交
2282 2283
	file_accessed(file);
	vma->vm_ops = &shmem_vm_ops;
2284
	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
2285 2286 2287 2288
			((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
			(vma->vm_end & HPAGE_PMD_MASK)) {
		khugepaged_enter(vma, vma->vm_flags);
	}
L
Linus Torvalds 已提交
2289 2290 2291
	return 0;
}

2292
static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2293
				     umode_t mode, dev_t dev, unsigned long flags)
L
Linus Torvalds 已提交
2294 2295 2296 2297
{
	struct inode *inode;
	struct shmem_inode_info *info;
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
C
Chris Down 已提交
2298
	ino_t ino;
L
Linus Torvalds 已提交
2299

C
Chris Down 已提交
2300
	if (shmem_reserve_inode(sb, &ino))
2301
		return NULL;
L
Linus Torvalds 已提交
2302 2303 2304

	inode = new_inode(sb);
	if (inode) {
C
Chris Down 已提交
2305
		inode->i_ino = ino;
2306
		inode_init_owner(inode, dir, mode);
L
Linus Torvalds 已提交
2307
		inode->i_blocks = 0;
2308
		inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2309
		inode->i_generation = prandom_u32();
L
Linus Torvalds 已提交
2310 2311 2312
		info = SHMEM_I(inode);
		memset(info, 0, (char *)inode - (char *)info);
		spin_lock_init(&info->lock);
2313
		atomic_set(&info->stop_eviction, 0);
D
David Herrmann 已提交
2314
		info->seals = F_SEAL_SEAL;
2315
		info->flags = flags & VM_NORESERVE;
2316
		INIT_LIST_HEAD(&info->shrinklist);
L
Linus Torvalds 已提交
2317
		INIT_LIST_HEAD(&info->swaplist);
2318
		simple_xattrs_init(&info->xattrs);
2319
		cache_no_acl(inode);
L
Linus Torvalds 已提交
2320 2321 2322

		switch (mode & S_IFMT) {
		default:
2323
			inode->i_op = &shmem_special_inode_operations;
L
Linus Torvalds 已提交
2324 2325 2326
			init_special_inode(inode, mode, dev);
			break;
		case S_IFREG:
2327
			inode->i_mapping->a_ops = &shmem_aops;
L
Linus Torvalds 已提交
2328 2329
			inode->i_op = &shmem_inode_operations;
			inode->i_fop = &shmem_file_operations;
2330 2331
			mpol_shared_policy_init(&info->policy,
						 shmem_get_sbmpol(sbinfo));
L
Linus Torvalds 已提交
2332 2333
			break;
		case S_IFDIR:
2334
			inc_nlink(inode);
L
Linus Torvalds 已提交
2335 2336 2337 2338 2339 2340 2341 2342 2343 2344
			/* Some things misbehave if size == 0 on a directory */
			inode->i_size = 2 * BOGO_DIRENT_SIZE;
			inode->i_op = &shmem_dir_inode_operations;
			inode->i_fop = &simple_dir_operations;
			break;
		case S_IFLNK:
			/*
			 * Must not load anything in the rbtree,
			 * mpol_free_shared_policy will not be called.
			 */
2345
			mpol_shared_policy_init(&info->policy, NULL);
L
Linus Torvalds 已提交
2346 2347
			break;
		}
2348 2349

		lockdep_annotate_inode_mutex_key(inode);
2350 2351
	} else
		shmem_free_inode(sb);
L
Linus Torvalds 已提交
2352 2353 2354
	return inode;
}

2355 2356 2357 2358 2359 2360 2361
static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
				  pmd_t *dst_pmd,
				  struct vm_area_struct *dst_vma,
				  unsigned long dst_addr,
				  unsigned long src_addr,
				  bool zeropage,
				  struct page **pagep)
2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372
{
	struct inode *inode = file_inode(dst_vma->vm_file);
	struct shmem_inode_info *info = SHMEM_I(inode);
	struct address_space *mapping = inode->i_mapping;
	gfp_t gfp = mapping_gfp_mask(mapping);
	pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
	spinlock_t *ptl;
	void *page_kaddr;
	struct page *page;
	pte_t _dst_pte, *dst_pte;
	int ret;
2373
	pgoff_t offset, max_off;
2374

2375
	ret = -ENOMEM;
2376
	if (!shmem_inode_acct_block(inode, 1))
2377
		goto out;
2378

2379
	if (!*pagep) {
2380 2381
		page = shmem_alloc_page(gfp, info, pgoff);
		if (!page)
2382
			goto out_unacct_blocks;
2383

2384 2385 2386 2387 2388 2389 2390
		if (!zeropage) {	/* mcopy_atomic */
			page_kaddr = kmap_atomic(page);
			ret = copy_from_user(page_kaddr,
					     (const void __user *)src_addr,
					     PAGE_SIZE);
			kunmap_atomic(page_kaddr);

2391
			/* fallback to copy_from_user outside mmap_lock */
2392 2393 2394 2395
			if (unlikely(ret)) {
				*pagep = page;
				shmem_inode_unacct_blocks(inode, 1);
				/* don't free the page */
2396
				return -ENOENT;
2397 2398 2399
			}
		} else {		/* mfill_zeropage_atomic */
			clear_highpage(page);
2400 2401 2402 2403 2404 2405
		}
	} else {
		page = *pagep;
		*pagep = NULL;
	}

2406 2407 2408
	VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
	__SetPageLocked(page);
	__SetPageSwapBacked(page);
2409
	__SetPageUptodate(page);
2410

2411 2412 2413 2414 2415 2416
	ret = -EFAULT;
	offset = linear_page_index(dst_vma, dst_addr);
	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
	if (unlikely(offset >= max_off))
		goto out_release;

2417
	ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
2418
				      gfp & GFP_RECLAIM_MASK, dst_mm);
2419
	if (ret)
2420
		goto out_release;
2421 2422 2423 2424

	_dst_pte = mk_pte(page, dst_vma->vm_page_prot);
	if (dst_vma->vm_flags & VM_WRITE)
		_dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2425 2426 2427 2428 2429 2430 2431 2432 2433 2434
	else {
		/*
		 * We don't set the pte dirty if the vma has no
		 * VM_WRITE permission, so mark the page dirty or it
		 * could be freed from under us. We could do it
		 * unconditionally before unlock_page(), but doing it
		 * only if VM_WRITE is not set is faster.
		 */
		set_page_dirty(page);
	}
2435 2436

	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2437 2438 2439 2440

	ret = -EFAULT;
	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
	if (unlikely(offset >= max_off))
2441
		goto out_release_unlock;
2442 2443

	ret = -EEXIST;
2444
	if (!pte_none(*dst_pte))
2445
		goto out_release_unlock;
2446

2447
	lru_cache_add(page);
2448

2449
	spin_lock_irq(&info->lock);
2450 2451 2452
	info->alloced++;
	inode->i_blocks += BLOCKS_PER_PAGE;
	shmem_recalc_inode(inode);
2453
	spin_unlock_irq(&info->lock);
2454 2455 2456 2457 2458 2459 2460 2461

	inc_mm_counter(dst_mm, mm_counter_file(page));
	page_add_file_rmap(page, false);
	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);

	/* No need to invalidate - it was non-present before */
	update_mmu_cache(dst_vma, dst_addr, dst_pte);
	pte_unmap_unlock(dst_pte, ptl);
2462
	unlock_page(page);
2463 2464 2465
	ret = 0;
out:
	return ret;
2466
out_release_unlock:
2467
	pte_unmap_unlock(dst_pte, ptl);
2468
	ClearPageDirty(page);
2469
	delete_from_page_cache(page);
2470
out_release:
2471
	unlock_page(page);
2472 2473
	put_page(page);
out_unacct_blocks:
2474
	shmem_inode_unacct_blocks(inode, 1);
2475 2476 2477
	goto out;
}

2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499
int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
			   pmd_t *dst_pmd,
			   struct vm_area_struct *dst_vma,
			   unsigned long dst_addr,
			   unsigned long src_addr,
			   struct page **pagep)
{
	return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
				      dst_addr, src_addr, false, pagep);
}

int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm,
			     pmd_t *dst_pmd,
			     struct vm_area_struct *dst_vma,
			     unsigned long dst_addr)
{
	struct page *page = NULL;

	return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
				      dst_addr, 0, true, &page);
}

L
Linus Torvalds 已提交
2500
#ifdef CONFIG_TMPFS
2501
static const struct inode_operations shmem_symlink_inode_operations;
2502
static const struct inode_operations shmem_short_symlink_operations;
L
Linus Torvalds 已提交
2503

2504 2505 2506 2507 2508 2509
#ifdef CONFIG_TMPFS_XATTR
static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
#else
#define shmem_initxattrs NULL
#endif

L
Linus Torvalds 已提交
2510
static int
N
Nick Piggin 已提交
2511 2512 2513
shmem_write_begin(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned flags,
			struct page **pagep, void **fsdata)
L
Linus Torvalds 已提交
2514
{
N
Nick Piggin 已提交
2515
	struct inode *inode = mapping->host;
D
David Herrmann 已提交
2516
	struct shmem_inode_info *info = SHMEM_I(inode);
2517
	pgoff_t index = pos >> PAGE_SHIFT;
D
David Herrmann 已提交
2518 2519

	/* i_mutex is held by caller */
2520 2521 2522
	if (unlikely(info->seals & (F_SEAL_GROW |
				   F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
		if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
D
David Herrmann 已提交
2523 2524 2525 2526 2527
			return -EPERM;
		if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
			return -EPERM;
	}

2528
	return shmem_getpage(inode, index, pagep, SGP_WRITE);
N
Nick Piggin 已提交
2529 2530 2531 2532 2533 2534 2535 2536 2537
}

static int
shmem_write_end(struct file *file, struct address_space *mapping,
			loff_t pos, unsigned len, unsigned copied,
			struct page *page, void *fsdata)
{
	struct inode *inode = mapping->host;

H
Hugh Dickins 已提交
2538 2539 2540
	if (pos + copied > inode->i_size)
		i_size_write(inode, pos + copied);

2541
	if (!PageUptodate(page)) {
2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552
		struct page *head = compound_head(page);
		if (PageTransCompound(page)) {
			int i;

			for (i = 0; i < HPAGE_PMD_NR; i++) {
				if (head + i == page)
					continue;
				clear_highpage(head + i);
				flush_dcache_page(head + i);
			}
		}
2553 2554
		if (copied < PAGE_SIZE) {
			unsigned from = pos & (PAGE_SIZE - 1);
2555
			zero_user_segments(page, 0, from,
2556
					from + copied, PAGE_SIZE);
2557
		}
2558
		SetPageUptodate(head);
2559
	}
N
Nick Piggin 已提交
2560
	set_page_dirty(page);
2561
	unlock_page(page);
2562
	put_page(page);
N
Nick Piggin 已提交
2563 2564

	return copied;
L
Linus Torvalds 已提交
2565 2566
}

A
Al Viro 已提交
2567
static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
L
Linus Torvalds 已提交
2568
{
2569 2570
	struct file *file = iocb->ki_filp;
	struct inode *inode = file_inode(file);
L
Linus Torvalds 已提交
2571
	struct address_space *mapping = inode->i_mapping;
2572 2573
	pgoff_t index;
	unsigned long offset;
2574
	enum sgp_type sgp = SGP_READ;
2575
	int error = 0;
A
Al Viro 已提交
2576
	ssize_t retval = 0;
2577
	loff_t *ppos = &iocb->ki_pos;
2578 2579 2580 2581 2582 2583

	/*
	 * Might this read be for a stacking filesystem?  Then when reading
	 * holes of a sparse file, we actually need to allocate those pages,
	 * and even mark them dirty, so it cannot exceed the max_blocks limit.
	 */
A
Al Viro 已提交
2584
	if (!iter_is_iovec(to))
H
Hugh Dickins 已提交
2585
		sgp = SGP_CACHE;
L
Linus Torvalds 已提交
2586

2587 2588
	index = *ppos >> PAGE_SHIFT;
	offset = *ppos & ~PAGE_MASK;
L
Linus Torvalds 已提交
2589 2590 2591

	for (;;) {
		struct page *page = NULL;
2592 2593
		pgoff_t end_index;
		unsigned long nr, ret;
L
Linus Torvalds 已提交
2594 2595
		loff_t i_size = i_size_read(inode);

2596
		end_index = i_size >> PAGE_SHIFT;
L
Linus Torvalds 已提交
2597 2598 2599
		if (index > end_index)
			break;
		if (index == end_index) {
2600
			nr = i_size & ~PAGE_MASK;
L
Linus Torvalds 已提交
2601 2602 2603 2604
			if (nr <= offset)
				break;
		}

2605
		error = shmem_getpage(inode, index, &page, sgp);
2606 2607 2608
		if (error) {
			if (error == -EINVAL)
				error = 0;
L
Linus Torvalds 已提交
2609 2610
			break;
		}
H
Hugh Dickins 已提交
2611 2612 2613
		if (page) {
			if (sgp == SGP_CACHE)
				set_page_dirty(page);
H
Hugh Dickins 已提交
2614
			unlock_page(page);
H
Hugh Dickins 已提交
2615
		}
L
Linus Torvalds 已提交
2616 2617 2618

		/*
		 * We must evaluate after, since reads (unlike writes)
2619
		 * are called without i_mutex protection against truncate
L
Linus Torvalds 已提交
2620
		 */
2621
		nr = PAGE_SIZE;
L
Linus Torvalds 已提交
2622
		i_size = i_size_read(inode);
2623
		end_index = i_size >> PAGE_SHIFT;
L
Linus Torvalds 已提交
2624
		if (index == end_index) {
2625
			nr = i_size & ~PAGE_MASK;
L
Linus Torvalds 已提交
2626 2627
			if (nr <= offset) {
				if (page)
2628
					put_page(page);
L
Linus Torvalds 已提交
2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646
				break;
			}
		}
		nr -= offset;

		if (page) {
			/*
			 * If users can be writing to this page using arbitrary
			 * virtual addresses, take care about potential aliasing
			 * before reading the page on the kernel side.
			 */
			if (mapping_writably_mapped(mapping))
				flush_dcache_page(page);
			/*
			 * Mark the page accessed if we read the beginning.
			 */
			if (!offset)
				mark_page_accessed(page);
N
Nick Piggin 已提交
2647
		} else {
L
Linus Torvalds 已提交
2648
			page = ZERO_PAGE(0);
2649
			get_page(page);
N
Nick Piggin 已提交
2650
		}
L
Linus Torvalds 已提交
2651 2652 2653 2654 2655

		/*
		 * Ok, we have the page, and it's up-to-date, so
		 * now we can copy it to user space...
		 */
A
Al Viro 已提交
2656
		ret = copy_page_to_iter(page, offset, nr, to);
2657
		retval += ret;
L
Linus Torvalds 已提交
2658
		offset += ret;
2659 2660
		index += offset >> PAGE_SHIFT;
		offset &= ~PAGE_MASK;
L
Linus Torvalds 已提交
2661

2662
		put_page(page);
A
Al Viro 已提交
2663
		if (!iov_iter_count(to))
L
Linus Torvalds 已提交
2664
			break;
2665 2666 2667 2668
		if (ret < nr) {
			error = -EFAULT;
			break;
		}
L
Linus Torvalds 已提交
2669 2670 2671
		cond_resched();
	}

2672
	*ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2673 2674
	file_accessed(file);
	return retval ? retval : error;
L
Linus Torvalds 已提交
2675 2676
}

2677
/*
M
Matthew Wilcox 已提交
2678
 * llseek SEEK_DATA or SEEK_HOLE through the page cache.
2679 2680
 */
static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2681
				    pgoff_t index, pgoff_t end, int whence)
2682 2683 2684 2685 2686 2687 2688
{
	struct page *page;
	struct pagevec pvec;
	pgoff_t indices[PAGEVEC_SIZE];
	bool done = false;
	int i;

2689
	pagevec_init(&pvec);
2690 2691
	pvec.nr = 1;		/* start small: we may be there already */
	while (!done) {
2692
		pvec.nr = find_get_entries(mapping, index,
2693 2694
					pvec.nr, pvec.pages, indices);
		if (!pvec.nr) {
2695
			if (whence == SEEK_DATA)
2696 2697 2698 2699 2700
				index = end;
			break;
		}
		for (i = 0; i < pvec.nr; i++, index++) {
			if (index < indices[i]) {
2701
				if (whence == SEEK_HOLE) {
2702 2703 2704 2705 2706 2707
					done = true;
					break;
				}
				index = indices[i];
			}
			page = pvec.pages[i];
2708
			if (page && !xa_is_value(page)) {
2709 2710 2711 2712
				if (!PageUptodate(page))
					page = NULL;
			}
			if (index >= end ||
2713 2714
			    (page && whence == SEEK_DATA) ||
			    (!page && whence == SEEK_HOLE)) {
2715 2716 2717 2718
				done = true;
				break;
			}
		}
2719
		pagevec_remove_exceptionals(&pvec);
2720 2721 2722 2723 2724 2725 2726
		pagevec_release(&pvec);
		pvec.nr = PAGEVEC_SIZE;
		cond_resched();
	}
	return index;
}

2727
static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2728 2729 2730 2731 2732 2733
{
	struct address_space *mapping = file->f_mapping;
	struct inode *inode = mapping->host;
	pgoff_t start, end;
	loff_t new_offset;

2734 2735
	if (whence != SEEK_DATA && whence != SEEK_HOLE)
		return generic_file_llseek_size(file, offset, whence,
2736
					MAX_LFS_FILESIZE, i_size_read(inode));
A
Al Viro 已提交
2737
	inode_lock(inode);
2738 2739
	/* We're holding i_mutex so we can access i_size directly */

2740
	if (offset < 0 || offset >= inode->i_size)
2741 2742
		offset = -ENXIO;
	else {
2743 2744
		start = offset >> PAGE_SHIFT;
		end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2745
		new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2746
		new_offset <<= PAGE_SHIFT;
2747 2748 2749
		if (new_offset > offset) {
			if (new_offset < inode->i_size)
				offset = new_offset;
2750
			else if (whence == SEEK_DATA)
2751 2752 2753 2754 2755 2756
				offset = -ENXIO;
			else
				offset = inode->i_size;
		}
	}

2757 2758
	if (offset >= 0)
		offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
A
Al Viro 已提交
2759
	inode_unlock(inode);
2760 2761 2762
	return offset;
}

2763 2764 2765
static long shmem_fallocate(struct file *file, int mode, loff_t offset,
							 loff_t len)
{
A
Al Viro 已提交
2766
	struct inode *inode = file_inode(file);
2767
	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
D
David Herrmann 已提交
2768
	struct shmem_inode_info *info = SHMEM_I(inode);
2769
	struct shmem_falloc shmem_falloc;
2770 2771
	pgoff_t start, index, end;
	int error;
2772

2773 2774 2775
	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
		return -EOPNOTSUPP;

A
Al Viro 已提交
2776
	inode_lock(inode);
2777 2778 2779 2780 2781

	if (mode & FALLOC_FL_PUNCH_HOLE) {
		struct address_space *mapping = file->f_mapping;
		loff_t unmap_start = round_up(offset, PAGE_SIZE);
		loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2782
		DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2783

D
David Herrmann 已提交
2784
		/* protected by i_mutex */
2785
		if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
D
David Herrmann 已提交
2786 2787 2788 2789
			error = -EPERM;
			goto out;
		}

2790
		shmem_falloc.waitq = &shmem_falloc_waitq;
2791
		shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
2792 2793 2794 2795 2796
		shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
		spin_lock(&inode->i_lock);
		inode->i_private = &shmem_falloc;
		spin_unlock(&inode->i_lock);

2797 2798 2799 2800 2801
		if ((u64)unmap_end > (u64)unmap_start)
			unmap_mapping_range(mapping, unmap_start,
					    1 + unmap_end - unmap_start, 0);
		shmem_truncate_range(inode, offset, offset + len - 1);
		/* No need to unmap again: hole-punching leaves COWed pages */
2802 2803 2804 2805

		spin_lock(&inode->i_lock);
		inode->i_private = NULL;
		wake_up_all(&shmem_falloc_waitq);
2806
		WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2807
		spin_unlock(&inode->i_lock);
2808
		error = 0;
2809
		goto out;
2810 2811 2812 2813 2814 2815 2816
	}

	/* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
	error = inode_newsize_ok(inode, offset + len);
	if (error)
		goto out;

D
David Herrmann 已提交
2817 2818 2819 2820 2821
	if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
		error = -EPERM;
		goto out;
	}

2822 2823
	start = offset >> PAGE_SHIFT;
	end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2824 2825 2826 2827
	/* Try to avoid a swapstorm if len is impossible to satisfy */
	if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
		error = -ENOSPC;
		goto out;
2828 2829
	}

2830
	shmem_falloc.waitq = NULL;
2831 2832 2833 2834 2835 2836 2837 2838
	shmem_falloc.start = start;
	shmem_falloc.next  = start;
	shmem_falloc.nr_falloced = 0;
	shmem_falloc.nr_unswapped = 0;
	spin_lock(&inode->i_lock);
	inode->i_private = &shmem_falloc;
	spin_unlock(&inode->i_lock);

2839 2840 2841 2842 2843 2844 2845 2846 2847
	for (index = start; index < end; index++) {
		struct page *page;

		/*
		 * Good, the fallocate(2) manpage permits EINTR: we may have
		 * been interrupted because we are using up too much memory.
		 */
		if (signal_pending(current))
			error = -EINTR;
2848 2849
		else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
			error = -ENOMEM;
2850
		else
2851
			error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2852
		if (error) {
2853
			/* Remove the !PageUptodate pages we added */
2854 2855 2856 2857 2858
			if (index > start) {
				shmem_undo_range(inode,
				    (loff_t)start << PAGE_SHIFT,
				    ((loff_t)index << PAGE_SHIFT) - 1, true);
			}
2859
			goto undone;
2860 2861
		}

2862 2863 2864 2865 2866 2867 2868 2869
		/*
		 * Inform shmem_writepage() how far we have reached.
		 * No need for lock or barrier: we have the page lock.
		 */
		shmem_falloc.next++;
		if (!PageUptodate(page))
			shmem_falloc.nr_falloced++;

2870
		/*
2871 2872 2873
		 * If !PageUptodate, leave it that way so that freeable pages
		 * can be recognized if we need to rollback on error later.
		 * But set_page_dirty so that memory pressure will swap rather
2874 2875 2876 2877 2878
		 * than free the pages we are allocating (and SGP_CACHE pages
		 * might still be clean: we now need to mark those dirty too).
		 */
		set_page_dirty(page);
		unlock_page(page);
2879
		put_page(page);
2880 2881 2882 2883 2884
		cond_resched();
	}

	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
		i_size_write(inode, offset + len);
2885
	inode->i_ctime = current_time(inode);
2886 2887 2888 2889
undone:
	spin_lock(&inode->i_lock);
	inode->i_private = NULL;
	spin_unlock(&inode->i_lock);
2890
out:
A
Al Viro 已提交
2891
	inode_unlock(inode);
2892 2893 2894
	return error;
}

2895
static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
L
Linus Torvalds 已提交
2896
{
2897
	struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
L
Linus Torvalds 已提交
2898 2899

	buf->f_type = TMPFS_MAGIC;
2900
	buf->f_bsize = PAGE_SIZE;
L
Linus Torvalds 已提交
2901
	buf->f_namelen = NAME_MAX;
2902
	if (sbinfo->max_blocks) {
L
Linus Torvalds 已提交
2903
		buf->f_blocks = sbinfo->max_blocks;
2904 2905 2906
		buf->f_bavail =
		buf->f_bfree  = sbinfo->max_blocks -
				percpu_counter_sum(&sbinfo->used_blocks);
2907 2908
	}
	if (sbinfo->max_inodes) {
L
Linus Torvalds 已提交
2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919
		buf->f_files = sbinfo->max_inodes;
		buf->f_ffree = sbinfo->free_inodes;
	}
	/* else leave those fields 0 like simple_statfs */
	return 0;
}

/*
 * File creation. Allocate an inode, and we're done..
 */
static int
A
Al Viro 已提交
2920
shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
L
Linus Torvalds 已提交
2921
{
2922
	struct inode *inode;
L
Linus Torvalds 已提交
2923 2924
	int error = -ENOSPC;

2925
	inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
L
Linus Torvalds 已提交
2926
	if (inode) {
C
Christoph Hellwig 已提交
2927 2928 2929
		error = simple_acl_create(dir, inode);
		if (error)
			goto out_iput;
2930
		error = security_inode_init_security(inode, dir,
2931
						     &dentry->d_name,
2932
						     shmem_initxattrs, NULL);
C
Christoph Hellwig 已提交
2933 2934
		if (error && error != -EOPNOTSUPP)
			goto out_iput;
2935

A
Al Viro 已提交
2936
		error = 0;
L
Linus Torvalds 已提交
2937
		dir->i_size += BOGO_DIRENT_SIZE;
2938
		dir->i_ctime = dir->i_mtime = current_time(dir);
L
Linus Torvalds 已提交
2939 2940 2941 2942
		d_instantiate(dentry, inode);
		dget(dentry); /* Extra count - pin the dentry in core */
	}
	return error;
C
Christoph Hellwig 已提交
2943 2944 2945
out_iput:
	iput(inode);
	return error;
L
Linus Torvalds 已提交
2946 2947
}

2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958
static int
shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
{
	struct inode *inode;
	int error = -ENOSPC;

	inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
	if (inode) {
		error = security_inode_init_security(inode, dir,
						     NULL,
						     shmem_initxattrs, NULL);
C
Christoph Hellwig 已提交
2959 2960 2961 2962 2963
		if (error && error != -EOPNOTSUPP)
			goto out_iput;
		error = simple_acl_create(dir, inode);
		if (error)
			goto out_iput;
2964 2965 2966
		d_tmpfile(dentry, inode);
	}
	return error;
C
Christoph Hellwig 已提交
2967 2968 2969
out_iput:
	iput(inode);
	return error;
2970 2971
}

2972
static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
L
Linus Torvalds 已提交
2973 2974 2975 2976 2977
{
	int error;

	if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
		return error;
2978
	inc_nlink(dir);
L
Linus Torvalds 已提交
2979 2980 2981
	return 0;
}

A
Al Viro 已提交
2982
static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
A
Al Viro 已提交
2983
		bool excl)
L
Linus Torvalds 已提交
2984 2985 2986 2987 2988 2989 2990 2991 2992
{
	return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
}

/*
 * Link a file..
 */
static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
{
2993
	struct inode *inode = d_inode(old_dentry);
2994
	int ret = 0;
L
Linus Torvalds 已提交
2995 2996 2997 2998 2999

	/*
	 * No ordinary (disk based) filesystem counts links as inodes;
	 * but each new link needs a new dentry, pinning lowmem, and
	 * tmpfs dentries cannot be pruned until they are unlinked.
3000 3001
	 * But if an O_TMPFILE file is linked into the tmpfs, the
	 * first link must skip that, to get the accounting right.
L
Linus Torvalds 已提交
3002
	 */
3003
	if (inode->i_nlink) {
C
Chris Down 已提交
3004
		ret = shmem_reserve_inode(inode->i_sb, NULL);
3005 3006 3007
		if (ret)
			goto out;
	}
L
Linus Torvalds 已提交
3008 3009

	dir->i_size += BOGO_DIRENT_SIZE;
3010
	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3011
	inc_nlink(inode);
A
Al Viro 已提交
3012
	ihold(inode);	/* New dentry reference */
L
Linus Torvalds 已提交
3013 3014
	dget(dentry);		/* Extra pinning count for the created dentry */
	d_instantiate(dentry, inode);
3015 3016
out:
	return ret;
L
Linus Torvalds 已提交
3017 3018 3019 3020
}

static int shmem_unlink(struct inode *dir, struct dentry *dentry)
{
3021
	struct inode *inode = d_inode(dentry);
L
Linus Torvalds 已提交
3022

3023 3024
	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
		shmem_free_inode(inode->i_sb);
L
Linus Torvalds 已提交
3025 3026

	dir->i_size -= BOGO_DIRENT_SIZE;
3027
	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3028
	drop_nlink(inode);
L
Linus Torvalds 已提交
3029 3030 3031 3032 3033 3034 3035 3036 3037
	dput(dentry);	/* Undo the count from "create" - this does all the work */
	return 0;
}

static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
{
	if (!simple_empty(dentry))
		return -ENOTEMPTY;

3038
	drop_nlink(d_inode(dentry));
3039
	drop_nlink(dir);
L
Linus Torvalds 已提交
3040 3041 3042
	return shmem_unlink(dir, dentry);
}

M
Miklos Szeredi 已提交
3043 3044
static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
{
3045 3046
	bool old_is_dir = d_is_dir(old_dentry);
	bool new_is_dir = d_is_dir(new_dentry);
M
Miklos Szeredi 已提交
3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058

	if (old_dir != new_dir && old_is_dir != new_is_dir) {
		if (old_is_dir) {
			drop_nlink(old_dir);
			inc_nlink(new_dir);
		} else {
			drop_nlink(new_dir);
			inc_nlink(old_dir);
		}
	}
	old_dir->i_ctime = old_dir->i_mtime =
	new_dir->i_ctime = new_dir->i_mtime =
3059
	d_inode(old_dentry)->i_ctime =
3060
	d_inode(new_dentry)->i_ctime = current_time(old_dir);
M
Miklos Szeredi 已提交
3061 3062 3063 3064

	return 0;
}

M
Miklos Szeredi 已提交
3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090
static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
{
	struct dentry *whiteout;
	int error;

	whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
	if (!whiteout)
		return -ENOMEM;

	error = shmem_mknod(old_dir, whiteout,
			    S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
	dput(whiteout);
	if (error)
		return error;

	/*
	 * Cheat and hash the whiteout while the old dentry is still in
	 * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
	 *
	 * d_lookup() will consistently find one of them at this point,
	 * not sure which one, but that isn't even important.
	 */
	d_rehash(whiteout);
	return 0;
}

L
Linus Torvalds 已提交
3091 3092 3093 3094 3095 3096
/*
 * The VFS layer already does all the dentry stuff for rename,
 * we just have to decrement the usage count for the target if
 * it exists so that the VFS layer correctly free's it when it
 * gets overwritten.
 */
M
Miklos Szeredi 已提交
3097
static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
L
Linus Torvalds 已提交
3098
{
3099
	struct inode *inode = d_inode(old_dentry);
L
Linus Torvalds 已提交
3100 3101
	int they_are_dirs = S_ISDIR(inode->i_mode);

M
Miklos Szeredi 已提交
3102
	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
M
Miklos Szeredi 已提交
3103 3104
		return -EINVAL;

M
Miklos Szeredi 已提交
3105 3106 3107
	if (flags & RENAME_EXCHANGE)
		return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);

L
Linus Torvalds 已提交
3108 3109 3110
	if (!simple_empty(new_dentry))
		return -ENOTEMPTY;

M
Miklos Szeredi 已提交
3111 3112 3113 3114 3115 3116 3117 3118
	if (flags & RENAME_WHITEOUT) {
		int error;

		error = shmem_whiteout(old_dir, old_dentry);
		if (error)
			return error;
	}

3119
	if (d_really_is_positive(new_dentry)) {
L
Linus Torvalds 已提交
3120
		(void) shmem_unlink(new_dir, new_dentry);
3121
		if (they_are_dirs) {
3122
			drop_nlink(d_inode(new_dentry));
3123
			drop_nlink(old_dir);
3124
		}
L
Linus Torvalds 已提交
3125
	} else if (they_are_dirs) {
3126
		drop_nlink(old_dir);
3127
		inc_nlink(new_dir);
L
Linus Torvalds 已提交
3128 3129 3130 3131 3132 3133
	}

	old_dir->i_size -= BOGO_DIRENT_SIZE;
	new_dir->i_size += BOGO_DIRENT_SIZE;
	old_dir->i_ctime = old_dir->i_mtime =
	new_dir->i_ctime = new_dir->i_mtime =
3134
	inode->i_ctime = current_time(old_dir);
L
Linus Torvalds 已提交
3135 3136 3137 3138 3139 3140 3141 3142
	return 0;
}

static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
{
	int error;
	int len;
	struct inode *inode;
H
Hugh Dickins 已提交
3143
	struct page *page;
L
Linus Torvalds 已提交
3144 3145

	len = strlen(symname) + 1;
3146
	if (len > PAGE_SIZE)
L
Linus Torvalds 已提交
3147 3148
		return -ENAMETOOLONG;

3149 3150
	inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
				VM_NORESERVE);
L
Linus Torvalds 已提交
3151 3152 3153
	if (!inode)
		return -ENOSPC;

3154
	error = security_inode_init_security(inode, dir, &dentry->d_name,
3155
					     shmem_initxattrs, NULL);
3156 3157 3158
	if (error && error != -EOPNOTSUPP) {
		iput(inode);
		return error;
3159 3160
	}

L
Linus Torvalds 已提交
3161
	inode->i_size = len-1;
3162
	if (len <= SHORT_SYMLINK_LEN) {
3163 3164
		inode->i_link = kmemdup(symname, len, GFP_KERNEL);
		if (!inode->i_link) {
3165 3166 3167 3168
			iput(inode);
			return -ENOMEM;
		}
		inode->i_op = &shmem_short_symlink_operations;
L
Linus Torvalds 已提交
3169
	} else {
3170
		inode_nohighmem(inode);
3171
		error = shmem_getpage(inode, 0, &page, SGP_WRITE);
L
Linus Torvalds 已提交
3172 3173 3174 3175
		if (error) {
			iput(inode);
			return error;
		}
3176
		inode->i_mapping->a_ops = &shmem_aops;
L
Linus Torvalds 已提交
3177
		inode->i_op = &shmem_symlink_inode_operations;
3178
		memcpy(page_address(page), symname, len);
3179
		SetPageUptodate(page);
L
Linus Torvalds 已提交
3180
		set_page_dirty(page);
3181
		unlock_page(page);
3182
		put_page(page);
L
Linus Torvalds 已提交
3183 3184
	}
	dir->i_size += BOGO_DIRENT_SIZE;
3185
	dir->i_ctime = dir->i_mtime = current_time(dir);
L
Linus Torvalds 已提交
3186 3187 3188 3189 3190
	d_instantiate(dentry, inode);
	dget(dentry);
	return 0;
}

3191
static void shmem_put_link(void *arg)
L
Linus Torvalds 已提交
3192
{
3193 3194
	mark_page_accessed(arg);
	put_page(arg);
L
Linus Torvalds 已提交
3195 3196
}

3197
static const char *shmem_get_link(struct dentry *dentry,
3198 3199
				  struct inode *inode,
				  struct delayed_call *done)
L
Linus Torvalds 已提交
3200 3201
{
	struct page *page = NULL;
3202
	int error;
3203 3204 3205 3206 3207 3208 3209 3210 3211
	if (!dentry) {
		page = find_get_page(inode->i_mapping, 0);
		if (!page)
			return ERR_PTR(-ECHILD);
		if (!PageUptodate(page)) {
			put_page(page);
			return ERR_PTR(-ECHILD);
		}
	} else {
3212
		error = shmem_getpage(inode, 0, &page, SGP_READ);
3213 3214 3215 3216
		if (error)
			return ERR_PTR(error);
		unlock_page(page);
	}
3217
	set_delayed_call(done, shmem_put_link, page);
3218
	return page_address(page);
L
Linus Torvalds 已提交
3219 3220
}

3221
#ifdef CONFIG_TMPFS_XATTR
3222
/*
3223 3224
 * Superblocks without xattr inode operations may get some security.* xattr
 * support from the LSM "for free". As soon as we have any other xattrs
3225 3226 3227 3228
 * like ACLs, we also need to implement the security.* handlers at
 * filesystem level, though.
 */

3229 3230 3231 3232 3233 3234 3235 3236 3237
/*
 * Callback for security_inode_init_security() for acquiring xattrs.
 */
static int shmem_initxattrs(struct inode *inode,
			    const struct xattr *xattr_array,
			    void *fs_info)
{
	struct shmem_inode_info *info = SHMEM_I(inode);
	const struct xattr *xattr;
3238
	struct simple_xattr *new_xattr;
3239 3240 3241
	size_t len;

	for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3242
		new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3243 3244 3245 3246 3247 3248 3249
		if (!new_xattr)
			return -ENOMEM;

		len = strlen(xattr->name) + 1;
		new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
					  GFP_KERNEL);
		if (!new_xattr->name) {
3250
			kvfree(new_xattr);
3251 3252 3253 3254 3255 3256 3257 3258
			return -ENOMEM;
		}

		memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
		       XATTR_SECURITY_PREFIX_LEN);
		memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
		       xattr->name, len);

3259
		simple_xattr_list_add(&info->xattrs, new_xattr);
3260 3261 3262 3263 3264
	}

	return 0;
}

3265
static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3266 3267
				   struct dentry *unused, struct inode *inode,
				   const char *name, void *buffer, size_t size)
3268
{
3269
	struct shmem_inode_info *info = SHMEM_I(inode);
3270

3271
	name = xattr_full_name(handler, name);
3272
	return simple_xattr_get(&info->xattrs, name, buffer, size);
3273 3274
}

3275
static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3276 3277 3278
				   struct dentry *unused, struct inode *inode,
				   const char *name, const void *value,
				   size_t size, int flags)
3279
{
3280
	struct shmem_inode_info *info = SHMEM_I(inode);
3281

3282
	name = xattr_full_name(handler, name);
3283
	return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
3284 3285
}

3286 3287 3288 3289 3290
static const struct xattr_handler shmem_security_xattr_handler = {
	.prefix = XATTR_SECURITY_PREFIX,
	.get = shmem_xattr_handler_get,
	.set = shmem_xattr_handler_set,
};
3291

3292 3293 3294 3295 3296
static const struct xattr_handler shmem_trusted_xattr_handler = {
	.prefix = XATTR_TRUSTED_PREFIX,
	.get = shmem_xattr_handler_get,
	.set = shmem_xattr_handler_set,
};
3297

3298 3299 3300 3301 3302 3303 3304 3305 3306
static const struct xattr_handler *shmem_xattr_handlers[] = {
#ifdef CONFIG_TMPFS_POSIX_ACL
	&posix_acl_access_xattr_handler,
	&posix_acl_default_xattr_handler,
#endif
	&shmem_security_xattr_handler,
	&shmem_trusted_xattr_handler,
	NULL
};
3307 3308 3309

static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
{
3310
	struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3311
	return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3312 3313 3314
}
#endif /* CONFIG_TMPFS_XATTR */

3315
static const struct inode_operations shmem_short_symlink_operations = {
3316
	.get_link	= simple_get_link,
3317 3318 3319 3320 3321 3322
#ifdef CONFIG_TMPFS_XATTR
	.listxattr	= shmem_listxattr,
#endif
};

static const struct inode_operations shmem_symlink_inode_operations = {
3323
	.get_link	= shmem_get_link,
3324 3325
#ifdef CONFIG_TMPFS_XATTR
	.listxattr	= shmem_listxattr,
3326
#endif
3327
};
3328

3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341
static struct dentry *shmem_get_parent(struct dentry *child)
{
	return ERR_PTR(-ESTALE);
}

static int shmem_match(struct inode *ino, void *vfh)
{
	__u32 *fh = vfh;
	__u64 inum = fh[2];
	inum = (inum << 32) | fh[1];
	return ino->i_ino == inum && fh[0] == ino->i_generation;
}

3342 3343 3344 3345 3346 3347 3348 3349 3350
/* Find any alias of inode, but prefer a hashed alias */
static struct dentry *shmem_find_alias(struct inode *inode)
{
	struct dentry *alias = d_find_alias(inode);

	return alias ?: d_find_any_alias(inode);
}


C
Christoph Hellwig 已提交
3351 3352
static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
		struct fid *fid, int fh_len, int fh_type)
3353 3354
{
	struct inode *inode;
C
Christoph Hellwig 已提交
3355
	struct dentry *dentry = NULL;
3356
	u64 inum;
C
Christoph Hellwig 已提交
3357 3358 3359

	if (fh_len < 3)
		return NULL;
3360

3361 3362 3363
	inum = fid->raw[2];
	inum = (inum << 32) | fid->raw[1];

C
Christoph Hellwig 已提交
3364 3365
	inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
			shmem_match, fid->raw);
3366
	if (inode) {
3367
		dentry = shmem_find_alias(inode);
3368 3369 3370
		iput(inode);
	}

C
Christoph Hellwig 已提交
3371
	return dentry;
3372 3373
}

A
Al Viro 已提交
3374 3375
static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
				struct inode *parent)
3376
{
3377 3378
	if (*len < 3) {
		*len = 3;
3379
		return FILEID_INVALID;
3380
	}
3381

A
Al Viro 已提交
3382
	if (inode_unhashed(inode)) {
3383 3384 3385 3386 3387 3388 3389
		/* Unfortunately insert_inode_hash is not idempotent,
		 * so as we hash inodes here rather than at creation
		 * time, we need a lock to ensure we only try
		 * to do it once
		 */
		static DEFINE_SPINLOCK(lock);
		spin_lock(&lock);
A
Al Viro 已提交
3390
		if (inode_unhashed(inode))
3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403
			__insert_inode_hash(inode,
					    inode->i_ino + inode->i_generation);
		spin_unlock(&lock);
	}

	fh[0] = inode->i_generation;
	fh[1] = inode->i_ino;
	fh[2] = ((__u64)inode->i_ino) >> 32;

	*len = 3;
	return 1;
}

3404
static const struct export_operations shmem_export_ops = {
3405 3406
	.get_parent     = shmem_get_parent,
	.encode_fh      = shmem_encode_fh,
C
Christoph Hellwig 已提交
3407
	.fh_to_dentry	= shmem_fh_to_dentry,
3408 3409
};

3410 3411 3412 3413 3414 3415 3416 3417 3418
enum shmem_param {
	Opt_gid,
	Opt_huge,
	Opt_mode,
	Opt_mpol,
	Opt_nr_blocks,
	Opt_nr_inodes,
	Opt_size,
	Opt_uid,
C
Chris Down 已提交
3419 3420
	Opt_inode32,
	Opt_inode64,
3421 3422
};

3423
static const struct constant_table shmem_param_enums_huge[] = {
A
Al Viro 已提交
3424 3425 3426 3427 3428 3429 3430
	{"never",	SHMEM_HUGE_NEVER },
	{"always",	SHMEM_HUGE_ALWAYS },
	{"within_size",	SHMEM_HUGE_WITHIN_SIZE },
	{"advise",	SHMEM_HUGE_ADVISE },
	{}
};

3431
const struct fs_parameter_spec shmem_fs_parameters[] = {
3432
	fsparam_u32   ("gid",		Opt_gid),
A
Al Viro 已提交
3433
	fsparam_enum  ("huge",		Opt_huge,  shmem_param_enums_huge),
3434 3435 3436 3437 3438 3439
	fsparam_u32oct("mode",		Opt_mode),
	fsparam_string("mpol",		Opt_mpol),
	fsparam_string("nr_blocks",	Opt_nr_blocks),
	fsparam_string("nr_inodes",	Opt_nr_inodes),
	fsparam_string("size",		Opt_size),
	fsparam_u32   ("uid",		Opt_uid),
C
Chris Down 已提交
3440 3441
	fsparam_flag  ("inode32",	Opt_inode32),
	fsparam_flag  ("inode64",	Opt_inode64),
3442 3443 3444
	{}
};

3445
static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
L
Linus Torvalds 已提交
3446
{
3447
	struct shmem_options *ctx = fc->fs_private;
3448 3449
	struct fs_parse_result result;
	unsigned long long size;
3450
	char *rest;
3451 3452
	int opt;

3453
	opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3454
	if (opt < 0)
3455
		return opt;
L
Linus Torvalds 已提交
3456

3457 3458 3459
	switch (opt) {
	case Opt_size:
		size = memparse(param->string, &rest);
3460 3461 3462 3463 3464 3465 3466
		if (*rest == '%') {
			size <<= PAGE_SHIFT;
			size *= totalram_pages();
			do_div(size, 100);
			rest++;
		}
		if (*rest)
3467
			goto bad_value;
3468 3469
		ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
		ctx->seen |= SHMEM_SEEN_BLOCKS;
3470 3471 3472
		break;
	case Opt_nr_blocks:
		ctx->blocks = memparse(param->string, &rest);
3473
		if (*rest)
3474
			goto bad_value;
3475
		ctx->seen |= SHMEM_SEEN_BLOCKS;
3476 3477 3478
		break;
	case Opt_nr_inodes:
		ctx->inodes = memparse(param->string, &rest);
3479
		if (*rest)
3480
			goto bad_value;
3481
		ctx->seen |= SHMEM_SEEN_INODES;
3482 3483 3484 3485 3486 3487
		break;
	case Opt_mode:
		ctx->mode = result.uint_32 & 07777;
		break;
	case Opt_uid:
		ctx->uid = make_kuid(current_user_ns(), result.uint_32);
3488
		if (!uid_valid(ctx->uid))
3489 3490 3491 3492
			goto bad_value;
		break;
	case Opt_gid:
		ctx->gid = make_kgid(current_user_ns(), result.uint_32);
3493
		if (!gid_valid(ctx->gid))
3494 3495 3496 3497 3498
			goto bad_value;
		break;
	case Opt_huge:
		ctx->huge = result.uint_32;
		if (ctx->huge != SHMEM_HUGE_NEVER &&
3499
		    !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3500 3501
		      has_transparent_hugepage()))
			goto unsupported_parameter;
3502
		ctx->seen |= SHMEM_SEEN_HUGE;
3503 3504 3505 3506 3507 3508 3509 3510 3511 3512
		break;
	case Opt_mpol:
		if (IS_ENABLED(CONFIG_NUMA)) {
			mpol_put(ctx->mpol);
			ctx->mpol = NULL;
			if (mpol_parse_str(param->string, &ctx->mpol))
				goto bad_value;
			break;
		}
		goto unsupported_parameter;
C
Chris Down 已提交
3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524
	case Opt_inode32:
		ctx->full_inums = false;
		ctx->seen |= SHMEM_SEEN_INUMS;
		break;
	case Opt_inode64:
		if (sizeof(ino_t) < 8) {
			return invalfc(fc,
				       "Cannot use inode64 with <64bit inums in kernel\n");
		}
		ctx->full_inums = true;
		ctx->seen |= SHMEM_SEEN_INUMS;
		break;
3525 3526 3527
	}
	return 0;

3528
unsupported_parameter:
A
Al Viro 已提交
3529
	return invalfc(fc, "Unsupported parameter '%s'", param->key);
3530
bad_value:
A
Al Viro 已提交
3531
	return invalfc(fc, "Bad value for '%s'", param->key);
3532 3533
}

3534
static int shmem_parse_options(struct fs_context *fc, void *data)
3535
{
3536 3537
	char *options = data;

A
Al Viro 已提交
3538 3539 3540 3541 3542 3543
	if (options) {
		int err = security_sb_eat_lsm_opts(options, &fc->security);
		if (err)
			return err;
	}

3544
	while (options != NULL) {
3545
		char *this_char = options;
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560
		for (;;) {
			/*
			 * NUL-terminate this option: unfortunately,
			 * mount options form a comma-separated list,
			 * but mpol's nodelist may also contain commas.
			 */
			options = strchr(options, ',');
			if (options == NULL)
				break;
			options++;
			if (!isdigit(*options)) {
				options[-1] = '\0';
				break;
			}
		}
3561 3562
		if (*this_char) {
			char *value = strchr(this_char,'=');
3563
			size_t len = 0;
3564 3565 3566 3567
			int err;

			if (value) {
				*value++ = '\0';
3568
				len = strlen(value);
3569
			}
3570 3571 3572
			err = vfs_parse_fs_string(fc, this_char, value, len);
			if (err < 0)
				return err;
L
Linus Torvalds 已提交
3573 3574 3575 3576 3577
		}
	}
	return 0;
}

3578 3579 3580 3581 3582 3583 3584 3585
/*
 * Reconfigure a shmem filesystem.
 *
 * Note that we disallow change from limited->unlimited blocks/inodes while any
 * are in use; but we must separately disallow unlimited->limited, because in
 * that case we have no record of how much is already in use.
 */
static int shmem_reconfigure(struct fs_context *fc)
L
Linus Torvalds 已提交
3586
{
3587 3588
	struct shmem_options *ctx = fc->fs_private;
	struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3589
	unsigned long inodes;
3590
	const char *err;
L
Linus Torvalds 已提交
3591

3592 3593
	spin_lock(&sbinfo->stat_lock);
	inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3594 3595 3596
	if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
		if (!sbinfo->max_blocks) {
			err = "Cannot retroactively limit size";
3597
			goto out;
3598
		}
3599
		if (percpu_counter_compare(&sbinfo->used_blocks,
3600 3601
					   ctx->blocks) > 0) {
			err = "Too small a size for current use";
3602
			goto out;
3603
		}
3604
	}
3605 3606 3607
	if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
		if (!sbinfo->max_inodes) {
			err = "Cannot retroactively limit inodes";
3608
			goto out;
3609 3610 3611
		}
		if (ctx->inodes < inodes) {
			err = "Too few inodes for current use";
3612
			goto out;
3613
		}
3614
	}
3615

C
Chris Down 已提交
3616 3617 3618 3619 3620 3621
	if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
	    sbinfo->next_ino > UINT_MAX) {
		err = "Current inum too high to switch to 32-bit inums";
		goto out;
	}

3622 3623
	if (ctx->seen & SHMEM_SEEN_HUGE)
		sbinfo->huge = ctx->huge;
C
Chris Down 已提交
3624 3625
	if (ctx->seen & SHMEM_SEEN_INUMS)
		sbinfo->full_inums = ctx->full_inums;
3626 3627 3628 3629 3630
	if (ctx->seen & SHMEM_SEEN_BLOCKS)
		sbinfo->max_blocks  = ctx->blocks;
	if (ctx->seen & SHMEM_SEEN_INODES) {
		sbinfo->max_inodes  = ctx->inodes;
		sbinfo->free_inodes = ctx->inodes - inodes;
3631
	}
3632

3633 3634 3635
	/*
	 * Preserve previous mempolicy unless mpol remount option was specified.
	 */
3636
	if (ctx->mpol) {
3637
		mpol_put(sbinfo->mpol);
3638 3639
		sbinfo->mpol = ctx->mpol;	/* transfers initial ref */
		ctx->mpol = NULL;
3640
	}
3641 3642
	spin_unlock(&sbinfo->stat_lock);
	return 0;
3643 3644
out:
	spin_unlock(&sbinfo->stat_lock);
A
Al Viro 已提交
3645
	return invalfc(fc, "%s", err);
L
Linus Torvalds 已提交
3646
}
3647

3648
static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3649
{
3650
	struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3651 3652 3653

	if (sbinfo->max_blocks != shmem_default_max_blocks())
		seq_printf(seq, ",size=%luk",
3654
			sbinfo->max_blocks << (PAGE_SHIFT - 10));
3655 3656
	if (sbinfo->max_inodes != shmem_default_max_inodes())
		seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3657
	if (sbinfo->mode != (0777 | S_ISVTX))
3658
		seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3659 3660 3661 3662 3663 3664
	if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
		seq_printf(seq, ",uid=%u",
				from_kuid_munged(&init_user_ns, sbinfo->uid));
	if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
		seq_printf(seq, ",gid=%u",
				from_kgid_munged(&init_user_ns, sbinfo->gid));
C
Chris Down 已提交
3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687

	/*
	 * Showing inode{64,32} might be useful even if it's the system default,
	 * since then people don't have to resort to checking both here and
	 * /proc/config.gz to confirm 64-bit inums were successfully applied
	 * (which may not even exist if IKCONFIG_PROC isn't enabled).
	 *
	 * We hide it when inode64 isn't the default and we are using 32-bit
	 * inodes, since that probably just means the feature isn't even under
	 * consideration.
	 *
	 * As such:
	 *
	 *                     +-----------------+-----------------+
	 *                     | TMPFS_INODE64=y | TMPFS_INODE64=n |
	 *  +------------------+-----------------+-----------------+
	 *  | full_inums=true  | show            | show            |
	 *  | full_inums=false | show            | hide            |
	 *  +------------------+-----------------+-----------------+
	 *
	 */
	if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
		seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
3688
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3689 3690 3691 3692
	/* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
	if (sbinfo->huge)
		seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
#endif
3693
	shmem_show_mpol(seq, sbinfo->mpol);
3694 3695
	return 0;
}
D
David Herrmann 已提交
3696

3697
#endif /* CONFIG_TMPFS */
L
Linus Torvalds 已提交
3698 3699 3700

static void shmem_put_super(struct super_block *sb)
{
3701 3702
	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);

C
Chris Down 已提交
3703
	free_percpu(sbinfo->ino_batch);
3704
	percpu_counter_destroy(&sbinfo->used_blocks);
G
Greg Thelen 已提交
3705
	mpol_put(sbinfo->mpol);
3706
	kfree(sbinfo);
L
Linus Torvalds 已提交
3707 3708 3709
	sb->s_fs_info = NULL;
}

3710
static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
L
Linus Torvalds 已提交
3711
{
3712
	struct shmem_options *ctx = fc->fs_private;
L
Linus Torvalds 已提交
3713
	struct inode *inode;
3714
	struct shmem_sb_info *sbinfo;
3715 3716 3717
	int err = -ENOMEM;

	/* Round up to L1_CACHE_BYTES to resist false sharing */
3718
	sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3719 3720 3721 3722 3723
				L1_CACHE_BYTES), GFP_KERNEL);
	if (!sbinfo)
		return -ENOMEM;

	sb->s_fs_info = sbinfo;
L
Linus Torvalds 已提交
3724

3725
#ifdef CONFIG_TMPFS
L
Linus Torvalds 已提交
3726 3727 3728 3729 3730
	/*
	 * Per default we only allow half of the physical ram per
	 * tmpfs instance, limiting inodes to one per page of lowmem;
	 * but the internal instance is left unlimited.
	 */
3731
	if (!(sb->s_flags & SB_KERNMOUNT)) {
3732 3733 3734 3735
		if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
			ctx->blocks = shmem_default_max_blocks();
		if (!(ctx->seen & SHMEM_SEEN_INODES))
			ctx->inodes = shmem_default_max_inodes();
C
Chris Down 已提交
3736 3737
		if (!(ctx->seen & SHMEM_SEEN_INUMS))
			ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
3738
	} else {
3739
		sb->s_flags |= SB_NOUSER;
L
Linus Torvalds 已提交
3740
	}
3741
	sb->s_export_op = &shmem_export_ops;
3742
	sb->s_flags |= SB_NOSEC;
L
Linus Torvalds 已提交
3743
#else
3744
	sb->s_flags |= SB_NOUSER;
L
Linus Torvalds 已提交
3745
#endif
3746 3747
	sbinfo->max_blocks = ctx->blocks;
	sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes;
C
Chris Down 已提交
3748 3749 3750 3751 3752
	if (sb->s_flags & SB_KERNMOUNT) {
		sbinfo->ino_batch = alloc_percpu(ino_t);
		if (!sbinfo->ino_batch)
			goto failed;
	}
3753 3754
	sbinfo->uid = ctx->uid;
	sbinfo->gid = ctx->gid;
C
Chris Down 已提交
3755
	sbinfo->full_inums = ctx->full_inums;
3756 3757 3758 3759
	sbinfo->mode = ctx->mode;
	sbinfo->huge = ctx->huge;
	sbinfo->mpol = ctx->mpol;
	ctx->mpol = NULL;
L
Linus Torvalds 已提交
3760

3761
	spin_lock_init(&sbinfo->stat_lock);
3762
	if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3763
		goto failed;
3764 3765
	spin_lock_init(&sbinfo->shrinklist_lock);
	INIT_LIST_HEAD(&sbinfo->shrinklist);
3766

3767
	sb->s_maxbytes = MAX_LFS_FILESIZE;
3768 3769
	sb->s_blocksize = PAGE_SIZE;
	sb->s_blocksize_bits = PAGE_SHIFT;
L
Linus Torvalds 已提交
3770 3771
	sb->s_magic = TMPFS_MAGIC;
	sb->s_op = &shmem_ops;
3772
	sb->s_time_gran = 1;
3773
#ifdef CONFIG_TMPFS_XATTR
3774
	sb->s_xattr = shmem_xattr_handlers;
3775 3776
#endif
#ifdef CONFIG_TMPFS_POSIX_ACL
3777
	sb->s_flags |= SB_POSIXACL;
3778
#endif
3779
	uuid_gen(&sb->s_uuid);
3780

3781
	inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
L
Linus Torvalds 已提交
3782 3783
	if (!inode)
		goto failed;
3784 3785
	inode->i_uid = sbinfo->uid;
	inode->i_gid = sbinfo->gid;
A
Al Viro 已提交
3786 3787
	sb->s_root = d_make_root(inode);
	if (!sb->s_root)
3788
		goto failed;
L
Linus Torvalds 已提交
3789 3790 3791 3792 3793 3794 3795
	return 0;

failed:
	shmem_put_super(sb);
	return err;
}

3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820
static int shmem_get_tree(struct fs_context *fc)
{
	return get_tree_nodev(fc, shmem_fill_super);
}

static void shmem_free_fc(struct fs_context *fc)
{
	struct shmem_options *ctx = fc->fs_private;

	if (ctx) {
		mpol_put(ctx->mpol);
		kfree(ctx);
	}
}

static const struct fs_context_operations shmem_fs_context_ops = {
	.free			= shmem_free_fc,
	.get_tree		= shmem_get_tree,
#ifdef CONFIG_TMPFS
	.parse_monolithic	= shmem_parse_options,
	.parse_param		= shmem_parse_one,
	.reconfigure		= shmem_reconfigure,
#endif
};

3821
static struct kmem_cache *shmem_inode_cachep;
L
Linus Torvalds 已提交
3822 3823 3824

static struct inode *shmem_alloc_inode(struct super_block *sb)
{
3825 3826 3827
	struct shmem_inode_info *info;
	info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
	if (!info)
L
Linus Torvalds 已提交
3828
		return NULL;
3829
	return &info->vfs_inode;
L
Linus Torvalds 已提交
3830 3831
}

A
Al Viro 已提交
3832
static void shmem_free_in_core_inode(struct inode *inode)
N
Nick Piggin 已提交
3833
{
A
Al Viro 已提交
3834 3835
	if (S_ISLNK(inode->i_mode))
		kfree(inode->i_link);
N
Nick Piggin 已提交
3836 3837 3838
	kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
}

L
Linus Torvalds 已提交
3839 3840
static void shmem_destroy_inode(struct inode *inode)
{
3841
	if (S_ISREG(inode->i_mode))
L
Linus Torvalds 已提交
3842 3843 3844
		mpol_free_shared_policy(&SHMEM_I(inode)->policy);
}

3845
static void shmem_init_inode(void *foo)
L
Linus Torvalds 已提交
3846
{
3847 3848
	struct shmem_inode_info *info = foo;
	inode_init_once(&info->vfs_inode);
L
Linus Torvalds 已提交
3849 3850
}

3851
static void shmem_init_inodecache(void)
L
Linus Torvalds 已提交
3852 3853 3854
{
	shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
				sizeof(struct shmem_inode_info),
3855
				0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
L
Linus Torvalds 已提交
3856 3857
}

3858
static void shmem_destroy_inodecache(void)
L
Linus Torvalds 已提交
3859
{
3860
	kmem_cache_destroy(shmem_inode_cachep);
L
Linus Torvalds 已提交
3861 3862
}

3863
const struct address_space_operations shmem_aops = {
L
Linus Torvalds 已提交
3864
	.writepage	= shmem_writepage,
3865
	.set_page_dirty	= __set_page_dirty_no_writeback,
L
Linus Torvalds 已提交
3866
#ifdef CONFIG_TMPFS
N
Nick Piggin 已提交
3867 3868
	.write_begin	= shmem_write_begin,
	.write_end	= shmem_write_end,
L
Linus Torvalds 已提交
3869
#endif
3870
#ifdef CONFIG_MIGRATION
3871
	.migratepage	= migrate_page,
3872
#endif
3873
	.error_remove_page = generic_error_remove_page,
L
Linus Torvalds 已提交
3874
};
3875
EXPORT_SYMBOL(shmem_aops);
L
Linus Torvalds 已提交
3876

3877
static const struct file_operations shmem_file_operations = {
L
Linus Torvalds 已提交
3878
	.mmap		= shmem_mmap,
3879
	.get_unmapped_area = shmem_get_unmapped_area,
L
Linus Torvalds 已提交
3880
#ifdef CONFIG_TMPFS
3881
	.llseek		= shmem_file_llseek,
A
Al Viro 已提交
3882
	.read_iter	= shmem_file_read_iter,
3883
	.write_iter	= generic_file_write_iter,
3884
	.fsync		= noop_fsync,
3885
	.splice_read	= generic_file_splice_read,
3886
	.splice_write	= iter_file_splice_write,
3887
	.fallocate	= shmem_fallocate,
L
Linus Torvalds 已提交
3888 3889 3890
#endif
};

3891
static const struct inode_operations shmem_inode_operations = {
3892
	.getattr	= shmem_getattr,
3893
	.setattr	= shmem_setattr,
3894 3895
#ifdef CONFIG_TMPFS_XATTR
	.listxattr	= shmem_listxattr,
C
Christoph Hellwig 已提交
3896
	.set_acl	= simple_set_acl,
3897
#endif
L
Linus Torvalds 已提交
3898 3899
};

3900
static const struct inode_operations shmem_dir_inode_operations = {
L
Linus Torvalds 已提交
3901 3902 3903 3904 3905 3906 3907 3908 3909
#ifdef CONFIG_TMPFS
	.create		= shmem_create,
	.lookup		= simple_lookup,
	.link		= shmem_link,
	.unlink		= shmem_unlink,
	.symlink	= shmem_symlink,
	.mkdir		= shmem_mkdir,
	.rmdir		= shmem_rmdir,
	.mknod		= shmem_mknod,
3910
	.rename		= shmem_rename2,
3911
	.tmpfile	= shmem_tmpfile,
L
Linus Torvalds 已提交
3912
#endif
3913 3914 3915
#ifdef CONFIG_TMPFS_XATTR
	.listxattr	= shmem_listxattr,
#endif
3916
#ifdef CONFIG_TMPFS_POSIX_ACL
3917
	.setattr	= shmem_setattr,
C
Christoph Hellwig 已提交
3918
	.set_acl	= simple_set_acl,
3919 3920 3921
#endif
};

3922
static const struct inode_operations shmem_special_inode_operations = {
3923 3924 3925
#ifdef CONFIG_TMPFS_XATTR
	.listxattr	= shmem_listxattr,
#endif
3926
#ifdef CONFIG_TMPFS_POSIX_ACL
3927
	.setattr	= shmem_setattr,
C
Christoph Hellwig 已提交
3928
	.set_acl	= simple_set_acl,
3929
#endif
L
Linus Torvalds 已提交
3930 3931
};

3932
static const struct super_operations shmem_ops = {
L
Linus Torvalds 已提交
3933
	.alloc_inode	= shmem_alloc_inode,
A
Al Viro 已提交
3934
	.free_inode	= shmem_free_in_core_inode,
L
Linus Torvalds 已提交
3935 3936 3937
	.destroy_inode	= shmem_destroy_inode,
#ifdef CONFIG_TMPFS
	.statfs		= shmem_statfs,
3938
	.show_options	= shmem_show_options,
L
Linus Torvalds 已提交
3939
#endif
A
Al Viro 已提交
3940
	.evict_inode	= shmem_evict_inode,
L
Linus Torvalds 已提交
3941 3942
	.drop_inode	= generic_delete_inode,
	.put_super	= shmem_put_super,
3943
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3944 3945 3946
	.nr_cached_objects	= shmem_unused_huge_count,
	.free_cached_objects	= shmem_unused_huge_scan,
#endif
L
Linus Torvalds 已提交
3947 3948
};

3949
static const struct vm_operations_struct shmem_vm_ops = {
3950
	.fault		= shmem_fault,
3951
	.map_pages	= filemap_map_pages,
L
Linus Torvalds 已提交
3952 3953 3954 3955 3956 3957
#ifdef CONFIG_NUMA
	.set_policy     = shmem_set_policy,
	.get_policy     = shmem_get_policy,
#endif
};

3958
int shmem_init_fs_context(struct fs_context *fc)
L
Linus Torvalds 已提交
3959
{
3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972
	struct shmem_options *ctx;

	ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
	if (!ctx)
		return -ENOMEM;

	ctx->mode = 0777 | S_ISVTX;
	ctx->uid = current_fsuid();
	ctx->gid = current_fsgid();

	fc->fs_private = ctx;
	fc->ops = &shmem_fs_context_ops;
	return 0;
L
Linus Torvalds 已提交
3973 3974
}

3975
static struct file_system_type shmem_fs_type = {
L
Linus Torvalds 已提交
3976 3977
	.owner		= THIS_MODULE,
	.name		= "tmpfs",
3978 3979
	.init_fs_context = shmem_init_fs_context,
#ifdef CONFIG_TMPFS
3980
	.parameters	= shmem_fs_parameters,
3981
#endif
L
Linus Torvalds 已提交
3982
	.kill_sb	= kill_litter_super,
3983
	.fs_flags	= FS_USERNS_MOUNT | FS_THP_SUPPORT,
L
Linus Torvalds 已提交
3984 3985
};

3986
int __init shmem_init(void)
L
Linus Torvalds 已提交
3987 3988 3989
{
	int error;

3990
	shmem_init_inodecache();
L
Linus Torvalds 已提交
3991

3992
	error = register_filesystem(&shmem_fs_type);
L
Linus Torvalds 已提交
3993
	if (error) {
3994
		pr_err("Could not register tmpfs\n");
L
Linus Torvalds 已提交
3995 3996
		goto out2;
	}
3997

3998
	shm_mnt = kern_mount(&shmem_fs_type);
L
Linus Torvalds 已提交
3999 4000
	if (IS_ERR(shm_mnt)) {
		error = PTR_ERR(shm_mnt);
4001
		pr_err("Could not kern_mount tmpfs\n");
L
Linus Torvalds 已提交
4002 4003
		goto out1;
	}
4004

4005
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4006
	if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4007 4008 4009 4010
		SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
	else
		shmem_huge = 0; /* just in case it was patched */
#endif
L
Linus Torvalds 已提交
4011 4012 4013
	return 0;

out1:
4014
	unregister_filesystem(&shmem_fs_type);
L
Linus Torvalds 已提交
4015
out2:
4016
	shmem_destroy_inodecache();
L
Linus Torvalds 已提交
4017 4018 4019
	shm_mnt = ERR_PTR(error);
	return error;
}
4020

4021
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
4022 4023 4024
static ssize_t shmem_enabled_show(struct kobject *kobj,
		struct kobj_attribute *attr, char *buf)
{
4025
	static const int values[] = {
4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065
		SHMEM_HUGE_ALWAYS,
		SHMEM_HUGE_WITHIN_SIZE,
		SHMEM_HUGE_ADVISE,
		SHMEM_HUGE_NEVER,
		SHMEM_HUGE_DENY,
		SHMEM_HUGE_FORCE,
	};
	int i, count;

	for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) {
		const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s ";

		count += sprintf(buf + count, fmt,
				shmem_format_huge(values[i]));
	}
	buf[count - 1] = '\n';
	return count;
}

static ssize_t shmem_enabled_store(struct kobject *kobj,
		struct kobj_attribute *attr, const char *buf, size_t count)
{
	char tmp[16];
	int huge;

	if (count + 1 > sizeof(tmp))
		return -EINVAL;
	memcpy(tmp, buf, count);
	tmp[count] = '\0';
	if (count && tmp[count - 1] == '\n')
		tmp[count - 1] = '\0';

	huge = shmem_parse_huge(tmp);
	if (huge == -EINVAL)
		return -EINVAL;
	if (!has_transparent_hugepage() &&
			huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
		return -EINVAL;

	shmem_huge = huge;
4066
	if (shmem_huge > SHMEM_HUGE_DENY)
4067 4068 4069 4070 4071 4072
		SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
	return count;
}

struct kobj_attribute shmem_enabled_attr =
	__ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store);
4073
#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
4074

4075
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4076 4077 4078 4079 4080 4081 4082
bool shmem_huge_enabled(struct vm_area_struct *vma)
{
	struct inode *inode = file_inode(vma->vm_file);
	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
	loff_t i_size;
	pgoff_t off;

4083 4084 4085
	if ((vma->vm_flags & VM_NOHUGEPAGE) ||
	    test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
		return false;
4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100
	if (shmem_huge == SHMEM_HUGE_FORCE)
		return true;
	if (shmem_huge == SHMEM_HUGE_DENY)
		return false;
	switch (sbinfo->huge) {
		case SHMEM_HUGE_NEVER:
			return false;
		case SHMEM_HUGE_ALWAYS:
			return true;
		case SHMEM_HUGE_WITHIN_SIZE:
			off = round_up(vma->vm_pgoff, HPAGE_PMD_NR);
			i_size = round_up(i_size_read(inode), PAGE_SIZE);
			if (i_size >= HPAGE_PMD_SIZE &&
					i_size >> PAGE_SHIFT >= off)
				return true;
J
Joe Perches 已提交
4101
			fallthrough;
4102 4103 4104 4105 4106 4107 4108 4109
		case SHMEM_HUGE_ADVISE:
			/* TODO: implement fadvise() hints */
			return (vma->vm_flags & VM_HUGEPAGE);
		default:
			VM_BUG_ON(1);
			return false;
	}
}
4110
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
4111

4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122
#else /* !CONFIG_SHMEM */

/*
 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
 *
 * This is intended for small system where the benefits of the full
 * shmem code (swap-backed and resource-limited) are outweighed by
 * their complexity. On systems without swap this code should be
 * effectively equivalent, but much lighter weight.
 */

4123
static struct file_system_type shmem_fs_type = {
4124
	.name		= "tmpfs",
4125
	.init_fs_context = ramfs_init_fs_context,
4126
	.parameters	= ramfs_fs_parameters,
4127
	.kill_sb	= kill_litter_super,
4128
	.fs_flags	= FS_USERNS_MOUNT,
4129 4130
};

4131
int __init shmem_init(void)
4132
{
4133
	BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4134

4135
	shm_mnt = kern_mount(&shmem_fs_type);
4136 4137 4138 4139 4140
	BUG_ON(IS_ERR(shm_mnt));

	return 0;
}

4141 4142
int shmem_unuse(unsigned int type, bool frontswap,
		unsigned long *fs_pages_to_unuse)
4143 4144 4145 4146
{
	return 0;
}

H
Hugh Dickins 已提交
4147 4148 4149 4150 4151
int shmem_lock(struct file *file, int lock, struct user_struct *user)
{
	return 0;
}

4152 4153 4154 4155
void shmem_unlock_mapping(struct address_space *mapping)
{
}

4156 4157 4158 4159 4160 4161 4162 4163 4164
#ifdef CONFIG_MMU
unsigned long shmem_get_unmapped_area(struct file *file,
				      unsigned long addr, unsigned long len,
				      unsigned long pgoff, unsigned long flags)
{
	return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
}
#endif

4165
void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4166
{
4167
	truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4168 4169 4170
}
EXPORT_SYMBOL_GPL(shmem_truncate_range);

4171 4172
#define shmem_vm_ops				generic_file_vm_ops
#define shmem_file_operations			ramfs_file_operations
4173
#define shmem_get_inode(sb, dir, mode, dev, flags)	ramfs_get_inode(sb, dir, mode, dev)
4174 4175
#define shmem_acct_size(flags, size)		0
#define shmem_unacct_size(flags, size)		do {} while (0)
4176 4177 4178 4179

#endif /* CONFIG_SHMEM */

/* common code */
L
Linus Torvalds 已提交
4180

4181
static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4182
				       unsigned long flags, unsigned int i_flags)
L
Linus Torvalds 已提交
4183 4184
{
	struct inode *inode;
4185
	struct file *res;
L
Linus Torvalds 已提交
4186

4187 4188
	if (IS_ERR(mnt))
		return ERR_CAST(mnt);
L
Linus Torvalds 已提交
4189

4190
	if (size < 0 || size > MAX_LFS_FILESIZE)
L
Linus Torvalds 已提交
4191 4192 4193 4194 4195
		return ERR_PTR(-EINVAL);

	if (shmem_acct_size(flags, size))
		return ERR_PTR(-ENOMEM);

4196 4197
	inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0,
				flags);
4198 4199 4200 4201
	if (unlikely(!inode)) {
		shmem_unacct_size(flags, size);
		return ERR_PTR(-ENOSPC);
	}
4202
	inode->i_flags |= i_flags;
L
Linus Torvalds 已提交
4203
	inode->i_size = size;
4204
	clear_nlink(inode);	/* It is unlinked */
A
Al Viro 已提交
4205
	res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4206 4207 4208
	if (!IS_ERR(res))
		res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
				&shmem_file_operations);
A
Al Viro 已提交
4209
	if (IS_ERR(res))
4210
		iput(inode);
A
Al Viro 已提交
4211
	return res;
L
Linus Torvalds 已提交
4212
}
4213 4214 4215 4216 4217

/**
 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
 * 	kernel internal.  There will be NO LSM permission checks against the
 * 	underlying inode.  So users of this interface must do LSM checks at a
4218 4219
 *	higher layer.  The users are the big_key and shm implementations.  LSM
 *	checks are provided at the key or shm level rather than the inode.
4220 4221 4222 4223 4224 4225
 * @name: name for dentry (to be seen in /proc/<pid>/maps
 * @size: size to be set for the file
 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
 */
struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
{
4226
	return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4227 4228 4229 4230 4231 4232 4233 4234 4235 4236
}

/**
 * shmem_file_setup - get an unlinked file living in tmpfs
 * @name: name for dentry (to be seen in /proc/<pid>/maps
 * @size: size to be set for the file
 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
 */
struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
{
4237
	return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4238
}
4239
EXPORT_SYMBOL_GPL(shmem_file_setup);
L
Linus Torvalds 已提交
4240

4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254
/**
 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
 * @mnt: the tmpfs mount where the file will be created
 * @name: name for dentry (to be seen in /proc/<pid>/maps
 * @size: size to be set for the file
 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
 */
struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
				       loff_t size, unsigned long flags)
{
	return __shmem_file_setup(mnt, name, size, flags, 0);
}
EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);

4255
/**
L
Linus Torvalds 已提交
4256
 * shmem_zero_setup - setup a shared anonymous mapping
4257
 * @vma: the vma to be mmapped is prepared by do_mmap
L
Linus Torvalds 已提交
4258 4259 4260 4261 4262 4263
 */
int shmem_zero_setup(struct vm_area_struct *vma)
{
	struct file *file;
	loff_t size = vma->vm_end - vma->vm_start;

4264
	/*
4265
	 * Cloning a new file under mmap_lock leads to a lock ordering conflict
4266 4267 4268 4269
	 * between XFS directory reading and selinux: since this file is only
	 * accessible to the user through its mapping, use S_PRIVATE flag to
	 * bypass file security, in the same way as shmem_kernel_file_setup().
	 */
4270
	file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
L
Linus Torvalds 已提交
4271 4272 4273 4274 4275 4276 4277
	if (IS_ERR(file))
		return PTR_ERR(file);

	if (vma->vm_file)
		fput(vma->vm_file);
	vma->vm_file = file;
	vma->vm_ops = &shmem_vm_ops;
4278

4279
	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
4280 4281 4282 4283 4284
			((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
			(vma->vm_end & HPAGE_PMD_MASK)) {
		khugepaged_enter(vma, vma->vm_flags);
	}

L
Linus Torvalds 已提交
4285 4286
	return 0;
}
4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299

/**
 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
 * @mapping:	the page's address_space
 * @index:	the page index
 * @gfp:	the page allocator flags to use if allocating
 *
 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
 * with any new page allocations done using the specified allocation flags.
 * But read_cache_page_gfp() uses the ->readpage() method: which does not
 * suit tmpfs, since it may have pages in swapcache, and needs to find those
 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
 *
4300 4301
 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4302 4303 4304 4305
 */
struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
					 pgoff_t index, gfp_t gfp)
{
4306 4307
#ifdef CONFIG_SHMEM
	struct inode *inode = mapping->host;
H
Hugh Dickins 已提交
4308
	struct page *page;
4309 4310
	int error;

4311
	BUG_ON(!shmem_mapping(mapping));
4312
	error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
4313
				  gfp, NULL, NULL, NULL);
4314 4315 4316 4317 4318 4319 4320 4321 4322
	if (error)
		page = ERR_PTR(error);
	else
		unlock_page(page);
	return page;
#else
	/*
	 * The tiny !SHMEM case uses ramfs without swap
	 */
4323
	return read_cache_page_gfp(mapping, index, gfp);
4324
#endif
4325 4326
}
EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);