mmap.c 99.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 * mm/mmap.c
 *
 * Written by obz.
 *
A
Alan Cox 已提交
6
 * Address space accounting code	<alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
7 8
 */

9 10
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

11
#include <linux/kernel.h>
L
Linus Torvalds 已提交
12
#include <linux/slab.h>
A
Alexey Dobriyan 已提交
13
#include <linux/backing-dev.h>
L
Linus Torvalds 已提交
14
#include <linux/mm.h>
D
Davidlohr Bueso 已提交
15
#include <linux/vmacache.h>
L
Linus Torvalds 已提交
16 17 18 19 20
#include <linux/shm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/syscalls.h>
21
#include <linux/capability.h>
L
Linus Torvalds 已提交
22 23 24 25 26 27
#include <linux/init.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/personality.h>
#include <linux/security.h>
#include <linux/hugetlb.h>
28
#include <linux/shmem_fs.h>
L
Linus Torvalds 已提交
29
#include <linux/profile.h>
30
#include <linux/export.h>
L
Linus Torvalds 已提交
31 32 33
#include <linux/mount.h>
#include <linux/mempolicy.h>
#include <linux/rmap.h>
A
Andrea Arcangeli 已提交
34
#include <linux/mmu_notifier.h>
35
#include <linux/mmdebug.h>
36
#include <linux/perf_event.h>
A
Al Viro 已提交
37
#include <linux/audit.h>
A
Andrea Arcangeli 已提交
38
#include <linux/khugepaged.h>
39
#include <linux/uprobes.h>
40
#include <linux/rbtree_augmented.h>
41 42
#include <linux/notifier.h>
#include <linux/memory.h>
43
#include <linux/printk.h>
44
#include <linux/userfaultfd_k.h>
45
#include <linux/moduleparam.h>
46
#include <linux/pkeys.h>
47
#include <linux/oom.h>
L
Linus Torvalds 已提交
48

49
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
50 51
#include <asm/cacheflush.h>
#include <asm/tlb.h>
52
#include <asm/mmu_context.h>
L
Linus Torvalds 已提交
53

54 55
#include "internal.h"

56 57 58 59
#ifndef arch_mmap_check
#define arch_mmap_check(addr, len, flags)	(0)
#endif

60 61 62 63 64 65 66 67 68 69 70
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX;
int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
#endif
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
#endif

71
static bool ignore_rlimit_data;
72
core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
73

74 75 76 77
static void unmap_region(struct mm_struct *mm,
		struct vm_area_struct *vma, struct vm_area_struct *prev,
		unsigned long start, unsigned long end);

L
Linus Torvalds 已提交
78 79 80 81 82 83 84 85 86
/* description of effects of mapping type and prot in current implementation.
 * this is due to the limited x86 page protection hardware.  The expected
 * behavior is in parens:
 *
 * map_type	prot
 *		PROT_NONE	PROT_READ	PROT_WRITE	PROT_EXEC
 * MAP_SHARED	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
 *		w: (no) no	w: (no) no	w: (yes) yes	w: (no) no
 *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
V
vishnu.ps 已提交
87
 *
L
Linus Torvalds 已提交
88 89 90 91
 * MAP_PRIVATE	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
 *		w: (no) no	w: (no) no	w: (copy) copy	w: (no) no
 *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
 *
92 93 94 95 96
 * On arm64, PROT_EXEC has the following behaviour for both MAP_SHARED and
 * MAP_PRIVATE:
 *								r: (no) no
 *								w: (no) no
 *								x: (yes) yes
L
Linus Torvalds 已提交
97
 */
98
pgprot_t protection_map[16] __ro_after_init = {
L
Linus Torvalds 已提交
99 100 101 102
	__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
	__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
};

103 104 105 106 107 108 109
#ifndef CONFIG_ARCH_HAS_FILTER_PGPROT
static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
{
	return prot;
}
#endif

110 111
pgprot_t vm_get_page_prot(unsigned long vm_flags)
{
112
	pgprot_t ret = __pgprot(pgprot_val(protection_map[vm_flags &
113 114
				(VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
			pgprot_val(arch_vm_get_page_prot(vm_flags)));
115 116

	return arch_filter_pgprot(ret);
117 118 119
}
EXPORT_SYMBOL(vm_get_page_prot);

120 121 122 123 124 125 126 127 128
static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
{
	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
}

/* Update vma->vm_page_prot to reflect vma->vm_flags. */
void vma_set_page_prot(struct vm_area_struct *vma)
{
	unsigned long vm_flags = vma->vm_flags;
129
	pgprot_t vm_page_prot;
130

131 132
	vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
	if (vma_wants_writenotify(vma, vm_page_prot)) {
133
		vm_flags &= ~VM_SHARED;
134
		vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags);
135
	}
136 137
	/* remove_protection_ptes reads vma->vm_page_prot without mmap_sem */
	WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
138 139
}

L
Linus Torvalds 已提交
140
/*
141
 * Requires inode->i_mapping->i_mmap_rwsem
L
Linus Torvalds 已提交
142 143 144 145 146
 */
static void __remove_shared_vm_struct(struct vm_area_struct *vma,
		struct file *file, struct address_space *mapping)
{
	if (vma->vm_flags & VM_DENYWRITE)
A
Al Viro 已提交
147
		atomic_inc(&file_inode(file)->i_writecount);
L
Linus Torvalds 已提交
148
	if (vma->vm_flags & VM_SHARED)
149
		mapping_unmap_writable(mapping);
L
Linus Torvalds 已提交
150 151

	flush_dcache_mmap_lock(mapping);
152
	vma_interval_tree_remove(vma, &mapping->i_mmap);
L
Linus Torvalds 已提交
153 154 155 156
	flush_dcache_mmap_unlock(mapping);
}

/*
157
 * Unlink a file-based vm structure from its interval tree, to hide
158
 * vma from rmap and vmtruncate before freeing its page tables.
L
Linus Torvalds 已提交
159
 */
160
void unlink_file_vma(struct vm_area_struct *vma)
L
Linus Torvalds 已提交
161 162 163 164 165
{
	struct file *file = vma->vm_file;

	if (file) {
		struct address_space *mapping = file->f_mapping;
166
		i_mmap_lock_write(mapping);
L
Linus Torvalds 已提交
167
		__remove_shared_vm_struct(vma, file, mapping);
168
		i_mmap_unlock_write(mapping);
L
Linus Torvalds 已提交
169
	}
170 171 172 173 174 175 176 177 178 179
}

/*
 * Close a vm structure and free it, returning the next.
 */
static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
{
	struct vm_area_struct *next = vma->vm_next;

	might_sleep();
L
Linus Torvalds 已提交
180 181
	if (vma->vm_ops && vma->vm_ops->close)
		vma->vm_ops->close(vma);
182
	if (vma->vm_file)
183
		fput(vma->vm_file);
184
	mpol_put(vma_policy(vma));
L
Linus Torvalds 已提交
185
	kmem_cache_free(vm_area_cachep, vma);
186
	return next;
L
Linus Torvalds 已提交
187 188
}

189
static int do_brk(unsigned long addr, unsigned long len, struct list_head *uf);
190

191
SYSCALL_DEFINE1(brk, unsigned long, brk)
L
Linus Torvalds 已提交
192
{
C
Cyrill Gorcunov 已提交
193
	unsigned long retval;
L
Linus Torvalds 已提交
194 195
	unsigned long newbrk, oldbrk;
	struct mm_struct *mm = current->mm;
196
	struct vm_area_struct *next;
197
	unsigned long min_brk;
198
	bool populate;
199
	LIST_HEAD(uf);
L
Linus Torvalds 已提交
200

201 202
	if (down_write_killable(&mm->mmap_sem))
		return -EINTR;
L
Linus Torvalds 已提交
203

204
#ifdef CONFIG_COMPAT_BRK
205 206 207 208 209
	/*
	 * CONFIG_COMPAT_BRK can still be overridden by setting
	 * randomize_va_space to 2, which will still cause mm->start_brk
	 * to be arbitrarily shifted
	 */
210
	if (current->brk_randomized)
211 212 213
		min_brk = mm->start_brk;
	else
		min_brk = mm->end_data;
214 215 216 217
#else
	min_brk = mm->start_brk;
#endif
	if (brk < min_brk)
L
Linus Torvalds 已提交
218
		goto out;
R
Ram Gupta 已提交
219 220 221 222 223 224 225

	/*
	 * Check against rlimit here. If this check is done later after the test
	 * of oldbrk with newbrk then it can escape the test and let the data
	 * segment grow beyond its set limit the in case where the limit is
	 * not page aligned -Ram Gupta
	 */
C
Cyrill Gorcunov 已提交
226 227
	if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
			      mm->end_data, mm->start_data))
R
Ram Gupta 已提交
228 229
		goto out;

L
Linus Torvalds 已提交
230 231 232 233 234 235 236
	newbrk = PAGE_ALIGN(brk);
	oldbrk = PAGE_ALIGN(mm->brk);
	if (oldbrk == newbrk)
		goto set_brk;

	/* Always allow shrinking brk. */
	if (brk <= mm->brk) {
237
		if (!do_munmap(mm, newbrk, oldbrk-newbrk, &uf))
L
Linus Torvalds 已提交
238 239 240 241 242
			goto set_brk;
		goto out;
	}

	/* Check against existing mmap mappings. */
243 244
	next = find_vma(mm, oldbrk);
	if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
L
Linus Torvalds 已提交
245 246 247
		goto out;

	/* Ok, looks good - let it rip. */
248
	if (do_brk(oldbrk, newbrk-oldbrk, &uf) < 0)
L
Linus Torvalds 已提交
249
		goto out;
250

L
Linus Torvalds 已提交
251 252
set_brk:
	mm->brk = brk;
253 254
	populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
	up_write(&mm->mmap_sem);
255
	userfaultfd_unmap_complete(mm, &uf);
256 257 258 259
	if (populate)
		mm_populate(oldbrk, newbrk - oldbrk);
	return brk;

L
Linus Torvalds 已提交
260 261 262 263 264 265
out:
	retval = mm->brk;
	up_write(&mm->mmap_sem);
	return retval;
}

266 267
static long vma_compute_subtree_gap(struct vm_area_struct *vma)
{
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
	unsigned long max, prev_end, subtree_gap;

	/*
	 * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we
	 * allow two stack_guard_gaps between them here, and when choosing
	 * an unmapped area; whereas when expanding we only require one.
	 * That's a little inconsistent, but keeps the code here simpler.
	 */
	max = vm_start_gap(vma);
	if (vma->vm_prev) {
		prev_end = vm_end_gap(vma->vm_prev);
		if (max > prev_end)
			max -= prev_end;
		else
			max = 0;
	}
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
	if (vma->vm_rb.rb_left) {
		subtree_gap = rb_entry(vma->vm_rb.rb_left,
				struct vm_area_struct, vm_rb)->rb_subtree_gap;
		if (subtree_gap > max)
			max = subtree_gap;
	}
	if (vma->vm_rb.rb_right) {
		subtree_gap = rb_entry(vma->vm_rb.rb_right,
				struct vm_area_struct, vm_rb)->rb_subtree_gap;
		if (subtree_gap > max)
			max = subtree_gap;
	}
	return max;
}

299
#ifdef CONFIG_DEBUG_VM_RB
300
static int browse_rb(struct mm_struct *mm)
L
Linus Torvalds 已提交
301
{
302
	struct rb_root *root = &mm->mm_rb;
303
	int i = 0, j, bug = 0;
L
Linus Torvalds 已提交
304 305 306 307 308 309
	struct rb_node *nd, *pn = NULL;
	unsigned long prev = 0, pend = 0;

	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
		struct vm_area_struct *vma;
		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
310
		if (vma->vm_start < prev) {
311 312
			pr_emerg("vm_start %lx < prev %lx\n",
				  vma->vm_start, prev);
313 314 315
			bug = 1;
		}
		if (vma->vm_start < pend) {
316 317
			pr_emerg("vm_start %lx < pend %lx\n",
				  vma->vm_start, pend);
318 319 320
			bug = 1;
		}
		if (vma->vm_start > vma->vm_end) {
321 322
			pr_emerg("vm_start %lx > vm_end %lx\n",
				  vma->vm_start, vma->vm_end);
323 324
			bug = 1;
		}
325
		spin_lock(&mm->page_table_lock);
326
		if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
327
			pr_emerg("free gap %lx, correct %lx\n",
328 329 330 331
			       vma->rb_subtree_gap,
			       vma_compute_subtree_gap(vma));
			bug = 1;
		}
332
		spin_unlock(&mm->page_table_lock);
L
Linus Torvalds 已提交
333 334
		i++;
		pn = nd;
D
David Miller 已提交
335 336
		prev = vma->vm_start;
		pend = vma->vm_end;
L
Linus Torvalds 已提交
337 338
	}
	j = 0;
339
	for (nd = pn; nd; nd = rb_prev(nd))
L
Linus Torvalds 已提交
340
		j++;
341
	if (i != j) {
342
		pr_emerg("backwards %d, forwards %d\n", j, i);
343
		bug = 1;
L
Linus Torvalds 已提交
344
	}
345
	return bug ? -1 : i;
L
Linus Torvalds 已提交
346 347
}

348 349 350 351 352 353 354
static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
{
	struct rb_node *nd;

	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
		struct vm_area_struct *vma;
		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
S
Sasha Levin 已提交
355 356 357
		VM_BUG_ON_VMA(vma != ignore &&
			vma->rb_subtree_gap != vma_compute_subtree_gap(vma),
			vma);
L
Linus Torvalds 已提交
358 359 360
	}
}

361
static void validate_mm(struct mm_struct *mm)
L
Linus Torvalds 已提交
362 363 364
{
	int bug = 0;
	int i = 0;
365
	unsigned long highest_address = 0;
366
	struct vm_area_struct *vma = mm->mmap;
367

368
	while (vma) {
369
		struct anon_vma *anon_vma = vma->anon_vma;
370
		struct anon_vma_chain *avc;
371

372 373 374 375 376 377 378
		if (anon_vma) {
			anon_vma_lock_read(anon_vma);
			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
				anon_vma_interval_tree_verify(avc);
			anon_vma_unlock_read(anon_vma);
		}

379
		highest_address = vm_end_gap(vma);
380
		vma = vma->vm_next;
L
Linus Torvalds 已提交
381 382
		i++;
	}
383
	if (i != mm->map_count) {
384
		pr_emerg("map_count %d vm_next %d\n", mm->map_count, i);
385 386 387
		bug = 1;
	}
	if (highest_address != mm->highest_vm_end) {
388
		pr_emerg("mm->highest_vm_end %lx, found %lx\n",
389
			  mm->highest_vm_end, highest_address);
390 391
		bug = 1;
	}
392
	i = browse_rb(mm);
393
	if (i != mm->map_count) {
394 395
		if (i != -1)
			pr_emerg("map_count %d rb %d\n", mm->map_count, i);
396 397
		bug = 1;
	}
S
Sasha Levin 已提交
398
	VM_BUG_ON_MM(bug, mm);
L
Linus Torvalds 已提交
399 400
}
#else
401
#define validate_mm_rb(root, ignore) do { } while (0)
L
Linus Torvalds 已提交
402 403 404
#define validate_mm(mm) do { } while (0)
#endif

405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430
RB_DECLARE_CALLBACKS(static, vma_gap_callbacks, struct vm_area_struct, vm_rb,
		     unsigned long, rb_subtree_gap, vma_compute_subtree_gap)

/*
 * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
 * vma->vm_prev->vm_end values changed, without modifying the vma's position
 * in the rbtree.
 */
static void vma_gap_update(struct vm_area_struct *vma)
{
	/*
	 * As it turns out, RB_DECLARE_CALLBACKS() already created a callback
	 * function that does exacltly what we want.
	 */
	vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
}

static inline void vma_rb_insert(struct vm_area_struct *vma,
				 struct rb_root *root)
{
	/* All rb_subtree_gap values must be consistent prior to insertion */
	validate_mm_rb(root, NULL);

	rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
}

431
static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
432 433 434 435 436 437 438 439 440
{
	/*
	 * Note rb_erase_augmented is a fairly large inline function,
	 * so make sure we instantiate it only once with our desired
	 * augmented rbtree callbacks.
	 */
	rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
}

441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma,
						struct rb_root *root,
						struct vm_area_struct *ignore)
{
	/*
	 * All rb_subtree_gap values must be consistent prior to erase,
	 * with the possible exception of the "next" vma being erased if
	 * next->vm_start was reduced.
	 */
	validate_mm_rb(root, ignore);

	__vma_rb_erase(vma, root);
}

static __always_inline void vma_rb_erase(struct vm_area_struct *vma,
					 struct rb_root *root)
{
	/*
	 * All rb_subtree_gap values must be consistent prior to erase,
	 * with the possible exception of the vma being erased.
	 */
	validate_mm_rb(root, vma);

	__vma_rb_erase(vma, root);
}

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
/*
 * vma has some anon_vma assigned, and is already inserted on that
 * anon_vma's interval trees.
 *
 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
 * vma must be removed from the anon_vma's interval trees using
 * anon_vma_interval_tree_pre_update_vma().
 *
 * After the update, the vma will be reinserted using
 * anon_vma_interval_tree_post_update_vma().
 *
 * The entire update must be protected by exclusive mmap_sem and by
 * the root anon_vma's mutex.
 */
static inline void
anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
{
	struct anon_vma_chain *avc;

	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
}

static inline void
anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
{
	struct anon_vma_chain *avc;

	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
}

499 500 501
static int find_vma_links(struct mm_struct *mm, unsigned long addr,
		unsigned long end, struct vm_area_struct **pprev,
		struct rb_node ***rb_link, struct rb_node **rb_parent)
L
Linus Torvalds 已提交
502
{
503
	struct rb_node **__rb_link, *__rb_parent, *rb_prev;
L
Linus Torvalds 已提交
504 505 506 507 508 509 510 511 512 513 514

	__rb_link = &mm->mm_rb.rb_node;
	rb_prev = __rb_parent = NULL;

	while (*__rb_link) {
		struct vm_area_struct *vma_tmp;

		__rb_parent = *__rb_link;
		vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);

		if (vma_tmp->vm_end > addr) {
515 516 517
			/* Fail if an existing vma overlaps the area */
			if (vma_tmp->vm_start < end)
				return -ENOMEM;
L
Linus Torvalds 已提交
518 519 520 521 522 523 524 525 526 527 528 529
			__rb_link = &__rb_parent->rb_left;
		} else {
			rb_prev = __rb_parent;
			__rb_link = &__rb_parent->rb_right;
		}
	}

	*pprev = NULL;
	if (rb_prev)
		*pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
	*rb_link = __rb_link;
	*rb_parent = __rb_parent;
530
	return 0;
L
Linus Torvalds 已提交
531 532
}

533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
static unsigned long count_vma_pages_range(struct mm_struct *mm,
		unsigned long addr, unsigned long end)
{
	unsigned long nr_pages = 0;
	struct vm_area_struct *vma;

	/* Find first overlaping mapping */
	vma = find_vma_intersection(mm, addr, end);
	if (!vma)
		return 0;

	nr_pages = (min(end, vma->vm_end) -
		max(addr, vma->vm_start)) >> PAGE_SHIFT;

	/* Iterate over the rest of the overlaps */
	for (vma = vma->vm_next; vma; vma = vma->vm_next) {
		unsigned long overlap_len;

		if (vma->vm_start > end)
			break;

		overlap_len = min(end, vma->vm_end) - vma->vm_start;
		nr_pages += overlap_len >> PAGE_SHIFT;
	}

	return nr_pages;
}

L
Linus Torvalds 已提交
561 562 563
void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
		struct rb_node **rb_link, struct rb_node *rb_parent)
{
564 565 566 567
	/* Update tracking information for the gap following the new vma. */
	if (vma->vm_next)
		vma_gap_update(vma->vm_next);
	else
568
		mm->highest_vm_end = vm_end_gap(vma);
569 570 571 572 573 574 575 576 577 578

	/*
	 * vma->vm_prev wasn't known when we followed the rbtree to find the
	 * correct insertion point for that vma. As a result, we could not
	 * update the vma vm_rb parents rb_subtree_gap values on the way down.
	 * So, we first insert the vma with a zero rb_subtree_gap value
	 * (to be consistent with what we did on the way down), and then
	 * immediately update the gap to the correct value. Finally we
	 * rebalance the rbtree after all augmented values have been set.
	 */
L
Linus Torvalds 已提交
579
	rb_link_node(&vma->vm_rb, rb_parent, rb_link);
580 581 582
	vma->rb_subtree_gap = 0;
	vma_gap_update(vma);
	vma_rb_insert(vma, &mm->mm_rb);
L
Linus Torvalds 已提交
583 584
}

D
Denys Vlasenko 已提交
585
static void __vma_link_file(struct vm_area_struct *vma)
L
Linus Torvalds 已提交
586
{
Z
ZhenwenXu 已提交
587
	struct file *file;
L
Linus Torvalds 已提交
588 589 590 591 592 593

	file = vma->vm_file;
	if (file) {
		struct address_space *mapping = file->f_mapping;

		if (vma->vm_flags & VM_DENYWRITE)
A
Al Viro 已提交
594
			atomic_dec(&file_inode(file)->i_writecount);
L
Linus Torvalds 已提交
595
		if (vma->vm_flags & VM_SHARED)
596
			atomic_inc(&mapping->i_mmap_writable);
L
Linus Torvalds 已提交
597 598

		flush_dcache_mmap_lock(mapping);
599
		vma_interval_tree_insert(vma, &mapping->i_mmap);
L
Linus Torvalds 已提交
600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
		flush_dcache_mmap_unlock(mapping);
	}
}

static void
__vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
	struct vm_area_struct *prev, struct rb_node **rb_link,
	struct rb_node *rb_parent)
{
	__vma_link_list(mm, vma, prev, rb_parent);
	__vma_link_rb(mm, vma, rb_link, rb_parent);
}

static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
			struct vm_area_struct *prev, struct rb_node **rb_link,
			struct rb_node *rb_parent)
{
	struct address_space *mapping = NULL;

619
	if (vma->vm_file) {
L
Linus Torvalds 已提交
620
		mapping = vma->vm_file->f_mapping;
621
		i_mmap_lock_write(mapping);
622
	}
L
Linus Torvalds 已提交
623 624 625 626 627

	__vma_link(mm, vma, prev, rb_link, rb_parent);
	__vma_link_file(vma);

	if (mapping)
628
		i_mmap_unlock_write(mapping);
L
Linus Torvalds 已提交
629 630 631 632 633 634

	mm->map_count++;
	validate_mm(mm);
}

/*
635
 * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
636
 * mm's list and rbtree.  It has already been inserted into the interval tree.
L
Linus Torvalds 已提交
637
 */
Z
ZhenwenXu 已提交
638
static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
L
Linus Torvalds 已提交
639
{
640
	struct vm_area_struct *prev;
Z
ZhenwenXu 已提交
641
	struct rb_node **rb_link, *rb_parent;
L
Linus Torvalds 已提交
642

643 644 645
	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
			   &prev, &rb_link, &rb_parent))
		BUG();
L
Linus Torvalds 已提交
646 647 648 649
	__vma_link(mm, vma, prev, rb_link, rb_parent);
	mm->map_count++;
}

650 651 652
static __always_inline void __vma_unlink_common(struct mm_struct *mm,
						struct vm_area_struct *vma,
						struct vm_area_struct *prev,
653 654
						bool has_prev,
						struct vm_area_struct *ignore)
L
Linus Torvalds 已提交
655
{
656
	struct vm_area_struct *next;
657

658
	vma_rb_erase_ignore(vma, &mm->mm_rb, ignore);
659 660 661 662 663 664 665 666 667 668
	next = vma->vm_next;
	if (has_prev)
		prev->vm_next = next;
	else {
		prev = vma->vm_prev;
		if (prev)
			prev->vm_next = next;
		else
			mm->mmap = next;
	}
669 670
	if (next)
		next->vm_prev = prev;
D
Davidlohr Bueso 已提交
671 672 673

	/* Kill the cache */
	vmacache_invalidate(mm);
L
Linus Torvalds 已提交
674 675
}

676 677 678 679
static inline void __vma_unlink_prev(struct mm_struct *mm,
				     struct vm_area_struct *vma,
				     struct vm_area_struct *prev)
{
680
	__vma_unlink_common(mm, vma, prev, true, vma);
681 682
}

L
Linus Torvalds 已提交
683 684 685 686 687 688 689
/*
 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
 * is already present in an i_mmap tree without adjusting the tree.
 * The following helper function should be used when such adjustments
 * are necessary.  The "insert" vma (if any) is to be inserted
 * before we drop the necessary locks.
 */
690 691 692
int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
	struct vm_area_struct *expand)
L
Linus Torvalds 已提交
693 694
{
	struct mm_struct *mm = vma->vm_mm;
695
	struct vm_area_struct *next = vma->vm_next, *orig_vma = vma;
L
Linus Torvalds 已提交
696
	struct address_space *mapping = NULL;
697
	struct rb_root_cached *root = NULL;
698
	struct anon_vma *anon_vma = NULL;
L
Linus Torvalds 已提交
699
	struct file *file = vma->vm_file;
700
	bool start_changed = false, end_changed = false;
L
Linus Torvalds 已提交
701 702 703 704
	long adjust_next = 0;
	int remove_next = 0;

	if (next && !insert) {
705
		struct vm_area_struct *exporter = NULL, *importer = NULL;
706

L
Linus Torvalds 已提交
707 708 709 710
		if (end >= next->vm_end) {
			/*
			 * vma expands, overlapping all the next, and
			 * perhaps the one after too (mprotect case 6).
711
			 * The only other cases that gets here are
712
			 * case 1, case 7 and case 8.
L
Linus Torvalds 已提交
713
			 */
714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
			if (next == expand) {
				/*
				 * The only case where we don't expand "vma"
				 * and we expand "next" instead is case 8.
				 */
				VM_WARN_ON(end != next->vm_end);
				/*
				 * remove_next == 3 means we're
				 * removing "vma" and that to do so we
				 * swapped "vma" and "next".
				 */
				remove_next = 3;
				VM_WARN_ON(file != next->vm_file);
				swap(vma, next);
			} else {
				VM_WARN_ON(expand != vma);
				/*
				 * case 1, 6, 7, remove_next == 2 is case 6,
				 * remove_next == 1 is case 1 or 7.
				 */
				remove_next = 1 + (end > next->vm_end);
				VM_WARN_ON(remove_next == 2 &&
					   end != next->vm_next->vm_end);
				VM_WARN_ON(remove_next == 1 &&
					   end != next->vm_end);
				/* trim end to next, for case 6 first pass */
				end = next->vm_end;
			}

743
			exporter = next;
L
Linus Torvalds 已提交
744
			importer = vma;
745 746 747 748 749

			/*
			 * If next doesn't have anon_vma, import from vma after
			 * next, if the vma overlaps with it.
			 */
750
			if (remove_next == 2 && !next->anon_vma)
751 752
				exporter = next->vm_next;

L
Linus Torvalds 已提交
753 754 755 756 757 758
		} else if (end > next->vm_start) {
			/*
			 * vma expands, overlapping part of the next:
			 * mprotect case 5 shifting the boundary up.
			 */
			adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
759
			exporter = next;
L
Linus Torvalds 已提交
760
			importer = vma;
761
			VM_WARN_ON(expand != importer);
L
Linus Torvalds 已提交
762 763 764 765 766 767
		} else if (end < vma->vm_end) {
			/*
			 * vma shrinks, and !insert tells it's not
			 * split_vma inserting another: so it must be
			 * mprotect case 4 shifting the boundary down.
			 */
V
vishnu.ps 已提交
768
			adjust_next = -((vma->vm_end - end) >> PAGE_SHIFT);
769
			exporter = vma;
L
Linus Torvalds 已提交
770
			importer = next;
771
			VM_WARN_ON(expand != importer);
L
Linus Torvalds 已提交
772 773
		}

774 775 776 777 778
		/*
		 * Easily overlooked: when mprotect shifts the boundary,
		 * make sure the expanding vma has anon_vma set if the
		 * shrinking vma had, to cover any anon pages imported.
		 */
779
		if (exporter && exporter->anon_vma && !importer->anon_vma) {
780 781
			int error;

782
			importer->anon_vma = exporter->anon_vma;
783
			error = anon_vma_clone(importer, exporter);
784
			if (error)
785
				return error;
786 787
		}
	}
788
again:
789
	vma_adjust_trans_huge(orig_vma, start, end, adjust_next);
790

L
Linus Torvalds 已提交
791 792
	if (file) {
		mapping = file->f_mapping;
793 794
		root = &mapping->i_mmap;
		uprobe_munmap(vma, vma->vm_start, vma->vm_end);
795

796 797
		if (adjust_next)
			uprobe_munmap(next, next->vm_start, next->vm_end);
798

799
		i_mmap_lock_write(mapping);
L
Linus Torvalds 已提交
800 801
		if (insert) {
			/*
802
			 * Put into interval tree now, so instantiated pages
L
Linus Torvalds 已提交
803 804 805 806 807 808 809 810
			 * are visible to arm/parisc __flush_dcache_page
			 * throughout; but we cannot insert into address
			 * space until vma start or end is updated.
			 */
			__vma_link_file(insert);
		}
	}

811 812 813 814
	anon_vma = vma->anon_vma;
	if (!anon_vma && adjust_next)
		anon_vma = next->anon_vma;
	if (anon_vma) {
815 816
		VM_WARN_ON(adjust_next && next->anon_vma &&
			   anon_vma != next->anon_vma);
817
		anon_vma_lock_write(anon_vma);
818 819 820 821
		anon_vma_interval_tree_pre_update_vma(vma);
		if (adjust_next)
			anon_vma_interval_tree_pre_update_vma(next);
	}
822

L
Linus Torvalds 已提交
823 824
	if (root) {
		flush_dcache_mmap_lock(mapping);
825
		vma_interval_tree_remove(vma, root);
L
Linus Torvalds 已提交
826
		if (adjust_next)
827
			vma_interval_tree_remove(next, root);
L
Linus Torvalds 已提交
828 829
	}

830 831 832 833 834 835 836 837
	if (start != vma->vm_start) {
		vma->vm_start = start;
		start_changed = true;
	}
	if (end != vma->vm_end) {
		vma->vm_end = end;
		end_changed = true;
	}
L
Linus Torvalds 已提交
838 839 840 841 842 843 844 845
	vma->vm_pgoff = pgoff;
	if (adjust_next) {
		next->vm_start += adjust_next << PAGE_SHIFT;
		next->vm_pgoff += adjust_next;
	}

	if (root) {
		if (adjust_next)
846 847
			vma_interval_tree_insert(next, root);
		vma_interval_tree_insert(vma, root);
L
Linus Torvalds 已提交
848 849 850 851 852 853 854 855
		flush_dcache_mmap_unlock(mapping);
	}

	if (remove_next) {
		/*
		 * vma_merge has merged next into vma, and needs
		 * us to remove next before dropping the locks.
		 */
856 857 858
		if (remove_next != 3)
			__vma_unlink_prev(mm, next, vma);
		else
859 860 861 862 863 864 865 866 867 868
			/*
			 * vma is not before next if they've been
			 * swapped.
			 *
			 * pre-swap() next->vm_start was reduced so
			 * tell validate_mm_rb to ignore pre-swap()
			 * "next" (which is stored in post-swap()
			 * "vma").
			 */
			__vma_unlink_common(mm, next, NULL, false, vma);
L
Linus Torvalds 已提交
869 870 871 872 873 874 875 876 877
		if (file)
			__remove_shared_vm_struct(next, file, mapping);
	} else if (insert) {
		/*
		 * split_vma has split insert from vma, and needs
		 * us to insert it before dropping the locks
		 * (it may either follow vma or precede it).
		 */
		__insert_vm_struct(mm, insert);
878 879 880 881 882
	} else {
		if (start_changed)
			vma_gap_update(vma);
		if (end_changed) {
			if (!next)
883
				mm->highest_vm_end = vm_end_gap(vma);
884 885 886
			else if (!adjust_next)
				vma_gap_update(next);
		}
L
Linus Torvalds 已提交
887 888
	}

889 890 891 892
	if (anon_vma) {
		anon_vma_interval_tree_post_update_vma(vma);
		if (adjust_next)
			anon_vma_interval_tree_post_update_vma(next);
893
		anon_vma_unlock_write(anon_vma);
894
	}
L
Linus Torvalds 已提交
895
	if (mapping)
896
		i_mmap_unlock_write(mapping);
L
Linus Torvalds 已提交
897

898
	if (root) {
899
		uprobe_mmap(vma);
900 901

		if (adjust_next)
902
			uprobe_mmap(next);
903 904
	}

L
Linus Torvalds 已提交
905
	if (remove_next) {
M
Matt Helsley 已提交
906
		if (file) {
907
			uprobe_munmap(next, next->vm_start, next->vm_end);
L
Linus Torvalds 已提交
908
			fput(file);
M
Matt Helsley 已提交
909
		}
910 911
		if (next->anon_vma)
			anon_vma_merge(vma, next);
L
Linus Torvalds 已提交
912
		mm->map_count--;
913
		mpol_put(vma_policy(next));
L
Linus Torvalds 已提交
914 915 916 917 918 919
		kmem_cache_free(vm_area_cachep, next);
		/*
		 * In mprotect's case 6 (see comments on vma_merge),
		 * we must remove another next too. It would clutter
		 * up the code too much to do both in one go.
		 */
920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
		if (remove_next != 3) {
			/*
			 * If "next" was removed and vma->vm_end was
			 * expanded (up) over it, in turn
			 * "next->vm_prev->vm_end" changed and the
			 * "vma->vm_next" gap must be updated.
			 */
			next = vma->vm_next;
		} else {
			/*
			 * For the scope of the comment "next" and
			 * "vma" considered pre-swap(): if "vma" was
			 * removed, next->vm_start was expanded (down)
			 * over it and the "next" gap must be updated.
			 * Because of the swap() the post-swap() "vma"
			 * actually points to pre-swap() "next"
			 * (post-swap() "next" as opposed is now a
			 * dangling pointer).
			 */
			next = vma;
		}
941 942 943
		if (remove_next == 2) {
			remove_next = 1;
			end = next->vm_end;
L
Linus Torvalds 已提交
944
			goto again;
945
		}
946 947
		else if (next)
			vma_gap_update(next);
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
		else {
			/*
			 * If remove_next == 2 we obviously can't
			 * reach this path.
			 *
			 * If remove_next == 3 we can't reach this
			 * path because pre-swap() next is always not
			 * NULL. pre-swap() "next" is not being
			 * removed and its next->vm_end is not altered
			 * (and furthermore "end" already matches
			 * next->vm_end in remove_next == 3).
			 *
			 * We reach this only in the remove_next == 1
			 * case if the "next" vma that was removed was
			 * the highest vma of the mm. However in such
			 * case next->vm_end == "end" and the extended
			 * "vma" has vma->vm_end == next->vm_end so
			 * mm->highest_vm_end doesn't need any update
			 * in remove_next == 1 case.
			 */
968
			VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma));
969
		}
L
Linus Torvalds 已提交
970
	}
971
	if (insert && file)
972
		uprobe_mmap(insert);
L
Linus Torvalds 已提交
973 974

	validate_mm(mm);
975 976

	return 0;
L
Linus Torvalds 已提交
977 978 979 980 981 982 983
}

/*
 * If the vma has a ->close operation then the driver probably needs to release
 * per-vma resources, so we don't attempt to merge those.
 */
static inline int is_mergeable_vma(struct vm_area_struct *vma,
984 985
				struct file *file, unsigned long vm_flags,
				struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
L
Linus Torvalds 已提交
986
{
987 988 989 990 991 992 993 994 995
	/*
	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
	 * match the flags but dirty bit -- the caller should mark
	 * merged VMA as dirty. If dirty bit won't be excluded from
	 * comparison, we increase pressue on the memory system forcing
	 * the kernel to generate new VMAs when old one could be
	 * extended instead.
	 */
	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
L
Linus Torvalds 已提交
996 997 998 999 1000
		return 0;
	if (vma->vm_file != file)
		return 0;
	if (vma->vm_ops && vma->vm_ops->close)
		return 0;
1001 1002
	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
		return 0;
L
Linus Torvalds 已提交
1003 1004 1005 1006
	return 1;
}

static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
S
Shaohua Li 已提交
1007 1008
					struct anon_vma *anon_vma2,
					struct vm_area_struct *vma)
L
Linus Torvalds 已提交
1009
{
S
Shaohua Li 已提交
1010 1011 1012 1013 1014 1015 1016 1017
	/*
	 * The list_is_singular() test is to avoid merging VMA cloned from
	 * parents. This can improve scalability caused by anon_vma lock.
	 */
	if ((!anon_vma1 || !anon_vma2) && (!vma ||
		list_is_singular(&vma->anon_vma_chain)))
		return 1;
	return anon_vma1 == anon_vma2;
L
Linus Torvalds 已提交
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
}

/*
 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
 * in front of (at a lower virtual address and file offset than) the vma.
 *
 * We cannot merge two vmas if they have differently assigned (non-NULL)
 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
 *
 * We don't check here for the merged mmap wrapping around the end of pagecache
 * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
 * wrap, nor mmaps which cover the final page at index -1UL.
 */
static int
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
1033 1034 1035
		     struct anon_vma *anon_vma, struct file *file,
		     pgoff_t vm_pgoff,
		     struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
L
Linus Torvalds 已提交
1036
{
1037
	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
S
Shaohua Li 已提交
1038
	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
L
Linus Torvalds 已提交
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
		if (vma->vm_pgoff == vm_pgoff)
			return 1;
	}
	return 0;
}

/*
 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
 * beyond (at a higher virtual address and file offset than) the vma.
 *
 * We cannot merge two vmas if they have differently assigned (non-NULL)
 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
 */
static int
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
1054 1055 1056
		    struct anon_vma *anon_vma, struct file *file,
		    pgoff_t vm_pgoff,
		    struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
L
Linus Torvalds 已提交
1057
{
1058
	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
S
Shaohua Li 已提交
1059
	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
L
Linus Torvalds 已提交
1060
		pgoff_t vm_pglen;
1061
		vm_pglen = vma_pages(vma);
L
Linus Torvalds 已提交
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
			return 1;
	}
	return 0;
}

/*
 * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
 * whether that can be merged with its predecessor or its successor.
 * Or both (it neatly fills a hole).
 *
 * In most cases - when called for mmap, brk or mremap - [addr,end) is
 * certain not to be mapped by the time vma_merge is called; but when
 * called for mprotect, it is certain to be already mapped (either at
 * an offset within prev, or at the start of next), and the flags of
 * this area are about to be changed to vm_flags - and the no-change
 * case has already been eliminated.
 *
 * The following mprotect cases have to be considered, where AAAA is
 * the area passed down from mprotect_fixup, never extending beyond one
 * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
 *
 *     AAAA             AAAA                AAAA          AAAA
 *    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPNNNNXXXX
 *    cannot merge    might become    might become    might become
 *                    PPNNNNNNNNNN    PPPPPPPPPPNN    PPPPPPPPPPPP 6 or
 *    mmap, brk or    case 4 below    case 5 below    PPPPPPPPXXXX 7 or
1089
 *    mremap move:                                    PPPPXXXXXXXX 8
L
Linus Torvalds 已提交
1090 1091 1092 1093
 *        AAAA
 *    PPPP    NNNN    PPPPPPPPPPPP    PPPPPPPPNNNN    PPPPNNNNNNNN
 *    might become    case 1 below    case 2 below    case 3 below
 *
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
 * It is important for case 8 that the the vma NNNN overlapping the
 * region AAAA is never going to extended over XXXX. Instead XXXX must
 * be extended in region AAAA and NNNN must be removed. This way in
 * all cases where vma_merge succeeds, the moment vma_adjust drops the
 * rmap_locks, the properties of the merged vma will be already
 * correct for the whole merged range. Some of those properties like
 * vm_page_prot/vm_flags may be accessed by rmap_walks and they must
 * be correct for the whole merged range immediately after the
 * rmap_locks are released. Otherwise if XXXX would be removed and
 * NNNN would be extended over the XXXX range, remove_migration_ptes
 * or other rmap walkers (if working on addresses beyond the "end"
 * parameter) may establish ptes with the wrong permissions of NNNN
 * instead of the right permissions of XXXX.
L
Linus Torvalds 已提交
1107 1108 1109 1110
 */
struct vm_area_struct *vma_merge(struct mm_struct *mm,
			struct vm_area_struct *prev, unsigned long addr,
			unsigned long end, unsigned long vm_flags,
V
vishnu.ps 已提交
1111
			struct anon_vma *anon_vma, struct file *file,
1112 1113
			pgoff_t pgoff, struct mempolicy *policy,
			struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
L
Linus Torvalds 已提交
1114 1115 1116
{
	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
	struct vm_area_struct *area, *next;
1117
	int err;
L
Linus Torvalds 已提交
1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130

	/*
	 * We later require that vma->vm_flags == vm_flags,
	 * so this tests vma->vm_flags & VM_SPECIAL, too.
	 */
	if (vm_flags & VM_SPECIAL)
		return NULL;

	if (prev)
		next = prev->vm_next;
	else
		next = mm->mmap;
	area = next;
1131
	if (area && area->vm_end == end)		/* cases 6, 7, 8 */
L
Linus Torvalds 已提交
1132 1133
		next = next->vm_next;

1134 1135 1136 1137 1138
	/* verify some invariant that must be enforced by the caller */
	VM_WARN_ON(prev && addr <= prev->vm_start);
	VM_WARN_ON(area && end > area->vm_end);
	VM_WARN_ON(addr >= end);

L
Linus Torvalds 已提交
1139 1140 1141 1142
	/*
	 * Can it merge with the predecessor?
	 */
	if (prev && prev->vm_end == addr &&
V
vishnu.ps 已提交
1143
			mpol_equal(vma_policy(prev), policy) &&
L
Linus Torvalds 已提交
1144
			can_vma_merge_after(prev, vm_flags,
1145 1146
					    anon_vma, file, pgoff,
					    vm_userfaultfd_ctx)) {
L
Linus Torvalds 已提交
1147 1148 1149 1150 1151 1152
		/*
		 * OK, it can.  Can we now merge in the successor as well?
		 */
		if (next && end == next->vm_start &&
				mpol_equal(policy, vma_policy(next)) &&
				can_vma_merge_before(next, vm_flags,
1153 1154 1155
						     anon_vma, file,
						     pgoff+pglen,
						     vm_userfaultfd_ctx) &&
L
Linus Torvalds 已提交
1156
				is_mergeable_anon_vma(prev->anon_vma,
S
Shaohua Li 已提交
1157
						      next->anon_vma, NULL)) {
L
Linus Torvalds 已提交
1158
							/* cases 1, 6 */
1159 1160 1161
			err = __vma_adjust(prev, prev->vm_start,
					 next->vm_end, prev->vm_pgoff, NULL,
					 prev);
L
Linus Torvalds 已提交
1162
		} else					/* cases 2, 5, 7 */
1163 1164
			err = __vma_adjust(prev, prev->vm_start,
					 end, prev->vm_pgoff, NULL, prev);
1165 1166
		if (err)
			return NULL;
1167
		khugepaged_enter_vma_merge(prev, vm_flags);
L
Linus Torvalds 已提交
1168 1169 1170 1171 1172 1173 1174
		return prev;
	}

	/*
	 * Can this new request be merged in front of next?
	 */
	if (next && end == next->vm_start &&
V
vishnu.ps 已提交
1175
			mpol_equal(policy, vma_policy(next)) &&
L
Linus Torvalds 已提交
1176
			can_vma_merge_before(next, vm_flags,
1177 1178
					     anon_vma, file, pgoff+pglen,
					     vm_userfaultfd_ctx)) {
L
Linus Torvalds 已提交
1179
		if (prev && addr < prev->vm_end)	/* case 4 */
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
			err = __vma_adjust(prev, prev->vm_start,
					 addr, prev->vm_pgoff, NULL, next);
		else {					/* cases 3, 8 */
			err = __vma_adjust(area, addr, next->vm_end,
					 next->vm_pgoff - pglen, NULL, next);
			/*
			 * In case 3 area is already equal to next and
			 * this is a noop, but in case 8 "area" has
			 * been removed and next was expanded over it.
			 */
			area = next;
		}
1192 1193
		if (err)
			return NULL;
1194
		khugepaged_enter_vma_merge(area, vm_flags);
L
Linus Torvalds 已提交
1195 1196 1197 1198 1199 1200
		return area;
	}

	return NULL;
}

1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
/*
 * Rough compatbility check to quickly see if it's even worth looking
 * at sharing an anon_vma.
 *
 * They need to have the same vm_file, and the flags can only differ
 * in things that mprotect may change.
 *
 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
 * we can merge the two vma's. For example, we refuse to merge a vma if
 * there is a vm_ops->close() function, because that indicates that the
 * driver is doing some kind of reference counting. But that doesn't
 * really matter for the anon_vma sharing case.
 */
static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
{
	return a->vm_end == b->vm_start &&
		mpol_equal(vma_policy(a), vma_policy(b)) &&
		a->vm_file == b->vm_file &&
1219
		!((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC|VM_SOFTDIRTY)) &&
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
}

/*
 * Do some basic sanity checking to see if we can re-use the anon_vma
 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
 * the same as 'old', the other will be the new one that is trying
 * to share the anon_vma.
 *
 * NOTE! This runs with mm_sem held for reading, so it is possible that
 * the anon_vma of 'old' is concurrently in the process of being set up
 * by another page fault trying to merge _that_. But that's ok: if it
 * is being set up, that automatically means that it will be a singleton
 * acceptable for merging, so we can do all of this optimistically. But
1234
 * we do that READ_ONCE() to make sure that we never re-load the pointer.
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
 *
 * IOW: that the "list_is_singular()" test on the anon_vma_chain only
 * matters for the 'stable anon_vma' case (ie the thing we want to avoid
 * is to return an anon_vma that is "complex" due to having gone through
 * a fork).
 *
 * We also make sure that the two vma's are compatible (adjacent,
 * and with the same memory policies). That's all stable, even with just
 * a read lock on the mm_sem.
 */
static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
{
	if (anon_vma_compatible(a, b)) {
1248
		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
1249 1250 1251 1252 1253 1254 1255

		if (anon_vma && list_is_singular(&old->anon_vma_chain))
			return anon_vma;
	}
	return NULL;
}

L
Linus Torvalds 已提交
1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
/*
 * find_mergeable_anon_vma is used by anon_vma_prepare, to check
 * neighbouring vmas for a suitable anon_vma, before it goes off
 * to allocate a new anon_vma.  It checks because a repetitive
 * sequence of mprotects and faults may otherwise lead to distinct
 * anon_vmas being allocated, preventing vma merge in subsequent
 * mprotect.
 */
struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
{
1266
	struct anon_vma *anon_vma;
L
Linus Torvalds 已提交
1267 1268 1269 1270 1271 1272
	struct vm_area_struct *near;

	near = vma->vm_next;
	if (!near)
		goto try_prev;

1273 1274 1275
	anon_vma = reusable_anon_vma(near, vma, near);
	if (anon_vma)
		return anon_vma;
L
Linus Torvalds 已提交
1276
try_prev:
1277
	near = vma->vm_prev;
L
Linus Torvalds 已提交
1278 1279 1280
	if (!near)
		goto none;

1281 1282 1283
	anon_vma = reusable_anon_vma(near, near, vma);
	if (anon_vma)
		return anon_vma;
L
Linus Torvalds 已提交
1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
none:
	/*
	 * There's no absolute need to look only at touching neighbours:
	 * we could search further afield for "compatible" anon_vmas.
	 * But it would probably just be a waste of time searching,
	 * or lead to too many vmas hanging off the same anon_vma.
	 * We're trying to allow mprotect remerging later on,
	 * not trying to minimize memory used for anon_vmas.
	 */
	return NULL;
}

A
Al Viro 已提交
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
/*
 * If a hint addr is less than mmap_min_addr change hint to be as
 * low as possible but still greater than mmap_min_addr
 */
static inline unsigned long round_hint_to_min(unsigned long hint)
{
	hint &= PAGE_MASK;
	if (((void *)hint != NULL) &&
	    (hint < mmap_min_addr))
		return PAGE_ALIGN(mmap_min_addr);
	return hint;
}

1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
static inline int mlock_future_check(struct mm_struct *mm,
				     unsigned long flags,
				     unsigned long len)
{
	unsigned long locked, lock_limit;

	/*  mlock MCL_FUTURE? */
	if (flags & VM_LOCKED) {
		locked = len >> PAGE_SHIFT;
		locked += mm->locked_vm;
		lock_limit = rlimit(RLIMIT_MEMLOCK);
		lock_limit >>= PAGE_SHIFT;
		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
			return -EAGAIN;
	}
	return 0;
}

1327 1328 1329
static inline u64 file_mmap_size_max(struct file *file, struct inode *inode)
{
	if (S_ISREG(inode->i_mode))
1330
		return MAX_LFS_FILESIZE;
1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355

	if (S_ISBLK(inode->i_mode))
		return MAX_LFS_FILESIZE;

	/* Special "we do even unsigned file positions" case */
	if (file->f_mode & FMODE_UNSIGNED_OFFSET)
		return 0;

	/* Yes, random drivers might want more. But I'm tired of buggy drivers */
	return ULONG_MAX;
}

static inline bool file_mmap_ok(struct file *file, struct inode *inode,
				unsigned long pgoff, unsigned long len)
{
	u64 maxsize = file_mmap_size_max(file, inode);

	if (maxsize && len > maxsize)
		return false;
	maxsize -= len;
	if (pgoff > maxsize >> PAGE_SHIFT)
		return false;
	return true;
}

L
Linus Torvalds 已提交
1356
/*
1357
 * The caller must hold down_write(&current->mm->mmap_sem).
L
Linus Torvalds 已提交
1358
 */
1359
unsigned long do_mmap(struct file *file, unsigned long addr,
L
Linus Torvalds 已提交
1360
			unsigned long len, unsigned long prot,
1361
			unsigned long flags, vm_flags_t vm_flags,
1362 1363
			unsigned long pgoff, unsigned long *populate,
			struct list_head *uf)
L
Linus Torvalds 已提交
1364
{
V
vishnu.ps 已提交
1365
	struct mm_struct *mm = current->mm;
1366
	int pkey = 0;
L
Linus Torvalds 已提交
1367

1368
	*populate = 0;
1369

1370 1371 1372
	if (!len)
		return -EINVAL;

L
Linus Torvalds 已提交
1373 1374 1375 1376 1377 1378 1379
	/*
	 * Does the application expect PROT_READ to imply PROT_EXEC?
	 *
	 * (the exception is when the underlying filesystem is noexec
	 *  mounted, in which case we dont add PROT_EXEC.)
	 */
	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
1380
		if (!(file && path_noexec(&file->f_path)))
L
Linus Torvalds 已提交
1381 1382
			prot |= PROT_EXEC;

M
Michal Hocko 已提交
1383 1384 1385 1386
	/* force arch specific MAP_FIXED handling in get_unmapped_area */
	if (flags & MAP_FIXED_NOREPLACE)
		flags |= MAP_FIXED;

1387 1388 1389
	if (!(flags & MAP_FIXED))
		addr = round_hint_to_min(addr);

L
Linus Torvalds 已提交
1390 1391
	/* Careful about overflows.. */
	len = PAGE_ALIGN(len);
1392
	if (!len)
L
Linus Torvalds 已提交
1393 1394 1395 1396
		return -ENOMEM;

	/* offset overflow? */
	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
V
vishnu.ps 已提交
1397
		return -EOVERFLOW;
L
Linus Torvalds 已提交
1398 1399 1400 1401 1402 1403 1404 1405 1406

	/* Too many mappings? */
	if (mm->map_count > sysctl_max_map_count)
		return -ENOMEM;

	/* Obtain the address to map to. we verify (or select) it and ensure
	 * that it represents a valid section of the address space.
	 */
	addr = get_unmapped_area(file, addr, len, pgoff, flags);
1407
	if (offset_in_page(addr))
L
Linus Torvalds 已提交
1408 1409
		return addr;

M
Michal Hocko 已提交
1410 1411 1412 1413 1414 1415 1416
	if (flags & MAP_FIXED_NOREPLACE) {
		struct vm_area_struct *vma = find_vma(mm, addr);

		if (vma && vma->vm_start <= addr)
			return -EEXIST;
	}

1417 1418 1419 1420 1421 1422
	if (prot == PROT_EXEC) {
		pkey = execute_only_pkey(mm);
		if (pkey < 0)
			pkey = 0;
	}

L
Linus Torvalds 已提交
1423 1424 1425 1426
	/* Do simple checking here so the lower-level routines won't have
	 * to. we assume access permissions have been handled by the open
	 * of the memory object, so we don't do any here.
	 */
1427
	vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
L
Linus Torvalds 已提交
1428 1429
			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;

H
Huang Shijie 已提交
1430
	if (flags & MAP_LOCKED)
L
Linus Torvalds 已提交
1431 1432
		if (!can_do_mlock())
			return -EPERM;
1433

1434 1435
	if (mlock_future_check(mm, vm_flags, len))
		return -EAGAIN;
L
Linus Torvalds 已提交
1436 1437

	if (file) {
1438
		struct inode *inode = file_inode(file);
1439 1440
		unsigned long flags_mask;

1441 1442 1443
		if (!file_mmap_ok(file, inode, pgoff, len))
			return -EOVERFLOW;

1444
		flags_mask = LEGACY_MAP_MASK | file->f_op->mmap_supported_flags;
1445

L
Linus Torvalds 已提交
1446 1447
		switch (flags & MAP_TYPE) {
		case MAP_SHARED:
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
			/*
			 * Force use of MAP_SHARED_VALIDATE with non-legacy
			 * flags. E.g. MAP_SYNC is dangerous to use with
			 * MAP_SHARED as you don't know which consistency model
			 * you will get. We silently ignore unsupported flags
			 * with MAP_SHARED to preserve backward compatibility.
			 */
			flags &= LEGACY_MAP_MASK;
			/* fall through */
		case MAP_SHARED_VALIDATE:
			if (flags & ~flags_mask)
				return -EOPNOTSUPP;
L
Linus Torvalds 已提交
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472
			if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
				return -EACCES;

			/*
			 * Make sure we don't allow writing to an append-only
			 * file..
			 */
			if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
				return -EACCES;

			/*
			 * Make sure there are no mandatory locks on the file.
			 */
1473
			if (locks_verify_locked(file))
L
Linus Torvalds 已提交
1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
				return -EAGAIN;

			vm_flags |= VM_SHARED | VM_MAYSHARE;
			if (!(file->f_mode & FMODE_WRITE))
				vm_flags &= ~(VM_MAYWRITE | VM_SHARED);

			/* fall through */
		case MAP_PRIVATE:
			if (!(file->f_mode & FMODE_READ))
				return -EACCES;
1484
			if (path_noexec(&file->f_path)) {
L
Linus Torvalds 已提交
1485 1486 1487 1488 1489
				if (vm_flags & VM_EXEC)
					return -EPERM;
				vm_flags &= ~VM_MAYEXEC;
			}

A
Al Viro 已提交
1490
			if (!file->f_op->mmap)
L
Linus Torvalds 已提交
1491
				return -ENODEV;
1492 1493
			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
				return -EINVAL;
L
Linus Torvalds 已提交
1494 1495 1496 1497 1498 1499 1500 1501
			break;

		default:
			return -EINVAL;
		}
	} else {
		switch (flags & MAP_TYPE) {
		case MAP_SHARED:
1502 1503
			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
				return -EINVAL;
1504 1505 1506 1507
			/*
			 * Ignore pgoff.
			 */
			pgoff = 0;
L
Linus Torvalds 已提交
1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
			vm_flags |= VM_SHARED | VM_MAYSHARE;
			break;
		case MAP_PRIVATE:
			/*
			 * Set pgoff according to addr for anon_vma.
			 */
			pgoff = addr >> PAGE_SHIFT;
			break;
		default:
			return -EINVAL;
		}
	}

1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
	/*
	 * Set 'VM_NORESERVE' if we should not account for the
	 * memory use of this mapping.
	 */
	if (flags & MAP_NORESERVE) {
		/* We honor MAP_NORESERVE if allowed to overcommit */
		if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
			vm_flags |= VM_NORESERVE;

		/* hugetlb applies strict overcommit unless MAP_NORESERVE */
		if (file && is_file_hugepages(file))
			vm_flags |= VM_NORESERVE;
	}

1535
	addr = mmap_region(file, addr, len, vm_flags, pgoff, uf);
1536 1537 1538
	if (!IS_ERR_VALUE(addr) &&
	    ((vm_flags & VM_LOCKED) ||
	     (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
1539
		*populate = len;
1540
	return addr;
M
Miklos Szeredi 已提交
1541
}
1542

1543 1544 1545
unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
			      unsigned long prot, unsigned long flags,
			      unsigned long fd, unsigned long pgoff)
H
Hugh Dickins 已提交
1546 1547
{
	struct file *file = NULL;
1548
	unsigned long retval;
H
Hugh Dickins 已提交
1549 1550

	if (!(flags & MAP_ANONYMOUS)) {
A
Al Viro 已提交
1551
		audit_mmap_fd(fd, flags);
H
Hugh Dickins 已提交
1552 1553
		file = fget(fd);
		if (!file)
1554
			return -EBADF;
1555 1556
		if (is_file_hugepages(file))
			len = ALIGN(len, huge_page_size(hstate_file(file)));
1557 1558 1559
		retval = -EINVAL;
		if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
			goto out_fput;
H
Hugh Dickins 已提交
1560 1561
	} else if (flags & MAP_HUGETLB) {
		struct user_struct *user = NULL;
1562
		struct hstate *hs;
1563

1564
		hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
1565 1566 1567 1568
		if (!hs)
			return -EINVAL;

		len = ALIGN(len, huge_page_size(hs));
H
Hugh Dickins 已提交
1569 1570 1571 1572 1573 1574
		/*
		 * VM_NORESERVE is used because the reservations will be
		 * taken when vm_ops->mmap() is called
		 * A dummy user value is used because we are not locking
		 * memory so no accounting is necessary
		 */
1575
		file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
1576 1577 1578
				VM_NORESERVE,
				&user, HUGETLB_ANONHUGE_INODE,
				(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
H
Hugh Dickins 已提交
1579 1580 1581 1582 1583 1584
		if (IS_ERR(file))
			return PTR_ERR(file);
	}

	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);

M
Michal Hocko 已提交
1585
	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1586
out_fput:
H
Hugh Dickins 已提交
1587 1588 1589 1590 1591
	if (file)
		fput(file);
	return retval;
}

1592 1593 1594 1595 1596 1597 1598
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
		unsigned long, prot, unsigned long, flags,
		unsigned long, fd, unsigned long, pgoff)
{
	return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
}

C
Christoph Hellwig 已提交
1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
#ifdef __ARCH_WANT_SYS_OLD_MMAP
struct mmap_arg_struct {
	unsigned long addr;
	unsigned long len;
	unsigned long prot;
	unsigned long flags;
	unsigned long fd;
	unsigned long offset;
};

SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
{
	struct mmap_arg_struct a;

	if (copy_from_user(&a, arg, sizeof(a)))
		return -EFAULT;
1615
	if (offset_in_page(a.offset))
C
Christoph Hellwig 已提交
1616 1617
		return -EINVAL;

1618 1619
	return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
			       a.offset >> PAGE_SHIFT);
C
Christoph Hellwig 已提交
1620 1621 1622
}
#endif /* __ARCH_WANT_SYS_OLD_MMAP */

A
Alexey Dobriyan 已提交
1623 1624 1625 1626 1627 1628
/*
 * Some shared mappigns will want the pages marked read-only
 * to track write events. If so, we'll downgrade vm_page_prot
 * to the private version (using protection_map[] without the
 * VM_SHARED bit).
 */
1629
int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
A
Alexey Dobriyan 已提交
1630
{
1631
	vm_flags_t vm_flags = vma->vm_flags;
1632
	const struct vm_operations_struct *vm_ops = vma->vm_ops;
A
Alexey Dobriyan 已提交
1633 1634 1635 1636 1637 1638

	/* If it was private or non-writable, the write bit is already clear */
	if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
		return 0;

	/* The backer wishes to know when pages are first written to? */
1639
	if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite))
A
Alexey Dobriyan 已提交
1640 1641
		return 1;

1642 1643
	/* The open routine did something to the protections that pgprot_modify
	 * won't preserve? */
1644 1645
	if (pgprot_val(vm_page_prot) !=
	    pgprot_val(vm_pgprot_modify(vm_page_prot, vm_flags)))
A
Alexey Dobriyan 已提交
1646 1647
		return 0;

1648 1649 1650 1651
	/* Do we need to track softdirty? */
	if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))
		return 1;

A
Alexey Dobriyan 已提交
1652
	/* Specialty mapping? */
1653
	if (vm_flags & VM_PFNMAP)
A
Alexey Dobriyan 已提交
1654 1655 1656 1657 1658 1659 1660
		return 0;

	/* Can the mapping track the dirty pages? */
	return vma->vm_file && vma->vm_file->f_mapping &&
		mapping_cap_account_dirty(vma->vm_file->f_mapping);
}

1661 1662
/*
 * We account for memory if it's a private writeable mapping,
1663
 * not hugepages and VM_NORESERVE wasn't set.
1664
 */
1665
static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
1666
{
1667 1668 1669 1670 1671 1672 1673
	/*
	 * hugetlb has its own accounting separate from the core VM
	 * VM_HUGETLB may not be set yet so we cannot check for that flag.
	 */
	if (file && is_file_hugepages(file))
		return 0;

1674 1675 1676
	return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
}

M
Miklos Szeredi 已提交
1677
unsigned long mmap_region(struct file *file, unsigned long addr,
1678 1679
		unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
		struct list_head *uf)
M
Miklos Szeredi 已提交
1680 1681 1682 1683 1684 1685 1686
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma, *prev;
	int error;
	struct rb_node **rb_link, *rb_parent;
	unsigned long charged = 0;

1687
	/* Check against address space limit. */
1688
	if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
1689 1690 1691 1692 1693 1694 1695 1696
		unsigned long nr_pages;

		/*
		 * MAP_FIXED may remove pages of mappings that intersects with
		 * requested mapping. Account for the pages it would unmap.
		 */
		nr_pages = count_vma_pages_range(mm, addr, addr + len);

1697 1698
		if (!may_expand_vm(mm, vm_flags,
					(len >> PAGE_SHIFT) - nr_pages))
1699 1700 1701
			return -ENOMEM;
	}

L
Linus Torvalds 已提交
1702
	/* Clear old maps */
1703 1704
	while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
			      &rb_parent)) {
1705
		if (do_munmap(mm, addr, len, uf))
L
Linus Torvalds 已提交
1706 1707 1708
			return -ENOMEM;
	}

1709 1710 1711
	/*
	 * Private writable mapping: check memory availability
	 */
1712
	if (accountable_mapping(file, vm_flags)) {
1713
		charged = len >> PAGE_SHIFT;
1714
		if (security_vm_enough_memory_mm(mm, charged))
1715 1716
			return -ENOMEM;
		vm_flags |= VM_ACCOUNT;
L
Linus Torvalds 已提交
1717 1718 1719
	}

	/*
1720
	 * Can we just expand an old mapping?
L
Linus Torvalds 已提交
1721
	 */
1722 1723
	vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
			NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
1724 1725
	if (vma)
		goto out;
L
Linus Torvalds 已提交
1726 1727 1728 1729 1730 1731

	/*
	 * Determine the object being mapped and call the appropriate
	 * specific mapper. the address has already been validated, but
	 * not unmapped, but the maps are removed from the list.
	 */
P
Pekka Enberg 已提交
1732
	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
L
Linus Torvalds 已提交
1733 1734 1735 1736 1737 1738 1739 1740 1741
	if (!vma) {
		error = -ENOMEM;
		goto unacct_error;
	}

	vma->vm_mm = mm;
	vma->vm_start = addr;
	vma->vm_end = addr + len;
	vma->vm_flags = vm_flags;
1742
	vma->vm_page_prot = vm_get_page_prot(vm_flags);
L
Linus Torvalds 已提交
1743
	vma->vm_pgoff = pgoff;
1744
	INIT_LIST_HEAD(&vma->anon_vma_chain);
L
Linus Torvalds 已提交
1745 1746 1747 1748 1749 1750 1751

	if (file) {
		if (vm_flags & VM_DENYWRITE) {
			error = deny_write_access(file);
			if (error)
				goto free_vma;
		}
1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762
		if (vm_flags & VM_SHARED) {
			error = mapping_map_writable(file->f_mapping);
			if (error)
				goto allow_write_and_free_vma;
		}

		/* ->mmap() can change vma->vm_file, but must guarantee that
		 * vma_link() below can deny write-access if VM_DENYWRITE is set
		 * and map writably if VM_SHARED is set. This usually means the
		 * new file must not have been exposed to user-space, yet.
		 */
A
Al Viro 已提交
1763
		vma->vm_file = get_file(file);
1764
		error = call_mmap(file, vma);
L
Linus Torvalds 已提交
1765 1766
		if (error)
			goto unmap_and_free_vma;
1767 1768 1769 1770 1771

		/* Can addr have changed??
		 *
		 * Answer: Yes, several device drivers can do it in their
		 *         f_op->mmap method. -DaveM
1772 1773
		 * Bug: If addr is changed, prev, rb_link, rb_parent should
		 *      be updated for vma_link()
1774
		 */
1775 1776
		WARN_ON_ONCE(addr != vma->vm_start);

1777 1778
		addr = vma->vm_start;
		vm_flags = vma->vm_flags;
L
Linus Torvalds 已提交
1779 1780 1781 1782 1783 1784
	} else if (vm_flags & VM_SHARED) {
		error = shmem_zero_setup(vma);
		if (error)
			goto free_vma;
	}

1785
	vma_link(mm, vma, prev, rb_link, rb_parent);
1786
	/* Once vma denies write, undo our temporary denial count */
1787 1788 1789 1790 1791 1792
	if (file) {
		if (vm_flags & VM_SHARED)
			mapping_unmap_writable(file->f_mapping);
		if (vm_flags & VM_DENYWRITE)
			allow_write_access(file);
	}
1793
	file = vma->vm_file;
1794
out:
1795
	perf_event_mmap(vma);
1796

1797
	vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
L
Linus Torvalds 已提交
1798
	if (vm_flags & VM_LOCKED) {
1799 1800
		if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
					vma == get_gate_vma(current->mm)))
1801
			mm->locked_vm += (len >> PAGE_SHIFT);
1802
		else
E
Eric B Munson 已提交
1803
			vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
1804
	}
1805

1806 1807
	if (file)
		uprobe_mmap(vma);
1808

1809 1810 1811 1812 1813 1814 1815 1816 1817
	/*
	 * New (or expanded) vma always get soft dirty status.
	 * Otherwise user-space soft-dirty page tracker won't
	 * be able to distinguish situation when vma area unmapped,
	 * then new mapped in-place (which must be aimed as
	 * a completely new data area).
	 */
	vma->vm_flags |= VM_SOFTDIRTY;

1818 1819
	vma_set_page_prot(vma);

L
Linus Torvalds 已提交
1820 1821 1822 1823 1824 1825 1826
	return addr;

unmap_and_free_vma:
	vma->vm_file = NULL;
	fput(file);

	/* Undo any partial mapping done by a device driver. */
1827 1828
	unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
	charged = 0;
1829 1830 1831 1832 1833
	if (vm_flags & VM_SHARED)
		mapping_unmap_writable(file->f_mapping);
allow_write_and_free_vma:
	if (vm_flags & VM_DENYWRITE)
		allow_write_access(file);
L
Linus Torvalds 已提交
1834 1835 1836 1837 1838 1839 1840 1841
free_vma:
	kmem_cache_free(vm_area_cachep, vma);
unacct_error:
	if (charged)
		vm_unacct_memory(charged);
	return error;
}

1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
unsigned long unmapped_area(struct vm_unmapped_area_info *info)
{
	/*
	 * We implement the search by looking for an rbtree node that
	 * immediately follows a suitable gap. That is,
	 * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
	 * - gap_end   = vma->vm_start        >= info->low_limit  + length;
	 * - gap_end - gap_start >= length
	 */

	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	unsigned long length, low_limit, high_limit, gap_start, gap_end;

	/* Adjust search length to account for worst case alignment overhead */
	length = info->length + info->align_mask;
	if (length < info->length)
		return -ENOMEM;

	/* Adjust search limits by the desired length */
	if (info->high_limit < length)
		return -ENOMEM;
	high_limit = info->high_limit - length;

	if (info->low_limit > high_limit)
		return -ENOMEM;
	low_limit = info->low_limit + length;

	/* Check if rbtree root looks promising */
	if (RB_EMPTY_ROOT(&mm->mm_rb))
		goto check_highest;
	vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
	if (vma->rb_subtree_gap < length)
		goto check_highest;

	while (true) {
		/* Visit left subtree if it looks promising */
1879
		gap_end = vm_start_gap(vma);
1880 1881 1882 1883 1884 1885 1886 1887 1888 1889
		if (gap_end >= low_limit && vma->vm_rb.rb_left) {
			struct vm_area_struct *left =
				rb_entry(vma->vm_rb.rb_left,
					 struct vm_area_struct, vm_rb);
			if (left->rb_subtree_gap >= length) {
				vma = left;
				continue;
			}
		}

1890
		gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
1891 1892 1893 1894
check_current:
		/* Check if current node has a suitable gap */
		if (gap_start > high_limit)
			return -ENOMEM;
1895 1896
		if (gap_end >= low_limit &&
		    gap_end > gap_start && gap_end - gap_start >= length)
1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917
			goto found;

		/* Visit right subtree if it looks promising */
		if (vma->vm_rb.rb_right) {
			struct vm_area_struct *right =
				rb_entry(vma->vm_rb.rb_right,
					 struct vm_area_struct, vm_rb);
			if (right->rb_subtree_gap >= length) {
				vma = right;
				continue;
			}
		}

		/* Go back up the rbtree to find next candidate node */
		while (true) {
			struct rb_node *prev = &vma->vm_rb;
			if (!rb_parent(prev))
				goto check_highest;
			vma = rb_entry(rb_parent(prev),
				       struct vm_area_struct, vm_rb);
			if (prev == vma->vm_rb.rb_left) {
1918 1919
				gap_start = vm_end_gap(vma->vm_prev);
				gap_end = vm_start_gap(vma);
1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982
				goto check_current;
			}
		}
	}

check_highest:
	/* Check highest gap, which does not precede any rbtree node */
	gap_start = mm->highest_vm_end;
	gap_end = ULONG_MAX;  /* Only for VM_BUG_ON below */
	if (gap_start > high_limit)
		return -ENOMEM;

found:
	/* We found a suitable gap. Clip it with the original low_limit. */
	if (gap_start < info->low_limit)
		gap_start = info->low_limit;

	/* Adjust gap address to the desired alignment */
	gap_start += (info->align_offset - gap_start) & info->align_mask;

	VM_BUG_ON(gap_start + info->length > info->high_limit);
	VM_BUG_ON(gap_start + info->length > gap_end);
	return gap_start;
}

unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	unsigned long length, low_limit, high_limit, gap_start, gap_end;

	/* Adjust search length to account for worst case alignment overhead */
	length = info->length + info->align_mask;
	if (length < info->length)
		return -ENOMEM;

	/*
	 * Adjust search limits by the desired length.
	 * See implementation comment at top of unmapped_area().
	 */
	gap_end = info->high_limit;
	if (gap_end < length)
		return -ENOMEM;
	high_limit = gap_end - length;

	if (info->low_limit > high_limit)
		return -ENOMEM;
	low_limit = info->low_limit + length;

	/* Check highest gap, which does not precede any rbtree node */
	gap_start = mm->highest_vm_end;
	if (gap_start <= high_limit)
		goto found_highest;

	/* Check if rbtree root looks promising */
	if (RB_EMPTY_ROOT(&mm->mm_rb))
		return -ENOMEM;
	vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
	if (vma->rb_subtree_gap < length)
		return -ENOMEM;

	while (true) {
		/* Visit right subtree if it looks promising */
1983
		gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995
		if (gap_start <= high_limit && vma->vm_rb.rb_right) {
			struct vm_area_struct *right =
				rb_entry(vma->vm_rb.rb_right,
					 struct vm_area_struct, vm_rb);
			if (right->rb_subtree_gap >= length) {
				vma = right;
				continue;
			}
		}

check_current:
		/* Check if current node has a suitable gap */
1996
		gap_end = vm_start_gap(vma);
1997 1998
		if (gap_end < low_limit)
			return -ENOMEM;
1999 2000
		if (gap_start <= high_limit &&
		    gap_end > gap_start && gap_end - gap_start >= length)
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022
			goto found;

		/* Visit left subtree if it looks promising */
		if (vma->vm_rb.rb_left) {
			struct vm_area_struct *left =
				rb_entry(vma->vm_rb.rb_left,
					 struct vm_area_struct, vm_rb);
			if (left->rb_subtree_gap >= length) {
				vma = left;
				continue;
			}
		}

		/* Go back up the rbtree to find next candidate node */
		while (true) {
			struct rb_node *prev = &vma->vm_rb;
			if (!rb_parent(prev))
				return -ENOMEM;
			vma = rb_entry(rb_parent(prev),
				       struct vm_area_struct, vm_rb);
			if (prev == vma->vm_rb.rb_right) {
				gap_start = vma->vm_prev ?
2023
					vm_end_gap(vma->vm_prev) : 0;
2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
				goto check_current;
			}
		}
	}

found:
	/* We found a suitable gap. Clip it with the original high_limit. */
	if (gap_end > info->high_limit)
		gap_end = info->high_limit;

found_highest:
	/* Compute highest gap address at the desired alignment */
	gap_end -= info->length;
	gap_end -= (gap_end - info->align_offset) & info->align_mask;

	VM_BUG_ON(gap_end < info->low_limit);
	VM_BUG_ON(gap_end < gap_start);
	return gap_end;
}

L
Linus Torvalds 已提交
2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060
/* Get an address range which is currently unmapped.
 * For shmat() with addr=0.
 *
 * Ugly calling convention alert:
 * Return value with the low bits set means error value,
 * ie
 *	if (ret & ~PAGE_MASK)
 *		error = ret;
 *
 * This function "knows" that -ENOMEM has the bits set.
 */
#ifndef HAVE_ARCH_UNMAPPED_AREA
unsigned long
arch_get_unmapped_area(struct file *filp, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct mm_struct *mm = current->mm;
2061
	struct vm_area_struct *vma, *prev;
2062
	struct vm_unmapped_area_info info;
L
Linus Torvalds 已提交
2063

2064
	if (len > TASK_SIZE - mmap_min_addr)
L
Linus Torvalds 已提交
2065 2066
		return -ENOMEM;

2067 2068 2069
	if (flags & MAP_FIXED)
		return addr;

L
Linus Torvalds 已提交
2070 2071
	if (addr) {
		addr = PAGE_ALIGN(addr);
2072
		vma = find_vma_prev(mm, addr, &prev);
2073
		if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
2074 2075
		    (!vma || addr + len <= vm_start_gap(vma)) &&
		    (!prev || addr >= vm_end_gap(prev)))
L
Linus Torvalds 已提交
2076 2077 2078
			return addr;
	}

2079 2080
	info.flags = 0;
	info.length = len;
2081
	info.low_limit = mm->mmap_base;
2082 2083 2084
	info.high_limit = TASK_SIZE;
	info.align_mask = 0;
	return vm_unmapped_area(&info);
L
Linus Torvalds 已提交
2085
}
V
vishnu.ps 已提交
2086
#endif
L
Linus Torvalds 已提交
2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097

/*
 * This mmap-allocator allocates new areas top-down from below the
 * stack's low limit (the base):
 */
#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
unsigned long
arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
			  const unsigned long len, const unsigned long pgoff,
			  const unsigned long flags)
{
2098
	struct vm_area_struct *vma, *prev;
L
Linus Torvalds 已提交
2099
	struct mm_struct *mm = current->mm;
2100 2101
	unsigned long addr = addr0;
	struct vm_unmapped_area_info info;
L
Linus Torvalds 已提交
2102 2103

	/* requested length too big for entire address space */
2104
	if (len > TASK_SIZE - mmap_min_addr)
L
Linus Torvalds 已提交
2105 2106
		return -ENOMEM;

2107 2108 2109
	if (flags & MAP_FIXED)
		return addr;

L
Linus Torvalds 已提交
2110 2111 2112
	/* requesting a specific address */
	if (addr) {
		addr = PAGE_ALIGN(addr);
2113
		vma = find_vma_prev(mm, addr, &prev);
2114
		if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
2115 2116
				(!vma || addr + len <= vm_start_gap(vma)) &&
				(!prev || addr >= vm_end_gap(prev)))
L
Linus Torvalds 已提交
2117 2118 2119
			return addr;
	}

2120 2121
	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	info.length = len;
2122
	info.low_limit = max(PAGE_SIZE, mmap_min_addr);
2123 2124 2125
	info.high_limit = mm->mmap_base;
	info.align_mask = 0;
	addr = vm_unmapped_area(&info);
2126

L
Linus Torvalds 已提交
2127 2128 2129 2130 2131 2132
	/*
	 * A failed mmap() very likely causes application failure,
	 * so fall back to the bottom-up function here. This scenario
	 * can happen with large stack limits and large mmap()
	 * allocations.
	 */
2133
	if (offset_in_page(addr)) {
2134 2135 2136 2137 2138 2139
		VM_BUG_ON(addr != -ENOMEM);
		info.flags = 0;
		info.low_limit = TASK_UNMAPPED_BASE;
		info.high_limit = TASK_SIZE;
		addr = vm_unmapped_area(&info);
	}
L
Linus Torvalds 已提交
2140 2141 2142 2143 2144 2145 2146 2147 2148

	return addr;
}
#endif

unsigned long
get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
2149 2150 2151
	unsigned long (*get_area)(struct file *, unsigned long,
				  unsigned long, unsigned long, unsigned long);

2152 2153 2154 2155 2156 2157 2158 2159
	unsigned long error = arch_mmap_check(addr, len, flags);
	if (error)
		return error;

	/* Careful about overflows.. */
	if (len > TASK_SIZE)
		return -ENOMEM;

2160
	get_area = current->mm->get_unmapped_area;
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173
	if (file) {
		if (file->f_op->get_unmapped_area)
			get_area = file->f_op->get_unmapped_area;
	} else if (flags & MAP_SHARED) {
		/*
		 * mmap_region() will call shmem_zero_setup() to create a file,
		 * so use shmem's get_unmapped_area in case it can be huge.
		 * do_mmap_pgoff() will clear pgoff, so match alignment.
		 */
		pgoff = 0;
		get_area = shmem_get_unmapped_area;
	}

2174 2175 2176
	addr = get_area(file, addr, len, pgoff, flags);
	if (IS_ERR_VALUE(addr))
		return addr;
L
Linus Torvalds 已提交
2177

2178 2179
	if (addr > TASK_SIZE - len)
		return -ENOMEM;
2180
	if (offset_in_page(addr))
2181
		return -EINVAL;
2182

2183 2184
	error = security_mmap_addr(addr);
	return error ? error : addr;
L
Linus Torvalds 已提交
2185 2186 2187 2188 2189
}

EXPORT_SYMBOL(get_unmapped_area);

/* Look up the first VMA which satisfies  addr < vm_end,  NULL if none. */
Z
ZhenwenXu 已提交
2190
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
L
Linus Torvalds 已提交
2191
{
D
Davidlohr Bueso 已提交
2192 2193
	struct rb_node *rb_node;
	struct vm_area_struct *vma;
L
Linus Torvalds 已提交
2194

2195
	/* Check the cache first. */
D
Davidlohr Bueso 已提交
2196 2197 2198
	vma = vmacache_find(mm, addr);
	if (likely(vma))
		return vma;
2199

D
Davidlohr Bueso 已提交
2200
	rb_node = mm->mm_rb.rb_node;
2201

D
Davidlohr Bueso 已提交
2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213
	while (rb_node) {
		struct vm_area_struct *tmp;

		tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);

		if (tmp->vm_end > addr) {
			vma = tmp;
			if (tmp->vm_start <= addr)
				break;
			rb_node = rb_node->rb_left;
		} else
			rb_node = rb_node->rb_right;
L
Linus Torvalds 已提交
2214
	}
D
Davidlohr Bueso 已提交
2215 2216 2217

	if (vma)
		vmacache_update(addr, vma);
L
Linus Torvalds 已提交
2218 2219 2220 2221 2222
	return vma;
}

EXPORT_SYMBOL(find_vma);

K
KOSAKI Motohiro 已提交
2223 2224 2225
/*
 * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
 */
L
Linus Torvalds 已提交
2226 2227 2228 2229
struct vm_area_struct *
find_vma_prev(struct mm_struct *mm, unsigned long addr,
			struct vm_area_struct **pprev)
{
K
KOSAKI Motohiro 已提交
2230
	struct vm_area_struct *vma;
L
Linus Torvalds 已提交
2231

K
KOSAKI Motohiro 已提交
2232
	vma = find_vma(mm, addr);
M
Mikulas Patocka 已提交
2233 2234 2235 2236 2237 2238 2239 2240 2241 2242
	if (vma) {
		*pprev = vma->vm_prev;
	} else {
		struct rb_node *rb_node = mm->mm_rb.rb_node;
		*pprev = NULL;
		while (rb_node) {
			*pprev = rb_entry(rb_node, struct vm_area_struct, vm_rb);
			rb_node = rb_node->rb_right;
		}
	}
K
KOSAKI Motohiro 已提交
2243
	return vma;
L
Linus Torvalds 已提交
2244 2245 2246 2247 2248 2249 2250
}

/*
 * Verify that the stack growth is acceptable and
 * update accounting. This is shared with both the
 * grow-up and grow-down cases.
 */
2251 2252
static int acct_stack_growth(struct vm_area_struct *vma,
			     unsigned long size, unsigned long grow)
L
Linus Torvalds 已提交
2253 2254
{
	struct mm_struct *mm = vma->vm_mm;
2255
	unsigned long new_start;
L
Linus Torvalds 已提交
2256 2257

	/* address space limit tests */
2258
	if (!may_expand_vm(mm, vma->vm_flags, grow))
L
Linus Torvalds 已提交
2259 2260 2261
		return -ENOMEM;

	/* Stack limit test */
2262
	if (size > rlimit(RLIMIT_STACK))
L
Linus Torvalds 已提交
2263 2264 2265 2266 2267 2268 2269
		return -ENOMEM;

	/* mlock limit tests */
	if (vma->vm_flags & VM_LOCKED) {
		unsigned long locked;
		unsigned long limit;
		locked = mm->locked_vm + grow;
2270
		limit = rlimit(RLIMIT_MEMLOCK);
J
Jiri Slaby 已提交
2271
		limit >>= PAGE_SHIFT;
L
Linus Torvalds 已提交
2272 2273 2274 2275
		if (locked > limit && !capable(CAP_IPC_LOCK))
			return -ENOMEM;
	}

2276 2277 2278 2279 2280 2281
	/* Check to ensure the stack will not grow into a hugetlb-only region */
	new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
			vma->vm_end - size;
	if (is_hugepage_only_range(vma->vm_mm, new_start, size))
		return -EFAULT;

L
Linus Torvalds 已提交
2282 2283 2284 2285
	/*
	 * Overcommit..  This must be the final test, as it will
	 * update security statistics.
	 */
2286
	if (security_vm_enough_memory_mm(mm, grow))
L
Linus Torvalds 已提交
2287 2288 2289 2290 2291
		return -ENOMEM;

	return 0;
}

H
Hugh Dickins 已提交
2292
#if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
L
Linus Torvalds 已提交
2293
/*
H
Hugh Dickins 已提交
2294 2295
 * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
 * vma is the last one with address > vma->vm_end.  Have to extend vma.
L
Linus Torvalds 已提交
2296
 */
H
Hugh Dickins 已提交
2297
int expand_upwards(struct vm_area_struct *vma, unsigned long address)
L
Linus Torvalds 已提交
2298
{
2299
	struct mm_struct *mm = vma->vm_mm;
2300 2301
	struct vm_area_struct *next;
	unsigned long gap_addr;
2302
	int error = 0;
L
Linus Torvalds 已提交
2303 2304 2305 2306

	if (!(vma->vm_flags & VM_GROWSUP))
		return -EFAULT;

2307
	/* Guard against exceeding limits of the address space. */
2308
	address &= PAGE_MASK;
2309
	if (address >= (TASK_SIZE & PAGE_MASK))
2310
		return -ENOMEM;
2311
	address += PAGE_SIZE;
2312

2313 2314
	/* Enforce stack_guard_gap */
	gap_addr = address + stack_guard_gap;
2315 2316 2317 2318 2319

	/* Guard against overflow */
	if (gap_addr < address || gap_addr > TASK_SIZE)
		gap_addr = TASK_SIZE;

2320
	next = vma->vm_next;
2321 2322
	if (next && next->vm_start < gap_addr &&
			(next->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) {
2323 2324 2325 2326 2327
		if (!(next->vm_flags & VM_GROWSUP))
			return -ENOMEM;
		/* Check that both stack segments have the same anon_vma? */
	}

2328
	/* We must make sure the anon_vma is allocated. */
L
Linus Torvalds 已提交
2329 2330 2331 2332 2333 2334 2335 2336
	if (unlikely(anon_vma_prepare(vma)))
		return -ENOMEM;

	/*
	 * vma->vm_start/vm_end cannot change under us because the caller
	 * is required to hold the mmap_sem in read mode.  We need the
	 * anon_vma lock to serialize against concurrent expand_stacks.
	 */
2337
	anon_vma_lock_write(vma->anon_vma);
L
Linus Torvalds 已提交
2338 2339 2340 2341 2342 2343 2344 2345

	/* Somebody else might have raced and expanded it already */
	if (address > vma->vm_end) {
		unsigned long size, grow;

		size = address - vma->vm_start;
		grow = (address - vma->vm_end) >> PAGE_SHIFT;

2346 2347 2348 2349
		error = -ENOMEM;
		if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
			error = acct_stack_growth(vma, size, grow);
			if (!error) {
2350 2351 2352 2353 2354
				/*
				 * vma_gap_update() doesn't support concurrent
				 * updates, but we only hold a shared mmap_sem
				 * lock here, so we need to protect against
				 * concurrent vma expansions.
2355
				 * anon_vma_lock_write() doesn't help here, as
2356 2357 2358 2359 2360
				 * we don't guarantee that all growable vmas
				 * in a mm share the same root anon vma.
				 * So, we reuse mm->page_table_lock to guard
				 * against concurrent vma expansions.
				 */
2361
				spin_lock(&mm->page_table_lock);
2362
				if (vma->vm_flags & VM_LOCKED)
2363
					mm->locked_vm += grow;
2364
				vm_stat_account(mm, vma->vm_flags, grow);
2365
				anon_vma_interval_tree_pre_update_vma(vma);
2366
				vma->vm_end = address;
2367
				anon_vma_interval_tree_post_update_vma(vma);
2368 2369 2370
				if (vma->vm_next)
					vma_gap_update(vma->vm_next);
				else
2371
					mm->highest_vm_end = vm_end_gap(vma);
2372
				spin_unlock(&mm->page_table_lock);
2373

2374 2375
				perf_event_mmap(vma);
			}
2376
		}
L
Linus Torvalds 已提交
2377
	}
2378
	anon_vma_unlock_write(vma->anon_vma);
2379
	khugepaged_enter_vma_merge(vma, vma->vm_flags);
2380
	validate_mm(mm);
L
Linus Torvalds 已提交
2381 2382
	return error;
}
H
Hugh Dickins 已提交
2383 2384
#endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */

L
Linus Torvalds 已提交
2385 2386 2387
/*
 * vma is the first one with address < vma->vm_start.  Have to extend vma.
 */
2388
int expand_downwards(struct vm_area_struct *vma,
2389
				   unsigned long address)
L
Linus Torvalds 已提交
2390
{
2391
	struct mm_struct *mm = vma->vm_mm;
2392
	struct vm_area_struct *prev;
L
Linus Torvalds 已提交
2393 2394
	int error;

2395
	address &= PAGE_MASK;
2396
	error = security_mmap_addr(address);
2397 2398 2399
	if (error)
		return error;

2400 2401
	/* Enforce stack_guard_gap */
	prev = vma->vm_prev;
2402 2403
	/* Check that both stack segments have the same anon_vma? */
	if (prev && !(prev->vm_flags & VM_GROWSDOWN) &&
2404
			(prev->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) {
2405
		if (address - prev->vm_end < stack_guard_gap)
2406 2407 2408
			return -ENOMEM;
	}

2409 2410 2411
	/* We must make sure the anon_vma is allocated. */
	if (unlikely(anon_vma_prepare(vma)))
		return -ENOMEM;
L
Linus Torvalds 已提交
2412 2413 2414 2415 2416 2417

	/*
	 * vma->vm_start/vm_end cannot change under us because the caller
	 * is required to hold the mmap_sem in read mode.  We need the
	 * anon_vma lock to serialize against concurrent expand_stacks.
	 */
2418
	anon_vma_lock_write(vma->anon_vma);
L
Linus Torvalds 已提交
2419 2420 2421 2422 2423 2424 2425 2426

	/* Somebody else might have raced and expanded it already */
	if (address < vma->vm_start) {
		unsigned long size, grow;

		size = vma->vm_end - address;
		grow = (vma->vm_start - address) >> PAGE_SHIFT;

2427 2428 2429 2430
		error = -ENOMEM;
		if (grow <= vma->vm_pgoff) {
			error = acct_stack_growth(vma, size, grow);
			if (!error) {
2431 2432 2433 2434 2435
				/*
				 * vma_gap_update() doesn't support concurrent
				 * updates, but we only hold a shared mmap_sem
				 * lock here, so we need to protect against
				 * concurrent vma expansions.
2436
				 * anon_vma_lock_write() doesn't help here, as
2437 2438 2439 2440 2441
				 * we don't guarantee that all growable vmas
				 * in a mm share the same root anon vma.
				 * So, we reuse mm->page_table_lock to guard
				 * against concurrent vma expansions.
				 */
2442
				spin_lock(&mm->page_table_lock);
2443
				if (vma->vm_flags & VM_LOCKED)
2444
					mm->locked_vm += grow;
2445
				vm_stat_account(mm, vma->vm_flags, grow);
2446
				anon_vma_interval_tree_pre_update_vma(vma);
2447 2448
				vma->vm_start = address;
				vma->vm_pgoff -= grow;
2449
				anon_vma_interval_tree_post_update_vma(vma);
2450
				vma_gap_update(vma);
2451
				spin_unlock(&mm->page_table_lock);
2452

2453 2454
				perf_event_mmap(vma);
			}
L
Linus Torvalds 已提交
2455 2456
		}
	}
2457
	anon_vma_unlock_write(vma->anon_vma);
2458
	khugepaged_enter_vma_merge(vma, vma->vm_flags);
2459
	validate_mm(mm);
L
Linus Torvalds 已提交
2460 2461 2462
	return error;
}

2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478
/* enforced gap between the expanding stack and other mappings. */
unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;

static int __init cmdline_parse_stack_guard_gap(char *p)
{
	unsigned long val;
	char *endptr;

	val = simple_strtoul(p, &endptr, 10);
	if (!*endptr)
		stack_guard_gap = val << PAGE_SHIFT;

	return 0;
}
__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);

2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493
#ifdef CONFIG_STACK_GROWSUP
int expand_stack(struct vm_area_struct *vma, unsigned long address)
{
	return expand_upwards(vma, address);
}

struct vm_area_struct *
find_extend_vma(struct mm_struct *mm, unsigned long addr)
{
	struct vm_area_struct *vma, *prev;

	addr &= PAGE_MASK;
	vma = find_vma_prev(mm, addr, &prev);
	if (vma && (vma->vm_start <= addr))
		return vma;
2494
	if (!prev || expand_stack(prev, addr))
2495
		return NULL;
2496
	if (prev->vm_flags & VM_LOCKED)
2497
		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
2498 2499 2500 2501 2502 2503 2504 2505
	return prev;
}
#else
int expand_stack(struct vm_area_struct *vma, unsigned long address)
{
	return expand_downwards(vma, address);
}

L
Linus Torvalds 已提交
2506
struct vm_area_struct *
V
vishnu.ps 已提交
2507
find_extend_vma(struct mm_struct *mm, unsigned long addr)
L
Linus Torvalds 已提交
2508
{
V
vishnu.ps 已提交
2509
	struct vm_area_struct *vma;
L
Linus Torvalds 已提交
2510 2511 2512
	unsigned long start;

	addr &= PAGE_MASK;
V
vishnu.ps 已提交
2513
	vma = find_vma(mm, addr);
L
Linus Torvalds 已提交
2514 2515 2516 2517 2518 2519 2520 2521 2522
	if (!vma)
		return NULL;
	if (vma->vm_start <= addr)
		return vma;
	if (!(vma->vm_flags & VM_GROWSDOWN))
		return NULL;
	start = vma->vm_start;
	if (expand_stack(vma, addr))
		return NULL;
2523
	if (vma->vm_flags & VM_LOCKED)
2524
		populate_vma_page_range(vma, addr, start, NULL);
L
Linus Torvalds 已提交
2525 2526 2527 2528
	return vma;
}
#endif

2529 2530
EXPORT_SYMBOL_GPL(find_extend_vma);

L
Linus Torvalds 已提交
2531
/*
2532
 * Ok - we have the memory areas we should free on the vma list,
L
Linus Torvalds 已提交
2533
 * so release them, and do the vma updates.
2534 2535
 *
 * Called with the mm semaphore held.
L
Linus Torvalds 已提交
2536
 */
2537
static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
L
Linus Torvalds 已提交
2538
{
2539 2540
	unsigned long nr_accounted = 0;

2541 2542
	/* Update high watermark before we lower total_vm */
	update_hiwater_vm(mm);
L
Linus Torvalds 已提交
2543
	do {
2544 2545
		long nrpages = vma_pages(vma);

2546 2547
		if (vma->vm_flags & VM_ACCOUNT)
			nr_accounted += nrpages;
2548
		vm_stat_account(mm, vma->vm_flags, -nrpages);
2549
		vma = remove_vma(vma);
2550
	} while (vma);
2551
	vm_unacct_memory(nr_accounted);
L
Linus Torvalds 已提交
2552 2553 2554 2555 2556 2557
	validate_mm(mm);
}

/*
 * Get rid of page table information in the indicated region.
 *
2558
 * Called with the mm semaphore held.
L
Linus Torvalds 已提交
2559 2560
 */
static void unmap_region(struct mm_struct *mm,
2561 2562
		struct vm_area_struct *vma, struct vm_area_struct *prev,
		unsigned long start, unsigned long end)
L
Linus Torvalds 已提交
2563
{
V
vishnu.ps 已提交
2564
	struct vm_area_struct *next = prev ? prev->vm_next : mm->mmap;
P
Peter Zijlstra 已提交
2565
	struct mmu_gather tlb;
L
Linus Torvalds 已提交
2566 2567

	lru_add_drain();
2568
	tlb_gather_mmu(&tlb, mm, start, end);
2569
	update_hiwater_rss(mm);
2570
	unmap_vmas(&tlb, vma, start, end);
P
Peter Zijlstra 已提交
2571
	free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
2572
				 next ? next->vm_start : USER_PGTABLES_CEILING);
P
Peter Zijlstra 已提交
2573
	tlb_finish_mmu(&tlb, start, end);
L
Linus Torvalds 已提交
2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587
}

/*
 * Create a list of vma's touched by the unmap, removing them from the mm's
 * vma list as we go..
 */
static void
detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
	struct vm_area_struct *prev, unsigned long end)
{
	struct vm_area_struct **insertion_point;
	struct vm_area_struct *tail_vma = NULL;

	insertion_point = (prev ? &prev->vm_next : &mm->mmap);
2588
	vma->vm_prev = NULL;
L
Linus Torvalds 已提交
2589
	do {
2590
		vma_rb_erase(vma, &mm->mm_rb);
L
Linus Torvalds 已提交
2591 2592 2593 2594 2595
		mm->map_count--;
		tail_vma = vma;
		vma = vma->vm_next;
	} while (vma && vma->vm_start < end);
	*insertion_point = vma;
2596
	if (vma) {
2597
		vma->vm_prev = prev;
2598 2599
		vma_gap_update(vma);
	} else
2600
		mm->highest_vm_end = prev ? vm_end_gap(prev) : 0;
L
Linus Torvalds 已提交
2601
	tail_vma->vm_next = NULL;
D
Davidlohr Bueso 已提交
2602 2603 2604

	/* Kill the cache */
	vmacache_invalidate(mm);
L
Linus Torvalds 已提交
2605 2606 2607
}

/*
2608 2609
 * __split_vma() bypasses sysctl_max_map_count checking.  We use this where it
 * has already been checked or doesn't make sense to fail.
L
Linus Torvalds 已提交
2610
 */
2611 2612
int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
		unsigned long addr, int new_below)
L
Linus Torvalds 已提交
2613 2614
{
	struct vm_area_struct *new;
2615
	int err;
L
Linus Torvalds 已提交
2616

2617 2618 2619 2620 2621
	if (vma->vm_ops && vma->vm_ops->split) {
		err = vma->vm_ops->split(vma, addr);
		if (err)
			return err;
	}
L
Linus Torvalds 已提交
2622

2623
	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
L
Linus Torvalds 已提交
2624
	if (!new)
2625
		return -ENOMEM;
L
Linus Torvalds 已提交
2626 2627 2628 2629

	/* most fields are the same, copy all, and then fixup */
	*new = *vma;

2630 2631
	INIT_LIST_HEAD(&new->anon_vma_chain);

L
Linus Torvalds 已提交
2632 2633 2634 2635 2636 2637 2638
	if (new_below)
		new->vm_end = addr;
	else {
		new->vm_start = addr;
		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
	}

2639 2640
	err = vma_dup_policy(vma, new);
	if (err)
2641
		goto out_free_vma;
L
Linus Torvalds 已提交
2642

2643 2644
	err = anon_vma_clone(new, vma);
	if (err)
2645 2646
		goto out_free_mpol;

2647
	if (new->vm_file)
L
Linus Torvalds 已提交
2648 2649 2650 2651 2652 2653
		get_file(new->vm_file);

	if (new->vm_ops && new->vm_ops->open)
		new->vm_ops->open(new);

	if (new_below)
2654
		err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
L
Linus Torvalds 已提交
2655 2656
			((addr - new->vm_start) >> PAGE_SHIFT), new);
	else
2657
		err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
L
Linus Torvalds 已提交
2658

2659 2660 2661 2662 2663
	/* Success. */
	if (!err)
		return 0;

	/* Clean everything up if vma_adjust failed. */
2664 2665
	if (new->vm_ops && new->vm_ops->close)
		new->vm_ops->close(new);
2666
	if (new->vm_file)
2667
		fput(new->vm_file);
2668
	unlink_anon_vmas(new);
2669
 out_free_mpol:
2670
	mpol_put(vma_policy(new));
2671 2672 2673
 out_free_vma:
	kmem_cache_free(vm_area_cachep, new);
	return err;
L
Linus Torvalds 已提交
2674 2675
}

2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688
/*
 * Split a vma into two pieces at address 'addr', a new vma is allocated
 * either for the first part or the tail.
 */
int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
	      unsigned long addr, int new_below)
{
	if (mm->map_count >= sysctl_max_map_count)
		return -ENOMEM;

	return __split_vma(mm, vma, addr, new_below);
}

L
Linus Torvalds 已提交
2689 2690 2691 2692 2693
/* Munmap is split into 2 main parts -- this part which finds
 * what needs doing, and the areas themselves, which do the
 * work.  This now handles partial unmappings.
 * Jeremy Fitzhardinge <jeremy@goop.org>
 */
2694 2695
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
	      struct list_head *uf)
L
Linus Torvalds 已提交
2696 2697
{
	unsigned long end;
2698
	struct vm_area_struct *vma, *prev, *last;
L
Linus Torvalds 已提交
2699

2700
	if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
L
Linus Torvalds 已提交
2701 2702
		return -EINVAL;

V
vishnu.ps 已提交
2703 2704
	len = PAGE_ALIGN(len);
	if (len == 0)
L
Linus Torvalds 已提交
2705 2706 2707
		return -EINVAL;

	/* Find the first overlapping VMA */
2708
	vma = find_vma(mm, start);
2709
	if (!vma)
L
Linus Torvalds 已提交
2710
		return 0;
2711
	prev = vma->vm_prev;
2712
	/* we have  start < vma->vm_end  */
L
Linus Torvalds 已提交
2713 2714 2715

	/* if it doesn't overlap, we have nothing.. */
	end = start + len;
2716
	if (vma->vm_start >= end)
L
Linus Torvalds 已提交
2717 2718 2719 2720 2721 2722 2723 2724 2725
		return 0;

	/*
	 * If we need to split any vma, do it now to save pain later.
	 *
	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
	 * unmapped vm_area_struct will remain in use: so lower split_vma
	 * places tmp vma above, and higher split_vma places tmp vma below.
	 */
2726
	if (start > vma->vm_start) {
2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737
		int error;

		/*
		 * Make sure that map_count on return from munmap() will
		 * not exceed its limit; but let map_count go just above
		 * its limit temporarily, to help free resources as expected.
		 */
		if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
			return -ENOMEM;

		error = __split_vma(mm, vma, start, 0);
L
Linus Torvalds 已提交
2738 2739
		if (error)
			return error;
2740
		prev = vma;
L
Linus Torvalds 已提交
2741 2742 2743 2744 2745
	}

	/* Does it split the last one? */
	last = find_vma(mm, end);
	if (last && end > last->vm_start) {
2746
		int error = __split_vma(mm, last, end, 1);
L
Linus Torvalds 已提交
2747 2748 2749
		if (error)
			return error;
	}
V
vishnu.ps 已提交
2750
	vma = prev ? prev->vm_next : mm->mmap;
L
Linus Torvalds 已提交
2751

2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766
	if (unlikely(uf)) {
		/*
		 * If userfaultfd_unmap_prep returns an error the vmas
		 * will remain splitted, but userland will get a
		 * highly unexpected error anyway. This is no
		 * different than the case where the first of the two
		 * __split_vma fails, but we don't undo the first
		 * split, despite we could. This is unlikely enough
		 * failure that it's not worth optimizing it for.
		 */
		int error = userfaultfd_unmap_prep(vma, start, end, uf);
		if (error)
			return error;
	}

2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780
	/*
	 * unlock any mlock()ed ranges before detaching vmas
	 */
	if (mm->locked_vm) {
		struct vm_area_struct *tmp = vma;
		while (tmp && tmp->vm_start < end) {
			if (tmp->vm_flags & VM_LOCKED) {
				mm->locked_vm -= vma_pages(tmp);
				munlock_vma_pages_all(tmp);
			}
			tmp = tmp->vm_next;
		}
	}

L
Linus Torvalds 已提交
2781 2782 2783
	/*
	 * Remove the vma's, and unmap the actual pages
	 */
2784 2785
	detach_vmas_to_be_unmapped(mm, vma, prev, end);
	unmap_region(mm, vma, prev, start, end);
L
Linus Torvalds 已提交
2786

2787 2788
	arch_unmap(mm, vma, start, end);

L
Linus Torvalds 已提交
2789
	/* Fix up all other VM information */
2790
	remove_vma_list(mm, vma);
L
Linus Torvalds 已提交
2791 2792 2793 2794

	return 0;
}

A
Al Viro 已提交
2795
int vm_munmap(unsigned long start, size_t len)
L
Linus Torvalds 已提交
2796 2797
{
	int ret;
A
Al Viro 已提交
2798
	struct mm_struct *mm = current->mm;
2799
	LIST_HEAD(uf);
L
Linus Torvalds 已提交
2800

M
Michal Hocko 已提交
2801 2802 2803
	if (down_write_killable(&mm->mmap_sem))
		return -EINTR;

2804
	ret = do_munmap(mm, start, len, &uf);
L
Linus Torvalds 已提交
2805
	up_write(&mm->mmap_sem);
2806
	userfaultfd_unmap_complete(mm, &uf);
L
Linus Torvalds 已提交
2807 2808
	return ret;
}
2809 2810 2811 2812 2813
EXPORT_SYMBOL(vm_munmap);

SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
{
	profile_munmap(addr);
2814
	return vm_munmap(addr, len);
2815
}
L
Linus Torvalds 已提交
2816

2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830

/*
 * Emulation of deprecated remap_file_pages() syscall.
 */
SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
		unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
{

	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	unsigned long populate = 0;
	unsigned long ret = -EINVAL;
	struct file *file;

2831
	pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.rst.\n",
J
Joe Perches 已提交
2832
		     current->comm, current->pid);
2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845

	if (prot)
		return ret;
	start = start & PAGE_MASK;
	size = size & PAGE_MASK;

	if (start + size <= start)
		return ret;

	/* Does pgoff wrap? */
	if (pgoff + (size >> PAGE_SHIFT) < pgoff)
		return ret;

2846 2847 2848
	if (down_write_killable(&mm->mmap_sem))
		return -EINTR;

2849 2850 2851 2852 2853
	vma = find_vma(mm, start);

	if (!vma || !(vma->vm_flags & VM_SHARED))
		goto out;

2854
	if (start < vma->vm_start)
2855 2856
		goto out;

2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876
	if (start + size > vma->vm_end) {
		struct vm_area_struct *next;

		for (next = vma->vm_next; next; next = next->vm_next) {
			/* hole between vmas ? */
			if (next->vm_start != next->vm_prev->vm_end)
				goto out;

			if (next->vm_file != vma->vm_file)
				goto out;

			if (next->vm_flags != vma->vm_flags)
				goto out;

			if (start + size <= next->vm_end)
				break;
		}

		if (!next)
			goto out;
2877 2878 2879 2880 2881 2882 2883 2884 2885
	}

	prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
	prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
	prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;

	flags &= MAP_NONBLOCK;
	flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
	if (vma->vm_flags & VM_LOCKED) {
2886
		struct vm_area_struct *tmp;
2887
		flags |= MAP_LOCKED;
2888

2889
		/* drop PG_Mlocked flag for over-mapped range */
2890 2891
		for (tmp = vma; tmp->vm_start >= start + size;
				tmp = tmp->vm_next) {
2892 2893 2894 2895 2896 2897
			/*
			 * Split pmd and munlock page on the border
			 * of the range.
			 */
			vma_adjust_trans_huge(tmp, start, start + size, 0);

2898 2899 2900 2901
			munlock_vma_pages_range(tmp,
					max(tmp->vm_start, start),
					min(tmp->vm_end, start + size));
		}
2902 2903 2904 2905
	}

	file = get_file(vma->vm_file);
	ret = do_mmap_pgoff(vma->vm_file, start, size,
2906
			prot, flags, pgoff, &populate, NULL);
2907 2908 2909 2910 2911 2912 2913 2914 2915 2916
	fput(file);
out:
	up_write(&mm->mmap_sem);
	if (populate)
		mm_populate(ret, populate);
	if (!IS_ERR_VALUE(ret))
		ret = 0;
	return ret;
}

L
Linus Torvalds 已提交
2917 2918
static inline void verify_mm_writelocked(struct mm_struct *mm)
{
2919
#ifdef CONFIG_DEBUG_VM
L
Linus Torvalds 已提交
2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931
	if (unlikely(down_read_trylock(&mm->mmap_sem))) {
		WARN_ON(1);
		up_read(&mm->mmap_sem);
	}
#endif
}

/*
 *  this is really a simplified "do_mmap".  it only handles
 *  anonymous maps.  eventually we may be able to do some
 *  brk-specific accounting here.
 */
2932
static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, struct list_head *uf)
L
Linus Torvalds 已提交
2933
{
V
vishnu.ps 已提交
2934 2935
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma, *prev;
2936
	unsigned long len;
V
vishnu.ps 已提交
2937
	struct rb_node **rb_link, *rb_parent;
L
Linus Torvalds 已提交
2938
	pgoff_t pgoff = addr >> PAGE_SHIFT;
2939
	int error;
L
Linus Torvalds 已提交
2940

K
Kees Cook 已提交
2941 2942 2943
	len = PAGE_ALIGN(request);
	if (len < request)
		return -ENOMEM;
L
Linus Torvalds 已提交
2944
	if (!len)
2945
		return 0;
L
Linus Torvalds 已提交
2946

2947 2948 2949 2950
	/* Until we need other flags, refuse anything except VM_EXEC. */
	if ((flags & (~VM_EXEC)) != 0)
		return -EINVAL;
	flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
2951

A
Al Viro 已提交
2952
	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2953
	if (offset_in_page(error))
2954 2955
		return error;

2956 2957 2958
	error = mlock_future_check(mm, mm->def_flags, len);
	if (error)
		return error;
L
Linus Torvalds 已提交
2959 2960 2961 2962 2963 2964 2965 2966 2967 2968

	/*
	 * mm->mmap_sem is required to protect against another thread
	 * changing the mappings in case we sleep.
	 */
	verify_mm_writelocked(mm);

	/*
	 * Clear old maps.  this also does some error checking for us
	 */
2969 2970
	while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
			      &rb_parent)) {
2971
		if (do_munmap(mm, addr, len, uf))
L
Linus Torvalds 已提交
2972 2973 2974 2975
			return -ENOMEM;
	}

	/* Check against address space limits *after* clearing old maps... */
2976
	if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
L
Linus Torvalds 已提交
2977 2978 2979 2980 2981
		return -ENOMEM;

	if (mm->map_count > sysctl_max_map_count)
		return -ENOMEM;

2982
	if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
L
Linus Torvalds 已提交
2983 2984 2985
		return -ENOMEM;

	/* Can we just expand an old private anonymous mapping? */
2986
	vma = vma_merge(mm, prev, addr, addr + len, flags,
2987
			NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
2988
	if (vma)
L
Linus Torvalds 已提交
2989 2990 2991 2992 2993
		goto out;

	/*
	 * create a vma struct for an anonymous mapping
	 */
P
Pekka Enberg 已提交
2994
	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
L
Linus Torvalds 已提交
2995 2996 2997 2998 2999
	if (!vma) {
		vm_unacct_memory(len >> PAGE_SHIFT);
		return -ENOMEM;
	}

3000
	INIT_LIST_HEAD(&vma->anon_vma_chain);
L
Linus Torvalds 已提交
3001 3002 3003 3004 3005
	vma->vm_mm = mm;
	vma->vm_start = addr;
	vma->vm_end = addr + len;
	vma->vm_pgoff = pgoff;
	vma->vm_flags = flags;
3006
	vma->vm_page_prot = vm_get_page_prot(flags);
L
Linus Torvalds 已提交
3007 3008
	vma_link(mm, vma, prev, rb_link, rb_parent);
out:
3009
	perf_event_mmap(vma);
L
Linus Torvalds 已提交
3010
	mm->total_vm += len >> PAGE_SHIFT;
3011
	mm->data_vm += len >> PAGE_SHIFT;
3012 3013
	if (flags & VM_LOCKED)
		mm->locked_vm += (len >> PAGE_SHIFT);
3014
	vma->vm_flags |= VM_SOFTDIRTY;
3015
	return 0;
L
Linus Torvalds 已提交
3016 3017
}

3018
static int do_brk(unsigned long addr, unsigned long len, struct list_head *uf)
3019
{
3020
	return do_brk_flags(addr, len, 0, uf);
3021 3022 3023
}

int vm_brk_flags(unsigned long addr, unsigned long len, unsigned long flags)
3024 3025
{
	struct mm_struct *mm = current->mm;
3026
	int ret;
3027
	bool populate;
3028
	LIST_HEAD(uf);
3029

M
Michal Hocko 已提交
3030 3031 3032
	if (down_write_killable(&mm->mmap_sem))
		return -EINTR;

3033
	ret = do_brk_flags(addr, len, flags, &uf);
3034
	populate = ((mm->def_flags & VM_LOCKED) != 0);
3035
	up_write(&mm->mmap_sem);
3036
	userfaultfd_unmap_complete(mm, &uf);
3037
	if (populate && !ret)
3038
		mm_populate(addr, len);
3039 3040
	return ret;
}
3041 3042 3043 3044 3045 3046
EXPORT_SYMBOL(vm_brk_flags);

int vm_brk(unsigned long addr, unsigned long len)
{
	return vm_brk_flags(addr, len, 0);
}
3047
EXPORT_SYMBOL(vm_brk);
L
Linus Torvalds 已提交
3048 3049 3050 3051

/* Release all mmaps. */
void exit_mmap(struct mm_struct *mm)
{
P
Peter Zijlstra 已提交
3052
	struct mmu_gather tlb;
3053
	struct vm_area_struct *vma;
L
Linus Torvalds 已提交
3054 3055
	unsigned long nr_accounted = 0;

3056
	/* mm's last user has gone, and its about to be pulled down */
A
Andrea Arcangeli 已提交
3057
	mmu_notifier_release(mm);
3058

3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084
	if (unlikely(mm_is_oom_victim(mm))) {
		/*
		 * Manually reap the mm to free as much memory as possible.
		 * Then, as the oom reaper does, set MMF_OOM_SKIP to disregard
		 * this mm from further consideration.  Taking mm->mmap_sem for
		 * write after setting MMF_OOM_SKIP will guarantee that the oom
		 * reaper will not run on this mm again after mmap_sem is
		 * dropped.
		 *
		 * Nothing can be holding mm->mmap_sem here and the above call
		 * to mmu_notifier_release(mm) ensures mmu notifier callbacks in
		 * __oom_reap_task_mm() will not block.
		 *
		 * This needs to be done before calling munlock_vma_pages_all(),
		 * which clears VM_LOCKED, otherwise the oom reaper cannot
		 * reliably test it.
		 */
		mutex_lock(&oom_lock);
		__oom_reap_task_mm(mm);
		mutex_unlock(&oom_lock);

		set_bit(MMF_OOM_SKIP, &mm->flags);
		down_write(&mm->mmap_sem);
		up_write(&mm->mmap_sem);
	}

3085 3086 3087 3088 3089 3090 3091 3092
	if (mm->locked_vm) {
		vma = mm->mmap;
		while (vma) {
			if (vma->vm_flags & VM_LOCKED)
				munlock_vma_pages_all(vma);
			vma = vma->vm_next;
		}
	}
3093 3094 3095

	arch_exit_mmap(mm);

3096
	vma = mm->mmap;
3097 3098 3099
	if (!vma)	/* Can happen if dup_mmap() received an OOM */
		return;

L
Linus Torvalds 已提交
3100 3101
	lru_add_drain();
	flush_cache_mm(mm);
3102
	tlb_gather_mmu(&tlb, mm, 0, -1);
3103
	/* update_hiwater_rss(mm) here? but nobody should be looking */
3104
	/* Use -1 here to ensure all VMAs in the mm are unmapped */
3105
	unmap_vmas(&tlb, vma, 0, -1);
3106
	free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
3107
	tlb_finish_mmu(&tlb, 0, -1);
L
Linus Torvalds 已提交
3108 3109

	/*
3110 3111
	 * Walk the list again, actually closing and freeing it,
	 * with preemption enabled, without holding any MM locks.
L
Linus Torvalds 已提交
3112
	 */
3113 3114 3115
	while (vma) {
		if (vma->vm_flags & VM_ACCOUNT)
			nr_accounted += vma_pages(vma);
3116
		vma = remove_vma(vma);
3117 3118
	}
	vm_unacct_memory(nr_accounted);
L
Linus Torvalds 已提交
3119 3120 3121 3122
}

/* Insert vm structure into process list sorted by address
 * and into the inode's i_mmap tree.  If vm_file is non-NULL
3123
 * then i_mmap_rwsem is taken here.
L
Linus Torvalds 已提交
3124
 */
3125
int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
L
Linus Torvalds 已提交
3126
{
3127 3128
	struct vm_area_struct *prev;
	struct rb_node **rb_link, *rb_parent;
L
Linus Torvalds 已提交
3129

3130 3131 3132 3133 3134 3135 3136
	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
			   &prev, &rb_link, &rb_parent))
		return -ENOMEM;
	if ((vma->vm_flags & VM_ACCOUNT) &&
	     security_vm_enough_memory_mm(mm, vma_pages(vma)))
		return -ENOMEM;

L
Linus Torvalds 已提交
3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148
	/*
	 * The vm_pgoff of a purely anonymous vma should be irrelevant
	 * until its first write fault, when page's anon_vma and index
	 * are set.  But now set the vm_pgoff it will almost certainly
	 * end up with (unless mremap moves it elsewhere before that
	 * first wfault), so /proc/pid/maps tells a consistent story.
	 *
	 * By setting it to reflect the virtual start address of the
	 * vma, merges and splits can happen in a seamless way, just
	 * using the existing file pgoff checks and manipulations.
	 * Similarly in do_mmap_pgoff and in do_brk.
	 */
3149
	if (vma_is_anonymous(vma)) {
L
Linus Torvalds 已提交
3150 3151 3152
		BUG_ON(vma->anon_vma);
		vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
	}
3153

L
Linus Torvalds 已提交
3154 3155 3156 3157 3158 3159 3160 3161 3162
	vma_link(mm, vma, prev, rb_link, rb_parent);
	return 0;
}

/*
 * Copy the vma structure to a new location in the same mm,
 * prior to moving page table entries, to effect an mremap move.
 */
struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
3163 3164
	unsigned long addr, unsigned long len, pgoff_t pgoff,
	bool *need_rmap_locks)
L
Linus Torvalds 已提交
3165 3166 3167 3168 3169 3170
{
	struct vm_area_struct *vma = *vmap;
	unsigned long vma_start = vma->vm_start;
	struct mm_struct *mm = vma->vm_mm;
	struct vm_area_struct *new_vma, *prev;
	struct rb_node **rb_link, *rb_parent;
3171
	bool faulted_in_anon_vma = true;
L
Linus Torvalds 已提交
3172 3173 3174 3175 3176

	/*
	 * If anonymous vma has not yet been faulted, update new pgoff
	 * to match new location, to increase its chance of merging.
	 */
3177
	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
L
Linus Torvalds 已提交
3178
		pgoff = addr >> PAGE_SHIFT;
3179 3180
		faulted_in_anon_vma = false;
	}
L
Linus Torvalds 已提交
3181

3182 3183
	if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
		return NULL;	/* should never get here */
L
Linus Torvalds 已提交
3184
	new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
3185 3186
			    vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
			    vma->vm_userfaultfd_ctx);
L
Linus Torvalds 已提交
3187 3188 3189 3190
	if (new_vma) {
		/*
		 * Source vma may have been merged into new_vma
		 */
3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204
		if (unlikely(vma_start >= new_vma->vm_start &&
			     vma_start < new_vma->vm_end)) {
			/*
			 * The only way we can get a vma_merge with
			 * self during an mremap is if the vma hasn't
			 * been faulted in yet and we were allowed to
			 * reset the dst vma->vm_pgoff to the
			 * destination address of the mremap to allow
			 * the merge to happen. mremap must change the
			 * vm_pgoff linearity between src and dst vmas
			 * (in turn preventing a vma_merge) to be
			 * safe. It is only safe to keep the vm_pgoff
			 * linear if there are no pages mapped yet.
			 */
3205
			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
3206
			*vmap = vma = new_vma;
3207
		}
3208
		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
L
Linus Torvalds 已提交
3209
	} else {
3210
		new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227
		if (!new_vma)
			goto out;
		*new_vma = *vma;
		new_vma->vm_start = addr;
		new_vma->vm_end = addr + len;
		new_vma->vm_pgoff = pgoff;
		if (vma_dup_policy(vma, new_vma))
			goto out_free_vma;
		INIT_LIST_HEAD(&new_vma->anon_vma_chain);
		if (anon_vma_clone(new_vma, vma))
			goto out_free_mempol;
		if (new_vma->vm_file)
			get_file(new_vma->vm_file);
		if (new_vma->vm_ops && new_vma->vm_ops->open)
			new_vma->vm_ops->open(new_vma);
		vma_link(mm, new_vma, prev, rb_link, rb_parent);
		*need_rmap_locks = false;
L
Linus Torvalds 已提交
3228 3229
	}
	return new_vma;
3230

3231
out_free_mempol:
3232
	mpol_put(vma_policy(new_vma));
3233
out_free_vma:
3234
	kmem_cache_free(vm_area_cachep, new_vma);
3235
out:
3236
	return NULL;
L
Linus Torvalds 已提交
3237
}
A
akpm@osdl.org 已提交
3238 3239 3240 3241 3242

/*
 * Return true if the calling process may expand its vm space by the passed
 * number of pages
 */
3243
bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages)
A
akpm@osdl.org 已提交
3244
{
3245 3246
	if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
		return false;
A
akpm@osdl.org 已提交
3247

3248 3249
	if (is_data_mapping(flags) &&
	    mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
3250 3251 3252 3253
		/* Workaround for Valgrind */
		if (rlimit(RLIMIT_DATA) == 0 &&
		    mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT)
			return true;
3254 3255 3256 3257 3258 3259 3260 3261

		pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n",
			     current->comm, current->pid,
			     (mm->data_vm + npages) << PAGE_SHIFT,
			     rlimit(RLIMIT_DATA),
			     ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data");

		if (!ignore_rlimit_data)
3262 3263
			return false;
	}
A
akpm@osdl.org 已提交
3264

3265 3266 3267 3268 3269 3270 3271
	return true;
}

void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
{
	mm->total_vm += npages;

3272
	if (is_exec_mapping(flags))
3273
		mm->exec_vm += npages;
3274
	else if (is_stack_mapping(flags))
3275
		mm->stack_vm += npages;
3276
	else if (is_data_mapping(flags))
3277
		mm->data_vm += npages;
A
akpm@osdl.org 已提交
3278
}
3279

3280
static vm_fault_t special_mapping_fault(struct vm_fault *vmf);
3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293

/*
 * Having a close hook prevents vma merging regardless of flags.
 */
static void special_mapping_close(struct vm_area_struct *vma)
{
}

static const char *special_mapping_name(struct vm_area_struct *vma)
{
	return ((struct vm_special_mapping *)vma->vm_private_data)->name;
}

3294 3295 3296 3297
static int special_mapping_mremap(struct vm_area_struct *new_vma)
{
	struct vm_special_mapping *sm = new_vma->vm_private_data;

3298 3299 3300
	if (WARN_ON_ONCE(current->mm != new_vma->vm_mm))
		return -EFAULT;

3301 3302
	if (sm->mremap)
		return sm->mremap(sm, new_vma);
3303

3304 3305 3306
	return 0;
}

3307 3308 3309
static const struct vm_operations_struct special_mapping_vmops = {
	.close = special_mapping_close,
	.fault = special_mapping_fault,
3310
	.mremap = special_mapping_mremap,
3311 3312 3313 3314 3315 3316 3317
	.name = special_mapping_name,
};

static const struct vm_operations_struct legacy_special_mapping_vmops = {
	.close = special_mapping_close,
	.fault = special_mapping_fault,
};
3318

3319
static vm_fault_t special_mapping_fault(struct vm_fault *vmf)
3320
{
3321
	struct vm_area_struct *vma = vmf->vma;
N
Nick Piggin 已提交
3322
	pgoff_t pgoff;
3323 3324
	struct page **pages;

3325
	if (vma->vm_ops == &legacy_special_mapping_vmops) {
3326
		pages = vma->vm_private_data;
3327 3328 3329 3330
	} else {
		struct vm_special_mapping *sm = vma->vm_private_data;

		if (sm->fault)
3331
			return sm->fault(sm, vmf->vma, vmf);
3332 3333 3334

		pages = sm->pages;
	}
3335

3336
	for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
N
Nick Piggin 已提交
3337
		pgoff--;
3338 3339 3340 3341

	if (*pages) {
		struct page *page = *pages;
		get_page(page);
N
Nick Piggin 已提交
3342 3343
		vmf->page = page;
		return 0;
3344 3345
	}

N
Nick Piggin 已提交
3346
	return VM_FAULT_SIGBUS;
3347 3348
}

3349 3350 3351
static struct vm_area_struct *__install_special_mapping(
	struct mm_struct *mm,
	unsigned long addr, unsigned long len,
3352 3353
	unsigned long vm_flags, void *priv,
	const struct vm_operations_struct *ops)
3354
{
3355
	int ret;
3356 3357 3358 3359
	struct vm_area_struct *vma;

	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
	if (unlikely(vma == NULL))
3360
		return ERR_PTR(-ENOMEM);
3361

3362
	INIT_LIST_HEAD(&vma->anon_vma_chain);
3363 3364 3365 3366
	vma->vm_mm = mm;
	vma->vm_start = addr;
	vma->vm_end = addr + len;

3367
	vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY;
3368
	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
3369

3370 3371
	vma->vm_ops = ops;
	vma->vm_private_data = priv;
3372

3373 3374 3375
	ret = insert_vm_struct(mm, vma);
	if (ret)
		goto out;
3376

3377
	vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
3378

3379
	perf_event_mmap(vma);
3380

3381
	return vma;
3382 3383 3384

out:
	kmem_cache_free(vm_area_cachep, vma);
3385 3386 3387
	return ERR_PTR(ret);
}

3388 3389 3390 3391 3392 3393 3394 3395
bool vma_is_special_mapping(const struct vm_area_struct *vma,
	const struct vm_special_mapping *sm)
{
	return vma->vm_private_data == sm &&
		(vma->vm_ops == &special_mapping_vmops ||
		 vma->vm_ops == &legacy_special_mapping_vmops);
}

3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409
/*
 * Called with mm->mmap_sem held for writing.
 * Insert a new vma covering the given region, with the given flags.
 * Its pages are supplied by the given array of struct page *.
 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
 * The region past the last page supplied will always produce SIGBUS.
 * The array pointer and the pages it points to are assumed to stay alive
 * for as long as this mapping might exist.
 */
struct vm_area_struct *_install_special_mapping(
	struct mm_struct *mm,
	unsigned long addr, unsigned long len,
	unsigned long vm_flags, const struct vm_special_mapping *spec)
{
3410 3411
	return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
					&special_mapping_vmops);
3412 3413
}

3414 3415 3416 3417
int install_special_mapping(struct mm_struct *mm,
			    unsigned long addr, unsigned long len,
			    unsigned long vm_flags, struct page **pages)
{
3418
	struct vm_area_struct *vma = __install_special_mapping(
3419 3420
		mm, addr, len, vm_flags, (void *)pages,
		&legacy_special_mapping_vmops);
3421

3422
	return PTR_ERR_OR_ZERO(vma);
3423
}
3424 3425 3426

static DEFINE_MUTEX(mm_all_locks_mutex);

3427
static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
3428
{
3429
	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
3430 3431 3432 3433
		/*
		 * The LSB of head.next can't change from under us
		 * because we hold the mm_all_locks_mutex.
		 */
3434
		down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem);
3435 3436
		/*
		 * We can safely modify head.next after taking the
3437
		 * anon_vma->root->rwsem. If some other vma in this mm shares
3438 3439 3440 3441
		 * the same anon_vma we won't take it again.
		 *
		 * No need of atomic instructions here, head.next
		 * can't change from under us thanks to the
3442
		 * anon_vma->root->rwsem.
3443 3444
		 */
		if (__test_and_set_bit(0, (unsigned long *)
3445
				       &anon_vma->root->rb_root.rb_root.rb_node))
3446 3447 3448 3449
			BUG();
	}
}

3450
static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463
{
	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
		/*
		 * AS_MM_ALL_LOCKS can't change from under us because
		 * we hold the mm_all_locks_mutex.
		 *
		 * Operations on ->flags have to be atomic because
		 * even if AS_MM_ALL_LOCKS is stable thanks to the
		 * mm_all_locks_mutex, there may be other cpus
		 * changing other bitflags in parallel to us.
		 */
		if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
			BUG();
3464
		down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_sem);
3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478
	}
}

/*
 * This operation locks against the VM for all pte/vma/mm related
 * operations that could ever happen on a certain mm. This includes
 * vmtruncate, try_to_unmap, and all page faults.
 *
 * The caller must take the mmap_sem in write mode before calling
 * mm_take_all_locks(). The caller isn't allowed to release the
 * mmap_sem until mm_drop_all_locks() returns.
 *
 * mmap_sem in write mode is required in order to block all operations
 * that could modify pagetables and free pages without need of
3479
 * altering the vma layout. It's also needed in write mode to avoid new
3480 3481 3482 3483 3484
 * anon_vmas to be associated with existing vmas.
 *
 * A single task can't take more than one mm_take_all_locks() in a row
 * or it would deadlock.
 *
3485
 * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3486 3487 3488
 * mapping->flags avoid to take the same lock twice, if more than one
 * vma in this mm is backed by the same anon_vma or address_space.
 *
3489 3490 3491 3492 3493 3494 3495 3496 3497 3498
 * We take locks in following order, accordingly to comment at beginning
 * of mm/rmap.c:
 *   - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
 *     hugetlb mapping);
 *   - all i_mmap_rwsem locks;
 *   - all anon_vma->rwseml
 *
 * We can take all locks within these types randomly because the VM code
 * doesn't nest them and we protected from parallel mm_take_all_locks() by
 * mm_all_locks_mutex.
3499 3500 3501 3502 3503 3504 3505 3506 3507
 *
 * mm_take_all_locks() and mm_drop_all_locks are expensive operations
 * that may have to take thousand of locks.
 *
 * mm_take_all_locks() can fail if it's interrupted by signals.
 */
int mm_take_all_locks(struct mm_struct *mm)
{
	struct vm_area_struct *vma;
3508
	struct anon_vma_chain *avc;
3509 3510 3511 3512 3513 3514 3515 3516

	BUG_ON(down_read_trylock(&mm->mmap_sem));

	mutex_lock(&mm_all_locks_mutex);

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (signal_pending(current))
			goto out_unlock;
3517 3518 3519 3520 3521 3522 3523 3524 3525 3526
		if (vma->vm_file && vma->vm_file->f_mapping &&
				is_vm_hugetlb_page(vma))
			vm_lock_mapping(mm, vma->vm_file->f_mapping);
	}

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (signal_pending(current))
			goto out_unlock;
		if (vma->vm_file && vma->vm_file->f_mapping &&
				!is_vm_hugetlb_page(vma))
3527
			vm_lock_mapping(mm, vma->vm_file->f_mapping);
3528
	}
3529 3530 3531 3532 3533

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (signal_pending(current))
			goto out_unlock;
		if (vma->anon_vma)
3534 3535
			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
				vm_lock_anon_vma(mm, avc->anon_vma);
3536
	}
3537

3538
	return 0;
3539 3540

out_unlock:
3541 3542
	mm_drop_all_locks(mm);
	return -EINTR;
3543 3544 3545 3546
}

static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
{
3547
	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
3548 3549 3550 3551 3552
		/*
		 * The LSB of head.next can't change to 0 from under
		 * us because we hold the mm_all_locks_mutex.
		 *
		 * We must however clear the bitflag before unlocking
3553
		 * the vma so the users using the anon_vma->rb_root will
3554 3555 3556 3557
		 * never see our bitflag.
		 *
		 * No need of atomic instructions here, head.next
		 * can't change from under us until we release the
3558
		 * anon_vma->root->rwsem.
3559 3560
		 */
		if (!__test_and_clear_bit(0, (unsigned long *)
3561
					  &anon_vma->root->rb_root.rb_root.rb_node))
3562
			BUG();
3563
		anon_vma_unlock_write(anon_vma);
3564 3565 3566 3567 3568 3569 3570 3571 3572 3573
	}
}

static void vm_unlock_mapping(struct address_space *mapping)
{
	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
		/*
		 * AS_MM_ALL_LOCKS can't change to 0 from under us
		 * because we hold the mm_all_locks_mutex.
		 */
3574
		i_mmap_unlock_write(mapping);
3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587
		if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
					&mapping->flags))
			BUG();
	}
}

/*
 * The mmap_sem cannot be released by the caller until
 * mm_drop_all_locks() returns.
 */
void mm_drop_all_locks(struct mm_struct *mm)
{
	struct vm_area_struct *vma;
3588
	struct anon_vma_chain *avc;
3589 3590 3591 3592 3593 3594

	BUG_ON(down_read_trylock(&mm->mmap_sem));
	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (vma->anon_vma)
3595 3596
			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
				vm_unlock_anon_vma(avc->anon_vma);
3597 3598 3599 3600 3601 3602
		if (vma->vm_file && vma->vm_file->f_mapping)
			vm_unlock_mapping(vma->vm_file->f_mapping);
	}

	mutex_unlock(&mm_all_locks_mutex);
}
3603 3604

/*
S
seokhoon.yoon 已提交
3605
 * initialise the percpu counter for VM
3606 3607 3608
 */
void __init mmap_init(void)
{
3609 3610
	int ret;

3611
	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
3612
	VM_BUG_ON(ret);
3613
}
3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624

/*
 * Initialise sysctl_user_reserve_kbytes.
 *
 * This is intended to prevent a user from starting a single memory hogging
 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
 * mode.
 *
 * The default value is min(3% of free memory, 128MB)
 * 128MB is enough to recover with sshd/login, bash, and top/kill.
 */
3625
static int init_user_reserve(void)
3626 3627 3628
{
	unsigned long free_kbytes;

3629
	free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3630 3631 3632 3633

	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
	return 0;
}
3634
subsys_initcall(init_user_reserve);
3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645

/*
 * Initialise sysctl_admin_reserve_kbytes.
 *
 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
 * to log in and kill a memory hogging process.
 *
 * Systems with more than 256MB will reserve 8MB, enough to recover
 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
 * only reserve 3% of free pages by default.
 */
3646
static int init_admin_reserve(void)
3647 3648 3649
{
	unsigned long free_kbytes;

3650
	free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3651 3652 3653 3654

	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
	return 0;
}
3655
subsys_initcall(init_admin_reserve);
3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693

/*
 * Reinititalise user and admin reserves if memory is added or removed.
 *
 * The default user reserve max is 128MB, and the default max for the
 * admin reserve is 8MB. These are usually, but not always, enough to
 * enable recovery from a memory hogging process using login/sshd, a shell,
 * and tools like top. It may make sense to increase or even disable the
 * reserve depending on the existence of swap or variations in the recovery
 * tools. So, the admin may have changed them.
 *
 * If memory is added and the reserves have been eliminated or increased above
 * the default max, then we'll trust the admin.
 *
 * If memory is removed and there isn't enough free memory, then we
 * need to reset the reserves.
 *
 * Otherwise keep the reserve set by the admin.
 */
static int reserve_mem_notifier(struct notifier_block *nb,
			     unsigned long action, void *data)
{
	unsigned long tmp, free_kbytes;

	switch (action) {
	case MEM_ONLINE:
		/* Default max is 128MB. Leave alone if modified by operator. */
		tmp = sysctl_user_reserve_kbytes;
		if (0 < tmp && tmp < (1UL << 17))
			init_user_reserve();

		/* Default max is 8MB.  Leave alone if modified by operator. */
		tmp = sysctl_admin_reserve_kbytes;
		if (0 < tmp && tmp < (1UL << 13))
			init_admin_reserve();

		break;
	case MEM_OFFLINE:
3694
		free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720

		if (sysctl_user_reserve_kbytes > free_kbytes) {
			init_user_reserve();
			pr_info("vm.user_reserve_kbytes reset to %lu\n",
				sysctl_user_reserve_kbytes);
		}

		if (sysctl_admin_reserve_kbytes > free_kbytes) {
			init_admin_reserve();
			pr_info("vm.admin_reserve_kbytes reset to %lu\n",
				sysctl_admin_reserve_kbytes);
		}
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block reserve_mem_nb = {
	.notifier_call = reserve_mem_notifier,
};

static int __meminit init_reserve_notifier(void)
{
	if (register_hotmemory_notifier(&reserve_mem_nb))
3721
		pr_err("Failed registering memory add/remove notifier for admin reserve\n");
3722 3723 3724

	return 0;
}
3725
subsys_initcall(init_reserve_notifier);