nommu.c 53.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8
/*
 *  linux/mm/nommu.c
 *
 *  Replacement code for mm functions to support CPU's that don't
 *  have any form of memory management unit (thus no virtual memory).
 *
 *  See Documentation/nommu-mmap.txt
 *
9
 *  Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
L
Linus Torvalds 已提交
10 11 12
 *  Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
 *  Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
 *  Copyright (c) 2002      Greg Ungerer <gerg@snapgear.com>
13
 *  Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
L
Linus Torvalds 已提交
14 15
 */

16 17
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

18
#include <linux/export.h>
L
Linus Torvalds 已提交
19
#include <linux/mm.h>
D
Davidlohr Bueso 已提交
20
#include <linux/vmacache.h>
L
Linus Torvalds 已提交
21 22 23 24 25 26 27 28 29
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/file.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
30
#include <linux/compiler.h>
L
Linus Torvalds 已提交
31 32 33 34
#include <linux/mount.h>
#include <linux/personality.h>
#include <linux/security.h>
#include <linux/syscalls.h>
A
Al Viro 已提交
35
#include <linux/audit.h>
36
#include <linux/sched/sysctl.h>
37
#include <linux/printk.h>
L
Linus Torvalds 已提交
38 39 40 41

#include <asm/uaccess.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
42
#include <asm/mmu_context.h>
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
#include "internal.h"

#if 0
#define kenter(FMT, ...) \
	printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
#define kleave(FMT, ...) \
	printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
#define kdebug(FMT, ...) \
	printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__)
#else
#define kenter(FMT, ...) \
	no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
#define kleave(FMT, ...) \
	no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
#define kdebug(FMT, ...) \
	no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__)
#endif
L
Linus Torvalds 已提交
60 61

void *high_memory;
62
EXPORT_SYMBOL(high_memory);
L
Linus Torvalds 已提交
63 64
struct page *mem_map;
unsigned long max_mapnr;
H
Hugh Dickins 已提交
65
unsigned long highest_memmap_pfn;
66
struct percpu_counter vm_committed_as;
L
Linus Torvalds 已提交
67 68
int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
int sysctl_overcommit_ratio = 50; /* default is 50% */
69
unsigned long sysctl_overcommit_kbytes __read_mostly;
L
Linus Torvalds 已提交
70
int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
71
int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
72
unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
73
unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
L
Linus Torvalds 已提交
74 75
int heap_stack_gap = 0;

76
atomic_long_t mmap_pages_allocated;
77

78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
/*
 * The global memory commitment made in the system can be a metric
 * that can be used to drive ballooning decisions when Linux is hosted
 * as a guest. On Hyper-V, the host implements a policy engine for dynamically
 * balancing memory across competing virtual machines that are hosted.
 * Several metrics drive this policy engine including the guest reported
 * memory commitment.
 */
unsigned long vm_memory_committed(void)
{
	return percpu_counter_read_positive(&vm_committed_as);
}

EXPORT_SYMBOL_GPL(vm_memory_committed);

L
Linus Torvalds 已提交
93 94
EXPORT_SYMBOL(mem_map);

95 96 97 98
/* list of mapped, potentially shareable regions */
static struct kmem_cache *vm_region_jar;
struct rb_root nommu_region_tree = RB_ROOT;
DECLARE_RWSEM(nommu_region_sem);
L
Linus Torvalds 已提交
99

100
const struct vm_operations_struct generic_file_vm_ops = {
L
Linus Torvalds 已提交
101 102 103 104 105 106 107 108 109 110 111 112
};

/*
 * Return the total memory allocated for this pointer, not
 * just what the caller asked for.
 *
 * Doesn't have to be accurate, i.e. may have races.
 */
unsigned int kobjsize(const void *objp)
{
	struct page *page;

113 114 115 116
	/*
	 * If the object we have should not have ksize performed on it,
	 * return size of 0
	 */
117
	if (!objp || !virt_addr_valid(objp))
118 119 120 121 122 123 124 125
		return 0;

	page = virt_to_head_page(objp);

	/*
	 * If the allocator sets PageSlab, we know the pointer came from
	 * kmalloc().
	 */
L
Linus Torvalds 已提交
126 127 128
	if (PageSlab(page))
		return ksize(objp);

129 130 131 132 133 134 135 136 137 138 139 140 141 142
	/*
	 * If it's not a compound page, see if we have a matching VMA
	 * region. This test is intentionally done in reverse order,
	 * so if there's no VMA, we still fall through and hand back
	 * PAGE_SIZE for 0-order pages.
	 */
	if (!PageCompound(page)) {
		struct vm_area_struct *vma;

		vma = find_vma(current->mm, (unsigned long)objp);
		if (vma)
			return vma->vm_end - vma->vm_start;
	}

143 144
	/*
	 * The ksize() function is only guaranteed to work for pointers
145
	 * returned by kmalloc(). So handle arbitrary pointers here.
146
	 */
147
	return PAGE_SIZE << compound_order(page);
L
Linus Torvalds 已提交
148 149
}

150 151 152 153
long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
		      unsigned long start, unsigned long nr_pages,
		      unsigned int foll_flags, struct page **pages,
		      struct vm_area_struct **vmas, int *nonblocking)
L
Linus Torvalds 已提交
154
{
155
	struct vm_area_struct *vma;
156 157 158 159
	unsigned long vm_flags;
	int i;

	/* calculate required read or write permissions.
H
Hugh Dickins 已提交
160
	 * If FOLL_FORCE is set, we only require the "MAY" flags.
161
	 */
H
Hugh Dickins 已提交
162 163 164 165
	vm_flags  = (foll_flags & FOLL_WRITE) ?
			(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
	vm_flags &= (foll_flags & FOLL_FORCE) ?
			(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
L
Linus Torvalds 已提交
166

167
	for (i = 0; i < nr_pages; i++) {
168
		vma = find_vma(mm, start);
169 170 171 172
		if (!vma)
			goto finish_or_fault;

		/* protect what we can, including chardevs */
H
Hugh Dickins 已提交
173 174
		if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
		    !(vm_flags & vma->vm_flags))
175
			goto finish_or_fault;
176

L
Linus Torvalds 已提交
177 178 179 180 181 182
		if (pages) {
			pages[i] = virt_to_page(start);
			if (pages[i])
				page_cache_get(pages[i]);
		}
		if (vmas)
183
			vmas[i] = vma;
184
		start = (start + PAGE_SIZE) & PAGE_MASK;
L
Linus Torvalds 已提交
185
	}
186 187 188 189 190

	return i;

finish_or_fault:
	return i ? : -EFAULT;
L
Linus Torvalds 已提交
191
}
N
Nick Piggin 已提交
192 193 194 195 196 197 198 199

/*
 * get a list of pages in an address range belonging to the specified process
 * and indicate the VMA that covers each page
 * - this is potentially dodgy as we may end incrementing the page count of a
 *   slab page or a secondary page from a compound page
 * - don't permit access to VMAs that don't support it, such as I/O mappings
 */
200 201 202 203
long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
		    unsigned long start, unsigned long nr_pages,
		    int write, int force, struct page **pages,
		    struct vm_area_struct **vmas)
N
Nick Piggin 已提交
204 205 206 207
{
	int flags = 0;

	if (write)
H
Hugh Dickins 已提交
208
		flags |= FOLL_WRITE;
N
Nick Piggin 已提交
209
	if (force)
H
Hugh Dickins 已提交
210
		flags |= FOLL_FORCE;
N
Nick Piggin 已提交
211

212 213
	return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas,
				NULL);
N
Nick Piggin 已提交
214
}
215 216
EXPORT_SYMBOL(get_user_pages);

217 218 219 220 221 222 223 224 225 226
long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
			   unsigned long start, unsigned long nr_pages,
			   int write, int force, struct page **pages,
			   int *locked)
{
	return get_user_pages(tsk, mm, start, nr_pages, write, force,
			      pages, NULL);
}
EXPORT_SYMBOL(get_user_pages_locked);

227 228 229 230
long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
			       unsigned long start, unsigned long nr_pages,
			       int write, int force, struct page **pages,
			       unsigned int gup_flags)
231 232 233 234 235 236 237 238
{
	long ret;
	down_read(&mm->mmap_sem);
	ret = get_user_pages(tsk, mm, start, nr_pages, write, force,
			     pages, NULL);
	up_read(&mm->mmap_sem);
	return ret;
}
239 240 241 242 243 244 245 246 247
EXPORT_SYMBOL(__get_user_pages_unlocked);

long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
			     unsigned long start, unsigned long nr_pages,
			     int write, int force, struct page **pages)
{
	return __get_user_pages_unlocked(tsk, mm, start, nr_pages, write,
					 force, pages, 0);
}
248 249
EXPORT_SYMBOL(get_user_pages_unlocked);

P
Paul Mundt 已提交
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
/**
 * follow_pfn - look up PFN at a user virtual address
 * @vma: memory mapping
 * @address: user virtual address
 * @pfn: location to store found PFN
 *
 * Only IO mappings and raw PFN mappings are allowed.
 *
 * Returns zero and the pfn at @pfn on success, -ve otherwise.
 */
int follow_pfn(struct vm_area_struct *vma, unsigned long address,
	unsigned long *pfn)
{
	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
		return -EINVAL;

	*pfn = address >> PAGE_SHIFT;
	return 0;
}
EXPORT_SYMBOL(follow_pfn);

271
LIST_HEAD(vmap_area_list);
L
Linus Torvalds 已提交
272

273
void vfree(const void *addr)
L
Linus Torvalds 已提交
274 275 276
{
	kfree(addr);
}
277
EXPORT_SYMBOL(vfree);
L
Linus Torvalds 已提交
278

A
Al Viro 已提交
279
void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
L
Linus Torvalds 已提交
280 281
{
	/*
282 283
	 *  You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
	 * returns only a logical address.
L
Linus Torvalds 已提交
284
	 */
285
	return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
L
Linus Torvalds 已提交
286
}
287
EXPORT_SYMBOL(__vmalloc);
L
Linus Torvalds 已提交
288

289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
void *vmalloc_user(unsigned long size)
{
	void *ret;

	ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
			PAGE_KERNEL);
	if (ret) {
		struct vm_area_struct *vma;

		down_write(&current->mm->mmap_sem);
		vma = find_vma(current->mm, (unsigned long)ret);
		if (vma)
			vma->vm_flags |= VM_USERMAP;
		up_write(&current->mm->mmap_sem);
	}

	return ret;
}
EXPORT_SYMBOL(vmalloc_user);

309
struct page *vmalloc_to_page(const void *addr)
L
Linus Torvalds 已提交
310 311 312
{
	return virt_to_page(addr);
}
313
EXPORT_SYMBOL(vmalloc_to_page);
L
Linus Torvalds 已提交
314

315
unsigned long vmalloc_to_pfn(const void *addr)
L
Linus Torvalds 已提交
316 317 318
{
	return page_to_pfn(virt_to_page(addr));
}
319
EXPORT_SYMBOL(vmalloc_to_pfn);
L
Linus Torvalds 已提交
320 321 322

long vread(char *buf, char *addr, unsigned long count)
{
323 324 325 326
	/* Don't allow overflow */
	if ((unsigned long) buf + count < count)
		count = -(unsigned long) buf;

L
Linus Torvalds 已提交
327 328 329 330 331 332 333 334 335 336 337
	memcpy(buf, addr, count);
	return count;
}

long vwrite(char *buf, char *addr, unsigned long count)
{
	/* Don't allow overflow */
	if ((unsigned long) addr + count < count)
		count = -(unsigned long) addr;

	memcpy(addr, buf, count);
338
	return count;
L
Linus Torvalds 已提交
339 340 341 342 343 344 345 346 347 348
}

/*
 *	vmalloc  -  allocate virtually continguos memory
 *
 *	@size:		allocation size
 *
 *	Allocate enough pages to cover @size from the page level
 *	allocator and map them into continguos kernel virtual space.
 *
349
 *	For tight control over page level allocator and protection flags
L
Linus Torvalds 已提交
350 351 352 353 354 355
 *	use __vmalloc() instead.
 */
void *vmalloc(unsigned long size)
{
       return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
}
356 357
EXPORT_SYMBOL(vmalloc);

358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
/*
 *	vzalloc - allocate virtually continguos memory with zero fill
 *
 *	@size:		allocation size
 *
 *	Allocate enough pages to cover @size from the page level
 *	allocator and map them into continguos kernel virtual space.
 *	The memory allocated is set to zero.
 *
 *	For tight control over page level allocator and protection flags
 *	use __vmalloc() instead.
 */
void *vzalloc(unsigned long size)
{
	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
			PAGE_KERNEL);
}
EXPORT_SYMBOL(vzalloc);

/**
 * vmalloc_node - allocate memory on a specific node
 * @size:	allocation size
 * @node:	numa node
 *
 * Allocate enough pages to cover @size from the page level
 * allocator and map them into contiguous kernel virtual space.
 *
 * For tight control over page level allocator and protection flags
 * use __vmalloc() instead.
 */
388 389 390 391
void *vmalloc_node(unsigned long size, int node)
{
	return vmalloc(size);
}
392
EXPORT_SYMBOL(vmalloc_node);
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410

/**
 * vzalloc_node - allocate memory on a specific node with zero fill
 * @size:	allocation size
 * @node:	numa node
 *
 * Allocate enough pages to cover @size from the page level
 * allocator and map them into contiguous kernel virtual space.
 * The memory allocated is set to zero.
 *
 * For tight control over page level allocator and protection flags
 * use __vmalloc() instead.
 */
void *vzalloc_node(unsigned long size, int node)
{
	return vzalloc(size);
}
EXPORT_SYMBOL(vzalloc_node);
L
Linus Torvalds 已提交
411

P
Paul Mundt 已提交
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
#ifndef PAGE_KERNEL_EXEC
# define PAGE_KERNEL_EXEC PAGE_KERNEL
#endif

/**
 *	vmalloc_exec  -  allocate virtually contiguous, executable memory
 *	@size:		allocation size
 *
 *	Kernel-internal function to allocate enough pages to cover @size
 *	the page level allocator and map them into contiguous and
 *	executable kernel virtual space.
 *
 *	For tight control over page level allocator and protection flags
 *	use __vmalloc() instead.
 */

void *vmalloc_exec(unsigned long size)
{
	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
}

433 434
/**
 * vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
L
Linus Torvalds 已提交
435 436 437 438 439 440 441 442 443
 *	@size:		allocation size
 *
 *	Allocate enough 32bit PA addressable pages to cover @size from the
 *	page level allocator and map them into continguos kernel virtual space.
 */
void *vmalloc_32(unsigned long size)
{
	return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
}
444 445 446 447 448 449 450 451
EXPORT_SYMBOL(vmalloc_32);

/**
 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
 *	@size:		allocation size
 *
 * The resulting memory area is 32bit addressable and zeroed so it can be
 * mapped to userspace without leaking data.
452 453 454
 *
 * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
 * remap_vmalloc_range() are permissible.
455 456 457
 */
void *vmalloc_32_user(unsigned long size)
{
458 459 460 461 462
	/*
	 * We'll have to sort out the ZONE_DMA bits for 64-bit,
	 * but for now this can simply use vmalloc_user() directly.
	 */
	return vmalloc_user(size);
463 464
}
EXPORT_SYMBOL(vmalloc_32_user);
L
Linus Torvalds 已提交
465 466 467 468 469 470

void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
{
	BUG();
	return NULL;
}
471
EXPORT_SYMBOL(vmap);
L
Linus Torvalds 已提交
472

473
void vunmap(const void *addr)
L
Linus Torvalds 已提交
474 475 476
{
	BUG();
}
477
EXPORT_SYMBOL(vunmap);
L
Linus Torvalds 已提交
478

479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
{
	BUG();
	return NULL;
}
EXPORT_SYMBOL(vm_map_ram);

void vm_unmap_ram(const void *mem, unsigned int count)
{
	BUG();
}
EXPORT_SYMBOL(vm_unmap_ram);

void vm_unmap_aliases(void)
{
}
EXPORT_SYMBOL_GPL(vm_unmap_aliases);

497 498 499 500
/*
 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
 * have one.
 */
501
void __weak vmalloc_sync_all(void)
502 503 504
{
}

505 506 507 508 509 510 511 512 513 514 515 516
/**
 *	alloc_vm_area - allocate a range of kernel address space
 *	@size:		size of the area
 *
 *	Returns:	NULL on failure, vm_struct on success
 *
 *	This function reserves a range of kernel address space, and
 *	allocates pagetables to map that range.  No actual mappings
 *	are created.  If the kernel address space is not shared
 *	between processes, it syncs the pagetable across all
 *	processes.
 */
517
struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
518 519 520 521 522 523 524 525 526 527 528 529
{
	BUG();
	return NULL;
}
EXPORT_SYMBOL_GPL(alloc_vm_area);

void free_vm_area(struct vm_struct *area)
{
	BUG();
}
EXPORT_SYMBOL_GPL(free_vm_area);

530 531 532 533 534 535 536
int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
		   struct page *page)
{
	return -EINVAL;
}
EXPORT_SYMBOL(vm_insert_page);

L
Linus Torvalds 已提交
537 538 539 540 541 542 543
/*
 *  sys_brk() for the most part doesn't need the global kernel
 *  lock, except when an application is doing something nasty
 *  like trying to un-brk an area that has already been mapped
 *  to a regular file.  in this case, the unmapping will need
 *  to invoke file system routines that need the global lock.
 */
544
SYSCALL_DEFINE1(brk, unsigned long, brk)
L
Linus Torvalds 已提交
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
{
	struct mm_struct *mm = current->mm;

	if (brk < mm->start_brk || brk > mm->context.end_brk)
		return mm->brk;

	if (mm->brk == brk)
		return mm->brk;

	/*
	 * Always allow shrinking brk
	 */
	if (brk <= mm->brk) {
		mm->brk = brk;
		return brk;
	}

	/*
	 * Ok, looks good - let it rip.
	 */
565
	flush_icache_range(mm->brk, brk);
L
Linus Torvalds 已提交
566 567 568
	return mm->brk = brk;
}

569 570 571 572
/*
 * initialise the VMA and region record slabs
 */
void __init mmap_init(void)
L
Linus Torvalds 已提交
573
{
574 575
	int ret;

576
	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
577
	VM_BUG_ON(ret);
578
	vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC);
L
Linus Torvalds 已提交
579 580
}

581
/*
582 583
 * validate the region tree
 * - the caller must hold the region lock
584
 */
585 586
#ifdef CONFIG_DEBUG_NOMMU_REGIONS
static noinline void validate_nommu_regions(void)
587
{
588 589
	struct vm_region *region, *last;
	struct rb_node *p, *lastp;
590

591 592 593 594 595
	lastp = rb_first(&nommu_region_tree);
	if (!lastp)
		return;

	last = rb_entry(lastp, struct vm_region, vm_rb);
596 597
	BUG_ON(unlikely(last->vm_end <= last->vm_start));
	BUG_ON(unlikely(last->vm_top < last->vm_end));
598 599 600 601 602

	while ((p = rb_next(lastp))) {
		region = rb_entry(p, struct vm_region, vm_rb);
		last = rb_entry(lastp, struct vm_region, vm_rb);

603 604 605
		BUG_ON(unlikely(region->vm_end <= region->vm_start));
		BUG_ON(unlikely(region->vm_top < region->vm_end));
		BUG_ON(unlikely(region->vm_start < last->vm_top));
606

607 608
		lastp = p;
	}
609
}
610
#else
611 612 613
static void validate_nommu_regions(void)
{
}
614
#endif
615 616

/*
617
 * add a region into the global tree
618
 */
619
static void add_nommu_region(struct vm_region *region)
620
{
621 622
	struct vm_region *pregion;
	struct rb_node **p, *parent;
623

624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
	validate_nommu_regions();

	parent = NULL;
	p = &nommu_region_tree.rb_node;
	while (*p) {
		parent = *p;
		pregion = rb_entry(parent, struct vm_region, vm_rb);
		if (region->vm_start < pregion->vm_start)
			p = &(*p)->rb_left;
		else if (region->vm_start > pregion->vm_start)
			p = &(*p)->rb_right;
		else if (pregion == region)
			return;
		else
			BUG();
639 640
	}

641 642
	rb_link_node(&region->vm_rb, parent, p);
	rb_insert_color(&region->vm_rb, &nommu_region_tree);
643

644
	validate_nommu_regions();
645 646
}

647
/*
648
 * delete a region from the global tree
649
 */
650
static void delete_nommu_region(struct vm_region *region)
651
{
652
	BUG_ON(!nommu_region_tree.rb_node);
653

654 655 656
	validate_nommu_regions();
	rb_erase(&region->vm_rb, &nommu_region_tree);
	validate_nommu_regions();
657 658
}

659
/*
660
 * free a contiguous series of pages
661
 */
662
static void free_page_series(unsigned long from, unsigned long to)
663
{
664 665 666 667
	for (; from < to; from += PAGE_SIZE) {
		struct page *page = virt_to_page(from);

		kdebug("- free %lx", from);
668
		atomic_long_dec(&mmap_pages_allocated);
669
		if (page_count(page) != 1)
670 671
			kdebug("free page %p: refcount not one: %d",
			       page, page_count(page));
672
		put_page(page);
673 674 675
	}
}

676
/*
677
 * release a reference to a region
678
 * - the caller must hold the region semaphore for writing, which this releases
679
 * - the region may not have been added to the tree yet, in which case vm_top
680
 *   will equal vm_start
681
 */
682 683
static void __put_nommu_region(struct vm_region *region)
	__releases(nommu_region_sem)
L
Linus Torvalds 已提交
684
{
685
	kenter("%p{%d}", region, region->vm_usage);
L
Linus Torvalds 已提交
686

687
	BUG_ON(!nommu_region_tree.rb_node);
L
Linus Torvalds 已提交
688

689
	if (--region->vm_usage == 0) {
690
		if (region->vm_top > region->vm_start)
691 692 693 694 695 696 697 698 699 700
			delete_nommu_region(region);
		up_write(&nommu_region_sem);

		if (region->vm_file)
			fput(region->vm_file);

		/* IO memory and memory shared directly out of the pagecache
		 * from ramfs/tmpfs mustn't be released here */
		if (region->vm_flags & VM_MAPPED_COPY) {
			kdebug("free series");
701
			free_page_series(region->vm_start, region->vm_top);
702 703 704 705
		}
		kmem_cache_free(vm_region_jar, region);
	} else {
		up_write(&nommu_region_sem);
L
Linus Torvalds 已提交
706
	}
707
}
L
Linus Torvalds 已提交
708

709 710 711 712 713 714 715
/*
 * release a reference to a region
 */
static void put_nommu_region(struct vm_region *region)
{
	down_write(&nommu_region_sem);
	__put_nommu_region(region);
L
Linus Torvalds 已提交
716 717
}

718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
/*
 * update protection on a vma
 */
static void protect_vma(struct vm_area_struct *vma, unsigned long flags)
{
#ifdef CONFIG_MPU
	struct mm_struct *mm = vma->vm_mm;
	long start = vma->vm_start & PAGE_MASK;
	while (start < vma->vm_end) {
		protect_page(mm, start, flags);
		start += PAGE_SIZE;
	}
	update_protections(mm);
#endif
}

734
/*
735 736 737 738
 * add a VMA into a process's mm_struct in the appropriate place in the list
 * and tree and add to the address space's page tree also if not an anonymous
 * page
 * - should be called with mm->mmap_sem held writelocked
739
 */
740
static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
L
Linus Torvalds 已提交
741
{
742
	struct vm_area_struct *pvma, *prev;
L
Linus Torvalds 已提交
743
	struct address_space *mapping;
744
	struct rb_node **p, *parent, *rb_prev;
745 746 747 748 749 750 751

	kenter(",%p", vma);

	BUG_ON(!vma->vm_region);

	mm->map_count++;
	vma->vm_mm = mm;
L
Linus Torvalds 已提交
752

753 754
	protect_vma(vma, vma->vm_flags);

L
Linus Torvalds 已提交
755 756 757 758
	/* add the VMA to the mapping */
	if (vma->vm_file) {
		mapping = vma->vm_file->f_mapping;

759
		i_mmap_lock_write(mapping);
L
Linus Torvalds 已提交
760
		flush_dcache_mmap_lock(mapping);
761
		vma_interval_tree_insert(vma, &mapping->i_mmap);
L
Linus Torvalds 已提交
762
		flush_dcache_mmap_unlock(mapping);
763
		i_mmap_unlock_write(mapping);
L
Linus Torvalds 已提交
764 765
	}

766
	/* add the VMA to the tree */
767
	parent = rb_prev = NULL;
768
	p = &mm->mm_rb.rb_node;
L
Linus Torvalds 已提交
769 770 771 772
	while (*p) {
		parent = *p;
		pvma = rb_entry(parent, struct vm_area_struct, vm_rb);

773 774 775
		/* sort by: start addr, end addr, VMA struct addr in that order
		 * (the latter is necessary as we may get identical VMAs) */
		if (vma->vm_start < pvma->vm_start)
L
Linus Torvalds 已提交
776
			p = &(*p)->rb_left;
777 778
		else if (vma->vm_start > pvma->vm_start) {
			rb_prev = parent;
L
Linus Torvalds 已提交
779
			p = &(*p)->rb_right;
780
		} else if (vma->vm_end < pvma->vm_end)
781
			p = &(*p)->rb_left;
782 783
		else if (vma->vm_end > pvma->vm_end) {
			rb_prev = parent;
784
			p = &(*p)->rb_right;
785
		} else if (vma < pvma)
786
			p = &(*p)->rb_left;
787 788
		else if (vma > pvma) {
			rb_prev = parent;
789
			p = &(*p)->rb_right;
790
		} else
791
			BUG();
L
Linus Torvalds 已提交
792 793 794
	}

	rb_link_node(&vma->vm_rb, parent, p);
795 796 797
	rb_insert_color(&vma->vm_rb, &mm->mm_rb);

	/* add VMA to the VMA list also */
798 799 800
	prev = NULL;
	if (rb_prev)
		prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
801

802
	__vma_link_list(mm, vma, prev, parent);
L
Linus Torvalds 已提交
803 804
}

805
/*
806
 * delete a VMA from its owning mm_struct and address space
807
 */
808
static void delete_vma_from_mm(struct vm_area_struct *vma)
L
Linus Torvalds 已提交
809
{
D
Davidlohr Bueso 已提交
810
	int i;
L
Linus Torvalds 已提交
811
	struct address_space *mapping;
812
	struct mm_struct *mm = vma->vm_mm;
D
Davidlohr Bueso 已提交
813
	struct task_struct *curr = current;
814 815 816

	kenter("%p", vma);

817 818
	protect_vma(vma, 0);

819
	mm->map_count--;
D
Davidlohr Bueso 已提交
820 821 822
	for (i = 0; i < VMACACHE_SIZE; i++) {
		/* if the vma is cached, invalidate the entire cache */
		if (curr->vmacache[i] == vma) {
823
			vmacache_invalidate(mm);
D
Davidlohr Bueso 已提交
824 825 826
			break;
		}
	}
L
Linus Torvalds 已提交
827 828 829 830 831

	/* remove the VMA from the mapping */
	if (vma->vm_file) {
		mapping = vma->vm_file->f_mapping;

832
		i_mmap_lock_write(mapping);
L
Linus Torvalds 已提交
833
		flush_dcache_mmap_lock(mapping);
834
		vma_interval_tree_remove(vma, &mapping->i_mmap);
L
Linus Torvalds 已提交
835
		flush_dcache_mmap_unlock(mapping);
836
		i_mmap_unlock_write(mapping);
L
Linus Torvalds 已提交
837 838
	}

839 840
	/* remove from the MM's tree and list */
	rb_erase(&vma->vm_rb, &mm->mm_rb);
841 842 843 844 845 846 847 848

	if (vma->vm_prev)
		vma->vm_prev->vm_next = vma->vm_next;
	else
		mm->mmap = vma->vm_next;

	if (vma->vm_next)
		vma->vm_next->vm_prev = vma->vm_prev;
849 850 851 852 853 854 855 856 857 858
}

/*
 * destroy a VMA record
 */
static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
{
	kenter("%p", vma);
	if (vma->vm_ops && vma->vm_ops->close)
		vma->vm_ops->close(vma);
859
	if (vma->vm_file)
860 861 862 863 864 865 866 867 868 869 870 871 872 873
		fput(vma->vm_file);
	put_nommu_region(vma->vm_region);
	kmem_cache_free(vm_area_cachep, vma);
}

/*
 * look up the first VMA in which addr resides, NULL if none
 * - should be called with mm->mmap_sem at least held readlocked
 */
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
{
	struct vm_area_struct *vma;

	/* check the cache first */
D
Davidlohr Bueso 已提交
874 875
	vma = vmacache_find(mm, addr);
	if (likely(vma))
876 877
		return vma;

878
	/* trawl the list (there may be multiple mappings in which addr
879
	 * resides) */
880
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
881 882 883
		if (vma->vm_start > addr)
			return NULL;
		if (vma->vm_end > addr) {
D
Davidlohr Bueso 已提交
884
			vmacache_update(addr, vma);
885 886 887 888 889 890 891 892 893 894 895 896 897 898
			return vma;
		}
	}

	return NULL;
}
EXPORT_SYMBOL(find_vma);

/*
 * find a VMA
 * - we don't extend stack VMAs under NOMMU conditions
 */
struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
{
899
	return find_vma(mm, addr);
900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
}

/*
 * expand a stack to a given address
 * - not supported under NOMMU conditions
 */
int expand_stack(struct vm_area_struct *vma, unsigned long address)
{
	return -ENOMEM;
}

/*
 * look up the first VMA exactly that exactly matches addr
 * - should be called with mm->mmap_sem at least held readlocked
 */
static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
					     unsigned long addr,
					     unsigned long len)
{
	struct vm_area_struct *vma;
	unsigned long end = addr + len;

	/* check the cache first */
D
Davidlohr Bueso 已提交
923 924
	vma = vmacache_find_exact(mm, addr, end);
	if (vma)
925 926
		return vma;

927
	/* trawl the list (there may be multiple mappings in which addr
928
	 * resides) */
929
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
930 931 932 933 934
		if (vma->vm_start < addr)
			continue;
		if (vma->vm_start > addr)
			return NULL;
		if (vma->vm_end == end) {
D
Davidlohr Bueso 已提交
935
			vmacache_update(addr, vma);
936 937 938 939 940
			return vma;
		}
	}

	return NULL;
L
Linus Torvalds 已提交
941 942 943 944 945 946 947 948 949 950 951 952 953 954
}

/*
 * determine whether a mapping should be permitted and, if so, what sort of
 * mapping we're capable of supporting
 */
static int validate_mmap_request(struct file *file,
				 unsigned long addr,
				 unsigned long len,
				 unsigned long prot,
				 unsigned long flags,
				 unsigned long pgoff,
				 unsigned long *_capabilities)
{
955
	unsigned long capabilities, rlen;
L
Linus Torvalds 已提交
956 957 958
	int ret;

	/* do the simple checks first */
959
	if (flags & MAP_FIXED) {
L
Linus Torvalds 已提交
960 961 962 963 964 965 966 967 968 969
		printk(KERN_DEBUG
		       "%d: Can't do fixed-address/overlay mmap of RAM\n",
		       current->pid);
		return -EINVAL;
	}

	if ((flags & MAP_TYPE) != MAP_PRIVATE &&
	    (flags & MAP_TYPE) != MAP_SHARED)
		return -EINVAL;

970
	if (!len)
L
Linus Torvalds 已提交
971 972
		return -EINVAL;

973
	/* Careful about overflows.. */
974 975
	rlen = PAGE_ALIGN(len);
	if (!rlen || rlen > TASK_SIZE)
976 977
		return -ENOMEM;

L
Linus Torvalds 已提交
978
	/* offset overflow? */
979
	if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
980
		return -EOVERFLOW;
L
Linus Torvalds 已提交
981 982 983 984 985 986

	if (file) {
		/* validate file mapping requests */
		struct address_space *mapping;

		/* files must support mmap */
A
Al Viro 已提交
987
		if (!file->f_op->mmap)
L
Linus Torvalds 已提交
988 989 990 991 992 993 994 995
			return -ENODEV;

		/* work out if what we've got could possibly be shared
		 * - we support chardevs that provide their own "memory"
		 * - we support files/blockdevs that are memory backed
		 */
		mapping = file->f_mapping;
		if (!mapping)
A
Al Viro 已提交
996
			mapping = file_inode(file)->i_mapping;
L
Linus Torvalds 已提交
997 998 999 1000 1001 1002 1003 1004

		capabilities = 0;
		if (mapping && mapping->backing_dev_info)
			capabilities = mapping->backing_dev_info->capabilities;

		if (!capabilities) {
			/* no explicit capabilities set, so assume some
			 * defaults */
A
Al Viro 已提交
1005
			switch (file_inode(file)->i_mode & S_IFMT) {
L
Linus Torvalds 已提交
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
			case S_IFREG:
			case S_IFBLK:
				capabilities = BDI_CAP_MAP_COPY;
				break;

			case S_IFCHR:
				capabilities =
					BDI_CAP_MAP_DIRECT |
					BDI_CAP_READ_MAP |
					BDI_CAP_WRITE_MAP;
				break;

			default:
				return -EINVAL;
			}
		}

		/* eliminate any capabilities that we can't support on this
		 * device */
		if (!file->f_op->get_unmapped_area)
			capabilities &= ~BDI_CAP_MAP_DIRECT;
		if (!file->f_op->read)
			capabilities &= ~BDI_CAP_MAP_COPY;

1030 1031 1032 1033
		/* The file shall have been opened with read permission. */
		if (!(file->f_mode & FMODE_READ))
			return -EACCES;

L
Linus Torvalds 已提交
1034 1035 1036 1037 1038 1039
		if (flags & MAP_SHARED) {
			/* do checks for writing, appending and locking */
			if ((prot & PROT_WRITE) &&
			    !(file->f_mode & FMODE_WRITE))
				return -EACCES;

A
Al Viro 已提交
1040
			if (IS_APPEND(file_inode(file)) &&
L
Linus Torvalds 已提交
1041 1042 1043
			    (file->f_mode & FMODE_WRITE))
				return -EACCES;

1044
			if (locks_verify_locked(file))
L
Linus Torvalds 已提交
1045 1046 1047 1048 1049 1050 1051
				return -EAGAIN;

			if (!(capabilities & BDI_CAP_MAP_DIRECT))
				return -ENODEV;

			/* we mustn't privatise shared mappings */
			capabilities &= ~BDI_CAP_MAP_COPY;
1052
		} else {
L
Linus Torvalds 已提交
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
			/* we're going to read the file into private memory we
			 * allocate */
			if (!(capabilities & BDI_CAP_MAP_COPY))
				return -ENODEV;

			/* we don't permit a private writable mapping to be
			 * shared with the backing device */
			if (prot & PROT_WRITE)
				capabilities &= ~BDI_CAP_MAP_DIRECT;
		}

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
		if (capabilities & BDI_CAP_MAP_DIRECT) {
			if (((prot & PROT_READ)  && !(capabilities & BDI_CAP_READ_MAP))  ||
			    ((prot & PROT_WRITE) && !(capabilities & BDI_CAP_WRITE_MAP)) ||
			    ((prot & PROT_EXEC)  && !(capabilities & BDI_CAP_EXEC_MAP))
			    ) {
				capabilities &= ~BDI_CAP_MAP_DIRECT;
				if (flags & MAP_SHARED) {
					printk(KERN_WARNING
					       "MAP_SHARED not completely supported on !MMU\n");
					return -EINVAL;
				}
			}
		}

L
Linus Torvalds 已提交
1078 1079
		/* handle executable mappings and implied executable
		 * mappings */
J
Josef Sipek 已提交
1080
		if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
L
Linus Torvalds 已提交
1081 1082
			if (prot & PROT_EXEC)
				return -EPERM;
1083
		} else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
L
Linus Torvalds 已提交
1084 1085 1086 1087 1088
			/* handle implication of PROT_EXEC by PROT_READ */
			if (current->personality & READ_IMPLIES_EXEC) {
				if (capabilities & BDI_CAP_EXEC_MAP)
					prot |= PROT_EXEC;
			}
1089
		} else if ((prot & PROT_READ) &&
L
Linus Torvalds 已提交
1090 1091 1092 1093 1094 1095
			 (prot & PROT_EXEC) &&
			 !(capabilities & BDI_CAP_EXEC_MAP)
			 ) {
			/* backing file is not executable, try to copy */
			capabilities &= ~BDI_CAP_MAP_DIRECT;
		}
1096
	} else {
L
Linus Torvalds 已提交
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
		/* anonymous mappings are always memory backed and can be
		 * privately mapped
		 */
		capabilities = BDI_CAP_MAP_COPY;

		/* handle PROT_EXEC implication by PROT_READ */
		if ((prot & PROT_READ) &&
		    (current->personality & READ_IMPLIES_EXEC))
			prot |= PROT_EXEC;
	}

	/* allow the security API to have its say */
1109
	ret = security_mmap_addr(addr);
L
Linus Torvalds 已提交
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
	if (ret < 0)
		return ret;

	/* looks okay */
	*_capabilities = capabilities;
	return 0;
}

/*
 * we've determined that we can make the mapping, now translate what we
 * now know into VMA flags
 */
static unsigned long determine_vm_flags(struct file *file,
					unsigned long prot,
					unsigned long flags,
					unsigned long capabilities)
{
	unsigned long vm_flags;

	vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags);
	/* vm_flags |= mm->def_flags; */

	if (!(capabilities & BDI_CAP_MAP_DIRECT)) {
		/* attempt to share read-only copies of mapped file chunks */
1134
		vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
L
Linus Torvalds 已提交
1135 1136
		if (file && !(prot & PROT_WRITE))
			vm_flags |= VM_MAYSHARE;
1137
	} else {
L
Linus Torvalds 已提交
1138 1139 1140
		/* overlay a shareable mapping on the backing device or inode
		 * if possible - used for chardevs, ramfs/tmpfs/shmfs and
		 * romfs/cramfs */
1141
		vm_flags |= VM_MAYSHARE | (capabilities & BDI_CAP_VMFLAGS);
L
Linus Torvalds 已提交
1142
		if (flags & MAP_SHARED)
1143
			vm_flags |= VM_SHARED;
L
Linus Torvalds 已提交
1144 1145 1146 1147 1148 1149
	}

	/* refuse to let anyone share private mappings with this process if
	 * it's being traced - otherwise breakpoints set in it may interfere
	 * with another untraced process
	 */
T
Tejun Heo 已提交
1150
	if ((flags & MAP_PRIVATE) && current->ptrace)
L
Linus Torvalds 已提交
1151 1152 1153 1154 1155 1156
		vm_flags &= ~VM_MAYSHARE;

	return vm_flags;
}

/*
1157 1158
 * set up a shared mapping on a file (the driver or filesystem provides and
 * pins the storage)
L
Linus Torvalds 已提交
1159
 */
1160
static int do_mmap_shared_file(struct vm_area_struct *vma)
L
Linus Torvalds 已提交
1161 1162 1163 1164
{
	int ret;

	ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
1165 1166
	if (ret == 0) {
		vma->vm_region->vm_top = vma->vm_region->vm_end;
1167
		return 0;
1168
	}
L
Linus Torvalds 已提交
1169 1170 1171
	if (ret != -ENOSYS)
		return ret;

1172 1173 1174
	/* getting -ENOSYS indicates that direct mmap isn't possible (as
	 * opposed to tried but failed) so we can only give a suitable error as
	 * it's not possible to make a private copy if MAP_SHARED was given */
L
Linus Torvalds 已提交
1175 1176 1177 1178 1179 1180
	return -ENODEV;
}

/*
 * set up a private mapping or an anonymous shared mapping
 */
1181 1182
static int do_mmap_private(struct vm_area_struct *vma,
			   struct vm_region *region,
1183 1184
			   unsigned long len,
			   unsigned long capabilities)
L
Linus Torvalds 已提交
1185
{
1186
	unsigned long total, point;
L
Linus Torvalds 已提交
1187
	void *base;
1188
	int ret, order;
L
Linus Torvalds 已提交
1189 1190 1191 1192 1193

	/* invoke the file's mapping function so that it can keep track of
	 * shared mappings on devices or memory
	 * - VM_MAYSHARE will be set if it may attempt to share
	 */
1194
	if (capabilities & BDI_CAP_MAP_DIRECT) {
L
Linus Torvalds 已提交
1195
		ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
1196
		if (ret == 0) {
L
Linus Torvalds 已提交
1197
			/* shouldn't return success if we're not sharing */
1198 1199
			BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
			vma->vm_region->vm_top = vma->vm_region->vm_end;
1200
			return 0;
L
Linus Torvalds 已提交
1201
		}
1202 1203
		if (ret != -ENOSYS)
			return ret;
L
Linus Torvalds 已提交
1204 1205 1206 1207 1208 1209

		/* getting an ENOSYS error indicates that direct mmap isn't
		 * possible (as opposed to tried but failed) so we'll try to
		 * make a private copy of the data and map that instead */
	}

1210

L
Linus Torvalds 已提交
1211 1212 1213 1214
	/* allocate some memory to hold the mapping
	 * - note that this may not return a page-aligned address if the object
	 *   we're allocating is smaller than a page
	 */
B
Bob Liu 已提交
1215
	order = get_order(len);
1216 1217 1218
	kdebug("alloc order %d for %lx", order, len);

	total = 1 << order;
B
Bob Liu 已提交
1219
	point = len >> PAGE_SHIFT;
1220

1221
	/* we don't want to allocate a power-of-2 sized page set */
1222
	if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages) {
1223 1224 1225 1226 1227
		total = point;
		kdebug("try to alloc exact %lu pages", total);
		base = alloc_pages_exact(len, GFP_KERNEL);
	} else {
		base = (void *)__get_free_pages(GFP_KERNEL, order);
1228 1229
	}

1230 1231 1232 1233
	if (!base)
		goto enomem;

	atomic_long_add(total, &mmap_pages_allocated);
L
Linus Torvalds 已提交
1234

1235 1236
	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
	region->vm_start = (unsigned long) base;
B
Bob Liu 已提交
1237
	region->vm_end   = region->vm_start + len;
1238
	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
1239 1240 1241

	vma->vm_start = region->vm_start;
	vma->vm_end   = region->vm_start + len;
L
Linus Torvalds 已提交
1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252

	if (vma->vm_file) {
		/* read the contents of a file into the copy */
		mm_segment_t old_fs;
		loff_t fpos;

		fpos = vma->vm_pgoff;
		fpos <<= PAGE_SHIFT;

		old_fs = get_fs();
		set_fs(KERNEL_DS);
B
Bob Liu 已提交
1253
		ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos);
L
Linus Torvalds 已提交
1254 1255 1256 1257 1258 1259
		set_fs(old_fs);

		if (ret < 0)
			goto error_free;

		/* clear the last little bit */
B
Bob Liu 已提交
1260 1261
		if (ret < len)
			memset(base + ret, 0, len - ret);
L
Linus Torvalds 已提交
1262 1263 1264 1265 1266 1267

	}

	return 0;

error_free:
1268
	free_page_series(region->vm_start, region->vm_top);
1269 1270
	region->vm_start = vma->vm_start = 0;
	region->vm_end   = vma->vm_end = 0;
1271
	region->vm_top   = 0;
L
Linus Torvalds 已提交
1272 1273 1274
	return ret;

enomem:
1275
	pr_err("Allocation of length %lu from process %d (%s) failed\n",
1276
	       len, current->pid, current->comm);
1277
	show_free_areas(0);
L
Linus Torvalds 已提交
1278 1279 1280 1281 1282 1283
	return -ENOMEM;
}

/*
 * handle mapping creation for uClinux
 */
1284
unsigned long do_mmap_pgoff(struct file *file,
L
Linus Torvalds 已提交
1285 1286 1287 1288
			    unsigned long addr,
			    unsigned long len,
			    unsigned long prot,
			    unsigned long flags,
1289
			    unsigned long pgoff,
1290
			    unsigned long *populate)
L
Linus Torvalds 已提交
1291
{
1292 1293
	struct vm_area_struct *vma;
	struct vm_region *region;
L
Linus Torvalds 已提交
1294
	struct rb_node *rb;
1295
	unsigned long capabilities, vm_flags, result;
L
Linus Torvalds 已提交
1296 1297
	int ret;

1298 1299
	kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff);

1300
	*populate = 0;
1301

L
Linus Torvalds 已提交
1302 1303 1304 1305
	/* decide whether we should attempt the mapping, and if so what sort of
	 * mapping */
	ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
				    &capabilities);
1306 1307
	if (ret < 0) {
		kleave(" = %d [val]", ret);
L
Linus Torvalds 已提交
1308
		return ret;
1309
	}
L
Linus Torvalds 已提交
1310

1311 1312
	/* we ignore the address hint */
	addr = 0;
B
Bob Liu 已提交
1313
	len = PAGE_ALIGN(len);
1314

L
Linus Torvalds 已提交
1315 1316 1317 1318
	/* we've determined that we can make the mapping, now translate what we
	 * now know into VMA flags */
	vm_flags = determine_vm_flags(file, prot, flags, capabilities);

1319 1320 1321 1322 1323 1324 1325 1326
	/* we're going to need to record the mapping */
	region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
	if (!region)
		goto error_getting_region;

	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
	if (!vma)
		goto error_getting_vma;
L
Linus Torvalds 已提交
1327

1328
	region->vm_usage = 1;
1329 1330 1331
	region->vm_flags = vm_flags;
	region->vm_pgoff = pgoff;

1332
	INIT_LIST_HEAD(&vma->anon_vma_chain);
1333 1334
	vma->vm_flags = vm_flags;
	vma->vm_pgoff = pgoff;
L
Linus Torvalds 已提交
1335

1336
	if (file) {
A
Al Viro 已提交
1337 1338
		region->vm_file = get_file(file);
		vma->vm_file = get_file(file);
1339 1340 1341 1342 1343
	}

	down_write(&nommu_region_sem);

	/* if we want to share, we need to check for regions created by other
L
Linus Torvalds 已提交
1344
	 * mmap() calls that overlap with our proposed mapping
1345
	 * - we can only share with a superset match on most regular files
L
Linus Torvalds 已提交
1346 1347 1348 1349 1350 1351
	 * - shared mappings on character devices and memory backed files are
	 *   permitted to overlap inexactly as far as we are concerned for in
	 *   these cases, sharing is handled in the driver or filesystem rather
	 *   than here
	 */
	if (vm_flags & VM_MAYSHARE) {
1352 1353
		struct vm_region *pregion;
		unsigned long pglen, rpglen, pgend, rpgend, start;
L
Linus Torvalds 已提交
1354

1355 1356
		pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
		pgend = pgoff + pglen;
1357

1358 1359
		for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
			pregion = rb_entry(rb, struct vm_region, vm_rb);
L
Linus Torvalds 已提交
1360

1361
			if (!(pregion->vm_flags & VM_MAYSHARE))
L
Linus Torvalds 已提交
1362 1363 1364
				continue;

			/* search for overlapping mappings on the same file */
A
Al Viro 已提交
1365 1366
			if (file_inode(pregion->vm_file) !=
			    file_inode(file))
L
Linus Torvalds 已提交
1367 1368
				continue;

1369
			if (pregion->vm_pgoff >= pgend)
L
Linus Torvalds 已提交
1370 1371
				continue;

1372 1373 1374 1375
			rpglen = pregion->vm_end - pregion->vm_start;
			rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
			rpgend = pregion->vm_pgoff + rpglen;
			if (pgoff >= rpgend)
L
Linus Torvalds 已提交
1376 1377
				continue;

1378 1379 1380 1381 1382
			/* handle inexactly overlapping matches between
			 * mappings */
			if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
			    !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
				/* new mapping is not a subset of the region */
L
Linus Torvalds 已提交
1383 1384 1385 1386 1387
				if (!(capabilities & BDI_CAP_MAP_DIRECT))
					goto sharing_violation;
				continue;
			}

1388
			/* we've found a region we can share */
1389
			pregion->vm_usage++;
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405
			vma->vm_region = pregion;
			start = pregion->vm_start;
			start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
			vma->vm_start = start;
			vma->vm_end = start + len;

			if (pregion->vm_flags & VM_MAPPED_COPY) {
				kdebug("share copy");
				vma->vm_flags |= VM_MAPPED_COPY;
			} else {
				kdebug("share mmap");
				ret = do_mmap_shared_file(vma);
				if (ret < 0) {
					vma->vm_region = NULL;
					vma->vm_start = 0;
					vma->vm_end = 0;
1406
					pregion->vm_usage--;
1407 1408 1409 1410 1411 1412 1413 1414 1415
					pregion = NULL;
					goto error_just_free;
				}
			}
			fput(region->vm_file);
			kmem_cache_free(vm_region_jar, region);
			region = pregion;
			result = start;
			goto share;
L
Linus Torvalds 已提交
1416 1417 1418 1419 1420 1421
		}

		/* obtain the address at which to make a shared mapping
		 * - this is the hook for quasi-memory character devices to
		 *   tell us the location of a shared mapping
		 */
1422
		if (capabilities & BDI_CAP_MAP_DIRECT) {
L
Linus Torvalds 已提交
1423 1424
			addr = file->f_op->get_unmapped_area(file, addr, len,
							     pgoff, flags);
1425
			if (IS_ERR_VALUE(addr)) {
L
Linus Torvalds 已提交
1426
				ret = addr;
1427
				if (ret != -ENOSYS)
1428
					goto error_just_free;
L
Linus Torvalds 已提交
1429 1430 1431 1432

				/* the driver refused to tell us where to site
				 * the mapping so we'll have to attempt to copy
				 * it */
1433
				ret = -ENODEV;
L
Linus Torvalds 已提交
1434
				if (!(capabilities & BDI_CAP_MAP_COPY))
1435
					goto error_just_free;
L
Linus Torvalds 已提交
1436 1437

				capabilities &= ~BDI_CAP_MAP_DIRECT;
1438 1439 1440
			} else {
				vma->vm_start = region->vm_start = addr;
				vma->vm_end = region->vm_end = addr + len;
L
Linus Torvalds 已提交
1441 1442 1443 1444
			}
		}
	}

1445
	vma->vm_region = region;
L
Linus Torvalds 已提交
1446

1447 1448 1449
	/* set up the mapping
	 * - the region is filled in if BDI_CAP_MAP_DIRECT is still set
	 */
L
Linus Torvalds 已提交
1450
	if (file && vma->vm_flags & VM_SHARED)
1451
		ret = do_mmap_shared_file(vma);
L
Linus Torvalds 已提交
1452
	else
1453
		ret = do_mmap_private(vma, region, len, capabilities);
L
Linus Torvalds 已提交
1454
	if (ret < 0)
1455 1456
		goto error_just_free;
	add_nommu_region(region);
1457

1458 1459 1460 1461 1462
	/* clear anonymous mappings that don't ask for uninitialized data */
	if (!vma->vm_file && !(flags & MAP_UNINITIALIZED))
		memset((void *)region->vm_start, 0,
		       region->vm_end - region->vm_start);

L
Linus Torvalds 已提交
1463
	/* okay... we have a mapping; now we have to register it */
1464
	result = vma->vm_start;
L
Linus Torvalds 已提交
1465 1466 1467

	current->mm->total_vm += len >> PAGE_SHIFT;

1468 1469
share:
	add_vma_to_mm(current->mm, vma);
L
Linus Torvalds 已提交
1470

1471 1472 1473 1474 1475 1476
	/* we flush the region from the icache only when the first executable
	 * mapping of it is made  */
	if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
		flush_icache_range(region->vm_start, region->vm_end);
		region->vm_icache_flushed = true;
	}
L
Linus Torvalds 已提交
1477

1478
	up_write(&nommu_region_sem);
L
Linus Torvalds 已提交
1479

1480 1481
	kleave(" = %lx", result);
	return result;
L
Linus Torvalds 已提交
1482

1483 1484 1485
error_just_free:
	up_write(&nommu_region_sem);
error:
1486 1487
	if (region->vm_file)
		fput(region->vm_file);
1488
	kmem_cache_free(vm_region_jar, region);
1489 1490
	if (vma->vm_file)
		fput(vma->vm_file);
1491 1492 1493 1494 1495 1496 1497 1498 1499
	kmem_cache_free(vm_area_cachep, vma);
	kleave(" = %d", ret);
	return ret;

sharing_violation:
	up_write(&nommu_region_sem);
	printk(KERN_WARNING "Attempt to share mismatched mappings\n");
	ret = -EINVAL;
	goto error;
L
Linus Torvalds 已提交
1500

1501 1502 1503 1504
error_getting_vma:
	kmem_cache_free(vm_region_jar, region);
	printk(KERN_WARNING "Allocation of vma for %lu byte allocation"
	       " from process %d failed\n",
L
Linus Torvalds 已提交
1505
	       len, current->pid);
1506
	show_free_areas(0);
L
Linus Torvalds 已提交
1507 1508
	return -ENOMEM;

1509 1510 1511
error_getting_region:
	printk(KERN_WARNING "Allocation of vm region for %lu byte allocation"
	       " from process %d failed\n",
L
Linus Torvalds 已提交
1512
	       len, current->pid);
1513
	show_free_areas(0);
L
Linus Torvalds 已提交
1514 1515
	return -ENOMEM;
}
1516

H
Hugh Dickins 已提交
1517 1518 1519 1520 1521 1522 1523
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
		unsigned long, prot, unsigned long, flags,
		unsigned long, fd, unsigned long, pgoff)
{
	struct file *file = NULL;
	unsigned long retval = -EBADF;

A
Al Viro 已提交
1524
	audit_mmap_fd(fd, flags);
H
Hugh Dickins 已提交
1525 1526 1527 1528 1529 1530 1531 1532
	if (!(flags & MAP_ANONYMOUS)) {
		file = fget(fd);
		if (!file)
			goto out;
	}

	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);

G
Greg Ungerer 已提交
1533
	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
H
Hugh Dickins 已提交
1534 1535 1536 1537 1538 1539 1540

	if (file)
		fput(file);
out:
	return retval;
}

C
Christoph Hellwig 已提交
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
#ifdef __ARCH_WANT_SYS_OLD_MMAP
struct mmap_arg_struct {
	unsigned long addr;
	unsigned long len;
	unsigned long prot;
	unsigned long flags;
	unsigned long fd;
	unsigned long offset;
};

SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
{
	struct mmap_arg_struct a;

	if (copy_from_user(&a, arg, sizeof(a)))
		return -EFAULT;
	if (a.offset & ~PAGE_MASK)
		return -EINVAL;

	return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
			      a.offset >> PAGE_SHIFT);
}
#endif /* __ARCH_WANT_SYS_OLD_MMAP */

L
Linus Torvalds 已提交
1565
/*
1566 1567
 * split a vma into two pieces at address 'addr', a new vma is allocated either
 * for the first part or the tail.
L
Linus Torvalds 已提交
1568
 */
1569 1570
int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
	      unsigned long addr, int new_below)
L
Linus Torvalds 已提交
1571
{
1572 1573 1574
	struct vm_area_struct *new;
	struct vm_region *region;
	unsigned long npages;
L
Linus Torvalds 已提交
1575

1576
	kenter("");
L
Linus Torvalds 已提交
1577

1578 1579 1580
	/* we're only permitted to split anonymous regions (these should have
	 * only a single usage on the region) */
	if (vma->vm_file)
1581
		return -ENOMEM;
L
Linus Torvalds 已提交
1582

1583 1584
	if (mm->map_count >= sysctl_max_map_count)
		return -ENOMEM;
L
Linus Torvalds 已提交
1585

1586 1587 1588
	region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
	if (!region)
		return -ENOMEM;
L
Linus Torvalds 已提交
1589

1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
	if (!new) {
		kmem_cache_free(vm_region_jar, region);
		return -ENOMEM;
	}

	/* most fields are the same, copy all, and then fixup */
	*new = *vma;
	*region = *vma->vm_region;
	new->vm_region = region;

	npages = (addr - vma->vm_start) >> PAGE_SHIFT;

	if (new_below) {
1604
		region->vm_top = region->vm_end = new->vm_end = addr;
1605 1606 1607
	} else {
		region->vm_start = new->vm_start = addr;
		region->vm_pgoff = new->vm_pgoff += npages;
L
Linus Torvalds 已提交
1608
	}
1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620

	if (new->vm_ops && new->vm_ops->open)
		new->vm_ops->open(new);

	delete_vma_from_mm(vma);
	down_write(&nommu_region_sem);
	delete_nommu_region(vma->vm_region);
	if (new_below) {
		vma->vm_region->vm_start = vma->vm_start = addr;
		vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
	} else {
		vma->vm_region->vm_end = vma->vm_end = addr;
1621
		vma->vm_region->vm_top = addr;
1622 1623 1624 1625 1626 1627 1628
	}
	add_nommu_region(vma->vm_region);
	add_nommu_region(new->vm_region);
	up_write(&nommu_region_sem);
	add_vma_to_mm(mm, vma);
	add_vma_to_mm(mm, new);
	return 0;
L
Linus Torvalds 已提交
1629 1630
}

1631
/*
1632 1633
 * shrink a VMA by removing the specified chunk from either the beginning or
 * the end
1634
 */
1635 1636 1637
static int shrink_vma(struct mm_struct *mm,
		      struct vm_area_struct *vma,
		      unsigned long from, unsigned long to)
L
Linus Torvalds 已提交
1638
{
1639
	struct vm_region *region;
L
Linus Torvalds 已提交
1640

1641
	kenter("");
L
Linus Torvalds 已提交
1642

1643 1644 1645 1646 1647 1648 1649 1650
	/* adjust the VMA's pointers, which may reposition it in the MM's tree
	 * and list */
	delete_vma_from_mm(vma);
	if (from > vma->vm_start)
		vma->vm_end = from;
	else
		vma->vm_start = to;
	add_vma_to_mm(mm, vma);
L
Linus Torvalds 已提交
1651

1652 1653
	/* cut the backing region down to size */
	region = vma->vm_region;
1654
	BUG_ON(region->vm_usage != 1);
1655 1656 1657

	down_write(&nommu_region_sem);
	delete_nommu_region(region);
1658 1659 1660 1661
	if (from > region->vm_start) {
		to = region->vm_top;
		region->vm_top = region->vm_end = from;
	} else {
1662
		region->vm_start = to;
1663
	}
1664 1665 1666 1667 1668 1669
	add_nommu_region(region);
	up_write(&nommu_region_sem);

	free_page_series(from, to);
	return 0;
}
L
Linus Torvalds 已提交
1670

1671 1672 1673 1674 1675 1676 1677 1678
/*
 * release a mapping
 * - under NOMMU conditions the chunk to be unmapped must be backed by a single
 *   VMA, though it need not cover the whole VMA
 */
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
{
	struct vm_area_struct *vma;
B
Bob Liu 已提交
1679
	unsigned long end;
1680
	int ret;
L
Linus Torvalds 已提交
1681

1682
	kenter(",%lx,%zx", start, len);
L
Linus Torvalds 已提交
1683

B
Bob Liu 已提交
1684
	len = PAGE_ALIGN(len);
1685 1686
	if (len == 0)
		return -EINVAL;
1687

B
Bob Liu 已提交
1688 1689
	end = start + len;

1690 1691 1692
	/* find the first potentially overlapping VMA */
	vma = find_vma(mm, start);
	if (!vma) {
1693
		static int limit;
1694 1695 1696 1697 1698 1699 1700 1701
		if (limit < 5) {
			printk(KERN_WARNING
			       "munmap of memory not mmapped by process %d"
			       " (%s): 0x%lx-0x%lx\n",
			       current->pid, current->comm,
			       start, start + len - 1);
			limit++;
		}
1702 1703
		return -EINVAL;
	}
L
Linus Torvalds 已提交
1704

1705 1706 1707 1708 1709 1710 1711 1712 1713
	/* we're allowed to split an anonymous VMA but not a file-backed one */
	if (vma->vm_file) {
		do {
			if (start > vma->vm_start) {
				kleave(" = -EINVAL [miss]");
				return -EINVAL;
			}
			if (end == vma->vm_end)
				goto erase_whole_vma;
1714 1715
			vma = vma->vm_next;
		} while (vma);
1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742
		kleave(" = -EINVAL [split file]");
		return -EINVAL;
	} else {
		/* the chunk must be a subset of the VMA found */
		if (start == vma->vm_start && end == vma->vm_end)
			goto erase_whole_vma;
		if (start < vma->vm_start || end > vma->vm_end) {
			kleave(" = -EINVAL [superset]");
			return -EINVAL;
		}
		if (start & ~PAGE_MASK) {
			kleave(" = -EINVAL [unaligned start]");
			return -EINVAL;
		}
		if (end != vma->vm_end && end & ~PAGE_MASK) {
			kleave(" = -EINVAL [unaligned split]");
			return -EINVAL;
		}
		if (start != vma->vm_start && end != vma->vm_end) {
			ret = split_vma(mm, vma, start, 1);
			if (ret < 0) {
				kleave(" = %d [split]", ret);
				return ret;
			}
		}
		return shrink_vma(mm, vma, start, end);
	}
L
Linus Torvalds 已提交
1743

1744 1745 1746 1747
erase_whole_vma:
	delete_vma_from_mm(vma);
	delete_vma(mm, vma);
	kleave(" = 0");
L
Linus Torvalds 已提交
1748 1749
	return 0;
}
1750
EXPORT_SYMBOL(do_munmap);
L
Linus Torvalds 已提交
1751

A
Al Viro 已提交
1752
int vm_munmap(unsigned long addr, size_t len)
1753
{
A
Al Viro 已提交
1754
	struct mm_struct *mm = current->mm;
1755 1756 1757 1758 1759 1760 1761
	int ret;

	down_write(&mm->mmap_sem);
	ret = do_munmap(mm, addr, len);
	up_write(&mm->mmap_sem);
	return ret;
}
1762 1763 1764 1765
EXPORT_SYMBOL(vm_munmap);

SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
{
A
Al Viro 已提交
1766
	return vm_munmap(addr, len);
1767
}
1768 1769

/*
1770
 * release all the mappings made in a process's VM space
1771
 */
1772
void exit_mmap(struct mm_struct *mm)
L
Linus Torvalds 已提交
1773
{
1774
	struct vm_area_struct *vma;
L
Linus Torvalds 已提交
1775

1776 1777
	if (!mm)
		return;
L
Linus Torvalds 已提交
1778

1779
	kenter("");
L
Linus Torvalds 已提交
1780

1781
	mm->total_vm = 0;
L
Linus Torvalds 已提交
1782

1783 1784 1785 1786
	while ((vma = mm->mmap)) {
		mm->mmap = vma->vm_next;
		delete_vma_from_mm(vma);
		delete_vma(mm, vma);
1787
		cond_resched();
L
Linus Torvalds 已提交
1788
	}
1789 1790

	kleave("");
L
Linus Torvalds 已提交
1791 1792
}

1793
unsigned long vm_brk(unsigned long addr, unsigned long len)
L
Linus Torvalds 已提交
1794 1795 1796 1797 1798
{
	return -ENOMEM;
}

/*
1799 1800
 * expand (or shrink) an existing mapping, potentially moving it at the same
 * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
L
Linus Torvalds 已提交
1801
 *
1802
 * under NOMMU conditions, we only permit changing a mapping's size, and only
1803 1804
 * as long as it stays within the region allocated by do_mmap_private() and the
 * block is not shareable
L
Linus Torvalds 已提交
1805
 *
1806
 * MREMAP_FIXED is not supported under NOMMU conditions
L
Linus Torvalds 已提交
1807
 */
A
Al Viro 已提交
1808
static unsigned long do_mremap(unsigned long addr,
L
Linus Torvalds 已提交
1809 1810 1811
			unsigned long old_len, unsigned long new_len,
			unsigned long flags, unsigned long new_addr)
{
1812
	struct vm_area_struct *vma;
L
Linus Torvalds 已提交
1813 1814

	/* insanity checks first */
B
Bob Liu 已提交
1815 1816
	old_len = PAGE_ALIGN(old_len);
	new_len = PAGE_ALIGN(new_len);
1817
	if (old_len == 0 || new_len == 0)
L
Linus Torvalds 已提交
1818 1819
		return (unsigned long) -EINVAL;

1820 1821 1822
	if (addr & ~PAGE_MASK)
		return -EINVAL;

L
Linus Torvalds 已提交
1823 1824 1825
	if (flags & MREMAP_FIXED && new_addr != addr)
		return (unsigned long) -EINVAL;

1826
	vma = find_vma_exact(current->mm, addr, old_len);
1827 1828
	if (!vma)
		return (unsigned long) -EINVAL;
L
Linus Torvalds 已提交
1829

1830
	if (vma->vm_end != vma->vm_start + old_len)
L
Linus Torvalds 已提交
1831 1832
		return (unsigned long) -EFAULT;

1833
	if (vma->vm_flags & VM_MAYSHARE)
L
Linus Torvalds 已提交
1834 1835
		return (unsigned long) -EPERM;

1836
	if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
L
Linus Torvalds 已提交
1837 1838 1839
		return (unsigned long) -ENOMEM;

	/* all checks complete - do it */
1840 1841 1842 1843
	vma->vm_end = vma->vm_start + new_len;
	return vma->vm_start;
}

1844 1845 1846
SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
		unsigned long, new_len, unsigned long, flags,
		unsigned long, new_addr)
1847 1848 1849 1850 1851 1852 1853
{
	unsigned long ret;

	down_write(&current->mm->mmap_sem);
	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
	up_write(&current->mm->mmap_sem);
	return ret;
L
Linus Torvalds 已提交
1854 1855
}

1856 1857 1858
struct page *follow_page_mask(struct vm_area_struct *vma,
			      unsigned long address, unsigned int flags,
			      unsigned int *page_mask)
L
Linus Torvalds 已提交
1859
{
1860
	*page_mask = 0;
L
Linus Torvalds 已提交
1861 1862 1863
	return NULL;
}

B
Bob Liu 已提交
1864 1865
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
		unsigned long pfn, unsigned long size, pgprot_t prot)
L
Linus Torvalds 已提交
1866
{
B
Bob Liu 已提交
1867 1868 1869
	if (addr != (pfn << PAGE_SHIFT))
		return -EINVAL;

1870
	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1871
	return 0;
L
Linus Torvalds 已提交
1872
}
1873
EXPORT_SYMBOL(remap_pfn_range);
L
Linus Torvalds 已提交
1874

1875 1876 1877 1878 1879 1880 1881 1882 1883 1884
int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
{
	unsigned long pfn = start >> PAGE_SHIFT;
	unsigned long vm_len = vma->vm_end - vma->vm_start;

	pfn += vma->vm_pgoff;
	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_iomap_memory);

1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899
int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
			unsigned long pgoff)
{
	unsigned int size = vma->vm_end - vma->vm_start;

	if (!(vma->vm_flags & VM_USERMAP))
		return -EINVAL;

	vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
	vma->vm_end = vma->vm_start + size;

	return 0;
}
EXPORT_SYMBOL(remap_vmalloc_range);

L
Linus Torvalds 已提交
1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910
unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
	unsigned long len, unsigned long pgoff, unsigned long flags)
{
	return -ENOMEM;
}

void unmap_mapping_range(struct address_space *mapping,
			 loff_t const holebegin, loff_t const holelen,
			 int even_cows)
{
}
1911
EXPORT_SYMBOL(unmap_mapping_range);
L
Linus Torvalds 已提交
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928

/*
 * Check that a process has enough memory to allocate a new virtual
 * mapping. 0 means there is enough memory for the allocation to
 * succeed and -ENOMEM implies there is not.
 *
 * We currently support three overcommit policies, which are set via the
 * vm.overcommit_memory sysctl.  See Documentation/vm/overcommit-accounting
 *
 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
 * Additional code 2002 Jul 20 by Robert Love.
 *
 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
 *
 * Note this is a helper function intended to be used by LSMs which
 * wish to use this logic.
 */
1929
int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
L
Linus Torvalds 已提交
1930
{
1931
	long free, allowed, reserve;
L
Linus Torvalds 已提交
1932 1933 1934 1935 1936 1937 1938 1939 1940 1941

	vm_acct_memory(pages);

	/*
	 * Sometimes we want to use more memory than we have
	 */
	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
		return 0;

	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
1942 1943 1944 1945 1946 1947 1948 1949 1950 1951
		free = global_page_state(NR_FREE_PAGES);
		free += global_page_state(NR_FILE_PAGES);

		/*
		 * shmem pages shouldn't be counted as free in this
		 * case, they can't be purged, only swapped out, and
		 * that won't affect the overall amount of available
		 * memory in the system.
		 */
		free -= global_page_state(NR_SHMEM);
L
Linus Torvalds 已提交
1952

1953
		free += get_nr_swap_pages();
L
Linus Torvalds 已提交
1954 1955 1956 1957 1958 1959 1960

		/*
		 * Any slabs which are created with the
		 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
		 * which are reclaimable, under pressure.  The dentry
		 * cache and most inode caches should fall into this
		 */
1961
		free += global_page_state(NR_SLAB_RECLAIMABLE);
L
Linus Torvalds 已提交
1962

1963 1964 1965
		/*
		 * Leave reserved pages. The pages are not for anonymous pages.
		 */
1966
		if (free <= totalreserve_pages)
1967 1968
			goto error;
		else
1969
			free -= totalreserve_pages;
1970 1971

		/*
1972
		 * Reserve some for root
1973
		 */
L
Linus Torvalds 已提交
1974
		if (!cap_sys_admin)
1975
			free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
L
Linus Torvalds 已提交
1976 1977 1978

		if (free > pages)
			return 0;
1979 1980

		goto error;
L
Linus Torvalds 已提交
1981 1982
	}

1983
	allowed = vm_commit_limit();
L
Linus Torvalds 已提交
1984
	/*
1985
	 * Reserve some 3% for root
L
Linus Torvalds 已提交
1986 1987
	 */
	if (!cap_sys_admin)
1988
		allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
L
Linus Torvalds 已提交
1989

1990 1991 1992 1993 1994
	/*
	 * Don't let a single process grow so big a user can't recover
	 */
	if (mm) {
		reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
1995
		allowed -= min_t(long, mm->total_vm / 32, reserve);
1996
	}
L
Linus Torvalds 已提交
1997

1998
	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
L
Linus Torvalds 已提交
1999
		return 0;
2000

2001
error:
L
Linus Torvalds 已提交
2002 2003 2004 2005 2006
	vm_unacct_memory(pages);

	return -ENOMEM;
}

N
Nick Piggin 已提交
2007
int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2008 2009
{
	BUG();
N
Nick Piggin 已提交
2010
	return 0;
2011
}
2012
EXPORT_SYMBOL(filemap_fault);
2013

2014 2015 2016 2017 2018 2019
void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	BUG();
}
EXPORT_SYMBOL(filemap_map_pages);

2020 2021
static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
		unsigned long addr, void *buf, int len, int write)
2022 2023 2024 2025 2026 2027
{
	struct vm_area_struct *vma;

	down_read(&mm->mmap_sem);

	/* the access must start within one of the target process's mappings */
2028 2029
	vma = find_vma(mm, addr);
	if (vma) {
2030 2031 2032 2033 2034
		/* don't overrun this mapping */
		if (addr + len >= vma->vm_end)
			len = vma->vm_end - addr;

		/* only read or write mappings where it is permitted */
2035
		if (write && vma->vm_flags & VM_MAYWRITE)
2036 2037
			copy_to_user_page(vma, NULL, addr,
					 (void *) addr, buf, len);
2038
		else if (!write && vma->vm_flags & VM_MAYREAD)
2039 2040
			copy_from_user_page(vma, NULL, addr,
					    buf, (void *) addr, len);
2041 2042 2043 2044 2045 2046 2047
		else
			len = 0;
	} else {
		len = 0;
	}

	up_read(&mm->mmap_sem);
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084

	return len;
}

/**
 * @access_remote_vm - access another process' address space
 * @mm:		the mm_struct of the target address space
 * @addr:	start address to access
 * @buf:	source or destination buffer
 * @len:	number of bytes to transfer
 * @write:	whether the access is a write
 *
 * The caller must hold a reference on @mm.
 */
int access_remote_vm(struct mm_struct *mm, unsigned long addr,
		void *buf, int len, int write)
{
	return __access_remote_vm(NULL, mm, addr, buf, len, write);
}

/*
 * Access another process' address space.
 * - source/target buffer must be kernel space
 */
int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
{
	struct mm_struct *mm;

	if (addr + len < addr)
		return 0;

	mm = get_task_mm(tsk);
	if (!mm)
		return 0;

	len = __access_remote_vm(tsk, mm, addr, buf, len, write);

2085 2086 2087
	mmput(mm);
	return len;
}
2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111

/**
 * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
 * @inode: The inode to check
 * @size: The current filesize of the inode
 * @newsize: The proposed filesize of the inode
 *
 * Check the shared mappings on an inode on behalf of a shrinking truncate to
 * make sure that that any outstanding VMAs aren't broken and then shrink the
 * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't
 * automatically grant mappings that are too large.
 */
int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
				size_t newsize)
{
	struct vm_area_struct *vma;
	struct vm_region *region;
	pgoff_t low, high;
	size_t r_size, r_top;

	low = newsize >> PAGE_SHIFT;
	high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;

	down_write(&nommu_region_sem);
D
Davidlohr Bueso 已提交
2112
	i_mmap_lock_read(inode->i_mapping);
2113 2114

	/* search for VMAs that fall within the dead zone */
2115
	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
2116 2117 2118
		/* found one - only interested if it's shared out of the page
		 * cache */
		if (vma->vm_flags & VM_SHARED) {
D
Davidlohr Bueso 已提交
2119
			i_mmap_unlock_read(inode->i_mapping);
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
			up_write(&nommu_region_sem);
			return -ETXTBSY; /* not quite true, but near enough */
		}
	}

	/* reduce any regions that overlap the dead zone - if in existence,
	 * these will be pointed to by VMAs that don't overlap the dead zone
	 *
	 * we don't check for any regions that start beyond the EOF as there
	 * shouldn't be any
	 */
D
Davidlohr Bueso 已提交
2131
	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145
		if (!(vma->vm_flags & VM_SHARED))
			continue;

		region = vma->vm_region;
		r_size = region->vm_top - region->vm_start;
		r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;

		if (r_top > newsize) {
			region->vm_top -= r_top - newsize;
			if (region->vm_end > region->vm_top)
				region->vm_end = region->vm_top;
		}
	}

D
Davidlohr Bueso 已提交
2146
	i_mmap_unlock_read(inode->i_mapping);
2147 2148 2149
	up_write(&nommu_region_sem);
	return 0;
}
2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170

/*
 * Initialise sysctl_user_reserve_kbytes.
 *
 * This is intended to prevent a user from starting a single memory hogging
 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
 * mode.
 *
 * The default value is min(3% of free memory, 128MB)
 * 128MB is enough to recover with sshd/login, bash, and top/kill.
 */
static int __meminit init_user_reserve(void)
{
	unsigned long free_kbytes;

	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);

	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
	return 0;
}
module_init(init_user_reserve)
2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191

/*
 * Initialise sysctl_admin_reserve_kbytes.
 *
 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
 * to log in and kill a memory hogging process.
 *
 * Systems with more than 256MB will reserve 8MB, enough to recover
 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
 * only reserve 3% of free pages by default.
 */
static int __meminit init_admin_reserve(void)
{
	unsigned long free_kbytes;

	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);

	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
	return 0;
}
module_init(init_admin_reserve)