mremap.c 13.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *	mm/mremap.c
 *
 *	(C) Copyright 1996 Linus Torvalds
 *
A
Alan Cox 已提交
6
 *	Address space accounting code	<alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
7 8 9 10 11 12
 *	(C) Copyright 2002 Red Hat Inc, All Rights Reserved
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
13
#include <linux/ksm.h>
L
Linus Torvalds 已提交
14 15
#include <linux/mman.h>
#include <linux/swap.h>
16
#include <linux/capability.h>
L
Linus Torvalds 已提交
17 18 19 20
#include <linux/fs.h>
#include <linux/highmem.h>
#include <linux/security.h>
#include <linux/syscalls.h>
A
Andrea Arcangeli 已提交
21
#include <linux/mmu_notifier.h>
L
Linus Torvalds 已提交
22 23 24 25 26

#include <asm/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>

27 28
#include "internal.h"

29
static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
L
Linus Torvalds 已提交
30 31 32 33 34 35 36 37 38 39 40 41 42 43
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	pgd = pgd_offset(mm, addr);
	if (pgd_none_or_clear_bad(pgd))
		return NULL;

	pud = pud_offset(pgd, addr);
	if (pud_none_or_clear_bad(pud))
		return NULL;

	pmd = pmd_offset(pud, addr);
A
Andrea Arcangeli 已提交
44
	split_huge_page_pmd(mm, pmd);
L
Linus Torvalds 已提交
45 46 47
	if (pmd_none_or_clear_bad(pmd))
		return NULL;

48
	return pmd;
L
Linus Torvalds 已提交
49 50
}

51 52
static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
			    unsigned long addr)
L
Linus Torvalds 已提交
53 54 55
{
	pgd_t *pgd;
	pud_t *pud;
H
Hugh Dickins 已提交
56
	pmd_t *pmd;
L
Linus Torvalds 已提交
57 58 59 60

	pgd = pgd_offset(mm, addr);
	pud = pud_alloc(mm, pgd, addr);
	if (!pud)
H
Hugh Dickins 已提交
61
		return NULL;
62

L
Linus Torvalds 已提交
63
	pmd = pmd_alloc(mm, pud, addr);
64
	if (!pmd)
H
Hugh Dickins 已提交
65
		return NULL;
66

67 68
	VM_BUG_ON(pmd_trans_huge(*pmd));
	if (pmd_none(*pmd) && __pte_alloc(mm, vma, pmd, addr))
H
Hugh Dickins 已提交
69 70
		return NULL;

71
	return pmd;
L
Linus Torvalds 已提交
72 73
}

74 75 76 77
static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
		unsigned long old_addr, unsigned long old_end,
		struct vm_area_struct *new_vma, pmd_t *new_pmd,
		unsigned long new_addr)
L
Linus Torvalds 已提交
78 79 80
{
	struct address_space *mapping = NULL;
	struct mm_struct *mm = vma->vm_mm;
81
	pte_t *old_pte, *new_pte, pte;
H
Hugh Dickins 已提交
82
	spinlock_t *old_ptl, *new_ptl;
A
Andrea Arcangeli 已提交
83
	unsigned long old_start;
L
Linus Torvalds 已提交
84

A
Andrea Arcangeli 已提交
85 86 87
	old_start = old_addr;
	mmu_notifier_invalidate_range_start(vma->vm_mm,
					    old_start, old_end);
L
Linus Torvalds 已提交
88 89 90
	if (vma->vm_file) {
		/*
		 * Subtle point from Rajesh Venkatasubramanian: before
N
npiggin@suse.de 已提交
91 92
		 * moving file-based ptes, we must lock truncate_pagecache
		 * out, since it might clean the dst vma before the src vma,
L
Linus Torvalds 已提交
93 94 95 96
		 * and we propagate stale pages into the dst afterward.
		 */
		mapping = vma->vm_file->f_mapping;
		spin_lock(&mapping->i_mmap_lock);
97
		new_vma->vm_truncate_count = 0;
L
Linus Torvalds 已提交
98 99
	}

H
Hugh Dickins 已提交
100 101 102 103
	/*
	 * We don't have to worry about the ordering of src and dst
	 * pte locks because exclusive mmap_sem prevents deadlock.
	 */
H
Hugh Dickins 已提交
104
	old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
P
Peter Zijlstra 已提交
105
	new_pte = pte_offset_map(new_pmd, new_addr);
H
Hugh Dickins 已提交
106 107
	new_ptl = pte_lockptr(mm, new_pmd);
	if (new_ptl != old_ptl)
I
Ingo Molnar 已提交
108
		spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
109
	arch_enter_lazy_mmu_mode();
110 111 112 113 114 115 116 117

	for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
				   new_pte++, new_addr += PAGE_SIZE) {
		if (pte_none(*old_pte))
			continue;
		pte = ptep_clear_flush(vma, old_addr, old_pte);
		pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
		set_pte_at(mm, new_addr, new_pte, pte);
L
Linus Torvalds 已提交
118
	}
119

120
	arch_leave_lazy_mmu_mode();
H
Hugh Dickins 已提交
121 122
	if (new_ptl != old_ptl)
		spin_unlock(new_ptl);
P
Peter Zijlstra 已提交
123
	pte_unmap(new_pte - 1);
H
Hugh Dickins 已提交
124
	pte_unmap_unlock(old_pte - 1, old_ptl);
L
Linus Torvalds 已提交
125 126
	if (mapping)
		spin_unlock(&mapping->i_mmap_lock);
A
Andrea Arcangeli 已提交
127
	mmu_notifier_invalidate_range_end(vma->vm_mm, old_start, old_end);
L
Linus Torvalds 已提交
128 129
}

130 131
#define LATENCY_LIMIT	(64 * PAGE_SIZE)

132
unsigned long move_page_tables(struct vm_area_struct *vma,
L
Linus Torvalds 已提交
133 134 135
		unsigned long old_addr, struct vm_area_struct *new_vma,
		unsigned long new_addr, unsigned long len)
{
136 137
	unsigned long extent, next, old_end;
	pmd_t *old_pmd, *new_pmd;
L
Linus Torvalds 已提交
138

139 140
	old_end = old_addr + len;
	flush_cache_range(vma, old_addr, old_end);
L
Linus Torvalds 已提交
141

142
	for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
L
Linus Torvalds 已提交
143
		cond_resched();
144 145 146 147 148 149 150
		next = (old_addr + PMD_SIZE) & PMD_MASK;
		if (next - 1 > old_end)
			next = old_end;
		extent = next - old_addr;
		old_pmd = get_old_pmd(vma->vm_mm, old_addr);
		if (!old_pmd)
			continue;
151
		new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
152 153 154 155 156 157 158 159 160
		if (!new_pmd)
			break;
		next = (new_addr + PMD_SIZE) & PMD_MASK;
		if (extent > next - new_addr)
			extent = next - new_addr;
		if (extent > LATENCY_LIMIT)
			extent = LATENCY_LIMIT;
		move_ptes(vma, old_pmd, old_addr, old_addr + extent,
				new_vma, new_pmd, new_addr);
L
Linus Torvalds 已提交
161
	}
162 163

	return len + old_addr - old_end;	/* how much done */
L
Linus Torvalds 已提交
164 165 166 167 168 169 170 171 172 173 174 175
}

static unsigned long move_vma(struct vm_area_struct *vma,
		unsigned long old_addr, unsigned long old_len,
		unsigned long new_len, unsigned long new_addr)
{
	struct mm_struct *mm = vma->vm_mm;
	struct vm_area_struct *new_vma;
	unsigned long vm_flags = vma->vm_flags;
	unsigned long new_pgoff;
	unsigned long moved_len;
	unsigned long excess = 0;
176
	unsigned long hiwater_vm;
L
Linus Torvalds 已提交
177
	int split = 0;
178
	int err;
L
Linus Torvalds 已提交
179 180 181 182 183 184 185 186

	/*
	 * We'd prefer to avoid failure later on in do_munmap:
	 * which may split one vma into three before unmapping.
	 */
	if (mm->map_count >= sysctl_max_map_count - 3)
		return -ENOMEM;

187 188 189 190 191 192 193
	/*
	 * Advise KSM to break any KSM pages in the area to be moved:
	 * it would be confusing if they were to turn up at the new
	 * location, where they happen to coincide with different KSM
	 * pages recently unmapped.  But leave vma->vm_flags as it was,
	 * so KSM can come around to merge on vma and new_vma afterwards.
	 */
194 195 196 197
	err = ksm_madvise(vma, old_addr, old_addr + old_len,
						MADV_UNMERGEABLE, &vm_flags);
	if (err)
		return err;
198

L
Linus Torvalds 已提交
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
	new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
	new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
	if (!new_vma)
		return -ENOMEM;

	moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len);
	if (moved_len < old_len) {
		/*
		 * On error, move entries back from new area to old,
		 * which will succeed since page tables still there,
		 * and then proceed to unmap new area instead of old.
		 */
		move_page_tables(new_vma, new_addr, vma, old_addr, moved_len);
		vma = new_vma;
		old_len = new_len;
		old_addr = new_addr;
		new_addr = -ENOMEM;
	}

	/* Conceal VM_ACCOUNT so old reservation is not undone */
	if (vm_flags & VM_ACCOUNT) {
		vma->vm_flags &= ~VM_ACCOUNT;
		excess = vma->vm_end - vma->vm_start - old_len;
		if (old_addr > vma->vm_start &&
		    old_addr + old_len < vma->vm_end)
			split = 1;
	}

K
Kirill Korotaev 已提交
227
	/*
228 229 230 231 232 233 234
	 * If we failed to move page tables we still do total_vm increment
	 * since do_munmap() will decrement it by old_len == new_len.
	 *
	 * Since total_vm is about to be raised artificially high for a
	 * moment, we need to restore high watermark afterwards: if stats
	 * are taken meanwhile, total_vm and hiwater_vm appear too high.
	 * If this were a serious issue, we'd add a flag to do_munmap().
K
Kirill Korotaev 已提交
235
	 */
236
	hiwater_vm = mm->hiwater_vm;
K
Kirill Korotaev 已提交
237
	mm->total_vm += new_len >> PAGE_SHIFT;
238
	vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
K
Kirill Korotaev 已提交
239

L
Linus Torvalds 已提交
240 241 242 243 244
	if (do_munmap(mm, old_addr, old_len) < 0) {
		/* OOM: unable to split vma, just get accounts right */
		vm_unacct_memory(excess >> PAGE_SHIFT);
		excess = 0;
	}
245
	mm->hiwater_vm = hiwater_vm;
L
Linus Torvalds 已提交
246 247 248 249 250 251 252 253 254 255 256

	/* Restore VM_ACCOUNT if one or two pieces of vma left */
	if (excess) {
		vma->vm_flags |= VM_ACCOUNT;
		if (split)
			vma->vm_next->vm_flags |= VM_ACCOUNT;
	}

	if (vm_flags & VM_LOCKED) {
		mm->locked_vm += new_len >> PAGE_SHIFT;
		if (new_len > old_len)
257 258
			mlock_vma_pages_range(new_vma, new_addr + old_len,
						       new_addr + new_len);
L
Linus Torvalds 已提交
259 260 261 262 263
	}

	return new_addr;
}

A
Al Viro 已提交
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
static struct vm_area_struct *vma_to_resize(unsigned long addr,
	unsigned long old_len, unsigned long new_len, unsigned long *p)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma = find_vma(mm, addr);

	if (!vma || vma->vm_start > addr)
		goto Efault;

	if (is_vm_hugetlb_page(vma))
		goto Einval;

	/* We can't remap across vm area boundaries */
	if (old_len > vma->vm_end - addr)
		goto Efault;

	if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) {
		if (new_len > old_len)
			goto Efault;
	}

	if (vma->vm_flags & VM_LOCKED) {
		unsigned long locked, lock_limit;
		locked = mm->locked_vm << PAGE_SHIFT;
J
Jiri Slaby 已提交
288
		lock_limit = rlimit(RLIMIT_MEMLOCK);
A
Al Viro 已提交
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
		locked += new_len - old_len;
		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
			goto Eagain;
	}

	if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT))
		goto Enomem;

	if (vma->vm_flags & VM_ACCOUNT) {
		unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
		if (security_vm_enough_memory(charged))
			goto Efault;
		*p = charged;
	}

	return vma;

Efault:	/* very odd choice for most of the cases, but... */
	return ERR_PTR(-EFAULT);
Einval:
	return ERR_PTR(-EINVAL);
Enomem:
	return ERR_PTR(-ENOMEM);
Eagain:
	return ERR_PTR(-EAGAIN);
}

A
Al Viro 已提交
316 317 318 319 320 321 322 323
static unsigned long mremap_to(unsigned long addr,
	unsigned long old_len, unsigned long new_addr,
	unsigned long new_len)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	unsigned long ret = -EINVAL;
	unsigned long charged = 0;
A
Al Viro 已提交
324
	unsigned long map_flags;
A
Al Viro 已提交
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361

	if (new_addr & ~PAGE_MASK)
		goto out;

	if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
		goto out;

	/* Check if the location we're moving into overlaps the
	 * old location at all, and fail if it does.
	 */
	if ((new_addr <= addr) && (new_addr+new_len) > addr)
		goto out;

	if ((addr <= new_addr) && (addr+old_len) > new_addr)
		goto out;

	ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1);
	if (ret)
		goto out;

	ret = do_munmap(mm, new_addr, new_len);
	if (ret)
		goto out;

	if (old_len >= new_len) {
		ret = do_munmap(mm, addr+new_len, old_len - new_len);
		if (ret && old_len != new_len)
			goto out;
		old_len = new_len;
	}

	vma = vma_to_resize(addr, old_len, new_len, &charged);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto out;
	}

A
Al Viro 已提交
362 363 364
	map_flags = MAP_FIXED;
	if (vma->vm_flags & VM_MAYSHARE)
		map_flags |= MAP_SHARED;
365

A
Al Viro 已提交
366 367 368
	ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
				((addr - vma->vm_start) >> PAGE_SHIFT),
				map_flags);
A
Al Viro 已提交
369
	if (ret & ~PAGE_MASK)
A
Al Viro 已提交
370 371 372 373 374 375 376
		goto out1;

	ret = move_vma(vma, addr, old_len, new_len, new_addr);
	if (!(ret & ~PAGE_MASK))
		goto out;
out1:
	vm_unacct_memory(charged);
A
Al Viro 已提交
377 378 379 380 381

out:
	return ret;
}

A
Al Viro 已提交
382 383
static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
{
A
Al Viro 已提交
384
	unsigned long end = vma->vm_end + delta;
385
	if (end < vma->vm_end) /* overflow */
A
Al Viro 已提交
386
		return 0;
387
	if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */
A
Al Viro 已提交
388 389 390
		return 0;
	if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
			      0, MAP_FIXED) & ~PAGE_MASK)
A
Al Viro 已提交
391 392 393 394
		return 0;
	return 1;
}

L
Linus Torvalds 已提交
395 396 397 398 399 400 401 402 403 404 405
/*
 * Expand (or shrink) an existing mapping, potentially moving it at the
 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
 *
 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
 * This option implies MREMAP_MAYMOVE.
 */
unsigned long do_mremap(unsigned long addr,
	unsigned long old_len, unsigned long new_len,
	unsigned long flags, unsigned long new_addr)
{
H
Hugh Dickins 已提交
406
	struct mm_struct *mm = current->mm;
L
Linus Torvalds 已提交
407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
	struct vm_area_struct *vma;
	unsigned long ret = -EINVAL;
	unsigned long charged = 0;

	if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
		goto out;

	if (addr & ~PAGE_MASK)
		goto out;

	old_len = PAGE_ALIGN(old_len);
	new_len = PAGE_ALIGN(new_len);

	/*
	 * We allow a zero old-len as a special case
	 * for DOS-emu "duplicate shm area" thing. But
	 * a zero new-len is nonsensical.
	 */
	if (!new_len)
		goto out;

	if (flags & MREMAP_FIXED) {
A
Al Viro 已提交
429 430 431
		if (flags & MREMAP_MAYMOVE)
			ret = mremap_to(addr, old_len, new_addr, new_len);
		goto out;
L
Linus Torvalds 已提交
432 433 434 435 436 437 438 439
	}

	/*
	 * Always allow a shrinking remap: that just unmaps
	 * the unnecessary pages..
	 * do_munmap does all the needed commit accounting
	 */
	if (old_len >= new_len) {
H
Hugh Dickins 已提交
440
		ret = do_munmap(mm, addr+new_len, old_len - new_len);
L
Linus Torvalds 已提交
441 442 443
		if (ret && old_len != new_len)
			goto out;
		ret = addr;
A
Al Viro 已提交
444
		goto out;
L
Linus Torvalds 已提交
445 446 447
	}

	/*
A
Al Viro 已提交
448
	 * Ok, we need to grow..
L
Linus Torvalds 已提交
449
	 */
A
Al Viro 已提交
450 451 452
	vma = vma_to_resize(addr, old_len, new_len, &charged);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
L
Linus Torvalds 已提交
453
		goto out;
A
akpm@osdl.org 已提交
454
	}
L
Linus Torvalds 已提交
455 456 457

	/* old_len exactly to the end of the area..
	 */
A
Al Viro 已提交
458
	if (old_len == vma->vm_end - addr) {
L
Linus Torvalds 已提交
459
		/* can we just expand the current mapping? */
A
Al Viro 已提交
460
		if (vma_expandable(vma, new_len - old_len)) {
L
Linus Torvalds 已提交
461 462
			int pages = (new_len - old_len) >> PAGE_SHIFT;

463 464 465 466 467
			if (vma_adjust(vma, vma->vm_start, addr + new_len,
				       vma->vm_pgoff, NULL)) {
				ret = -ENOMEM;
				goto out;
			}
L
Linus Torvalds 已提交
468

H
Hugh Dickins 已提交
469 470
			mm->total_vm += pages;
			vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
L
Linus Torvalds 已提交
471
			if (vma->vm_flags & VM_LOCKED) {
H
Hugh Dickins 已提交
472
				mm->locked_vm += pages;
473
				mlock_vma_pages_range(vma, addr + old_len,
L
Linus Torvalds 已提交
474 475 476 477 478 479 480 481 482 483 484 485 486
						   addr + new_len);
			}
			ret = addr;
			goto out;
		}
	}

	/*
	 * We weren't able to just expand or shrink the area,
	 * we need to create a new one and move it..
	 */
	ret = -ENOMEM;
	if (flags & MREMAP_MAYMOVE) {
A
Al Viro 已提交
487 488 489 490 491
		unsigned long map_flags = 0;
		if (vma->vm_flags & VM_MAYSHARE)
			map_flags |= MAP_SHARED;

		new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
492 493 494
					vma->vm_pgoff +
					((addr - vma->vm_start) >> PAGE_SHIFT),
					map_flags);
A
Al Viro 已提交
495 496 497
		if (new_addr & ~PAGE_MASK) {
			ret = new_addr;
			goto out;
L
Linus Torvalds 已提交
498
		}
A
Al Viro 已提交
499 500 501 502

		ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1);
		if (ret)
			goto out;
L
Linus Torvalds 已提交
503 504 505 506 507 508 509 510
		ret = move_vma(vma, addr, old_len, new_len, new_addr);
	}
out:
	if (ret & ~PAGE_MASK)
		vm_unacct_memory(charged);
	return ret;
}

511 512 513
SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
		unsigned long, new_len, unsigned long, flags,
		unsigned long, new_addr)
L
Linus Torvalds 已提交
514 515 516 517 518 519 520 521
{
	unsigned long ret;

	down_write(&current->mm->mmap_sem);
	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
	up_write(&current->mm->mmap_sem);
	return ret;
}