mremap.c 13.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *	mm/mremap.c
 *
 *	(C) Copyright 1996 Linus Torvalds
 *
A
Alan Cox 已提交
6
 *	Address space accounting code	<alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
7 8 9 10 11 12
 *	(C) Copyright 2002 Red Hat Inc, All Rights Reserved
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
13
#include <linux/ksm.h>
L
Linus Torvalds 已提交
14 15
#include <linux/mman.h>
#include <linux/swap.h>
16
#include <linux/capability.h>
L
Linus Torvalds 已提交
17 18 19 20
#include <linux/fs.h>
#include <linux/highmem.h>
#include <linux/security.h>
#include <linux/syscalls.h>
A
Andrea Arcangeli 已提交
21
#include <linux/mmu_notifier.h>
L
Linus Torvalds 已提交
22 23 24 25 26

#include <asm/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>

27 28
#include "internal.h"

29
static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
L
Linus Torvalds 已提交
30 31 32 33 34 35 36 37 38 39 40 41 42 43
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	pgd = pgd_offset(mm, addr);
	if (pgd_none_or_clear_bad(pgd))
		return NULL;

	pud = pud_offset(pgd, addr);
	if (pud_none_or_clear_bad(pud))
		return NULL;

	pmd = pmd_offset(pud, addr);
44
	if (pmd_none(*pmd))
L
Linus Torvalds 已提交
45 46
		return NULL;

47
	return pmd;
L
Linus Torvalds 已提交
48 49
}

50 51
static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
			    unsigned long addr)
L
Linus Torvalds 已提交
52 53 54
{
	pgd_t *pgd;
	pud_t *pud;
H
Hugh Dickins 已提交
55
	pmd_t *pmd;
L
Linus Torvalds 已提交
56 57 58 59

	pgd = pgd_offset(mm, addr);
	pud = pud_alloc(mm, pgd, addr);
	if (!pud)
H
Hugh Dickins 已提交
60
		return NULL;
61

L
Linus Torvalds 已提交
62
	pmd = pmd_alloc(mm, pud, addr);
63
	if (!pmd)
H
Hugh Dickins 已提交
64
		return NULL;
65

66
	VM_BUG_ON(pmd_trans_huge(*pmd));
H
Hugh Dickins 已提交
67

68
	return pmd;
L
Linus Torvalds 已提交
69 70
}

71 72 73 74
static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
		unsigned long old_addr, unsigned long old_end,
		struct vm_area_struct *new_vma, pmd_t *new_pmd,
		unsigned long new_addr)
L
Linus Torvalds 已提交
75 76
{
	struct address_space *mapping = NULL;
77
	struct anon_vma *anon_vma = vma->anon_vma;
L
Linus Torvalds 已提交
78
	struct mm_struct *mm = vma->vm_mm;
79
	pte_t *old_pte, *new_pte, pte;
H
Hugh Dickins 已提交
80
	spinlock_t *old_ptl, *new_ptl;
L
Linus Torvalds 已提交
81 82 83 84

	if (vma->vm_file) {
		/*
		 * Subtle point from Rajesh Venkatasubramanian: before
N
npiggin@suse.de 已提交
85 86
		 * moving file-based ptes, we must lock truncate_pagecache
		 * out, since it might clean the dst vma before the src vma,
L
Linus Torvalds 已提交
87 88 89
		 * and we propagate stale pages into the dst afterward.
		 */
		mapping = vma->vm_file->f_mapping;
90
		mutex_lock(&mapping->i_mmap_mutex);
L
Linus Torvalds 已提交
91
	}
92 93
	if (anon_vma)
		anon_vma_lock(anon_vma);
L
Linus Torvalds 已提交
94

H
Hugh Dickins 已提交
95 96 97 98
	/*
	 * We don't have to worry about the ordering of src and dst
	 * pte locks because exclusive mmap_sem prevents deadlock.
	 */
H
Hugh Dickins 已提交
99
	old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
P
Peter Zijlstra 已提交
100
	new_pte = pte_offset_map(new_pmd, new_addr);
H
Hugh Dickins 已提交
101 102
	new_ptl = pte_lockptr(mm, new_pmd);
	if (new_ptl != old_ptl)
I
Ingo Molnar 已提交
103
		spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
104
	arch_enter_lazy_mmu_mode();
105 106 107 108 109

	for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
				   new_pte++, new_addr += PAGE_SIZE) {
		if (pte_none(*old_pte))
			continue;
110
		pte = ptep_get_and_clear(mm, old_addr, old_pte);
111 112
		pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
		set_pte_at(mm, new_addr, new_pte, pte);
L
Linus Torvalds 已提交
113
	}
114

115
	arch_leave_lazy_mmu_mode();
H
Hugh Dickins 已提交
116 117
	if (new_ptl != old_ptl)
		spin_unlock(new_ptl);
P
Peter Zijlstra 已提交
118
	pte_unmap(new_pte - 1);
H
Hugh Dickins 已提交
119
	pte_unmap_unlock(old_pte - 1, old_ptl);
120 121
	if (anon_vma)
		anon_vma_unlock(anon_vma);
L
Linus Torvalds 已提交
122
	if (mapping)
123
		mutex_unlock(&mapping->i_mmap_mutex);
L
Linus Torvalds 已提交
124 125
}

126 127
#define LATENCY_LIMIT	(64 * PAGE_SIZE)

128
unsigned long move_page_tables(struct vm_area_struct *vma,
L
Linus Torvalds 已提交
129 130 131
		unsigned long old_addr, struct vm_area_struct *new_vma,
		unsigned long new_addr, unsigned long len)
{
132 133
	unsigned long extent, next, old_end;
	pmd_t *old_pmd, *new_pmd;
134
	bool need_flush = false;
L
Linus Torvalds 已提交
135

136 137
	old_end = old_addr + len;
	flush_cache_range(vma, old_addr, old_end);
L
Linus Torvalds 已提交
138

139 140
	mmu_notifier_invalidate_range_start(vma->vm_mm, old_addr, old_end);

141
	for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
L
Linus Torvalds 已提交
142
		cond_resched();
143
		next = (old_addr + PMD_SIZE) & PMD_MASK;
144
		/* even if next overflowed, extent below will be ok */
145
		extent = next - old_addr;
146 147
		if (extent > old_end - old_addr)
			extent = old_end - old_addr;
148 149 150
		old_pmd = get_old_pmd(vma->vm_mm, old_addr);
		if (!old_pmd)
			continue;
151
		new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
152 153
		if (!new_pmd)
			break;
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
		if (pmd_trans_huge(*old_pmd)) {
			int err = 0;
			if (extent == HPAGE_PMD_SIZE)
				err = move_huge_pmd(vma, new_vma, old_addr,
						    new_addr, old_end,
						    old_pmd, new_pmd);
			if (err > 0) {
				need_flush = true;
				continue;
			} else if (!err) {
				split_huge_page_pmd(vma->vm_mm, old_pmd);
			}
			VM_BUG_ON(pmd_trans_huge(*old_pmd));
		}
		if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma,
						      new_pmd, new_addr))
			break;
171 172 173 174 175 176 177
		next = (new_addr + PMD_SIZE) & PMD_MASK;
		if (extent > next - new_addr)
			extent = next - new_addr;
		if (extent > LATENCY_LIMIT)
			extent = LATENCY_LIMIT;
		move_ptes(vma, old_pmd, old_addr, old_addr + extent,
				new_vma, new_pmd, new_addr);
178
		need_flush = true;
L
Linus Torvalds 已提交
179
	}
180 181 182 183
	if (likely(need_flush))
		flush_tlb_range(vma, old_end-len, old_addr);

	mmu_notifier_invalidate_range_end(vma->vm_mm, old_end-len, old_end);
184 185

	return len + old_addr - old_end;	/* how much done */
L
Linus Torvalds 已提交
186 187 188 189 190 191 192 193 194 195 196 197
}

static unsigned long move_vma(struct vm_area_struct *vma,
		unsigned long old_addr, unsigned long old_len,
		unsigned long new_len, unsigned long new_addr)
{
	struct mm_struct *mm = vma->vm_mm;
	struct vm_area_struct *new_vma;
	unsigned long vm_flags = vma->vm_flags;
	unsigned long new_pgoff;
	unsigned long moved_len;
	unsigned long excess = 0;
198
	unsigned long hiwater_vm;
L
Linus Torvalds 已提交
199
	int split = 0;
200
	int err;
L
Linus Torvalds 已提交
201 202 203 204 205 206 207 208

	/*
	 * We'd prefer to avoid failure later on in do_munmap:
	 * which may split one vma into three before unmapping.
	 */
	if (mm->map_count >= sysctl_max_map_count - 3)
		return -ENOMEM;

209 210 211 212 213 214 215
	/*
	 * Advise KSM to break any KSM pages in the area to be moved:
	 * it would be confusing if they were to turn up at the new
	 * location, where they happen to coincide with different KSM
	 * pages recently unmapped.  But leave vma->vm_flags as it was,
	 * so KSM can come around to merge on vma and new_vma afterwards.
	 */
216 217 218 219
	err = ksm_madvise(vma, old_addr, old_addr + old_len,
						MADV_UNMERGEABLE, &vm_flags);
	if (err)
		return err;
220

L
Linus Torvalds 已提交
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
	new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
	new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
	if (!new_vma)
		return -ENOMEM;

	moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len);
	if (moved_len < old_len) {
		/*
		 * On error, move entries back from new area to old,
		 * which will succeed since page tables still there,
		 * and then proceed to unmap new area instead of old.
		 */
		move_page_tables(new_vma, new_addr, vma, old_addr, moved_len);
		vma = new_vma;
		old_len = new_len;
		old_addr = new_addr;
		new_addr = -ENOMEM;
	}

	/* Conceal VM_ACCOUNT so old reservation is not undone */
	if (vm_flags & VM_ACCOUNT) {
		vma->vm_flags &= ~VM_ACCOUNT;
		excess = vma->vm_end - vma->vm_start - old_len;
		if (old_addr > vma->vm_start &&
		    old_addr + old_len < vma->vm_end)
			split = 1;
	}

K
Kirill Korotaev 已提交
249
	/*
250 251 252 253 254 255 256
	 * If we failed to move page tables we still do total_vm increment
	 * since do_munmap() will decrement it by old_len == new_len.
	 *
	 * Since total_vm is about to be raised artificially high for a
	 * moment, we need to restore high watermark afterwards: if stats
	 * are taken meanwhile, total_vm and hiwater_vm appear too high.
	 * If this were a serious issue, we'd add a flag to do_munmap().
K
Kirill Korotaev 已提交
257
	 */
258
	hiwater_vm = mm->hiwater_vm;
259
	vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
K
Kirill Korotaev 已提交
260

L
Linus Torvalds 已提交
261 262 263 264 265
	if (do_munmap(mm, old_addr, old_len) < 0) {
		/* OOM: unable to split vma, just get accounts right */
		vm_unacct_memory(excess >> PAGE_SHIFT);
		excess = 0;
	}
266
	mm->hiwater_vm = hiwater_vm;
L
Linus Torvalds 已提交
267 268 269 270 271 272 273 274 275 276 277

	/* Restore VM_ACCOUNT if one or two pieces of vma left */
	if (excess) {
		vma->vm_flags |= VM_ACCOUNT;
		if (split)
			vma->vm_next->vm_flags |= VM_ACCOUNT;
	}

	if (vm_flags & VM_LOCKED) {
		mm->locked_vm += new_len >> PAGE_SHIFT;
		if (new_len > old_len)
278 279
			mlock_vma_pages_range(new_vma, new_addr + old_len,
						       new_addr + new_len);
L
Linus Torvalds 已提交
280 281 282 283 284
	}

	return new_addr;
}

A
Al Viro 已提交
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
static struct vm_area_struct *vma_to_resize(unsigned long addr,
	unsigned long old_len, unsigned long new_len, unsigned long *p)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma = find_vma(mm, addr);

	if (!vma || vma->vm_start > addr)
		goto Efault;

	if (is_vm_hugetlb_page(vma))
		goto Einval;

	/* We can't remap across vm area boundaries */
	if (old_len > vma->vm_end - addr)
		goto Efault;

301 302 303 304 305
	/* Need to be careful about a growing mapping */
	if (new_len > old_len) {
		unsigned long pgoff;

		if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))
A
Al Viro 已提交
306
			goto Efault;
307 308 309 310
		pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
		pgoff += vma->vm_pgoff;
		if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
			goto Einval;
A
Al Viro 已提交
311 312 313 314 315
	}

	if (vma->vm_flags & VM_LOCKED) {
		unsigned long locked, lock_limit;
		locked = mm->locked_vm << PAGE_SHIFT;
J
Jiri Slaby 已提交
316
		lock_limit = rlimit(RLIMIT_MEMLOCK);
A
Al Viro 已提交
317 318 319 320 321 322 323 324 325 326
		locked += new_len - old_len;
		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
			goto Eagain;
	}

	if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT))
		goto Enomem;

	if (vma->vm_flags & VM_ACCOUNT) {
		unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
327
		if (security_vm_enough_memory_mm(mm, charged))
A
Al Viro 已提交
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
			goto Efault;
		*p = charged;
	}

	return vma;

Efault:	/* very odd choice for most of the cases, but... */
	return ERR_PTR(-EFAULT);
Einval:
	return ERR_PTR(-EINVAL);
Enomem:
	return ERR_PTR(-ENOMEM);
Eagain:
	return ERR_PTR(-EAGAIN);
}

A
Al Viro 已提交
344 345 346 347 348 349 350 351
static unsigned long mremap_to(unsigned long addr,
	unsigned long old_len, unsigned long new_addr,
	unsigned long new_len)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	unsigned long ret = -EINVAL;
	unsigned long charged = 0;
A
Al Viro 已提交
352
	unsigned long map_flags;
A
Al Viro 已提交
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385

	if (new_addr & ~PAGE_MASK)
		goto out;

	if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
		goto out;

	/* Check if the location we're moving into overlaps the
	 * old location at all, and fail if it does.
	 */
	if ((new_addr <= addr) && (new_addr+new_len) > addr)
		goto out;

	if ((addr <= new_addr) && (addr+old_len) > new_addr)
		goto out;

	ret = do_munmap(mm, new_addr, new_len);
	if (ret)
		goto out;

	if (old_len >= new_len) {
		ret = do_munmap(mm, addr+new_len, old_len - new_len);
		if (ret && old_len != new_len)
			goto out;
		old_len = new_len;
	}

	vma = vma_to_resize(addr, old_len, new_len, &charged);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto out;
	}

A
Al Viro 已提交
386 387 388
	map_flags = MAP_FIXED;
	if (vma->vm_flags & VM_MAYSHARE)
		map_flags |= MAP_SHARED;
389

A
Al Viro 已提交
390 391 392
	ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
				((addr - vma->vm_start) >> PAGE_SHIFT),
				map_flags);
A
Al Viro 已提交
393
	if (ret & ~PAGE_MASK)
A
Al Viro 已提交
394 395 396 397 398 399 400
		goto out1;

	ret = move_vma(vma, addr, old_len, new_len, new_addr);
	if (!(ret & ~PAGE_MASK))
		goto out;
out1:
	vm_unacct_memory(charged);
A
Al Viro 已提交
401 402 403 404 405

out:
	return ret;
}

A
Al Viro 已提交
406 407
static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
{
A
Al Viro 已提交
408
	unsigned long end = vma->vm_end + delta;
409
	if (end < vma->vm_end) /* overflow */
A
Al Viro 已提交
410
		return 0;
411
	if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */
A
Al Viro 已提交
412 413 414
		return 0;
	if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
			      0, MAP_FIXED) & ~PAGE_MASK)
A
Al Viro 已提交
415 416 417 418
		return 0;
	return 1;
}

L
Linus Torvalds 已提交
419 420 421 422 423 424 425
/*
 * Expand (or shrink) an existing mapping, potentially moving it at the
 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
 *
 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
 * This option implies MREMAP_MAYMOVE.
 */
A
Al Viro 已提交
426 427 428
SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
		unsigned long, new_len, unsigned long, flags,
		unsigned long, new_addr)
L
Linus Torvalds 已提交
429
{
H
Hugh Dickins 已提交
430
	struct mm_struct *mm = current->mm;
L
Linus Torvalds 已提交
431 432 433 434
	struct vm_area_struct *vma;
	unsigned long ret = -EINVAL;
	unsigned long charged = 0;

A
Al Viro 已提交
435 436
	down_write(&current->mm->mmap_sem);

L
Linus Torvalds 已提交
437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
	if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
		goto out;

	if (addr & ~PAGE_MASK)
		goto out;

	old_len = PAGE_ALIGN(old_len);
	new_len = PAGE_ALIGN(new_len);

	/*
	 * We allow a zero old-len as a special case
	 * for DOS-emu "duplicate shm area" thing. But
	 * a zero new-len is nonsensical.
	 */
	if (!new_len)
		goto out;

	if (flags & MREMAP_FIXED) {
A
Al Viro 已提交
455 456 457
		if (flags & MREMAP_MAYMOVE)
			ret = mremap_to(addr, old_len, new_addr, new_len);
		goto out;
L
Linus Torvalds 已提交
458 459 460 461 462 463 464 465
	}

	/*
	 * Always allow a shrinking remap: that just unmaps
	 * the unnecessary pages..
	 * do_munmap does all the needed commit accounting
	 */
	if (old_len >= new_len) {
H
Hugh Dickins 已提交
466
		ret = do_munmap(mm, addr+new_len, old_len - new_len);
L
Linus Torvalds 已提交
467 468 469
		if (ret && old_len != new_len)
			goto out;
		ret = addr;
A
Al Viro 已提交
470
		goto out;
L
Linus Torvalds 已提交
471 472 473
	}

	/*
A
Al Viro 已提交
474
	 * Ok, we need to grow..
L
Linus Torvalds 已提交
475
	 */
A
Al Viro 已提交
476 477 478
	vma = vma_to_resize(addr, old_len, new_len, &charged);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
L
Linus Torvalds 已提交
479
		goto out;
A
akpm@osdl.org 已提交
480
	}
L
Linus Torvalds 已提交
481 482 483

	/* old_len exactly to the end of the area..
	 */
A
Al Viro 已提交
484
	if (old_len == vma->vm_end - addr) {
L
Linus Torvalds 已提交
485
		/* can we just expand the current mapping? */
A
Al Viro 已提交
486
		if (vma_expandable(vma, new_len - old_len)) {
L
Linus Torvalds 已提交
487 488
			int pages = (new_len - old_len) >> PAGE_SHIFT;

489 490 491 492 493
			if (vma_adjust(vma, vma->vm_start, addr + new_len,
				       vma->vm_pgoff, NULL)) {
				ret = -ENOMEM;
				goto out;
			}
L
Linus Torvalds 已提交
494

H
Hugh Dickins 已提交
495
			vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
L
Linus Torvalds 已提交
496
			if (vma->vm_flags & VM_LOCKED) {
H
Hugh Dickins 已提交
497
				mm->locked_vm += pages;
498
				mlock_vma_pages_range(vma, addr + old_len,
L
Linus Torvalds 已提交
499 500 501 502 503 504 505 506 507 508 509 510 511
						   addr + new_len);
			}
			ret = addr;
			goto out;
		}
	}

	/*
	 * We weren't able to just expand or shrink the area,
	 * we need to create a new one and move it..
	 */
	ret = -ENOMEM;
	if (flags & MREMAP_MAYMOVE) {
A
Al Viro 已提交
512 513 514 515 516
		unsigned long map_flags = 0;
		if (vma->vm_flags & VM_MAYSHARE)
			map_flags |= MAP_SHARED;

		new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
517 518 519
					vma->vm_pgoff +
					((addr - vma->vm_start) >> PAGE_SHIFT),
					map_flags);
A
Al Viro 已提交
520 521 522
		if (new_addr & ~PAGE_MASK) {
			ret = new_addr;
			goto out;
L
Linus Torvalds 已提交
523
		}
A
Al Viro 已提交
524

L
Linus Torvalds 已提交
525 526 527 528 529 530 531 532
		ret = move_vma(vma, addr, old_len, new_len, new_addr);
	}
out:
	if (ret & ~PAGE_MASK)
		vm_unacct_memory(charged);
	up_write(&current->mm->mmap_sem);
	return ret;
}