mremap.c 13.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *	mm/mremap.c
 *
 *	(C) Copyright 1996 Linus Torvalds
 *
A
Alan Cox 已提交
6
 *	Address space accounting code	<alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
7 8 9 10 11 12
 *	(C) Copyright 2002 Red Hat Inc, All Rights Reserved
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
13
#include <linux/ksm.h>
L
Linus Torvalds 已提交
14 15
#include <linux/mman.h>
#include <linux/swap.h>
16
#include <linux/capability.h>
L
Linus Torvalds 已提交
17 18 19 20
#include <linux/fs.h>
#include <linux/highmem.h>
#include <linux/security.h>
#include <linux/syscalls.h>
A
Andrea Arcangeli 已提交
21
#include <linux/mmu_notifier.h>
L
Linus Torvalds 已提交
22 23 24 25 26

#include <asm/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>

27 28
#include "internal.h"

29
static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
L
Linus Torvalds 已提交
30 31 32 33 34 35 36 37 38 39 40 41 42 43
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	pgd = pgd_offset(mm, addr);
	if (pgd_none_or_clear_bad(pgd))
		return NULL;

	pud = pud_offset(pgd, addr);
	if (pud_none_or_clear_bad(pud))
		return NULL;

	pmd = pmd_offset(pud, addr);
A
Andrea Arcangeli 已提交
44
	split_huge_page_pmd(mm, pmd);
L
Linus Torvalds 已提交
45 46 47
	if (pmd_none_or_clear_bad(pmd))
		return NULL;

48
	return pmd;
L
Linus Torvalds 已提交
49 50
}

51 52
static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
			    unsigned long addr)
L
Linus Torvalds 已提交
53 54 55
{
	pgd_t *pgd;
	pud_t *pud;
H
Hugh Dickins 已提交
56
	pmd_t *pmd;
L
Linus Torvalds 已提交
57 58 59 60

	pgd = pgd_offset(mm, addr);
	pud = pud_alloc(mm, pgd, addr);
	if (!pud)
H
Hugh Dickins 已提交
61
		return NULL;
62

L
Linus Torvalds 已提交
63
	pmd = pmd_alloc(mm, pud, addr);
64
	if (!pmd)
H
Hugh Dickins 已提交
65
		return NULL;
66

67 68
	VM_BUG_ON(pmd_trans_huge(*pmd));
	if (pmd_none(*pmd) && __pte_alloc(mm, vma, pmd, addr))
H
Hugh Dickins 已提交
69 70
		return NULL;

71
	return pmd;
L
Linus Torvalds 已提交
72 73
}

74 75 76 77
static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
		unsigned long old_addr, unsigned long old_end,
		struct vm_area_struct *new_vma, pmd_t *new_pmd,
		unsigned long new_addr)
L
Linus Torvalds 已提交
78 79 80
{
	struct address_space *mapping = NULL;
	struct mm_struct *mm = vma->vm_mm;
81
	pte_t *old_pte, *new_pte, pte;
H
Hugh Dickins 已提交
82
	spinlock_t *old_ptl, *new_ptl;
L
Linus Torvalds 已提交
83 84 85 86

	if (vma->vm_file) {
		/*
		 * Subtle point from Rajesh Venkatasubramanian: before
N
npiggin@suse.de 已提交
87 88
		 * moving file-based ptes, we must lock truncate_pagecache
		 * out, since it might clean the dst vma before the src vma,
L
Linus Torvalds 已提交
89 90 91
		 * and we propagate stale pages into the dst afterward.
		 */
		mapping = vma->vm_file->f_mapping;
92
		mutex_lock(&mapping->i_mmap_mutex);
L
Linus Torvalds 已提交
93 94
	}

H
Hugh Dickins 已提交
95 96 97 98
	/*
	 * We don't have to worry about the ordering of src and dst
	 * pte locks because exclusive mmap_sem prevents deadlock.
	 */
H
Hugh Dickins 已提交
99
	old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
P
Peter Zijlstra 已提交
100
	new_pte = pte_offset_map(new_pmd, new_addr);
H
Hugh Dickins 已提交
101 102
	new_ptl = pte_lockptr(mm, new_pmd);
	if (new_ptl != old_ptl)
I
Ingo Molnar 已提交
103
		spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
104
	arch_enter_lazy_mmu_mode();
105 106 107 108 109

	for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
				   new_pte++, new_addr += PAGE_SIZE) {
		if (pte_none(*old_pte))
			continue;
110
		pte = ptep_get_and_clear(mm, old_addr, old_pte);
111 112
		pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
		set_pte_at(mm, new_addr, new_pte, pte);
L
Linus Torvalds 已提交
113
	}
114

115
	arch_leave_lazy_mmu_mode();
H
Hugh Dickins 已提交
116 117
	if (new_ptl != old_ptl)
		spin_unlock(new_ptl);
P
Peter Zijlstra 已提交
118
	pte_unmap(new_pte - 1);
H
Hugh Dickins 已提交
119
	pte_unmap_unlock(old_pte - 1, old_ptl);
L
Linus Torvalds 已提交
120
	if (mapping)
121
		mutex_unlock(&mapping->i_mmap_mutex);
L
Linus Torvalds 已提交
122 123
}

124 125
#define LATENCY_LIMIT	(64 * PAGE_SIZE)

126
unsigned long move_page_tables(struct vm_area_struct *vma,
L
Linus Torvalds 已提交
127 128 129
		unsigned long old_addr, struct vm_area_struct *new_vma,
		unsigned long new_addr, unsigned long len)
{
130 131
	unsigned long extent, next, old_end;
	pmd_t *old_pmd, *new_pmd;
132
	bool need_flush = false;
L
Linus Torvalds 已提交
133

134 135
	old_end = old_addr + len;
	flush_cache_range(vma, old_addr, old_end);
L
Linus Torvalds 已提交
136

137 138
	mmu_notifier_invalidate_range_start(vma->vm_mm, old_addr, old_end);

139
	for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
L
Linus Torvalds 已提交
140
		cond_resched();
141
		next = (old_addr + PMD_SIZE) & PMD_MASK;
142
		/* even if next overflowed, extent below will be ok */
143
		extent = next - old_addr;
144 145
		if (extent > old_end - old_addr)
			extent = old_end - old_addr;
146 147 148
		old_pmd = get_old_pmd(vma->vm_mm, old_addr);
		if (!old_pmd)
			continue;
149
		new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
150 151 152 153 154 155 156 157 158
		if (!new_pmd)
			break;
		next = (new_addr + PMD_SIZE) & PMD_MASK;
		if (extent > next - new_addr)
			extent = next - new_addr;
		if (extent > LATENCY_LIMIT)
			extent = LATENCY_LIMIT;
		move_ptes(vma, old_pmd, old_addr, old_addr + extent,
				new_vma, new_pmd, new_addr);
159
		need_flush = true;
L
Linus Torvalds 已提交
160
	}
161 162 163 164
	if (likely(need_flush))
		flush_tlb_range(vma, old_end-len, old_addr);

	mmu_notifier_invalidate_range_end(vma->vm_mm, old_end-len, old_end);
165 166

	return len + old_addr - old_end;	/* how much done */
L
Linus Torvalds 已提交
167 168 169 170 171 172 173 174 175 176 177 178
}

static unsigned long move_vma(struct vm_area_struct *vma,
		unsigned long old_addr, unsigned long old_len,
		unsigned long new_len, unsigned long new_addr)
{
	struct mm_struct *mm = vma->vm_mm;
	struct vm_area_struct *new_vma;
	unsigned long vm_flags = vma->vm_flags;
	unsigned long new_pgoff;
	unsigned long moved_len;
	unsigned long excess = 0;
179
	unsigned long hiwater_vm;
L
Linus Torvalds 已提交
180
	int split = 0;
181
	int err;
L
Linus Torvalds 已提交
182 183 184 185 186 187 188 189

	/*
	 * We'd prefer to avoid failure later on in do_munmap:
	 * which may split one vma into three before unmapping.
	 */
	if (mm->map_count >= sysctl_max_map_count - 3)
		return -ENOMEM;

190 191 192 193 194 195 196
	/*
	 * Advise KSM to break any KSM pages in the area to be moved:
	 * it would be confusing if they were to turn up at the new
	 * location, where they happen to coincide with different KSM
	 * pages recently unmapped.  But leave vma->vm_flags as it was,
	 * so KSM can come around to merge on vma and new_vma afterwards.
	 */
197 198 199 200
	err = ksm_madvise(vma, old_addr, old_addr + old_len,
						MADV_UNMERGEABLE, &vm_flags);
	if (err)
		return err;
201

L
Linus Torvalds 已提交
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
	new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
	new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
	if (!new_vma)
		return -ENOMEM;

	moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len);
	if (moved_len < old_len) {
		/*
		 * On error, move entries back from new area to old,
		 * which will succeed since page tables still there,
		 * and then proceed to unmap new area instead of old.
		 */
		move_page_tables(new_vma, new_addr, vma, old_addr, moved_len);
		vma = new_vma;
		old_len = new_len;
		old_addr = new_addr;
		new_addr = -ENOMEM;
	}

	/* Conceal VM_ACCOUNT so old reservation is not undone */
	if (vm_flags & VM_ACCOUNT) {
		vma->vm_flags &= ~VM_ACCOUNT;
		excess = vma->vm_end - vma->vm_start - old_len;
		if (old_addr > vma->vm_start &&
		    old_addr + old_len < vma->vm_end)
			split = 1;
	}

K
Kirill Korotaev 已提交
230
	/*
231 232 233 234 235 236 237
	 * If we failed to move page tables we still do total_vm increment
	 * since do_munmap() will decrement it by old_len == new_len.
	 *
	 * Since total_vm is about to be raised artificially high for a
	 * moment, we need to restore high watermark afterwards: if stats
	 * are taken meanwhile, total_vm and hiwater_vm appear too high.
	 * If this were a serious issue, we'd add a flag to do_munmap().
K
Kirill Korotaev 已提交
238
	 */
239
	hiwater_vm = mm->hiwater_vm;
K
Kirill Korotaev 已提交
240
	mm->total_vm += new_len >> PAGE_SHIFT;
241
	vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
K
Kirill Korotaev 已提交
242

L
Linus Torvalds 已提交
243 244 245 246 247
	if (do_munmap(mm, old_addr, old_len) < 0) {
		/* OOM: unable to split vma, just get accounts right */
		vm_unacct_memory(excess >> PAGE_SHIFT);
		excess = 0;
	}
248
	mm->hiwater_vm = hiwater_vm;
L
Linus Torvalds 已提交
249 250 251 252 253 254 255 256 257 258 259

	/* Restore VM_ACCOUNT if one or two pieces of vma left */
	if (excess) {
		vma->vm_flags |= VM_ACCOUNT;
		if (split)
			vma->vm_next->vm_flags |= VM_ACCOUNT;
	}

	if (vm_flags & VM_LOCKED) {
		mm->locked_vm += new_len >> PAGE_SHIFT;
		if (new_len > old_len)
260 261
			mlock_vma_pages_range(new_vma, new_addr + old_len,
						       new_addr + new_len);
L
Linus Torvalds 已提交
262 263 264 265 266
	}

	return new_addr;
}

A
Al Viro 已提交
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
static struct vm_area_struct *vma_to_resize(unsigned long addr,
	unsigned long old_len, unsigned long new_len, unsigned long *p)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma = find_vma(mm, addr);

	if (!vma || vma->vm_start > addr)
		goto Efault;

	if (is_vm_hugetlb_page(vma))
		goto Einval;

	/* We can't remap across vm area boundaries */
	if (old_len > vma->vm_end - addr)
		goto Efault;

283 284 285 286 287
	/* Need to be careful about a growing mapping */
	if (new_len > old_len) {
		unsigned long pgoff;

		if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))
A
Al Viro 已提交
288
			goto Efault;
289 290 291 292
		pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
		pgoff += vma->vm_pgoff;
		if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
			goto Einval;
A
Al Viro 已提交
293 294 295 296 297
	}

	if (vma->vm_flags & VM_LOCKED) {
		unsigned long locked, lock_limit;
		locked = mm->locked_vm << PAGE_SHIFT;
J
Jiri Slaby 已提交
298
		lock_limit = rlimit(RLIMIT_MEMLOCK);
A
Al Viro 已提交
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
		locked += new_len - old_len;
		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
			goto Eagain;
	}

	if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT))
		goto Enomem;

	if (vma->vm_flags & VM_ACCOUNT) {
		unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
		if (security_vm_enough_memory(charged))
			goto Efault;
		*p = charged;
	}

	return vma;

Efault:	/* very odd choice for most of the cases, but... */
	return ERR_PTR(-EFAULT);
Einval:
	return ERR_PTR(-EINVAL);
Enomem:
	return ERR_PTR(-ENOMEM);
Eagain:
	return ERR_PTR(-EAGAIN);
}

A
Al Viro 已提交
326 327 328 329 330 331 332 333
static unsigned long mremap_to(unsigned long addr,
	unsigned long old_len, unsigned long new_addr,
	unsigned long new_len)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	unsigned long ret = -EINVAL;
	unsigned long charged = 0;
A
Al Viro 已提交
334
	unsigned long map_flags;
A
Al Viro 已提交
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371

	if (new_addr & ~PAGE_MASK)
		goto out;

	if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
		goto out;

	/* Check if the location we're moving into overlaps the
	 * old location at all, and fail if it does.
	 */
	if ((new_addr <= addr) && (new_addr+new_len) > addr)
		goto out;

	if ((addr <= new_addr) && (addr+old_len) > new_addr)
		goto out;

	ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1);
	if (ret)
		goto out;

	ret = do_munmap(mm, new_addr, new_len);
	if (ret)
		goto out;

	if (old_len >= new_len) {
		ret = do_munmap(mm, addr+new_len, old_len - new_len);
		if (ret && old_len != new_len)
			goto out;
		old_len = new_len;
	}

	vma = vma_to_resize(addr, old_len, new_len, &charged);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto out;
	}

A
Al Viro 已提交
372 373 374
	map_flags = MAP_FIXED;
	if (vma->vm_flags & VM_MAYSHARE)
		map_flags |= MAP_SHARED;
375

A
Al Viro 已提交
376 377 378
	ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
				((addr - vma->vm_start) >> PAGE_SHIFT),
				map_flags);
A
Al Viro 已提交
379
	if (ret & ~PAGE_MASK)
A
Al Viro 已提交
380 381 382 383 384 385 386
		goto out1;

	ret = move_vma(vma, addr, old_len, new_len, new_addr);
	if (!(ret & ~PAGE_MASK))
		goto out;
out1:
	vm_unacct_memory(charged);
A
Al Viro 已提交
387 388 389 390 391

out:
	return ret;
}

A
Al Viro 已提交
392 393
static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
{
A
Al Viro 已提交
394
	unsigned long end = vma->vm_end + delta;
395
	if (end < vma->vm_end) /* overflow */
A
Al Viro 已提交
396
		return 0;
397
	if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */
A
Al Viro 已提交
398 399 400
		return 0;
	if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
			      0, MAP_FIXED) & ~PAGE_MASK)
A
Al Viro 已提交
401 402 403 404
		return 0;
	return 1;
}

L
Linus Torvalds 已提交
405 406 407 408 409 410 411 412 413 414 415
/*
 * Expand (or shrink) an existing mapping, potentially moving it at the
 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
 *
 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
 * This option implies MREMAP_MAYMOVE.
 */
unsigned long do_mremap(unsigned long addr,
	unsigned long old_len, unsigned long new_len,
	unsigned long flags, unsigned long new_addr)
{
H
Hugh Dickins 已提交
416
	struct mm_struct *mm = current->mm;
L
Linus Torvalds 已提交
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
	struct vm_area_struct *vma;
	unsigned long ret = -EINVAL;
	unsigned long charged = 0;

	if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
		goto out;

	if (addr & ~PAGE_MASK)
		goto out;

	old_len = PAGE_ALIGN(old_len);
	new_len = PAGE_ALIGN(new_len);

	/*
	 * We allow a zero old-len as a special case
	 * for DOS-emu "duplicate shm area" thing. But
	 * a zero new-len is nonsensical.
	 */
	if (!new_len)
		goto out;

	if (flags & MREMAP_FIXED) {
A
Al Viro 已提交
439 440 441
		if (flags & MREMAP_MAYMOVE)
			ret = mremap_to(addr, old_len, new_addr, new_len);
		goto out;
L
Linus Torvalds 已提交
442 443 444 445 446 447 448 449
	}

	/*
	 * Always allow a shrinking remap: that just unmaps
	 * the unnecessary pages..
	 * do_munmap does all the needed commit accounting
	 */
	if (old_len >= new_len) {
H
Hugh Dickins 已提交
450
		ret = do_munmap(mm, addr+new_len, old_len - new_len);
L
Linus Torvalds 已提交
451 452 453
		if (ret && old_len != new_len)
			goto out;
		ret = addr;
A
Al Viro 已提交
454
		goto out;
L
Linus Torvalds 已提交
455 456 457
	}

	/*
A
Al Viro 已提交
458
	 * Ok, we need to grow..
L
Linus Torvalds 已提交
459
	 */
A
Al Viro 已提交
460 461 462
	vma = vma_to_resize(addr, old_len, new_len, &charged);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
L
Linus Torvalds 已提交
463
		goto out;
A
akpm@osdl.org 已提交
464
	}
L
Linus Torvalds 已提交
465 466 467

	/* old_len exactly to the end of the area..
	 */
A
Al Viro 已提交
468
	if (old_len == vma->vm_end - addr) {
L
Linus Torvalds 已提交
469
		/* can we just expand the current mapping? */
A
Al Viro 已提交
470
		if (vma_expandable(vma, new_len - old_len)) {
L
Linus Torvalds 已提交
471 472
			int pages = (new_len - old_len) >> PAGE_SHIFT;

473 474 475 476 477
			if (vma_adjust(vma, vma->vm_start, addr + new_len,
				       vma->vm_pgoff, NULL)) {
				ret = -ENOMEM;
				goto out;
			}
L
Linus Torvalds 已提交
478

H
Hugh Dickins 已提交
479 480
			mm->total_vm += pages;
			vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
L
Linus Torvalds 已提交
481
			if (vma->vm_flags & VM_LOCKED) {
H
Hugh Dickins 已提交
482
				mm->locked_vm += pages;
483
				mlock_vma_pages_range(vma, addr + old_len,
L
Linus Torvalds 已提交
484 485 486 487 488 489 490 491 492 493 494 495 496
						   addr + new_len);
			}
			ret = addr;
			goto out;
		}
	}

	/*
	 * We weren't able to just expand or shrink the area,
	 * we need to create a new one and move it..
	 */
	ret = -ENOMEM;
	if (flags & MREMAP_MAYMOVE) {
A
Al Viro 已提交
497 498 499 500 501
		unsigned long map_flags = 0;
		if (vma->vm_flags & VM_MAYSHARE)
			map_flags |= MAP_SHARED;

		new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
502 503 504
					vma->vm_pgoff +
					((addr - vma->vm_start) >> PAGE_SHIFT),
					map_flags);
A
Al Viro 已提交
505 506 507
		if (new_addr & ~PAGE_MASK) {
			ret = new_addr;
			goto out;
L
Linus Torvalds 已提交
508
		}
A
Al Viro 已提交
509 510 511 512

		ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1);
		if (ret)
			goto out;
L
Linus Torvalds 已提交
513 514 515 516 517 518 519 520
		ret = move_vma(vma, addr, old_len, new_len, new_addr);
	}
out:
	if (ret & ~PAGE_MASK)
		vm_unacct_memory(charged);
	return ret;
}

521 522 523
SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
		unsigned long, new_len, unsigned long, flags,
		unsigned long, new_addr)
L
Linus Torvalds 已提交
524 525 526 527 528 529 530 531
{
	unsigned long ret;

	down_write(&current->mm->mmap_sem);
	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
	up_write(&current->mm->mmap_sem);
	return ret;
}