mremap.c 11.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 *	mm/mremap.c
 *
 *	(C) Copyright 1996 Linus Torvalds
 *
A
Alan Cox 已提交
6
 *	Address space accounting code	<alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
7 8 9 10 11 12 13
 *	(C) Copyright 2002 Red Hat Inc, All Rights Reserved
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/slab.h>
#include <linux/shm.h>
14
#include <linux/ksm.h>
L
Linus Torvalds 已提交
15 16
#include <linux/mman.h>
#include <linux/swap.h>
17
#include <linux/capability.h>
L
Linus Torvalds 已提交
18 19 20 21
#include <linux/fs.h>
#include <linux/highmem.h>
#include <linux/security.h>
#include <linux/syscalls.h>
A
Andrea Arcangeli 已提交
22
#include <linux/mmu_notifier.h>
L
Linus Torvalds 已提交
23 24 25 26 27

#include <asm/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>

28 29
#include "internal.h"

30
static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
L
Linus Torvalds 已提交
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	pgd = pgd_offset(mm, addr);
	if (pgd_none_or_clear_bad(pgd))
		return NULL;

	pud = pud_offset(pgd, addr);
	if (pud_none_or_clear_bad(pud))
		return NULL;

	pmd = pmd_offset(pud, addr);
	if (pmd_none_or_clear_bad(pmd))
		return NULL;

48
	return pmd;
L
Linus Torvalds 已提交
49 50
}

51
static pmd_t *alloc_new_pmd(struct mm_struct *mm, unsigned long addr)
L
Linus Torvalds 已提交
52 53 54
{
	pgd_t *pgd;
	pud_t *pud;
H
Hugh Dickins 已提交
55
	pmd_t *pmd;
L
Linus Torvalds 已提交
56 57 58 59

	pgd = pgd_offset(mm, addr);
	pud = pud_alloc(mm, pgd, addr);
	if (!pud)
H
Hugh Dickins 已提交
60
		return NULL;
61

L
Linus Torvalds 已提交
62
	pmd = pmd_alloc(mm, pud, addr);
63
	if (!pmd)
H
Hugh Dickins 已提交
64
		return NULL;
65

66
	if (!pmd_present(*pmd) && __pte_alloc(mm, pmd, addr))
H
Hugh Dickins 已提交
67 68
		return NULL;

69
	return pmd;
L
Linus Torvalds 已提交
70 71
}

72 73 74 75
static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
		unsigned long old_addr, unsigned long old_end,
		struct vm_area_struct *new_vma, pmd_t *new_pmd,
		unsigned long new_addr)
L
Linus Torvalds 已提交
76 77 78
{
	struct address_space *mapping = NULL;
	struct mm_struct *mm = vma->vm_mm;
79
	pte_t *old_pte, *new_pte, pte;
H
Hugh Dickins 已提交
80
	spinlock_t *old_ptl, *new_ptl;
A
Andrea Arcangeli 已提交
81
	unsigned long old_start;
L
Linus Torvalds 已提交
82

A
Andrea Arcangeli 已提交
83 84 85
	old_start = old_addr;
	mmu_notifier_invalidate_range_start(vma->vm_mm,
					    old_start, old_end);
L
Linus Torvalds 已提交
86 87 88
	if (vma->vm_file) {
		/*
		 * Subtle point from Rajesh Venkatasubramanian: before
N
npiggin@suse.de 已提交
89 90
		 * moving file-based ptes, we must lock truncate_pagecache
		 * out, since it might clean the dst vma before the src vma,
L
Linus Torvalds 已提交
91 92 93 94 95 96 97 98 99
		 * and we propagate stale pages into the dst afterward.
		 */
		mapping = vma->vm_file->f_mapping;
		spin_lock(&mapping->i_mmap_lock);
		if (new_vma->vm_truncate_count &&
		    new_vma->vm_truncate_count != vma->vm_truncate_count)
			new_vma->vm_truncate_count = 0;
	}

H
Hugh Dickins 已提交
100 101 102 103
	/*
	 * We don't have to worry about the ordering of src and dst
	 * pte locks because exclusive mmap_sem prevents deadlock.
	 */
H
Hugh Dickins 已提交
104 105
	old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
 	new_pte = pte_offset_map_nested(new_pmd, new_addr);
H
Hugh Dickins 已提交
106 107
	new_ptl = pte_lockptr(mm, new_pmd);
	if (new_ptl != old_ptl)
I
Ingo Molnar 已提交
108
		spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
109
	arch_enter_lazy_mmu_mode();
110 111 112 113 114 115 116 117

	for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
				   new_pte++, new_addr += PAGE_SIZE) {
		if (pte_none(*old_pte))
			continue;
		pte = ptep_clear_flush(vma, old_addr, old_pte);
		pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
		set_pte_at(mm, new_addr, new_pte, pte);
L
Linus Torvalds 已提交
118
	}
119

120
	arch_leave_lazy_mmu_mode();
H
Hugh Dickins 已提交
121 122
	if (new_ptl != old_ptl)
		spin_unlock(new_ptl);
123
	pte_unmap_nested(new_pte - 1);
H
Hugh Dickins 已提交
124
	pte_unmap_unlock(old_pte - 1, old_ptl);
L
Linus Torvalds 已提交
125 126
	if (mapping)
		spin_unlock(&mapping->i_mmap_lock);
A
Andrea Arcangeli 已提交
127
	mmu_notifier_invalidate_range_end(vma->vm_mm, old_start, old_end);
L
Linus Torvalds 已提交
128 129
}

130 131
#define LATENCY_LIMIT	(64 * PAGE_SIZE)

132
unsigned long move_page_tables(struct vm_area_struct *vma,
L
Linus Torvalds 已提交
133 134 135
		unsigned long old_addr, struct vm_area_struct *new_vma,
		unsigned long new_addr, unsigned long len)
{
136 137
	unsigned long extent, next, old_end;
	pmd_t *old_pmd, *new_pmd;
L
Linus Torvalds 已提交
138

139 140
	old_end = old_addr + len;
	flush_cache_range(vma, old_addr, old_end);
L
Linus Torvalds 已提交
141

142
	for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
L
Linus Torvalds 已提交
143
		cond_resched();
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
		next = (old_addr + PMD_SIZE) & PMD_MASK;
		if (next - 1 > old_end)
			next = old_end;
		extent = next - old_addr;
		old_pmd = get_old_pmd(vma->vm_mm, old_addr);
		if (!old_pmd)
			continue;
		new_pmd = alloc_new_pmd(vma->vm_mm, new_addr);
		if (!new_pmd)
			break;
		next = (new_addr + PMD_SIZE) & PMD_MASK;
		if (extent > next - new_addr)
			extent = next - new_addr;
		if (extent > LATENCY_LIMIT)
			extent = LATENCY_LIMIT;
		move_ptes(vma, old_pmd, old_addr, old_addr + extent,
				new_vma, new_pmd, new_addr);
L
Linus Torvalds 已提交
161
	}
162 163

	return len + old_addr - old_end;	/* how much done */
L
Linus Torvalds 已提交
164 165 166 167 168 169 170 171 172 173 174 175
}

static unsigned long move_vma(struct vm_area_struct *vma,
		unsigned long old_addr, unsigned long old_len,
		unsigned long new_len, unsigned long new_addr)
{
	struct mm_struct *mm = vma->vm_mm;
	struct vm_area_struct *new_vma;
	unsigned long vm_flags = vma->vm_flags;
	unsigned long new_pgoff;
	unsigned long moved_len;
	unsigned long excess = 0;
176
	unsigned long hiwater_vm;
L
Linus Torvalds 已提交
177
	int split = 0;
178
	int err;
L
Linus Torvalds 已提交
179 180 181 182 183 184 185 186

	/*
	 * We'd prefer to avoid failure later on in do_munmap:
	 * which may split one vma into three before unmapping.
	 */
	if (mm->map_count >= sysctl_max_map_count - 3)
		return -ENOMEM;

187 188 189 190 191 192 193
	/*
	 * Advise KSM to break any KSM pages in the area to be moved:
	 * it would be confusing if they were to turn up at the new
	 * location, where they happen to coincide with different KSM
	 * pages recently unmapped.  But leave vma->vm_flags as it was,
	 * so KSM can come around to merge on vma and new_vma afterwards.
	 */
194 195 196 197
	err = ksm_madvise(vma, old_addr, old_addr + old_len,
						MADV_UNMERGEABLE, &vm_flags);
	if (err)
		return err;
198

L
Linus Torvalds 已提交
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
	new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
	new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
	if (!new_vma)
		return -ENOMEM;

	moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len);
	if (moved_len < old_len) {
		/*
		 * On error, move entries back from new area to old,
		 * which will succeed since page tables still there,
		 * and then proceed to unmap new area instead of old.
		 */
		move_page_tables(new_vma, new_addr, vma, old_addr, moved_len);
		vma = new_vma;
		old_len = new_len;
		old_addr = new_addr;
		new_addr = -ENOMEM;
	}

	/* Conceal VM_ACCOUNT so old reservation is not undone */
	if (vm_flags & VM_ACCOUNT) {
		vma->vm_flags &= ~VM_ACCOUNT;
		excess = vma->vm_end - vma->vm_start - old_len;
		if (old_addr > vma->vm_start &&
		    old_addr + old_len < vma->vm_end)
			split = 1;
	}

K
Kirill Korotaev 已提交
227
	/*
228 229 230 231 232 233 234
	 * If we failed to move page tables we still do total_vm increment
	 * since do_munmap() will decrement it by old_len == new_len.
	 *
	 * Since total_vm is about to be raised artificially high for a
	 * moment, we need to restore high watermark afterwards: if stats
	 * are taken meanwhile, total_vm and hiwater_vm appear too high.
	 * If this were a serious issue, we'd add a flag to do_munmap().
K
Kirill Korotaev 已提交
235
	 */
236
	hiwater_vm = mm->hiwater_vm;
K
Kirill Korotaev 已提交
237
	mm->total_vm += new_len >> PAGE_SHIFT;
238
	vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
K
Kirill Korotaev 已提交
239

L
Linus Torvalds 已提交
240 241 242 243 244
	if (do_munmap(mm, old_addr, old_len) < 0) {
		/* OOM: unable to split vma, just get accounts right */
		vm_unacct_memory(excess >> PAGE_SHIFT);
		excess = 0;
	}
245
	mm->hiwater_vm = hiwater_vm;
L
Linus Torvalds 已提交
246 247 248 249 250 251 252 253 254 255 256

	/* Restore VM_ACCOUNT if one or two pieces of vma left */
	if (excess) {
		vma->vm_flags |= VM_ACCOUNT;
		if (split)
			vma->vm_next->vm_flags |= VM_ACCOUNT;
	}

	if (vm_flags & VM_LOCKED) {
		mm->locked_vm += new_len >> PAGE_SHIFT;
		if (new_len > old_len)
257 258
			mlock_vma_pages_range(new_vma, new_addr + old_len,
						       new_addr + new_len);
L
Linus Torvalds 已提交
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
	}

	return new_addr;
}

/*
 * Expand (or shrink) an existing mapping, potentially moving it at the
 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
 *
 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
 * This option implies MREMAP_MAYMOVE.
 */
unsigned long do_mremap(unsigned long addr,
	unsigned long old_len, unsigned long new_len,
	unsigned long flags, unsigned long new_addr)
{
H
Hugh Dickins 已提交
275
	struct mm_struct *mm = current->mm;
L
Linus Torvalds 已提交
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
	struct vm_area_struct *vma;
	unsigned long ret = -EINVAL;
	unsigned long charged = 0;

	if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
		goto out;

	if (addr & ~PAGE_MASK)
		goto out;

	old_len = PAGE_ALIGN(old_len);
	new_len = PAGE_ALIGN(new_len);

	/*
	 * We allow a zero old-len as a special case
	 * for DOS-emu "duplicate shm area" thing. But
	 * a zero new-len is nonsensical.
	 */
	if (!new_len)
		goto out;

	/* new_addr is only valid if MREMAP_FIXED is specified */
	if (flags & MREMAP_FIXED) {
		if (new_addr & ~PAGE_MASK)
			goto out;
		if (!(flags & MREMAP_MAYMOVE))
			goto out;

		if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
			goto out;

		/* Check if the location we're moving into overlaps the
		 * old location at all, and fail if it does.
		 */
		if ((new_addr <= addr) && (new_addr+new_len) > addr)
			goto out;

		if ((addr <= new_addr) && (addr+old_len) > new_addr)
			goto out;

316
		ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1);
317 318 319
		if (ret)
			goto out;

H
Hugh Dickins 已提交
320
		ret = do_munmap(mm, new_addr, new_len);
L
Linus Torvalds 已提交
321 322 323 324 325 326 327 328 329 330
		if (ret)
			goto out;
	}

	/*
	 * Always allow a shrinking remap: that just unmaps
	 * the unnecessary pages..
	 * do_munmap does all the needed commit accounting
	 */
	if (old_len >= new_len) {
H
Hugh Dickins 已提交
331
		ret = do_munmap(mm, addr+new_len, old_len - new_len);
L
Linus Torvalds 已提交
332 333 334 335 336 337 338 339 340 341 342 343
		if (ret && old_len != new_len)
			goto out;
		ret = addr;
		if (!(flags & MREMAP_FIXED) || (new_addr == addr))
			goto out;
		old_len = new_len;
	}

	/*
	 * Ok, we need to grow..  or relocate.
	 */
	ret = -EFAULT;
H
Hugh Dickins 已提交
344
	vma = find_vma(mm, addr);
L
Linus Torvalds 已提交
345 346 347 348 349 350 351 352 353
	if (!vma || vma->vm_start > addr)
		goto out;
	if (is_vm_hugetlb_page(vma)) {
		ret = -EINVAL;
		goto out;
	}
	/* We can't remap across vm area boundaries */
	if (old_len > vma->vm_end - addr)
		goto out;
354
	if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) {
L
Linus Torvalds 已提交
355 356 357 358 359
		if (new_len > old_len)
			goto out;
	}
	if (vma->vm_flags & VM_LOCKED) {
		unsigned long locked, lock_limit;
H
Hugh Dickins 已提交
360
		locked = mm->locked_vm << PAGE_SHIFT;
L
Linus Torvalds 已提交
361 362 363 364 365 366
		lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
		locked += new_len - old_len;
		ret = -EAGAIN;
		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
			goto out;
	}
H
Hugh Dickins 已提交
367
	if (!may_expand_vm(mm, (new_len - old_len) >> PAGE_SHIFT)) {
A
akpm@osdl.org 已提交
368
		ret = -ENOMEM;
L
Linus Torvalds 已提交
369
		goto out;
A
akpm@osdl.org 已提交
370
	}
L
Linus Torvalds 已提交
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393

	if (vma->vm_flags & VM_ACCOUNT) {
		charged = (new_len - old_len) >> PAGE_SHIFT;
		if (security_vm_enough_memory(charged))
			goto out_nc;
	}

	/* old_len exactly to the end of the area..
	 * And we're not relocating the area.
	 */
	if (old_len == vma->vm_end - addr &&
	    !((flags & MREMAP_FIXED) && (addr != new_addr)) &&
	    (old_len != new_len || !(flags & MREMAP_MAYMOVE))) {
		unsigned long max_addr = TASK_SIZE;
		if (vma->vm_next)
			max_addr = vma->vm_next->vm_start;
		/* can we just expand the current mapping? */
		if (max_addr - addr >= new_len) {
			int pages = (new_len - old_len) >> PAGE_SHIFT;

			vma_adjust(vma, vma->vm_start,
				addr + new_len, vma->vm_pgoff, NULL);

H
Hugh Dickins 已提交
394 395
			mm->total_vm += pages;
			vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
L
Linus Torvalds 已提交
396
			if (vma->vm_flags & VM_LOCKED) {
H
Hugh Dickins 已提交
397
				mm->locked_vm += pages;
398
				mlock_vma_pages_range(vma, addr + old_len,
L
Linus Torvalds 已提交
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
						   addr + new_len);
			}
			ret = addr;
			goto out;
		}
	}

	/*
	 * We weren't able to just expand or shrink the area,
	 * we need to create a new one and move it..
	 */
	ret = -ENOMEM;
	if (flags & MREMAP_MAYMOVE) {
		if (!(flags & MREMAP_FIXED)) {
			unsigned long map_flags = 0;
			if (vma->vm_flags & VM_MAYSHARE)
				map_flags |= MAP_SHARED;

			new_addr = get_unmapped_area(vma->vm_file, 0, new_len,
						vma->vm_pgoff, map_flags);
419 420 421 422 423
			if (new_addr & ~PAGE_MASK) {
				ret = new_addr;
				goto out;
			}

424
			ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1);
425
			if (ret)
L
Linus Torvalds 已提交
426 427 428 429 430 431 432 433 434 435 436
				goto out;
		}
		ret = move_vma(vma, addr, old_len, new_len, new_addr);
	}
out:
	if (ret & ~PAGE_MASK)
		vm_unacct_memory(charged);
out_nc:
	return ret;
}

437 438 439
SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
		unsigned long, new_len, unsigned long, flags,
		unsigned long, new_addr)
L
Linus Torvalds 已提交
440 441 442 443 444 445 446 447
{
	unsigned long ret;

	down_write(&current->mm->mmap_sem);
	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
	up_write(&current->mm->mmap_sem);
	return ret;
}