madvise.c 10.6 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10
/*
 *	linux/mm/madvise.c
 *
 * Copyright (C) 1999  Linus Torvalds
 * Copyright (C) 2002  Christoph Hellwig
 */

#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/syscalls.h>
P
Prasanna Meda 已提交
11
#include <linux/mempolicy.h>
L
Linus Torvalds 已提交
12
#include <linux/hugetlb.h>
A
Alexey Dobriyan 已提交
13
#include <linux/sched.h>
L
Linus Torvalds 已提交
14

15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Any behaviour which results in changes to the vma->vm_flags needs to
 * take mmap_sem for writing. Others, which simply traverse vmas, need
 * to only take it for reading.
 */
static int madvise_need_mmap_write(int behavior)
{
	switch (behavior) {
	case MADV_REMOVE:
	case MADV_WILLNEED:
	case MADV_DONTNEED:
		return 0;
	default:
		/* be safe, default to 1. list exceptions explicitly */
		return 1;
	}
}

L
Linus Torvalds 已提交
33 34 35 36
/*
 * We can potentially split a vm area into separate
 * areas, each area with its own behavior.
 */
P
Prasanna Meda 已提交
37 38 39
static long madvise_behavior(struct vm_area_struct * vma,
		     struct vm_area_struct **prev,
		     unsigned long start, unsigned long end, int behavior)
L
Linus Torvalds 已提交
40 41 42
{
	struct mm_struct * mm = vma->vm_mm;
	int error = 0;
P
Prasanna Meda 已提交
43
	pgoff_t pgoff;
44
	int new_flags = vma->vm_flags;
45 46

	switch (behavior) {
47 48 49
	case MADV_NORMAL:
		new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
		break;
50
	case MADV_SEQUENTIAL:
51
		new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
52 53
		break;
	case MADV_RANDOM:
54
		new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
55
		break;
56 57 58 59 60
	case MADV_DONTFORK:
		new_flags |= VM_DONTCOPY;
		break;
	case MADV_DOFORK:
		new_flags &= ~VM_DONTCOPY;
61 62 63
		break;
	}

P
Prasanna Meda 已提交
64 65
	if (new_flags == vma->vm_flags) {
		*prev = vma;
66
		goto out;
P
Prasanna Meda 已提交
67 68 69 70 71 72 73 74 75 76 77
	}

	pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
	*prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
				vma->vm_file, pgoff, vma_policy(vma));
	if (*prev) {
		vma = *prev;
		goto success;
	}

	*prev = vma;
L
Linus Torvalds 已提交
78 79 80 81 82 83 84 85 86 87 88 89 90

	if (start != vma->vm_start) {
		error = split_vma(mm, vma, start, 1);
		if (error)
			goto out;
	}

	if (end != vma->vm_end) {
		error = split_vma(mm, vma, end, 0);
		if (error)
			goto out;
	}

91
success:
L
Linus Torvalds 已提交
92 93 94
	/*
	 * vm_flags is protected by the mmap_sem held in write mode.
	 */
95
	vma->vm_flags = new_flags;
L
Linus Torvalds 已提交
96 97 98 99 100 101 102 103 104 105 106

out:
	if (error == -ENOMEM)
		error = -EAGAIN;
	return error;
}

/*
 * Schedule all required I/O operations.  Do not wait for completion.
 */
static long madvise_willneed(struct vm_area_struct * vma,
P
Prasanna Meda 已提交
107
			     struct vm_area_struct ** prev,
L
Linus Torvalds 已提交
108 109 110 111
			     unsigned long start, unsigned long end)
{
	struct file *file = vma->vm_file;

112 113 114
	if (!file)
		return -EBADF;

115
	if (file->f_mapping->a_ops->get_xip_mem) {
116 117 118 119
		/* no bad return value, but ignore advice */
		return 0;
	}

P
Prasanna Meda 已提交
120
	*prev = vma;
L
Linus Torvalds 已提交
121 122 123 124 125
	start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
	if (end > vma->vm_end)
		end = vma->vm_end;
	end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;

126
	force_page_cache_readahead(file->f_mapping, file, start, end - start);
L
Linus Torvalds 已提交
127 128 129 130 131 132 133
	return 0;
}

/*
 * Application no longer needs these pages.  If the pages are dirty,
 * it's OK to just throw them away.  The app will be more careful about
 * data it wants to keep.  Be sure to free swap resources too.  The
134
 * zap_page_range call sets things up for shrink_active_list to actually free
L
Linus Torvalds 已提交
135 136
 * these pages later if no one else has touched them in the meantime,
 * although we could add these pages to a global reuse list for
137
 * shrink_active_list to pick up before reclaiming other pages.
L
Linus Torvalds 已提交
138 139 140 141 142 143 144 145 146 147 148 149
 *
 * NB: This interface discards data rather than pushes it out to swap,
 * as some implementations do.  This has performance implications for
 * applications like large transactional databases which want to discard
 * pages in anonymous maps after committing to backing store the data
 * that was kept in them.  There is no reason to write this data out to
 * the swap area if the application is discarding it.
 *
 * An interface that causes the system to free clean pages and flush
 * dirty pages is already available as msync(MS_INVALIDATE).
 */
static long madvise_dontneed(struct vm_area_struct * vma,
P
Prasanna Meda 已提交
150
			     struct vm_area_struct ** prev,
L
Linus Torvalds 已提交
151 152
			     unsigned long start, unsigned long end)
{
P
Prasanna Meda 已提交
153
	*prev = vma;
154
	if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
L
Linus Torvalds 已提交
155 156 157 158 159 160 161 162 163 164 165 166 167
		return -EINVAL;

	if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
		struct zap_details details = {
			.nonlinear_vma = vma,
			.last_index = ULONG_MAX,
		};
		zap_page_range(vma, start, end - start, &details);
	} else
		zap_page_range(vma, start, end - start, NULL);
	return 0;
}

168 169 170 171 172 173 174 175
/*
 * Application wants to free up the pages and associated backing store.
 * This is effectively punching a hole into the middle of a file.
 *
 * NOTE: Currently, only shmfs/tmpfs is supported for this operation.
 * Other filesystems return -ENOSYS.
 */
static long madvise_remove(struct vm_area_struct *vma,
176
				struct vm_area_struct **prev,
177 178 179
				unsigned long start, unsigned long end)
{
	struct address_space *mapping;
180 181
	loff_t offset, endoff;
	int error;
182

183
	*prev = NULL;	/* tell sys_madvise we drop mmap_sem */
184

185 186 187 188 189 190 191 192
	if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB))
		return -EINVAL;

	if (!vma->vm_file || !vma->vm_file->f_mapping
		|| !vma->vm_file->f_mapping->host) {
			return -EINVAL;
	}

193 194 195
	if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
		return -EACCES;

196 197 198 199 200 201
	mapping = vma->vm_file->f_mapping;

	offset = (loff_t)(start - vma->vm_start)
			+ ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
	endoff = (loff_t)(end - vma->vm_start - 1)
			+ ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
202 203

	/* vmtruncate_range needs to take i_mutex and i_alloc_sem */
204
	up_read(&current->mm->mmap_sem);
205
	error = vmtruncate_range(mapping->host, offset, endoff);
206
	down_read(&current->mm->mmap_sem);
207
	return error;
208 209
}

210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
#ifdef CONFIG_MEMORY_FAILURE
/*
 * Error injection support for memory error handling.
 */
static int madvise_hwpoison(unsigned long start, unsigned long end)
{
	int ret = 0;

	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;
	for (; start < end; start += PAGE_SIZE) {
		struct page *p;
		int ret = get_user_pages(current, current->mm, start, 1,
						0, 0, &p, NULL);
		if (ret != 1)
			return ret;
		printk(KERN_INFO "Injecting memory failure for page %lx at %lx\n",
		       page_to_pfn(p), start);
		/* Ignore return value for now */
		__memory_failure(page_to_pfn(p), 0, 1);
		put_page(p);
	}
	return ret;
}
#endif

236 237 238
static long
madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
		unsigned long start, unsigned long end, int behavior)
L
Linus Torvalds 已提交
239
{
240
	long error;
241

L
Linus Torvalds 已提交
242
	switch (behavior) {
243 244 245 246 247 248
	case MADV_DOFORK:
		if (vma->vm_flags & VM_IO) {
			error = -EINVAL;
			break;
		}
	case MADV_DONTFORK:
L
Linus Torvalds 已提交
249 250 251
	case MADV_NORMAL:
	case MADV_SEQUENTIAL:
	case MADV_RANDOM:
P
Prasanna Meda 已提交
252
		error = madvise_behavior(vma, prev, start, end, behavior);
L
Linus Torvalds 已提交
253
		break;
254
	case MADV_REMOVE:
255
		error = madvise_remove(vma, prev, start, end);
256
		break;
L
Linus Torvalds 已提交
257 258

	case MADV_WILLNEED:
P
Prasanna Meda 已提交
259
		error = madvise_willneed(vma, prev, start, end);
L
Linus Torvalds 已提交
260 261 262
		break;

	case MADV_DONTNEED:
P
Prasanna Meda 已提交
263
		error = madvise_dontneed(vma, prev, start, end);
L
Linus Torvalds 已提交
264 265 266
		break;

	default:
N
Nick Piggin 已提交
267
		BUG();
L
Linus Torvalds 已提交
268 269 270 271 272
		break;
	}
	return error;
}

N
Nick Piggin 已提交
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
static int
madvise_behavior_valid(int behavior)
{
	switch (behavior) {
	case MADV_DOFORK:
	case MADV_DONTFORK:
	case MADV_NORMAL:
	case MADV_SEQUENTIAL:
	case MADV_RANDOM:
	case MADV_REMOVE:
	case MADV_WILLNEED:
	case MADV_DONTNEED:
		return 1;

	default:
		return 0;
	}
}
L
Linus Torvalds 已提交
291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
/*
 * The madvise(2) system call.
 *
 * Applications can use madvise() to advise the kernel how it should
 * handle paging I/O in this VM area.  The idea is to help the kernel
 * use appropriate read-ahead and caching techniques.  The information
 * provided is advisory only, and can be safely disregarded by the
 * kernel without affecting the correct operation of the application.
 *
 * behavior values:
 *  MADV_NORMAL - the default behavior is to read clusters.  This
 *		results in some read-ahead and read-behind.
 *  MADV_RANDOM - the system should read the minimum amount of data
 *		on any access, since it is unlikely that the appli-
 *		cation will need more than what it asks for.
 *  MADV_SEQUENTIAL - pages in the given range will probably be accessed
 *		once, so they can be aggressively read ahead, and
 *		can be freed soon after they are accessed.
 *  MADV_WILLNEED - the application is notifying the system to read
 *		some pages ahead.
 *  MADV_DONTNEED - the application is finished with the given range,
 *		so the kernel can free resources associated with it.
313 314
 *  MADV_REMOVE - the application wants to free up the given range of
 *		pages and associated backing store.
L
Linus Torvalds 已提交
315 316 317 318 319 320 321 322 323 324 325 326
 *
 * return values:
 *  zero    - success
 *  -EINVAL - start + len < 0, start is not page-aligned,
 *		"behavior" is not a valid value, or application
 *		is attempting to release locked or shared pages.
 *  -ENOMEM - addresses in the specified range are not currently
 *		mapped, or are outside the AS of the process.
 *  -EIO    - an I/O error occurred while paging in data.
 *  -EBADF  - map exists, but area maps something that isn't a file.
 *  -EAGAIN - a kernel resource was temporarily unavailable.
 */
327
SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
L
Linus Torvalds 已提交
328
{
P
Prasanna Meda 已提交
329 330
	unsigned long end, tmp;
	struct vm_area_struct * vma, *prev;
L
Linus Torvalds 已提交
331 332
	int unmapped_error = 0;
	int error = -EINVAL;
333
	int write;
L
Linus Torvalds 已提交
334 335
	size_t len;

336 337 338 339
#ifdef CONFIG_MEMORY_FAILURE
	if (behavior == MADV_HWPOISON)
		return madvise_hwpoison(start, start+len_in);
#endif
N
Nick Piggin 已提交
340 341 342
	if (!madvise_behavior_valid(behavior))
		return error;

343 344
	write = madvise_need_mmap_write(behavior);
	if (write)
345 346 347
		down_write(&current->mm->mmap_sem);
	else
		down_read(&current->mm->mmap_sem);
L
Linus Torvalds 已提交
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367

	if (start & ~PAGE_MASK)
		goto out;
	len = (len_in + ~PAGE_MASK) & PAGE_MASK;

	/* Check to see whether len was rounded up from small -ve to zero */
	if (len_in && !len)
		goto out;

	end = start + len;
	if (end < start)
		goto out;

	error = 0;
	if (end == start)
		goto out;

	/*
	 * If the interval [start,end) covers some unmapped address
	 * ranges, just ignore them, but return -ENOMEM at the end.
P
Prasanna Meda 已提交
368
	 * - different from the way of handling in mlock etc.
L
Linus Torvalds 已提交
369
	 */
P
Prasanna Meda 已提交
370
	vma = find_vma_prev(current->mm, start, &prev);
371 372 373
	if (vma && start > vma->vm_start)
		prev = vma;

L
Linus Torvalds 已提交
374 375 376 377 378 379
	for (;;) {
		/* Still start < end. */
		error = -ENOMEM;
		if (!vma)
			goto out;

P
Prasanna Meda 已提交
380
		/* Here start < (end|vma->vm_end). */
L
Linus Torvalds 已提交
381 382 383
		if (start < vma->vm_start) {
			unmapped_error = -ENOMEM;
			start = vma->vm_start;
P
Prasanna Meda 已提交
384 385
			if (start >= end)
				goto out;
L
Linus Torvalds 已提交
386 387
		}

P
Prasanna Meda 已提交
388 389 390 391
		/* Here vma->vm_start <= start < (end|vma->vm_end) */
		tmp = vma->vm_end;
		if (end < tmp)
			tmp = end;
L
Linus Torvalds 已提交
392

P
Prasanna Meda 已提交
393 394
		/* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
		error = madvise_vma(vma, &prev, start, tmp, behavior);
L
Linus Torvalds 已提交
395 396
		if (error)
			goto out;
P
Prasanna Meda 已提交
397
		start = tmp;
398
		if (prev && start < prev->vm_end)
P
Prasanna Meda 已提交
399 400 401 402
			start = prev->vm_end;
		error = unmapped_error;
		if (start >= end)
			goto out;
403 404 405 406
		if (prev)
			vma = prev->vm_next;
		else	/* madvise_remove dropped mmap_sem */
			vma = find_vma(current->mm, start);
L
Linus Torvalds 已提交
407 408
	}
out:
409
	if (write)
410 411 412 413
		up_write(&current->mm->mmap_sem);
	else
		up_read(&current->mm->mmap_sem);

L
Linus Torvalds 已提交
414 415
	return error;
}