filemap.c 71.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 *	linux/mm/filemap.c
 *
 * Copyright (C) 1994-1999  Linus Torvalds
 */

/*
 * This file handles the generic file mmap semantics used by
 * most "normal" filesystems (but you don't /have/ to use this:
 * the NFS filesystem used to do this differently, for example)
 */
12
#include <linux/export.h>
L
Linus Torvalds 已提交
13 14
#include <linux/compiler.h>
#include <linux/fs.h>
15
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
16
#include <linux/aio.h>
17
#include <linux/capability.h>
L
Linus Torvalds 已提交
18
#include <linux/kernel_stat.h>
19
#include <linux/gfp.h>
L
Linus Torvalds 已提交
20 21 22 23 24 25 26 27
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/hash.h>
#include <linux/writeback.h>
28
#include <linux/backing-dev.h>
L
Linus Torvalds 已提交
29 30 31
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/security.h>
32
#include <linux/cpuset.h>
N
Nick Piggin 已提交
33
#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
34
#include <linux/memcontrol.h>
35
#include <linux/cleancache.h>
36
#include <linux/rmap.h>
37 38
#include "internal.h"

R
Robert Jarzmik 已提交
39 40 41
#define CREATE_TRACE_POINTS
#include <trace/events/filemap.h>

L
Linus Torvalds 已提交
42 43 44
/*
 * FIXME: remove all knowledge of the buffer layer from the core VM
 */
45
#include <linux/buffer_head.h> /* for try_to_free_buffers */
L
Linus Torvalds 已提交
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63

#include <asm/mman.h>

/*
 * Shared mappings implemented 30.11.1994. It's not fully working yet,
 * though.
 *
 * Shared mappings now work. 15.8.1995  Bruno.
 *
 * finished 'unifying' the page and buffer cache and SMP-threaded the
 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
 *
 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
 */

/*
 * Lock ordering:
 *
64
 *  ->i_mmap_mutex		(truncate_pagecache)
L
Linus Torvalds 已提交
65
 *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
66 67
 *      ->swap_lock		(exclusive_swap_page, others)
 *        ->mapping->tree_lock
L
Linus Torvalds 已提交
68
 *
69
 *  ->i_mutex
70
 *    ->i_mmap_mutex		(truncate->unmap_mapping_range)
L
Linus Torvalds 已提交
71 72
 *
 *  ->mmap_sem
73
 *    ->i_mmap_mutex
74
 *      ->page_table_lock or pte_lock	(various, mainly in memory.c)
L
Linus Torvalds 已提交
75 76 77 78 79
 *        ->mapping->tree_lock	(arch-dependent flush_dcache_mmap_lock)
 *
 *  ->mmap_sem
 *    ->lock_page		(access_process_vm)
 *
A
Al Viro 已提交
80
 *  ->i_mutex			(generic_perform_write)
81
 *    ->mmap_sem		(fault_in_pages_readable->do_page_fault)
L
Linus Torvalds 已提交
82
 *
83
 *  bdi->wb.list_lock
84
 *    sb_lock			(fs/fs-writeback.c)
L
Linus Torvalds 已提交
85 86
 *    ->mapping->tree_lock	(__sync_single_inode)
 *
87
 *  ->i_mmap_mutex
L
Linus Torvalds 已提交
88 89 90
 *    ->anon_vma.lock		(vma_adjust)
 *
 *  ->anon_vma.lock
91
 *    ->page_table_lock or pte_lock	(anon_vma_prepare and various)
L
Linus Torvalds 已提交
92
 *
93
 *  ->page_table_lock or pte_lock
94
 *    ->swap_lock		(try_to_unmap_one)
L
Linus Torvalds 已提交
95 96 97
 *    ->private_lock		(try_to_unmap_one)
 *    ->tree_lock		(try_to_unmap_one)
 *    ->zone.lru_lock		(follow_page->mark_page_accessed)
98
 *    ->zone.lru_lock		(check_pte_range->isolate_lru_page)
L
Linus Torvalds 已提交
99 100
 *    ->private_lock		(page_remove_rmap->set_page_dirty)
 *    ->tree_lock		(page_remove_rmap->set_page_dirty)
101
 *    bdi.wb->list_lock		(page_remove_rmap->set_page_dirty)
102
 *    ->inode->i_lock		(page_remove_rmap->set_page_dirty)
103
 *    bdi.wb->list_lock		(zap_pte_range->set_page_dirty)
104
 *    ->inode->i_lock		(zap_pte_range->set_page_dirty)
L
Linus Torvalds 已提交
105 106
 *    ->private_lock		(zap_pte_range->__set_page_dirty_buffers)
 *
107 108
 * ->i_mmap_mutex
 *   ->tasklist_lock            (memory_failure, collect_procs_ao)
L
Linus Torvalds 已提交
109 110
 */

111 112 113
static void page_cache_tree_delete(struct address_space *mapping,
				   struct page *page, void *shadow)
{
114 115 116 117 118
	struct radix_tree_node *node;
	unsigned long index;
	unsigned int offset;
	unsigned int tag;
	void **slot;
119

120 121 122 123 124
	VM_BUG_ON(!PageLocked(page));

	__radix_tree_lookup(&mapping->page_tree, page->index, &node, &slot);

	if (shadow) {
125 126 127 128 129 130 131 132
		mapping->nrshadows++;
		/*
		 * Make sure the nrshadows update is committed before
		 * the nrpages update so that final truncate racing
		 * with reclaim does not see both counters 0 at the
		 * same time and miss a shadow entry.
		 */
		smp_wmb();
133
	}
134
	mapping->nrpages--;
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171

	if (!node) {
		/* Clear direct pointer tags in root node */
		mapping->page_tree.gfp_mask &= __GFP_BITS_MASK;
		radix_tree_replace_slot(slot, shadow);
		return;
	}

	/* Clear tree tags for the removed page */
	index = page->index;
	offset = index & RADIX_TREE_MAP_MASK;
	for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {
		if (test_bit(offset, node->tags[tag]))
			radix_tree_tag_clear(&mapping->page_tree, index, tag);
	}

	/* Delete page, swap shadow entry */
	radix_tree_replace_slot(slot, shadow);
	workingset_node_pages_dec(node);
	if (shadow)
		workingset_node_shadows_inc(node);
	else
		if (__radix_tree_delete_node(&mapping->page_tree, node))
			return;

	/*
	 * Track node that only contains shadow entries.
	 *
	 * Avoid acquiring the list_lru lock if already tracked.  The
	 * list_empty() test is safe as node->private_list is
	 * protected by mapping->tree_lock.
	 */
	if (!workingset_node_pages(node) &&
	    list_empty(&node->private_list)) {
		node->private_data = mapping;
		list_lru_add(&workingset_shadow_nodes, &node->private_list);
	}
172 173
}

L
Linus Torvalds 已提交
174
/*
175
 * Delete a page from the page cache and free it. Caller has to make
L
Linus Torvalds 已提交
176
 * sure the page is locked and that nobody else uses it - or that usage
N
Nick Piggin 已提交
177
 * is safe.  The caller must hold the mapping's tree_lock.
L
Linus Torvalds 已提交
178
 */
179
void __delete_from_page_cache(struct page *page, void *shadow)
L
Linus Torvalds 已提交
180 181 182
{
	struct address_space *mapping = page->mapping;

R
Robert Jarzmik 已提交
183
	trace_mm_filemap_delete_from_page_cache(page);
184 185 186 187 188 189 190 191
	/*
	 * if we're uptodate, flush out into the cleancache, otherwise
	 * invalidate any existing cleancache entries.  We can't leave
	 * stale data around in the cleancache once our page is gone
	 */
	if (PageUptodate(page) && PageMappedToDisk(page))
		cleancache_put_page(page);
	else
192
		cleancache_invalidate_page(mapping, page);
193

194 195
	page_cache_tree_delete(mapping, page, shadow);

L
Linus Torvalds 已提交
196
	page->mapping = NULL;
197
	/* Leave page->index set: truncation lookup relies upon it */
198

199
	__dec_zone_page_state(page, NR_FILE_PAGES);
200 201
	if (PageSwapBacked(page))
		__dec_zone_page_state(page, NR_SHMEM);
202
	BUG_ON(page_mapped(page));
203 204 205 206 207 208 209 210 211 212 213 214

	/*
	 * Some filesystems seem to re-dirty the page even after
	 * the VM has canceled the dirty bit (eg ext3 journaling).
	 *
	 * Fix it up by doing a final dirty accounting check after
	 * having removed the page entirely.
	 */
	if (PageDirty(page) && mapping_cap_account_dirty(mapping)) {
		dec_zone_page_state(page, NR_FILE_DIRTY);
		dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
	}
L
Linus Torvalds 已提交
215 216
}

217 218 219 220 221 222 223 224 225
/**
 * delete_from_page_cache - delete page from page cache
 * @page: the page which the kernel is trying to remove from page cache
 *
 * This must be called only on pages that have been verified to be in the page
 * cache and locked.  It will never put the page into the free list, the caller
 * has a reference on the page.
 */
void delete_from_page_cache(struct page *page)
L
Linus Torvalds 已提交
226 227
{
	struct address_space *mapping = page->mapping;
228
	void (*freepage)(struct page *);
L
Linus Torvalds 已提交
229

M
Matt Mackall 已提交
230
	BUG_ON(!PageLocked(page));
L
Linus Torvalds 已提交
231

232
	freepage = mapping->a_ops->freepage;
N
Nick Piggin 已提交
233
	spin_lock_irq(&mapping->tree_lock);
234
	__delete_from_page_cache(page, NULL);
N
Nick Piggin 已提交
235
	spin_unlock_irq(&mapping->tree_lock);
236
	mem_cgroup_uncharge_cache_page(page);
237 238 239

	if (freepage)
		freepage(page);
240 241 242 243
	page_cache_release(page);
}
EXPORT_SYMBOL(delete_from_page_cache);

J
Jens Axboe 已提交
244
static int sleep_on_page(void *word)
L
Linus Torvalds 已提交
245 246 247 248 249
{
	io_schedule();
	return 0;
}

J
Jens Axboe 已提交
250
static int sleep_on_page_killable(void *word)
M
Matthew Wilcox 已提交
251
{
J
Jens Axboe 已提交
252
	sleep_on_page(word);
M
Matthew Wilcox 已提交
253 254 255
	return fatal_signal_pending(current) ? -EINTR : 0;
}

256 257 258 259 260 261 262 263 264 265 266
static int filemap_check_errors(struct address_space *mapping)
{
	int ret = 0;
	/* Check for outstanding write errors */
	if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
		ret = -ENOSPC;
	if (test_and_clear_bit(AS_EIO, &mapping->flags))
		ret = -EIO;
	return ret;
}

L
Linus Torvalds 已提交
267
/**
268
 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
269 270
 * @mapping:	address space structure to write
 * @start:	offset in bytes where the range starts
271
 * @end:	offset in bytes where the range ends (inclusive)
272
 * @sync_mode:	enable synchronous operation
L
Linus Torvalds 已提交
273
 *
274 275 276
 * Start writeback against all of a mapping's dirty pages that lie
 * within the byte offsets <start, end> inclusive.
 *
L
Linus Torvalds 已提交
277
 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
278
 * opposed to a regular memory cleansing writeback.  The difference between
L
Linus Torvalds 已提交
279 280 281
 * these two operations is that if a dirty page/buffer is encountered, it must
 * be waited upon, and not just skipped over.
 */
282 283
int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
				loff_t end, int sync_mode)
L
Linus Torvalds 已提交
284 285 286 287
{
	int ret;
	struct writeback_control wbc = {
		.sync_mode = sync_mode,
288
		.nr_to_write = LONG_MAX,
289 290
		.range_start = start,
		.range_end = end,
L
Linus Torvalds 已提交
291 292 293 294 295 296 297 298 299 300 301 302
	};

	if (!mapping_cap_writeback_dirty(mapping))
		return 0;

	ret = do_writepages(mapping, &wbc);
	return ret;
}

static inline int __filemap_fdatawrite(struct address_space *mapping,
	int sync_mode)
{
303
	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
L
Linus Torvalds 已提交
304 305 306 307 308 309 310 311
}

int filemap_fdatawrite(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
}
EXPORT_SYMBOL(filemap_fdatawrite);

312
int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
313
				loff_t end)
L
Linus Torvalds 已提交
314 315 316
{
	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
}
317
EXPORT_SYMBOL(filemap_fdatawrite_range);
L
Linus Torvalds 已提交
318

319 320 321 322
/**
 * filemap_flush - mostly a non-blocking flush
 * @mapping:	target address_space
 *
L
Linus Torvalds 已提交
323 324 325 326 327 328 329 330 331
 * This is a mostly non-blocking flush.  Not suitable for data-integrity
 * purposes - I/O may not be started against all dirty pages.
 */
int filemap_flush(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
}
EXPORT_SYMBOL(filemap_flush);

332
/**
333 334 335 336
 * filemap_fdatawait_range - wait for writeback to complete
 * @mapping:		address space structure to wait for
 * @start_byte:		offset in bytes where the range starts
 * @end_byte:		offset in bytes where the range ends (inclusive)
337
 *
338 339
 * Walk the list of under-writeback pages of the given address space
 * in the given range and wait for all of them.
L
Linus Torvalds 已提交
340
 */
341 342
int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
			    loff_t end_byte)
L
Linus Torvalds 已提交
343
{
344 345
	pgoff_t index = start_byte >> PAGE_CACHE_SHIFT;
	pgoff_t end = end_byte >> PAGE_CACHE_SHIFT;
L
Linus Torvalds 已提交
346 347
	struct pagevec pvec;
	int nr_pages;
348
	int ret2, ret = 0;
L
Linus Torvalds 已提交
349

350
	if (end_byte < start_byte)
351
		goto out;
L
Linus Torvalds 已提交
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367

	pagevec_init(&pvec, 0);
	while ((index <= end) &&
			(nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
			PAGECACHE_TAG_WRITEBACK,
			min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
		unsigned i;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/* until radix tree lookup accepts end_index */
			if (page->index > end)
				continue;

			wait_on_page_writeback(page);
368
			if (TestClearPageError(page))
L
Linus Torvalds 已提交
369 370 371 372 373
				ret = -EIO;
		}
		pagevec_release(&pvec);
		cond_resched();
	}
374 375 376 377
out:
	ret2 = filemap_check_errors(mapping);
	if (!ret)
		ret = ret2;
L
Linus Torvalds 已提交
378 379 380

	return ret;
}
381 382
EXPORT_SYMBOL(filemap_fdatawait_range);

L
Linus Torvalds 已提交
383
/**
384
 * filemap_fdatawait - wait for all under-writeback pages to complete
L
Linus Torvalds 已提交
385
 * @mapping: address space structure to wait for
386 387 388
 *
 * Walk the list of under-writeback pages of the given address space
 * and wait for all of them.
L
Linus Torvalds 已提交
389 390 391 392 393 394 395 396
 */
int filemap_fdatawait(struct address_space *mapping)
{
	loff_t i_size = i_size_read(mapping->host);

	if (i_size == 0)
		return 0;

397
	return filemap_fdatawait_range(mapping, 0, i_size - 1);
L
Linus Torvalds 已提交
398 399 400 401 402
}
EXPORT_SYMBOL(filemap_fdatawait);

int filemap_write_and_wait(struct address_space *mapping)
{
403
	int err = 0;
L
Linus Torvalds 已提交
404 405

	if (mapping->nrpages) {
406 407 408 409 410 411 412 413 414 415 416 417
		err = filemap_fdatawrite(mapping);
		/*
		 * Even if the above returned error, the pages may be
		 * written partially (e.g. -ENOSPC), so we wait for it.
		 * But the -EIO is special case, it may indicate the worst
		 * thing (e.g. bug) happened, so we avoid waiting for it.
		 */
		if (err != -EIO) {
			int err2 = filemap_fdatawait(mapping);
			if (!err)
				err = err2;
		}
418 419
	} else {
		err = filemap_check_errors(mapping);
L
Linus Torvalds 已提交
420
	}
421
	return err;
L
Linus Torvalds 已提交
422
}
423
EXPORT_SYMBOL(filemap_write_and_wait);
L
Linus Torvalds 已提交
424

425 426 427 428 429 430
/**
 * filemap_write_and_wait_range - write out & wait on a file range
 * @mapping:	the address_space for the pages
 * @lstart:	offset in bytes where the range starts
 * @lend:	offset in bytes where the range ends (inclusive)
 *
431 432 433 434 435
 * Write out and wait upon file offsets lstart->lend, inclusive.
 *
 * Note that `lend' is inclusive (describes the last byte to be written) so
 * that this function can be used to write to the very end-of-file (end = -1).
 */
L
Linus Torvalds 已提交
436 437 438
int filemap_write_and_wait_range(struct address_space *mapping,
				 loff_t lstart, loff_t lend)
{
439
	int err = 0;
L
Linus Torvalds 已提交
440 441

	if (mapping->nrpages) {
442 443 444 445
		err = __filemap_fdatawrite_range(mapping, lstart, lend,
						 WB_SYNC_ALL);
		/* See comment of filemap_write_and_wait() */
		if (err != -EIO) {
446 447
			int err2 = filemap_fdatawait_range(mapping,
						lstart, lend);
448 449 450
			if (!err)
				err = err2;
		}
451 452
	} else {
		err = filemap_check_errors(mapping);
L
Linus Torvalds 已提交
453
	}
454
	return err;
L
Linus Torvalds 已提交
455
}
456
EXPORT_SYMBOL(filemap_write_and_wait_range);
L
Linus Torvalds 已提交
457

458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
/**
 * replace_page_cache_page - replace a pagecache page with a new one
 * @old:	page to be replaced
 * @new:	page to replace with
 * @gfp_mask:	allocation mode
 *
 * This function replaces a page in the pagecache with a new one.  On
 * success it acquires the pagecache reference for the new page and
 * drops it for the old page.  Both the old and new pages must be
 * locked.  This function does not add the new page to the LRU, the
 * caller must do that.
 *
 * The remove + add is atomic.  The only way this function can fail is
 * memory allocation failure.
 */
int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
{
	int error;

477 478 479
	VM_BUG_ON_PAGE(!PageLocked(old), old);
	VM_BUG_ON_PAGE(!PageLocked(new), new);
	VM_BUG_ON_PAGE(new->mapping, new);
480 481 482 483 484 485 486 487 488 489 490 491 492 493

	error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
	if (!error) {
		struct address_space *mapping = old->mapping;
		void (*freepage)(struct page *);

		pgoff_t offset = old->index;
		freepage = mapping->a_ops->freepage;

		page_cache_get(new);
		new->mapping = mapping;
		new->index = offset;

		spin_lock_irq(&mapping->tree_lock);
494
		__delete_from_page_cache(old, NULL);
495 496 497 498 499 500 501
		error = radix_tree_insert(&mapping->page_tree, offset, new);
		BUG_ON(error);
		mapping->nrpages++;
		__inc_zone_page_state(new, NR_FILE_PAGES);
		if (PageSwapBacked(new))
			__inc_zone_page_state(new, NR_SHMEM);
		spin_unlock_irq(&mapping->tree_lock);
502 503
		/* mem_cgroup codes must not be called under tree_lock */
		mem_cgroup_replace_page_cache(old, new);
504 505 506 507 508 509 510 511 512 513
		radix_tree_preload_end();
		if (freepage)
			freepage(old);
		page_cache_release(old);
	}

	return error;
}
EXPORT_SYMBOL_GPL(replace_page_cache_page);

514
static int page_cache_tree_insert(struct address_space *mapping,
515
				  struct page *page, void **shadowp)
516
{
517
	struct radix_tree_node *node;
518 519 520
	void **slot;
	int error;

521 522 523 524 525
	error = __radix_tree_create(&mapping->page_tree, page->index,
				    &node, &slot);
	if (error)
		return error;
	if (*slot) {
526 527 528 529 530
		void *p;

		p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
		if (!radix_tree_exceptional_entry(p))
			return -EEXIST;
531 532
		if (shadowp)
			*shadowp = p;
533 534 535
		mapping->nrshadows--;
		if (node)
			workingset_node_shadows_dec(node);
536
	}
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553
	radix_tree_replace_slot(slot, page);
	mapping->nrpages++;
	if (node) {
		workingset_node_pages_inc(node);
		/*
		 * Don't track node that contains actual pages.
		 *
		 * Avoid acquiring the list_lru lock if already
		 * untracked.  The list_empty() test is safe as
		 * node->private_list is protected by
		 * mapping->tree_lock.
		 */
		if (!list_empty(&node->private_list))
			list_lru_del(&workingset_shadow_nodes,
				     &node->private_list);
	}
	return 0;
554 555
}

556 557 558 559
static int __add_to_page_cache_locked(struct page *page,
				      struct address_space *mapping,
				      pgoff_t offset, gfp_t gfp_mask,
				      void **shadowp)
L
Linus Torvalds 已提交
560
{
N
Nick Piggin 已提交
561 562
	int error;

563 564
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(PageSwapBacked(page), page);
N
Nick Piggin 已提交
565

566
	error = mem_cgroup_charge_file(page, current->mm,
K
KAMEZAWA Hiroyuki 已提交
567
					gfp_mask & GFP_RECLAIM_MASK);
B
Balbir Singh 已提交
568
	if (error)
569
		return error;
L
Linus Torvalds 已提交
570

571
	error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM);
572
	if (error) {
573
		mem_cgroup_uncharge_cache_page(page);
574 575 576 577 578 579 580 581
		return error;
	}

	page_cache_get(page);
	page->mapping = mapping;
	page->index = offset;

	spin_lock_irq(&mapping->tree_lock);
582
	error = page_cache_tree_insert(mapping, page, shadowp);
583 584 585 586 587 588 589 590 591 592 593 594 595
	radix_tree_preload_end();
	if (unlikely(error))
		goto err_insert;
	__inc_zone_page_state(page, NR_FILE_PAGES);
	spin_unlock_irq(&mapping->tree_lock);
	trace_mm_filemap_add_to_page_cache(page);
	return 0;
err_insert:
	page->mapping = NULL;
	/* Leave page->index set: truncation relies upon it */
	spin_unlock_irq(&mapping->tree_lock);
	mem_cgroup_uncharge_cache_page(page);
	page_cache_release(page);
L
Linus Torvalds 已提交
596 597
	return error;
}
598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614

/**
 * add_to_page_cache_locked - add a locked page to the pagecache
 * @page:	page to add
 * @mapping:	the page's address_space
 * @offset:	page index
 * @gfp_mask:	page allocation mode
 *
 * This function is used to add a page to the pagecache. It must be locked.
 * This function does not add the page to the LRU.  The caller must do that.
 */
int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
		pgoff_t offset, gfp_t gfp_mask)
{
	return __add_to_page_cache_locked(page, mapping, offset,
					  gfp_mask, NULL);
}
N
Nick Piggin 已提交
615
EXPORT_SYMBOL(add_to_page_cache_locked);
L
Linus Torvalds 已提交
616 617

int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
A
Al Viro 已提交
618
				pgoff_t offset, gfp_t gfp_mask)
L
Linus Torvalds 已提交
619
{
620
	void *shadow = NULL;
621 622
	int ret;

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
	__set_page_locked(page);
	ret = __add_to_page_cache_locked(page, mapping, offset,
					 gfp_mask, &shadow);
	if (unlikely(ret))
		__clear_page_locked(page);
	else {
		/*
		 * The page might have been evicted from cache only
		 * recently, in which case it should be activated like
		 * any other repeatedly accessed page.
		 */
		if (shadow && workingset_refault(shadow)) {
			SetPageActive(page);
			workingset_activation(page);
		} else
			ClearPageActive(page);
		lru_cache_add(page);
	}
L
Linus Torvalds 已提交
641 642
	return ret;
}
643
EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
L
Linus Torvalds 已提交
644

645
#ifdef CONFIG_NUMA
646
struct page *__page_cache_alloc(gfp_t gfp)
647
{
648 649 650
	int n;
	struct page *page;

651
	if (cpuset_do_page_mem_spread()) {
652 653
		unsigned int cpuset_mems_cookie;
		do {
654
			cpuset_mems_cookie = read_mems_allowed_begin();
655 656
			n = cpuset_mem_spread_node();
			page = alloc_pages_exact_node(n, gfp, 0);
657
		} while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
658

659
		return page;
660
	}
661
	return alloc_pages(gfp, 0);
662
}
663
EXPORT_SYMBOL(__page_cache_alloc);
664 665
#endif

L
Linus Torvalds 已提交
666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
/*
 * In order to wait for pages to become available there must be
 * waitqueues associated with pages. By using a hash table of
 * waitqueues where the bucket discipline is to maintain all
 * waiters on the same queue and wake all when any of the pages
 * become available, and for the woken contexts to check to be
 * sure the appropriate page became available, this saves space
 * at a cost of "thundering herd" phenomena during rare hash
 * collisions.
 */
static wait_queue_head_t *page_waitqueue(struct page *page)
{
	const struct zone *zone = page_zone(page);

	return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
}

static inline void wake_up_page(struct page *page, int bit)
{
	__wake_up_bit(page_waitqueue(page), &page->flags, bit);
}

H
Harvey Harrison 已提交
688
void wait_on_page_bit(struct page *page, int bit_nr)
L
Linus Torvalds 已提交
689 690 691 692
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	if (test_bit(bit_nr, &page->flags))
J
Jens Axboe 已提交
693
		__wait_on_bit(page_waitqueue(page), &wait, sleep_on_page,
L
Linus Torvalds 已提交
694 695 696 697
							TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(wait_on_page_bit);

698 699 700 701 702 703 704 705 706 707 708
int wait_on_page_bit_killable(struct page *page, int bit_nr)
{
	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

	if (!test_bit(bit_nr, &page->flags))
		return 0;

	return __wait_on_bit(page_waitqueue(page), &wait,
			     sleep_on_page_killable, TASK_KILLABLE);
}

709 710
/**
 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
R
Randy Dunlap 已提交
711 712
 * @page: Page defining the wait queue of interest
 * @waiter: Waiter to add to the queue
713 714 715 716 717 718 719 720 721 722 723 724 725 726
 *
 * Add an arbitrary @waiter to the wait queue for the nominated @page.
 */
void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
{
	wait_queue_head_t *q = page_waitqueue(page);
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	__add_wait_queue(q, waiter);
	spin_unlock_irqrestore(&q->lock, flags);
}
EXPORT_SYMBOL_GPL(add_page_wait_queue);

L
Linus Torvalds 已提交
727
/**
728
 * unlock_page - unlock a locked page
L
Linus Torvalds 已提交
729 730 731 732 733 734 735
 * @page: the page
 *
 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
 * mechananism between PageLocked pages and PageWriteback pages is shared.
 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
 *
N
Nick Piggin 已提交
736 737
 * The mb is necessary to enforce ordering between the clear_bit and the read
 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
L
Linus Torvalds 已提交
738
 */
H
Harvey Harrison 已提交
739
void unlock_page(struct page *page)
L
Linus Torvalds 已提交
740
{
741
	VM_BUG_ON_PAGE(!PageLocked(page), page);
N
Nick Piggin 已提交
742 743
	clear_bit_unlock(PG_locked, &page->flags);
	smp_mb__after_clear_bit();
L
Linus Torvalds 已提交
744 745 746 747
	wake_up_page(page, PG_locked);
}
EXPORT_SYMBOL(unlock_page);

748 749 750
/**
 * end_page_writeback - end writeback against a page
 * @page: the page
L
Linus Torvalds 已提交
751 752 753
 */
void end_page_writeback(struct page *page)
{
754 755 756 757 758 759
	if (TestClearPageReclaim(page))
		rotate_reclaimable_page(page);

	if (!test_clear_page_writeback(page))
		BUG();

L
Linus Torvalds 已提交
760 761 762 763 764
	smp_mb__after_clear_bit();
	wake_up_page(page, PG_writeback);
}
EXPORT_SYMBOL(end_page_writeback);

765 766 767
/**
 * __lock_page - get a lock on the page, assuming we need to sleep to get it
 * @page: the page to lock
L
Linus Torvalds 已提交
768
 */
H
Harvey Harrison 已提交
769
void __lock_page(struct page *page)
L
Linus Torvalds 已提交
770 771 772
{
	DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);

J
Jens Axboe 已提交
773
	__wait_on_bit_lock(page_waitqueue(page), &wait, sleep_on_page,
L
Linus Torvalds 已提交
774 775 776 777
							TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__lock_page);

H
Harvey Harrison 已提交
778
int __lock_page_killable(struct page *page)
M
Matthew Wilcox 已提交
779 780 781 782
{
	DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);

	return __wait_on_bit_lock(page_waitqueue(page), &wait,
J
Jens Axboe 已提交
783
					sleep_on_page_killable, TASK_KILLABLE);
M
Matthew Wilcox 已提交
784
}
785
EXPORT_SYMBOL_GPL(__lock_page_killable);
M
Matthew Wilcox 已提交
786

787 788 789
int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
			 unsigned int flags)
{
790 791 792 793 794 795 796 797 798 799 800 801
	if (flags & FAULT_FLAG_ALLOW_RETRY) {
		/*
		 * CAUTION! In this case, mmap_sem is not released
		 * even though return 0.
		 */
		if (flags & FAULT_FLAG_RETRY_NOWAIT)
			return 0;

		up_read(&mm->mmap_sem);
		if (flags & FAULT_FLAG_KILLABLE)
			wait_on_page_locked_killable(page);
		else
802
			wait_on_page_locked(page);
803
		return 0;
804 805 806 807 808 809 810 811 812 813 814 815
	} else {
		if (flags & FAULT_FLAG_KILLABLE) {
			int ret;

			ret = __lock_page_killable(page);
			if (ret) {
				up_read(&mm->mmap_sem);
				return 0;
			}
		} else
			__lock_page(page);
		return 1;
816 817 818
	}
}

819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845
/**
 * page_cache_next_hole - find the next hole (not-present entry)
 * @mapping: mapping
 * @index: index
 * @max_scan: maximum range to search
 *
 * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
 * lowest indexed hole.
 *
 * Returns: the index of the hole if found, otherwise returns an index
 * outside of the set specified (in which case 'return - index >=
 * max_scan' will be true). In rare cases of index wrap-around, 0 will
 * be returned.
 *
 * page_cache_next_hole may be called under rcu_read_lock. However,
 * like radix_tree_gang_lookup, this will not atomically search a
 * snapshot of the tree at a single point in time. For example, if a
 * hole is created at index 5, then subsequently a hole is created at
 * index 10, page_cache_next_hole covering both indexes may return 10
 * if called under rcu_read_lock.
 */
pgoff_t page_cache_next_hole(struct address_space *mapping,
			     pgoff_t index, unsigned long max_scan)
{
	unsigned long i;

	for (i = 0; i < max_scan; i++) {
846 847 848 849
		struct page *page;

		page = radix_tree_lookup(&mapping->page_tree, index);
		if (!page || radix_tree_exceptional_entry(page))
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
			break;
		index++;
		if (index == 0)
			break;
	}

	return index;
}
EXPORT_SYMBOL(page_cache_next_hole);

/**
 * page_cache_prev_hole - find the prev hole (not-present entry)
 * @mapping: mapping
 * @index: index
 * @max_scan: maximum range to search
 *
 * Search backwards in the range [max(index-max_scan+1, 0), index] for
 * the first hole.
 *
 * Returns: the index of the hole if found, otherwise returns an index
 * outside of the set specified (in which case 'index - return >=
 * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
 * will be returned.
 *
 * page_cache_prev_hole may be called under rcu_read_lock. However,
 * like radix_tree_gang_lookup, this will not atomically search a
 * snapshot of the tree at a single point in time. For example, if a
 * hole is created at index 10, then subsequently a hole is created at
 * index 5, page_cache_prev_hole covering both indexes may return 5 if
 * called under rcu_read_lock.
 */
pgoff_t page_cache_prev_hole(struct address_space *mapping,
			     pgoff_t index, unsigned long max_scan)
{
	unsigned long i;

	for (i = 0; i < max_scan; i++) {
887 888 889 890
		struct page *page;

		page = radix_tree_lookup(&mapping->page_tree, index);
		if (!page || radix_tree_exceptional_entry(page))
891 892 893 894 895 896 897 898 899 900
			break;
		index--;
		if (index == ULONG_MAX)
			break;
	}

	return index;
}
EXPORT_SYMBOL(page_cache_prev_hole);

901
/**
902
 * find_get_entry - find and get a page cache entry
903
 * @mapping: the address_space to search
904 905 906 907
 * @offset: the page cache index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned with an increased refcount.
908
 *
909 910
 * If the slot holds a shadow entry of a previously evicted page, or a
 * swap entry from shmem/tmpfs, it is returned.
911 912
 *
 * Otherwise, %NULL is returned.
L
Linus Torvalds 已提交
913
 */
914
struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
L
Linus Torvalds 已提交
915
{
N
Nick Piggin 已提交
916
	void **pagep;
L
Linus Torvalds 已提交
917 918
	struct page *page;

N
Nick Piggin 已提交
919 920 921 922 923 924
	rcu_read_lock();
repeat:
	page = NULL;
	pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
	if (pagep) {
		page = radix_tree_deref_slot(pagep);
N
Nick Piggin 已提交
925 926
		if (unlikely(!page))
			goto out;
927
		if (radix_tree_exception(page)) {
928 929 930
			if (radix_tree_deref_retry(page))
				goto repeat;
			/*
931 932 933
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Return
			 * it without attempting to raise page count.
934 935
			 */
			goto out;
936
		}
N
Nick Piggin 已提交
937 938 939 940 941 942 943 944 945 946 947 948 949
		if (!page_cache_get_speculative(page))
			goto repeat;

		/*
		 * Has the page moved?
		 * This is part of the lockless pagecache protocol. See
		 * include/linux/pagemap.h for details.
		 */
		if (unlikely(page != *pagep)) {
			page_cache_release(page);
			goto repeat;
		}
	}
N
Nick Piggin 已提交
950
out:
N
Nick Piggin 已提交
951 952
	rcu_read_unlock();

L
Linus Torvalds 已提交
953 954
	return page;
}
955
EXPORT_SYMBOL(find_get_entry);
L
Linus Torvalds 已提交
956 957

/**
958
 * find_get_page - find and get a page reference
959 960
 * @mapping: the address_space to search
 * @offset: the page index
L
Linus Torvalds 已提交
961
 *
962 963
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned with an increased refcount.
L
Linus Torvalds 已提交
964
 *
965
 * Otherwise, %NULL is returned.
L
Linus Torvalds 已提交
966
 */
967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
{
	struct page *page = find_get_entry(mapping, offset);

	if (radix_tree_exceptional_entry(page))
		page = NULL;
	return page;
}
EXPORT_SYMBOL(find_get_page);

/**
 * find_lock_entry - locate, pin and lock a page cache entry
 * @mapping: the address_space to search
 * @offset: the page cache index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned locked and with an increased
 * refcount.
 *
986 987
 * If the slot holds a shadow entry of a previously evicted page, or a
 * swap entry from shmem/tmpfs, it is returned.
988 989 990 991 992 993
 *
 * Otherwise, %NULL is returned.
 *
 * find_lock_entry() may sleep.
 */
struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
L
Linus Torvalds 已提交
994 995 996 997
{
	struct page *page;

repeat:
998
	page = find_get_entry(mapping, offset);
999
	if (page && !radix_tree_exception(page)) {
N
Nick Piggin 已提交
1000 1001 1002 1003 1004 1005
		lock_page(page);
		/* Has the page been truncated? */
		if (unlikely(page->mapping != mapping)) {
			unlock_page(page);
			page_cache_release(page);
			goto repeat;
L
Linus Torvalds 已提交
1006
		}
1007
		VM_BUG_ON_PAGE(page->index != offset, page);
L
Linus Torvalds 已提交
1008 1009 1010
	}
	return page;
}
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
EXPORT_SYMBOL(find_lock_entry);

/**
 * find_lock_page - locate, pin and lock a pagecache page
 * @mapping: the address_space to search
 * @offset: the page index
 *
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned locked and with an increased
 * refcount.
 *
 * Otherwise, %NULL is returned.
 *
 * find_lock_page() may sleep.
 */
struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
{
	struct page *page = find_lock_entry(mapping, offset);

	if (radix_tree_exceptional_entry(page))
		page = NULL;
	return page;
}
L
Linus Torvalds 已提交
1034 1035 1036 1037
EXPORT_SYMBOL(find_lock_page);

/**
 * find_or_create_page - locate or add a pagecache page
1038 1039 1040
 * @mapping: the page's address_space
 * @index: the page's index into the mapping
 * @gfp_mask: page allocation mode
L
Linus Torvalds 已提交
1041
 *
1042 1043 1044 1045 1046 1047 1048
 * Looks up the page cache slot at @mapping & @offset.  If there is a
 * page cache page, it is returned locked and with an increased
 * refcount.
 *
 * If the page is not present, a new page is allocated using @gfp_mask
 * and added to the page cache and the VM's LRU list.  The page is
 * returned locked and with an increased refcount.
L
Linus Torvalds 已提交
1049
 *
1050
 * On memory exhaustion, %NULL is returned.
L
Linus Torvalds 已提交
1051
 *
1052 1053
 * find_or_create_page() may sleep, even if @gfp_flags specifies an
 * atomic allocation!
L
Linus Torvalds 已提交
1054 1055
 */
struct page *find_or_create_page(struct address_space *mapping,
1056
		pgoff_t index, gfp_t gfp_mask)
L
Linus Torvalds 已提交
1057
{
N
Nick Piggin 已提交
1058
	struct page *page;
L
Linus Torvalds 已提交
1059 1060 1061 1062
	int err;
repeat:
	page = find_lock_page(mapping, index);
	if (!page) {
N
Nick Piggin 已提交
1063 1064 1065
		page = __page_cache_alloc(gfp_mask);
		if (!page)
			return NULL;
N
Nick Piggin 已提交
1066 1067 1068 1069 1070 1071 1072 1073
		/*
		 * We want a regular kernel memory (not highmem or DMA etc)
		 * allocation for the radix tree nodes, but we need to honour
		 * the context-specific requirements the caller has asked for.
		 * GFP_RECLAIM_MASK collects those requirements.
		 */
		err = add_to_page_cache_lru(page, mapping, index,
			(gfp_mask & GFP_RECLAIM_MASK));
N
Nick Piggin 已提交
1074 1075 1076 1077 1078
		if (unlikely(err)) {
			page_cache_release(page);
			page = NULL;
			if (err == -EEXIST)
				goto repeat;
L
Linus Torvalds 已提交
1079 1080 1081 1082 1083 1084
		}
	}
	return page;
}
EXPORT_SYMBOL(find_or_create_page);

1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
/**
 * find_get_entries - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page cache index
 * @nr_entries:	The maximum number of entries
 * @entries:	Where the resulting entries are placed
 * @indices:	The cache indices corresponding to the entries in @entries
 *
 * find_get_entries() will search for and return a group of up to
 * @nr_entries entries in the mapping.  The entries are placed at
 * @entries.  find_get_entries() takes a reference against any actual
 * pages it returns.
 *
 * The search returns a group of mapping-contiguous page cache entries
 * with ascending indexes.  There may be holes in the indices due to
 * not-present pages.
 *
1102 1103
 * Any shadow entries of evicted pages, or swap entries from
 * shmem/tmpfs, are included in the returned array.
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
 *
 * find_get_entries() returns the number of pages and shadow entries
 * which were found.
 */
unsigned find_get_entries(struct address_space *mapping,
			  pgoff_t start, unsigned int nr_entries,
			  struct page **entries, pgoff_t *indices)
{
	void **slot;
	unsigned int ret = 0;
	struct radix_tree_iter iter;

	if (!nr_entries)
		return 0;

	rcu_read_lock();
restart:
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
		struct page *page;
repeat:
		page = radix_tree_deref_slot(slot);
		if (unlikely(!page))
			continue;
		if (radix_tree_exception(page)) {
			if (radix_tree_deref_retry(page))
				goto restart;
			/*
1131 1132 1133
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Return
			 * it without attempting to raise page count.
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
			 */
			goto export;
		}
		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
		if (unlikely(page != *slot)) {
			page_cache_release(page);
			goto repeat;
		}
export:
		indices[ret] = iter.index;
		entries[ret] = page;
		if (++ret == nr_entries)
			break;
	}
	rcu_read_unlock();
	return ret;
}

L
Linus Torvalds 已提交
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
/**
 * find_get_pages - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page index
 * @nr_pages:	The maximum number of pages
 * @pages:	Where the resulting pages are placed
 *
 * find_get_pages() will search for and return a group of up to
 * @nr_pages pages in the mapping.  The pages are placed at @pages.
 * find_get_pages() takes a reference against the returned pages.
 *
 * The search returns a group of mapping-contiguous pages with ascending
 * indexes.  There may be holes in the indices due to not-present pages.
 *
 * find_get_pages() returns the number of pages which were found.
 */
unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
			    unsigned int nr_pages, struct page **pages)
{
1174 1175 1176 1177 1178 1179
	struct radix_tree_iter iter;
	void **slot;
	unsigned ret = 0;

	if (unlikely(!nr_pages))
		return 0;
N
Nick Piggin 已提交
1180 1181 1182

	rcu_read_lock();
restart:
1183
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
N
Nick Piggin 已提交
1184 1185
		struct page *page;
repeat:
1186
		page = radix_tree_deref_slot(slot);
N
Nick Piggin 已提交
1187 1188
		if (unlikely(!page))
			continue;
1189

1190
		if (radix_tree_exception(page)) {
1191 1192 1193 1194 1195 1196
			if (radix_tree_deref_retry(page)) {
				/*
				 * Transient condition which can only trigger
				 * when entry at index 0 moves out of or back
				 * to root: none yet gotten, safe to restart.
				 */
1197
				WARN_ON(iter.index);
1198 1199
				goto restart;
			}
1200
			/*
1201 1202 1203
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Skip
			 * over it.
1204
			 */
1205
			continue;
N
Nick Piggin 已提交
1206
		}
N
Nick Piggin 已提交
1207 1208 1209 1210 1211

		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
1212
		if (unlikely(page != *slot)) {
N
Nick Piggin 已提交
1213 1214 1215
			page_cache_release(page);
			goto repeat;
		}
L
Linus Torvalds 已提交
1216

N
Nick Piggin 已提交
1217
		pages[ret] = page;
1218 1219
		if (++ret == nr_pages)
			break;
N
Nick Piggin 已提交
1220
	}
1221

N
Nick Piggin 已提交
1222
	rcu_read_unlock();
L
Linus Torvalds 已提交
1223 1224 1225
	return ret;
}

1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
/**
 * find_get_pages_contig - gang contiguous pagecache lookup
 * @mapping:	The address_space to search
 * @index:	The starting page index
 * @nr_pages:	The maximum number of pages
 * @pages:	Where the resulting pages are placed
 *
 * find_get_pages_contig() works exactly like find_get_pages(), except
 * that the returned number of pages are guaranteed to be contiguous.
 *
 * find_get_pages_contig() returns the number of pages which were found.
 */
unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
			       unsigned int nr_pages, struct page **pages)
{
1241 1242 1243 1244 1245 1246
	struct radix_tree_iter iter;
	void **slot;
	unsigned int ret = 0;

	if (unlikely(!nr_pages))
		return 0;
N
Nick Piggin 已提交
1247 1248 1249

	rcu_read_lock();
restart:
1250
	radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
N
Nick Piggin 已提交
1251 1252
		struct page *page;
repeat:
1253 1254
		page = radix_tree_deref_slot(slot);
		/* The hole, there no reason to continue */
N
Nick Piggin 已提交
1255
		if (unlikely(!page))
1256
			break;
1257

1258
		if (radix_tree_exception(page)) {
1259 1260 1261 1262 1263 1264 1265 1266
			if (radix_tree_deref_retry(page)) {
				/*
				 * Transient condition which can only trigger
				 * when entry at index 0 moves out of or back
				 * to root: none yet gotten, safe to restart.
				 */
				goto restart;
			}
1267
			/*
1268 1269 1270
			 * A shadow entry of a recently evicted page,
			 * or a swap entry from shmem/tmpfs.  Stop
			 * looking for contiguous pages.
1271
			 */
1272
			break;
1273
		}
1274

N
Nick Piggin 已提交
1275 1276 1277 1278
		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
1279
		if (unlikely(page != *slot)) {
N
Nick Piggin 已提交
1280 1281 1282 1283
			page_cache_release(page);
			goto repeat;
		}

N
Nick Piggin 已提交
1284 1285 1286 1287 1288
		/*
		 * must check mapping and index after taking the ref.
		 * otherwise we can get both false positives and false
		 * negatives, which is just confusing to the caller.
		 */
1289
		if (page->mapping == NULL || page->index != iter.index) {
N
Nick Piggin 已提交
1290 1291 1292 1293
			page_cache_release(page);
			break;
		}

N
Nick Piggin 已提交
1294
		pages[ret] = page;
1295 1296
		if (++ret == nr_pages)
			break;
1297
	}
N
Nick Piggin 已提交
1298 1299
	rcu_read_unlock();
	return ret;
1300
}
1301
EXPORT_SYMBOL(find_get_pages_contig);
1302

1303 1304 1305 1306 1307 1308 1309 1310
/**
 * find_get_pages_tag - find and return pages that match @tag
 * @mapping:	the address_space to search
 * @index:	the starting page index
 * @tag:	the tag index
 * @nr_pages:	the maximum number of pages
 * @pages:	where the resulting pages are placed
 *
L
Linus Torvalds 已提交
1311
 * Like find_get_pages, except we only return pages which are tagged with
1312
 * @tag.   We update @index to index the next page for the traversal.
L
Linus Torvalds 已提交
1313 1314 1315 1316
 */
unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
			int tag, unsigned int nr_pages, struct page **pages)
{
1317 1318 1319 1320 1321 1322
	struct radix_tree_iter iter;
	void **slot;
	unsigned ret = 0;

	if (unlikely(!nr_pages))
		return 0;
N
Nick Piggin 已提交
1323 1324 1325

	rcu_read_lock();
restart:
1326 1327
	radix_tree_for_each_tagged(slot, &mapping->page_tree,
				   &iter, *index, tag) {
N
Nick Piggin 已提交
1328 1329
		struct page *page;
repeat:
1330
		page = radix_tree_deref_slot(slot);
N
Nick Piggin 已提交
1331 1332
		if (unlikely(!page))
			continue;
1333

1334
		if (radix_tree_exception(page)) {
1335 1336 1337 1338 1339 1340 1341 1342
			if (radix_tree_deref_retry(page)) {
				/*
				 * Transient condition which can only trigger
				 * when entry at index 0 moves out of or back
				 * to root: none yet gotten, safe to restart.
				 */
				goto restart;
			}
1343
			/*
1344 1345 1346 1347 1348 1349 1350 1351 1352
			 * A shadow entry of a recently evicted page.
			 *
			 * Those entries should never be tagged, but
			 * this tree walk is lockless and the tags are
			 * looked up in bulk, one radix tree node at a
			 * time, so there is a sizable window for page
			 * reclaim to evict a page we saw tagged.
			 *
			 * Skip over it.
1353
			 */
1354
			continue;
1355
		}
N
Nick Piggin 已提交
1356 1357 1358 1359 1360

		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
1361
		if (unlikely(page != *slot)) {
N
Nick Piggin 已提交
1362 1363 1364 1365 1366
			page_cache_release(page);
			goto repeat;
		}

		pages[ret] = page;
1367 1368
		if (++ret == nr_pages)
			break;
N
Nick Piggin 已提交
1369
	}
1370

N
Nick Piggin 已提交
1371
	rcu_read_unlock();
L
Linus Torvalds 已提交
1372 1373 1374

	if (ret)
		*index = pages[ret - 1]->index + 1;
N
Nick Piggin 已提交
1375

L
Linus Torvalds 已提交
1376 1377
	return ret;
}
1378
EXPORT_SYMBOL(find_get_pages_tag);
L
Linus Torvalds 已提交
1379

1380 1381 1382 1383 1384
/**
 * grab_cache_page_nowait - returns locked page at given index in given cache
 * @mapping: target address_space
 * @index: the page index
 *
1385
 * Same as grab_cache_page(), but do not wait if the page is unavailable.
L
Linus Torvalds 已提交
1386 1387 1388 1389 1390 1391 1392 1393
 * This is intended for speculative data generators, where the data can
 * be regenerated if the page couldn't be grabbed.  This routine should
 * be safe to call while holding the lock for another page.
 *
 * Clear __GFP_FS when allocating the page to avoid recursion into the fs
 * and deadlock against the caller's locked page.
 */
struct page *
1394
grab_cache_page_nowait(struct address_space *mapping, pgoff_t index)
L
Linus Torvalds 已提交
1395 1396 1397 1398
{
	struct page *page = find_get_page(mapping, index);

	if (page) {
N
Nick Piggin 已提交
1399
		if (trylock_page(page))
L
Linus Torvalds 已提交
1400 1401 1402 1403
			return page;
		page_cache_release(page);
		return NULL;
	}
1404
	page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS);
N
Nick Piggin 已提交
1405
	if (page && add_to_page_cache_lru(page, mapping, index, GFP_NOFS)) {
L
Linus Torvalds 已提交
1406 1407 1408 1409 1410 1411 1412
		page_cache_release(page);
		page = NULL;
	}
	return page;
}
EXPORT_SYMBOL(grab_cache_page_nowait);

1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
/*
 * CD/DVDs are error prone. When a medium error occurs, the driver may fail
 * a _large_ part of the i/o request. Imagine the worst scenario:
 *
 *      ---R__________________________________________B__________
 *         ^ reading here                             ^ bad block(assume 4k)
 *
 * read(R) => miss => readahead(R...B) => media error => frustrating retries
 * => failing the whole request => read(R) => read(R+1) =>
 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
 *
 * It is going insane. Fix it by quickly scaling down the readahead size.
 */
static void shrink_readahead_size_eio(struct file *filp,
					struct file_ra_state *ra)
{
	ra->ra_pages /= 4;
}

1434
/**
C
Christoph Hellwig 已提交
1435
 * do_generic_file_read - generic file read routine
1436 1437
 * @filp:	the file to read
 * @ppos:	current file position
1438 1439
 * @iter:	data destination
 * @written:	already copied
1440
 *
L
Linus Torvalds 已提交
1441
 * This is a generic file read routine, and uses the
1442
 * mapping->a_ops->readpage() function for the actual low-level stuff.
L
Linus Torvalds 已提交
1443 1444 1445 1446
 *
 * This is really ugly. But the goto's actually try to clarify some
 * of the logic when it comes to error handling etc.
 */
1447 1448
static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos,
		struct iov_iter *iter, ssize_t written)
L
Linus Torvalds 已提交
1449
{
C
Christoph Hellwig 已提交
1450
	struct address_space *mapping = filp->f_mapping;
L
Linus Torvalds 已提交
1451
	struct inode *inode = mapping->host;
C
Christoph Hellwig 已提交
1452
	struct file_ra_state *ra = &filp->f_ra;
1453 1454 1455 1456
	pgoff_t index;
	pgoff_t last_index;
	pgoff_t prev_index;
	unsigned long offset;      /* offset into pagecache page */
1457
	unsigned int prev_offset;
1458
	int error = 0;
L
Linus Torvalds 已提交
1459 1460

	index = *ppos >> PAGE_CACHE_SHIFT;
1461 1462
	prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT;
	prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1);
1463
	last_index = (*ppos + iter->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
L
Linus Torvalds 已提交
1464 1465 1466 1467
	offset = *ppos & ~PAGE_CACHE_MASK;

	for (;;) {
		struct page *page;
1468
		pgoff_t end_index;
N
NeilBrown 已提交
1469
		loff_t isize;
L
Linus Torvalds 已提交
1470 1471 1472 1473 1474
		unsigned long nr, ret;

		cond_resched();
find_page:
		page = find_get_page(mapping, index);
1475
		if (!page) {
1476
			page_cache_sync_readahead(mapping,
1477
					ra, filp,
1478 1479 1480 1481 1482 1483
					index, last_index - index);
			page = find_get_page(mapping, index);
			if (unlikely(page == NULL))
				goto no_cached_page;
		}
		if (PageReadahead(page)) {
1484
			page_cache_async_readahead(mapping,
1485
					ra, filp, page,
1486
					index, last_index - index);
L
Linus Torvalds 已提交
1487
		}
1488 1489 1490 1491
		if (!PageUptodate(page)) {
			if (inode->i_blkbits == PAGE_CACHE_SHIFT ||
					!mapping->a_ops->is_partially_uptodate)
				goto page_not_up_to_date;
N
Nick Piggin 已提交
1492
			if (!trylock_page(page))
1493
				goto page_not_up_to_date;
1494 1495 1496
			/* Did it get truncated before we got the lock? */
			if (!page->mapping)
				goto page_not_up_to_date_locked;
1497
			if (!mapping->a_ops->is_partially_uptodate(page,
1498
							offset, iter->count))
1499 1500 1501
				goto page_not_up_to_date_locked;
			unlock_page(page);
		}
L
Linus Torvalds 已提交
1502
page_ok:
N
NeilBrown 已提交
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528
		/*
		 * i_size must be checked after we know the page is Uptodate.
		 *
		 * Checking i_size after the check allows us to calculate
		 * the correct value for "nr", which means the zero-filled
		 * part of the page is not copied back to userspace (unless
		 * another truncate extends the file - this is desired though).
		 */

		isize = i_size_read(inode);
		end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
		if (unlikely(!isize || index > end_index)) {
			page_cache_release(page);
			goto out;
		}

		/* nr is the maximum number of bytes to copy from this page */
		nr = PAGE_CACHE_SIZE;
		if (index == end_index) {
			nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
			if (nr <= offset) {
				page_cache_release(page);
				goto out;
			}
		}
		nr = nr - offset;
L
Linus Torvalds 已提交
1529 1530 1531 1532 1533 1534 1535 1536 1537

		/* If users can be writing to this page using arbitrary
		 * virtual addresses, take care about potential aliasing
		 * before reading the page on the kernel side.
		 */
		if (mapping_writably_mapped(mapping))
			flush_dcache_page(page);

		/*
1538 1539
		 * When a sequential read accesses a page several times,
		 * only mark it as accessed the first time.
L
Linus Torvalds 已提交
1540
		 */
1541
		if (prev_index != index || offset != prev_offset)
L
Linus Torvalds 已提交
1542 1543 1544 1545 1546 1547 1548
			mark_page_accessed(page);
		prev_index = index;

		/*
		 * Ok, we have the page, and it's up-to-date, so
		 * now we can copy it to user space...
		 */
1549 1550

		ret = copy_page_to_iter(page, offset, nr, iter);
L
Linus Torvalds 已提交
1551 1552 1553
		offset += ret;
		index += offset >> PAGE_CACHE_SHIFT;
		offset &= ~PAGE_CACHE_MASK;
J
Jan Kara 已提交
1554
		prev_offset = offset;
L
Linus Torvalds 已提交
1555 1556

		page_cache_release(page);
1557 1558 1559 1560 1561 1562 1563 1564
		written += ret;
		if (!iov_iter_count(iter))
			goto out;
		if (ret < nr) {
			error = -EFAULT;
			goto out;
		}
		continue;
L
Linus Torvalds 已提交
1565 1566 1567

page_not_up_to_date:
		/* Get exclusive access to the page ... */
1568 1569 1570
		error = lock_page_killable(page);
		if (unlikely(error))
			goto readpage_error;
L
Linus Torvalds 已提交
1571

1572
page_not_up_to_date_locked:
N
Nick Piggin 已提交
1573
		/* Did it get truncated before we got the lock? */
L
Linus Torvalds 已提交
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
		if (!page->mapping) {
			unlock_page(page);
			page_cache_release(page);
			continue;
		}

		/* Did somebody else fill it already? */
		if (PageUptodate(page)) {
			unlock_page(page);
			goto page_ok;
		}

readpage:
1587 1588 1589 1590 1591 1592
		/*
		 * A previous I/O error may have been due to temporary
		 * failures, eg. multipath errors.
		 * PG_error will be set again if readpage fails.
		 */
		ClearPageError(page);
L
Linus Torvalds 已提交
1593 1594 1595
		/* Start the actual read. The read will unlock the page. */
		error = mapping->a_ops->readpage(filp, page);

1596 1597 1598
		if (unlikely(error)) {
			if (error == AOP_TRUNCATED_PAGE) {
				page_cache_release(page);
1599
				error = 0;
1600 1601
				goto find_page;
			}
L
Linus Torvalds 已提交
1602
			goto readpage_error;
1603
		}
L
Linus Torvalds 已提交
1604 1605

		if (!PageUptodate(page)) {
1606 1607 1608
			error = lock_page_killable(page);
			if (unlikely(error))
				goto readpage_error;
L
Linus Torvalds 已提交
1609 1610 1611
			if (!PageUptodate(page)) {
				if (page->mapping == NULL) {
					/*
1612
					 * invalidate_mapping_pages got it
L
Linus Torvalds 已提交
1613 1614 1615 1616 1617 1618
					 */
					unlock_page(page);
					page_cache_release(page);
					goto find_page;
				}
				unlock_page(page);
1619
				shrink_readahead_size_eio(filp, ra);
1620 1621
				error = -EIO;
				goto readpage_error;
L
Linus Torvalds 已提交
1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
			}
			unlock_page(page);
		}

		goto page_ok;

readpage_error:
		/* UHHUH! A synchronous read error occurred. Report it */
		page_cache_release(page);
		goto out;

no_cached_page:
		/*
		 * Ok, it wasn't cached, so we need to create a new
		 * page..
		 */
N
Nick Piggin 已提交
1638 1639
		page = page_cache_alloc_cold(mapping);
		if (!page) {
1640
			error = -ENOMEM;
N
Nick Piggin 已提交
1641
			goto out;
L
Linus Torvalds 已提交
1642
		}
N
Nick Piggin 已提交
1643
		error = add_to_page_cache_lru(page, mapping,
L
Linus Torvalds 已提交
1644 1645
						index, GFP_KERNEL);
		if (error) {
N
Nick Piggin 已提交
1646
			page_cache_release(page);
1647 1648
			if (error == -EEXIST) {
				error = 0;
L
Linus Torvalds 已提交
1649
				goto find_page;
1650
			}
L
Linus Torvalds 已提交
1651 1652 1653 1654 1655 1656
			goto out;
		}
		goto readpage;
	}

out:
1657 1658 1659
	ra->prev_pos = prev_index;
	ra->prev_pos <<= PAGE_CACHE_SHIFT;
	ra->prev_pos |= prev_offset;
L
Linus Torvalds 已提交
1660

1661
	*ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset;
1662
	file_accessed(filp);
1663
	return written ? written : error;
L
Linus Torvalds 已提交
1664 1665
}

1666
/**
H
Henrik Kretzschmar 已提交
1667
 * generic_file_aio_read - generic filesystem read routine
1668 1669 1670
 * @iocb:	kernel I/O control block
 * @iov:	io vector request
 * @nr_segs:	number of segments in the iovec
H
Henrik Kretzschmar 已提交
1671
 * @pos:	current file position
1672
 *
L
Linus Torvalds 已提交
1673 1674 1675 1676
 * This is the "read()" routine for all filesystems
 * that can use the page cache directly.
 */
ssize_t
1677 1678
generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
		unsigned long nr_segs, loff_t pos)
L
Linus Torvalds 已提交
1679 1680
{
	struct file *filp = iocb->ki_filp;
A
Al Viro 已提交
1681
	ssize_t retval = 0;
L
Linus Torvalds 已提交
1682
	size_t count;
1683
	loff_t *ppos = &iocb->ki_pos;
1684
	struct iov_iter i;
L
Linus Torvalds 已提交
1685

A
Al Viro 已提交
1686
	count = iov_length(iov, nr_segs);
1687
	iov_iter_init(&i, iov, nr_segs, count, 0);
L
Linus Torvalds 已提交
1688 1689 1690

	/* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
	if (filp->f_flags & O_DIRECT) {
1691
		loff_t size;
L
Linus Torvalds 已提交
1692 1693 1694 1695 1696 1697 1698 1699
		struct address_space *mapping;
		struct inode *inode;

		mapping = filp->f_mapping;
		inode = mapping->host;
		if (!count)
			goto out; /* skip atime */
		size = i_size_read(inode);
1700
		retval = filemap_write_and_wait_range(mapping, pos,
1701
					pos + iov_length(iov, nr_segs) - 1);
A
Al Viro 已提交
1702 1703 1704
		if (!retval)
			retval = mapping->a_ops->direct_IO(READ, iocb, &i, pos);

1705 1706 1707
		if (retval > 0) {
			*ppos = pos + retval;
			count -= retval;
1708 1709 1710 1711 1712
			/*
			 * If we did a short DIO read we need to skip the
			 * section of the iov that we've already read data into.
			 */
			iov_iter_advance(&i, retval);
1713
		}
1714

1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725
		/*
		 * Btrfs can have a short DIO read if we encounter
		 * compressed extents, so if there was an error, or if
		 * we've already read everything we wanted to, or if
		 * there was a short read because we hit EOF, go ahead
		 * and return.  Otherwise fallthrough to buffered io for
		 * the rest of the read.
		 */
		if (retval < 0 || !count || *ppos >= size) {
			file_accessed(filp);
			goto out;
1726
		}
L
Linus Torvalds 已提交
1727 1728
	}

1729
	retval = do_generic_file_read(filp, ppos, &i, retval);
L
Linus Torvalds 已提交
1730 1731 1732 1733 1734 1735
out:
	return retval;
}
EXPORT_SYMBOL(generic_file_aio_read);

#ifdef CONFIG_MMU
1736 1737 1738 1739 1740
/**
 * page_cache_read - adds requested page to the page cache if not already there
 * @file:	file to read
 * @offset:	page index
 *
L
Linus Torvalds 已提交
1741 1742 1743
 * This adds the requested page to the page cache if it isn't already there,
 * and schedules an I/O to read in its contents from disk.
 */
H
Harvey Harrison 已提交
1744
static int page_cache_read(struct file *file, pgoff_t offset)
L
Linus Torvalds 已提交
1745 1746 1747
{
	struct address_space *mapping = file->f_mapping;
	struct page *page; 
1748
	int ret;
L
Linus Torvalds 已提交
1749

1750 1751 1752 1753 1754 1755 1756 1757 1758 1759
	do {
		page = page_cache_alloc_cold(mapping);
		if (!page)
			return -ENOMEM;

		ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
		if (ret == 0)
			ret = mapping->a_ops->readpage(file, page);
		else if (ret == -EEXIST)
			ret = 0; /* losing race to add is OK */
L
Linus Torvalds 已提交
1760 1761 1762

		page_cache_release(page);

1763 1764 1765
	} while (ret == AOP_TRUNCATED_PAGE);
		
	return ret;
L
Linus Torvalds 已提交
1766 1767 1768 1769
}

#define MMAP_LOTSAMISS  (100)

1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782
/*
 * Synchronous readahead happens when we don't even find
 * a page in the page cache at all.
 */
static void do_sync_mmap_readahead(struct vm_area_struct *vma,
				   struct file_ra_state *ra,
				   struct file *file,
				   pgoff_t offset)
{
	unsigned long ra_pages;
	struct address_space *mapping = file->f_mapping;

	/* If we don't want any read-ahead, don't bother */
1783
	if (vma->vm_flags & VM_RAND_READ)
1784
		return;
1785 1786
	if (!ra->ra_pages)
		return;
1787

1788
	if (vma->vm_flags & VM_SEQ_READ) {
1789 1790
		page_cache_sync_readahead(mapping, ra, file, offset,
					  ra->ra_pages);
1791 1792 1793
		return;
	}

1794 1795
	/* Avoid banging the cache line if not needed */
	if (ra->mmap_miss < MMAP_LOTSAMISS * 10)
1796 1797 1798 1799 1800 1801 1802 1803 1804
		ra->mmap_miss++;

	/*
	 * Do we miss much more than hit in this file? If so,
	 * stop bothering with read-ahead. It will only hurt.
	 */
	if (ra->mmap_miss > MMAP_LOTSAMISS)
		return;

1805 1806 1807
	/*
	 * mmap read-around
	 */
1808
	ra_pages = max_sane_readahead(ra->ra_pages);
1809 1810
	ra->start = max_t(long, 0, offset - ra_pages / 2);
	ra->size = ra_pages;
1811
	ra->async_size = ra_pages / 4;
1812
	ra_submit(ra, mapping, file);
1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827
}

/*
 * Asynchronous readahead happens when we find the page and PG_readahead,
 * so we want to possibly extend the readahead further..
 */
static void do_async_mmap_readahead(struct vm_area_struct *vma,
				    struct file_ra_state *ra,
				    struct file *file,
				    struct page *page,
				    pgoff_t offset)
{
	struct address_space *mapping = file->f_mapping;

	/* If we don't want any read-ahead, don't bother */
1828
	if (vma->vm_flags & VM_RAND_READ)
1829 1830 1831 1832
		return;
	if (ra->mmap_miss > 0)
		ra->mmap_miss--;
	if (PageReadahead(page))
1833 1834
		page_cache_async_readahead(mapping, ra, file,
					   page, offset, ra->ra_pages);
1835 1836
}

1837
/**
1838
 * filemap_fault - read in file data for page fault handling
N
Nick Piggin 已提交
1839 1840
 * @vma:	vma in which the fault was taken
 * @vmf:	struct vm_fault containing details of the fault
1841
 *
1842
 * filemap_fault() is invoked via the vma operations vector for a
L
Linus Torvalds 已提交
1843 1844 1845 1846 1847 1848
 * mapped memory region to read in file data during a page fault.
 *
 * The goto's are kind of ugly, but this streamlines the normal case of having
 * it in the page cache, and handles the special cases reasonably without
 * having a lot of duplicated code.
 */
N
Nick Piggin 已提交
1849
int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
L
Linus Torvalds 已提交
1850 1851
{
	int error;
1852
	struct file *file = vma->vm_file;
L
Linus Torvalds 已提交
1853 1854 1855
	struct address_space *mapping = file->f_mapping;
	struct file_ra_state *ra = &file->f_ra;
	struct inode *inode = mapping->host;
1856
	pgoff_t offset = vmf->pgoff;
L
Linus Torvalds 已提交
1857
	struct page *page;
1858
	loff_t size;
N
Nick Piggin 已提交
1859
	int ret = 0;
L
Linus Torvalds 已提交
1860

1861 1862
	size = round_up(i_size_read(inode), PAGE_CACHE_SIZE);
	if (offset >= size >> PAGE_CACHE_SHIFT)
1863
		return VM_FAULT_SIGBUS;
L
Linus Torvalds 已提交
1864 1865

	/*
1866
	 * Do we have something in the page cache already?
L
Linus Torvalds 已提交
1867
	 */
1868
	page = find_get_page(mapping, offset);
1869
	if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
L
Linus Torvalds 已提交
1870
		/*
1871 1872
		 * We found the page, so try async readahead before
		 * waiting for the lock.
L
Linus Torvalds 已提交
1873
		 */
1874
		do_async_mmap_readahead(vma, ra, file, page, offset);
1875
	} else if (!page) {
1876 1877 1878
		/* No page in the page cache at all */
		do_sync_mmap_readahead(vma, ra, file, offset);
		count_vm_event(PGMAJFAULT);
1879
		mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1880 1881
		ret = VM_FAULT_MAJOR;
retry_find:
1882
		page = find_get_page(mapping, offset);
L
Linus Torvalds 已提交
1883 1884 1885 1886
		if (!page)
			goto no_cached_page;
	}

1887 1888
	if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) {
		page_cache_release(page);
1889
		return ret | VM_FAULT_RETRY;
1890
	}
1891 1892 1893 1894 1895 1896 1897

	/* Did it get truncated? */
	if (unlikely(page->mapping != mapping)) {
		unlock_page(page);
		put_page(page);
		goto retry_find;
	}
1898
	VM_BUG_ON_PAGE(page->index != offset, page);
1899

L
Linus Torvalds 已提交
1900
	/*
1901 1902
	 * We have a locked page in the page cache, now we need to check
	 * that it's up-to-date. If not, it is going to be due to an error.
L
Linus Torvalds 已提交
1903
	 */
1904
	if (unlikely(!PageUptodate(page)))
L
Linus Torvalds 已提交
1905 1906
		goto page_not_uptodate;

1907 1908 1909 1910
	/*
	 * Found the page and have a reference on it.
	 * We must recheck i_size under page lock.
	 */
1911 1912
	size = round_up(i_size_read(inode), PAGE_CACHE_SIZE);
	if (unlikely(offset >= size >> PAGE_CACHE_SHIFT)) {
1913
		unlock_page(page);
1914
		page_cache_release(page);
1915
		return VM_FAULT_SIGBUS;
1916 1917
	}

N
Nick Piggin 已提交
1918
	vmf->page = page;
N
Nick Piggin 已提交
1919
	return ret | VM_FAULT_LOCKED;
L
Linus Torvalds 已提交
1920 1921 1922 1923 1924 1925

no_cached_page:
	/*
	 * We're only likely to ever get here if MADV_RANDOM is in
	 * effect.
	 */
1926
	error = page_cache_read(file, offset);
L
Linus Torvalds 已提交
1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941

	/*
	 * The page we want has now been added to the page cache.
	 * In the unlikely event that someone removed it in the
	 * meantime, we'll just come back here and read it again.
	 */
	if (error >= 0)
		goto retry_find;

	/*
	 * An error return from page_cache_read can result if the
	 * system is low on memory, or a problem occurs while trying
	 * to schedule I/O.
	 */
	if (error == -ENOMEM)
N
Nick Piggin 已提交
1942 1943
		return VM_FAULT_OOM;
	return VM_FAULT_SIGBUS;
L
Linus Torvalds 已提交
1944 1945 1946 1947 1948 1949 1950 1951 1952

page_not_uptodate:
	/*
	 * Umm, take care of errors if the page isn't up-to-date.
	 * Try to re-read it _once_. We do this synchronously,
	 * because there really aren't any performance issues here
	 * and we need to check for errors.
	 */
	ClearPageError(page);
1953
	error = mapping->a_ops->readpage(file, page);
1954 1955 1956 1957 1958
	if (!error) {
		wait_on_page_locked(page);
		if (!PageUptodate(page))
			error = -EIO;
	}
1959 1960 1961
	page_cache_release(page);

	if (!error || error == AOP_TRUNCATED_PAGE)
1962
		goto retry_find;
L
Linus Torvalds 已提交
1963

1964
	/* Things didn't work out. Return zero to tell the mm layer so. */
1965
	shrink_readahead_size_eio(file, ra);
N
Nick Piggin 已提交
1966
	return VM_FAULT_SIGBUS;
1967 1968 1969
}
EXPORT_SYMBOL(filemap_fault);

1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015
void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct radix_tree_iter iter;
	void **slot;
	struct file *file = vma->vm_file;
	struct address_space *mapping = file->f_mapping;
	loff_t size;
	struct page *page;
	unsigned long address = (unsigned long) vmf->virtual_address;
	unsigned long addr;
	pte_t *pte;

	rcu_read_lock();
	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, vmf->pgoff) {
		if (iter.index > vmf->max_pgoff)
			break;
repeat:
		page = radix_tree_deref_slot(slot);
		if (unlikely(!page))
			goto next;
		if (radix_tree_exception(page)) {
			if (radix_tree_deref_retry(page))
				break;
			else
				goto next;
		}

		if (!page_cache_get_speculative(page))
			goto repeat;

		/* Has the page moved? */
		if (unlikely(page != *slot)) {
			page_cache_release(page);
			goto repeat;
		}

		if (!PageUptodate(page) ||
				PageReadahead(page) ||
				PageHWPoison(page))
			goto skip;
		if (!trylock_page(page))
			goto skip;

		if (page->mapping != mapping || !PageUptodate(page))
			goto unlock;

2016 2017
		size = round_up(i_size_read(mapping->host), PAGE_CACHE_SIZE);
		if (page->index >= size >> PAGE_CACHE_SHIFT)
2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041
			goto unlock;

		pte = vmf->pte + page->index - vmf->pgoff;
		if (!pte_none(*pte))
			goto unlock;

		if (file->f_ra.mmap_miss > 0)
			file->f_ra.mmap_miss--;
		addr = address + (page->index - vmf->pgoff) * PAGE_SIZE;
		do_set_pte(vma, addr, page, pte, false, false);
		unlock_page(page);
		goto next;
unlock:
		unlock_page(page);
skip:
		page_cache_release(page);
next:
		if (iter.index == vmf->max_pgoff)
			break;
	}
	rcu_read_unlock();
}
EXPORT_SYMBOL(filemap_map_pages);

2042 2043 2044
int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct page *page = vmf->page;
A
Al Viro 已提交
2045
	struct inode *inode = file_inode(vma->vm_file);
2046 2047
	int ret = VM_FAULT_LOCKED;

2048
	sb_start_pagefault(inode->i_sb);
2049 2050 2051 2052 2053 2054 2055
	file_update_time(vma->vm_file);
	lock_page(page);
	if (page->mapping != inode->i_mapping) {
		unlock_page(page);
		ret = VM_FAULT_NOPAGE;
		goto out;
	}
2056 2057 2058 2059 2060 2061
	/*
	 * We mark the page dirty already here so that when freeze is in
	 * progress, we are guaranteed that writeback during freezing will
	 * see the dirty page and writeprotect it again.
	 */
	set_page_dirty(page);
2062
	wait_for_stable_page(page);
2063
out:
2064
	sb_end_pagefault(inode->i_sb);
2065 2066 2067 2068
	return ret;
}
EXPORT_SYMBOL(filemap_page_mkwrite);

2069
const struct vm_operations_struct generic_file_vm_ops = {
2070
	.fault		= filemap_fault,
2071
	.map_pages	= filemap_map_pages,
2072
	.page_mkwrite	= filemap_page_mkwrite,
2073
	.remap_pages	= generic_file_remap_pages,
L
Linus Torvalds 已提交
2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111
};

/* This is used for a general mmap of a disk file */

int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
{
	struct address_space *mapping = file->f_mapping;

	if (!mapping->a_ops->readpage)
		return -ENOEXEC;
	file_accessed(file);
	vma->vm_ops = &generic_file_vm_ops;
	return 0;
}

/*
 * This is for filesystems which do not implement ->writepage.
 */
int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
{
	if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
		return -EINVAL;
	return generic_file_mmap(file, vma);
}
#else
int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
{
	return -ENOSYS;
}
int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
{
	return -ENOSYS;
}
#endif /* CONFIG_MMU */

EXPORT_SYMBOL(generic_file_mmap);
EXPORT_SYMBOL(generic_file_readonly_mmap);

S
Sasha Levin 已提交
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123
static struct page *wait_on_page_read(struct page *page)
{
	if (!IS_ERR(page)) {
		wait_on_page_locked(page);
		if (!PageUptodate(page)) {
			page_cache_release(page);
			page = ERR_PTR(-EIO);
		}
	}
	return page;
}

2124
static struct page *__read_cache_page(struct address_space *mapping,
2125
				pgoff_t index,
2126
				int (*filler)(void *, struct page *),
2127 2128
				void *data,
				gfp_t gfp)
L
Linus Torvalds 已提交
2129
{
N
Nick Piggin 已提交
2130
	struct page *page;
L
Linus Torvalds 已提交
2131 2132 2133 2134
	int err;
repeat:
	page = find_get_page(mapping, index);
	if (!page) {
2135
		page = __page_cache_alloc(gfp | __GFP_COLD);
N
Nick Piggin 已提交
2136 2137
		if (!page)
			return ERR_PTR(-ENOMEM);
2138
		err = add_to_page_cache_lru(page, mapping, index, gfp);
N
Nick Piggin 已提交
2139 2140 2141 2142
		if (unlikely(err)) {
			page_cache_release(page);
			if (err == -EEXIST)
				goto repeat;
L
Linus Torvalds 已提交
2143 2144 2145 2146 2147 2148 2149
			/* Presumably ENOMEM for radix tree node */
			return ERR_PTR(err);
		}
		err = filler(data, page);
		if (err < 0) {
			page_cache_release(page);
			page = ERR_PTR(err);
S
Sasha Levin 已提交
2150 2151
		} else {
			page = wait_on_page_read(page);
L
Linus Torvalds 已提交
2152 2153 2154 2155 2156
		}
	}
	return page;
}

2157
static struct page *do_read_cache_page(struct address_space *mapping,
2158
				pgoff_t index,
2159
				int (*filler)(void *, struct page *),
2160 2161 2162
				void *data,
				gfp_t gfp)

L
Linus Torvalds 已提交
2163 2164 2165 2166 2167
{
	struct page *page;
	int err;

retry:
2168
	page = __read_cache_page(mapping, index, filler, data, gfp);
L
Linus Torvalds 已提交
2169
	if (IS_ERR(page))
2170
		return page;
L
Linus Torvalds 已提交
2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
	if (PageUptodate(page))
		goto out;

	lock_page(page);
	if (!page->mapping) {
		unlock_page(page);
		page_cache_release(page);
		goto retry;
	}
	if (PageUptodate(page)) {
		unlock_page(page);
		goto out;
	}
	err = filler(data, page);
	if (err < 0) {
		page_cache_release(page);
2187
		return ERR_PTR(err);
S
Sasha Levin 已提交
2188 2189 2190 2191
	} else {
		page = wait_on_page_read(page);
		if (IS_ERR(page))
			return page;
L
Linus Torvalds 已提交
2192
	}
2193
out:
2194 2195 2196
	mark_page_accessed(page);
	return page;
}
2197 2198

/**
S
Sasha Levin 已提交
2199
 * read_cache_page - read into page cache, fill it if needed
2200 2201 2202
 * @mapping:	the page's address_space
 * @index:	the page index
 * @filler:	function to perform the read
2203
 * @data:	first arg to filler(data, page) function, often left as NULL
2204 2205
 *
 * Read into the page cache. If a page already exists, and PageUptodate() is
S
Sasha Levin 已提交
2206
 * not set, try to fill the page and wait for it to become unlocked.
2207 2208 2209
 *
 * If the page does not get brought uptodate, return -EIO.
 */
S
Sasha Levin 已提交
2210
struct page *read_cache_page(struct address_space *mapping,
2211
				pgoff_t index,
2212
				int (*filler)(void *, struct page *),
2213 2214 2215 2216
				void *data)
{
	return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
}
S
Sasha Levin 已提交
2217
EXPORT_SYMBOL(read_cache_page);
2218 2219 2220 2221 2222 2223 2224 2225

/**
 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
 * @mapping:	the page's address_space
 * @index:	the page index
 * @gfp:	the page allocator flags to use if allocating
 *
 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
2226
 * any new page allocations done using the specified allocation flags.
2227 2228 2229 2230 2231 2232 2233 2234 2235
 *
 * If the page does not get brought uptodate, return -EIO.
 */
struct page *read_cache_page_gfp(struct address_space *mapping,
				pgoff_t index,
				gfp_t gfp)
{
	filler_t *filler = (filler_t *)mapping->a_ops->readpage;

S
Sasha Levin 已提交
2236
	return do_read_cache_page(mapping, index, filler, NULL, gfp);
2237 2238 2239
}
EXPORT_SYMBOL(read_cache_page_gfp);

L
Linus Torvalds 已提交
2240 2241 2242
/*
 * Performs necessary checks before doing a write
 *
2243
 * Can adjust writing position or amount of bytes to write.
L
Linus Torvalds 已提交
2244 2245 2246 2247 2248 2249
 * Returns appropriate error code that caller should return or
 * zero in case that write should be allowed.
 */
inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
{
	struct inode *inode = file->f_mapping->host;
J
Jiri Slaby 已提交
2250
	unsigned long limit = rlimit(RLIMIT_FSIZE);
L
Linus Torvalds 已提交
2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301

        if (unlikely(*pos < 0))
                return -EINVAL;

	if (!isblk) {
		/* FIXME: this is for backwards compatibility with 2.4 */
		if (file->f_flags & O_APPEND)
                        *pos = i_size_read(inode);

		if (limit != RLIM_INFINITY) {
			if (*pos >= limit) {
				send_sig(SIGXFSZ, current, 0);
				return -EFBIG;
			}
			if (*count > limit - (typeof(limit))*pos) {
				*count = limit - (typeof(limit))*pos;
			}
		}
	}

	/*
	 * LFS rule
	 */
	if (unlikely(*pos + *count > MAX_NON_LFS &&
				!(file->f_flags & O_LARGEFILE))) {
		if (*pos >= MAX_NON_LFS) {
			return -EFBIG;
		}
		if (*count > MAX_NON_LFS - (unsigned long)*pos) {
			*count = MAX_NON_LFS - (unsigned long)*pos;
		}
	}

	/*
	 * Are we about to exceed the fs block limit ?
	 *
	 * If we have written data it becomes a short write.  If we have
	 * exceeded without writing data we send a signal and return EFBIG.
	 * Linus frestrict idea will clean these up nicely..
	 */
	if (likely(!isblk)) {
		if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
			if (*count || *pos > inode->i_sb->s_maxbytes) {
				return -EFBIG;
			}
			/* zero-length writes at ->s_maxbytes are OK */
		}

		if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
			*count = inode->i_sb->s_maxbytes - *pos;
	} else {
2302
#ifdef CONFIG_BLOCK
L
Linus Torvalds 已提交
2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313
		loff_t isize;
		if (bdev_read_only(I_BDEV(inode)))
			return -EPERM;
		isize = i_size_read(inode);
		if (*pos >= isize) {
			if (*count || *pos > isize)
				return -ENOSPC;
		}

		if (*pos + *count > isize)
			*count = isize - *pos;
2314 2315 2316
#else
		return -EPERM;
#endif
L
Linus Torvalds 已提交
2317 2318 2319 2320 2321
	}
	return 0;
}
EXPORT_SYMBOL(generic_write_checks);

2322 2323 2324 2325 2326 2327
int pagecache_write_begin(struct file *file, struct address_space *mapping,
				loff_t pos, unsigned len, unsigned flags,
				struct page **pagep, void **fsdata)
{
	const struct address_space_operations *aops = mapping->a_ops;

2328
	return aops->write_begin(file, mapping, pos, len, flags,
2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
							pagep, fsdata);
}
EXPORT_SYMBOL(pagecache_write_begin);

int pagecache_write_end(struct file *file, struct address_space *mapping,
				loff_t pos, unsigned len, unsigned copied,
				struct page *page, void *fsdata)
{
	const struct address_space_operations *aops = mapping->a_ops;

2339 2340
	mark_page_accessed(page);
	return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
2341 2342 2343
}
EXPORT_SYMBOL(pagecache_write_end);

L
Linus Torvalds 已提交
2344
ssize_t
2345 2346
generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from,
		loff_t pos, size_t count, size_t ocount)
L
Linus Torvalds 已提交
2347 2348 2349 2350 2351
{
	struct file	*file = iocb->ki_filp;
	struct address_space *mapping = file->f_mapping;
	struct inode	*inode = mapping->host;
	ssize_t		written;
2352 2353
	size_t		write_len;
	pgoff_t		end;
L
Linus Torvalds 已提交
2354 2355

	if (count != ocount)
2356
		from->nr_segs = iov_shorten((struct iovec *)from->iov, from->nr_segs, count);
L
Linus Torvalds 已提交
2357

2358
	write_len = iov_length(from->iov, from->nr_segs);
2359 2360
	end = (pos + write_len - 1) >> PAGE_CACHE_SHIFT;

2361
	written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);
2362 2363 2364 2365 2366 2367 2368
	if (written)
		goto out;

	/*
	 * After a write we want buffered reads to be sure to go to disk to get
	 * the new data.  We invalidate clean cached page from the region we're
	 * about to write.  We do this *before* the write so that we can return
2369
	 * without clobbering -EIOCBQUEUED from ->direct_IO().
2370 2371 2372 2373
	 */
	if (mapping->nrpages) {
		written = invalidate_inode_pages2_range(mapping,
					pos >> PAGE_CACHE_SHIFT, end);
2374 2375 2376 2377 2378 2379 2380
		/*
		 * If a page can not be invalidated, return 0 to fall back
		 * to buffered write.
		 */
		if (written) {
			if (written == -EBUSY)
				return 0;
2381
			goto out;
2382
		}
2383 2384
	}

A
Al Viro 已提交
2385
	written = mapping->a_ops->direct_IO(WRITE, iocb, from, pos);
2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399

	/*
	 * Finally, try again to invalidate clean pages which might have been
	 * cached by non-direct readahead, or faulted in by get_user_pages()
	 * if the source of the write was an mmap'ed region of the file
	 * we're writing.  Either one is a pretty crazy thing to do,
	 * so we don't support it 100%.  If this invalidation
	 * fails, tough, the write still worked...
	 */
	if (mapping->nrpages) {
		invalidate_inode_pages2_range(mapping,
					      pos >> PAGE_CACHE_SHIFT, end);
	}

L
Linus Torvalds 已提交
2400
	if (written > 0) {
2401
		pos += written;
2402
		iov_iter_advance(from, written);
2403 2404
		if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
			i_size_write(inode, pos);
L
Linus Torvalds 已提交
2405 2406
			mark_inode_dirty(inode);
		}
2407
		iocb->ki_pos = pos;
L
Linus Torvalds 已提交
2408
	}
2409
out:
L
Linus Torvalds 已提交
2410 2411 2412 2413
	return written;
}
EXPORT_SYMBOL(generic_file_direct_write);

N
Nick Piggin 已提交
2414 2415 2416 2417
/*
 * Find or create a page at the given pagecache position. Return the locked
 * page. This function is specifically for buffered writes.
 */
2418 2419
struct page *grab_cache_page_write_begin(struct address_space *mapping,
					pgoff_t index, unsigned flags)
N
Nick Piggin 已提交
2420 2421
{
	int status;
2422
	gfp_t gfp_mask;
N
Nick Piggin 已提交
2423
	struct page *page;
2424
	gfp_t gfp_notmask = 0;
2425

2426 2427 2428
	gfp_mask = mapping_gfp_mask(mapping);
	if (mapping_cap_account_dirty(mapping))
		gfp_mask |= __GFP_WRITE;
2429 2430
	if (flags & AOP_FLAG_NOFS)
		gfp_notmask = __GFP_FS;
N
Nick Piggin 已提交
2431 2432
repeat:
	page = find_lock_page(mapping, index);
2433
	if (page)
2434
		goto found;
N
Nick Piggin 已提交
2435

2436
	page = __page_cache_alloc(gfp_mask & ~gfp_notmask);
N
Nick Piggin 已提交
2437 2438
	if (!page)
		return NULL;
2439 2440
	status = add_to_page_cache_lru(page, mapping, index,
						GFP_KERNEL & ~gfp_notmask);
N
Nick Piggin 已提交
2441 2442 2443 2444 2445 2446
	if (unlikely(status)) {
		page_cache_release(page);
		if (status == -EEXIST)
			goto repeat;
		return NULL;
	}
2447
found:
2448
	wait_for_stable_page(page);
N
Nick Piggin 已提交
2449 2450
	return page;
}
2451
EXPORT_SYMBOL(grab_cache_page_write_begin);
N
Nick Piggin 已提交
2452

2453
ssize_t generic_perform_write(struct file *file,
2454 2455 2456 2457 2458 2459
				struct iov_iter *i, loff_t pos)
{
	struct address_space *mapping = file->f_mapping;
	const struct address_space_operations *a_ops = mapping->a_ops;
	long status = 0;
	ssize_t written = 0;
N
Nick Piggin 已提交
2460 2461 2462 2463 2464 2465 2466
	unsigned int flags = 0;

	/*
	 * Copies from kernel address space cannot fail (NFSD is a big user).
	 */
	if (segment_eq(get_fs(), KERNEL_DS))
		flags |= AOP_FLAG_UNINTERRUPTIBLE;
2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494

	do {
		struct page *page;
		unsigned long offset;	/* Offset into pagecache page */
		unsigned long bytes;	/* Bytes to write to page */
		size_t copied;		/* Bytes copied from user */
		void *fsdata;

		offset = (pos & (PAGE_CACHE_SIZE - 1));
		bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
						iov_iter_count(i));

again:
		/*
		 * Bring in the user page that we will copy from _first_.
		 * Otherwise there's a nasty deadlock on copying from the
		 * same page as we're writing to, without it being marked
		 * up-to-date.
		 *
		 * Not only is this an optimisation, but it is also required
		 * to check that the address is actually valid, when atomic
		 * usercopies are used, below.
		 */
		if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
			status = -EFAULT;
			break;
		}

N
Nick Piggin 已提交
2495
		status = a_ops->write_begin(file, mapping, pos, bytes, flags,
2496 2497 2498 2499
						&page, &fsdata);
		if (unlikely(status))
			break;

2500 2501 2502
		if (mapping_writably_mapped(mapping))
			flush_dcache_page(page);

2503 2504 2505
		copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
		flush_dcache_page(page);

2506
		mark_page_accessed(page);
2507 2508 2509 2510 2511 2512 2513 2514
		status = a_ops->write_end(file, mapping, pos, bytes, copied,
						page, fsdata);
		if (unlikely(status < 0))
			break;
		copied = status;

		cond_resched();

2515
		iov_iter_advance(i, copied);
2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
		if (unlikely(copied == 0)) {
			/*
			 * If we were unable to copy any data at all, we must
			 * fall back to a single segment length write.
			 *
			 * If we didn't fallback here, we could livelock
			 * because not all segments in the iov can be copied at
			 * once without a pagefault.
			 */
			bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
						iov_iter_single_seg_count(i));
			goto again;
		}
		pos += copied;
		written += copied;

		balance_dirty_pages_ratelimited(mapping);
2533 2534 2535 2536
		if (fatal_signal_pending(current)) {
			status = -EINTR;
			break;
		}
2537 2538 2539 2540
	} while (iov_iter_count(i));

	return written ? written : status;
}
2541
EXPORT_SYMBOL(generic_perform_write);
L
Linus Torvalds 已提交
2542

2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561
/**
 * __generic_file_aio_write - write data to a file
 * @iocb:	IO state structure (file, offset, etc.)
 * @iov:	vector with data to write
 * @nr_segs:	number of segments in the vector
 *
 * This function does all the work needed for actually writing data to a
 * file. It does all basic checks, removes SUID from the file, updates
 * modification times and calls proper subroutines depending on whether we
 * do direct IO or a standard buffered write.
 *
 * It expects i_mutex to be grabbed unless we work on a block device or similar
 * object which does not need locking at all.
 *
 * This function does *not* take care of syncing data in case of O_SYNC write.
 * A caller has to handle it. This is mainly due to the fact that we want to
 * avoid syncing under i_mutex.
 */
ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
2562
				 unsigned long nr_segs)
L
Linus Torvalds 已提交
2563 2564
{
	struct file *file = iocb->ki_filp;
2565
	struct address_space * mapping = file->f_mapping;
L
Linus Torvalds 已提交
2566 2567 2568
	size_t ocount;		/* original count */
	size_t count;		/* after file limit checks */
	struct inode 	*inode = mapping->host;
2569
	loff_t		pos = iocb->ki_pos;
2570
	ssize_t		written = 0;
L
Linus Torvalds 已提交
2571
	ssize_t		err;
2572 2573
	ssize_t		status;
	struct iov_iter from;
L
Linus Torvalds 已提交
2574

A
Al Viro 已提交
2575
	count = ocount = iov_length(iov, nr_segs);
L
Linus Torvalds 已提交
2576 2577 2578 2579 2580 2581 2582 2583 2584 2585

	/* We can write back this queue in page reclaim */
	current->backing_dev_info = mapping->backing_dev_info;
	err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
	if (err)
		goto out;

	if (count == 0)
		goto out;

2586
	err = file_remove_suid(file);
L
Linus Torvalds 已提交
2587 2588 2589
	if (err)
		goto out;

2590 2591 2592
	err = file_update_time(file);
	if (err)
		goto out;
L
Linus Torvalds 已提交
2593

2594 2595
	iov_iter_init(&from, iov, nr_segs, count, 0);

L
Linus Torvalds 已提交
2596 2597
	/* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
	if (unlikely(file->f_flags & O_DIRECT)) {
2598 2599
		loff_t endbyte;

2600
		written = generic_file_direct_write(iocb, &from, pos,
2601
							count, ocount);
L
Linus Torvalds 已提交
2602 2603
		if (written < 0 || written == count)
			goto out;
2604

L
Linus Torvalds 已提交
2605 2606 2607 2608 2609 2610
		/*
		 * direct-io write to a hole: fall through to buffered I/O
		 * for completing the rest of the request.
		 */
		pos += written;
		count -= written;
2611 2612

		status = generic_perform_write(file, &from, pos);
2613
		/*
2614
		 * If generic_perform_write() returned a synchronous error
2615 2616 2617 2618 2619
		 * then we want to return the number of bytes which were
		 * direct-written, or the error code if that was zero.  Note
		 * that this differs from normal direct-io semantics, which
		 * will return -EFOO even if some bytes were written.
		 */
2620 2621
		if (unlikely(status < 0) && !written) {
			err = status;
2622 2623
			goto out;
		}
2624
		iocb->ki_pos = pos + status;
2625 2626 2627 2628 2629
		/*
		 * We need to ensure that the page cache pages are written to
		 * disk and invalidated to preserve the expected O_DIRECT
		 * semantics.
		 */
2630
		endbyte = pos + status - 1;
2631
		err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte);
2632
		if (err == 0) {
2633
			written += status;
2634 2635 2636 2637 2638 2639 2640 2641 2642 2643
			invalidate_mapping_pages(mapping,
						 pos >> PAGE_CACHE_SHIFT,
						 endbyte >> PAGE_CACHE_SHIFT);
		} else {
			/*
			 * We don't know how much we wrote, so just return
			 * the number of bytes which were direct-written
			 */
		}
	} else {
2644 2645 2646
		written = generic_perform_write(file, &from, pos);
		if (likely(written >= 0))
			iocb->ki_pos = pos + written;
2647
	}
L
Linus Torvalds 已提交
2648 2649 2650 2651
out:
	current->backing_dev_info = NULL;
	return written ? written : err;
}
2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664
EXPORT_SYMBOL(__generic_file_aio_write);

/**
 * generic_file_aio_write - write data to a file
 * @iocb:	IO state structure
 * @iov:	vector with data to write
 * @nr_segs:	number of segments in the vector
 * @pos:	position in file where to write
 *
 * This is a wrapper around __generic_file_aio_write() to be used by most
 * filesystems. It takes care of syncing the file in case of O_SYNC file
 * and acquires i_mutex as needed.
 */
2665 2666
ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
		unsigned long nr_segs, loff_t pos)
L
Linus Torvalds 已提交
2667 2668
{
	struct file *file = iocb->ki_filp;
2669
	struct inode *inode = file->f_mapping->host;
L
Linus Torvalds 已提交
2670 2671 2672 2673
	ssize_t ret;

	BUG_ON(iocb->ki_pos != pos);

2674
	mutex_lock(&inode->i_mutex);
2675
	ret = __generic_file_aio_write(iocb, iov, nr_segs);
2676
	mutex_unlock(&inode->i_mutex);
L
Linus Torvalds 已提交
2677

2678
	if (ret > 0) {
L
Linus Torvalds 已提交
2679 2680
		ssize_t err;

2681 2682
		err = generic_write_sync(file, iocb->ki_pos - ret, ret);
		if (err < 0)
L
Linus Torvalds 已提交
2683 2684 2685 2686 2687 2688
			ret = err;
	}
	return ret;
}
EXPORT_SYMBOL(generic_file_aio_write);

2689 2690 2691 2692 2693 2694 2695 2696 2697 2698
/**
 * try_to_release_page() - release old fs-specific metadata on a page
 *
 * @page: the page which the kernel is trying to free
 * @gfp_mask: memory allocation flags (and I/O mode)
 *
 * The address_space is to try to release any data against the page
 * (presumably at page->private).  If the release was successful, return `1'.
 * Otherwise return zero.
 *
2699 2700 2701
 * This may also be called if PG_fscache is set on a page, indicating that the
 * page is known to the local caching routines.
 *
2702
 * The @gfp_mask argument specifies whether I/O may be performed to release
2703
 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT & __GFP_FS).
2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719
 *
 */
int try_to_release_page(struct page *page, gfp_t gfp_mask)
{
	struct address_space * const mapping = page->mapping;

	BUG_ON(!PageLocked(page));
	if (PageWriteback(page))
		return 0;

	if (mapping && mapping->a_ops->releasepage)
		return mapping->a_ops->releasepage(page, gfp_mask);
	return try_to_free_buffers(page);
}

EXPORT_SYMBOL(try_to_release_page);