write.c 56.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * linux/fs/nfs/write.c
 *
4
 * Write file data over NFS.
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14
 *
 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/writeback.h>
15
#include <linux/swap.h>
16
#include <linux/migrate.h>
L
Linus Torvalds 已提交
17 18 19 20 21

#include <linux/sunrpc/clnt.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/nfs_page.h>
22
#include <linux/backing-dev.h>
23
#include <linux/export.h>
24 25
#include <linux/freezer.h>
#include <linux/wait.h>
26

27
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
28 29

#include "delegation.h"
30
#include "internal.h"
31
#include "iostat.h"
32
#include "nfs4_fs.h"
33
#include "fscache.h"
34
#include "pnfs.h"
L
Linus Torvalds 已提交
35

36 37
#include "nfstrace.h"

L
Linus Torvalds 已提交
38 39 40 41 42
#define NFSDBG_FACILITY		NFSDBG_PAGECACHE

#define MIN_POOL_WRITE		(32)
#define MIN_POOL_COMMIT		(4)

43 44 45 46 47 48
struct nfs_io_completion {
	void (*complete)(void *data);
	void *data;
	struct kref refcount;
};

L
Linus Torvalds 已提交
49 50 51
/*
 * Local function declarations
 */
52
static void nfs_redirty_request(struct nfs_page *req);
53
static const struct rpc_call_ops nfs_commit_ops;
54
static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
55
static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
56
static const struct nfs_rw_ops nfs_rw_write_ops;
57
static void nfs_clear_request_commit(struct nfs_page *req);
58 59
static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
				      struct inode *inode);
60 61 62
static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
						struct page *page);
L
Linus Torvalds 已提交
63

64
static struct kmem_cache *nfs_wdata_cachep;
65
static mempool_t *nfs_wdata_mempool;
66
static struct kmem_cache *nfs_cdata_cachep;
L
Linus Torvalds 已提交
67 68
static mempool_t *nfs_commit_mempool;

N
NeilBrown 已提交
69
struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail)
L
Linus Torvalds 已提交
70
{
N
NeilBrown 已提交
71
	struct nfs_commit_data *p;
72

N
NeilBrown 已提交
73 74 75 76 77 78 79 80 81 82 83 84 85 86
	if (never_fail)
		p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
	else {
		/* It is OK to do some reclaim, not no safe to wait
		 * for anything to be returned to the pool.
		 * mempool_alloc() cannot handle that particular combination,
		 * so we need two separate attempts.
		 */
		p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
		if (!p)
			p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO |
					     __GFP_NOWARN | __GFP_NORETRY);
		if (!p)
			return NULL;
L
Linus Torvalds 已提交
87
	}
N
NeilBrown 已提交
88 89 90

	memset(p, 0, sizeof(*p));
	INIT_LIST_HEAD(&p->pages);
L
Linus Torvalds 已提交
91 92
	return p;
}
93
EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
L
Linus Torvalds 已提交
94

95
void nfs_commit_free(struct nfs_commit_data *p)
L
Linus Torvalds 已提交
96 97 98
{
	mempool_free(p, nfs_commit_mempool);
}
99
EXPORT_SYMBOL_GPL(nfs_commit_free);
L
Linus Torvalds 已提交
100

101
static struct nfs_pgio_header *nfs_writehdr_alloc(void)
102
{
103
	struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
104

105 106
	memset(p, 0, sizeof(*p));
	p->rw_mode = FMODE_WRITE;
107 108
	return p;
}
109

110
static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
111
{
112
	mempool_free(hdr, nfs_wdata_mempool);
113
}
L
Linus Torvalds 已提交
114

115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
{
	return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
}

static void nfs_io_completion_init(struct nfs_io_completion *ioc,
		void (*complete)(void *), void *data)
{
	ioc->complete = complete;
	ioc->data = data;
	kref_init(&ioc->refcount);
}

static void nfs_io_completion_release(struct kref *kref)
{
	struct nfs_io_completion *ioc = container_of(kref,
			struct nfs_io_completion, refcount);
	ioc->complete(ioc->data);
	kfree(ioc);
}

static void nfs_io_completion_get(struct nfs_io_completion *ioc)
{
	if (ioc != NULL)
		kref_get(&ioc->refcount);
}

static void nfs_io_completion_put(struct nfs_io_completion *ioc)
{
	if (ioc != NULL)
		kref_put(&ioc->refcount, nfs_io_completion_release);
}

148 149 150 151 152 153 154
static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
{
	ctx->error = error;
	smp_wmb();
	set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
}

155 156 157 158 159 160 161 162
static struct nfs_page *
nfs_page_private_request(struct page *page)
{
	if (!PagePrivate(page))
		return NULL;
	return (struct nfs_page *)page_private(page);
}

163 164 165 166 167 168 169
/*
 * nfs_page_find_head_request_locked - find head request associated with @page
 *
 * must be called while holding the inode lock.
 *
 * returns matching head request with reference held, or NULL if not found.
 */
170
static struct nfs_page *
171
nfs_page_find_private_request(struct page *page)
172
{
173
	struct address_space *mapping = page_file_mapping(page);
174
	struct nfs_page *req;
175

176 177
	if (!PagePrivate(page))
		return NULL;
178
	spin_lock(&mapping->private_lock);
179
	req = nfs_page_private_request(page);
180 181
	if (req) {
		WARN_ON_ONCE(req->wb_head != req);
182
		kref_get(&req->wb_kref);
183
	}
184
	spin_unlock(&mapping->private_lock);
185 186
	return req;
}
187

188 189 190 191 192 193 194 195
static struct nfs_page *
nfs_page_find_swap_request(struct page *page)
{
	struct inode *inode = page_file_mapping(page)->host;
	struct nfs_inode *nfsi = NFS_I(inode);
	struct nfs_page *req = NULL;
	if (!PageSwapCache(page))
		return NULL;
196
	mutex_lock(&nfsi->commit_mutex);
197 198 199 200 201 202 203 204
	if (PageSwapCache(page)) {
		req = nfs_page_search_commits_for_head_request_locked(nfsi,
			page);
		if (req) {
			WARN_ON_ONCE(req->wb_head != req);
			kref_get(&req->wb_kref);
		}
	}
205
	mutex_unlock(&nfsi->commit_mutex);
206 207 208
	return req;
}

209 210 211 212 213 214
/*
 * nfs_page_find_head_request - find head request associated with @page
 *
 * returns matching head request with reference held, or NULL if not found.
 */
static struct nfs_page *nfs_page_find_head_request(struct page *page)
215
{
216
	struct nfs_page *req;
217

218 219 220
	req = nfs_page_find_private_request(page);
	if (!req)
		req = nfs_page_find_swap_request(page);
221 222 223
	return req;
}

L
Linus Torvalds 已提交
224 225 226
/* Adjust the file length if we're writing beyond the end */
static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
{
227
	struct inode *inode = page_file_mapping(page)->host;
228 229
	loff_t end, i_size;
	pgoff_t end_index;
L
Linus Torvalds 已提交
230

231 232
	spin_lock(&inode->i_lock);
	i_size = i_size_read(inode);
233
	end_index = (i_size - 1) >> PAGE_SHIFT;
H
Huang Ying 已提交
234
	if (i_size > 0 && page_index(page) < end_index)
235
		goto out;
236
	end = page_file_offset(page) + ((loff_t)offset+count);
L
Linus Torvalds 已提交
237
	if (i_size >= end)
238
		goto out;
L
Linus Torvalds 已提交
239
	i_size_write(inode, end);
240 241 242
	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
out:
	spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
243 244
}

245 246 247
/* A writeback failed: mark the page as bad, and invalidate the page cache */
static void nfs_set_pageerror(struct page *page)
{
248
	nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page));
249 250
}

251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
/*
 * nfs_page_group_search_locked
 * @head - head request of page group
 * @page_offset - offset into page
 *
 * Search page group with head @head to find a request that contains the
 * page offset @page_offset.
 *
 * Returns a pointer to the first matching nfs request, or NULL if no
 * match is found.
 *
 * Must be called with the page group lock held
 */
static struct nfs_page *
nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
{
	struct nfs_page *req;

	req = head;
	do {
		if (page_offset >= req->wb_pgbase &&
		    page_offset < (req->wb_pgbase + req->wb_bytes))
			return req;

		req = req->wb_this_page;
	} while (req != head);

	return NULL;
}

/*
 * nfs_page_group_covers_page
 * @head - head request of page group
 *
 * Return true if the page group with head @head covers the whole page,
 * returns false otherwise
 */
static bool nfs_page_group_covers_page(struct nfs_page *req)
{
	struct nfs_page *tmp;
	unsigned int pos = 0;
	unsigned int len = nfs_page_length(req->wb_page);

294
	nfs_page_group_lock(req);
295

296
	for (;;) {
297
		tmp = nfs_page_group_search_locked(req->wb_head, pos);
298 299 300 301
		if (!tmp)
			break;
		pos = tmp->wb_pgbase + tmp->wb_bytes;
	}
302 303

	nfs_page_group_unlock(req);
304
	return pos >= len;
305 306
}

L
Linus Torvalds 已提交
307 308 309
/* We can set the PG_uptodate flag if we see that a write request
 * covers the full page.
 */
310
static void nfs_mark_uptodate(struct nfs_page *req)
L
Linus Torvalds 已提交
311
{
312
	if (PageUptodate(req->wb_page))
L
Linus Torvalds 已提交
313
		return;
314
	if (!nfs_page_group_covers_page(req))
L
Linus Torvalds 已提交
315
		return;
316
	SetPageUptodate(req->wb_page);
L
Linus Torvalds 已提交
317 318 319 320
}

static int wb_priority(struct writeback_control *wbc)
{
321
	int ret = 0;
322

323 324 325
	if (wbc->sync_mode == WB_SYNC_ALL)
		ret = FLUSH_COND_STABLE;
	return ret;
L
Linus Torvalds 已提交
326 327
}

328 329 330 331 332 333 334 335 336 337
/*
 * NFS congestion control
 */

int nfs_congestion_kb;

#define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
#define NFS_CONGESTION_OFF_THRESH	\
	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))

338
static void nfs_set_page_writeback(struct page *page)
339
{
340 341
	struct inode *inode = page_file_mapping(page)->host;
	struct nfs_server *nfss = NFS_SERVER(inode);
342 343
	int ret = test_set_page_writeback(page);

344
	WARN_ON_ONCE(ret != 0);
345

346
	if (atomic_long_inc_return(&nfss->writeback) >
347 348
			NFS_CONGESTION_ON_THRESH)
		set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
349 350
}

351
static void nfs_end_page_writeback(struct nfs_page *req)
352
{
353
	struct inode *inode = page_file_mapping(req->wb_page)->host;
354
	struct nfs_server *nfss = NFS_SERVER(inode);
355
	bool is_done;
356

357 358 359
	is_done = nfs_page_group_sync_on_bit(req, PG_WB_END);
	nfs_unlock_request(req);
	if (!is_done)
360 361 362
		return;

	end_page_writeback(req->wb_page);
363
	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
364
		clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
365 366
}

367 368 369 370 371 372 373 374 375
/*
 * nfs_unroll_locks_and_wait -  unlock all newly locked reqs and wait on @req
 *
 * this is a helper function for nfs_lock_and_join_requests
 *
 * @inode - inode associated with request page group, must be holding inode lock
 * @head  - head request of page group, must be holding head lock
 * @req   - request that couldn't lock and needs to wait on the req bit lock
 *
376 377
 * NOTE: this must be called holding page_group bit lock
 *       which will be released before returning.
378 379 380
 *
 * returns 0 on success, < 0 on error.
 */
381 382
static void
nfs_unroll_locks(struct inode *inode, struct nfs_page *head,
383
			  struct nfs_page *req)
384 385 386 387
{
	struct nfs_page *tmp;

	/* relinquish all the locks successfully grabbed this run */
388 389 390 391 392
	for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
		if (!kref_read(&tmp->wb_kref))
			continue;
		nfs_unlock_and_release_request(tmp);
	}
393 394 395 396 397 398 399 400 401 402 403 404 405 406
}

/*
 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
 *
 * @destroy_list - request list (using wb_this_page) terminated by @old_head
 * @old_head - the old head of the list
 *
 * All subrequests must be locked and removed from all lists, so at this point
 * they are only "active" in this function, and possibly in nfs_wait_on_request
 * with a reference held by some other context.
 */
static void
nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
407 408
				 struct nfs_page *old_head,
				 struct inode *inode)
409 410 411 412 413 414 415 416 417 418 419 420
{
	while (destroy_list) {
		struct nfs_page *subreq = destroy_list;

		destroy_list = (subreq->wb_this_page == old_head) ?
				   NULL : subreq->wb_this_page;

		WARN_ON_ONCE(old_head != subreq->wb_head);

		/* make sure old group is not used */
		subreq->wb_this_page = subreq;

421 422
		clear_bit(PG_REMOVE, &subreq->wb_flags);

423 424 425
		/* Note: races with nfs_page_group_destroy() */
		if (!kref_read(&subreq->wb_kref)) {
			/* Check if we raced with nfs_page_group_destroy() */
426
			if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags))
427 428 429
				nfs_free_request(subreq);
			continue;
		}
430

431 432 433 434
		subreq->wb_head = subreq;

		if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) {
			nfs_release_request(subreq);
435
			atomic_long_dec(&NFS_I(inode)->nrequests);
436
		}
437 438 439 440

		/* subreq is now totally disconnected from page group or any
		 * write / commit lists. last chance to wake any waiters */
		nfs_unlock_and_release_request(subreq);
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
	}
}

/*
 * nfs_lock_and_join_requests - join all subreqs to the head req and return
 *                              a locked reference, cancelling any pending
 *                              operations for this page.
 *
 * @page - the page used to lookup the "page group" of nfs_page structures
 *
 * This function joins all sub requests to the head request by first
 * locking all requests in the group, cancelling any pending operations
 * and finally updating the head request to cover the whole range covered by
 * the (former) group.  All subrequests are removed from any write or commit
 * lists, unlinked from the group and destroyed.
 *
 * Returns a locked, referenced pointer to the head request - which after
 * this call is guaranteed to be the only request associated with the page.
 * Returns NULL if no requests are found for @page, or a ERR_PTR if an
 * error was encountered.
 */
static struct nfs_page *
463
nfs_lock_and_join_requests(struct page *page)
464
{
465
	struct inode *inode = page_file_mapping(page)->host;
466 467 468
	struct nfs_page *head, *subreq;
	struct nfs_page *destroy_list = NULL;
	unsigned int total_bytes;
469 470
	int ret;

471 472 473 474 475 476
try_again:
	/*
	 * A reference is taken only on the head request which acts as a
	 * reference to the whole page group - the group will not be destroyed
	 * until the head reference is released.
	 */
477 478
	head = nfs_page_find_head_request(page);
	if (!head)
479 480
		return NULL;

481 482 483 484 485 486 487 488
	/* lock the page head first in order to avoid an ABBA inefficiency */
	if (!nfs_lock_request(head)) {
		ret = nfs_wait_on_request(head);
		nfs_release_request(head);
		if (ret < 0)
			return ERR_PTR(ret);
		goto try_again;
	}
489 490 491 492 493 494

	/* Ensure that nobody removed the request before we locked it */
	if (head != nfs_page_private_request(page) && !PageSwapCache(page)) {
		nfs_unlock_and_release_request(head);
		goto try_again;
	}
495

496
	ret = nfs_page_group_lock(head);
497
	if (ret < 0) {
498
		nfs_unlock_and_release_request(head);
499
		return ERR_PTR(ret);
500
	}
501 502

	/* lock each request in the page group */
503 504 505
	total_bytes = head->wb_bytes;
	for (subreq = head->wb_this_page; subreq != head;
			subreq = subreq->wb_this_page) {
506

507 508
		if (!kref_get_unless_zero(&subreq->wb_kref))
			continue;
509
		while (!nfs_lock_request(subreq)) {
510
			/*
511 512
			 * Unlock page to allow nfs_page_group_sync_on_bit()
			 * to succeed
513
			 */
514 515 516
			nfs_page_group_unlock(head);
			ret = nfs_wait_on_request(subreq);
			if (!ret)
517
				ret = nfs_page_group_lock(head);
518 519
			if (ret < 0) {
				nfs_unroll_locks(inode, head, subreq);
520
				nfs_release_request(subreq);
521 522 523
				nfs_unlock_and_release_request(head);
				return ERR_PTR(ret);
			}
524
		}
525 526
		/*
		 * Subrequests are always contiguous, non overlapping
527
		 * and in order - but may be repeated (mirrored writes).
528
		 */
529 530 531 532 533 534
		if (subreq->wb_offset == (head->wb_offset + total_bytes)) {
			/* keep track of how many bytes this group covers */
			total_bytes += subreq->wb_bytes;
		} else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset ||
			    ((subreq->wb_offset + subreq->wb_bytes) >
			     (head->wb_offset + total_bytes)))) {
535
			nfs_unroll_locks(inode, head, subreq);
536
			nfs_unlock_and_release_request(subreq);
537 538
			nfs_page_group_unlock(head);
			nfs_unlock_and_release_request(head);
539 540
			return ERR_PTR(-EIO);
		}
541
	}
542 543 544 545 546

	/* Now that all requests are locked, make sure they aren't on any list.
	 * Commit list removal accounting is done after locks are dropped */
	subreq = head;
	do {
547
		nfs_clear_request_commit(subreq);
548 549 550 551 552 553 554 555 556 557 558 559
		subreq = subreq->wb_this_page;
	} while (subreq != head);

	/* unlink subrequests from head, destroy them later */
	if (head->wb_this_page != head) {
		/* destroy list will be terminated by head */
		destroy_list = head->wb_this_page;
		head->wb_this_page = head;

		/* change head request to cover whole range that
		 * the former page group covered */
		head->wb_bytes = total_bytes;
560
	}
561

562 563 564 565
	/* Postpone destruction of this request */
	if (test_and_clear_bit(PG_REMOVE, &head->wb_flags)) {
		set_bit(PG_INODE_REF, &head->wb_flags);
		kref_get(&head->wb_kref);
566
		atomic_long_inc(&NFS_I(inode)->nrequests);
567 568
	}

569 570
	nfs_page_group_unlock(head);

571
	nfs_destroy_unlinked_subrequests(destroy_list, head, inode);
572

573 574 575 576 577 578
	/* Did we lose a race with nfs_inode_remove_request()? */
	if (!(PagePrivate(page) || PageSwapCache(page))) {
		nfs_unlock_and_release_request(head);
		return NULL;
	}

579
	/* still holds ref on head from nfs_page_find_head_request
580 581
	 * and still has lock on head from lock loop */
	return head;
582 583
}

584 585 586 587 588
static void nfs_write_error_remove_page(struct nfs_page *req)
{
	nfs_end_page_writeback(req);
	generic_error_remove_page(page_file_mapping(req->wb_page),
				  req->wb_page);
589
	nfs_release_request(req);
590 591
}

592 593 594 595 596 597 598 599 600 601
static bool
nfs_error_is_fatal_on_server(int err)
{
	switch (err) {
	case 0:
	case -ERESTARTSYS:
	case -EINTR:
		return false;
	}
	return nfs_error_is_fatal(err);
602 603
}

604 605 606 607 608
/*
 * Find an associated nfs write request, and prepare to flush it out
 * May return an error if the user signalled nfs_wait_on_request().
 */
static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
609
				struct page *page)
610 611 612 613
{
	struct nfs_page *req;
	int ret = 0;

614
	req = nfs_lock_and_join_requests(page);
615 616 617 618 619 620
	if (!req)
		goto out;
	ret = PTR_ERR(req);
	if (IS_ERR(req))
		goto out;

621 622
	nfs_set_page_writeback(page);
	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
623

624
	ret = 0;
625 626 627 628
	/* If there is a fatal error that covers this write, just exit */
	if (nfs_error_is_fatal_on_server(req->wb_context->error))
		goto out_launder;

629
	if (!nfs_pageio_add_request(pgio, req)) {
630
		ret = pgio->pg_error;
631
		/*
632
		 * Remove the problematic req upon fatal errors on the server
633 634 635
		 */
		if (nfs_error_is_fatal(ret)) {
			nfs_context_set_write_error(req->wb_context, ret);
636
			if (nfs_error_is_fatal_on_server(ret))
637
				goto out_launder;
638
		}
639 640
		nfs_redirty_request(req);
		ret = -EAGAIN;
641 642 643
	} else
		nfs_add_stats(page_file_mapping(page)->host,
				NFSIOS_WRITEPAGES, 1);
644 645
out:
	return ret;
646 647 648
out_launder:
	nfs_write_error_remove_page(req);
	return ret;
649 650
}

651
static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
652
			    struct nfs_pageio_descriptor *pgio)
L
Linus Torvalds 已提交
653
{
654
	int ret;
L
Linus Torvalds 已提交
655

H
Huang Ying 已提交
656
	nfs_pageio_cond_complete(pgio, page_index(page));
657
	ret = nfs_page_async_flush(pgio, page);
658 659 660 661 662
	if (ret == -EAGAIN) {
		redirty_page_for_writepage(wbc, page);
		ret = 0;
	}
	return ret;
663
}
664

665 666 667
/*
 * Write an mmapped page to the server.
 */
668
static int nfs_writepage_locked(struct page *page,
669
				struct writeback_control *wbc)
670 671
{
	struct nfs_pageio_descriptor pgio;
672
	struct inode *inode = page_file_mapping(page)->host;
673
	int err;
674

675
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
676
	nfs_pageio_init_write(&pgio, inode, 0,
677
				false, &nfs_async_write_completion_ops);
678
	err = nfs_do_writepage(page, wbc, &pgio);
679 680 681 682 683 684
	nfs_pageio_complete(&pgio);
	if (err < 0)
		return err;
	if (pgio.pg_error < 0)
		return pgio.pg_error;
	return 0;
685 686 687 688
}

int nfs_writepage(struct page *page, struct writeback_control *wbc)
{
689
	int ret;
690

691
	ret = nfs_writepage_locked(page, wbc);
L
Linus Torvalds 已提交
692
	unlock_page(page);
693 694 695 696 697 698 699
	return ret;
}

static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
{
	int ret;

700
	ret = nfs_do_writepage(page, wbc, data);
701 702
	unlock_page(page);
	return ret;
L
Linus Torvalds 已提交
703 704
}

705 706 707 708 709
static void nfs_io_completion_commit(void *inode)
{
	nfs_commit_inode(inode, 0);
}

L
Linus Torvalds 已提交
710 711 712
int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;
713
	struct nfs_pageio_descriptor pgio;
714
	struct nfs_io_completion *ioc = nfs_io_completion_alloc(GFP_NOFS);
L
Linus Torvalds 已提交
715 716
	int err;

717 718
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);

719 720 721
	if (ioc)
		nfs_io_completion_init(ioc, nfs_io_completion_commit, inode);

722 723
	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
				&nfs_async_write_completion_ops);
724
	pgio.pg_io_completion = ioc;
725
	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
726
	nfs_pageio_complete(&pgio);
727
	nfs_io_completion_put(ioc);
728

729
	if (err < 0)
730 731 732 733
		goto out_err;
	err = pgio.pg_error;
	if (err < 0)
		goto out_err;
734
	return 0;
735 736
out_err:
	return err;
L
Linus Torvalds 已提交
737 738 739 740 741
}

/*
 * Insert a write request into an inode
 */
742
static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
L
Linus Torvalds 已提交
743
{
744
	struct address_space *mapping = page_file_mapping(req->wb_page);
L
Linus Torvalds 已提交
745
	struct nfs_inode *nfsi = NFS_I(inode);
746

747 748
	WARN_ON_ONCE(req->wb_this_page != req);

749
	/* Lock the request! */
750
	nfs_lock_request(req);
751

752 753 754 755
	/*
	 * Swap-space should not get truncated. Hence no need to plug the race
	 * with invalidate/truncate.
	 */
756 757 758 759 760 761 762
	spin_lock(&mapping->private_lock);
	if (!nfs_have_writebacks(inode) &&
	    NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) {
		spin_lock(&inode->i_lock);
		inode->i_version++;
		spin_unlock(&inode->i_lock);
	}
763 764 765 766 767
	if (likely(!PageSwapCache(req->wb_page))) {
		set_bit(PG_MAPPED, &req->wb_flags);
		SetPagePrivate(req->wb_page);
		set_page_private(req->wb_page, (unsigned long)req);
	}
768
	spin_unlock(&mapping->private_lock);
769
	atomic_long_inc(&nfsi->nrequests);
770
	/* this a head request for a page group - mark it as having an
771 772 773
	 * extra reference so sub groups can follow suit.
	 * This flag also informs pgio layer when to bump nrequests when
	 * adding subrequests. */
774
	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
775
	kref_get(&req->wb_kref);
L
Linus Torvalds 已提交
776 777 778
}

/*
779
 * Remove a write request from an inode
L
Linus Torvalds 已提交
780 781 782
 */
static void nfs_inode_remove_request(struct nfs_page *req)
{
783 784
	struct address_space *mapping = page_file_mapping(req->wb_page);
	struct inode *inode = mapping->host;
L
Linus Torvalds 已提交
785
	struct nfs_inode *nfsi = NFS_I(inode);
786
	struct nfs_page *head;
L
Linus Torvalds 已提交
787

788
	atomic_long_dec(&nfsi->nrequests);
789 790 791
	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
		head = req->wb_head;

792
		spin_lock(&mapping->private_lock);
793
		if (likely(head->wb_page && !PageSwapCache(head->wb_page))) {
794 795 796 797
			set_page_private(head->wb_page, 0);
			ClearPagePrivate(head->wb_page);
			clear_bit(PG_MAPPED, &head->wb_flags);
		}
798
		spin_unlock(&mapping->private_lock);
799
	}
800 801 802

	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags))
		nfs_release_request(req);
L
Linus Torvalds 已提交
803 804
}

805
static void
F
Fred 已提交
806
nfs_mark_request_dirty(struct nfs_page *req)
807
{
808 809
	if (req->wb_page)
		__set_page_dirty_nobuffers(req->wb_page);
810 811
}

812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
/*
 * nfs_page_search_commits_for_head_request_locked
 *
 * Search through commit lists on @inode for the head request for @page.
 * Must be called while holding the inode (which is cinfo) lock.
 *
 * Returns the head request if found, or NULL if not found.
 */
static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
						struct page *page)
{
	struct nfs_page *freq, *t;
	struct nfs_commit_info cinfo;
	struct inode *inode = &nfsi->vfs_inode;

	nfs_init_cinfo_from_inode(&cinfo, inode);

	/* search through pnfs commit lists */
	freq = pnfs_search_commit_reqs(inode, &cinfo, page);
	if (freq)
		return freq->wb_head;

	/* Linearly search the commit list for the correct request */
	list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) {
		if (freq->wb_page == page)
			return freq->wb_head;
	}

	return NULL;
}

844 845 846 847 848 849 850 851 852 853
/**
 * nfs_request_add_commit_list_locked - add request to a commit list
 * @req: pointer to a struct nfs_page
 * @dst: commit list head
 * @cinfo: holds list lock and accounting info
 *
 * This sets the PG_CLEAN bit, updates the cinfo count of
 * number of outstanding requests requiring a commit as well as
 * the MM page stats.
 *
854 855
 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
 * nfs_page lock.
856 857 858 859 860 861 862
 */
void
nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
			    struct nfs_commit_info *cinfo)
{
	set_bit(PG_CLEAN, &req->wb_flags);
	nfs_list_add_request(req, dst);
863
	atomic_long_inc(&cinfo->mds->ncommit);
864 865 866
}
EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);

867 868 869
/**
 * nfs_request_add_commit_list - add request to a commit list
 * @req: pointer to a struct nfs_page
870 871
 * @dst: commit list head
 * @cinfo: holds list lock and accounting info
872
 *
873
 * This sets the PG_CLEAN bit, updates the cinfo count of
874 875 876
 * number of outstanding requests requiring a commit as well as
 * the MM page stats.
 *
877
 * The caller must _not_ hold the cinfo->lock, but must be
878
 * holding the nfs_page lock.
L
Linus Torvalds 已提交
879
 */
880
void
881
nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
882
{
883
	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
884
	nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
885
	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
886 887
	if (req->wb_page)
		nfs_mark_page_unstable(req->wb_page, cinfo);
L
Linus Torvalds 已提交
888
}
889 890 891 892 893
EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);

/**
 * nfs_request_remove_commit_list - Remove request from a commit list
 * @req: pointer to a nfs_page
894
 * @cinfo: holds list lock and accounting info
895
 *
896
 * This clears the PG_CLEAN bit, and updates the cinfo's count of
897 898 899
 * number of outstanding requests requiring a commit
 * It does not update the MM page stats.
 *
900
 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
901 902
 */
void
903 904
nfs_request_remove_commit_list(struct nfs_page *req,
			       struct nfs_commit_info *cinfo)
905 906 907 908
{
	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
		return;
	nfs_list_remove_request(req);
909
	atomic_long_dec(&cinfo->mds->ncommit);
910 911 912
}
EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);

913 914 915
static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
				      struct inode *inode)
{
916
	cinfo->inode = inode;
917 918
	cinfo->mds = &NFS_I(inode)->commit_info;
	cinfo->ds = pnfs_get_ds_info(inode);
919
	cinfo->dreq = NULL;
920
	cinfo->completion_ops = &nfs_commit_completion_ops;
921 922 923 924 925 926
}

void nfs_init_cinfo(struct nfs_commit_info *cinfo,
		    struct inode *inode,
		    struct nfs_direct_req *dreq)
{
927 928 929 930
	if (dreq)
		nfs_init_cinfo_from_dreq(cinfo, dreq);
	else
		nfs_init_cinfo_from_inode(cinfo, inode);
931 932
}
EXPORT_SYMBOL_GPL(nfs_init_cinfo);
933 934 935 936

/*
 * Add a request to the inode's commit list.
 */
937
void
938
nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
939
			struct nfs_commit_info *cinfo, u32 ds_commit_idx)
940
{
941
	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
942
		return;
943
	nfs_request_add_commit_list(req, cinfo);
944
}
945

946 947 948
static void
nfs_clear_page_commit(struct page *page)
{
949
	dec_node_page_state(page, NR_UNSTABLE_NFS);
950 951
	dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb,
		    WB_RECLAIMABLE);
952 953
}

954
/* Called holding the request lock on @req */
955
static void
956 957
nfs_clear_request_commit(struct nfs_page *req)
{
958
	if (test_bit(PG_CLEAN, &req->wb_flags)) {
959
		struct inode *inode = d_inode(req->wb_context->dentry);
960
		struct nfs_commit_info cinfo;
961

962
		nfs_init_cinfo_from_inode(&cinfo, inode);
963
		mutex_lock(&NFS_I(inode)->commit_mutex);
964 965
		if (!pnfs_clear_request_commit(req, &cinfo)) {
			nfs_request_remove_commit_list(req, &cinfo);
966
		}
967
		mutex_unlock(&NFS_I(inode)->commit_mutex);
968
		nfs_clear_page_commit(req->wb_page);
969 970 971
	}
}

972
int nfs_write_need_commit(struct nfs_pgio_header *hdr)
973
{
974
	if (hdr->verf.committed == NFS_DATA_SYNC)
975
		return hdr->lseg == NULL;
976
	return hdr->verf.committed != NFS_FILE_SYNC;
977 978
}

979 980 981 982 983
static void nfs_async_write_init(struct nfs_pgio_header *hdr)
{
	nfs_io_completion_get(hdr->io_completion);
}

984
static void nfs_write_completion(struct nfs_pgio_header *hdr)
985
{
986
	struct nfs_commit_info cinfo;
987 988 989 990
	unsigned long bytes = 0;

	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
		goto out;
991
	nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
992 993 994 995 996 997 998
	while (!list_empty(&hdr->pages)) {
		struct nfs_page *req = nfs_list_entry(hdr->pages.next);

		bytes += req->wb_bytes;
		nfs_list_remove_request(req);
		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
		    (hdr->good_bytes < bytes)) {
999
			nfs_set_pageerror(req->wb_page);
1000 1001 1002
			nfs_context_set_write_error(req->wb_context, hdr->error);
			goto remove_req;
		}
1003
		if (nfs_write_need_commit(hdr)) {
1004
			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
1005
			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
1006
				hdr->pgio_mirror_idx);
1007 1008 1009 1010 1011
			goto next;
		}
remove_req:
		nfs_inode_remove_request(req);
next:
1012
		nfs_end_page_writeback(req);
1013
		nfs_release_request(req);
1014 1015
	}
out:
1016
	nfs_io_completion_put(hdr->io_completion);
1017
	hdr->release(hdr);
1018
}
L
Linus Torvalds 已提交
1019

1020
unsigned long
1021
nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
1022
{
1023
	return atomic_long_read(&cinfo->mds->ncommit);
1024 1025
}

1026
/* NFS_I(cinfo->inode)->commit_mutex held by caller */
1027
int
1028 1029
nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
		     struct nfs_commit_info *cinfo, int max)
1030
{
1031
	struct nfs_page *req, *tmp;
1032 1033
	int ret = 0;

1034 1035
restart:
	list_for_each_entry_safe(req, tmp, src, wb_list) {
1036
		kref_get(&req->wb_kref);
1037 1038
		if (!nfs_lock_request(req)) {
			int status;
1039 1040 1041 1042 1043 1044 1045

			/* Prevent deadlock with nfs_lock_and_join_requests */
			if (!list_empty(dst)) {
				nfs_release_request(req);
				continue;
			}
			/* Ensure we make progress to prevent livelock */
1046 1047 1048 1049 1050 1051
			mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
			status = nfs_wait_on_request(req);
			nfs_release_request(req);
			mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
			if (status < 0)
				break;
1052
			goto restart;
1053
		}
1054
		nfs_request_remove_commit_list(req, cinfo);
1055
		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
1056 1057
		nfs_list_add_request(req, dst);
		ret++;
1058
		if ((ret == max) && !cinfo->dreq)
1059
			break;
1060
		cond_resched();
1061 1062
	}
	return ret;
1063
}
1064
EXPORT_SYMBOL_GPL(nfs_scan_commit_list);
1065

L
Linus Torvalds 已提交
1066 1067 1068
/*
 * nfs_scan_commit - Scan an inode for commit requests
 * @inode: NFS inode to scan
1069 1070
 * @dst: mds destination list
 * @cinfo: mds and ds lists of reqs ready to commit
L
Linus Torvalds 已提交
1071 1072 1073 1074
 *
 * Moves requests from the inode's 'commit' request list.
 * The requests are *not* checked to ensure that they form a contiguous set.
 */
1075
int
1076 1077
nfs_scan_commit(struct inode *inode, struct list_head *dst,
		struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
1078
{
1079
	int ret = 0;
1080

1081 1082
	if (!atomic_long_read(&cinfo->mds->ncommit))
		return 0;
1083
	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1084
	if (atomic_long_read(&cinfo->mds->ncommit) > 0) {
1085
		const int max = INT_MAX;
1086

1087 1088 1089
		ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
					   cinfo, max);
		ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
1090
	}
1091
	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1092
	return ret;
L
Linus Torvalds 已提交
1093
}
1094

L
Linus Torvalds 已提交
1095
/*
1096 1097
 * Search for an existing write request, and attempt to update
 * it to reflect a new dirty region on a given page.
L
Linus Torvalds 已提交
1098
 *
1099 1100
 * If the attempt fails, then the existing request is flushed out
 * to disk.
L
Linus Torvalds 已提交
1101
 */
1102 1103 1104 1105
static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
		struct page *page,
		unsigned int offset,
		unsigned int bytes)
L
Linus Torvalds 已提交
1106
{
1107 1108 1109 1110 1111
	struct nfs_page *req;
	unsigned int rqend;
	unsigned int end;
	int error;

L
Linus Torvalds 已提交
1112 1113
	end = offset + bytes;

1114 1115 1116
	req = nfs_lock_and_join_requests(page);
	if (IS_ERR_OR_NULL(req))
		return req;
L
Linus Torvalds 已提交
1117

1118 1119 1120 1121 1122 1123 1124 1125 1126
	rqend = req->wb_offset + req->wb_bytes;
	/*
	 * Tell the caller to flush out the request if
	 * the offsets are non-contiguous.
	 * Note: nfs_flush_incompatible() will already
	 * have flushed out requests having wrong owners.
	 */
	if (offset > rqend || end < req->wb_offset)
		goto out_flushme;
L
Linus Torvalds 已提交
1127 1128 1129 1130 1131 1132 1133 1134

	/* Okay, the request matches. Update the region */
	if (offset < req->wb_offset) {
		req->wb_offset = offset;
		req->wb_pgbase = offset;
	}
	if (end > rqend)
		req->wb_bytes = end - req->wb_offset;
1135 1136 1137 1138
	else
		req->wb_bytes = rqend - req->wb_offset;
	return req;
out_flushme:
1139 1140 1141 1142 1143 1144 1145
	/*
	 * Note: we mark the request dirty here because
	 * nfs_lock_and_join_requests() cannot preserve
	 * commit flags, so we have to replay the write.
	 */
	nfs_mark_request_dirty(req);
	nfs_unlock_and_release_request(req);
1146
	error = nfs_wb_page(inode, page);
1147
	return (error < 0) ? ERR_PTR(error) : NULL;
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
}

/*
 * Try to update an existing write request, or create one if there is none.
 *
 * Note: Should always be called with the Page Lock held to prevent races
 * if we have to add a new request. Also assumes that the caller has
 * already called nfs_flush_incompatible() if necessary.
 */
static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
		struct page *page, unsigned int offset, unsigned int bytes)
{
1160
	struct inode *inode = page_file_mapping(page)->host;
1161
	struct nfs_page	*req;
L
Linus Torvalds 已提交
1162

1163 1164 1165
	req = nfs_try_to_update_request(inode, page, offset, bytes);
	if (req != NULL)
		goto out;
1166
	req = nfs_create_request(ctx, page, NULL, offset, bytes);
1167 1168
	if (IS_ERR(req))
		goto out;
1169
	nfs_inode_add_request(inode, req);
1170
out:
1171
	return req;
L
Linus Torvalds 已提交
1172 1173
}

1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
		unsigned int offset, unsigned int count)
{
	struct nfs_page	*req;

	req = nfs_setup_write_request(ctx, page, offset, count);
	if (IS_ERR(req))
		return PTR_ERR(req);
	/* Update file length */
	nfs_grow_file(page, offset, count);
1184
	nfs_mark_uptodate(req);
1185
	nfs_mark_request_dirty(req);
1186
	nfs_unlock_and_release_request(req);
1187 1188 1189
	return 0;
}

L
Linus Torvalds 已提交
1190 1191
int nfs_flush_incompatible(struct file *file, struct page *page)
{
1192
	struct nfs_open_context *ctx = nfs_file_open_context(file);
1193
	struct nfs_lock_context *l_ctx;
1194
	struct file_lock_context *flctx = file_inode(file)->i_flctx;
L
Linus Torvalds 已提交
1195
	struct nfs_page	*req;
1196
	int do_flush, status;
L
Linus Torvalds 已提交
1197 1198 1199 1200 1201 1202 1203 1204
	/*
	 * Look for a request corresponding to this page. If there
	 * is one, and it belongs to another file, we flush it out
	 * before we try to copy anything into the page. Do this
	 * due to the lack of an ACCESS-type call in NFSv2.
	 * Also do the same if we find a request from an existing
	 * dropped page.
	 */
1205
	do {
1206
		req = nfs_page_find_head_request(page);
1207 1208
		if (req == NULL)
			return 0;
1209
		l_ctx = req->wb_lock_context;
1210 1211
		do_flush = req->wb_page != page ||
			!nfs_match_open_context(req->wb_context, ctx);
1212 1213 1214
		if (l_ctx && flctx &&
		    !(list_empty_careful(&flctx->flc_posix) &&
		      list_empty_careful(&flctx->flc_flock))) {
1215
			do_flush |= l_ctx->lockowner != current->files;
1216
		}
L
Linus Torvalds 已提交
1217
		nfs_release_request(req);
1218 1219
		if (!do_flush)
			return 0;
1220
		status = nfs_wb_page(page_file_mapping(page)->host, page);
1221 1222
	} while (status == 0);
	return status;
L
Linus Torvalds 已提交
1223 1224
}

1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
/*
 * Avoid buffered writes when a open context credential's key would
 * expire soon.
 *
 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
 *
 * Return 0 and set a credential flag which triggers the inode to flush
 * and performs  NFS_FILE_SYNC writes if the key will expired within
 * RPC_KEY_EXPIRE_TIMEO.
 */
int
nfs_key_timeout_notify(struct file *filp, struct inode *inode)
{
	struct nfs_open_context *ctx = nfs_file_open_context(filp);
	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;

	return rpcauth_key_timeout_notify(auth, ctx->cred);
}

/*
 * Test if the open context credential key is marked to expire soon.
 */
1247
bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
1248
{
1249 1250 1251
	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;

	return rpcauth_cred_key_to_expire(auth, ctx->cred);
1252 1253
}

1254 1255 1256 1257 1258
/*
 * If the page cache is marked as unsafe or invalid, then we can't rely on
 * the PageUptodate() flag. In this case, we will need to turn off
 * write optimisations that depend on the page contents being correct.
 */
1259
static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
1260
{
1261 1262
	struct nfs_inode *nfsi = NFS_I(inode);

1263 1264
	if (nfs_have_delegated_attributes(inode))
		goto out;
1265
	if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
1266
		return false;
1267
	smp_rmb();
1268
	if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
1269 1270
		return false;
out:
1271 1272
	if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
		return false;
1273
	return PageUptodate(page) != 0;
1274 1275
}

1276 1277 1278 1279 1280 1281 1282
static bool
is_whole_file_wrlock(struct file_lock *fl)
{
	return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
			fl->fl_type == F_WRLCK;
}

1283 1284 1285 1286 1287
/* If we know the page is up to date, and we're not using byte range locks (or
 * if we have the whole file locked for writing), it may be more efficient to
 * extend the write to cover the entire page in order to avoid fragmentation
 * inefficiencies.
 *
1288 1289
 * If the file is opened for synchronous writes then we can just skip the rest
 * of the checks.
1290 1291 1292
 */
static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
{
1293 1294 1295 1296
	int ret;
	struct file_lock_context *flctx = inode->i_flctx;
	struct file_lock *fl;

1297 1298
	if (file->f_flags & O_DSYNC)
		return 0;
1299 1300
	if (!nfs_write_pageuptodate(page, inode))
		return 0;
1301 1302
	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
		return 1;
1303 1304
	if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
		       list_empty_careful(&flctx->flc_posix)))
1305
		return 1;
1306 1307 1308

	/* Check to see if there are whole file write locks */
	ret = 0;
1309
	spin_lock(&flctx->flc_lock);
1310 1311 1312 1313 1314 1315
	if (!list_empty(&flctx->flc_posix)) {
		fl = list_first_entry(&flctx->flc_posix, struct file_lock,
					fl_list);
		if (is_whole_file_wrlock(fl))
			ret = 1;
	} else if (!list_empty(&flctx->flc_flock)) {
1316 1317 1318 1319 1320
		fl = list_first_entry(&flctx->flc_flock, struct file_lock,
					fl_list);
		if (fl->fl_type == F_WRLCK)
			ret = 1;
	}
1321
	spin_unlock(&flctx->flc_lock);
1322
	return ret;
1323 1324
}

L
Linus Torvalds 已提交
1325 1326 1327 1328 1329 1330 1331 1332 1333
/*
 * Update and possibly write a cached page of an NFS file.
 *
 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
 * things with a page scheduled for an RPC call (e.g. invalidate it).
 */
int nfs_updatepage(struct file *file, struct page *page,
		unsigned int offset, unsigned int count)
{
1334
	struct nfs_open_context *ctx = nfs_file_open_context(file);
1335
	struct inode	*inode = page_file_mapping(page)->host;
L
Linus Torvalds 已提交
1336 1337
	int		status = 0;

1338 1339
	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);

1340 1341
	dprintk("NFS:       nfs_updatepage(%pD2 %d@%lld)\n",
		file, count, (long long)(page_file_offset(page) + offset));
L
Linus Torvalds 已提交
1342

1343 1344 1345
	if (!count)
		goto out;

1346
	if (nfs_can_extend_write(file, page, inode)) {
1347
		count = max(count + offset, nfs_page_length(page));
L
Linus Torvalds 已提交
1348 1349 1350
		offset = 0;
	}

1351
	status = nfs_writepage_setup(ctx, page, offset, count);
1352 1353
	if (status < 0)
		nfs_set_pageerror(page);
1354 1355
	else
		__set_page_dirty_nobuffers(page);
1356
out:
1357
	dprintk("NFS:       nfs_updatepage returns %d (isize %lld)\n",
L
Linus Torvalds 已提交
1358 1359 1360 1361
			status, (long long)i_size_read(inode));
	return status;
}

1362
static int flush_task_priority(int how)
L
Linus Torvalds 已提交
1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
{
	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
		case FLUSH_HIGHPRI:
			return RPC_PRIORITY_HIGH;
		case FLUSH_LOWPRI:
			return RPC_PRIORITY_LOW;
	}
	return RPC_PRIORITY_NORMAL;
}

1373 1374
static void nfs_initiate_write(struct nfs_pgio_header *hdr,
			       struct rpc_message *msg,
1375
			       const struct nfs_rpc_ops *rpc_ops,
1376
			       struct rpc_task_setup *task_setup_data, int how)
L
Linus Torvalds 已提交
1377
{
1378
	int priority = flush_task_priority(how);
1379

1380
	task_setup_data->priority = priority;
1381
	rpc_ops->write_setup(hdr, msg);
1382

1383
	nfs4_state_protect_write(NFS_SERVER(hdr->inode)->nfs_client,
1384
				 &task_setup_data->rpc_client, msg, hdr);
1385 1386
}

F
Fred 已提交
1387 1388 1389 1390 1391 1392 1393
/* If a nfs_flush_* function fails, it should remove reqs from @head and
 * call this on each, which will prepare them to be retried on next
 * writeback using standard nfs.
 */
static void nfs_redirty_request(struct nfs_page *req)
{
	nfs_mark_request_dirty(req);
1394
	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1395
	nfs_end_page_writeback(req);
1396
	nfs_release_request(req);
F
Fred 已提交
1397 1398
}

1399
static void nfs_async_write_error(struct list_head *head)
1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
{
	struct nfs_page	*req;

	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_redirty_request(req);
	}
}

1410 1411 1412 1413 1414
static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
{
	nfs_async_write_error(&hdr->pages);
}

1415
static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1416
	.init_hdr = nfs_async_write_init,
1417 1418
	.error_cleanup = nfs_async_write_error,
	.completion = nfs_write_completion,
1419
	.reschedule_io = nfs_async_write_reschedule_io,
1420 1421
};

1422
void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1423
			       struct inode *inode, int ioflags, bool force_mds,
1424
			       const struct nfs_pgio_completion_ops *compl_ops)
L
Linus Torvalds 已提交
1425
{
1426
	struct nfs_server *server = NFS_SERVER(inode);
1427
	const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
1428 1429 1430 1431 1432

#ifdef CONFIG_NFS_V4_1
	if (server->pnfs_curr_ld && !force_mds)
		pg_ops = server->pnfs_curr_ld->pg_write_ops;
#endif
1433
	nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1434
			server->wsize, ioflags);
1435
}
1436
EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
L
Linus Torvalds 已提交
1437

1438 1439
void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
{
1440 1441
	struct nfs_pgio_mirror *mirror;

1442 1443 1444
	if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
		pgio->pg_ops->pg_cleanup(pgio);

1445
	pgio->pg_ops = &nfs_pgio_rw_ops;
1446 1447 1448 1449 1450

	nfs_pageio_stop_mirroring(pgio);

	mirror = &pgio->pg_mirrors[0];
	mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1451
}
1452
EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1453

L
Linus Torvalds 已提交
1454

1455 1456 1457 1458 1459 1460 1461
void nfs_commit_prepare(struct rpc_task *task, void *calldata)
{
	struct nfs_commit_data *data = calldata;

	NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
}

1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472
/*
 * Special version of should_remove_suid() that ignores capabilities.
 */
static int nfs_should_remove_suid(const struct inode *inode)
{
	umode_t mode = inode->i_mode;
	int kill = 0;

	/* suid always must be killed */
	if (unlikely(mode & S_ISUID))
		kill = ATTR_KILL_SUID;
1473

1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485
	/*
	 * sgid without any exec bits is just a mandatory locking mark; leave
	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
	 */
	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
		kill |= ATTR_KILL_SGID;

	if (unlikely(kill && S_ISREG(mode)))
		return kill;

	return 0;
}
1486

1487 1488 1489 1490 1491
static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
		struct nfs_fattr *fattr)
{
	struct nfs_pgio_args *argp = &hdr->args;
	struct nfs_pgio_res *resp = &hdr->res;
1492
	u64 size = argp->offset + resp->count;
1493 1494

	if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
1495 1496 1497
		fattr->size = size;
	if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
		fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
1498
		return;
1499 1500
	}
	if (size != fattr->size)
1501 1502 1503
		return;
	/* Set attribute barrier */
	nfs_fattr_set_barrier(fattr);
1504 1505
	/* ...and update size */
	fattr->valid |= NFS_ATTR_FATTR_SIZE;
1506 1507 1508 1509
}

void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
{
1510
	struct nfs_fattr *fattr = &hdr->fattr;
1511 1512 1513 1514 1515 1516 1517 1518 1519
	struct inode *inode = hdr->inode;

	spin_lock(&inode->i_lock);
	nfs_writeback_check_extend(hdr, fattr);
	nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
	spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);

L
Linus Torvalds 已提交
1520 1521 1522
/*
 * This function is called when the WRITE call is complete.
 */
1523 1524
static int nfs_writeback_done(struct rpc_task *task,
			      struct nfs_pgio_header *hdr,
1525
			      struct inode *inode)
L
Linus Torvalds 已提交
1526
{
1527
	int status;
L
Linus Torvalds 已提交
1528

1529 1530 1531 1532 1533 1534 1535
	/*
	 * ->write_done will attempt to use post-op attributes to detect
	 * conflicting writes by other clients.  A strict interpretation
	 * of close-to-open would allow us to continue caching even if
	 * another writer had changed the file, but some applications
	 * depend on tighter cache coherency when writing.
	 */
1536
	status = NFS_PROTO(inode)->write_done(task, hdr);
1537
	if (status != 0)
1538
		return status;
1539
	nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
1540

1541 1542
	if (hdr->res.verf->committed < hdr->args.stable &&
	    task->tk_status >= 0) {
L
Linus Torvalds 已提交
1543 1544 1545 1546 1547 1548 1549 1550 1551 1552
		/* We tried a write call, but the server did not
		 * commit data to stable storage even though we
		 * requested it.
		 * Note: There is a known bug in Tru64 < 5.0 in which
		 *	 the server reports NFS_DATA_SYNC, but performs
		 *	 NFS_FILE_SYNC. We therefore implement this checking
		 *	 as a dprintk() in order to avoid filling syslog.
		 */
		static unsigned long    complain;

1553
		/* Note this will print the MDS for a DS write */
L
Linus Torvalds 已提交
1554
		if (time_before(complain, jiffies)) {
1555
			dprintk("NFS:       faulty NFS server %s:"
L
Linus Torvalds 已提交
1556
				" (committed = %d) != (stable = %d)\n",
1557
				NFS_SERVER(inode)->nfs_client->cl_hostname,
1558
				hdr->res.verf->committed, hdr->args.stable);
L
Linus Torvalds 已提交
1559 1560 1561
			complain = jiffies + 300 * HZ;
		}
	}
1562 1563 1564 1565

	/* Deal with the suid/sgid bit corner case */
	if (nfs_should_remove_suid(inode))
		nfs_mark_for_revalidate(inode);
1566 1567 1568 1569 1570 1571
	return 0;
}

/*
 * This function is called when the WRITE call is complete.
 */
1572 1573
static void nfs_writeback_result(struct rpc_task *task,
				 struct nfs_pgio_header *hdr)
1574
{
1575 1576
	struct nfs_pgio_args	*argp = &hdr->args;
	struct nfs_pgio_res	*resp = &hdr->res;
1577 1578

	if (resp->count < argp->count) {
L
Linus Torvalds 已提交
1579 1580
		static unsigned long    complain;

1581
		/* This a short write! */
1582
		nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
1583

L
Linus Torvalds 已提交
1584
		/* Has the server at least made some progress? */
1585 1586 1587 1588 1589 1590
		if (resp->count == 0) {
			if (time_before(complain, jiffies)) {
				printk(KERN_WARNING
				       "NFS: Server wrote zero bytes, expected %u.\n",
				       argp->count);
				complain = jiffies + 300 * HZ;
L
Linus Torvalds 已提交
1591
			}
1592
			nfs_set_pgio_error(hdr, -EIO, argp->offset);
1593
			task->tk_status = -EIO;
1594
			return;
L
Linus Torvalds 已提交
1595
		}
1596 1597 1598 1599 1600 1601 1602

		/* For non rpc-based layout drivers, retry-through-MDS */
		if (!task->tk_ops) {
			hdr->pnfs_error = -EAGAIN;
			return;
		}

1603 1604 1605
		/* Was this an NFSv2 write or an NFSv3 stable write? */
		if (resp->verf->committed != NFS_UNSTABLE) {
			/* Resend from where the server left off */
1606
			hdr->mds_offset += resp->count;
1607 1608 1609 1610 1611 1612 1613 1614
			argp->offset += resp->count;
			argp->pgbase += resp->count;
			argp->count -= resp->count;
		} else {
			/* Resend as a stable write in order to avoid
			 * headaches in the case of a server crash.
			 */
			argp->stable = NFS_FILE_SYNC;
L
Linus Torvalds 已提交
1615
		}
1616
		rpc_restart_call_prepare(task);
L
Linus Torvalds 已提交
1617 1618 1619
	}
}

1620
static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
1621
{
1622 1623 1624
	return wait_on_atomic_t(&cinfo->rpcs_out,
			nfs_wait_atomic_killable, TASK_KILLABLE);
}
1625

1626 1627 1628
static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
{
	atomic_inc(&cinfo->rpcs_out);
1629 1630
}

1631
static void nfs_commit_end(struct nfs_mds_commit_info *cinfo)
1632
{
1633 1634
	if (atomic_dec_and_test(&cinfo->rpcs_out))
		wake_up_atomic_t(&cinfo->rpcs_out);
1635 1636
}

1637
void nfs_commitdata_release(struct nfs_commit_data *data)
L
Linus Torvalds 已提交
1638
{
1639 1640
	put_nfs_open_context(data->context);
	nfs_commit_free(data);
L
Linus Torvalds 已提交
1641
}
1642
EXPORT_SYMBOL_GPL(nfs_commitdata_release);
L
Linus Torvalds 已提交
1643

1644
int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1645
			const struct nfs_rpc_ops *nfs_ops,
1646
			const struct rpc_call_ops *call_ops,
1647
			int how, int flags)
L
Linus Torvalds 已提交
1648
{
1649
	struct rpc_task *task;
1650
	int priority = flush_task_priority(how);
1651 1652 1653
	struct rpc_message msg = {
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
1654
		.rpc_cred = data->cred,
1655
	};
1656
	struct rpc_task_setup task_setup_data = {
1657
		.task = &data->task,
1658
		.rpc_client = clnt,
1659
		.rpc_message = &msg,
1660
		.callback_ops = call_ops,
1661
		.callback_data = data,
1662
		.workqueue = nfsiod_workqueue,
1663
		.flags = RPC_TASK_ASYNC | flags,
1664
		.priority = priority,
1665
	};
1666
	/* Set up the initial task struct.  */
1667
	nfs_ops->commit_setup(data, &msg);
1668

1669
	dprintk("NFS: initiated commit call\n");
1670

1671 1672 1673
	nfs4_state_protect(NFS_SERVER(data->inode)->nfs_client,
		NFS_SP4_MACH_CRED_COMMIT, &task_setup_data.rpc_client, &msg);

1674 1675 1676 1677 1678 1679 1680 1681
	task = rpc_run_task(&task_setup_data);
	if (IS_ERR(task))
		return PTR_ERR(task);
	if (how & FLUSH_SYNC)
		rpc_wait_for_completion_task(task);
	rpc_put_task(task);
	return 0;
}
1682
EXPORT_SYMBOL_GPL(nfs_initiate_commit);
1683

1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
static loff_t nfs_get_lwb(struct list_head *head)
{
	loff_t lwb = 0;
	struct nfs_page *req;

	list_for_each_entry(req, head, wb_list)
		if (lwb < (req_offset(req) + req->wb_bytes))
			lwb = req_offset(req) + req->wb_bytes;

	return lwb;
}

1696 1697 1698
/*
 * Set up the argument/result storage required for the RPC call.
 */
1699
void nfs_init_commit(struct nfs_commit_data *data,
1700 1701 1702
		     struct list_head *head,
		     struct pnfs_layout_segment *lseg,
		     struct nfs_commit_info *cinfo)
1703 1704
{
	struct nfs_page *first = nfs_list_entry(head->next);
1705
	struct inode *inode = d_inode(first->wb_context->dentry);
L
Linus Torvalds 已提交
1706 1707 1708 1709 1710 1711 1712

	/* Set up the RPC argument and reply structs
	 * NB: take care not to mess about with data->commit et al. */

	list_splice_init(head, &data->pages);

	data->inode	  = inode;
1713
	data->cred	  = first->wb_context->cred;
1714
	data->lseg	  = lseg; /* reference transferred */
1715 1716 1717
	/* only set lwb for pnfs commit */
	if (lseg)
		data->lwb = nfs_get_lwb(&data->pages);
1718
	data->mds_ops     = &nfs_commit_ops;
1719
	data->completion_ops = cinfo->completion_ops;
1720
	data->dreq	  = cinfo->dreq;
L
Linus Torvalds 已提交
1721 1722

	data->args.fh     = NFS_FH(data->inode);
1723 1724 1725
	/* Note: we always request a commit of the entire inode */
	data->args.offset = 0;
	data->args.count  = 0;
1726
	data->context     = get_nfs_open_context(first->wb_context);
L
Linus Torvalds 已提交
1727 1728
	data->res.fattr   = &data->fattr;
	data->res.verf    = &data->verf;
1729
	nfs_fattr_init(&data->fattr);
L
Linus Torvalds 已提交
1730
}
1731
EXPORT_SYMBOL_GPL(nfs_init_commit);
L
Linus Torvalds 已提交
1732

1733
void nfs_retry_commit(struct list_head *page_list,
1734
		      struct pnfs_layout_segment *lseg,
1735 1736
		      struct nfs_commit_info *cinfo,
		      u32 ds_commit_idx)
1737 1738 1739 1740 1741 1742
{
	struct nfs_page *req;

	while (!list_empty(page_list)) {
		req = nfs_list_entry(page_list->next);
		nfs_list_remove_request(req);
1743
		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
1744 1745
		if (!cinfo->dreq)
			nfs_clear_page_commit(req->wb_page);
1746
		nfs_unlock_and_release_request(req);
1747 1748
	}
}
1749
EXPORT_SYMBOL_GPL(nfs_retry_commit);
1750

1751 1752 1753 1754 1755 1756 1757
static void
nfs_commit_resched_write(struct nfs_commit_info *cinfo,
		struct nfs_page *req)
{
	__set_page_dirty_nobuffers(req->wb_page);
}

L
Linus Torvalds 已提交
1758 1759 1760 1761
/*
 * Commit dirty pages
 */
static int
1762 1763
nfs_commit_list(struct inode *inode, struct list_head *head, int how,
		struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
1764
{
1765
	struct nfs_commit_data	*data;
L
Linus Torvalds 已提交
1766

1767 1768 1769 1770
	/* another commit raced with us */
	if (list_empty(head))
		return 0;

N
NeilBrown 已提交
1771
	data = nfs_commitdata_alloc(true);
L
Linus Torvalds 已提交
1772 1773

	/* Set up the argument struct */
1774 1775
	nfs_init_commit(data, head, NULL, cinfo);
	atomic_inc(&cinfo->mds->rpcs_out);
1776 1777
	return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
				   data->mds_ops, how, 0);
1778 1779
}

L
Linus Torvalds 已提交
1780 1781 1782
/*
 * COMMIT call returned
 */
1783
static void nfs_commit_done(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
1784
{
1785
	struct nfs_commit_data	*data = calldata;
L
Linus Torvalds 已提交
1786

1787
        dprintk("NFS: %5u nfs_commit_done (status %d)\n",
L
Linus Torvalds 已提交
1788 1789
                                task->tk_pid, task->tk_status);

1790
	/* Call the NFS version-specific code */
1791
	NFS_PROTO(data->inode)->commit_done(task, data);
1792 1793
}

1794
static void nfs_commit_release_pages(struct nfs_commit_data *data)
1795
{
1796
	struct nfs_page	*req;
1797
	int status = data->task.tk_status;
1798
	struct nfs_commit_info cinfo;
1799
	struct nfs_server *nfss;
1800

L
Linus Torvalds 已提交
1801 1802 1803
	while (!list_empty(&data->pages)) {
		req = nfs_list_entry(data->pages.next);
		nfs_list_remove_request(req);
1804 1805
		if (req->wb_page)
			nfs_clear_page_commit(req->wb_page);
L
Linus Torvalds 已提交
1806

1807
		dprintk("NFS:       commit (%s/%llu %d@%lld)",
1808
			req->wb_context->dentry->d_sb->s_id,
1809
			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),
L
Linus Torvalds 已提交
1810 1811
			req->wb_bytes,
			(long long)req_offset(req));
1812 1813
		if (status < 0) {
			nfs_context_set_write_error(req->wb_context, status);
1814 1815
			if (req->wb_page)
				nfs_inode_remove_request(req);
1816
			dprintk_cont(", error = %d\n", status);
L
Linus Torvalds 已提交
1817 1818 1819 1820 1821
			goto next;
		}

		/* Okay, COMMIT succeeded, apparently. Check the verifier
		 * returned by the server against all stored verfs. */
1822
		if (!nfs_write_verifier_cmp(&req->wb_verf, &data->verf.verifier)) {
L
Linus Torvalds 已提交
1823
			/* We have a match */
1824 1825
			if (req->wb_page)
				nfs_inode_remove_request(req);
1826
			dprintk_cont(" OK\n");
L
Linus Torvalds 已提交
1827 1828 1829
			goto next;
		}
		/* We have a mismatch. Write the page again */
1830
		dprintk_cont(" mismatch\n");
F
Fred 已提交
1831
		nfs_mark_request_dirty(req);
1832
		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
L
Linus Torvalds 已提交
1833
	next:
1834
		nfs_unlock_and_release_request(req);
L
Linus Torvalds 已提交
1835
	}
1836 1837
	nfss = NFS_SERVER(data->inode);
	if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
1838
		clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC);
1839

1840
	nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1841
	nfs_commit_end(cinfo.mds);
1842 1843 1844 1845
}

static void nfs_commit_release(void *calldata)
{
1846
	struct nfs_commit_data *data = calldata;
1847

1848
	data->completion_ops->completion(data);
1849
	nfs_commitdata_release(calldata);
L
Linus Torvalds 已提交
1850
}
1851 1852

static const struct rpc_call_ops nfs_commit_ops = {
1853
	.rpc_call_prepare = nfs_commit_prepare,
1854 1855 1856
	.rpc_call_done = nfs_commit_done,
	.rpc_release = nfs_commit_release,
};
L
Linus Torvalds 已提交
1857

1858 1859
static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
	.completion = nfs_commit_release_pages,
1860
	.resched_write = nfs_commit_resched_write,
1861 1862
};

1863 1864
int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
			    int how, struct nfs_commit_info *cinfo)
1865 1866 1867
{
	int status;

1868
	status = pnfs_commit_list(inode, head, how, cinfo);
1869
	if (status == PNFS_NOT_ATTEMPTED)
1870
		status = nfs_commit_list(inode, head, how, cinfo);
1871 1872 1873
	return status;
}

1874
int nfs_commit_inode(struct inode *inode, int how)
L
Linus Torvalds 已提交
1875 1876
{
	LIST_HEAD(head);
1877
	struct nfs_commit_info cinfo;
1878
	int may_wait = how & FLUSH_SYNC;
1879
	int error = 0;
1880
	int res;
L
Linus Torvalds 已提交
1881

1882
	nfs_init_cinfo_from_inode(&cinfo, inode);
1883
	nfs_commit_begin(cinfo.mds);
1884
	res = nfs_scan_commit(inode, &head, &cinfo);
1885
	if (res)
1886
		error = nfs_generic_commit_list(inode, &head, how, &cinfo);
1887 1888 1889 1890 1891 1892 1893 1894
	nfs_commit_end(cinfo.mds);
	if (error < 0)
		goto out_error;
	if (!may_wait)
		goto out_mark_dirty;
	error = wait_on_commit(cinfo.mds);
	if (error < 0)
		return error;
1895
	return res;
1896 1897
out_error:
	res = error;
1898 1899 1900 1901 1902 1903 1904
	/* Note: If we exit without ensuring that the commit is complete,
	 * we must mark the inode as dirty. Otherwise, future calls to
	 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure
	 * that the data is on the disk.
	 */
out_mark_dirty:
	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
L
Linus Torvalds 已提交
1905 1906
	return res;
}
1907
EXPORT_SYMBOL_GPL(nfs_commit_inode);
1908

1909
int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1910
{
1911 1912 1913
	struct nfs_inode *nfsi = NFS_I(inode);
	int flags = FLUSH_SYNC;
	int ret = 0;
1914

1915
	/* no commits means nothing needs to be done */
1916
	if (!atomic_long_read(&nfsi->commit_info.ncommit))
1917 1918
		return ret;

1919 1920 1921 1922
	if (wbc->sync_mode == WB_SYNC_NONE) {
		/* Don't commit yet if this is a non-blocking flush and there
		 * are a lot of outstanding writes for this mapping.
		 */
1923
		if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
1924
			goto out_mark_dirty;
1925

1926
		/* don't wait for the COMMIT response */
1927
		flags = 0;
1928 1929
	}

1930 1931 1932 1933 1934 1935 1936 1937
	ret = nfs_commit_inode(inode, flags);
	if (ret >= 0) {
		if (wbc->sync_mode == WB_SYNC_NONE) {
			if (ret < wbc->nr_to_write)
				wbc->nr_to_write -= ret;
			else
				wbc->nr_to_write = 0;
		}
1938
		return 0;
1939 1940
	}
out_mark_dirty:
1941 1942 1943
	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
	return ret;
}
1944
EXPORT_SYMBOL_GPL(nfs_write_inode);
1945

1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963
/*
 * Wrapper for filemap_write_and_wait_range()
 *
 * Needed for pNFS in order to ensure data becomes visible to the
 * client.
 */
int nfs_filemap_write_and_wait_range(struct address_space *mapping,
		loff_t lstart, loff_t lend)
{
	int ret;

	ret = filemap_write_and_wait_range(mapping, lstart, lend);
	if (ret == 0)
		ret = pnfs_sync_inode(mapping->host, true);
	return ret;
}
EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);

1964 1965 1966 1967
/*
 * flush the inode to disk.
 */
int nfs_wb_all(struct inode *inode)
1968
{
1969 1970 1971 1972
	int ret;

	trace_nfs_writeback_inode_enter(inode);

1973
	ret = filemap_write_and_wait(inode->i_mapping);
1974 1975 1976 1977 1978 1979 1980
	if (ret)
		goto out;
	ret = nfs_commit_inode(inode, FLUSH_SYNC);
	if (ret < 0)
		goto out;
	pnfs_sync_inode(inode, true);
	ret = 0;
1981

1982
out:
1983 1984
	trace_nfs_writeback_inode_exit(inode, ret);
	return ret;
1985
}
1986
EXPORT_SYMBOL_GPL(nfs_wb_all);
1987

1988 1989 1990 1991 1992
int nfs_wb_page_cancel(struct inode *inode, struct page *page)
{
	struct nfs_page *req;
	int ret = 0;

1993 1994 1995 1996
	wait_on_page_writeback(page);

	/* blocking call to cancel all requests and join to a single (head)
	 * request */
1997
	req = nfs_lock_and_join_requests(page);
1998 1999 2000 2001 2002 2003 2004 2005 2006 2007

	if (IS_ERR(req)) {
		ret = PTR_ERR(req);
	} else if (req) {
		/* all requests from this page have been cancelled by
		 * nfs_lock_and_join_requests, so just remove the head
		 * request from the inode / page_private pointer and
		 * release it */
		nfs_inode_remove_request(req);
		nfs_unlock_and_release_request(req);
2008
	}
2009

2010 2011 2012
	return ret;
}

2013 2014 2015
/*
 * Write back all requests on one page - we do this before reading it.
 */
2016
int nfs_wb_page(struct inode *inode, struct page *page)
2017
{
2018
	loff_t range_start = page_file_offset(page);
2019
	loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1);
2020 2021
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_ALL,
2022
		.nr_to_write = 0,
2023 2024 2025 2026
		.range_start = range_start,
		.range_end = range_end,
	};
	int ret;
2027

2028 2029
	trace_nfs_writeback_page_enter(inode);

2030
	for (;;) {
2031
		wait_on_page_writeback(page);
2032
		if (clear_page_dirty_for_io(page)) {
2033
			ret = nfs_writepage_locked(page, &wbc);
2034 2035
			if (ret < 0)
				goto out_error;
2036
			continue;
2037
		}
2038
		ret = 0;
2039 2040 2041
		if (!PagePrivate(page))
			break;
		ret = nfs_commit_inode(inode, FLUSH_SYNC);
2042
		if (ret < 0)
2043
			goto out_error;
2044
	}
2045
out_error:
2046
	trace_nfs_writeback_page_exit(inode, ret);
2047
	return ret;
2048 2049
}

2050 2051
#ifdef CONFIG_MIGRATION
int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
2052
		struct page *page, enum migrate_mode mode)
2053
{
2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
	/*
	 * If PagePrivate is set, then the page is currently associated with
	 * an in-progress read or write request. Don't try to migrate it.
	 *
	 * FIXME: we could do this in principle, but we'll need a way to ensure
	 *        that we can safely release the inode reference while holding
	 *        the page lock.
	 */
	if (PagePrivate(page))
		return -EBUSY;
2064

2065 2066
	if (!nfs_fscache_release_page(page, GFP_KERNEL))
		return -EBUSY;
2067

2068
	return migrate_page(mapping, newpage, page, mode);
2069 2070 2071
}
#endif

D
David Howells 已提交
2072
int __init nfs_init_writepagecache(void)
L
Linus Torvalds 已提交
2073 2074
{
	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
2075
					     sizeof(struct nfs_pgio_header),
L
Linus Torvalds 已提交
2076
					     0, SLAB_HWCACHE_ALIGN,
2077
					     NULL);
L
Linus Torvalds 已提交
2078 2079 2080
	if (nfs_wdata_cachep == NULL)
		return -ENOMEM;

2081 2082
	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
						     nfs_wdata_cachep);
L
Linus Torvalds 已提交
2083
	if (nfs_wdata_mempool == NULL)
2084
		goto out_destroy_write_cache;
L
Linus Torvalds 已提交
2085

2086 2087 2088 2089 2090
	nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
					     sizeof(struct nfs_commit_data),
					     0, SLAB_HWCACHE_ALIGN,
					     NULL);
	if (nfs_cdata_cachep == NULL)
2091
		goto out_destroy_write_mempool;
2092

2093
	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
2094
						      nfs_cdata_cachep);
L
Linus Torvalds 已提交
2095
	if (nfs_commit_mempool == NULL)
2096
		goto out_destroy_commit_cache;
L
Linus Torvalds 已提交
2097

2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117
	/*
	 * NFS congestion size, scale with available memory.
	 *
	 *  64MB:    8192k
	 * 128MB:   11585k
	 * 256MB:   16384k
	 * 512MB:   23170k
	 *   1GB:   32768k
	 *   2GB:   46340k
	 *   4GB:   65536k
	 *   8GB:   92681k
	 *  16GB:  131072k
	 *
	 * This allows larger machines to have larger/more transfers.
	 * Limit the default to 256M
	 */
	nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
	if (nfs_congestion_kb > 256*1024)
		nfs_congestion_kb = 256*1024;

L
Linus Torvalds 已提交
2118
	return 0;
2119 2120 2121 2122 2123 2124 2125 2126

out_destroy_commit_cache:
	kmem_cache_destroy(nfs_cdata_cachep);
out_destroy_write_mempool:
	mempool_destroy(nfs_wdata_mempool);
out_destroy_write_cache:
	kmem_cache_destroy(nfs_wdata_cachep);
	return -ENOMEM;
L
Linus Torvalds 已提交
2127 2128
}

2129
void nfs_destroy_writepagecache(void)
L
Linus Torvalds 已提交
2130 2131
{
	mempool_destroy(nfs_commit_mempool);
2132
	kmem_cache_destroy(nfs_cdata_cachep);
L
Linus Torvalds 已提交
2133
	mempool_destroy(nfs_wdata_mempool);
2134
	kmem_cache_destroy(nfs_wdata_cachep);
L
Linus Torvalds 已提交
2135 2136
}

2137 2138 2139
static const struct nfs_rw_ops nfs_rw_write_ops = {
	.rw_alloc_header	= nfs_writehdr_alloc,
	.rw_free_header		= nfs_writehdr_free,
2140 2141
	.rw_done		= nfs_writeback_done,
	.rw_result		= nfs_writeback_result,
2142
	.rw_initiate		= nfs_initiate_write,
2143
};
反馈
建议
客服 返回
顶部