write.c 56.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * linux/fs/nfs/write.c
 *
4
 * Write file data over NFS.
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14
 *
 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/writeback.h>
15
#include <linux/swap.h>
16
#include <linux/migrate.h>
L
Linus Torvalds 已提交
17 18 19 20 21

#include <linux/sunrpc/clnt.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/nfs_page.h>
22
#include <linux/backing-dev.h>
23
#include <linux/export.h>
24 25
#include <linux/freezer.h>
#include <linux/wait.h>
J
Jeff Layton 已提交
26
#include <linux/iversion.h>
27

28
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
29 30

#include "delegation.h"
31
#include "internal.h"
C
Chuck Lever 已提交
32
#include "iostat.h"
33
#include "nfs4_fs.h"
34
#include "fscache.h"
35
#include "pnfs.h"
L
Linus Torvalds 已提交
36

37 38
#include "nfstrace.h"

L
Linus Torvalds 已提交
39 40 41 42 43
#define NFSDBG_FACILITY		NFSDBG_PAGECACHE

#define MIN_POOL_WRITE		(32)
#define MIN_POOL_COMMIT		(4)

44 45 46 47 48 49
struct nfs_io_completion {
	void (*complete)(void *data);
	void *data;
	struct kref refcount;
};

L
Linus Torvalds 已提交
50 51 52
/*
 * Local function declarations
 */
53
static void nfs_redirty_request(struct nfs_page *req);
54
static const struct rpc_call_ops nfs_commit_ops;
55
static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
56
static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
57
static const struct nfs_rw_ops nfs_rw_write_ops;
58
static void nfs_clear_request_commit(struct nfs_page *req);
59 60
static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
				      struct inode *inode);
61 62 63
static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
						struct page *page);
L
Linus Torvalds 已提交
64

65
static struct kmem_cache *nfs_wdata_cachep;
66
static mempool_t *nfs_wdata_mempool;
67
static struct kmem_cache *nfs_cdata_cachep;
L
Linus Torvalds 已提交
68 69
static mempool_t *nfs_commit_mempool;

N
NeilBrown 已提交
70
struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail)
L
Linus Torvalds 已提交
71
{
N
NeilBrown 已提交
72
	struct nfs_commit_data *p;
73

N
NeilBrown 已提交
74 75 76 77 78 79 80 81 82 83 84 85 86 87
	if (never_fail)
		p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
	else {
		/* It is OK to do some reclaim, not no safe to wait
		 * for anything to be returned to the pool.
		 * mempool_alloc() cannot handle that particular combination,
		 * so we need two separate attempts.
		 */
		p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
		if (!p)
			p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO |
					     __GFP_NOWARN | __GFP_NORETRY);
		if (!p)
			return NULL;
L
Linus Torvalds 已提交
88
	}
N
NeilBrown 已提交
89 90 91

	memset(p, 0, sizeof(*p));
	INIT_LIST_HEAD(&p->pages);
L
Linus Torvalds 已提交
92 93
	return p;
}
94
EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
L
Linus Torvalds 已提交
95

96
void nfs_commit_free(struct nfs_commit_data *p)
L
Linus Torvalds 已提交
97 98 99
{
	mempool_free(p, nfs_commit_mempool);
}
100
EXPORT_SYMBOL_GPL(nfs_commit_free);
L
Linus Torvalds 已提交
101

102
static struct nfs_pgio_header *nfs_writehdr_alloc(void)
103
{
104
	struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
105

106 107
	memset(p, 0, sizeof(*p));
	p->rw_mode = FMODE_WRITE;
108 109
	return p;
}
110

111
static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
112
{
113
	mempool_free(hdr, nfs_wdata_mempool);
114
}
L
Linus Torvalds 已提交
115

116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
{
	return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
}

static void nfs_io_completion_init(struct nfs_io_completion *ioc,
		void (*complete)(void *), void *data)
{
	ioc->complete = complete;
	ioc->data = data;
	kref_init(&ioc->refcount);
}

static void nfs_io_completion_release(struct kref *kref)
{
	struct nfs_io_completion *ioc = container_of(kref,
			struct nfs_io_completion, refcount);
	ioc->complete(ioc->data);
	kfree(ioc);
}

static void nfs_io_completion_get(struct nfs_io_completion *ioc)
{
	if (ioc != NULL)
		kref_get(&ioc->refcount);
}

static void nfs_io_completion_put(struct nfs_io_completion *ioc)
{
	if (ioc != NULL)
		kref_put(&ioc->refcount, nfs_io_completion_release);
}

149 150 151 152 153 154 155 156
static struct nfs_page *
nfs_page_private_request(struct page *page)
{
	if (!PagePrivate(page))
		return NULL;
	return (struct nfs_page *)page_private(page);
}

157 158 159 160 161 162 163
/*
 * nfs_page_find_head_request_locked - find head request associated with @page
 *
 * must be called while holding the inode lock.
 *
 * returns matching head request with reference held, or NULL if not found.
 */
164
static struct nfs_page *
165
nfs_page_find_private_request(struct page *page)
166
{
167
	struct address_space *mapping = page_file_mapping(page);
168
	struct nfs_page *req;
169

170 171
	if (!PagePrivate(page))
		return NULL;
172
	spin_lock(&mapping->private_lock);
173
	req = nfs_page_private_request(page);
174 175
	if (req) {
		WARN_ON_ONCE(req->wb_head != req);
176
		kref_get(&req->wb_kref);
177
	}
178
	spin_unlock(&mapping->private_lock);
179 180
	return req;
}
181

182 183 184 185 186 187 188 189
static struct nfs_page *
nfs_page_find_swap_request(struct page *page)
{
	struct inode *inode = page_file_mapping(page)->host;
	struct nfs_inode *nfsi = NFS_I(inode);
	struct nfs_page *req = NULL;
	if (!PageSwapCache(page))
		return NULL;
190
	mutex_lock(&nfsi->commit_mutex);
191 192 193 194 195 196 197 198
	if (PageSwapCache(page)) {
		req = nfs_page_search_commits_for_head_request_locked(nfsi,
			page);
		if (req) {
			WARN_ON_ONCE(req->wb_head != req);
			kref_get(&req->wb_kref);
		}
	}
199
	mutex_unlock(&nfsi->commit_mutex);
200 201 202
	return req;
}

203 204 205 206 207 208
/*
 * nfs_page_find_head_request - find head request associated with @page
 *
 * returns matching head request with reference held, or NULL if not found.
 */
static struct nfs_page *nfs_page_find_head_request(struct page *page)
209
{
210
	struct nfs_page *req;
211

212 213 214
	req = nfs_page_find_private_request(page);
	if (!req)
		req = nfs_page_find_swap_request(page);
215 216 217
	return req;
}

L
Linus Torvalds 已提交
218 219 220
/* Adjust the file length if we're writing beyond the end */
static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
{
221
	struct inode *inode = page_file_mapping(page)->host;
222 223
	loff_t end, i_size;
	pgoff_t end_index;
L
Linus Torvalds 已提交
224

225 226
	spin_lock(&inode->i_lock);
	i_size = i_size_read(inode);
227
	end_index = (i_size - 1) >> PAGE_SHIFT;
H
Huang Ying 已提交
228
	if (i_size > 0 && page_index(page) < end_index)
229
		goto out;
230
	end = page_file_offset(page) + ((loff_t)offset+count);
L
Linus Torvalds 已提交
231
	if (i_size >= end)
232
		goto out;
L
Linus Torvalds 已提交
233
	i_size_write(inode, end);
234
	NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
235 236 237
	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
out:
	spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
238 239
}

240
/* A writeback failed: mark the page as bad, and invalidate the page cache */
241
static void nfs_set_pageerror(struct address_space *mapping)
242
{
243
	nfs_zap_mapping(mapping->host, mapping);
244 245
}

246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
/*
 * nfs_page_group_search_locked
 * @head - head request of page group
 * @page_offset - offset into page
 *
 * Search page group with head @head to find a request that contains the
 * page offset @page_offset.
 *
 * Returns a pointer to the first matching nfs request, or NULL if no
 * match is found.
 *
 * Must be called with the page group lock held
 */
static struct nfs_page *
nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
{
	struct nfs_page *req;

	req = head;
	do {
		if (page_offset >= req->wb_pgbase &&
		    page_offset < (req->wb_pgbase + req->wb_bytes))
			return req;

		req = req->wb_this_page;
	} while (req != head);

	return NULL;
}

/*
 * nfs_page_group_covers_page
 * @head - head request of page group
 *
 * Return true if the page group with head @head covers the whole page,
 * returns false otherwise
 */
static bool nfs_page_group_covers_page(struct nfs_page *req)
{
	struct nfs_page *tmp;
	unsigned int pos = 0;
	unsigned int len = nfs_page_length(req->wb_page);

289
	nfs_page_group_lock(req);
290

291
	for (;;) {
292
		tmp = nfs_page_group_search_locked(req->wb_head, pos);
293 294 295 296
		if (!tmp)
			break;
		pos = tmp->wb_pgbase + tmp->wb_bytes;
	}
297 298

	nfs_page_group_unlock(req);
299
	return pos >= len;
300 301
}

L
Linus Torvalds 已提交
302 303 304
/* We can set the PG_uptodate flag if we see that a write request
 * covers the full page.
 */
305
static void nfs_mark_uptodate(struct nfs_page *req)
L
Linus Torvalds 已提交
306
{
307
	if (PageUptodate(req->wb_page))
L
Linus Torvalds 已提交
308
		return;
309
	if (!nfs_page_group_covers_page(req))
L
Linus Torvalds 已提交
310
		return;
311
	SetPageUptodate(req->wb_page);
L
Linus Torvalds 已提交
312 313 314 315
}

static int wb_priority(struct writeback_control *wbc)
{
316
	int ret = 0;
317

318 319 320
	if (wbc->sync_mode == WB_SYNC_ALL)
		ret = FLUSH_COND_STABLE;
	return ret;
L
Linus Torvalds 已提交
321 322
}

323 324 325 326 327 328 329 330 331 332
/*
 * NFS congestion control
 */

int nfs_congestion_kb;

#define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
#define NFS_CONGESTION_OFF_THRESH	\
	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))

333
static void nfs_set_page_writeback(struct page *page)
334
{
335 336
	struct inode *inode = page_file_mapping(page)->host;
	struct nfs_server *nfss = NFS_SERVER(inode);
337 338
	int ret = test_set_page_writeback(page);

339
	WARN_ON_ONCE(ret != 0);
340

341
	if (atomic_long_inc_return(&nfss->writeback) >
342 343
			NFS_CONGESTION_ON_THRESH)
		set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
344 345
}

346
static void nfs_end_page_writeback(struct nfs_page *req)
347
{
348
	struct inode *inode = page_file_mapping(req->wb_page)->host;
349
	struct nfs_server *nfss = NFS_SERVER(inode);
350
	bool is_done;
351

352 353 354
	is_done = nfs_page_group_sync_on_bit(req, PG_WB_END);
	nfs_unlock_request(req);
	if (!is_done)
355 356 357
		return;

	end_page_writeback(req->wb_page);
P
Peter Zijlstra 已提交
358
	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
359
		clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
360 361
}

362 363 364 365 366 367 368 369 370
/*
 * nfs_unroll_locks_and_wait -  unlock all newly locked reqs and wait on @req
 *
 * this is a helper function for nfs_lock_and_join_requests
 *
 * @inode - inode associated with request page group, must be holding inode lock
 * @head  - head request of page group, must be holding head lock
 * @req   - request that couldn't lock and needs to wait on the req bit lock
 *
371 372
 * NOTE: this must be called holding page_group bit lock
 *       which will be released before returning.
373 374 375
 *
 * returns 0 on success, < 0 on error.
 */
376 377
static void
nfs_unroll_locks(struct inode *inode, struct nfs_page *head,
T
Trond Myklebust 已提交
378
			  struct nfs_page *req)
379 380 381 382
{
	struct nfs_page *tmp;

	/* relinquish all the locks successfully grabbed this run */
383 384 385 386 387
	for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
		if (!kref_read(&tmp->wb_kref))
			continue;
		nfs_unlock_and_release_request(tmp);
	}
388 389 390 391 392 393 394 395 396 397 398 399 400 401
}

/*
 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
 *
 * @destroy_list - request list (using wb_this_page) terminated by @old_head
 * @old_head - the old head of the list
 *
 * All subrequests must be locked and removed from all lists, so at this point
 * they are only "active" in this function, and possibly in nfs_wait_on_request
 * with a reference held by some other context.
 */
static void
nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
402 403
				 struct nfs_page *old_head,
				 struct inode *inode)
404 405 406 407 408 409 410 411 412 413 414 415
{
	while (destroy_list) {
		struct nfs_page *subreq = destroy_list;

		destroy_list = (subreq->wb_this_page == old_head) ?
				   NULL : subreq->wb_this_page;

		WARN_ON_ONCE(old_head != subreq->wb_head);

		/* make sure old group is not used */
		subreq->wb_this_page = subreq;

416 417
		clear_bit(PG_REMOVE, &subreq->wb_flags);

418 419 420
		/* Note: races with nfs_page_group_destroy() */
		if (!kref_read(&subreq->wb_kref)) {
			/* Check if we raced with nfs_page_group_destroy() */
421
			if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags))
422 423 424
				nfs_free_request(subreq);
			continue;
		}
425

426 427 428 429
		subreq->wb_head = subreq;

		if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) {
			nfs_release_request(subreq);
430
			atomic_long_dec(&NFS_I(inode)->nrequests);
431
		}
432 433 434 435

		/* subreq is now totally disconnected from page group or any
		 * write / commit lists. last chance to wake any waiters */
		nfs_unlock_and_release_request(subreq);
436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
	}
}

/*
 * nfs_lock_and_join_requests - join all subreqs to the head req and return
 *                              a locked reference, cancelling any pending
 *                              operations for this page.
 *
 * @page - the page used to lookup the "page group" of nfs_page structures
 *
 * This function joins all sub requests to the head request by first
 * locking all requests in the group, cancelling any pending operations
 * and finally updating the head request to cover the whole range covered by
 * the (former) group.  All subrequests are removed from any write or commit
 * lists, unlinked from the group and destroyed.
 *
 * Returns a locked, referenced pointer to the head request - which after
 * this call is guaranteed to be the only request associated with the page.
 * Returns NULL if no requests are found for @page, or a ERR_PTR if an
 * error was encountered.
 */
static struct nfs_page *
T
Trond Myklebust 已提交
458
nfs_lock_and_join_requests(struct page *page)
459
{
460
	struct inode *inode = page_file_mapping(page)->host;
461 462 463
	struct nfs_page *head, *subreq;
	struct nfs_page *destroy_list = NULL;
	unsigned int total_bytes;
464 465
	int ret;

466 467 468 469 470 471
try_again:
	/*
	 * A reference is taken only on the head request which acts as a
	 * reference to the whole page group - the group will not be destroyed
	 * until the head reference is released.
	 */
472 473
	head = nfs_page_find_head_request(page);
	if (!head)
474 475
		return NULL;

476 477 478 479 480 481 482 483
	/* lock the page head first in order to avoid an ABBA inefficiency */
	if (!nfs_lock_request(head)) {
		ret = nfs_wait_on_request(head);
		nfs_release_request(head);
		if (ret < 0)
			return ERR_PTR(ret);
		goto try_again;
	}
484 485 486 487 488 489

	/* Ensure that nobody removed the request before we locked it */
	if (head != nfs_page_private_request(page) && !PageSwapCache(page)) {
		nfs_unlock_and_release_request(head);
		goto try_again;
	}
490

491
	ret = nfs_page_group_lock(head);
492 493
	if (ret < 0)
		goto release_request;
494 495

	/* lock each request in the page group */
496 497 498
	total_bytes = head->wb_bytes;
	for (subreq = head->wb_this_page; subreq != head;
			subreq = subreq->wb_this_page) {
499

500 501 502
		if (!kref_get_unless_zero(&subreq->wb_kref)) {
			if (subreq->wb_offset == head->wb_offset + total_bytes)
				total_bytes += subreq->wb_bytes;
503
			continue;
504 505
		}

506
		while (!nfs_lock_request(subreq)) {
507
			/*
508 509
			 * Unlock page to allow nfs_page_group_sync_on_bit()
			 * to succeed
510
			 */
511 512 513
			nfs_page_group_unlock(head);
			ret = nfs_wait_on_request(subreq);
			if (!ret)
514
				ret = nfs_page_group_lock(head);
515 516
			if (ret < 0) {
				nfs_unroll_locks(inode, head, subreq);
517
				nfs_release_request(subreq);
518
				goto release_request;
519
			}
520
		}
521 522
		/*
		 * Subrequests are always contiguous, non overlapping
523
		 * and in order - but may be repeated (mirrored writes).
524
		 */
525 526 527 528 529 530
		if (subreq->wb_offset == (head->wb_offset + total_bytes)) {
			/* keep track of how many bytes this group covers */
			total_bytes += subreq->wb_bytes;
		} else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset ||
			    ((subreq->wb_offset + subreq->wb_bytes) >
			     (head->wb_offset + total_bytes)))) {
531
			nfs_page_group_unlock(head);
532
			nfs_unroll_locks(inode, head, subreq);
533
			nfs_unlock_and_release_request(subreq);
534 535
			ret = -EIO;
			goto release_request;
536
		}
537
	}
538 539 540 541 542

	/* Now that all requests are locked, make sure they aren't on any list.
	 * Commit list removal accounting is done after locks are dropped */
	subreq = head;
	do {
543
		nfs_clear_request_commit(subreq);
544 545 546 547 548 549 550 551 552 553 554 555
		subreq = subreq->wb_this_page;
	} while (subreq != head);

	/* unlink subrequests from head, destroy them later */
	if (head->wb_this_page != head) {
		/* destroy list will be terminated by head */
		destroy_list = head->wb_this_page;
		head->wb_this_page = head;

		/* change head request to cover whole range that
		 * the former page group covered */
		head->wb_bytes = total_bytes;
556
	}
557

558 559 560 561
	/* Postpone destruction of this request */
	if (test_and_clear_bit(PG_REMOVE, &head->wb_flags)) {
		set_bit(PG_INODE_REF, &head->wb_flags);
		kref_get(&head->wb_kref);
562
		atomic_long_inc(&NFS_I(inode)->nrequests);
563 564
	}

565 566
	nfs_page_group_unlock(head);

567
	nfs_destroy_unlinked_subrequests(destroy_list, head, inode);
568

569 570 571 572 573 574
	/* Did we lose a race with nfs_inode_remove_request()? */
	if (!(PagePrivate(page) || PageSwapCache(page))) {
		nfs_unlock_and_release_request(head);
		return NULL;
	}

575
	/* still holds ref on head from nfs_page_find_head_request
576 577
	 * and still has lock on head from lock loop */
	return head;
578 579 580 581

release_request:
	nfs_unlock_and_release_request(head);
	return ERR_PTR(ret);
582 583
}

584 585 586 587 588
static void nfs_write_error_remove_page(struct nfs_page *req)
{
	nfs_end_page_writeback(req);
	generic_error_remove_page(page_file_mapping(req->wb_page),
				  req->wb_page);
589
	nfs_release_request(req);
590 591
}

592 593 594 595 596 597 598 599 600 601
static bool
nfs_error_is_fatal_on_server(int err)
{
	switch (err) {
	case 0:
	case -ERESTARTSYS:
	case -EINTR:
		return false;
	}
	return nfs_error_is_fatal(err);
602 603
}

604 605 606 607 608
/*
 * Find an associated nfs write request, and prepare to flush it out
 * May return an error if the user signalled nfs_wait_on_request().
 */
static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
T
Trond Myklebust 已提交
609
				struct page *page)
610 611 612 613
{
	struct nfs_page *req;
	int ret = 0;

T
Trond Myklebust 已提交
614
	req = nfs_lock_and_join_requests(page);
615 616 617 618 619 620
	if (!req)
		goto out;
	ret = PTR_ERR(req);
	if (IS_ERR(req))
		goto out;

621 622
	nfs_set_page_writeback(page);
	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
623

624
	ret = req->wb_context->error;
625
	/* If there is a fatal error that covers this write, just exit */
626
	if (nfs_error_is_fatal_on_server(ret))
627 628
		goto out_launder;

629
	ret = 0;
630
	if (!nfs_pageio_add_request(pgio, req)) {
631
		ret = pgio->pg_error;
632
		/*
633
		 * Remove the problematic req upon fatal errors on the server
634 635 636
		 */
		if (nfs_error_is_fatal(ret)) {
			nfs_context_set_write_error(req->wb_context, ret);
637
			if (nfs_error_is_fatal_on_server(ret))
638
				goto out_launder;
639 640
		} else
			ret = -EAGAIN;
641
		nfs_redirty_request(req);
642 643 644
	} else
		nfs_add_stats(page_file_mapping(page)->host,
				NFSIOS_WRITEPAGES, 1);
645 646
out:
	return ret;
647 648 649
out_launder:
	nfs_write_error_remove_page(req);
	return ret;
650 651
}

652
static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
653
			    struct nfs_pageio_descriptor *pgio)
L
Linus Torvalds 已提交
654
{
655
	int ret;
L
Linus Torvalds 已提交
656

H
Huang Ying 已提交
657
	nfs_pageio_cond_complete(pgio, page_index(page));
T
Trond Myklebust 已提交
658
	ret = nfs_page_async_flush(pgio, page);
659 660 661 662 663
	if (ret == -EAGAIN) {
		redirty_page_for_writepage(wbc, page);
		ret = 0;
	}
	return ret;
T
Trond Myklebust 已提交
664
}
665

T
Trond Myklebust 已提交
666 667 668
/*
 * Write an mmapped page to the server.
 */
669
static int nfs_writepage_locked(struct page *page,
670
				struct writeback_control *wbc)
T
Trond Myklebust 已提交
671 672
{
	struct nfs_pageio_descriptor pgio;
673
	struct inode *inode = page_file_mapping(page)->host;
T
Trond Myklebust 已提交
674
	int err;
675

676
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
677
	nfs_pageio_init_write(&pgio, inode, 0,
678
				false, &nfs_async_write_completion_ops);
679
	err = nfs_do_writepage(page, wbc, &pgio);
T
Trond Myklebust 已提交
680 681 682 683 684 685
	nfs_pageio_complete(&pgio);
	if (err < 0)
		return err;
	if (pgio.pg_error < 0)
		return pgio.pg_error;
	return 0;
686 687 688 689
}

int nfs_writepage(struct page *page, struct writeback_control *wbc)
{
T
Trond Myklebust 已提交
690
	int ret;
691

692
	ret = nfs_writepage_locked(page, wbc);
L
Linus Torvalds 已提交
693
	unlock_page(page);
T
Trond Myklebust 已提交
694 695 696 697 698 699 700
	return ret;
}

static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
{
	int ret;

701
	ret = nfs_do_writepage(page, wbc, data);
T
Trond Myklebust 已提交
702 703
	unlock_page(page);
	return ret;
L
Linus Torvalds 已提交
704 705
}

706 707 708 709 710
static void nfs_io_completion_commit(void *inode)
{
	nfs_commit_inode(inode, 0);
}

L
Linus Torvalds 已提交
711 712 713
int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;
714
	struct nfs_pageio_descriptor pgio;
715
	struct nfs_io_completion *ioc = nfs_io_completion_alloc(GFP_NOFS);
L
Linus Torvalds 已提交
716 717
	int err;

C
Chuck Lever 已提交
718 719
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);

720 721 722
	if (ioc)
		nfs_io_completion_init(ioc, nfs_io_completion_commit, inode);

723 724
	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
				&nfs_async_write_completion_ops);
725
	pgio.pg_io_completion = ioc;
T
Trond Myklebust 已提交
726
	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
727
	nfs_pageio_complete(&pgio);
728
	nfs_io_completion_put(ioc);
729

T
Trond Myklebust 已提交
730
	if (err < 0)
731 732 733 734
		goto out_err;
	err = pgio.pg_error;
	if (err < 0)
		goto out_err;
735
	return 0;
736 737
out_err:
	return err;
L
Linus Torvalds 已提交
738 739 740 741 742
}

/*
 * Insert a write request into an inode
 */
F
Fred Isaman 已提交
743
static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
L
Linus Torvalds 已提交
744
{
745
	struct address_space *mapping = page_file_mapping(req->wb_page);
L
Linus Torvalds 已提交
746
	struct nfs_inode *nfsi = NFS_I(inode);
747

748 749
	WARN_ON_ONCE(req->wb_this_page != req);

750
	/* Lock the request! */
751
	nfs_lock_request(req);
752

753 754 755 756
	/*
	 * Swap-space should not get truncated. Hence no need to plug the race
	 * with invalidate/truncate.
	 */
757 758
	spin_lock(&mapping->private_lock);
	if (!nfs_have_writebacks(inode) &&
J
Jeff Layton 已提交
759 760
	    NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
		inode_inc_iversion_raw(inode);
761 762 763 764 765
	if (likely(!PageSwapCache(req->wb_page))) {
		set_bit(PG_MAPPED, &req->wb_flags);
		SetPagePrivate(req->wb_page);
		set_page_private(req->wb_page, (unsigned long)req);
	}
766
	spin_unlock(&mapping->private_lock);
767
	atomic_long_inc(&nfsi->nrequests);
768
	/* this a head request for a page group - mark it as having an
769 770 771
	 * extra reference so sub groups can follow suit.
	 * This flag also informs pgio layer when to bump nrequests when
	 * adding subrequests. */
772
	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
773
	kref_get(&req->wb_kref);
L
Linus Torvalds 已提交
774 775 776
}

/*
777
 * Remove a write request from an inode
L
Linus Torvalds 已提交
778 779 780
 */
static void nfs_inode_remove_request(struct nfs_page *req)
{
781 782
	struct address_space *mapping = page_file_mapping(req->wb_page);
	struct inode *inode = mapping->host;
L
Linus Torvalds 已提交
783
	struct nfs_inode *nfsi = NFS_I(inode);
784
	struct nfs_page *head;
L
Linus Torvalds 已提交
785

786
	atomic_long_dec(&nfsi->nrequests);
787 788 789
	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
		head = req->wb_head;

790
		spin_lock(&mapping->private_lock);
A
Anna Schumaker 已提交
791
		if (likely(head->wb_page && !PageSwapCache(head->wb_page))) {
792 793 794 795
			set_page_private(head->wb_page, 0);
			ClearPagePrivate(head->wb_page);
			clear_bit(PG_MAPPED, &head->wb_flags);
		}
796
		spin_unlock(&mapping->private_lock);
797
	}
798 799 800

	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags))
		nfs_release_request(req);
L
Linus Torvalds 已提交
801 802
}

803
static void
F
Fred 已提交
804
nfs_mark_request_dirty(struct nfs_page *req)
805
{
A
Anna Schumaker 已提交
806 807
	if (req->wb_page)
		__set_page_dirty_nobuffers(req->wb_page);
808 809
}

810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
/*
 * nfs_page_search_commits_for_head_request_locked
 *
 * Search through commit lists on @inode for the head request for @page.
 * Must be called while holding the inode (which is cinfo) lock.
 *
 * Returns the head request if found, or NULL if not found.
 */
static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
						struct page *page)
{
	struct nfs_page *freq, *t;
	struct nfs_commit_info cinfo;
	struct inode *inode = &nfsi->vfs_inode;

	nfs_init_cinfo_from_inode(&cinfo, inode);

	/* search through pnfs commit lists */
	freq = pnfs_search_commit_reqs(inode, &cinfo, page);
	if (freq)
		return freq->wb_head;

	/* Linearly search the commit list for the correct request */
	list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) {
		if (freq->wb_page == page)
			return freq->wb_head;
	}

	return NULL;
}

842 843 844 845 846 847 848 849 850 851
/**
 * nfs_request_add_commit_list_locked - add request to a commit list
 * @req: pointer to a struct nfs_page
 * @dst: commit list head
 * @cinfo: holds list lock and accounting info
 *
 * This sets the PG_CLEAN bit, updates the cinfo count of
 * number of outstanding requests requiring a commit as well as
 * the MM page stats.
 *
852 853
 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
 * nfs_page lock.
854 855 856 857 858 859 860
 */
void
nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
			    struct nfs_commit_info *cinfo)
{
	set_bit(PG_CLEAN, &req->wb_flags);
	nfs_list_add_request(req, dst);
861
	atomic_long_inc(&cinfo->mds->ncommit);
862 863 864
}
EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);

865 866 867
/**
 * nfs_request_add_commit_list - add request to a commit list
 * @req: pointer to a struct nfs_page
F
Fred Isaman 已提交
868 869
 * @dst: commit list head
 * @cinfo: holds list lock and accounting info
870
 *
F
Fred Isaman 已提交
871
 * This sets the PG_CLEAN bit, updates the cinfo count of
872 873 874
 * number of outstanding requests requiring a commit as well as
 * the MM page stats.
 *
F
Fred Isaman 已提交
875
 * The caller must _not_ hold the cinfo->lock, but must be
876
 * holding the nfs_page lock.
L
Linus Torvalds 已提交
877
 */
878
void
879
nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
880
{
881
	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
882
	nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
883
	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
A
Anna Schumaker 已提交
884 885
	if (req->wb_page)
		nfs_mark_page_unstable(req->wb_page, cinfo);
L
Linus Torvalds 已提交
886
}
887 888 889 890 891
EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);

/**
 * nfs_request_remove_commit_list - Remove request from a commit list
 * @req: pointer to a nfs_page
F
Fred Isaman 已提交
892
 * @cinfo: holds list lock and accounting info
893
 *
F
Fred Isaman 已提交
894
 * This clears the PG_CLEAN bit, and updates the cinfo's count of
895 896 897
 * number of outstanding requests requiring a commit
 * It does not update the MM page stats.
 *
F
Fred Isaman 已提交
898
 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
899 900
 */
void
F
Fred Isaman 已提交
901 902
nfs_request_remove_commit_list(struct nfs_page *req,
			       struct nfs_commit_info *cinfo)
903 904 905 906
{
	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
		return;
	nfs_list_remove_request(req);
907
	atomic_long_dec(&cinfo->mds->ncommit);
908 909 910
}
EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);

F
Fred Isaman 已提交
911 912 913
static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
				      struct inode *inode)
{
914
	cinfo->inode = inode;
F
Fred Isaman 已提交
915 916
	cinfo->mds = &NFS_I(inode)->commit_info;
	cinfo->ds = pnfs_get_ds_info(inode);
F
Fred Isaman 已提交
917
	cinfo->dreq = NULL;
918
	cinfo->completion_ops = &nfs_commit_completion_ops;
F
Fred Isaman 已提交
919 920 921 922 923 924
}

void nfs_init_cinfo(struct nfs_commit_info *cinfo,
		    struct inode *inode,
		    struct nfs_direct_req *dreq)
{
925 926 927 928
	if (dreq)
		nfs_init_cinfo_from_dreq(cinfo, dreq);
	else
		nfs_init_cinfo_from_inode(cinfo, inode);
F
Fred Isaman 已提交
929 930
}
EXPORT_SYMBOL_GPL(nfs_init_cinfo);
931 932 933 934

/*
 * Add a request to the inode's commit list.
 */
935
void
F
Fred Isaman 已提交
936
nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
937
			struct nfs_commit_info *cinfo, u32 ds_commit_idx)
938
{
939
	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
940
		return;
941
	nfs_request_add_commit_list(req, cinfo);
942
}
943

F
Fred Isaman 已提交
944 945 946
static void
nfs_clear_page_commit(struct page *page)
{
947
	dec_node_page_state(page, NR_UNSTABLE_NFS);
948 949
	dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb,
		    WB_RECLAIMABLE);
F
Fred Isaman 已提交
950 951
}

952
/* Called holding the request lock on @req */
953
static void
954 955
nfs_clear_request_commit(struct nfs_page *req)
{
956
	if (test_bit(PG_CLEAN, &req->wb_flags)) {
957
		struct inode *inode = d_inode(req->wb_context->dentry);
F
Fred Isaman 已提交
958
		struct nfs_commit_info cinfo;
959

F
Fred Isaman 已提交
960
		nfs_init_cinfo_from_inode(&cinfo, inode);
961
		mutex_lock(&NFS_I(inode)->commit_mutex);
F
Fred Isaman 已提交
962 963
		if (!pnfs_clear_request_commit(req, &cinfo)) {
			nfs_request_remove_commit_list(req, &cinfo);
964
		}
965
		mutex_unlock(&NFS_I(inode)->commit_mutex);
F
Fred Isaman 已提交
966
		nfs_clear_page_commit(req->wb_page);
967 968 969
	}
}

970
int nfs_write_need_commit(struct nfs_pgio_header *hdr)
971
{
972
	if (hdr->verf.committed == NFS_DATA_SYNC)
973
		return hdr->lseg == NULL;
974
	return hdr->verf.committed != NFS_FILE_SYNC;
975 976
}

977 978 979 980 981
static void nfs_async_write_init(struct nfs_pgio_header *hdr)
{
	nfs_io_completion_get(hdr->io_completion);
}

982
static void nfs_write_completion(struct nfs_pgio_header *hdr)
983
{
F
Fred Isaman 已提交
984
	struct nfs_commit_info cinfo;
985 986 987 988
	unsigned long bytes = 0;

	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
		goto out;
F
Fred Isaman 已提交
989
	nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
990 991 992 993 994 995 996
	while (!list_empty(&hdr->pages)) {
		struct nfs_page *req = nfs_list_entry(hdr->pages.next);

		bytes += req->wb_bytes;
		nfs_list_remove_request(req);
		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
		    (hdr->good_bytes < bytes)) {
997
			nfs_set_pageerror(page_file_mapping(req->wb_page));
998 999 1000
			nfs_context_set_write_error(req->wb_context, hdr->error);
			goto remove_req;
		}
1001
		if (nfs_write_need_commit(hdr)) {
1002
			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
1003
			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
1004
				hdr->pgio_mirror_idx);
1005 1006 1007 1008 1009
			goto next;
		}
remove_req:
		nfs_inode_remove_request(req);
next:
1010
		nfs_end_page_writeback(req);
1011
		nfs_release_request(req);
1012 1013
	}
out:
1014
	nfs_io_completion_put(hdr->io_completion);
1015
	hdr->release(hdr);
1016
}
L
Linus Torvalds 已提交
1017

1018
unsigned long
F
Fred Isaman 已提交
1019
nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
1020
{
1021
	return atomic_long_read(&cinfo->mds->ncommit);
F
Fred Isaman 已提交
1022 1023
}

1024
/* NFS_I(cinfo->inode)->commit_mutex held by caller */
1025
int
F
Fred Isaman 已提交
1026 1027
nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
		     struct nfs_commit_info *cinfo, int max)
F
Fred Isaman 已提交
1028
{
1029
	struct nfs_page *req, *tmp;
F
Fred Isaman 已提交
1030 1031
	int ret = 0;

1032 1033
restart:
	list_for_each_entry_safe(req, tmp, src, wb_list) {
1034
		kref_get(&req->wb_kref);
1035 1036
		if (!nfs_lock_request(req)) {
			int status;
1037 1038 1039 1040 1041 1042 1043

			/* Prevent deadlock with nfs_lock_and_join_requests */
			if (!list_empty(dst)) {
				nfs_release_request(req);
				continue;
			}
			/* Ensure we make progress to prevent livelock */
1044 1045 1046 1047 1048 1049
			mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
			status = nfs_wait_on_request(req);
			nfs_release_request(req);
			mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
			if (status < 0)
				break;
1050
			goto restart;
1051
		}
F
Fred Isaman 已提交
1052
		nfs_request_remove_commit_list(req, cinfo);
1053
		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
1054 1055
		nfs_list_add_request(req, dst);
		ret++;
1056
		if ((ret == max) && !cinfo->dreq)
1057
			break;
1058
		cond_resched();
F
Fred Isaman 已提交
1059 1060
	}
	return ret;
1061
}
1062
EXPORT_SYMBOL_GPL(nfs_scan_commit_list);
1063

L
Linus Torvalds 已提交
1064 1065 1066
/*
 * nfs_scan_commit - Scan an inode for commit requests
 * @inode: NFS inode to scan
F
Fred Isaman 已提交
1067 1068
 * @dst: mds destination list
 * @cinfo: mds and ds lists of reqs ready to commit
L
Linus Torvalds 已提交
1069 1070 1071 1072
 *
 * Moves requests from the inode's 'commit' request list.
 * The requests are *not* checked to ensure that they form a contiguous set.
 */
1073
int
F
Fred Isaman 已提交
1074 1075
nfs_scan_commit(struct inode *inode, struct list_head *dst,
		struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
1076
{
F
Fred Isaman 已提交
1077
	int ret = 0;
1078

1079 1080
	if (!atomic_long_read(&cinfo->mds->ncommit))
		return 0;
1081
	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1082
	if (atomic_long_read(&cinfo->mds->ncommit) > 0) {
1083
		const int max = INT_MAX;
F
Fred Isaman 已提交
1084

F
Fred Isaman 已提交
1085 1086 1087
		ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
					   cinfo, max);
		ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
F
Fred Isaman 已提交
1088
	}
1089
	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1090
	return ret;
L
Linus Torvalds 已提交
1091
}
F
Fred Isaman 已提交
1092

L
Linus Torvalds 已提交
1093
/*
1094 1095
 * Search for an existing write request, and attempt to update
 * it to reflect a new dirty region on a given page.
L
Linus Torvalds 已提交
1096
 *
1097 1098
 * If the attempt fails, then the existing request is flushed out
 * to disk.
L
Linus Torvalds 已提交
1099
 */
1100 1101 1102 1103
static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
		struct page *page,
		unsigned int offset,
		unsigned int bytes)
L
Linus Torvalds 已提交
1104
{
1105 1106 1107 1108 1109
	struct nfs_page *req;
	unsigned int rqend;
	unsigned int end;
	int error;

L
Linus Torvalds 已提交
1110 1111
	end = offset + bytes;

1112 1113 1114
	req = nfs_lock_and_join_requests(page);
	if (IS_ERR_OR_NULL(req))
		return req;
L
Linus Torvalds 已提交
1115

1116 1117 1118 1119 1120 1121 1122 1123 1124
	rqend = req->wb_offset + req->wb_bytes;
	/*
	 * Tell the caller to flush out the request if
	 * the offsets are non-contiguous.
	 * Note: nfs_flush_incompatible() will already
	 * have flushed out requests having wrong owners.
	 */
	if (offset > rqend || end < req->wb_offset)
		goto out_flushme;
L
Linus Torvalds 已提交
1125 1126 1127 1128 1129 1130 1131 1132

	/* Okay, the request matches. Update the region */
	if (offset < req->wb_offset) {
		req->wb_offset = offset;
		req->wb_pgbase = offset;
	}
	if (end > rqend)
		req->wb_bytes = end - req->wb_offset;
1133 1134 1135 1136
	else
		req->wb_bytes = rqend - req->wb_offset;
	return req;
out_flushme:
1137 1138 1139 1140 1141 1142 1143
	/*
	 * Note: we mark the request dirty here because
	 * nfs_lock_and_join_requests() cannot preserve
	 * commit flags, so we have to replay the write.
	 */
	nfs_mark_request_dirty(req);
	nfs_unlock_and_release_request(req);
1144
	error = nfs_wb_page(inode, page);
1145
	return (error < 0) ? ERR_PTR(error) : NULL;
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
}

/*
 * Try to update an existing write request, or create one if there is none.
 *
 * Note: Should always be called with the Page Lock held to prevent races
 * if we have to add a new request. Also assumes that the caller has
 * already called nfs_flush_incompatible() if necessary.
 */
static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
		struct page *page, unsigned int offset, unsigned int bytes)
{
1158
	struct inode *inode = page_file_mapping(page)->host;
1159
	struct nfs_page	*req;
L
Linus Torvalds 已提交
1160

1161 1162 1163
	req = nfs_try_to_update_request(inode, page, offset, bytes);
	if (req != NULL)
		goto out;
1164
	req = nfs_create_request(ctx, page, NULL, offset, bytes);
1165 1166
	if (IS_ERR(req))
		goto out;
F
Fred Isaman 已提交
1167
	nfs_inode_add_request(inode, req);
1168
out:
T
Trond Myklebust 已提交
1169
	return req;
L
Linus Torvalds 已提交
1170 1171
}

1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
		unsigned int offset, unsigned int count)
{
	struct nfs_page	*req;

	req = nfs_setup_write_request(ctx, page, offset, count);
	if (IS_ERR(req))
		return PTR_ERR(req);
	/* Update file length */
	nfs_grow_file(page, offset, count);
1182
	nfs_mark_uptodate(req);
1183
	nfs_mark_request_dirty(req);
1184
	nfs_unlock_and_release_request(req);
1185 1186 1187
	return 0;
}

L
Linus Torvalds 已提交
1188 1189
int nfs_flush_incompatible(struct file *file, struct page *page)
{
1190
	struct nfs_open_context *ctx = nfs_file_open_context(file);
1191
	struct nfs_lock_context *l_ctx;
1192
	struct file_lock_context *flctx = file_inode(file)->i_flctx;
L
Linus Torvalds 已提交
1193
	struct nfs_page	*req;
T
Trond Myklebust 已提交
1194
	int do_flush, status;
L
Linus Torvalds 已提交
1195 1196 1197 1198 1199 1200 1201 1202
	/*
	 * Look for a request corresponding to this page. If there
	 * is one, and it belongs to another file, we flush it out
	 * before we try to copy anything into the page. Do this
	 * due to the lack of an ACCESS-type call in NFSv2.
	 * Also do the same if we find a request from an existing
	 * dropped page.
	 */
T
Trond Myklebust 已提交
1203
	do {
1204
		req = nfs_page_find_head_request(page);
T
Trond Myklebust 已提交
1205 1206
		if (req == NULL)
			return 0;
1207
		l_ctx = req->wb_lock_context;
1208 1209
		do_flush = req->wb_page != page ||
			!nfs_match_open_context(req->wb_context, ctx);
1210 1211 1212
		if (l_ctx && flctx &&
		    !(list_empty_careful(&flctx->flc_posix) &&
		      list_empty_careful(&flctx->flc_flock))) {
N
NeilBrown 已提交
1213
			do_flush |= l_ctx->lockowner != current->files;
1214
		}
L
Linus Torvalds 已提交
1215
		nfs_release_request(req);
T
Trond Myklebust 已提交
1216 1217
		if (!do_flush)
			return 0;
1218
		status = nfs_wb_page(page_file_mapping(page)->host, page);
T
Trond Myklebust 已提交
1219 1220
	} while (status == 0);
	return status;
L
Linus Torvalds 已提交
1221 1222
}

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
/*
 * Avoid buffered writes when a open context credential's key would
 * expire soon.
 *
 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
 *
 * Return 0 and set a credential flag which triggers the inode to flush
 * and performs  NFS_FILE_SYNC writes if the key will expired within
 * RPC_KEY_EXPIRE_TIMEO.
 */
int
nfs_key_timeout_notify(struct file *filp, struct inode *inode)
{
	struct nfs_open_context *ctx = nfs_file_open_context(filp);

1238 1239 1240 1241 1242
	if (nfs_ctx_key_to_expire(ctx, inode) &&
	    !ctx->ll_cred)
		/* Already expired! */
		return -EACCES;
	return 0;
1243 1244 1245 1246 1247
}

/*
 * Test if the open context credential key is marked to expire soon.
 */
1248
bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
1249
{
1250
	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
1251 1252
	struct rpc_cred *cred = ctx->ll_cred;
	struct auth_cred acred = {
1253
		.cred = ctx->cred,
1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
	};

	if (cred && !cred->cr_ops->crmatch(&acred, cred, 0)) {
		put_rpccred(cred);
		ctx->ll_cred = NULL;
		cred = NULL;
	}
	if (!cred)
		cred = auth->au_ops->lookup_cred(auth, &acred, 0);
	if (!cred || IS_ERR(cred))
		return true;
	ctx->ll_cred = cred;
	return !!(cred->cr_ops->crkey_timeout &&
		  cred->cr_ops->crkey_timeout(cred));
1268 1269
}

1270 1271 1272 1273 1274
/*
 * If the page cache is marked as unsafe or invalid, then we can't rely on
 * the PageUptodate() flag. In this case, we will need to turn off
 * write optimisations that depend on the page contents being correct.
 */
1275
static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
1276
{
1277 1278
	struct nfs_inode *nfsi = NFS_I(inode);

1279 1280
	if (nfs_have_delegated_attributes(inode))
		goto out;
1281
	if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
1282
		return false;
1283
	smp_rmb();
1284
	if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
1285 1286
		return false;
out:
1287 1288
	if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
		return false;
1289
	return PageUptodate(page) != 0;
1290 1291
}

1292 1293 1294 1295 1296 1297 1298
static bool
is_whole_file_wrlock(struct file_lock *fl)
{
	return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
			fl->fl_type == F_WRLCK;
}

1299 1300 1301 1302 1303
/* If we know the page is up to date, and we're not using byte range locks (or
 * if we have the whole file locked for writing), it may be more efficient to
 * extend the write to cover the entire page in order to avoid fragmentation
 * inefficiencies.
 *
1304 1305
 * If the file is opened for synchronous writes then we can just skip the rest
 * of the checks.
1306 1307 1308
 */
static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
{
1309 1310 1311 1312
	int ret;
	struct file_lock_context *flctx = inode->i_flctx;
	struct file_lock *fl;

1313 1314
	if (file->f_flags & O_DSYNC)
		return 0;
1315 1316
	if (!nfs_write_pageuptodate(page, inode))
		return 0;
1317 1318
	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
		return 1;
1319 1320
	if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
		       list_empty_careful(&flctx->flc_posix)))
1321
		return 1;
1322 1323 1324

	/* Check to see if there are whole file write locks */
	ret = 0;
1325
	spin_lock(&flctx->flc_lock);
1326 1327 1328 1329 1330 1331
	if (!list_empty(&flctx->flc_posix)) {
		fl = list_first_entry(&flctx->flc_posix, struct file_lock,
					fl_list);
		if (is_whole_file_wrlock(fl))
			ret = 1;
	} else if (!list_empty(&flctx->flc_flock)) {
1332 1333 1334 1335 1336
		fl = list_first_entry(&flctx->flc_flock, struct file_lock,
					fl_list);
		if (fl->fl_type == F_WRLCK)
			ret = 1;
	}
1337
	spin_unlock(&flctx->flc_lock);
1338
	return ret;
1339 1340
}

L
Linus Torvalds 已提交
1341 1342 1343 1344 1345 1346 1347 1348 1349
/*
 * Update and possibly write a cached page of an NFS file.
 *
 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
 * things with a page scheduled for an RPC call (e.g. invalidate it).
 */
int nfs_updatepage(struct file *file, struct page *page,
		unsigned int offset, unsigned int count)
{
1350
	struct nfs_open_context *ctx = nfs_file_open_context(file);
1351 1352
	struct address_space *mapping = page_file_mapping(page);
	struct inode	*inode = mapping->host;
L
Linus Torvalds 已提交
1353 1354
	int		status = 0;

C
Chuck Lever 已提交
1355 1356
	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);

1357 1358
	dprintk("NFS:       nfs_updatepage(%pD2 %d@%lld)\n",
		file, count, (long long)(page_file_offset(page) + offset));
L
Linus Torvalds 已提交
1359

1360 1361 1362
	if (!count)
		goto out;

1363
	if (nfs_can_extend_write(file, page, inode)) {
1364
		count = max(count + offset, nfs_page_length(page));
L
Linus Torvalds 已提交
1365 1366 1367
		offset = 0;
	}

1368
	status = nfs_writepage_setup(ctx, page, offset, count);
1369
	if (status < 0)
1370
		nfs_set_pageerror(mapping);
1371 1372
	else
		__set_page_dirty_nobuffers(page);
1373
out:
1374
	dprintk("NFS:       nfs_updatepage returns %d (isize %lld)\n",
L
Linus Torvalds 已提交
1375 1376 1377 1378
			status, (long long)i_size_read(inode));
	return status;
}

1379
static int flush_task_priority(int how)
L
Linus Torvalds 已提交
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
{
	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
		case FLUSH_HIGHPRI:
			return RPC_PRIORITY_HIGH;
		case FLUSH_LOWPRI:
			return RPC_PRIORITY_LOW;
	}
	return RPC_PRIORITY_NORMAL;
}

1390 1391
static void nfs_initiate_write(struct nfs_pgio_header *hdr,
			       struct rpc_message *msg,
1392
			       const struct nfs_rpc_ops *rpc_ops,
1393
			       struct rpc_task_setup *task_setup_data, int how)
L
Linus Torvalds 已提交
1394
{
1395
	int priority = flush_task_priority(how);
1396

1397
	task_setup_data->priority = priority;
1398
	rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client);
1399 1400
	trace_nfs_initiate_write(hdr->inode, hdr->io_start, hdr->good_bytes,
				 hdr->args.stable);
1401 1402
}

F
Fred 已提交
1403 1404 1405 1406 1407 1408 1409
/* If a nfs_flush_* function fails, it should remove reqs from @head and
 * call this on each, which will prepare them to be retried on next
 * writeback using standard nfs.
 */
static void nfs_redirty_request(struct nfs_page *req)
{
	nfs_mark_request_dirty(req);
1410
	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1411
	nfs_end_page_writeback(req);
1412
	nfs_release_request(req);
F
Fred 已提交
1413 1414
}

1415
static void nfs_async_write_error(struct list_head *head)
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
{
	struct nfs_page	*req;

	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_redirty_request(req);
	}
}

1426 1427 1428
static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
{
	nfs_async_write_error(&hdr->pages);
1429 1430
	filemap_fdatawrite_range(hdr->inode->i_mapping, hdr->args.offset,
			hdr->args.offset + hdr->args.count - 1);
1431 1432
}

1433
static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1434
	.init_hdr = nfs_async_write_init,
1435 1436
	.error_cleanup = nfs_async_write_error,
	.completion = nfs_write_completion,
1437
	.reschedule_io = nfs_async_write_reschedule_io,
1438 1439
};

1440
void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1441
			       struct inode *inode, int ioflags, bool force_mds,
1442
			       const struct nfs_pgio_completion_ops *compl_ops)
L
Linus Torvalds 已提交
1443
{
1444
	struct nfs_server *server = NFS_SERVER(inode);
1445
	const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
1446 1447 1448 1449 1450

#ifdef CONFIG_NFS_V4_1
	if (server->pnfs_curr_ld && !force_mds)
		pg_ops = server->pnfs_curr_ld->pg_write_ops;
#endif
1451
	nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1452
			server->wsize, ioflags);
1453
}
B
Bryan Schumaker 已提交
1454
EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
L
Linus Torvalds 已提交
1455

1456 1457
void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
{
1458 1459
	struct nfs_pgio_mirror *mirror;

1460 1461 1462
	if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
		pgio->pg_ops->pg_cleanup(pgio);

1463
	pgio->pg_ops = &nfs_pgio_rw_ops;
1464 1465 1466 1467 1468

	nfs_pageio_stop_mirroring(pgio);

	mirror = &pgio->pg_mirrors[0];
	mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1469
}
1470
EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1471

L
Linus Torvalds 已提交
1472

1473 1474 1475 1476 1477 1478 1479
void nfs_commit_prepare(struct rpc_task *task, void *calldata)
{
	struct nfs_commit_data *data = calldata;

	NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
}

1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
/*
 * Special version of should_remove_suid() that ignores capabilities.
 */
static int nfs_should_remove_suid(const struct inode *inode)
{
	umode_t mode = inode->i_mode;
	int kill = 0;

	/* suid always must be killed */
	if (unlikely(mode & S_ISUID))
		kill = ATTR_KILL_SUID;
1491

1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
	/*
	 * sgid without any exec bits is just a mandatory locking mark; leave
	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
	 */
	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
		kill |= ATTR_KILL_SGID;

	if (unlikely(kill && S_ISREG(mode)))
		return kill;

	return 0;
}
1504

1505 1506 1507 1508 1509
static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
		struct nfs_fattr *fattr)
{
	struct nfs_pgio_args *argp = &hdr->args;
	struct nfs_pgio_res *resp = &hdr->res;
1510
	u64 size = argp->offset + resp->count;
1511 1512

	if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
1513 1514 1515
		fattr->size = size;
	if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
		fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
1516
		return;
1517 1518
	}
	if (size != fattr->size)
1519 1520 1521
		return;
	/* Set attribute barrier */
	nfs_fattr_set_barrier(fattr);
1522 1523
	/* ...and update size */
	fattr->valid |= NFS_ATTR_FATTR_SIZE;
1524 1525 1526 1527
}

void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
{
1528
	struct nfs_fattr *fattr = &hdr->fattr;
1529 1530 1531 1532 1533 1534 1535 1536 1537
	struct inode *inode = hdr->inode;

	spin_lock(&inode->i_lock);
	nfs_writeback_check_extend(hdr, fattr);
	nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
	spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);

L
Linus Torvalds 已提交
1538 1539 1540
/*
 * This function is called when the WRITE call is complete.
 */
1541 1542
static int nfs_writeback_done(struct rpc_task *task,
			      struct nfs_pgio_header *hdr,
1543
			      struct inode *inode)
L
Linus Torvalds 已提交
1544
{
1545
	int status;
L
Linus Torvalds 已提交
1546

1547 1548 1549 1550 1551 1552 1553
	/*
	 * ->write_done will attempt to use post-op attributes to detect
	 * conflicting writes by other clients.  A strict interpretation
	 * of close-to-open would allow us to continue caching even if
	 * another writer had changed the file, but some applications
	 * depend on tighter cache coherency when writing.
	 */
1554
	status = NFS_PROTO(inode)->write_done(task, hdr);
1555
	if (status != 0)
1556
		return status;
1557

1558
	nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
1559 1560
	trace_nfs_writeback_done(inode, task->tk_status,
				 hdr->args.offset, hdr->res.verf);
C
Chuck Lever 已提交
1561

1562 1563
	if (hdr->res.verf->committed < hdr->args.stable &&
	    task->tk_status >= 0) {
L
Linus Torvalds 已提交
1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
		/* We tried a write call, but the server did not
		 * commit data to stable storage even though we
		 * requested it.
		 * Note: There is a known bug in Tru64 < 5.0 in which
		 *	 the server reports NFS_DATA_SYNC, but performs
		 *	 NFS_FILE_SYNC. We therefore implement this checking
		 *	 as a dprintk() in order to avoid filling syslog.
		 */
		static unsigned long    complain;

1574
		/* Note this will print the MDS for a DS write */
L
Linus Torvalds 已提交
1575
		if (time_before(complain, jiffies)) {
1576
			dprintk("NFS:       faulty NFS server %s:"
L
Linus Torvalds 已提交
1577
				" (committed = %d) != (stable = %d)\n",
1578
				NFS_SERVER(inode)->nfs_client->cl_hostname,
1579
				hdr->res.verf->committed, hdr->args.stable);
L
Linus Torvalds 已提交
1580 1581 1582
			complain = jiffies + 300 * HZ;
		}
	}
1583 1584

	/* Deal with the suid/sgid bit corner case */
1585 1586 1587 1588 1589
	if (nfs_should_remove_suid(inode)) {
		spin_lock(&inode->i_lock);
		NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER;
		spin_unlock(&inode->i_lock);
	}
1590 1591 1592 1593 1594 1595
	return 0;
}

/*
 * This function is called when the WRITE call is complete.
 */
1596 1597
static void nfs_writeback_result(struct rpc_task *task,
				 struct nfs_pgio_header *hdr)
1598
{
1599 1600
	struct nfs_pgio_args	*argp = &hdr->args;
	struct nfs_pgio_res	*resp = &hdr->res;
1601 1602

	if (resp->count < argp->count) {
L
Linus Torvalds 已提交
1603 1604
		static unsigned long    complain;

1605
		/* This a short write! */
1606
		nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
C
Chuck Lever 已提交
1607

L
Linus Torvalds 已提交
1608
		/* Has the server at least made some progress? */
1609 1610 1611 1612 1613 1614
		if (resp->count == 0) {
			if (time_before(complain, jiffies)) {
				printk(KERN_WARNING
				       "NFS: Server wrote zero bytes, expected %u.\n",
				       argp->count);
				complain = jiffies + 300 * HZ;
L
Linus Torvalds 已提交
1615
			}
1616
			nfs_set_pgio_error(hdr, -EIO, argp->offset);
1617
			task->tk_status = -EIO;
1618
			return;
L
Linus Torvalds 已提交
1619
		}
1620 1621 1622 1623 1624 1625 1626

		/* For non rpc-based layout drivers, retry-through-MDS */
		if (!task->tk_ops) {
			hdr->pnfs_error = -EAGAIN;
			return;
		}

1627 1628 1629
		/* Was this an NFSv2 write or an NFSv3 stable write? */
		if (resp->verf->committed != NFS_UNSTABLE) {
			/* Resend from where the server left off */
1630
			hdr->mds_offset += resp->count;
1631 1632 1633 1634 1635 1636 1637 1638
			argp->offset += resp->count;
			argp->pgbase += resp->count;
			argp->count -= resp->count;
		} else {
			/* Resend as a stable write in order to avoid
			 * headaches in the case of a server crash.
			 */
			argp->stable = NFS_FILE_SYNC;
L
Linus Torvalds 已提交
1639
		}
1640
		rpc_restart_call_prepare(task);
L
Linus Torvalds 已提交
1641 1642 1643
	}
}

1644
static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
1645
{
1646 1647
	return wait_var_event_killable(&cinfo->rpcs_out,
				       !atomic_read(&cinfo->rpcs_out));
1648
}
1649

1650 1651 1652
static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
{
	atomic_inc(&cinfo->rpcs_out);
1653 1654
}

1655
static void nfs_commit_end(struct nfs_mds_commit_info *cinfo)
1656
{
1657
	if (atomic_dec_and_test(&cinfo->rpcs_out))
1658
		wake_up_var(&cinfo->rpcs_out);
1659 1660
}

1661
void nfs_commitdata_release(struct nfs_commit_data *data)
L
Linus Torvalds 已提交
1662
{
1663 1664
	put_nfs_open_context(data->context);
	nfs_commit_free(data);
L
Linus Torvalds 已提交
1665
}
1666
EXPORT_SYMBOL_GPL(nfs_commitdata_release);
L
Linus Torvalds 已提交
1667

1668
int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1669
			const struct nfs_rpc_ops *nfs_ops,
1670
			const struct rpc_call_ops *call_ops,
1671
			int how, int flags)
L
Linus Torvalds 已提交
1672
{
1673
	struct rpc_task *task;
1674
	int priority = flush_task_priority(how);
1675 1676 1677
	struct rpc_message msg = {
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
1678
		.rpc_cred = data->cred,
1679
	};
1680
	struct rpc_task_setup task_setup_data = {
1681
		.task = &data->task,
1682
		.rpc_client = clnt,
1683
		.rpc_message = &msg,
1684
		.callback_ops = call_ops,
1685
		.callback_data = data,
1686
		.workqueue = nfsiod_workqueue,
1687
		.flags = RPC_TASK_ASYNC | flags,
1688
		.priority = priority,
1689
	};
1690
	/* Set up the initial task struct.  */
1691
	nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client);
1692
	trace_nfs_initiate_commit(data);
1693

1694
	dprintk("NFS: initiated commit call\n");
1695 1696 1697 1698 1699 1700 1701 1702 1703

	task = rpc_run_task(&task_setup_data);
	if (IS_ERR(task))
		return PTR_ERR(task);
	if (how & FLUSH_SYNC)
		rpc_wait_for_completion_task(task);
	rpc_put_task(task);
	return 0;
}
1704
EXPORT_SYMBOL_GPL(nfs_initiate_commit);
1705

1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
static loff_t nfs_get_lwb(struct list_head *head)
{
	loff_t lwb = 0;
	struct nfs_page *req;

	list_for_each_entry(req, head, wb_list)
		if (lwb < (req_offset(req) + req->wb_bytes))
			lwb = req_offset(req) + req->wb_bytes;

	return lwb;
}

1718 1719 1720
/*
 * Set up the argument/result storage required for the RPC call.
 */
1721
void nfs_init_commit(struct nfs_commit_data *data,
1722 1723 1724
		     struct list_head *head,
		     struct pnfs_layout_segment *lseg,
		     struct nfs_commit_info *cinfo)
1725 1726
{
	struct nfs_page *first = nfs_list_entry(head->next);
1727
	struct inode *inode = d_inode(first->wb_context->dentry);
L
Linus Torvalds 已提交
1728 1729 1730 1731 1732 1733 1734

	/* Set up the RPC argument and reply structs
	 * NB: take care not to mess about with data->commit et al. */

	list_splice_init(head, &data->pages);

	data->inode	  = inode;
1735
	data->cred	  = first->wb_context->cred;
1736
	data->lseg	  = lseg; /* reference transferred */
1737 1738 1739
	/* only set lwb for pnfs commit */
	if (lseg)
		data->lwb = nfs_get_lwb(&data->pages);
1740
	data->mds_ops     = &nfs_commit_ops;
1741
	data->completion_ops = cinfo->completion_ops;
F
Fred Isaman 已提交
1742
	data->dreq	  = cinfo->dreq;
L
Linus Torvalds 已提交
1743 1744

	data->args.fh     = NFS_FH(data->inode);
1745 1746 1747
	/* Note: we always request a commit of the entire inode */
	data->args.offset = 0;
	data->args.count  = 0;
1748
	data->context     = get_nfs_open_context(first->wb_context);
L
Linus Torvalds 已提交
1749 1750
	data->res.fattr   = &data->fattr;
	data->res.verf    = &data->verf;
1751
	nfs_fattr_init(&data->fattr);
L
Linus Torvalds 已提交
1752
}
1753
EXPORT_SYMBOL_GPL(nfs_init_commit);
L
Linus Torvalds 已提交
1754

1755
void nfs_retry_commit(struct list_head *page_list,
F
Fred Isaman 已提交
1756
		      struct pnfs_layout_segment *lseg,
1757 1758
		      struct nfs_commit_info *cinfo,
		      u32 ds_commit_idx)
1759 1760 1761 1762 1763 1764
{
	struct nfs_page *req;

	while (!list_empty(page_list)) {
		req = nfs_list_entry(page_list->next);
		nfs_list_remove_request(req);
1765
		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
1766 1767
		if (!cinfo->dreq)
			nfs_clear_page_commit(req->wb_page);
1768
		nfs_unlock_and_release_request(req);
1769 1770
	}
}
1771
EXPORT_SYMBOL_GPL(nfs_retry_commit);
1772

1773 1774 1775 1776 1777 1778 1779
static void
nfs_commit_resched_write(struct nfs_commit_info *cinfo,
		struct nfs_page *req)
{
	__set_page_dirty_nobuffers(req->wb_page);
}

L
Linus Torvalds 已提交
1780 1781 1782 1783
/*
 * Commit dirty pages
 */
static int
F
Fred Isaman 已提交
1784 1785
nfs_commit_list(struct inode *inode, struct list_head *head, int how,
		struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
1786
{
1787
	struct nfs_commit_data	*data;
L
Linus Torvalds 已提交
1788

1789 1790 1791 1792
	/* another commit raced with us */
	if (list_empty(head))
		return 0;

N
NeilBrown 已提交
1793
	data = nfs_commitdata_alloc(true);
L
Linus Torvalds 已提交
1794 1795

	/* Set up the argument struct */
1796 1797
	nfs_init_commit(data, head, NULL, cinfo);
	atomic_inc(&cinfo->mds->rpcs_out);
1798 1799
	return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
				   data->mds_ops, how, 0);
A
Anna Schumaker 已提交
1800 1801
}

L
Linus Torvalds 已提交
1802 1803 1804
/*
 * COMMIT call returned
 */
1805
static void nfs_commit_done(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
1806
{
1807
	struct nfs_commit_data	*data = calldata;
L
Linus Torvalds 已提交
1808

C
Chuck Lever 已提交
1809
        dprintk("NFS: %5u nfs_commit_done (status %d)\n",
L
Linus Torvalds 已提交
1810 1811
                                task->tk_pid, task->tk_status);

1812
	/* Call the NFS version-specific code */
1813
	NFS_PROTO(data->inode)->commit_done(task, data);
1814
	trace_nfs_commit_done(data);
1815 1816
}

1817
static void nfs_commit_release_pages(struct nfs_commit_data *data)
1818
{
1819
	struct nfs_page	*req;
1820
	int status = data->task.tk_status;
1821
	struct nfs_commit_info cinfo;
1822
	struct nfs_server *nfss;
1823

L
Linus Torvalds 已提交
1824 1825 1826
	while (!list_empty(&data->pages)) {
		req = nfs_list_entry(data->pages.next);
		nfs_list_remove_request(req);
A
Anna Schumaker 已提交
1827 1828
		if (req->wb_page)
			nfs_clear_page_commit(req->wb_page);
L
Linus Torvalds 已提交
1829

1830
		dprintk("NFS:       commit (%s/%llu %d@%lld)",
1831
			req->wb_context->dentry->d_sb->s_id,
1832
			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),
L
Linus Torvalds 已提交
1833 1834
			req->wb_bytes,
			(long long)req_offset(req));
1835 1836
		if (status < 0) {
			nfs_context_set_write_error(req->wb_context, status);
1837 1838
			if (req->wb_page)
				nfs_inode_remove_request(req);
1839
			dprintk_cont(", error = %d\n", status);
L
Linus Torvalds 已提交
1840 1841 1842 1843 1844
			goto next;
		}

		/* Okay, COMMIT succeeded, apparently. Check the verifier
		 * returned by the server against all stored verfs. */
1845
		if (!nfs_write_verifier_cmp(&req->wb_verf, &data->verf.verifier)) {
L
Linus Torvalds 已提交
1846
			/* We have a match */
1847 1848
			if (req->wb_page)
				nfs_inode_remove_request(req);
1849
			dprintk_cont(" OK\n");
L
Linus Torvalds 已提交
1850 1851 1852
			goto next;
		}
		/* We have a mismatch. Write the page again */
1853
		dprintk_cont(" mismatch\n");
F
Fred 已提交
1854
		nfs_mark_request_dirty(req);
1855
		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
L
Linus Torvalds 已提交
1856
	next:
1857
		nfs_unlock_and_release_request(req);
1858 1859
		/* Latency breaker */
		cond_resched();
L
Linus Torvalds 已提交
1860
	}
1861 1862
	nfss = NFS_SERVER(data->inode);
	if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
1863
		clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC);
1864

1865
	nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1866
	nfs_commit_end(cinfo.mds);
1867 1868 1869 1870
}

static void nfs_commit_release(void *calldata)
{
1871
	struct nfs_commit_data *data = calldata;
1872

1873
	data->completion_ops->completion(data);
1874
	nfs_commitdata_release(calldata);
L
Linus Torvalds 已提交
1875
}
1876 1877

static const struct rpc_call_ops nfs_commit_ops = {
1878
	.rpc_call_prepare = nfs_commit_prepare,
1879 1880 1881
	.rpc_call_done = nfs_commit_done,
	.rpc_release = nfs_commit_release,
};
L
Linus Torvalds 已提交
1882

1883 1884
static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
	.completion = nfs_commit_release_pages,
1885
	.resched_write = nfs_commit_resched_write,
1886 1887
};

1888 1889
int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
			    int how, struct nfs_commit_info *cinfo)
1890 1891 1892
{
	int status;

F
Fred Isaman 已提交
1893
	status = pnfs_commit_list(inode, head, how, cinfo);
1894
	if (status == PNFS_NOT_ATTEMPTED)
F
Fred Isaman 已提交
1895
		status = nfs_commit_list(inode, head, how, cinfo);
1896 1897 1898
	return status;
}

1899 1900
static int __nfs_commit_inode(struct inode *inode, int how,
		struct writeback_control *wbc)
L
Linus Torvalds 已提交
1901 1902
{
	LIST_HEAD(head);
F
Fred Isaman 已提交
1903
	struct nfs_commit_info cinfo;
1904
	int may_wait = how & FLUSH_SYNC;
1905
	int ret, nscan;
L
Linus Torvalds 已提交
1906

F
Fred Isaman 已提交
1907
	nfs_init_cinfo_from_inode(&cinfo, inode);
1908
	nfs_commit_begin(cinfo.mds);
1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926
	for (;;) {
		ret = nscan = nfs_scan_commit(inode, &head, &cinfo);
		if (ret <= 0)
			break;
		ret = nfs_generic_commit_list(inode, &head, how, &cinfo);
		if (ret < 0)
			break;
		ret = 0;
		if (wbc && wbc->sync_mode == WB_SYNC_NONE) {
			if (nscan < wbc->nr_to_write)
				wbc->nr_to_write -= nscan;
			else
				wbc->nr_to_write = 0;
		}
		if (nscan < INT_MAX)
			break;
		cond_resched();
	}
1927
	nfs_commit_end(cinfo.mds);
1928 1929 1930 1931 1932 1933 1934 1935
	if (ret || !may_wait)
		return ret;
	return wait_on_commit(cinfo.mds);
}

int nfs_commit_inode(struct inode *inode, int how)
{
	return __nfs_commit_inode(inode, how, NULL);
L
Linus Torvalds 已提交
1936
}
1937
EXPORT_SYMBOL_GPL(nfs_commit_inode);
1938

1939
int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1940
{
1941 1942 1943
	struct nfs_inode *nfsi = NFS_I(inode);
	int flags = FLUSH_SYNC;
	int ret = 0;
1944

1945
	if (wbc->sync_mode == WB_SYNC_NONE) {
1946 1947 1948 1949
		/* no commits means nothing needs to be done */
		if (!atomic_long_read(&nfsi->commit_info.ncommit))
			goto check_requests_outstanding;

1950 1951 1952
		/* Don't commit yet if this is a non-blocking flush and there
		 * are a lot of outstanding writes for this mapping.
		 */
1953
		if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
1954
			goto out_mark_dirty;
1955

1956
		/* don't wait for the COMMIT response */
1957
		flags = 0;
1958 1959
	}

1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
	ret = __nfs_commit_inode(inode, flags, wbc);
	if (!ret) {
		if (flags & FLUSH_SYNC)
			return 0;
	} else if (atomic_long_read(&nfsi->commit_info.ncommit))
		goto out_mark_dirty;

check_requests_outstanding:
	if (!atomic_read(&nfsi->commit_info.rpcs_out))
		return ret;
1970
out_mark_dirty:
1971 1972 1973
	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
	return ret;
}
B
Bryan Schumaker 已提交
1974
EXPORT_SYMBOL_GPL(nfs_write_inode);
1975

1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993
/*
 * Wrapper for filemap_write_and_wait_range()
 *
 * Needed for pNFS in order to ensure data becomes visible to the
 * client.
 */
int nfs_filemap_write_and_wait_range(struct address_space *mapping,
		loff_t lstart, loff_t lend)
{
	int ret;

	ret = filemap_write_and_wait_range(mapping, lstart, lend);
	if (ret == 0)
		ret = pnfs_sync_inode(mapping->host, true);
	return ret;
}
EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);

1994 1995 1996 1997
/*
 * flush the inode to disk.
 */
int nfs_wb_all(struct inode *inode)
T
Trond Myklebust 已提交
1998
{
1999 2000 2001 2002
	int ret;

	trace_nfs_writeback_inode_enter(inode);

2003
	ret = filemap_write_and_wait(inode->i_mapping);
2004 2005 2006 2007 2008 2009 2010
	if (ret)
		goto out;
	ret = nfs_commit_inode(inode, FLUSH_SYNC);
	if (ret < 0)
		goto out;
	pnfs_sync_inode(inode, true);
	ret = 0;
T
Trond Myklebust 已提交
2011

2012
out:
2013 2014
	trace_nfs_writeback_inode_exit(inode, ret);
	return ret;
2015
}
B
Bryan Schumaker 已提交
2016
EXPORT_SYMBOL_GPL(nfs_wb_all);
2017

2018 2019 2020 2021 2022
int nfs_wb_page_cancel(struct inode *inode, struct page *page)
{
	struct nfs_page *req;
	int ret = 0;

2023 2024 2025 2026
	wait_on_page_writeback(page);

	/* blocking call to cancel all requests and join to a single (head)
	 * request */
T
Trond Myklebust 已提交
2027
	req = nfs_lock_and_join_requests(page);
2028 2029 2030 2031 2032 2033 2034 2035 2036 2037

	if (IS_ERR(req)) {
		ret = PTR_ERR(req);
	} else if (req) {
		/* all requests from this page have been cancelled by
		 * nfs_lock_and_join_requests, so just remove the head
		 * request from the inode / page_private pointer and
		 * release it */
		nfs_inode_remove_request(req);
		nfs_unlock_and_release_request(req);
2038
	}
2039

2040 2041 2042
	return ret;
}

T
Trond Myklebust 已提交
2043 2044 2045
/*
 * Write back all requests on one page - we do this before reading it.
 */
2046
int nfs_wb_page(struct inode *inode, struct page *page)
2047
{
2048
	loff_t range_start = page_file_offset(page);
2049
	loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1);
2050 2051
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_ALL,
T
Trond Myklebust 已提交
2052
		.nr_to_write = 0,
2053 2054 2055 2056
		.range_start = range_start,
		.range_end = range_end,
	};
	int ret;
2057

2058 2059
	trace_nfs_writeback_page_enter(inode);

2060
	for (;;) {
2061
		wait_on_page_writeback(page);
2062
		if (clear_page_dirty_for_io(page)) {
2063
			ret = nfs_writepage_locked(page, &wbc);
2064 2065
			if (ret < 0)
				goto out_error;
2066
			continue;
T
Trond Myklebust 已提交
2067
		}
2068
		ret = 0;
2069 2070 2071
		if (!PagePrivate(page))
			break;
		ret = nfs_commit_inode(inode, FLUSH_SYNC);
2072
		if (ret < 0)
2073
			goto out_error;
T
Trond Myklebust 已提交
2074
	}
2075
out_error:
2076
	trace_nfs_writeback_page_exit(inode, ret);
2077
	return ret;
2078 2079
}

2080 2081
#ifdef CONFIG_MIGRATION
int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
2082
		struct page *page, enum migrate_mode mode)
2083
{
2084 2085 2086 2087 2088 2089 2090 2091 2092 2093
	/*
	 * If PagePrivate is set, then the page is currently associated with
	 * an in-progress read or write request. Don't try to migrate it.
	 *
	 * FIXME: we could do this in principle, but we'll need a way to ensure
	 *        that we can safely release the inode reference while holding
	 *        the page lock.
	 */
	if (PagePrivate(page))
		return -EBUSY;
2094

2095 2096
	if (!nfs_fscache_release_page(page, GFP_KERNEL))
		return -EBUSY;
2097

2098
	return migrate_page(mapping, newpage, page, mode);
2099 2100 2101
}
#endif

D
David Howells 已提交
2102
int __init nfs_init_writepagecache(void)
L
Linus Torvalds 已提交
2103 2104
{
	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
2105
					     sizeof(struct nfs_pgio_header),
L
Linus Torvalds 已提交
2106
					     0, SLAB_HWCACHE_ALIGN,
2107
					     NULL);
L
Linus Torvalds 已提交
2108 2109 2110
	if (nfs_wdata_cachep == NULL)
		return -ENOMEM;

2111 2112
	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
						     nfs_wdata_cachep);
L
Linus Torvalds 已提交
2113
	if (nfs_wdata_mempool == NULL)
2114
		goto out_destroy_write_cache;
L
Linus Torvalds 已提交
2115

2116 2117 2118 2119 2120
	nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
					     sizeof(struct nfs_commit_data),
					     0, SLAB_HWCACHE_ALIGN,
					     NULL);
	if (nfs_cdata_cachep == NULL)
2121
		goto out_destroy_write_mempool;
2122

2123
	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
2124
						      nfs_cdata_cachep);
L
Linus Torvalds 已提交
2125
	if (nfs_commit_mempool == NULL)
2126
		goto out_destroy_commit_cache;
L
Linus Torvalds 已提交
2127

2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143
	/*
	 * NFS congestion size, scale with available memory.
	 *
	 *  64MB:    8192k
	 * 128MB:   11585k
	 * 256MB:   16384k
	 * 512MB:   23170k
	 *   1GB:   32768k
	 *   2GB:   46340k
	 *   4GB:   65536k
	 *   8GB:   92681k
	 *  16GB:  131072k
	 *
	 * This allows larger machines to have larger/more transfers.
	 * Limit the default to 256M
	 */
2144
	nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10);
2145 2146 2147
	if (nfs_congestion_kb > 256*1024)
		nfs_congestion_kb = 256*1024;

L
Linus Torvalds 已提交
2148
	return 0;
2149 2150 2151 2152 2153 2154 2155 2156

out_destroy_commit_cache:
	kmem_cache_destroy(nfs_cdata_cachep);
out_destroy_write_mempool:
	mempool_destroy(nfs_wdata_mempool);
out_destroy_write_cache:
	kmem_cache_destroy(nfs_wdata_cachep);
	return -ENOMEM;
L
Linus Torvalds 已提交
2157 2158
}

2159
void nfs_destroy_writepagecache(void)
L
Linus Torvalds 已提交
2160 2161
{
	mempool_destroy(nfs_commit_mempool);
2162
	kmem_cache_destroy(nfs_cdata_cachep);
L
Linus Torvalds 已提交
2163
	mempool_destroy(nfs_wdata_mempool);
2164
	kmem_cache_destroy(nfs_wdata_cachep);
L
Linus Torvalds 已提交
2165 2166
}

2167 2168 2169
static const struct nfs_rw_ops nfs_rw_write_ops = {
	.rw_alloc_header	= nfs_writehdr_alloc,
	.rw_free_header		= nfs_writehdr_free,
2170 2171
	.rw_done		= nfs_writeback_done,
	.rw_result		= nfs_writeback_result,
2172
	.rw_initiate		= nfs_initiate_write,
2173
};