write.c 57.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * linux/fs/nfs/write.c
 *
4
 * Write file data over NFS.
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14
 *
 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/writeback.h>
15
#include <linux/swap.h>
16
#include <linux/migrate.h>
L
Linus Torvalds 已提交
17 18 19 20 21

#include <linux/sunrpc/clnt.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/nfs_page.h>
22
#include <linux/backing-dev.h>
23
#include <linux/export.h>
24 25
#include <linux/freezer.h>
#include <linux/wait.h>
26
#include <linux/iversion.h>
27

28
#include <linux/uaccess.h>
29
#include <linux/sched/mm.h>
L
Linus Torvalds 已提交
30 31

#include "delegation.h"
32
#include "internal.h"
33
#include "iostat.h"
34
#include "nfs4_fs.h"
35
#include "fscache.h"
36
#include "pnfs.h"
L
Linus Torvalds 已提交
37

38 39
#include "nfstrace.h"

L
Linus Torvalds 已提交
40 41 42 43 44
#define NFSDBG_FACILITY		NFSDBG_PAGECACHE

#define MIN_POOL_WRITE		(32)
#define MIN_POOL_COMMIT		(4)

45 46 47 48 49 50
struct nfs_io_completion {
	void (*complete)(void *data);
	void *data;
	struct kref refcount;
};

L
Linus Torvalds 已提交
51 52 53
/*
 * Local function declarations
 */
54
static void nfs_redirty_request(struct nfs_page *req);
55
static const struct rpc_call_ops nfs_commit_ops;
56
static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
57
static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
58
static const struct nfs_rw_ops nfs_rw_write_ops;
59
static void nfs_clear_request_commit(struct nfs_page *req);
60 61
static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
				      struct inode *inode);
62 63 64
static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
						struct page *page);
L
Linus Torvalds 已提交
65

66
static struct kmem_cache *nfs_wdata_cachep;
67
static mempool_t *nfs_wdata_mempool;
68
static struct kmem_cache *nfs_cdata_cachep;
L
Linus Torvalds 已提交
69 70
static mempool_t *nfs_commit_mempool;

N
NeilBrown 已提交
71
struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail)
L
Linus Torvalds 已提交
72
{
N
NeilBrown 已提交
73
	struct nfs_commit_data *p;
74

N
NeilBrown 已提交
75 76 77 78 79 80 81 82 83 84 85 86 87 88
	if (never_fail)
		p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
	else {
		/* It is OK to do some reclaim, not no safe to wait
		 * for anything to be returned to the pool.
		 * mempool_alloc() cannot handle that particular combination,
		 * so we need two separate attempts.
		 */
		p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
		if (!p)
			p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO |
					     __GFP_NOWARN | __GFP_NORETRY);
		if (!p)
			return NULL;
L
Linus Torvalds 已提交
89
	}
N
NeilBrown 已提交
90 91 92

	memset(p, 0, sizeof(*p));
	INIT_LIST_HEAD(&p->pages);
L
Linus Torvalds 已提交
93 94
	return p;
}
95
EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
L
Linus Torvalds 已提交
96

97
void nfs_commit_free(struct nfs_commit_data *p)
L
Linus Torvalds 已提交
98 99 100
{
	mempool_free(p, nfs_commit_mempool);
}
101
EXPORT_SYMBOL_GPL(nfs_commit_free);
L
Linus Torvalds 已提交
102

103
static struct nfs_pgio_header *nfs_writehdr_alloc(void)
104
{
105
	struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
106

107 108
	memset(p, 0, sizeof(*p));
	p->rw_mode = FMODE_WRITE;
109 110
	return p;
}
111

112
static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
113
{
114
	mempool_free(hdr, nfs_wdata_mempool);
115
}
L
Linus Torvalds 已提交
116

117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
{
	return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
}

static void nfs_io_completion_init(struct nfs_io_completion *ioc,
		void (*complete)(void *), void *data)
{
	ioc->complete = complete;
	ioc->data = data;
	kref_init(&ioc->refcount);
}

static void nfs_io_completion_release(struct kref *kref)
{
	struct nfs_io_completion *ioc = container_of(kref,
			struct nfs_io_completion, refcount);
	ioc->complete(ioc->data);
	kfree(ioc);
}

static void nfs_io_completion_get(struct nfs_io_completion *ioc)
{
	if (ioc != NULL)
		kref_get(&ioc->refcount);
}

static void nfs_io_completion_put(struct nfs_io_completion *ioc)
{
	if (ioc != NULL)
		kref_put(&ioc->refcount, nfs_io_completion_release);
}

150 151 152 153 154 155 156 157
static struct nfs_page *
nfs_page_private_request(struct page *page)
{
	if (!PagePrivate(page))
		return NULL;
	return (struct nfs_page *)page_private(page);
}

158 159 160 161 162 163 164
/*
 * nfs_page_find_head_request_locked - find head request associated with @page
 *
 * must be called while holding the inode lock.
 *
 * returns matching head request with reference held, or NULL if not found.
 */
165
static struct nfs_page *
166
nfs_page_find_private_request(struct page *page)
167
{
168
	struct address_space *mapping = page_file_mapping(page);
169
	struct nfs_page *req;
170

171 172
	if (!PagePrivate(page))
		return NULL;
173
	spin_lock(&mapping->private_lock);
174
	req = nfs_page_private_request(page);
175 176
	if (req) {
		WARN_ON_ONCE(req->wb_head != req);
177
		kref_get(&req->wb_kref);
178
	}
179
	spin_unlock(&mapping->private_lock);
180 181
	return req;
}
182

183 184 185 186 187 188 189 190
static struct nfs_page *
nfs_page_find_swap_request(struct page *page)
{
	struct inode *inode = page_file_mapping(page)->host;
	struct nfs_inode *nfsi = NFS_I(inode);
	struct nfs_page *req = NULL;
	if (!PageSwapCache(page))
		return NULL;
191
	mutex_lock(&nfsi->commit_mutex);
192 193 194 195 196 197 198 199
	if (PageSwapCache(page)) {
		req = nfs_page_search_commits_for_head_request_locked(nfsi,
			page);
		if (req) {
			WARN_ON_ONCE(req->wb_head != req);
			kref_get(&req->wb_kref);
		}
	}
200
	mutex_unlock(&nfsi->commit_mutex);
201 202 203
	return req;
}

204 205 206 207 208 209
/*
 * nfs_page_find_head_request - find head request associated with @page
 *
 * returns matching head request with reference held, or NULL if not found.
 */
static struct nfs_page *nfs_page_find_head_request(struct page *page)
210
{
211
	struct nfs_page *req;
212

213 214 215
	req = nfs_page_find_private_request(page);
	if (!req)
		req = nfs_page_find_swap_request(page);
216 217 218
	return req;
}

L
Linus Torvalds 已提交
219 220 221
/* Adjust the file length if we're writing beyond the end */
static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
{
222
	struct inode *inode = page_file_mapping(page)->host;
223 224
	loff_t end, i_size;
	pgoff_t end_index;
L
Linus Torvalds 已提交
225

226 227
	spin_lock(&inode->i_lock);
	i_size = i_size_read(inode);
228
	end_index = (i_size - 1) >> PAGE_SHIFT;
H
Huang Ying 已提交
229
	if (i_size > 0 && page_index(page) < end_index)
230
		goto out;
231
	end = page_file_offset(page) + ((loff_t)offset+count);
L
Linus Torvalds 已提交
232
	if (i_size >= end)
233
		goto out;
L
Linus Torvalds 已提交
234
	i_size_write(inode, end);
235
	NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
236 237 238
	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
out:
	spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
239 240
}

241
/* A writeback failed: mark the page as bad, and invalidate the page cache */
242
static void nfs_set_pageerror(struct address_space *mapping)
243
{
244
	nfs_zap_mapping(mapping->host, mapping);
245 246
}

247 248 249 250 251 252
static void nfs_mapping_set_error(struct page *page, int error)
{
	SetPageError(page);
	mapping_set_error(page_file_mapping(page), error);
}

253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
/*
 * nfs_page_group_search_locked
 * @head - head request of page group
 * @page_offset - offset into page
 *
 * Search page group with head @head to find a request that contains the
 * page offset @page_offset.
 *
 * Returns a pointer to the first matching nfs request, or NULL if no
 * match is found.
 *
 * Must be called with the page group lock held
 */
static struct nfs_page *
nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
{
	struct nfs_page *req;

	req = head;
	do {
		if (page_offset >= req->wb_pgbase &&
		    page_offset < (req->wb_pgbase + req->wb_bytes))
			return req;

		req = req->wb_this_page;
	} while (req != head);

	return NULL;
}

/*
 * nfs_page_group_covers_page
 * @head - head request of page group
 *
 * Return true if the page group with head @head covers the whole page,
 * returns false otherwise
 */
static bool nfs_page_group_covers_page(struct nfs_page *req)
{
	struct nfs_page *tmp;
	unsigned int pos = 0;
	unsigned int len = nfs_page_length(req->wb_page);

296
	nfs_page_group_lock(req);
297

298
	for (;;) {
299
		tmp = nfs_page_group_search_locked(req->wb_head, pos);
300 301 302 303
		if (!tmp)
			break;
		pos = tmp->wb_pgbase + tmp->wb_bytes;
	}
304 305

	nfs_page_group_unlock(req);
306
	return pos >= len;
307 308
}

L
Linus Torvalds 已提交
309 310 311
/* We can set the PG_uptodate flag if we see that a write request
 * covers the full page.
 */
312
static void nfs_mark_uptodate(struct nfs_page *req)
L
Linus Torvalds 已提交
313
{
314
	if (PageUptodate(req->wb_page))
L
Linus Torvalds 已提交
315
		return;
316
	if (!nfs_page_group_covers_page(req))
L
Linus Torvalds 已提交
317
		return;
318
	SetPageUptodate(req->wb_page);
L
Linus Torvalds 已提交
319 320 321 322
}

static int wb_priority(struct writeback_control *wbc)
{
323
	int ret = 0;
324

325 326 327
	if (wbc->sync_mode == WB_SYNC_ALL)
		ret = FLUSH_COND_STABLE;
	return ret;
L
Linus Torvalds 已提交
328 329
}

330 331 332 333 334 335 336 337 338 339
/*
 * NFS congestion control
 */

int nfs_congestion_kb;

#define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
#define NFS_CONGESTION_OFF_THRESH	\
	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))

340
static void nfs_set_page_writeback(struct page *page)
341
{
342 343
	struct inode *inode = page_file_mapping(page)->host;
	struct nfs_server *nfss = NFS_SERVER(inode);
344 345
	int ret = test_set_page_writeback(page);

346
	WARN_ON_ONCE(ret != 0);
347

348
	if (atomic_long_inc_return(&nfss->writeback) >
349 350
			NFS_CONGESTION_ON_THRESH)
		set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
351 352
}

353
static void nfs_end_page_writeback(struct nfs_page *req)
354
{
355
	struct inode *inode = page_file_mapping(req->wb_page)->host;
356
	struct nfs_server *nfss = NFS_SERVER(inode);
357
	bool is_done;
358

359 360 361
	is_done = nfs_page_group_sync_on_bit(req, PG_WB_END);
	nfs_unlock_request(req);
	if (!is_done)
362 363 364
		return;

	end_page_writeback(req->wb_page);
365
	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
366
		clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
367 368
}

369 370 371 372 373 374 375 376 377
/*
 * nfs_unroll_locks_and_wait -  unlock all newly locked reqs and wait on @req
 *
 * this is a helper function for nfs_lock_and_join_requests
 *
 * @inode - inode associated with request page group, must be holding inode lock
 * @head  - head request of page group, must be holding head lock
 * @req   - request that couldn't lock and needs to wait on the req bit lock
 *
378 379
 * NOTE: this must be called holding page_group bit lock
 *       which will be released before returning.
380 381 382
 *
 * returns 0 on success, < 0 on error.
 */
383 384
static void
nfs_unroll_locks(struct inode *inode, struct nfs_page *head,
385
			  struct nfs_page *req)
386 387 388 389
{
	struct nfs_page *tmp;

	/* relinquish all the locks successfully grabbed this run */
390 391 392 393 394
	for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
		if (!kref_read(&tmp->wb_kref))
			continue;
		nfs_unlock_and_release_request(tmp);
	}
395 396 397 398 399 400 401 402 403 404 405 406 407 408
}

/*
 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
 *
 * @destroy_list - request list (using wb_this_page) terminated by @old_head
 * @old_head - the old head of the list
 *
 * All subrequests must be locked and removed from all lists, so at this point
 * they are only "active" in this function, and possibly in nfs_wait_on_request
 * with a reference held by some other context.
 */
static void
nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
409 410
				 struct nfs_page *old_head,
				 struct inode *inode)
411 412 413 414 415 416 417 418 419 420 421 422
{
	while (destroy_list) {
		struct nfs_page *subreq = destroy_list;

		destroy_list = (subreq->wb_this_page == old_head) ?
				   NULL : subreq->wb_this_page;

		WARN_ON_ONCE(old_head != subreq->wb_head);

		/* make sure old group is not used */
		subreq->wb_this_page = subreq;

423 424
		clear_bit(PG_REMOVE, &subreq->wb_flags);

425 426 427
		/* Note: races with nfs_page_group_destroy() */
		if (!kref_read(&subreq->wb_kref)) {
			/* Check if we raced with nfs_page_group_destroy() */
428
			if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags))
429 430 431
				nfs_free_request(subreq);
			continue;
		}
432

433 434 435 436
		subreq->wb_head = subreq;

		if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) {
			nfs_release_request(subreq);
437
			atomic_long_dec(&NFS_I(inode)->nrequests);
438
		}
439 440 441 442

		/* subreq is now totally disconnected from page group or any
		 * write / commit lists. last chance to wake any waiters */
		nfs_unlock_and_release_request(subreq);
443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
	}
}

/*
 * nfs_lock_and_join_requests - join all subreqs to the head req and return
 *                              a locked reference, cancelling any pending
 *                              operations for this page.
 *
 * @page - the page used to lookup the "page group" of nfs_page structures
 *
 * This function joins all sub requests to the head request by first
 * locking all requests in the group, cancelling any pending operations
 * and finally updating the head request to cover the whole range covered by
 * the (former) group.  All subrequests are removed from any write or commit
 * lists, unlinked from the group and destroyed.
 *
 * Returns a locked, referenced pointer to the head request - which after
 * this call is guaranteed to be the only request associated with the page.
 * Returns NULL if no requests are found for @page, or a ERR_PTR if an
 * error was encountered.
 */
static struct nfs_page *
465
nfs_lock_and_join_requests(struct page *page)
466
{
467
	struct inode *inode = page_file_mapping(page)->host;
468 469 470
	struct nfs_page *head, *subreq;
	struct nfs_page *destroy_list = NULL;
	unsigned int total_bytes;
471 472
	int ret;

473 474 475 476 477 478
try_again:
	/*
	 * A reference is taken only on the head request which acts as a
	 * reference to the whole page group - the group will not be destroyed
	 * until the head reference is released.
	 */
479 480
	head = nfs_page_find_head_request(page);
	if (!head)
481 482
		return NULL;

483 484 485 486 487 488 489 490
	/* lock the page head first in order to avoid an ABBA inefficiency */
	if (!nfs_lock_request(head)) {
		ret = nfs_wait_on_request(head);
		nfs_release_request(head);
		if (ret < 0)
			return ERR_PTR(ret);
		goto try_again;
	}
491 492 493 494 495 496

	/* Ensure that nobody removed the request before we locked it */
	if (head != nfs_page_private_request(page) && !PageSwapCache(page)) {
		nfs_unlock_and_release_request(head);
		goto try_again;
	}
497

498
	ret = nfs_page_group_lock(head);
499 500
	if (ret < 0)
		goto release_request;
501 502

	/* lock each request in the page group */
503 504 505
	total_bytes = head->wb_bytes;
	for (subreq = head->wb_this_page; subreq != head;
			subreq = subreq->wb_this_page) {
506

507 508 509
		if (!kref_get_unless_zero(&subreq->wb_kref)) {
			if (subreq->wb_offset == head->wb_offset + total_bytes)
				total_bytes += subreq->wb_bytes;
510
			continue;
511 512
		}

513
		while (!nfs_lock_request(subreq)) {
514
			/*
515 516
			 * Unlock page to allow nfs_page_group_sync_on_bit()
			 * to succeed
517
			 */
518 519 520
			nfs_page_group_unlock(head);
			ret = nfs_wait_on_request(subreq);
			if (!ret)
521
				ret = nfs_page_group_lock(head);
522 523
			if (ret < 0) {
				nfs_unroll_locks(inode, head, subreq);
524
				nfs_release_request(subreq);
525
				goto release_request;
526
			}
527
		}
528 529
		/*
		 * Subrequests are always contiguous, non overlapping
530
		 * and in order - but may be repeated (mirrored writes).
531
		 */
532 533 534 535 536 537
		if (subreq->wb_offset == (head->wb_offset + total_bytes)) {
			/* keep track of how many bytes this group covers */
			total_bytes += subreq->wb_bytes;
		} else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset ||
			    ((subreq->wb_offset + subreq->wb_bytes) >
			     (head->wb_offset + total_bytes)))) {
538
			nfs_page_group_unlock(head);
539
			nfs_unroll_locks(inode, head, subreq);
540
			nfs_unlock_and_release_request(subreq);
541 542
			ret = -EIO;
			goto release_request;
543
		}
544
	}
545 546 547 548 549

	/* Now that all requests are locked, make sure they aren't on any list.
	 * Commit list removal accounting is done after locks are dropped */
	subreq = head;
	do {
550
		nfs_clear_request_commit(subreq);
551 552 553 554 555 556 557 558 559 560 561 562
		subreq = subreq->wb_this_page;
	} while (subreq != head);

	/* unlink subrequests from head, destroy them later */
	if (head->wb_this_page != head) {
		/* destroy list will be terminated by head */
		destroy_list = head->wb_this_page;
		head->wb_this_page = head;

		/* change head request to cover whole range that
		 * the former page group covered */
		head->wb_bytes = total_bytes;
563
	}
564

565 566 567 568
	/* Postpone destruction of this request */
	if (test_and_clear_bit(PG_REMOVE, &head->wb_flags)) {
		set_bit(PG_INODE_REF, &head->wb_flags);
		kref_get(&head->wb_kref);
569
		atomic_long_inc(&NFS_I(inode)->nrequests);
570 571
	}

572 573
	nfs_page_group_unlock(head);

574
	nfs_destroy_unlinked_subrequests(destroy_list, head, inode);
575

576 577 578 579 580 581
	/* Did we lose a race with nfs_inode_remove_request()? */
	if (!(PagePrivate(page) || PageSwapCache(page))) {
		nfs_unlock_and_release_request(head);
		return NULL;
	}

582
	/* still holds ref on head from nfs_page_find_head_request
583 584
	 * and still has lock on head from lock loop */
	return head;
585 586 587 588

release_request:
	nfs_unlock_and_release_request(head);
	return ERR_PTR(ret);
589 590
}

591
static void nfs_write_error(struct nfs_page *req, int error)
592
{
593
	nfs_mapping_set_error(req->wb_page, error);
594
	nfs_end_page_writeback(req);
595
	nfs_release_request(req);
596 597
}

598 599 600 601 602 603 604 605 606 607
static bool
nfs_error_is_fatal_on_server(int err)
{
	switch (err) {
	case 0:
	case -ERESTARTSYS:
	case -EINTR:
		return false;
	}
	return nfs_error_is_fatal(err);
608 609
}

610 611 612 613 614
/*
 * Find an associated nfs write request, and prepare to flush it out
 * May return an error if the user signalled nfs_wait_on_request().
 */
static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
615
				struct page *page)
616
{
617
	struct address_space *mapping;
618 619 620
	struct nfs_page *req;
	int ret = 0;

621
	req = nfs_lock_and_join_requests(page);
622 623 624 625 626 627
	if (!req)
		goto out;
	ret = PTR_ERR(req);
	if (IS_ERR(req))
		goto out;

628 629
	nfs_set_page_writeback(page);
	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
630

631
	/* If there is a fatal error that covers this write, just exit */
632 633 634 635
	ret = 0;
	mapping = page_file_mapping(page);
	if (test_bit(AS_ENOSPC, &mapping->flags) ||
	    test_bit(AS_EIO, &mapping->flags))
636 637
		goto out_launder;

638
	if (!nfs_pageio_add_request(pgio, req)) {
639
		ret = pgio->pg_error;
640
		/*
641
		 * Remove the problematic req upon fatal errors on the server
642 643
		 */
		if (nfs_error_is_fatal(ret)) {
644
			if (nfs_error_is_fatal_on_server(ret))
645
				goto out_launder;
646 647
		} else
			ret = -EAGAIN;
648
		nfs_redirty_request(req);
649 650 651
	} else
		nfs_add_stats(page_file_mapping(page)->host,
				NFSIOS_WRITEPAGES, 1);
652 653
out:
	return ret;
654
out_launder:
655
	nfs_write_error(req, ret);
656
	return 0;
657 658
}

659
static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
660
			    struct nfs_pageio_descriptor *pgio)
L
Linus Torvalds 已提交
661
{
662
	int ret;
L
Linus Torvalds 已提交
663

H
Huang Ying 已提交
664
	nfs_pageio_cond_complete(pgio, page_index(page));
665
	ret = nfs_page_async_flush(pgio, page);
666 667 668 669 670
	if (ret == -EAGAIN) {
		redirty_page_for_writepage(wbc, page);
		ret = 0;
	}
	return ret;
671
}
672

673 674 675
/*
 * Write an mmapped page to the server.
 */
676
static int nfs_writepage_locked(struct page *page,
677
				struct writeback_control *wbc)
678 679
{
	struct nfs_pageio_descriptor pgio;
680
	struct inode *inode = page_file_mapping(page)->host;
681
	int err;
682

683
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
684
	nfs_pageio_init_write(&pgio, inode, 0,
685
				false, &nfs_async_write_completion_ops);
686
	err = nfs_do_writepage(page, wbc, &pgio);
687 688 689 690 691 692
	nfs_pageio_complete(&pgio);
	if (err < 0)
		return err;
	if (pgio.pg_error < 0)
		return pgio.pg_error;
	return 0;
693 694 695 696
}

int nfs_writepage(struct page *page, struct writeback_control *wbc)
{
697
	int ret;
698

699
	ret = nfs_writepage_locked(page, wbc);
L
Linus Torvalds 已提交
700
	unlock_page(page);
701 702 703 704 705 706 707
	return ret;
}

static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
{
	int ret;

708
	ret = nfs_do_writepage(page, wbc, data);
709 710
	unlock_page(page);
	return ret;
L
Linus Torvalds 已提交
711 712
}

713 714 715 716 717
static void nfs_io_completion_commit(void *inode)
{
	nfs_commit_inode(inode, 0);
}

L
Linus Torvalds 已提交
718 719 720
int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;
721
	struct nfs_pageio_descriptor pgio;
722 723
	struct nfs_io_completion *ioc;
	unsigned int pflags = memalloc_nofs_save();
L
Linus Torvalds 已提交
724 725
	int err;

726 727
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);

728
	ioc = nfs_io_completion_alloc(GFP_NOFS);
729 730 731
	if (ioc)
		nfs_io_completion_init(ioc, nfs_io_completion_commit, inode);

732 733
	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
				&nfs_async_write_completion_ops);
734
	pgio.pg_io_completion = ioc;
735
	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
736
	nfs_pageio_complete(&pgio);
737
	nfs_io_completion_put(ioc);
738

739 740
	memalloc_nofs_restore(pflags);

741
	if (err < 0)
742 743 744 745
		goto out_err;
	err = pgio.pg_error;
	if (err < 0)
		goto out_err;
746
	return 0;
747 748
out_err:
	return err;
L
Linus Torvalds 已提交
749 750 751 752 753
}

/*
 * Insert a write request into an inode
 */
754
static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
L
Linus Torvalds 已提交
755
{
756
	struct address_space *mapping = page_file_mapping(req->wb_page);
L
Linus Torvalds 已提交
757
	struct nfs_inode *nfsi = NFS_I(inode);
758

759 760
	WARN_ON_ONCE(req->wb_this_page != req);

761
	/* Lock the request! */
762
	nfs_lock_request(req);
763

764 765 766 767
	/*
	 * Swap-space should not get truncated. Hence no need to plug the race
	 * with invalidate/truncate.
	 */
768 769
	spin_lock(&mapping->private_lock);
	if (!nfs_have_writebacks(inode) &&
770 771
	    NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
		inode_inc_iversion_raw(inode);
772 773 774 775 776
	if (likely(!PageSwapCache(req->wb_page))) {
		set_bit(PG_MAPPED, &req->wb_flags);
		SetPagePrivate(req->wb_page);
		set_page_private(req->wb_page, (unsigned long)req);
	}
777
	spin_unlock(&mapping->private_lock);
778
	atomic_long_inc(&nfsi->nrequests);
779
	/* this a head request for a page group - mark it as having an
780 781 782
	 * extra reference so sub groups can follow suit.
	 * This flag also informs pgio layer when to bump nrequests when
	 * adding subrequests. */
783
	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
784
	kref_get(&req->wb_kref);
L
Linus Torvalds 已提交
785 786 787
}

/*
788
 * Remove a write request from an inode
L
Linus Torvalds 已提交
789 790 791
 */
static void nfs_inode_remove_request(struct nfs_page *req)
{
792 793
	struct address_space *mapping = page_file_mapping(req->wb_page);
	struct inode *inode = mapping->host;
L
Linus Torvalds 已提交
794
	struct nfs_inode *nfsi = NFS_I(inode);
795
	struct nfs_page *head;
L
Linus Torvalds 已提交
796

797
	atomic_long_dec(&nfsi->nrequests);
798 799 800
	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
		head = req->wb_head;

801
		spin_lock(&mapping->private_lock);
802
		if (likely(head->wb_page && !PageSwapCache(head->wb_page))) {
803 804 805 806
			set_page_private(head->wb_page, 0);
			ClearPagePrivate(head->wb_page);
			clear_bit(PG_MAPPED, &head->wb_flags);
		}
807
		spin_unlock(&mapping->private_lock);
808
	}
809 810 811

	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags))
		nfs_release_request(req);
L
Linus Torvalds 已提交
812 813
}

814
static void
F
Fred 已提交
815
nfs_mark_request_dirty(struct nfs_page *req)
816
{
817 818
	if (req->wb_page)
		__set_page_dirty_nobuffers(req->wb_page);
819 820
}

821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
/*
 * nfs_page_search_commits_for_head_request_locked
 *
 * Search through commit lists on @inode for the head request for @page.
 * Must be called while holding the inode (which is cinfo) lock.
 *
 * Returns the head request if found, or NULL if not found.
 */
static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
						struct page *page)
{
	struct nfs_page *freq, *t;
	struct nfs_commit_info cinfo;
	struct inode *inode = &nfsi->vfs_inode;

	nfs_init_cinfo_from_inode(&cinfo, inode);

	/* search through pnfs commit lists */
	freq = pnfs_search_commit_reqs(inode, &cinfo, page);
	if (freq)
		return freq->wb_head;

	/* Linearly search the commit list for the correct request */
	list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) {
		if (freq->wb_page == page)
			return freq->wb_head;
	}

	return NULL;
}

853 854 855 856 857 858 859 860 861 862
/**
 * nfs_request_add_commit_list_locked - add request to a commit list
 * @req: pointer to a struct nfs_page
 * @dst: commit list head
 * @cinfo: holds list lock and accounting info
 *
 * This sets the PG_CLEAN bit, updates the cinfo count of
 * number of outstanding requests requiring a commit as well as
 * the MM page stats.
 *
863 864
 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
 * nfs_page lock.
865 866 867 868 869 870 871
 */
void
nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
			    struct nfs_commit_info *cinfo)
{
	set_bit(PG_CLEAN, &req->wb_flags);
	nfs_list_add_request(req, dst);
872
	atomic_long_inc(&cinfo->mds->ncommit);
873 874 875
}
EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);

876 877 878
/**
 * nfs_request_add_commit_list - add request to a commit list
 * @req: pointer to a struct nfs_page
879
 * @cinfo: holds list lock and accounting info
880
 *
881
 * This sets the PG_CLEAN bit, updates the cinfo count of
882 883 884
 * number of outstanding requests requiring a commit as well as
 * the MM page stats.
 *
885
 * The caller must _not_ hold the cinfo->lock, but must be
886
 * holding the nfs_page lock.
L
Linus Torvalds 已提交
887
 */
888
void
889
nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
890
{
891
	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
892
	nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
893
	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
894 895
	if (req->wb_page)
		nfs_mark_page_unstable(req->wb_page, cinfo);
L
Linus Torvalds 已提交
896
}
897 898 899 900 901
EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);

/**
 * nfs_request_remove_commit_list - Remove request from a commit list
 * @req: pointer to a nfs_page
902
 * @cinfo: holds list lock and accounting info
903
 *
904
 * This clears the PG_CLEAN bit, and updates the cinfo's count of
905 906 907
 * number of outstanding requests requiring a commit
 * It does not update the MM page stats.
 *
908
 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
909 910
 */
void
911 912
nfs_request_remove_commit_list(struct nfs_page *req,
			       struct nfs_commit_info *cinfo)
913 914 915 916
{
	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
		return;
	nfs_list_remove_request(req);
917
	atomic_long_dec(&cinfo->mds->ncommit);
918 919 920
}
EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);

921 922 923
static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
				      struct inode *inode)
{
924
	cinfo->inode = inode;
925 926
	cinfo->mds = &NFS_I(inode)->commit_info;
	cinfo->ds = pnfs_get_ds_info(inode);
927
	cinfo->dreq = NULL;
928
	cinfo->completion_ops = &nfs_commit_completion_ops;
929 930 931 932 933 934
}

void nfs_init_cinfo(struct nfs_commit_info *cinfo,
		    struct inode *inode,
		    struct nfs_direct_req *dreq)
{
935 936 937 938
	if (dreq)
		nfs_init_cinfo_from_dreq(cinfo, dreq);
	else
		nfs_init_cinfo_from_inode(cinfo, inode);
939 940
}
EXPORT_SYMBOL_GPL(nfs_init_cinfo);
941 942 943 944

/*
 * Add a request to the inode's commit list.
 */
945
void
946
nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
947
			struct nfs_commit_info *cinfo, u32 ds_commit_idx)
948
{
949
	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
950
		return;
951
	nfs_request_add_commit_list(req, cinfo);
952
}
953

954 955 956
static void
nfs_clear_page_commit(struct page *page)
{
957
	dec_node_page_state(page, NR_UNSTABLE_NFS);
958 959
	dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb,
		    WB_RECLAIMABLE);
960 961
}

962
/* Called holding the request lock on @req */
963
static void
964 965
nfs_clear_request_commit(struct nfs_page *req)
{
966
	if (test_bit(PG_CLEAN, &req->wb_flags)) {
967
		struct inode *inode = d_inode(req->wb_context->dentry);
968
		struct nfs_commit_info cinfo;
969

970
		nfs_init_cinfo_from_inode(&cinfo, inode);
971
		mutex_lock(&NFS_I(inode)->commit_mutex);
972 973
		if (!pnfs_clear_request_commit(req, &cinfo)) {
			nfs_request_remove_commit_list(req, &cinfo);
974
		}
975
		mutex_unlock(&NFS_I(inode)->commit_mutex);
976
		nfs_clear_page_commit(req->wb_page);
977 978 979
	}
}

980
int nfs_write_need_commit(struct nfs_pgio_header *hdr)
981
{
982
	if (hdr->verf.committed == NFS_DATA_SYNC)
983
		return hdr->lseg == NULL;
984
	return hdr->verf.committed != NFS_FILE_SYNC;
985 986
}

987 988 989 990 991
static void nfs_async_write_init(struct nfs_pgio_header *hdr)
{
	nfs_io_completion_get(hdr->io_completion);
}

992
static void nfs_write_completion(struct nfs_pgio_header *hdr)
993
{
994
	struct nfs_commit_info cinfo;
995 996 997 998
	unsigned long bytes = 0;

	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
		goto out;
999
	nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
1000 1001 1002 1003 1004 1005 1006
	while (!list_empty(&hdr->pages)) {
		struct nfs_page *req = nfs_list_entry(hdr->pages.next);

		bytes += req->wb_bytes;
		nfs_list_remove_request(req);
		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
		    (hdr->good_bytes < bytes)) {
1007
			nfs_set_pageerror(page_file_mapping(req->wb_page));
1008
			nfs_mapping_set_error(req->wb_page, hdr->error);
1009 1010
			goto remove_req;
		}
1011
		if (nfs_write_need_commit(hdr)) {
1012
			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
1013
			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
1014
				hdr->pgio_mirror_idx);
1015 1016 1017 1018 1019
			goto next;
		}
remove_req:
		nfs_inode_remove_request(req);
next:
1020
		nfs_end_page_writeback(req);
1021
		nfs_release_request(req);
1022 1023
	}
out:
1024
	nfs_io_completion_put(hdr->io_completion);
1025
	hdr->release(hdr);
1026
}
L
Linus Torvalds 已提交
1027

1028
unsigned long
1029
nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
1030
{
1031
	return atomic_long_read(&cinfo->mds->ncommit);
1032 1033
}

1034
/* NFS_I(cinfo->inode)->commit_mutex held by caller */
1035
int
1036 1037
nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
		     struct nfs_commit_info *cinfo, int max)
1038
{
1039
	struct nfs_page *req, *tmp;
1040 1041
	int ret = 0;

1042 1043
restart:
	list_for_each_entry_safe(req, tmp, src, wb_list) {
1044
		kref_get(&req->wb_kref);
1045 1046
		if (!nfs_lock_request(req)) {
			int status;
1047 1048 1049 1050 1051 1052 1053

			/* Prevent deadlock with nfs_lock_and_join_requests */
			if (!list_empty(dst)) {
				nfs_release_request(req);
				continue;
			}
			/* Ensure we make progress to prevent livelock */
1054 1055 1056 1057 1058 1059
			mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
			status = nfs_wait_on_request(req);
			nfs_release_request(req);
			mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
			if (status < 0)
				break;
1060
			goto restart;
1061
		}
1062
		nfs_request_remove_commit_list(req, cinfo);
1063
		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
1064 1065
		nfs_list_add_request(req, dst);
		ret++;
1066
		if ((ret == max) && !cinfo->dreq)
1067
			break;
1068
		cond_resched();
1069 1070
	}
	return ret;
1071
}
1072
EXPORT_SYMBOL_GPL(nfs_scan_commit_list);
1073

L
Linus Torvalds 已提交
1074 1075 1076
/*
 * nfs_scan_commit - Scan an inode for commit requests
 * @inode: NFS inode to scan
1077 1078
 * @dst: mds destination list
 * @cinfo: mds and ds lists of reqs ready to commit
L
Linus Torvalds 已提交
1079 1080 1081 1082
 *
 * Moves requests from the inode's 'commit' request list.
 * The requests are *not* checked to ensure that they form a contiguous set.
 */
1083
int
1084 1085
nfs_scan_commit(struct inode *inode, struct list_head *dst,
		struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
1086
{
1087
	int ret = 0;
1088

1089 1090
	if (!atomic_long_read(&cinfo->mds->ncommit))
		return 0;
1091
	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1092
	if (atomic_long_read(&cinfo->mds->ncommit) > 0) {
1093
		const int max = INT_MAX;
1094

1095 1096 1097
		ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
					   cinfo, max);
		ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
1098
	}
1099
	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1100
	return ret;
L
Linus Torvalds 已提交
1101
}
1102

L
Linus Torvalds 已提交
1103
/*
1104 1105
 * Search for an existing write request, and attempt to update
 * it to reflect a new dirty region on a given page.
L
Linus Torvalds 已提交
1106
 *
1107 1108
 * If the attempt fails, then the existing request is flushed out
 * to disk.
L
Linus Torvalds 已提交
1109
 */
1110 1111 1112 1113
static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
		struct page *page,
		unsigned int offset,
		unsigned int bytes)
L
Linus Torvalds 已提交
1114
{
1115 1116 1117 1118 1119
	struct nfs_page *req;
	unsigned int rqend;
	unsigned int end;
	int error;

L
Linus Torvalds 已提交
1120 1121
	end = offset + bytes;

1122 1123 1124
	req = nfs_lock_and_join_requests(page);
	if (IS_ERR_OR_NULL(req))
		return req;
L
Linus Torvalds 已提交
1125

1126 1127 1128 1129 1130 1131 1132 1133 1134
	rqend = req->wb_offset + req->wb_bytes;
	/*
	 * Tell the caller to flush out the request if
	 * the offsets are non-contiguous.
	 * Note: nfs_flush_incompatible() will already
	 * have flushed out requests having wrong owners.
	 */
	if (offset > rqend || end < req->wb_offset)
		goto out_flushme;
L
Linus Torvalds 已提交
1135 1136 1137 1138 1139 1140 1141 1142

	/* Okay, the request matches. Update the region */
	if (offset < req->wb_offset) {
		req->wb_offset = offset;
		req->wb_pgbase = offset;
	}
	if (end > rqend)
		req->wb_bytes = end - req->wb_offset;
1143 1144 1145 1146
	else
		req->wb_bytes = rqend - req->wb_offset;
	return req;
out_flushme:
1147 1148 1149 1150 1151 1152 1153
	/*
	 * Note: we mark the request dirty here because
	 * nfs_lock_and_join_requests() cannot preserve
	 * commit flags, so we have to replay the write.
	 */
	nfs_mark_request_dirty(req);
	nfs_unlock_and_release_request(req);
1154
	error = nfs_wb_page(inode, page);
1155
	return (error < 0) ? ERR_PTR(error) : NULL;
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
}

/*
 * Try to update an existing write request, or create one if there is none.
 *
 * Note: Should always be called with the Page Lock held to prevent races
 * if we have to add a new request. Also assumes that the caller has
 * already called nfs_flush_incompatible() if necessary.
 */
static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
		struct page *page, unsigned int offset, unsigned int bytes)
{
1168
	struct inode *inode = page_file_mapping(page)->host;
1169
	struct nfs_page	*req;
L
Linus Torvalds 已提交
1170

1171 1172 1173
	req = nfs_try_to_update_request(inode, page, offset, bytes);
	if (req != NULL)
		goto out;
1174
	req = nfs_create_request(ctx, page, offset, bytes);
1175 1176
	if (IS_ERR(req))
		goto out;
1177
	nfs_inode_add_request(inode, req);
1178
out:
1179
	return req;
L
Linus Torvalds 已提交
1180 1181
}

1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
		unsigned int offset, unsigned int count)
{
	struct nfs_page	*req;

	req = nfs_setup_write_request(ctx, page, offset, count);
	if (IS_ERR(req))
		return PTR_ERR(req);
	/* Update file length */
	nfs_grow_file(page, offset, count);
1192
	nfs_mark_uptodate(req);
1193
	nfs_mark_request_dirty(req);
1194
	nfs_unlock_and_release_request(req);
1195 1196 1197
	return 0;
}

L
Linus Torvalds 已提交
1198 1199
int nfs_flush_incompatible(struct file *file, struct page *page)
{
1200
	struct nfs_open_context *ctx = nfs_file_open_context(file);
1201
	struct nfs_lock_context *l_ctx;
1202
	struct file_lock_context *flctx = file_inode(file)->i_flctx;
L
Linus Torvalds 已提交
1203
	struct nfs_page	*req;
1204
	int do_flush, status;
L
Linus Torvalds 已提交
1205 1206 1207 1208 1209 1210 1211 1212
	/*
	 * Look for a request corresponding to this page. If there
	 * is one, and it belongs to another file, we flush it out
	 * before we try to copy anything into the page. Do this
	 * due to the lack of an ACCESS-type call in NFSv2.
	 * Also do the same if we find a request from an existing
	 * dropped page.
	 */
1213
	do {
1214
		req = nfs_page_find_head_request(page);
1215 1216
		if (req == NULL)
			return 0;
1217
		l_ctx = req->wb_lock_context;
1218 1219
		do_flush = req->wb_page != page ||
			!nfs_match_open_context(req->wb_context, ctx);
1220 1221 1222
		if (l_ctx && flctx &&
		    !(list_empty_careful(&flctx->flc_posix) &&
		      list_empty_careful(&flctx->flc_flock))) {
1223
			do_flush |= l_ctx->lockowner != current->files;
1224
		}
L
Linus Torvalds 已提交
1225
		nfs_release_request(req);
1226 1227
		if (!do_flush)
			return 0;
1228
		status = nfs_wb_page(page_file_mapping(page)->host, page);
1229 1230
	} while (status == 0);
	return status;
L
Linus Torvalds 已提交
1231 1232
}

1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
/*
 * Avoid buffered writes when a open context credential's key would
 * expire soon.
 *
 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
 *
 * Return 0 and set a credential flag which triggers the inode to flush
 * and performs  NFS_FILE_SYNC writes if the key will expired within
 * RPC_KEY_EXPIRE_TIMEO.
 */
int
nfs_key_timeout_notify(struct file *filp, struct inode *inode)
{
	struct nfs_open_context *ctx = nfs_file_open_context(filp);

1248 1249 1250 1251 1252
	if (nfs_ctx_key_to_expire(ctx, inode) &&
	    !ctx->ll_cred)
		/* Already expired! */
		return -EACCES;
	return 0;
1253 1254 1255 1256 1257
}

/*
 * Test if the open context credential key is marked to expire soon.
 */
1258
bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
1259
{
1260
	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
1261 1262
	struct rpc_cred *cred = ctx->ll_cred;
	struct auth_cred acred = {
1263
		.cred = ctx->cred,
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
	};

	if (cred && !cred->cr_ops->crmatch(&acred, cred, 0)) {
		put_rpccred(cred);
		ctx->ll_cred = NULL;
		cred = NULL;
	}
	if (!cred)
		cred = auth->au_ops->lookup_cred(auth, &acred, 0);
	if (!cred || IS_ERR(cred))
		return true;
	ctx->ll_cred = cred;
	return !!(cred->cr_ops->crkey_timeout &&
		  cred->cr_ops->crkey_timeout(cred));
1278 1279
}

1280 1281 1282 1283 1284
/*
 * If the page cache is marked as unsafe or invalid, then we can't rely on
 * the PageUptodate() flag. In this case, we will need to turn off
 * write optimisations that depend on the page contents being correct.
 */
1285
static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
1286
{
1287 1288
	struct nfs_inode *nfsi = NFS_I(inode);

1289 1290
	if (nfs_have_delegated_attributes(inode))
		goto out;
1291
	if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
1292
		return false;
1293
	smp_rmb();
1294
	if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
1295 1296
		return false;
out:
1297 1298
	if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
		return false;
1299
	return PageUptodate(page) != 0;
1300 1301
}

1302 1303 1304 1305 1306 1307 1308
static bool
is_whole_file_wrlock(struct file_lock *fl)
{
	return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
			fl->fl_type == F_WRLCK;
}

1309 1310 1311 1312 1313
/* If we know the page is up to date, and we're not using byte range locks (or
 * if we have the whole file locked for writing), it may be more efficient to
 * extend the write to cover the entire page in order to avoid fragmentation
 * inefficiencies.
 *
1314 1315
 * If the file is opened for synchronous writes then we can just skip the rest
 * of the checks.
1316 1317 1318
 */
static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
{
1319 1320 1321 1322
	int ret;
	struct file_lock_context *flctx = inode->i_flctx;
	struct file_lock *fl;

1323 1324
	if (file->f_flags & O_DSYNC)
		return 0;
1325 1326
	if (!nfs_write_pageuptodate(page, inode))
		return 0;
1327 1328
	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
		return 1;
1329 1330
	if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
		       list_empty_careful(&flctx->flc_posix)))
1331
		return 1;
1332 1333 1334

	/* Check to see if there are whole file write locks */
	ret = 0;
1335
	spin_lock(&flctx->flc_lock);
1336 1337 1338 1339 1340 1341
	if (!list_empty(&flctx->flc_posix)) {
		fl = list_first_entry(&flctx->flc_posix, struct file_lock,
					fl_list);
		if (is_whole_file_wrlock(fl))
			ret = 1;
	} else if (!list_empty(&flctx->flc_flock)) {
1342 1343 1344 1345 1346
		fl = list_first_entry(&flctx->flc_flock, struct file_lock,
					fl_list);
		if (fl->fl_type == F_WRLCK)
			ret = 1;
	}
1347
	spin_unlock(&flctx->flc_lock);
1348
	return ret;
1349 1350
}

L
Linus Torvalds 已提交
1351 1352 1353 1354 1355 1356 1357 1358 1359
/*
 * Update and possibly write a cached page of an NFS file.
 *
 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
 * things with a page scheduled for an RPC call (e.g. invalidate it).
 */
int nfs_updatepage(struct file *file, struct page *page,
		unsigned int offset, unsigned int count)
{
1360
	struct nfs_open_context *ctx = nfs_file_open_context(file);
1361 1362
	struct address_space *mapping = page_file_mapping(page);
	struct inode	*inode = mapping->host;
L
Linus Torvalds 已提交
1363 1364
	int		status = 0;

1365 1366
	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);

1367 1368
	dprintk("NFS:       nfs_updatepage(%pD2 %d@%lld)\n",
		file, count, (long long)(page_file_offset(page) + offset));
L
Linus Torvalds 已提交
1369

1370 1371 1372
	if (!count)
		goto out;

1373
	if (nfs_can_extend_write(file, page, inode)) {
1374
		count = max(count + offset, nfs_page_length(page));
L
Linus Torvalds 已提交
1375 1376 1377
		offset = 0;
	}

1378
	status = nfs_writepage_setup(ctx, page, offset, count);
1379
	if (status < 0)
1380
		nfs_set_pageerror(mapping);
1381 1382
	else
		__set_page_dirty_nobuffers(page);
1383
out:
1384
	dprintk("NFS:       nfs_updatepage returns %d (isize %lld)\n",
L
Linus Torvalds 已提交
1385 1386 1387 1388
			status, (long long)i_size_read(inode));
	return status;
}

1389
static int flush_task_priority(int how)
L
Linus Torvalds 已提交
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
{
	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
		case FLUSH_HIGHPRI:
			return RPC_PRIORITY_HIGH;
		case FLUSH_LOWPRI:
			return RPC_PRIORITY_LOW;
	}
	return RPC_PRIORITY_NORMAL;
}

1400 1401
static void nfs_initiate_write(struct nfs_pgio_header *hdr,
			       struct rpc_message *msg,
1402
			       const struct nfs_rpc_ops *rpc_ops,
1403
			       struct rpc_task_setup *task_setup_data, int how)
L
Linus Torvalds 已提交
1404
{
1405
	int priority = flush_task_priority(how);
1406

1407
	task_setup_data->priority = priority;
1408
	rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client);
1409 1410
	trace_nfs_initiate_write(hdr->inode, hdr->io_start, hdr->good_bytes,
				 hdr->args.stable);
1411 1412
}

F
Fred 已提交
1413 1414 1415 1416 1417 1418 1419
/* If a nfs_flush_* function fails, it should remove reqs from @head and
 * call this on each, which will prepare them to be retried on next
 * writeback using standard nfs.
 */
static void nfs_redirty_request(struct nfs_page *req)
{
	nfs_mark_request_dirty(req);
1420
	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1421
	nfs_end_page_writeback(req);
1422
	nfs_release_request(req);
F
Fred 已提交
1423 1424
}

1425
static void nfs_async_write_error(struct list_head *head, int error)
1426 1427 1428 1429 1430 1431
{
	struct nfs_page	*req;

	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
1432 1433 1434 1435
		if (nfs_error_is_fatal(error))
			nfs_write_error(req, error);
		else
			nfs_redirty_request(req);
1436 1437 1438
	}
}

1439 1440
static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
{
1441
	nfs_async_write_error(&hdr->pages, 0);
1442 1443
	filemap_fdatawrite_range(hdr->inode->i_mapping, hdr->args.offset,
			hdr->args.offset + hdr->args.count - 1);
1444 1445
}

1446
static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1447
	.init_hdr = nfs_async_write_init,
1448 1449
	.error_cleanup = nfs_async_write_error,
	.completion = nfs_write_completion,
1450
	.reschedule_io = nfs_async_write_reschedule_io,
1451 1452
};

1453
void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1454
			       struct inode *inode, int ioflags, bool force_mds,
1455
			       const struct nfs_pgio_completion_ops *compl_ops)
L
Linus Torvalds 已提交
1456
{
1457
	struct nfs_server *server = NFS_SERVER(inode);
1458
	const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
1459 1460 1461 1462 1463

#ifdef CONFIG_NFS_V4_1
	if (server->pnfs_curr_ld && !force_mds)
		pg_ops = server->pnfs_curr_ld->pg_write_ops;
#endif
1464
	nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1465
			server->wsize, ioflags);
1466
}
1467
EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
L
Linus Torvalds 已提交
1468

1469 1470
void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
{
1471 1472
	struct nfs_pgio_mirror *mirror;

1473 1474 1475
	if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
		pgio->pg_ops->pg_cleanup(pgio);

1476
	pgio->pg_ops = &nfs_pgio_rw_ops;
1477 1478 1479 1480 1481

	nfs_pageio_stop_mirroring(pgio);

	mirror = &pgio->pg_mirrors[0];
	mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1482
}
1483
EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1484

L
Linus Torvalds 已提交
1485

1486 1487 1488 1489 1490 1491 1492
void nfs_commit_prepare(struct rpc_task *task, void *calldata)
{
	struct nfs_commit_data *data = calldata;

	NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
}

1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
/*
 * Special version of should_remove_suid() that ignores capabilities.
 */
static int nfs_should_remove_suid(const struct inode *inode)
{
	umode_t mode = inode->i_mode;
	int kill = 0;

	/* suid always must be killed */
	if (unlikely(mode & S_ISUID))
		kill = ATTR_KILL_SUID;
1504

1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
	/*
	 * sgid without any exec bits is just a mandatory locking mark; leave
	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
	 */
	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
		kill |= ATTR_KILL_SGID;

	if (unlikely(kill && S_ISREG(mode)))
		return kill;

	return 0;
}
1517

1518 1519 1520 1521 1522
static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
		struct nfs_fattr *fattr)
{
	struct nfs_pgio_args *argp = &hdr->args;
	struct nfs_pgio_res *resp = &hdr->res;
1523
	u64 size = argp->offset + resp->count;
1524 1525

	if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
1526 1527 1528
		fattr->size = size;
	if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
		fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
1529
		return;
1530 1531
	}
	if (size != fattr->size)
1532 1533 1534
		return;
	/* Set attribute barrier */
	nfs_fattr_set_barrier(fattr);
1535 1536
	/* ...and update size */
	fattr->valid |= NFS_ATTR_FATTR_SIZE;
1537 1538 1539 1540
}

void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
{
1541
	struct nfs_fattr *fattr = &hdr->fattr;
1542 1543 1544 1545 1546 1547 1548 1549 1550
	struct inode *inode = hdr->inode;

	spin_lock(&inode->i_lock);
	nfs_writeback_check_extend(hdr, fattr);
	nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
	spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);

L
Linus Torvalds 已提交
1551 1552 1553
/*
 * This function is called when the WRITE call is complete.
 */
1554 1555
static int nfs_writeback_done(struct rpc_task *task,
			      struct nfs_pgio_header *hdr,
1556
			      struct inode *inode)
L
Linus Torvalds 已提交
1557
{
1558
	int status;
L
Linus Torvalds 已提交
1559

1560 1561 1562 1563 1564 1565 1566
	/*
	 * ->write_done will attempt to use post-op attributes to detect
	 * conflicting writes by other clients.  A strict interpretation
	 * of close-to-open would allow us to continue caching even if
	 * another writer had changed the file, but some applications
	 * depend on tighter cache coherency when writing.
	 */
1567
	status = NFS_PROTO(inode)->write_done(task, hdr);
1568
	if (status != 0)
1569
		return status;
1570

1571
	nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
1572 1573
	trace_nfs_writeback_done(inode, task->tk_status,
				 hdr->args.offset, hdr->res.verf);
1574

1575 1576
	if (hdr->res.verf->committed < hdr->args.stable &&
	    task->tk_status >= 0) {
L
Linus Torvalds 已提交
1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
		/* We tried a write call, but the server did not
		 * commit data to stable storage even though we
		 * requested it.
		 * Note: There is a known bug in Tru64 < 5.0 in which
		 *	 the server reports NFS_DATA_SYNC, but performs
		 *	 NFS_FILE_SYNC. We therefore implement this checking
		 *	 as a dprintk() in order to avoid filling syslog.
		 */
		static unsigned long    complain;

1587
		/* Note this will print the MDS for a DS write */
L
Linus Torvalds 已提交
1588
		if (time_before(complain, jiffies)) {
1589
			dprintk("NFS:       faulty NFS server %s:"
L
Linus Torvalds 已提交
1590
				" (committed = %d) != (stable = %d)\n",
1591
				NFS_SERVER(inode)->nfs_client->cl_hostname,
1592
				hdr->res.verf->committed, hdr->args.stable);
L
Linus Torvalds 已提交
1593 1594 1595
			complain = jiffies + 300 * HZ;
		}
	}
1596 1597

	/* Deal with the suid/sgid bit corner case */
1598 1599 1600 1601 1602
	if (nfs_should_remove_suid(inode)) {
		spin_lock(&inode->i_lock);
		NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER;
		spin_unlock(&inode->i_lock);
	}
1603 1604 1605 1606 1607 1608
	return 0;
}

/*
 * This function is called when the WRITE call is complete.
 */
1609 1610
static void nfs_writeback_result(struct rpc_task *task,
				 struct nfs_pgio_header *hdr)
1611
{
1612 1613
	struct nfs_pgio_args	*argp = &hdr->args;
	struct nfs_pgio_res	*resp = &hdr->res;
1614 1615

	if (resp->count < argp->count) {
L
Linus Torvalds 已提交
1616 1617
		static unsigned long    complain;

1618
		/* This a short write! */
1619
		nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
1620

L
Linus Torvalds 已提交
1621
		/* Has the server at least made some progress? */
1622 1623 1624 1625 1626 1627
		if (resp->count == 0) {
			if (time_before(complain, jiffies)) {
				printk(KERN_WARNING
				       "NFS: Server wrote zero bytes, expected %u.\n",
				       argp->count);
				complain = jiffies + 300 * HZ;
L
Linus Torvalds 已提交
1628
			}
1629
			nfs_set_pgio_error(hdr, -EIO, argp->offset);
1630
			task->tk_status = -EIO;
1631
			return;
L
Linus Torvalds 已提交
1632
		}
1633 1634 1635 1636 1637 1638 1639

		/* For non rpc-based layout drivers, retry-through-MDS */
		if (!task->tk_ops) {
			hdr->pnfs_error = -EAGAIN;
			return;
		}

1640 1641 1642
		/* Was this an NFSv2 write or an NFSv3 stable write? */
		if (resp->verf->committed != NFS_UNSTABLE) {
			/* Resend from where the server left off */
1643
			hdr->mds_offset += resp->count;
1644 1645 1646 1647 1648 1649 1650 1651
			argp->offset += resp->count;
			argp->pgbase += resp->count;
			argp->count -= resp->count;
		} else {
			/* Resend as a stable write in order to avoid
			 * headaches in the case of a server crash.
			 */
			argp->stable = NFS_FILE_SYNC;
L
Linus Torvalds 已提交
1652
		}
1653
		rpc_restart_call_prepare(task);
L
Linus Torvalds 已提交
1654 1655 1656
	}
}

1657
static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
1658
{
1659 1660
	return wait_var_event_killable(&cinfo->rpcs_out,
				       !atomic_read(&cinfo->rpcs_out));
1661
}
1662

1663 1664 1665
static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
{
	atomic_inc(&cinfo->rpcs_out);
1666 1667
}

1668
static void nfs_commit_end(struct nfs_mds_commit_info *cinfo)
1669
{
1670
	if (atomic_dec_and_test(&cinfo->rpcs_out))
1671
		wake_up_var(&cinfo->rpcs_out);
1672 1673
}

1674
void nfs_commitdata_release(struct nfs_commit_data *data)
L
Linus Torvalds 已提交
1675
{
1676 1677
	put_nfs_open_context(data->context);
	nfs_commit_free(data);
L
Linus Torvalds 已提交
1678
}
1679
EXPORT_SYMBOL_GPL(nfs_commitdata_release);
L
Linus Torvalds 已提交
1680

1681
int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1682
			const struct nfs_rpc_ops *nfs_ops,
1683
			const struct rpc_call_ops *call_ops,
1684
			int how, int flags)
L
Linus Torvalds 已提交
1685
{
1686
	struct rpc_task *task;
1687
	int priority = flush_task_priority(how);
1688 1689 1690
	struct rpc_message msg = {
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
1691
		.rpc_cred = data->cred,
1692
	};
1693
	struct rpc_task_setup task_setup_data = {
1694
		.task = &data->task,
1695
		.rpc_client = clnt,
1696
		.rpc_message = &msg,
1697
		.callback_ops = call_ops,
1698
		.callback_data = data,
1699
		.workqueue = nfsiod_workqueue,
1700
		.flags = RPC_TASK_ASYNC | flags,
1701
		.priority = priority,
1702
	};
1703
	/* Set up the initial task struct.  */
1704
	nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client);
1705
	trace_nfs_initiate_commit(data);
1706

1707
	dprintk("NFS: initiated commit call\n");
1708 1709 1710 1711 1712 1713 1714 1715 1716

	task = rpc_run_task(&task_setup_data);
	if (IS_ERR(task))
		return PTR_ERR(task);
	if (how & FLUSH_SYNC)
		rpc_wait_for_completion_task(task);
	rpc_put_task(task);
	return 0;
}
1717
EXPORT_SYMBOL_GPL(nfs_initiate_commit);
1718

1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
static loff_t nfs_get_lwb(struct list_head *head)
{
	loff_t lwb = 0;
	struct nfs_page *req;

	list_for_each_entry(req, head, wb_list)
		if (lwb < (req_offset(req) + req->wb_bytes))
			lwb = req_offset(req) + req->wb_bytes;

	return lwb;
}

1731 1732 1733
/*
 * Set up the argument/result storage required for the RPC call.
 */
1734
void nfs_init_commit(struct nfs_commit_data *data,
1735 1736 1737
		     struct list_head *head,
		     struct pnfs_layout_segment *lseg,
		     struct nfs_commit_info *cinfo)
1738 1739
{
	struct nfs_page *first = nfs_list_entry(head->next);
1740
	struct inode *inode = d_inode(first->wb_context->dentry);
L
Linus Torvalds 已提交
1741 1742 1743 1744 1745 1746 1747

	/* Set up the RPC argument and reply structs
	 * NB: take care not to mess about with data->commit et al. */

	list_splice_init(head, &data->pages);

	data->inode	  = inode;
1748
	data->cred	  = first->wb_context->cred;
1749
	data->lseg	  = lseg; /* reference transferred */
1750 1751 1752
	/* only set lwb for pnfs commit */
	if (lseg)
		data->lwb = nfs_get_lwb(&data->pages);
1753
	data->mds_ops     = &nfs_commit_ops;
1754
	data->completion_ops = cinfo->completion_ops;
1755
	data->dreq	  = cinfo->dreq;
L
Linus Torvalds 已提交
1756 1757

	data->args.fh     = NFS_FH(data->inode);
1758 1759 1760
	/* Note: we always request a commit of the entire inode */
	data->args.offset = 0;
	data->args.count  = 0;
1761
	data->context     = get_nfs_open_context(first->wb_context);
L
Linus Torvalds 已提交
1762 1763
	data->res.fattr   = &data->fattr;
	data->res.verf    = &data->verf;
1764
	nfs_fattr_init(&data->fattr);
L
Linus Torvalds 已提交
1765
}
1766
EXPORT_SYMBOL_GPL(nfs_init_commit);
L
Linus Torvalds 已提交
1767

1768
void nfs_retry_commit(struct list_head *page_list,
1769
		      struct pnfs_layout_segment *lseg,
1770 1771
		      struct nfs_commit_info *cinfo,
		      u32 ds_commit_idx)
1772 1773 1774 1775 1776 1777
{
	struct nfs_page *req;

	while (!list_empty(page_list)) {
		req = nfs_list_entry(page_list->next);
		nfs_list_remove_request(req);
1778
		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
1779 1780
		if (!cinfo->dreq)
			nfs_clear_page_commit(req->wb_page);
1781
		nfs_unlock_and_release_request(req);
1782 1783
	}
}
1784
EXPORT_SYMBOL_GPL(nfs_retry_commit);
1785

1786 1787 1788 1789 1790 1791 1792
static void
nfs_commit_resched_write(struct nfs_commit_info *cinfo,
		struct nfs_page *req)
{
	__set_page_dirty_nobuffers(req->wb_page);
}

L
Linus Torvalds 已提交
1793 1794 1795 1796
/*
 * Commit dirty pages
 */
static int
1797 1798
nfs_commit_list(struct inode *inode, struct list_head *head, int how,
		struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
1799
{
1800
	struct nfs_commit_data	*data;
L
Linus Torvalds 已提交
1801

1802 1803 1804 1805
	/* another commit raced with us */
	if (list_empty(head))
		return 0;

N
NeilBrown 已提交
1806
	data = nfs_commitdata_alloc(true);
L
Linus Torvalds 已提交
1807 1808

	/* Set up the argument struct */
1809 1810
	nfs_init_commit(data, head, NULL, cinfo);
	atomic_inc(&cinfo->mds->rpcs_out);
1811 1812
	return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
				   data->mds_ops, how, 0);
1813 1814
}

L
Linus Torvalds 已提交
1815 1816 1817
/*
 * COMMIT call returned
 */
1818
static void nfs_commit_done(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
1819
{
1820
	struct nfs_commit_data	*data = calldata;
L
Linus Torvalds 已提交
1821

1822
        dprintk("NFS: %5u nfs_commit_done (status %d)\n",
L
Linus Torvalds 已提交
1823 1824
                                task->tk_pid, task->tk_status);

1825
	/* Call the NFS version-specific code */
1826
	NFS_PROTO(data->inode)->commit_done(task, data);
1827
	trace_nfs_commit_done(data);
1828 1829
}

1830
static void nfs_commit_release_pages(struct nfs_commit_data *data)
1831
{
1832
	struct nfs_page	*req;
1833
	int status = data->task.tk_status;
1834
	struct nfs_commit_info cinfo;
1835
	struct nfs_server *nfss;
1836

L
Linus Torvalds 已提交
1837 1838 1839
	while (!list_empty(&data->pages)) {
		req = nfs_list_entry(data->pages.next);
		nfs_list_remove_request(req);
1840 1841
		if (req->wb_page)
			nfs_clear_page_commit(req->wb_page);
L
Linus Torvalds 已提交
1842

1843
		dprintk("NFS:       commit (%s/%llu %d@%lld)",
1844
			req->wb_context->dentry->d_sb->s_id,
1845
			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),
L
Linus Torvalds 已提交
1846 1847
			req->wb_bytes,
			(long long)req_offset(req));
1848
		if (status < 0) {
1849 1850
			if (req->wb_page) {
				nfs_mapping_set_error(req->wb_page, status);
1851
				nfs_inode_remove_request(req);
1852
			}
1853
			dprintk_cont(", error = %d\n", status);
L
Linus Torvalds 已提交
1854 1855 1856 1857 1858
			goto next;
		}

		/* Okay, COMMIT succeeded, apparently. Check the verifier
		 * returned by the server against all stored verfs. */
1859
		if (!nfs_write_verifier_cmp(&req->wb_verf, &data->verf.verifier)) {
L
Linus Torvalds 已提交
1860
			/* We have a match */
1861 1862
			if (req->wb_page)
				nfs_inode_remove_request(req);
1863
			dprintk_cont(" OK\n");
L
Linus Torvalds 已提交
1864 1865 1866
			goto next;
		}
		/* We have a mismatch. Write the page again */
1867
		dprintk_cont(" mismatch\n");
F
Fred 已提交
1868
		nfs_mark_request_dirty(req);
1869
		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
L
Linus Torvalds 已提交
1870
	next:
1871
		nfs_unlock_and_release_request(req);
1872 1873
		/* Latency breaker */
		cond_resched();
L
Linus Torvalds 已提交
1874
	}
1875 1876
	nfss = NFS_SERVER(data->inode);
	if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
1877
		clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC);
1878

1879
	nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1880
	nfs_commit_end(cinfo.mds);
1881 1882 1883 1884
}

static void nfs_commit_release(void *calldata)
{
1885
	struct nfs_commit_data *data = calldata;
1886

1887
	data->completion_ops->completion(data);
1888
	nfs_commitdata_release(calldata);
L
Linus Torvalds 已提交
1889
}
1890 1891

static const struct rpc_call_ops nfs_commit_ops = {
1892
	.rpc_call_prepare = nfs_commit_prepare,
1893 1894 1895
	.rpc_call_done = nfs_commit_done,
	.rpc_release = nfs_commit_release,
};
L
Linus Torvalds 已提交
1896

1897 1898
static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
	.completion = nfs_commit_release_pages,
1899
	.resched_write = nfs_commit_resched_write,
1900 1901
};

1902 1903
int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
			    int how, struct nfs_commit_info *cinfo)
1904 1905 1906
{
	int status;

1907
	status = pnfs_commit_list(inode, head, how, cinfo);
1908
	if (status == PNFS_NOT_ATTEMPTED)
1909
		status = nfs_commit_list(inode, head, how, cinfo);
1910 1911 1912
	return status;
}

1913 1914
static int __nfs_commit_inode(struct inode *inode, int how,
		struct writeback_control *wbc)
L
Linus Torvalds 已提交
1915 1916
{
	LIST_HEAD(head);
1917
	struct nfs_commit_info cinfo;
1918
	int may_wait = how & FLUSH_SYNC;
1919
	int ret, nscan;
L
Linus Torvalds 已提交
1920

1921
	nfs_init_cinfo_from_inode(&cinfo, inode);
1922
	nfs_commit_begin(cinfo.mds);
1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940
	for (;;) {
		ret = nscan = nfs_scan_commit(inode, &head, &cinfo);
		if (ret <= 0)
			break;
		ret = nfs_generic_commit_list(inode, &head, how, &cinfo);
		if (ret < 0)
			break;
		ret = 0;
		if (wbc && wbc->sync_mode == WB_SYNC_NONE) {
			if (nscan < wbc->nr_to_write)
				wbc->nr_to_write -= nscan;
			else
				wbc->nr_to_write = 0;
		}
		if (nscan < INT_MAX)
			break;
		cond_resched();
	}
1941
	nfs_commit_end(cinfo.mds);
1942 1943 1944 1945 1946 1947 1948 1949
	if (ret || !may_wait)
		return ret;
	return wait_on_commit(cinfo.mds);
}

int nfs_commit_inode(struct inode *inode, int how)
{
	return __nfs_commit_inode(inode, how, NULL);
L
Linus Torvalds 已提交
1950
}
1951
EXPORT_SYMBOL_GPL(nfs_commit_inode);
1952

1953
int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1954
{
1955 1956 1957
	struct nfs_inode *nfsi = NFS_I(inode);
	int flags = FLUSH_SYNC;
	int ret = 0;
1958

1959
	if (wbc->sync_mode == WB_SYNC_NONE) {
1960 1961 1962 1963
		/* no commits means nothing needs to be done */
		if (!atomic_long_read(&nfsi->commit_info.ncommit))
			goto check_requests_outstanding;

1964 1965 1966
		/* Don't commit yet if this is a non-blocking flush and there
		 * are a lot of outstanding writes for this mapping.
		 */
1967
		if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
1968
			goto out_mark_dirty;
1969

1970
		/* don't wait for the COMMIT response */
1971
		flags = 0;
1972 1973
	}

1974 1975 1976 1977 1978 1979 1980 1981 1982 1983
	ret = __nfs_commit_inode(inode, flags, wbc);
	if (!ret) {
		if (flags & FLUSH_SYNC)
			return 0;
	} else if (atomic_long_read(&nfsi->commit_info.ncommit))
		goto out_mark_dirty;

check_requests_outstanding:
	if (!atomic_read(&nfsi->commit_info.rpcs_out))
		return ret;
1984
out_mark_dirty:
1985 1986 1987
	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
	return ret;
}
1988
EXPORT_SYMBOL_GPL(nfs_write_inode);
1989

1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007
/*
 * Wrapper for filemap_write_and_wait_range()
 *
 * Needed for pNFS in order to ensure data becomes visible to the
 * client.
 */
int nfs_filemap_write_and_wait_range(struct address_space *mapping,
		loff_t lstart, loff_t lend)
{
	int ret;

	ret = filemap_write_and_wait_range(mapping, lstart, lend);
	if (ret == 0)
		ret = pnfs_sync_inode(mapping->host, true);
	return ret;
}
EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);

2008 2009 2010 2011
/*
 * flush the inode to disk.
 */
int nfs_wb_all(struct inode *inode)
2012
{
2013 2014 2015 2016
	int ret;

	trace_nfs_writeback_inode_enter(inode);

2017
	ret = filemap_write_and_wait(inode->i_mapping);
2018 2019 2020 2021 2022 2023 2024
	if (ret)
		goto out;
	ret = nfs_commit_inode(inode, FLUSH_SYNC);
	if (ret < 0)
		goto out;
	pnfs_sync_inode(inode, true);
	ret = 0;
2025

2026
out:
2027 2028
	trace_nfs_writeback_inode_exit(inode, ret);
	return ret;
2029
}
2030
EXPORT_SYMBOL_GPL(nfs_wb_all);
2031

2032 2033 2034 2035 2036
int nfs_wb_page_cancel(struct inode *inode, struct page *page)
{
	struct nfs_page *req;
	int ret = 0;

2037 2038 2039 2040
	wait_on_page_writeback(page);

	/* blocking call to cancel all requests and join to a single (head)
	 * request */
2041
	req = nfs_lock_and_join_requests(page);
2042 2043 2044 2045 2046 2047 2048 2049 2050 2051

	if (IS_ERR(req)) {
		ret = PTR_ERR(req);
	} else if (req) {
		/* all requests from this page have been cancelled by
		 * nfs_lock_and_join_requests, so just remove the head
		 * request from the inode / page_private pointer and
		 * release it */
		nfs_inode_remove_request(req);
		nfs_unlock_and_release_request(req);
2052
	}
2053

2054 2055 2056
	return ret;
}

2057 2058 2059
/*
 * Write back all requests on one page - we do this before reading it.
 */
2060
int nfs_wb_page(struct inode *inode, struct page *page)
2061
{
2062
	loff_t range_start = page_file_offset(page);
2063
	loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1);
2064 2065
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_ALL,
2066
		.nr_to_write = 0,
2067 2068 2069 2070
		.range_start = range_start,
		.range_end = range_end,
	};
	int ret;
2071

2072 2073
	trace_nfs_writeback_page_enter(inode);

2074
	for (;;) {
2075
		wait_on_page_writeback(page);
2076
		if (clear_page_dirty_for_io(page)) {
2077
			ret = nfs_writepage_locked(page, &wbc);
2078 2079
			if (ret < 0)
				goto out_error;
2080
			continue;
2081
		}
2082
		ret = 0;
2083 2084 2085
		if (!PagePrivate(page))
			break;
		ret = nfs_commit_inode(inode, FLUSH_SYNC);
2086
		if (ret < 0)
2087
			goto out_error;
2088
	}
2089
out_error:
2090
	trace_nfs_writeback_page_exit(inode, ret);
2091
	return ret;
2092 2093
}

2094 2095
#ifdef CONFIG_MIGRATION
int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
2096
		struct page *page, enum migrate_mode mode)
2097
{
2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
	/*
	 * If PagePrivate is set, then the page is currently associated with
	 * an in-progress read or write request. Don't try to migrate it.
	 *
	 * FIXME: we could do this in principle, but we'll need a way to ensure
	 *        that we can safely release the inode reference while holding
	 *        the page lock.
	 */
	if (PagePrivate(page))
		return -EBUSY;
2108

2109 2110
	if (!nfs_fscache_release_page(page, GFP_KERNEL))
		return -EBUSY;
2111

2112
	return migrate_page(mapping, newpage, page, mode);
2113 2114 2115
}
#endif

D
David Howells 已提交
2116
int __init nfs_init_writepagecache(void)
L
Linus Torvalds 已提交
2117 2118
{
	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
2119
					     sizeof(struct nfs_pgio_header),
L
Linus Torvalds 已提交
2120
					     0, SLAB_HWCACHE_ALIGN,
2121
					     NULL);
L
Linus Torvalds 已提交
2122 2123 2124
	if (nfs_wdata_cachep == NULL)
		return -ENOMEM;

2125 2126
	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
						     nfs_wdata_cachep);
L
Linus Torvalds 已提交
2127
	if (nfs_wdata_mempool == NULL)
2128
		goto out_destroy_write_cache;
L
Linus Torvalds 已提交
2129

2130 2131 2132 2133 2134
	nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
					     sizeof(struct nfs_commit_data),
					     0, SLAB_HWCACHE_ALIGN,
					     NULL);
	if (nfs_cdata_cachep == NULL)
2135
		goto out_destroy_write_mempool;
2136

2137
	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
2138
						      nfs_cdata_cachep);
L
Linus Torvalds 已提交
2139
	if (nfs_commit_mempool == NULL)
2140
		goto out_destroy_commit_cache;
L
Linus Torvalds 已提交
2141

2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157
	/*
	 * NFS congestion size, scale with available memory.
	 *
	 *  64MB:    8192k
	 * 128MB:   11585k
	 * 256MB:   16384k
	 * 512MB:   23170k
	 *   1GB:   32768k
	 *   2GB:   46340k
	 *   4GB:   65536k
	 *   8GB:   92681k
	 *  16GB:  131072k
	 *
	 * This allows larger machines to have larger/more transfers.
	 * Limit the default to 256M
	 */
2158
	nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10);
2159 2160 2161
	if (nfs_congestion_kb > 256*1024)
		nfs_congestion_kb = 256*1024;

L
Linus Torvalds 已提交
2162
	return 0;
2163 2164 2165 2166 2167 2168 2169 2170

out_destroy_commit_cache:
	kmem_cache_destroy(nfs_cdata_cachep);
out_destroy_write_mempool:
	mempool_destroy(nfs_wdata_mempool);
out_destroy_write_cache:
	kmem_cache_destroy(nfs_wdata_cachep);
	return -ENOMEM;
L
Linus Torvalds 已提交
2171 2172
}

2173
void nfs_destroy_writepagecache(void)
L
Linus Torvalds 已提交
2174 2175
{
	mempool_destroy(nfs_commit_mempool);
2176
	kmem_cache_destroy(nfs_cdata_cachep);
L
Linus Torvalds 已提交
2177
	mempool_destroy(nfs_wdata_mempool);
2178
	kmem_cache_destroy(nfs_wdata_cachep);
L
Linus Torvalds 已提交
2179 2180
}

2181 2182 2183
static const struct nfs_rw_ops nfs_rw_write_ops = {
	.rw_alloc_header	= nfs_writehdr_alloc,
	.rw_free_header		= nfs_writehdr_free,
2184 2185
	.rw_done		= nfs_writeback_done,
	.rw_result		= nfs_writeback_result,
2186
	.rw_initiate		= nfs_initiate_write,
2187
};
反馈
建议
客服 返回
顶部