write.c 56.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * linux/fs/nfs/write.c
 *
4
 * Write file data over NFS.
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14
 *
 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/writeback.h>
15
#include <linux/swap.h>
16
#include <linux/migrate.h>
L
Linus Torvalds 已提交
17 18 19 20 21

#include <linux/sunrpc/clnt.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/nfs_page.h>
22
#include <linux/backing-dev.h>
23
#include <linux/export.h>
24 25
#include <linux/freezer.h>
#include <linux/wait.h>
26

27
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
28 29

#include "delegation.h"
30
#include "internal.h"
31
#include "iostat.h"
32
#include "nfs4_fs.h"
33
#include "fscache.h"
34
#include "pnfs.h"
L
Linus Torvalds 已提交
35

36 37
#include "nfstrace.h"

L
Linus Torvalds 已提交
38 39 40 41 42
#define NFSDBG_FACILITY		NFSDBG_PAGECACHE

#define MIN_POOL_WRITE		(32)
#define MIN_POOL_COMMIT		(4)

43 44 45 46 47 48
struct nfs_io_completion {
	void (*complete)(void *data);
	void *data;
	struct kref refcount;
};

L
Linus Torvalds 已提交
49 50 51
/*
 * Local function declarations
 */
52
static void nfs_redirty_request(struct nfs_page *req);
53
static const struct rpc_call_ops nfs_commit_ops;
54
static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
55
static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
56
static const struct nfs_rw_ops nfs_rw_write_ops;
57
static void nfs_clear_request_commit(struct nfs_page *req);
58 59
static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
				      struct inode *inode);
60 61 62
static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
						struct page *page);
L
Linus Torvalds 已提交
63

64
static struct kmem_cache *nfs_wdata_cachep;
65
static mempool_t *nfs_wdata_mempool;
66
static struct kmem_cache *nfs_cdata_cachep;
L
Linus Torvalds 已提交
67 68
static mempool_t *nfs_commit_mempool;

N
NeilBrown 已提交
69
struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail)
L
Linus Torvalds 已提交
70
{
N
NeilBrown 已提交
71
	struct nfs_commit_data *p;
72

N
NeilBrown 已提交
73 74 75 76 77 78 79 80 81 82 83 84 85 86
	if (never_fail)
		p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
	else {
		/* It is OK to do some reclaim, not no safe to wait
		 * for anything to be returned to the pool.
		 * mempool_alloc() cannot handle that particular combination,
		 * so we need two separate attempts.
		 */
		p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
		if (!p)
			p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO |
					     __GFP_NOWARN | __GFP_NORETRY);
		if (!p)
			return NULL;
L
Linus Torvalds 已提交
87
	}
N
NeilBrown 已提交
88 89 90

	memset(p, 0, sizeof(*p));
	INIT_LIST_HEAD(&p->pages);
L
Linus Torvalds 已提交
91 92
	return p;
}
93
EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
L
Linus Torvalds 已提交
94

95
void nfs_commit_free(struct nfs_commit_data *p)
L
Linus Torvalds 已提交
96 97 98
{
	mempool_free(p, nfs_commit_mempool);
}
99
EXPORT_SYMBOL_GPL(nfs_commit_free);
L
Linus Torvalds 已提交
100

101
static struct nfs_pgio_header *nfs_writehdr_alloc(void)
102
{
103
	struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
104

105 106
	memset(p, 0, sizeof(*p));
	p->rw_mode = FMODE_WRITE;
107 108
	return p;
}
109

110
static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
111
{
112
	mempool_free(hdr, nfs_wdata_mempool);
113
}
L
Linus Torvalds 已提交
114

115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
{
	return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
}

static void nfs_io_completion_init(struct nfs_io_completion *ioc,
		void (*complete)(void *), void *data)
{
	ioc->complete = complete;
	ioc->data = data;
	kref_init(&ioc->refcount);
}

static void nfs_io_completion_release(struct kref *kref)
{
	struct nfs_io_completion *ioc = container_of(kref,
			struct nfs_io_completion, refcount);
	ioc->complete(ioc->data);
	kfree(ioc);
}

static void nfs_io_completion_get(struct nfs_io_completion *ioc)
{
	if (ioc != NULL)
		kref_get(&ioc->refcount);
}

static void nfs_io_completion_put(struct nfs_io_completion *ioc)
{
	if (ioc != NULL)
		kref_put(&ioc->refcount, nfs_io_completion_release);
}

148 149 150 151 152 153 154
static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
{
	ctx->error = error;
	smp_wmb();
	set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
}

155 156 157 158 159 160 161 162
static struct nfs_page *
nfs_page_private_request(struct page *page)
{
	if (!PagePrivate(page))
		return NULL;
	return (struct nfs_page *)page_private(page);
}

163 164 165 166 167 168 169
/*
 * nfs_page_find_head_request_locked - find head request associated with @page
 *
 * must be called while holding the inode lock.
 *
 * returns matching head request with reference held, or NULL if not found.
 */
170
static struct nfs_page *
171
nfs_page_find_private_request(struct page *page)
172
{
173
	struct address_space *mapping = page_file_mapping(page);
174
	struct nfs_page *req;
175

176 177
	if (!PagePrivate(page))
		return NULL;
178
	spin_lock(&mapping->private_lock);
179
	req = nfs_page_private_request(page);
180 181
	if (req) {
		WARN_ON_ONCE(req->wb_head != req);
182
		kref_get(&req->wb_kref);
183
	}
184
	spin_unlock(&mapping->private_lock);
185 186
	return req;
}
187

188 189 190 191 192 193 194 195
static struct nfs_page *
nfs_page_find_swap_request(struct page *page)
{
	struct inode *inode = page_file_mapping(page)->host;
	struct nfs_inode *nfsi = NFS_I(inode);
	struct nfs_page *req = NULL;
	if (!PageSwapCache(page))
		return NULL;
196
	mutex_lock(&nfsi->commit_mutex);
197 198 199 200 201 202 203 204
	if (PageSwapCache(page)) {
		req = nfs_page_search_commits_for_head_request_locked(nfsi,
			page);
		if (req) {
			WARN_ON_ONCE(req->wb_head != req);
			kref_get(&req->wb_kref);
		}
	}
205
	mutex_unlock(&nfsi->commit_mutex);
206 207 208
	return req;
}

209 210 211 212 213 214
/*
 * nfs_page_find_head_request - find head request associated with @page
 *
 * returns matching head request with reference held, or NULL if not found.
 */
static struct nfs_page *nfs_page_find_head_request(struct page *page)
215
{
216
	struct nfs_page *req;
217

218 219 220
	req = nfs_page_find_private_request(page);
	if (!req)
		req = nfs_page_find_swap_request(page);
221 222 223
	return req;
}

L
Linus Torvalds 已提交
224 225 226
/* Adjust the file length if we're writing beyond the end */
static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
{
227
	struct inode *inode = page_file_mapping(page)->host;
228 229
	loff_t end, i_size;
	pgoff_t end_index;
L
Linus Torvalds 已提交
230

231 232
	spin_lock(&inode->i_lock);
	i_size = i_size_read(inode);
233
	end_index = (i_size - 1) >> PAGE_SHIFT;
H
Huang Ying 已提交
234
	if (i_size > 0 && page_index(page) < end_index)
235
		goto out;
236
	end = page_file_offset(page) + ((loff_t)offset+count);
L
Linus Torvalds 已提交
237
	if (i_size >= end)
238
		goto out;
L
Linus Torvalds 已提交
239
	i_size_write(inode, end);
240 241 242
	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
out:
	spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
243 244
}

245 246 247
/* A writeback failed: mark the page as bad, and invalidate the page cache */
static void nfs_set_pageerror(struct page *page)
{
248
	nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page));
249 250
}

251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
/*
 * nfs_page_group_search_locked
 * @head - head request of page group
 * @page_offset - offset into page
 *
 * Search page group with head @head to find a request that contains the
 * page offset @page_offset.
 *
 * Returns a pointer to the first matching nfs request, or NULL if no
 * match is found.
 *
 * Must be called with the page group lock held
 */
static struct nfs_page *
nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
{
	struct nfs_page *req;

	req = head;
	do {
		if (page_offset >= req->wb_pgbase &&
		    page_offset < (req->wb_pgbase + req->wb_bytes))
			return req;

		req = req->wb_this_page;
	} while (req != head);

	return NULL;
}

/*
 * nfs_page_group_covers_page
 * @head - head request of page group
 *
 * Return true if the page group with head @head covers the whole page,
 * returns false otherwise
 */
static bool nfs_page_group_covers_page(struct nfs_page *req)
{
	struct nfs_page *tmp;
	unsigned int pos = 0;
	unsigned int len = nfs_page_length(req->wb_page);

294
	nfs_page_group_lock(req);
295

296
	for (;;) {
297
		tmp = nfs_page_group_search_locked(req->wb_head, pos);
298 299 300 301
		if (!tmp)
			break;
		pos = tmp->wb_pgbase + tmp->wb_bytes;
	}
302 303

	nfs_page_group_unlock(req);
304
	return pos >= len;
305 306
}

L
Linus Torvalds 已提交
307 308 309
/* We can set the PG_uptodate flag if we see that a write request
 * covers the full page.
 */
310
static void nfs_mark_uptodate(struct nfs_page *req)
L
Linus Torvalds 已提交
311
{
312
	if (PageUptodate(req->wb_page))
L
Linus Torvalds 已提交
313
		return;
314
	if (!nfs_page_group_covers_page(req))
L
Linus Torvalds 已提交
315
		return;
316
	SetPageUptodate(req->wb_page);
L
Linus Torvalds 已提交
317 318 319 320
}

static int wb_priority(struct writeback_control *wbc)
{
321
	int ret = 0;
322

323 324 325
	if (wbc->sync_mode == WB_SYNC_ALL)
		ret = FLUSH_COND_STABLE;
	return ret;
L
Linus Torvalds 已提交
326 327
}

328 329 330 331 332 333 334 335 336 337
/*
 * NFS congestion control
 */

int nfs_congestion_kb;

#define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
#define NFS_CONGESTION_OFF_THRESH	\
	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))

338
static void nfs_set_page_writeback(struct page *page)
339
{
340 341
	struct inode *inode = page_file_mapping(page)->host;
	struct nfs_server *nfss = NFS_SERVER(inode);
342 343
	int ret = test_set_page_writeback(page);

344
	WARN_ON_ONCE(ret != 0);
345

346
	if (atomic_long_inc_return(&nfss->writeback) >
347 348
			NFS_CONGESTION_ON_THRESH)
		set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
349 350
}

351
static void nfs_end_page_writeback(struct nfs_page *req)
352
{
353
	struct inode *inode = page_file_mapping(req->wb_page)->host;
354
	struct nfs_server *nfss = NFS_SERVER(inode);
355
	bool is_done;
356

357 358 359
	is_done = nfs_page_group_sync_on_bit(req, PG_WB_END);
	nfs_unlock_request(req);
	if (!is_done)
360 361 362
		return;

	end_page_writeback(req->wb_page);
363
	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
364
		clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
365 366
}

367 368 369 370 371 372 373 374 375
/*
 * nfs_unroll_locks_and_wait -  unlock all newly locked reqs and wait on @req
 *
 * this is a helper function for nfs_lock_and_join_requests
 *
 * @inode - inode associated with request page group, must be holding inode lock
 * @head  - head request of page group, must be holding head lock
 * @req   - request that couldn't lock and needs to wait on the req bit lock
 *
376 377
 * NOTE: this must be called holding page_group bit lock
 *       which will be released before returning.
378 379 380
 *
 * returns 0 on success, < 0 on error.
 */
381 382
static void
nfs_unroll_locks(struct inode *inode, struct nfs_page *head,
383
			  struct nfs_page *req)
384 385 386 387
{
	struct nfs_page *tmp;

	/* relinquish all the locks successfully grabbed this run */
388 389 390 391 392
	for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
		if (!kref_read(&tmp->wb_kref))
			continue;
		nfs_unlock_and_release_request(tmp);
	}
393 394 395 396 397 398 399 400 401 402 403 404 405 406
}

/*
 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
 *
 * @destroy_list - request list (using wb_this_page) terminated by @old_head
 * @old_head - the old head of the list
 *
 * All subrequests must be locked and removed from all lists, so at this point
 * they are only "active" in this function, and possibly in nfs_wait_on_request
 * with a reference held by some other context.
 */
static void
nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
407 408
				 struct nfs_page *old_head,
				 struct inode *inode)
409 410 411 412 413 414 415 416 417 418 419 420
{
	while (destroy_list) {
		struct nfs_page *subreq = destroy_list;

		destroy_list = (subreq->wb_this_page == old_head) ?
				   NULL : subreq->wb_this_page;

		WARN_ON_ONCE(old_head != subreq->wb_head);

		/* make sure old group is not used */
		subreq->wb_this_page = subreq;

421 422
		clear_bit(PG_REMOVE, &subreq->wb_flags);

423 424 425
		/* Note: races with nfs_page_group_destroy() */
		if (!kref_read(&subreq->wb_kref)) {
			/* Check if we raced with nfs_page_group_destroy() */
426
			if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags))
427 428 429
				nfs_free_request(subreq);
			continue;
		}
430

431 432 433 434
		subreq->wb_head = subreq;

		if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) {
			nfs_release_request(subreq);
435
			atomic_long_dec(&NFS_I(inode)->nrequests);
436
		}
437 438 439 440

		/* subreq is now totally disconnected from page group or any
		 * write / commit lists. last chance to wake any waiters */
		nfs_unlock_and_release_request(subreq);
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
	}
}

/*
 * nfs_lock_and_join_requests - join all subreqs to the head req and return
 *                              a locked reference, cancelling any pending
 *                              operations for this page.
 *
 * @page - the page used to lookup the "page group" of nfs_page structures
 *
 * This function joins all sub requests to the head request by first
 * locking all requests in the group, cancelling any pending operations
 * and finally updating the head request to cover the whole range covered by
 * the (former) group.  All subrequests are removed from any write or commit
 * lists, unlinked from the group and destroyed.
 *
 * Returns a locked, referenced pointer to the head request - which after
 * this call is guaranteed to be the only request associated with the page.
 * Returns NULL if no requests are found for @page, or a ERR_PTR if an
 * error was encountered.
 */
static struct nfs_page *
463
nfs_lock_and_join_requests(struct page *page)
464
{
465
	struct inode *inode = page_file_mapping(page)->host;
466 467 468
	struct nfs_page *head, *subreq;
	struct nfs_page *destroy_list = NULL;
	unsigned int total_bytes;
469 470
	int ret;

471 472 473 474 475 476
try_again:
	/*
	 * A reference is taken only on the head request which acts as a
	 * reference to the whole page group - the group will not be destroyed
	 * until the head reference is released.
	 */
477 478
	head = nfs_page_find_head_request(page);
	if (!head)
479 480
		return NULL;

481 482 483 484 485 486 487 488
	/* lock the page head first in order to avoid an ABBA inefficiency */
	if (!nfs_lock_request(head)) {
		ret = nfs_wait_on_request(head);
		nfs_release_request(head);
		if (ret < 0)
			return ERR_PTR(ret);
		goto try_again;
	}
489 490 491 492 493 494

	/* Ensure that nobody removed the request before we locked it */
	if (head != nfs_page_private_request(page) && !PageSwapCache(page)) {
		nfs_unlock_and_release_request(head);
		goto try_again;
	}
495

496
	ret = nfs_page_group_lock(head);
497
	if (ret < 0) {
498
		nfs_unlock_and_release_request(head);
499
		return ERR_PTR(ret);
500
	}
501 502

	/* lock each request in the page group */
503 504 505
	total_bytes = head->wb_bytes;
	for (subreq = head->wb_this_page; subreq != head;
			subreq = subreq->wb_this_page) {
506

507 508
		if (!kref_get_unless_zero(&subreq->wb_kref))
			continue;
509
		while (!nfs_lock_request(subreq)) {
510
			/*
511 512
			 * Unlock page to allow nfs_page_group_sync_on_bit()
			 * to succeed
513
			 */
514 515 516
			nfs_page_group_unlock(head);
			ret = nfs_wait_on_request(subreq);
			if (!ret)
517
				ret = nfs_page_group_lock(head);
518 519
			if (ret < 0) {
				nfs_unroll_locks(inode, head, subreq);
520
				nfs_release_request(subreq);
521 522 523
				nfs_unlock_and_release_request(head);
				return ERR_PTR(ret);
			}
524
		}
525 526
		/*
		 * Subrequests are always contiguous, non overlapping
527
		 * and in order - but may be repeated (mirrored writes).
528
		 */
529 530 531 532 533 534
		if (subreq->wb_offset == (head->wb_offset + total_bytes)) {
			/* keep track of how many bytes this group covers */
			total_bytes += subreq->wb_bytes;
		} else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset ||
			    ((subreq->wb_offset + subreq->wb_bytes) >
			     (head->wb_offset + total_bytes)))) {
535
			nfs_unroll_locks(inode, head, subreq);
536
			nfs_unlock_and_release_request(subreq);
537 538
			nfs_page_group_unlock(head);
			nfs_unlock_and_release_request(head);
539 540
			return ERR_PTR(-EIO);
		}
541
	}
542 543 544 545 546

	/* Now that all requests are locked, make sure they aren't on any list.
	 * Commit list removal accounting is done after locks are dropped */
	subreq = head;
	do {
547
		nfs_clear_request_commit(subreq);
548 549 550 551 552 553 554 555 556 557 558 559
		subreq = subreq->wb_this_page;
	} while (subreq != head);

	/* unlink subrequests from head, destroy them later */
	if (head->wb_this_page != head) {
		/* destroy list will be terminated by head */
		destroy_list = head->wb_this_page;
		head->wb_this_page = head;

		/* change head request to cover whole range that
		 * the former page group covered */
		head->wb_bytes = total_bytes;
560
	}
561

562 563 564 565
	/* Postpone destruction of this request */
	if (test_and_clear_bit(PG_REMOVE, &head->wb_flags)) {
		set_bit(PG_INODE_REF, &head->wb_flags);
		kref_get(&head->wb_kref);
566
		atomic_long_inc(&NFS_I(inode)->nrequests);
567 568
	}

569 570
	nfs_page_group_unlock(head);

571
	nfs_destroy_unlinked_subrequests(destroy_list, head, inode);
572

573 574 575 576 577 578
	/* Did we lose a race with nfs_inode_remove_request()? */
	if (!(PagePrivate(page) || PageSwapCache(page))) {
		nfs_unlock_and_release_request(head);
		return NULL;
	}

579
	/* still holds ref on head from nfs_page_find_head_request
580 581
	 * and still has lock on head from lock loop */
	return head;
582 583
}

584 585 586 587 588
static void nfs_write_error_remove_page(struct nfs_page *req)
{
	nfs_end_page_writeback(req);
	generic_error_remove_page(page_file_mapping(req->wb_page),
				  req->wb_page);
589
	nfs_release_request(req);
590 591
}

592 593 594 595 596 597 598 599 600 601
static bool
nfs_error_is_fatal_on_server(int err)
{
	switch (err) {
	case 0:
	case -ERESTARTSYS:
	case -EINTR:
		return false;
	}
	return nfs_error_is_fatal(err);
602 603
}

604 605 606 607 608
/*
 * Find an associated nfs write request, and prepare to flush it out
 * May return an error if the user signalled nfs_wait_on_request().
 */
static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
609
				struct page *page)
610 611 612 613
{
	struct nfs_page *req;
	int ret = 0;

614
	req = nfs_lock_and_join_requests(page);
615 616 617 618 619 620
	if (!req)
		goto out;
	ret = PTR_ERR(req);
	if (IS_ERR(req))
		goto out;

621 622
	nfs_set_page_writeback(page);
	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
623

624
	ret = 0;
625 626 627 628
	/* If there is a fatal error that covers this write, just exit */
	if (nfs_error_is_fatal_on_server(req->wb_context->error))
		goto out_launder;

629
	if (!nfs_pageio_add_request(pgio, req)) {
630
		ret = pgio->pg_error;
631
		/*
632
		 * Remove the problematic req upon fatal errors on the server
633 634 635
		 */
		if (nfs_error_is_fatal(ret)) {
			nfs_context_set_write_error(req->wb_context, ret);
636
			if (nfs_error_is_fatal_on_server(ret))
637
				goto out_launder;
638
		}
639 640
		nfs_redirty_request(req);
		ret = -EAGAIN;
641 642 643
	} else
		nfs_add_stats(page_file_mapping(page)->host,
				NFSIOS_WRITEPAGES, 1);
644 645
out:
	return ret;
646 647 648
out_launder:
	nfs_write_error_remove_page(req);
	return ret;
649 650
}

651
static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
652
			    struct nfs_pageio_descriptor *pgio)
L
Linus Torvalds 已提交
653
{
654
	int ret;
L
Linus Torvalds 已提交
655

H
Huang Ying 已提交
656
	nfs_pageio_cond_complete(pgio, page_index(page));
657
	ret = nfs_page_async_flush(pgio, page);
658 659 660 661 662
	if (ret == -EAGAIN) {
		redirty_page_for_writepage(wbc, page);
		ret = 0;
	}
	return ret;
663
}
664

665 666 667
/*
 * Write an mmapped page to the server.
 */
668
static int nfs_writepage_locked(struct page *page,
669
				struct writeback_control *wbc)
670 671
{
	struct nfs_pageio_descriptor pgio;
672
	struct inode *inode = page_file_mapping(page)->host;
673
	int err;
674

675
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
676
	nfs_pageio_init_write(&pgio, inode, 0,
677
				false, &nfs_async_write_completion_ops);
678
	err = nfs_do_writepage(page, wbc, &pgio);
679 680 681 682 683 684
	nfs_pageio_complete(&pgio);
	if (err < 0)
		return err;
	if (pgio.pg_error < 0)
		return pgio.pg_error;
	return 0;
685 686 687 688
}

int nfs_writepage(struct page *page, struct writeback_control *wbc)
{
689
	int ret;
690

691
	ret = nfs_writepage_locked(page, wbc);
L
Linus Torvalds 已提交
692
	unlock_page(page);
693 694 695 696 697 698 699
	return ret;
}

static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
{
	int ret;

700
	ret = nfs_do_writepage(page, wbc, data);
701 702
	unlock_page(page);
	return ret;
L
Linus Torvalds 已提交
703 704
}

705 706 707 708 709
static void nfs_io_completion_commit(void *inode)
{
	nfs_commit_inode(inode, 0);
}

L
Linus Torvalds 已提交
710 711 712
int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;
713
	struct nfs_pageio_descriptor pgio;
714
	struct nfs_io_completion *ioc = nfs_io_completion_alloc(GFP_NOFS);
L
Linus Torvalds 已提交
715 716
	int err;

717 718
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);

719 720 721
	if (ioc)
		nfs_io_completion_init(ioc, nfs_io_completion_commit, inode);

722 723
	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
				&nfs_async_write_completion_ops);
724
	pgio.pg_io_completion = ioc;
725
	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
726
	nfs_pageio_complete(&pgio);
727
	nfs_io_completion_put(ioc);
728

729
	if (err < 0)
730 731 732 733
		goto out_err;
	err = pgio.pg_error;
	if (err < 0)
		goto out_err;
734
	return 0;
735 736
out_err:
	return err;
L
Linus Torvalds 已提交
737 738 739 740 741
}

/*
 * Insert a write request into an inode
 */
742
static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
L
Linus Torvalds 已提交
743
{
744
	struct address_space *mapping = page_file_mapping(req->wb_page);
L
Linus Torvalds 已提交
745
	struct nfs_inode *nfsi = NFS_I(inode);
746

747 748
	WARN_ON_ONCE(req->wb_this_page != req);

749
	/* Lock the request! */
750
	nfs_lock_request(req);
751

752 753 754 755
	/*
	 * Swap-space should not get truncated. Hence no need to plug the race
	 * with invalidate/truncate.
	 */
756 757 758 759 760 761 762
	spin_lock(&mapping->private_lock);
	if (!nfs_have_writebacks(inode) &&
	    NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE)) {
		spin_lock(&inode->i_lock);
		inode->i_version++;
		spin_unlock(&inode->i_lock);
	}
763 764 765 766 767
	if (likely(!PageSwapCache(req->wb_page))) {
		set_bit(PG_MAPPED, &req->wb_flags);
		SetPagePrivate(req->wb_page);
		set_page_private(req->wb_page, (unsigned long)req);
	}
768
	spin_unlock(&mapping->private_lock);
769
	atomic_long_inc(&nfsi->nrequests);
770
	/* this a head request for a page group - mark it as having an
771 772 773
	 * extra reference so sub groups can follow suit.
	 * This flag also informs pgio layer when to bump nrequests when
	 * adding subrequests. */
774
	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
775
	kref_get(&req->wb_kref);
L
Linus Torvalds 已提交
776 777 778
}

/*
779
 * Remove a write request from an inode
L
Linus Torvalds 已提交
780 781 782
 */
static void nfs_inode_remove_request(struct nfs_page *req)
{
783 784
	struct address_space *mapping = page_file_mapping(req->wb_page);
	struct inode *inode = mapping->host;
L
Linus Torvalds 已提交
785
	struct nfs_inode *nfsi = NFS_I(inode);
786
	struct nfs_page *head;
L
Linus Torvalds 已提交
787

788
	atomic_long_dec(&nfsi->nrequests);
789 790 791
	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
		head = req->wb_head;

792
		spin_lock(&mapping->private_lock);
793
		if (likely(head->wb_page && !PageSwapCache(head->wb_page))) {
794 795 796 797
			set_page_private(head->wb_page, 0);
			ClearPagePrivate(head->wb_page);
			clear_bit(PG_MAPPED, &head->wb_flags);
		}
798
		spin_unlock(&mapping->private_lock);
799
	}
800 801 802

	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags))
		nfs_release_request(req);
L
Linus Torvalds 已提交
803 804
}

805
static void
F
Fred 已提交
806
nfs_mark_request_dirty(struct nfs_page *req)
807
{
808 809
	if (req->wb_page)
		__set_page_dirty_nobuffers(req->wb_page);
810 811
}

812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
/*
 * nfs_page_search_commits_for_head_request_locked
 *
 * Search through commit lists on @inode for the head request for @page.
 * Must be called while holding the inode (which is cinfo) lock.
 *
 * Returns the head request if found, or NULL if not found.
 */
static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
						struct page *page)
{
	struct nfs_page *freq, *t;
	struct nfs_commit_info cinfo;
	struct inode *inode = &nfsi->vfs_inode;

	nfs_init_cinfo_from_inode(&cinfo, inode);

	/* search through pnfs commit lists */
	freq = pnfs_search_commit_reqs(inode, &cinfo, page);
	if (freq)
		return freq->wb_head;

	/* Linearly search the commit list for the correct request */
	list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) {
		if (freq->wb_page == page)
			return freq->wb_head;
	}

	return NULL;
}

844 845 846 847 848 849 850 851 852 853
/**
 * nfs_request_add_commit_list_locked - add request to a commit list
 * @req: pointer to a struct nfs_page
 * @dst: commit list head
 * @cinfo: holds list lock and accounting info
 *
 * This sets the PG_CLEAN bit, updates the cinfo count of
 * number of outstanding requests requiring a commit as well as
 * the MM page stats.
 *
854 855
 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
 * nfs_page lock.
856 857 858 859 860 861 862
 */
void
nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
			    struct nfs_commit_info *cinfo)
{
	set_bit(PG_CLEAN, &req->wb_flags);
	nfs_list_add_request(req, dst);
863
	atomic_long_inc(&cinfo->mds->ncommit);
864 865 866
}
EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);

867 868 869
/**
 * nfs_request_add_commit_list - add request to a commit list
 * @req: pointer to a struct nfs_page
870 871
 * @dst: commit list head
 * @cinfo: holds list lock and accounting info
872
 *
873
 * This sets the PG_CLEAN bit, updates the cinfo count of
874 875 876
 * number of outstanding requests requiring a commit as well as
 * the MM page stats.
 *
877
 * The caller must _not_ hold the cinfo->lock, but must be
878
 * holding the nfs_page lock.
L
Linus Torvalds 已提交
879
 */
880
void
881
nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
882
{
883
	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
884
	nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
885
	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
886 887
	if (req->wb_page)
		nfs_mark_page_unstable(req->wb_page, cinfo);
L
Linus Torvalds 已提交
888
}
889 890 891 892 893
EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);

/**
 * nfs_request_remove_commit_list - Remove request from a commit list
 * @req: pointer to a nfs_page
894
 * @cinfo: holds list lock and accounting info
895
 *
896
 * This clears the PG_CLEAN bit, and updates the cinfo's count of
897 898 899
 * number of outstanding requests requiring a commit
 * It does not update the MM page stats.
 *
900
 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
901 902
 */
void
903 904
nfs_request_remove_commit_list(struct nfs_page *req,
			       struct nfs_commit_info *cinfo)
905 906 907 908
{
	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
		return;
	nfs_list_remove_request(req);
909
	atomic_long_dec(&cinfo->mds->ncommit);
910 911 912
}
EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);

913 914 915
static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
				      struct inode *inode)
{
916
	cinfo->inode = inode;
917 918
	cinfo->mds = &NFS_I(inode)->commit_info;
	cinfo->ds = pnfs_get_ds_info(inode);
919
	cinfo->dreq = NULL;
920
	cinfo->completion_ops = &nfs_commit_completion_ops;
921 922 923 924 925 926
}

void nfs_init_cinfo(struct nfs_commit_info *cinfo,
		    struct inode *inode,
		    struct nfs_direct_req *dreq)
{
927 928 929 930
	if (dreq)
		nfs_init_cinfo_from_dreq(cinfo, dreq);
	else
		nfs_init_cinfo_from_inode(cinfo, inode);
931 932
}
EXPORT_SYMBOL_GPL(nfs_init_cinfo);
933 934 935 936

/*
 * Add a request to the inode's commit list.
 */
937
void
938
nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
939
			struct nfs_commit_info *cinfo, u32 ds_commit_idx)
940
{
941
	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
942
		return;
943
	nfs_request_add_commit_list(req, cinfo);
944
}
945

946 947 948
static void
nfs_clear_page_commit(struct page *page)
{
949
	dec_node_page_state(page, NR_UNSTABLE_NFS);
950 951
	dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb,
		    WB_RECLAIMABLE);
952 953
}

954
/* Called holding the request lock on @req */
955
static void
956 957
nfs_clear_request_commit(struct nfs_page *req)
{
958
	if (test_bit(PG_CLEAN, &req->wb_flags)) {
959
		struct inode *inode = d_inode(req->wb_context->dentry);
960
		struct nfs_commit_info cinfo;
961

962
		nfs_init_cinfo_from_inode(&cinfo, inode);
963
		mutex_lock(&NFS_I(inode)->commit_mutex);
964 965
		if (!pnfs_clear_request_commit(req, &cinfo)) {
			nfs_request_remove_commit_list(req, &cinfo);
966
		}
967
		mutex_unlock(&NFS_I(inode)->commit_mutex);
968
		nfs_clear_page_commit(req->wb_page);
969 970 971
	}
}

972
int nfs_write_need_commit(struct nfs_pgio_header *hdr)
973
{
974
	if (hdr->verf.committed == NFS_DATA_SYNC)
975
		return hdr->lseg == NULL;
976
	return hdr->verf.committed != NFS_FILE_SYNC;
977 978
}

979 980 981 982 983
static void nfs_async_write_init(struct nfs_pgio_header *hdr)
{
	nfs_io_completion_get(hdr->io_completion);
}

984
static void nfs_write_completion(struct nfs_pgio_header *hdr)
985
{
986
	struct nfs_commit_info cinfo;
987 988 989 990
	unsigned long bytes = 0;

	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
		goto out;
991
	nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
992 993 994 995 996 997 998
	while (!list_empty(&hdr->pages)) {
		struct nfs_page *req = nfs_list_entry(hdr->pages.next);

		bytes += req->wb_bytes;
		nfs_list_remove_request(req);
		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
		    (hdr->good_bytes < bytes)) {
999
			nfs_set_pageerror(req->wb_page);
1000 1001 1002
			nfs_context_set_write_error(req->wb_context, hdr->error);
			goto remove_req;
		}
1003
		if (nfs_write_need_commit(hdr)) {
1004
			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
1005
			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
1006
				hdr->pgio_mirror_idx);
1007 1008 1009 1010 1011
			goto next;
		}
remove_req:
		nfs_inode_remove_request(req);
next:
1012
		nfs_end_page_writeback(req);
1013
		nfs_release_request(req);
1014 1015
	}
out:
1016
	nfs_io_completion_put(hdr->io_completion);
1017
	hdr->release(hdr);
1018
}
L
Linus Torvalds 已提交
1019

1020
unsigned long
1021
nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
1022
{
1023
	return atomic_long_read(&cinfo->mds->ncommit);
1024 1025
}

1026
/* NFS_I(cinfo->inode)->commit_mutex held by caller */
1027
int
1028 1029
nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
		     struct nfs_commit_info *cinfo, int max)
1030
{
1031
	struct nfs_page *req, *tmp;
1032 1033
	int ret = 0;

1034 1035
restart:
	list_for_each_entry_safe(req, tmp, src, wb_list) {
1036
		kref_get(&req->wb_kref);
1037 1038
		if (!nfs_lock_request(req)) {
			int status;
1039 1040 1041 1042 1043 1044 1045

			/* Prevent deadlock with nfs_lock_and_join_requests */
			if (!list_empty(dst)) {
				nfs_release_request(req);
				continue;
			}
			/* Ensure we make progress to prevent livelock */
1046 1047 1048 1049 1050 1051
			mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
			status = nfs_wait_on_request(req);
			nfs_release_request(req);
			mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
			if (status < 0)
				break;
1052
			goto restart;
1053
		}
1054
		nfs_request_remove_commit_list(req, cinfo);
1055 1056
		nfs_list_add_request(req, dst);
		ret++;
1057
		if ((ret == max) && !cinfo->dreq)
1058
			break;
1059
		cond_resched();
1060 1061
	}
	return ret;
1062 1063
}

L
Linus Torvalds 已提交
1064 1065 1066
/*
 * nfs_scan_commit - Scan an inode for commit requests
 * @inode: NFS inode to scan
1067 1068
 * @dst: mds destination list
 * @cinfo: mds and ds lists of reqs ready to commit
L
Linus Torvalds 已提交
1069 1070 1071 1072
 *
 * Moves requests from the inode's 'commit' request list.
 * The requests are *not* checked to ensure that they form a contiguous set.
 */
1073
int
1074 1075
nfs_scan_commit(struct inode *inode, struct list_head *dst,
		struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
1076
{
1077
	int ret = 0;
1078

1079 1080
	if (!atomic_long_read(&cinfo->mds->ncommit))
		return 0;
1081
	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1082
	if (atomic_long_read(&cinfo->mds->ncommit) > 0) {
1083
		const int max = INT_MAX;
1084

1085 1086 1087
		ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
					   cinfo, max);
		ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
1088
	}
1089
	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1090
	return ret;
L
Linus Torvalds 已提交
1091
}
1092

L
Linus Torvalds 已提交
1093
/*
1094 1095
 * Search for an existing write request, and attempt to update
 * it to reflect a new dirty region on a given page.
L
Linus Torvalds 已提交
1096
 *
1097 1098
 * If the attempt fails, then the existing request is flushed out
 * to disk.
L
Linus Torvalds 已提交
1099
 */
1100 1101 1102 1103
static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
		struct page *page,
		unsigned int offset,
		unsigned int bytes)
L
Linus Torvalds 已提交
1104
{
1105 1106 1107 1108 1109
	struct nfs_page *req;
	unsigned int rqend;
	unsigned int end;
	int error;

L
Linus Torvalds 已提交
1110 1111
	end = offset + bytes;

1112 1113 1114
	req = nfs_lock_and_join_requests(page);
	if (IS_ERR_OR_NULL(req))
		return req;
L
Linus Torvalds 已提交
1115

1116 1117 1118 1119 1120 1121 1122 1123 1124
	rqend = req->wb_offset + req->wb_bytes;
	/*
	 * Tell the caller to flush out the request if
	 * the offsets are non-contiguous.
	 * Note: nfs_flush_incompatible() will already
	 * have flushed out requests having wrong owners.
	 */
	if (offset > rqend || end < req->wb_offset)
		goto out_flushme;
L
Linus Torvalds 已提交
1125 1126 1127 1128 1129 1130 1131 1132

	/* Okay, the request matches. Update the region */
	if (offset < req->wb_offset) {
		req->wb_offset = offset;
		req->wb_pgbase = offset;
	}
	if (end > rqend)
		req->wb_bytes = end - req->wb_offset;
1133 1134 1135 1136
	else
		req->wb_bytes = rqend - req->wb_offset;
	return req;
out_flushme:
1137 1138 1139 1140 1141 1142 1143
	/*
	 * Note: we mark the request dirty here because
	 * nfs_lock_and_join_requests() cannot preserve
	 * commit flags, so we have to replay the write.
	 */
	nfs_mark_request_dirty(req);
	nfs_unlock_and_release_request(req);
1144
	error = nfs_wb_page(inode, page);
1145
	return (error < 0) ? ERR_PTR(error) : NULL;
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
}

/*
 * Try to update an existing write request, or create one if there is none.
 *
 * Note: Should always be called with the Page Lock held to prevent races
 * if we have to add a new request. Also assumes that the caller has
 * already called nfs_flush_incompatible() if necessary.
 */
static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
		struct page *page, unsigned int offset, unsigned int bytes)
{
1158
	struct inode *inode = page_file_mapping(page)->host;
1159
	struct nfs_page	*req;
L
Linus Torvalds 已提交
1160

1161 1162 1163
	req = nfs_try_to_update_request(inode, page, offset, bytes);
	if (req != NULL)
		goto out;
1164
	req = nfs_create_request(ctx, page, NULL, offset, bytes);
1165 1166
	if (IS_ERR(req))
		goto out;
1167
	nfs_inode_add_request(inode, req);
1168
out:
1169
	return req;
L
Linus Torvalds 已提交
1170 1171
}

1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
		unsigned int offset, unsigned int count)
{
	struct nfs_page	*req;

	req = nfs_setup_write_request(ctx, page, offset, count);
	if (IS_ERR(req))
		return PTR_ERR(req);
	/* Update file length */
	nfs_grow_file(page, offset, count);
1182
	nfs_mark_uptodate(req);
1183
	nfs_mark_request_dirty(req);
1184
	nfs_unlock_and_release_request(req);
1185 1186 1187
	return 0;
}

L
Linus Torvalds 已提交
1188 1189
int nfs_flush_incompatible(struct file *file, struct page *page)
{
1190
	struct nfs_open_context *ctx = nfs_file_open_context(file);
1191
	struct nfs_lock_context *l_ctx;
1192
	struct file_lock_context *flctx = file_inode(file)->i_flctx;
L
Linus Torvalds 已提交
1193
	struct nfs_page	*req;
1194
	int do_flush, status;
L
Linus Torvalds 已提交
1195 1196 1197 1198 1199 1200 1201 1202
	/*
	 * Look for a request corresponding to this page. If there
	 * is one, and it belongs to another file, we flush it out
	 * before we try to copy anything into the page. Do this
	 * due to the lack of an ACCESS-type call in NFSv2.
	 * Also do the same if we find a request from an existing
	 * dropped page.
	 */
1203
	do {
1204
		req = nfs_page_find_head_request(page);
1205 1206
		if (req == NULL)
			return 0;
1207
		l_ctx = req->wb_lock_context;
1208 1209
		do_flush = req->wb_page != page ||
			!nfs_match_open_context(req->wb_context, ctx);
1210 1211 1212
		if (l_ctx && flctx &&
		    !(list_empty_careful(&flctx->flc_posix) &&
		      list_empty_careful(&flctx->flc_flock))) {
1213
			do_flush |= l_ctx->lockowner != current->files;
1214
		}
L
Linus Torvalds 已提交
1215
		nfs_release_request(req);
1216 1217
		if (!do_flush)
			return 0;
1218
		status = nfs_wb_page(page_file_mapping(page)->host, page);
1219 1220
	} while (status == 0);
	return status;
L
Linus Torvalds 已提交
1221 1222
}

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
/*
 * Avoid buffered writes when a open context credential's key would
 * expire soon.
 *
 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
 *
 * Return 0 and set a credential flag which triggers the inode to flush
 * and performs  NFS_FILE_SYNC writes if the key will expired within
 * RPC_KEY_EXPIRE_TIMEO.
 */
int
nfs_key_timeout_notify(struct file *filp, struct inode *inode)
{
	struct nfs_open_context *ctx = nfs_file_open_context(filp);
	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;

	return rpcauth_key_timeout_notify(auth, ctx->cred);
}

/*
 * Test if the open context credential key is marked to expire soon.
 */
1245
bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
1246
{
1247 1248 1249
	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;

	return rpcauth_cred_key_to_expire(auth, ctx->cred);
1250 1251
}

1252 1253 1254 1255 1256
/*
 * If the page cache is marked as unsafe or invalid, then we can't rely on
 * the PageUptodate() flag. In this case, we will need to turn off
 * write optimisations that depend on the page contents being correct.
 */
1257
static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
1258
{
1259 1260
	struct nfs_inode *nfsi = NFS_I(inode);

1261 1262
	if (nfs_have_delegated_attributes(inode))
		goto out;
1263
	if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
1264
		return false;
1265
	smp_rmb();
1266
	if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
1267 1268
		return false;
out:
1269 1270
	if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
		return false;
1271
	return PageUptodate(page) != 0;
1272 1273
}

1274 1275 1276 1277 1278 1279 1280
static bool
is_whole_file_wrlock(struct file_lock *fl)
{
	return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
			fl->fl_type == F_WRLCK;
}

1281 1282 1283 1284 1285
/* If we know the page is up to date, and we're not using byte range locks (or
 * if we have the whole file locked for writing), it may be more efficient to
 * extend the write to cover the entire page in order to avoid fragmentation
 * inefficiencies.
 *
1286 1287
 * If the file is opened for synchronous writes then we can just skip the rest
 * of the checks.
1288 1289 1290
 */
static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
{
1291 1292 1293 1294
	int ret;
	struct file_lock_context *flctx = inode->i_flctx;
	struct file_lock *fl;

1295 1296
	if (file->f_flags & O_DSYNC)
		return 0;
1297 1298
	if (!nfs_write_pageuptodate(page, inode))
		return 0;
1299 1300
	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
		return 1;
1301 1302
	if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
		       list_empty_careful(&flctx->flc_posix)))
1303
		return 1;
1304 1305 1306

	/* Check to see if there are whole file write locks */
	ret = 0;
1307
	spin_lock(&flctx->flc_lock);
1308 1309 1310 1311 1312 1313
	if (!list_empty(&flctx->flc_posix)) {
		fl = list_first_entry(&flctx->flc_posix, struct file_lock,
					fl_list);
		if (is_whole_file_wrlock(fl))
			ret = 1;
	} else if (!list_empty(&flctx->flc_flock)) {
1314 1315 1316 1317 1318
		fl = list_first_entry(&flctx->flc_flock, struct file_lock,
					fl_list);
		if (fl->fl_type == F_WRLCK)
			ret = 1;
	}
1319
	spin_unlock(&flctx->flc_lock);
1320
	return ret;
1321 1322
}

L
Linus Torvalds 已提交
1323 1324 1325 1326 1327 1328 1329 1330 1331
/*
 * Update and possibly write a cached page of an NFS file.
 *
 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
 * things with a page scheduled for an RPC call (e.g. invalidate it).
 */
int nfs_updatepage(struct file *file, struct page *page,
		unsigned int offset, unsigned int count)
{
1332
	struct nfs_open_context *ctx = nfs_file_open_context(file);
1333
	struct inode	*inode = page_file_mapping(page)->host;
L
Linus Torvalds 已提交
1334 1335
	int		status = 0;

1336 1337
	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);

1338 1339
	dprintk("NFS:       nfs_updatepage(%pD2 %d@%lld)\n",
		file, count, (long long)(page_file_offset(page) + offset));
L
Linus Torvalds 已提交
1340

1341 1342 1343
	if (!count)
		goto out;

1344
	if (nfs_can_extend_write(file, page, inode)) {
1345
		count = max(count + offset, nfs_page_length(page));
L
Linus Torvalds 已提交
1346 1347 1348
		offset = 0;
	}

1349
	status = nfs_writepage_setup(ctx, page, offset, count);
1350 1351
	if (status < 0)
		nfs_set_pageerror(page);
1352 1353
	else
		__set_page_dirty_nobuffers(page);
1354
out:
1355
	dprintk("NFS:       nfs_updatepage returns %d (isize %lld)\n",
L
Linus Torvalds 已提交
1356 1357 1358 1359
			status, (long long)i_size_read(inode));
	return status;
}

1360
static int flush_task_priority(int how)
L
Linus Torvalds 已提交
1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
{
	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
		case FLUSH_HIGHPRI:
			return RPC_PRIORITY_HIGH;
		case FLUSH_LOWPRI:
			return RPC_PRIORITY_LOW;
	}
	return RPC_PRIORITY_NORMAL;
}

1371 1372
static void nfs_initiate_write(struct nfs_pgio_header *hdr,
			       struct rpc_message *msg,
1373
			       const struct nfs_rpc_ops *rpc_ops,
1374
			       struct rpc_task_setup *task_setup_data, int how)
L
Linus Torvalds 已提交
1375
{
1376
	int priority = flush_task_priority(how);
1377

1378
	task_setup_data->priority = priority;
1379
	rpc_ops->write_setup(hdr, msg);
1380

1381
	nfs4_state_protect_write(NFS_SERVER(hdr->inode)->nfs_client,
1382
				 &task_setup_data->rpc_client, msg, hdr);
1383 1384
}

F
Fred 已提交
1385 1386 1387 1388 1389 1390 1391
/* If a nfs_flush_* function fails, it should remove reqs from @head and
 * call this on each, which will prepare them to be retried on next
 * writeback using standard nfs.
 */
static void nfs_redirty_request(struct nfs_page *req)
{
	nfs_mark_request_dirty(req);
1392
	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1393
	nfs_end_page_writeback(req);
1394
	nfs_release_request(req);
F
Fred 已提交
1395 1396
}

1397
static void nfs_async_write_error(struct list_head *head)
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
{
	struct nfs_page	*req;

	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_redirty_request(req);
	}
}

1408 1409 1410 1411 1412
static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
{
	nfs_async_write_error(&hdr->pages);
}

1413
static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1414
	.init_hdr = nfs_async_write_init,
1415 1416
	.error_cleanup = nfs_async_write_error,
	.completion = nfs_write_completion,
1417
	.reschedule_io = nfs_async_write_reschedule_io,
1418 1419
};

1420
void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1421
			       struct inode *inode, int ioflags, bool force_mds,
1422
			       const struct nfs_pgio_completion_ops *compl_ops)
L
Linus Torvalds 已提交
1423
{
1424
	struct nfs_server *server = NFS_SERVER(inode);
1425
	const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
1426 1427 1428 1429 1430

#ifdef CONFIG_NFS_V4_1
	if (server->pnfs_curr_ld && !force_mds)
		pg_ops = server->pnfs_curr_ld->pg_write_ops;
#endif
1431
	nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1432
			server->wsize, ioflags);
1433
}
1434
EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
L
Linus Torvalds 已提交
1435

1436 1437
void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
{
1438 1439
	struct nfs_pgio_mirror *mirror;

1440 1441 1442
	if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
		pgio->pg_ops->pg_cleanup(pgio);

1443
	pgio->pg_ops = &nfs_pgio_rw_ops;
1444 1445 1446 1447 1448

	nfs_pageio_stop_mirroring(pgio);

	mirror = &pgio->pg_mirrors[0];
	mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1449
}
1450
EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1451

L
Linus Torvalds 已提交
1452

1453 1454 1455 1456 1457 1458 1459
void nfs_commit_prepare(struct rpc_task *task, void *calldata)
{
	struct nfs_commit_data *data = calldata;

	NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
}

1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
/*
 * Special version of should_remove_suid() that ignores capabilities.
 */
static int nfs_should_remove_suid(const struct inode *inode)
{
	umode_t mode = inode->i_mode;
	int kill = 0;

	/* suid always must be killed */
	if (unlikely(mode & S_ISUID))
		kill = ATTR_KILL_SUID;
1471

1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
	/*
	 * sgid without any exec bits is just a mandatory locking mark; leave
	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
	 */
	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
		kill |= ATTR_KILL_SGID;

	if (unlikely(kill && S_ISREG(mode)))
		return kill;

	return 0;
}
1484

1485 1486 1487 1488 1489
static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
		struct nfs_fattr *fattr)
{
	struct nfs_pgio_args *argp = &hdr->args;
	struct nfs_pgio_res *resp = &hdr->res;
1490
	u64 size = argp->offset + resp->count;
1491 1492

	if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
1493 1494 1495
		fattr->size = size;
	if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
		fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
1496
		return;
1497 1498
	}
	if (size != fattr->size)
1499 1500 1501
		return;
	/* Set attribute barrier */
	nfs_fattr_set_barrier(fattr);
1502 1503
	/* ...and update size */
	fattr->valid |= NFS_ATTR_FATTR_SIZE;
1504 1505 1506 1507
}

void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
{
1508
	struct nfs_fattr *fattr = &hdr->fattr;
1509 1510 1511 1512 1513 1514 1515 1516 1517
	struct inode *inode = hdr->inode;

	spin_lock(&inode->i_lock);
	nfs_writeback_check_extend(hdr, fattr);
	nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
	spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);

L
Linus Torvalds 已提交
1518 1519 1520
/*
 * This function is called when the WRITE call is complete.
 */
1521 1522
static int nfs_writeback_done(struct rpc_task *task,
			      struct nfs_pgio_header *hdr,
1523
			      struct inode *inode)
L
Linus Torvalds 已提交
1524
{
1525
	int status;
L
Linus Torvalds 已提交
1526

1527 1528 1529 1530 1531 1532 1533
	/*
	 * ->write_done will attempt to use post-op attributes to detect
	 * conflicting writes by other clients.  A strict interpretation
	 * of close-to-open would allow us to continue caching even if
	 * another writer had changed the file, but some applications
	 * depend on tighter cache coherency when writing.
	 */
1534
	status = NFS_PROTO(inode)->write_done(task, hdr);
1535
	if (status != 0)
1536
		return status;
1537
	nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
1538

1539 1540
	if (hdr->res.verf->committed < hdr->args.stable &&
	    task->tk_status >= 0) {
L
Linus Torvalds 已提交
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550
		/* We tried a write call, but the server did not
		 * commit data to stable storage even though we
		 * requested it.
		 * Note: There is a known bug in Tru64 < 5.0 in which
		 *	 the server reports NFS_DATA_SYNC, but performs
		 *	 NFS_FILE_SYNC. We therefore implement this checking
		 *	 as a dprintk() in order to avoid filling syslog.
		 */
		static unsigned long    complain;

1551
		/* Note this will print the MDS for a DS write */
L
Linus Torvalds 已提交
1552
		if (time_before(complain, jiffies)) {
1553
			dprintk("NFS:       faulty NFS server %s:"
L
Linus Torvalds 已提交
1554
				" (committed = %d) != (stable = %d)\n",
1555
				NFS_SERVER(inode)->nfs_client->cl_hostname,
1556
				hdr->res.verf->committed, hdr->args.stable);
L
Linus Torvalds 已提交
1557 1558 1559
			complain = jiffies + 300 * HZ;
		}
	}
1560 1561 1562 1563

	/* Deal with the suid/sgid bit corner case */
	if (nfs_should_remove_suid(inode))
		nfs_mark_for_revalidate(inode);
1564 1565 1566 1567 1568 1569
	return 0;
}

/*
 * This function is called when the WRITE call is complete.
 */
1570 1571
static void nfs_writeback_result(struct rpc_task *task,
				 struct nfs_pgio_header *hdr)
1572
{
1573 1574
	struct nfs_pgio_args	*argp = &hdr->args;
	struct nfs_pgio_res	*resp = &hdr->res;
1575 1576

	if (resp->count < argp->count) {
L
Linus Torvalds 已提交
1577 1578
		static unsigned long    complain;

1579
		/* This a short write! */
1580
		nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
1581

L
Linus Torvalds 已提交
1582
		/* Has the server at least made some progress? */
1583 1584 1585 1586 1587 1588
		if (resp->count == 0) {
			if (time_before(complain, jiffies)) {
				printk(KERN_WARNING
				       "NFS: Server wrote zero bytes, expected %u.\n",
				       argp->count);
				complain = jiffies + 300 * HZ;
L
Linus Torvalds 已提交
1589
			}
1590
			nfs_set_pgio_error(hdr, -EIO, argp->offset);
1591
			task->tk_status = -EIO;
1592
			return;
L
Linus Torvalds 已提交
1593
		}
1594 1595 1596 1597 1598 1599 1600

		/* For non rpc-based layout drivers, retry-through-MDS */
		if (!task->tk_ops) {
			hdr->pnfs_error = -EAGAIN;
			return;
		}

1601 1602 1603
		/* Was this an NFSv2 write or an NFSv3 stable write? */
		if (resp->verf->committed != NFS_UNSTABLE) {
			/* Resend from where the server left off */
1604
			hdr->mds_offset += resp->count;
1605 1606 1607 1608 1609 1610 1611 1612
			argp->offset += resp->count;
			argp->pgbase += resp->count;
			argp->count -= resp->count;
		} else {
			/* Resend as a stable write in order to avoid
			 * headaches in the case of a server crash.
			 */
			argp->stable = NFS_FILE_SYNC;
L
Linus Torvalds 已提交
1613
		}
1614
		rpc_restart_call_prepare(task);
L
Linus Torvalds 已提交
1615 1616 1617
	}
}

1618
static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
1619
{
1620 1621 1622
	return wait_on_atomic_t(&cinfo->rpcs_out,
			nfs_wait_atomic_killable, TASK_KILLABLE);
}
1623

1624 1625 1626
static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
{
	atomic_inc(&cinfo->rpcs_out);
1627 1628
}

1629
static void nfs_commit_end(struct nfs_mds_commit_info *cinfo)
1630
{
1631 1632
	if (atomic_dec_and_test(&cinfo->rpcs_out))
		wake_up_atomic_t(&cinfo->rpcs_out);
1633 1634
}

1635
void nfs_commitdata_release(struct nfs_commit_data *data)
L
Linus Torvalds 已提交
1636
{
1637 1638
	put_nfs_open_context(data->context);
	nfs_commit_free(data);
L
Linus Torvalds 已提交
1639
}
1640
EXPORT_SYMBOL_GPL(nfs_commitdata_release);
L
Linus Torvalds 已提交
1641

1642
int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1643
			const struct nfs_rpc_ops *nfs_ops,
1644
			const struct rpc_call_ops *call_ops,
1645
			int how, int flags)
L
Linus Torvalds 已提交
1646
{
1647
	struct rpc_task *task;
1648
	int priority = flush_task_priority(how);
1649 1650 1651
	struct rpc_message msg = {
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
1652
		.rpc_cred = data->cred,
1653
	};
1654
	struct rpc_task_setup task_setup_data = {
1655
		.task = &data->task,
1656
		.rpc_client = clnt,
1657
		.rpc_message = &msg,
1658
		.callback_ops = call_ops,
1659
		.callback_data = data,
1660
		.workqueue = nfsiod_workqueue,
1661
		.flags = RPC_TASK_ASYNC | flags,
1662
		.priority = priority,
1663
	};
1664
	/* Set up the initial task struct.  */
1665
	nfs_ops->commit_setup(data, &msg);
1666

1667
	dprintk("NFS: initiated commit call\n");
1668

1669 1670 1671
	nfs4_state_protect(NFS_SERVER(data->inode)->nfs_client,
		NFS_SP4_MACH_CRED_COMMIT, &task_setup_data.rpc_client, &msg);

1672 1673 1674 1675 1676 1677 1678 1679
	task = rpc_run_task(&task_setup_data);
	if (IS_ERR(task))
		return PTR_ERR(task);
	if (how & FLUSH_SYNC)
		rpc_wait_for_completion_task(task);
	rpc_put_task(task);
	return 0;
}
1680
EXPORT_SYMBOL_GPL(nfs_initiate_commit);
1681

1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693
static loff_t nfs_get_lwb(struct list_head *head)
{
	loff_t lwb = 0;
	struct nfs_page *req;

	list_for_each_entry(req, head, wb_list)
		if (lwb < (req_offset(req) + req->wb_bytes))
			lwb = req_offset(req) + req->wb_bytes;

	return lwb;
}

1694 1695 1696
/*
 * Set up the argument/result storage required for the RPC call.
 */
1697
void nfs_init_commit(struct nfs_commit_data *data,
1698 1699 1700
		     struct list_head *head,
		     struct pnfs_layout_segment *lseg,
		     struct nfs_commit_info *cinfo)
1701 1702
{
	struct nfs_page *first = nfs_list_entry(head->next);
1703
	struct inode *inode = d_inode(first->wb_context->dentry);
L
Linus Torvalds 已提交
1704 1705 1706 1707 1708 1709 1710

	/* Set up the RPC argument and reply structs
	 * NB: take care not to mess about with data->commit et al. */

	list_splice_init(head, &data->pages);

	data->inode	  = inode;
1711
	data->cred	  = first->wb_context->cred;
1712
	data->lseg	  = lseg; /* reference transferred */
1713 1714 1715
	/* only set lwb for pnfs commit */
	if (lseg)
		data->lwb = nfs_get_lwb(&data->pages);
1716
	data->mds_ops     = &nfs_commit_ops;
1717
	data->completion_ops = cinfo->completion_ops;
1718
	data->dreq	  = cinfo->dreq;
L
Linus Torvalds 已提交
1719 1720

	data->args.fh     = NFS_FH(data->inode);
1721 1722 1723
	/* Note: we always request a commit of the entire inode */
	data->args.offset = 0;
	data->args.count  = 0;
1724
	data->context     = get_nfs_open_context(first->wb_context);
L
Linus Torvalds 已提交
1725 1726
	data->res.fattr   = &data->fattr;
	data->res.verf    = &data->verf;
1727
	nfs_fattr_init(&data->fattr);
L
Linus Torvalds 已提交
1728
}
1729
EXPORT_SYMBOL_GPL(nfs_init_commit);
L
Linus Torvalds 已提交
1730

1731
void nfs_retry_commit(struct list_head *page_list,
1732
		      struct pnfs_layout_segment *lseg,
1733 1734
		      struct nfs_commit_info *cinfo,
		      u32 ds_commit_idx)
1735 1736 1737 1738 1739 1740
{
	struct nfs_page *req;

	while (!list_empty(page_list)) {
		req = nfs_list_entry(page_list->next);
		nfs_list_remove_request(req);
1741
		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
1742 1743
		if (!cinfo->dreq)
			nfs_clear_page_commit(req->wb_page);
1744
		nfs_unlock_and_release_request(req);
1745 1746
	}
}
1747
EXPORT_SYMBOL_GPL(nfs_retry_commit);
1748

1749 1750 1751 1752 1753 1754 1755
static void
nfs_commit_resched_write(struct nfs_commit_info *cinfo,
		struct nfs_page *req)
{
	__set_page_dirty_nobuffers(req->wb_page);
}

L
Linus Torvalds 已提交
1756 1757 1758 1759
/*
 * Commit dirty pages
 */
static int
1760 1761
nfs_commit_list(struct inode *inode, struct list_head *head, int how,
		struct nfs_commit_info *cinfo)
L
Linus Torvalds 已提交
1762
{
1763
	struct nfs_commit_data	*data;
L
Linus Torvalds 已提交
1764

1765 1766 1767 1768
	/* another commit raced with us */
	if (list_empty(head))
		return 0;

N
NeilBrown 已提交
1769
	data = nfs_commitdata_alloc(true);
L
Linus Torvalds 已提交
1770 1771

	/* Set up the argument struct */
1772 1773
	nfs_init_commit(data, head, NULL, cinfo);
	atomic_inc(&cinfo->mds->rpcs_out);
1774 1775
	return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
				   data->mds_ops, how, 0);
1776 1777
}

L
Linus Torvalds 已提交
1778 1779 1780
/*
 * COMMIT call returned
 */
1781
static void nfs_commit_done(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
1782
{
1783
	struct nfs_commit_data	*data = calldata;
L
Linus Torvalds 已提交
1784

1785
        dprintk("NFS: %5u nfs_commit_done (status %d)\n",
L
Linus Torvalds 已提交
1786 1787
                                task->tk_pid, task->tk_status);

1788
	/* Call the NFS version-specific code */
1789
	NFS_PROTO(data->inode)->commit_done(task, data);
1790 1791
}

1792
static void nfs_commit_release_pages(struct nfs_commit_data *data)
1793
{
1794
	struct nfs_page	*req;
1795
	int status = data->task.tk_status;
1796
	struct nfs_commit_info cinfo;
1797
	struct nfs_server *nfss;
1798

L
Linus Torvalds 已提交
1799 1800 1801
	while (!list_empty(&data->pages)) {
		req = nfs_list_entry(data->pages.next);
		nfs_list_remove_request(req);
1802 1803
		if (req->wb_page)
			nfs_clear_page_commit(req->wb_page);
L
Linus Torvalds 已提交
1804

1805
		dprintk("NFS:       commit (%s/%llu %d@%lld)",
1806
			req->wb_context->dentry->d_sb->s_id,
1807
			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),
L
Linus Torvalds 已提交
1808 1809
			req->wb_bytes,
			(long long)req_offset(req));
1810 1811
		if (status < 0) {
			nfs_context_set_write_error(req->wb_context, status);
1812 1813
			if (req->wb_page)
				nfs_inode_remove_request(req);
1814
			dprintk_cont(", error = %d\n", status);
L
Linus Torvalds 已提交
1815 1816 1817 1818 1819
			goto next;
		}

		/* Okay, COMMIT succeeded, apparently. Check the verifier
		 * returned by the server against all stored verfs. */
1820
		if (!nfs_write_verifier_cmp(&req->wb_verf, &data->verf.verifier)) {
L
Linus Torvalds 已提交
1821
			/* We have a match */
1822 1823
			if (req->wb_page)
				nfs_inode_remove_request(req);
1824
			dprintk_cont(" OK\n");
L
Linus Torvalds 已提交
1825 1826 1827
			goto next;
		}
		/* We have a mismatch. Write the page again */
1828
		dprintk_cont(" mismatch\n");
F
Fred 已提交
1829
		nfs_mark_request_dirty(req);
1830
		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
L
Linus Torvalds 已提交
1831
	next:
1832
		nfs_unlock_and_release_request(req);
L
Linus Torvalds 已提交
1833
	}
1834 1835
	nfss = NFS_SERVER(data->inode);
	if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
1836
		clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC);
1837

1838
	nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1839
	nfs_commit_end(cinfo.mds);
1840 1841 1842 1843
}

static void nfs_commit_release(void *calldata)
{
1844
	struct nfs_commit_data *data = calldata;
1845

1846
	data->completion_ops->completion(data);
1847
	nfs_commitdata_release(calldata);
L
Linus Torvalds 已提交
1848
}
1849 1850

static const struct rpc_call_ops nfs_commit_ops = {
1851
	.rpc_call_prepare = nfs_commit_prepare,
1852 1853 1854
	.rpc_call_done = nfs_commit_done,
	.rpc_release = nfs_commit_release,
};
L
Linus Torvalds 已提交
1855

1856 1857
static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
	.completion = nfs_commit_release_pages,
1858
	.resched_write = nfs_commit_resched_write,
1859 1860
};

1861 1862
int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
			    int how, struct nfs_commit_info *cinfo)
1863 1864 1865
{
	int status;

1866
	status = pnfs_commit_list(inode, head, how, cinfo);
1867
	if (status == PNFS_NOT_ATTEMPTED)
1868
		status = nfs_commit_list(inode, head, how, cinfo);
1869 1870 1871
	return status;
}

1872
int nfs_commit_inode(struct inode *inode, int how)
L
Linus Torvalds 已提交
1873 1874
{
	LIST_HEAD(head);
1875
	struct nfs_commit_info cinfo;
1876
	int may_wait = how & FLUSH_SYNC;
1877
	int error = 0;
1878
	int res;
L
Linus Torvalds 已提交
1879

1880
	nfs_init_cinfo_from_inode(&cinfo, inode);
1881
	nfs_commit_begin(cinfo.mds);
1882
	res = nfs_scan_commit(inode, &head, &cinfo);
1883
	if (res)
1884
		error = nfs_generic_commit_list(inode, &head, how, &cinfo);
1885 1886 1887 1888 1889 1890 1891 1892
	nfs_commit_end(cinfo.mds);
	if (error < 0)
		goto out_error;
	if (!may_wait)
		goto out_mark_dirty;
	error = wait_on_commit(cinfo.mds);
	if (error < 0)
		return error;
1893
	return res;
1894 1895
out_error:
	res = error;
1896 1897 1898 1899 1900 1901 1902
	/* Note: If we exit without ensuring that the commit is complete,
	 * we must mark the inode as dirty. Otherwise, future calls to
	 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure
	 * that the data is on the disk.
	 */
out_mark_dirty:
	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
L
Linus Torvalds 已提交
1903 1904
	return res;
}
1905
EXPORT_SYMBOL_GPL(nfs_commit_inode);
1906

1907
int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1908
{
1909 1910 1911
	struct nfs_inode *nfsi = NFS_I(inode);
	int flags = FLUSH_SYNC;
	int ret = 0;
1912

1913
	/* no commits means nothing needs to be done */
1914
	if (!atomic_long_read(&nfsi->commit_info.ncommit))
1915 1916
		return ret;

1917 1918 1919 1920
	if (wbc->sync_mode == WB_SYNC_NONE) {
		/* Don't commit yet if this is a non-blocking flush and there
		 * are a lot of outstanding writes for this mapping.
		 */
1921
		if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
1922
			goto out_mark_dirty;
1923

1924
		/* don't wait for the COMMIT response */
1925
		flags = 0;
1926 1927
	}

1928 1929 1930 1931 1932 1933 1934 1935
	ret = nfs_commit_inode(inode, flags);
	if (ret >= 0) {
		if (wbc->sync_mode == WB_SYNC_NONE) {
			if (ret < wbc->nr_to_write)
				wbc->nr_to_write -= ret;
			else
				wbc->nr_to_write = 0;
		}
1936
		return 0;
1937 1938
	}
out_mark_dirty:
1939 1940 1941
	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
	return ret;
}
1942
EXPORT_SYMBOL_GPL(nfs_write_inode);
1943

1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
/*
 * Wrapper for filemap_write_and_wait_range()
 *
 * Needed for pNFS in order to ensure data becomes visible to the
 * client.
 */
int nfs_filemap_write_and_wait_range(struct address_space *mapping,
		loff_t lstart, loff_t lend)
{
	int ret;

	ret = filemap_write_and_wait_range(mapping, lstart, lend);
	if (ret == 0)
		ret = pnfs_sync_inode(mapping->host, true);
	return ret;
}
EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);

1962 1963 1964 1965
/*
 * flush the inode to disk.
 */
int nfs_wb_all(struct inode *inode)
1966
{
1967 1968 1969 1970
	int ret;

	trace_nfs_writeback_inode_enter(inode);

1971
	ret = filemap_write_and_wait(inode->i_mapping);
1972 1973 1974 1975 1976 1977 1978
	if (ret)
		goto out;
	ret = nfs_commit_inode(inode, FLUSH_SYNC);
	if (ret < 0)
		goto out;
	pnfs_sync_inode(inode, true);
	ret = 0;
1979

1980
out:
1981 1982
	trace_nfs_writeback_inode_exit(inode, ret);
	return ret;
1983
}
1984
EXPORT_SYMBOL_GPL(nfs_wb_all);
1985

1986 1987 1988 1989 1990
int nfs_wb_page_cancel(struct inode *inode, struct page *page)
{
	struct nfs_page *req;
	int ret = 0;

1991 1992 1993 1994
	wait_on_page_writeback(page);

	/* blocking call to cancel all requests and join to a single (head)
	 * request */
1995
	req = nfs_lock_and_join_requests(page);
1996 1997 1998 1999 2000 2001 2002 2003 2004 2005

	if (IS_ERR(req)) {
		ret = PTR_ERR(req);
	} else if (req) {
		/* all requests from this page have been cancelled by
		 * nfs_lock_and_join_requests, so just remove the head
		 * request from the inode / page_private pointer and
		 * release it */
		nfs_inode_remove_request(req);
		nfs_unlock_and_release_request(req);
2006
	}
2007

2008 2009 2010
	return ret;
}

2011 2012 2013
/*
 * Write back all requests on one page - we do this before reading it.
 */
2014
int nfs_wb_page(struct inode *inode, struct page *page)
2015
{
2016
	loff_t range_start = page_file_offset(page);
2017
	loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1);
2018 2019
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_ALL,
2020
		.nr_to_write = 0,
2021 2022 2023 2024
		.range_start = range_start,
		.range_end = range_end,
	};
	int ret;
2025

2026 2027
	trace_nfs_writeback_page_enter(inode);

2028
	for (;;) {
2029
		wait_on_page_writeback(page);
2030
		if (clear_page_dirty_for_io(page)) {
2031
			ret = nfs_writepage_locked(page, &wbc);
2032 2033
			if (ret < 0)
				goto out_error;
2034
			continue;
2035
		}
2036
		ret = 0;
2037 2038 2039
		if (!PagePrivate(page))
			break;
		ret = nfs_commit_inode(inode, FLUSH_SYNC);
2040
		if (ret < 0)
2041
			goto out_error;
2042
	}
2043
out_error:
2044
	trace_nfs_writeback_page_exit(inode, ret);
2045
	return ret;
2046 2047
}

2048 2049
#ifdef CONFIG_MIGRATION
int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
2050
		struct page *page, enum migrate_mode mode)
2051
{
2052 2053 2054 2055 2056 2057 2058 2059 2060 2061
	/*
	 * If PagePrivate is set, then the page is currently associated with
	 * an in-progress read or write request. Don't try to migrate it.
	 *
	 * FIXME: we could do this in principle, but we'll need a way to ensure
	 *        that we can safely release the inode reference while holding
	 *        the page lock.
	 */
	if (PagePrivate(page))
		return -EBUSY;
2062

2063 2064
	if (!nfs_fscache_release_page(page, GFP_KERNEL))
		return -EBUSY;
2065

2066
	return migrate_page(mapping, newpage, page, mode);
2067 2068 2069
}
#endif

D
David Howells 已提交
2070
int __init nfs_init_writepagecache(void)
L
Linus Torvalds 已提交
2071 2072
{
	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
2073
					     sizeof(struct nfs_pgio_header),
L
Linus Torvalds 已提交
2074
					     0, SLAB_HWCACHE_ALIGN,
2075
					     NULL);
L
Linus Torvalds 已提交
2076 2077 2078
	if (nfs_wdata_cachep == NULL)
		return -ENOMEM;

2079 2080
	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
						     nfs_wdata_cachep);
L
Linus Torvalds 已提交
2081
	if (nfs_wdata_mempool == NULL)
2082
		goto out_destroy_write_cache;
L
Linus Torvalds 已提交
2083

2084 2085 2086 2087 2088
	nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
					     sizeof(struct nfs_commit_data),
					     0, SLAB_HWCACHE_ALIGN,
					     NULL);
	if (nfs_cdata_cachep == NULL)
2089
		goto out_destroy_write_mempool;
2090

2091
	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
2092
						      nfs_cdata_cachep);
L
Linus Torvalds 已提交
2093
	if (nfs_commit_mempool == NULL)
2094
		goto out_destroy_commit_cache;
L
Linus Torvalds 已提交
2095

2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115
	/*
	 * NFS congestion size, scale with available memory.
	 *
	 *  64MB:    8192k
	 * 128MB:   11585k
	 * 256MB:   16384k
	 * 512MB:   23170k
	 *   1GB:   32768k
	 *   2GB:   46340k
	 *   4GB:   65536k
	 *   8GB:   92681k
	 *  16GB:  131072k
	 *
	 * This allows larger machines to have larger/more transfers.
	 * Limit the default to 256M
	 */
	nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
	if (nfs_congestion_kb > 256*1024)
		nfs_congestion_kb = 256*1024;

L
Linus Torvalds 已提交
2116
	return 0;
2117 2118 2119 2120 2121 2122 2123 2124

out_destroy_commit_cache:
	kmem_cache_destroy(nfs_cdata_cachep);
out_destroy_write_mempool:
	mempool_destroy(nfs_wdata_mempool);
out_destroy_write_cache:
	kmem_cache_destroy(nfs_wdata_cachep);
	return -ENOMEM;
L
Linus Torvalds 已提交
2125 2126
}

2127
void nfs_destroy_writepagecache(void)
L
Linus Torvalds 已提交
2128 2129
{
	mempool_destroy(nfs_commit_mempool);
2130
	kmem_cache_destroy(nfs_cdata_cachep);
L
Linus Torvalds 已提交
2131
	mempool_destroy(nfs_wdata_mempool);
2132
	kmem_cache_destroy(nfs_wdata_cachep);
L
Linus Torvalds 已提交
2133 2134
}

2135 2136 2137
static const struct nfs_rw_ops nfs_rw_write_ops = {
	.rw_alloc_header	= nfs_writehdr_alloc,
	.rw_free_header		= nfs_writehdr_free,
2138 2139
	.rw_done		= nfs_writeback_done,
	.rw_result		= nfs_writeback_result,
2140
	.rw_initiate		= nfs_initiate_write,
2141
};
新手
引导
客服 返回
顶部