write.c 39.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * linux/fs/nfs/write.c
 *
4
 * Write file data over NFS.
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14
 *
 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/writeback.h>
15
#include <linux/swap.h>
L
Linus Torvalds 已提交
16 17 18 19 20

#include <linux/sunrpc/clnt.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/nfs_page.h>
21 22
#include <linux/backing-dev.h>

L
Linus Torvalds 已提交
23 24 25
#include <asm/uaccess.h>

#include "delegation.h"
26
#include "internal.h"
C
Chuck Lever 已提交
27
#include "iostat.h"
L
Linus Torvalds 已提交
28 29 30 31 32 33 34 35 36 37 38 39

#define NFSDBG_FACILITY		NFSDBG_PAGECACHE

#define MIN_POOL_WRITE		(32)
#define MIN_POOL_COMMIT		(4)

/*
 * Local function declarations
 */
static struct nfs_page * nfs_update_request(struct nfs_open_context*,
					    struct page *,
					    unsigned int, unsigned int);
40 41
static void nfs_pageio_init_write(struct nfs_pageio_descriptor *desc,
				  struct inode *inode, int ioflags);
42
static void nfs_redirty_request(struct nfs_page *req);
43 44 45
static const struct rpc_call_ops nfs_write_partial_ops;
static const struct rpc_call_ops nfs_write_full_ops;
static const struct rpc_call_ops nfs_commit_ops;
L
Linus Torvalds 已提交
46

47
static struct kmem_cache *nfs_wdata_cachep;
48
static mempool_t *nfs_wdata_mempool;
L
Linus Torvalds 已提交
49 50
static mempool_t *nfs_commit_mempool;

51
struct nfs_write_data *nfs_commitdata_alloc(void)
L
Linus Torvalds 已提交
52
{
C
Christoph Lameter 已提交
53
	struct nfs_write_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOFS);
54

L
Linus Torvalds 已提交
55 56 57 58 59 60 61
	if (p) {
		memset(p, 0, sizeof(*p));
		INIT_LIST_HEAD(&p->pages);
	}
	return p;
}

62
void nfs_commit_free(struct nfs_write_data *p)
L
Linus Torvalds 已提交
63
{
64 65
	if (p && (p->pagevec != &p->page_array[0]))
		kfree(p->pagevec);
L
Linus Torvalds 已提交
66 67 68
	mempool_free(p, nfs_commit_mempool);
}

69
struct nfs_write_data *nfs_writedata_alloc(unsigned int pagecount)
70
{
C
Christoph Lameter 已提交
71
	struct nfs_write_data *p = mempool_alloc(nfs_wdata_mempool, GFP_NOFS);
72 73 74 75

	if (p) {
		memset(p, 0, sizeof(*p));
		INIT_LIST_HEAD(&p->pages);
76
		p->npages = pagecount;
77 78
		if (pagecount <= ARRAY_SIZE(p->page_array))
			p->pagevec = p->page_array;
79
		else {
80 81
			p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_NOFS);
			if (!p->pagevec) {
82 83 84 85 86 87 88 89
				mempool_free(p, nfs_wdata_mempool);
				p = NULL;
			}
		}
	}
	return p;
}

90
static void nfs_writedata_free(struct nfs_write_data *p)
91 92 93 94 95 96
{
	if (p && (p->pagevec != &p->page_array[0]))
		kfree(p->pagevec);
	mempool_free(p, nfs_wdata_mempool);
}

97
void nfs_writedata_release(void *data)
L
Linus Torvalds 已提交
98
{
99 100 101
	struct nfs_write_data *wdata = data;

	put_nfs_open_context(wdata->args.context);
L
Linus Torvalds 已提交
102 103 104
	nfs_writedata_free(wdata);
}

105 106 107 108 109 110 111
static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
{
	ctx->error = error;
	smp_wmb();
	set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
}

112 113 114 115 116 117 118
static struct nfs_page *nfs_page_find_request_locked(struct page *page)
{
	struct nfs_page *req = NULL;

	if (PagePrivate(page)) {
		req = (struct nfs_page *)page_private(page);
		if (req != NULL)
119
			kref_get(&req->wb_kref);
120 121 122 123 124 125
	}
	return req;
}

static struct nfs_page *nfs_page_find_request(struct page *page)
{
126
	struct inode *inode = page->mapping->host;
127 128
	struct nfs_page *req = NULL;

129
	spin_lock(&inode->i_lock);
130
	req = nfs_page_find_request_locked(page);
131
	spin_unlock(&inode->i_lock);
132 133 134
	return req;
}

L
Linus Torvalds 已提交
135 136 137 138 139
/* Adjust the file length if we're writing beyond the end */
static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
{
	struct inode *inode = page->mapping->host;
	loff_t end, i_size = i_size_read(inode);
140
	pgoff_t end_index = (i_size - 1) >> PAGE_CACHE_SHIFT;
L
Linus Torvalds 已提交
141 142 143 144 145 146

	if (i_size > 0 && page->index < end_index)
		return;
	end = ((loff_t)page->index << PAGE_CACHE_SHIFT) + ((loff_t)offset+count);
	if (i_size >= end)
		return;
C
Chuck Lever 已提交
147
	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
L
Linus Torvalds 已提交
148 149 150
	i_size_write(inode, end);
}

151 152 153 154 155 156 157
/* A writeback failed: mark the page as bad, and invalidate the page cache */
static void nfs_set_pageerror(struct page *page)
{
	SetPageError(page);
	nfs_zap_mapping(page->mapping->host, page->mapping);
}

L
Linus Torvalds 已提交
158 159 160 161 162 163 164 165 166
/* We can set the PG_uptodate flag if we see that a write request
 * covers the full page.
 */
static void nfs_mark_uptodate(struct page *page, unsigned int base, unsigned int count)
{
	if (PageUptodate(page))
		return;
	if (base != 0)
		return;
167
	if (count != nfs_page_length(page))
L
Linus Torvalds 已提交
168
		return;
169
	SetPageUptodate(page);
L
Linus Torvalds 已提交
170 171
}

172
static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
L
Linus Torvalds 已提交
173 174 175
		unsigned int offset, unsigned int count)
{
	struct nfs_page	*req;
176
	int ret;
L
Linus Torvalds 已提交
177

178 179 180 181 182 183 184 185 186 187 188
	for (;;) {
		req = nfs_update_request(ctx, page, offset, count);
		if (!IS_ERR(req))
			break;
		ret = PTR_ERR(req);
		if (ret != -EBUSY)
			return ret;
		ret = nfs_wb_page(page->mapping->host, page);
		if (ret != 0)
			return ret;
	}
L
Linus Torvalds 已提交
189 190
	/* Update file length */
	nfs_grow_file(page, offset, count);
T
Trond Myklebust 已提交
191
	nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes);
192
	nfs_clear_page_tag_locked(req);
193
	return 0;
L
Linus Torvalds 已提交
194 195 196 197 198
}

static int wb_priority(struct writeback_control *wbc)
{
	if (wbc->for_reclaim)
199
		return FLUSH_HIGHPRI | FLUSH_STABLE;
L
Linus Torvalds 已提交
200 201 202 203 204
	if (wbc->for_kupdate)
		return FLUSH_LOWPRI;
	return 0;
}

205 206 207 208 209 210 211 212 213 214
/*
 * NFS congestion control
 */

int nfs_congestion_kb;

#define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
#define NFS_CONGESTION_OFF_THRESH	\
	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))

215
static int nfs_set_page_writeback(struct page *page)
216
{
217 218 219
	int ret = test_set_page_writeback(page);

	if (!ret) {
220 221 222
		struct inode *inode = page->mapping->host;
		struct nfs_server *nfss = NFS_SERVER(inode);

223
		if (atomic_long_inc_return(&nfss->writeback) >
224 225 226
				NFS_CONGESTION_ON_THRESH)
			set_bdi_congested(&nfss->backing_dev_info, WRITE);
	}
227
	return ret;
228 229 230 231 232 233 234 235
}

static void nfs_end_page_writeback(struct page *page)
{
	struct inode *inode = page->mapping->host;
	struct nfs_server *nfss = NFS_SERVER(inode);

	end_page_writeback(page);
P
Peter Zijlstra 已提交
236
	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
237 238 239
		clear_bdi_congested(&nfss->backing_dev_info, WRITE);
}

240 241
/*
 * Find an associated nfs write request, and prepare to flush it out
T
Trond Myklebust 已提交
242
 * May return an error if the user signalled nfs_wait_on_request().
243
 */
244 245
static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
				struct page *page)
246
{
247
	struct inode *inode = page->mapping->host;
248 249 250
	struct nfs_page *req;
	int ret;

251
	spin_lock(&inode->i_lock);
252 253 254
	for(;;) {
		req = nfs_page_find_request_locked(page);
		if (req == NULL) {
255
			spin_unlock(&inode->i_lock);
T
Trond Myklebust 已提交
256
			return 0;
257
		}
258
		if (nfs_set_page_tag_locked(req))
259 260
			break;
		/* Note: If we hold the page lock, as is the case in nfs_writepage,
261
		 *	 then the call to nfs_set_page_tag_locked() will always
262 263 264
		 *	 succeed provided that someone hasn't already marked the
		 *	 request as dirty (in which case we don't care).
		 */
265
		spin_unlock(&inode->i_lock);
266 267 268 269
		ret = nfs_wait_on_request(req);
		nfs_release_request(req);
		if (ret != 0)
			return ret;
270
		spin_lock(&inode->i_lock);
271
	}
272 273
	if (test_bit(PG_NEED_COMMIT, &req->wb_flags)) {
		/* This request is marked for commit */
274
		spin_unlock(&inode->i_lock);
275
		nfs_clear_page_tag_locked(req);
276
		nfs_pageio_complete(pgio);
T
Trond Myklebust 已提交
277
		return 0;
278
	}
279
	if (nfs_set_page_writeback(page) != 0) {
280
		spin_unlock(&inode->i_lock);
281 282
		BUG();
	}
283
	spin_unlock(&inode->i_lock);
284 285 286 287
	if (!nfs_pageio_add_request(pgio, req)) {
		nfs_redirty_request(req);
		return pgio->pg_error;
	}
T
Trond Myklebust 已提交
288
	return 0;
289 290
}

T
Trond Myklebust 已提交
291
static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
L
Linus Torvalds 已提交
292 293 294
{
	struct inode *inode = page->mapping->host;

C
Chuck Lever 已提交
295 296 297
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
	nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);

298
	nfs_pageio_cond_complete(pgio, page->index);
T
Trond Myklebust 已提交
299 300
	return nfs_page_async_flush(pgio, page);
}
301

T
Trond Myklebust 已提交
302 303 304 305 306 307 308
/*
 * Write an mmapped page to the server.
 */
static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc)
{
	struct nfs_pageio_descriptor pgio;
	int err;
309

T
Trond Myklebust 已提交
310 311 312 313 314 315 316 317
	nfs_pageio_init_write(&pgio, page->mapping->host, wb_priority(wbc));
	err = nfs_do_writepage(page, wbc, &pgio);
	nfs_pageio_complete(&pgio);
	if (err < 0)
		return err;
	if (pgio.pg_error < 0)
		return pgio.pg_error;
	return 0;
318 319 320 321
}

int nfs_writepage(struct page *page, struct writeback_control *wbc)
{
T
Trond Myklebust 已提交
322
	int ret;
323

T
Trond Myklebust 已提交
324
	ret = nfs_writepage_locked(page, wbc);
L
Linus Torvalds 已提交
325
	unlock_page(page);
T
Trond Myklebust 已提交
326 327 328 329 330 331 332 333 334 335
	return ret;
}

static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
{
	int ret;

	ret = nfs_do_writepage(page, wbc, data);
	unlock_page(page);
	return ret;
L
Linus Torvalds 已提交
336 337 338 339 340
}

int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;
341
	struct nfs_pageio_descriptor pgio;
L
Linus Torvalds 已提交
342 343
	int err;

C
Chuck Lever 已提交
344 345
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);

346
	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc));
T
Trond Myklebust 已提交
347
	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
348
	nfs_pageio_complete(&pgio);
T
Trond Myklebust 已提交
349
	if (err < 0)
L
Linus Torvalds 已提交
350
		return err;
T
Trond Myklebust 已提交
351
	if (pgio.pg_error < 0)
352 353
		return pgio.pg_error;
	return 0;
L
Linus Torvalds 已提交
354 355 356 357 358
}

/*
 * Insert a write request into an inode
 */
359
static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
L
Linus Torvalds 已提交
360 361 362 363 364
{
	struct nfs_inode *nfsi = NFS_I(inode);
	int error;

	error = radix_tree_insert(&nfsi->nfs_page_tree, req->wb_index, req);
365
	BUG_ON(error);
L
Linus Torvalds 已提交
366 367 368 369 370
	if (!nfsi->npages) {
		igrab(inode);
		if (nfs_have_delegation(inode, FMODE_WRITE))
			nfsi->change_attr++;
	}
371
	SetPagePrivate(req->wb_page);
372
	set_page_private(req->wb_page, (unsigned long)req);
L
Linus Torvalds 已提交
373
	nfsi->npages++;
374
	kref_get(&req->wb_kref);
375 376
	radix_tree_tag_set(&nfsi->nfs_page_tree, req->wb_index,
				NFS_PAGE_TAG_LOCKED);
L
Linus Torvalds 已提交
377 378 379
}

/*
380
 * Remove a write request from an inode
L
Linus Torvalds 已提交
381 382 383
 */
static void nfs_inode_remove_request(struct nfs_page *req)
{
384
	struct inode *inode = req->wb_context->path.dentry->d_inode;
L
Linus Torvalds 已提交
385 386 387 388
	struct nfs_inode *nfsi = NFS_I(inode);

	BUG_ON (!NFS_WBACK_BUSY(req));

389
	spin_lock(&inode->i_lock);
390
	set_page_private(req->wb_page, 0);
391
	ClearPagePrivate(req->wb_page);
L
Linus Torvalds 已提交
392 393 394
	radix_tree_delete(&nfsi->nfs_page_tree, req->wb_index);
	nfsi->npages--;
	if (!nfsi->npages) {
395
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
396 397
		iput(inode);
	} else
398
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
399 400 401 402
	nfs_clear_request(req);
	nfs_release_request(req);
}

403
static void
F
Fred 已提交
404
nfs_mark_request_dirty(struct nfs_page *req)
405 406 407 408
{
	__set_page_dirty_nobuffers(req->wb_page);
}

L
Linus Torvalds 已提交
409 410 411 412 413 414
/*
 * Check if a request is dirty
 */
static inline int
nfs_dirty_request(struct nfs_page *req)
{
415 416
	struct page *page = req->wb_page;

417
	if (page == NULL || test_bit(PG_NEED_COMMIT, &req->wb_flags))
418
		return 0;
F
Fred Isaman 已提交
419
	return !PageWriteback(page);
L
Linus Torvalds 已提交
420 421 422 423 424 425 426 427 428
}

#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
/*
 * Add a request to the inode's commit list.
 */
static void
nfs_mark_request_commit(struct nfs_page *req)
{
429
	struct inode *inode = req->wb_context->path.dentry->d_inode;
L
Linus Torvalds 已提交
430 431
	struct nfs_inode *nfsi = NFS_I(inode);

432
	spin_lock(&inode->i_lock);
L
Linus Torvalds 已提交
433
	nfsi->ncommit++;
434
	set_bit(PG_NEED_COMMIT, &(req)->wb_flags);
435 436 437
	radix_tree_tag_set(&nfsi->nfs_page_tree,
			req->wb_index,
			NFS_PAGE_TAG_COMMIT);
438
	spin_unlock(&inode->i_lock);
439
	inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
440
	inc_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_RECLAIMABLE);
441
	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
L
Linus Torvalds 已提交
442
}
443 444 445 446 447 448 449 450 451 452

static inline
int nfs_write_need_commit(struct nfs_write_data *data)
{
	return data->verf.committed != NFS_FILE_SYNC;
}

static inline
int nfs_reschedule_unstable_write(struct nfs_page *req)
{
453
	if (test_bit(PG_NEED_COMMIT, &req->wb_flags)) {
454 455 456 457
		nfs_mark_request_commit(req);
		return 1;
	}
	if (test_and_clear_bit(PG_NEED_RESCHED, &req->wb_flags)) {
F
Fred 已提交
458
		nfs_mark_request_dirty(req);
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
		return 1;
	}
	return 0;
}
#else
static inline void
nfs_mark_request_commit(struct nfs_page *req)
{
}

static inline
int nfs_write_need_commit(struct nfs_write_data *data)
{
	return 0;
}

static inline
int nfs_reschedule_unstable_write(struct nfs_page *req)
{
	return 0;
}
L
Linus Torvalds 已提交
480 481 482 483 484
#endif

/*
 * Wait for a request to complete.
 *
485
 * Interruptible by fatal signals only.
L
Linus Torvalds 已提交
486
 */
487
static int nfs_wait_on_requests_locked(struct inode *inode, pgoff_t idx_start, unsigned int npages)
L
Linus Torvalds 已提交
488 489 490
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct nfs_page *req;
491
	pgoff_t idx_end, next;
L
Linus Torvalds 已提交
492 493 494 495 496 497 498 499 500
	unsigned int		res = 0;
	int			error;

	if (npages == 0)
		idx_end = ~0;
	else
		idx_end = idx_start + npages - 1;

	next = idx_start;
501
	while (radix_tree_gang_lookup_tag(&nfsi->nfs_page_tree, (void **)&req, next, 1, NFS_PAGE_TAG_LOCKED)) {
L
Linus Torvalds 已提交
502 503 504 505
		if (req->wb_index > idx_end)
			break;

		next = req->wb_index + 1;
506
		BUG_ON(!NFS_WBACK_BUSY(req));
L
Linus Torvalds 已提交
507

508
		kref_get(&req->wb_kref);
509
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
510 511
		error = nfs_wait_on_request(req);
		nfs_release_request(req);
512
		spin_lock(&inode->i_lock);
L
Linus Torvalds 已提交
513 514 515 516 517 518 519
		if (error < 0)
			return error;
		res++;
	}
	return res;
}

T
Trond Myklebust 已提交
520 521 522 523 524 525
static void nfs_cancel_commit_list(struct list_head *head)
{
	struct nfs_page *req;

	while(!list_empty(head)) {
		req = nfs_list_entry(head->next);
526
		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
527 528
		dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
				BDI_RECLAIMABLE);
T
Trond Myklebust 已提交
529
		nfs_list_remove_request(req);
530
		clear_bit(PG_NEED_COMMIT, &(req)->wb_flags);
T
Trond Myklebust 已提交
531
		nfs_inode_remove_request(req);
532
		nfs_unlock_request(req);
T
Trond Myklebust 已提交
533 534 535
	}
}

L
Linus Torvalds 已提交
536 537 538 539 540 541 542 543 544 545 546 547
#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
/*
 * nfs_scan_commit - Scan an inode for commit requests
 * @inode: NFS inode to scan
 * @dst: destination list
 * @idx_start: lower bound of page->index to scan.
 * @npages: idx_start + npages sets the upper bound to scan.
 *
 * Moves requests from the inode's 'commit' request list.
 * The requests are *not* checked to ensure that they form a contiguous set.
 */
static int
548
nfs_scan_commit(struct inode *inode, struct list_head *dst, pgoff_t idx_start, unsigned int npages)
L
Linus Torvalds 已提交
549 550
{
	struct nfs_inode *nfsi = NFS_I(inode);
551 552 553
	int res = 0;

	if (nfsi->ncommit != 0) {
554 555
		res = nfs_scan_list(nfsi, dst, idx_start, npages,
				NFS_PAGE_TAG_COMMIT);
556 557
		nfsi->ncommit -= res;
	}
L
Linus Torvalds 已提交
558 559
	return res;
}
560
#else
561
static inline int nfs_scan_commit(struct inode *inode, struct list_head *dst, pgoff_t idx_start, unsigned int npages)
562 563 564
{
	return 0;
}
L
Linus Torvalds 已提交
565 566 567 568 569 570 571 572 573 574
#endif

/*
 * Try to update any existing write request, or create one if there is none.
 * In order to match, the request's credentials must match those of
 * the calling process.
 *
 * Note: Should always be called with the Page Lock held!
 */
static struct nfs_page * nfs_update_request(struct nfs_open_context* ctx,
575
		struct page *page, unsigned int offset, unsigned int bytes)
L
Linus Torvalds 已提交
576
{
577 578
	struct address_space *mapping = page->mapping;
	struct inode *inode = mapping->host;
L
Linus Torvalds 已提交
579
	struct nfs_page		*req, *new = NULL;
580
	pgoff_t		rqend, end;
L
Linus Torvalds 已提交
581 582 583 584 585 586 587

	end = offset + bytes;

	for (;;) {
		/* Loop over all inode entries and see if we find
		 * A request for the page we wish to update
		 */
588
		spin_lock(&inode->i_lock);
589
		req = nfs_page_find_request_locked(page);
L
Linus Torvalds 已提交
590
		if (req) {
591
			if (!nfs_set_page_tag_locked(req)) {
L
Linus Torvalds 已提交
592
				int error;
593

594
				spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
595 596
				error = nfs_wait_on_request(req);
				nfs_release_request(req);
N
Neil Brown 已提交
597
				if (error < 0) {
598 599
					if (new) {
						radix_tree_preload_end();
N
Neil Brown 已提交
600
						nfs_release_request(new);
601
					}
L
Linus Torvalds 已提交
602
					return ERR_PTR(error);
N
Neil Brown 已提交
603
				}
L
Linus Torvalds 已提交
604 605
				continue;
			}
606
			spin_unlock(&inode->i_lock);
607 608
			if (new) {
				radix_tree_preload_end();
L
Linus Torvalds 已提交
609
				nfs_release_request(new);
610
			}
L
Linus Torvalds 已提交
611 612 613 614 615
			break;
		}

		if (new) {
			nfs_lock_request_dontget(new);
616
			nfs_inode_add_request(inode, new);
617
			spin_unlock(&inode->i_lock);
618
			radix_tree_preload_end();
T
Trond Myklebust 已提交
619
			req = new;
620
			goto out;
L
Linus Torvalds 已提交
621
		}
622
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
623 624 625 626

		new = nfs_create_request(ctx, inode, page, offset, bytes);
		if (IS_ERR(new))
			return new;
627 628 629 630
		if (radix_tree_preload(GFP_NOFS)) {
			nfs_release_request(new);
			return ERR_PTR(-ENOMEM);
		}
L
Linus Torvalds 已提交
631 632 633 634 635 636 637 638 639 640 641 642 643
	}

	/* We have a request for our page.
	 * If the creds don't match, or the
	 * page addresses don't match,
	 * tell the caller to wait on the conflicting
	 * request.
	 */
	rqend = req->wb_offset + req->wb_bytes;
	if (req->wb_context != ctx
	    || req->wb_page != page
	    || !nfs_dirty_request(req)
	    || offset > rqend || end < req->wb_offset) {
644
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
645 646 647 648 649 650 651
		return ERR_PTR(-EBUSY);
	}

	/* Okay, the request matches. Update the region */
	if (offset < req->wb_offset) {
		req->wb_offset = offset;
		req->wb_pgbase = offset;
T
Trond Myklebust 已提交
652
		req->wb_bytes = max(end, rqend) - req->wb_offset;
653
		goto out;
L
Linus Torvalds 已提交
654 655 656 657 658
	}

	if (end > rqend)
		req->wb_bytes = end - req->wb_offset;

659
out:
T
Trond Myklebust 已提交
660
	return req;
L
Linus Torvalds 已提交
661 662 663 664
}

int nfs_flush_incompatible(struct file *file, struct page *page)
{
665
	struct nfs_open_context *ctx = nfs_file_open_context(file);
L
Linus Torvalds 已提交
666
	struct nfs_page	*req;
T
Trond Myklebust 已提交
667
	int do_flush, status;
L
Linus Torvalds 已提交
668 669 670 671 672 673 674 675
	/*
	 * Look for a request corresponding to this page. If there
	 * is one, and it belongs to another file, we flush it out
	 * before we try to copy anything into the page. Do this
	 * due to the lack of an ACCESS-type call in NFSv2.
	 * Also do the same if we find a request from an existing
	 * dropped page.
	 */
T
Trond Myklebust 已提交
676 677 678 679 680
	do {
		req = nfs_page_find_request(page);
		if (req == NULL)
			return 0;
		do_flush = req->wb_page != page || req->wb_context != ctx
681
			|| !nfs_dirty_request(req);
L
Linus Torvalds 已提交
682
		nfs_release_request(req);
T
Trond Myklebust 已提交
683 684 685 686 687
		if (!do_flush)
			return 0;
		status = nfs_wb_page(page->mapping->host, page);
	} while (status == 0);
	return status;
L
Linus Torvalds 已提交
688 689
}

690 691 692 693 694 695 696 697 698 699 700
/*
 * If the page cache is marked as unsafe or invalid, then we can't rely on
 * the PageUptodate() flag. In this case, we will need to turn off
 * write optimisations that depend on the page contents being correct.
 */
static int nfs_write_pageuptodate(struct page *page, struct inode *inode)
{
	return PageUptodate(page) &&
		!(NFS_I(inode)->cache_validity & (NFS_INO_REVAL_PAGECACHE|NFS_INO_INVALID_DATA));
}

L
Linus Torvalds 已提交
701 702 703 704 705 706 707 708 709
/*
 * Update and possibly write a cached page of an NFS file.
 *
 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
 * things with a page scheduled for an RPC call (e.g. invalidate it).
 */
int nfs_updatepage(struct file *file, struct page *page,
		unsigned int offset, unsigned int count)
{
710
	struct nfs_open_context *ctx = nfs_file_open_context(file);
L
Linus Torvalds 已提交
711 712 713
	struct inode	*inode = page->mapping->host;
	int		status = 0;

C
Chuck Lever 已提交
714 715
	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);

L
Linus Torvalds 已提交
716
	dprintk("NFS:      nfs_updatepage(%s/%s %d@%Ld)\n",
717 718
		file->f_path.dentry->d_parent->d_name.name,
		file->f_path.dentry->d_name.name, count,
719
		(long long)(page_offset(page) +offset));
L
Linus Torvalds 已提交
720 721

	/* If we're not using byte range locks, and we know the page
722 723 724
	 * is up to date, it may be more efficient to extend the write
	 * to cover the entire page in order to avoid fragmentation
	 * inefficiencies.
L
Linus Torvalds 已提交
725
	 */
726 727
	if (nfs_write_pageuptodate(page, inode) &&
			inode->i_flock == NULL &&
728
			!(file->f_flags & O_SYNC)) {
729
		count = max(count + offset, nfs_page_length(page));
L
Linus Torvalds 已提交
730 731 732
		offset = 0;
	}

733
	status = nfs_writepage_setup(ctx, page, offset, count);
734 735 736 737
	if (status < 0)
		nfs_set_pageerror(page);
	else
		__set_page_dirty_nobuffers(page);
L
Linus Torvalds 已提交
738 739 740 741 742 743 744 745 746

        dprintk("NFS:      nfs_updatepage returns %d (isize %Ld)\n",
			status, (long long)i_size_read(inode));
	return status;
}

static void nfs_writepage_release(struct nfs_page *req)
{

T
Trond Myklebust 已提交
747
	if (PageError(req->wb_page) || !nfs_reschedule_unstable_write(req)) {
748 749 750 751
		nfs_end_page_writeback(req->wb_page);
		nfs_inode_remove_request(req);
	} else
		nfs_end_page_writeback(req->wb_page);
752
	nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
753 754
}

755
static int flush_task_priority(int how)
L
Linus Torvalds 已提交
756 757 758 759 760 761 762 763 764 765 766 767 768
{
	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
		case FLUSH_HIGHPRI:
			return RPC_PRIORITY_HIGH;
		case FLUSH_LOWPRI:
			return RPC_PRIORITY_LOW;
	}
	return RPC_PRIORITY_NORMAL;
}

/*
 * Set up the argument/result storage required for the RPC call.
 */
769
static int nfs_write_rpcsetup(struct nfs_page *req,
L
Linus Torvalds 已提交
770
		struct nfs_write_data *data,
771
		const struct rpc_call_ops *call_ops,
L
Linus Torvalds 已提交
772 773 774
		unsigned int count, unsigned int offset,
		int how)
{
775 776
	struct inode *inode = req->wb_context->path.dentry->d_inode;
	int flags = (how & FLUSH_SYNC) ? 0 : RPC_TASK_ASYNC;
777
	int priority = flush_task_priority(how);
778
	struct rpc_task *task;
779 780 781 782 783
	struct rpc_message msg = {
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
		.rpc_cred = req->wb_context->cred,
	};
784 785
	struct rpc_task_setup task_setup_data = {
		.rpc_client = NFS_CLIENT(inode),
786
		.task = &data->task,
787
		.rpc_message = &msg,
788 789
		.callback_ops = call_ops,
		.callback_data = data,
790
		.workqueue = nfsiod_workqueue,
791
		.flags = flags,
792
		.priority = priority,
793
	};
L
Linus Torvalds 已提交
794 795 796 797 798

	/* Set up the RPC argument and reply structs
	 * NB: take care not to mess about with data->commit et al. */

	data->req = req;
799
	data->inode = inode = req->wb_context->path.dentry->d_inode;
800
	data->cred = msg.rpc_cred;
L
Linus Torvalds 已提交
801 802 803 804 805 806

	data->args.fh     = NFS_FH(inode);
	data->args.offset = req_offset(req) + offset;
	data->args.pgbase = req->wb_pgbase + offset;
	data->args.pages  = data->pagevec;
	data->args.count  = count;
807
	data->args.context = get_nfs_open_context(req->wb_context);
808 809 810 811 812 813
	data->args.stable  = NFS_UNSTABLE;
	if (how & FLUSH_STABLE) {
		data->args.stable = NFS_DATA_SYNC;
		if (!NFS_I(inode)->ncommit)
			data->args.stable = NFS_FILE_SYNC;
	}
L
Linus Torvalds 已提交
814 815 816 817

	data->res.fattr   = &data->fattr;
	data->res.count   = count;
	data->res.verf    = &data->verf;
818
	nfs_fattr_init(&data->fattr);
L
Linus Torvalds 已提交
819

820
	/* Set up the initial task struct.  */
821
	NFS_PROTO(inode)->write_setup(data, &msg);
L
Linus Torvalds 已提交
822

C
Chuck Lever 已提交
823 824
	dprintk("NFS: %5u initiated write call "
		"(req %s/%Ld, %u bytes @ offset %Lu)\n",
825
		data->task.tk_pid,
L
Linus Torvalds 已提交
826 827 828 829 830
		inode->i_sb->s_id,
		(long long)NFS_FILEID(inode),
		count,
		(unsigned long long)data->args.offset);

831
	task = rpc_run_task(&task_setup_data);
832 833 834 835
	if (IS_ERR(task))
		return PTR_ERR(task);
	rpc_put_task(task);
	return 0;
L
Linus Torvalds 已提交
836 837
}

F
Fred 已提交
838 839 840 841 842 843 844 845 846 847 848
/* If a nfs_flush_* function fails, it should remove reqs from @head and
 * call this on each, which will prepare them to be retried on next
 * writeback using standard nfs.
 */
static void nfs_redirty_request(struct nfs_page *req)
{
	nfs_mark_request_dirty(req);
	nfs_end_page_writeback(req->wb_page);
	nfs_clear_page_tag_locked(req);
}

L
Linus Torvalds 已提交
849 850 851 852
/*
 * Generate multiple small requests to write out a single
 * contiguous dirty area on one page.
 */
853
static int nfs_flush_multi(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int how)
L
Linus Torvalds 已提交
854 855 856 857
{
	struct nfs_page *req = nfs_list_entry(head->next);
	struct page *page = req->wb_page;
	struct nfs_write_data *data;
858 859
	size_t wsize = NFS_SERVER(inode)->wsize, nbytes;
	unsigned int offset;
L
Linus Torvalds 已提交
860
	int requests = 0;
861
	int ret = 0;
L
Linus Torvalds 已提交
862 863 864 865
	LIST_HEAD(list);

	nfs_list_remove_request(req);

866
	nbytes = count;
867 868 869
	do {
		size_t len = min(nbytes, wsize);

870
		data = nfs_writedata_alloc(1);
L
Linus Torvalds 已提交
871 872 873 874
		if (!data)
			goto out_bad;
		list_add(&data->pages, &list);
		requests++;
875 876
		nbytes -= len;
	} while (nbytes != 0);
L
Linus Torvalds 已提交
877 878 879 880
	atomic_set(&req->wb_complete, requests);

	ClearPageError(page);
	offset = 0;
881
	nbytes = count;
L
Linus Torvalds 已提交
882
	do {
883 884
		int ret2;

L
Linus Torvalds 已提交
885 886 887 888 889
		data = list_entry(list.next, struct nfs_write_data, pages);
		list_del_init(&data->pages);

		data->pagevec[0] = page;

890 891
		if (nbytes < wsize)
			wsize = nbytes;
892
		ret2 = nfs_write_rpcsetup(req, data, &nfs_write_partial_ops,
893
				   wsize, offset, how);
894 895
		if (ret == 0)
			ret = ret2;
896 897
		offset += wsize;
		nbytes -= wsize;
L
Linus Torvalds 已提交
898 899
	} while (nbytes != 0);

900
	return ret;
L
Linus Torvalds 已提交
901 902 903 904 905

out_bad:
	while (!list_empty(&list)) {
		data = list_entry(list.next, struct nfs_write_data, pages);
		list_del(&data->pages);
906
		nfs_writedata_release(data);
L
Linus Torvalds 已提交
907
	}
908
	nfs_redirty_request(req);
L
Linus Torvalds 已提交
909 910 911 912 913 914 915 916 917 918 919
	return -ENOMEM;
}

/*
 * Create an RPC task for the given write request and kick it.
 * The page must have been locked by the caller.
 *
 * It may happen that the page we're passed is not marked dirty.
 * This is the case if nfs_updatepage detects a conflicting request
 * that has been written but not committed.
 */
920
static int nfs_flush_one(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int how)
L
Linus Torvalds 已提交
921 922 923 924 925
{
	struct nfs_page		*req;
	struct page		**pages;
	struct nfs_write_data	*data;

926
	data = nfs_writedata_alloc(npages);
L
Linus Torvalds 已提交
927 928 929 930 931 932 933 934 935 936 937 938 939 940
	if (!data)
		goto out_bad;

	pages = data->pagevec;
	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_list_add_request(req, &data->pages);
		ClearPageError(req->wb_page);
		*pages++ = req->wb_page;
	}
	req = nfs_list_entry(data->pages.next);

	/* Set up the argument struct */
941
	return nfs_write_rpcsetup(req, data, &nfs_write_full_ops, count, 0, how);
L
Linus Torvalds 已提交
942 943
 out_bad:
	while (!list_empty(head)) {
944
		req = nfs_list_entry(head->next);
L
Linus Torvalds 已提交
945
		nfs_list_remove_request(req);
946
		nfs_redirty_request(req);
L
Linus Torvalds 已提交
947 948 949 950
	}
	return -ENOMEM;
}

951 952
static void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
				  struct inode *inode, int ioflags)
L
Linus Torvalds 已提交
953
{
954
	size_t wsize = NFS_SERVER(inode)->wsize;
L
Linus Torvalds 已提交
955

956
	if (wsize < PAGE_CACHE_SIZE)
957
		nfs_pageio_init(pgio, inode, nfs_flush_multi, wsize, ioflags);
958
	else
959
		nfs_pageio_init(pgio, inode, nfs_flush_one, wsize, ioflags);
L
Linus Torvalds 已提交
960 961 962 963 964
}

/*
 * Handle a write reply that flushed part of a page.
 */
965
static void nfs_writeback_done_partial(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
966
{
967
	struct nfs_write_data	*data = calldata;
L
Linus Torvalds 已提交
968 969 970
	struct nfs_page		*req = data->req;

	dprintk("NFS: write (%s/%Ld %d@%Ld)",
971 972
		req->wb_context->path.dentry->d_inode->i_sb->s_id,
		(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
L
Linus Torvalds 已提交
973 974 975
		req->wb_bytes,
		(long long)req_offset(req));

976 977
	nfs_writeback_done(task, data);
}
978

979 980 981 982 983 984 985 986
static void nfs_writeback_release_partial(void *calldata)
{
	struct nfs_write_data	*data = calldata;
	struct nfs_page		*req = data->req;
	struct page		*page = req->wb_page;
	int status = data->task.tk_status;

	if (status < 0) {
987
		nfs_set_pageerror(page);
988 989
		nfs_context_set_write_error(req->wb_context, status);
		dprintk(", error = %d\n", status);
990
		goto out;
L
Linus Torvalds 已提交
991 992
	}

993
	if (nfs_write_need_commit(data)) {
994
		struct inode *inode = page->mapping->host;
995

996
		spin_lock(&inode->i_lock);
997 998 999 1000 1001 1002 1003 1004 1005 1006
		if (test_bit(PG_NEED_RESCHED, &req->wb_flags)) {
			/* Do nothing we need to resend the writes */
		} else if (!test_and_set_bit(PG_NEED_COMMIT, &req->wb_flags)) {
			memcpy(&req->wb_verf, &data->verf, sizeof(req->wb_verf));
			dprintk(" defer commit\n");
		} else if (memcmp(&req->wb_verf, &data->verf, sizeof(req->wb_verf))) {
			set_bit(PG_NEED_RESCHED, &req->wb_flags);
			clear_bit(PG_NEED_COMMIT, &req->wb_flags);
			dprintk(" server reboot detected\n");
		}
1007
		spin_unlock(&inode->i_lock);
1008 1009 1010 1011
	} else
		dprintk(" OK\n");

out:
L
Linus Torvalds 已提交
1012 1013
	if (atomic_dec_and_test(&req->wb_complete))
		nfs_writepage_release(req);
1014
	nfs_writedata_release(calldata);
L
Linus Torvalds 已提交
1015 1016
}

1017 1018
static const struct rpc_call_ops nfs_write_partial_ops = {
	.rpc_call_done = nfs_writeback_done_partial,
1019
	.rpc_release = nfs_writeback_release_partial,
1020 1021
};

L
Linus Torvalds 已提交
1022 1023 1024 1025 1026 1027 1028
/*
 * Handle a write reply that flushes a whole page.
 *
 * FIXME: There is an inherent race with invalidate_inode_pages and
 *	  writebacks since the page->count is kept > 1 for as long
 *	  as the page has a write request pending.
 */
1029
static void nfs_writeback_done_full(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
1030
{
1031
	struct nfs_write_data	*data = calldata;
L
Linus Torvalds 已提交
1032

1033 1034 1035 1036 1037 1038 1039
	nfs_writeback_done(task, data);
}

static void nfs_writeback_release_full(void *calldata)
{
	struct nfs_write_data	*data = calldata;
	int status = data->task.tk_status;
1040

L
Linus Torvalds 已提交
1041 1042
	/* Update attributes as result of writeback. */
	while (!list_empty(&data->pages)) {
1043 1044 1045
		struct nfs_page *req = nfs_list_entry(data->pages.next);
		struct page *page = req->wb_page;

L
Linus Torvalds 已提交
1046 1047 1048
		nfs_list_remove_request(req);

		dprintk("NFS: write (%s/%Ld %d@%Ld)",
1049 1050
			req->wb_context->path.dentry->d_inode->i_sb->s_id,
			(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
L
Linus Torvalds 已提交
1051 1052 1053
			req->wb_bytes,
			(long long)req_offset(req));

1054
		if (status < 0) {
1055
			nfs_set_pageerror(page);
1056 1057
			nfs_context_set_write_error(req->wb_context, status);
			dprintk(", error = %d\n", status);
1058
			goto remove_request;
L
Linus Torvalds 已提交
1059 1060
		}

1061 1062 1063 1064 1065
		if (nfs_write_need_commit(data)) {
			memcpy(&req->wb_verf, &data->verf, sizeof(req->wb_verf));
			nfs_mark_request_commit(req);
			nfs_end_page_writeback(page);
			dprintk(" marked for commit\n");
L
Linus Torvalds 已提交
1066 1067
			goto next;
		}
1068 1069 1070
		dprintk(" OK\n");
remove_request:
		nfs_end_page_writeback(page);
L
Linus Torvalds 已提交
1071 1072
		nfs_inode_remove_request(req);
	next:
1073
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
1074
	}
1075
	nfs_writedata_release(calldata);
L
Linus Torvalds 已提交
1076 1077
}

1078 1079
static const struct rpc_call_ops nfs_write_full_ops = {
	.rpc_call_done = nfs_writeback_done_full,
1080
	.rpc_release = nfs_writeback_release_full,
1081 1082 1083
};


L
Linus Torvalds 已提交
1084 1085 1086
/*
 * This function is called when the WRITE call is complete.
 */
1087
int nfs_writeback_done(struct rpc_task *task, struct nfs_write_data *data)
L
Linus Torvalds 已提交
1088 1089 1090
{
	struct nfs_writeargs	*argp = &data->args;
	struct nfs_writeres	*resp = &data->res;
1091
	int status;
L
Linus Torvalds 已提交
1092

C
Chuck Lever 已提交
1093
	dprintk("NFS: %5u nfs_writeback_done (status %d)\n",
L
Linus Torvalds 已提交
1094 1095
		task->tk_pid, task->tk_status);

1096 1097 1098 1099 1100 1101 1102
	/*
	 * ->write_done will attempt to use post-op attributes to detect
	 * conflicting writes by other clients.  A strict interpretation
	 * of close-to-open would allow us to continue caching even if
	 * another writer had changed the file, but some applications
	 * depend on tighter cache coherency when writing.
	 */
1103 1104 1105
	status = NFS_PROTO(data->inode)->write_done(task, data);
	if (status != 0)
		return status;
C
Chuck Lever 已提交
1106 1107
	nfs_add_stats(data->inode, NFSIOS_SERVERWRITTENBYTES, resp->count);

L
Linus Torvalds 已提交
1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
	if (resp->verf->committed < argp->stable && task->tk_status >= 0) {
		/* We tried a write call, but the server did not
		 * commit data to stable storage even though we
		 * requested it.
		 * Note: There is a known bug in Tru64 < 5.0 in which
		 *	 the server reports NFS_DATA_SYNC, but performs
		 *	 NFS_FILE_SYNC. We therefore implement this checking
		 *	 as a dprintk() in order to avoid filling syslog.
		 */
		static unsigned long    complain;

		if (time_before(complain, jiffies)) {
			dprintk("NFS: faulty NFS server %s:"
				" (committed = %d) != (stable = %d)\n",
1123
				NFS_SERVER(data->inode)->nfs_client->cl_hostname,
L
Linus Torvalds 已提交
1124 1125 1126 1127 1128 1129 1130 1131 1132
				resp->verf->committed, argp->stable);
			complain = jiffies + 300 * HZ;
		}
	}
#endif
	/* Is this a short write? */
	if (task->tk_status >= 0 && resp->count < argp->count) {
		static unsigned long    complain;

C
Chuck Lever 已提交
1133 1134
		nfs_inc_stats(data->inode, NFSIOS_SHORTWRITE);

L
Linus Torvalds 已提交
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
		/* Has the server at least made some progress? */
		if (resp->count != 0) {
			/* Was this an NFSv2 write or an NFSv3 stable write? */
			if (resp->verf->committed != NFS_UNSTABLE) {
				/* Resend from where the server left off */
				argp->offset += resp->count;
				argp->pgbase += resp->count;
				argp->count -= resp->count;
			} else {
				/* Resend as a stable write in order to avoid
				 * headaches in the case of a server crash.
				 */
				argp->stable = NFS_FILE_SYNC;
			}
			rpc_restart_call(task);
1150
			return -EAGAIN;
L
Linus Torvalds 已提交
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
		}
		if (time_before(complain, jiffies)) {
			printk(KERN_WARNING
			       "NFS: Server wrote zero bytes, expected %u.\n",
					argp->count);
			complain = jiffies + 300 * HZ;
		}
		/* Can't do anything about it except throw an error. */
		task->tk_status = -EIO;
	}
1161
	return 0;
L
Linus Torvalds 已提交
1162 1163 1164 1165
}


#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
1166
void nfs_commitdata_release(void *data)
L
Linus Torvalds 已提交
1167
{
1168 1169 1170
	struct nfs_write_data *wdata = data;

	put_nfs_open_context(wdata->args.context);
L
Linus Torvalds 已提交
1171 1172 1173 1174 1175 1176
	nfs_commit_free(wdata);
}

/*
 * Set up the argument/result storage required for the RPC call.
 */
1177
static int nfs_commit_rpcsetup(struct list_head *head,
1178 1179
		struct nfs_write_data *data,
		int how)
L
Linus Torvalds 已提交
1180
{
1181 1182 1183
	struct nfs_page *first = nfs_list_entry(head->next);
	struct inode *inode = first->wb_context->path.dentry->d_inode;
	int flags = (how & FLUSH_SYNC) ? 0 : RPC_TASK_ASYNC;
1184
	int priority = flush_task_priority(how);
1185
	struct rpc_task *task;
1186 1187 1188 1189 1190
	struct rpc_message msg = {
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
		.rpc_cred = first->wb_context->cred,
	};
1191
	struct rpc_task_setup task_setup_data = {
1192
		.task = &data->task,
1193
		.rpc_client = NFS_CLIENT(inode),
1194
		.rpc_message = &msg,
1195 1196
		.callback_ops = &nfs_commit_ops,
		.callback_data = data,
1197
		.workqueue = nfsiod_workqueue,
1198
		.flags = flags,
1199
		.priority = priority,
1200
	};
L
Linus Torvalds 已提交
1201 1202 1203 1204 1205 1206 1207

	/* Set up the RPC argument and reply structs
	 * NB: take care not to mess about with data->commit et al. */

	list_splice_init(head, &data->pages);

	data->inode	  = inode;
1208
	data->cred	  = msg.rpc_cred;
L
Linus Torvalds 已提交
1209 1210

	data->args.fh     = NFS_FH(data->inode);
1211 1212 1213
	/* Note: we always request a commit of the entire inode */
	data->args.offset = 0;
	data->args.count  = 0;
1214
	data->args.context = get_nfs_open_context(first->wb_context);
1215
	data->res.count   = 0;
L
Linus Torvalds 已提交
1216 1217
	data->res.fattr   = &data->fattr;
	data->res.verf    = &data->verf;
1218
	nfs_fattr_init(&data->fattr);
1219 1220

	/* Set up the initial task struct.  */
1221
	NFS_PROTO(inode)->commit_setup(data, &msg);
L
Linus Torvalds 已提交
1222

C
Chuck Lever 已提交
1223
	dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
1224

1225
	task = rpc_run_task(&task_setup_data);
1226 1227 1228 1229
	if (IS_ERR(task))
		return PTR_ERR(task);
	rpc_put_task(task);
	return 0;
L
Linus Torvalds 已提交
1230 1231 1232 1233 1234 1235
}

/*
 * Commit dirty pages
 */
static int
1236
nfs_commit_list(struct inode *inode, struct list_head *head, int how)
L
Linus Torvalds 已提交
1237 1238 1239 1240
{
	struct nfs_write_data	*data;
	struct nfs_page         *req;

1241
	data = nfs_commitdata_alloc();
L
Linus Torvalds 已提交
1242 1243 1244 1245 1246

	if (!data)
		goto out_bad;

	/* Set up the argument struct */
1247
	return nfs_commit_rpcsetup(head, data, how);
L
Linus Torvalds 已提交
1248 1249 1250 1251 1252
 out_bad:
	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_mark_request_commit(req);
T
Trond Myklebust 已提交
1253
		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
1254 1255
		dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
				BDI_RECLAIMABLE);
1256
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
1257 1258 1259 1260 1261 1262 1263
	}
	return -ENOMEM;
}

/*
 * COMMIT call returned
 */
1264
static void nfs_commit_done(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
1265
{
1266
	struct nfs_write_data	*data = calldata;
L
Linus Torvalds 已提交
1267

C
Chuck Lever 已提交
1268
        dprintk("NFS: %5u nfs_commit_done (status %d)\n",
L
Linus Torvalds 已提交
1269 1270
                                task->tk_pid, task->tk_status);

1271 1272 1273
	/* Call the NFS version-specific code */
	if (NFS_PROTO(data->inode)->commit_done(task, data) != 0)
		return;
1274 1275 1276 1277 1278 1279 1280
}

static void nfs_commit_release(void *calldata)
{
	struct nfs_write_data	*data = calldata;
	struct nfs_page		*req;
	int status = data->task.tk_status;
1281

L
Linus Torvalds 已提交
1282 1283 1284
	while (!list_empty(&data->pages)) {
		req = nfs_list_entry(data->pages.next);
		nfs_list_remove_request(req);
1285
		clear_bit(PG_NEED_COMMIT, &(req)->wb_flags);
1286
		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
1287 1288
		dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
				BDI_RECLAIMABLE);
L
Linus Torvalds 已提交
1289 1290

		dprintk("NFS: commit (%s/%Ld %d@%Ld)",
1291 1292
			req->wb_context->path.dentry->d_inode->i_sb->s_id,
			(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
L
Linus Torvalds 已提交
1293 1294
			req->wb_bytes,
			(long long)req_offset(req));
1295 1296
		if (status < 0) {
			nfs_context_set_write_error(req->wb_context, status);
L
Linus Torvalds 已提交
1297
			nfs_inode_remove_request(req);
1298
			dprintk(", error = %d\n", status);
L
Linus Torvalds 已提交
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
			goto next;
		}

		/* Okay, COMMIT succeeded, apparently. Check the verifier
		 * returned by the server against all stored verfs. */
		if (!memcmp(req->wb_verf.verifier, data->verf.verifier, sizeof(data->verf.verifier))) {
			/* We have a match */
			nfs_inode_remove_request(req);
			dprintk(" OK\n");
			goto next;
		}
		/* We have a mismatch. Write the page again */
		dprintk(" mismatch\n");
F
Fred 已提交
1312
		nfs_mark_request_dirty(req);
L
Linus Torvalds 已提交
1313
	next:
1314
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
1315
	}
1316
	nfs_commitdata_release(calldata);
L
Linus Torvalds 已提交
1317
}
1318 1319 1320 1321 1322

static const struct rpc_call_ops nfs_commit_ops = {
	.rpc_call_done = nfs_commit_done,
	.rpc_release = nfs_commit_release,
};
L
Linus Torvalds 已提交
1323

1324
int nfs_commit_inode(struct inode *inode, int how)
L
Linus Torvalds 已提交
1325 1326
{
	LIST_HEAD(head);
T
Trond Myklebust 已提交
1327
	int res;
L
Linus Torvalds 已提交
1328

1329
	spin_lock(&inode->i_lock);
1330
	res = nfs_scan_commit(inode, &head, 0, 0);
1331
	spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
1332
	if (res) {
T
Trond Myklebust 已提交
1333
		int error = nfs_commit_list(inode, &head, how);
1334 1335 1336
		if (error < 0)
			return error;
	}
L
Linus Torvalds 已提交
1337 1338
	return res;
}
1339 1340 1341 1342 1343
#else
static inline int nfs_commit_list(struct inode *inode, struct list_head *head, int how)
{
	return 0;
}
L
Linus Torvalds 已提交
1344 1345
#endif

1346
long nfs_sync_mapping_wait(struct address_space *mapping, struct writeback_control *wbc, int how)
L
Linus Torvalds 已提交
1347
{
1348
	struct inode *inode = mapping->host;
1349
	pgoff_t idx_start, idx_end;
1350
	unsigned int npages = 0;
1351
	LIST_HEAD(head);
1352
	int nocommit = how & FLUSH_NOCOMMIT;
T
Trond Myklebust 已提交
1353
	long pages, ret;
L
Linus Torvalds 已提交
1354

1355 1356 1357 1358 1359 1360 1361
	/* FIXME */
	if (wbc->range_cyclic)
		idx_start = 0;
	else {
		idx_start = wbc->range_start >> PAGE_CACHE_SHIFT;
		idx_end = wbc->range_end >> PAGE_CACHE_SHIFT;
		if (idx_end > idx_start) {
1362
			pgoff_t l_npages = 1 + idx_end - idx_start;
1363 1364
			npages = l_npages;
			if (sizeof(npages) != sizeof(l_npages) &&
1365
					(pgoff_t)npages != l_npages)
1366 1367 1368
				npages = 0;
		}
	}
1369
	how &= ~FLUSH_NOCOMMIT;
1370
	spin_lock(&inode->i_lock);
L
Linus Torvalds 已提交
1371
	do {
1372 1373
		ret = nfs_wait_on_requests_locked(inode, idx_start, npages);
		if (ret != 0)
1374
			continue;
1375 1376
		if (nocommit)
			break;
1377
		pages = nfs_scan_commit(inode, &head, idx_start, npages);
1378
		if (pages == 0)
1379
			break;
1380
		if (how & FLUSH_INVALIDATE) {
1381
			spin_unlock(&inode->i_lock);
T
Trond Myklebust 已提交
1382
			nfs_cancel_commit_list(&head);
1383
			ret = pages;
1384
			spin_lock(&inode->i_lock);
1385 1386 1387
			continue;
		}
		pages += nfs_scan_commit(inode, &head, 0, 0);
1388
		spin_unlock(&inode->i_lock);
1389
		ret = nfs_commit_list(inode, &head, how);
1390 1391
		spin_lock(&inode->i_lock);

1392
	} while (ret >= 0);
1393
	spin_unlock(&inode->i_lock);
1394
	return ret;
L
Linus Torvalds 已提交
1395 1396
}

T
Trond Myklebust 已提交
1397
static int __nfs_write_mapping(struct address_space *mapping, struct writeback_control *wbc, int how)
1398 1399 1400
{
	int ret;

T
Trond Myklebust 已提交
1401
	ret = nfs_writepages(mapping, wbc);
1402 1403
	if (ret < 0)
		goto out;
T
Trond Myklebust 已提交
1404
	ret = nfs_sync_mapping_wait(mapping, wbc, how);
1405 1406 1407
	if (ret < 0)
		goto out;
	return 0;
1408
out:
1409
	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1410 1411 1412
	return ret;
}

T
Trond Myklebust 已提交
1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
/* Two pass sync: first using WB_SYNC_NONE, then WB_SYNC_ALL */
static int nfs_write_mapping(struct address_space *mapping, int how)
{
	struct writeback_control wbc = {
		.bdi = mapping->backing_dev_info,
		.sync_mode = WB_SYNC_NONE,
		.nr_to_write = LONG_MAX,
		.for_writepages = 1,
		.range_cyclic = 1,
	};
	int ret;

	ret = __nfs_write_mapping(mapping, &wbc, how);
	if (ret < 0)
		return ret;
	wbc.sync_mode = WB_SYNC_ALL;
	return __nfs_write_mapping(mapping, &wbc, how);
}

1432 1433 1434 1435
/*
 * flush the inode to disk.
 */
int nfs_wb_all(struct inode *inode)
1436
{
1437 1438
	return nfs_write_mapping(inode->i_mapping, 0);
}
1439

1440 1441 1442
int nfs_wb_nocommit(struct inode *inode)
{
	return nfs_write_mapping(inode->i_mapping, FLUSH_NOCOMMIT);
1443 1444
}

1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
int nfs_wb_page_cancel(struct inode *inode, struct page *page)
{
	struct nfs_page *req;
	loff_t range_start = page_offset(page);
	loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
	struct writeback_control wbc = {
		.bdi = page->mapping->backing_dev_info,
		.sync_mode = WB_SYNC_ALL,
		.nr_to_write = LONG_MAX,
		.range_start = range_start,
		.range_end = range_end,
	};
	int ret = 0;

	BUG_ON(!PageLocked(page));
	for (;;) {
		req = nfs_page_find_request(page);
		if (req == NULL)
			goto out;
		if (test_bit(PG_NEED_COMMIT, &req->wb_flags)) {
			nfs_release_request(req);
			break;
		}
		if (nfs_lock_request_dontget(req)) {
			nfs_inode_remove_request(req);
			/*
			 * In case nfs_inode_remove_request has marked the
			 * page as being dirty
			 */
			cancel_dirty_page(page, PAGE_CACHE_SIZE);
			nfs_unlock_request(req);
			break;
		}
		ret = nfs_wait_on_request(req);
		if (ret < 0)
			goto out;
	}
	if (!PagePrivate(page))
		return 0;
	ret = nfs_sync_mapping_wait(page->mapping, &wbc, FLUSH_INVALIDATE);
out:
	return ret;
}

1489 1490
static int nfs_wb_page_priority(struct inode *inode, struct page *page,
				int how)
1491 1492 1493
{
	loff_t range_start = page_offset(page);
	loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
1494 1495 1496 1497 1498 1499 1500 1501
	struct writeback_control wbc = {
		.bdi = page->mapping->backing_dev_info,
		.sync_mode = WB_SYNC_ALL,
		.nr_to_write = LONG_MAX,
		.range_start = range_start,
		.range_end = range_end,
	};
	int ret;
1502

1503 1504 1505 1506 1507 1508 1509 1510
	do {
		if (clear_page_dirty_for_io(page)) {
			ret = nfs_writepage_locked(page, &wbc);
			if (ret < 0)
				goto out_error;
		} else if (!PagePrivate(page))
			break;
		ret = nfs_sync_mapping_wait(page->mapping, &wbc, how);
1511
		if (ret < 0)
1512 1513 1514 1515
			goto out_error;
	} while (PagePrivate(page));
	return 0;
out_error:
1516
	__mark_inode_dirty(inode, I_DIRTY_PAGES);
1517
	return ret;
1518 1519 1520 1521 1522 1523 1524
}

/*
 * Write back all requests on one page - we do this before reading it.
 */
int nfs_wb_page(struct inode *inode, struct page* page)
{
1525
	return nfs_wb_page_priority(inode, page, FLUSH_STABLE);
1526 1527
}

D
David Howells 已提交
1528
int __init nfs_init_writepagecache(void)
L
Linus Torvalds 已提交
1529 1530 1531 1532
{
	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
					     sizeof(struct nfs_write_data),
					     0, SLAB_HWCACHE_ALIGN,
1533
					     NULL);
L
Linus Torvalds 已提交
1534 1535 1536
	if (nfs_wdata_cachep == NULL)
		return -ENOMEM;

1537 1538
	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
						     nfs_wdata_cachep);
L
Linus Torvalds 已提交
1539 1540 1541
	if (nfs_wdata_mempool == NULL)
		return -ENOMEM;

1542 1543
	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
						      nfs_wdata_cachep);
L
Linus Torvalds 已提交
1544 1545 1546
	if (nfs_commit_mempool == NULL)
		return -ENOMEM;

1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566
	/*
	 * NFS congestion size, scale with available memory.
	 *
	 *  64MB:    8192k
	 * 128MB:   11585k
	 * 256MB:   16384k
	 * 512MB:   23170k
	 *   1GB:   32768k
	 *   2GB:   46340k
	 *   4GB:   65536k
	 *   8GB:   92681k
	 *  16GB:  131072k
	 *
	 * This allows larger machines to have larger/more transfers.
	 * Limit the default to 256M
	 */
	nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
	if (nfs_congestion_kb > 256*1024)
		nfs_congestion_kb = 256*1024;

L
Linus Torvalds 已提交
1567 1568 1569
	return 0;
}

1570
void nfs_destroy_writepagecache(void)
L
Linus Torvalds 已提交
1571 1572 1573
{
	mempool_destroy(nfs_commit_mempool);
	mempool_destroy(nfs_wdata_mempool);
1574
	kmem_cache_destroy(nfs_wdata_cachep);
L
Linus Torvalds 已提交
1575 1576
}