write.c 38.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * linux/fs/nfs/write.c
 *
4
 * Write file data over NFS.
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14
 *
 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/writeback.h>
15
#include <linux/swap.h>
L
Linus Torvalds 已提交
16 17 18 19 20

#include <linux/sunrpc/clnt.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/nfs_page.h>
21 22
#include <linux/backing-dev.h>

L
Linus Torvalds 已提交
23 24 25
#include <asm/uaccess.h>

#include "delegation.h"
26
#include "internal.h"
C
Chuck Lever 已提交
27
#include "iostat.h"
L
Linus Torvalds 已提交
28 29 30 31 32 33 34 35 36 37 38 39

#define NFSDBG_FACILITY		NFSDBG_PAGECACHE

#define MIN_POOL_WRITE		(32)
#define MIN_POOL_COMMIT		(4)

/*
 * Local function declarations
 */
static struct nfs_page * nfs_update_request(struct nfs_open_context*,
					    struct page *,
					    unsigned int, unsigned int);
40 41
static void nfs_pageio_init_write(struct nfs_pageio_descriptor *desc,
				  struct inode *inode, int ioflags);
42 43 44
static const struct rpc_call_ops nfs_write_partial_ops;
static const struct rpc_call_ops nfs_write_full_ops;
static const struct rpc_call_ops nfs_commit_ops;
L
Linus Torvalds 已提交
45

46
static struct kmem_cache *nfs_wdata_cachep;
47
static mempool_t *nfs_wdata_mempool;
L
Linus Torvalds 已提交
48 49
static mempool_t *nfs_commit_mempool;

50
struct nfs_write_data *nfs_commit_alloc(void)
L
Linus Torvalds 已提交
51
{
C
Christoph Lameter 已提交
52
	struct nfs_write_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOFS);
53

L
Linus Torvalds 已提交
54 55 56 57 58 59 60
	if (p) {
		memset(p, 0, sizeof(*p));
		INIT_LIST_HEAD(&p->pages);
	}
	return p;
}

61
static void nfs_commit_rcu_free(struct rcu_head *head)
L
Linus Torvalds 已提交
62
{
63
	struct nfs_write_data *p = container_of(head, struct nfs_write_data, task.u.tk_rcu);
64 65
	if (p && (p->pagevec != &p->page_array[0]))
		kfree(p->pagevec);
L
Linus Torvalds 已提交
66 67 68
	mempool_free(p, nfs_commit_mempool);
}

69 70 71 72 73
void nfs_commit_free(struct nfs_write_data *wdata)
{
	call_rcu_bh(&wdata->task.u.tk_rcu, nfs_commit_rcu_free);
}

74
struct nfs_write_data *nfs_writedata_alloc(unsigned int pagecount)
75
{
C
Christoph Lameter 已提交
76
	struct nfs_write_data *p = mempool_alloc(nfs_wdata_mempool, GFP_NOFS);
77 78 79 80

	if (p) {
		memset(p, 0, sizeof(*p));
		INIT_LIST_HEAD(&p->pages);
81
		p->npages = pagecount;
82 83
		if (pagecount <= ARRAY_SIZE(p->page_array))
			p->pagevec = p->page_array;
84
		else {
85 86
			p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_NOFS);
			if (!p->pagevec) {
87 88 89 90 91 92 93 94
				mempool_free(p, nfs_wdata_mempool);
				p = NULL;
			}
		}
	}
	return p;
}

95
static void nfs_writedata_rcu_free(struct rcu_head *head)
96
{
97
	struct nfs_write_data *p = container_of(head, struct nfs_write_data, task.u.tk_rcu);
98 99 100 101 102
	if (p && (p->pagevec != &p->page_array[0]))
		kfree(p->pagevec);
	mempool_free(p, nfs_wdata_mempool);
}

103 104 105 106 107
static void nfs_writedata_free(struct nfs_write_data *wdata)
{
	call_rcu_bh(&wdata->task.u.tk_rcu, nfs_writedata_rcu_free);
}

108
void nfs_writedata_release(void *wdata)
L
Linus Torvalds 已提交
109 110 111 112
{
	nfs_writedata_free(wdata);
}

113 114 115 116 117 118 119
static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
{
	ctx->error = error;
	smp_wmb();
	set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
}

120 121 122 123 124 125 126
static struct nfs_page *nfs_page_find_request_locked(struct page *page)
{
	struct nfs_page *req = NULL;

	if (PagePrivate(page)) {
		req = (struct nfs_page *)page_private(page);
		if (req != NULL)
127
			kref_get(&req->wb_kref);
128 129 130 131 132 133
	}
	return req;
}

static struct nfs_page *nfs_page_find_request(struct page *page)
{
134
	struct inode *inode = page->mapping->host;
135 136
	struct nfs_page *req = NULL;

137
	spin_lock(&inode->i_lock);
138
	req = nfs_page_find_request_locked(page);
139
	spin_unlock(&inode->i_lock);
140 141 142
	return req;
}

L
Linus Torvalds 已提交
143 144 145 146 147
/* Adjust the file length if we're writing beyond the end */
static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
{
	struct inode *inode = page->mapping->host;
	loff_t end, i_size = i_size_read(inode);
148
	pgoff_t end_index = (i_size - 1) >> PAGE_CACHE_SHIFT;
L
Linus Torvalds 已提交
149 150 151 152 153 154

	if (i_size > 0 && page->index < end_index)
		return;
	end = ((loff_t)page->index << PAGE_CACHE_SHIFT) + ((loff_t)offset+count);
	if (i_size >= end)
		return;
C
Chuck Lever 已提交
155
	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
L
Linus Torvalds 已提交
156 157 158
	i_size_write(inode, end);
}

159 160 161 162 163 164 165
/* A writeback failed: mark the page as bad, and invalidate the page cache */
static void nfs_set_pageerror(struct page *page)
{
	SetPageError(page);
	nfs_zap_mapping(page->mapping->host, page->mapping);
}

L
Linus Torvalds 已提交
166 167 168 169 170 171 172 173 174
/* We can set the PG_uptodate flag if we see that a write request
 * covers the full page.
 */
static void nfs_mark_uptodate(struct page *page, unsigned int base, unsigned int count)
{
	if (PageUptodate(page))
		return;
	if (base != 0)
		return;
175
	if (count != nfs_page_length(page))
L
Linus Torvalds 已提交
176
		return;
177
	SetPageUptodate(page);
L
Linus Torvalds 已提交
178 179
}

180
static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
L
Linus Torvalds 已提交
181 182 183
		unsigned int offset, unsigned int count)
{
	struct nfs_page	*req;
184
	int ret;
L
Linus Torvalds 已提交
185

186 187 188 189 190 191 192 193 194 195 196
	for (;;) {
		req = nfs_update_request(ctx, page, offset, count);
		if (!IS_ERR(req))
			break;
		ret = PTR_ERR(req);
		if (ret != -EBUSY)
			return ret;
		ret = nfs_wb_page(page->mapping->host, page);
		if (ret != 0)
			return ret;
	}
L
Linus Torvalds 已提交
197 198
	/* Update file length */
	nfs_grow_file(page, offset, count);
199
	nfs_clear_page_tag_locked(req);
200
	return 0;
L
Linus Torvalds 已提交
201 202 203 204 205
}

static int wb_priority(struct writeback_control *wbc)
{
	if (wbc->for_reclaim)
206
		return FLUSH_HIGHPRI | FLUSH_STABLE;
L
Linus Torvalds 已提交
207 208 209 210 211
	if (wbc->for_kupdate)
		return FLUSH_LOWPRI;
	return 0;
}

212 213 214 215 216 217 218 219 220 221
/*
 * NFS congestion control
 */

int nfs_congestion_kb;

#define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
#define NFS_CONGESTION_OFF_THRESH	\
	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))

222
static int nfs_set_page_writeback(struct page *page)
223
{
224 225 226
	int ret = test_set_page_writeback(page);

	if (!ret) {
227 228 229
		struct inode *inode = page->mapping->host;
		struct nfs_server *nfss = NFS_SERVER(inode);

230
		if (atomic_long_inc_return(&nfss->writeback) >
231 232 233
				NFS_CONGESTION_ON_THRESH)
			set_bdi_congested(&nfss->backing_dev_info, WRITE);
	}
234
	return ret;
235 236 237 238 239 240 241 242
}

static void nfs_end_page_writeback(struct page *page)
{
	struct inode *inode = page->mapping->host;
	struct nfs_server *nfss = NFS_SERVER(inode);

	end_page_writeback(page);
P
Peter Zijlstra 已提交
243
	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
244 245 246
		clear_bdi_congested(&nfss->backing_dev_info, WRITE);
}

247 248
/*
 * Find an associated nfs write request, and prepare to flush it out
T
Trond Myklebust 已提交
249
 * May return an error if the user signalled nfs_wait_on_request().
250
 */
251 252
static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
				struct page *page)
253
{
254
	struct inode *inode = page->mapping->host;
255 256 257
	struct nfs_page *req;
	int ret;

258
	spin_lock(&inode->i_lock);
259 260 261
	for(;;) {
		req = nfs_page_find_request_locked(page);
		if (req == NULL) {
262
			spin_unlock(&inode->i_lock);
T
Trond Myklebust 已提交
263
			return 0;
264
		}
265
		if (nfs_set_page_tag_locked(req))
266 267
			break;
		/* Note: If we hold the page lock, as is the case in nfs_writepage,
268
		 *	 then the call to nfs_set_page_tag_locked() will always
269 270 271
		 *	 succeed provided that someone hasn't already marked the
		 *	 request as dirty (in which case we don't care).
		 */
272
		spin_unlock(&inode->i_lock);
273 274 275 276
		ret = nfs_wait_on_request(req);
		nfs_release_request(req);
		if (ret != 0)
			return ret;
277
		spin_lock(&inode->i_lock);
278
	}
279 280
	if (test_bit(PG_NEED_COMMIT, &req->wb_flags)) {
		/* This request is marked for commit */
281
		spin_unlock(&inode->i_lock);
282
		nfs_clear_page_tag_locked(req);
283
		nfs_pageio_complete(pgio);
T
Trond Myklebust 已提交
284
		return 0;
285
	}
286
	if (nfs_set_page_writeback(page) != 0) {
287
		spin_unlock(&inode->i_lock);
288 289
		BUG();
	}
290
	spin_unlock(&inode->i_lock);
291
	nfs_pageio_add_request(pgio, req);
T
Trond Myklebust 已提交
292
	return 0;
293 294
}

T
Trond Myklebust 已提交
295
static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
L
Linus Torvalds 已提交
296 297 298
{
	struct inode *inode = page->mapping->host;

C
Chuck Lever 已提交
299 300 301
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
	nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);

302
	nfs_pageio_cond_complete(pgio, page->index);
T
Trond Myklebust 已提交
303 304
	return nfs_page_async_flush(pgio, page);
}
305

T
Trond Myklebust 已提交
306 307 308 309 310 311 312
/*
 * Write an mmapped page to the server.
 */
static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc)
{
	struct nfs_pageio_descriptor pgio;
	int err;
313

T
Trond Myklebust 已提交
314 315 316 317 318 319 320 321
	nfs_pageio_init_write(&pgio, page->mapping->host, wb_priority(wbc));
	err = nfs_do_writepage(page, wbc, &pgio);
	nfs_pageio_complete(&pgio);
	if (err < 0)
		return err;
	if (pgio.pg_error < 0)
		return pgio.pg_error;
	return 0;
322 323 324 325
}

int nfs_writepage(struct page *page, struct writeback_control *wbc)
{
T
Trond Myklebust 已提交
326
	int ret;
327

T
Trond Myklebust 已提交
328
	ret = nfs_writepage_locked(page, wbc);
L
Linus Torvalds 已提交
329
	unlock_page(page);
T
Trond Myklebust 已提交
330 331 332 333 334 335 336 337 338 339
	return ret;
}

static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
{
	int ret;

	ret = nfs_do_writepage(page, wbc, data);
	unlock_page(page);
	return ret;
L
Linus Torvalds 已提交
340 341 342 343 344
}

int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
	struct inode *inode = mapping->host;
345
	struct nfs_pageio_descriptor pgio;
L
Linus Torvalds 已提交
346 347
	int err;

C
Chuck Lever 已提交
348 349
	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);

350
	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc));
T
Trond Myklebust 已提交
351
	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
352
	nfs_pageio_complete(&pgio);
T
Trond Myklebust 已提交
353
	if (err < 0)
L
Linus Torvalds 已提交
354
		return err;
T
Trond Myklebust 已提交
355
	if (pgio.pg_error < 0)
356 357
		return pgio.pg_error;
	return 0;
L
Linus Torvalds 已提交
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
}

/*
 * Insert a write request into an inode
 */
static int nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	int error;

	error = radix_tree_insert(&nfsi->nfs_page_tree, req->wb_index, req);
	BUG_ON(error == -EEXIST);
	if (error)
		return error;
	if (!nfsi->npages) {
		igrab(inode);
		if (nfs_have_delegation(inode, FMODE_WRITE))
			nfsi->change_attr++;
	}
377
	SetPagePrivate(req->wb_page);
378
	set_page_private(req->wb_page, (unsigned long)req);
L
Linus Torvalds 已提交
379
	nfsi->npages++;
380
	kref_get(&req->wb_kref);
381
	radix_tree_tag_set(&nfsi->nfs_page_tree, req->wb_index, NFS_PAGE_TAG_LOCKED);
L
Linus Torvalds 已提交
382 383 384 385
	return 0;
}

/*
386
 * Remove a write request from an inode
L
Linus Torvalds 已提交
387 388 389
 */
static void nfs_inode_remove_request(struct nfs_page *req)
{
390
	struct inode *inode = req->wb_context->path.dentry->d_inode;
L
Linus Torvalds 已提交
391 392 393 394
	struct nfs_inode *nfsi = NFS_I(inode);

	BUG_ON (!NFS_WBACK_BUSY(req));

395
	spin_lock(&inode->i_lock);
396
	set_page_private(req->wb_page, 0);
397
	ClearPagePrivate(req->wb_page);
L
Linus Torvalds 已提交
398 399 400
	radix_tree_delete(&nfsi->nfs_page_tree, req->wb_index);
	nfsi->npages--;
	if (!nfsi->npages) {
401
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
402 403
		iput(inode);
	} else
404
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
405 406 407 408
	nfs_clear_request(req);
	nfs_release_request(req);
}

409 410 411 412 413 414
static void
nfs_redirty_request(struct nfs_page *req)
{
	__set_page_dirty_nobuffers(req->wb_page);
}

L
Linus Torvalds 已提交
415 416 417 418 419 420
/*
 * Check if a request is dirty
 */
static inline int
nfs_dirty_request(struct nfs_page *req)
{
421 422
	struct page *page = req->wb_page;

423
	if (page == NULL || test_bit(PG_NEED_COMMIT, &req->wb_flags))
424 425
		return 0;
	return !PageWriteback(req->wb_page);
L
Linus Torvalds 已提交
426 427 428 429 430 431 432 433 434
}

#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
/*
 * Add a request to the inode's commit list.
 */
static void
nfs_mark_request_commit(struct nfs_page *req)
{
435
	struct inode *inode = req->wb_context->path.dentry->d_inode;
L
Linus Torvalds 已提交
436 437
	struct nfs_inode *nfsi = NFS_I(inode);

438
	spin_lock(&inode->i_lock);
L
Linus Torvalds 已提交
439
	nfsi->ncommit++;
440
	set_bit(PG_NEED_COMMIT, &(req)->wb_flags);
441 442 443
	radix_tree_tag_set(&nfsi->nfs_page_tree,
			req->wb_index,
			NFS_PAGE_TAG_COMMIT);
444
	spin_unlock(&inode->i_lock);
445
	inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
446
	inc_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_RECLAIMABLE);
447
	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
L
Linus Torvalds 已提交
448
}
449 450 451 452 453 454 455 456 457 458

static inline
int nfs_write_need_commit(struct nfs_write_data *data)
{
	return data->verf.committed != NFS_FILE_SYNC;
}

static inline
int nfs_reschedule_unstable_write(struct nfs_page *req)
{
459
	if (test_bit(PG_NEED_COMMIT, &req->wb_flags)) {
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
		nfs_mark_request_commit(req);
		return 1;
	}
	if (test_and_clear_bit(PG_NEED_RESCHED, &req->wb_flags)) {
		nfs_redirty_request(req);
		return 1;
	}
	return 0;
}
#else
static inline void
nfs_mark_request_commit(struct nfs_page *req)
{
}

static inline
int nfs_write_need_commit(struct nfs_write_data *data)
{
	return 0;
}

static inline
int nfs_reschedule_unstable_write(struct nfs_page *req)
{
	return 0;
}
L
Linus Torvalds 已提交
486 487 488 489 490 491 492
#endif

/*
 * Wait for a request to complete.
 *
 * Interruptible by signals only if mounted with intr flag.
 */
493
static int nfs_wait_on_requests_locked(struct inode *inode, pgoff_t idx_start, unsigned int npages)
L
Linus Torvalds 已提交
494 495 496
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct nfs_page *req;
497
	pgoff_t idx_end, next;
L
Linus Torvalds 已提交
498 499 500 501 502 503 504 505 506
	unsigned int		res = 0;
	int			error;

	if (npages == 0)
		idx_end = ~0;
	else
		idx_end = idx_start + npages - 1;

	next = idx_start;
507
	while (radix_tree_gang_lookup_tag(&nfsi->nfs_page_tree, (void **)&req, next, 1, NFS_PAGE_TAG_LOCKED)) {
L
Linus Torvalds 已提交
508 509 510 511
		if (req->wb_index > idx_end)
			break;

		next = req->wb_index + 1;
512
		BUG_ON(!NFS_WBACK_BUSY(req));
L
Linus Torvalds 已提交
513

514
		kref_get(&req->wb_kref);
515
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
516 517
		error = nfs_wait_on_request(req);
		nfs_release_request(req);
518
		spin_lock(&inode->i_lock);
L
Linus Torvalds 已提交
519 520 521 522 523 524 525
		if (error < 0)
			return error;
		res++;
	}
	return res;
}

T
Trond Myklebust 已提交
526 527 528 529 530 531
static void nfs_cancel_commit_list(struct list_head *head)
{
	struct nfs_page *req;

	while(!list_empty(head)) {
		req = nfs_list_entry(head->next);
532
		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
533 534
		dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
				BDI_RECLAIMABLE);
T
Trond Myklebust 已提交
535
		nfs_list_remove_request(req);
536
		clear_bit(PG_NEED_COMMIT, &(req)->wb_flags);
T
Trond Myklebust 已提交
537
		nfs_inode_remove_request(req);
538
		nfs_unlock_request(req);
T
Trond Myklebust 已提交
539 540 541
	}
}

L
Linus Torvalds 已提交
542 543 544 545 546 547 548 549 550 551 552 553
#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
/*
 * nfs_scan_commit - Scan an inode for commit requests
 * @inode: NFS inode to scan
 * @dst: destination list
 * @idx_start: lower bound of page->index to scan.
 * @npages: idx_start + npages sets the upper bound to scan.
 *
 * Moves requests from the inode's 'commit' request list.
 * The requests are *not* checked to ensure that they form a contiguous set.
 */
static int
554
nfs_scan_commit(struct inode *inode, struct list_head *dst, pgoff_t idx_start, unsigned int npages)
L
Linus Torvalds 已提交
555 556
{
	struct nfs_inode *nfsi = NFS_I(inode);
557 558 559
	int res = 0;

	if (nfsi->ncommit != 0) {
560 561
		res = nfs_scan_list(nfsi, dst, idx_start, npages,
				NFS_PAGE_TAG_COMMIT);
562 563
		nfsi->ncommit -= res;
	}
L
Linus Torvalds 已提交
564 565
	return res;
}
566
#else
567
static inline int nfs_scan_commit(struct inode *inode, struct list_head *dst, pgoff_t idx_start, unsigned int npages)
568 569 570
{
	return 0;
}
L
Linus Torvalds 已提交
571 572 573 574 575 576 577 578 579 580
#endif

/*
 * Try to update any existing write request, or create one if there is none.
 * In order to match, the request's credentials must match those of
 * the calling process.
 *
 * Note: Should always be called with the Page Lock held!
 */
static struct nfs_page * nfs_update_request(struct nfs_open_context* ctx,
581
		struct page *page, unsigned int offset, unsigned int bytes)
L
Linus Torvalds 已提交
582
{
583 584
	struct address_space *mapping = page->mapping;
	struct inode *inode = mapping->host;
L
Linus Torvalds 已提交
585
	struct nfs_page		*req, *new = NULL;
586
	pgoff_t		rqend, end;
L
Linus Torvalds 已提交
587 588 589 590 591 592 593

	end = offset + bytes;

	for (;;) {
		/* Loop over all inode entries and see if we find
		 * A request for the page we wish to update
		 */
594
		spin_lock(&inode->i_lock);
595
		req = nfs_page_find_request_locked(page);
L
Linus Torvalds 已提交
596
		if (req) {
597
			if (!nfs_set_page_tag_locked(req)) {
L
Linus Torvalds 已提交
598
				int error;
599

600
				spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
601 602
				error = nfs_wait_on_request(req);
				nfs_release_request(req);
N
Neil Brown 已提交
603 604 605
				if (error < 0) {
					if (new)
						nfs_release_request(new);
L
Linus Torvalds 已提交
606
					return ERR_PTR(error);
N
Neil Brown 已提交
607
				}
L
Linus Torvalds 已提交
608 609
				continue;
			}
610
			spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
611 612 613 614 615 616 617 618 619 620
			if (new)
				nfs_release_request(new);
			break;
		}

		if (new) {
			int error;
			nfs_lock_request_dontget(new);
			error = nfs_inode_add_request(inode, new);
			if (error) {
621
				spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
622 623 624
				nfs_unlock_request(new);
				return ERR_PTR(error);
			}
625
			spin_unlock(&inode->i_lock);
T
Trond Myklebust 已提交
626 627
			req = new;
			goto zero_page;
L
Linus Torvalds 已提交
628
		}
629
		spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646

		new = nfs_create_request(ctx, inode, page, offset, bytes);
		if (IS_ERR(new))
			return new;
	}

	/* We have a request for our page.
	 * If the creds don't match, or the
	 * page addresses don't match,
	 * tell the caller to wait on the conflicting
	 * request.
	 */
	rqend = req->wb_offset + req->wb_bytes;
	if (req->wb_context != ctx
	    || req->wb_page != page
	    || !nfs_dirty_request(req)
	    || offset > rqend || end < req->wb_offset) {
647
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
648 649 650 651 652 653 654
		return ERR_PTR(-EBUSY);
	}

	/* Okay, the request matches. Update the region */
	if (offset < req->wb_offset) {
		req->wb_offset = offset;
		req->wb_pgbase = offset;
T
Trond Myklebust 已提交
655 656
		req->wb_bytes = max(end, rqend) - req->wb_offset;
		goto zero_page;
L
Linus Torvalds 已提交
657 658 659 660 661 662
	}

	if (end > rqend)
		req->wb_bytes = end - req->wb_offset;

	return req;
T
Trond Myklebust 已提交
663 664 665 666 667 668 669 670 671
zero_page:
	/* If this page might potentially be marked as up to date,
	 * then we need to zero any uninitalised data. */
	if (req->wb_pgbase == 0 && req->wb_bytes != PAGE_CACHE_SIZE
			&& !PageUptodate(req->wb_page))
		zero_user_page(req->wb_page, req->wb_bytes,
				PAGE_CACHE_SIZE - req->wb_bytes,
				KM_USER0);
	return req;
L
Linus Torvalds 已提交
672 673 674 675
}

int nfs_flush_incompatible(struct file *file, struct page *page)
{
676
	struct nfs_open_context *ctx = nfs_file_open_context(file);
L
Linus Torvalds 已提交
677
	struct nfs_page	*req;
T
Trond Myklebust 已提交
678
	int do_flush, status;
L
Linus Torvalds 已提交
679 680 681 682 683 684 685 686
	/*
	 * Look for a request corresponding to this page. If there
	 * is one, and it belongs to another file, we flush it out
	 * before we try to copy anything into the page. Do this
	 * due to the lack of an ACCESS-type call in NFSv2.
	 * Also do the same if we find a request from an existing
	 * dropped page.
	 */
T
Trond Myklebust 已提交
687 688 689 690 691
	do {
		req = nfs_page_find_request(page);
		if (req == NULL)
			return 0;
		do_flush = req->wb_page != page || req->wb_context != ctx
692
			|| !nfs_dirty_request(req);
L
Linus Torvalds 已提交
693
		nfs_release_request(req);
T
Trond Myklebust 已提交
694 695 696 697 698
		if (!do_flush)
			return 0;
		status = nfs_wb_page(page->mapping->host, page);
	} while (status == 0);
	return status;
L
Linus Torvalds 已提交
699 700 701 702 703 704 705 706 707 708 709
}

/*
 * Update and possibly write a cached page of an NFS file.
 *
 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
 * things with a page scheduled for an RPC call (e.g. invalidate it).
 */
int nfs_updatepage(struct file *file, struct page *page,
		unsigned int offset, unsigned int count)
{
710
	struct nfs_open_context *ctx = nfs_file_open_context(file);
L
Linus Torvalds 已提交
711 712 713
	struct inode	*inode = page->mapping->host;
	int		status = 0;

C
Chuck Lever 已提交
714 715
	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);

L
Linus Torvalds 已提交
716
	dprintk("NFS:      nfs_updatepage(%s/%s %d@%Ld)\n",
717 718
		file->f_path.dentry->d_parent->d_name.name,
		file->f_path.dentry->d_name.name, count,
719
		(long long)(page_offset(page) +offset));
L
Linus Torvalds 已提交
720 721 722 723 724

	/* If we're not using byte range locks, and we know the page
	 * is entirely in cache, it may be more efficient to avoid
	 * fragmenting write requests.
	 */
725
	if (PageUptodate(page) && inode->i_flock == NULL && !(file->f_mode & O_SYNC)) {
726
		count = max(count + offset, nfs_page_length(page));
L
Linus Torvalds 已提交
727 728 729
		offset = 0;
	}

730
	status = nfs_writepage_setup(ctx, page, offset, count);
731
	__set_page_dirty_nobuffers(page);
L
Linus Torvalds 已提交
732 733 734 735

        dprintk("NFS:      nfs_updatepage returns %d (isize %Ld)\n",
			status, (long long)i_size_read(inode));
	if (status < 0)
736
		nfs_set_pageerror(page);
L
Linus Torvalds 已提交
737 738 739 740 741 742
	return status;
}

static void nfs_writepage_release(struct nfs_page *req)
{

743 744 745 746 747 748
	if (PageError(req->wb_page)) {
		nfs_end_page_writeback(req->wb_page);
		nfs_inode_remove_request(req);
	} else if (!nfs_reschedule_unstable_write(req)) {
		/* Set the PG_uptodate flag */
		nfs_mark_uptodate(req->wb_page, req->wb_pgbase, req->wb_bytes);
749 750 751 752
		nfs_end_page_writeback(req->wb_page);
		nfs_inode_remove_request(req);
	} else
		nfs_end_page_writeback(req->wb_page);
753
	nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
754 755
}

756
static int flush_task_priority(int how)
L
Linus Torvalds 已提交
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771
{
	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
		case FLUSH_HIGHPRI:
			return RPC_PRIORITY_HIGH;
		case FLUSH_LOWPRI:
			return RPC_PRIORITY_LOW;
	}
	return RPC_PRIORITY_NORMAL;
}

/*
 * Set up the argument/result storage required for the RPC call.
 */
static void nfs_write_rpcsetup(struct nfs_page *req,
		struct nfs_write_data *data,
772
		const struct rpc_call_ops *call_ops,
L
Linus Torvalds 已提交
773 774 775
		unsigned int count, unsigned int offset,
		int how)
{
776 777
	struct inode *inode = req->wb_context->path.dentry->d_inode;
	int flags = (how & FLUSH_SYNC) ? 0 : RPC_TASK_ASYNC;
778
	int priority = flush_task_priority(how);
779
	struct rpc_task *task;
780 781 782 783 784
	struct rpc_message msg = {
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
		.rpc_cred = req->wb_context->cred,
	};
785 786
	struct rpc_task_setup task_setup_data = {
		.rpc_client = NFS_CLIENT(inode),
787
		.task = &data->task,
788
		.rpc_message = &msg,
789 790 791
		.callback_ops = call_ops,
		.callback_data = data,
		.flags = flags,
792
		.priority = priority,
793
	};
L
Linus Torvalds 已提交
794 795 796 797 798

	/* Set up the RPC argument and reply structs
	 * NB: take care not to mess about with data->commit et al. */

	data->req = req;
799
	data->inode = inode = req->wb_context->path.dentry->d_inode;
800
	data->cred = msg.rpc_cred;
L
Linus Torvalds 已提交
801 802 803 804 805 806 807

	data->args.fh     = NFS_FH(inode);
	data->args.offset = req_offset(req) + offset;
	data->args.pgbase = req->wb_pgbase + offset;
	data->args.pages  = data->pagevec;
	data->args.count  = count;
	data->args.context = req->wb_context;
808 809 810 811 812 813
	data->args.stable  = NFS_UNSTABLE;
	if (how & FLUSH_STABLE) {
		data->args.stable = NFS_DATA_SYNC;
		if (!NFS_I(inode)->ncommit)
			data->args.stable = NFS_FILE_SYNC;
	}
L
Linus Torvalds 已提交
814 815 816 817

	data->res.fattr   = &data->fattr;
	data->res.count   = count;
	data->res.verf    = &data->verf;
818
	nfs_fattr_init(&data->fattr);
L
Linus Torvalds 已提交
819

820
	/* Set up the initial task struct.  */
821
	NFS_PROTO(inode)->write_setup(data, &msg);
L
Linus Torvalds 已提交
822

C
Chuck Lever 已提交
823 824
	dprintk("NFS: %5u initiated write call "
		"(req %s/%Ld, %u bytes @ offset %Lu)\n",
825
		data->task.tk_pid,
L
Linus Torvalds 已提交
826 827 828 829 830
		inode->i_sb->s_id,
		(long long)NFS_FILEID(inode),
		count,
		(unsigned long long)data->args.offset);

831 832 833
	task = rpc_run_task(&task_setup_data);
	if (!IS_ERR(task))
		rpc_put_task(task);
L
Linus Torvalds 已提交
834 835 836 837 838 839
}

/*
 * Generate multiple small requests to write out a single
 * contiguous dirty area on one page.
 */
840
static int nfs_flush_multi(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int how)
L
Linus Torvalds 已提交
841 842 843 844
{
	struct nfs_page *req = nfs_list_entry(head->next);
	struct page *page = req->wb_page;
	struct nfs_write_data *data;
845 846
	size_t wsize = NFS_SERVER(inode)->wsize, nbytes;
	unsigned int offset;
L
Linus Torvalds 已提交
847 848 849 850 851
	int requests = 0;
	LIST_HEAD(list);

	nfs_list_remove_request(req);

852
	nbytes = count;
853 854 855
	do {
		size_t len = min(nbytes, wsize);

856
		data = nfs_writedata_alloc(1);
L
Linus Torvalds 已提交
857 858 859 860
		if (!data)
			goto out_bad;
		list_add(&data->pages, &list);
		requests++;
861 862
		nbytes -= len;
	} while (nbytes != 0);
L
Linus Torvalds 已提交
863 864 865 866
	atomic_set(&req->wb_complete, requests);

	ClearPageError(page);
	offset = 0;
867
	nbytes = count;
L
Linus Torvalds 已提交
868 869 870 871 872 873
	do {
		data = list_entry(list.next, struct nfs_write_data, pages);
		list_del_init(&data->pages);

		data->pagevec[0] = page;

874 875 876 877 878 879
		if (nbytes < wsize)
			wsize = nbytes;
		nfs_write_rpcsetup(req, data, &nfs_write_partial_ops,
				   wsize, offset, how);
		offset += wsize;
		nbytes -= wsize;
L
Linus Torvalds 已提交
880 881 882 883 884 885 886 887
	} while (nbytes != 0);

	return 0;

out_bad:
	while (!list_empty(&list)) {
		data = list_entry(list.next, struct nfs_write_data, pages);
		list_del(&data->pages);
888
		nfs_writedata_release(data);
L
Linus Torvalds 已提交
889
	}
890
	nfs_redirty_request(req);
891
	nfs_end_page_writeback(req->wb_page);
892
	nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
893 894 895 896 897 898 899 900 901 902 903
	return -ENOMEM;
}

/*
 * Create an RPC task for the given write request and kick it.
 * The page must have been locked by the caller.
 *
 * It may happen that the page we're passed is not marked dirty.
 * This is the case if nfs_updatepage detects a conflicting request
 * that has been written but not committed.
 */
904
static int nfs_flush_one(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int how)
L
Linus Torvalds 已提交
905 906 907 908 909
{
	struct nfs_page		*req;
	struct page		**pages;
	struct nfs_write_data	*data;

910
	data = nfs_writedata_alloc(npages);
L
Linus Torvalds 已提交
911 912 913 914 915 916 917 918 919 920 921 922 923 924
	if (!data)
		goto out_bad;

	pages = data->pagevec;
	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_list_add_request(req, &data->pages);
		ClearPageError(req->wb_page);
		*pages++ = req->wb_page;
	}
	req = nfs_list_entry(data->pages.next);

	/* Set up the argument struct */
925
	nfs_write_rpcsetup(req, data, &nfs_write_full_ops, count, 0, how);
L
Linus Torvalds 已提交
926 927 928 929

	return 0;
 out_bad:
	while (!list_empty(head)) {
930
		req = nfs_list_entry(head->next);
L
Linus Torvalds 已提交
931
		nfs_list_remove_request(req);
932
		nfs_redirty_request(req);
933
		nfs_end_page_writeback(req->wb_page);
934
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
935 936 937 938
	}
	return -ENOMEM;
}

939 940
static void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
				  struct inode *inode, int ioflags)
L
Linus Torvalds 已提交
941
{
T
Trond Myklebust 已提交
942
	int wsize = NFS_SERVER(inode)->wsize;
L
Linus Torvalds 已提交
943

944
	if (wsize < PAGE_CACHE_SIZE)
945
		nfs_pageio_init(pgio, inode, nfs_flush_multi, wsize, ioflags);
946
	else
947
		nfs_pageio_init(pgio, inode, nfs_flush_one, wsize, ioflags);
L
Linus Torvalds 已提交
948 949 950 951 952
}

/*
 * Handle a write reply that flushed part of a page.
 */
953
static void nfs_writeback_done_partial(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
954
{
955
	struct nfs_write_data	*data = calldata;
L
Linus Torvalds 已提交
956 957 958 959
	struct nfs_page		*req = data->req;
	struct page		*page = req->wb_page;

	dprintk("NFS: write (%s/%Ld %d@%Ld)",
960 961
		req->wb_context->path.dentry->d_inode->i_sb->s_id,
		(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
L
Linus Torvalds 已提交
962 963 964
		req->wb_bytes,
		(long long)req_offset(req));

965 966 967 968
	if (nfs_writeback_done(task, data) != 0)
		return;

	if (task->tk_status < 0) {
969
		nfs_set_pageerror(page);
970
		nfs_context_set_write_error(req->wb_context, task->tk_status);
971
		dprintk(", error = %d\n", task->tk_status);
972
		goto out;
L
Linus Torvalds 已提交
973 974
	}

975
	if (nfs_write_need_commit(data)) {
976
		struct inode *inode = page->mapping->host;
977

978
		spin_lock(&inode->i_lock);
979 980 981 982 983 984 985 986 987 988
		if (test_bit(PG_NEED_RESCHED, &req->wb_flags)) {
			/* Do nothing we need to resend the writes */
		} else if (!test_and_set_bit(PG_NEED_COMMIT, &req->wb_flags)) {
			memcpy(&req->wb_verf, &data->verf, sizeof(req->wb_verf));
			dprintk(" defer commit\n");
		} else if (memcmp(&req->wb_verf, &data->verf, sizeof(req->wb_verf))) {
			set_bit(PG_NEED_RESCHED, &req->wb_flags);
			clear_bit(PG_NEED_COMMIT, &req->wb_flags);
			dprintk(" server reboot detected\n");
		}
989
		spin_unlock(&inode->i_lock);
990 991 992 993
	} else
		dprintk(" OK\n");

out:
L
Linus Torvalds 已提交
994 995 996 997
	if (atomic_dec_and_test(&req->wb_complete))
		nfs_writepage_release(req);
}

998 999 1000 1001 1002
static const struct rpc_call_ops nfs_write_partial_ops = {
	.rpc_call_done = nfs_writeback_done_partial,
	.rpc_release = nfs_writedata_release,
};

L
Linus Torvalds 已提交
1003 1004 1005 1006 1007 1008 1009
/*
 * Handle a write reply that flushes a whole page.
 *
 * FIXME: There is an inherent race with invalidate_inode_pages and
 *	  writebacks since the page->count is kept > 1 for as long
 *	  as the page has a write request pending.
 */
1010
static void nfs_writeback_done_full(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
1011
{
1012
	struct nfs_write_data	*data = calldata;
L
Linus Torvalds 已提交
1013 1014 1015
	struct nfs_page		*req;
	struct page		*page;

1016 1017 1018
	if (nfs_writeback_done(task, data) != 0)
		return;

L
Linus Torvalds 已提交
1019 1020 1021 1022 1023 1024 1025
	/* Update attributes as result of writeback. */
	while (!list_empty(&data->pages)) {
		req = nfs_list_entry(data->pages.next);
		nfs_list_remove_request(req);
		page = req->wb_page;

		dprintk("NFS: write (%s/%Ld %d@%Ld)",
1026 1027
			req->wb_context->path.dentry->d_inode->i_sb->s_id,
			(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
L
Linus Torvalds 已提交
1028 1029 1030
			req->wb_bytes,
			(long long)req_offset(req));

1031
		if (task->tk_status < 0) {
1032
			nfs_set_pageerror(page);
1033
			nfs_context_set_write_error(req->wb_context, task->tk_status);
1034
			dprintk(", error = %d\n", task->tk_status);
1035
			goto remove_request;
L
Linus Torvalds 已提交
1036 1037
		}

1038 1039 1040 1041 1042
		if (nfs_write_need_commit(data)) {
			memcpy(&req->wb_verf, &data->verf, sizeof(req->wb_verf));
			nfs_mark_request_commit(req);
			nfs_end_page_writeback(page);
			dprintk(" marked for commit\n");
L
Linus Torvalds 已提交
1043 1044
			goto next;
		}
1045 1046
		/* Set the PG_uptodate flag? */
		nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes);
1047 1048 1049
		dprintk(" OK\n");
remove_request:
		nfs_end_page_writeback(page);
L
Linus Torvalds 已提交
1050 1051
		nfs_inode_remove_request(req);
	next:
1052
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
1053 1054 1055
	}
}

1056 1057 1058 1059 1060 1061
static const struct rpc_call_ops nfs_write_full_ops = {
	.rpc_call_done = nfs_writeback_done_full,
	.rpc_release = nfs_writedata_release,
};


L
Linus Torvalds 已提交
1062 1063 1064
/*
 * This function is called when the WRITE call is complete.
 */
1065
int nfs_writeback_done(struct rpc_task *task, struct nfs_write_data *data)
L
Linus Torvalds 已提交
1066 1067 1068
{
	struct nfs_writeargs	*argp = &data->args;
	struct nfs_writeres	*resp = &data->res;
1069
	int status;
L
Linus Torvalds 已提交
1070

C
Chuck Lever 已提交
1071
	dprintk("NFS: %5u nfs_writeback_done (status %d)\n",
L
Linus Torvalds 已提交
1072 1073
		task->tk_pid, task->tk_status);

1074 1075 1076 1077 1078 1079 1080
	/*
	 * ->write_done will attempt to use post-op attributes to detect
	 * conflicting writes by other clients.  A strict interpretation
	 * of close-to-open would allow us to continue caching even if
	 * another writer had changed the file, but some applications
	 * depend on tighter cache coherency when writing.
	 */
1081 1082 1083
	status = NFS_PROTO(data->inode)->write_done(task, data);
	if (status != 0)
		return status;
C
Chuck Lever 已提交
1084 1085
	nfs_add_stats(data->inode, NFSIOS_SERVERWRITTENBYTES, resp->count);

L
Linus Torvalds 已提交
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
	if (resp->verf->committed < argp->stable && task->tk_status >= 0) {
		/* We tried a write call, but the server did not
		 * commit data to stable storage even though we
		 * requested it.
		 * Note: There is a known bug in Tru64 < 5.0 in which
		 *	 the server reports NFS_DATA_SYNC, but performs
		 *	 NFS_FILE_SYNC. We therefore implement this checking
		 *	 as a dprintk() in order to avoid filling syslog.
		 */
		static unsigned long    complain;

		if (time_before(complain, jiffies)) {
			dprintk("NFS: faulty NFS server %s:"
				" (committed = %d) != (stable = %d)\n",
1101
				NFS_SERVER(data->inode)->nfs_client->cl_hostname,
L
Linus Torvalds 已提交
1102 1103 1104 1105 1106 1107 1108 1109 1110
				resp->verf->committed, argp->stable);
			complain = jiffies + 300 * HZ;
		}
	}
#endif
	/* Is this a short write? */
	if (task->tk_status >= 0 && resp->count < argp->count) {
		static unsigned long    complain;

C
Chuck Lever 已提交
1111 1112
		nfs_inc_stats(data->inode, NFSIOS_SHORTWRITE);

L
Linus Torvalds 已提交
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
		/* Has the server at least made some progress? */
		if (resp->count != 0) {
			/* Was this an NFSv2 write or an NFSv3 stable write? */
			if (resp->verf->committed != NFS_UNSTABLE) {
				/* Resend from where the server left off */
				argp->offset += resp->count;
				argp->pgbase += resp->count;
				argp->count -= resp->count;
			} else {
				/* Resend as a stable write in order to avoid
				 * headaches in the case of a server crash.
				 */
				argp->stable = NFS_FILE_SYNC;
			}
			rpc_restart_call(task);
1128
			return -EAGAIN;
L
Linus Torvalds 已提交
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
		}
		if (time_before(complain, jiffies)) {
			printk(KERN_WARNING
			       "NFS: Server wrote zero bytes, expected %u.\n",
					argp->count);
			complain = jiffies + 300 * HZ;
		}
		/* Can't do anything about it except throw an error. */
		task->tk_status = -EIO;
	}
1139
	return 0;
L
Linus Torvalds 已提交
1140 1141 1142 1143
}


#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
1144
void nfs_commit_release(void *wdata)
L
Linus Torvalds 已提交
1145 1146 1147 1148 1149 1150 1151 1152
{
	nfs_commit_free(wdata);
}

/*
 * Set up the argument/result storage required for the RPC call.
 */
static void nfs_commit_rpcsetup(struct list_head *head,
1153 1154
		struct nfs_write_data *data,
		int how)
L
Linus Torvalds 已提交
1155
{
1156 1157 1158
	struct nfs_page *first = nfs_list_entry(head->next);
	struct inode *inode = first->wb_context->path.dentry->d_inode;
	int flags = (how & FLUSH_SYNC) ? 0 : RPC_TASK_ASYNC;
1159
	int priority = flush_task_priority(how);
1160
	struct rpc_task *task;
1161 1162 1163 1164 1165
	struct rpc_message msg = {
		.rpc_argp = &data->args,
		.rpc_resp = &data->res,
		.rpc_cred = first->wb_context->cred,
	};
1166
	struct rpc_task_setup task_setup_data = {
1167
		.task = &data->task,
1168
		.rpc_client = NFS_CLIENT(inode),
1169
		.rpc_message = &msg,
1170 1171 1172
		.callback_ops = &nfs_commit_ops,
		.callback_data = data,
		.flags = flags,
1173
		.priority = priority,
1174
	};
L
Linus Torvalds 已提交
1175 1176 1177 1178 1179 1180 1181

	/* Set up the RPC argument and reply structs
	 * NB: take care not to mess about with data->commit et al. */

	list_splice_init(head, &data->pages);

	data->inode	  = inode;
1182
	data->cred	  = msg.rpc_cred;
L
Linus Torvalds 已提交
1183 1184

	data->args.fh     = NFS_FH(data->inode);
1185 1186 1187 1188
	/* Note: we always request a commit of the entire inode */
	data->args.offset = 0;
	data->args.count  = 0;
	data->res.count   = 0;
L
Linus Torvalds 已提交
1189 1190
	data->res.fattr   = &data->fattr;
	data->res.verf    = &data->verf;
1191
	nfs_fattr_init(&data->fattr);
1192 1193

	/* Set up the initial task struct.  */
1194
	NFS_PROTO(inode)->commit_setup(data, &msg);
L
Linus Torvalds 已提交
1195

C
Chuck Lever 已提交
1196
	dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
1197

1198 1199 1200
	task = rpc_run_task(&task_setup_data);
	if (!IS_ERR(task))
		rpc_put_task(task);
L
Linus Torvalds 已提交
1201 1202 1203 1204 1205 1206
}

/*
 * Commit dirty pages
 */
static int
1207
nfs_commit_list(struct inode *inode, struct list_head *head, int how)
L
Linus Torvalds 已提交
1208 1209 1210 1211
{
	struct nfs_write_data	*data;
	struct nfs_page         *req;

1212
	data = nfs_commit_alloc();
L
Linus Torvalds 已提交
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225

	if (!data)
		goto out_bad;

	/* Set up the argument struct */
	nfs_commit_rpcsetup(head, data, how);

	return 0;
 out_bad:
	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_mark_request_commit(req);
T
Trond Myklebust 已提交
1226
		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
1227 1228
		dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
				BDI_RECLAIMABLE);
1229
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
1230 1231 1232 1233 1234 1235 1236
	}
	return -ENOMEM;
}

/*
 * COMMIT call returned
 */
1237
static void nfs_commit_done(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
1238
{
1239
	struct nfs_write_data	*data = calldata;
L
Linus Torvalds 已提交
1240 1241
	struct nfs_page		*req;

C
Chuck Lever 已提交
1242
        dprintk("NFS: %5u nfs_commit_done (status %d)\n",
L
Linus Torvalds 已提交
1243 1244
                                task->tk_pid, task->tk_status);

1245 1246 1247 1248
	/* Call the NFS version-specific code */
	if (NFS_PROTO(data->inode)->commit_done(task, data) != 0)
		return;

L
Linus Torvalds 已提交
1249 1250 1251
	while (!list_empty(&data->pages)) {
		req = nfs_list_entry(data->pages.next);
		nfs_list_remove_request(req);
1252
		clear_bit(PG_NEED_COMMIT, &(req)->wb_flags);
1253
		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
1254 1255
		dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
				BDI_RECLAIMABLE);
L
Linus Torvalds 已提交
1256 1257

		dprintk("NFS: commit (%s/%Ld %d@%Ld)",
1258 1259
			req->wb_context->path.dentry->d_inode->i_sb->s_id,
			(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
L
Linus Torvalds 已提交
1260 1261 1262
			req->wb_bytes,
			(long long)req_offset(req));
		if (task->tk_status < 0) {
1263
			nfs_context_set_write_error(req->wb_context, task->tk_status);
L
Linus Torvalds 已提交
1264 1265 1266 1267 1268 1269 1270 1271 1272
			nfs_inode_remove_request(req);
			dprintk(", error = %d\n", task->tk_status);
			goto next;
		}

		/* Okay, COMMIT succeeded, apparently. Check the verifier
		 * returned by the server against all stored verfs. */
		if (!memcmp(req->wb_verf.verifier, data->verf.verifier, sizeof(data->verf.verifier))) {
			/* We have a match */
1273 1274 1275
			/* Set the PG_uptodate flag */
			nfs_mark_uptodate(req->wb_page, req->wb_pgbase,
					req->wb_bytes);
L
Linus Torvalds 已提交
1276 1277 1278 1279 1280 1281
			nfs_inode_remove_request(req);
			dprintk(" OK\n");
			goto next;
		}
		/* We have a mismatch. Write the page again */
		dprintk(" mismatch\n");
1282
		nfs_redirty_request(req);
L
Linus Torvalds 已提交
1283
	next:
1284
		nfs_clear_page_tag_locked(req);
L
Linus Torvalds 已提交
1285 1286
	}
}
1287 1288 1289 1290 1291

static const struct rpc_call_ops nfs_commit_ops = {
	.rpc_call_done = nfs_commit_done,
	.rpc_release = nfs_commit_release,
};
L
Linus Torvalds 已提交
1292

1293
int nfs_commit_inode(struct inode *inode, int how)
L
Linus Torvalds 已提交
1294 1295
{
	LIST_HEAD(head);
T
Trond Myklebust 已提交
1296
	int res;
L
Linus Torvalds 已提交
1297

1298
	spin_lock(&inode->i_lock);
1299
	res = nfs_scan_commit(inode, &head, 0, 0);
1300
	spin_unlock(&inode->i_lock);
L
Linus Torvalds 已提交
1301
	if (res) {
T
Trond Myklebust 已提交
1302
		int error = nfs_commit_list(inode, &head, how);
1303 1304 1305
		if (error < 0)
			return error;
	}
L
Linus Torvalds 已提交
1306 1307
	return res;
}
1308 1309 1310 1311 1312
#else
static inline int nfs_commit_list(struct inode *inode, struct list_head *head, int how)
{
	return 0;
}
L
Linus Torvalds 已提交
1313 1314
#endif

1315
long nfs_sync_mapping_wait(struct address_space *mapping, struct writeback_control *wbc, int how)
L
Linus Torvalds 已提交
1316
{
1317
	struct inode *inode = mapping->host;
1318
	pgoff_t idx_start, idx_end;
1319
	unsigned int npages = 0;
1320
	LIST_HEAD(head);
1321
	int nocommit = how & FLUSH_NOCOMMIT;
T
Trond Myklebust 已提交
1322
	long pages, ret;
L
Linus Torvalds 已提交
1323

1324 1325 1326 1327 1328 1329 1330
	/* FIXME */
	if (wbc->range_cyclic)
		idx_start = 0;
	else {
		idx_start = wbc->range_start >> PAGE_CACHE_SHIFT;
		idx_end = wbc->range_end >> PAGE_CACHE_SHIFT;
		if (idx_end > idx_start) {
1331
			pgoff_t l_npages = 1 + idx_end - idx_start;
1332 1333
			npages = l_npages;
			if (sizeof(npages) != sizeof(l_npages) &&
1334
					(pgoff_t)npages != l_npages)
1335 1336 1337
				npages = 0;
		}
	}
1338
	how &= ~FLUSH_NOCOMMIT;
1339
	spin_lock(&inode->i_lock);
L
Linus Torvalds 已提交
1340
	do {
1341 1342
		ret = nfs_wait_on_requests_locked(inode, idx_start, npages);
		if (ret != 0)
1343
			continue;
1344 1345
		if (nocommit)
			break;
1346
		pages = nfs_scan_commit(inode, &head, idx_start, npages);
1347
		if (pages == 0)
1348
			break;
1349
		if (how & FLUSH_INVALIDATE) {
1350
			spin_unlock(&inode->i_lock);
T
Trond Myklebust 已提交
1351
			nfs_cancel_commit_list(&head);
1352
			ret = pages;
1353
			spin_lock(&inode->i_lock);
1354 1355 1356
			continue;
		}
		pages += nfs_scan_commit(inode, &head, 0, 0);
1357
		spin_unlock(&inode->i_lock);
1358
		ret = nfs_commit_list(inode, &head, how);
1359 1360
		spin_lock(&inode->i_lock);

1361
	} while (ret >= 0);
1362
	spin_unlock(&inode->i_lock);
1363
	return ret;
L
Linus Torvalds 已提交
1364 1365
}

T
Trond Myklebust 已提交
1366
static int __nfs_write_mapping(struct address_space *mapping, struct writeback_control *wbc, int how)
1367 1368 1369
{
	int ret;

T
Trond Myklebust 已提交
1370
	ret = nfs_writepages(mapping, wbc);
1371 1372
	if (ret < 0)
		goto out;
T
Trond Myklebust 已提交
1373
	ret = nfs_sync_mapping_wait(mapping, wbc, how);
1374 1375 1376
	if (ret < 0)
		goto out;
	return 0;
1377
out:
1378
	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1379 1380 1381
	return ret;
}

T
Trond Myklebust 已提交
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
/* Two pass sync: first using WB_SYNC_NONE, then WB_SYNC_ALL */
static int nfs_write_mapping(struct address_space *mapping, int how)
{
	struct writeback_control wbc = {
		.bdi = mapping->backing_dev_info,
		.sync_mode = WB_SYNC_NONE,
		.nr_to_write = LONG_MAX,
		.for_writepages = 1,
		.range_cyclic = 1,
	};
	int ret;

	ret = __nfs_write_mapping(mapping, &wbc, how);
	if (ret < 0)
		return ret;
	wbc.sync_mode = WB_SYNC_ALL;
	return __nfs_write_mapping(mapping, &wbc, how);
}

1401 1402 1403 1404
/*
 * flush the inode to disk.
 */
int nfs_wb_all(struct inode *inode)
1405
{
1406 1407
	return nfs_write_mapping(inode->i_mapping, 0);
}
1408

1409 1410 1411
int nfs_wb_nocommit(struct inode *inode)
{
	return nfs_write_mapping(inode->i_mapping, FLUSH_NOCOMMIT);
1412 1413
}

1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
int nfs_wb_page_cancel(struct inode *inode, struct page *page)
{
	struct nfs_page *req;
	loff_t range_start = page_offset(page);
	loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
	struct writeback_control wbc = {
		.bdi = page->mapping->backing_dev_info,
		.sync_mode = WB_SYNC_ALL,
		.nr_to_write = LONG_MAX,
		.range_start = range_start,
		.range_end = range_end,
	};
	int ret = 0;

	BUG_ON(!PageLocked(page));
	for (;;) {
		req = nfs_page_find_request(page);
		if (req == NULL)
			goto out;
		if (test_bit(PG_NEED_COMMIT, &req->wb_flags)) {
			nfs_release_request(req);
			break;
		}
		if (nfs_lock_request_dontget(req)) {
			nfs_inode_remove_request(req);
			/*
			 * In case nfs_inode_remove_request has marked the
			 * page as being dirty
			 */
			cancel_dirty_page(page, PAGE_CACHE_SIZE);
			nfs_unlock_request(req);
			break;
		}
		ret = nfs_wait_on_request(req);
		if (ret < 0)
			goto out;
	}
	if (!PagePrivate(page))
		return 0;
	ret = nfs_sync_mapping_wait(page->mapping, &wbc, FLUSH_INVALIDATE);
out:
	return ret;
}

1458 1459
static int nfs_wb_page_priority(struct inode *inode, struct page *page,
				int how)
1460 1461 1462
{
	loff_t range_start = page_offset(page);
	loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
1463 1464 1465 1466 1467 1468 1469 1470
	struct writeback_control wbc = {
		.bdi = page->mapping->backing_dev_info,
		.sync_mode = WB_SYNC_ALL,
		.nr_to_write = LONG_MAX,
		.range_start = range_start,
		.range_end = range_end,
	};
	int ret;
1471

1472
	BUG_ON(!PageLocked(page));
1473
	if (clear_page_dirty_for_io(page)) {
1474 1475 1476 1477
		ret = nfs_writepage_locked(page, &wbc);
		if (ret < 0)
			goto out;
	}
1478 1479
	if (!PagePrivate(page))
		return 0;
1480 1481 1482 1483
	ret = nfs_sync_mapping_wait(page->mapping, &wbc, how);
	if (ret >= 0)
		return 0;
out:
1484
	__mark_inode_dirty(inode, I_DIRTY_PAGES);
1485
	return ret;
1486 1487 1488 1489 1490 1491 1492
}

/*
 * Write back all requests on one page - we do this before reading it.
 */
int nfs_wb_page(struct inode *inode, struct page* page)
{
1493
	return nfs_wb_page_priority(inode, page, FLUSH_STABLE);
1494 1495
}

D
David Howells 已提交
1496
int __init nfs_init_writepagecache(void)
L
Linus Torvalds 已提交
1497 1498 1499 1500
{
	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
					     sizeof(struct nfs_write_data),
					     0, SLAB_HWCACHE_ALIGN,
1501
					     NULL);
L
Linus Torvalds 已提交
1502 1503 1504
	if (nfs_wdata_cachep == NULL)
		return -ENOMEM;

1505 1506
	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
						     nfs_wdata_cachep);
L
Linus Torvalds 已提交
1507 1508 1509
	if (nfs_wdata_mempool == NULL)
		return -ENOMEM;

1510 1511
	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
						      nfs_wdata_cachep);
L
Linus Torvalds 已提交
1512 1513 1514
	if (nfs_commit_mempool == NULL)
		return -ENOMEM;

1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
	/*
	 * NFS congestion size, scale with available memory.
	 *
	 *  64MB:    8192k
	 * 128MB:   11585k
	 * 256MB:   16384k
	 * 512MB:   23170k
	 *   1GB:   32768k
	 *   2GB:   46340k
	 *   4GB:   65536k
	 *   8GB:   92681k
	 *  16GB:  131072k
	 *
	 * This allows larger machines to have larger/more transfers.
	 * Limit the default to 256M
	 */
	nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
	if (nfs_congestion_kb > 256*1024)
		nfs_congestion_kb = 256*1024;

L
Linus Torvalds 已提交
1535 1536 1537
	return 0;
}

1538
void nfs_destroy_writepagecache(void)
L
Linus Torvalds 已提交
1539 1540 1541
{
	mempool_destroy(nfs_commit_mempool);
	mempool_destroy(nfs_wdata_mempool);
1542
	kmem_cache_destroy(nfs_wdata_cachep);
L
Linus Torvalds 已提交
1543 1544
}