pagelist.c 33.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * linux/fs/nfs/pagelist.c
 *
 * A set of helper functions for managing NFS read and write requests.
 * The main purpose of these routines is to provide support for the
 * coalescing of several requests into a single RPC call.
 *
 * Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no>
 *
 */

#include <linux/slab.h>
#include <linux/file.h>
A
Alexey Dobriyan 已提交
14
#include <linux/sched.h>
L
Linus Torvalds 已提交
15
#include <linux/sunrpc/clnt.h>
16
#include <linux/nfs.h>
L
Linus Torvalds 已提交
17 18 19 20 21
#include <linux/nfs3.h>
#include <linux/nfs4.h>
#include <linux/nfs_page.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
22
#include <linux/export.h>
L
Linus Torvalds 已提交
23

24
#include "internal.h"
25
#include "pnfs.h"
26

27 28
#define NFSDBG_FACILITY		NFSDBG_PAGECACHE

29
static struct kmem_cache *nfs_page_cachep;
30
static const struct rpc_call_ops nfs_pgio_common_ops;
L
Linus Torvalds 已提交
31

32
static bool nfs_pgarray_set(struct nfs_page_array *p, unsigned int pagecount)
F
Fred Isaman 已提交
33 34 35 36 37 38 39 40 41 42 43 44
{
	p->npages = pagecount;
	if (pagecount <= ARRAY_SIZE(p->page_array))
		p->pagevec = p->page_array;
	else {
		p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL);
		if (!p->pagevec)
			p->npages = 0;
	}
	return p->pagevec != NULL;
}

45 46 47 48
void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
		       struct nfs_pgio_header *hdr,
		       void (*release)(struct nfs_pgio_header *hdr))
{
49 50 51 52
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];


	hdr->req = nfs_list_entry(mirror->pg_list.next);
53 54 55
	hdr->inode = desc->pg_inode;
	hdr->cred = hdr->req->wb_context->cred;
	hdr->io_start = req_offset(hdr->req);
56
	hdr->good_bytes = mirror->pg_count;
57
	hdr->dreq = desc->pg_dreq;
58
	hdr->layout_private = desc->pg_layout_private;
59
	hdr->release = release;
60
	hdr->completion_ops = desc->pg_completion_ops;
61 62
	if (hdr->completion_ops->init_hdr)
		hdr->completion_ops->init_hdr(hdr);
63 64

	hdr->pgio_mirror_idx = desc->pg_mirror_idx;
65
}
B
Bryan Schumaker 已提交
66
EXPORT_SYMBOL_GPL(nfs_pgheader_init);
67 68 69 70 71 72 73 74 75 76 77 78 79

void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos)
{
	spin_lock(&hdr->lock);
	if (pos < hdr->io_start + hdr->good_bytes) {
		set_bit(NFS_IOHDR_ERROR, &hdr->flags);
		clear_bit(NFS_IOHDR_EOF, &hdr->flags);
		hdr->good_bytes = pos - hdr->io_start;
		hdr->error = error;
	}
	spin_unlock(&hdr->lock);
}

L
Linus Torvalds 已提交
80 81 82
static inline struct nfs_page *
nfs_page_alloc(void)
{
83
	struct nfs_page	*p = kmem_cache_zalloc(nfs_page_cachep, GFP_NOIO);
84
	if (p)
L
Linus Torvalds 已提交
85 86 87 88 89 90 91 92 93 94
		INIT_LIST_HEAD(&p->wb_list);
	return p;
}

static inline void
nfs_page_free(struct nfs_page *p)
{
	kmem_cache_free(nfs_page_cachep, p);
}

95 96 97 98 99 100 101 102 103 104 105
static void
nfs_iocounter_inc(struct nfs_io_counter *c)
{
	atomic_inc(&c->io_count);
}

static void
nfs_iocounter_dec(struct nfs_io_counter *c)
{
	if (atomic_dec_and_test(&c->io_count)) {
		clear_bit(NFS_IO_INPROGRESS, &c->flags);
106
		smp_mb__after_atomic();
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
		wake_up_bit(&c->flags, NFS_IO_INPROGRESS);
	}
}

static int
__nfs_iocounter_wait(struct nfs_io_counter *c)
{
	wait_queue_head_t *wq = bit_waitqueue(&c->flags, NFS_IO_INPROGRESS);
	DEFINE_WAIT_BIT(q, &c->flags, NFS_IO_INPROGRESS);
	int ret = 0;

	do {
		prepare_to_wait(wq, &q.wait, TASK_KILLABLE);
		set_bit(NFS_IO_INPROGRESS, &c->flags);
		if (atomic_read(&c->io_count) == 0)
			break;
123
		ret = nfs_wait_bit_killable(&q.key);
124
	} while (atomic_read(&c->io_count) != 0 && !ret);
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
	finish_wait(wq, &q.wait);
	return ret;
}

/**
 * nfs_iocounter_wait - wait for i/o to complete
 * @c: nfs_io_counter to use
 *
 * returns -ERESTARTSYS if interrupted by a fatal signal.
 * Otherwise returns 0 once the io_count hits 0.
 */
int
nfs_iocounter_wait(struct nfs_io_counter *c)
{
	if (atomic_read(&c->io_count) == 0)
		return 0;
	return __nfs_iocounter_wait(c);
}

144 145 146
/*
 * nfs_page_group_lock - lock the head of the page group
 * @req - request in group that is to be locked
147
 * @nonblock - if true don't block waiting for lock
148 149
 *
 * this lock must be held if modifying the page group list
150
 *
151 152 153 154 155 156
 * return 0 on success, < 0 on error: -EDELAY if nonblocking or the
 * result from wait_on_bit_lock
 *
 * NOTE: calling with nonblock=false should always have set the
 *       lock bit (see fs/buffer.c and other uses of wait_on_bit_lock
 *       with TASK_UNINTERRUPTIBLE), so there is no need to check the result.
157
 */
158
int
159
nfs_page_group_lock(struct nfs_page *req, bool nonblock)
160 161 162 163 164
{
	struct nfs_page *head = req->wb_head;

	WARN_ON_ONCE(head != head->wb_head);

165 166
	if (!test_and_set_bit(PG_HEADLOCK, &head->wb_flags))
		return 0;
167

168 169 170 171 172
	if (!nonblock)
		return wait_on_bit_lock(&head->wb_flags, PG_HEADLOCK,
				TASK_UNINTERRUPTIBLE);

	return -EAGAIN;
173 174
}

175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
/*
 * nfs_page_group_lock_wait - wait for the lock to clear, but don't grab it
 * @req - a request in the group
 *
 * This is a blocking call to wait for the group lock to be cleared.
 */
void
nfs_page_group_lock_wait(struct nfs_page *req)
{
	struct nfs_page *head = req->wb_head;

	WARN_ON_ONCE(head != head->wb_head);

	wait_on_bit(&head->wb_flags, PG_HEADLOCK,
		TASK_UNINTERRUPTIBLE);
}

192 193 194 195 196 197 198 199 200 201 202
/*
 * nfs_page_group_unlock - unlock the head of the page group
 * @req - request in group that is to be unlocked
 */
void
nfs_page_group_unlock(struct nfs_page *req)
{
	struct nfs_page *head = req->wb_head;

	WARN_ON_ONCE(head != head->wb_head);

203
	smp_mb__before_atomic();
204
	clear_bit(PG_HEADLOCK, &head->wb_flags);
205
	smp_mb__after_atomic();
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
	wake_up_bit(&head->wb_flags, PG_HEADLOCK);
}

/*
 * nfs_page_group_sync_on_bit_locked
 *
 * must be called with page group lock held
 */
static bool
nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit)
{
	struct nfs_page *head = req->wb_head;
	struct nfs_page *tmp;

	WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags));
	WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));

	tmp = req->wb_this_page;
	while (tmp != req) {
		if (!test_bit(bit, &tmp->wb_flags))
			return false;
		tmp = tmp->wb_this_page;
	}

	/* true! reset all bits */
	tmp = req;
	do {
		clear_bit(bit, &tmp->wb_flags);
		tmp = tmp->wb_this_page;
	} while (tmp != req);

	return true;
}

/*
 * nfs_page_group_sync_on_bit - set bit on current request, but only
 *   return true if the bit is set for all requests in page group
 * @req - request in page group
 * @bit - PG_* bit that is used to sync page group
 */
bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit)
{
	bool ret;

250
	nfs_page_group_lock(req, false);
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
	ret = nfs_page_group_sync_on_bit_locked(req, bit);
	nfs_page_group_unlock(req);

	return ret;
}

/*
 * nfs_page_group_init - Initialize the page group linkage for @req
 * @req - a new nfs request
 * @prev - the previous request in page group, or NULL if @req is the first
 *         or only request in the group (the head).
 */
static inline void
nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev)
{
266
	struct inode *inode;
267 268 269
	WARN_ON_ONCE(prev == req);

	if (!prev) {
270
		/* a head request */
271 272 273
		req->wb_head = req;
		req->wb_this_page = req;
	} else {
274
		/* a subrequest */
275 276 277 278 279 280
		WARN_ON_ONCE(prev->wb_this_page != prev->wb_head);
		WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags));
		req->wb_head = prev->wb_head;
		req->wb_this_page = prev->wb_this_page;
		prev->wb_this_page = req;

281 282 283 284
		/* All subrequests take a ref on the head request until
		 * nfs_page_group_destroy is called */
		kref_get(&req->wb_head->wb_kref);

285 286 287
		/* grab extra ref and bump the request count if head request
		 * has extra ref from the write/commit path to handle handoff
		 * between write and commit lists. */
288
		if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) {
289
			inode = page_file_mapping(req->wb_page)->host;
290
			set_bit(PG_INODE_REF, &req->wb_flags);
291
			kref_get(&req->wb_kref);
292 293 294
			spin_lock(&inode->i_lock);
			NFS_I(inode)->nrequests++;
			spin_unlock(&inode->i_lock);
295
		}
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
	}
}

/*
 * nfs_page_group_destroy - sync the destruction of page groups
 * @req - request that no longer needs the page group
 *
 * releases the page group reference from each member once all
 * members have called this function.
 */
static void
nfs_page_group_destroy(struct kref *kref)
{
	struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);
	struct nfs_page *tmp, *next;

312 313 314 315
	/* subrequests must release the ref on the head request */
	if (req->wb_head != req)
		nfs_release_request(req->wb_head);

316 317 318 319 320 321 322 323 324 325 326 327 328 329
	if (!nfs_page_group_sync_on_bit(req, PG_TEARDOWN))
		return;

	tmp = req;
	do {
		next = tmp->wb_this_page;
		/* unlink and free */
		tmp->wb_this_page = tmp;
		tmp->wb_head = tmp;
		nfs_free_request(tmp);
		tmp = next;
	} while (tmp != req);
}

L
Linus Torvalds 已提交
330 331
/**
 * nfs_create_request - Create an NFS read/write request.
332
 * @ctx: open context to use
L
Linus Torvalds 已提交
333
 * @page: page to write
334
 * @last: last nfs request created for this page group or NULL if head
L
Linus Torvalds 已提交
335 336 337 338
 * @offset: starting offset within the page for the write
 * @count: number of bytes to read/write
 *
 * The page must be locked by the caller. This makes sure we never
339
 * create two different requests for the same page.
L
Linus Torvalds 已提交
340 341 342
 * User should ensure it is safe to sleep in this function.
 */
struct nfs_page *
343
nfs_create_request(struct nfs_open_context *ctx, struct page *page,
344 345
		   struct nfs_page *last, unsigned int offset,
		   unsigned int count)
L
Linus Torvalds 已提交
346 347
{
	struct nfs_page		*req;
348
	struct nfs_lock_context *l_ctx;
L
Linus Torvalds 已提交
349

350 351
	if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))
		return ERR_PTR(-EBADF);
352 353 354 355
	/* try to allocate the request struct */
	req = nfs_page_alloc();
	if (req == NULL)
		return ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
356

357
	/* get lock context early so we can deal with alloc failures */
358 359
	l_ctx = nfs_get_lock_context(ctx);
	if (IS_ERR(l_ctx)) {
360
		nfs_page_free(req);
361
		return ERR_CAST(l_ctx);
362
	}
363
	req->wb_lock_context = l_ctx;
364
	nfs_iocounter_inc(&l_ctx->io_count);
365

L
Linus Torvalds 已提交
366 367 368 369
	/* Initialize the request struct. Initially, we assume a
	 * long write-back delay. This will be adjusted in
	 * update_nfs_request below if the region is not locked. */
	req->wb_page    = page;
370
	req->wb_index	= page_file_index(page);
L
Linus Torvalds 已提交
371 372 373 374 375
	page_cache_get(page);
	req->wb_offset  = offset;
	req->wb_pgbase	= offset;
	req->wb_bytes   = count;
	req->wb_context = get_nfs_open_context(ctx);
376
	kref_init(&req->wb_kref);
377
	nfs_page_group_init(req, last);
L
Linus Torvalds 已提交
378 379 380 381
	return req;
}

/**
382
 * nfs_unlock_request - Unlock request and wake up sleepers.
L
Linus Torvalds 已提交
383 384
 * @req:
 */
385
void nfs_unlock_request(struct nfs_page *req)
L
Linus Torvalds 已提交
386 387 388 389 390
{
	if (!NFS_WBACK_BUSY(req)) {
		printk(KERN_ERR "NFS: Invalid unlock attempted\n");
		BUG();
	}
391
	smp_mb__before_atomic();
L
Linus Torvalds 已提交
392
	clear_bit(PG_BUSY, &req->wb_flags);
393
	smp_mb__after_atomic();
394
	wake_up_bit(&req->wb_flags, PG_BUSY);
395 396 397
}

/**
398 399
 * nfs_unlock_and_release_request - Unlock request and release the nfs_page
 * @req:
400
 */
401
void nfs_unlock_and_release_request(struct nfs_page *req)
402
{
403
	nfs_unlock_request(req);
L
Linus Torvalds 已提交
404 405 406
	nfs_release_request(req);
}

407
/*
L
Linus Torvalds 已提交
408 409 410
 * nfs_clear_request - Free up all resources allocated to the request
 * @req:
 *
411 412
 * Release page and open context resources associated with a read/write
 * request after it has completed.
L
Linus Torvalds 已提交
413
 */
414
static void nfs_clear_request(struct nfs_page *req)
L
Linus Torvalds 已提交
415
{
416
	struct page *page = req->wb_page;
417
	struct nfs_open_context *ctx = req->wb_context;
418
	struct nfs_lock_context *l_ctx = req->wb_lock_context;
419

420 421
	if (page != NULL) {
		page_cache_release(page);
L
Linus Torvalds 已提交
422 423
		req->wb_page = NULL;
	}
424
	if (l_ctx != NULL) {
425
		nfs_iocounter_dec(&l_ctx->io_count);
426 427 428
		nfs_put_lock_context(l_ctx);
		req->wb_lock_context = NULL;
	}
429 430 431 432
	if (ctx != NULL) {
		put_nfs_open_context(ctx);
		req->wb_context = NULL;
	}
L
Linus Torvalds 已提交
433 434 435 436 437 438 439 440
}

/**
 * nfs_release_request - Release the count on an NFS read/write request
 * @req: request to release
 *
 * Note: Should never be called with the spinlock held!
 */
441
void nfs_free_request(struct nfs_page *req)
L
Linus Torvalds 已提交
442
{
443 444 445 446
	WARN_ON_ONCE(req->wb_this_page != req);

	/* extra debug: make sure no sync bits are still set */
	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));
447 448
	WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));
	WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));
449 450
	WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));
	WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));
L
Linus Torvalds 已提交
451

452
	/* Release struct file and open context */
L
Linus Torvalds 已提交
453 454 455 456
	nfs_clear_request(req);
	nfs_page_free(req);
}

457 458
void nfs_release_request(struct nfs_page *req)
{
459
	kref_put(&req->wb_kref, nfs_page_group_destroy);
460 461
}

L
Linus Torvalds 已提交
462 463 464 465
/**
 * nfs_wait_on_request - Wait for a request to complete.
 * @req: request to wait upon.
 *
466
 * Interruptible by fatal signals only.
L
Linus Torvalds 已提交
467 468 469 470 471
 * The user is responsible for holding a count on the request.
 */
int
nfs_wait_on_request(struct nfs_page *req)
{
472 473
	return wait_on_bit_io(&req->wb_flags, PG_BUSY,
			      TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
474 475
}

476 477 478 479 480 481 482 483 484 485 486
/*
 * nfs_generic_pg_test - determine if requests can be coalesced
 * @desc: pointer to descriptor
 * @prev: previous request in desc, or NULL
 * @req: this request
 *
 * Returns zero if @req can be coalesced into @desc, otherwise it returns
 * the size of the request.
 */
size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
			   struct nfs_page *prev, struct nfs_page *req)
487
{
488 489 490 491
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];


	if (mirror->pg_count > mirror->pg_bsize) {
492 493
		/* should never happen */
		WARN_ON_ONCE(1);
494
		return 0;
495
	}
496

497 498 499 500
	/*
	 * Limit the request size so that we can still allocate a page array
	 * for it without upsetting the slab allocator.
	 */
501
	if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
502 503 504
			sizeof(struct page) > PAGE_SIZE)
		return 0;

505
	return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);
506
}
507
EXPORT_SYMBOL_GPL(nfs_generic_pg_test);
508

509
struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *ops)
510
{
511
	struct nfs_pgio_header *hdr = ops->rw_alloc_header();
512

513
	if (hdr) {
514 515 516 517
		INIT_LIST_HEAD(&hdr->pages);
		spin_lock_init(&hdr->lock);
		hdr->rw_ops = ops;
	}
518
	return hdr;
519
}
520
EXPORT_SYMBOL_GPL(nfs_pgio_header_alloc);
521 522

/*
523
 * nfs_pgio_header_free - Free a read or write header
524 525
 * @hdr: The header to free
 */
526
void nfs_pgio_header_free(struct nfs_pgio_header *hdr)
527
{
528
	hdr->rw_ops->rw_free_header(hdr);
529
}
530
EXPORT_SYMBOL_GPL(nfs_pgio_header_free);
531

532
/**
533 534 535 536 537 538
 * nfs_pgio_data_destroy - make @hdr suitable for reuse
 *
 * Frees memory and releases refs from nfs_generic_pgio, so that it may
 * be called again.
 *
 * @hdr: A header that has had nfs_generic_pgio called
539
 */
540
void nfs_pgio_data_destroy(struct nfs_pgio_header *hdr)
541
{
542 543
	if (hdr->args.context)
		put_nfs_open_context(hdr->args.context);
544 545
	if (hdr->page_array.pagevec != hdr->page_array.page_array)
		kfree(hdr->page_array.pagevec);
546
}
547
EXPORT_SYMBOL_GPL(nfs_pgio_data_destroy);
548

549 550
/**
 * nfs_pgio_rpcsetup - Set up arguments for a pageio call
551
 * @hdr: The pageio hdr
552 553 554 555 556
 * @count: Number of bytes to read
 * @offset: Initial offset
 * @how: How to commit data (writes only)
 * @cinfo: Commit information for the call (writes only)
 */
557
static void nfs_pgio_rpcsetup(struct nfs_pgio_header *hdr,
558 559 560
			      unsigned int count, unsigned int offset,
			      int how, struct nfs_commit_info *cinfo)
{
561
	struct nfs_page *req = hdr->req;
562 563

	/* Set up the RPC argument and reply structs
564
	 * NB: take care not to mess about with hdr->commit et al. */
565

566 567
	hdr->args.fh     = NFS_FH(hdr->inode);
	hdr->args.offset = req_offset(req) + offset;
568
	/* pnfs_set_layoutcommit needs this */
569 570 571 572 573 574 575
	hdr->mds_offset = hdr->args.offset;
	hdr->args.pgbase = req->wb_pgbase + offset;
	hdr->args.pages  = hdr->page_array.pagevec;
	hdr->args.count  = count;
	hdr->args.context = get_nfs_open_context(req->wb_context);
	hdr->args.lock_context = req->wb_lock_context;
	hdr->args.stable  = NFS_UNSTABLE;
576 577 578 579 580 581 582
	switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
	case 0:
		break;
	case FLUSH_COND_STABLE:
		if (nfs_reqs_to_commit(cinfo))
			break;
	default:
583
		hdr->args.stable = NFS_FILE_SYNC;
584 585
	}

586 587 588
	hdr->res.fattr   = &hdr->fattr;
	hdr->res.count   = count;
	hdr->res.eof     = 0;
589
	hdr->res.verf    = &hdr->verf;
590
	nfs_fattr_init(&hdr->fattr);
591 592
}

593
/**
594
 * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
595
 * @task: The current task
596
 * @calldata: pageio header to prepare
597
 */
598
static void nfs_pgio_prepare(struct rpc_task *task, void *calldata)
599
{
600
	struct nfs_pgio_header *hdr = calldata;
601
	int err;
602
	err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr);
603 604 605 606
	if (err)
		rpc_exit(task, err);
}

607
int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
608
		      struct rpc_cred *cred, const struct nfs_rpc_ops *rpc_ops,
609 610 611 612
		      const struct rpc_call_ops *call_ops, int how, int flags)
{
	struct rpc_task *task;
	struct rpc_message msg = {
613 614
		.rpc_argp = &hdr->args,
		.rpc_resp = &hdr->res,
615
		.rpc_cred = cred,
616 617 618
	};
	struct rpc_task_setup task_setup_data = {
		.rpc_client = clnt,
619
		.task = &hdr->task,
620 621
		.rpc_message = &msg,
		.callback_ops = call_ops,
622
		.callback_data = hdr,
623 624 625 626 627
		.workqueue = nfsiod_workqueue,
		.flags = RPC_TASK_ASYNC | flags,
	};
	int ret = 0;

628
	hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);
629 630 631

	dprintk("NFS: %5u initiated pgio call "
		"(req %s/%llu, %u bytes @ offset %llu)\n",
632
		hdr->task.tk_pid,
A
Anna Schumaker 已提交
633 634
		hdr->inode->i_sb->s_id,
		(unsigned long long)NFS_FILEID(hdr->inode),
635 636
		hdr->args.count,
		(unsigned long long)hdr->args.offset);
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653

	task = rpc_run_task(&task_setup_data);
	if (IS_ERR(task)) {
		ret = PTR_ERR(task);
		goto out;
	}
	if (how & FLUSH_SYNC) {
		ret = rpc_wait_for_completion_task(task);
		if (ret == 0)
			ret = task->tk_status;
	}
	rpc_put_task(task);
out:
	return ret;
}
EXPORT_SYMBOL_GPL(nfs_initiate_pgio);

654 655 656 657 658
/**
 * nfs_pgio_error - Clean up from a pageio error
 * @desc: IO descriptor
 * @hdr: pageio header
 */
659
static int nfs_pgio_error(struct nfs_pageio_descriptor *desc,
660 661
			  struct nfs_pgio_header *hdr)
{
662 663 664
	struct nfs_pgio_mirror *mirror;
	u32 midx;

665
	set_bit(NFS_IOHDR_REDO, &hdr->flags);
666
	nfs_pgio_data_destroy(hdr);
667
	hdr->completion_ops->completion(hdr);
668 669 670 671 672 673
	/* TODO: Make sure it's right to clean up all mirrors here
	 *       and not just hdr->pgio_mirror_idx */
	for (midx = 0; midx < desc->pg_mirror_count; midx++) {
		mirror = &desc->pg_mirrors[midx];
		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
	}
674 675 676
	return -ENOMEM;
}

677 678
/**
 * nfs_pgio_release - Release pageio data
679
 * @calldata: The pageio header to release
680
 */
681
static void nfs_pgio_release(void *calldata)
682
{
683 684 685 686
	struct nfs_pgio_header *hdr = calldata;
	if (hdr->rw_ops->rw_release)
		hdr->rw_ops->rw_release(hdr);
	nfs_pgio_data_destroy(hdr);
687
	hdr->completion_ops->completion(hdr);
688 689
}

690 691 692 693 694 695 696 697 698 699 700
static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,
				   unsigned int bsize)
{
	INIT_LIST_HEAD(&mirror->pg_list);
	mirror->pg_bytes_written = 0;
	mirror->pg_count = 0;
	mirror->pg_bsize = bsize;
	mirror->pg_base = 0;
	mirror->pg_recoalesce = 0;
}

L
Linus Torvalds 已提交
701
/**
702 703
 * nfs_pageio_init - initialise a page io descriptor
 * @desc: pointer to descriptor
704 705 706 707
 * @inode: pointer to inode
 * @doio: pointer to io function
 * @bsize: io block size
 * @io_flags: extra parameters for the io function
708
 */
709 710
void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
		     struct inode *inode,
711
		     const struct nfs_pageio_ops *pg_ops,
712
		     const struct nfs_pgio_completion_ops *compl_ops,
713
		     const struct nfs_rw_ops *rw_ops,
714
		     size_t bsize,
715
		     int io_flags)
716
{
717 718 719
	struct nfs_pgio_mirror *new;
	int i;

720
	desc->pg_moreio = 0;
721
	desc->pg_inode = inode;
722
	desc->pg_ops = pg_ops;
723
	desc->pg_completion_ops = compl_ops;
724
	desc->pg_rw_ops = rw_ops;
725 726
	desc->pg_ioflags = io_flags;
	desc->pg_error = 0;
727
	desc->pg_lseg = NULL;
728
	desc->pg_dreq = NULL;
729
	desc->pg_layout_private = NULL;
730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749
	desc->pg_bsize = bsize;

	desc->pg_mirror_count = 1;
	desc->pg_mirror_idx = 0;

	if (pg_ops->pg_get_mirror_count) {
		/* until we have a request, we don't have an lseg and no
		 * idea how many mirrors there will be */
		new = kcalloc(NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX,
			      sizeof(struct nfs_pgio_mirror), GFP_KERNEL);
		desc->pg_mirrors_dynamic = new;
		desc->pg_mirrors = new;

		for (i = 0; i < NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX; i++)
			nfs_pageio_mirror_init(&desc->pg_mirrors[i], bsize);
	} else {
		desc->pg_mirrors_dynamic = NULL;
		desc->pg_mirrors = desc->pg_mirrors_static;
		nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);
	}
750
}
B
Bryan Schumaker 已提交
751
EXPORT_SYMBOL_GPL(nfs_pageio_init);
752

753 754 755
/**
 * nfs_pgio_result - Basic pageio error handling
 * @task: The task that ran
756
 * @calldata: Pageio header to check
757
 */
758
static void nfs_pgio_result(struct rpc_task *task, void *calldata)
759
{
760 761
	struct nfs_pgio_header *hdr = calldata;
	struct inode *inode = hdr->inode;
762 763 764 765

	dprintk("NFS: %s: %5u, (status %d)\n", __func__,
		task->tk_pid, task->tk_status);

766
	if (hdr->rw_ops->rw_done(task, hdr, inode) != 0)
767 768
		return;
	if (task->tk_status < 0)
769
		nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset);
770
	else
771
		hdr->rw_ops->rw_result(task, hdr);
772 773
}

774 775 776 777 778 779 780 781
/*
 * Create an RPC task for the given read or write request and kick it.
 * The page must have been locked by the caller.
 *
 * It may happen that the page we're passed is not marked dirty.
 * This is the case if nfs_updatepage detects a conflicting request
 * that has been written but not committed.
 */
782 783
int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
		     struct nfs_pgio_header *hdr)
784
{
785 786
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

787
	struct nfs_page		*req;
788 789
	struct page		**pages,
				*last_page;
790
	struct list_head *head = &mirror->pg_list;
791
	struct nfs_commit_info cinfo;
792
	unsigned int pagecount, pageused;
793

794
	pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);
795
	if (!nfs_pgarray_set(&hdr->page_array, pagecount))
796 797 798
		return nfs_pgio_error(desc, hdr);

	nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);
799
	pages = hdr->page_array.pagevec;
800 801
	last_page = NULL;
	pageused = 0;
802 803 804 805
	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_list_add_request(req, &hdr->pages);
806 807 808

		if (!last_page || last_page != req->wb_page) {
			pageused++;
809 810 811
			if (pageused > pagecount)
				break;
			*pages++ = last_page = req->wb_page;
812
		}
813
	}
814 815
	if (WARN_ON_ONCE(pageused != pagecount))
		return nfs_pgio_error(desc, hdr);
816 817 818 819 820 821

	if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
	    (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))
		desc->pg_ioflags &= ~FLUSH_COND_STABLE;

	/* Set up the argument struct */
822
	nfs_pgio_rpcsetup(hdr, mirror->pg_count, 0, desc->pg_ioflags, &cinfo);
823 824 825
	desc->pg_rpc_callops = &nfs_pgio_common_ops;
	return 0;
}
826
EXPORT_SYMBOL_GPL(nfs_generic_pgio);
827

828
static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)
829
{
830
	struct nfs_pgio_mirror *mirror;
831 832 833
	struct nfs_pgio_header *hdr;
	int ret;

834 835
	mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

836 837
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
838 839 840
		/* TODO: make sure this is right with mirroring - or
		 *       should it back out all mirrors? */
		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
841 842
		return -ENOMEM;
	}
843
	nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);
844 845
	ret = nfs_generic_pgio(desc, hdr);
	if (ret == 0)
846
		ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode),
847 848 849
					hdr,
					hdr->cred,
					NFS_PROTO(hdr->inode),
850
					desc->pg_rpc_callops,
851
					desc->pg_ioflags, 0);
852 853 854
	return ret;
}

855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
/*
 * nfs_pageio_setup_mirroring - determine if mirroring is to be used
 *				by calling the pg_get_mirror_count op
 */
static int nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio,
				       struct nfs_page *req)
{
	int mirror_count = 1;

	if (!pgio->pg_ops->pg_get_mirror_count)
		return 0;

	mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);

	if (!mirror_count || mirror_count > NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX)
		return -EINVAL;

	if (WARN_ON_ONCE(!pgio->pg_mirrors_dynamic))
		return -EINVAL;

	pgio->pg_mirror_count = mirror_count;

	return 0;
}

/*
 * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
 */
void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio)
{
	pgio->pg_mirror_count = 1;
	pgio->pg_mirror_idx = 0;
}

static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio)
{
	pgio->pg_mirror_count = 1;
	pgio->pg_mirror_idx = 0;
	pgio->pg_mirrors = pgio->pg_mirrors_static;
	kfree(pgio->pg_mirrors_dynamic);
	pgio->pg_mirrors_dynamic = NULL;
}

898 899 900 901 902 903 904 905 906 907 908 909 910
static bool nfs_match_open_context(const struct nfs_open_context *ctx1,
		const struct nfs_open_context *ctx2)
{
	return ctx1->cred == ctx2->cred && ctx1->state == ctx2->state;
}

static bool nfs_match_lock_context(const struct nfs_lock_context *l1,
		const struct nfs_lock_context *l2)
{
	return l1->lockowner.l_owner == l2->lockowner.l_owner
		&& l1->lockowner.l_pid == l2->lockowner.l_pid;
}

911 912 913 914 915 916 917 918 919 920 921
/**
 * nfs_can_coalesce_requests - test two requests for compatibility
 * @prev: pointer to nfs_page
 * @req: pointer to nfs_page
 *
 * The nfs_page structures 'prev' and 'req' are compared to ensure that the
 * page data area they describe is contiguous, and that their RPC
 * credentials, NFSv4 open state, and lockowners are the same.
 *
 * Return 'true' if this is the case, else return 'false'.
 */
922 923 924
static bool nfs_can_coalesce_requests(struct nfs_page *prev,
				      struct nfs_page *req,
				      struct nfs_pageio_descriptor *pgio)
925
{
926 927
	size_t size;

928 929 930 931 932 933 934 935 936
	if (prev) {
		if (!nfs_match_open_context(req->wb_context, prev->wb_context))
			return false;
		if (req->wb_context->dentry->d_inode->i_flock != NULL &&
		    !nfs_match_lock_context(req->wb_lock_context,
					    prev->wb_lock_context))
			return false;
		if (req_offset(req) != req_offset(prev) + prev->wb_bytes)
			return false;
937 938 939 940 941 942 943 944
		if (req->wb_page == prev->wb_page) {
			if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes)
				return false;
		} else {
			if (req->wb_pgbase != 0 ||
			    prev->wb_pgbase + prev->wb_bytes != PAGE_CACHE_SIZE)
				return false;
		}
945
	}
946
	size = pgio->pg_ops->pg_test(pgio, prev, req);
947 948 949
	WARN_ON_ONCE(size > req->wb_bytes);
	if (size && size < req->wb_bytes)
		req->wb_bytes = size;
950
	return size > 0;
951 952 953
}

/**
954
 * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
955 956 957 958 959 960
 * @desc: destination io descriptor
 * @req: request
 *
 * Returns true if the request 'req' was successfully coalesced into the
 * existing list of pages 'desc'.
 */
961 962
static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
				     struct nfs_page *req)
963
{
964 965
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

966
	struct nfs_page *prev = NULL;
967 968 969

	if (mirror->pg_count != 0) {
		prev = nfs_list_entry(mirror->pg_list.prev);
970
	} else {
971 972
		if (desc->pg_ops->pg_init)
			desc->pg_ops->pg_init(desc, req);
973
		mirror->pg_base = req->wb_pgbase;
974
	}
975 976
	if (!nfs_can_coalesce_requests(prev, req, desc))
		return 0;
977
	nfs_list_remove_request(req);
978 979
	nfs_list_add_request(req, &mirror->pg_list);
	mirror->pg_count += req->wb_bytes;
980 981 982
	return 1;
}

983 984 985 986 987
/*
 * Helper for nfs_pageio_add_request and nfs_pageio_complete
 */
static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
{
988 989 990 991
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];


	if (!list_empty(&mirror->pg_list)) {
992
		int error = desc->pg_ops->pg_doio(desc);
993 994 995
		if (error < 0)
			desc->pg_error = error;
		else
996
			mirror->pg_bytes_written += mirror->pg_count;
997
	}
998 999 1000
	if (list_empty(&mirror->pg_list)) {
		mirror->pg_count = 0;
		mirror->pg_base = 0;
1001 1002 1003 1004 1005 1006 1007 1008
	}
}

/**
 * nfs_pageio_add_request - Attempt to coalesce a request into a page list.
 * @desc: destination io descriptor
 * @req: request
 *
1009 1010 1011
 * This may split a request into subrequests which are all part of the
 * same page group.
 *
1012 1013 1014
 * Returns true if the request 'req' was successfully coalesced into the
 * existing list of pages 'desc'.
 */
1015
static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
1016
			   struct nfs_page *req)
1017
{
1018 1019
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

1020 1021 1022 1023
	struct nfs_page *subreq;
	unsigned int bytes_left = 0;
	unsigned int offset, pgbase;

1024 1025
	WARN_ON_ONCE(desc->pg_mirror_idx >= desc->pg_mirror_count);

1026
	nfs_page_group_lock(req, false);
1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044

	subreq = req;
	bytes_left = subreq->wb_bytes;
	offset = subreq->wb_offset;
	pgbase = subreq->wb_pgbase;

	do {
		if (!nfs_pageio_do_add_request(desc, subreq)) {
			/* make sure pg_test call(s) did nothing */
			WARN_ON_ONCE(subreq->wb_bytes != bytes_left);
			WARN_ON_ONCE(subreq->wb_offset != offset);
			WARN_ON_ONCE(subreq->wb_pgbase != pgbase);

			nfs_page_group_unlock(req);
			desc->pg_moreio = 1;
			nfs_pageio_doio(desc);
			if (desc->pg_error < 0)
				return 0;
1045
			if (mirror->pg_recoalesce)
1046 1047
				return 0;
			/* retry add_request for this subreq */
1048
			nfs_page_group_lock(req, false);
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
			continue;
		}

		/* check for buggy pg_test call(s) */
		WARN_ON_ONCE(subreq->wb_bytes + subreq->wb_pgbase > PAGE_SIZE);
		WARN_ON_ONCE(subreq->wb_bytes > bytes_left);
		WARN_ON_ONCE(subreq->wb_bytes == 0);

		bytes_left -= subreq->wb_bytes;
		offset += subreq->wb_bytes;
		pgbase += subreq->wb_bytes;

		if (bytes_left) {
			subreq = nfs_create_request(req->wb_context,
					req->wb_page,
					subreq, pgbase, bytes_left);
1065 1066
			if (IS_ERR(subreq))
				goto err_ptr;
1067 1068 1069 1070 1071 1072 1073
			nfs_lock_request(subreq);
			subreq->wb_offset  = offset;
			subreq->wb_index = req->wb_index;
		}
	} while (bytes_left > 0);

	nfs_page_group_unlock(req);
1074
	return 1;
1075 1076 1077 1078
err_ptr:
	desc->pg_error = PTR_ERR(subreq);
	nfs_page_group_unlock(req);
	return 0;
1079 1080
}

1081 1082
static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
{
1083
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
1084 1085 1086
	LIST_HEAD(head);

	do {
1087 1088 1089 1090 1091 1092
		list_splice_init(&mirror->pg_list, &head);
		mirror->pg_bytes_written -= mirror->pg_count;
		mirror->pg_count = 0;
		mirror->pg_base = 0;
		mirror->pg_recoalesce = 0;

1093
		desc->pg_moreio = 0;
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105

		while (!list_empty(&head)) {
			struct nfs_page *req;

			req = list_first_entry(&head, struct nfs_page, wb_list);
			nfs_list_remove_request(req);
			if (__nfs_pageio_add_request(desc, req))
				continue;
			if (desc->pg_error < 0)
				return 0;
			break;
		}
1106
	} while (mirror->pg_recoalesce);
1107 1108 1109
	return 1;
}

1110
static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc,
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
		struct nfs_page *req)
{
	int ret;

	do {
		ret = __nfs_pageio_add_request(desc, req);
		if (ret)
			break;
		if (desc->pg_error < 0)
			break;
		ret = nfs_do_recoalesce(desc);
	} while (ret);
1123

1124 1125 1126
	return ret;
}

1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
			   struct nfs_page *req)
{
	u32 midx;
	unsigned int pgbase, offset, bytes;
	struct nfs_page *dupreq, *lastreq;

	pgbase = req->wb_pgbase;
	offset = req->wb_offset;
	bytes = req->wb_bytes;

	nfs_pageio_setup_mirroring(desc, req);

	for (midx = 0; midx < desc->pg_mirror_count; midx++) {
		if (midx) {
			nfs_page_group_lock(req, false);

			/* find the last request */
			for (lastreq = req->wb_head;
			     lastreq->wb_this_page != req->wb_head;
			     lastreq = lastreq->wb_this_page)
				;

			dupreq = nfs_create_request(req->wb_context,
					req->wb_page, lastreq, pgbase, bytes);

			if (IS_ERR(dupreq)) {
				nfs_page_group_unlock(req);
				return 0;
			}

			nfs_lock_request(dupreq);
			nfs_page_group_unlock(req);
			dupreq->wb_offset = offset;
			dupreq->wb_index = req->wb_index;
		} else
			dupreq = req;

		desc->pg_mirror_idx = midx;
		if (!nfs_pageio_add_request_mirror(desc, dupreq))
			return 0;
	}

	return 1;
}

/*
 * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
 *				nfs_pageio_descriptor
 * @desc: pointer to io descriptor
 */
static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,
				       u32 mirror_idx)
{
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[mirror_idx];
	u32 restore_idx = desc->pg_mirror_idx;

	desc->pg_mirror_idx = mirror_idx;
	for (;;) {
		nfs_pageio_doio(desc);
		if (!mirror->pg_recoalesce)
			break;
		if (!nfs_do_recoalesce(desc))
			break;
	}
	desc->pg_mirror_idx = restore_idx;
}

1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
/*
 * nfs_pageio_resend - Transfer requests to new descriptor and resend
 * @hdr - the pgio header to move request from
 * @desc - the pageio descriptor to add requests to
 *
 * Try to move each request (nfs_page) from @hdr to @desc then attempt
 * to send them.
 *
 * Returns 0 on success and < 0 on error.
 */
int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,
		      struct nfs_pgio_header *hdr)
{
	LIST_HEAD(failed);

	desc->pg_dreq = hdr->dreq;
	while (!list_empty(&hdr->pages)) {
		struct nfs_page *req = nfs_list_entry(hdr->pages.next);

		nfs_list_remove_request(req);
		if (!nfs_pageio_add_request(desc, req))
			nfs_list_add_request(req, &failed);
	}
	nfs_pageio_complete(desc);
	if (!list_empty(&failed)) {
		list_move(&failed, &hdr->pages);
		return -EIO;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(nfs_pageio_resend);
1226

1227
/**
1228
 * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1229 1230 1231 1232
 * @desc: pointer to io descriptor
 */
void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
{
1233 1234 1235 1236
	u32 midx;

	for (midx = 0; midx < desc->pg_mirror_count; midx++)
		nfs_pageio_complete_mirror(desc, midx);
1237 1238 1239

	if (desc->pg_ops->pg_cleanup)
		desc->pg_ops->pg_cleanup(desc);
1240
	nfs_pageio_cleanup_mirroring(desc);
1241 1242
}

1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
/**
 * nfs_pageio_cond_complete - Conditional I/O completion
 * @desc: pointer to io descriptor
 * @index: page index
 *
 * It is important to ensure that processes don't try to take locks
 * on non-contiguous ranges of pages as that might deadlock. This
 * function should be called before attempting to wait on a locked
 * nfs_page. It will complete the I/O if the page index 'index'
 * is not contiguous with the existing list of pages in 'desc'.
 */
void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
{
1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
	struct nfs_pgio_mirror *mirror;
	struct nfs_page *prev;
	u32 midx;

	for (midx = 0; midx < desc->pg_mirror_count; midx++) {
		mirror = &desc->pg_mirrors[midx];
		if (!list_empty(&mirror->pg_list)) {
			prev = nfs_list_entry(mirror->pg_list.prev);
			if (index != prev->wb_index + 1)
				nfs_pageio_complete_mirror(desc, midx);
		}
1267 1268 1269
	}
}

D
David Howells 已提交
1270
int __init nfs_init_nfspagecache(void)
L
Linus Torvalds 已提交
1271 1272 1273 1274
{
	nfs_page_cachep = kmem_cache_create("nfs_page",
					    sizeof(struct nfs_page),
					    0, SLAB_HWCACHE_ALIGN,
1275
					    NULL);
L
Linus Torvalds 已提交
1276 1277 1278 1279 1280 1281
	if (nfs_page_cachep == NULL)
		return -ENOMEM;

	return 0;
}

1282
void nfs_destroy_nfspagecache(void)
L
Linus Torvalds 已提交
1283
{
1284
	kmem_cache_destroy(nfs_page_cachep);
L
Linus Torvalds 已提交
1285 1286
}

1287
static const struct rpc_call_ops nfs_pgio_common_ops = {
1288 1289 1290 1291
	.rpc_call_prepare = nfs_pgio_prepare,
	.rpc_call_done = nfs_pgio_result,
	.rpc_release = nfs_pgio_release,
};
1292 1293 1294 1295 1296

const struct nfs_pageio_ops nfs_pgio_rw_ops = {
	.pg_test = nfs_generic_pg_test,
	.pg_doio = nfs_generic_pg_pgios,
};