pagelist.c 13.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * linux/fs/nfs/pagelist.c
 *
 * A set of helper functions for managing NFS read and write requests.
 * The main purpose of these routines is to provide support for the
 * coalescing of several requests into a single RPC call.
 *
 * Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no>
 *
 */

#include <linux/slab.h>
#include <linux/file.h>
A
Alexey Dobriyan 已提交
14
#include <linux/sched.h>
L
Linus Torvalds 已提交
15
#include <linux/sunrpc/clnt.h>
16
#include <linux/nfs.h>
L
Linus Torvalds 已提交
17 18 19 20 21
#include <linux/nfs3.h>
#include <linux/nfs4.h>
#include <linux/nfs_page.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
22
#include <linux/export.h>
L
Linus Torvalds 已提交
23

24
#include "internal.h"
25
#include "pnfs.h"
26

27
static struct kmem_cache *nfs_page_cachep;
L
Linus Torvalds 已提交
28

F
Fred Isaman 已提交
29 30 31 32 33 34 35 36 37 38 39 40 41
bool nfs_pgarray_set(struct nfs_page_array *p, unsigned int pagecount)
{
	p->npages = pagecount;
	if (pagecount <= ARRAY_SIZE(p->page_array))
		p->pagevec = p->page_array;
	else {
		p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL);
		if (!p->pagevec)
			p->npages = 0;
	}
	return p->pagevec != NULL;
}

42 43 44 45 46 47 48 49 50
void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
		       struct nfs_pgio_header *hdr,
		       void (*release)(struct nfs_pgio_header *hdr))
{
	hdr->req = nfs_list_entry(desc->pg_list.next);
	hdr->inode = desc->pg_inode;
	hdr->cred = hdr->req->wb_context->cred;
	hdr->io_start = req_offset(hdr->req);
	hdr->good_bytes = desc->pg_count;
51
	hdr->dreq = desc->pg_dreq;
52
	hdr->layout_private = desc->pg_layout_private;
53
	hdr->release = release;
54
	hdr->completion_ops = desc->pg_completion_ops;
55 56
	if (hdr->completion_ops->init_hdr)
		hdr->completion_ops->init_hdr(hdr);
57
}
B
Bryan Schumaker 已提交
58
EXPORT_SYMBOL_GPL(nfs_pgheader_init);
59 60 61 62 63 64 65 66 67 68 69 70 71

void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos)
{
	spin_lock(&hdr->lock);
	if (pos < hdr->io_start + hdr->good_bytes) {
		set_bit(NFS_IOHDR_ERROR, &hdr->flags);
		clear_bit(NFS_IOHDR_EOF, &hdr->flags);
		hdr->good_bytes = pos - hdr->io_start;
		hdr->error = error;
	}
	spin_unlock(&hdr->lock);
}

L
Linus Torvalds 已提交
72 73 74
static inline struct nfs_page *
nfs_page_alloc(void)
{
75
	struct nfs_page	*p = kmem_cache_zalloc(nfs_page_cachep, GFP_NOIO);
76
	if (p)
L
Linus Torvalds 已提交
77 78 79 80 81 82 83 84 85 86
		INIT_LIST_HEAD(&p->wb_list);
	return p;
}

static inline void
nfs_page_free(struct nfs_page *p)
{
	kmem_cache_free(nfs_page_cachep, p);
}

87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
static void
nfs_iocounter_inc(struct nfs_io_counter *c)
{
	atomic_inc(&c->io_count);
}

static void
nfs_iocounter_dec(struct nfs_io_counter *c)
{
	if (atomic_dec_and_test(&c->io_count)) {
		clear_bit(NFS_IO_INPROGRESS, &c->flags);
		smp_mb__after_clear_bit();
		wake_up_bit(&c->flags, NFS_IO_INPROGRESS);
	}
}

static int
__nfs_iocounter_wait(struct nfs_io_counter *c)
{
	wait_queue_head_t *wq = bit_waitqueue(&c->flags, NFS_IO_INPROGRESS);
	DEFINE_WAIT_BIT(q, &c->flags, NFS_IO_INPROGRESS);
	int ret = 0;

	do {
		prepare_to_wait(wq, &q.wait, TASK_KILLABLE);
		set_bit(NFS_IO_INPROGRESS, &c->flags);
		if (atomic_read(&c->io_count) == 0)
			break;
		ret = nfs_wait_bit_killable(&c->flags);
	} while (atomic_read(&c->io_count) != 0);
	finish_wait(wq, &q.wait);
	return ret;
}

/**
 * nfs_iocounter_wait - wait for i/o to complete
 * @c: nfs_io_counter to use
 *
 * returns -ERESTARTSYS if interrupted by a fatal signal.
 * Otherwise returns 0 once the io_count hits 0.
 */
int
nfs_iocounter_wait(struct nfs_io_counter *c)
{
	if (atomic_read(&c->io_count) == 0)
		return 0;
	return __nfs_iocounter_wait(c);
}

L
Linus Torvalds 已提交
136 137
/**
 * nfs_create_request - Create an NFS read/write request.
138
 * @ctx: open context to use
L
Linus Torvalds 已提交
139 140 141 142 143 144
 * @inode: inode to which the request is attached
 * @page: page to write
 * @offset: starting offset within the page for the write
 * @count: number of bytes to read/write
 *
 * The page must be locked by the caller. This makes sure we never
145
 * create two different requests for the same page.
L
Linus Torvalds 已提交
146 147 148 149 150 151 152 153
 * User should ensure it is safe to sleep in this function.
 */
struct nfs_page *
nfs_create_request(struct nfs_open_context *ctx, struct inode *inode,
		   struct page *page,
		   unsigned int offset, unsigned int count)
{
	struct nfs_page		*req;
154
	struct nfs_lock_context *l_ctx;
L
Linus Torvalds 已提交
155

156 157
	if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))
		return ERR_PTR(-EBADF);
158 159 160 161
	/* try to allocate the request struct */
	req = nfs_page_alloc();
	if (req == NULL)
		return ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
162

163
	/* get lock context early so we can deal with alloc failures */
164 165
	l_ctx = nfs_get_lock_context(ctx);
	if (IS_ERR(l_ctx)) {
166
		nfs_page_free(req);
167
		return ERR_CAST(l_ctx);
168
	}
169
	req->wb_lock_context = l_ctx;
170
	nfs_iocounter_inc(&l_ctx->io_count);
171

L
Linus Torvalds 已提交
172 173 174 175
	/* Initialize the request struct. Initially, we assume a
	 * long write-back delay. This will be adjusted in
	 * update_nfs_request below if the region is not locked. */
	req->wb_page    = page;
176
	req->wb_index	= page_file_index(page);
L
Linus Torvalds 已提交
177 178 179 180 181
	page_cache_get(page);
	req->wb_offset  = offset;
	req->wb_pgbase	= offset;
	req->wb_bytes   = count;
	req->wb_context = get_nfs_open_context(ctx);
182
	kref_init(&req->wb_kref);
L
Linus Torvalds 已提交
183 184 185 186
	return req;
}

/**
187
 * nfs_unlock_request - Unlock request and wake up sleepers.
L
Linus Torvalds 已提交
188 189
 * @req:
 */
190
void nfs_unlock_request(struct nfs_page *req)
L
Linus Torvalds 已提交
191 192 193 194 195 196 197 198
{
	if (!NFS_WBACK_BUSY(req)) {
		printk(KERN_ERR "NFS: Invalid unlock attempted\n");
		BUG();
	}
	smp_mb__before_clear_bit();
	clear_bit(PG_BUSY, &req->wb_flags);
	smp_mb__after_clear_bit();
199
	wake_up_bit(&req->wb_flags, PG_BUSY);
200 201 202
}

/**
203 204
 * nfs_unlock_and_release_request - Unlock request and release the nfs_page
 * @req:
205
 */
206
void nfs_unlock_and_release_request(struct nfs_page *req)
207
{
208
	nfs_unlock_request(req);
L
Linus Torvalds 已提交
209 210 211
	nfs_release_request(req);
}

212
/*
L
Linus Torvalds 已提交
213 214 215
 * nfs_clear_request - Free up all resources allocated to the request
 * @req:
 *
216 217
 * Release page and open context resources associated with a read/write
 * request after it has completed.
L
Linus Torvalds 已提交
218
 */
219
static void nfs_clear_request(struct nfs_page *req)
L
Linus Torvalds 已提交
220
{
221
	struct page *page = req->wb_page;
222
	struct nfs_open_context *ctx = req->wb_context;
223
	struct nfs_lock_context *l_ctx = req->wb_lock_context;
224

225 226
	if (page != NULL) {
		page_cache_release(page);
L
Linus Torvalds 已提交
227 228
		req->wb_page = NULL;
	}
229
	if (l_ctx != NULL) {
230
		nfs_iocounter_dec(&l_ctx->io_count);
231 232 233
		nfs_put_lock_context(l_ctx);
		req->wb_lock_context = NULL;
	}
234 235 236 237
	if (ctx != NULL) {
		put_nfs_open_context(ctx);
		req->wb_context = NULL;
	}
L
Linus Torvalds 已提交
238 239 240 241 242 243 244 245 246
}


/**
 * nfs_release_request - Release the count on an NFS read/write request
 * @req: request to release
 *
 * Note: Should never be called with the spinlock held!
 */
247
static void nfs_free_request(struct kref *kref)
L
Linus Torvalds 已提交
248
{
249
	struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);
L
Linus Torvalds 已提交
250

251
	/* Release struct file and open context */
L
Linus Torvalds 已提交
252 253 254 255
	nfs_clear_request(req);
	nfs_page_free(req);
}

256 257 258 259 260
void nfs_release_request(struct nfs_page *req)
{
	kref_put(&req->wb_kref, nfs_free_request);
}

261 262 263 264 265 266
static int nfs_wait_bit_uninterruptible(void *word)
{
	io_schedule();
	return 0;
}

L
Linus Torvalds 已提交
267 268 269 270
/**
 * nfs_wait_on_request - Wait for a request to complete.
 * @req: request to wait upon.
 *
271
 * Interruptible by fatal signals only.
L
Linus Torvalds 已提交
272 273 274 275 276
 * The user is responsible for holding a count on the request.
 */
int
nfs_wait_on_request(struct nfs_page *req)
{
277 278 279
	return wait_on_bit(&req->wb_flags, PG_BUSY,
			nfs_wait_bit_uninterruptible,
			TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
280 281
}

282
bool nfs_generic_pg_test(struct nfs_pageio_descriptor *desc, struct nfs_page *prev, struct nfs_page *req)
283 284 285 286 287 288 289 290 291 292 293 294 295
{
	/*
	 * FIXME: ideally we should be able to coalesce all requests
	 * that are not block boundary aligned, but currently this
	 * is problematic for the case of bsize < PAGE_CACHE_SIZE,
	 * since nfs_flush_multi and nfs_pagein_multi assume you
	 * can have only one struct nfs_page.
	 */
	if (desc->pg_bsize < PAGE_SIZE)
		return 0;

	return desc->pg_count + req->wb_bytes <= desc->pg_bsize;
}
296
EXPORT_SYMBOL_GPL(nfs_generic_pg_test);
297

L
Linus Torvalds 已提交
298
/**
299 300
 * nfs_pageio_init - initialise a page io descriptor
 * @desc: pointer to descriptor
301 302 303 304
 * @inode: pointer to inode
 * @doio: pointer to io function
 * @bsize: io block size
 * @io_flags: extra parameters for the io function
305
 */
306 307
void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
		     struct inode *inode,
308
		     const struct nfs_pageio_ops *pg_ops,
309
		     const struct nfs_pgio_completion_ops *compl_ops,
310
		     size_t bsize,
311
		     int io_flags)
312 313
{
	INIT_LIST_HEAD(&desc->pg_list);
314
	desc->pg_bytes_written = 0;
315 316 317
	desc->pg_count = 0;
	desc->pg_bsize = bsize;
	desc->pg_base = 0;
318
	desc->pg_moreio = 0;
319
	desc->pg_recoalesce = 0;
320
	desc->pg_inode = inode;
321
	desc->pg_ops = pg_ops;
322
	desc->pg_completion_ops = compl_ops;
323 324
	desc->pg_ioflags = io_flags;
	desc->pg_error = 0;
325
	desc->pg_lseg = NULL;
326
	desc->pg_dreq = NULL;
327
	desc->pg_layout_private = NULL;
328
}
B
Bryan Schumaker 已提交
329
EXPORT_SYMBOL_GPL(nfs_pageio_init);
330 331 332 333 334 335 336 337 338 339 340 341

/**
 * nfs_can_coalesce_requests - test two requests for compatibility
 * @prev: pointer to nfs_page
 * @req: pointer to nfs_page
 *
 * The nfs_page structures 'prev' and 'req' are compared to ensure that the
 * page data area they describe is contiguous, and that their RPC
 * credentials, NFSv4 open state, and lockowners are the same.
 *
 * Return 'true' if this is the case, else return 'false'.
 */
342 343 344
static bool nfs_can_coalesce_requests(struct nfs_page *prev,
				      struct nfs_page *req,
				      struct nfs_pageio_descriptor *pgio)
345 346
{
	if (req->wb_context->cred != prev->wb_context->cred)
347
		return false;
348 349 350
	if (req->wb_lock_context->lockowner.l_owner != prev->wb_lock_context->lockowner.l_owner)
		return false;
	if (req->wb_lock_context->lockowner.l_pid != prev->wb_lock_context->lockowner.l_pid)
351
		return false;
352
	if (req->wb_context->state != prev->wb_context->state)
353
		return false;
354
	if (req->wb_pgbase != 0)
355
		return false;
356
	if (prev->wb_pgbase + prev->wb_bytes != PAGE_CACHE_SIZE)
357
		return false;
358 359
	if (req_offset(req) != req_offset(prev) + prev->wb_bytes)
		return false;
360
	return pgio->pg_ops->pg_test(pgio, prev, req);
361 362 363
}

/**
364
 * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
365 366 367 368 369 370
 * @desc: destination io descriptor
 * @req: request
 *
 * Returns true if the request 'req' was successfully coalesced into the
 * existing list of pages 'desc'.
 */
371 372
static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
				     struct nfs_page *req)
373 374 375 376 377
{
	if (desc->pg_count != 0) {
		struct nfs_page *prev;

		prev = nfs_list_entry(desc->pg_list.prev);
378
		if (!nfs_can_coalesce_requests(prev, req, desc))
379
			return 0;
380
	} else {
381 382
		if (desc->pg_ops->pg_init)
			desc->pg_ops->pg_init(desc, req);
383
		desc->pg_base = req->wb_pgbase;
384
	}
385 386
	nfs_list_remove_request(req);
	nfs_list_add_request(req, &desc->pg_list);
387
	desc->pg_count += req->wb_bytes;
388 389 390
	return 1;
}

391 392 393 394 395 396
/*
 * Helper for nfs_pageio_add_request and nfs_pageio_complete
 */
static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
{
	if (!list_empty(&desc->pg_list)) {
397
		int error = desc->pg_ops->pg_doio(desc);
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416
		if (error < 0)
			desc->pg_error = error;
		else
			desc->pg_bytes_written += desc->pg_count;
	}
	if (list_empty(&desc->pg_list)) {
		desc->pg_count = 0;
		desc->pg_base = 0;
	}
}

/**
 * nfs_pageio_add_request - Attempt to coalesce a request into a page list.
 * @desc: destination io descriptor
 * @req: request
 *
 * Returns true if the request 'req' was successfully coalesced into the
 * existing list of pages 'desc'.
 */
417
static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
418
			   struct nfs_page *req)
419 420
{
	while (!nfs_pageio_do_add_request(desc, req)) {
421
		desc->pg_moreio = 1;
422 423 424
		nfs_pageio_doio(desc);
		if (desc->pg_error < 0)
			return 0;
425
		desc->pg_moreio = 0;
426 427
		if (desc->pg_recoalesce)
			return 0;
428 429 430 431
	}
	return 1;
}

432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
{
	LIST_HEAD(head);

	do {
		list_splice_init(&desc->pg_list, &head);
		desc->pg_bytes_written -= desc->pg_count;
		desc->pg_count = 0;
		desc->pg_base = 0;
		desc->pg_recoalesce = 0;

		while (!list_empty(&head)) {
			struct nfs_page *req;

			req = list_first_entry(&head, struct nfs_page, wb_list);
			nfs_list_remove_request(req);
			if (__nfs_pageio_add_request(desc, req))
				continue;
			if (desc->pg_error < 0)
				return 0;
			break;
		}
	} while (desc->pg_recoalesce);
	return 1;
}

int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
		struct nfs_page *req)
{
	int ret;

	do {
		ret = __nfs_pageio_add_request(desc, req);
		if (ret)
			break;
		if (desc->pg_error < 0)
			break;
		ret = nfs_do_recoalesce(desc);
	} while (ret);
	return ret;
}
B
Bryan Schumaker 已提交
473
EXPORT_SYMBOL_GPL(nfs_pageio_add_request);
474

475 476 477 478 479 480
/**
 * nfs_pageio_complete - Complete I/O on an nfs_pageio_descriptor
 * @desc: pointer to io descriptor
 */
void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
{
481 482 483 484 485 486 487
	for (;;) {
		nfs_pageio_doio(desc);
		if (!desc->pg_recoalesce)
			break;
		if (!nfs_do_recoalesce(desc))
			break;
	}
488
}
B
Bryan Schumaker 已提交
489
EXPORT_SYMBOL_GPL(nfs_pageio_complete);
490

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
/**
 * nfs_pageio_cond_complete - Conditional I/O completion
 * @desc: pointer to io descriptor
 * @index: page index
 *
 * It is important to ensure that processes don't try to take locks
 * on non-contiguous ranges of pages as that might deadlock. This
 * function should be called before attempting to wait on a locked
 * nfs_page. It will complete the I/O if the page index 'index'
 * is not contiguous with the existing list of pages in 'desc'.
 */
void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
{
	if (!list_empty(&desc->pg_list)) {
		struct nfs_page *prev = nfs_list_entry(desc->pg_list.prev);
		if (index != prev->wb_index + 1)
507
			nfs_pageio_complete(desc);
508 509 510
	}
}

D
David Howells 已提交
511
int __init nfs_init_nfspagecache(void)
L
Linus Torvalds 已提交
512 513 514 515
{
	nfs_page_cachep = kmem_cache_create("nfs_page",
					    sizeof(struct nfs_page),
					    0, SLAB_HWCACHE_ALIGN,
516
					    NULL);
L
Linus Torvalds 已提交
517 518 519 520 521 522
	if (nfs_page_cachep == NULL)
		return -ENOMEM;

	return 0;
}

523
void nfs_destroy_nfspagecache(void)
L
Linus Torvalds 已提交
524
{
525
	kmem_cache_destroy(nfs_page_cachep);
L
Linus Torvalds 已提交
526 527
}