direct.c 19.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
/*
 * linux/fs/nfs/direct.c
 *
 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
 *
 * High-performance uncached I/O for the Linux NFS client
 *
 * There are important applications whose performance or correctness
 * depends on uncached access to file data.  Database clusters
 * (multiple copies of the same instance running on separate hosts) 
 * implement their own cache coherency protocol that subsumes file
 * system cache protocols.  Applications that process datasets 
 * considerably larger than the client's memory do not always benefit 
 * from a local cache.  A streaming video server, for instance, has no 
 * need to cache the contents of a file.
 *
 * When an application requests uncached I/O, all read and write requests
 * are made directly to the server; data stored or fetched via these
 * requests is not cached in the Linux page cache.  The client does not
 * correct unaligned requests from applications.  All requested bytes are
 * held on permanent storage before a direct write system call returns to
 * an application.
 *
 * Solaris implements an uncached I/O facility called directio() that
 * is used for backups and sequential I/O to very large files.  Solaris
 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
 * an undocumented mount option.
 *
 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
 * help from Andrew Morton.
 *
 * 18 Dec 2001	Initial implementation for 2.4  --cel
 * 08 Jul 2002	Version for 2.4.19, with bug fixes --trondmy
 * 08 Jun 2003	Port to 2.5 APIs  --cel
 * 31 Mar 2004	Handle direct I/O without VFS support  --cel
 * 15 Sep 2004	Parallel async reads  --cel
 *
 */

#include <linux/config.h>
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/smp_lock.h>
#include <linux/file.h>
#include <linux/pagemap.h>
#include <linux/kref.h>

#include <linux/nfs_fs.h>
#include <linux/nfs_page.h>
#include <linux/sunrpc/clnt.h>

#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/atomic.h>

C
Chuck Lever 已提交
57 58
#include "iostat.h"

L
Linus Torvalds 已提交
59 60 61
#define NFSDBG_FACILITY		NFSDBG_VFS
#define MAX_DIRECTIO_SIZE	(4096UL << PAGE_SHIFT)

62
static void nfs_free_user_pages(struct page **pages, int npages, int do_dirty);
L
Linus Torvalds 已提交
63 64 65 66 67 68 69 70
static kmem_cache_t *nfs_direct_cachep;

/*
 * This represents a set of asynchronous requests that we're waiting on
 */
struct nfs_direct_req {
	struct kref		kref;		/* release manager */
	struct list_head	list;		/* nfs_read_data structs */
71 72
	struct file *		filp;		/* file descriptor */
	struct kiocb *		iocb;		/* controlling i/o request */
L
Linus Torvalds 已提交
73
	wait_queue_head_t	wait;		/* wait for i/o completion */
C
Chuck Lever 已提交
74
	struct inode *		inode;		/* target file of I/O */
L
Linus Torvalds 已提交
75 76 77 78 79 80 81 82
	struct page **		pages;		/* pages in our buffer */
	unsigned int		npages;		/* count of pages */
	atomic_t		complete,	/* i/os we're waiting for */
				count,		/* bytes actually processed */
				error;		/* any reported error */
};


83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
/**
 * nfs_direct_IO - NFS address space operation for direct I/O
 * @rw: direction (read or write)
 * @iocb: target I/O control block
 * @iov: array of vectors that define I/O buffer
 * @pos: offset in file to begin the operation
 * @nr_segs: size of iovec array
 *
 * The presence of this routine in the address space ops vector means
 * the NFS client supports direct I/O.  However, we shunt off direct
 * read and write requests before the VFS gets them, so this method
 * should never be called.
 */
ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
{
	struct dentry *dentry = iocb->ki_filp->f_dentry;

	dprintk("NFS: nfs_direct_IO (%s) off/no(%Ld/%lu) EINVAL\n",
			dentry->d_name.name, (long long) pos, nr_segs);

	return -EINVAL;
}

106
static inline int nfs_get_user_pages(int rw, unsigned long user_addr, size_t size, struct page ***pages)
L
Linus Torvalds 已提交
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
{
	int result = -ENOMEM;
	unsigned long page_count;
	size_t array_size;

	/* set an arbitrary limit to prevent type overflow */
	/* XXX: this can probably be as large as INT_MAX */
	if (size > MAX_DIRECTIO_SIZE) {
		*pages = NULL;
		return -EFBIG;
	}

	page_count = (user_addr + size + PAGE_SIZE - 1) >> PAGE_SHIFT;
	page_count -= user_addr >> PAGE_SHIFT;

	array_size = (page_count * sizeof(struct page *));
	*pages = kmalloc(array_size, GFP_KERNEL);
	if (*pages) {
		down_read(&current->mm->mmap_sem);
		result = get_user_pages(current, current->mm, user_addr,
					page_count, (rw == READ), 0,
					*pages, NULL);
		up_read(&current->mm->mmap_sem);
130 131 132 133 134 135 136 137 138
		/*
		 * If we got fewer pages than expected from get_user_pages(),
		 * the user buffer runs off the end of a mapping; return EFAULT.
		 */
		if (result >= 0 && result < page_count) {
			nfs_free_user_pages(*pages, result, 0);
			*pages = NULL;
			result = -EFAULT;
		}
L
Linus Torvalds 已提交
139 140 141 142
	}
	return result;
}

143
static void nfs_free_user_pages(struct page **pages, int npages, int do_dirty)
L
Linus Torvalds 已提交
144 145 146
{
	int i;
	for (i = 0; i < npages; i++) {
147 148 149 150
		struct page *page = pages[i];
		if (do_dirty && !PageCompound(page))
			set_page_dirty_lock(page);
		page_cache_release(page);
L
Linus Torvalds 已提交
151 152 153 154
	}
	kfree(pages);
}

155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
{
	struct nfs_direct_req *dreq;

	dreq = kmem_cache_alloc(nfs_direct_cachep, SLAB_KERNEL);
	if (!dreq)
		return NULL;

	kref_init(&dreq->kref);
	init_waitqueue_head(&dreq->wait);
	INIT_LIST_HEAD(&dreq->list);
	dreq->iocb = NULL;
	atomic_set(&dreq->count, 0);
	atomic_set(&dreq->error, 0);

	return dreq;
}

L
Linus Torvalds 已提交
173 174 175 176 177 178
static void nfs_direct_req_release(struct kref *kref)
{
	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
	kmem_cache_free(nfs_direct_cachep, dreq);
}

179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
/*
 * Collects and returns the final error value/byte-count.
 */
static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
{
	int result = -EIOCBQUEUED;

	/* Async requests don't wait here */
	if (dreq->iocb)
		goto out;

	result = wait_event_interruptible(dreq->wait,
					(atomic_read(&dreq->complete) == 0));

	if (!result)
		result = atomic_read(&dreq->error);
	if (!result)
		result = atomic_read(&dreq->count);

out:
	kref_put(&dreq->kref, nfs_direct_req_release);
	return (ssize_t) result;
}

203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
/*
 * We must hold a reference to all the pages in this direct read request
 * until the RPCs complete.  This could be long *after* we are woken up in
 * nfs_direct_wait (for instance, if someone hits ^C on a slow server).
 *
 * In addition, synchronous I/O uses a stack-allocated iocb.  Thus we
 * can't trust the iocb is still valid here if this is a synchronous
 * request.  If the waiter is woken prematurely, the iocb is long gone.
 */
static void nfs_direct_complete(struct nfs_direct_req *dreq)
{
	nfs_free_user_pages(dreq->pages, dreq->npages, 1);

	if (dreq->iocb) {
		long res = atomic_read(&dreq->error);
		if (!res)
			res = atomic_read(&dreq->count);
		aio_complete(dreq->iocb, res, 0);
	} else
		wake_up(&dreq->wait);

	kref_put(&dreq->kref, nfs_direct_req_release);
}

227
/*
L
Linus Torvalds 已提交
228 229 230 231
 * Note we also set the number of requests we have in the dreq when we are
 * done.  This prevents races with I/O completion so we will always wait
 * until all requests have been dispatched and completed.
 */
232
static struct nfs_direct_req *nfs_direct_read_alloc(size_t nbytes, size_t rsize)
L
Linus Torvalds 已提交
233 234 235 236
{
	struct list_head *list;
	struct nfs_direct_req *dreq;
	unsigned int reads = 0;
237
	unsigned int rpages = (rsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
L
Linus Torvalds 已提交
238

239
	dreq = nfs_direct_req_alloc();
L
Linus Torvalds 已提交
240 241 242 243 244
	if (!dreq)
		return NULL;

	list = &dreq->list;
	for(;;) {
245
		struct nfs_read_data *data = nfs_readdata_alloc(rpages);
L
Linus Torvalds 已提交
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271

		if (unlikely(!data)) {
			while (!list_empty(list)) {
				data = list_entry(list->next,
						  struct nfs_read_data, pages);
				list_del(&data->pages);
				nfs_readdata_free(data);
			}
			kref_put(&dreq->kref, nfs_direct_req_release);
			return NULL;
		}

		INIT_LIST_HEAD(&data->pages);
		list_add(&data->pages, list);

		data->req = (struct nfs_page *) dreq;
		reads++;
		if (nbytes <= rsize)
			break;
		nbytes -= rsize;
	}
	kref_get(&dreq->kref);
	atomic_set(&dreq->complete, reads);
	return dreq;
}

T
Trond Myklebust 已提交
272
static void nfs_direct_read_result(struct rpc_task *task, void *calldata)
L
Linus Torvalds 已提交
273
{
T
Trond Myklebust 已提交
274
	struct nfs_read_data *data = calldata;
L
Linus Torvalds 已提交
275 276
	struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;

T
Trond Myklebust 已提交
277 278 279
	if (nfs_readpage_result(task, data) != 0)
		return;
	if (likely(task->tk_status >= 0))
L
Linus Torvalds 已提交
280 281
		atomic_add(data->res.count, &dreq->count);
	else
T
Trond Myklebust 已提交
282
		atomic_set(&dreq->error, task->tk_status);
L
Linus Torvalds 已提交
283

284 285
	if (unlikely(atomic_dec_and_test(&dreq->complete)))
		nfs_direct_complete(dreq);
L
Linus Torvalds 已提交
286 287
}

T
Trond Myklebust 已提交
288 289 290 291 292
static const struct rpc_call_ops nfs_read_direct_ops = {
	.rpc_call_done = nfs_direct_read_result,
	.rpc_release = nfs_readdata_release,
};

293
/*
L
Linus Torvalds 已提交
294 295 296
 * For each nfs_read_data struct that was allocated on the list, dispatch
 * an NFS READ operation
 */
297
static void nfs_direct_read_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t file_offset)
L
Linus Torvalds 已提交
298
{
299 300 301 302
	struct file *file = dreq->filp;
	struct inode *inode = file->f_mapping->host;
	struct nfs_open_context *ctx = (struct nfs_open_context *)
							file->private_data;
L
Linus Torvalds 已提交
303 304
	struct list_head *list = &dreq->list;
	struct page **pages = dreq->pages;
305
	size_t rsize = NFS_SERVER(inode)->rsize;
L
Linus Torvalds 已提交
306 307 308 309 310 311
	unsigned int curpage, pgbase;

	curpage = 0;
	pgbase = user_addr & ~PAGE_MASK;
	do {
		struct nfs_read_data *data;
312
		size_t bytes;
L
Linus Torvalds 已提交
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332

		bytes = rsize;
		if (count < rsize)
			bytes = count;

		data = list_entry(list->next, struct nfs_read_data, pages);
		list_del_init(&data->pages);

		data->inode = inode;
		data->cred = ctx->cred;
		data->args.fh = NFS_FH(inode);
		data->args.context = ctx;
		data->args.offset = file_offset;
		data->args.pgbase = pgbase;
		data->args.pages = &pages[curpage];
		data->args.count = bytes;
		data->res.fattr = &data->fattr;
		data->res.eof = 0;
		data->res.count = bytes;

T
Trond Myklebust 已提交
333 334
		rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
				&nfs_read_direct_ops, data);
L
Linus Torvalds 已提交
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
		NFS_PROTO(inode)->read_setup(data);

		data->task.tk_cookie = (unsigned long) inode;

		lock_kernel();
		rpc_execute(&data->task);
		unlock_kernel();

		dfprintk(VFS, "NFS: %4d initiated direct read call (req %s/%Ld, %u bytes @ offset %Lu)\n",
				data->task.tk_pid,
				inode->i_sb->s_id,
				(long long)NFS_FILEID(inode),
				bytes,
				(unsigned long long)data->args.offset);

		file_offset += bytes;
		pgbase += bytes;
		curpage += pgbase >> PAGE_SHIFT;
		pgbase &= ~PAGE_MASK;

		count -= bytes;
	} while (count != 0);
}

359
static ssize_t nfs_direct_read(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t file_offset, struct page **pages, unsigned int nr_pages)
L
Linus Torvalds 已提交
360 361 362
{
	ssize_t result;
	sigset_t oldset;
363
	struct inode *inode = iocb->ki_filp->f_mapping->host;
L
Linus Torvalds 已提交
364 365 366 367 368 369 370 371 372
	struct rpc_clnt *clnt = NFS_CLIENT(inode);
	struct nfs_direct_req *dreq;

	dreq = nfs_direct_read_alloc(count, NFS_SERVER(inode)->rsize);
	if (!dreq)
		return -ENOMEM;

	dreq->pages = pages;
	dreq->npages = nr_pages;
C
Chuck Lever 已提交
373
	dreq->inode = inode;
374
	dreq->filp = iocb->ki_filp;
375 376
	if (!is_sync_kiocb(iocb))
		dreq->iocb = iocb;
L
Linus Torvalds 已提交
377

C
Chuck Lever 已提交
378
	nfs_add_stats(inode, NFSIOS_DIRECTREADBYTES, count);
L
Linus Torvalds 已提交
379
	rpc_clnt_sigmask(clnt, &oldset);
380
	nfs_direct_read_schedule(dreq, user_addr, count, file_offset);
381
	result = nfs_direct_wait(dreq);
L
Linus Torvalds 已提交
382 383 384 385 386
	rpc_clnt_sigunmask(clnt, &oldset);

	return result;
}

387
static struct nfs_direct_req *nfs_direct_write_alloc(size_t nbytes, size_t wsize)
L
Linus Torvalds 已提交
388
{
389 390 391 392
	struct list_head *list;
	struct nfs_direct_req *dreq;
	unsigned int writes = 0;
	unsigned int wpages = (wsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
L
Linus Torvalds 已提交
393

394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
	dreq = nfs_direct_req_alloc();
	if (!dreq)
		return NULL;

	list = &dreq->list;
	for(;;) {
		struct nfs_write_data *data = nfs_writedata_alloc(wpages);

		if (unlikely(!data)) {
			while (!list_empty(list)) {
				data = list_entry(list->next,
						  struct nfs_write_data, pages);
				list_del(&data->pages);
				nfs_writedata_free(data);
			}
			kref_put(&dreq->kref, nfs_direct_req_release);
			return NULL;
		}

		INIT_LIST_HEAD(&data->pages);
		list_add(&data->pages, list);

		data->req = (struct nfs_page *) dreq;
		writes++;
		if (nbytes <= wsize)
			break;
		nbytes -= wsize;
	}
	kref_get(&dreq->kref);
	atomic_set(&dreq->complete, writes);
	return dreq;
}

static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
{
	struct nfs_write_data *data = calldata;
	struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
	int status = task->tk_status;

	if (nfs_writeback_done(task, data) != 0)
		return;
	/* If the server fell back to an UNSTABLE write, it's an error. */
	if (unlikely(data->res.verf->committed != NFS_FILE_SYNC))
		status = -EIO;

	if (likely(status >= 0))
		atomic_add(data->res.count, &dreq->count);
	else
		atomic_set(&dreq->error, status);

	if (unlikely(atomic_dec_and_test(&dreq->complete)))
		nfs_direct_complete(dreq);
}

static const struct rpc_call_ops nfs_write_direct_ops = {
	.rpc_call_done = nfs_direct_write_result,
	.rpc_release = nfs_writedata_release,
};

/*
 * For each nfs_write_data struct that was allocated on the list, dispatch
 * an NFS WRITE operation
 *
 * XXX: For now, support only FILE_SYNC writes.  Later we may add
 *      support for UNSTABLE + COMMIT.
 */
460
static void nfs_direct_write_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t file_offset)
461
{
462 463 464 465
	struct file *file = dreq->filp;
	struct inode *inode = file->f_mapping->host;
	struct nfs_open_context *ctx = (struct nfs_open_context *)
							file->private_data;
466 467 468 469
	struct list_head *list = &dreq->list;
	struct page **pages = dreq->pages;
	size_t wsize = NFS_SERVER(inode)->wsize;
	unsigned int curpage, pgbase;
L
Linus Torvalds 已提交
470 471

	curpage = 0;
472
	pgbase = user_addr & ~PAGE_MASK;
L
Linus Torvalds 已提交
473
	do {
474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493
		struct nfs_write_data *data;
		size_t bytes;

		bytes = wsize;
		if (count < wsize)
			bytes = count;

		data = list_entry(list->next, struct nfs_write_data, pages);
		list_del_init(&data->pages);

		data->inode = inode;
		data->cred = ctx->cred;
		data->args.fh = NFS_FH(inode);
		data->args.context = ctx;
		data->args.offset = file_offset;
		data->args.pgbase = pgbase;
		data->args.pages = &pages[curpage];
		data->args.count = bytes;
		data->res.fattr = &data->fattr;
		data->res.count = bytes;
494
		data->res.verf = &data->verf;
495 496 497 498

		rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
				&nfs_write_direct_ops, data);
		NFS_PROTO(inode)->write_setup(data, FLUSH_STABLE);
L
Linus Torvalds 已提交
499

500 501
		data->task.tk_priority = RPC_PRIORITY_NORMAL;
		data->task.tk_cookie = (unsigned long) inode;
L
Linus Torvalds 已提交
502 503

		lock_kernel();
504
		rpc_execute(&data->task);
L
Linus Torvalds 已提交
505 506
		unlock_kernel();

507 508 509 510 511 512
		dfprintk(VFS, "NFS: %4d initiated direct write call (req %s/%Ld, %u bytes @ offset %Lu)\n",
				data->task.tk_pid,
				inode->i_sb->s_id,
				(long long)NFS_FILEID(inode),
				bytes,
				(unsigned long long)data->args.offset);
L
Linus Torvalds 已提交
513

514 515 516 517
		file_offset += bytes;
		pgbase += bytes;
		curpage += pgbase >> PAGE_SHIFT;
		pgbase &= ~PAGE_MASK;
L
Linus Torvalds 已提交
518

519 520 521
		count -= bytes;
	} while (count != 0);
}
L
Linus Torvalds 已提交
522

523
static ssize_t nfs_direct_write(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t file_offset, struct page **pages, int nr_pages)
524 525 526
{
	ssize_t result;
	sigset_t oldset;
527
	struct inode *inode = iocb->ki_filp->f_mapping->host;
528 529
	struct rpc_clnt *clnt = NFS_CLIENT(inode);
	struct nfs_direct_req *dreq;
L
Linus Torvalds 已提交
530

531 532 533
	dreq = nfs_direct_write_alloc(count, NFS_SERVER(inode)->wsize);
	if (!dreq)
		return -ENOMEM;
L
Linus Torvalds 已提交
534

535 536
	dreq->pages = pages;
	dreq->npages = nr_pages;
537 538 539 540
	dreq->inode = inode;
	dreq->filp = iocb->ki_filp;
	if (!is_sync_kiocb(iocb))
		dreq->iocb = iocb;
L
Linus Torvalds 已提交
541

542 543
	nfs_add_stats(inode, NFSIOS_DIRECTWRITTENBYTES, count);

544
	nfs_begin_data_update(inode);
L
Linus Torvalds 已提交
545

546
	rpc_clnt_sigmask(clnt, &oldset);
547 548
	nfs_direct_write_schedule(dreq, user_addr, count, file_offset);
	result = nfs_direct_wait(dreq);
549
	rpc_clnt_sigunmask(clnt, &oldset);
L
Linus Torvalds 已提交
550

551
	nfs_end_data_update(inode);
L
Linus Torvalds 已提交
552

553
	return result;
L
Linus Torvalds 已提交
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
}

/**
 * nfs_file_direct_read - file direct read operation for NFS files
 * @iocb: target I/O control block
 * @buf: user's buffer into which to read data
 * count: number of bytes to read
 * pos: byte offset in file where reading starts
 *
 * We use this function for direct reads instead of calling
 * generic_file_aio_read() in order to avoid gfar's check to see if
 * the request starts before the end of the file.  For that check
 * to work, we must generate a GETATTR before each direct read, and
 * even then there is a window between the GETATTR and the subsequent
 * READ where the file size could change.  So our preference is simply
 * to do all reads the application wants, and the server will take
 * care of managing the end of file boundary.
 * 
 * This function also eliminates unnecessarily updating the file's
 * atime locally, as the NFS server sets the file's atime, and this
 * client must read the updated atime from the server back into its
 * cache.
 */
577
ssize_t nfs_file_direct_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos)
L
Linus Torvalds 已提交
578 579
{
	ssize_t retval = -EINVAL;
580 581
	int page_count;
	struct page **pages;
L
Linus Torvalds 已提交
582 583 584
	struct file *file = iocb->ki_filp;
	struct address_space *mapping = file->f_mapping;

585
	dprintk("nfs: direct read(%s/%s, %lu@%Ld)\n",
586 587
		file->f_dentry->d_parent->d_name.name,
		file->f_dentry->d_name.name,
588
		(unsigned long) count, (long long) pos);
L
Linus Torvalds 已提交
589 590 591 592

	if (count < 0)
		goto out;
	retval = -EFAULT;
593
	if (!access_ok(VERIFY_WRITE, buf, count))
L
Linus Torvalds 已提交
594 595 596 597 598
		goto out;
	retval = 0;
	if (!count)
		goto out;

T
Trond Myklebust 已提交
599 600 601
	retval = nfs_sync_mapping(mapping);
	if (retval)
		goto out;
L
Linus Torvalds 已提交
602

603 604 605 606 607 608 609 610
	page_count = nfs_get_user_pages(READ, (unsigned long) buf,
						count, &pages);
	if (page_count < 0) {
		nfs_free_user_pages(pages, 0, 0);
		retval = page_count;
		goto out;
	}

611
	retval = nfs_direct_read(iocb, (unsigned long) buf, count, pos,
612
						pages, page_count);
L
Linus Torvalds 已提交
613
	if (retval > 0)
614
		iocb->ki_pos = pos + retval;
L
Linus Torvalds 已提交
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644

out:
	return retval;
}

/**
 * nfs_file_direct_write - file direct write operation for NFS files
 * @iocb: target I/O control block
 * @buf: user's buffer from which to write data
 * count: number of bytes to write
 * pos: byte offset in file where writing starts
 *
 * We use this function for direct writes instead of calling
 * generic_file_aio_write() in order to avoid taking the inode
 * semaphore and updating the i_size.  The NFS server will set
 * the new i_size and this client must read the updated size
 * back into its cache.  We let the server do generic write
 * parameter checking and report problems.
 *
 * We also avoid an unnecessary invocation of generic_osync_inode(),
 * as it is fairly meaningless to sync the metadata of an NFS file.
 *
 * We eliminate local atime updates, see direct read above.
 *
 * We avoid unnecessary page cache invalidations for normal cached
 * readers of this file.
 *
 * Note that O_APPEND is not supported for NFS direct writes, as there
 * is no atomic O_APPEND write facility in the NFS protocol.
 */
645
ssize_t nfs_file_direct_write(struct kiocb *iocb, const char __user *buf, size_t count, loff_t pos)
L
Linus Torvalds 已提交
646
{
647
	ssize_t retval;
648 649
	int page_count;
	struct page **pages;
L
Linus Torvalds 已提交
650 651 652
	struct file *file = iocb->ki_filp;
	struct address_space *mapping = file->f_mapping;

653
	dfprintk(VFS, "nfs: direct write(%s/%s, %lu@%Ld)\n",
654
		file->f_dentry->d_parent->d_name.name,
655 656
		file->f_dentry->d_name.name,
		(unsigned long) count, (long long) pos);
L
Linus Torvalds 已提交
657

658
	retval = -EINVAL;
L
Linus Torvalds 已提交
659 660
	if (!is_sync_kiocb(iocb))
		goto out;
661 662 663

	retval = generic_write_checks(file, &pos, &count, 0);
	if (retval)
L
Linus Torvalds 已提交
664
		goto out;
665 666 667

	retval = -EINVAL;
	if ((ssize_t) count < 0)
L
Linus Torvalds 已提交
668 669 670 671
		goto out;
	retval = 0;
	if (!count)
		goto out;
672 673

	retval = -EFAULT;
674
	if (!access_ok(VERIFY_READ, buf, count))
675
		goto out;
L
Linus Torvalds 已提交
676

T
Trond Myklebust 已提交
677 678 679
	retval = nfs_sync_mapping(mapping);
	if (retval)
		goto out;
L
Linus Torvalds 已提交
680

681 682 683 684 685 686 687 688
	page_count = nfs_get_user_pages(WRITE, (unsigned long) buf,
						count, &pages);
	if (page_count < 0) {
		nfs_free_user_pages(pages, 0, 0);
		retval = page_count;
		goto out;
	}

689
	retval = nfs_direct_write(iocb, (unsigned long) buf, count,
690
					pos, pages, page_count);
L
Linus Torvalds 已提交
691 692 693
	if (mapping->nrpages)
		invalidate_inode_pages2(mapping);
	if (retval > 0)
694
		iocb->ki_pos = pos + retval;
L
Linus Torvalds 已提交
695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716

out:
	return retval;
}

int nfs_init_directcache(void)
{
	nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
						sizeof(struct nfs_direct_req),
						0, SLAB_RECLAIM_ACCOUNT,
						NULL, NULL);
	if (nfs_direct_cachep == NULL)
		return -ENOMEM;

	return 0;
}

void nfs_destroy_directcache(void)
{
	if (kmem_cache_destroy(nfs_direct_cachep))
		printk(KERN_INFO "nfs_direct_cache: not all structures were freed\n");
}