direct.c 28.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9
/*
 * linux/fs/nfs/direct.c
 *
 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
 *
 * High-performance uncached I/O for the Linux NFS client
 *
 * There are important applications whose performance or correctness
 * depends on uncached access to file data.  Database clusters
10
 * (multiple copies of the same instance running on separate hosts)
L
Linus Torvalds 已提交
11
 * implement their own cache coherency protocol that subsumes file
12 13 14
 * system cache protocols.  Applications that process datasets
 * considerably larger than the client's memory do not always benefit
 * from a local cache.  A streaming video server, for instance, has no
L
Linus Torvalds 已提交
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
 * need to cache the contents of a file.
 *
 * When an application requests uncached I/O, all read and write requests
 * are made directly to the server; data stored or fetched via these
 * requests is not cached in the Linux page cache.  The client does not
 * correct unaligned requests from applications.  All requested bytes are
 * held on permanent storage before a direct write system call returns to
 * an application.
 *
 * Solaris implements an uncached I/O facility called directio() that
 * is used for backups and sequential I/O to very large files.  Solaris
 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
 * an undocumented mount option.
 *
 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
 * help from Andrew Morton.
 *
 * 18 Dec 2001	Initial implementation for 2.4  --cel
 * 08 Jul 2002	Version for 2.4.19, with bug fixes --trondmy
 * 08 Jun 2003	Port to 2.5 APIs  --cel
 * 31 Mar 2004	Handle direct I/O without VFS support  --cel
 * 15 Sep 2004	Parallel async reads  --cel
37
 * 04 May 2005	support O_DIRECT with aio  --cel
L
Linus Torvalds 已提交
38 39 40 41 42 43 44 45 46
 *
 */

#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/file.h>
#include <linux/pagemap.h>
#include <linux/kref.h>
47
#include <linux/slab.h>
48
#include <linux/task_io_accounting_ops.h>
49
#include <linux/module.h>
L
Linus Torvalds 已提交
50 51 52 53 54 55

#include <linux/nfs_fs.h>
#include <linux/nfs_page.h>
#include <linux/sunrpc/clnt.h>

#include <asm/uaccess.h>
A
Arun Sharma 已提交
56
#include <linux/atomic.h>
L
Linus Torvalds 已提交
57

58
#include "internal.h"
C
Chuck Lever 已提交
59
#include "iostat.h"
60
#include "pnfs.h"
C
Chuck Lever 已提交
61

L
Linus Torvalds 已提交
62 63
#define NFSDBG_FACILITY		NFSDBG_VFS

64
static struct kmem_cache *nfs_direct_cachep;
L
Linus Torvalds 已提交
65 66 67 68 69 70

/*
 * This represents a set of asynchronous requests that we're waiting on
 */
struct nfs_direct_req {
	struct kref		kref;		/* release manager */
71 72

	/* I/O parameters */
73
	struct nfs_open_context	*ctx;		/* file open context info */
74
	struct nfs_lock_context *l_ctx;		/* Lock context info */
75
	struct kiocb *		iocb;		/* controlling i/o request */
76
	struct inode *		inode;		/* target file of i/o */
77 78

	/* completion state */
79
	atomic_t		io_count;	/* i/os we're waiting for */
80 81
	spinlock_t		lock;		/* protect completion state */
	ssize_t			count,		/* bytes actually processed */
P
Peng Tao 已提交
82
				bytes_left,	/* bytes left to be sent */
L
Linus Torvalds 已提交
83
				error;		/* any reported error */
84
	struct completion	completion;	/* wait for i/o completion */
85 86

	/* commit state */
87 88 89
	struct nfs_mds_commit_info mds_cinfo;	/* Storage for cinfo */
	struct pnfs_ds_commit_info ds_cinfo;	/* Storage for cinfo */
	struct work_struct	work;
90 91 92 93
	int			flags;
#define NFS_ODIRECT_DO_COMMIT		(1)	/* an unstable reply was received */
#define NFS_ODIRECT_RESCHED_WRITES	(2)	/* write verification failed */
	struct nfs_writeverf	verf;		/* unstable write verifier */
L
Linus Torvalds 已提交
94 95
};

96 97
static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
98
static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
99
static void nfs_direct_write_schedule_work(struct work_struct *work);
100 101 102 103 104 105 106 107 108 109 110

static inline void get_dreq(struct nfs_direct_req *dreq)
{
	atomic_inc(&dreq->io_count);
}

static inline int put_dreq(struct nfs_direct_req *dreq)
{
	return atomic_dec_and_test(&dreq->io_count);
}

L
Linus Torvalds 已提交
111
/**
112 113 114 115 116 117 118 119
 * nfs_direct_IO - NFS address space operation for direct I/O
 * @rw: direction (read or write)
 * @iocb: target I/O control block
 * @iov: array of vectors that define I/O buffer
 * @pos: offset in file to begin the operation
 * @nr_segs: size of iovec array
 *
 * The presence of this routine in the address space ops vector means
M
Mel Gorman 已提交
120 121 122
 * the NFS client supports direct I/O. However, for most direct IO, we
 * shunt off direct read and write requests before the VFS gets them,
 * so this method is only ever called for swap.
L
Linus Torvalds 已提交
123
 */
124 125
ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
{
M
Mel Gorman 已提交
126
#ifndef CONFIG_NFS_SWAP
127 128
	dprintk("NFS: nfs_direct_IO (%pD) off/no(%Ld/%lu) EINVAL\n",
			iocb->ki_filp, (long long) pos, nr_segs);
129 130

	return -EINVAL;
M
Mel Gorman 已提交
131 132 133 134 135 136 137 138 139
#else
	VM_BUG_ON(iocb->ki_nbytes != PAGE_SIZE);

	if (rw == READ || rw == KERNEL_READ)
		return nfs_file_direct_read(iocb, iov, nr_segs, pos,
				rw == READ ? true : false);
	return nfs_file_direct_write(iocb, iov, nr_segs, pos,
				rw == WRITE ? true : false);
#endif /* CONFIG_NFS_SWAP */
140 141
}

142
static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
143
{
144
	unsigned int i;
145 146
	for (i = 0; i < npages; i++)
		page_cache_release(pages[i]);
147 148
}

149 150 151 152 153 154 155 156 157 158
void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
			      struct nfs_direct_req *dreq)
{
	cinfo->lock = &dreq->lock;
	cinfo->mds = &dreq->mds_cinfo;
	cinfo->ds = &dreq->ds_cinfo;
	cinfo->dreq = dreq;
	cinfo->completion_ops = &nfs_direct_commit_completion_ops;
}

159
static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
L
Linus Torvalds 已提交
160
{
161 162
	struct nfs_direct_req *dreq;

163
	dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
164 165 166 167
	if (!dreq)
		return NULL;

	kref_init(&dreq->kref);
168
	kref_get(&dreq->kref);
169
	init_completion(&dreq->completion);
170 171
	INIT_LIST_HEAD(&dreq->mds_cinfo.list);
	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
172
	spin_lock_init(&dreq->lock);
173 174

	return dreq;
L
Linus Torvalds 已提交
175 176
}

177
static void nfs_direct_req_free(struct kref *kref)
L
Linus Torvalds 已提交
178 179
{
	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
180

181 182
	if (dreq->l_ctx != NULL)
		nfs_put_lock_context(dreq->l_ctx);
183 184
	if (dreq->ctx != NULL)
		put_nfs_open_context(dreq->ctx);
L
Linus Torvalds 已提交
185 186 187
	kmem_cache_free(nfs_direct_cachep, dreq);
}

188 189 190 191 192
static void nfs_direct_req_release(struct nfs_direct_req *dreq)
{
	kref_put(&dreq->kref, nfs_direct_req_free);
}

193 194 195 196 197 198
ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)
{
	return dreq->bytes_left;
}
EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);

199 200 201 202 203
/*
 * Collects and returns the final error value/byte-count.
 */
static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
{
204
	ssize_t result = -EIOCBQUEUED;
205 206 207 208 209

	/* Async requests don't wait here */
	if (dreq->iocb)
		goto out;

210
	result = wait_for_completion_killable(&dreq->completion);
211 212

	if (!result)
213
		result = dreq->error;
214
	if (!result)
215
		result = dreq->count;
216 217 218 219 220

out:
	return (ssize_t) result;
}

221
/*
222 223
 * Synchronous I/O uses a stack-allocated iocb.  Thus we can't trust
 * the iocb is still valid here if this is a synchronous request.
224
 */
225
static void nfs_direct_complete(struct nfs_direct_req *dreq, bool write)
226
{
227 228
	struct inode *inode = dreq->inode;

229
	if (dreq->iocb && write) {
230
		loff_t pos = dreq->iocb->ki_pos + dreq->count;
231 232 233 234 235 236 237

		spin_lock(&inode->i_lock);
		if (i_size_read(inode) < pos)
			i_size_write(inode, pos);
		spin_unlock(&inode->i_lock);
	}

238
	if (write)
239
		nfs_zap_mapping(inode, inode->i_mapping);
240 241

	inode_dio_done(inode);
242 243

	if (dreq->iocb) {
244
		long res = (long) dreq->error;
245
		if (!res)
246
			res = (long) dreq->count;
247
		aio_complete(dreq->iocb, res, 0);
248
	}
249

250
	complete_all(&dreq->completion);
251

252
	nfs_direct_req_release(dreq);
253 254
}

T
Trond Myklebust 已提交
255
static void nfs_direct_readpage_release(struct nfs_page *req)
L
Linus Torvalds 已提交
256
{
257
	dprintk("NFS: direct read done (%s/%llu %d@%lld)\n",
258
		req->wb_context->dentry->d_inode->i_sb->s_id,
259
		(unsigned long long)NFS_FILEID(req->wb_context->dentry->d_inode),
260 261 262
		req->wb_bytes,
		(long long)req_offset(req));
	nfs_release_request(req);
263 264
}

265
static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
266
{
267 268
	unsigned long bytes = 0;
	struct nfs_direct_req *dreq = hdr->dreq;
269

270 271
	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
		goto out_put;
272 273

	spin_lock(&dreq->lock);
274 275 276 277 278 279
	if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))
		dreq->error = hdr->error;
	else
		dreq->count += hdr->good_bytes;
	spin_unlock(&dreq->lock);

280 281 282 283
	while (!list_empty(&hdr->pages)) {
		struct nfs_page *req = nfs_list_entry(hdr->pages.next);
		struct page *page = req->wb_page;

284 285
		if (!PageCompound(page) && bytes < hdr->good_bytes)
			set_page_dirty(page);
286 287 288
		bytes += req->wb_bytes;
		nfs_list_remove_request(req);
		nfs_direct_readpage_release(req);
289
	}
290
out_put:
291
	if (put_dreq(dreq))
292
		nfs_direct_complete(dreq, false);
293
	hdr->release(hdr);
L
Linus Torvalds 已提交
294 295
}

296
static void nfs_read_sync_pgio_error(struct list_head *head)
297
{
298
	struct nfs_page *req;
299

300 301 302 303 304
	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
		nfs_release_request(req);
	}
305 306
}

307 308 309 310 311 312
static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
{
	get_dreq(hdr->dreq);
}

static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
313
	.error_cleanup = nfs_read_sync_pgio_error,
314 315 316 317
	.init_hdr = nfs_direct_pgio_init,
	.completion = nfs_direct_read_completion,
};

318
/*
319 320 321 322 323
 * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
 * operation.  If nfs_readdata_alloc() or get_user_pages() fails,
 * bail and stop sending more reads.  Read length accounting is
 * handled automatically by nfs_direct_read_result().  Otherwise, if
 * no requests have been sent, just return an error.
L
Linus Torvalds 已提交
324
 */
325
static ssize_t nfs_direct_read_schedule_segment(struct nfs_pageio_descriptor *desc,
326
						const struct iovec *iov,
M
Mel Gorman 已提交
327
						loff_t pos, bool uio)
L
Linus Torvalds 已提交
328
{
329
	struct nfs_direct_req *dreq = desc->pg_dreq;
330
	struct nfs_open_context *ctx = dreq->ctx;
331
	struct inode *inode = ctx->dentry->d_inode;
332 333
	unsigned long user_addr = (unsigned long)iov->iov_base;
	size_t count = iov->iov_len;
334
	size_t rsize = NFS_SERVER(inode)->rsize;
335 336 337
	unsigned int pgbase;
	int result;
	ssize_t started = 0;
338 339
	struct page **pagevec = NULL;
	unsigned int npages;
340

L
Linus Torvalds 已提交
341
	do {
342
		size_t bytes;
343
		int i;
L
Linus Torvalds 已提交
344

345
		pgbase = user_addr & ~PAGE_MASK;
346
		bytes = min(max_t(size_t, rsize, PAGE_SIZE), count);
347

348
		result = -ENOMEM;
349 350 351 352 353
		npages = nfs_page_array_len(pgbase, bytes);
		if (!pagevec)
			pagevec = kmalloc(npages * sizeof(struct page *),
					  GFP_KERNEL);
		if (!pagevec)
354
			break;
M
Mel Gorman 已提交
355 356 357
		if (uio) {
			down_read(&current->mm->mmap_sem);
			result = get_user_pages(current, current->mm, user_addr,
358
					npages, 1, 0, pagevec, NULL);
M
Mel Gorman 已提交
359 360 361 362 363 364 365 366 367 368
			up_read(&current->mm->mmap_sem);
			if (result < 0)
				break;
		} else {
			WARN_ON(npages != 1);
			result = get_kernel_page(user_addr, 1, pagevec);
			if (WARN_ON(result != 1))
				break;
		}

369
		if ((unsigned)result < npages) {
370 371
			bytes = result * PAGE_SIZE;
			if (bytes <= pgbase) {
372
				nfs_direct_release_pages(pagevec, result);
373 374 375
				break;
			}
			bytes -= pgbase;
376
			npages = result;
377 378
		}

379 380
		for (i = 0; i < npages; i++) {
			struct nfs_page *req;
381
			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
382
			/* XXX do we need to do the eof zeroing found in async_filler? */
383
			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
						 pgbase, req_len);
			if (IS_ERR(req)) {
				result = PTR_ERR(req);
				break;
			}
			req->wb_index = pos >> PAGE_SHIFT;
			req->wb_offset = pos & ~PAGE_MASK;
			if (!nfs_pageio_add_request(desc, req)) {
				result = desc->pg_error;
				nfs_release_request(req);
				break;
			}
			pgbase = 0;
			bytes -= req_len;
			started += req_len;
			user_addr += req_len;
			pos += req_len;
			count -= req_len;
P
Peng Tao 已提交
402
			dreq->bytes_left -= req_len;
403
		}
404 405
		/* The nfs_page now hold references to these pages */
		nfs_direct_release_pages(pagevec, npages);
406
	} while (count != 0 && result >= 0);
407

408 409
	kfree(pagevec);

410
	if (started)
411
		return started;
412
	return result < 0 ? (ssize_t) result : -EFAULT;
L
Linus Torvalds 已提交
413 414
}

415 416 417
static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
					      const struct iovec *iov,
					      unsigned long nr_segs,
M
Mel Gorman 已提交
418
					      loff_t pos, bool uio)
419
{
420
	struct nfs_pageio_descriptor desc;
421
	struct inode *inode = dreq->inode;
422 423 424 425
	ssize_t result = -EINVAL;
	size_t requested_bytes = 0;
	unsigned long seg;

426
	nfs_pageio_init_read(&desc, dreq->inode, false,
427
			     &nfs_direct_read_completion_ops);
428
	get_dreq(dreq);
429
	desc.pg_dreq = dreq;
430
	atomic_inc(&inode->i_dio_count);
431 432 433

	for (seg = 0; seg < nr_segs; seg++) {
		const struct iovec *vec = &iov[seg];
M
Mel Gorman 已提交
434
		result = nfs_direct_read_schedule_segment(&desc, vec, pos, uio);
435 436 437 438 439 440 441 442
		if (result < 0)
			break;
		requested_bytes += result;
		if ((size_t)result < vec->iov_len)
			break;
		pos += vec->iov_len;
	}

443 444
	nfs_pageio_complete(&desc);

445 446 447 448 449
	/*
	 * If no bytes were started, return the error, and let the
	 * generic layer handle the completion.
	 */
	if (requested_bytes == 0) {
450
		inode_dio_done(inode);
451 452 453 454
		nfs_direct_req_release(dreq);
		return result < 0 ? result : -EIO;
	}

455
	if (put_dreq(dreq))
456
		nfs_direct_complete(dreq, false);
457
	return 0;
458 459
}

460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
/**
 * nfs_file_direct_read - file direct read operation for NFS files
 * @iocb: target I/O control block
 * @iov: vector of user buffers into which to read data
 * @nr_segs: size of iov vector
 * @pos: byte offset in file where reading starts
 *
 * We use this function for direct reads instead of calling
 * generic_file_aio_read() in order to avoid gfar's check to see if
 * the request starts before the end of the file.  For that check
 * to work, we must generate a GETATTR before each direct read, and
 * even then there is a window between the GETATTR and the subsequent
 * READ where the file size could change.  Our preference is simply
 * to do all reads the application wants, and the server will take
 * care of managing the end of file boundary.
 *
 * This function also eliminates unnecessarily updating the file's
 * atime locally, as the NFS server sets the file's atime, and this
 * client must read the updated atime from the server back into its
 * cache.
 */
ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov,
				unsigned long nr_segs, loff_t pos, bool uio)
L
Linus Torvalds 已提交
483
{
484 485 486
	struct file *file = iocb->ki_filp;
	struct address_space *mapping = file->f_mapping;
	struct inode *inode = mapping->host;
L
Linus Torvalds 已提交
487
	struct nfs_direct_req *dreq;
488
	struct nfs_lock_context *l_ctx;
489 490 491 492 493 494 495 496 497 498 499 500 501
	ssize_t result = -EINVAL;
	size_t count;

	count = iov_length(iov, nr_segs);
	nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);

	dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
		file, count, (long long) pos);

	result = 0;
	if (!count)
		goto out;

502
	mutex_lock(&inode->i_mutex);
503 504
	result = nfs_sync_mapping(mapping);
	if (result)
505
		goto out_unlock;
L
Linus Torvalds 已提交
506

507 508 509
	task_io_account_read(count);

	result = -ENOMEM;
510
	dreq = nfs_direct_req_alloc();
511
	if (dreq == NULL)
512
		goto out_unlock;
L
Linus Torvalds 已提交
513

C
Chuck Lever 已提交
514
	dreq->inode = inode;
P
Peng Tao 已提交
515
	dreq->bytes_left = iov_length(iov, nr_segs);
516
	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
517 518 519
	l_ctx = nfs_get_lock_context(dreq->ctx);
	if (IS_ERR(l_ctx)) {
		result = PTR_ERR(l_ctx);
520
		goto out_release;
521 522
	}
	dreq->l_ctx = l_ctx;
523 524
	if (!is_sync_kiocb(iocb))
		dreq->iocb = iocb;
L
Linus Torvalds 已提交
525

P
Peng Tao 已提交
526
	NFS_I(inode)->read_io += iov_length(iov, nr_segs);
M
Mel Gorman 已提交
527
	result = nfs_direct_read_schedule_iovec(dreq, iov, nr_segs, pos, uio);
528 529 530

	mutex_unlock(&inode->i_mutex);

531
	if (!result) {
532
		result = nfs_direct_wait(dreq);
533 534 535
		if (result > 0)
			iocb->ki_pos = pos + result;
	}
536 537 538 539

	nfs_direct_req_release(dreq);
	return result;

540
out_release:
541
	nfs_direct_req_release(dreq);
542 543
out_unlock:
	mutex_unlock(&inode->i_mutex);
544
out:
L
Linus Torvalds 已提交
545 546 547
	return result;
}

B
Bryan Schumaker 已提交
548
#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
549 550
static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
{
551 552 553 554 555 556 557 558 559 560 561
	struct nfs_pageio_descriptor desc;
	struct nfs_page *req, *tmp;
	LIST_HEAD(reqs);
	struct nfs_commit_info cinfo;
	LIST_HEAD(failed);

	nfs_init_cinfo_from_dreq(&cinfo, dreq);
	pnfs_recover_commit_reqs(dreq->inode, &reqs, &cinfo);
	spin_lock(cinfo.lock);
	nfs_scan_commit_list(&cinfo.mds->list, &reqs, &cinfo, 0);
	spin_unlock(cinfo.lock);
L
Linus Torvalds 已提交
562

563
	dreq->count = 0;
564 565
	get_dreq(dreq);

566
	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
567 568
			      &nfs_direct_write_completion_ops);
	desc.pg_dreq = dreq;
569

570 571
	list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
		if (!nfs_pageio_add_request(&desc, req)) {
572
			nfs_list_remove_request(req);
573 574 575 576 577 578
			nfs_list_add_request(req, &failed);
			spin_lock(cinfo.lock);
			dreq->flags = 0;
			dreq->error = -EIO;
			spin_unlock(cinfo.lock);
		}
579
		nfs_release_request(req);
580 581
	}
	nfs_pageio_complete(&desc);
582

583 584 585
	while (!list_empty(&failed)) {
		req = nfs_list_entry(failed.next);
		nfs_list_remove_request(req);
586
		nfs_unlock_and_release_request(req);
587
	}
588

589 590
	if (put_dreq(dreq))
		nfs_direct_write_complete(dreq, dreq->inode);
591 592
}

593
static void nfs_direct_commit_complete(struct nfs_commit_data *data)
594
{
595
	struct nfs_direct_req *dreq = data->dreq;
596 597
	struct nfs_commit_info cinfo;
	struct nfs_page *req;
598 599
	int status = data->task.tk_status;

600
	nfs_init_cinfo_from_dreq(&cinfo, dreq);
601
	if (status < 0) {
602
		dprintk("NFS: %5u commit failed with error %d.\n",
603
			data->task.tk_pid, status);
604
		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
605
	} else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) {
606
		dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid);
607
		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
L
Linus Torvalds 已提交
608 609
	}

610
	dprintk("NFS: %5u commit returned %d\n", data->task.tk_pid, status);
611 612 613 614 615 616
	while (!list_empty(&data->pages)) {
		req = nfs_list_entry(data->pages.next);
		nfs_list_remove_request(req);
		if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
			/* Note the rewrite will go through mds */
			nfs_mark_request_commit(req, NULL, &cinfo);
617 618
		} else
			nfs_release_request(req);
619
		nfs_unlock_and_release_request(req);
620 621 622 623
	}

	if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
		nfs_direct_write_complete(dreq, data->inode);
L
Linus Torvalds 已提交
624 625
}

626 627 628 629 630 631 632 633
static void nfs_direct_error_cleanup(struct nfs_inode *nfsi)
{
	/* There is no lock to clear */
}

static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
	.completion = nfs_direct_commit_complete,
	.error_cleanup = nfs_direct_error_cleanup,
634 635 636
};

static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
L
Linus Torvalds 已提交
637
{
638 639 640 641 642 643 644 645 646
	int res;
	struct nfs_commit_info cinfo;
	LIST_HEAD(mds_list);

	nfs_init_cinfo_from_dreq(&cinfo, dreq);
	nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
	res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
	if (res < 0) /* res == -ENOMEM */
		nfs_direct_write_reschedule(dreq);
647
}
L
Linus Torvalds 已提交
648

649
static void nfs_direct_write_schedule_work(struct work_struct *work)
650
{
651
	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
652
	int flags = dreq->flags;
L
Linus Torvalds 已提交
653

654 655 656 657
	dreq->flags = 0;
	switch (flags) {
		case NFS_ODIRECT_DO_COMMIT:
			nfs_direct_commit_schedule(dreq);
L
Linus Torvalds 已提交
658
			break;
659 660 661 662
		case NFS_ODIRECT_RESCHED_WRITES:
			nfs_direct_write_reschedule(dreq);
			break;
		default:
663
			nfs_direct_complete(dreq, true);
664 665
	}
}
L
Linus Torvalds 已提交
666

667
static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
668
{
669
	schedule_work(&dreq->work); /* Calls nfs_direct_write_schedule_work */
670
}
671

672
#else
673 674 675
static void nfs_direct_write_schedule_work(struct work_struct *work)
{
}
L
Linus Torvalds 已提交
676

677 678
static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
{
679
	nfs_direct_complete(dreq, true);
680 681
}
#endif
L
Linus Torvalds 已提交
682

683 684 685 686
/*
 * NB: Return the value of the first error return code.  Subsequent
 *     errors after the first one are ignored.
 */
687
/*
688 689 690 691 692
 * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
 * operation.  If nfs_writedata_alloc() or get_user_pages() fails,
 * bail and stop sending more writes.  Write length accounting is
 * handled automatically by nfs_direct_write_result().  Otherwise, if
 * no requests have been sent, just return an error.
693
 */
694
static ssize_t nfs_direct_write_schedule_segment(struct nfs_pageio_descriptor *desc,
695
						 const struct iovec *iov,
M
Mel Gorman 已提交
696
						 loff_t pos, bool uio)
697
{
698
	struct nfs_direct_req *dreq = desc->pg_dreq;
699
	struct nfs_open_context *ctx = dreq->ctx;
700
	struct inode *inode = ctx->dentry->d_inode;
701 702
	unsigned long user_addr = (unsigned long)iov->iov_base;
	size_t count = iov->iov_len;
703
	size_t wsize = NFS_SERVER(inode)->wsize;
704 705 706
	unsigned int pgbase;
	int result;
	ssize_t started = 0;
707 708
	struct page **pagevec = NULL;
	unsigned int npages;
709

L
Linus Torvalds 已提交
710
	do {
711
		size_t bytes;
712
		int i;
713

714
		pgbase = user_addr & ~PAGE_MASK;
715
		bytes = min(max_t(size_t, wsize, PAGE_SIZE), count);
716

717
		result = -ENOMEM;
718 719 720 721
		npages = nfs_page_array_len(pgbase, bytes);
		if (!pagevec)
			pagevec = kmalloc(npages * sizeof(struct page *), GFP_KERNEL);
		if (!pagevec)
722 723
			break;

M
Mel Gorman 已提交
724 725 726 727 728 729 730 731 732 733 734 735 736
		if (uio) {
			down_read(&current->mm->mmap_sem);
			result = get_user_pages(current, current->mm, user_addr,
						npages, 0, 0, pagevec, NULL);
			up_read(&current->mm->mmap_sem);
			if (result < 0)
				break;
		} else {
			WARN_ON(npages != 1);
			result = get_kernel_page(user_addr, 0, pagevec);
			if (WARN_ON(result != 1))
				break;
		}
737 738

		if ((unsigned)result < npages) {
739 740
			bytes = result * PAGE_SIZE;
			if (bytes <= pgbase) {
741
				nfs_direct_release_pages(pagevec, result);
742 743 744
				break;
			}
			bytes -= pgbase;
745
			npages = result;
746 747
		}

748 749
		for (i = 0; i < npages; i++) {
			struct nfs_page *req;
750
			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
L
Linus Torvalds 已提交
751

752
			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
753 754 755 756 757 758 759 760 761 762
						 pgbase, req_len);
			if (IS_ERR(req)) {
				result = PTR_ERR(req);
				break;
			}
			nfs_lock_request(req);
			req->wb_index = pos >> PAGE_SHIFT;
			req->wb_offset = pos & ~PAGE_MASK;
			if (!nfs_pageio_add_request(desc, req)) {
				result = desc->pg_error;
763
				nfs_unlock_and_release_request(req);
764
				break;
765 766 767 768 769 770 771
			}
			pgbase = 0;
			bytes -= req_len;
			started += req_len;
			user_addr += req_len;
			pos += req_len;
			count -= req_len;
P
Peng Tao 已提交
772
			dreq->bytes_left -= req_len;
773
		}
774 775
		/* The nfs_page now hold references to these pages */
		nfs_direct_release_pages(pagevec, npages);
776
	} while (count != 0 && result >= 0);
777

778 779
	kfree(pagevec);

780
	if (started)
781
		return started;
782
	return result < 0 ? (ssize_t) result : -EFAULT;
783
}
L
Linus Torvalds 已提交
784

785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
{
	struct nfs_direct_req *dreq = hdr->dreq;
	struct nfs_commit_info cinfo;
	int bit = -1;
	struct nfs_page *req = nfs_list_entry(hdr->pages.next);

	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
		goto out_put;

	nfs_init_cinfo_from_dreq(&cinfo, dreq);

	spin_lock(&dreq->lock);

	if (test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
		dreq->flags = 0;
		dreq->error = hdr->error;
	}
	if (dreq->error != 0)
		bit = NFS_IOHDR_ERROR;
	else {
		dreq->count += hdr->good_bytes;
		if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) {
			dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
			bit = NFS_IOHDR_NEED_RESCHED;
		} else if (test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) {
			if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)
				bit = NFS_IOHDR_NEED_RESCHED;
			else if (dreq->flags == 0) {
814
				memcpy(&dreq->verf, &hdr->verf,
815 816 817 818
				       sizeof(dreq->verf));
				bit = NFS_IOHDR_NEED_COMMIT;
				dreq->flags = NFS_ODIRECT_DO_COMMIT;
			} else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {
819
				if (memcmp(&dreq->verf, &hdr->verf, sizeof(dreq->verf))) {
820 821 822 823 824 825 826 827 828 829
					dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
					bit = NFS_IOHDR_NEED_RESCHED;
				} else
					bit = NFS_IOHDR_NEED_COMMIT;
			}
		}
	}
	spin_unlock(&dreq->lock);

	while (!list_empty(&hdr->pages)) {
830 831
		bool do_destroy = true;

832 833 834 835 836
		req = nfs_list_entry(hdr->pages.next);
		nfs_list_remove_request(req);
		switch (bit) {
		case NFS_IOHDR_NEED_RESCHED:
		case NFS_IOHDR_NEED_COMMIT:
837
			kref_get(&req->wb_kref);
838
			nfs_mark_request_commit(req, hdr->lseg, &cinfo);
839
			do_destroy = false;
840
		}
841
		nfs_unlock_and_release_request(req);
842 843 844 845 846 847 848 849
	}

out_put:
	if (put_dreq(dreq))
		nfs_direct_write_complete(dreq, hdr->inode);
	hdr->release(hdr);
}

850 851 852 853 854 855 856
static void nfs_write_sync_pgio_error(struct list_head *head)
{
	struct nfs_page *req;

	while (!list_empty(head)) {
		req = nfs_list_entry(head->next);
		nfs_list_remove_request(req);
857
		nfs_unlock_and_release_request(req);
858 859 860
	}
}

861
static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
862
	.error_cleanup = nfs_write_sync_pgio_error,
863 864 865 866
	.init_hdr = nfs_direct_pgio_init,
	.completion = nfs_direct_write_completion,
};

867 868 869
static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
					       const struct iovec *iov,
					       unsigned long nr_segs,
M
Mel Gorman 已提交
870
					       loff_t pos, bool uio)
871
{
872
	struct nfs_pageio_descriptor desc;
873
	struct inode *inode = dreq->inode;
874 875 876 877
	ssize_t result = 0;
	size_t requested_bytes = 0;
	unsigned long seg;

878
	nfs_pageio_init_write(&desc, inode, FLUSH_COND_STABLE, false,
879 880
			      &nfs_direct_write_completion_ops);
	desc.pg_dreq = dreq;
881
	get_dreq(dreq);
882
	atomic_inc(&inode->i_dio_count);
883

P
Peng Tao 已提交
884
	NFS_I(dreq->inode)->write_io += iov_length(iov, nr_segs);
885 886
	for (seg = 0; seg < nr_segs; seg++) {
		const struct iovec *vec = &iov[seg];
M
Mel Gorman 已提交
887
		result = nfs_direct_write_schedule_segment(&desc, vec, pos, uio);
888 889 890 891 892 893 894
		if (result < 0)
			break;
		requested_bytes += result;
		if ((size_t)result < vec->iov_len)
			break;
		pos += vec->iov_len;
	}
895
	nfs_pageio_complete(&desc);
896

897 898 899 900 901
	/*
	 * If no bytes were started, return the error, and let the
	 * generic layer handle the completion.
	 */
	if (requested_bytes == 0) {
902
		inode_dio_done(inode);
903 904 905 906
		nfs_direct_req_release(dreq);
		return result < 0 ? result : -EIO;
	}

907 908
	if (put_dreq(dreq))
		nfs_direct_write_complete(dreq, dreq->inode);
909
	return 0;
910 911
}

L
Linus Torvalds 已提交
912 913 914
/**
 * nfs_file_direct_write - file direct write operation for NFS files
 * @iocb: target I/O control block
915 916
 * @iov: vector of user buffers from which to write data
 * @nr_segs: size of iov vector
917
 * @pos: byte offset in file where writing starts
L
Linus Torvalds 已提交
918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933
 *
 * We use this function for direct writes instead of calling
 * generic_file_aio_write() in order to avoid taking the inode
 * semaphore and updating the i_size.  The NFS server will set
 * the new i_size and this client must read the updated size
 * back into its cache.  We let the server do generic write
 * parameter checking and report problems.
 *
 * We eliminate local atime updates, see direct read above.
 *
 * We avoid unnecessary page cache invalidations for normal cached
 * readers of this file.
 *
 * Note that O_APPEND is not supported for NFS direct writes, as there
 * is no atomic O_APPEND write facility in the NFS protocol.
 */
934
ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
M
Mel Gorman 已提交
935
				unsigned long nr_segs, loff_t pos, bool uio)
L
Linus Torvalds 已提交
936
{
937
	ssize_t result = -EINVAL;
L
Linus Torvalds 已提交
938 939
	struct file *file = iocb->ki_filp;
	struct address_space *mapping = file->f_mapping;
940 941 942
	struct inode *inode = mapping->host;
	struct nfs_direct_req *dreq;
	struct nfs_lock_context *l_ctx;
943
	loff_t end;
944
	size_t count;
L
Linus Torvalds 已提交
945

946
	count = iov_length(iov, nr_segs);
947 948
	end = (pos + count - 1) >> PAGE_CACHE_SHIFT;

949 950
	nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);

951 952
	dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
		file, count, (long long) pos);
953

954 955
	result = generic_write_checks(file, &pos, &count, 0);
	if (result)
L
Linus Torvalds 已提交
956
		goto out;
957

958
	result = -EINVAL;
959
	if ((ssize_t) count < 0)
L
Linus Torvalds 已提交
960
		goto out;
961
	result = 0;
L
Linus Torvalds 已提交
962 963
	if (!count)
		goto out;
964

965 966
	mutex_lock(&inode->i_mutex);

967 968
	result = nfs_sync_mapping(mapping);
	if (result)
969 970 971 972 973 974 975 976
		goto out_unlock;

	if (mapping->nrpages) {
		result = invalidate_inode_pages2_range(mapping,
					pos >> PAGE_CACHE_SHIFT, end);
		if (result)
			goto out_unlock;
	}
L
Linus Torvalds 已提交
977

978 979
	task_io_account_write(count);

980 981 982
	result = -ENOMEM;
	dreq = nfs_direct_req_alloc();
	if (!dreq)
983
		goto out_unlock;
984

985 986 987 988 989 990 991 992 993 994 995 996 997
	dreq->inode = inode;
	dreq->bytes_left = count;
	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
	l_ctx = nfs_get_lock_context(dreq->ctx);
	if (IS_ERR(l_ctx)) {
		result = PTR_ERR(l_ctx);
		goto out_release;
	}
	dreq->l_ctx = l_ctx;
	if (!is_sync_kiocb(iocb))
		dreq->iocb = iocb;

	result = nfs_direct_write_schedule_iovec(dreq, iov, nr_segs, pos, uio);
998 999 1000 1001 1002 1003 1004 1005

	if (mapping->nrpages) {
		invalidate_inode_pages2_range(mapping,
					      pos >> PAGE_CACHE_SHIFT, end);
	}

	mutex_unlock(&inode->i_mutex);

1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
	if (!result) {
		result = nfs_direct_wait(dreq);
		if (result > 0) {
			struct inode *inode = mapping->host;

			iocb->ki_pos = pos + result;
			spin_lock(&inode->i_lock);
			if (i_size_read(inode) < iocb->ki_pos)
				i_size_write(inode, iocb->ki_pos);
			spin_unlock(&inode->i_lock);
		}
1017
	}
1018 1019 1020
	nfs_direct_req_release(dreq);
	return result;

1021 1022
out_release:
	nfs_direct_req_release(dreq);
1023 1024
out_unlock:
	mutex_unlock(&inode->i_mutex);
L
Linus Torvalds 已提交
1025
out:
1026
	return result;
L
Linus Torvalds 已提交
1027 1028
}

1029 1030 1031 1032
/**
 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
 *
 */
D
David Howells 已提交
1033
int __init nfs_init_directcache(void)
L
Linus Torvalds 已提交
1034 1035 1036
{
	nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
						sizeof(struct nfs_direct_req),
1037 1038
						0, (SLAB_RECLAIM_ACCOUNT|
							SLAB_MEM_SPREAD),
1039
						NULL);
L
Linus Torvalds 已提交
1040 1041 1042 1043 1044 1045
	if (nfs_direct_cachep == NULL)
		return -ENOMEM;

	return 0;
}

1046
/**
D
David Howells 已提交
1047
 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
1048 1049
 *
 */
1050
void nfs_destroy_directcache(void)
L
Linus Torvalds 已提交
1051
{
1052
	kmem_cache_destroy(nfs_direct_cachep);
L
Linus Torvalds 已提交
1053
}