pnfs.c 56.1 KB
Newer Older
R
Ricardo Labiaga 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 *  pNFS functions to call and manage layout drivers.
 *
 *  Copyright (c) 2002 [year of first publication]
 *  The Regents of the University of Michigan
 *  All Rights Reserved
 *
 *  Dean Hildebrand <dhildebz@umich.edu>
 *
 *  Permission is granted to use, copy, create derivative works, and
 *  redistribute this software and such derivative works for any purpose,
 *  so long as the name of the University of Michigan is not used in
 *  any advertising or publicity pertaining to the use or distribution
 *  of this software without specific, written prior authorization. If
 *  the above copyright notice or any other identification of the
 *  University of Michigan is included in any copy of any portion of
 *  this software, then the disclaimer below must also be included.
 *
 *  This software is provided as is, without representation or warranty
 *  of any kind either express or implied, including without limitation
 *  the implied warranties of merchantability, fitness for a particular
 *  purpose, or noninfringement.  The Regents of the University of
 *  Michigan shall not be liable for any damages, including special,
 *  indirect, incidental, or consequential damages, with respect to any
 *  claim arising out of or in connection with the use of the software,
 *  even if it has been or is hereafter advised of the possibility of
 *  such damages.
 */

#include <linux/nfs_fs.h>
31
#include <linux/nfs_page.h>
32
#include <linux/module.h>
33
#include "internal.h"
R
Ricardo Labiaga 已提交
34
#include "pnfs.h"
A
Andy Adamson 已提交
35
#include "iostat.h"
36
#include "nfs4trace.h"
R
Ricardo Labiaga 已提交
37 38

#define NFSDBG_FACILITY		NFSDBG_PNFS
39
#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
R
Ricardo Labiaga 已提交
40

41 42 43 44 45 46 47 48 49 50 51 52
/* Locking:
 *
 * pnfs_spinlock:
 *      protects pnfs_modules_tbl.
 */
static DEFINE_SPINLOCK(pnfs_spinlock);

/*
 * pnfs_modules_tbl holds all pnfs modules
 */
static LIST_HEAD(pnfs_modules_tbl);

53 54 55 56
static int
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid,
		       enum pnfs_iomode iomode);

57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
/* Return the registered pnfs layout driver module matching given id */
static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)
{
	struct pnfs_layoutdriver_type *local;

	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
		if (local->id == id)
			goto out;
	local = NULL;
out:
	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
	return local;
}

R
Ricardo Labiaga 已提交
72 73 74
static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)
{
75 76 77 78
	struct pnfs_layoutdriver_type *local;

	spin_lock(&pnfs_spinlock);
	local = find_pnfs_driver_locked(id);
79 80 81 82
	if (local != NULL && !try_module_get(local->owner)) {
		dprintk("%s: Could not grab reference on module\n", __func__);
		local = NULL;
	}
83 84
	spin_unlock(&pnfs_spinlock);
	return local;
R
Ricardo Labiaga 已提交
85 86 87 88 89
}

void
unset_pnfs_layoutdriver(struct nfs_server *nfss)
{
90 91 92
	if (nfss->pnfs_curr_ld) {
		if (nfss->pnfs_curr_ld->clear_layoutdriver)
			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
93 94 95
		/* Decrement the MDS count. Purge the deviceid cache if zero */
		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
			nfs4_deviceid_purge_client(nfss->nfs_client);
96
		module_put(nfss->pnfs_curr_ld->owner);
97
	}
R
Ricardo Labiaga 已提交
98 99 100 101 102 103 104 105 106 107
	nfss->pnfs_curr_ld = NULL;
}

/*
 * Try to set the server's pnfs module to the pnfs layout type specified by id.
 * Currently only one pNFS layout driver per filesystem is supported.
 *
 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
 */
void
108 109
set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
		      u32 id)
R
Ricardo Labiaga 已提交
110 111 112 113 114 115 116
{
	struct pnfs_layoutdriver_type *ld_type = NULL;

	if (id == 0)
		goto out_no_driver;
	if (!(server->nfs_client->cl_exchange_flags &
		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
117 118
		printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
			__func__, id, server->nfs_client->cl_exchange_flags);
R
Ricardo Labiaga 已提交
119 120 121 122 123 124 125 126 127 128 129 130 131
		goto out_no_driver;
	}
	ld_type = find_pnfs_driver(id);
	if (!ld_type) {
		request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
		ld_type = find_pnfs_driver(id);
		if (!ld_type) {
			dprintk("%s: No pNFS module found for %u.\n",
				__func__, id);
			goto out_no_driver;
		}
	}
	server->pnfs_curr_ld = ld_type;
132 133
	if (ld_type->set_layoutdriver
	    && ld_type->set_layoutdriver(server, mntfh)) {
134 135
		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
			"driver %u.\n", __func__, id);
136 137 138
		module_put(ld_type->owner);
		goto out_no_driver;
	}
139 140
	/* Bump the MDS count */
	atomic_inc(&server->nfs_client->cl_mds_count);
141

R
Ricardo Labiaga 已提交
142 143 144 145 146 147 148
	dprintk("%s: pNFS module for %u set\n", __func__, id);
	return;

out_no_driver:
	dprintk("%s: Using NFSv4 I/O\n", __func__);
	server->pnfs_curr_ld = NULL;
}
149 150 151 152 153 154 155 156

int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	int status = -EINVAL;
	struct pnfs_layoutdriver_type *tmp;

	if (ld_type->id == 0) {
157
		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
158 159
		return status;
	}
160
	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
161
		printk(KERN_ERR "NFS: %s Layout driver must provide "
162 163 164
		       "alloc_lseg and free_lseg.\n", __func__);
		return status;
	}
165 166 167 168 169 170 171 172 173

	spin_lock(&pnfs_spinlock);
	tmp = find_pnfs_driver_locked(ld_type->id);
	if (!tmp) {
		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
		status = 0;
		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
			ld_type->name);
	} else {
174
		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
			__func__, ld_type->id);
	}
	spin_unlock(&pnfs_spinlock);

	return status;
}
EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);

void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
	spin_lock(&pnfs_spinlock);
	list_del(&ld_type->pnfs_tblid);
	spin_unlock(&pnfs_spinlock);
}
EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
192

193 194 195 196
/*
 * pNFS client layout cache
 */

197
/* Need to hold i_lock if caller does not already hold reference */
F
Fred Isaman 已提交
198
void
199
pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
200
{
201
	atomic_inc(&lo->plh_refcount);
202 203
}

204 205 206 207
static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
208
	return ld->alloc_layout_hdr(ino, gfp_flags);
209 210 211 212 213
}

static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
214 215 216 217 218 219 220 221 222 223
	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;

	if (!list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		spin_lock(&clp->cl_lock);
		list_del_init(&lo->plh_layouts);
		spin_unlock(&clp->cl_lock);
	}
224
	put_rpccred(lo->plh_lc_cred);
225
	return ld->free_layout_hdr(lo);
226 227
}

228
static void
229
pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
230
{
231
	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
232
	dprintk("%s: freeing layout cache %p\n", __func__, lo);
233 234 235 236
	nfsi->layout = NULL;
	/* Reset MDS Threshold I/O counters */
	nfsi->write_io = 0;
	nfsi->read_io = 0;
237 238
}

239
void
240
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
241
{
242 243 244
	struct inode *inode = lo->plh_inode;

	if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
245
		pnfs_detach_layout_hdr(lo);
246
		spin_unlock(&inode->i_lock);
247
		pnfs_free_layout_hdr(lo);
248
	}
249 250
}

251 252 253 254 255 256 257 258
static int
pnfs_iomode_to_fail_bit(u32 iomode)
{
	return iomode == IOMODE_RW ?
		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
}

static void
259
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
260
{
261
	lo->plh_retry_timestamp = jiffies;
262
	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
263 264 265 266 267 268 269 270 271 272 273 274 275 276
		atomic_inc(&lo->plh_refcount);
}

static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
{
	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
		atomic_dec(&lo->plh_refcount);
}

static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
	struct inode *inode = lo->plh_inode;
277 278 279 280 281 282
	struct pnfs_layout_range range = {
		.iomode = iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(head);
283 284 285

	spin_lock(&inode->i_lock);
	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
286
	pnfs_mark_matching_lsegs_invalid(lo, &head, &range);
287
	spin_unlock(&inode->i_lock);
288
	pnfs_free_lseg_list(&head);
289 290 291 292 293 294 295
	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
			iomode == IOMODE_RW ?  "RW" : "READ");
}

static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
296
	unsigned long start, end;
297 298 299
	int fail_bit = pnfs_iomode_to_fail_bit(iomode);

	if (test_bit(fail_bit, &lo->plh_flags) == 0)
300 301 302 303 304
		return false;
	end = jiffies;
	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
		/* It is time to retry the failed layoutgets */
305
		pnfs_layout_clear_fail_bit(lo, fail_bit);
306 307 308
		return false;
	}
	return true;
309 310
}

311 312 313
static void
init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
{
314
	INIT_LIST_HEAD(&lseg->pls_list);
315
	INIT_LIST_HEAD(&lseg->pls_lc_list);
316 317 318
	atomic_set(&lseg->pls_refcount, 1);
	smp_mb();
	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
319
	lseg->pls_layout = lo;
320 321
}

322
static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
323
{
324
	struct inode *ino = lseg->pls_layout->plh_inode;
325

326
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
327 328
}

F
Fred Isaman 已提交
329
static void
330 331
pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
F
Fred Isaman 已提交
332
{
333
	struct inode *inode = lo->plh_inode;
F
Fred Isaman 已提交
334

335
	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
F
Fred Isaman 已提交
336
	list_del_init(&lseg->pls_list);
337 338
	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
	atomic_dec(&lo->plh_refcount);
339 340
	if (list_empty(&lo->plh_segs))
		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
F
Fred Isaman 已提交
341 342 343
	rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}

344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
/* Return true if layoutreturn is needed */
static bool
pnfs_layout_need_return(struct pnfs_layout_hdr *lo,
			struct pnfs_layout_segment *lseg,
			nfs4_stateid *stateid, enum pnfs_iomode *iomode)
{
	struct pnfs_layout_segment *s;

	if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
		return false;

	list_for_each_entry(s, &lo->plh_segs, pls_list)
		if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
			return false;

	*stateid = lo->plh_stateid;
	*iomode = lo->plh_return_iomode;
	/* decreased in pnfs_send_layoutreturn() */
	lo->plh_block_lgets++;
	lo->plh_return_iomode = 0;
	return true;
}

367
void
368
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
369
{
370
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
371 372 373 374 375
	struct inode *inode;

	if (!lseg)
		return;

376 377 378
	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
379 380
	lo = lseg->pls_layout;
	inode = lo->plh_inode;
F
Fred Isaman 已提交
381
	if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
382 383 384 385
		bool need_return;
		nfs4_stateid stateid;
		enum pnfs_iomode iomode;

386
		pnfs_get_layout_hdr(lo);
387
		pnfs_layout_remove_lseg(lo, lseg);
388 389
		need_return = pnfs_layout_need_return(lo, lseg,
						      &stateid, &iomode);
F
Fred Isaman 已提交
390
		spin_unlock(&inode->i_lock);
391
		pnfs_free_lseg(lseg);
392 393 394 395
		if (need_return)
			pnfs_send_layoutreturn(lo, stateid, iomode);
		else
			pnfs_put_layout_hdr(lo);
396 397
	}
}
398
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
399

400
static void pnfs_free_lseg_async_work(struct work_struct *work)
401 402
{
	struct pnfs_layout_segment *lseg;
403
	struct pnfs_layout_hdr *lo;
404 405

	lseg = container_of(work, struct pnfs_layout_segment, pls_work);
406
	lo = lseg->pls_layout;
407

408 409
	pnfs_free_lseg(lseg);
	pnfs_put_layout_hdr(lo);
410 411
}

412
static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg)
413
{
414
	INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work);
415 416
	schedule_work(&lseg->pls_work);
}
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436

void
pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
{
	if (!lseg)
		return;

	assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock);

	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
	if (atomic_dec_and_test(&lseg->pls_refcount)) {
		struct pnfs_layout_hdr *lo = lseg->pls_layout;
		pnfs_get_layout_hdr(lo);
		pnfs_layout_remove_lseg(lo, lseg);
		pnfs_free_lseg_async(lseg);
	}
}
EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
437

438
static u64
439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
end_offset(u64 start, u64 len)
{
	u64 end;

	end = start + len;
	return end >= start ? end : NFS4_MAX_UINT64;
}

/*
 * is l2 fully contained in l1?
 *   start1                             end1
 *   [----------------------------------)
 *           start2           end2
 *           [----------------)
 */
454
static bool
455
pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
456
		 const struct pnfs_layout_range *l2)
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (start1 <= start2) && (end1 >= end2);
}

/*
 * is l1 and l2 intersecting?
 *   start1                             end1
 *   [----------------------------------)
 *                              start2           end2
 *                              [----------------)
 */
473
static bool
474
pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
475
		    const struct pnfs_layout_range *l2)
476 477 478 479 480 481 482 483 484 485
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
	       (end2 == NFS4_MAX_UINT64 || end2 > start1);
}

486
static bool
487 488
should_free_lseg(const struct pnfs_layout_range *lseg_range,
		 const struct pnfs_layout_range *recall_range)
489
{
490 491
	return (recall_range->iomode == IOMODE_ANY ||
		lseg_range->iomode == recall_range->iomode) &&
492
	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
493 494
}

495 496 497 498 499 500 501 502 503 504
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		struct list_head *tmp_list)
{
	if (!atomic_dec_and_test(&lseg->pls_refcount))
		return false;
	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
	list_add(&lseg->pls_list, tmp_list);
	return true;
}

505 506 507 508 509 510 511 512 513 514 515
/* Returns 1 if lseg is removed from list, 0 otherwise */
static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
			     struct list_head *tmp_list)
{
	int rv = 0;

	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
		/* Remove the reference keeping the lseg in the
		 * list.  It will now be removed when all
		 * outstanding io is finished.
		 */
F
Fred Isaman 已提交
516 517
		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
			atomic_read(&lseg->pls_refcount));
518
		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
F
Fred Isaman 已提交
519
			rv = 1;
520 521 522 523 524 525 526
	}
	return rv;
}

/* Returns count of number of matching invalid lsegs remaining in list
 * after call.
 */
F
Fred Isaman 已提交
527
int
528
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
529
			    struct list_head *tmp_list,
530
			    struct pnfs_layout_range *recall_range)
531 532
{
	struct pnfs_layout_segment *lseg, *next;
533
	int invalid = 0, removed = 0;
534 535 536

	dprintk("%s:Begin lo %p\n", __func__, lo);

537
	if (list_empty(&lo->plh_segs))
538
		return 0;
539
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
540 541
		if (!recall_range ||
		    should_free_lseg(&lseg->pls_range, recall_range)) {
542 543 544 545 546 547 548 549 550
			dprintk("%s: freeing lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode, lseg->pls_range.offset,
				lseg->pls_range.length);
			invalid++;
			removed += mark_lseg_invalid(lseg, tmp_list);
		}
	dprintk("%s:Return %i\n", __func__, invalid - removed);
	return invalid - removed;
551 552
}

553
/* note free_me must contain lsegs from a single layout_hdr */
F
Fred Isaman 已提交
554
void
555
pnfs_free_lseg_list(struct list_head *free_me)
556
{
557
	struct pnfs_layout_segment *lseg, *tmp;
558 559 560 561

	if (list_empty(free_me))
		return;

562
	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
563
		list_del(&lseg->pls_list);
564
		pnfs_free_lseg(lseg);
565 566 567
	}
}

568 569 570 571
void
pnfs_destroy_layout(struct nfs_inode *nfsi)
{
	struct pnfs_layout_hdr *lo;
572
	LIST_HEAD(tmp_list);
573 574 575 576

	spin_lock(&nfsi->vfs_inode.i_lock);
	lo = nfsi->layout;
	if (lo) {
577
		lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
578
		pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
579 580 581 582 583 584 585 586
		pnfs_get_layout_hdr(lo);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
		spin_unlock(&nfsi->vfs_inode.i_lock);
		pnfs_free_lseg_list(&tmp_list);
		pnfs_put_layout_hdr(lo);
	} else
		spin_unlock(&nfsi->vfs_inode.i_lock);
587
}
588
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
589

590 591 592
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
		struct list_head *layout_list)
593 594
{
	struct pnfs_layout_hdr *lo;
595
	bool ret = false;
596

597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653
	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
		pnfs_get_layout_hdr(lo);
		list_add(&lo->plh_bulk_destroy, layout_list);
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

/* Caller must hold rcu_read_lock and clp->cl_lock */
static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
		struct nfs_server *server,
		struct list_head *layout_list)
{
	struct pnfs_layout_hdr *lo, *next;
	struct inode *inode;

	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
		inode = igrab(lo->plh_inode);
		if (inode == NULL)
			continue;
		list_del_init(&lo->plh_layouts);
		if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
			continue;
		rcu_read_unlock();
		spin_unlock(&clp->cl_lock);
		iput(inode);
		spin_lock(&clp->cl_lock);
		rcu_read_lock();
		return -EAGAIN;
	}
	return 0;
}

static int
pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
		bool is_bulk_recall)
{
	struct pnfs_layout_hdr *lo;
	struct inode *inode;
	struct pnfs_layout_range range = {
		.iomode = IOMODE_ANY,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(lseg_list);
	int ret = 0;

	while (!list_empty(layout_list)) {
		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
				plh_bulk_destroy);
		dprintk("%s freeing layout for inode %lu\n", __func__,
			lo->plh_inode->i_ino);
		inode = lo->plh_inode;
654 655 656

		pnfs_layoutcommit_inode(inode, false);

657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
		spin_lock(&inode->i_lock);
		list_del_init(&lo->plh_bulk_destroy);
		lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
		if (is_bulk_recall)
			set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
		if (pnfs_mark_matching_lsegs_invalid(lo, &lseg_list, &range))
			ret = -EAGAIN;
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg_list(&lseg_list);
		pnfs_put_layout_hdr(lo);
		iput(inode);
	}
	return ret;
}

int
pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
		struct nfs_fsid *fsid,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);
679

680
	spin_lock(&clp->cl_lock);
681
	rcu_read_lock();
682
restart:
683
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
684 685 686 687 688 689
		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
			continue;
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
				server,
				&layout_list) != 0)
			goto restart;
690 691
	}
	rcu_read_unlock();
692 693
	spin_unlock(&clp->cl_lock);

694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

int
pnfs_destroy_layouts_byclid(struct nfs_client *clp,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);

	spin_lock(&clp->cl_lock);
	rcu_read_lock();
restart:
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
					server,
					&layout_list) != 0)
			goto restart;
714
	}
715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
	rcu_read_unlock();
	spin_unlock(&clp->cl_lock);

	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

/*
 * Called by the state manger to remove all layouts established under an
 * expired lease.
 */
void
pnfs_destroy_all_layouts(struct nfs_client *clp)
{
	nfs4_deviceid_mark_client_invalid(clp);
	nfs4_deviceid_purge_client(clp);

	pnfs_destroy_layouts_byclid(clp, false);
734 735
}

736 737 738 739 740 741
/*
 * Compare 2 layout stateid sequence ids, to see which is newer,
 * taking into account wraparound issues.
 */
static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
{
742
	return (s32)(s1 - s2) > 0;
743 744
}

745
/* update lo->plh_stateid with new if is more recent */
F
Fred Isaman 已提交
746 747 748
void
pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
			bool update_barrier)
749
{
750 751
	u32 oldseq, newseq, new_barrier;
	int empty = list_empty(&lo->plh_segs);
752

753 754
	oldseq = be32_to_cpu(lo->plh_stateid.seqid);
	newseq = be32_to_cpu(new->seqid);
755
	if (empty || pnfs_seqid_is_newer(newseq, oldseq)) {
756
		nfs4_stateid_copy(&lo->plh_stateid, new);
F
Fred Isaman 已提交
757
		if (update_barrier) {
758
			new_barrier = be32_to_cpu(new->seqid);
F
Fred Isaman 已提交
759 760
		} else {
			/* Because of wraparound, we want to keep the barrier
761
			 * "close" to the current seqids.
F
Fred Isaman 已提交
762
			 */
763
			new_barrier = newseq - atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
764
		}
765 766
		if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
			lo->plh_barrier = new_barrier;
F
Fred Isaman 已提交
767
	}
768 769
}

770
static bool
771 772
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid)
F
Fred Isaman 已提交
773
{
774
	u32 seqid = be32_to_cpu(stateid->seqid);
775

776 777 778
	return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
}

779 780 781 782 783 784 785 786 787
static bool
pnfs_layout_returning(const struct pnfs_layout_hdr *lo,
		      struct pnfs_layout_range *range)
{
	return test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) &&
		(lo->plh_return_iomode == IOMODE_ANY ||
		 lo->plh_return_iomode == range->iomode);
}

788 789
/* lget is set to 1 if called from inside send_layoutget call chain */
static bool
790 791
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo,
			struct pnfs_layout_range *range, int lget)
792
{
F
Fred Isaman 已提交
793 794
	return lo->plh_block_lgets ||
		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags) ||
F
Fred Isaman 已提交
795
		(list_empty(&lo->plh_segs) &&
796 797
		 (atomic_read(&lo->plh_outstanding) > lget)) ||
		pnfs_layout_returning(lo, range);
798 799
}

800 801
int
pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
802
			      struct pnfs_layout_range *range,
803
			      struct nfs4_state *open_state)
804
{
805
	int status = 0;
806

807
	dprintk("--> %s\n", __func__);
808
	spin_lock(&lo->plh_inode->i_lock);
809
	if (pnfs_layoutgets_blocked(lo, range, 1)) {
810
		status = -EAGAIN;
811 812
	} else if (!nfs4_valid_open_stateid(open_state)) {
		status = -EBADF;
813 814
	} else if (list_empty(&lo->plh_segs) ||
		   test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
815 816 817 818
		int seq;

		do {
			seq = read_seqbegin(&open_state->seqlock);
819
			nfs4_stateid_copy(dst, &open_state->stateid);
820 821
		} while (read_seqretry(&open_state->seqlock, seq));
	} else
822
		nfs4_stateid_copy(dst, &lo->plh_stateid);
823
	spin_unlock(&lo->plh_inode->i_lock);
824
	dprintk("<-- %s\n", __func__);
825
	return status;
826 827 828 829 830 831 832 833
}

/*
* Get layout from server.
*    for now, assume that whole file layouts are requested.
*    arg->offset: 0
*    arg->length: all ones
*/
834 835 836
static struct pnfs_layout_segment *
send_layoutget(struct pnfs_layout_hdr *lo,
	   struct nfs_open_context *ctx,
837
	   struct pnfs_layout_range *range,
838
	   gfp_t gfp_flags)
839
{
840
	struct inode *ino = lo->plh_inode;
841 842
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs4_layoutget *lgp;
843
	struct pnfs_layout_segment *lseg;
844 845

	dprintk("--> %s\n", __func__);
846

847
	lgp = kzalloc(sizeof(*lgp), gfp_flags);
848
	if (lgp == NULL)
849
		return NULL;
850

851 852 853
	lgp->args.minlength = PAGE_CACHE_SIZE;
	if (lgp->args.minlength > range->length)
		lgp->args.minlength = range->length;
854
	lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
855
	lgp->args.range = *range;
856 857 858
	lgp->args.type = server->pnfs_curr_ld->id;
	lgp->args.inode = ino;
	lgp->args.ctx = get_nfs_open_context(ctx);
859
	lgp->gfp_flags = gfp_flags;
860
	lgp->cred = lo->plh_lc_cred;
861 862 863 864

	/* Synchronously retrieve layout information from server and
	 * store in lseg.
	 */
865 866 867 868 869 870 871 872
	lseg = nfs4_proc_layoutget(lgp, gfp_flags);
	if (IS_ERR(lseg)) {
		switch (PTR_ERR(lseg)) {
		case -ENOMEM:
		case -ERESTARTSYS:
			break;
		default:
			/* remember that LAYOUTGET failed and suspend trying */
873
			pnfs_layout_io_set_failed(lo, range->iomode);
874 875
		}
		return NULL;
876
	}
877

878 879 880
	return lseg;
}

881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
static void pnfs_clear_layoutcommit(struct inode *inode,
		struct list_head *head)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct pnfs_layout_segment *lseg, *tmp;

	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		return;
	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
			continue;
		pnfs_lseg_dec_and_remove_zero(lseg, head);
	}
}

896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
static int
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid,
		       enum pnfs_iomode iomode)
{
	struct inode *ino = lo->plh_inode;
	struct nfs4_layoutreturn *lrp;
	int status = 0;

	lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
	if (unlikely(lrp == NULL)) {
		status = -ENOMEM;
		spin_lock(&ino->i_lock);
		lo->plh_block_lgets--;
		spin_unlock(&ino->i_lock);
		pnfs_put_layout_hdr(lo);
		goto out;
	}

	lrp->args.stateid = stateid;
	lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
	lrp->args.inode = ino;
	lrp->args.iomode = iomode;
	lrp->args.layout = lo;
	lrp->clp = NFS_SERVER(ino)->nfs_client;
	lrp->cred = lo->plh_lc_cred;

	status = nfs4_proc_layoutreturn(lrp);
out:
924 925 926 927 928
	if (status) {
		spin_lock(&ino->i_lock);
		clear_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
		spin_unlock(&ino->i_lock);
	}
929 930 931 932
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

933 934 935 936 937 938 939 940
/*
 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
 * when the layout segment list is empty.
 *
 * Note that a pnfs_layout_hdr can exist with an empty layout segment
 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
 * deviceid is marked invalid.
 */
B
Benny Halevy 已提交
941 942 943 944 945 946 947
int
_pnfs_return_layout(struct inode *ino)
{
	struct pnfs_layout_hdr *lo = NULL;
	struct nfs_inode *nfsi = NFS_I(ino);
	LIST_HEAD(tmp_list);
	nfs4_stateid stateid;
948
	int status = 0, empty;
B
Benny Halevy 已提交
949

950
	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
B
Benny Halevy 已提交
951 952 953

	spin_lock(&ino->i_lock);
	lo = nfsi->layout;
954
	if (!lo) {
B
Benny Halevy 已提交
955
		spin_unlock(&ino->i_lock);
956 957
		dprintk("NFS: %s no layout to return\n", __func__);
		goto out;
B
Benny Halevy 已提交
958 959 960
	}
	stateid = nfsi->layout->plh_stateid;
	/* Reference matched in nfs4_layoutreturn_release */
961
	pnfs_get_layout_hdr(lo);
962
	empty = list_empty(&lo->plh_segs);
963
	pnfs_clear_layoutcommit(ino, &tmp_list);
964
	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
965 966 967 968 969 970 971 972 973 974

	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
		struct pnfs_layout_range range = {
			.iomode		= IOMODE_ANY,
			.offset		= 0,
			.length		= NFS4_MAX_UINT64,
		};
		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
	}

975 976 977
	/* Don't send a LAYOUTRETURN if list was initially empty */
	if (empty) {
		spin_unlock(&ino->i_lock);
978
		pnfs_put_layout_hdr(lo);
979 980 981
		dprintk("NFS: %s no layout segments to return\n", __func__);
		goto out;
	}
982 983

	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
984
	lo->plh_block_lgets++;
B
Benny Halevy 已提交
985 986 987
	spin_unlock(&ino->i_lock);
	pnfs_free_lseg_list(&tmp_list);

988
	status = pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY);
B
Benny Halevy 已提交
989 990 991 992
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}
993
EXPORT_SYMBOL_GPL(_pnfs_return_layout);
B
Benny Halevy 已提交
994

995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
int
pnfs_commit_and_return_layout(struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	int ret;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo == NULL) {
		spin_unlock(&inode->i_lock);
		return 0;
	}
	pnfs_get_layout_hdr(lo);
	/* Block new layoutgets and read/write to ds */
	lo->plh_block_lgets++;
	spin_unlock(&inode->i_lock);
	filemap_fdatawait(inode->i_mapping);
	ret = pnfs_layoutcommit_inode(inode, true);
	if (ret == 0)
		ret = _pnfs_return_layout(inode);
	spin_lock(&inode->i_lock);
	lo->plh_block_lgets--;
	spin_unlock(&inode->i_lock);
	pnfs_put_layout_hdr(lo);
	return ret;
}

F
Fred Isaman 已提交
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
bool pnfs_roc(struct inode *ino)
{
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg, *tmp;
	LIST_HEAD(tmp_list);
	bool found = false;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
	if (!lo || !test_and_clear_bit(NFS_LAYOUT_ROC, &lo->plh_flags) ||
	    test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
		goto out_nolayout;
	list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
		if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
			mark_lseg_invalid(lseg, &tmp_list);
			found = true;
		}
	if (!found)
		goto out_nolayout;
	lo->plh_block_lgets++;
1042
	pnfs_get_layout_hdr(lo); /* matched in pnfs_roc_release */
F
Fred Isaman 已提交
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
	spin_unlock(&ino->i_lock);
	pnfs_free_lseg_list(&tmp_list);
	return true;

out_nolayout:
	spin_unlock(&ino->i_lock);
	return false;
}

void pnfs_roc_release(struct inode *ino)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
	lo->plh_block_lgets--;
1059 1060 1061 1062 1063 1064
	if (atomic_dec_and_test(&lo->plh_refcount)) {
		pnfs_detach_layout_hdr(lo);
		spin_unlock(&ino->i_lock);
		pnfs_free_layout_hdr(lo);
	} else
		spin_unlock(&ino->i_lock);
F
Fred Isaman 已提交
1065 1066 1067 1068 1069 1070 1071 1072
}

void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1073
	if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
F
Fred Isaman 已提交
1074 1075 1076 1077
		lo->plh_barrier = barrier;
	spin_unlock(&ino->i_lock);
}

1078
bool pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task)
F
Fred Isaman 已提交
1079 1080
{
	struct nfs_inode *nfsi = NFS_I(ino);
1081
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
1082
	struct pnfs_layout_segment *lseg;
1083
	u32 current_seqid;
F
Fred Isaman 已提交
1084 1085 1086 1087 1088
	bool found = false;

	spin_lock(&ino->i_lock);
	list_for_each_entry(lseg, &nfsi->layout->plh_segs, pls_list)
		if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
1089
			rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
F
Fred Isaman 已提交
1090
			found = true;
1091
			goto out;
F
Fred Isaman 已提交
1092
		}
1093 1094
	lo = nfsi->layout;
	current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
F
Fred Isaman 已提交
1095

1096 1097 1098 1099 1100
	/* Since close does not return a layout stateid for use as
	 * a barrier, we choose the worst-case barrier.
	 */
	*barrier = current_seqid + atomic_read(&lo->plh_outstanding);
out:
F
Fred Isaman 已提交
1101 1102 1103 1104
	spin_unlock(&ino->i_lock);
	return found;
}

1105 1106 1107 1108 1109 1110
/*
 * Compare two layout segments for sorting into layout cache.
 * We want to preferentially return RW over RO layouts, so ensure those
 * are seen first.
 */
static s64
1111
pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1112
	   const struct pnfs_layout_range *l2)
1113
{
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
	s64 d;

	/* high offset > low offset */
	d = l1->offset - l2->offset;
	if (d)
		return d;

	/* short length > long length */
	d = l2->length - l1->length;
	if (d)
		return d;

1126
	/* read > read/write */
1127
	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1128 1129
}

1130
static void
1131
pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
1132 1133
		   struct pnfs_layout_segment *lseg)
{
1134 1135
	struct pnfs_layout_segment *lp;

1136 1137
	dprintk("%s:Begin\n", __func__);

1138
	list_for_each_entry(lp, &lo->plh_segs, pls_list) {
1139
		if (pnfs_lseg_range_cmp(&lseg->pls_range, &lp->pls_range) > 0)
1140
			continue;
1141
		list_add_tail(&lseg->pls_list, &lp->pls_list);
1142 1143 1144
		dprintk("%s: inserted lseg %p "
			"iomode %d offset %llu length %llu before "
			"lp %p iomode %d offset %llu length %llu\n",
1145 1146 1147 1148
			__func__, lseg, lseg->pls_range.iomode,
			lseg->pls_range.offset, lseg->pls_range.length,
			lp, lp->pls_range.iomode, lp->pls_range.offset,
			lp->pls_range.length);
1149
		goto out;
1150
	}
1151 1152 1153 1154 1155 1156
	list_add_tail(&lseg->pls_list, &lo->plh_segs);
	dprintk("%s: inserted lseg %p "
		"iomode %d offset %llu length %llu at tail\n",
		__func__, lseg, lseg->pls_range.iomode,
		lseg->pls_range.offset, lseg->pls_range.length);
out:
1157
	pnfs_get_layout_hdr(lo);
1158 1159

	dprintk("%s:Return\n", __func__);
1160 1161 1162
}

static struct pnfs_layout_hdr *
1163 1164 1165
alloc_init_layout_hdr(struct inode *ino,
		      struct nfs_open_context *ctx,
		      gfp_t gfp_flags)
1166 1167 1168
{
	struct pnfs_layout_hdr *lo;

1169
	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1170 1171
	if (!lo)
		return NULL;
1172
	atomic_set(&lo->plh_refcount, 1);
1173 1174
	INIT_LIST_HEAD(&lo->plh_layouts);
	INIT_LIST_HEAD(&lo->plh_segs);
1175
	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1176
	lo->plh_inode = ino;
1177
	lo->plh_lc_cred = get_rpccred(ctx->cred);
1178 1179 1180 1181
	return lo;
}

static struct pnfs_layout_hdr *
1182 1183 1184
pnfs_find_alloc_layout(struct inode *ino,
		       struct nfs_open_context *ctx,
		       gfp_t gfp_flags)
1185 1186 1187 1188 1189 1190
{
	struct nfs_inode *nfsi = NFS_I(ino);
	struct pnfs_layout_hdr *new = NULL;

	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

1191 1192
	if (nfsi->layout != NULL)
		goto out_existing;
1193
	spin_unlock(&ino->i_lock);
1194
	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1195 1196
	spin_lock(&ino->i_lock);

1197
	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1198
		nfsi->layout = new;
1199
		return new;
1200 1201
	} else if (new != NULL)
		pnfs_free_layout_hdr(new);
1202 1203
out_existing:
	pnfs_get_layout_hdr(nfsi->layout);
1204 1205 1206
	return nfsi->layout;
}

1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217
/*
 * iomode matching rules:
 * iomode	lseg	match
 * -----	-----	-----
 * ANY		READ	true
 * ANY		RW	true
 * RW		READ	false
 * RW		RW	true
 * READ		READ	true
 * READ		RW	true
 */
1218
static bool
1219
pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1220
		 const struct pnfs_layout_range *range)
1221
{
1222 1223 1224 1225
	struct pnfs_layout_range range1;

	if ((range->iomode == IOMODE_RW &&
	     ls_range->iomode != IOMODE_RW) ||
1226
	    !pnfs_lseg_range_intersecting(ls_range, range))
1227 1228 1229 1230 1231
		return 0;

	/* range1 covers only the first byte in the range */
	range1 = *range;
	range1.length = 1;
1232
	return pnfs_lseg_range_contained(ls_range, &range1);
1233 1234 1235 1236 1237
}

/*
 * lookup range in layout
 */
1238
static struct pnfs_layout_segment *
1239 1240
pnfs_find_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_range *range)
1241
{
1242 1243 1244 1245
	struct pnfs_layout_segment *lseg, *ret = NULL;

	dprintk("%s:Begin\n", __func__);

1246
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1247
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1248
		    !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
1249
		    pnfs_lseg_range_match(&lseg->pls_range, range)) {
1250
			ret = pnfs_get_lseg(lseg);
1251 1252
			break;
		}
1253
		if (lseg->pls_range.offset > range->offset)
1254 1255 1256 1257
			break;
	}

	dprintk("%s:Return lseg %p ref %d\n",
1258
		__func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
1259
	return ret;
1260 1261
}

1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
/*
 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
 * to the MDS or over pNFS
 *
 * The nfs_inode read_io and write_io fields are cumulative counters reset
 * when there are no layout segments. Note that in pnfs_update_layout iomode
 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
 * WRITE request.
 *
 * A return of true means use MDS I/O.
 *
 * From rfc 5661:
 * If a file's size is smaller than the file size threshold, data accesses
 * SHOULD be sent to the metadata server.  If an I/O request has a length that
 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
 * server.  If both file size and I/O size are provided, the client SHOULD
 * reach or exceed  both thresholds before sending its read or write
 * requests to the data server.
 */
static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
				     struct inode *ino, int iomode)
{
	struct nfs4_threshold *t = ctx->mdsthreshold;
	struct nfs_inode *nfsi = NFS_I(ino);
	loff_t fsize = i_size_read(ino);
	bool size = false, size_set = false, io = false, io_set = false, ret = false;

	if (t == NULL)
		return ret;

	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);

	switch (iomode) {
	case IOMODE_READ:
		if (t->bm & THRESHOLD_RD) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->rd_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_RD_IO) {
			dprintk("%s nfsi->read_io %llu\n", __func__,
				nfsi->read_io);
			io_set = true;
			if (nfsi->read_io < t->rd_io_sz)
				io = true;
		}
		break;
	case IOMODE_RW:
		if (t->bm & THRESHOLD_WR) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->wr_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_WR_IO) {
			dprintk("%s nfsi->write_io %llu\n", __func__,
				nfsi->write_io);
			io_set = true;
			if (nfsi->write_io < t->wr_io_sz)
				io = true;
		}
		break;
	}
	if (size_set && io_set) {
		if (size && io)
			ret = true;
	} else if (size || io)
		ret = true;

	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
	return ret;
}

1337 1338 1339 1340
/*
 * Layout segment is retreived from the server if not cached.
 * The appropriate layout segment is referenced and returned to the caller.
 */
1341
struct pnfs_layout_segment *
1342 1343
pnfs_update_layout(struct inode *ino,
		   struct nfs_open_context *ctx,
1344 1345
		   loff_t pos,
		   u64 count,
1346 1347
		   enum pnfs_iomode iomode,
		   gfp_t gfp_flags)
1348
{
1349 1350 1351 1352 1353
	struct pnfs_layout_range arg = {
		.iomode = iomode,
		.offset = pos,
		.length = count,
	};
1354
	unsigned pg_offset;
1355 1356
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs_client *clp = server->nfs_client;
1357 1358
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg = NULL;
1359
	bool first;
1360 1361

	if (!pnfs_enabled_sb(NFS_SERVER(ino)))
1362
		goto out;
1363 1364

	if (pnfs_within_mdsthreshold(ctx, ino, iomode))
1365
		goto out;
1366

1367 1368
lookup_again:
	first = false;
1369
	spin_lock(&ino->i_lock);
1370
	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
1371 1372 1373 1374
	if (lo == NULL) {
		spin_unlock(&ino->i_lock);
		goto out;
	}
1375

F
Fred Isaman 已提交
1376
	/* Do we even need to bother with this? */
1377
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
F
Fred Isaman 已提交
1378
		dprintk("%s matches recall, use MDS\n", __func__);
1379 1380 1381 1382
		goto out_unlock;
	}

	/* if LAYOUTGET already failed once we don't try again */
1383
	if (pnfs_layout_io_test_failed(lo, iomode))
1384 1385
		goto out_unlock;

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
	first = list_empty(&lo->plh_segs);
	if (first) {
		/* The first layoutget for the file. Need to serialize per
		 * RFC 5661 Errata 3208.
		 */
		if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
				     &lo->plh_flags)) {
			spin_unlock(&ino->i_lock);
			wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
				    TASK_UNINTERRUPTIBLE);
			pnfs_put_layout_hdr(lo);
			goto lookup_again;
		}
	} else {
		/* Check to see if the layout for the given range
		 * already exists
		 */
		lseg = pnfs_find_lseg(lo, &arg);
		if (lseg)
			goto out_unlock;
	}
1407

1408
	if (pnfs_layoutgets_blocked(lo, &arg, 0))
1409 1410
		goto out_unlock;
	atomic_inc(&lo->plh_outstanding);
1411
	spin_unlock(&ino->i_lock);
1412

1413
	if (list_empty(&lo->plh_layouts)) {
1414 1415 1416 1417
		/* The lo must be on the clp list if there is any
		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
		 */
		spin_lock(&clp->cl_lock);
1418 1419
		if (list_empty(&lo->plh_layouts))
			list_add_tail(&lo->plh_layouts, &server->layouts);
1420 1421
		spin_unlock(&clp->cl_lock);
	}
1422

1423 1424 1425 1426 1427
	pg_offset = arg.offset & ~PAGE_CACHE_MASK;
	if (pg_offset) {
		arg.offset -= pg_offset;
		arg.length += pg_offset;
	}
1428 1429
	if (arg.length != NFS4_MAX_UINT64)
		arg.length = PAGE_CACHE_ALIGN(arg.length);
1430

1431
	lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
1432
	atomic_dec(&lo->plh_outstanding);
1433
out_put_layout_hdr:
1434 1435 1436 1437 1438 1439 1440
	if (first) {
		unsigned long *bitlock = &lo->plh_flags;

		clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
		smp_mb__after_atomic();
		wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
	}
1441
	pnfs_put_layout_hdr(lo);
1442
out:
1443 1444 1445 1446 1447 1448 1449 1450
	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
			"(%s, offset: %llu, length: %llu)\n",
			__func__, ino->i_sb->s_id,
			(unsigned long long)NFS_FILEID(ino),
			lseg == NULL ? "not found" : "found",
			iomode==IOMODE_RW ?  "read/write" : "read-only",
			(unsigned long long)pos,
			(unsigned long long)count);
1451 1452 1453
	return lseg;
out_unlock:
	spin_unlock(&ino->i_lock);
1454
	goto out_put_layout_hdr;
1455
}
1456
EXPORT_SYMBOL_GPL(pnfs_update_layout);
1457

1458
struct pnfs_layout_segment *
1459 1460 1461 1462 1463
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
	struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
	struct nfs4_layoutget_res *res = &lgp->res;
	struct pnfs_layout_segment *lseg;
1464
	struct inode *ino = lo->plh_inode;
1465
	LIST_HEAD(free_me);
1466 1467 1468
	int status = 0;

	/* Inject layout blob into I/O device driver */
1469
	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
1470 1471 1472 1473 1474 1475 1476 1477 1478 1479
	if (!lseg || IS_ERR(lseg)) {
		if (!lseg)
			status = -ENOMEM;
		else
			status = PTR_ERR(lseg);
		dprintk("%s: Could not allocate layout: error %d\n",
		       __func__, status);
		goto out;
	}

1480 1481 1482
	init_lseg(lo, lseg);
	lseg->pls_range = res->range;

1483
	spin_lock(&ino->i_lock);
1484
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
F
Fred Isaman 已提交
1485 1486 1487 1488
		dprintk("%s forget reply due to recall\n", __func__);
		goto out_forget_reply;
	}

1489
	if (pnfs_layoutgets_blocked(lo, &lgp->args.range, 1)) {
F
Fred Isaman 已提交
1490 1491 1492
		dprintk("%s forget reply due to state\n", __func__);
		goto out_forget_reply;
	}
1493

1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511
	if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
		/* existing state ID, make sure the sequence number matches. */
		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
			dprintk("%s forget reply due to sequence\n", __func__);
			goto out_forget_reply;
		}
		pnfs_set_layout_stateid(lo, &res->stateid, false);
	} else {
		/*
		 * We got an entirely new state ID.  Mark all segments for the
		 * inode invalid, and don't bother validating the stateid
		 * sequence number.
		 */
		pnfs_mark_matching_lsegs_invalid(lo, &free_me, NULL);

		nfs4_stateid_copy(&lo->plh_stateid, &res->stateid);
		lo->plh_barrier = be32_to_cpu(res->stateid.seqid);
	}
1512

1513 1514
	clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);

1515
	pnfs_get_lseg(lseg);
1516
	pnfs_layout_insert_lseg(lo, lseg);
1517

F
Fred Isaman 已提交
1518 1519 1520 1521 1522
	if (res->return_on_close) {
		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
		set_bit(NFS_LAYOUT_ROC, &lo->plh_flags);
	}

1523
	spin_unlock(&ino->i_lock);
1524
	pnfs_free_lseg_list(&free_me);
1525
	return lseg;
1526
out:
1527
	return ERR_PTR(status);
F
Fred Isaman 已提交
1528 1529 1530 1531 1532 1533

out_forget_reply:
	spin_unlock(&ino->i_lock);
	lseg->pls_layout = lo;
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
	goto out;
1534 1535
}

1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590
static void
pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
				struct list_head *tmp_list,
				struct pnfs_layout_range *return_range)
{
	struct pnfs_layout_segment *lseg, *next;

	dprintk("%s:Begin lo %p\n", __func__, lo);

	if (list_empty(&lo->plh_segs))
		return;

	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
		if (should_free_lseg(&lseg->pls_range, return_range)) {
			dprintk("%s: marking lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode,
				lseg->pls_range.offset,
				lseg->pls_range.length);
			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
			mark_lseg_invalid(lseg, tmp_list);
		}
}

void pnfs_error_mark_layout_for_return(struct inode *inode,
				       struct pnfs_layout_segment *lseg)
{
	struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
	int iomode = pnfs_iomode_to_fail_bit(lseg->pls_range.iomode);
	struct pnfs_layout_range range = {
		.iomode = lseg->pls_range.iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(free_me);

	spin_lock(&inode->i_lock);
	/* set failure bit so that pnfs path will be retried later */
	pnfs_layout_set_fail_bit(lo, iomode);
	set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
	if (lo->plh_return_iomode == 0)
		lo->plh_return_iomode = range.iomode;
	else if (lo->plh_return_iomode != range.iomode)
		lo->plh_return_iomode = IOMODE_ANY;
	/*
	 * mark all matching lsegs so that we are sure to have no live
	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
	 * for how it works.
	 */
	pnfs_mark_matching_lsegs_return(lo, &free_me, &range);
	spin_unlock(&inode->i_lock);
	pnfs_free_lseg_list(&free_me);
}
EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);

1591 1592 1593
void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
P
Peng Tao 已提交
1594 1595
	u64 rd_size = req->wb_bytes;

1596
	WARN_ON_ONCE(pgio->pg_lseg != NULL);
1597

P
Peng Tao 已提交
1598 1599 1600 1601 1602
	if (pgio->pg_dreq == NULL)
		rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
	else
		rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);

1603 1604 1605
	pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
					   req->wb_context,
					   req_offset(req),
P
Peng Tao 已提交
1606
					   rd_size,
1607 1608
					   IOMODE_READ,
					   GFP_KERNEL);
1609 1610
	/* If no lseg, fall back to read through mds */
	if (pgio->pg_lseg == NULL)
1611
		nfs_pageio_reset_read_mds(pgio);
1612

1613 1614 1615 1616
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);

void
1617 1618
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
			   struct nfs_page *req, u64 wb_size)
1619
{
1620
	WARN_ON_ONCE(pgio->pg_lseg != NULL);
1621 1622 1623 1624

	pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
					   req->wb_context,
					   req_offset(req),
1625
					   wb_size,
1626 1627
					   IOMODE_RW,
					   GFP_NOFS);
1628 1629
	/* If no lseg, fall back to write through mds */
	if (pgio->pg_lseg == NULL)
1630
		nfs_pageio_reset_write_mds(pgio);
1631 1632 1633
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);

1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
void
pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
{
	if (desc->pg_lseg) {
		pnfs_put_lseg(desc->pg_lseg);
		desc->pg_lseg = NULL;
	}
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);

1644 1645 1646 1647 1648
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
size_t
1649 1650
pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
		     struct nfs_page *prev, struct nfs_page *req)
1651
{
1652
	unsigned int size;
1653
	u64 seg_end, req_start, seg_left;
1654 1655 1656 1657

	size = nfs_generic_pg_test(pgio, prev, req);
	if (!size)
		return 0;
1658

1659
	/*
1660 1661 1662 1663 1664
	 * 'size' contains the number of bytes left in the current page (up
	 * to the original size asked for in @req->wb_bytes).
	 *
	 * Calculate how many bytes are left in the layout segment
	 * and if there are less bytes than 'size', return that instead.
1665 1666 1667 1668 1669
	 *
	 * Please also note that 'end_offset' is actually the offset of the
	 * first byte that lies outside the pnfs_layout_range. FIXME?
	 *
	 */
1670
	if (pgio->pg_lseg) {
1671 1672 1673 1674 1675 1676
		seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
				     pgio->pg_lseg->pls_range.length);
		req_start = req_offset(req);
		WARN_ON_ONCE(req_start > seg_end);
		/* start of request is past the last byte of this segment */
		if (req_start >= seg_end)
1677
			return 0;
1678 1679 1680 1681 1682 1683

		/* adjust 'size' iff there are fewer bytes left in the
		 * segment than what nfs_generic_pg_test returned */
		seg_left = seg_end - req_start;
		if (seg_left < size)
			size = (unsigned int)seg_left;
1684
	}
1685

1686
	return size;
1687
}
1688
EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
1689

1690
int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
1691 1692 1693 1694
{
	struct nfs_pageio_descriptor pgio;

	/* Resend all requests through the MDS */
1695 1696 1697
	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
			      hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
1698
}
1699
EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
1700

1701
static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
1702
{
1703 1704 1705

	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
1706
	    PNFS_LAYOUTRET_ON_ERROR) {
1707
		pnfs_return_layout(hdr->inode);
1708
	}
1709
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1710
		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
1711 1712
}

1713 1714 1715
/*
 * Called by non rpc-based layout drivers
 */
1716
void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
1717
{
1718
	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
1719
	if (!hdr->pnfs_error) {
1720 1721
		pnfs_set_layoutcommit(hdr);
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
1722
	} else
1723 1724
		pnfs_ld_handle_write_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
1725
}
1726
EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
1727

1728 1729
static void
pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
1730
		struct nfs_pgio_header *hdr)
1731
{
1732 1733
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

1734
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1735
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
1736
		nfs_pageio_reset_write_mds(desc);
1737
		mirror->pg_recoalesce = 1;
1738
	}
1739
	nfs_pgio_data_destroy(hdr);
1740 1741 1742
}

static enum pnfs_try_status
1743
pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
1744 1745 1746
			const struct rpc_call_ops *call_ops,
			struct pnfs_layout_segment *lseg,
			int how)
1747
{
1748
	struct inode *inode = hdr->inode;
1749 1750 1751
	enum pnfs_try_status trypnfs;
	struct nfs_server *nfss = NFS_SERVER(inode);

1752
	hdr->mds_ops = call_ops;
1753 1754

	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
1755 1756
		inode->i_ino, hdr->args.count, hdr->args.offset, how);
	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
1757
	if (trypnfs != PNFS_NOT_ATTEMPTED)
1758 1759 1760 1761 1762
		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}

1763
static void
1764 1765
pnfs_do_write(struct nfs_pageio_descriptor *desc,
	      struct nfs_pgio_header *hdr, int how)
1766 1767 1768
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
1769
	enum pnfs_try_status trypnfs;
1770

1771
	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
1772
	if (trypnfs == PNFS_NOT_ATTEMPTED)
1773
		pnfs_write_through_mds(desc, hdr);
1774 1775
}

1776 1777
static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
1778
	pnfs_put_lseg(hdr->lseg);
1779
	nfs_pgio_header_free(hdr);
1780
}
B
Bryan Schumaker 已提交
1781
EXPORT_SYMBOL_GPL(pnfs_writehdr_free);
1782

1783 1784 1785
int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
{
1786 1787
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

1788
	struct nfs_pgio_header *hdr;
1789 1790
	int ret;

1791 1792
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
1793
		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
1794
		return -ENOMEM;
1795
	}
1796
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
1797

1798
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
1799
	ret = nfs_generic_pgio(desc, hdr);
1800
	if (!ret)
1801
		pnfs_do_write(desc, hdr, desc->pg_ioflags);
1802

1803
	return ret;
1804 1805 1806
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);

1807
int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
1808 1809 1810
{
	struct nfs_pageio_descriptor pgio;

1811
	/* Resend all requests through the MDS */
1812 1813
	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
1814
}
1815
EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
1816

1817
static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
1818
{
1819 1820
	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
1821
	    PNFS_LAYOUTRET_ON_ERROR) {
1822
		pnfs_return_layout(hdr->inode);
1823
	}
1824
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1825
		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
1826 1827
}

1828 1829 1830
/*
 * Called by non rpc-based layout drivers
 */
1831
void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
1832
{
1833
	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
1834
	if (likely(!hdr->pnfs_error)) {
1835 1836
		__nfs4_read_done_cb(hdr);
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
1837
	} else
1838 1839
		pnfs_ld_handle_read_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
1840 1841 1842
}
EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

1843 1844
static void
pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
1845
		struct nfs_pgio_header *hdr)
1846
{
1847 1848 1849
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];


1850
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1851
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
1852
		nfs_pageio_reset_read_mds(desc);
1853
		mirror->pg_recoalesce = 1;
1854
	}
1855
	nfs_pgio_data_destroy(hdr);
1856 1857
}

A
Andy Adamson 已提交
1858 1859 1860
/*
 * Call the appropriate parallel I/O subsystem read function.
 */
1861
static enum pnfs_try_status
1862
pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
1863 1864
		       const struct rpc_call_ops *call_ops,
		       struct pnfs_layout_segment *lseg)
A
Andy Adamson 已提交
1865
{
1866
	struct inode *inode = hdr->inode;
A
Andy Adamson 已提交
1867 1868 1869
	struct nfs_server *nfss = NFS_SERVER(inode);
	enum pnfs_try_status trypnfs;

1870
	hdr->mds_ops = call_ops;
A
Andy Adamson 已提交
1871 1872

	dprintk("%s: Reading ino:%lu %u@%llu\n",
1873
		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
A
Andy Adamson 已提交
1874

1875
	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
1876
	if (trypnfs != PNFS_NOT_ATTEMPTED)
A
Andy Adamson 已提交
1877 1878 1879 1880
		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}
A
Andy Adamson 已提交
1881

1882
static void
1883
pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
1884 1885 1886
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
1887
	enum pnfs_try_status trypnfs;
1888

1889
	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
1890
	if (trypnfs == PNFS_NOT_ATTEMPTED)
1891
		pnfs_read_through_mds(desc, hdr);
1892 1893
}

1894 1895
static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
1896
	pnfs_put_lseg(hdr->lseg);
1897
	nfs_pgio_header_free(hdr);
1898
}
B
Bryan Schumaker 已提交
1899
EXPORT_SYMBOL_GPL(pnfs_readhdr_free);
1900

1901 1902 1903
int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
{
1904 1905
	struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];

1906
	struct nfs_pgio_header *hdr;
1907 1908
	int ret;

1909 1910
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
1911
		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
1912
		return -ENOMEM;
1913
	}
1914
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
1915
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
1916
	ret = nfs_generic_pgio(desc, hdr);
1917
	if (!ret)
1918
		pnfs_do_read(desc, hdr);
1919
	return ret;
1920 1921 1922
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);

1923 1924 1925 1926 1927
static void pnfs_clear_layoutcommitting(struct inode *inode)
{
	unsigned long *bitlock = &NFS_I(inode)->flags;

	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
1928
	smp_mb__after_atomic();
1929 1930 1931
	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}

A
Andy Adamson 已提交
1932
/*
1933
 * There can be multiple RW segments.
A
Andy Adamson 已提交
1934
 */
1935
static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
A
Andy Adamson 已提交
1936
{
1937
	struct pnfs_layout_segment *lseg;
A
Andy Adamson 已提交
1938

1939 1940
	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
		if (lseg->pls_range.iomode == IOMODE_RW &&
1941
		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
1942 1943
			list_add(&lseg->pls_lc_list, listp);
	}
A
Andy Adamson 已提交
1944 1945
}

1946 1947 1948 1949 1950 1951 1952 1953 1954 1955
static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
{
	struct pnfs_layout_segment *lseg, *tmp;

	/* Matched by references in pnfs_set_layoutcommit */
	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
		list_del_init(&lseg->pls_lc_list);
		pnfs_put_lseg(lseg);
	}

1956
	pnfs_clear_layoutcommitting(inode);
1957 1958
}

P
Peng Tao 已提交
1959 1960
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
1961
	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
P
Peng Tao 已提交
1962 1963 1964
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

A
Andy Adamson 已提交
1965
void
1966
pnfs_set_layoutcommit(struct nfs_pgio_header *hdr)
A
Andy Adamson 已提交
1967
{
1968 1969
	struct inode *inode = hdr->inode;
	struct nfs_inode *nfsi = NFS_I(inode);
1970
	loff_t end_pos = hdr->mds_offset + hdr->res.count;
1971
	bool mark_as_dirty = false;
A
Andy Adamson 已提交
1972

1973
	spin_lock(&inode->i_lock);
A
Andy Adamson 已提交
1974
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
1975
		mark_as_dirty = true;
A
Andy Adamson 已提交
1976
		dprintk("%s: Set layoutcommit for inode %lu ",
1977
			__func__, inode->i_ino);
A
Andy Adamson 已提交
1978
	}
1979
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &hdr->lseg->pls_flags)) {
1980
		/* references matched in nfs4_layoutcommit_release */
1981
		pnfs_get_lseg(hdr->lseg);
1982
	}
1983 1984
	if (end_pos > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = end_pos;
1985
	spin_unlock(&inode->i_lock);
1986
	dprintk("%s: lseg %p end_pos %llu\n",
1987
		__func__, hdr->lseg, nfsi->layout->plh_lwb);
1988 1989 1990 1991

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
1992
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
1993 1994 1995
}
EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);

1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024
void pnfs_commit_set_layoutcommit(struct nfs_commit_data *data)
{
	struct inode *inode = data->inode;
	struct nfs_inode *nfsi = NFS_I(inode);
	bool mark_as_dirty = false;

	spin_lock(&inode->i_lock);
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
		mark_as_dirty = true;
		dprintk("%s: Set layoutcommit for inode %lu ",
			__func__, inode->i_ino);
	}
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &data->lseg->pls_flags)) {
		/* references matched in nfs4_layoutcommit_release */
		pnfs_get_lseg(data->lseg);
	}
	if (data->lwb > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = data->lwb;
	spin_unlock(&inode->i_lock);
	dprintk("%s: lseg %p end_pos %llu\n",
		__func__, data->lseg, nfsi->layout->plh_lwb);

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
		mark_inode_dirty_sync(inode);
}
EXPORT_SYMBOL_GPL(pnfs_commit_set_layoutcommit);

A
Andy Adamson 已提交
2025 2026 2027 2028 2029 2030
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
{
	struct nfs_server *nfss = NFS_SERVER(data->args.inode);

	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
2031
	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
A
Andy Adamson 已提交
2032 2033
}

2034 2035 2036 2037 2038 2039 2040 2041
/*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
 * data to disk to allow the server to recover the data if it crashes.
 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
 * is off, and a COMMIT is sent to a data server, or
 * if WRITEs to a data server return NFS_DATA_SYNC.
 */
A
Andy Adamson 已提交
2042
int
2043
pnfs_layoutcommit_inode(struct inode *inode, bool sync)
A
Andy Adamson 已提交
2044
{
2045
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
A
Andy Adamson 已提交
2046 2047 2048
	struct nfs4_layoutcommit_data *data;
	struct nfs_inode *nfsi = NFS_I(inode);
	loff_t end_pos;
2049
	int status;
A
Andy Adamson 已提交
2050

2051
	if (!pnfs_layoutcommit_outstanding(inode))
2052 2053
		return 0;

2054
	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
P
Peng Tao 已提交
2055

2056
	status = -EAGAIN;
P
Peng Tao 已提交
2057
	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2058 2059
		if (!sync)
			goto out;
2060
		status = wait_on_bit_lock_action(&nfsi->flags,
2061 2062 2063
				NFS_INO_LAYOUTCOMMITTING,
				nfs_wait_bit_killable,
				TASK_KILLABLE);
P
Peng Tao 已提交
2064
		if (status)
2065
			goto out;
P
Peng Tao 已提交
2066 2067
	}

2068 2069 2070 2071 2072 2073 2074
	status = -ENOMEM;
	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (!data)
		goto clear_layoutcommitting;

	status = 0;
2075
	spin_lock(&inode->i_lock);
2076 2077
	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		goto out_unlock;
2078

2079
	INIT_LIST_HEAD(&data->lseg_list);
2080
	pnfs_list_write_lseg(inode, &data->lseg_list);
A
Andy Adamson 已提交
2081

2082 2083
	end_pos = nfsi->layout->plh_lwb;
	nfsi->layout->plh_lwb = 0;
A
Andy Adamson 已提交
2084

2085
	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
A
Andy Adamson 已提交
2086 2087 2088
	spin_unlock(&inode->i_lock);

	data->args.inode = inode;
2089
	data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
A
Andy Adamson 已提交
2090 2091 2092 2093 2094 2095
	nfs_fattr_init(&data->fattr);
	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
	data->res.fattr = &data->fattr;
	data->args.lastbytewritten = end_pos - 1;
	data->res.server = NFS_SERVER(inode);

2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
	if (ld->prepare_layoutcommit) {
		status = ld->prepare_layoutcommit(&data->args);
		if (status) {
			spin_lock(&inode->i_lock);
			if (end_pos < nfsi->layout->plh_lwb)
				nfsi->layout->plh_lwb = end_pos;
			spin_unlock(&inode->i_lock);
			put_rpccred(data->cred);
			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
			goto clear_layoutcommitting;
		}
	}


A
Andy Adamson 已提交
2110 2111
	status = nfs4_proc_layoutcommit(data, sync);
out:
P
Peng Tao 已提交
2112 2113
	if (status)
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2114 2115
	dprintk("<-- %s status %d\n", __func__, status);
	return status;
2116 2117
out_unlock:
	spin_unlock(&inode->i_lock);
P
Peng Tao 已提交
2118
	kfree(data);
2119 2120
clear_layoutcommitting:
	pnfs_clear_layoutcommitting(inode);
P
Peng Tao 已提交
2121
	goto out;
A
Andy Adamson 已提交
2122
}
2123
EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135

struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
{
	struct nfs4_threshold *thp;

	thp = kzalloc(sizeof(*thp), GFP_NOFS);
	if (!thp) {
		dprintk("%s mdsthreshold allocation failed\n", __func__);
		return NULL;
	}
	return thp;
}