pnfs.c 52.2 KB
Newer Older
R
Ricardo Labiaga 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 *  pNFS functions to call and manage layout drivers.
 *
 *  Copyright (c) 2002 [year of first publication]
 *  The Regents of the University of Michigan
 *  All Rights Reserved
 *
 *  Dean Hildebrand <dhildebz@umich.edu>
 *
 *  Permission is granted to use, copy, create derivative works, and
 *  redistribute this software and such derivative works for any purpose,
 *  so long as the name of the University of Michigan is not used in
 *  any advertising or publicity pertaining to the use or distribution
 *  of this software without specific, written prior authorization. If
 *  the above copyright notice or any other identification of the
 *  University of Michigan is included in any copy of any portion of
 *  this software, then the disclaimer below must also be included.
 *
 *  This software is provided as is, without representation or warranty
 *  of any kind either express or implied, including without limitation
 *  the implied warranties of merchantability, fitness for a particular
 *  purpose, or noninfringement.  The Regents of the University of
 *  Michigan shall not be liable for any damages, including special,
 *  indirect, incidental, or consequential damages, with respect to any
 *  claim arising out of or in connection with the use of the software,
 *  even if it has been or is hereafter advised of the possibility of
 *  such damages.
 */

#include <linux/nfs_fs.h>
31
#include <linux/nfs_page.h>
32
#include <linux/module.h>
33
#include "internal.h"
R
Ricardo Labiaga 已提交
34
#include "pnfs.h"
A
Andy Adamson 已提交
35
#include "iostat.h"
36
#include "nfs4trace.h"
37
#include "delegation.h"
R
Ricardo Labiaga 已提交
38 39

#define NFSDBG_FACILITY		NFSDBG_PNFS
40
#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
R
Ricardo Labiaga 已提交
41

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
/* Locking:
 *
 * pnfs_spinlock:
 *      protects pnfs_modules_tbl.
 */
static DEFINE_SPINLOCK(pnfs_spinlock);

/*
 * pnfs_modules_tbl holds all pnfs modules
 */
static LIST_HEAD(pnfs_modules_tbl);

/* Return the registered pnfs layout driver module matching given id */
static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)
{
	struct pnfs_layoutdriver_type *local;

	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
		if (local->id == id)
			goto out;
	local = NULL;
out:
	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
	return local;
}

R
Ricardo Labiaga 已提交
69 70 71
static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)
{
72 73 74 75
	struct pnfs_layoutdriver_type *local;

	spin_lock(&pnfs_spinlock);
	local = find_pnfs_driver_locked(id);
76 77 78 79
	if (local != NULL && !try_module_get(local->owner)) {
		dprintk("%s: Could not grab reference on module\n", __func__);
		local = NULL;
	}
80 81
	spin_unlock(&pnfs_spinlock);
	return local;
R
Ricardo Labiaga 已提交
82 83 84 85 86
}

void
unset_pnfs_layoutdriver(struct nfs_server *nfss)
{
87 88 89
	if (nfss->pnfs_curr_ld) {
		if (nfss->pnfs_curr_ld->clear_layoutdriver)
			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
90 91 92
		/* Decrement the MDS count. Purge the deviceid cache if zero */
		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
			nfs4_deviceid_purge_client(nfss->nfs_client);
93
		module_put(nfss->pnfs_curr_ld->owner);
94
	}
R
Ricardo Labiaga 已提交
95 96 97 98 99 100 101 102 103 104
	nfss->pnfs_curr_ld = NULL;
}

/*
 * Try to set the server's pnfs module to the pnfs layout type specified by id.
 * Currently only one pNFS layout driver per filesystem is supported.
 *
 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
 */
void
105 106
set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
		      u32 id)
R
Ricardo Labiaga 已提交
107 108 109 110 111 112 113
{
	struct pnfs_layoutdriver_type *ld_type = NULL;

	if (id == 0)
		goto out_no_driver;
	if (!(server->nfs_client->cl_exchange_flags &
		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
114 115
		printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
			__func__, id, server->nfs_client->cl_exchange_flags);
R
Ricardo Labiaga 已提交
116 117 118 119 120 121 122 123 124 125 126 127 128
		goto out_no_driver;
	}
	ld_type = find_pnfs_driver(id);
	if (!ld_type) {
		request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
		ld_type = find_pnfs_driver(id);
		if (!ld_type) {
			dprintk("%s: No pNFS module found for %u.\n",
				__func__, id);
			goto out_no_driver;
		}
	}
	server->pnfs_curr_ld = ld_type;
129 130
	if (ld_type->set_layoutdriver
	    && ld_type->set_layoutdriver(server, mntfh)) {
131 132
		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
			"driver %u.\n", __func__, id);
133 134 135
		module_put(ld_type->owner);
		goto out_no_driver;
	}
136 137
	/* Bump the MDS count */
	atomic_inc(&server->nfs_client->cl_mds_count);
138

R
Ricardo Labiaga 已提交
139 140 141 142 143 144 145
	dprintk("%s: pNFS module for %u set\n", __func__, id);
	return;

out_no_driver:
	dprintk("%s: Using NFSv4 I/O\n", __func__);
	server->pnfs_curr_ld = NULL;
}
146 147 148 149 150 151 152 153

int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	int status = -EINVAL;
	struct pnfs_layoutdriver_type *tmp;

	if (ld_type->id == 0) {
154
		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
155 156
		return status;
	}
157
	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
158
		printk(KERN_ERR "NFS: %s Layout driver must provide "
159 160 161
		       "alloc_lseg and free_lseg.\n", __func__);
		return status;
	}
162 163 164 165 166 167 168 169 170

	spin_lock(&pnfs_spinlock);
	tmp = find_pnfs_driver_locked(ld_type->id);
	if (!tmp) {
		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
		status = 0;
		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
			ld_type->name);
	} else {
171
		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
			__func__, ld_type->id);
	}
	spin_unlock(&pnfs_spinlock);

	return status;
}
EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);

void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
	spin_lock(&pnfs_spinlock);
	list_del(&ld_type->pnfs_tblid);
	spin_unlock(&pnfs_spinlock);
}
EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
189

190 191 192 193
/*
 * pNFS client layout cache
 */

194
/* Need to hold i_lock if caller does not already hold reference */
F
Fred Isaman 已提交
195
void
196
pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
197
{
198
	atomic_inc(&lo->plh_refcount);
199 200
}

201 202 203 204
static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
205
	return ld->alloc_layout_hdr(ino, gfp_flags);
206 207 208 209 210
}

static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
211 212 213 214 215 216 217 218 219 220
	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;

	if (!list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		spin_lock(&clp->cl_lock);
		list_del_init(&lo->plh_layouts);
		spin_unlock(&clp->cl_lock);
	}
221
	put_rpccred(lo->plh_lc_cred);
222
	return ld->free_layout_hdr(lo);
223 224
}

225
static void
226
pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
227
{
228
	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
229
	dprintk("%s: freeing layout cache %p\n", __func__, lo);
230 231 232 233
	nfsi->layout = NULL;
	/* Reset MDS Threshold I/O counters */
	nfsi->write_io = 0;
	nfsi->read_io = 0;
234 235
}

236
void
237
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
238
{
239 240 241
	struct inode *inode = lo->plh_inode;

	if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
242
		pnfs_detach_layout_hdr(lo);
243
		spin_unlock(&inode->i_lock);
244
		pnfs_free_layout_hdr(lo);
245
	}
246 247
}

248 249 250 251 252 253 254 255
static int
pnfs_iomode_to_fail_bit(u32 iomode)
{
	return iomode == IOMODE_RW ?
		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
}

static void
256
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
257
{
258
	lo->plh_retry_timestamp = jiffies;
259
	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
260 261 262 263 264 265 266 267 268 269 270 271 272 273
		atomic_inc(&lo->plh_refcount);
}

static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
{
	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
		atomic_dec(&lo->plh_refcount);
}

static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
	struct inode *inode = lo->plh_inode;
274 275 276 277 278 279
	struct pnfs_layout_range range = {
		.iomode = iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(head);
280 281 282

	spin_lock(&inode->i_lock);
	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
283
	pnfs_mark_matching_lsegs_invalid(lo, &head, &range);
284
	spin_unlock(&inode->i_lock);
285
	pnfs_free_lseg_list(&head);
286 287 288 289 290 291 292
	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
			iomode == IOMODE_RW ?  "RW" : "READ");
}

static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
293
	unsigned long start, end;
294 295 296
	int fail_bit = pnfs_iomode_to_fail_bit(iomode);

	if (test_bit(fail_bit, &lo->plh_flags) == 0)
297 298 299 300 301
		return false;
	end = jiffies;
	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
		/* It is time to retry the failed layoutgets */
302
		pnfs_layout_clear_fail_bit(lo, fail_bit);
303 304 305
		return false;
	}
	return true;
306 307
}

308 309 310
static void
init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
{
311
	INIT_LIST_HEAD(&lseg->pls_list);
312
	INIT_LIST_HEAD(&lseg->pls_lc_list);
313 314 315
	atomic_set(&lseg->pls_refcount, 1);
	smp_mb();
	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
316
	lseg->pls_layout = lo;
317 318
}

319
static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
320
{
321
	struct inode *ino = lseg->pls_layout->plh_inode;
322

323
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
324 325
}

F
Fred Isaman 已提交
326
static void
327 328
pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
F
Fred Isaman 已提交
329
{
330
	struct inode *inode = lo->plh_inode;
F
Fred Isaman 已提交
331

332
	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
F
Fred Isaman 已提交
333
	list_del_init(&lseg->pls_list);
334 335
	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
	atomic_dec(&lo->plh_refcount);
336 337
	if (list_empty(&lo->plh_segs))
		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
F
Fred Isaman 已提交
338 339 340
	rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}

341
void
342
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
343
{
344
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
345 346 347 348 349
	struct inode *inode;

	if (!lseg)
		return;

350 351 352
	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
353 354
	lo = lseg->pls_layout;
	inode = lo->plh_inode;
F
Fred Isaman 已提交
355
	if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
356
		pnfs_get_layout_hdr(lo);
357
		pnfs_layout_remove_lseg(lo, lseg);
F
Fred Isaman 已提交
358
		spin_unlock(&inode->i_lock);
359
		pnfs_free_lseg(lseg);
360
		pnfs_put_layout_hdr(lo);
361 362
	}
}
363
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
364

365
static void pnfs_free_lseg_async_work(struct work_struct *work)
366 367
{
	struct pnfs_layout_segment *lseg;
368
	struct pnfs_layout_hdr *lo;
369 370

	lseg = container_of(work, struct pnfs_layout_segment, pls_work);
371
	lo = lseg->pls_layout;
372

373 374
	pnfs_free_lseg(lseg);
	pnfs_put_layout_hdr(lo);
375 376
}

377
static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg)
378
{
379
	INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work);
380 381
	schedule_work(&lseg->pls_work);
}
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401

void
pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
{
	if (!lseg)
		return;

	assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock);

	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
	if (atomic_dec_and_test(&lseg->pls_refcount)) {
		struct pnfs_layout_hdr *lo = lseg->pls_layout;
		pnfs_get_layout_hdr(lo);
		pnfs_layout_remove_lseg(lo, lseg);
		pnfs_free_lseg_async(lseg);
	}
}
EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
402

403
static u64
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
end_offset(u64 start, u64 len)
{
	u64 end;

	end = start + len;
	return end >= start ? end : NFS4_MAX_UINT64;
}

/*
 * is l2 fully contained in l1?
 *   start1                             end1
 *   [----------------------------------)
 *           start2           end2
 *           [----------------)
 */
419
static bool
420
pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
421
		 const struct pnfs_layout_range *l2)
422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (start1 <= start2) && (end1 >= end2);
}

/*
 * is l1 and l2 intersecting?
 *   start1                             end1
 *   [----------------------------------)
 *                              start2           end2
 *                              [----------------)
 */
438
static bool
439
pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
440
		    const struct pnfs_layout_range *l2)
441 442 443 444 445 446 447 448 449 450
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
	       (end2 == NFS4_MAX_UINT64 || end2 > start1);
}

451
static bool
452 453
should_free_lseg(const struct pnfs_layout_range *lseg_range,
		 const struct pnfs_layout_range *recall_range)
454
{
455 456
	return (recall_range->iomode == IOMODE_ANY ||
		lseg_range->iomode == recall_range->iomode) &&
457
	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
458 459
}

460 461 462 463 464 465 466 467 468 469
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		struct list_head *tmp_list)
{
	if (!atomic_dec_and_test(&lseg->pls_refcount))
		return false;
	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
	list_add(&lseg->pls_list, tmp_list);
	return true;
}

470 471 472 473 474 475 476 477 478 479 480
/* Returns 1 if lseg is removed from list, 0 otherwise */
static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
			     struct list_head *tmp_list)
{
	int rv = 0;

	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
		/* Remove the reference keeping the lseg in the
		 * list.  It will now be removed when all
		 * outstanding io is finished.
		 */
F
Fred Isaman 已提交
481 482
		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
			atomic_read(&lseg->pls_refcount));
483
		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
F
Fred Isaman 已提交
484
			rv = 1;
485 486 487 488 489 490 491
	}
	return rv;
}

/* Returns count of number of matching invalid lsegs remaining in list
 * after call.
 */
F
Fred Isaman 已提交
492
int
493
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
494
			    struct list_head *tmp_list,
495
			    struct pnfs_layout_range *recall_range)
496 497
{
	struct pnfs_layout_segment *lseg, *next;
498
	int invalid = 0, removed = 0;
499 500 501

	dprintk("%s:Begin lo %p\n", __func__, lo);

502
	if (list_empty(&lo->plh_segs))
503
		return 0;
504
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
505 506
		if (!recall_range ||
		    should_free_lseg(&lseg->pls_range, recall_range)) {
507 508 509 510 511 512 513 514 515
			dprintk("%s: freeing lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode, lseg->pls_range.offset,
				lseg->pls_range.length);
			invalid++;
			removed += mark_lseg_invalid(lseg, tmp_list);
		}
	dprintk("%s:Return %i\n", __func__, invalid - removed);
	return invalid - removed;
516 517
}

518
/* note free_me must contain lsegs from a single layout_hdr */
F
Fred Isaman 已提交
519
void
520
pnfs_free_lseg_list(struct list_head *free_me)
521
{
522
	struct pnfs_layout_segment *lseg, *tmp;
523 524 525 526

	if (list_empty(free_me))
		return;

527
	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
528
		list_del(&lseg->pls_list);
529
		pnfs_free_lseg(lseg);
530 531 532
	}
}

533 534 535 536
void
pnfs_destroy_layout(struct nfs_inode *nfsi)
{
	struct pnfs_layout_hdr *lo;
537
	LIST_HEAD(tmp_list);
538 539 540 541

	spin_lock(&nfsi->vfs_inode.i_lock);
	lo = nfsi->layout;
	if (lo) {
542
		lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
543
		pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
544 545 546 547 548 549 550 551
		pnfs_get_layout_hdr(lo);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
		spin_unlock(&nfsi->vfs_inode.i_lock);
		pnfs_free_lseg_list(&tmp_list);
		pnfs_put_layout_hdr(lo);
	} else
		spin_unlock(&nfsi->vfs_inode.i_lock);
552
}
553
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
554

555 556 557
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
		struct list_head *layout_list)
558 559
{
	struct pnfs_layout_hdr *lo;
560
	bool ret = false;
561

562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
		pnfs_get_layout_hdr(lo);
		list_add(&lo->plh_bulk_destroy, layout_list);
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

/* Caller must hold rcu_read_lock and clp->cl_lock */
static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
		struct nfs_server *server,
		struct list_head *layout_list)
{
	struct pnfs_layout_hdr *lo, *next;
	struct inode *inode;

	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
		inode = igrab(lo->plh_inode);
		if (inode == NULL)
			continue;
		list_del_init(&lo->plh_layouts);
		if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
			continue;
		rcu_read_unlock();
		spin_unlock(&clp->cl_lock);
		iput(inode);
		spin_lock(&clp->cl_lock);
		rcu_read_lock();
		return -EAGAIN;
	}
	return 0;
}

static int
pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
		bool is_bulk_recall)
{
	struct pnfs_layout_hdr *lo;
	struct inode *inode;
	struct pnfs_layout_range range = {
		.iomode = IOMODE_ANY,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(lseg_list);
	int ret = 0;

	while (!list_empty(layout_list)) {
		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
				plh_bulk_destroy);
		dprintk("%s freeing layout for inode %lu\n", __func__,
			lo->plh_inode->i_ino);
		inode = lo->plh_inode;
619 620 621

		pnfs_layoutcommit_inode(inode, false);

622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
		spin_lock(&inode->i_lock);
		list_del_init(&lo->plh_bulk_destroy);
		lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
		if (is_bulk_recall)
			set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
		if (pnfs_mark_matching_lsegs_invalid(lo, &lseg_list, &range))
			ret = -EAGAIN;
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg_list(&lseg_list);
		pnfs_put_layout_hdr(lo);
		iput(inode);
	}
	return ret;
}

int
pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
		struct nfs_fsid *fsid,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);
644

645
	spin_lock(&clp->cl_lock);
646
	rcu_read_lock();
647
restart:
648
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
649 650 651 652 653 654
		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
			continue;
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
				server,
				&layout_list) != 0)
			goto restart;
655 656
	}
	rcu_read_unlock();
657 658
	spin_unlock(&clp->cl_lock);

659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

int
pnfs_destroy_layouts_byclid(struct nfs_client *clp,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);

	spin_lock(&clp->cl_lock);
	rcu_read_lock();
restart:
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
					server,
					&layout_list) != 0)
			goto restart;
679
	}
680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698
	rcu_read_unlock();
	spin_unlock(&clp->cl_lock);

	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

/*
 * Called by the state manger to remove all layouts established under an
 * expired lease.
 */
void
pnfs_destroy_all_layouts(struct nfs_client *clp)
{
	nfs4_deviceid_mark_client_invalid(clp);
	nfs4_deviceid_purge_client(clp);

	pnfs_destroy_layouts_byclid(clp, false);
699 700
}

701 702 703 704 705 706
/*
 * Compare 2 layout stateid sequence ids, to see which is newer,
 * taking into account wraparound issues.
 */
static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
{
707
	return (s32)(s1 - s2) > 0;
708 709
}

710
/* update lo->plh_stateid with new if is more recent */
F
Fred Isaman 已提交
711 712 713
void
pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
			bool update_barrier)
714
{
715 716
	u32 oldseq, newseq, new_barrier;
	int empty = list_empty(&lo->plh_segs);
717

718 719
	oldseq = be32_to_cpu(lo->plh_stateid.seqid);
	newseq = be32_to_cpu(new->seqid);
720
	if (empty || pnfs_seqid_is_newer(newseq, oldseq)) {
721
		nfs4_stateid_copy(&lo->plh_stateid, new);
F
Fred Isaman 已提交
722
		if (update_barrier) {
723
			new_barrier = be32_to_cpu(new->seqid);
F
Fred Isaman 已提交
724 725
		} else {
			/* Because of wraparound, we want to keep the barrier
726
			 * "close" to the current seqids.
F
Fred Isaman 已提交
727
			 */
728
			new_barrier = newseq - atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
729
		}
730 731
		if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
			lo->plh_barrier = new_barrier;
F
Fred Isaman 已提交
732
	}
733 734
}

735
static bool
736 737
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid)
F
Fred Isaman 已提交
738
{
739
	u32 seqid = be32_to_cpu(stateid->seqid);
740

741 742 743 744 745 746 747
	return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
}

/* lget is set to 1 if called from inside send_layoutget call chain */
static bool
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo, int lget)
{
F
Fred Isaman 已提交
748 749
	return lo->plh_block_lgets ||
		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags) ||
F
Fred Isaman 已提交
750
		(list_empty(&lo->plh_segs) &&
751 752 753
		 (atomic_read(&lo->plh_outstanding) > lget));
}

754 755 756
int
pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
			      struct nfs4_state *open_state)
757
{
758
	int status = 0;
759

760
	dprintk("--> %s\n", __func__);
761
	spin_lock(&lo->plh_inode->i_lock);
762
	if (pnfs_layoutgets_blocked(lo, 1)) {
763
		status = -EAGAIN;
764 765
	} else if (!nfs4_valid_open_stateid(open_state)) {
		status = -EBADF;
766 767
	} else if (list_empty(&lo->plh_segs) ||
		   test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
768 769 770 771
		int seq;

		do {
			seq = read_seqbegin(&open_state->seqlock);
772
			nfs4_stateid_copy(dst, &open_state->stateid);
773 774
		} while (read_seqretry(&open_state->seqlock, seq));
	} else
775
		nfs4_stateid_copy(dst, &lo->plh_stateid);
776
	spin_unlock(&lo->plh_inode->i_lock);
777
	dprintk("<-- %s\n", __func__);
778
	return status;
779 780 781 782 783 784 785 786
}

/*
* Get layout from server.
*    for now, assume that whole file layouts are requested.
*    arg->offset: 0
*    arg->length: all ones
*/
787 788 789
static struct pnfs_layout_segment *
send_layoutget(struct pnfs_layout_hdr *lo,
	   struct nfs_open_context *ctx,
790
	   struct pnfs_layout_range *range,
791
	   gfp_t gfp_flags)
792
{
793
	struct inode *ino = lo->plh_inode;
794 795
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs4_layoutget *lgp;
796
	struct pnfs_layout_segment *lseg;
797 798

	dprintk("--> %s\n", __func__);
799

800
	lgp = kzalloc(sizeof(*lgp), gfp_flags);
801
	if (lgp == NULL)
802
		return NULL;
803

804 805 806
	lgp->args.minlength = PAGE_CACHE_SIZE;
	if (lgp->args.minlength > range->length)
		lgp->args.minlength = range->length;
807
	lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
808
	lgp->args.range = *range;
809 810 811
	lgp->args.type = server->pnfs_curr_ld->id;
	lgp->args.inode = ino;
	lgp->args.ctx = get_nfs_open_context(ctx);
812
	lgp->gfp_flags = gfp_flags;
813
	lgp->cred = lo->plh_lc_cred;
814 815 816 817

	/* Synchronously retrieve layout information from server and
	 * store in lseg.
	 */
818 819 820 821 822 823 824 825
	lseg = nfs4_proc_layoutget(lgp, gfp_flags);
	if (IS_ERR(lseg)) {
		switch (PTR_ERR(lseg)) {
		case -ENOMEM:
		case -ERESTARTSYS:
			break;
		default:
			/* remember that LAYOUTGET failed and suspend trying */
826
			pnfs_layout_io_set_failed(lo, range->iomode);
827 828
		}
		return NULL;
829
	}
830

831 832 833
	return lseg;
}

834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
static void pnfs_clear_layoutcommit(struct inode *inode,
		struct list_head *head)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct pnfs_layout_segment *lseg, *tmp;

	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		return;
	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
			continue;
		pnfs_lseg_dec_and_remove_zero(lseg, head);
	}
}

849 850 851 852 853 854 855 856
/*
 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
 * when the layout segment list is empty.
 *
 * Note that a pnfs_layout_hdr can exist with an empty layout segment
 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
 * deviceid is marked invalid.
 */
B
Benny Halevy 已提交
857 858 859 860 861 862 863 864
int
_pnfs_return_layout(struct inode *ino)
{
	struct pnfs_layout_hdr *lo = NULL;
	struct nfs_inode *nfsi = NFS_I(ino);
	LIST_HEAD(tmp_list);
	struct nfs4_layoutreturn *lrp;
	nfs4_stateid stateid;
865
	int status = 0, empty;
B
Benny Halevy 已提交
866

867
	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
B
Benny Halevy 已提交
868 869 870

	spin_lock(&ino->i_lock);
	lo = nfsi->layout;
871
	if (!lo) {
B
Benny Halevy 已提交
872
		spin_unlock(&ino->i_lock);
873 874
		dprintk("NFS: %s no layout to return\n", __func__);
		goto out;
B
Benny Halevy 已提交
875 876 877
	}
	stateid = nfsi->layout->plh_stateid;
	/* Reference matched in nfs4_layoutreturn_release */
878
	pnfs_get_layout_hdr(lo);
879
	empty = list_empty(&lo->plh_segs);
880
	pnfs_clear_layoutcommit(ino, &tmp_list);
881
	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
882 883 884 885 886 887 888 889 890 891

	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
		struct pnfs_layout_range range = {
			.iomode		= IOMODE_ANY,
			.offset		= 0,
			.length		= NFS4_MAX_UINT64,
		};
		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
	}

892 893 894
	/* Don't send a LAYOUTRETURN if list was initially empty */
	if (empty) {
		spin_unlock(&ino->i_lock);
895
		pnfs_put_layout_hdr(lo);
896 897 898
		dprintk("NFS: %s no layout segments to return\n", __func__);
		goto out;
	}
899 900

	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
901
	lo->plh_block_lgets++;
B
Benny Halevy 已提交
902 903 904 905 906 907
	spin_unlock(&ino->i_lock);
	pnfs_free_lseg_list(&tmp_list);

	lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
	if (unlikely(lrp == NULL)) {
		status = -ENOMEM;
908 909 910
		spin_lock(&ino->i_lock);
		lo->plh_block_lgets--;
		spin_unlock(&ino->i_lock);
911
		pnfs_put_layout_hdr(lo);
B
Benny Halevy 已提交
912 913 914 915 916 917
		goto out;
	}

	lrp->args.stateid = stateid;
	lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
	lrp->args.inode = ino;
T
Trond Myklebust 已提交
918
	lrp->args.layout = lo;
B
Benny Halevy 已提交
919
	lrp->clp = NFS_SERVER(ino)->nfs_client;
920
	lrp->cred = lo->plh_lc_cred;
B
Benny Halevy 已提交
921 922 923 924 925 926

	status = nfs4_proc_layoutreturn(lrp);
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}
927
EXPORT_SYMBOL_GPL(_pnfs_return_layout);
B
Benny Halevy 已提交
928

929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
int
pnfs_commit_and_return_layout(struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	int ret;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo == NULL) {
		spin_unlock(&inode->i_lock);
		return 0;
	}
	pnfs_get_layout_hdr(lo);
	/* Block new layoutgets and read/write to ds */
	lo->plh_block_lgets++;
	spin_unlock(&inode->i_lock);
	filemap_fdatawait(inode->i_mapping);
	ret = pnfs_layoutcommit_inode(inode, true);
	if (ret == 0)
		ret = _pnfs_return_layout(inode);
	spin_lock(&inode->i_lock);
	lo->plh_block_lgets--;
	spin_unlock(&inode->i_lock);
	pnfs_put_layout_hdr(lo);
	return ret;
}

F
Fred Isaman 已提交
956 957
bool pnfs_roc(struct inode *ino)
{
958 959 960
	struct nfs_inode *nfsi = NFS_I(ino);
	struct nfs_open_context *ctx;
	struct nfs4_state *state;
F
Fred Isaman 已提交
961 962 963 964 965 966
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg, *tmp;
	LIST_HEAD(tmp_list);
	bool found = false;

	spin_lock(&ino->i_lock);
967
	lo = nfsi->layout;
F
Fred Isaman 已提交
968 969
	if (!lo || !test_and_clear_bit(NFS_LAYOUT_ROC, &lo->plh_flags) ||
	    test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
970 971 972 973 974 975 976 977 978 979 980 981 982
		goto out_noroc;

	/* Don't return layout if we hold a delegation */
	if (nfs4_check_delegation(ino, FMODE_READ))
		goto out_noroc;

	list_for_each_entry(ctx, &nfsi->open_files, list) {
		state = ctx->state;
		/* Don't return layout if there is open file state */
		if (state != NULL && state->state != 0)
			goto out_noroc;
	}

F
Fred Isaman 已提交
983 984 985 986 987 988
	list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
		if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
			mark_lseg_invalid(lseg, &tmp_list);
			found = true;
		}
	if (!found)
989
		goto out_noroc;
F
Fred Isaman 已提交
990
	lo->plh_block_lgets++;
991
	pnfs_get_layout_hdr(lo); /* matched in pnfs_roc_release */
F
Fred Isaman 已提交
992 993 994 995
	spin_unlock(&ino->i_lock);
	pnfs_free_lseg_list(&tmp_list);
	return true;

996
out_noroc:
F
Fred Isaman 已提交
997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
	spin_unlock(&ino->i_lock);
	return false;
}

void pnfs_roc_release(struct inode *ino)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
	lo->plh_block_lgets--;
1008 1009 1010 1011 1012 1013
	if (atomic_dec_and_test(&lo->plh_refcount)) {
		pnfs_detach_layout_hdr(lo);
		spin_unlock(&ino->i_lock);
		pnfs_free_layout_hdr(lo);
	} else
		spin_unlock(&ino->i_lock);
F
Fred Isaman 已提交
1014 1015 1016 1017 1018 1019 1020 1021
}

void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1022
	if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
F
Fred Isaman 已提交
1023 1024 1025 1026
		lo->plh_barrier = barrier;
	spin_unlock(&ino->i_lock);
}

1027
bool pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task)
F
Fred Isaman 已提交
1028 1029
{
	struct nfs_inode *nfsi = NFS_I(ino);
1030
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
1031
	struct pnfs_layout_segment *lseg;
1032
	u32 current_seqid;
F
Fred Isaman 已提交
1033 1034 1035 1036 1037
	bool found = false;

	spin_lock(&ino->i_lock);
	list_for_each_entry(lseg, &nfsi->layout->plh_segs, pls_list)
		if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
1038
			rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
F
Fred Isaman 已提交
1039
			found = true;
1040
			goto out;
F
Fred Isaman 已提交
1041
		}
1042 1043
	lo = nfsi->layout;
	current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
F
Fred Isaman 已提交
1044

1045 1046 1047 1048 1049
	/* Since close does not return a layout stateid for use as
	 * a barrier, we choose the worst-case barrier.
	 */
	*barrier = current_seqid + atomic_read(&lo->plh_outstanding);
out:
F
Fred Isaman 已提交
1050 1051 1052 1053
	spin_unlock(&ino->i_lock);
	return found;
}

1054 1055 1056 1057 1058 1059
/*
 * Compare two layout segments for sorting into layout cache.
 * We want to preferentially return RW over RO layouts, so ensure those
 * are seen first.
 */
static s64
1060
pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1061
	   const struct pnfs_layout_range *l2)
1062
{
1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
	s64 d;

	/* high offset > low offset */
	d = l1->offset - l2->offset;
	if (d)
		return d;

	/* short length > long length */
	d = l2->length - l1->length;
	if (d)
		return d;

1075
	/* read > read/write */
1076
	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1077 1078
}

1079
static void
1080
pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
1081 1082
		   struct pnfs_layout_segment *lseg)
{
1083 1084
	struct pnfs_layout_segment *lp;

1085 1086
	dprintk("%s:Begin\n", __func__);

1087
	list_for_each_entry(lp, &lo->plh_segs, pls_list) {
1088
		if (pnfs_lseg_range_cmp(&lseg->pls_range, &lp->pls_range) > 0)
1089
			continue;
1090
		list_add_tail(&lseg->pls_list, &lp->pls_list);
1091 1092 1093
		dprintk("%s: inserted lseg %p "
			"iomode %d offset %llu length %llu before "
			"lp %p iomode %d offset %llu length %llu\n",
1094 1095 1096 1097
			__func__, lseg, lseg->pls_range.iomode,
			lseg->pls_range.offset, lseg->pls_range.length,
			lp, lp->pls_range.iomode, lp->pls_range.offset,
			lp->pls_range.length);
1098
		goto out;
1099
	}
1100 1101 1102 1103 1104 1105
	list_add_tail(&lseg->pls_list, &lo->plh_segs);
	dprintk("%s: inserted lseg %p "
		"iomode %d offset %llu length %llu at tail\n",
		__func__, lseg, lseg->pls_range.iomode,
		lseg->pls_range.offset, lseg->pls_range.length);
out:
1106
	pnfs_get_layout_hdr(lo);
1107 1108

	dprintk("%s:Return\n", __func__);
1109 1110 1111
}

static struct pnfs_layout_hdr *
1112 1113 1114
alloc_init_layout_hdr(struct inode *ino,
		      struct nfs_open_context *ctx,
		      gfp_t gfp_flags)
1115 1116 1117
{
	struct pnfs_layout_hdr *lo;

1118
	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1119 1120
	if (!lo)
		return NULL;
1121
	atomic_set(&lo->plh_refcount, 1);
1122 1123
	INIT_LIST_HEAD(&lo->plh_layouts);
	INIT_LIST_HEAD(&lo->plh_segs);
1124
	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1125
	lo->plh_inode = ino;
1126
	lo->plh_lc_cred = get_rpccred(ctx->cred);
1127 1128 1129 1130
	return lo;
}

static struct pnfs_layout_hdr *
1131 1132 1133
pnfs_find_alloc_layout(struct inode *ino,
		       struct nfs_open_context *ctx,
		       gfp_t gfp_flags)
1134 1135 1136 1137 1138 1139
{
	struct nfs_inode *nfsi = NFS_I(ino);
	struct pnfs_layout_hdr *new = NULL;

	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

1140 1141
	if (nfsi->layout != NULL)
		goto out_existing;
1142
	spin_unlock(&ino->i_lock);
1143
	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1144 1145
	spin_lock(&ino->i_lock);

1146
	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1147
		nfsi->layout = new;
1148
		return new;
1149 1150
	} else if (new != NULL)
		pnfs_free_layout_hdr(new);
1151 1152
out_existing:
	pnfs_get_layout_hdr(nfsi->layout);
1153 1154 1155
	return nfsi->layout;
}

1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
/*
 * iomode matching rules:
 * iomode	lseg	match
 * -----	-----	-----
 * ANY		READ	true
 * ANY		RW	true
 * RW		READ	false
 * RW		RW	true
 * READ		READ	true
 * READ		RW	true
 */
1167
static bool
1168
pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1169
		 const struct pnfs_layout_range *range)
1170
{
1171 1172 1173 1174
	struct pnfs_layout_range range1;

	if ((range->iomode == IOMODE_RW &&
	     ls_range->iomode != IOMODE_RW) ||
1175
	    !pnfs_lseg_range_intersecting(ls_range, range))
1176 1177 1178 1179 1180
		return 0;

	/* range1 covers only the first byte in the range */
	range1 = *range;
	range1.length = 1;
1181
	return pnfs_lseg_range_contained(ls_range, &range1);
1182 1183 1184 1185 1186
}

/*
 * lookup range in layout
 */
1187
static struct pnfs_layout_segment *
1188 1189
pnfs_find_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_range *range)
1190
{
1191 1192 1193 1194
	struct pnfs_layout_segment *lseg, *ret = NULL;

	dprintk("%s:Begin\n", __func__);

1195
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1196
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1197
		    pnfs_lseg_range_match(&lseg->pls_range, range)) {
1198
			ret = pnfs_get_lseg(lseg);
1199 1200
			break;
		}
1201
		if (lseg->pls_range.offset > range->offset)
1202 1203 1204 1205
			break;
	}

	dprintk("%s:Return lseg %p ref %d\n",
1206
		__func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
1207
	return ret;
1208 1209
}

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
/*
 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
 * to the MDS or over pNFS
 *
 * The nfs_inode read_io and write_io fields are cumulative counters reset
 * when there are no layout segments. Note that in pnfs_update_layout iomode
 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
 * WRITE request.
 *
 * A return of true means use MDS I/O.
 *
 * From rfc 5661:
 * If a file's size is smaller than the file size threshold, data accesses
 * SHOULD be sent to the metadata server.  If an I/O request has a length that
 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
 * server.  If both file size and I/O size are provided, the client SHOULD
 * reach or exceed  both thresholds before sending its read or write
 * requests to the data server.
 */
static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
				     struct inode *ino, int iomode)
{
	struct nfs4_threshold *t = ctx->mdsthreshold;
	struct nfs_inode *nfsi = NFS_I(ino);
	loff_t fsize = i_size_read(ino);
	bool size = false, size_set = false, io = false, io_set = false, ret = false;

	if (t == NULL)
		return ret;

	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);

	switch (iomode) {
	case IOMODE_READ:
		if (t->bm & THRESHOLD_RD) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->rd_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_RD_IO) {
			dprintk("%s nfsi->read_io %llu\n", __func__,
				nfsi->read_io);
			io_set = true;
			if (nfsi->read_io < t->rd_io_sz)
				io = true;
		}
		break;
	case IOMODE_RW:
		if (t->bm & THRESHOLD_WR) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->wr_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_WR_IO) {
			dprintk("%s nfsi->write_io %llu\n", __func__,
				nfsi->write_io);
			io_set = true;
			if (nfsi->write_io < t->wr_io_sz)
				io = true;
		}
		break;
	}
	if (size_set && io_set) {
		if (size && io)
			ret = true;
	} else if (size || io)
		ret = true;

	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
	return ret;
}

1285 1286 1287 1288
/*
 * Layout segment is retreived from the server if not cached.
 * The appropriate layout segment is referenced and returned to the caller.
 */
1289
struct pnfs_layout_segment *
1290 1291
pnfs_update_layout(struct inode *ino,
		   struct nfs_open_context *ctx,
1292 1293
		   loff_t pos,
		   u64 count,
1294 1295
		   enum pnfs_iomode iomode,
		   gfp_t gfp_flags)
1296
{
1297 1298 1299 1300 1301
	struct pnfs_layout_range arg = {
		.iomode = iomode,
		.offset = pos,
		.length = count,
	};
1302
	unsigned pg_offset;
1303 1304
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs_client *clp = server->nfs_client;
1305 1306
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg = NULL;
1307
	bool first;
1308 1309

	if (!pnfs_enabled_sb(NFS_SERVER(ino)))
1310
		goto out;
1311 1312

	if (pnfs_within_mdsthreshold(ctx, ino, iomode))
1313
		goto out;
1314

1315
	spin_lock(&ino->i_lock);
1316
	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
1317 1318 1319 1320
	if (lo == NULL) {
		spin_unlock(&ino->i_lock);
		goto out;
	}
1321

F
Fred Isaman 已提交
1322
	/* Do we even need to bother with this? */
1323
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
F
Fred Isaman 已提交
1324
		dprintk("%s matches recall, use MDS\n", __func__);
1325 1326 1327 1328
		goto out_unlock;
	}

	/* if LAYOUTGET already failed once we don't try again */
1329
	if (pnfs_layout_io_test_failed(lo, iomode))
1330 1331
		goto out_unlock;

1332
	/* Check to see if the layout for the given range already exists */
1333
	lseg = pnfs_find_lseg(lo, &arg);
1334 1335 1336
	if (lseg)
		goto out_unlock;

1337
	if (pnfs_layoutgets_blocked(lo, 0))
1338 1339 1340
		goto out_unlock;
	atomic_inc(&lo->plh_outstanding);

1341
	first = list_empty(&lo->plh_layouts) ? true : false;
1342
	spin_unlock(&ino->i_lock);
1343

1344
	if (first) {
1345 1346 1347 1348
		/* The lo must be on the clp list if there is any
		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
		 */
		spin_lock(&clp->cl_lock);
1349
		list_add_tail(&lo->plh_layouts, &server->layouts);
1350 1351
		spin_unlock(&clp->cl_lock);
	}
1352

1353 1354 1355 1356 1357
	pg_offset = arg.offset & ~PAGE_CACHE_MASK;
	if (pg_offset) {
		arg.offset -= pg_offset;
		arg.length += pg_offset;
	}
1358 1359
	if (arg.length != NFS4_MAX_UINT64)
		arg.length = PAGE_CACHE_ALIGN(arg.length);
1360

1361
	lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
1362
	atomic_dec(&lo->plh_outstanding);
1363
out_put_layout_hdr:
1364
	pnfs_put_layout_hdr(lo);
1365
out:
1366 1367 1368 1369 1370 1371 1372 1373
	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
			"(%s, offset: %llu, length: %llu)\n",
			__func__, ino->i_sb->s_id,
			(unsigned long long)NFS_FILEID(ino),
			lseg == NULL ? "not found" : "found",
			iomode==IOMODE_RW ?  "read/write" : "read-only",
			(unsigned long long)pos,
			(unsigned long long)count);
1374 1375 1376
	return lseg;
out_unlock:
	spin_unlock(&ino->i_lock);
1377
	goto out_put_layout_hdr;
1378
}
1379
EXPORT_SYMBOL_GPL(pnfs_update_layout);
1380

1381
struct pnfs_layout_segment *
1382 1383 1384 1385 1386
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
	struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
	struct nfs4_layoutget_res *res = &lgp->res;
	struct pnfs_layout_segment *lseg;
1387
	struct inode *ino = lo->plh_inode;
1388
	LIST_HEAD(free_me);
1389 1390 1391
	int status = 0;

	/* Inject layout blob into I/O device driver */
1392
	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402
	if (!lseg || IS_ERR(lseg)) {
		if (!lseg)
			status = -ENOMEM;
		else
			status = PTR_ERR(lseg);
		dprintk("%s: Could not allocate layout: error %d\n",
		       __func__, status);
		goto out;
	}

1403 1404 1405
	init_lseg(lo, lseg);
	lseg->pls_range = res->range;

1406
	spin_lock(&ino->i_lock);
1407
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
F
Fred Isaman 已提交
1408 1409 1410 1411
		dprintk("%s forget reply due to recall\n", __func__);
		goto out_forget_reply;
	}

1412
	if (pnfs_layoutgets_blocked(lo, 1)) {
F
Fred Isaman 已提交
1413 1414 1415
		dprintk("%s forget reply due to state\n", __func__);
		goto out_forget_reply;
	}
1416

1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434
	if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
		/* existing state ID, make sure the sequence number matches. */
		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
			dprintk("%s forget reply due to sequence\n", __func__);
			goto out_forget_reply;
		}
		pnfs_set_layout_stateid(lo, &res->stateid, false);
	} else {
		/*
		 * We got an entirely new state ID.  Mark all segments for the
		 * inode invalid, and don't bother validating the stateid
		 * sequence number.
		 */
		pnfs_mark_matching_lsegs_invalid(lo, &free_me, NULL);

		nfs4_stateid_copy(&lo->plh_stateid, &res->stateid);
		lo->plh_barrier = be32_to_cpu(res->stateid.seqid);
	}
1435

1436 1437
	clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);

1438
	pnfs_get_lseg(lseg);
1439
	pnfs_layout_insert_lseg(lo, lseg);
1440

F
Fred Isaman 已提交
1441 1442 1443 1444 1445
	if (res->return_on_close) {
		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
		set_bit(NFS_LAYOUT_ROC, &lo->plh_flags);
	}

1446
	spin_unlock(&ino->i_lock);
1447
	pnfs_free_lseg_list(&free_me);
1448
	return lseg;
1449
out:
1450
	return ERR_PTR(status);
F
Fred Isaman 已提交
1451 1452 1453 1454 1455 1456

out_forget_reply:
	spin_unlock(&ino->i_lock);
	lseg->pls_layout = lo;
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
	goto out;
1457 1458
}

1459 1460 1461
void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
P
Peng Tao 已提交
1462 1463
	u64 rd_size = req->wb_bytes;

1464
	WARN_ON_ONCE(pgio->pg_lseg != NULL);
1465

P
Peng Tao 已提交
1466 1467 1468 1469 1470
	if (pgio->pg_dreq == NULL)
		rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
	else
		rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);

1471 1472 1473
	pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
					   req->wb_context,
					   req_offset(req),
P
Peng Tao 已提交
1474
					   rd_size,
1475 1476
					   IOMODE_READ,
					   GFP_KERNEL);
1477 1478
	/* If no lseg, fall back to read through mds */
	if (pgio->pg_lseg == NULL)
1479
		nfs_pageio_reset_read_mds(pgio);
1480

1481 1482 1483 1484
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);

void
1485 1486
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
			   struct nfs_page *req, u64 wb_size)
1487
{
1488
	WARN_ON_ONCE(pgio->pg_lseg != NULL);
1489 1490 1491 1492

	pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
					   req->wb_context,
					   req_offset(req),
1493
					   wb_size,
1494 1495
					   IOMODE_RW,
					   GFP_NOFS);
1496 1497
	/* If no lseg, fall back to write through mds */
	if (pgio->pg_lseg == NULL)
1498
		nfs_pageio_reset_write_mds(pgio);
1499 1500 1501
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);

1502 1503 1504 1505 1506
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
size_t
1507 1508
pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
		     struct nfs_page *req)
1509
{
1510
	unsigned int size;
1511
	u64 seg_end, req_start, seg_left;
1512 1513 1514 1515

	size = nfs_generic_pg_test(pgio, prev, req);
	if (!size)
		return 0;
1516

1517
	/*
1518 1519 1520 1521 1522
	 * 'size' contains the number of bytes left in the current page (up
	 * to the original size asked for in @req->wb_bytes).
	 *
	 * Calculate how many bytes are left in the layout segment
	 * and if there are less bytes than 'size', return that instead.
1523 1524 1525 1526 1527
	 *
	 * Please also note that 'end_offset' is actually the offset of the
	 * first byte that lies outside the pnfs_layout_range. FIXME?
	 *
	 */
1528
	if (pgio->pg_lseg) {
1529 1530 1531 1532 1533 1534
		seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
				     pgio->pg_lseg->pls_range.length);
		req_start = req_offset(req);
		WARN_ON_ONCE(req_start > seg_end);
		/* start of request is past the last byte of this segment */
		if (req_start >= seg_end)
1535
			return 0;
1536 1537 1538 1539 1540 1541

		/* adjust 'size' iff there are fewer bytes left in the
		 * segment than what nfs_generic_pg_test returned */
		seg_left = seg_end - req_start;
		if (seg_left < size)
			size = (unsigned int)seg_left;
1542
	}
1543

1544
	return size;
1545
}
1546
EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
1547

1548
int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
1549 1550 1551 1552
{
	struct nfs_pageio_descriptor pgio;

	/* Resend all requests through the MDS */
1553 1554 1555
	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
			      hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
1556
}
1557
EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
1558

1559
static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
1560
{
1561 1562 1563

	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
1564
	    PNFS_LAYOUTRET_ON_ERROR) {
1565
		pnfs_return_layout(hdr->inode);
1566
	}
1567
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1568
		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
1569 1570
}

1571 1572 1573
/*
 * Called by non rpc-based layout drivers
 */
1574
void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
1575
{
1576
	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
1577
	if (!hdr->pnfs_error) {
1578 1579
		pnfs_set_layoutcommit(hdr);
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
1580
	} else
1581 1582
		pnfs_ld_handle_write_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
1583
}
1584
EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
1585

1586 1587
static void
pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
1588
		struct nfs_pgio_header *hdr)
1589
{
1590 1591 1592 1593 1594
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
		list_splice_tail_init(&hdr->pages, &desc->pg_list);
		nfs_pageio_reset_write_mds(desc);
		desc->pg_recoalesce = 1;
	}
1595
	nfs_pgio_data_destroy(hdr);
1596 1597 1598
}

static enum pnfs_try_status
1599
pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
1600 1601 1602
			const struct rpc_call_ops *call_ops,
			struct pnfs_layout_segment *lseg,
			int how)
1603
{
1604
	struct inode *inode = hdr->inode;
1605 1606 1607
	enum pnfs_try_status trypnfs;
	struct nfs_server *nfss = NFS_SERVER(inode);

1608
	hdr->mds_ops = call_ops;
1609 1610

	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
1611 1612
		inode->i_ino, hdr->args.count, hdr->args.offset, how);
	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
1613
	if (trypnfs != PNFS_NOT_ATTEMPTED)
1614 1615 1616 1617 1618
		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}

1619
static void
1620 1621
pnfs_do_write(struct nfs_pageio_descriptor *desc,
	      struct nfs_pgio_header *hdr, int how)
1622 1623 1624
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
1625
	enum pnfs_try_status trypnfs;
1626 1627

	desc->pg_lseg = NULL;
1628
	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
1629
	if (trypnfs == PNFS_NOT_ATTEMPTED)
1630
		pnfs_write_through_mds(desc, hdr);
1631
	pnfs_put_lseg(lseg);
1632 1633
}

1634 1635
static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
1636
	pnfs_put_lseg(hdr->lseg);
1637
	nfs_pgio_header_free(hdr);
1638
}
B
Bryan Schumaker 已提交
1639
EXPORT_SYMBOL_GPL(pnfs_writehdr_free);
1640

1641 1642 1643
int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
{
1644
	struct nfs_pgio_header *hdr;
1645 1646
	int ret;

1647 1648
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
1649
		desc->pg_completion_ops->error_cleanup(&desc->pg_list);
1650
		pnfs_put_lseg(desc->pg_lseg);
1651
		desc->pg_lseg = NULL;
1652
		return -ENOMEM;
1653
	}
1654
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
1655
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
1656
	ret = nfs_generic_pgio(desc, hdr);
1657
	if (ret != 0) {
1658
		pnfs_put_lseg(desc->pg_lseg);
1659 1660
		desc->pg_lseg = NULL;
	} else
1661
		pnfs_do_write(desc, hdr, desc->pg_ioflags);
1662
	return ret;
1663 1664 1665
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);

1666
int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
1667 1668 1669
{
	struct nfs_pageio_descriptor pgio;

1670
	/* Resend all requests through the MDS */
1671 1672
	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
1673
}
1674
EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
1675

1676
static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
1677
{
1678 1679
	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
1680
	    PNFS_LAYOUTRET_ON_ERROR) {
1681
		pnfs_return_layout(hdr->inode);
1682
	}
1683
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1684
		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
1685 1686
}

1687 1688 1689
/*
 * Called by non rpc-based layout drivers
 */
1690
void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
1691
{
1692
	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
1693
	if (likely(!hdr->pnfs_error)) {
1694 1695
		__nfs4_read_done_cb(hdr);
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
1696
	} else
1697 1698
		pnfs_ld_handle_read_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
1699 1700 1701
}
EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

1702 1703
static void
pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
1704
		struct nfs_pgio_header *hdr)
1705
{
1706 1707 1708 1709 1710
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
		list_splice_tail_init(&hdr->pages, &desc->pg_list);
		nfs_pageio_reset_read_mds(desc);
		desc->pg_recoalesce = 1;
	}
1711
	nfs_pgio_data_destroy(hdr);
1712 1713
}

A
Andy Adamson 已提交
1714 1715 1716
/*
 * Call the appropriate parallel I/O subsystem read function.
 */
1717
static enum pnfs_try_status
1718
pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
1719 1720
		       const struct rpc_call_ops *call_ops,
		       struct pnfs_layout_segment *lseg)
A
Andy Adamson 已提交
1721
{
1722
	struct inode *inode = hdr->inode;
A
Andy Adamson 已提交
1723 1724 1725
	struct nfs_server *nfss = NFS_SERVER(inode);
	enum pnfs_try_status trypnfs;

1726
	hdr->mds_ops = call_ops;
A
Andy Adamson 已提交
1727 1728

	dprintk("%s: Reading ino:%lu %u@%llu\n",
1729
		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
A
Andy Adamson 已提交
1730

1731
	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
1732
	if (trypnfs != PNFS_NOT_ATTEMPTED)
A
Andy Adamson 已提交
1733 1734 1735 1736
		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}
A
Andy Adamson 已提交
1737

1738
static void
1739
pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
1740 1741 1742
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
1743
	enum pnfs_try_status trypnfs;
1744 1745

	desc->pg_lseg = NULL;
1746
	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
1747
	if (trypnfs == PNFS_NOT_ATTEMPTED)
1748
		pnfs_read_through_mds(desc, hdr);
1749
	pnfs_put_lseg(lseg);
1750 1751
}

1752 1753
static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
1754
	pnfs_put_lseg(hdr->lseg);
1755
	nfs_pgio_header_free(hdr);
1756
}
B
Bryan Schumaker 已提交
1757
EXPORT_SYMBOL_GPL(pnfs_readhdr_free);
1758

1759 1760 1761
int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
{
1762
	struct nfs_pgio_header *hdr;
1763 1764
	int ret;

1765 1766
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
1767
		desc->pg_completion_ops->error_cleanup(&desc->pg_list);
1768
		ret = -ENOMEM;
1769
		pnfs_put_lseg(desc->pg_lseg);
1770 1771 1772
		desc->pg_lseg = NULL;
		return ret;
	}
1773
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
1774
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
1775
	ret = nfs_generic_pgio(desc, hdr);
1776
	if (ret != 0) {
1777
		pnfs_put_lseg(desc->pg_lseg);
1778 1779
		desc->pg_lseg = NULL;
	} else
1780
		pnfs_do_read(desc, hdr);
1781
	return ret;
1782 1783 1784
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);

1785 1786 1787 1788 1789
static void pnfs_clear_layoutcommitting(struct inode *inode)
{
	unsigned long *bitlock = &NFS_I(inode)->flags;

	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
1790
	smp_mb__after_atomic();
1791 1792 1793
	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}

A
Andy Adamson 已提交
1794
/*
1795
 * There can be multiple RW segments.
A
Andy Adamson 已提交
1796
 */
1797
static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
A
Andy Adamson 已提交
1798
{
1799
	struct pnfs_layout_segment *lseg;
A
Andy Adamson 已提交
1800

1801 1802
	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
		if (lseg->pls_range.iomode == IOMODE_RW &&
1803
		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
1804 1805
			list_add(&lseg->pls_lc_list, listp);
	}
A
Andy Adamson 已提交
1806 1807
}

1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
{
	struct pnfs_layout_segment *lseg, *tmp;

	/* Matched by references in pnfs_set_layoutcommit */
	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
		list_del_init(&lseg->pls_lc_list);
		pnfs_put_lseg(lseg);
	}

1818
	pnfs_clear_layoutcommitting(inode);
1819 1820
}

P
Peng Tao 已提交
1821 1822
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
1823
	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
P
Peng Tao 已提交
1824 1825 1826
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

A
Andy Adamson 已提交
1827
void
1828
pnfs_set_layoutcommit(struct nfs_pgio_header *hdr)
A
Andy Adamson 已提交
1829
{
1830 1831
	struct inode *inode = hdr->inode;
	struct nfs_inode *nfsi = NFS_I(inode);
1832
	loff_t end_pos = hdr->mds_offset + hdr->res.count;
1833
	bool mark_as_dirty = false;
A
Andy Adamson 已提交
1834

1835
	spin_lock(&inode->i_lock);
A
Andy Adamson 已提交
1836
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
1837
		mark_as_dirty = true;
A
Andy Adamson 已提交
1838
		dprintk("%s: Set layoutcommit for inode %lu ",
1839
			__func__, inode->i_ino);
A
Andy Adamson 已提交
1840
	}
1841
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &hdr->lseg->pls_flags)) {
1842
		/* references matched in nfs4_layoutcommit_release */
1843
		pnfs_get_lseg(hdr->lseg);
1844
	}
1845 1846
	if (end_pos > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = end_pos;
1847
	spin_unlock(&inode->i_lock);
1848
	dprintk("%s: lseg %p end_pos %llu\n",
1849
		__func__, hdr->lseg, nfsi->layout->plh_lwb);
1850 1851 1852 1853

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
1854
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
1855 1856 1857
}
EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);

1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
void pnfs_commit_set_layoutcommit(struct nfs_commit_data *data)
{
	struct inode *inode = data->inode;
	struct nfs_inode *nfsi = NFS_I(inode);
	bool mark_as_dirty = false;

	spin_lock(&inode->i_lock);
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
		mark_as_dirty = true;
		dprintk("%s: Set layoutcommit for inode %lu ",
			__func__, inode->i_ino);
	}
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &data->lseg->pls_flags)) {
		/* references matched in nfs4_layoutcommit_release */
		pnfs_get_lseg(data->lseg);
	}
	if (data->lwb > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = data->lwb;
	spin_unlock(&inode->i_lock);
	dprintk("%s: lseg %p end_pos %llu\n",
		__func__, data->lseg, nfsi->layout->plh_lwb);

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
		mark_inode_dirty_sync(inode);
}
EXPORT_SYMBOL_GPL(pnfs_commit_set_layoutcommit);

A
Andy Adamson 已提交
1887 1888 1889 1890 1891 1892
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
{
	struct nfs_server *nfss = NFS_SERVER(data->args.inode);

	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
1893
	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
A
Andy Adamson 已提交
1894 1895
}

1896 1897 1898 1899 1900 1901 1902 1903
/*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
 * data to disk to allow the server to recover the data if it crashes.
 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
 * is off, and a COMMIT is sent to a data server, or
 * if WRITEs to a data server return NFS_DATA_SYNC.
 */
A
Andy Adamson 已提交
1904
int
1905
pnfs_layoutcommit_inode(struct inode *inode, bool sync)
A
Andy Adamson 已提交
1906
{
1907
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
A
Andy Adamson 已提交
1908 1909 1910
	struct nfs4_layoutcommit_data *data;
	struct nfs_inode *nfsi = NFS_I(inode);
	loff_t end_pos;
1911
	int status;
A
Andy Adamson 已提交
1912

1913
	if (!pnfs_layoutcommit_outstanding(inode))
1914 1915
		return 0;

1916
	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
P
Peng Tao 已提交
1917

1918
	status = -EAGAIN;
P
Peng Tao 已提交
1919
	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
1920 1921
		if (!sync)
			goto out;
1922
		status = wait_on_bit_lock_action(&nfsi->flags,
1923 1924 1925
				NFS_INO_LAYOUTCOMMITTING,
				nfs_wait_bit_killable,
				TASK_KILLABLE);
P
Peng Tao 已提交
1926
		if (status)
1927
			goto out;
P
Peng Tao 已提交
1928 1929
	}

1930 1931 1932 1933 1934 1935 1936
	status = -ENOMEM;
	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (!data)
		goto clear_layoutcommitting;

	status = 0;
1937
	spin_lock(&inode->i_lock);
1938 1939
	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		goto out_unlock;
1940

1941
	INIT_LIST_HEAD(&data->lseg_list);
1942
	pnfs_list_write_lseg(inode, &data->lseg_list);
A
Andy Adamson 已提交
1943

1944 1945
	end_pos = nfsi->layout->plh_lwb;
	nfsi->layout->plh_lwb = 0;
A
Andy Adamson 已提交
1946

1947
	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
A
Andy Adamson 已提交
1948 1949 1950
	spin_unlock(&inode->i_lock);

	data->args.inode = inode;
1951
	data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
A
Andy Adamson 已提交
1952 1953 1954 1955 1956 1957
	nfs_fattr_init(&data->fattr);
	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
	data->res.fattr = &data->fattr;
	data->args.lastbytewritten = end_pos - 1;
	data->res.server = NFS_SERVER(inode);

1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
	if (ld->prepare_layoutcommit) {
		status = ld->prepare_layoutcommit(&data->args);
		if (status) {
			spin_lock(&inode->i_lock);
			if (end_pos < nfsi->layout->plh_lwb)
				nfsi->layout->plh_lwb = end_pos;
			spin_unlock(&inode->i_lock);
			put_rpccred(data->cred);
			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
			goto clear_layoutcommitting;
		}
	}


A
Andy Adamson 已提交
1972 1973
	status = nfs4_proc_layoutcommit(data, sync);
out:
P
Peng Tao 已提交
1974 1975
	if (status)
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
1976 1977
	dprintk("<-- %s status %d\n", __func__, status);
	return status;
1978 1979
out_unlock:
	spin_unlock(&inode->i_lock);
P
Peng Tao 已提交
1980
	kfree(data);
1981 1982
clear_layoutcommitting:
	pnfs_clear_layoutcommitting(inode);
P
Peng Tao 已提交
1983
	goto out;
A
Andy Adamson 已提交
1984
}
1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996

struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
{
	struct nfs4_threshold *thp;

	thp = kzalloc(sizeof(*thp), GFP_NOFS);
	if (!thp) {
		dprintk("%s mdsthreshold allocation failed\n", __func__);
		return NULL;
	}
	return thp;
}