pnfs.c 67.1 KB
Newer Older
R
Ricardo Labiaga 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 *  pNFS functions to call and manage layout drivers.
 *
 *  Copyright (c) 2002 [year of first publication]
 *  The Regents of the University of Michigan
 *  All Rights Reserved
 *
 *  Dean Hildebrand <dhildebz@umich.edu>
 *
 *  Permission is granted to use, copy, create derivative works, and
 *  redistribute this software and such derivative works for any purpose,
 *  so long as the name of the University of Michigan is not used in
 *  any advertising or publicity pertaining to the use or distribution
 *  of this software without specific, written prior authorization. If
 *  the above copyright notice or any other identification of the
 *  University of Michigan is included in any copy of any portion of
 *  this software, then the disclaimer below must also be included.
 *
 *  This software is provided as is, without representation or warranty
 *  of any kind either express or implied, including without limitation
 *  the implied warranties of merchantability, fitness for a particular
 *  purpose, or noninfringement.  The Regents of the University of
 *  Michigan shall not be liable for any damages, including special,
 *  indirect, incidental, or consequential damages, with respect to any
 *  claim arising out of or in connection with the use of the software,
 *  even if it has been or is hereafter advised of the possibility of
 *  such damages.
 */

#include <linux/nfs_fs.h>
31
#include <linux/nfs_page.h>
32
#include <linux/module.h>
33
#include "internal.h"
R
Ricardo Labiaga 已提交
34
#include "pnfs.h"
A
Andy Adamson 已提交
35
#include "iostat.h"
36
#include "nfs4trace.h"
37
#include "delegation.h"
38
#include "nfs42.h"
R
Ricardo Labiaga 已提交
39 40

#define NFSDBG_FACILITY		NFSDBG_PNFS
41
#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
R
Ricardo Labiaga 已提交
42

43 44 45 46 47 48 49 50 51 52 53 54
/* Locking:
 *
 * pnfs_spinlock:
 *      protects pnfs_modules_tbl.
 */
static DEFINE_SPINLOCK(pnfs_spinlock);

/*
 * pnfs_modules_tbl holds all pnfs modules
 */
static LIST_HEAD(pnfs_modules_tbl);

55
static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo);
56

57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
/* Return the registered pnfs layout driver module matching given id */
static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)
{
	struct pnfs_layoutdriver_type *local;

	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
		if (local->id == id)
			goto out;
	local = NULL;
out:
	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
	return local;
}

R
Ricardo Labiaga 已提交
72 73 74
static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)
{
75 76 77 78
	struct pnfs_layoutdriver_type *local;

	spin_lock(&pnfs_spinlock);
	local = find_pnfs_driver_locked(id);
79 80 81 82
	if (local != NULL && !try_module_get(local->owner)) {
		dprintk("%s: Could not grab reference on module\n", __func__);
		local = NULL;
	}
83 84
	spin_unlock(&pnfs_spinlock);
	return local;
R
Ricardo Labiaga 已提交
85 86 87 88 89
}

void
unset_pnfs_layoutdriver(struct nfs_server *nfss)
{
90 91 92
	if (nfss->pnfs_curr_ld) {
		if (nfss->pnfs_curr_ld->clear_layoutdriver)
			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
93 94 95
		/* Decrement the MDS count. Purge the deviceid cache if zero */
		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
			nfs4_deviceid_purge_client(nfss->nfs_client);
96
		module_put(nfss->pnfs_curr_ld->owner);
97
	}
R
Ricardo Labiaga 已提交
98 99 100 101 102 103 104 105 106 107
	nfss->pnfs_curr_ld = NULL;
}

/*
 * Try to set the server's pnfs module to the pnfs layout type specified by id.
 * Currently only one pNFS layout driver per filesystem is supported.
 *
 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
 */
void
108 109
set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
		      u32 id)
R
Ricardo Labiaga 已提交
110 111 112 113 114 115 116
{
	struct pnfs_layoutdriver_type *ld_type = NULL;

	if (id == 0)
		goto out_no_driver;
	if (!(server->nfs_client->cl_exchange_flags &
		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
117 118
		printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
			__func__, id, server->nfs_client->cl_exchange_flags);
R
Ricardo Labiaga 已提交
119 120 121 122 123 124 125 126 127 128 129 130 131
		goto out_no_driver;
	}
	ld_type = find_pnfs_driver(id);
	if (!ld_type) {
		request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
		ld_type = find_pnfs_driver(id);
		if (!ld_type) {
			dprintk("%s: No pNFS module found for %u.\n",
				__func__, id);
			goto out_no_driver;
		}
	}
	server->pnfs_curr_ld = ld_type;
132 133
	if (ld_type->set_layoutdriver
	    && ld_type->set_layoutdriver(server, mntfh)) {
134 135
		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
			"driver %u.\n", __func__, id);
136 137 138
		module_put(ld_type->owner);
		goto out_no_driver;
	}
139 140
	/* Bump the MDS count */
	atomic_inc(&server->nfs_client->cl_mds_count);
141

R
Ricardo Labiaga 已提交
142 143 144 145 146 147 148
	dprintk("%s: pNFS module for %u set\n", __func__, id);
	return;

out_no_driver:
	dprintk("%s: Using NFSv4 I/O\n", __func__);
	server->pnfs_curr_ld = NULL;
}
149 150 151 152 153 154 155 156

int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	int status = -EINVAL;
	struct pnfs_layoutdriver_type *tmp;

	if (ld_type->id == 0) {
157
		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
158 159
		return status;
	}
160
	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
161
		printk(KERN_ERR "NFS: %s Layout driver must provide "
162 163 164
		       "alloc_lseg and free_lseg.\n", __func__);
		return status;
	}
165 166 167 168 169 170 171 172 173

	spin_lock(&pnfs_spinlock);
	tmp = find_pnfs_driver_locked(ld_type->id);
	if (!tmp) {
		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
		status = 0;
		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
			ld_type->name);
	} else {
174
		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
			__func__, ld_type->id);
	}
	spin_unlock(&pnfs_spinlock);

	return status;
}
EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);

void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
	spin_lock(&pnfs_spinlock);
	list_del(&ld_type->pnfs_tblid);
	spin_unlock(&pnfs_spinlock);
}
EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
192

193 194 195 196
/*
 * pNFS client layout cache
 */

197
/* Need to hold i_lock if caller does not already hold reference */
F
Fred Isaman 已提交
198
void
199
pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
200
{
201
	atomic_inc(&lo->plh_refcount);
202 203
}

204 205 206 207
static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
208
	return ld->alloc_layout_hdr(ino, gfp_flags);
209 210 211 212 213
}

static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
214 215 216 217 218 219 220 221 222 223
	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;

	if (!list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		spin_lock(&clp->cl_lock);
		list_del_init(&lo->plh_layouts);
		spin_unlock(&clp->cl_lock);
	}
224
	put_rpccred(lo->plh_lc_cred);
225
	return ld->free_layout_hdr(lo);
226 227
}

228
static void
229
pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
230
{
231
	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
232
	dprintk("%s: freeing layout cache %p\n", __func__, lo);
233 234 235 236
	nfsi->layout = NULL;
	/* Reset MDS Threshold I/O counters */
	nfsi->write_io = 0;
	nfsi->read_io = 0;
237 238
}

239
void
240
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
241
{
242 243
	struct inode *inode = lo->plh_inode;

244 245
	pnfs_layoutreturn_before_put_layout_hdr(lo);

246
	if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
247 248
		if (!list_empty(&lo->plh_segs))
			WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
249
		pnfs_detach_layout_hdr(lo);
250
		spin_unlock(&inode->i_lock);
251
		pnfs_free_layout_hdr(lo);
252
	}
253 254
}

255 256 257 258 259 260 261
/*
 * Mark a pnfs_layout_hdr and all associated layout segments as invalid
 *
 * In order to continue using the pnfs_layout_hdr, a full recovery
 * is required.
 * Note that caller must hold inode->i_lock.
 */
262
int
263 264 265 266 267 268 269 270 271 272
pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
		struct list_head *lseg_list)
{
	struct pnfs_layout_range range = {
		.iomode = IOMODE_ANY,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};

	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
273
	return pnfs_mark_matching_lsegs_invalid(lo, lseg_list, &range, 0);
274 275
}

276 277 278 279 280 281 282 283
static int
pnfs_iomode_to_fail_bit(u32 iomode)
{
	return iomode == IOMODE_RW ?
		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
}

static void
284
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
285
{
286
	lo->plh_retry_timestamp = jiffies;
287
	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
288 289 290 291 292 293 294 295 296 297 298 299 300 301
		atomic_inc(&lo->plh_refcount);
}

static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
{
	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
		atomic_dec(&lo->plh_refcount);
}

static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
	struct inode *inode = lo->plh_inode;
302 303 304 305 306 307
	struct pnfs_layout_range range = {
		.iomode = iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(head);
308 309 310

	spin_lock(&inode->i_lock);
	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
311
	pnfs_mark_matching_lsegs_invalid(lo, &head, &range, 0);
312
	spin_unlock(&inode->i_lock);
313
	pnfs_free_lseg_list(&head);
314 315 316 317 318 319 320
	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
			iomode == IOMODE_RW ?  "RW" : "READ");
}

static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
321
	unsigned long start, end;
322 323 324
	int fail_bit = pnfs_iomode_to_fail_bit(iomode);

	if (test_bit(fail_bit, &lo->plh_flags) == 0)
325 326 327 328 329
		return false;
	end = jiffies;
	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
		/* It is time to retry the failed layoutgets */
330
		pnfs_layout_clear_fail_bit(lo, fail_bit);
331 332 333
		return false;
	}
	return true;
334 335
}

336
static void
337 338 339
pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,
		const struct pnfs_layout_range *range,
		const nfs4_stateid *stateid)
340
{
341
	INIT_LIST_HEAD(&lseg->pls_list);
342
	INIT_LIST_HEAD(&lseg->pls_lc_list);
343 344
	atomic_set(&lseg->pls_refcount, 1);
	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
345
	lseg->pls_layout = lo;
346 347
	lseg->pls_range = *range;
	lseg->pls_seq = be32_to_cpu(stateid->seqid);
348 349
}

350
static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
351
{
352
	struct inode *ino = lseg->pls_layout->plh_inode;
353

354
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
355 356
}

F
Fred Isaman 已提交
357
static void
358 359
pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
F
Fred Isaman 已提交
360
{
361
	struct inode *inode = lo->plh_inode;
F
Fred Isaman 已提交
362

363
	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
F
Fred Isaman 已提交
364
	list_del_init(&lseg->pls_list);
365 366
	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
	atomic_dec(&lo->plh_refcount);
367 368
	if (list_empty(&lo->plh_segs)) {
		set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
369
		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
370
	}
F
Fred Isaman 已提交
371 372 373
	rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}

374
void
375
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
376
{
377
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
378 379 380 381 382
	struct inode *inode;

	if (!lseg)
		return;

383 384 385
	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
386

387 388
	lo = lseg->pls_layout;
	inode = lo->plh_inode;
389

F
Fred Isaman 已提交
390
	if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
391 392 393 394
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
			spin_unlock(&inode->i_lock);
			return;
		}
395
		pnfs_get_layout_hdr(lo);
396 397 398 399
		pnfs_layout_remove_lseg(lo, lseg);
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg(lseg);
		pnfs_put_layout_hdr(lo);
400 401
	}
}
402
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
403

404
static void pnfs_free_lseg_async_work(struct work_struct *work)
405 406
{
	struct pnfs_layout_segment *lseg;
407
	struct pnfs_layout_hdr *lo;
408 409

	lseg = container_of(work, struct pnfs_layout_segment, pls_work);
410
	lo = lseg->pls_layout;
411

412 413
	pnfs_free_lseg(lseg);
	pnfs_put_layout_hdr(lo);
414 415
}

416
static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg)
417
{
418
	INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work);
419 420
	schedule_work(&lseg->pls_work);
}
421 422 423 424 425 426 427 428 429 430 431 432 433 434

void
pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
{
	if (!lseg)
		return;

	assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock);

	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
	if (atomic_dec_and_test(&lseg->pls_refcount)) {
		struct pnfs_layout_hdr *lo = lseg->pls_layout;
435 436
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags))
			return;
437 438 439 440 441 442
		pnfs_get_layout_hdr(lo);
		pnfs_layout_remove_lseg(lo, lseg);
		pnfs_free_lseg_async(lseg);
	}
}
EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
443

444
static u64
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
end_offset(u64 start, u64 len)
{
	u64 end;

	end = start + len;
	return end >= start ? end : NFS4_MAX_UINT64;
}

/*
 * is l2 fully contained in l1?
 *   start1                             end1
 *   [----------------------------------)
 *           start2           end2
 *           [----------------)
 */
460
static bool
461
pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
462
		 const struct pnfs_layout_range *l2)
463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (start1 <= start2) && (end1 >= end2);
}

/*
 * is l1 and l2 intersecting?
 *   start1                             end1
 *   [----------------------------------)
 *                              start2           end2
 *                              [----------------)
 */
479
static bool
480
pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
481
		    const struct pnfs_layout_range *l2)
482 483 484 485 486 487 488 489 490 491
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
	       (end2 == NFS4_MAX_UINT64 || end2 > start1);
}

492 493 494 495 496 497 498 499 500 501
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		struct list_head *tmp_list)
{
	if (!atomic_dec_and_test(&lseg->pls_refcount))
		return false;
	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
	list_add(&lseg->pls_list, tmp_list);
	return true;
}

502 503 504 505 506 507 508 509 510 511 512
/* Returns 1 if lseg is removed from list, 0 otherwise */
static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
			     struct list_head *tmp_list)
{
	int rv = 0;

	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
		/* Remove the reference keeping the lseg in the
		 * list.  It will now be removed when all
		 * outstanding io is finished.
		 */
F
Fred Isaman 已提交
513 514
		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
			atomic_read(&lseg->pls_refcount));
515
		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
F
Fred Isaman 已提交
516
			rv = 1;
517 518 519 520
	}
	return rv;
}

521 522 523 524 525 526 527 528 529
/*
 * Compare 2 layout stateid sequence ids, to see which is newer,
 * taking into account wraparound issues.
 */
static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
{
	return (s32)(s1 - s2) > 0;
}

530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550
static bool
pnfs_should_free_range(const struct pnfs_layout_range *lseg_range,
		 const struct pnfs_layout_range *recall_range)
{
	return (recall_range->iomode == IOMODE_ANY ||
		lseg_range->iomode == recall_range->iomode) &&
	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
}

static bool
pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg,
		const struct pnfs_layout_range *recall_range,
		u32 seq)
{
	if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq))
		return false;
	if (recall_range == NULL)
		return true;
	return pnfs_should_free_range(&lseg->pls_range, recall_range);
}

551 552 553 554 555 556 557 558 559 560 561 562 563 564
/**
 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later
 * @lo: layout header containing the lsegs
 * @tmp_list: list head where doomed lsegs should go
 * @recall_range: optional recall range argument to match (may be NULL)
 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0)
 *
 * Walk the list of lsegs in the layout header, and tear down any that should
 * be destroyed. If "recall_range" is specified then the segment must match
 * that range. If "seq" is non-zero, then only match segments that were handed
 * out at or before that sequence.
 *
 * Returns number of matching invalid lsegs remaining in list after scanning
 * it and purging them.
565
 */
F
Fred Isaman 已提交
566
int
567
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
568
			    struct list_head *tmp_list,
569 570
			    const struct pnfs_layout_range *recall_range,
			    u32 seq)
571 572
{
	struct pnfs_layout_segment *lseg, *next;
573
	int remaining = 0;
574 575 576

	dprintk("%s:Begin lo %p\n", __func__, lo);

577
	if (list_empty(&lo->plh_segs))
578
		return 0;
579
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
580
		if (pnfs_match_lseg_recall(lseg, recall_range, seq)) {
581
			dprintk("%s: freeing lseg %p iomode %d seq %u"
582
				"offset %llu length %llu\n", __func__,
583 584
				lseg, lseg->pls_range.iomode, lseg->pls_seq,
				lseg->pls_range.offset, lseg->pls_range.length);
585 586
			if (!mark_lseg_invalid(lseg, tmp_list))
				remaining++;
587
		}
588 589
	dprintk("%s:Return %i\n", __func__, remaining);
	return remaining;
590 591
}

592
/* note free_me must contain lsegs from a single layout_hdr */
F
Fred Isaman 已提交
593
void
594
pnfs_free_lseg_list(struct list_head *free_me)
595
{
596
	struct pnfs_layout_segment *lseg, *tmp;
597 598 599 600

	if (list_empty(free_me))
		return;

601
	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
602
		list_del(&lseg->pls_list);
603
		pnfs_free_lseg(lseg);
604 605 606
	}
}

607 608 609 610
void
pnfs_destroy_layout(struct nfs_inode *nfsi)
{
	struct pnfs_layout_hdr *lo;
611
	LIST_HEAD(tmp_list);
612 613 614 615

	spin_lock(&nfsi->vfs_inode.i_lock);
	lo = nfsi->layout;
	if (lo) {
616
		pnfs_get_layout_hdr(lo);
617
		pnfs_mark_layout_stateid_invalid(lo, &tmp_list);
618 619 620 621 622 623 624
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
		spin_unlock(&nfsi->vfs_inode.i_lock);
		pnfs_free_lseg_list(&tmp_list);
		pnfs_put_layout_hdr(lo);
	} else
		spin_unlock(&nfsi->vfs_inode.i_lock);
625
}
626
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
627

628 629 630
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
		struct list_head *layout_list)
631 632
{
	struct pnfs_layout_hdr *lo;
633
	bool ret = false;
634

635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
		pnfs_get_layout_hdr(lo);
		list_add(&lo->plh_bulk_destroy, layout_list);
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

/* Caller must hold rcu_read_lock and clp->cl_lock */
static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
		struct nfs_server *server,
		struct list_head *layout_list)
{
	struct pnfs_layout_hdr *lo, *next;
	struct inode *inode;

	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
		inode = igrab(lo->plh_inode);
		if (inode == NULL)
			continue;
		list_del_init(&lo->plh_layouts);
		if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
			continue;
		rcu_read_unlock();
		spin_unlock(&clp->cl_lock);
		iput(inode);
		spin_lock(&clp->cl_lock);
		rcu_read_lock();
		return -EAGAIN;
	}
	return 0;
}

static int
pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
		bool is_bulk_recall)
{
	struct pnfs_layout_hdr *lo;
	struct inode *inode;
	LIST_HEAD(lseg_list);
	int ret = 0;

	while (!list_empty(layout_list)) {
		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
				plh_bulk_destroy);
		dprintk("%s freeing layout for inode %lu\n", __func__,
			lo->plh_inode->i_ino);
		inode = lo->plh_inode;
687 688 689

		pnfs_layoutcommit_inode(inode, false);

690 691
		spin_lock(&inode->i_lock);
		list_del_init(&lo->plh_bulk_destroy);
692 693 694
		if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) {
			if (is_bulk_recall)
				set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
695
			ret = -EAGAIN;
696
		}
697 698
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg_list(&lseg_list);
699 700
		/* Free all lsegs that are attached to commit buckets */
		nfs_commit_inode(inode, 0);
701 702 703 704 705 706 707 708 709 710 711 712 713
		pnfs_put_layout_hdr(lo);
		iput(inode);
	}
	return ret;
}

int
pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
		struct nfs_fsid *fsid,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);
714

715
	spin_lock(&clp->cl_lock);
716
	rcu_read_lock();
717
restart:
718
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
719 720 721 722 723 724
		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
			continue;
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
				server,
				&layout_list) != 0)
			goto restart;
725 726
	}
	rcu_read_unlock();
727 728
	spin_unlock(&clp->cl_lock);

729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

int
pnfs_destroy_layouts_byclid(struct nfs_client *clp,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);

	spin_lock(&clp->cl_lock);
	rcu_read_lock();
restart:
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
					server,
					&layout_list) != 0)
			goto restart;
749
	}
750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
	rcu_read_unlock();
	spin_unlock(&clp->cl_lock);

	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

/*
 * Called by the state manger to remove all layouts established under an
 * expired lease.
 */
void
pnfs_destroy_all_layouts(struct nfs_client *clp)
{
	nfs4_deviceid_mark_client_invalid(clp);
	nfs4_deviceid_purge_client(clp);

	pnfs_destroy_layouts_byclid(clp, false);
769 770
}

771
/* update lo->plh_stateid with new if is more recent */
F
Fred Isaman 已提交
772 773 774
void
pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
			bool update_barrier)
775
{
776 777
	u32 oldseq, newseq, new_barrier = 0;
	bool invalid = !pnfs_layout_is_valid(lo);
778

779 780
	oldseq = be32_to_cpu(lo->plh_stateid.seqid);
	newseq = be32_to_cpu(new->seqid);
781
	if (invalid || pnfs_seqid_is_newer(newseq, oldseq)) {
782
		nfs4_stateid_copy(&lo->plh_stateid, new);
783 784 785 786 787
		/*
		 * Because of wraparound, we want to keep the barrier
		 * "close" to the current seqids.
		 */
		new_barrier = newseq - atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
788
	}
789 790 791 792 793 794
	if (update_barrier)
		new_barrier = be32_to_cpu(new->seqid);
	else if (new_barrier == 0)
		return;
	if (invalid || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
		lo->plh_barrier = new_barrier;
795 796
}

797
static bool
798 799
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid)
F
Fred Isaman 已提交
800
{
801
	u32 seqid = be32_to_cpu(stateid->seqid);
802

803 804 805 806 807
	return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
}

/* lget is set to 1 if called from inside send_layoutget call chain */
static bool
808
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
809
{
F
Fred Isaman 已提交
810
	return lo->plh_block_lgets ||
811
		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
812 813
}

814
/*
815 816 817 818 819
 * Get layout from server.
 *    for now, assume that whole file layouts are requested.
 *    arg->offset: 0
 *    arg->length: all ones
 */
820 821 822
static struct pnfs_layout_segment *
send_layoutget(struct pnfs_layout_hdr *lo,
	   struct nfs_open_context *ctx,
823
	   nfs4_stateid *stateid,
824
	   const struct pnfs_layout_range *range,
825
	   long *timeout, gfp_t gfp_flags)
826
{
827
	struct inode *ino = lo->plh_inode;
828 829
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs4_layoutget *lgp;
830
	loff_t i_size;
831 832

	dprintk("--> %s\n", __func__);
833

834 835 836 837 838
	/*
	 * Synchronously retrieve layout information from server and
	 * store in lseg. If we race with a concurrent seqid morphing
	 * op, then re-send the LAYOUTGET.
	 */
839 840 841 842 843 844 845 846 847 848 849 850 851 852
	lgp = kzalloc(sizeof(*lgp), gfp_flags);
	if (lgp == NULL)
		return ERR_PTR(-ENOMEM);

	i_size = i_size_read(ino);

	lgp->args.minlength = PAGE_SIZE;
	if (lgp->args.minlength > range->length)
		lgp->args.minlength = range->length;
	if (range->iomode == IOMODE_READ) {
		if (range->offset >= i_size)
			lgp->args.minlength = 0;
		else if (i_size - range->offset < lgp->args.minlength)
			lgp->args.minlength = i_size - range->offset;
853
	}
854 855 856 857 858
	lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
	pnfs_copy_range(&lgp->args.range, range);
	lgp->args.type = server->pnfs_curr_ld->id;
	lgp->args.inode = ino;
	lgp->args.ctx = get_nfs_open_context(ctx);
859
	nfs4_stateid_copy(&lgp->args.stateid, stateid);
860 861
	lgp->gfp_flags = gfp_flags;
	lgp->cred = lo->plh_lc_cred;
862

863
	return nfs4_proc_layoutget(lgp, timeout, gfp_flags);
864 865
}

866 867 868 869 870 871 872 873 874 875 876 877 878 879 880
static void pnfs_clear_layoutcommit(struct inode *inode,
		struct list_head *head)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct pnfs_layout_segment *lseg, *tmp;

	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		return;
	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
			continue;
		pnfs_lseg_dec_and_remove_zero(lseg, head);
	}
}

881 882 883 884 885
void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
{
	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
	smp_mb__after_atomic();
	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
886
	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
887 888
}

889 890 891 892 893 894 895 896
static void
pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo)
{
	lo->plh_return_iomode = 0;
	lo->plh_return_seq = 0;
	clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
}

897
static bool
898 899 900
pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo,
		nfs4_stateid *stateid,
		enum pnfs_iomode *iomode)
901
{
902 903 904
	/* Serialise LAYOUTGET/LAYOUTRETURN */
	if (atomic_read(&lo->plh_outstanding) != 0)
		return false;
905 906 907
	if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
		return false;
	pnfs_get_layout_hdr(lo);
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
		if (stateid != NULL) {
			nfs4_stateid_copy(stateid, &lo->plh_stateid);
			if (lo->plh_return_seq != 0)
				stateid->seqid = cpu_to_be32(lo->plh_return_seq);
		}
		if (iomode != NULL)
			*iomode = lo->plh_return_iomode;
		pnfs_clear_layoutreturn_info(lo);
		return true;
	}
	if (stateid != NULL)
		nfs4_stateid_copy(stateid, &lo->plh_stateid);
	if (iomode != NULL)
		*iomode = IOMODE_ANY;
923 924 925
	return true;
}

926
static int
927
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
928
		       enum pnfs_iomode iomode, bool sync)
929 930 931 932 933
{
	struct inode *ino = lo->plh_inode;
	struct nfs4_layoutreturn *lrp;
	int status = 0;

934
	lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
935 936 937
	if (unlikely(lrp == NULL)) {
		status = -ENOMEM;
		spin_lock(&ino->i_lock);
938
		pnfs_clear_layoutreturn_waitbit(lo);
939 940 941 942 943
		spin_unlock(&ino->i_lock);
		pnfs_put_layout_hdr(lo);
		goto out;
	}

944
	nfs4_stateid_copy(&lrp->args.stateid, stateid);
945 946
	lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
	lrp->args.inode = ino;
P
Peng Tao 已提交
947 948 949
	lrp->args.range.iomode = iomode;
	lrp->args.range.offset = 0;
	lrp->args.range.length = NFS4_MAX_UINT64;
950 951 952 953
	lrp->args.layout = lo;
	lrp->clp = NFS_SERVER(ino)->nfs_client;
	lrp->cred = lo->plh_lc_cred;

954
	status = nfs4_proc_layoutreturn(lrp, sync);
955 956 957 958 959
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

960 961 962 963 964 965
/* Return true if layoutreturn is needed */
static bool
pnfs_layout_need_return(struct pnfs_layout_hdr *lo)
{
	struct pnfs_layout_segment *s;

966
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
967 968 969 970 971 972 973 974 975 976 977 978 979 980 981
		return false;

	/* Defer layoutreturn until all lsegs are done */
	list_for_each_entry(s, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
			return false;
	}

	return true;
}

static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo)
{
	struct inode *inode= lo->plh_inode;

982
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
983 984 985 986 987 988 989
		return;
	spin_lock(&inode->i_lock);
	if (pnfs_layout_need_return(lo)) {
		nfs4_stateid stateid;
		enum pnfs_iomode iomode;
		bool send;

990
		send = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
991 992 993 994 995 996 997 998 999
		spin_unlock(&inode->i_lock);
		if (send) {
			/* Send an async layoutreturn so we dont deadlock */
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
		}
	} else
		spin_unlock(&inode->i_lock);
}

1000 1001 1002 1003 1004 1005 1006 1007
/*
 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
 * when the layout segment list is empty.
 *
 * Note that a pnfs_layout_hdr can exist with an empty layout segment
 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
 * deviceid is marked invalid.
 */
B
Benny Halevy 已提交
1008 1009 1010 1011 1012 1013 1014
int
_pnfs_return_layout(struct inode *ino)
{
	struct pnfs_layout_hdr *lo = NULL;
	struct nfs_inode *nfsi = NFS_I(ino);
	LIST_HEAD(tmp_list);
	nfs4_stateid stateid;
1015
	int status = 0, empty;
1016
	bool send;
B
Benny Halevy 已提交
1017

1018
	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
B
Benny Halevy 已提交
1019 1020 1021

	spin_lock(&ino->i_lock);
	lo = nfsi->layout;
1022
	if (!lo) {
B
Benny Halevy 已提交
1023
		spin_unlock(&ino->i_lock);
1024 1025
		dprintk("NFS: %s no layout to return\n", __func__);
		goto out;
B
Benny Halevy 已提交
1026 1027
	}
	/* Reference matched in nfs4_layoutreturn_release */
1028
	pnfs_get_layout_hdr(lo);
1029
	empty = list_empty(&lo->plh_segs);
1030
	pnfs_clear_layoutcommit(ino, &tmp_list);
1031
	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0);
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041

	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
		struct pnfs_layout_range range = {
			.iomode		= IOMODE_ANY,
			.offset		= 0,
			.length		= NFS4_MAX_UINT64,
		};
		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
	}

1042 1043 1044 1045
	/* Don't send a LAYOUTRETURN if list was initially empty */
	if (empty) {
		spin_unlock(&ino->i_lock);
		dprintk("NFS: %s no layout segments to return\n", __func__);
1046
		goto out_put_layout_hdr;
1047
	}
1048

1049
	send = pnfs_prepare_layoutreturn(lo, &stateid, NULL);
B
Benny Halevy 已提交
1050 1051
	spin_unlock(&ino->i_lock);
	pnfs_free_lseg_list(&tmp_list);
1052
	if (send)
1053
		status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1054 1055
out_put_layout_hdr:
	pnfs_put_layout_hdr(lo);
B
Benny Halevy 已提交
1056 1057 1058 1059
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}
1060
EXPORT_SYMBOL_GPL(_pnfs_return_layout);
B
Benny Halevy 已提交
1061

1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
int
pnfs_commit_and_return_layout(struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	int ret;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo == NULL) {
		spin_unlock(&inode->i_lock);
		return 0;
	}
	pnfs_get_layout_hdr(lo);
	/* Block new layoutgets and read/write to ds */
	lo->plh_block_lgets++;
	spin_unlock(&inode->i_lock);
	filemap_fdatawait(inode->i_mapping);
	ret = pnfs_layoutcommit_inode(inode, true);
	if (ret == 0)
		ret = _pnfs_return_layout(inode);
	spin_lock(&inode->i_lock);
	lo->plh_block_lgets--;
	spin_unlock(&inode->i_lock);
	pnfs_put_layout_hdr(lo);
	return ret;
}

F
Fred Isaman 已提交
1089 1090
bool pnfs_roc(struct inode *ino)
{
1091 1092 1093
	struct nfs_inode *nfsi = NFS_I(ino);
	struct nfs_open_context *ctx;
	struct nfs4_state *state;
F
Fred Isaman 已提交
1094 1095
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg, *tmp;
1096
	nfs4_stateid stateid;
F
Fred Isaman 已提交
1097
	LIST_HEAD(tmp_list);
1098
	bool found = false, layoutreturn = false, roc = false;
F
Fred Isaman 已提交
1099 1100

	spin_lock(&ino->i_lock);
1101
	lo = nfsi->layout;
P
Peng Tao 已提交
1102
	if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
1103 1104
		goto out_noroc;

1105
	/* no roc if we hold a delegation */
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115
	if (nfs4_check_delegation(ino, FMODE_READ))
		goto out_noroc;

	list_for_each_entry(ctx, &nfsi->open_files, list) {
		state = ctx->state;
		/* Don't return layout if there is open file state */
		if (state != NULL && state->state != 0)
			goto out_noroc;
	}

1116
	/* always send layoutreturn if being marked so */
1117 1118 1119
	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
		layoutreturn = pnfs_prepare_layoutreturn(lo,
				&stateid, NULL);
1120

F
Fred Isaman 已提交
1121
	list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
1122 1123
		/* If we are sending layoutreturn, invalidate all valid lsegs */
		if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
F
Fred Isaman 已提交
1124 1125 1126
			mark_lseg_invalid(lseg, &tmp_list);
			found = true;
		}
1127
	/* ROC in two conditions:
1128 1129 1130
	 * 1. there are ROC lsegs
	 * 2. we don't send layoutreturn
	 */
1131 1132 1133
	if (found && !layoutreturn) {
		/* lo ref dropped in pnfs_roc_release() */
		pnfs_get_layout_hdr(lo);
1134
		roc = true;
1135
	}
F
Fred Isaman 已提交
1136

1137
out_noroc:
F
Fred Isaman 已提交
1138
	spin_unlock(&ino->i_lock);
1139 1140 1141
	pnfs_free_lseg_list(&tmp_list);
	pnfs_layoutcommit_inode(ino, true);
	if (layoutreturn)
1142
		pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1143
	return roc;
F
Fred Isaman 已提交
1144 1145 1146 1147 1148 1149 1150 1151
}

void pnfs_roc_release(struct inode *ino)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1152
	pnfs_clear_layoutreturn_waitbit(lo);
1153 1154 1155 1156 1157 1158
	if (atomic_dec_and_test(&lo->plh_refcount)) {
		pnfs_detach_layout_hdr(lo);
		spin_unlock(&ino->i_lock);
		pnfs_free_layout_hdr(lo);
	} else
		spin_unlock(&ino->i_lock);
F
Fred Isaman 已提交
1159 1160 1161 1162 1163 1164 1165 1166
}

void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1167
	if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
F
Fred Isaman 已提交
1168 1169
		lo->plh_barrier = barrier;
	spin_unlock(&ino->i_lock);
1170
	trace_nfs4_layoutreturn_on_close(ino, 0);
F
Fred Isaman 已提交
1171 1172
}

1173
void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier)
F
Fred Isaman 已提交
1174 1175
{
	struct nfs_inode *nfsi = NFS_I(ino);
1176 1177
	struct pnfs_layout_hdr *lo;
	u32 current_seqid;
F
Fred Isaman 已提交
1178 1179

	spin_lock(&ino->i_lock);
1180 1181
	lo = nfsi->layout;
	current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
F
Fred Isaman 已提交
1182

1183 1184 1185 1186
	/* Since close does not return a layout stateid for use as
	 * a barrier, we choose the worst-case barrier.
	 */
	*barrier = current_seqid + atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
1187 1188 1189
	spin_unlock(&ino->i_lock);
}

1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
{
	struct nfs_inode *nfsi = NFS_I(ino);
        struct pnfs_layout_hdr *lo;
        bool sleep = false;

	/* we might not have grabbed lo reference. so need to check under
	 * i_lock */
        spin_lock(&ino->i_lock);
        lo = nfsi->layout;
        if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
                sleep = true;
        spin_unlock(&ino->i_lock);

        if (sleep)
                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);

        return sleep;
}

1210 1211 1212 1213 1214 1215
/*
 * Compare two layout segments for sorting into layout cache.
 * We want to preferentially return RW over RO layouts, so ensure those
 * are seen first.
 */
static s64
1216
pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1217
	   const struct pnfs_layout_range *l2)
1218
{
1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
	s64 d;

	/* high offset > low offset */
	d = l1->offset - l2->offset;
	if (d)
		return d;

	/* short length > long length */
	d = l2->length - l1->length;
	if (d)
		return d;

1231
	/* read > read/write */
1232
	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1233 1234
}

1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
static bool
pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
		const struct pnfs_layout_range *l2)
{
	return pnfs_lseg_range_cmp(l1, l2) > 0;
}

static bool
pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
		struct pnfs_layout_segment *old)
{
	return false;
}

void
pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   bool (*is_after)(const struct pnfs_layout_range *,
			   const struct pnfs_layout_range *),
		   bool (*do_merge)(struct pnfs_layout_segment *,
			   struct pnfs_layout_segment *),
		   struct list_head *free_me)
1257
{
1258
	struct pnfs_layout_segment *lp, *tmp;
1259

1260 1261
	dprintk("%s:Begin\n", __func__);

1262 1263 1264 1265 1266 1267 1268 1269
	list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
			continue;
		if (do_merge(lseg, lp)) {
			mark_lseg_invalid(lp, free_me);
			continue;
		}
		if (is_after(&lseg->pls_range, &lp->pls_range))
1270
			continue;
1271
		list_add_tail(&lseg->pls_list, &lp->pls_list);
1272 1273 1274
		dprintk("%s: inserted lseg %p "
			"iomode %d offset %llu length %llu before "
			"lp %p iomode %d offset %llu length %llu\n",
1275 1276 1277 1278
			__func__, lseg, lseg->pls_range.iomode,
			lseg->pls_range.offset, lseg->pls_range.length,
			lp, lp->pls_range.iomode, lp->pls_range.offset,
			lp->pls_range.length);
1279
		goto out;
1280
	}
1281 1282 1283 1284 1285 1286
	list_add_tail(&lseg->pls_list, &lo->plh_segs);
	dprintk("%s: inserted lseg %p "
		"iomode %d offset %llu length %llu at tail\n",
		__func__, lseg, lseg->pls_range.iomode,
		lseg->pls_range.offset, lseg->pls_range.length);
out:
1287
	pnfs_get_layout_hdr(lo);
1288 1289

	dprintk("%s:Return\n", __func__);
1290
}
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);

static void
pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   struct list_head *free_me)
{
	struct inode *inode = lo->plh_inode;
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;

	if (ld->add_lseg != NULL)
		ld->add_lseg(lo, lseg, free_me);
	else
		pnfs_generic_layout_insert_lseg(lo, lseg,
				pnfs_lseg_range_is_after,
				pnfs_lseg_no_merge,
				free_me);
}
1309 1310

static struct pnfs_layout_hdr *
1311 1312 1313
alloc_init_layout_hdr(struct inode *ino,
		      struct nfs_open_context *ctx,
		      gfp_t gfp_flags)
1314 1315 1316
{
	struct pnfs_layout_hdr *lo;

1317
	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1318 1319
	if (!lo)
		return NULL;
1320
	atomic_set(&lo->plh_refcount, 1);
1321 1322
	INIT_LIST_HEAD(&lo->plh_layouts);
	INIT_LIST_HEAD(&lo->plh_segs);
1323
	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1324
	lo->plh_inode = ino;
1325
	lo->plh_lc_cred = get_rpccred(ctx->cred);
1326
	lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID;
1327 1328 1329 1330
	return lo;
}

static struct pnfs_layout_hdr *
1331 1332 1333
pnfs_find_alloc_layout(struct inode *ino,
		       struct nfs_open_context *ctx,
		       gfp_t gfp_flags)
1334 1335
	__releases(&ino->i_lock)
	__acquires(&ino->i_lock)
1336 1337 1338 1339 1340 1341
{
	struct nfs_inode *nfsi = NFS_I(ino);
	struct pnfs_layout_hdr *new = NULL;

	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

1342 1343
	if (nfsi->layout != NULL)
		goto out_existing;
1344
	spin_unlock(&ino->i_lock);
1345
	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1346 1347
	spin_lock(&ino->i_lock);

1348
	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1349
		nfsi->layout = new;
1350
		return new;
1351 1352
	} else if (new != NULL)
		pnfs_free_layout_hdr(new);
1353 1354
out_existing:
	pnfs_get_layout_hdr(nfsi->layout);
1355 1356 1357
	return nfsi->layout;
}

1358 1359
/*
 * iomode matching rules:
1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
 * iomode	lseg	strict match
 *                      iomode
 * -----	-----	------ -----
 * ANY		READ	N/A    true
 * ANY		RW	N/A    true
 * RW		READ	N/A    false
 * RW		RW	N/A    true
 * READ		READ	N/A    true
 * READ		RW	true   false
 * READ		RW	false  true
1370
 */
1371
static bool
1372
pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1373 1374
		 const struct pnfs_layout_range *range,
		 bool strict_iomode)
1375
{
1376 1377 1378 1379
	struct pnfs_layout_range range1;

	if ((range->iomode == IOMODE_RW &&
	     ls_range->iomode != IOMODE_RW) ||
1380 1381
	    (range->iomode != ls_range->iomode &&
	     strict_iomode == true) ||
1382
	    !pnfs_lseg_range_intersecting(ls_range, range))
1383 1384 1385 1386 1387
		return 0;

	/* range1 covers only the first byte in the range */
	range1 = *range;
	range1.length = 1;
1388
	return pnfs_lseg_range_contained(ls_range, &range1);
1389 1390 1391 1392 1393
}

/*
 * lookup range in layout
 */
1394
static struct pnfs_layout_segment *
1395
pnfs_find_lseg(struct pnfs_layout_hdr *lo,
1396 1397
		struct pnfs_layout_range *range,
		bool strict_iomode)
1398
{
1399 1400 1401 1402
	struct pnfs_layout_segment *lseg, *ret = NULL;

	dprintk("%s:Begin\n", __func__);

1403
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1404
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1405
		    !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
1406 1407
		    pnfs_lseg_range_match(&lseg->pls_range, range,
					  strict_iomode)) {
1408
			ret = pnfs_get_lseg(lseg);
1409 1410 1411 1412 1413
			break;
		}
	}

	dprintk("%s:Return lseg %p ref %d\n",
1414
		__func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
1415
	return ret;
1416 1417
}

1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
/*
 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
 * to the MDS or over pNFS
 *
 * The nfs_inode read_io and write_io fields are cumulative counters reset
 * when there are no layout segments. Note that in pnfs_update_layout iomode
 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
 * WRITE request.
 *
 * A return of true means use MDS I/O.
 *
 * From rfc 5661:
 * If a file's size is smaller than the file size threshold, data accesses
 * SHOULD be sent to the metadata server.  If an I/O request has a length that
 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
 * server.  If both file size and I/O size are provided, the client SHOULD
 * reach or exceed  both thresholds before sending its read or write
 * requests to the data server.
 */
static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
				     struct inode *ino, int iomode)
{
	struct nfs4_threshold *t = ctx->mdsthreshold;
	struct nfs_inode *nfsi = NFS_I(ino);
	loff_t fsize = i_size_read(ino);
	bool size = false, size_set = false, io = false, io_set = false, ret = false;

	if (t == NULL)
		return ret;

	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);

	switch (iomode) {
	case IOMODE_READ:
		if (t->bm & THRESHOLD_RD) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->rd_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_RD_IO) {
			dprintk("%s nfsi->read_io %llu\n", __func__,
				nfsi->read_io);
			io_set = true;
			if (nfsi->read_io < t->rd_io_sz)
				io = true;
		}
		break;
	case IOMODE_RW:
		if (t->bm & THRESHOLD_WR) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->wr_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_WR_IO) {
			dprintk("%s nfsi->write_io %llu\n", __func__,
				nfsi->write_io);
			io_set = true;
			if (nfsi->write_io < t->wr_io_sz)
				io = true;
		}
		break;
	}
	if (size_set && io_set) {
		if (size && io)
			ret = true;
	} else if (size || io)
		ret = true;

	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
	return ret;
}

1493 1494 1495 1496 1497 1498 1499 1500
static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
{
	/*
	 * send layoutcommit as it can hold up layoutreturn due to lseg
	 * reference
	 */
	pnfs_layoutcommit_inode(lo->plh_inode, false);
	return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
1501
				   nfs_wait_bit_killable,
1502 1503 1504
				   TASK_UNINTERRUPTIBLE);
}

1505 1506 1507 1508 1509 1510 1511 1512 1513
static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
{
	unsigned long *bitlock = &lo->plh_flags;

	clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
	smp_mb__after_atomic();
	wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
}

1514 1515 1516 1517
/*
 * Layout segment is retreived from the server if not cached.
 * The appropriate layout segment is referenced and returned to the caller.
 */
1518
struct pnfs_layout_segment *
1519 1520
pnfs_update_layout(struct inode *ino,
		   struct nfs_open_context *ctx,
1521 1522
		   loff_t pos,
		   u64 count,
1523
		   enum pnfs_iomode iomode,
1524
		   bool strict_iomode,
1525
		   gfp_t gfp_flags)
1526
{
1527 1528 1529 1530 1531
	struct pnfs_layout_range arg = {
		.iomode = iomode,
		.offset = pos,
		.length = count,
	};
1532
	unsigned pg_offset, seq;
1533 1534
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs_client *clp = server->nfs_client;
1535
	struct pnfs_layout_hdr *lo = NULL;
1536
	struct pnfs_layout_segment *lseg = NULL;
1537 1538
	nfs4_stateid stateid;
	long timeout = 0;
1539
	unsigned long giveup = jiffies + (clp->cl_lease_time << 1);
1540
	bool first;
1541

1542
	if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
1543
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1544
				 PNFS_UPDATE_LAYOUT_NO_PNFS);
1545
		goto out;
1546
	}
1547

1548
	if (iomode == IOMODE_READ && i_size_read(ino) == 0) {
1549
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1550
				 PNFS_UPDATE_LAYOUT_RD_ZEROLEN);
1551
		goto out;
1552
	}
1553

1554
	if (pnfs_within_mdsthreshold(ctx, ino, iomode)) {
1555
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1556
				 PNFS_UPDATE_LAYOUT_MDSTHRESH);
1557
		goto out;
1558
	}
1559

1560
lookup_again:
1561
	nfs4_client_recover_expired_lease(clp);
1562
	first = false;
1563
	spin_lock(&ino->i_lock);
1564
	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
1565 1566
	if (lo == NULL) {
		spin_unlock(&ino->i_lock);
1567
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1568
				 PNFS_UPDATE_LAYOUT_NOMEM);
1569 1570
		goto out;
	}
1571

F
Fred Isaman 已提交
1572
	/* Do we even need to bother with this? */
1573
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
1574
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1575
				 PNFS_UPDATE_LAYOUT_BULK_RECALL);
F
Fred Isaman 已提交
1576
		dprintk("%s matches recall, use MDS\n", __func__);
1577 1578 1579 1580
		goto out_unlock;
	}

	/* if LAYOUTGET already failed once we don't try again */
1581
	if (pnfs_layout_io_test_failed(lo, iomode)) {
1582
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1583
				 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL);
1584
		goto out_unlock;
1585
	}
1586

1587
	lseg = pnfs_find_lseg(lo, &arg, strict_iomode);
1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604
	if (lseg) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				PNFS_UPDATE_LAYOUT_FOUND_CACHED);
		goto out_unlock;
	}

	if (!nfs4_valid_open_stateid(ctx->state)) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				PNFS_UPDATE_LAYOUT_INVALID_OPEN);
		goto out_unlock;
	}

	/*
	 * Choose a stateid for the LAYOUTGET. If we don't have a layout
	 * stateid, or it has been invalidated, then we must use the open
	 * stateid.
	 */
1605
	if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
1606 1607 1608

		/*
		 * The first layoutget for the file. Need to serialize per
1609 1610 1611 1612 1613 1614 1615 1616
		 * RFC 5661 Errata 3208.
		 */
		if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
				     &lo->plh_flags)) {
			spin_unlock(&ino->i_lock);
			wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
				    TASK_UNINTERRUPTIBLE);
			pnfs_put_layout_hdr(lo);
1617
			dprintk("%s retrying\n", __func__);
1618 1619
			goto lookup_again;
		}
1620 1621 1622 1623 1624 1625

		first = true;
		do {
			seq = read_seqbegin(&ctx->state->seqlock);
			nfs4_stateid_copy(&stateid, &ctx->state->stateid);
		} while (read_seqretry(&ctx->state->seqlock, seq));
1626
	} else {
1627
		nfs4_stateid_copy(&stateid, &lo->plh_stateid);
1628
	}
1629

1630 1631 1632 1633
	/*
	 * Because we free lsegs before sending LAYOUTRETURN, we need to wait
	 * for LAYOUTRETURN even if first is true.
	 */
1634
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
1635 1636 1637
		spin_unlock(&ino->i_lock);
		dprintk("%s wait for layoutreturn\n", __func__);
		if (pnfs_prepare_to_retry_layoutget(lo)) {
1638 1639
			if (first)
				pnfs_clear_first_layoutget(lo);
1640 1641
			pnfs_put_layout_hdr(lo);
			dprintk("%s retrying\n", __func__);
1642 1643
			trace_pnfs_update_layout(ino, pos, count, iomode, lo,
					lseg, PNFS_UPDATE_LAYOUT_RETRY);
1644 1645
			goto lookup_again;
		}
1646
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1647
				PNFS_UPDATE_LAYOUT_RETURN);
1648 1649 1650
		goto out_put_layout_hdr;
	}

1651
	if (pnfs_layoutgets_blocked(lo)) {
1652
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1653
				PNFS_UPDATE_LAYOUT_BLOCKED);
1654
		goto out_unlock;
1655
	}
1656
	atomic_inc(&lo->plh_outstanding);
1657
	spin_unlock(&ino->i_lock);
1658

1659
	if (list_empty(&lo->plh_layouts)) {
1660 1661 1662 1663
		/* The lo must be on the clp list if there is any
		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
		 */
		spin_lock(&clp->cl_lock);
1664 1665
		if (list_empty(&lo->plh_layouts))
			list_add_tail(&lo->plh_layouts, &server->layouts);
1666 1667
		spin_unlock(&clp->cl_lock);
	}
1668

1669
	pg_offset = arg.offset & ~PAGE_MASK;
1670 1671 1672 1673
	if (pg_offset) {
		arg.offset -= pg_offset;
		arg.length += pg_offset;
	}
1674
	if (arg.length != NFS4_MAX_UINT64)
1675
		arg.length = PAGE_ALIGN(arg.length);
1676

1677 1678 1679
	lseg = send_layoutget(lo, ctx, &stateid, &arg, &timeout, gfp_flags);
	trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
1680
	atomic_dec(&lo->plh_outstanding);
1681
	if (IS_ERR(lseg)) {
1682
		switch(PTR_ERR(lseg)) {
1683
		case -EBUSY:
1684 1685
			if (time_after(jiffies, giveup))
				lseg = NULL;
1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
			break;
		case -ERECALLCONFLICT:
			/* Huh? We hold no layouts, how is there a recall? */
			if (first) {
				lseg = NULL;
				break;
			}
			/* Destroy the existing layout and start over */
			if (time_after(jiffies, giveup))
				pnfs_destroy_layout(NFS_I(ino));
1696 1697
			/* Fallthrough */
		case -EAGAIN:
1698
			break;
1699 1700 1701 1702 1703
		default:
			if (!nfs_error_is_fatal(PTR_ERR(lseg))) {
				pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
				lseg = NULL;
			}
1704 1705 1706 1707 1708 1709 1710 1711 1712
			goto out_put_layout_hdr;
		}
		if (lseg) {
			if (first)
				pnfs_clear_first_layoutget(lo);
			trace_pnfs_update_layout(ino, pos, count,
				iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY);
			pnfs_put_layout_hdr(lo);
			goto lookup_again;
1713 1714 1715 1716 1717
		}
	} else {
		pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
	}

1718
out_put_layout_hdr:
1719 1720
	if (first)
		pnfs_clear_first_layoutget(lo);
1721
	pnfs_put_layout_hdr(lo);
1722
out:
1723 1724 1725 1726
	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
			"(%s, offset: %llu, length: %llu)\n",
			__func__, ino->i_sb->s_id,
			(unsigned long long)NFS_FILEID(ino),
1727
			IS_ERR_OR_NULL(lseg) ? "not found" : "found",
1728 1729 1730
			iomode==IOMODE_RW ?  "read/write" : "read-only",
			(unsigned long long)pos,
			(unsigned long long)count);
1731 1732 1733
	return lseg;
out_unlock:
	spin_unlock(&ino->i_lock);
1734
	goto out_put_layout_hdr;
1735
}
1736
EXPORT_SYMBOL_GPL(pnfs_update_layout);
1737

1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757
static bool
pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
{
	switch (range->iomode) {
	case IOMODE_READ:
	case IOMODE_RW:
		break;
	default:
		return false;
	}
	if (range->offset == NFS4_MAX_UINT64)
		return false;
	if (range->length == 0)
		return false;
	if (range->length != NFS4_MAX_UINT64 &&
	    range->length > NFS4_MAX_UINT64 - range->offset)
		return false;
	return true;
}

1758
struct pnfs_layout_segment *
1759 1760 1761 1762 1763
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
	struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
	struct nfs4_layoutget_res *res = &lgp->res;
	struct pnfs_layout_segment *lseg;
1764
	struct inode *ino = lo->plh_inode;
1765
	LIST_HEAD(free_me);
1766 1767

	if (!pnfs_sanity_check_layout_range(&res->range))
1768
		return ERR_PTR(-EINVAL);
1769 1770

	/* Inject layout blob into I/O device driver */
1771
	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
1772
	if (IS_ERR_OR_NULL(lseg)) {
1773
		if (!lseg)
1774 1775 1776 1777 1778
			lseg = ERR_PTR(-ENOMEM);

		dprintk("%s: Could not allocate layout: error %ld\n",
		       __func__, PTR_ERR(lseg));
		return lseg;
1779 1780
	}

1781
	pnfs_init_lseg(lo, lseg, &res->range, &res->stateid);
1782

1783
	spin_lock(&ino->i_lock);
1784
	if (pnfs_layoutgets_blocked(lo)) {
F
Fred Isaman 已提交
1785
		dprintk("%s forget reply due to state\n", __func__);
1786
		goto out_forget;
F
Fred Isaman 已提交
1787
	}
1788

1789 1790 1791 1792
	if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
		/* existing state ID, make sure the sequence number matches. */
		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
			dprintk("%s forget reply due to sequence\n", __func__);
1793
			goto out_forget;
1794 1795 1796 1797 1798 1799 1800 1801
		}
		pnfs_set_layout_stateid(lo, &res->stateid, false);
	} else {
		/*
		 * We got an entirely new state ID.  Mark all segments for the
		 * inode invalid, and don't bother validating the stateid
		 * sequence number.
		 */
1802
		pnfs_mark_layout_stateid_invalid(lo, &free_me);
1803 1804 1805 1806

		nfs4_stateid_copy(&lo->plh_stateid, &res->stateid);
		lo->plh_barrier = be32_to_cpu(res->stateid.seqid);
	}
1807

1808
	pnfs_get_lseg(lseg);
1809
	pnfs_layout_insert_lseg(lo, lseg, &free_me);
1810 1811 1812 1813 1814
	if (!pnfs_layout_is_valid(lo)) {
		pnfs_clear_layoutreturn_info(lo);
		clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
	}

1815

P
Peng Tao 已提交
1816
	if (res->return_on_close)
F
Fred Isaman 已提交
1817 1818
		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);

1819
	spin_unlock(&ino->i_lock);
1820
	pnfs_free_lseg_list(&free_me);
1821
	return lseg;
F
Fred Isaman 已提交
1822

1823
out_forget:
F
Fred Isaman 已提交
1824 1825 1826
	spin_unlock(&ino->i_lock);
	lseg->pls_layout = lo;
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
1827
	return ERR_PTR(-EAGAIN);
1828 1829
}

1830
static void
1831 1832
pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode,
			 u32 seq)
1833
{
1834
	if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode)
1835 1836
		iomode = IOMODE_ANY;
	lo->plh_return_iomode = iomode;
1837
	set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
1838 1839
	if (seq != 0) {
		WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq);
1840
		lo->plh_return_seq = seq;
1841
	}
1842 1843
}

1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
/**
 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments
 * @lo: pointer to layout header
 * @tmp_list: list header to be used with pnfs_free_lseg_list()
 * @return_range: describe layout segment ranges to be returned
 *
 * This function is mainly intended for use by layoutrecall. It attempts
 * to free the layout segment immediately, or else to mark it for return
 * as soon as its reference count drops to zero.
 */
1854
int
1855 1856
pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
				struct list_head *tmp_list,
1857 1858
				const struct pnfs_layout_range *return_range,
				u32 seq)
1859 1860
{
	struct pnfs_layout_segment *lseg, *next;
1861
	int remaining = 0;
1862 1863 1864 1865

	dprintk("%s:Begin lo %p\n", __func__, lo);

	if (list_empty(&lo->plh_segs))
1866
		return 0;
1867

1868
	assert_spin_locked(&lo->plh_inode->i_lock);
1869 1870

	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
1871
		if (pnfs_match_lseg_recall(lseg, return_range, seq)) {
1872 1873 1874 1875 1876
			dprintk("%s: marking lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode,
				lseg->pls_range.offset,
				lseg->pls_range.length);
1877 1878 1879
			if (mark_lseg_invalid(lseg, tmp_list))
				continue;
			remaining++;
1880 1881
			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
		}
1882 1883 1884 1885

	if (remaining)
		pnfs_set_plh_return_info(lo, return_range->iomode, seq);

1886
	return remaining;
1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898
}

void pnfs_error_mark_layout_for_return(struct inode *inode,
				       struct pnfs_layout_segment *lseg)
{
	struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
	struct pnfs_layout_range range = {
		.iomode = lseg->pls_range.iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(free_me);
1899
	bool return_now = false;
1900 1901

	spin_lock(&inode->i_lock);
1902
	pnfs_set_plh_return_info(lo, range.iomode, 0);
1903 1904 1905 1906 1907
	/*
	 * mark all matching lsegs so that we are sure to have no live
	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
	 * for how it works.
	 */
1908
	if (!pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0)) {
1909
		nfs4_stateid stateid;
1910
		enum pnfs_iomode iomode;
1911

1912
		return_now = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
1913 1914 1915 1916 1917 1918 1919
		spin_unlock(&inode->i_lock);
		if (return_now)
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
	} else {
		spin_unlock(&inode->i_lock);
		nfs_commit_inode(inode, 0);
	}
1920 1921 1922 1923
	pnfs_free_lseg_list(&free_me);
}
EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);

1924 1925 1926
void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
P
Peng Tao 已提交
1927 1928
	u64 rd_size = req->wb_bytes;

1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939
	if (pgio->pg_lseg == NULL) {
		if (pgio->pg_dreq == NULL)
			rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
		else
			rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);

		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   rd_size,
						   IOMODE_READ,
1940
						   false,
1941
						   GFP_KERNEL);
1942 1943 1944 1945 1946
		if (IS_ERR(pgio->pg_lseg)) {
			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
			pgio->pg_lseg = NULL;
			return;
		}
1947
	}
1948 1949
	/* If no lseg, fall back to read through mds */
	if (pgio->pg_lseg == NULL)
1950
		nfs_pageio_reset_read_mds(pgio);
1951

1952 1953 1954 1955
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);

void
1956 1957
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
			   struct nfs_page *req, u64 wb_size)
1958
{
1959
	if (pgio->pg_lseg == NULL) {
1960 1961 1962 1963 1964
		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   wb_size,
						   IOMODE_RW,
1965
						   false,
1966
						   GFP_NOFS);
1967 1968 1969 1970 1971 1972
		if (IS_ERR(pgio->pg_lseg)) {
			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
			pgio->pg_lseg = NULL;
			return;
		}
	}
1973 1974
	/* If no lseg, fall back to write through mds */
	if (pgio->pg_lseg == NULL)
1975
		nfs_pageio_reset_write_mds(pgio);
1976 1977 1978
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);

1979 1980 1981 1982 1983 1984 1985 1986 1987 1988
void
pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
{
	if (desc->pg_lseg) {
		pnfs_put_lseg(desc->pg_lseg);
		desc->pg_lseg = NULL;
	}
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);

1989 1990 1991 1992 1993
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
size_t
1994 1995
pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
		     struct nfs_page *prev, struct nfs_page *req)
1996
{
1997
	unsigned int size;
1998
	u64 seg_end, req_start, seg_left;
1999 2000 2001 2002

	size = nfs_generic_pg_test(pgio, prev, req);
	if (!size)
		return 0;
2003

2004
	/*
2005 2006 2007 2008 2009
	 * 'size' contains the number of bytes left in the current page (up
	 * to the original size asked for in @req->wb_bytes).
	 *
	 * Calculate how many bytes are left in the layout segment
	 * and if there are less bytes than 'size', return that instead.
2010 2011 2012 2013 2014
	 *
	 * Please also note that 'end_offset' is actually the offset of the
	 * first byte that lies outside the pnfs_layout_range. FIXME?
	 *
	 */
2015
	if (pgio->pg_lseg) {
2016 2017 2018
		seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
				     pgio->pg_lseg->pls_range.length);
		req_start = req_offset(req);
2019
		WARN_ON_ONCE(req_start >= seg_end);
2020
		/* start of request is past the last byte of this segment */
2021 2022 2023 2024 2025 2026
		if (req_start >= seg_end) {
			/* reference the new lseg */
			if (pgio->pg_ops->pg_cleanup)
				pgio->pg_ops->pg_cleanup(pgio);
			if (pgio->pg_ops->pg_init)
				pgio->pg_ops->pg_init(pgio, req);
2027
			return 0;
2028
		}
2029 2030 2031 2032 2033 2034

		/* adjust 'size' iff there are fewer bytes left in the
		 * segment than what nfs_generic_pg_test returned */
		seg_left = seg_end - req_start;
		if (seg_left < size)
			size = (unsigned int)seg_left;
2035
	}
2036

2037
	return size;
2038
}
2039
EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
2040

2041
int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
2042 2043 2044 2045
{
	struct nfs_pageio_descriptor pgio;

	/* Resend all requests through the MDS */
2046 2047
	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
			      hdr->completion_ops);
2048
	set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags);
2049
	return nfs_pageio_resend(&pgio, hdr);
2050
}
2051
EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
2052

2053
static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
2054
{
2055 2056 2057

	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2058
	    PNFS_LAYOUTRET_ON_ERROR) {
2059
		pnfs_return_layout(hdr->inode);
2060
	}
2061
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2062
		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
2063 2064
}

2065 2066 2067
/*
 * Called by non rpc-based layout drivers
 */
2068
void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
2069
{
2070
	if (likely(!hdr->pnfs_error)) {
2071 2072
		pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
				hdr->mds_offset + hdr->res.count);
2073
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2074 2075 2076
	}
	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2077 2078
		pnfs_ld_handle_write_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2079
}
2080
EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
2081

2082 2083
static void
pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
2084
		struct nfs_pgio_header *hdr)
2085
{
2086
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2087

2088
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2089
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2090
		nfs_pageio_reset_write_mds(desc);
2091
		mirror->pg_recoalesce = 1;
2092
	}
2093
	nfs_pgio_data_destroy(hdr);
2094
	hdr->release(hdr);
2095 2096 2097
}

static enum pnfs_try_status
2098
pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
2099 2100 2101
			const struct rpc_call_ops *call_ops,
			struct pnfs_layout_segment *lseg,
			int how)
2102
{
2103
	struct inode *inode = hdr->inode;
2104 2105 2106
	enum pnfs_try_status trypnfs;
	struct nfs_server *nfss = NFS_SERVER(inode);

2107
	hdr->mds_ops = call_ops;
2108 2109

	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
2110 2111
		inode->i_ino, hdr->args.count, hdr->args.offset, how);
	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
2112
	if (trypnfs != PNFS_NOT_ATTEMPTED)
2113 2114 2115 2116 2117
		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}

2118
static void
2119 2120
pnfs_do_write(struct nfs_pageio_descriptor *desc,
	      struct nfs_pgio_header *hdr, int how)
2121 2122 2123
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2124
	enum pnfs_try_status trypnfs;
2125

2126
	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
2127
	if (trypnfs == PNFS_NOT_ATTEMPTED)
2128
		pnfs_write_through_mds(desc, hdr);
2129 2130
}

2131 2132
static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
2133
	pnfs_put_lseg(hdr->lseg);
2134
	nfs_pgio_header_free(hdr);
2135 2136
}

2137 2138 2139
int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
{
2140
	struct nfs_pgio_header *hdr;
2141 2142
	int ret;

2143 2144
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
P
Peng Tao 已提交
2145 2146
		desc->pg_error = -ENOMEM;
		return desc->pg_error;
2147
	}
2148
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
2149

2150
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2151
	ret = nfs_generic_pgio(desc, hdr);
2152
	if (!ret)
2153
		pnfs_do_write(desc, hdr, desc->pg_ioflags);
2154

2155
	return ret;
2156 2157 2158
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);

2159
int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
2160 2161 2162
{
	struct nfs_pageio_descriptor pgio;

2163
	/* Resend all requests through the MDS */
2164 2165
	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
2166
}
2167
EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
2168

2169
static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
2170
{
2171 2172
	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2173
	    PNFS_LAYOUTRET_ON_ERROR) {
2174
		pnfs_return_layout(hdr->inode);
2175
	}
2176
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2177
		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
2178 2179
}

2180 2181 2182
/*
 * Called by non rpc-based layout drivers
 */
2183
void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
2184
{
2185
	if (likely(!hdr->pnfs_error)) {
2186 2187
		__nfs4_read_done_cb(hdr);
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2188 2189 2190
	}
	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2191 2192
		pnfs_ld_handle_read_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2193 2194 2195
}
EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

2196 2197
static void
pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
2198
		struct nfs_pgio_header *hdr)
2199
{
2200
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2201

2202
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2203
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2204
		nfs_pageio_reset_read_mds(desc);
2205
		mirror->pg_recoalesce = 1;
2206
	}
2207
	nfs_pgio_data_destroy(hdr);
2208
	hdr->release(hdr);
2209 2210
}

A
Andy Adamson 已提交
2211 2212 2213
/*
 * Call the appropriate parallel I/O subsystem read function.
 */
2214
static enum pnfs_try_status
2215
pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
2216 2217
		       const struct rpc_call_ops *call_ops,
		       struct pnfs_layout_segment *lseg)
A
Andy Adamson 已提交
2218
{
2219
	struct inode *inode = hdr->inode;
A
Andy Adamson 已提交
2220 2221 2222
	struct nfs_server *nfss = NFS_SERVER(inode);
	enum pnfs_try_status trypnfs;

2223
	hdr->mds_ops = call_ops;
A
Andy Adamson 已提交
2224 2225

	dprintk("%s: Reading ino:%lu %u@%llu\n",
2226
		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
A
Andy Adamson 已提交
2227

2228
	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
2229
	if (trypnfs != PNFS_NOT_ATTEMPTED)
A
Andy Adamson 已提交
2230 2231 2232 2233
		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}
A
Andy Adamson 已提交
2234

2235
/* Resend all requests through pnfs. */
2236
void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
2237 2238 2239
{
	struct nfs_pageio_descriptor pgio;

2240 2241 2242 2243 2244
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
		nfs_pageio_init_read(&pgio, hdr->inode, false,
					hdr->completion_ops);
		hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr);
	}
2245 2246 2247
}
EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);

2248
static void
2249
pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
2250 2251 2252
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2253
	enum pnfs_try_status trypnfs;
2254

2255
	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
2256
	if (trypnfs == PNFS_TRY_AGAIN)
2257 2258
		pnfs_read_resend_pnfs(hdr);
	if (trypnfs == PNFS_NOT_ATTEMPTED || hdr->task.tk_status)
2259
		pnfs_read_through_mds(desc, hdr);
2260 2261
}

2262 2263
static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
2264
	pnfs_put_lseg(hdr->lseg);
2265
	nfs_pgio_header_free(hdr);
2266 2267
}

2268 2269 2270
int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
{
2271
	struct nfs_pgio_header *hdr;
2272 2273
	int ret;

2274 2275
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
P
Peng Tao 已提交
2276 2277
		desc->pg_error = -ENOMEM;
		return desc->pg_error;
2278
	}
2279
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
2280
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2281
	ret = nfs_generic_pgio(desc, hdr);
2282
	if (!ret)
2283
		pnfs_do_read(desc, hdr);
2284
	return ret;
2285 2286 2287
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);

2288 2289 2290 2291 2292
static void pnfs_clear_layoutcommitting(struct inode *inode)
{
	unsigned long *bitlock = &NFS_I(inode)->flags;

	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
2293
	smp_mb__after_atomic();
2294 2295 2296
	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}

A
Andy Adamson 已提交
2297
/*
2298
 * There can be multiple RW segments.
A
Andy Adamson 已提交
2299
 */
2300
static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
A
Andy Adamson 已提交
2301
{
2302
	struct pnfs_layout_segment *lseg;
A
Andy Adamson 已提交
2303

2304 2305
	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
		if (lseg->pls_range.iomode == IOMODE_RW &&
2306
		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
2307 2308
			list_add(&lseg->pls_lc_list, listp);
	}
A
Andy Adamson 已提交
2309 2310
}

2311 2312 2313 2314 2315 2316 2317 2318 2319 2320
static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
{
	struct pnfs_layout_segment *lseg, *tmp;

	/* Matched by references in pnfs_set_layoutcommit */
	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
		list_del_init(&lseg->pls_lc_list);
		pnfs_put_lseg(lseg);
	}

2321
	pnfs_clear_layoutcommitting(inode);
2322 2323
}

P
Peng Tao 已提交
2324 2325
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
2326
	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
P
Peng Tao 已提交
2327 2328 2329
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

A
Andy Adamson 已提交
2330
void
2331 2332
pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
		loff_t end_pos)
A
Andy Adamson 已提交
2333
{
2334
	struct nfs_inode *nfsi = NFS_I(inode);
2335
	bool mark_as_dirty = false;
A
Andy Adamson 已提交
2336

2337
	spin_lock(&inode->i_lock);
A
Andy Adamson 已提交
2338
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
2339
		nfsi->layout->plh_lwb = end_pos;
2340
		mark_as_dirty = true;
A
Andy Adamson 已提交
2341
		dprintk("%s: Set layoutcommit for inode %lu ",
2342
			__func__, inode->i_ino);
2343 2344
	} else if (end_pos > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = end_pos;
2345
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
2346
		/* references matched in nfs4_layoutcommit_release */
2347
		pnfs_get_lseg(lseg);
2348
	}
2349
	spin_unlock(&inode->i_lock);
2350
	dprintk("%s: lseg %p end_pos %llu\n",
2351
		__func__, lseg, nfsi->layout->plh_lwb);
2352 2353 2354 2355

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
2356
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2357 2358 2359
}
EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);

A
Andy Adamson 已提交
2360 2361 2362 2363 2364 2365
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
{
	struct nfs_server *nfss = NFS_SERVER(data->args.inode);

	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
2366
	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
A
Andy Adamson 已提交
2367 2368
}

2369 2370 2371 2372 2373 2374 2375 2376
/*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
 * data to disk to allow the server to recover the data if it crashes.
 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
 * is off, and a COMMIT is sent to a data server, or
 * if WRITEs to a data server return NFS_DATA_SYNC.
 */
A
Andy Adamson 已提交
2377
int
2378
pnfs_layoutcommit_inode(struct inode *inode, bool sync)
A
Andy Adamson 已提交
2379
{
2380
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
A
Andy Adamson 已提交
2381 2382 2383
	struct nfs4_layoutcommit_data *data;
	struct nfs_inode *nfsi = NFS_I(inode);
	loff_t end_pos;
2384
	int status;
A
Andy Adamson 已提交
2385

2386
	if (!pnfs_layoutcommit_outstanding(inode))
2387 2388
		return 0;

2389
	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
P
Peng Tao 已提交
2390

2391
	status = -EAGAIN;
P
Peng Tao 已提交
2392
	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2393 2394
		if (!sync)
			goto out;
2395
		status = wait_on_bit_lock_action(&nfsi->flags,
2396 2397 2398
				NFS_INO_LAYOUTCOMMITTING,
				nfs_wait_bit_killable,
				TASK_KILLABLE);
P
Peng Tao 已提交
2399
		if (status)
2400
			goto out;
P
Peng Tao 已提交
2401 2402
	}

2403 2404 2405 2406 2407 2408 2409
	status = -ENOMEM;
	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (!data)
		goto clear_layoutcommitting;

	status = 0;
2410
	spin_lock(&inode->i_lock);
2411 2412
	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		goto out_unlock;
2413

2414
	INIT_LIST_HEAD(&data->lseg_list);
2415
	pnfs_list_write_lseg(inode, &data->lseg_list);
A
Andy Adamson 已提交
2416

2417
	end_pos = nfsi->layout->plh_lwb;
A
Andy Adamson 已提交
2418

2419
	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
A
Andy Adamson 已提交
2420 2421 2422
	spin_unlock(&inode->i_lock);

	data->args.inode = inode;
2423
	data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
A
Andy Adamson 已提交
2424 2425 2426
	nfs_fattr_init(&data->fattr);
	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
	data->res.fattr = &data->fattr;
2427 2428 2429 2430
	if (end_pos != 0)
		data->args.lastbytewritten = end_pos - 1;
	else
		data->args.lastbytewritten = U64_MAX;
A
Andy Adamson 已提交
2431 2432
	data->res.server = NFS_SERVER(inode);

2433 2434 2435
	if (ld->prepare_layoutcommit) {
		status = ld->prepare_layoutcommit(&data->args);
		if (status) {
2436
			put_rpccred(data->cred);
2437
			spin_lock(&inode->i_lock);
2438 2439
			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
			if (end_pos > nfsi->layout->plh_lwb)
2440
				nfsi->layout->plh_lwb = end_pos;
2441
			goto out_unlock;
2442 2443 2444 2445
		}
	}


A
Andy Adamson 已提交
2446 2447
	status = nfs4_proc_layoutcommit(data, sync);
out:
P
Peng Tao 已提交
2448 2449
	if (status)
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2450 2451
	dprintk("<-- %s status %d\n", __func__, status);
	return status;
2452 2453
out_unlock:
	spin_unlock(&inode->i_lock);
P
Peng Tao 已提交
2454
	kfree(data);
2455 2456
clear_layoutcommitting:
	pnfs_clear_layoutcommitting(inode);
P
Peng Tao 已提交
2457
	goto out;
A
Andy Adamson 已提交
2458
}
2459
EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
2460

2461 2462 2463 2464 2465 2466 2467
int
pnfs_generic_sync(struct inode *inode, bool datasync)
{
	return pnfs_layoutcommit_inode(inode, true);
}
EXPORT_SYMBOL_GPL(pnfs_generic_sync);

2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478
struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
{
	struct nfs4_threshold *thp;

	thp = kzalloc(sizeof(*thp), GFP_NOFS);
	if (!thp) {
		dprintk("%s mdsthreshold allocation failed\n", __func__);
		return NULL;
	}
	return thp;
}
2479

2480
#if IS_ENABLED(CONFIG_NFS_V4_2)
2481
int
2482
pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
2483 2484 2485
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
	struct nfs_server *server = NFS_SERVER(inode);
2486
	struct nfs_inode *nfsi = NFS_I(inode);
2487 2488 2489 2490 2491 2492 2493
	struct nfs42_layoutstat_data *data;
	struct pnfs_layout_hdr *hdr;
	int status = 0;

	if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
		goto out;

2494 2495 2496
	if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
		goto out;

2497 2498 2499
	if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
		goto out;

2500 2501 2502
	spin_lock(&inode->i_lock);
	if (!NFS_I(inode)->layout) {
		spin_unlock(&inode->i_lock);
2503
		goto out_clear_layoutstats;
2504 2505 2506 2507 2508
	}
	hdr = NFS_I(inode)->layout;
	pnfs_get_layout_hdr(hdr);
	spin_unlock(&inode->i_lock);

2509
	data = kzalloc(sizeof(*data), gfp_flags);
2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
	if (!data) {
		status = -ENOMEM;
		goto out_put;
	}

	data->args.fh = NFS_FH(inode);
	data->args.inode = inode;
	status = ld->prepare_layoutstats(&data->args);
	if (status)
		goto out_free;

	status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);

out:
	dprintk("%s returns %d\n", __func__, status);
	return status;

out_free:
	kfree(data);
out_put:
	pnfs_put_layout_hdr(hdr);
2531
out_clear_layoutstats:
2532 2533 2534
	smp_mb__before_atomic();
	clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
	smp_mb__after_atomic();
2535 2536 2537
	goto out;
}
EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
2538
#endif
2539 2540 2541 2542

unsigned int layoutstats_timer;
module_param(layoutstats_timer, uint, 0644);
EXPORT_SYMBOL_GPL(layoutstats_timer);