pnfs.c 78.9 KB
Newer Older
R
Ricardo Labiaga 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 *  pNFS functions to call and manage layout drivers.
 *
 *  Copyright (c) 2002 [year of first publication]
 *  The Regents of the University of Michigan
 *  All Rights Reserved
 *
 *  Dean Hildebrand <dhildebz@umich.edu>
 *
 *  Permission is granted to use, copy, create derivative works, and
 *  redistribute this software and such derivative works for any purpose,
 *  so long as the name of the University of Michigan is not used in
 *  any advertising or publicity pertaining to the use or distribution
 *  of this software without specific, written prior authorization. If
 *  the above copyright notice or any other identification of the
 *  University of Michigan is included in any copy of any portion of
 *  this software, then the disclaimer below must also be included.
 *
 *  This software is provided as is, without representation or warranty
 *  of any kind either express or implied, including without limitation
 *  the implied warranties of merchantability, fitness for a particular
 *  purpose, or noninfringement.  The Regents of the University of
 *  Michigan shall not be liable for any damages, including special,
 *  indirect, incidental, or consequential damages, with respect to any
 *  claim arising out of or in connection with the use of the software,
 *  even if it has been or is hereafter advised of the possibility of
 *  such damages.
 */

#include <linux/nfs_fs.h>
31
#include <linux/nfs_page.h>
32
#include <linux/module.h>
33
#include <linux/sort.h>
34
#include "internal.h"
R
Ricardo Labiaga 已提交
35
#include "pnfs.h"
A
Andy Adamson 已提交
36
#include "iostat.h"
37
#include "nfs4trace.h"
38
#include "delegation.h"
39
#include "nfs42.h"
40
#include "nfs4_fs.h"
R
Ricardo Labiaga 已提交
41 42

#define NFSDBG_FACILITY		NFSDBG_PNFS
43
#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
R
Ricardo Labiaga 已提交
44

45 46 47 48 49 50 51 52 53 54 55 56
/* Locking:
 *
 * pnfs_spinlock:
 *      protects pnfs_modules_tbl.
 */
static DEFINE_SPINLOCK(pnfs_spinlock);

/*
 * pnfs_modules_tbl holds all pnfs modules
 */
static LIST_HEAD(pnfs_modules_tbl);

57
static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo);
58 59 60 61
static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
		struct list_head *free_me,
		const struct pnfs_layout_range *range,
		u32 seq);
62 63
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		                struct list_head *tmp_list);
64

65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
/* Return the registered pnfs layout driver module matching given id */
static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)
{
	struct pnfs_layoutdriver_type *local;

	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
		if (local->id == id)
			goto out;
	local = NULL;
out:
	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
	return local;
}

R
Ricardo Labiaga 已提交
80 81 82
static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)
{
83 84 85 86
	struct pnfs_layoutdriver_type *local;

	spin_lock(&pnfs_spinlock);
	local = find_pnfs_driver_locked(id);
87 88 89 90
	if (local != NULL && !try_module_get(local->owner)) {
		dprintk("%s: Could not grab reference on module\n", __func__);
		local = NULL;
	}
91 92
	spin_unlock(&pnfs_spinlock);
	return local;
R
Ricardo Labiaga 已提交
93 94 95 96 97
}

void
unset_pnfs_layoutdriver(struct nfs_server *nfss)
{
98 99 100
	if (nfss->pnfs_curr_ld) {
		if (nfss->pnfs_curr_ld->clear_layoutdriver)
			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
101 102 103
		/* Decrement the MDS count. Purge the deviceid cache if zero */
		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
			nfs4_deviceid_purge_client(nfss->nfs_client);
104
		module_put(nfss->pnfs_curr_ld->owner);
105
	}
R
Ricardo Labiaga 已提交
106 107 108
	nfss->pnfs_curr_ld = NULL;
}

109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
/*
 * When the server sends a list of layout types, we choose one in the order
 * given in the list below.
 *
 * FIXME: should this list be configurable in some fashion? module param?
 * 	  mount option? something else?
 */
static const u32 ld_prefs[] = {
	LAYOUT_SCSI,
	LAYOUT_BLOCK_VOLUME,
	LAYOUT_OSD2_OBJECTS,
	LAYOUT_FLEX_FILES,
	LAYOUT_NFSV4_1_FILES,
	0
};

static int
ld_cmp(const void *e1, const void *e2)
{
	u32 ld1 = *((u32 *)e1);
	u32 ld2 = *((u32 *)e2);
	int i;

	for (i = 0; ld_prefs[i] != 0; i++) {
		if (ld1 == ld_prefs[i])
			return -1;

		if (ld2 == ld_prefs[i])
			return 1;
	}
	return 0;
}

R
Ricardo Labiaga 已提交
142 143 144 145
/*
 * Try to set the server's pnfs module to the pnfs layout type specified by id.
 * Currently only one pNFS layout driver per filesystem is supported.
 *
146
 * @ids array of layout types supported by MDS.
R
Ricardo Labiaga 已提交
147 148
 */
void
149
set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
150
		      struct nfs_fsinfo *fsinfo)
R
Ricardo Labiaga 已提交
151 152
{
	struct pnfs_layoutdriver_type *ld_type = NULL;
153
	u32 id;
154
	int i;
R
Ricardo Labiaga 已提交
155

156 157
	if (fsinfo->nlayouttypes == 0)
		goto out_no_driver;
R
Ricardo Labiaga 已提交
158 159
	if (!(server->nfs_client->cl_exchange_flags &
		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
160 161
		printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n",
			__func__, server->nfs_client->cl_exchange_flags);
R
Ricardo Labiaga 已提交
162 163
		goto out_no_driver;
	}
164

165 166
	sort(fsinfo->layouttype, fsinfo->nlayouttypes,
		sizeof(*fsinfo->layouttype), ld_cmp, NULL);
167

168 169
	for (i = 0; i < fsinfo->nlayouttypes; i++) {
		id = fsinfo->layouttype[i];
R
Ricardo Labiaga 已提交
170
		ld_type = find_pnfs_driver(id);
171 172 173 174 175 176 177
		if (!ld_type) {
			request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX,
					id);
			ld_type = find_pnfs_driver(id);
		}
		if (ld_type)
			break;
R
Ricardo Labiaga 已提交
178
	}
179 180

	if (!ld_type) {
181
		dprintk("%s: No pNFS module found!\n", __func__);
182 183 184
		goto out_no_driver;
	}

R
Ricardo Labiaga 已提交
185
	server->pnfs_curr_ld = ld_type;
186 187
	if (ld_type->set_layoutdriver
	    && ld_type->set_layoutdriver(server, mntfh)) {
188 189
		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
			"driver %u.\n", __func__, id);
190 191 192
		module_put(ld_type->owner);
		goto out_no_driver;
	}
193 194
	/* Bump the MDS count */
	atomic_inc(&server->nfs_client->cl_mds_count);
195

R
Ricardo Labiaga 已提交
196 197 198 199 200 201 202
	dprintk("%s: pNFS module for %u set\n", __func__, id);
	return;

out_no_driver:
	dprintk("%s: Using NFSv4 I/O\n", __func__);
	server->pnfs_curr_ld = NULL;
}
203 204 205 206 207 208 209 210

int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	int status = -EINVAL;
	struct pnfs_layoutdriver_type *tmp;

	if (ld_type->id == 0) {
211
		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
212 213
		return status;
	}
214
	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
215
		printk(KERN_ERR "NFS: %s Layout driver must provide "
216 217 218
		       "alloc_lseg and free_lseg.\n", __func__);
		return status;
	}
219 220 221 222 223 224 225 226 227

	spin_lock(&pnfs_spinlock);
	tmp = find_pnfs_driver_locked(ld_type->id);
	if (!tmp) {
		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
		status = 0;
		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
			ld_type->name);
	} else {
228
		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
			__func__, ld_type->id);
	}
	spin_unlock(&pnfs_spinlock);

	return status;
}
EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);

void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
	spin_lock(&pnfs_spinlock);
	list_del(&ld_type->pnfs_tblid);
	spin_unlock(&pnfs_spinlock);
}
EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
246

247 248 249 250
/*
 * pNFS client layout cache
 */

251
/* Need to hold i_lock if caller does not already hold reference */
F
Fred Isaman 已提交
252
void
253
pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
254
{
255
	refcount_inc(&lo->plh_refcount);
256 257
}

258 259 260 261
static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
262
	return ld->alloc_layout_hdr(ino, gfp_flags);
263 264 265 266 267
}

static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
268 269 270 271 272 273 274 275 276 277
	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;

	if (!list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		spin_lock(&clp->cl_lock);
		list_del_init(&lo->plh_layouts);
		spin_unlock(&clp->cl_lock);
	}
278
	put_rpccred(lo->plh_lc_cred);
279
	return ld->free_layout_hdr(lo);
280 281
}

282
static void
283
pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
284
{
285
	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
286
	dprintk("%s: freeing layout cache %p\n", __func__, lo);
287 288 289 290
	nfsi->layout = NULL;
	/* Reset MDS Threshold I/O counters */
	nfsi->write_io = 0;
	nfsi->read_io = 0;
291 292
}

293
void
294
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
295
{
296
	struct inode *inode;
297

298 299 300
	if (!lo)
		return;
	inode = lo->plh_inode;
301 302
	pnfs_layoutreturn_before_put_layout_hdr(lo);

303
	if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
304 305
		if (!list_empty(&lo->plh_segs))
			WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
306
		pnfs_detach_layout_hdr(lo);
307
		spin_unlock(&inode->i_lock);
308
		pnfs_free_layout_hdr(lo);
309
	}
310 311
}

312 313 314 315 316 317 318 319 320 321 322 323 324 325
static void
pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode,
			 u32 seq)
{
	if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode)
		iomode = IOMODE_ANY;
	lo->plh_return_iomode = iomode;
	set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
	if (seq != 0) {
		WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq);
		lo->plh_return_seq = seq;
	}
}

326 327 328
static void
pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo)
{
329
	struct pnfs_layout_segment *lseg;
330 331 332
	lo->plh_return_iomode = 0;
	lo->plh_return_seq = 0;
	clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
333 334 335 336 337
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
		if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
			continue;
		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
	}
338 339
}

340 341 342 343 344 345 346 347 348
static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
{
	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
	clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags);
	smp_mb__after_atomic();
	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
}

349 350 351 352 353 354 355 356 357 358 359 360
static void
pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg,
		struct list_head *free_me)
{
	clear_bit(NFS_LSEG_ROC, &lseg->pls_flags);
	clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags))
		pnfs_lseg_dec_and_remove_zero(lseg, free_me);
	if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
		pnfs_lseg_dec_and_remove_zero(lseg, free_me);
}

361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
/*
 * Update the seqid of a layout stateid
 */
bool nfs4_refresh_layout_stateid(nfs4_stateid *dst, struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	bool ret = false;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo && nfs4_stateid_match_other(dst, &lo->plh_stateid)) {
		dst->seqid = lo->plh_stateid.seqid;
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

379 380 381 382 383 384 385
/*
 * Mark a pnfs_layout_hdr and all associated layout segments as invalid
 *
 * In order to continue using the pnfs_layout_hdr, a full recovery
 * is required.
 * Note that caller must hold inode->i_lock.
 */
386
int
387 388 389 390 391 392 393 394
pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
		struct list_head *lseg_list)
{
	struct pnfs_layout_range range = {
		.iomode = IOMODE_ANY,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
395
	struct pnfs_layout_segment *lseg, *next;
396 397

	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
398 399
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
		pnfs_clear_lseg_state(lseg, lseg_list);
400
	pnfs_clear_layoutreturn_info(lo);
401
	pnfs_free_returned_lsegs(lo, lseg_list, &range, 0);
402 403 404
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) &&
	    !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
		pnfs_clear_layoutreturn_waitbit(lo);
405
	return !list_empty(&lo->plh_segs);
406 407
}

408 409 410 411 412 413 414 415
static int
pnfs_iomode_to_fail_bit(u32 iomode)
{
	return iomode == IOMODE_RW ?
		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
}

static void
416
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
417
{
418
	lo->plh_retry_timestamp = jiffies;
419
	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
420
		refcount_inc(&lo->plh_refcount);
421 422 423 424 425 426
}

static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
{
	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
427
		refcount_dec(&lo->plh_refcount);
428 429 430 431 432 433
}

static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
	struct inode *inode = lo->plh_inode;
434 435 436 437 438 439
	struct pnfs_layout_range range = {
		.iomode = iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(head);
440 441 442

	spin_lock(&inode->i_lock);
	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
443
	pnfs_mark_matching_lsegs_invalid(lo, &head, &range, 0);
444
	spin_unlock(&inode->i_lock);
445
	pnfs_free_lseg_list(&head);
446 447 448 449 450 451 452
	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
			iomode == IOMODE_RW ?  "RW" : "READ");
}

static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
453
	unsigned long start, end;
454 455 456
	int fail_bit = pnfs_iomode_to_fail_bit(iomode);

	if (test_bit(fail_bit, &lo->plh_flags) == 0)
457 458 459 460 461
		return false;
	end = jiffies;
	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
		/* It is time to retry the failed layoutgets */
462
		pnfs_layout_clear_fail_bit(lo, fail_bit);
463 464 465
		return false;
	}
	return true;
466 467
}

468
static void
469 470 471
pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,
		const struct pnfs_layout_range *range,
		const nfs4_stateid *stateid)
472
{
473
	INIT_LIST_HEAD(&lseg->pls_list);
474
	INIT_LIST_HEAD(&lseg->pls_lc_list);
475
	refcount_set(&lseg->pls_refcount, 1);
476
	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
477
	lseg->pls_layout = lo;
478 479
	lseg->pls_range = *range;
	lseg->pls_seq = be32_to_cpu(stateid->seqid);
480 481
}

482
static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
483
{
484 485 486 487
	if (lseg != NULL) {
		struct inode *inode = lseg->pls_layout->plh_inode;
		NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg);
	}
488 489
}

F
Fred Isaman 已提交
490
static void
491 492
pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
F
Fred Isaman 已提交
493
{
494
	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
F
Fred Isaman 已提交
495
	list_del_init(&lseg->pls_list);
496
	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
497
	refcount_dec(&lo->plh_refcount);
498 499
	if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
		return;
500 501 502
	if (list_empty(&lo->plh_segs) &&
	    !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) &&
	    !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
503 504
		if (atomic_read(&lo->plh_outstanding) == 0)
			set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
505
		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
506
	}
F
Fred Isaman 已提交
507 508
}

509 510 511 512 513 514
static bool
pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
{
	if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
	    pnfs_layout_is_valid(lo)) {
515
		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
516 517 518 519 520 521
		list_move_tail(&lseg->pls_list, &lo->plh_return_segs);
		return true;
	}
	return false;
}

522
void
523
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
524
{
525
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
526 527 528 529 530
	struct inode *inode;

	if (!lseg)
		return;

531
	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
532
		refcount_read(&lseg->pls_refcount),
533
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
534

535 536
	lo = lseg->pls_layout;
	inode = lo->plh_inode;
537

538
	if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
539 540 541 542
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
			spin_unlock(&inode->i_lock);
			return;
		}
543
		pnfs_get_layout_hdr(lo);
544
		pnfs_layout_remove_lseg(lo, lseg);
545 546
		if (pnfs_cache_lseg_for_layoutreturn(lo, lseg))
			lseg = NULL;
547 548 549
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg(lseg);
		pnfs_put_layout_hdr(lo);
550 551
	}
}
552
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
553

554 555 556 557 558 559 560
/*
 * is l2 fully contained in l1?
 *   start1                             end1
 *   [----------------------------------)
 *           start2           end2
 *           [----------------)
 */
561
static bool
562
pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
563
		 const struct pnfs_layout_range *l2)
564 565
{
	u64 start1 = l1->offset;
566
	u64 end1 = pnfs_end_offset(start1, l1->length);
567
	u64 start2 = l2->offset;
568
	u64 end2 = pnfs_end_offset(start2, l2->length);
569 570 571 572

	return (start1 <= start2) && (end1 >= end2);
}

573 574 575
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		struct list_head *tmp_list)
{
576
	if (!refcount_dec_and_test(&lseg->pls_refcount))
577 578 579 580 581 582
		return false;
	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
	list_add(&lseg->pls_list, tmp_list);
	return true;
}

583 584 585 586 587 588 589 590 591 592 593
/* Returns 1 if lseg is removed from list, 0 otherwise */
static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
			     struct list_head *tmp_list)
{
	int rv = 0;

	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
		/* Remove the reference keeping the lseg in the
		 * list.  It will now be removed when all
		 * outstanding io is finished.
		 */
F
Fred Isaman 已提交
594
		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
595
			refcount_read(&lseg->pls_refcount));
596
		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
F
Fred Isaman 已提交
597
			rv = 1;
598 599 600 601
	}
	return rv;
}

602 603 604 605 606 607 608 609 610
/*
 * Compare 2 layout stateid sequence ids, to see which is newer,
 * taking into account wraparound issues.
 */
static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
{
	return (s32)(s1 - s2) > 0;
}

611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
static bool
pnfs_should_free_range(const struct pnfs_layout_range *lseg_range,
		 const struct pnfs_layout_range *recall_range)
{
	return (recall_range->iomode == IOMODE_ANY ||
		lseg_range->iomode == recall_range->iomode) &&
	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
}

static bool
pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg,
		const struct pnfs_layout_range *recall_range,
		u32 seq)
{
	if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq))
		return false;
	if (recall_range == NULL)
		return true;
	return pnfs_should_free_range(&lseg->pls_range, recall_range);
}

632 633 634 635 636 637 638 639 640 641 642 643 644 645
/**
 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later
 * @lo: layout header containing the lsegs
 * @tmp_list: list head where doomed lsegs should go
 * @recall_range: optional recall range argument to match (may be NULL)
 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0)
 *
 * Walk the list of lsegs in the layout header, and tear down any that should
 * be destroyed. If "recall_range" is specified then the segment must match
 * that range. If "seq" is non-zero, then only match segments that were handed
 * out at or before that sequence.
 *
 * Returns number of matching invalid lsegs remaining in list after scanning
 * it and purging them.
646
 */
F
Fred Isaman 已提交
647
int
648
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
649
			    struct list_head *tmp_list,
650 651
			    const struct pnfs_layout_range *recall_range,
			    u32 seq)
652 653
{
	struct pnfs_layout_segment *lseg, *next;
654
	int remaining = 0;
655 656 657

	dprintk("%s:Begin lo %p\n", __func__, lo);

658
	if (list_empty(&lo->plh_segs))
659
		return 0;
660
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
661
		if (pnfs_match_lseg_recall(lseg, recall_range, seq)) {
662
			dprintk("%s: freeing lseg %p iomode %d seq %u "
663
				"offset %llu length %llu\n", __func__,
664 665
				lseg, lseg->pls_range.iomode, lseg->pls_seq,
				lseg->pls_range.offset, lseg->pls_range.length);
666 667
			if (!mark_lseg_invalid(lseg, tmp_list))
				remaining++;
668
		}
669 670
	dprintk("%s:Return %i\n", __func__, remaining);
	return remaining;
671 672
}

673 674 675 676 677 678 679 680 681 682 683 684 685 686
static void
pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
		struct list_head *free_me,
		const struct pnfs_layout_range *range,
		u32 seq)
{
	struct pnfs_layout_segment *lseg, *next;

	list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) {
		if (pnfs_match_lseg_recall(lseg, range, seq))
			list_move_tail(&lseg->pls_list, free_me);
	}
}

687
/* note free_me must contain lsegs from a single layout_hdr */
F
Fred Isaman 已提交
688
void
689
pnfs_free_lseg_list(struct list_head *free_me)
690
{
691
	struct pnfs_layout_segment *lseg, *tmp;
692 693 694 695

	if (list_empty(free_me))
		return;

696
	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
697
		list_del(&lseg->pls_list);
698
		pnfs_free_lseg(lseg);
699 700 701
	}
}

702 703 704 705
void
pnfs_destroy_layout(struct nfs_inode *nfsi)
{
	struct pnfs_layout_hdr *lo;
706
	LIST_HEAD(tmp_list);
707 708 709 710

	spin_lock(&nfsi->vfs_inode.i_lock);
	lo = nfsi->layout;
	if (lo) {
711
		pnfs_get_layout_hdr(lo);
712
		pnfs_mark_layout_stateid_invalid(lo, &tmp_list);
713 714 715 716
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
		spin_unlock(&nfsi->vfs_inode.i_lock);
		pnfs_free_lseg_list(&tmp_list);
717
		nfs_commit_inode(&nfsi->vfs_inode, 0);
718 719 720
		pnfs_put_layout_hdr(lo);
	} else
		spin_unlock(&nfsi->vfs_inode.i_lock);
721
}
722
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
723

724 725 726
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
		struct list_head *layout_list)
727 728
{
	struct pnfs_layout_hdr *lo;
729
	bool ret = false;
730

731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
		pnfs_get_layout_hdr(lo);
		list_add(&lo->plh_bulk_destroy, layout_list);
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

/* Caller must hold rcu_read_lock and clp->cl_lock */
static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
		struct nfs_server *server,
		struct list_head *layout_list)
{
	struct pnfs_layout_hdr *lo, *next;
	struct inode *inode;

	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
752 753
		if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags))
			continue;
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
		inode = igrab(lo->plh_inode);
		if (inode == NULL)
			continue;
		list_del_init(&lo->plh_layouts);
		if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
			continue;
		rcu_read_unlock();
		spin_unlock(&clp->cl_lock);
		iput(inode);
		spin_lock(&clp->cl_lock);
		rcu_read_lock();
		return -EAGAIN;
	}
	return 0;
}

static int
pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
		bool is_bulk_recall)
{
	struct pnfs_layout_hdr *lo;
	struct inode *inode;
	LIST_HEAD(lseg_list);
	int ret = 0;

	while (!list_empty(layout_list)) {
		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
				plh_bulk_destroy);
		dprintk("%s freeing layout for inode %lu\n", __func__,
			lo->plh_inode->i_ino);
		inode = lo->plh_inode;
785 786 787

		pnfs_layoutcommit_inode(inode, false);

788 789
		spin_lock(&inode->i_lock);
		list_del_init(&lo->plh_bulk_destroy);
790 791 792
		if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) {
			if (is_bulk_recall)
				set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
793
			ret = -EAGAIN;
794
		}
795 796
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg_list(&lseg_list);
797 798
		/* Free all lsegs that are attached to commit buckets */
		nfs_commit_inode(inode, 0);
799 800 801 802 803 804 805 806 807 808 809 810 811
		pnfs_put_layout_hdr(lo);
		iput(inode);
	}
	return ret;
}

int
pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
		struct nfs_fsid *fsid,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);
812

813
	spin_lock(&clp->cl_lock);
814
	rcu_read_lock();
815
restart:
816
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
817 818 819 820 821 822
		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
			continue;
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
				server,
				&layout_list) != 0)
			goto restart;
823 824
	}
	rcu_read_unlock();
825 826
	spin_unlock(&clp->cl_lock);

827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

int
pnfs_destroy_layouts_byclid(struct nfs_client *clp,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);

	spin_lock(&clp->cl_lock);
	rcu_read_lock();
restart:
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
					server,
					&layout_list) != 0)
			goto restart;
847
	}
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866
	rcu_read_unlock();
	spin_unlock(&clp->cl_lock);

	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

/*
 * Called by the state manger to remove all layouts established under an
 * expired lease.
 */
void
pnfs_destroy_all_layouts(struct nfs_client *clp)
{
	nfs4_deviceid_mark_client_invalid(clp);
	nfs4_deviceid_purge_client(clp);

	pnfs_destroy_layouts_byclid(clp, false);
867 868
}

869
/* update lo->plh_stateid with new if is more recent */
F
Fred Isaman 已提交
870 871 872
void
pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
			bool update_barrier)
873
{
874
	u32 oldseq, newseq, new_barrier = 0;
875

876 877
	oldseq = be32_to_cpu(lo->plh_stateid.seqid);
	newseq = be32_to_cpu(new->seqid);
878 879 880 881 882 883 884 885 886

	if (!pnfs_layout_is_valid(lo)) {
		nfs4_stateid_copy(&lo->plh_stateid, new);
		lo->plh_barrier = newseq;
		pnfs_clear_layoutreturn_info(lo);
		clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
		return;
	}
	if (pnfs_seqid_is_newer(newseq, oldseq)) {
887
		nfs4_stateid_copy(&lo->plh_stateid, new);
888 889 890 891 892
		/*
		 * Because of wraparound, we want to keep the barrier
		 * "close" to the current seqids.
		 */
		new_barrier = newseq - atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
893
	}
894 895 896 897
	if (update_barrier)
		new_barrier = be32_to_cpu(new->seqid);
	else if (new_barrier == 0)
		return;
898
	if (pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
899
		lo->plh_barrier = new_barrier;
900 901
}

902
static bool
903 904
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid)
F
Fred Isaman 已提交
905
{
906
	u32 seqid = be32_to_cpu(stateid->seqid);
907

908 909 910 911 912
	return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
}

/* lget is set to 1 if called from inside send_layoutget call chain */
static bool
913
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
914
{
F
Fred Isaman 已提交
915
	return lo->plh_block_lgets ||
916
		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
917 918
}

919 920 921 922 923
static struct nfs_server *
pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx)
{
	struct nfs_server *server;

924
	if (inode) {
925
		server = NFS_SERVER(inode);
926
	} else {
927 928 929 930 931 932 933
		struct dentry *parent_dir = dget_parent(ctx->dentry);
		server = NFS_SERVER(parent_dir->d_inode);
		dput(parent_dir);
	}
	return server;
}

934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971
static void nfs4_free_pages(struct page **pages, size_t size)
{
	int i;

	if (!pages)
		return;

	for (i = 0; i < size; i++) {
		if (!pages[i])
			break;
		__free_page(pages[i]);
	}
	kfree(pages);
}

static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags)
{
	struct page **pages;
	int i;

	pages = kcalloc(size, sizeof(struct page *), gfp_flags);
	if (!pages) {
		dprintk("%s: can't alloc array of %zu pages\n", __func__, size);
		return NULL;
	}

	for (i = 0; i < size; i++) {
		pages[i] = alloc_page(gfp_flags);
		if (!pages[i]) {
			dprintk("%s: failed to allocate page\n", __func__);
			nfs4_free_pages(pages, size);
			return NULL;
		}
	}

	return pages;
}

F
Fred Isaman 已提交
972
static struct nfs4_layoutget *
973
pnfs_alloc_init_layoutget_args(struct inode *ino,
974
	   struct nfs_open_context *ctx,
975
	   const nfs4_stateid *stateid,
976
	   const struct pnfs_layout_range *range,
F
Fred Isaman 已提交
977
	   gfp_t gfp_flags)
978
{
979
	struct nfs_server *server = pnfs_find_server(ino, ctx);
980
	size_t max_pages = max_response_pages(server);
981 982 983
	struct nfs4_layoutget *lgp;

	dprintk("--> %s\n", __func__);
984

985 986
	lgp = kzalloc(sizeof(*lgp), gfp_flags);
	if (lgp == NULL)
F
Fred Isaman 已提交
987
		return NULL;
988

989 990 991 992 993 994 995 996
	lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags);
	if (!lgp->args.layout.pages) {
		kfree(lgp);
		return NULL;
	}
	lgp->args.layout.pglen = max_pages * PAGE_SIZE;
	lgp->res.layoutp = &lgp->args.layout;

997 998
	/* Don't confuse uninitialised result and success */
	lgp->res.status = -NFS4ERR_DELAY;
999 1000 1001 1002

	lgp->args.minlength = PAGE_SIZE;
	if (lgp->args.minlength > range->length)
		lgp->args.minlength = range->length;
1003 1004 1005 1006 1007 1008 1009 1010 1011
	if (ino) {
		loff_t i_size = i_size_read(ino);

		if (range->iomode == IOMODE_READ) {
			if (range->offset >= i_size)
				lgp->args.minlength = 0;
			else if (i_size - range->offset < lgp->args.minlength)
				lgp->args.minlength = i_size - range->offset;
		}
1012
	}
1013 1014 1015 1016 1017
	lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
	pnfs_copy_range(&lgp->args.range, range);
	lgp->args.type = server->pnfs_curr_ld->id;
	lgp->args.inode = ino;
	lgp->args.ctx = get_nfs_open_context(ctx);
1018
	nfs4_stateid_copy(&lgp->args.stateid, stateid);
1019
	lgp->gfp_flags = gfp_flags;
1020
	lgp->cred = get_rpccred(ctx->cred);
1021
	lgp->callback_count = raw_seqcount_begin(&server->nfs_client->cl_callback_count);
F
Fred Isaman 已提交
1022
	return lgp;
1023 1024
}

1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
void pnfs_layoutget_free(struct nfs4_layoutget *lgp)
{
	size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE;

	nfs4_free_pages(lgp->args.layout.pages, max_pages);
	if (lgp->args.inode)
		pnfs_put_layout_hdr(NFS_I(lgp->args.inode)->layout);
	put_rpccred(lgp->cred);
	put_nfs_open_context(lgp->args.ctx);
	kfree(lgp);
}

1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
static void pnfs_clear_layoutcommit(struct inode *inode,
		struct list_head *head)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct pnfs_layout_segment *lseg, *tmp;

	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		return;
	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
			continue;
		pnfs_lseg_dec_and_remove_zero(lseg, head);
	}
}

1052
void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo,
1053
		const nfs4_stateid *arg_stateid,
1054 1055 1056 1057 1058 1059 1060
		const struct pnfs_layout_range *range,
		const nfs4_stateid *stateid)
{
	struct inode *inode = lo->plh_inode;
	LIST_HEAD(freeme);

	spin_lock(&inode->i_lock);
1061 1062 1063
	if (!pnfs_layout_is_valid(lo) || !arg_stateid ||
	    !nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid))
		goto out_unlock;
1064
	if (stateid) {
1065 1066
		u32 seq = be32_to_cpu(arg_stateid->seqid);

1067 1068 1069 1070 1071
		pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq);
		pnfs_free_returned_lsegs(lo, &freeme, range, seq);
		pnfs_set_layout_stateid(lo, stateid, true);
	} else
		pnfs_mark_layout_stateid_invalid(lo, &freeme);
1072
out_unlock:
1073 1074 1075 1076 1077 1078
	pnfs_clear_layoutreturn_waitbit(lo);
	spin_unlock(&inode->i_lock);
	pnfs_free_lseg_list(&freeme);

}

1079
static bool
1080 1081 1082
pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo,
		nfs4_stateid *stateid,
		enum pnfs_iomode *iomode)
1083
{
1084 1085 1086
	/* Serialise LAYOUTGET/LAYOUTRETURN */
	if (atomic_read(&lo->plh_outstanding) != 0)
		return false;
1087
	if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
1088
		return false;
1089
	set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
1090
	pnfs_get_layout_hdr(lo);
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
		if (stateid != NULL) {
			nfs4_stateid_copy(stateid, &lo->plh_stateid);
			if (lo->plh_return_seq != 0)
				stateid->seqid = cpu_to_be32(lo->plh_return_seq);
		}
		if (iomode != NULL)
			*iomode = lo->plh_return_iomode;
		pnfs_clear_layoutreturn_info(lo);
		return true;
	}
	if (stateid != NULL)
		nfs4_stateid_copy(stateid, &lo->plh_stateid);
	if (iomode != NULL)
		*iomode = IOMODE_ANY;
1106 1107 1108
	return true;
}

1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
static void
pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args,
		struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid,
		enum pnfs_iomode iomode)
{
	struct inode *inode = lo->plh_inode;

	args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id;
	args->inode = inode;
	args->range.iomode = iomode;
	args->range.offset = 0;
	args->range.length = NFS4_MAX_UINT64;
	args->layout = lo;
	nfs4_stateid_copy(&args->stateid, stateid);
}

1126
static int
1127
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
1128
		       enum pnfs_iomode iomode, bool sync)
1129 1130
{
	struct inode *ino = lo->plh_inode;
1131
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
1132 1133 1134
	struct nfs4_layoutreturn *lrp;
	int status = 0;

1135
	lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
1136 1137 1138
	if (unlikely(lrp == NULL)) {
		status = -ENOMEM;
		spin_lock(&ino->i_lock);
1139
		pnfs_clear_layoutreturn_waitbit(lo);
1140 1141 1142 1143 1144
		spin_unlock(&ino->i_lock);
		pnfs_put_layout_hdr(lo);
		goto out;
	}

1145
	pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode);
1146
	lrp->args.ld_private = &lrp->ld_private;
1147 1148
	lrp->clp = NFS_SERVER(ino)->nfs_client;
	lrp->cred = lo->plh_lc_cred;
1149 1150
	if (ld->prepare_layoutreturn)
		ld->prepare_layoutreturn(&lrp->args);
1151

1152
	status = nfs4_proc_layoutreturn(lrp, sync);
1153 1154 1155 1156 1157
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

1158 1159 1160 1161 1162 1163
/* Return true if layoutreturn is needed */
static bool
pnfs_layout_need_return(struct pnfs_layout_hdr *lo)
{
	struct pnfs_layout_segment *s;

1164
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
		return false;

	/* Defer layoutreturn until all lsegs are done */
	list_for_each_entry(s, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
			return false;
	}

	return true;
}

static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo)
{
	struct inode *inode= lo->plh_inode;

1180
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1181 1182 1183 1184 1185 1186 1187
		return;
	spin_lock(&inode->i_lock);
	if (pnfs_layout_need_return(lo)) {
		nfs4_stateid stateid;
		enum pnfs_iomode iomode;
		bool send;

1188
		send = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
1189 1190 1191 1192 1193 1194 1195 1196 1197
		spin_unlock(&inode->i_lock);
		if (send) {
			/* Send an async layoutreturn so we dont deadlock */
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
		}
	} else
		spin_unlock(&inode->i_lock);
}

1198 1199 1200 1201 1202 1203 1204 1205
/*
 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
 * when the layout segment list is empty.
 *
 * Note that a pnfs_layout_hdr can exist with an empty layout segment
 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
 * deviceid is marked invalid.
 */
B
Benny Halevy 已提交
1206 1207 1208 1209 1210 1211 1212
int
_pnfs_return_layout(struct inode *ino)
{
	struct pnfs_layout_hdr *lo = NULL;
	struct nfs_inode *nfsi = NFS_I(ino);
	LIST_HEAD(tmp_list);
	nfs4_stateid stateid;
1213
	int status = 0;
1214
	bool send;
B
Benny Halevy 已提交
1215

1216
	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
B
Benny Halevy 已提交
1217 1218 1219

	spin_lock(&ino->i_lock);
	lo = nfsi->layout;
1220
	if (!lo) {
B
Benny Halevy 已提交
1221
		spin_unlock(&ino->i_lock);
1222 1223
		dprintk("NFS: %s no layout to return\n", __func__);
		goto out;
B
Benny Halevy 已提交
1224 1225
	}
	/* Reference matched in nfs4_layoutreturn_release */
1226
	pnfs_get_layout_hdr(lo);
1227 1228 1229 1230 1231 1232 1233 1234
	/* Is there an outstanding layoutreturn ? */
	if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) {
		spin_unlock(&ino->i_lock);
		if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN,
					TASK_UNINTERRUPTIBLE))
			goto out_put_layout_hdr;
		spin_lock(&ino->i_lock);
	}
1235
	pnfs_clear_layoutcommit(ino, &tmp_list);
1236
	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0);
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246

	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
		struct pnfs_layout_range range = {
			.iomode		= IOMODE_ANY,
			.offset		= 0,
			.length		= NFS4_MAX_UINT64,
		};
		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
	}

1247
	/* Don't send a LAYOUTRETURN if list was initially empty */
1248
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
1249 1250
		spin_unlock(&ino->i_lock);
		dprintk("NFS: %s no layout segments to return\n", __func__);
1251
		goto out_put_layout_hdr;
1252
	}
1253

1254
	send = pnfs_prepare_layoutreturn(lo, &stateid, NULL);
B
Benny Halevy 已提交
1255
	spin_unlock(&ino->i_lock);
1256
	if (send)
1257
		status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1258
out_put_layout_hdr:
1259
	pnfs_free_lseg_list(&tmp_list);
1260
	pnfs_put_layout_hdr(lo);
B
Benny Halevy 已提交
1261 1262 1263 1264 1265
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
int
pnfs_commit_and_return_layout(struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	int ret;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo == NULL) {
		spin_unlock(&inode->i_lock);
		return 0;
	}
	pnfs_get_layout_hdr(lo);
	/* Block new layoutgets and read/write to ds */
	lo->plh_block_lgets++;
	spin_unlock(&inode->i_lock);
	filemap_fdatawait(inode->i_mapping);
	ret = pnfs_layoutcommit_inode(inode, true);
	if (ret == 0)
		ret = _pnfs_return_layout(inode);
	spin_lock(&inode->i_lock);
	lo->plh_block_lgets--;
	spin_unlock(&inode->i_lock);
	pnfs_put_layout_hdr(lo);
	return ret;
}

1293 1294 1295 1296
bool pnfs_roc(struct inode *ino,
		struct nfs4_layoutreturn_args *args,
		struct nfs4_layoutreturn_res *res,
		const struct rpc_cred *cred)
F
Fred Isaman 已提交
1297
{
1298 1299 1300
	struct nfs_inode *nfsi = NFS_I(ino);
	struct nfs_open_context *ctx;
	struct nfs4_state *state;
F
Fred Isaman 已提交
1301
	struct pnfs_layout_hdr *lo;
1302
	struct pnfs_layout_segment *lseg, *next;
1303
	nfs4_stateid stateid;
1304 1305
	enum pnfs_iomode iomode = 0;
	bool layoutreturn = false, roc = false;
1306
	bool skip_read = false;
F
Fred Isaman 已提交
1307

1308 1309
	if (!nfs_have_layout(ino))
		return false;
1310
retry:
F
Fred Isaman 已提交
1311
	spin_lock(&ino->i_lock);
1312
	lo = nfsi->layout;
1313
	if (!lo || !pnfs_layout_is_valid(lo) ||
1314 1315
	    test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
		lo = NULL;
1316
		goto out_noroc;
1317 1318
	}
	pnfs_get_layout_hdr(lo);
1319 1320 1321 1322 1323 1324 1325
	if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) {
		spin_unlock(&ino->i_lock);
		wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN,
				TASK_UNINTERRUPTIBLE);
		pnfs_put_layout_hdr(lo);
		goto retry;
	}
1326

1327
	/* no roc if we hold a delegation */
1328 1329 1330 1331 1332
	if (nfs4_check_delegation(ino, FMODE_READ)) {
		if (nfs4_check_delegation(ino, FMODE_WRITE))
			goto out_noroc;
		skip_read = true;
	}
1333 1334 1335

	list_for_each_entry(ctx, &nfsi->open_files, list) {
		state = ctx->state;
1336 1337
		if (state == NULL)
			continue;
1338
		/* Don't return layout if there is open file state */
1339
		if (state->state & FMODE_WRITE)
1340
			goto out_noroc;
1341 1342
		if (state->state & FMODE_READ)
			skip_read = true;
1343 1344
	}

1345

1346
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) {
1347 1348
		if (skip_read && lseg->pls_range.iomode == IOMODE_READ)
			continue;
1349
		/* If we are sending layoutreturn, invalidate all valid lsegs */
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
		if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags))
			continue;
		/*
		 * Note: mark lseg for return so pnfs_layout_remove_lseg
		 * doesn't invalidate the layout for us.
		 */
		set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
		if (!mark_lseg_invalid(lseg, &lo->plh_return_segs))
			continue;
		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
1360 1361
	}

1362 1363
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
		goto out_noroc;
1364

1365
	/* ROC in two conditions:
1366 1367 1368
	 * 1. there are ROC lsegs
	 * 2. we don't send layoutreturn
	 */
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378
	/* lo ref dropped in pnfs_roc_release() */
	layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
	/* If the creds don't match, we can't compound the layoutreturn */
	if (!layoutreturn || cred != lo->plh_lc_cred)
		goto out_noroc;

	roc = layoutreturn;
	pnfs_init_layoutreturn_args(args, lo, &stateid, iomode);
	res->lrs_present = 0;
	layoutreturn = false;
F
Fred Isaman 已提交
1379

1380
out_noroc:
F
Fred Isaman 已提交
1381
	spin_unlock(&ino->i_lock);
1382
	pnfs_layoutcommit_inode(ino, true);
1383 1384 1385 1386
	if (roc) {
		struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
		if (ld->prepare_layoutreturn)
			ld->prepare_layoutreturn(args);
1387
		pnfs_put_layout_hdr(lo);
1388 1389
		return true;
	}
1390
	if (layoutreturn)
1391
		pnfs_send_layoutreturn(lo, &stateid, iomode, true);
1392
	pnfs_put_layout_hdr(lo);
1393
	return false;
F
Fred Isaman 已提交
1394 1395
}

1396 1397 1398
void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
		struct nfs4_layoutreturn_res *res,
		int ret)
F
Fred Isaman 已提交
1399
{
1400 1401 1402
	struct pnfs_layout_hdr *lo = args->layout;
	const nfs4_stateid *arg_stateid = NULL;
	const nfs4_stateid *res_stateid = NULL;
1403
	struct nfs4_xdr_opaque_data *ld_private = args->ld_private;
F
Fred Isaman 已提交
1404

1405 1406 1407 1408 1409 1410 1411
	if (ret == 0) {
		arg_stateid = &args->stateid;
		if (res->lrs_present)
			res_stateid = &res->stateid;
	}
	pnfs_layoutreturn_free_lsegs(lo, arg_stateid, &args->range,
			res_stateid);
1412 1413
	if (ld_private && ld_private->ops && ld_private->ops->free)
		ld_private->ops->free(ld_private);
1414 1415
	pnfs_put_layout_hdr(lo);
	trace_nfs4_layoutreturn_on_close(args->inode, 0);
F
Fred Isaman 已提交
1416 1417
}

1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
{
	struct nfs_inode *nfsi = NFS_I(ino);
        struct pnfs_layout_hdr *lo;
        bool sleep = false;

	/* we might not have grabbed lo reference. so need to check under
	 * i_lock */
        spin_lock(&ino->i_lock);
        lo = nfsi->layout;
1428 1429
        if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
1430
                sleep = true;
1431
	}
1432 1433 1434 1435
        spin_unlock(&ino->i_lock);
        return sleep;
}

1436 1437 1438 1439 1440 1441
/*
 * Compare two layout segments for sorting into layout cache.
 * We want to preferentially return RW over RO layouts, so ensure those
 * are seen first.
 */
static s64
1442
pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1443
	   const struct pnfs_layout_range *l2)
1444
{
1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
	s64 d;

	/* high offset > low offset */
	d = l1->offset - l2->offset;
	if (d)
		return d;

	/* short length > long length */
	d = l2->length - l1->length;
	if (d)
		return d;

1457
	/* read > read/write */
1458
	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1459 1460
}

1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482
static bool
pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
		const struct pnfs_layout_range *l2)
{
	return pnfs_lseg_range_cmp(l1, l2) > 0;
}

static bool
pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
		struct pnfs_layout_segment *old)
{
	return false;
}

void
pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   bool (*is_after)(const struct pnfs_layout_range *,
			   const struct pnfs_layout_range *),
		   bool (*do_merge)(struct pnfs_layout_segment *,
			   struct pnfs_layout_segment *),
		   struct list_head *free_me)
1483
{
1484
	struct pnfs_layout_segment *lp, *tmp;
1485

1486 1487
	dprintk("%s:Begin\n", __func__);

1488 1489 1490 1491 1492 1493 1494 1495
	list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
			continue;
		if (do_merge(lseg, lp)) {
			mark_lseg_invalid(lp, free_me);
			continue;
		}
		if (is_after(&lseg->pls_range, &lp->pls_range))
1496
			continue;
1497
		list_add_tail(&lseg->pls_list, &lp->pls_list);
1498 1499 1500
		dprintk("%s: inserted lseg %p "
			"iomode %d offset %llu length %llu before "
			"lp %p iomode %d offset %llu length %llu\n",
1501 1502 1503 1504
			__func__, lseg, lseg->pls_range.iomode,
			lseg->pls_range.offset, lseg->pls_range.length,
			lp, lp->pls_range.iomode, lp->pls_range.offset,
			lp->pls_range.length);
1505
		goto out;
1506
	}
1507 1508 1509 1510 1511 1512
	list_add_tail(&lseg->pls_list, &lo->plh_segs);
	dprintk("%s: inserted lseg %p "
		"iomode %d offset %llu length %llu at tail\n",
		__func__, lseg, lseg->pls_range.iomode,
		lseg->pls_range.offset, lseg->pls_range.length);
out:
1513
	pnfs_get_layout_hdr(lo);
1514 1515

	dprintk("%s:Return\n", __func__);
1516
}
1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);

static void
pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   struct list_head *free_me)
{
	struct inode *inode = lo->plh_inode;
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;

	if (ld->add_lseg != NULL)
		ld->add_lseg(lo, lseg, free_me);
	else
		pnfs_generic_layout_insert_lseg(lo, lseg,
				pnfs_lseg_range_is_after,
				pnfs_lseg_no_merge,
				free_me);
}
1535 1536

static struct pnfs_layout_hdr *
1537 1538 1539
alloc_init_layout_hdr(struct inode *ino,
		      struct nfs_open_context *ctx,
		      gfp_t gfp_flags)
1540 1541 1542
{
	struct pnfs_layout_hdr *lo;

1543
	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1544 1545
	if (!lo)
		return NULL;
1546
	refcount_set(&lo->plh_refcount, 1);
1547 1548
	INIT_LIST_HEAD(&lo->plh_layouts);
	INIT_LIST_HEAD(&lo->plh_segs);
1549
	INIT_LIST_HEAD(&lo->plh_return_segs);
1550
	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1551
	lo->plh_inode = ino;
1552
	lo->plh_lc_cred = get_rpccred(ctx->cred);
1553
	lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID;
1554 1555 1556 1557
	return lo;
}

static struct pnfs_layout_hdr *
1558 1559 1560
pnfs_find_alloc_layout(struct inode *ino,
		       struct nfs_open_context *ctx,
		       gfp_t gfp_flags)
1561 1562
	__releases(&ino->i_lock)
	__acquires(&ino->i_lock)
1563 1564 1565 1566 1567 1568
{
	struct nfs_inode *nfsi = NFS_I(ino);
	struct pnfs_layout_hdr *new = NULL;

	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

1569 1570
	if (nfsi->layout != NULL)
		goto out_existing;
1571
	spin_unlock(&ino->i_lock);
1572
	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1573 1574
	spin_lock(&ino->i_lock);

1575
	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1576
		nfsi->layout = new;
1577
		return new;
1578 1579
	} else if (new != NULL)
		pnfs_free_layout_hdr(new);
1580 1581
out_existing:
	pnfs_get_layout_hdr(nfsi->layout);
1582 1583 1584
	return nfsi->layout;
}

1585 1586
/*
 * iomode matching rules:
1587 1588 1589 1590 1591 1592 1593 1594 1595 1596
 * iomode	lseg	strict match
 *                      iomode
 * -----	-----	------ -----
 * ANY		READ	N/A    true
 * ANY		RW	N/A    true
 * RW		READ	N/A    false
 * RW		RW	N/A    true
 * READ		READ	N/A    true
 * READ		RW	true   false
 * READ		RW	false  true
1597
 */
1598
static bool
1599
pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1600 1601
		 const struct pnfs_layout_range *range,
		 bool strict_iomode)
1602
{
1603 1604 1605 1606
	struct pnfs_layout_range range1;

	if ((range->iomode == IOMODE_RW &&
	     ls_range->iomode != IOMODE_RW) ||
1607
	    (range->iomode != ls_range->iomode &&
1608
	     strict_iomode) ||
1609
	    !pnfs_lseg_range_intersecting(ls_range, range))
1610 1611 1612 1613 1614
		return 0;

	/* range1 covers only the first byte in the range */
	range1 = *range;
	range1.length = 1;
1615
	return pnfs_lseg_range_contained(ls_range, &range1);
1616 1617 1618 1619 1620
}

/*
 * lookup range in layout
 */
1621
static struct pnfs_layout_segment *
1622
pnfs_find_lseg(struct pnfs_layout_hdr *lo,
1623 1624
		struct pnfs_layout_range *range,
		bool strict_iomode)
1625
{
1626 1627 1628 1629
	struct pnfs_layout_segment *lseg, *ret = NULL;

	dprintk("%s:Begin\n", __func__);

1630
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1631
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1632
		    !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
1633 1634
		    pnfs_lseg_range_match(&lseg->pls_range, range,
					  strict_iomode)) {
1635
			ret = pnfs_get_lseg(lseg);
1636 1637 1638 1639 1640
			break;
		}
	}

	dprintk("%s:Return lseg %p ref %d\n",
1641
		__func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0);
1642
	return ret;
1643 1644
}

1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
/*
 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
 * to the MDS or over pNFS
 *
 * The nfs_inode read_io and write_io fields are cumulative counters reset
 * when there are no layout segments. Note that in pnfs_update_layout iomode
 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
 * WRITE request.
 *
 * A return of true means use MDS I/O.
 *
 * From rfc 5661:
 * If a file's size is smaller than the file size threshold, data accesses
 * SHOULD be sent to the metadata server.  If an I/O request has a length that
 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
 * server.  If both file size and I/O size are provided, the client SHOULD
 * reach or exceed  both thresholds before sending its read or write
 * requests to the data server.
 */
static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
				     struct inode *ino, int iomode)
{
	struct nfs4_threshold *t = ctx->mdsthreshold;
	struct nfs_inode *nfsi = NFS_I(ino);
	loff_t fsize = i_size_read(ino);
	bool size = false, size_set = false, io = false, io_set = false, ret = false;

	if (t == NULL)
		return ret;

	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);

	switch (iomode) {
	case IOMODE_READ:
		if (t->bm & THRESHOLD_RD) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->rd_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_RD_IO) {
			dprintk("%s nfsi->read_io %llu\n", __func__,
				nfsi->read_io);
			io_set = true;
			if (nfsi->read_io < t->rd_io_sz)
				io = true;
		}
		break;
	case IOMODE_RW:
		if (t->bm & THRESHOLD_WR) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->wr_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_WR_IO) {
			dprintk("%s nfsi->write_io %llu\n", __func__,
				nfsi->write_io);
			io_set = true;
			if (nfsi->write_io < t->wr_io_sz)
				io = true;
		}
		break;
	}
	if (size_set && io_set) {
		if (size && io)
			ret = true;
	} else if (size || io)
		ret = true;

	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
	return ret;
}

1720 1721 1722 1723 1724 1725 1726 1727
static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
{
	/*
	 * send layoutcommit as it can hold up layoutreturn due to lseg
	 * reference
	 */
	pnfs_layoutcommit_inode(lo->plh_inode, false);
	return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
1728
				   nfs_wait_bit_killable,
1729 1730 1731
				   TASK_UNINTERRUPTIBLE);
}

1732 1733 1734 1735 1736 1737 1738 1739 1740
static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
{
	unsigned long *bitlock = &lo->plh_flags;

	clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
	smp_mb__after_atomic();
	wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
}

1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
static void _add_to_server_list(struct pnfs_layout_hdr *lo,
				struct nfs_server *server)
{
	if (list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		/* The lo must be on the clp list if there is any
		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
		 */
		spin_lock(&clp->cl_lock);
		if (list_empty(&lo->plh_layouts))
			list_add_tail(&lo->plh_layouts, &server->layouts);
		spin_unlock(&clp->cl_lock);
	}
}

1757 1758 1759 1760
/*
 * Layout segment is retreived from the server if not cached.
 * The appropriate layout segment is referenced and returned to the caller.
 */
1761
struct pnfs_layout_segment *
1762 1763
pnfs_update_layout(struct inode *ino,
		   struct nfs_open_context *ctx,
1764 1765
		   loff_t pos,
		   u64 count,
1766
		   enum pnfs_iomode iomode,
1767
		   bool strict_iomode,
1768
		   gfp_t gfp_flags)
1769
{
1770 1771 1772 1773 1774
	struct pnfs_layout_range arg = {
		.iomode = iomode,
		.offset = pos,
		.length = count,
	};
1775
	unsigned pg_offset;
1776 1777
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs_client *clp = server->nfs_client;
1778
	struct pnfs_layout_hdr *lo = NULL;
1779
	struct pnfs_layout_segment *lseg = NULL;
F
Fred Isaman 已提交
1780
	struct nfs4_layoutget *lgp;
1781 1782
	nfs4_stateid stateid;
	long timeout = 0;
1783
	unsigned long giveup = jiffies + (clp->cl_lease_time << 1);
1784
	bool first;
1785

1786
	if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
1787
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1788
				 PNFS_UPDATE_LAYOUT_NO_PNFS);
1789
		goto out;
1790
	}
1791

1792
	if (iomode == IOMODE_READ && i_size_read(ino) == 0) {
1793
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1794
				 PNFS_UPDATE_LAYOUT_RD_ZEROLEN);
1795
		goto out;
1796
	}
1797

1798
	if (pnfs_within_mdsthreshold(ctx, ino, iomode)) {
1799
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1800
				 PNFS_UPDATE_LAYOUT_MDSTHRESH);
1801
		goto out;
1802
	}
1803

1804
lookup_again:
1805
	nfs4_client_recover_expired_lease(clp);
1806
	first = false;
1807
	spin_lock(&ino->i_lock);
1808
	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
1809 1810
	if (lo == NULL) {
		spin_unlock(&ino->i_lock);
1811
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1812
				 PNFS_UPDATE_LAYOUT_NOMEM);
1813 1814
		goto out;
	}
1815

F
Fred Isaman 已提交
1816
	/* Do we even need to bother with this? */
1817
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
1818
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1819
				 PNFS_UPDATE_LAYOUT_BULK_RECALL);
F
Fred Isaman 已提交
1820
		dprintk("%s matches recall, use MDS\n", __func__);
1821 1822 1823 1824
		goto out_unlock;
	}

	/* if LAYOUTGET already failed once we don't try again */
1825
	if (pnfs_layout_io_test_failed(lo, iomode)) {
1826
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1827
				 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL);
1828
		goto out_unlock;
1829
	}
1830

1831
	lseg = pnfs_find_lseg(lo, &arg, strict_iomode);
1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848
	if (lseg) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				PNFS_UPDATE_LAYOUT_FOUND_CACHED);
		goto out_unlock;
	}

	if (!nfs4_valid_open_stateid(ctx->state)) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				PNFS_UPDATE_LAYOUT_INVALID_OPEN);
		goto out_unlock;
	}

	/*
	 * Choose a stateid for the LAYOUTGET. If we don't have a layout
	 * stateid, or it has been invalidated, then we must use the open
	 * stateid.
	 */
1849
	if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
1850 1851 1852

		/*
		 * The first layoutget for the file. Need to serialize per
1853 1854 1855 1856 1857 1858 1859 1860
		 * RFC 5661 Errata 3208.
		 */
		if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
				     &lo->plh_flags)) {
			spin_unlock(&ino->i_lock);
			wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
				    TASK_UNINTERRUPTIBLE);
			pnfs_put_layout_hdr(lo);
1861
			dprintk("%s retrying\n", __func__);
1862 1863
			goto lookup_again;
		}
1864 1865

		first = true;
1866 1867 1868 1869 1870 1871 1872 1873
		if (nfs4_select_rw_stateid(ctx->state,
					iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ,
					NULL, &stateid, NULL) != 0) {
			trace_pnfs_update_layout(ino, pos, count,
					iomode, lo, lseg,
					PNFS_UPDATE_LAYOUT_INVALID_OPEN);
			goto out_unlock;
		}
1874
	} else {
1875
		nfs4_stateid_copy(&stateid, &lo->plh_stateid);
1876
	}
1877

1878 1879 1880 1881
	/*
	 * Because we free lsegs before sending LAYOUTRETURN, we need to wait
	 * for LAYOUTRETURN even if first is true.
	 */
1882
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
1883 1884 1885
		spin_unlock(&ino->i_lock);
		dprintk("%s wait for layoutreturn\n", __func__);
		if (pnfs_prepare_to_retry_layoutget(lo)) {
1886 1887
			if (first)
				pnfs_clear_first_layoutget(lo);
1888 1889
			pnfs_put_layout_hdr(lo);
			dprintk("%s retrying\n", __func__);
1890 1891
			trace_pnfs_update_layout(ino, pos, count, iomode, lo,
					lseg, PNFS_UPDATE_LAYOUT_RETRY);
1892 1893
			goto lookup_again;
		}
1894
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1895
				PNFS_UPDATE_LAYOUT_RETURN);
1896 1897 1898
		goto out_put_layout_hdr;
	}

1899
	if (pnfs_layoutgets_blocked(lo)) {
1900
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1901
				PNFS_UPDATE_LAYOUT_BLOCKED);
1902
		goto out_unlock;
1903
	}
1904
	atomic_inc(&lo->plh_outstanding);
1905
	spin_unlock(&ino->i_lock);
1906

1907
	_add_to_server_list(lo, server);
1908

1909
	pg_offset = arg.offset & ~PAGE_MASK;
1910 1911 1912 1913
	if (pg_offset) {
		arg.offset -= pg_offset;
		arg.length += pg_offset;
	}
1914
	if (arg.length != NFS4_MAX_UINT64)
1915
		arg.length = PAGE_ALIGN(arg.length);
1916

1917
	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags);
F
Fred Isaman 已提交
1918 1919 1920 1921 1922 1923 1924
	if (!lgp) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL,
					 PNFS_UPDATE_LAYOUT_NOMEM);
		atomic_dec(&lo->plh_outstanding);
		goto out_put_layout_hdr;
	}

1925
	lseg = nfs4_proc_layoutget(lgp, &timeout);
1926 1927
	trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
1928
	atomic_dec(&lo->plh_outstanding);
1929
	if (IS_ERR(lseg)) {
1930
		switch(PTR_ERR(lseg)) {
1931
		case -EBUSY:
1932 1933
			if (time_after(jiffies, giveup))
				lseg = NULL;
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943
			break;
		case -ERECALLCONFLICT:
			/* Huh? We hold no layouts, how is there a recall? */
			if (first) {
				lseg = NULL;
				break;
			}
			/* Destroy the existing layout and start over */
			if (time_after(jiffies, giveup))
				pnfs_destroy_layout(NFS_I(ino));
1944 1945
			/* Fallthrough */
		case -EAGAIN:
1946
			break;
1947 1948 1949 1950 1951
		default:
			if (!nfs_error_is_fatal(PTR_ERR(lseg))) {
				pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
				lseg = NULL;
			}
1952 1953 1954 1955 1956 1957 1958 1959 1960
			goto out_put_layout_hdr;
		}
		if (lseg) {
			if (first)
				pnfs_clear_first_layoutget(lo);
			trace_pnfs_update_layout(ino, pos, count,
				iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY);
			pnfs_put_layout_hdr(lo);
			goto lookup_again;
1961 1962 1963 1964 1965
		}
	} else {
		pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
	}

1966
out_put_layout_hdr:
1967 1968
	if (first)
		pnfs_clear_first_layoutget(lo);
1969
	pnfs_put_layout_hdr(lo);
1970
out:
1971 1972 1973 1974
	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
			"(%s, offset: %llu, length: %llu)\n",
			__func__, ino->i_sb->s_id,
			(unsigned long long)NFS_FILEID(ino),
1975
			IS_ERR_OR_NULL(lseg) ? "not found" : "found",
1976 1977 1978
			iomode==IOMODE_RW ?  "read/write" : "read-only",
			(unsigned long long)pos,
			(unsigned long long)count);
1979 1980 1981
	return lseg;
out_unlock:
	spin_unlock(&ino->i_lock);
1982
	goto out_put_layout_hdr;
1983
}
1984
EXPORT_SYMBOL_GPL(pnfs_update_layout);
1985

1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
static bool
pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
{
	switch (range->iomode) {
	case IOMODE_READ:
	case IOMODE_RW:
		break;
	default:
		return false;
	}
	if (range->offset == NFS4_MAX_UINT64)
		return false;
	if (range->length == 0)
		return false;
	if (range->length != NFS4_MAX_UINT64 &&
	    range->length > NFS4_MAX_UINT64 - range->offset)
		return false;
	return true;
}

2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033
static struct pnfs_layout_hdr *
_pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = pnfs_find_alloc_layout(ino, ctx, GFP_KERNEL);
	if (!lo)
		goto out_unlock;
	if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags))
		goto out_unlock;
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
		goto out_unlock;
	if (pnfs_layoutgets_blocked(lo))
		goto out_unlock;
	if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags))
		goto out_unlock;
	atomic_inc(&lo->plh_outstanding);
	spin_unlock(&ino->i_lock);
	_add_to_server_list(lo, NFS_SERVER(ino));
	return lo;

out_unlock:
	spin_unlock(&ino->i_lock);
	pnfs_put_layout_hdr(lo);
	return NULL;
}

2034 2035 2036 2037 2038
extern const nfs4_stateid current_stateid;

static void _lgopen_prepare_attached(struct nfs4_opendata *data,
				     struct nfs_open_context *ctx)
{
2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
	struct inode *ino = data->dentry->d_inode;
	struct pnfs_layout_range rng = {
		.iomode = (data->o_arg.fmode & FMODE_WRITE) ?
			  IOMODE_RW: IOMODE_READ,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	struct nfs4_layoutget *lgp;
	struct pnfs_layout_hdr *lo;

2049 2050 2051 2052 2053
	/* Heuristic: don't send layoutget if we have cached data */
	if (rng.iomode == IOMODE_READ &&
	   (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0))
		return;

2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066
	lo = _pnfs_grab_empty_layout(ino, ctx);
	if (!lo)
		return;
	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &current_stateid,
					     &rng, GFP_KERNEL);
	if (!lgp) {
		pnfs_clear_first_layoutget(lo);
		pnfs_put_layout_hdr(lo);
		return;
	}
	data->lgp = lgp;
	data->o_arg.lg_args = &lgp->args;
	data->o_res.lg_res = &lgp->res;
2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
}

static void _lgopen_prepare_floating(struct nfs4_opendata *data,
				     struct nfs_open_context *ctx)
{
	struct pnfs_layout_range rng = {
		.iomode = (data->o_arg.fmode & FMODE_WRITE) ?
			  IOMODE_RW: IOMODE_READ,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	struct nfs4_layoutget *lgp;

	lgp = pnfs_alloc_init_layoutget_args(NULL, ctx, &current_stateid,
					     &rng, GFP_KERNEL);
	if (!lgp)
		return;
	data->lgp = lgp;
	data->o_arg.lg_args = &lgp->args;
	data->o_res.lg_res = &lgp->res;
}

void pnfs_lgopen_prepare(struct nfs4_opendata *data,
			 struct nfs_open_context *ctx)
{
	struct nfs_server *server = NFS_SERVER(data->dir->d_inode);

	if (!(pnfs_enabled_sb(server) &&
	      server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN))
		return;
	/* Could check on max_ops, but currently hardcoded high enough */
2098 2099
	if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN))
		return;
2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110
	if (data->state)
		_lgopen_prepare_attached(data, ctx);
	else
		_lgopen_prepare_floating(data, ctx);
}

void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp,
		       struct nfs_open_context *ctx)
{
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg;
2111
	struct nfs_server *srv = NFS_SERVER(ino);
2112 2113
	u32 iomode;

2114
	if (!lgp)
2115
		return;
2116 2117 2118 2119
	dprintk("%s: entered with status %i\n", __func__, lgp->res.status);
	if (lgp->res.status) {
		switch (lgp->res.status) {
		default:
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
			break;
		/*
		 * Halt lgopen attempts if the server doesn't recognise
		 * the "current stateid" value, the layout type, or the
		 * layoutget operation as being valid.
		 * Also if it complains about too many ops in the compound
		 * or of the request/reply being too big.
		 */
		case -NFS4ERR_BAD_STATEID:
		case -NFS4ERR_NOTSUPP:
		case -NFS4ERR_REP_TOO_BIG:
		case -NFS4ERR_REP_TOO_BIG_TO_CACHE:
		case -NFS4ERR_REQ_TOO_BIG:
		case -NFS4ERR_TOO_MANY_OPS:
		case -NFS4ERR_UNKNOWN_LAYOUTTYPE:
2135
			srv->caps &= ~NFS_CAP_LGOPEN;
2136 2137 2138
		}
		return;
	}
2139
	if (!lgp->args.inode) {
2140 2141 2142
		lo = _pnfs_grab_empty_layout(ino, ctx);
		if (!lo)
			return;
2143 2144 2145 2146
		lgp->args.inode = ino;
	} else
		lo = NFS_I(lgp->args.inode)->layout;

2147 2148
	if (read_seqcount_retry(&srv->nfs_client->cl_callback_count,
				lgp->callback_count))
2149
		return;
2150 2151 2152 2153 2154 2155 2156 2157 2158
	lseg = pnfs_layout_process(lgp);
	if (IS_ERR(lseg)) {
		/* ignore lseg, but would like to mark not to try lgopen */
		/* clear some lo flags - first and fail ???? */
	} else {
		iomode = lgp->args.range.iomode;
		pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
		pnfs_put_lseg(lseg);
	}
2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
}

void nfs4_lgopen_release(struct nfs4_layoutget *lgp)
{
	if (lgp != NULL) {
		struct inode *inode = lgp->args.inode;
		if (inode) {
			struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
			atomic_dec(&lo->plh_outstanding);
			pnfs_clear_first_layoutget(lo);
		}
		pnfs_layoutget_free(lgp);
	}
2172 2173
}

2174
struct pnfs_layout_segment *
2175 2176 2177 2178 2179
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
	struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
	struct nfs4_layoutget_res *res = &lgp->res;
	struct pnfs_layout_segment *lseg;
2180
	struct inode *ino = lo->plh_inode;
2181
	LIST_HEAD(free_me);
2182 2183

	if (!pnfs_sanity_check_layout_range(&res->range))
2184
		return ERR_PTR(-EINVAL);
2185 2186

	/* Inject layout blob into I/O device driver */
2187
	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
2188
	if (IS_ERR_OR_NULL(lseg)) {
2189
		if (!lseg)
2190 2191 2192 2193 2194
			lseg = ERR_PTR(-ENOMEM);

		dprintk("%s: Could not allocate layout: error %ld\n",
		       __func__, PTR_ERR(lseg));
		return lseg;
2195 2196
	}

2197
	pnfs_init_lseg(lo, lseg, &res->range, &res->stateid);
2198

2199
	spin_lock(&ino->i_lock);
2200
	if (pnfs_layoutgets_blocked(lo)) {
F
Fred Isaman 已提交
2201
		dprintk("%s forget reply due to state\n", __func__);
2202
		goto out_forget;
F
Fred Isaman 已提交
2203
	}
2204

2205 2206 2207 2208
	if (!pnfs_layout_is_valid(lo)) {
		/* We have a completely new layout */
		pnfs_set_layout_stateid(lo, &res->stateid, true);
	} else if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
2209 2210 2211
		/* existing state ID, make sure the sequence number matches. */
		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
			dprintk("%s forget reply due to sequence\n", __func__);
2212
			goto out_forget;
2213 2214 2215 2216 2217
		}
		pnfs_set_layout_stateid(lo, &res->stateid, false);
	} else {
		/*
		 * We got an entirely new state ID.  Mark all segments for the
2218
		 * inode invalid, and retry the layoutget
2219
		 */
2220
		pnfs_mark_layout_stateid_invalid(lo, &free_me);
2221
		goto out_forget;
2222
	}
2223

2224
	pnfs_get_lseg(lseg);
2225
	pnfs_layout_insert_lseg(lo, lseg, &free_me);
2226

2227

P
Peng Tao 已提交
2228
	if (res->return_on_close)
F
Fred Isaman 已提交
2229 2230
		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);

2231
	spin_unlock(&ino->i_lock);
2232
	pnfs_free_lseg_list(&free_me);
2233
	return lseg;
F
Fred Isaman 已提交
2234

2235
out_forget:
F
Fred Isaman 已提交
2236 2237 2238
	spin_unlock(&ino->i_lock);
	lseg->pls_layout = lo;
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
2239 2240
	if (!pnfs_layout_is_valid(lo))
		nfs_commit_inode(ino, 0);
2241
	return ERR_PTR(-EAGAIN);
2242 2243
}

2244 2245 2246 2247 2248 2249 2250 2251 2252 2253
/**
 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments
 * @lo: pointer to layout header
 * @tmp_list: list header to be used with pnfs_free_lseg_list()
 * @return_range: describe layout segment ranges to be returned
 *
 * This function is mainly intended for use by layoutrecall. It attempts
 * to free the layout segment immediately, or else to mark it for return
 * as soon as its reference count drops to zero.
 */
2254
int
2255 2256
pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
				struct list_head *tmp_list,
2257 2258
				const struct pnfs_layout_range *return_range,
				u32 seq)
2259 2260
{
	struct pnfs_layout_segment *lseg, *next;
2261
	int remaining = 0;
2262 2263 2264 2265

	dprintk("%s:Begin lo %p\n", __func__, lo);

	if (list_empty(&lo->plh_segs))
2266
		return 0;
2267

2268
	assert_spin_locked(&lo->plh_inode->i_lock);
2269 2270

	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
2271
		if (pnfs_match_lseg_recall(lseg, return_range, seq)) {
2272 2273 2274 2275 2276
			dprintk("%s: marking lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode,
				lseg->pls_range.offset,
				lseg->pls_range.length);
2277 2278 2279
			if (mark_lseg_invalid(lseg, tmp_list))
				continue;
			remaining++;
2280 2281
			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
		}
2282 2283 2284 2285

	if (remaining)
		pnfs_set_plh_return_info(lo, return_range->iomode, seq);

2286
	return remaining;
2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297
}

void pnfs_error_mark_layout_for_return(struct inode *inode,
				       struct pnfs_layout_segment *lseg)
{
	struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
	struct pnfs_layout_range range = {
		.iomode = lseg->pls_range.iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
2298
	bool return_now = false;
2299 2300

	spin_lock(&inode->i_lock);
2301 2302 2303 2304
	if (!pnfs_layout_is_valid(lo)) {
		spin_unlock(&inode->i_lock);
		return;
	}
2305
	pnfs_set_plh_return_info(lo, range.iomode, 0);
2306 2307 2308 2309 2310
	/*
	 * mark all matching lsegs so that we are sure to have no live
	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
	 * for how it works.
	 */
2311
	if (!pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, &range, 0)) {
2312
		nfs4_stateid stateid;
2313
		enum pnfs_iomode iomode;
2314

2315
		return_now = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
2316 2317 2318 2319 2320 2321 2322
		spin_unlock(&inode->i_lock);
		if (return_now)
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
	} else {
		spin_unlock(&inode->i_lock);
		nfs_commit_inode(inode, 0);
	}
2323 2324 2325
}
EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);

2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
void
pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio)
{
	if (pgio->pg_lseg == NULL ||
	    test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags))
		return;
	pnfs_put_lseg(pgio->pg_lseg);
	pgio->pg_lseg = NULL;
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout);

2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349
/*
 * Check for any intersection between the request and the pgio->pg_lseg,
 * and if none, put this pgio->pg_lseg away.
 */
static void
pnfs_generic_pg_check_range(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
	if (pgio->pg_lseg && !pnfs_lseg_request_intersecting(pgio->pg_lseg, req)) {
		pnfs_put_lseg(pgio->pg_lseg);
		pgio->pg_lseg = NULL;
	}
}

2350 2351 2352
void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
P
Peng Tao 已提交
2353 2354
	u64 rd_size = req->wb_bytes;

2355
	pnfs_generic_pg_check_layout(pgio);
2356
	pnfs_generic_pg_check_range(pgio, req);
2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
	if (pgio->pg_lseg == NULL) {
		if (pgio->pg_dreq == NULL)
			rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
		else
			rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);

		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   rd_size,
						   IOMODE_READ,
2368
						   false,
2369
						   GFP_KERNEL);
2370 2371 2372 2373 2374
		if (IS_ERR(pgio->pg_lseg)) {
			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
			pgio->pg_lseg = NULL;
			return;
		}
2375
	}
2376 2377
	/* If no lseg, fall back to read through mds */
	if (pgio->pg_lseg == NULL)
2378
		nfs_pageio_reset_read_mds(pgio);
2379

2380 2381 2382 2383
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);

void
2384 2385
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
			   struct nfs_page *req, u64 wb_size)
2386
{
2387
	pnfs_generic_pg_check_layout(pgio);
2388
	pnfs_generic_pg_check_range(pgio, req);
2389
	if (pgio->pg_lseg == NULL) {
2390 2391 2392 2393 2394
		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   wb_size,
						   IOMODE_RW,
2395
						   false,
2396
						   GFP_NOFS);
2397 2398 2399 2400 2401 2402
		if (IS_ERR(pgio->pg_lseg)) {
			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
			pgio->pg_lseg = NULL;
			return;
		}
	}
2403 2404
	/* If no lseg, fall back to write through mds */
	if (pgio->pg_lseg == NULL)
2405
		nfs_pageio_reset_write_mds(pgio);
2406 2407 2408
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);

2409 2410 2411 2412 2413 2414 2415 2416 2417 2418
void
pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
{
	if (desc->pg_lseg) {
		pnfs_put_lseg(desc->pg_lseg);
		desc->pg_lseg = NULL;
	}
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);

2419 2420 2421 2422 2423
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
size_t
2424 2425
pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
		     struct nfs_page *prev, struct nfs_page *req)
2426
{
2427
	unsigned int size;
2428
	u64 seg_end, req_start, seg_left;
2429 2430 2431 2432

	size = nfs_generic_pg_test(pgio, prev, req);
	if (!size)
		return 0;
2433

2434
	/*
2435 2436 2437 2438 2439
	 * 'size' contains the number of bytes left in the current page (up
	 * to the original size asked for in @req->wb_bytes).
	 *
	 * Calculate how many bytes are left in the layout segment
	 * and if there are less bytes than 'size', return that instead.
2440 2441 2442 2443 2444
	 *
	 * Please also note that 'end_offset' is actually the offset of the
	 * first byte that lies outside the pnfs_layout_range. FIXME?
	 *
	 */
2445
	if (pgio->pg_lseg) {
2446
		seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset,
2447 2448
				     pgio->pg_lseg->pls_range.length);
		req_start = req_offset(req);
2449

2450
		/* start of request is past the last byte of this segment */
2451
		if (req_start >= seg_end)
2452
			return 0;
2453 2454 2455 2456 2457 2458

		/* adjust 'size' iff there are fewer bytes left in the
		 * segment than what nfs_generic_pg_test returned */
		seg_left = seg_end - req_start;
		if (seg_left < size)
			size = (unsigned int)seg_left;
2459
	}
2460

2461
	return size;
2462
}
2463
EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
2464

2465
int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
2466 2467 2468 2469
{
	struct nfs_pageio_descriptor pgio;

	/* Resend all requests through the MDS */
2470 2471
	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
			      hdr->completion_ops);
2472
	set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags);
2473
	return nfs_pageio_resend(&pgio, hdr);
2474
}
2475
EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
2476

2477
static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
2478
{
2479 2480 2481

	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2482
	    PNFS_LAYOUTRET_ON_ERROR) {
2483
		pnfs_return_layout(hdr->inode);
2484
	}
2485
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2486
		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
2487 2488
}

2489 2490 2491
/*
 * Called by non rpc-based layout drivers
 */
2492
void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
2493
{
2494
	if (likely(!hdr->pnfs_error)) {
2495 2496
		pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
				hdr->mds_offset + hdr->res.count);
2497
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2498 2499 2500
	}
	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2501 2502
		pnfs_ld_handle_write_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2503
}
2504
EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
2505

2506 2507
static void
pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
2508
		struct nfs_pgio_header *hdr)
2509
{
2510
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2511

2512
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2513
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2514
		nfs_pageio_reset_write_mds(desc);
2515
		mirror->pg_recoalesce = 1;
2516
	}
2517
	hdr->completion_ops->completion(hdr);
2518 2519 2520
}

static enum pnfs_try_status
2521
pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
2522 2523 2524
			const struct rpc_call_ops *call_ops,
			struct pnfs_layout_segment *lseg,
			int how)
2525
{
2526
	struct inode *inode = hdr->inode;
2527 2528 2529
	enum pnfs_try_status trypnfs;
	struct nfs_server *nfss = NFS_SERVER(inode);

2530
	hdr->mds_ops = call_ops;
2531 2532

	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
2533 2534
		inode->i_ino, hdr->args.count, hdr->args.offset, how);
	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
2535
	if (trypnfs != PNFS_NOT_ATTEMPTED)
2536 2537 2538 2539 2540
		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}

2541
static void
2542 2543
pnfs_do_write(struct nfs_pageio_descriptor *desc,
	      struct nfs_pgio_header *hdr, int how)
2544 2545 2546
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2547
	enum pnfs_try_status trypnfs;
2548

2549
	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
2550 2551
	switch (trypnfs) {
	case PNFS_NOT_ATTEMPTED:
2552
		pnfs_write_through_mds(desc, hdr);
2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563
	case PNFS_ATTEMPTED:
		break;
	case PNFS_TRY_AGAIN:
		/* cleanup hdr and prepare to redo pnfs */
		if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
			struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
			list_splice_init(&hdr->pages, &mirror->pg_list);
			mirror->pg_recoalesce = 1;
		}
		hdr->mds_ops->rpc_release(hdr);
	}
2564 2565
}

2566 2567
static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
2568
	pnfs_put_lseg(hdr->lseg);
2569
	nfs_pgio_header_free(hdr);
2570 2571
}

2572 2573 2574
int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
{
2575
	struct nfs_pgio_header *hdr;
2576 2577
	int ret;

2578 2579
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
P
Peng Tao 已提交
2580 2581
		desc->pg_error = -ENOMEM;
		return desc->pg_error;
2582
	}
2583
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
2584

2585
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2586
	ret = nfs_generic_pgio(desc, hdr);
2587
	if (!ret)
2588
		pnfs_do_write(desc, hdr, desc->pg_ioflags);
2589

2590
	return ret;
2591 2592 2593
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);

2594
int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
2595 2596 2597
{
	struct nfs_pageio_descriptor pgio;

2598
	/* Resend all requests through the MDS */
2599 2600
	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
2601
}
2602
EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
2603

2604
static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
2605
{
2606 2607
	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2608
	    PNFS_LAYOUTRET_ON_ERROR) {
2609
		pnfs_return_layout(hdr->inode);
2610
	}
2611
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2612
		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
2613 2614
}

2615 2616 2617
/*
 * Called by non rpc-based layout drivers
 */
2618
void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
2619
{
2620
	if (likely(!hdr->pnfs_error))
2621
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2622 2623
	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2624 2625
		pnfs_ld_handle_read_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2626 2627 2628
}
EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

2629 2630
static void
pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
2631
		struct nfs_pgio_header *hdr)
2632
{
2633
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2634

2635
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2636
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2637
		nfs_pageio_reset_read_mds(desc);
2638
		mirror->pg_recoalesce = 1;
2639
	}
2640
	hdr->completion_ops->completion(hdr);
2641 2642
}

A
Andy Adamson 已提交
2643 2644 2645
/*
 * Call the appropriate parallel I/O subsystem read function.
 */
2646
static enum pnfs_try_status
2647
pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
2648 2649
		       const struct rpc_call_ops *call_ops,
		       struct pnfs_layout_segment *lseg)
A
Andy Adamson 已提交
2650
{
2651
	struct inode *inode = hdr->inode;
A
Andy Adamson 已提交
2652 2653 2654
	struct nfs_server *nfss = NFS_SERVER(inode);
	enum pnfs_try_status trypnfs;

2655
	hdr->mds_ops = call_ops;
A
Andy Adamson 已提交
2656 2657

	dprintk("%s: Reading ino:%lu %u@%llu\n",
2658
		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
A
Andy Adamson 已提交
2659

2660
	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
2661
	if (trypnfs != PNFS_NOT_ATTEMPTED)
A
Andy Adamson 已提交
2662 2663 2664 2665
		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}
A
Andy Adamson 已提交
2666

2667
/* Resend all requests through pnfs. */
2668
void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
2669 2670 2671
{
	struct nfs_pageio_descriptor pgio;

2672
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2673 2674 2675 2676
		/* Prevent deadlocks with layoutreturn! */
		pnfs_put_lseg(hdr->lseg);
		hdr->lseg = NULL;

2677 2678 2679 2680
		nfs_pageio_init_read(&pgio, hdr->inode, false,
					hdr->completion_ops);
		hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr);
	}
2681 2682 2683
}
EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);

2684
static void
2685
pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
2686 2687 2688
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2689
	enum pnfs_try_status trypnfs;
2690

2691
	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
2692 2693
	switch (trypnfs) {
	case PNFS_NOT_ATTEMPTED:
2694
		pnfs_read_through_mds(desc, hdr);
2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705
	case PNFS_ATTEMPTED:
		break;
	case PNFS_TRY_AGAIN:
		/* cleanup hdr and prepare to redo pnfs */
		if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
			struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
			list_splice_init(&hdr->pages, &mirror->pg_list);
			mirror->pg_recoalesce = 1;
		}
		hdr->mds_ops->rpc_release(hdr);
	}
2706 2707
}

2708 2709
static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
2710
	pnfs_put_lseg(hdr->lseg);
2711
	nfs_pgio_header_free(hdr);
2712 2713
}

2714 2715 2716
int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
{
2717
	struct nfs_pgio_header *hdr;
2718 2719
	int ret;

2720 2721
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
P
Peng Tao 已提交
2722 2723
		desc->pg_error = -ENOMEM;
		return desc->pg_error;
2724
	}
2725
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
2726
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2727
	ret = nfs_generic_pgio(desc, hdr);
2728
	if (!ret)
2729
		pnfs_do_read(desc, hdr);
2730
	return ret;
2731 2732 2733
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);

2734 2735 2736 2737 2738
static void pnfs_clear_layoutcommitting(struct inode *inode)
{
	unsigned long *bitlock = &NFS_I(inode)->flags;

	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
2739
	smp_mb__after_atomic();
2740 2741 2742
	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}

A
Andy Adamson 已提交
2743
/*
2744
 * There can be multiple RW segments.
A
Andy Adamson 已提交
2745
 */
2746
static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
A
Andy Adamson 已提交
2747
{
2748
	struct pnfs_layout_segment *lseg;
A
Andy Adamson 已提交
2749

2750 2751
	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
		if (lseg->pls_range.iomode == IOMODE_RW &&
2752
		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
2753 2754
			list_add(&lseg->pls_lc_list, listp);
	}
A
Andy Adamson 已提交
2755 2756
}

2757 2758 2759 2760 2761 2762 2763 2764 2765 2766
static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
{
	struct pnfs_layout_segment *lseg, *tmp;

	/* Matched by references in pnfs_set_layoutcommit */
	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
		list_del_init(&lseg->pls_lc_list);
		pnfs_put_lseg(lseg);
	}

2767
	pnfs_clear_layoutcommitting(inode);
2768 2769
}

P
Peng Tao 已提交
2770 2771
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
2772
	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
P
Peng Tao 已提交
2773 2774 2775
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

A
Andy Adamson 已提交
2776
void
2777 2778
pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
		loff_t end_pos)
A
Andy Adamson 已提交
2779
{
2780
	struct nfs_inode *nfsi = NFS_I(inode);
2781
	bool mark_as_dirty = false;
A
Andy Adamson 已提交
2782

2783
	spin_lock(&inode->i_lock);
A
Andy Adamson 已提交
2784
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
2785
		nfsi->layout->plh_lwb = end_pos;
2786
		mark_as_dirty = true;
A
Andy Adamson 已提交
2787
		dprintk("%s: Set layoutcommit for inode %lu ",
2788
			__func__, inode->i_ino);
2789 2790
	} else if (end_pos > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = end_pos;
2791
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
2792
		/* references matched in nfs4_layoutcommit_release */
2793
		pnfs_get_lseg(lseg);
2794
	}
2795
	spin_unlock(&inode->i_lock);
2796
	dprintk("%s: lseg %p end_pos %llu\n",
2797
		__func__, lseg, nfsi->layout->plh_lwb);
2798 2799 2800 2801

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
2802
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2803 2804 2805
}
EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);

A
Andy Adamson 已提交
2806 2807 2808 2809 2810 2811
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
{
	struct nfs_server *nfss = NFS_SERVER(data->args.inode);

	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
2812
	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
A
Andy Adamson 已提交
2813 2814
}

2815 2816 2817 2818 2819 2820 2821 2822
/*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
 * data to disk to allow the server to recover the data if it crashes.
 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
 * is off, and a COMMIT is sent to a data server, or
 * if WRITEs to a data server return NFS_DATA_SYNC.
 */
A
Andy Adamson 已提交
2823
int
2824
pnfs_layoutcommit_inode(struct inode *inode, bool sync)
A
Andy Adamson 已提交
2825
{
2826
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
A
Andy Adamson 已提交
2827 2828 2829
	struct nfs4_layoutcommit_data *data;
	struct nfs_inode *nfsi = NFS_I(inode);
	loff_t end_pos;
2830
	int status;
A
Andy Adamson 已提交
2831

2832
	if (!pnfs_layoutcommit_outstanding(inode))
2833 2834
		return 0;

2835
	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
P
Peng Tao 已提交
2836

2837
	status = -EAGAIN;
P
Peng Tao 已提交
2838
	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2839 2840
		if (!sync)
			goto out;
2841
		status = wait_on_bit_lock_action(&nfsi->flags,
2842 2843 2844
				NFS_INO_LAYOUTCOMMITTING,
				nfs_wait_bit_killable,
				TASK_KILLABLE);
P
Peng Tao 已提交
2845
		if (status)
2846
			goto out;
P
Peng Tao 已提交
2847 2848
	}

2849 2850 2851 2852 2853 2854 2855
	status = -ENOMEM;
	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (!data)
		goto clear_layoutcommitting;

	status = 0;
2856
	spin_lock(&inode->i_lock);
2857 2858
	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		goto out_unlock;
2859

2860
	INIT_LIST_HEAD(&data->lseg_list);
2861
	pnfs_list_write_lseg(inode, &data->lseg_list);
A
Andy Adamson 已提交
2862

2863
	end_pos = nfsi->layout->plh_lwb;
A
Andy Adamson 已提交
2864

2865
	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
A
Andy Adamson 已提交
2866 2867 2868
	spin_unlock(&inode->i_lock);

	data->args.inode = inode;
2869
	data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
A
Andy Adamson 已提交
2870 2871 2872
	nfs_fattr_init(&data->fattr);
	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
	data->res.fattr = &data->fattr;
2873 2874 2875 2876
	if (end_pos != 0)
		data->args.lastbytewritten = end_pos - 1;
	else
		data->args.lastbytewritten = U64_MAX;
A
Andy Adamson 已提交
2877 2878
	data->res.server = NFS_SERVER(inode);

2879 2880 2881
	if (ld->prepare_layoutcommit) {
		status = ld->prepare_layoutcommit(&data->args);
		if (status) {
2882
			put_rpccred(data->cred);
2883
			spin_lock(&inode->i_lock);
2884 2885
			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
			if (end_pos > nfsi->layout->plh_lwb)
2886
				nfsi->layout->plh_lwb = end_pos;
2887
			goto out_unlock;
2888 2889 2890 2891
		}
	}


A
Andy Adamson 已提交
2892 2893
	status = nfs4_proc_layoutcommit(data, sync);
out:
P
Peng Tao 已提交
2894 2895
	if (status)
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2896 2897
	dprintk("<-- %s status %d\n", __func__, status);
	return status;
2898 2899
out_unlock:
	spin_unlock(&inode->i_lock);
P
Peng Tao 已提交
2900
	kfree(data);
2901 2902
clear_layoutcommitting:
	pnfs_clear_layoutcommitting(inode);
P
Peng Tao 已提交
2903
	goto out;
A
Andy Adamson 已提交
2904
}
2905
EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
2906

2907 2908 2909 2910 2911 2912 2913
int
pnfs_generic_sync(struct inode *inode, bool datasync)
{
	return pnfs_layoutcommit_inode(inode, true);
}
EXPORT_SYMBOL_GPL(pnfs_generic_sync);

2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924
struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
{
	struct nfs4_threshold *thp;

	thp = kzalloc(sizeof(*thp), GFP_NOFS);
	if (!thp) {
		dprintk("%s mdsthreshold allocation failed\n", __func__);
		return NULL;
	}
	return thp;
}
2925

2926
#if IS_ENABLED(CONFIG_NFS_V4_2)
2927
int
2928
pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
2929 2930 2931
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
	struct nfs_server *server = NFS_SERVER(inode);
2932
	struct nfs_inode *nfsi = NFS_I(inode);
2933 2934 2935 2936 2937 2938 2939
	struct nfs42_layoutstat_data *data;
	struct pnfs_layout_hdr *hdr;
	int status = 0;

	if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
		goto out;

2940 2941 2942
	if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
		goto out;

2943 2944 2945
	if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
		goto out;

2946 2947 2948
	spin_lock(&inode->i_lock);
	if (!NFS_I(inode)->layout) {
		spin_unlock(&inode->i_lock);
2949
		goto out_clear_layoutstats;
2950 2951 2952 2953 2954
	}
	hdr = NFS_I(inode)->layout;
	pnfs_get_layout_hdr(hdr);
	spin_unlock(&inode->i_lock);

2955
	data = kzalloc(sizeof(*data), gfp_flags);
2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976
	if (!data) {
		status = -ENOMEM;
		goto out_put;
	}

	data->args.fh = NFS_FH(inode);
	data->args.inode = inode;
	status = ld->prepare_layoutstats(&data->args);
	if (status)
		goto out_free;

	status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);

out:
	dprintk("%s returns %d\n", __func__, status);
	return status;

out_free:
	kfree(data);
out_put:
	pnfs_put_layout_hdr(hdr);
2977
out_clear_layoutstats:
2978 2979 2980
	smp_mb__before_atomic();
	clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
	smp_mb__after_atomic();
2981 2982 2983
	goto out;
}
EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
2984
#endif
2985 2986 2987 2988

unsigned int layoutstats_timer;
module_param(layoutstats_timer, uint, 0644);
EXPORT_SYMBOL_GPL(layoutstats_timer);