pnfs.c 72.8 KB
Newer Older
R
Ricardo Labiaga 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 *  pNFS functions to call and manage layout drivers.
 *
 *  Copyright (c) 2002 [year of first publication]
 *  The Regents of the University of Michigan
 *  All Rights Reserved
 *
 *  Dean Hildebrand <dhildebz@umich.edu>
 *
 *  Permission is granted to use, copy, create derivative works, and
 *  redistribute this software and such derivative works for any purpose,
 *  so long as the name of the University of Michigan is not used in
 *  any advertising or publicity pertaining to the use or distribution
 *  of this software without specific, written prior authorization. If
 *  the above copyright notice or any other identification of the
 *  University of Michigan is included in any copy of any portion of
 *  this software, then the disclaimer below must also be included.
 *
 *  This software is provided as is, without representation or warranty
 *  of any kind either express or implied, including without limitation
 *  the implied warranties of merchantability, fitness for a particular
 *  purpose, or noninfringement.  The Regents of the University of
 *  Michigan shall not be liable for any damages, including special,
 *  indirect, incidental, or consequential damages, with respect to any
 *  claim arising out of or in connection with the use of the software,
 *  even if it has been or is hereafter advised of the possibility of
 *  such damages.
 */

#include <linux/nfs_fs.h>
31
#include <linux/nfs_page.h>
32
#include <linux/module.h>
33
#include <linux/sort.h>
34
#include "internal.h"
R
Ricardo Labiaga 已提交
35
#include "pnfs.h"
A
Andy Adamson 已提交
36
#include "iostat.h"
37
#include "nfs4trace.h"
38
#include "delegation.h"
39
#include "nfs42.h"
R
Ricardo Labiaga 已提交
40 41

#define NFSDBG_FACILITY		NFSDBG_PNFS
42
#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
R
Ricardo Labiaga 已提交
43

44 45 46 47 48 49 50 51 52 53 54 55
/* Locking:
 *
 * pnfs_spinlock:
 *      protects pnfs_modules_tbl.
 */
static DEFINE_SPINLOCK(pnfs_spinlock);

/*
 * pnfs_modules_tbl holds all pnfs modules
 */
static LIST_HEAD(pnfs_modules_tbl);

56
static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo);
57 58 59 60
static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
		struct list_head *free_me,
		const struct pnfs_layout_range *range,
		u32 seq);
61 62
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		                struct list_head *tmp_list);
63

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
/* Return the registered pnfs layout driver module matching given id */
static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)
{
	struct pnfs_layoutdriver_type *local;

	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
		if (local->id == id)
			goto out;
	local = NULL;
out:
	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
	return local;
}

R
Ricardo Labiaga 已提交
79 80 81
static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)
{
82 83 84 85
	struct pnfs_layoutdriver_type *local;

	spin_lock(&pnfs_spinlock);
	local = find_pnfs_driver_locked(id);
86 87 88 89
	if (local != NULL && !try_module_get(local->owner)) {
		dprintk("%s: Could not grab reference on module\n", __func__);
		local = NULL;
	}
90 91
	spin_unlock(&pnfs_spinlock);
	return local;
R
Ricardo Labiaga 已提交
92 93 94 95 96
}

void
unset_pnfs_layoutdriver(struct nfs_server *nfss)
{
97 98 99
	if (nfss->pnfs_curr_ld) {
		if (nfss->pnfs_curr_ld->clear_layoutdriver)
			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
100 101 102
		/* Decrement the MDS count. Purge the deviceid cache if zero */
		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
			nfs4_deviceid_purge_client(nfss->nfs_client);
103
		module_put(nfss->pnfs_curr_ld->owner);
104
	}
R
Ricardo Labiaga 已提交
105 106 107
	nfss->pnfs_curr_ld = NULL;
}

108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
/*
 * When the server sends a list of layout types, we choose one in the order
 * given in the list below.
 *
 * FIXME: should this list be configurable in some fashion? module param?
 * 	  mount option? something else?
 */
static const u32 ld_prefs[] = {
	LAYOUT_SCSI,
	LAYOUT_BLOCK_VOLUME,
	LAYOUT_OSD2_OBJECTS,
	LAYOUT_FLEX_FILES,
	LAYOUT_NFSV4_1_FILES,
	0
};

static int
ld_cmp(const void *e1, const void *e2)
{
	u32 ld1 = *((u32 *)e1);
	u32 ld2 = *((u32 *)e2);
	int i;

	for (i = 0; ld_prefs[i] != 0; i++) {
		if (ld1 == ld_prefs[i])
			return -1;

		if (ld2 == ld_prefs[i])
			return 1;
	}
	return 0;
}

R
Ricardo Labiaga 已提交
141 142 143 144
/*
 * Try to set the server's pnfs module to the pnfs layout type specified by id.
 * Currently only one pNFS layout driver per filesystem is supported.
 *
145
 * @ids array of layout types supported by MDS.
R
Ricardo Labiaga 已提交
146 147
 */
void
148
set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
149
		      struct nfs_fsinfo *fsinfo)
R
Ricardo Labiaga 已提交
150 151
{
	struct pnfs_layoutdriver_type *ld_type = NULL;
152
	u32 id;
153
	int i;
R
Ricardo Labiaga 已提交
154

155 156
	if (fsinfo->nlayouttypes == 0)
		goto out_no_driver;
R
Ricardo Labiaga 已提交
157 158
	if (!(server->nfs_client->cl_exchange_flags &
		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
159 160
		printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n",
			__func__, server->nfs_client->cl_exchange_flags);
R
Ricardo Labiaga 已提交
161 162
		goto out_no_driver;
	}
163

164 165
	sort(fsinfo->layouttype, fsinfo->nlayouttypes,
		sizeof(*fsinfo->layouttype), ld_cmp, NULL);
166

167 168
	for (i = 0; i < fsinfo->nlayouttypes; i++) {
		id = fsinfo->layouttype[i];
R
Ricardo Labiaga 已提交
169
		ld_type = find_pnfs_driver(id);
170 171 172 173 174 175 176
		if (!ld_type) {
			request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX,
					id);
			ld_type = find_pnfs_driver(id);
		}
		if (ld_type)
			break;
R
Ricardo Labiaga 已提交
177
	}
178 179

	if (!ld_type) {
180
		dprintk("%s: No pNFS module found!\n", __func__);
181 182 183
		goto out_no_driver;
	}

R
Ricardo Labiaga 已提交
184
	server->pnfs_curr_ld = ld_type;
185 186
	if (ld_type->set_layoutdriver
	    && ld_type->set_layoutdriver(server, mntfh)) {
187 188
		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
			"driver %u.\n", __func__, id);
189 190 191
		module_put(ld_type->owner);
		goto out_no_driver;
	}
192 193
	/* Bump the MDS count */
	atomic_inc(&server->nfs_client->cl_mds_count);
194

R
Ricardo Labiaga 已提交
195 196 197 198 199 200 201
	dprintk("%s: pNFS module for %u set\n", __func__, id);
	return;

out_no_driver:
	dprintk("%s: Using NFSv4 I/O\n", __func__);
	server->pnfs_curr_ld = NULL;
}
202 203 204 205 206 207 208 209

int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	int status = -EINVAL;
	struct pnfs_layoutdriver_type *tmp;

	if (ld_type->id == 0) {
210
		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
211 212
		return status;
	}
213
	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
214
		printk(KERN_ERR "NFS: %s Layout driver must provide "
215 216 217
		       "alloc_lseg and free_lseg.\n", __func__);
		return status;
	}
218 219 220 221 222 223 224 225 226

	spin_lock(&pnfs_spinlock);
	tmp = find_pnfs_driver_locked(ld_type->id);
	if (!tmp) {
		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
		status = 0;
		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
			ld_type->name);
	} else {
227
		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
			__func__, ld_type->id);
	}
	spin_unlock(&pnfs_spinlock);

	return status;
}
EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);

void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
	spin_lock(&pnfs_spinlock);
	list_del(&ld_type->pnfs_tblid);
	spin_unlock(&pnfs_spinlock);
}
EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
245

246 247 248 249
/*
 * pNFS client layout cache
 */

250
/* Need to hold i_lock if caller does not already hold reference */
F
Fred Isaman 已提交
251
void
252
pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
253
{
254
	refcount_inc(&lo->plh_refcount);
255 256
}

257 258 259 260
static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
261
	return ld->alloc_layout_hdr(ino, gfp_flags);
262 263 264 265 266
}

static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
267 268 269 270 271 272 273 274 275 276
	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;

	if (!list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		spin_lock(&clp->cl_lock);
		list_del_init(&lo->plh_layouts);
		spin_unlock(&clp->cl_lock);
	}
277
	put_rpccred(lo->plh_lc_cred);
278
	return ld->free_layout_hdr(lo);
279 280
}

281
static void
282
pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
283
{
284
	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
285
	dprintk("%s: freeing layout cache %p\n", __func__, lo);
286 287 288 289
	nfsi->layout = NULL;
	/* Reset MDS Threshold I/O counters */
	nfsi->write_io = 0;
	nfsi->read_io = 0;
290 291
}

292
void
293
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
294
{
295
	struct inode *inode;
296

297 298 299
	if (!lo)
		return;
	inode = lo->plh_inode;
300 301
	pnfs_layoutreturn_before_put_layout_hdr(lo);

302
	if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
303 304
		if (!list_empty(&lo->plh_segs))
			WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
305
		pnfs_detach_layout_hdr(lo);
306
		spin_unlock(&inode->i_lock);
307
		pnfs_free_layout_hdr(lo);
308
	}
309 310
}

311 312 313 314 315 316 317 318 319 320 321 322 323 324
static void
pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode,
			 u32 seq)
{
	if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode)
		iomode = IOMODE_ANY;
	lo->plh_return_iomode = iomode;
	set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
	if (seq != 0) {
		WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq);
		lo->plh_return_seq = seq;
	}
}

325 326 327
static void
pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo)
{
328
	struct pnfs_layout_segment *lseg;
329 330 331
	lo->plh_return_iomode = 0;
	lo->plh_return_seq = 0;
	clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
332 333 334 335 336
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
		if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
			continue;
		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
	}
337 338
}

339 340 341 342 343 344 345 346 347
static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
{
	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
	clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags);
	smp_mb__after_atomic();
	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
}

348 349 350 351 352 353 354 355 356 357 358 359
static void
pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg,
		struct list_head *free_me)
{
	clear_bit(NFS_LSEG_ROC, &lseg->pls_flags);
	clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags))
		pnfs_lseg_dec_and_remove_zero(lseg, free_me);
	if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
		pnfs_lseg_dec_and_remove_zero(lseg, free_me);
}

360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
/*
 * Update the seqid of a layout stateid
 */
bool nfs4_refresh_layout_stateid(nfs4_stateid *dst, struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	bool ret = false;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo && nfs4_stateid_match_other(dst, &lo->plh_stateid)) {
		dst->seqid = lo->plh_stateid.seqid;
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

378 379 380 381 382 383 384
/*
 * Mark a pnfs_layout_hdr and all associated layout segments as invalid
 *
 * In order to continue using the pnfs_layout_hdr, a full recovery
 * is required.
 * Note that caller must hold inode->i_lock.
 */
385
int
386 387 388 389 390 391 392 393
pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
		struct list_head *lseg_list)
{
	struct pnfs_layout_range range = {
		.iomode = IOMODE_ANY,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
394
	struct pnfs_layout_segment *lseg, *next;
395 396

	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
397 398
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
		pnfs_clear_lseg_state(lseg, lseg_list);
399
	pnfs_clear_layoutreturn_info(lo);
400
	pnfs_free_returned_lsegs(lo, lseg_list, &range, 0);
401 402 403
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) &&
	    !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
		pnfs_clear_layoutreturn_waitbit(lo);
404
	return !list_empty(&lo->plh_segs);
405 406
}

407 408 409 410 411 412 413 414
static int
pnfs_iomode_to_fail_bit(u32 iomode)
{
	return iomode == IOMODE_RW ?
		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
}

static void
415
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
416
{
417
	lo->plh_retry_timestamp = jiffies;
418
	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
419
		refcount_inc(&lo->plh_refcount);
420 421 422 423 424 425
}

static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
{
	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
426
		refcount_dec(&lo->plh_refcount);
427 428 429 430 431 432
}

static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
	struct inode *inode = lo->plh_inode;
433 434 435 436 437 438
	struct pnfs_layout_range range = {
		.iomode = iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(head);
439 440 441

	spin_lock(&inode->i_lock);
	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
442
	pnfs_mark_matching_lsegs_invalid(lo, &head, &range, 0);
443
	spin_unlock(&inode->i_lock);
444
	pnfs_free_lseg_list(&head);
445 446 447 448 449 450 451
	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
			iomode == IOMODE_RW ?  "RW" : "READ");
}

static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
452
	unsigned long start, end;
453 454 455
	int fail_bit = pnfs_iomode_to_fail_bit(iomode);

	if (test_bit(fail_bit, &lo->plh_flags) == 0)
456 457 458 459 460
		return false;
	end = jiffies;
	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
		/* It is time to retry the failed layoutgets */
461
		pnfs_layout_clear_fail_bit(lo, fail_bit);
462 463 464
		return false;
	}
	return true;
465 466
}

467
static void
468 469 470
pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,
		const struct pnfs_layout_range *range,
		const nfs4_stateid *stateid)
471
{
472
	INIT_LIST_HEAD(&lseg->pls_list);
473
	INIT_LIST_HEAD(&lseg->pls_lc_list);
474
	refcount_set(&lseg->pls_refcount, 1);
475
	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
476
	lseg->pls_layout = lo;
477 478
	lseg->pls_range = *range;
	lseg->pls_seq = be32_to_cpu(stateid->seqid);
479 480
}

481
static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
482
{
483 484 485 486
	if (lseg != NULL) {
		struct inode *inode = lseg->pls_layout->plh_inode;
		NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg);
	}
487 488
}

F
Fred Isaman 已提交
489
static void
490 491
pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
F
Fred Isaman 已提交
492
{
493
	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
F
Fred Isaman 已提交
494
	list_del_init(&lseg->pls_list);
495
	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
496
	refcount_dec(&lo->plh_refcount);
497 498
	if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
		return;
499 500 501
	if (list_empty(&lo->plh_segs) &&
	    !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) &&
	    !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
502 503
		if (atomic_read(&lo->plh_outstanding) == 0)
			set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
504
		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
505
	}
F
Fred Isaman 已提交
506 507
}

508 509 510 511 512 513
static bool
pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
{
	if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
	    pnfs_layout_is_valid(lo)) {
514
		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
515 516 517 518 519 520
		list_move_tail(&lseg->pls_list, &lo->plh_return_segs);
		return true;
	}
	return false;
}

521
void
522
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
523
{
524
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
525 526 527 528 529
	struct inode *inode;

	if (!lseg)
		return;

530
	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
531
		refcount_read(&lseg->pls_refcount),
532
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
533

534 535
	lo = lseg->pls_layout;
	inode = lo->plh_inode;
536

537
	if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
538 539 540 541
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
			spin_unlock(&inode->i_lock);
			return;
		}
542
		pnfs_get_layout_hdr(lo);
543
		pnfs_layout_remove_lseg(lo, lseg);
544 545
		if (pnfs_cache_lseg_for_layoutreturn(lo, lseg))
			lseg = NULL;
546 547 548
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg(lseg);
		pnfs_put_layout_hdr(lo);
549 550
	}
}
551
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
552

553 554 555 556 557 558 559
/*
 * is l2 fully contained in l1?
 *   start1                             end1
 *   [----------------------------------)
 *           start2           end2
 *           [----------------)
 */
560
static bool
561
pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
562
		 const struct pnfs_layout_range *l2)
563 564
{
	u64 start1 = l1->offset;
565
	u64 end1 = pnfs_end_offset(start1, l1->length);
566
	u64 start2 = l2->offset;
567
	u64 end2 = pnfs_end_offset(start2, l2->length);
568 569 570 571

	return (start1 <= start2) && (end1 >= end2);
}

572 573 574
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		struct list_head *tmp_list)
{
575
	if (!refcount_dec_and_test(&lseg->pls_refcount))
576 577 578 579 580 581
		return false;
	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
	list_add(&lseg->pls_list, tmp_list);
	return true;
}

582 583 584 585 586 587 588 589 590 591 592
/* Returns 1 if lseg is removed from list, 0 otherwise */
static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
			     struct list_head *tmp_list)
{
	int rv = 0;

	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
		/* Remove the reference keeping the lseg in the
		 * list.  It will now be removed when all
		 * outstanding io is finished.
		 */
F
Fred Isaman 已提交
593
		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
594
			refcount_read(&lseg->pls_refcount));
595
		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
F
Fred Isaman 已提交
596
			rv = 1;
597 598 599 600
	}
	return rv;
}

601 602 603 604 605 606 607 608 609
/*
 * Compare 2 layout stateid sequence ids, to see which is newer,
 * taking into account wraparound issues.
 */
static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
{
	return (s32)(s1 - s2) > 0;
}

610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630
static bool
pnfs_should_free_range(const struct pnfs_layout_range *lseg_range,
		 const struct pnfs_layout_range *recall_range)
{
	return (recall_range->iomode == IOMODE_ANY ||
		lseg_range->iomode == recall_range->iomode) &&
	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
}

static bool
pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg,
		const struct pnfs_layout_range *recall_range,
		u32 seq)
{
	if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq))
		return false;
	if (recall_range == NULL)
		return true;
	return pnfs_should_free_range(&lseg->pls_range, recall_range);
}

631 632 633 634 635 636 637 638 639 640 641 642 643 644
/**
 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later
 * @lo: layout header containing the lsegs
 * @tmp_list: list head where doomed lsegs should go
 * @recall_range: optional recall range argument to match (may be NULL)
 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0)
 *
 * Walk the list of lsegs in the layout header, and tear down any that should
 * be destroyed. If "recall_range" is specified then the segment must match
 * that range. If "seq" is non-zero, then only match segments that were handed
 * out at or before that sequence.
 *
 * Returns number of matching invalid lsegs remaining in list after scanning
 * it and purging them.
645
 */
F
Fred Isaman 已提交
646
int
647
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
648
			    struct list_head *tmp_list,
649 650
			    const struct pnfs_layout_range *recall_range,
			    u32 seq)
651 652
{
	struct pnfs_layout_segment *lseg, *next;
653
	int remaining = 0;
654 655 656

	dprintk("%s:Begin lo %p\n", __func__, lo);

657
	if (list_empty(&lo->plh_segs))
658
		return 0;
659
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
660
		if (pnfs_match_lseg_recall(lseg, recall_range, seq)) {
661
			dprintk("%s: freeing lseg %p iomode %d seq %u "
662
				"offset %llu length %llu\n", __func__,
663 664
				lseg, lseg->pls_range.iomode, lseg->pls_seq,
				lseg->pls_range.offset, lseg->pls_range.length);
665 666
			if (!mark_lseg_invalid(lseg, tmp_list))
				remaining++;
667
		}
668 669
	dprintk("%s:Return %i\n", __func__, remaining);
	return remaining;
670 671
}

672 673 674 675 676 677 678 679 680 681 682 683 684 685
static void
pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
		struct list_head *free_me,
		const struct pnfs_layout_range *range,
		u32 seq)
{
	struct pnfs_layout_segment *lseg, *next;

	list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) {
		if (pnfs_match_lseg_recall(lseg, range, seq))
			list_move_tail(&lseg->pls_list, free_me);
	}
}

686
/* note free_me must contain lsegs from a single layout_hdr */
F
Fred Isaman 已提交
687
void
688
pnfs_free_lseg_list(struct list_head *free_me)
689
{
690
	struct pnfs_layout_segment *lseg, *tmp;
691 692 693 694

	if (list_empty(free_me))
		return;

695
	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
696
		list_del(&lseg->pls_list);
697
		pnfs_free_lseg(lseg);
698 699 700
	}
}

701 702 703 704
void
pnfs_destroy_layout(struct nfs_inode *nfsi)
{
	struct pnfs_layout_hdr *lo;
705
	LIST_HEAD(tmp_list);
706 707 708 709

	spin_lock(&nfsi->vfs_inode.i_lock);
	lo = nfsi->layout;
	if (lo) {
710
		pnfs_get_layout_hdr(lo);
711
		pnfs_mark_layout_stateid_invalid(lo, &tmp_list);
712 713 714 715
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
		spin_unlock(&nfsi->vfs_inode.i_lock);
		pnfs_free_lseg_list(&tmp_list);
716
		nfs_commit_inode(&nfsi->vfs_inode, 0);
717 718 719
		pnfs_put_layout_hdr(lo);
	} else
		spin_unlock(&nfsi->vfs_inode.i_lock);
720
}
721
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
722

723 724 725
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
		struct list_head *layout_list)
726 727
{
	struct pnfs_layout_hdr *lo;
728
	bool ret = false;
729

730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750
	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
		pnfs_get_layout_hdr(lo);
		list_add(&lo->plh_bulk_destroy, layout_list);
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

/* Caller must hold rcu_read_lock and clp->cl_lock */
static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
		struct nfs_server *server,
		struct list_head *layout_list)
{
	struct pnfs_layout_hdr *lo, *next;
	struct inode *inode;

	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
751 752
		if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags))
			continue;
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783
		inode = igrab(lo->plh_inode);
		if (inode == NULL)
			continue;
		list_del_init(&lo->plh_layouts);
		if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
			continue;
		rcu_read_unlock();
		spin_unlock(&clp->cl_lock);
		iput(inode);
		spin_lock(&clp->cl_lock);
		rcu_read_lock();
		return -EAGAIN;
	}
	return 0;
}

static int
pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
		bool is_bulk_recall)
{
	struct pnfs_layout_hdr *lo;
	struct inode *inode;
	LIST_HEAD(lseg_list);
	int ret = 0;

	while (!list_empty(layout_list)) {
		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
				plh_bulk_destroy);
		dprintk("%s freeing layout for inode %lu\n", __func__,
			lo->plh_inode->i_ino);
		inode = lo->plh_inode;
784 785 786

		pnfs_layoutcommit_inode(inode, false);

787 788
		spin_lock(&inode->i_lock);
		list_del_init(&lo->plh_bulk_destroy);
789 790 791
		if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) {
			if (is_bulk_recall)
				set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
792
			ret = -EAGAIN;
793
		}
794 795
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg_list(&lseg_list);
796 797
		/* Free all lsegs that are attached to commit buckets */
		nfs_commit_inode(inode, 0);
798 799 800 801 802 803 804 805 806 807 808 809 810
		pnfs_put_layout_hdr(lo);
		iput(inode);
	}
	return ret;
}

int
pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
		struct nfs_fsid *fsid,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);
811

812
	spin_lock(&clp->cl_lock);
813
	rcu_read_lock();
814
restart:
815
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
816 817 818 819 820 821
		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
			continue;
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
				server,
				&layout_list) != 0)
			goto restart;
822 823
	}
	rcu_read_unlock();
824 825
	spin_unlock(&clp->cl_lock);

826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845
	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

int
pnfs_destroy_layouts_byclid(struct nfs_client *clp,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);

	spin_lock(&clp->cl_lock);
	rcu_read_lock();
restart:
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
					server,
					&layout_list) != 0)
			goto restart;
846
	}
847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
	rcu_read_unlock();
	spin_unlock(&clp->cl_lock);

	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

/*
 * Called by the state manger to remove all layouts established under an
 * expired lease.
 */
void
pnfs_destroy_all_layouts(struct nfs_client *clp)
{
	nfs4_deviceid_mark_client_invalid(clp);
	nfs4_deviceid_purge_client(clp);

	pnfs_destroy_layouts_byclid(clp, false);
866 867
}

868
/* update lo->plh_stateid with new if is more recent */
F
Fred Isaman 已提交
869 870 871
void
pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
			bool update_barrier)
872
{
873
	u32 oldseq, newseq, new_barrier = 0;
874

875 876
	oldseq = be32_to_cpu(lo->plh_stateid.seqid);
	newseq = be32_to_cpu(new->seqid);
877 878 879 880 881 882 883 884 885

	if (!pnfs_layout_is_valid(lo)) {
		nfs4_stateid_copy(&lo->plh_stateid, new);
		lo->plh_barrier = newseq;
		pnfs_clear_layoutreturn_info(lo);
		clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
		return;
	}
	if (pnfs_seqid_is_newer(newseq, oldseq)) {
886
		nfs4_stateid_copy(&lo->plh_stateid, new);
887 888 889 890 891
		/*
		 * Because of wraparound, we want to keep the barrier
		 * "close" to the current seqids.
		 */
		new_barrier = newseq - atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
892
	}
893 894 895 896
	if (update_barrier)
		new_barrier = be32_to_cpu(new->seqid);
	else if (new_barrier == 0)
		return;
897
	if (pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
898
		lo->plh_barrier = new_barrier;
899 900
}

901
static bool
902 903
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid)
F
Fred Isaman 已提交
904
{
905
	u32 seqid = be32_to_cpu(stateid->seqid);
906

907 908 909 910 911
	return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
}

/* lget is set to 1 if called from inside send_layoutget call chain */
static bool
912
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
913
{
F
Fred Isaman 已提交
914
	return lo->plh_block_lgets ||
915
		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
916 917
}

918
/*
919 920 921 922 923
 * Get layout from server.
 *    for now, assume that whole file layouts are requested.
 *    arg->offset: 0
 *    arg->length: all ones
 */
924 925 926
static struct pnfs_layout_segment *
send_layoutget(struct pnfs_layout_hdr *lo,
	   struct nfs_open_context *ctx,
927
	   nfs4_stateid *stateid,
928
	   const struct pnfs_layout_range *range,
929
	   long *timeout, gfp_t gfp_flags)
930
{
931
	struct inode *ino = lo->plh_inode;
932 933
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs4_layoutget *lgp;
934
	loff_t i_size;
935 936

	dprintk("--> %s\n", __func__);
937

938 939 940 941 942
	/*
	 * Synchronously retrieve layout information from server and
	 * store in lseg. If we race with a concurrent seqid morphing
	 * op, then re-send the LAYOUTGET.
	 */
943 944 945 946 947 948 949 950 951 952 953 954 955 956
	lgp = kzalloc(sizeof(*lgp), gfp_flags);
	if (lgp == NULL)
		return ERR_PTR(-ENOMEM);

	i_size = i_size_read(ino);

	lgp->args.minlength = PAGE_SIZE;
	if (lgp->args.minlength > range->length)
		lgp->args.minlength = range->length;
	if (range->iomode == IOMODE_READ) {
		if (range->offset >= i_size)
			lgp->args.minlength = 0;
		else if (i_size - range->offset < lgp->args.minlength)
			lgp->args.minlength = i_size - range->offset;
957
	}
958 959 960 961 962
	lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
	pnfs_copy_range(&lgp->args.range, range);
	lgp->args.type = server->pnfs_curr_ld->id;
	lgp->args.inode = ino;
	lgp->args.ctx = get_nfs_open_context(ctx);
963
	nfs4_stateid_copy(&lgp->args.stateid, stateid);
964 965
	lgp->gfp_flags = gfp_flags;
	lgp->cred = lo->plh_lc_cred;
966

967
	return nfs4_proc_layoutget(lgp, timeout, gfp_flags);
968 969
}

970 971 972 973 974 975 976 977 978 979 980 981 982 983 984
static void pnfs_clear_layoutcommit(struct inode *inode,
		struct list_head *head)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct pnfs_layout_segment *lseg, *tmp;

	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		return;
	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
			continue;
		pnfs_lseg_dec_and_remove_zero(lseg, head);
	}
}

985
void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo,
986
		const nfs4_stateid *arg_stateid,
987 988 989 990 991 992 993
		const struct pnfs_layout_range *range,
		const nfs4_stateid *stateid)
{
	struct inode *inode = lo->plh_inode;
	LIST_HEAD(freeme);

	spin_lock(&inode->i_lock);
994 995 996
	if (!pnfs_layout_is_valid(lo) || !arg_stateid ||
	    !nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid))
		goto out_unlock;
997
	if (stateid) {
998 999
		u32 seq = be32_to_cpu(arg_stateid->seqid);

1000 1001 1002 1003 1004
		pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq);
		pnfs_free_returned_lsegs(lo, &freeme, range, seq);
		pnfs_set_layout_stateid(lo, stateid, true);
	} else
		pnfs_mark_layout_stateid_invalid(lo, &freeme);
1005
out_unlock:
1006 1007 1008 1009 1010 1011
	pnfs_clear_layoutreturn_waitbit(lo);
	spin_unlock(&inode->i_lock);
	pnfs_free_lseg_list(&freeme);

}

1012
static bool
1013 1014 1015
pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo,
		nfs4_stateid *stateid,
		enum pnfs_iomode *iomode)
1016
{
1017 1018 1019
	/* Serialise LAYOUTGET/LAYOUTRETURN */
	if (atomic_read(&lo->plh_outstanding) != 0)
		return false;
1020
	if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
1021
		return false;
1022
	set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
1023
	pnfs_get_layout_hdr(lo);
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
		if (stateid != NULL) {
			nfs4_stateid_copy(stateid, &lo->plh_stateid);
			if (lo->plh_return_seq != 0)
				stateid->seqid = cpu_to_be32(lo->plh_return_seq);
		}
		if (iomode != NULL)
			*iomode = lo->plh_return_iomode;
		pnfs_clear_layoutreturn_info(lo);
		return true;
	}
	if (stateid != NULL)
		nfs4_stateid_copy(stateid, &lo->plh_stateid);
	if (iomode != NULL)
		*iomode = IOMODE_ANY;
1039 1040 1041
	return true;
}

1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
static void
pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args,
		struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid,
		enum pnfs_iomode iomode)
{
	struct inode *inode = lo->plh_inode;

	args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id;
	args->inode = inode;
	args->range.iomode = iomode;
	args->range.offset = 0;
	args->range.length = NFS4_MAX_UINT64;
	args->layout = lo;
	nfs4_stateid_copy(&args->stateid, stateid);
}

1059
static int
1060
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
1061
		       enum pnfs_iomode iomode, bool sync)
1062 1063
{
	struct inode *ino = lo->plh_inode;
1064
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
1065 1066 1067
	struct nfs4_layoutreturn *lrp;
	int status = 0;

1068
	lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
1069 1070 1071
	if (unlikely(lrp == NULL)) {
		status = -ENOMEM;
		spin_lock(&ino->i_lock);
1072
		pnfs_clear_layoutreturn_waitbit(lo);
1073 1074 1075 1076 1077
		spin_unlock(&ino->i_lock);
		pnfs_put_layout_hdr(lo);
		goto out;
	}

1078
	pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode);
1079
	lrp->args.ld_private = &lrp->ld_private;
1080 1081
	lrp->clp = NFS_SERVER(ino)->nfs_client;
	lrp->cred = lo->plh_lc_cred;
1082 1083
	if (ld->prepare_layoutreturn)
		ld->prepare_layoutreturn(&lrp->args);
1084

1085
	status = nfs4_proc_layoutreturn(lrp, sync);
1086 1087 1088 1089 1090
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

1091 1092 1093 1094 1095 1096
/* Return true if layoutreturn is needed */
static bool
pnfs_layout_need_return(struct pnfs_layout_hdr *lo)
{
	struct pnfs_layout_segment *s;

1097
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
		return false;

	/* Defer layoutreturn until all lsegs are done */
	list_for_each_entry(s, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
			return false;
	}

	return true;
}

static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo)
{
	struct inode *inode= lo->plh_inode;

1113
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1114 1115 1116 1117 1118 1119 1120
		return;
	spin_lock(&inode->i_lock);
	if (pnfs_layout_need_return(lo)) {
		nfs4_stateid stateid;
		enum pnfs_iomode iomode;
		bool send;

1121
		send = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
1122 1123 1124 1125 1126 1127 1128 1129 1130
		spin_unlock(&inode->i_lock);
		if (send) {
			/* Send an async layoutreturn so we dont deadlock */
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
		}
	} else
		spin_unlock(&inode->i_lock);
}

1131 1132 1133 1134 1135 1136 1137 1138
/*
 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
 * when the layout segment list is empty.
 *
 * Note that a pnfs_layout_hdr can exist with an empty layout segment
 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
 * deviceid is marked invalid.
 */
B
Benny Halevy 已提交
1139 1140 1141 1142 1143 1144 1145
int
_pnfs_return_layout(struct inode *ino)
{
	struct pnfs_layout_hdr *lo = NULL;
	struct nfs_inode *nfsi = NFS_I(ino);
	LIST_HEAD(tmp_list);
	nfs4_stateid stateid;
1146
	int status = 0;
1147
	bool send;
B
Benny Halevy 已提交
1148

1149
	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
B
Benny Halevy 已提交
1150 1151 1152

	spin_lock(&ino->i_lock);
	lo = nfsi->layout;
1153
	if (!lo) {
B
Benny Halevy 已提交
1154
		spin_unlock(&ino->i_lock);
1155 1156
		dprintk("NFS: %s no layout to return\n", __func__);
		goto out;
B
Benny Halevy 已提交
1157 1158
	}
	/* Reference matched in nfs4_layoutreturn_release */
1159
	pnfs_get_layout_hdr(lo);
1160 1161 1162 1163 1164 1165 1166 1167
	/* Is there an outstanding layoutreturn ? */
	if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) {
		spin_unlock(&ino->i_lock);
		if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN,
					TASK_UNINTERRUPTIBLE))
			goto out_put_layout_hdr;
		spin_lock(&ino->i_lock);
	}
1168
	pnfs_clear_layoutcommit(ino, &tmp_list);
1169
	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0);
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179

	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
		struct pnfs_layout_range range = {
			.iomode		= IOMODE_ANY,
			.offset		= 0,
			.length		= NFS4_MAX_UINT64,
		};
		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
	}

1180
	/* Don't send a LAYOUTRETURN if list was initially empty */
1181
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
1182 1183
		spin_unlock(&ino->i_lock);
		dprintk("NFS: %s no layout segments to return\n", __func__);
1184
		goto out_put_layout_hdr;
1185
	}
1186

1187
	send = pnfs_prepare_layoutreturn(lo, &stateid, NULL);
B
Benny Halevy 已提交
1188
	spin_unlock(&ino->i_lock);
1189
	if (send)
1190
		status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1191
out_put_layout_hdr:
1192
	pnfs_free_lseg_list(&tmp_list);
1193
	pnfs_put_layout_hdr(lo);
B
Benny Halevy 已提交
1194 1195 1196 1197 1198
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
int
pnfs_commit_and_return_layout(struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	int ret;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo == NULL) {
		spin_unlock(&inode->i_lock);
		return 0;
	}
	pnfs_get_layout_hdr(lo);
	/* Block new layoutgets and read/write to ds */
	lo->plh_block_lgets++;
	spin_unlock(&inode->i_lock);
	filemap_fdatawait(inode->i_mapping);
	ret = pnfs_layoutcommit_inode(inode, true);
	if (ret == 0)
		ret = _pnfs_return_layout(inode);
	spin_lock(&inode->i_lock);
	lo->plh_block_lgets--;
	spin_unlock(&inode->i_lock);
	pnfs_put_layout_hdr(lo);
	return ret;
}

1226 1227 1228 1229
bool pnfs_roc(struct inode *ino,
		struct nfs4_layoutreturn_args *args,
		struct nfs4_layoutreturn_res *res,
		const struct rpc_cred *cred)
F
Fred Isaman 已提交
1230
{
1231 1232 1233
	struct nfs_inode *nfsi = NFS_I(ino);
	struct nfs_open_context *ctx;
	struct nfs4_state *state;
F
Fred Isaman 已提交
1234
	struct pnfs_layout_hdr *lo;
1235
	struct pnfs_layout_segment *lseg, *next;
1236
	nfs4_stateid stateid;
1237 1238
	enum pnfs_iomode iomode = 0;
	bool layoutreturn = false, roc = false;
1239
	bool skip_read = false;
F
Fred Isaman 已提交
1240

1241 1242
	if (!nfs_have_layout(ino))
		return false;
1243
retry:
F
Fred Isaman 已提交
1244
	spin_lock(&ino->i_lock);
1245
	lo = nfsi->layout;
1246
	if (!lo || !pnfs_layout_is_valid(lo) ||
1247 1248
	    test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
		lo = NULL;
1249
		goto out_noroc;
1250 1251
	}
	pnfs_get_layout_hdr(lo);
1252 1253 1254 1255 1256 1257 1258
	if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) {
		spin_unlock(&ino->i_lock);
		wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN,
				TASK_UNINTERRUPTIBLE);
		pnfs_put_layout_hdr(lo);
		goto retry;
	}
1259

1260
	/* no roc if we hold a delegation */
1261 1262 1263 1264 1265
	if (nfs4_check_delegation(ino, FMODE_READ)) {
		if (nfs4_check_delegation(ino, FMODE_WRITE))
			goto out_noroc;
		skip_read = true;
	}
1266 1267 1268

	list_for_each_entry(ctx, &nfsi->open_files, list) {
		state = ctx->state;
1269 1270
		if (state == NULL)
			continue;
1271
		/* Don't return layout if there is open file state */
1272
		if (state->state & FMODE_WRITE)
1273
			goto out_noroc;
1274 1275
		if (state->state & FMODE_READ)
			skip_read = true;
1276 1277
	}

1278

1279
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) {
1280 1281
		if (skip_read && lseg->pls_range.iomode == IOMODE_READ)
			continue;
1282
		/* If we are sending layoutreturn, invalidate all valid lsegs */
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
		if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags))
			continue;
		/*
		 * Note: mark lseg for return so pnfs_layout_remove_lseg
		 * doesn't invalidate the layout for us.
		 */
		set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
		if (!mark_lseg_invalid(lseg, &lo->plh_return_segs))
			continue;
		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
1293 1294
	}

1295 1296
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
		goto out_noroc;
1297

1298
	/* ROC in two conditions:
1299 1300 1301
	 * 1. there are ROC lsegs
	 * 2. we don't send layoutreturn
	 */
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
	/* lo ref dropped in pnfs_roc_release() */
	layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
	/* If the creds don't match, we can't compound the layoutreturn */
	if (!layoutreturn || cred != lo->plh_lc_cred)
		goto out_noroc;

	roc = layoutreturn;
	pnfs_init_layoutreturn_args(args, lo, &stateid, iomode);
	res->lrs_present = 0;
	layoutreturn = false;
F
Fred Isaman 已提交
1312

1313
out_noroc:
F
Fred Isaman 已提交
1314
	spin_unlock(&ino->i_lock);
1315
	pnfs_layoutcommit_inode(ino, true);
1316 1317 1318 1319
	if (roc) {
		struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
		if (ld->prepare_layoutreturn)
			ld->prepare_layoutreturn(args);
1320
		pnfs_put_layout_hdr(lo);
1321 1322
		return true;
	}
1323
	if (layoutreturn)
1324
		pnfs_send_layoutreturn(lo, &stateid, iomode, true);
1325
	pnfs_put_layout_hdr(lo);
1326
	return false;
F
Fred Isaman 已提交
1327 1328
}

1329 1330 1331
void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
		struct nfs4_layoutreturn_res *res,
		int ret)
F
Fred Isaman 已提交
1332
{
1333 1334 1335
	struct pnfs_layout_hdr *lo = args->layout;
	const nfs4_stateid *arg_stateid = NULL;
	const nfs4_stateid *res_stateid = NULL;
1336
	struct nfs4_xdr_opaque_data *ld_private = args->ld_private;
F
Fred Isaman 已提交
1337

1338 1339 1340 1341 1342 1343 1344
	if (ret == 0) {
		arg_stateid = &args->stateid;
		if (res->lrs_present)
			res_stateid = &res->stateid;
	}
	pnfs_layoutreturn_free_lsegs(lo, arg_stateid, &args->range,
			res_stateid);
1345 1346
	if (ld_private && ld_private->ops && ld_private->ops->free)
		ld_private->ops->free(ld_private);
1347 1348
	pnfs_put_layout_hdr(lo);
	trace_nfs4_layoutreturn_on_close(args->inode, 0);
F
Fred Isaman 已提交
1349 1350
}

1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
{
	struct nfs_inode *nfsi = NFS_I(ino);
        struct pnfs_layout_hdr *lo;
        bool sleep = false;

	/* we might not have grabbed lo reference. so need to check under
	 * i_lock */
        spin_lock(&ino->i_lock);
        lo = nfsi->layout;
1361 1362
        if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
1363
                sleep = true;
1364
	}
1365 1366 1367 1368
        spin_unlock(&ino->i_lock);
        return sleep;
}

1369 1370 1371 1372 1373 1374
/*
 * Compare two layout segments for sorting into layout cache.
 * We want to preferentially return RW over RO layouts, so ensure those
 * are seen first.
 */
static s64
1375
pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1376
	   const struct pnfs_layout_range *l2)
1377
{
1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
	s64 d;

	/* high offset > low offset */
	d = l1->offset - l2->offset;
	if (d)
		return d;

	/* short length > long length */
	d = l2->length - l1->length;
	if (d)
		return d;

1390
	/* read > read/write */
1391
	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1392 1393
}

1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
static bool
pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
		const struct pnfs_layout_range *l2)
{
	return pnfs_lseg_range_cmp(l1, l2) > 0;
}

static bool
pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
		struct pnfs_layout_segment *old)
{
	return false;
}

void
pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   bool (*is_after)(const struct pnfs_layout_range *,
			   const struct pnfs_layout_range *),
		   bool (*do_merge)(struct pnfs_layout_segment *,
			   struct pnfs_layout_segment *),
		   struct list_head *free_me)
1416
{
1417
	struct pnfs_layout_segment *lp, *tmp;
1418

1419 1420
	dprintk("%s:Begin\n", __func__);

1421 1422 1423 1424 1425 1426 1427 1428
	list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
			continue;
		if (do_merge(lseg, lp)) {
			mark_lseg_invalid(lp, free_me);
			continue;
		}
		if (is_after(&lseg->pls_range, &lp->pls_range))
1429
			continue;
1430
		list_add_tail(&lseg->pls_list, &lp->pls_list);
1431 1432 1433
		dprintk("%s: inserted lseg %p "
			"iomode %d offset %llu length %llu before "
			"lp %p iomode %d offset %llu length %llu\n",
1434 1435 1436 1437
			__func__, lseg, lseg->pls_range.iomode,
			lseg->pls_range.offset, lseg->pls_range.length,
			lp, lp->pls_range.iomode, lp->pls_range.offset,
			lp->pls_range.length);
1438
		goto out;
1439
	}
1440 1441 1442 1443 1444 1445
	list_add_tail(&lseg->pls_list, &lo->plh_segs);
	dprintk("%s: inserted lseg %p "
		"iomode %d offset %llu length %llu at tail\n",
		__func__, lseg, lseg->pls_range.iomode,
		lseg->pls_range.offset, lseg->pls_range.length);
out:
1446
	pnfs_get_layout_hdr(lo);
1447 1448

	dprintk("%s:Return\n", __func__);
1449
}
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467
EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);

static void
pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   struct list_head *free_me)
{
	struct inode *inode = lo->plh_inode;
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;

	if (ld->add_lseg != NULL)
		ld->add_lseg(lo, lseg, free_me);
	else
		pnfs_generic_layout_insert_lseg(lo, lseg,
				pnfs_lseg_range_is_after,
				pnfs_lseg_no_merge,
				free_me);
}
1468 1469

static struct pnfs_layout_hdr *
1470 1471 1472
alloc_init_layout_hdr(struct inode *ino,
		      struct nfs_open_context *ctx,
		      gfp_t gfp_flags)
1473 1474 1475
{
	struct pnfs_layout_hdr *lo;

1476
	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1477 1478
	if (!lo)
		return NULL;
1479
	refcount_set(&lo->plh_refcount, 1);
1480 1481
	INIT_LIST_HEAD(&lo->plh_layouts);
	INIT_LIST_HEAD(&lo->plh_segs);
1482
	INIT_LIST_HEAD(&lo->plh_return_segs);
1483
	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1484
	lo->plh_inode = ino;
1485
	lo->plh_lc_cred = get_rpccred(ctx->cred);
1486
	lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID;
1487 1488 1489 1490
	return lo;
}

static struct pnfs_layout_hdr *
1491 1492 1493
pnfs_find_alloc_layout(struct inode *ino,
		       struct nfs_open_context *ctx,
		       gfp_t gfp_flags)
1494 1495
	__releases(&ino->i_lock)
	__acquires(&ino->i_lock)
1496 1497 1498 1499 1500 1501
{
	struct nfs_inode *nfsi = NFS_I(ino);
	struct pnfs_layout_hdr *new = NULL;

	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

1502 1503
	if (nfsi->layout != NULL)
		goto out_existing;
1504
	spin_unlock(&ino->i_lock);
1505
	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1506 1507
	spin_lock(&ino->i_lock);

1508
	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1509
		nfsi->layout = new;
1510
		return new;
1511 1512
	} else if (new != NULL)
		pnfs_free_layout_hdr(new);
1513 1514
out_existing:
	pnfs_get_layout_hdr(nfsi->layout);
1515 1516 1517
	return nfsi->layout;
}

1518 1519
/*
 * iomode matching rules:
1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
 * iomode	lseg	strict match
 *                      iomode
 * -----	-----	------ -----
 * ANY		READ	N/A    true
 * ANY		RW	N/A    true
 * RW		READ	N/A    false
 * RW		RW	N/A    true
 * READ		READ	N/A    true
 * READ		RW	true   false
 * READ		RW	false  true
1530
 */
1531
static bool
1532
pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1533 1534
		 const struct pnfs_layout_range *range,
		 bool strict_iomode)
1535
{
1536 1537 1538 1539
	struct pnfs_layout_range range1;

	if ((range->iomode == IOMODE_RW &&
	     ls_range->iomode != IOMODE_RW) ||
1540
	    (range->iomode != ls_range->iomode &&
1541
	     strict_iomode) ||
1542
	    !pnfs_lseg_range_intersecting(ls_range, range))
1543 1544 1545 1546 1547
		return 0;

	/* range1 covers only the first byte in the range */
	range1 = *range;
	range1.length = 1;
1548
	return pnfs_lseg_range_contained(ls_range, &range1);
1549 1550 1551 1552 1553
}

/*
 * lookup range in layout
 */
1554
static struct pnfs_layout_segment *
1555
pnfs_find_lseg(struct pnfs_layout_hdr *lo,
1556 1557
		struct pnfs_layout_range *range,
		bool strict_iomode)
1558
{
1559 1560 1561 1562
	struct pnfs_layout_segment *lseg, *ret = NULL;

	dprintk("%s:Begin\n", __func__);

1563
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1564
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1565
		    !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
1566 1567
		    pnfs_lseg_range_match(&lseg->pls_range, range,
					  strict_iomode)) {
1568
			ret = pnfs_get_lseg(lseg);
1569 1570 1571 1572 1573
			break;
		}
	}

	dprintk("%s:Return lseg %p ref %d\n",
1574
		__func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0);
1575
	return ret;
1576 1577
}

1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652
/*
 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
 * to the MDS or over pNFS
 *
 * The nfs_inode read_io and write_io fields are cumulative counters reset
 * when there are no layout segments. Note that in pnfs_update_layout iomode
 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
 * WRITE request.
 *
 * A return of true means use MDS I/O.
 *
 * From rfc 5661:
 * If a file's size is smaller than the file size threshold, data accesses
 * SHOULD be sent to the metadata server.  If an I/O request has a length that
 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
 * server.  If both file size and I/O size are provided, the client SHOULD
 * reach or exceed  both thresholds before sending its read or write
 * requests to the data server.
 */
static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
				     struct inode *ino, int iomode)
{
	struct nfs4_threshold *t = ctx->mdsthreshold;
	struct nfs_inode *nfsi = NFS_I(ino);
	loff_t fsize = i_size_read(ino);
	bool size = false, size_set = false, io = false, io_set = false, ret = false;

	if (t == NULL)
		return ret;

	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);

	switch (iomode) {
	case IOMODE_READ:
		if (t->bm & THRESHOLD_RD) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->rd_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_RD_IO) {
			dprintk("%s nfsi->read_io %llu\n", __func__,
				nfsi->read_io);
			io_set = true;
			if (nfsi->read_io < t->rd_io_sz)
				io = true;
		}
		break;
	case IOMODE_RW:
		if (t->bm & THRESHOLD_WR) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->wr_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_WR_IO) {
			dprintk("%s nfsi->write_io %llu\n", __func__,
				nfsi->write_io);
			io_set = true;
			if (nfsi->write_io < t->wr_io_sz)
				io = true;
		}
		break;
	}
	if (size_set && io_set) {
		if (size && io)
			ret = true;
	} else if (size || io)
		ret = true;

	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
	return ret;
}

1653 1654 1655 1656 1657 1658 1659 1660
static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
{
	/*
	 * send layoutcommit as it can hold up layoutreturn due to lseg
	 * reference
	 */
	pnfs_layoutcommit_inode(lo->plh_inode, false);
	return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
1661
				   nfs_wait_bit_killable,
1662 1663 1664
				   TASK_UNINTERRUPTIBLE);
}

1665 1666 1667 1668 1669 1670 1671 1672 1673
static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
{
	unsigned long *bitlock = &lo->plh_flags;

	clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
	smp_mb__after_atomic();
	wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
}

1674 1675 1676 1677
/*
 * Layout segment is retreived from the server if not cached.
 * The appropriate layout segment is referenced and returned to the caller.
 */
1678
struct pnfs_layout_segment *
1679 1680
pnfs_update_layout(struct inode *ino,
		   struct nfs_open_context *ctx,
1681 1682
		   loff_t pos,
		   u64 count,
1683
		   enum pnfs_iomode iomode,
1684
		   bool strict_iomode,
1685
		   gfp_t gfp_flags)
1686
{
1687 1688 1689 1690 1691
	struct pnfs_layout_range arg = {
		.iomode = iomode,
		.offset = pos,
		.length = count,
	};
1692
	unsigned pg_offset;
1693 1694
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs_client *clp = server->nfs_client;
1695
	struct pnfs_layout_hdr *lo = NULL;
1696
	struct pnfs_layout_segment *lseg = NULL;
1697 1698
	nfs4_stateid stateid;
	long timeout = 0;
1699
	unsigned long giveup = jiffies + (clp->cl_lease_time << 1);
1700
	bool first;
1701

1702
	if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
1703
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1704
				 PNFS_UPDATE_LAYOUT_NO_PNFS);
1705
		goto out;
1706
	}
1707

1708
	if (iomode == IOMODE_READ && i_size_read(ino) == 0) {
1709
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1710
				 PNFS_UPDATE_LAYOUT_RD_ZEROLEN);
1711
		goto out;
1712
	}
1713

1714
	if (pnfs_within_mdsthreshold(ctx, ino, iomode)) {
1715
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1716
				 PNFS_UPDATE_LAYOUT_MDSTHRESH);
1717
		goto out;
1718
	}
1719

1720
lookup_again:
1721
	nfs4_client_recover_expired_lease(clp);
1722
	first = false;
1723
	spin_lock(&ino->i_lock);
1724
	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
1725 1726
	if (lo == NULL) {
		spin_unlock(&ino->i_lock);
1727
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1728
				 PNFS_UPDATE_LAYOUT_NOMEM);
1729 1730
		goto out;
	}
1731

F
Fred Isaman 已提交
1732
	/* Do we even need to bother with this? */
1733
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
1734
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1735
				 PNFS_UPDATE_LAYOUT_BULK_RECALL);
F
Fred Isaman 已提交
1736
		dprintk("%s matches recall, use MDS\n", __func__);
1737 1738 1739 1740
		goto out_unlock;
	}

	/* if LAYOUTGET already failed once we don't try again */
1741
	if (pnfs_layout_io_test_failed(lo, iomode)) {
1742
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1743
				 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL);
1744
		goto out_unlock;
1745
	}
1746

1747
	lseg = pnfs_find_lseg(lo, &arg, strict_iomode);
1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
	if (lseg) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				PNFS_UPDATE_LAYOUT_FOUND_CACHED);
		goto out_unlock;
	}

	if (!nfs4_valid_open_stateid(ctx->state)) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				PNFS_UPDATE_LAYOUT_INVALID_OPEN);
		goto out_unlock;
	}

	/*
	 * Choose a stateid for the LAYOUTGET. If we don't have a layout
	 * stateid, or it has been invalidated, then we must use the open
	 * stateid.
	 */
1765
	if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
1766 1767 1768

		/*
		 * The first layoutget for the file. Need to serialize per
1769 1770 1771 1772 1773 1774 1775 1776
		 * RFC 5661 Errata 3208.
		 */
		if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
				     &lo->plh_flags)) {
			spin_unlock(&ino->i_lock);
			wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
				    TASK_UNINTERRUPTIBLE);
			pnfs_put_layout_hdr(lo);
1777
			dprintk("%s retrying\n", __func__);
1778 1779
			goto lookup_again;
		}
1780 1781

		first = true;
1782 1783 1784 1785 1786 1787 1788 1789
		if (nfs4_select_rw_stateid(ctx->state,
					iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ,
					NULL, &stateid, NULL) != 0) {
			trace_pnfs_update_layout(ino, pos, count,
					iomode, lo, lseg,
					PNFS_UPDATE_LAYOUT_INVALID_OPEN);
			goto out_unlock;
		}
1790
	} else {
1791
		nfs4_stateid_copy(&stateid, &lo->plh_stateid);
1792
	}
1793

1794 1795 1796 1797
	/*
	 * Because we free lsegs before sending LAYOUTRETURN, we need to wait
	 * for LAYOUTRETURN even if first is true.
	 */
1798
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
1799 1800 1801
		spin_unlock(&ino->i_lock);
		dprintk("%s wait for layoutreturn\n", __func__);
		if (pnfs_prepare_to_retry_layoutget(lo)) {
1802 1803
			if (first)
				pnfs_clear_first_layoutget(lo);
1804 1805
			pnfs_put_layout_hdr(lo);
			dprintk("%s retrying\n", __func__);
1806 1807
			trace_pnfs_update_layout(ino, pos, count, iomode, lo,
					lseg, PNFS_UPDATE_LAYOUT_RETRY);
1808 1809
			goto lookup_again;
		}
1810
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1811
				PNFS_UPDATE_LAYOUT_RETURN);
1812 1813 1814
		goto out_put_layout_hdr;
	}

1815
	if (pnfs_layoutgets_blocked(lo)) {
1816
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1817
				PNFS_UPDATE_LAYOUT_BLOCKED);
1818
		goto out_unlock;
1819
	}
1820
	atomic_inc(&lo->plh_outstanding);
1821
	spin_unlock(&ino->i_lock);
1822

1823
	if (list_empty(&lo->plh_layouts)) {
1824 1825 1826 1827
		/* The lo must be on the clp list if there is any
		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
		 */
		spin_lock(&clp->cl_lock);
1828 1829
		if (list_empty(&lo->plh_layouts))
			list_add_tail(&lo->plh_layouts, &server->layouts);
1830 1831
		spin_unlock(&clp->cl_lock);
	}
1832

1833
	pg_offset = arg.offset & ~PAGE_MASK;
1834 1835 1836 1837
	if (pg_offset) {
		arg.offset -= pg_offset;
		arg.length += pg_offset;
	}
1838
	if (arg.length != NFS4_MAX_UINT64)
1839
		arg.length = PAGE_ALIGN(arg.length);
1840

1841 1842 1843
	lseg = send_layoutget(lo, ctx, &stateid, &arg, &timeout, gfp_flags);
	trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
1844
	atomic_dec(&lo->plh_outstanding);
1845
	if (IS_ERR(lseg)) {
1846
		switch(PTR_ERR(lseg)) {
1847
		case -EBUSY:
1848 1849
			if (time_after(jiffies, giveup))
				lseg = NULL;
1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
			break;
		case -ERECALLCONFLICT:
			/* Huh? We hold no layouts, how is there a recall? */
			if (first) {
				lseg = NULL;
				break;
			}
			/* Destroy the existing layout and start over */
			if (time_after(jiffies, giveup))
				pnfs_destroy_layout(NFS_I(ino));
1860 1861
			/* Fallthrough */
		case -EAGAIN:
1862
			break;
1863 1864 1865 1866 1867
		default:
			if (!nfs_error_is_fatal(PTR_ERR(lseg))) {
				pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
				lseg = NULL;
			}
1868 1869 1870 1871 1872 1873 1874 1875 1876
			goto out_put_layout_hdr;
		}
		if (lseg) {
			if (first)
				pnfs_clear_first_layoutget(lo);
			trace_pnfs_update_layout(ino, pos, count,
				iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY);
			pnfs_put_layout_hdr(lo);
			goto lookup_again;
1877 1878 1879 1880 1881
		}
	} else {
		pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
	}

1882
out_put_layout_hdr:
1883 1884
	if (first)
		pnfs_clear_first_layoutget(lo);
1885
	pnfs_put_layout_hdr(lo);
1886
out:
1887 1888 1889 1890
	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
			"(%s, offset: %llu, length: %llu)\n",
			__func__, ino->i_sb->s_id,
			(unsigned long long)NFS_FILEID(ino),
1891
			IS_ERR_OR_NULL(lseg) ? "not found" : "found",
1892 1893 1894
			iomode==IOMODE_RW ?  "read/write" : "read-only",
			(unsigned long long)pos,
			(unsigned long long)count);
1895 1896 1897
	return lseg;
out_unlock:
	spin_unlock(&ino->i_lock);
1898
	goto out_put_layout_hdr;
1899
}
1900
EXPORT_SYMBOL_GPL(pnfs_update_layout);
1901

1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921
static bool
pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
{
	switch (range->iomode) {
	case IOMODE_READ:
	case IOMODE_RW:
		break;
	default:
		return false;
	}
	if (range->offset == NFS4_MAX_UINT64)
		return false;
	if (range->length == 0)
		return false;
	if (range->length != NFS4_MAX_UINT64 &&
	    range->length > NFS4_MAX_UINT64 - range->offset)
		return false;
	return true;
}

1922
struct pnfs_layout_segment *
1923 1924 1925 1926 1927
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
	struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
	struct nfs4_layoutget_res *res = &lgp->res;
	struct pnfs_layout_segment *lseg;
1928
	struct inode *ino = lo->plh_inode;
1929
	LIST_HEAD(free_me);
1930 1931

	if (!pnfs_sanity_check_layout_range(&res->range))
1932
		return ERR_PTR(-EINVAL);
1933 1934

	/* Inject layout blob into I/O device driver */
1935
	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
1936
	if (IS_ERR_OR_NULL(lseg)) {
1937
		if (!lseg)
1938 1939 1940 1941 1942
			lseg = ERR_PTR(-ENOMEM);

		dprintk("%s: Could not allocate layout: error %ld\n",
		       __func__, PTR_ERR(lseg));
		return lseg;
1943 1944
	}

1945
	pnfs_init_lseg(lo, lseg, &res->range, &res->stateid);
1946

1947
	spin_lock(&ino->i_lock);
1948
	if (pnfs_layoutgets_blocked(lo)) {
F
Fred Isaman 已提交
1949
		dprintk("%s forget reply due to state\n", __func__);
1950
		goto out_forget;
F
Fred Isaman 已提交
1951
	}
1952

1953 1954 1955 1956
	if (!pnfs_layout_is_valid(lo)) {
		/* We have a completely new layout */
		pnfs_set_layout_stateid(lo, &res->stateid, true);
	} else if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
1957 1958 1959
		/* existing state ID, make sure the sequence number matches. */
		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
			dprintk("%s forget reply due to sequence\n", __func__);
1960
			goto out_forget;
1961 1962 1963 1964 1965
		}
		pnfs_set_layout_stateid(lo, &res->stateid, false);
	} else {
		/*
		 * We got an entirely new state ID.  Mark all segments for the
1966
		 * inode invalid, and retry the layoutget
1967
		 */
1968
		pnfs_mark_layout_stateid_invalid(lo, &free_me);
1969
		goto out_forget;
1970
	}
1971

1972
	pnfs_get_lseg(lseg);
1973
	pnfs_layout_insert_lseg(lo, lseg, &free_me);
1974

1975

P
Peng Tao 已提交
1976
	if (res->return_on_close)
F
Fred Isaman 已提交
1977 1978
		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);

1979
	spin_unlock(&ino->i_lock);
1980
	pnfs_free_lseg_list(&free_me);
1981
	return lseg;
F
Fred Isaman 已提交
1982

1983
out_forget:
F
Fred Isaman 已提交
1984 1985 1986
	spin_unlock(&ino->i_lock);
	lseg->pls_layout = lo;
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
1987 1988
	if (!pnfs_layout_is_valid(lo))
		nfs_commit_inode(ino, 0);
1989
	return ERR_PTR(-EAGAIN);
1990 1991
}

1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
/**
 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments
 * @lo: pointer to layout header
 * @tmp_list: list header to be used with pnfs_free_lseg_list()
 * @return_range: describe layout segment ranges to be returned
 *
 * This function is mainly intended for use by layoutrecall. It attempts
 * to free the layout segment immediately, or else to mark it for return
 * as soon as its reference count drops to zero.
 */
2002
int
2003 2004
pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
				struct list_head *tmp_list,
2005 2006
				const struct pnfs_layout_range *return_range,
				u32 seq)
2007 2008
{
	struct pnfs_layout_segment *lseg, *next;
2009
	int remaining = 0;
2010 2011 2012 2013

	dprintk("%s:Begin lo %p\n", __func__, lo);

	if (list_empty(&lo->plh_segs))
2014
		return 0;
2015

2016
	assert_spin_locked(&lo->plh_inode->i_lock);
2017 2018

	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
2019
		if (pnfs_match_lseg_recall(lseg, return_range, seq)) {
2020 2021 2022 2023 2024
			dprintk("%s: marking lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode,
				lseg->pls_range.offset,
				lseg->pls_range.length);
2025 2026 2027
			if (mark_lseg_invalid(lseg, tmp_list))
				continue;
			remaining++;
2028 2029
			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
		}
2030 2031 2032 2033

	if (remaining)
		pnfs_set_plh_return_info(lo, return_range->iomode, seq);

2034
	return remaining;
2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045
}

void pnfs_error_mark_layout_for_return(struct inode *inode,
				       struct pnfs_layout_segment *lseg)
{
	struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
	struct pnfs_layout_range range = {
		.iomode = lseg->pls_range.iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
2046
	bool return_now = false;
2047 2048

	spin_lock(&inode->i_lock);
2049 2050 2051 2052
	if (!pnfs_layout_is_valid(lo)) {
		spin_unlock(&inode->i_lock);
		return;
	}
2053
	pnfs_set_plh_return_info(lo, range.iomode, 0);
2054 2055 2056 2057 2058
	/*
	 * mark all matching lsegs so that we are sure to have no live
	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
	 * for how it works.
	 */
2059
	if (!pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, &range, 0)) {
2060
		nfs4_stateid stateid;
2061
		enum pnfs_iomode iomode;
2062

2063
		return_now = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
2064 2065 2066 2067 2068 2069 2070
		spin_unlock(&inode->i_lock);
		if (return_now)
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
	} else {
		spin_unlock(&inode->i_lock);
		nfs_commit_inode(inode, 0);
	}
2071 2072 2073
}
EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);

2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
void
pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio)
{
	if (pgio->pg_lseg == NULL ||
	    test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags))
		return;
	pnfs_put_lseg(pgio->pg_lseg);
	pgio->pg_lseg = NULL;
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout);

2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
/*
 * Check for any intersection between the request and the pgio->pg_lseg,
 * and if none, put this pgio->pg_lseg away.
 */
static void
pnfs_generic_pg_check_range(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
	if (pgio->pg_lseg && !pnfs_lseg_request_intersecting(pgio->pg_lseg, req)) {
		pnfs_put_lseg(pgio->pg_lseg);
		pgio->pg_lseg = NULL;
	}
}

2098 2099 2100
void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
P
Peng Tao 已提交
2101 2102
	u64 rd_size = req->wb_bytes;

2103
	pnfs_generic_pg_check_layout(pgio);
2104
	pnfs_generic_pg_check_range(pgio, req);
2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115
	if (pgio->pg_lseg == NULL) {
		if (pgio->pg_dreq == NULL)
			rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
		else
			rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);

		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   rd_size,
						   IOMODE_READ,
2116
						   false,
2117
						   GFP_KERNEL);
2118 2119 2120 2121 2122
		if (IS_ERR(pgio->pg_lseg)) {
			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
			pgio->pg_lseg = NULL;
			return;
		}
2123
	}
2124 2125
	/* If no lseg, fall back to read through mds */
	if (pgio->pg_lseg == NULL)
2126
		nfs_pageio_reset_read_mds(pgio);
2127

2128 2129 2130 2131
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);

void
2132 2133
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
			   struct nfs_page *req, u64 wb_size)
2134
{
2135
	pnfs_generic_pg_check_layout(pgio);
2136
	pnfs_generic_pg_check_range(pgio, req);
2137
	if (pgio->pg_lseg == NULL) {
2138 2139 2140 2141 2142
		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   wb_size,
						   IOMODE_RW,
2143
						   false,
2144
						   GFP_NOFS);
2145 2146 2147 2148 2149 2150
		if (IS_ERR(pgio->pg_lseg)) {
			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
			pgio->pg_lseg = NULL;
			return;
		}
	}
2151 2152
	/* If no lseg, fall back to write through mds */
	if (pgio->pg_lseg == NULL)
2153
		nfs_pageio_reset_write_mds(pgio);
2154 2155 2156
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);

2157 2158 2159 2160 2161 2162 2163 2164 2165 2166
void
pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
{
	if (desc->pg_lseg) {
		pnfs_put_lseg(desc->pg_lseg);
		desc->pg_lseg = NULL;
	}
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);

2167 2168 2169 2170 2171
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
size_t
2172 2173
pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
		     struct nfs_page *prev, struct nfs_page *req)
2174
{
2175
	unsigned int size;
2176
	u64 seg_end, req_start, seg_left;
2177 2178 2179 2180

	size = nfs_generic_pg_test(pgio, prev, req);
	if (!size)
		return 0;
2181

2182
	/*
2183 2184 2185 2186 2187
	 * 'size' contains the number of bytes left in the current page (up
	 * to the original size asked for in @req->wb_bytes).
	 *
	 * Calculate how many bytes are left in the layout segment
	 * and if there are less bytes than 'size', return that instead.
2188 2189 2190 2191 2192
	 *
	 * Please also note that 'end_offset' is actually the offset of the
	 * first byte that lies outside the pnfs_layout_range. FIXME?
	 *
	 */
2193
	if (pgio->pg_lseg) {
2194
		seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset,
2195 2196
				     pgio->pg_lseg->pls_range.length);
		req_start = req_offset(req);
2197

2198
		/* start of request is past the last byte of this segment */
2199
		if (req_start >= seg_end)
2200
			return 0;
2201 2202 2203 2204 2205 2206

		/* adjust 'size' iff there are fewer bytes left in the
		 * segment than what nfs_generic_pg_test returned */
		seg_left = seg_end - req_start;
		if (seg_left < size)
			size = (unsigned int)seg_left;
2207
	}
2208

2209
	return size;
2210
}
2211
EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
2212

2213
int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
2214 2215 2216 2217
{
	struct nfs_pageio_descriptor pgio;

	/* Resend all requests through the MDS */
2218 2219
	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
			      hdr->completion_ops);
2220
	set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags);
2221
	return nfs_pageio_resend(&pgio, hdr);
2222
}
2223
EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
2224

2225
static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
2226
{
2227 2228 2229

	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2230
	    PNFS_LAYOUTRET_ON_ERROR) {
2231
		pnfs_return_layout(hdr->inode);
2232
	}
2233
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2234
		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
2235 2236
}

2237 2238 2239
/*
 * Called by non rpc-based layout drivers
 */
2240
void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
2241
{
2242
	if (likely(!hdr->pnfs_error)) {
2243 2244
		pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
				hdr->mds_offset + hdr->res.count);
2245
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2246 2247 2248
	}
	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2249 2250
		pnfs_ld_handle_write_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2251
}
2252
EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
2253

2254 2255
static void
pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
2256
		struct nfs_pgio_header *hdr)
2257
{
2258
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2259

2260
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2261
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2262
		nfs_pageio_reset_write_mds(desc);
2263
		mirror->pg_recoalesce = 1;
2264
	}
2265
	hdr->completion_ops->completion(hdr);
2266 2267 2268
}

static enum pnfs_try_status
2269
pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
2270 2271 2272
			const struct rpc_call_ops *call_ops,
			struct pnfs_layout_segment *lseg,
			int how)
2273
{
2274
	struct inode *inode = hdr->inode;
2275 2276 2277
	enum pnfs_try_status trypnfs;
	struct nfs_server *nfss = NFS_SERVER(inode);

2278
	hdr->mds_ops = call_ops;
2279 2280

	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
2281 2282
		inode->i_ino, hdr->args.count, hdr->args.offset, how);
	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
2283
	if (trypnfs != PNFS_NOT_ATTEMPTED)
2284 2285 2286 2287 2288
		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}

2289
static void
2290 2291
pnfs_do_write(struct nfs_pageio_descriptor *desc,
	      struct nfs_pgio_header *hdr, int how)
2292 2293 2294
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2295
	enum pnfs_try_status trypnfs;
2296

2297
	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
2298 2299
	switch (trypnfs) {
	case PNFS_NOT_ATTEMPTED:
2300
		pnfs_write_through_mds(desc, hdr);
2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311
	case PNFS_ATTEMPTED:
		break;
	case PNFS_TRY_AGAIN:
		/* cleanup hdr and prepare to redo pnfs */
		if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
			struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
			list_splice_init(&hdr->pages, &mirror->pg_list);
			mirror->pg_recoalesce = 1;
		}
		hdr->mds_ops->rpc_release(hdr);
	}
2312 2313
}

2314 2315
static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
2316
	pnfs_put_lseg(hdr->lseg);
2317
	nfs_pgio_header_free(hdr);
2318 2319
}

2320 2321 2322
int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
{
2323
	struct nfs_pgio_header *hdr;
2324 2325
	int ret;

2326 2327
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
P
Peng Tao 已提交
2328 2329
		desc->pg_error = -ENOMEM;
		return desc->pg_error;
2330
	}
2331
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
2332

2333
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2334
	ret = nfs_generic_pgio(desc, hdr);
2335
	if (!ret)
2336
		pnfs_do_write(desc, hdr, desc->pg_ioflags);
2337

2338
	return ret;
2339 2340 2341
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);

2342
int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
2343 2344 2345
{
	struct nfs_pageio_descriptor pgio;

2346
	/* Resend all requests through the MDS */
2347 2348
	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
2349
}
2350
EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
2351

2352
static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
2353
{
2354 2355
	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2356
	    PNFS_LAYOUTRET_ON_ERROR) {
2357
		pnfs_return_layout(hdr->inode);
2358
	}
2359
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2360
		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
2361 2362
}

2363 2364 2365
/*
 * Called by non rpc-based layout drivers
 */
2366
void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
2367
{
2368
	if (likely(!hdr->pnfs_error))
2369
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2370 2371
	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2372 2373
		pnfs_ld_handle_read_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2374 2375 2376
}
EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

2377 2378
static void
pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
2379
		struct nfs_pgio_header *hdr)
2380
{
2381
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2382

2383
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2384
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2385
		nfs_pageio_reset_read_mds(desc);
2386
		mirror->pg_recoalesce = 1;
2387
	}
2388
	hdr->completion_ops->completion(hdr);
2389 2390
}

A
Andy Adamson 已提交
2391 2392 2393
/*
 * Call the appropriate parallel I/O subsystem read function.
 */
2394
static enum pnfs_try_status
2395
pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
2396 2397
		       const struct rpc_call_ops *call_ops,
		       struct pnfs_layout_segment *lseg)
A
Andy Adamson 已提交
2398
{
2399
	struct inode *inode = hdr->inode;
A
Andy Adamson 已提交
2400 2401 2402
	struct nfs_server *nfss = NFS_SERVER(inode);
	enum pnfs_try_status trypnfs;

2403
	hdr->mds_ops = call_ops;
A
Andy Adamson 已提交
2404 2405

	dprintk("%s: Reading ino:%lu %u@%llu\n",
2406
		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
A
Andy Adamson 已提交
2407

2408
	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
2409
	if (trypnfs != PNFS_NOT_ATTEMPTED)
A
Andy Adamson 已提交
2410 2411 2412 2413
		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}
A
Andy Adamson 已提交
2414

2415
/* Resend all requests through pnfs. */
2416
void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
2417 2418 2419
{
	struct nfs_pageio_descriptor pgio;

2420
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2421 2422 2423 2424
		/* Prevent deadlocks with layoutreturn! */
		pnfs_put_lseg(hdr->lseg);
		hdr->lseg = NULL;

2425 2426 2427 2428
		nfs_pageio_init_read(&pgio, hdr->inode, false,
					hdr->completion_ops);
		hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr);
	}
2429 2430 2431
}
EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);

2432
static void
2433
pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
2434 2435 2436
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2437
	enum pnfs_try_status trypnfs;
2438

2439
	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
2440 2441
	switch (trypnfs) {
	case PNFS_NOT_ATTEMPTED:
2442
		pnfs_read_through_mds(desc, hdr);
2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453
	case PNFS_ATTEMPTED:
		break;
	case PNFS_TRY_AGAIN:
		/* cleanup hdr and prepare to redo pnfs */
		if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
			struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
			list_splice_init(&hdr->pages, &mirror->pg_list);
			mirror->pg_recoalesce = 1;
		}
		hdr->mds_ops->rpc_release(hdr);
	}
2454 2455
}

2456 2457
static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
2458
	pnfs_put_lseg(hdr->lseg);
2459
	nfs_pgio_header_free(hdr);
2460 2461
}

2462 2463 2464
int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
{
2465
	struct nfs_pgio_header *hdr;
2466 2467
	int ret;

2468 2469
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
P
Peng Tao 已提交
2470 2471
		desc->pg_error = -ENOMEM;
		return desc->pg_error;
2472
	}
2473
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
2474
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2475
	ret = nfs_generic_pgio(desc, hdr);
2476
	if (!ret)
2477
		pnfs_do_read(desc, hdr);
2478
	return ret;
2479 2480 2481
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);

2482 2483 2484 2485 2486
static void pnfs_clear_layoutcommitting(struct inode *inode)
{
	unsigned long *bitlock = &NFS_I(inode)->flags;

	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
2487
	smp_mb__after_atomic();
2488 2489 2490
	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}

A
Andy Adamson 已提交
2491
/*
2492
 * There can be multiple RW segments.
A
Andy Adamson 已提交
2493
 */
2494
static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
A
Andy Adamson 已提交
2495
{
2496
	struct pnfs_layout_segment *lseg;
A
Andy Adamson 已提交
2497

2498 2499
	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
		if (lseg->pls_range.iomode == IOMODE_RW &&
2500
		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
2501 2502
			list_add(&lseg->pls_lc_list, listp);
	}
A
Andy Adamson 已提交
2503 2504
}

2505 2506 2507 2508 2509 2510 2511 2512 2513 2514
static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
{
	struct pnfs_layout_segment *lseg, *tmp;

	/* Matched by references in pnfs_set_layoutcommit */
	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
		list_del_init(&lseg->pls_lc_list);
		pnfs_put_lseg(lseg);
	}

2515
	pnfs_clear_layoutcommitting(inode);
2516 2517
}

P
Peng Tao 已提交
2518 2519
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
2520
	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
P
Peng Tao 已提交
2521 2522 2523
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

A
Andy Adamson 已提交
2524
void
2525 2526
pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
		loff_t end_pos)
A
Andy Adamson 已提交
2527
{
2528
	struct nfs_inode *nfsi = NFS_I(inode);
2529
	bool mark_as_dirty = false;
A
Andy Adamson 已提交
2530

2531
	spin_lock(&inode->i_lock);
A
Andy Adamson 已提交
2532
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
2533
		nfsi->layout->plh_lwb = end_pos;
2534
		mark_as_dirty = true;
A
Andy Adamson 已提交
2535
		dprintk("%s: Set layoutcommit for inode %lu ",
2536
			__func__, inode->i_ino);
2537 2538
	} else if (end_pos > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = end_pos;
2539
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
2540
		/* references matched in nfs4_layoutcommit_release */
2541
		pnfs_get_lseg(lseg);
2542
	}
2543
	spin_unlock(&inode->i_lock);
2544
	dprintk("%s: lseg %p end_pos %llu\n",
2545
		__func__, lseg, nfsi->layout->plh_lwb);
2546 2547 2548 2549

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
2550
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2551 2552 2553
}
EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);

A
Andy Adamson 已提交
2554 2555 2556 2557 2558 2559
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
{
	struct nfs_server *nfss = NFS_SERVER(data->args.inode);

	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
2560
	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
A
Andy Adamson 已提交
2561 2562
}

2563 2564 2565 2566 2567 2568 2569 2570
/*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
 * data to disk to allow the server to recover the data if it crashes.
 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
 * is off, and a COMMIT is sent to a data server, or
 * if WRITEs to a data server return NFS_DATA_SYNC.
 */
A
Andy Adamson 已提交
2571
int
2572
pnfs_layoutcommit_inode(struct inode *inode, bool sync)
A
Andy Adamson 已提交
2573
{
2574
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
A
Andy Adamson 已提交
2575 2576 2577
	struct nfs4_layoutcommit_data *data;
	struct nfs_inode *nfsi = NFS_I(inode);
	loff_t end_pos;
2578
	int status;
A
Andy Adamson 已提交
2579

2580
	if (!pnfs_layoutcommit_outstanding(inode))
2581 2582
		return 0;

2583
	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
P
Peng Tao 已提交
2584

2585
	status = -EAGAIN;
P
Peng Tao 已提交
2586
	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2587 2588
		if (!sync)
			goto out;
2589
		status = wait_on_bit_lock_action(&nfsi->flags,
2590 2591 2592
				NFS_INO_LAYOUTCOMMITTING,
				nfs_wait_bit_killable,
				TASK_KILLABLE);
P
Peng Tao 已提交
2593
		if (status)
2594
			goto out;
P
Peng Tao 已提交
2595 2596
	}

2597 2598 2599 2600 2601 2602 2603
	status = -ENOMEM;
	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (!data)
		goto clear_layoutcommitting;

	status = 0;
2604
	spin_lock(&inode->i_lock);
2605 2606
	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		goto out_unlock;
2607

2608
	INIT_LIST_HEAD(&data->lseg_list);
2609
	pnfs_list_write_lseg(inode, &data->lseg_list);
A
Andy Adamson 已提交
2610

2611
	end_pos = nfsi->layout->plh_lwb;
A
Andy Adamson 已提交
2612

2613
	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
A
Andy Adamson 已提交
2614 2615 2616
	spin_unlock(&inode->i_lock);

	data->args.inode = inode;
2617
	data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
A
Andy Adamson 已提交
2618 2619 2620
	nfs_fattr_init(&data->fattr);
	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
	data->res.fattr = &data->fattr;
2621 2622 2623 2624
	if (end_pos != 0)
		data->args.lastbytewritten = end_pos - 1;
	else
		data->args.lastbytewritten = U64_MAX;
A
Andy Adamson 已提交
2625 2626
	data->res.server = NFS_SERVER(inode);

2627 2628 2629
	if (ld->prepare_layoutcommit) {
		status = ld->prepare_layoutcommit(&data->args);
		if (status) {
2630
			put_rpccred(data->cred);
2631
			spin_lock(&inode->i_lock);
2632 2633
			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
			if (end_pos > nfsi->layout->plh_lwb)
2634
				nfsi->layout->plh_lwb = end_pos;
2635
			goto out_unlock;
2636 2637 2638 2639
		}
	}


A
Andy Adamson 已提交
2640 2641
	status = nfs4_proc_layoutcommit(data, sync);
out:
P
Peng Tao 已提交
2642 2643
	if (status)
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2644 2645
	dprintk("<-- %s status %d\n", __func__, status);
	return status;
2646 2647
out_unlock:
	spin_unlock(&inode->i_lock);
P
Peng Tao 已提交
2648
	kfree(data);
2649 2650
clear_layoutcommitting:
	pnfs_clear_layoutcommitting(inode);
P
Peng Tao 已提交
2651
	goto out;
A
Andy Adamson 已提交
2652
}
2653
EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
2654

2655 2656 2657 2658 2659 2660 2661
int
pnfs_generic_sync(struct inode *inode, bool datasync)
{
	return pnfs_layoutcommit_inode(inode, true);
}
EXPORT_SYMBOL_GPL(pnfs_generic_sync);

2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672
struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
{
	struct nfs4_threshold *thp;

	thp = kzalloc(sizeof(*thp), GFP_NOFS);
	if (!thp) {
		dprintk("%s mdsthreshold allocation failed\n", __func__);
		return NULL;
	}
	return thp;
}
2673

2674
#if IS_ENABLED(CONFIG_NFS_V4_2)
2675
int
2676
pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
2677 2678 2679
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
	struct nfs_server *server = NFS_SERVER(inode);
2680
	struct nfs_inode *nfsi = NFS_I(inode);
2681 2682 2683 2684 2685 2686 2687
	struct nfs42_layoutstat_data *data;
	struct pnfs_layout_hdr *hdr;
	int status = 0;

	if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
		goto out;

2688 2689 2690
	if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
		goto out;

2691 2692 2693
	if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
		goto out;

2694 2695 2696
	spin_lock(&inode->i_lock);
	if (!NFS_I(inode)->layout) {
		spin_unlock(&inode->i_lock);
2697
		goto out_clear_layoutstats;
2698 2699 2700 2701 2702
	}
	hdr = NFS_I(inode)->layout;
	pnfs_get_layout_hdr(hdr);
	spin_unlock(&inode->i_lock);

2703
	data = kzalloc(sizeof(*data), gfp_flags);
2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
	if (!data) {
		status = -ENOMEM;
		goto out_put;
	}

	data->args.fh = NFS_FH(inode);
	data->args.inode = inode;
	status = ld->prepare_layoutstats(&data->args);
	if (status)
		goto out_free;

	status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);

out:
	dprintk("%s returns %d\n", __func__, status);
	return status;

out_free:
	kfree(data);
out_put:
	pnfs_put_layout_hdr(hdr);
2725
out_clear_layoutstats:
2726 2727 2728
	smp_mb__before_atomic();
	clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
	smp_mb__after_atomic();
2729 2730 2731
	goto out;
}
EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
2732
#endif
2733 2734 2735 2736

unsigned int layoutstats_timer;
module_param(layoutstats_timer, uint, 0644);
EXPORT_SYMBOL_GPL(layoutstats_timer);