pnfs.c 69.4 KB
Newer Older
R
Ricardo Labiaga 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 *  pNFS functions to call and manage layout drivers.
 *
 *  Copyright (c) 2002 [year of first publication]
 *  The Regents of the University of Michigan
 *  All Rights Reserved
 *
 *  Dean Hildebrand <dhildebz@umich.edu>
 *
 *  Permission is granted to use, copy, create derivative works, and
 *  redistribute this software and such derivative works for any purpose,
 *  so long as the name of the University of Michigan is not used in
 *  any advertising or publicity pertaining to the use or distribution
 *  of this software without specific, written prior authorization. If
 *  the above copyright notice or any other identification of the
 *  University of Michigan is included in any copy of any portion of
 *  this software, then the disclaimer below must also be included.
 *
 *  This software is provided as is, without representation or warranty
 *  of any kind either express or implied, including without limitation
 *  the implied warranties of merchantability, fitness for a particular
 *  purpose, or noninfringement.  The Regents of the University of
 *  Michigan shall not be liable for any damages, including special,
 *  indirect, incidental, or consequential damages, with respect to any
 *  claim arising out of or in connection with the use of the software,
 *  even if it has been or is hereafter advised of the possibility of
 *  such damages.
 */

#include <linux/nfs_fs.h>
31
#include <linux/nfs_page.h>
32
#include <linux/module.h>
33
#include <linux/sort.h>
34
#include "internal.h"
R
Ricardo Labiaga 已提交
35
#include "pnfs.h"
A
Andy Adamson 已提交
36
#include "iostat.h"
37
#include "nfs4trace.h"
38
#include "delegation.h"
39
#include "nfs42.h"
R
Ricardo Labiaga 已提交
40 41

#define NFSDBG_FACILITY		NFSDBG_PNFS
42
#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
R
Ricardo Labiaga 已提交
43

44 45 46 47 48 49 50 51 52 53 54 55
/* Locking:
 *
 * pnfs_spinlock:
 *      protects pnfs_modules_tbl.
 */
static DEFINE_SPINLOCK(pnfs_spinlock);

/*
 * pnfs_modules_tbl holds all pnfs modules
 */
static LIST_HEAD(pnfs_modules_tbl);

56
static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo);
57 58 59 60
static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
		struct list_head *free_me,
		const struct pnfs_layout_range *range,
		u32 seq);
61

62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
/* Return the registered pnfs layout driver module matching given id */
static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)
{
	struct pnfs_layoutdriver_type *local;

	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
		if (local->id == id)
			goto out;
	local = NULL;
out:
	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
	return local;
}

R
Ricardo Labiaga 已提交
77 78 79
static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)
{
80 81 82 83
	struct pnfs_layoutdriver_type *local;

	spin_lock(&pnfs_spinlock);
	local = find_pnfs_driver_locked(id);
84 85 86 87
	if (local != NULL && !try_module_get(local->owner)) {
		dprintk("%s: Could not grab reference on module\n", __func__);
		local = NULL;
	}
88 89
	spin_unlock(&pnfs_spinlock);
	return local;
R
Ricardo Labiaga 已提交
90 91 92 93 94
}

void
unset_pnfs_layoutdriver(struct nfs_server *nfss)
{
95 96 97
	if (nfss->pnfs_curr_ld) {
		if (nfss->pnfs_curr_ld->clear_layoutdriver)
			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
98 99 100
		/* Decrement the MDS count. Purge the deviceid cache if zero */
		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
			nfs4_deviceid_purge_client(nfss->nfs_client);
101
		module_put(nfss->pnfs_curr_ld->owner);
102
	}
R
Ricardo Labiaga 已提交
103 104 105
	nfss->pnfs_curr_ld = NULL;
}

106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
/*
 * When the server sends a list of layout types, we choose one in the order
 * given in the list below.
 *
 * FIXME: should this list be configurable in some fashion? module param?
 * 	  mount option? something else?
 */
static const u32 ld_prefs[] = {
	LAYOUT_SCSI,
	LAYOUT_BLOCK_VOLUME,
	LAYOUT_OSD2_OBJECTS,
	LAYOUT_FLEX_FILES,
	LAYOUT_NFSV4_1_FILES,
	0
};

static int
ld_cmp(const void *e1, const void *e2)
{
	u32 ld1 = *((u32 *)e1);
	u32 ld2 = *((u32 *)e2);
	int i;

	for (i = 0; ld_prefs[i] != 0; i++) {
		if (ld1 == ld_prefs[i])
			return -1;

		if (ld2 == ld_prefs[i])
			return 1;
	}
	return 0;
}

R
Ricardo Labiaga 已提交
139 140 141 142
/*
 * Try to set the server's pnfs module to the pnfs layout type specified by id.
 * Currently only one pNFS layout driver per filesystem is supported.
 *
143
 * @ids array of layout types supported by MDS.
R
Ricardo Labiaga 已提交
144 145
 */
void
146
set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
147
		      struct nfs_fsinfo *fsinfo)
R
Ricardo Labiaga 已提交
148 149
{
	struct pnfs_layoutdriver_type *ld_type = NULL;
150
	u32 id;
151
	int i;
R
Ricardo Labiaga 已提交
152

153 154
	if (fsinfo->nlayouttypes == 0)
		goto out_no_driver;
R
Ricardo Labiaga 已提交
155 156
	if (!(server->nfs_client->cl_exchange_flags &
		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
157 158
		printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n",
			__func__, server->nfs_client->cl_exchange_flags);
R
Ricardo Labiaga 已提交
159 160
		goto out_no_driver;
	}
161

162 163
	sort(fsinfo->layouttype, fsinfo->nlayouttypes,
		sizeof(*fsinfo->layouttype), ld_cmp, NULL);
164

165 166
	for (i = 0; i < fsinfo->nlayouttypes; i++) {
		id = fsinfo->layouttype[i];
R
Ricardo Labiaga 已提交
167
		ld_type = find_pnfs_driver(id);
168 169 170 171 172 173 174
		if (!ld_type) {
			request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX,
					id);
			ld_type = find_pnfs_driver(id);
		}
		if (ld_type)
			break;
R
Ricardo Labiaga 已提交
175
	}
176 177

	if (!ld_type) {
178
		dprintk("%s: No pNFS module found!\n", __func__);
179 180 181
		goto out_no_driver;
	}

R
Ricardo Labiaga 已提交
182
	server->pnfs_curr_ld = ld_type;
183 184
	if (ld_type->set_layoutdriver
	    && ld_type->set_layoutdriver(server, mntfh)) {
185 186
		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
			"driver %u.\n", __func__, id);
187 188 189
		module_put(ld_type->owner);
		goto out_no_driver;
	}
190 191
	/* Bump the MDS count */
	atomic_inc(&server->nfs_client->cl_mds_count);
192

R
Ricardo Labiaga 已提交
193 194 195 196 197 198 199
	dprintk("%s: pNFS module for %u set\n", __func__, id);
	return;

out_no_driver:
	dprintk("%s: Using NFSv4 I/O\n", __func__);
	server->pnfs_curr_ld = NULL;
}
200 201 202 203 204 205 206 207

int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	int status = -EINVAL;
	struct pnfs_layoutdriver_type *tmp;

	if (ld_type->id == 0) {
208
		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
209 210
		return status;
	}
211
	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
212
		printk(KERN_ERR "NFS: %s Layout driver must provide "
213 214 215
		       "alloc_lseg and free_lseg.\n", __func__);
		return status;
	}
216 217 218 219 220 221 222 223 224

	spin_lock(&pnfs_spinlock);
	tmp = find_pnfs_driver_locked(ld_type->id);
	if (!tmp) {
		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
		status = 0;
		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
			ld_type->name);
	} else {
225
		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
			__func__, ld_type->id);
	}
	spin_unlock(&pnfs_spinlock);

	return status;
}
EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);

void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
	spin_lock(&pnfs_spinlock);
	list_del(&ld_type->pnfs_tblid);
	spin_unlock(&pnfs_spinlock);
}
EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
243

244 245 246 247
/*
 * pNFS client layout cache
 */

248
/* Need to hold i_lock if caller does not already hold reference */
F
Fred Isaman 已提交
249
void
250
pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
251
{
252
	atomic_inc(&lo->plh_refcount);
253 254
}

255 256 257 258
static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
259
	return ld->alloc_layout_hdr(ino, gfp_flags);
260 261 262 263 264
}

static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
265 266 267 268 269 270 271 272 273 274
	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;

	if (!list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		spin_lock(&clp->cl_lock);
		list_del_init(&lo->plh_layouts);
		spin_unlock(&clp->cl_lock);
	}
275
	put_rpccred(lo->plh_lc_cred);
276
	return ld->free_layout_hdr(lo);
277 278
}

279
static void
280
pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
281
{
282
	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
283
	dprintk("%s: freeing layout cache %p\n", __func__, lo);
284 285 286 287
	nfsi->layout = NULL;
	/* Reset MDS Threshold I/O counters */
	nfsi->write_io = 0;
	nfsi->read_io = 0;
288 289
}

290
void
291
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
292
{
293 294
	struct inode *inode = lo->plh_inode;

295 296
	pnfs_layoutreturn_before_put_layout_hdr(lo);

297
	if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
298 299
		if (!list_empty(&lo->plh_segs))
			WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
300
		pnfs_detach_layout_hdr(lo);
301
		spin_unlock(&inode->i_lock);
302
		pnfs_free_layout_hdr(lo);
303
	}
304 305
}

306 307 308 309 310 311 312 313
static void
pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo)
{
	lo->plh_return_iomode = 0;
	lo->plh_return_seq = 0;
	clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
}

314 315 316 317 318 319 320
/*
 * Mark a pnfs_layout_hdr and all associated layout segments as invalid
 *
 * In order to continue using the pnfs_layout_hdr, a full recovery
 * is required.
 * Note that caller must hold inode->i_lock.
 */
321
int
322 323 324 325 326 327 328 329 330 331
pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
		struct list_head *lseg_list)
{
	struct pnfs_layout_range range = {
		.iomode = IOMODE_ANY,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};

	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
332
	pnfs_clear_layoutreturn_info(lo);
333
	pnfs_free_returned_lsegs(lo, lseg_list, &range, 0);
334
	return pnfs_mark_matching_lsegs_invalid(lo, lseg_list, &range, 0);
335 336
}

337 338 339 340 341 342 343 344
static int
pnfs_iomode_to_fail_bit(u32 iomode)
{
	return iomode == IOMODE_RW ?
		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
}

static void
345
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
346
{
347
	lo->plh_retry_timestamp = jiffies;
348
	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
349 350 351 352 353 354 355 356 357 358 359 360 361 362
		atomic_inc(&lo->plh_refcount);
}

static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
{
	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
		atomic_dec(&lo->plh_refcount);
}

static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
	struct inode *inode = lo->plh_inode;
363 364 365 366 367 368
	struct pnfs_layout_range range = {
		.iomode = iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(head);
369 370 371

	spin_lock(&inode->i_lock);
	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
372
	pnfs_mark_matching_lsegs_invalid(lo, &head, &range, 0);
373
	spin_unlock(&inode->i_lock);
374
	pnfs_free_lseg_list(&head);
375 376 377 378 379 380 381
	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
			iomode == IOMODE_RW ?  "RW" : "READ");
}

static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
382
	unsigned long start, end;
383 384 385
	int fail_bit = pnfs_iomode_to_fail_bit(iomode);

	if (test_bit(fail_bit, &lo->plh_flags) == 0)
386 387 388 389 390
		return false;
	end = jiffies;
	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
		/* It is time to retry the failed layoutgets */
391
		pnfs_layout_clear_fail_bit(lo, fail_bit);
392 393 394
		return false;
	}
	return true;
395 396
}

397
static void
398 399 400
pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,
		const struct pnfs_layout_range *range,
		const nfs4_stateid *stateid)
401
{
402
	INIT_LIST_HEAD(&lseg->pls_list);
403
	INIT_LIST_HEAD(&lseg->pls_lc_list);
404 405
	atomic_set(&lseg->pls_refcount, 1);
	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
406
	lseg->pls_layout = lo;
407 408
	lseg->pls_range = *range;
	lseg->pls_seq = be32_to_cpu(stateid->seqid);
409 410
}

411
static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
412
{
413 414 415 416
	if (lseg != NULL) {
		struct inode *inode = lseg->pls_layout->plh_inode;
		NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg);
	}
417 418
}

F
Fred Isaman 已提交
419
static void
420 421
pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
F
Fred Isaman 已提交
422
{
423
	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
F
Fred Isaman 已提交
424
	list_del_init(&lseg->pls_list);
425 426
	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
	atomic_dec(&lo->plh_refcount);
427 428 429
	if (list_empty(&lo->plh_segs) &&
	    !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) &&
	    !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
430 431
		if (atomic_read(&lo->plh_outstanding) == 0)
			set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
432
		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
433
	}
F
Fred Isaman 已提交
434 435
}

436 437 438 439 440 441 442 443 444 445 446 447
static bool
pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
{
	if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
	    pnfs_layout_is_valid(lo)) {
		list_move_tail(&lseg->pls_list, &lo->plh_return_segs);
		return true;
	}
	return false;
}

448
void
449
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
450
{
451
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
452 453 454 455 456
	struct inode *inode;

	if (!lseg)
		return;

457 458 459
	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
460

461 462
	lo = lseg->pls_layout;
	inode = lo->plh_inode;
463

F
Fred Isaman 已提交
464
	if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
465 466 467 468
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
			spin_unlock(&inode->i_lock);
			return;
		}
469
		pnfs_get_layout_hdr(lo);
470
		pnfs_layout_remove_lseg(lo, lseg);
471 472
		if (pnfs_cache_lseg_for_layoutreturn(lo, lseg))
			lseg = NULL;
473 474 475
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg(lseg);
		pnfs_put_layout_hdr(lo);
476 477
	}
}
478
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
479

480
static void pnfs_free_lseg_async_work(struct work_struct *work)
481 482
{
	struct pnfs_layout_segment *lseg;
483
	struct pnfs_layout_hdr *lo;
484 485

	lseg = container_of(work, struct pnfs_layout_segment, pls_work);
486
	lo = lseg->pls_layout;
487

488 489
	pnfs_free_lseg(lseg);
	pnfs_put_layout_hdr(lo);
490 491
}

492
static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg)
493
{
494
	INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work);
495 496
	schedule_work(&lseg->pls_work);
}
497 498 499 500 501 502 503 504 505 506 507 508 509 510

void
pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
{
	if (!lseg)
		return;

	assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock);

	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
	if (atomic_dec_and_test(&lseg->pls_refcount)) {
		struct pnfs_layout_hdr *lo = lseg->pls_layout;
511 512
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags))
			return;
513
		pnfs_layout_remove_lseg(lo, lseg);
514 515 516 517
		if (!pnfs_cache_lseg_for_layoutreturn(lo, lseg)) {
			pnfs_get_layout_hdr(lo);
			pnfs_free_lseg_async(lseg);
		}
518 519 520
	}
}
EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
521

522 523 524 525 526 527 528
/*
 * is l2 fully contained in l1?
 *   start1                             end1
 *   [----------------------------------)
 *           start2           end2
 *           [----------------)
 */
529
static bool
530
pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
531
		 const struct pnfs_layout_range *l2)
532 533
{
	u64 start1 = l1->offset;
534
	u64 end1 = pnfs_end_offset(start1, l1->length);
535
	u64 start2 = l2->offset;
536
	u64 end2 = pnfs_end_offset(start2, l2->length);
537 538 539 540

	return (start1 <= start2) && (end1 >= end2);
}

541 542 543 544 545 546 547 548 549 550
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		struct list_head *tmp_list)
{
	if (!atomic_dec_and_test(&lseg->pls_refcount))
		return false;
	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
	list_add(&lseg->pls_list, tmp_list);
	return true;
}

551 552 553 554 555 556 557 558 559 560 561
/* Returns 1 if lseg is removed from list, 0 otherwise */
static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
			     struct list_head *tmp_list)
{
	int rv = 0;

	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
		/* Remove the reference keeping the lseg in the
		 * list.  It will now be removed when all
		 * outstanding io is finished.
		 */
F
Fred Isaman 已提交
562 563
		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
			atomic_read(&lseg->pls_refcount));
564
		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
F
Fred Isaman 已提交
565
			rv = 1;
566 567 568 569
	}
	return rv;
}

570 571 572 573 574 575 576 577 578
/*
 * Compare 2 layout stateid sequence ids, to see which is newer,
 * taking into account wraparound issues.
 */
static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
{
	return (s32)(s1 - s2) > 0;
}

579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
static bool
pnfs_should_free_range(const struct pnfs_layout_range *lseg_range,
		 const struct pnfs_layout_range *recall_range)
{
	return (recall_range->iomode == IOMODE_ANY ||
		lseg_range->iomode == recall_range->iomode) &&
	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
}

static bool
pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg,
		const struct pnfs_layout_range *recall_range,
		u32 seq)
{
	if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq))
		return false;
	if (recall_range == NULL)
		return true;
	return pnfs_should_free_range(&lseg->pls_range, recall_range);
}

600 601 602 603 604 605 606 607 608 609 610 611 612 613
/**
 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later
 * @lo: layout header containing the lsegs
 * @tmp_list: list head where doomed lsegs should go
 * @recall_range: optional recall range argument to match (may be NULL)
 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0)
 *
 * Walk the list of lsegs in the layout header, and tear down any that should
 * be destroyed. If "recall_range" is specified then the segment must match
 * that range. If "seq" is non-zero, then only match segments that were handed
 * out at or before that sequence.
 *
 * Returns number of matching invalid lsegs remaining in list after scanning
 * it and purging them.
614
 */
F
Fred Isaman 已提交
615
int
616
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
617
			    struct list_head *tmp_list,
618 619
			    const struct pnfs_layout_range *recall_range,
			    u32 seq)
620 621
{
	struct pnfs_layout_segment *lseg, *next;
622
	int remaining = 0;
623 624 625

	dprintk("%s:Begin lo %p\n", __func__, lo);

626
	if (list_empty(&lo->plh_segs))
627
		return 0;
628
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
629
		if (pnfs_match_lseg_recall(lseg, recall_range, seq)) {
630
			dprintk("%s: freeing lseg %p iomode %d seq %u"
631
				"offset %llu length %llu\n", __func__,
632 633
				lseg, lseg->pls_range.iomode, lseg->pls_seq,
				lseg->pls_range.offset, lseg->pls_range.length);
634 635
			if (!mark_lseg_invalid(lseg, tmp_list))
				remaining++;
636
		}
637 638
	dprintk("%s:Return %i\n", __func__, remaining);
	return remaining;
639 640
}

641 642 643 644 645 646 647 648 649 650 651 652 653 654
static void
pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
		struct list_head *free_me,
		const struct pnfs_layout_range *range,
		u32 seq)
{
	struct pnfs_layout_segment *lseg, *next;

	list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) {
		if (pnfs_match_lseg_recall(lseg, range, seq))
			list_move_tail(&lseg->pls_list, free_me);
	}
}

655
/* note free_me must contain lsegs from a single layout_hdr */
F
Fred Isaman 已提交
656
void
657
pnfs_free_lseg_list(struct list_head *free_me)
658
{
659
	struct pnfs_layout_segment *lseg, *tmp;
660 661 662 663

	if (list_empty(free_me))
		return;

664
	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
665
		list_del(&lseg->pls_list);
666
		pnfs_free_lseg(lseg);
667 668 669
	}
}

670 671 672 673
void
pnfs_destroy_layout(struct nfs_inode *nfsi)
{
	struct pnfs_layout_hdr *lo;
674
	LIST_HEAD(tmp_list);
675 676 677 678

	spin_lock(&nfsi->vfs_inode.i_lock);
	lo = nfsi->layout;
	if (lo) {
679
		pnfs_get_layout_hdr(lo);
680
		pnfs_mark_layout_stateid_invalid(lo, &tmp_list);
681 682 683 684 685 686 687
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
		spin_unlock(&nfsi->vfs_inode.i_lock);
		pnfs_free_lseg_list(&tmp_list);
		pnfs_put_layout_hdr(lo);
	} else
		spin_unlock(&nfsi->vfs_inode.i_lock);
688
}
689
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
690

691 692 693
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
		struct list_head *layout_list)
694 695
{
	struct pnfs_layout_hdr *lo;
696
	bool ret = false;
697

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749
	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
		pnfs_get_layout_hdr(lo);
		list_add(&lo->plh_bulk_destroy, layout_list);
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

/* Caller must hold rcu_read_lock and clp->cl_lock */
static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
		struct nfs_server *server,
		struct list_head *layout_list)
{
	struct pnfs_layout_hdr *lo, *next;
	struct inode *inode;

	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
		inode = igrab(lo->plh_inode);
		if (inode == NULL)
			continue;
		list_del_init(&lo->plh_layouts);
		if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
			continue;
		rcu_read_unlock();
		spin_unlock(&clp->cl_lock);
		iput(inode);
		spin_lock(&clp->cl_lock);
		rcu_read_lock();
		return -EAGAIN;
	}
	return 0;
}

static int
pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
		bool is_bulk_recall)
{
	struct pnfs_layout_hdr *lo;
	struct inode *inode;
	LIST_HEAD(lseg_list);
	int ret = 0;

	while (!list_empty(layout_list)) {
		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
				plh_bulk_destroy);
		dprintk("%s freeing layout for inode %lu\n", __func__,
			lo->plh_inode->i_ino);
		inode = lo->plh_inode;
750 751 752

		pnfs_layoutcommit_inode(inode, false);

753 754
		spin_lock(&inode->i_lock);
		list_del_init(&lo->plh_bulk_destroy);
755 756 757
		if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) {
			if (is_bulk_recall)
				set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
758
			ret = -EAGAIN;
759
		}
760 761
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg_list(&lseg_list);
762 763
		/* Free all lsegs that are attached to commit buckets */
		nfs_commit_inode(inode, 0);
764 765 766 767 768 769 770 771 772 773 774 775 776
		pnfs_put_layout_hdr(lo);
		iput(inode);
	}
	return ret;
}

int
pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
		struct nfs_fsid *fsid,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);
777

778
	spin_lock(&clp->cl_lock);
779
	rcu_read_lock();
780
restart:
781
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
782 783 784 785 786 787
		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
			continue;
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
				server,
				&layout_list) != 0)
			goto restart;
788 789
	}
	rcu_read_unlock();
790 791
	spin_unlock(&clp->cl_lock);

792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

int
pnfs_destroy_layouts_byclid(struct nfs_client *clp,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);

	spin_lock(&clp->cl_lock);
	rcu_read_lock();
restart:
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
					server,
					&layout_list) != 0)
			goto restart;
812
	}
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
	rcu_read_unlock();
	spin_unlock(&clp->cl_lock);

	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

/*
 * Called by the state manger to remove all layouts established under an
 * expired lease.
 */
void
pnfs_destroy_all_layouts(struct nfs_client *clp)
{
	nfs4_deviceid_mark_client_invalid(clp);
	nfs4_deviceid_purge_client(clp);

	pnfs_destroy_layouts_byclid(clp, false);
832 833
}

834
/* update lo->plh_stateid with new if is more recent */
F
Fred Isaman 已提交
835 836 837
void
pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
			bool update_barrier)
838
{
839
	u32 oldseq, newseq, new_barrier = 0;
840

841 842
	oldseq = be32_to_cpu(lo->plh_stateid.seqid);
	newseq = be32_to_cpu(new->seqid);
843 844 845 846 847 848 849 850 851

	if (!pnfs_layout_is_valid(lo)) {
		nfs4_stateid_copy(&lo->plh_stateid, new);
		lo->plh_barrier = newseq;
		pnfs_clear_layoutreturn_info(lo);
		clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
		return;
	}
	if (pnfs_seqid_is_newer(newseq, oldseq)) {
852
		nfs4_stateid_copy(&lo->plh_stateid, new);
853 854 855 856 857
		/*
		 * Because of wraparound, we want to keep the barrier
		 * "close" to the current seqids.
		 */
		new_barrier = newseq - atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
858
	}
859 860 861 862
	if (update_barrier)
		new_barrier = be32_to_cpu(new->seqid);
	else if (new_barrier == 0)
		return;
863
	if (pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
864
		lo->plh_barrier = new_barrier;
865 866
}

867
static bool
868 869
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid)
F
Fred Isaman 已提交
870
{
871
	u32 seqid = be32_to_cpu(stateid->seqid);
872

873 874 875 876 877
	return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
}

/* lget is set to 1 if called from inside send_layoutget call chain */
static bool
878
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
879
{
F
Fred Isaman 已提交
880
	return lo->plh_block_lgets ||
881
		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
882 883
}

884
/*
885 886 887 888 889
 * Get layout from server.
 *    for now, assume that whole file layouts are requested.
 *    arg->offset: 0
 *    arg->length: all ones
 */
890 891 892
static struct pnfs_layout_segment *
send_layoutget(struct pnfs_layout_hdr *lo,
	   struct nfs_open_context *ctx,
893
	   nfs4_stateid *stateid,
894
	   const struct pnfs_layout_range *range,
895
	   long *timeout, gfp_t gfp_flags)
896
{
897
	struct inode *ino = lo->plh_inode;
898 899
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs4_layoutget *lgp;
900
	loff_t i_size;
901 902

	dprintk("--> %s\n", __func__);
903

904 905 906 907 908
	/*
	 * Synchronously retrieve layout information from server and
	 * store in lseg. If we race with a concurrent seqid morphing
	 * op, then re-send the LAYOUTGET.
	 */
909 910 911 912 913 914 915 916 917 918 919 920 921 922
	lgp = kzalloc(sizeof(*lgp), gfp_flags);
	if (lgp == NULL)
		return ERR_PTR(-ENOMEM);

	i_size = i_size_read(ino);

	lgp->args.minlength = PAGE_SIZE;
	if (lgp->args.minlength > range->length)
		lgp->args.minlength = range->length;
	if (range->iomode == IOMODE_READ) {
		if (range->offset >= i_size)
			lgp->args.minlength = 0;
		else if (i_size - range->offset < lgp->args.minlength)
			lgp->args.minlength = i_size - range->offset;
923
	}
924 925 926 927 928
	lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
	pnfs_copy_range(&lgp->args.range, range);
	lgp->args.type = server->pnfs_curr_ld->id;
	lgp->args.inode = ino;
	lgp->args.ctx = get_nfs_open_context(ctx);
929
	nfs4_stateid_copy(&lgp->args.stateid, stateid);
930 931
	lgp->gfp_flags = gfp_flags;
	lgp->cred = lo->plh_lc_cred;
932

933
	return nfs4_proc_layoutget(lgp, timeout, gfp_flags);
934 935
}

936 937 938 939 940 941 942 943 944 945 946 947 948 949 950
static void pnfs_clear_layoutcommit(struct inode *inode,
		struct list_head *head)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct pnfs_layout_segment *lseg, *tmp;

	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		return;
	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
			continue;
		pnfs_lseg_dec_and_remove_zero(lseg, head);
	}
}

951
static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
952 953
{
	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
954
	clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags);
955 956
	smp_mb__after_atomic();
	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
957
	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
958 959
}

960
void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo,
961
		const nfs4_stateid *arg_stateid,
962 963 964 965 966 967 968
		const struct pnfs_layout_range *range,
		const nfs4_stateid *stateid)
{
	struct inode *inode = lo->plh_inode;
	LIST_HEAD(freeme);

	spin_lock(&inode->i_lock);
969 970 971
	if (!pnfs_layout_is_valid(lo) || !arg_stateid ||
	    !nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid))
		goto out_unlock;
972
	if (stateid) {
973 974
		u32 seq = be32_to_cpu(arg_stateid->seqid);

975 976 977 978 979
		pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq);
		pnfs_free_returned_lsegs(lo, &freeme, range, seq);
		pnfs_set_layout_stateid(lo, stateid, true);
	} else
		pnfs_mark_layout_stateid_invalid(lo, &freeme);
980
out_unlock:
981 982 983 984 985 986
	pnfs_clear_layoutreturn_waitbit(lo);
	spin_unlock(&inode->i_lock);
	pnfs_free_lseg_list(&freeme);

}

987
static bool
988 989 990
pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo,
		nfs4_stateid *stateid,
		enum pnfs_iomode *iomode)
991
{
992 993 994
	/* Serialise LAYOUTGET/LAYOUTRETURN */
	if (atomic_read(&lo->plh_outstanding) != 0)
		return false;
995
	if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
996
		return false;
997
	set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
998
	pnfs_get_layout_hdr(lo);
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
		if (stateid != NULL) {
			nfs4_stateid_copy(stateid, &lo->plh_stateid);
			if (lo->plh_return_seq != 0)
				stateid->seqid = cpu_to_be32(lo->plh_return_seq);
		}
		if (iomode != NULL)
			*iomode = lo->plh_return_iomode;
		pnfs_clear_layoutreturn_info(lo);
		return true;
	}
	if (stateid != NULL)
		nfs4_stateid_copy(stateid, &lo->plh_stateid);
	if (iomode != NULL)
		*iomode = IOMODE_ANY;
1014 1015 1016
	return true;
}

1017
static int
1018
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
1019
		       enum pnfs_iomode iomode, bool sync)
1020 1021 1022 1023 1024
{
	struct inode *ino = lo->plh_inode;
	struct nfs4_layoutreturn *lrp;
	int status = 0;

1025
	lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
1026 1027 1028
	if (unlikely(lrp == NULL)) {
		status = -ENOMEM;
		spin_lock(&ino->i_lock);
1029
		pnfs_clear_layoutreturn_waitbit(lo);
1030 1031 1032 1033 1034
		spin_unlock(&ino->i_lock);
		pnfs_put_layout_hdr(lo);
		goto out;
	}

1035
	nfs4_stateid_copy(&lrp->args.stateid, stateid);
1036 1037
	lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
	lrp->args.inode = ino;
P
Peng Tao 已提交
1038 1039 1040
	lrp->args.range.iomode = iomode;
	lrp->args.range.offset = 0;
	lrp->args.range.length = NFS4_MAX_UINT64;
1041 1042 1043 1044
	lrp->args.layout = lo;
	lrp->clp = NFS_SERVER(ino)->nfs_client;
	lrp->cred = lo->plh_lc_cred;

1045
	status = nfs4_proc_layoutreturn(lrp, sync);
1046 1047 1048 1049 1050
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

1051 1052 1053 1054 1055 1056
/* Return true if layoutreturn is needed */
static bool
pnfs_layout_need_return(struct pnfs_layout_hdr *lo)
{
	struct pnfs_layout_segment *s;

1057
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
		return false;

	/* Defer layoutreturn until all lsegs are done */
	list_for_each_entry(s, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
			return false;
	}

	return true;
}

static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo)
{
	struct inode *inode= lo->plh_inode;

1073
	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1074 1075 1076 1077 1078 1079 1080
		return;
	spin_lock(&inode->i_lock);
	if (pnfs_layout_need_return(lo)) {
		nfs4_stateid stateid;
		enum pnfs_iomode iomode;
		bool send;

1081
		send = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
1082 1083 1084 1085 1086 1087 1088 1089 1090
		spin_unlock(&inode->i_lock);
		if (send) {
			/* Send an async layoutreturn so we dont deadlock */
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
		}
	} else
		spin_unlock(&inode->i_lock);
}

1091 1092 1093 1094 1095 1096 1097 1098
/*
 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
 * when the layout segment list is empty.
 *
 * Note that a pnfs_layout_hdr can exist with an empty layout segment
 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
 * deviceid is marked invalid.
 */
B
Benny Halevy 已提交
1099 1100 1101 1102 1103 1104 1105
int
_pnfs_return_layout(struct inode *ino)
{
	struct pnfs_layout_hdr *lo = NULL;
	struct nfs_inode *nfsi = NFS_I(ino);
	LIST_HEAD(tmp_list);
	nfs4_stateid stateid;
1106
	int status = 0, empty;
1107
	bool send;
B
Benny Halevy 已提交
1108

1109
	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
B
Benny Halevy 已提交
1110 1111 1112

	spin_lock(&ino->i_lock);
	lo = nfsi->layout;
1113
	if (!lo) {
B
Benny Halevy 已提交
1114
		spin_unlock(&ino->i_lock);
1115 1116
		dprintk("NFS: %s no layout to return\n", __func__);
		goto out;
B
Benny Halevy 已提交
1117 1118
	}
	/* Reference matched in nfs4_layoutreturn_release */
1119
	pnfs_get_layout_hdr(lo);
1120
	empty = list_empty(&lo->plh_segs);
1121
	pnfs_clear_layoutcommit(ino, &tmp_list);
1122
	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0);
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132

	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
		struct pnfs_layout_range range = {
			.iomode		= IOMODE_ANY,
			.offset		= 0,
			.length		= NFS4_MAX_UINT64,
		};
		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
	}

1133 1134 1135 1136
	/* Don't send a LAYOUTRETURN if list was initially empty */
	if (empty) {
		spin_unlock(&ino->i_lock);
		dprintk("NFS: %s no layout segments to return\n", __func__);
1137
		goto out_put_layout_hdr;
1138
	}
1139

1140
	send = pnfs_prepare_layoutreturn(lo, &stateid, NULL);
B
Benny Halevy 已提交
1141 1142
	spin_unlock(&ino->i_lock);
	pnfs_free_lseg_list(&tmp_list);
1143
	if (send)
1144
		status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1145 1146
out_put_layout_hdr:
	pnfs_put_layout_hdr(lo);
B
Benny Halevy 已提交
1147 1148 1149 1150
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}
1151
EXPORT_SYMBOL_GPL(_pnfs_return_layout);
B
Benny Halevy 已提交
1152

1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
int
pnfs_commit_and_return_layout(struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	int ret;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo == NULL) {
		spin_unlock(&inode->i_lock);
		return 0;
	}
	pnfs_get_layout_hdr(lo);
	/* Block new layoutgets and read/write to ds */
	lo->plh_block_lgets++;
	spin_unlock(&inode->i_lock);
	filemap_fdatawait(inode->i_mapping);
	ret = pnfs_layoutcommit_inode(inode, true);
	if (ret == 0)
		ret = _pnfs_return_layout(inode);
	spin_lock(&inode->i_lock);
	lo->plh_block_lgets--;
	spin_unlock(&inode->i_lock);
	pnfs_put_layout_hdr(lo);
	return ret;
}

F
Fred Isaman 已提交
1180 1181
bool pnfs_roc(struct inode *ino)
{
1182 1183 1184
	struct nfs_inode *nfsi = NFS_I(ino);
	struct nfs_open_context *ctx;
	struct nfs4_state *state;
F
Fred Isaman 已提交
1185 1186
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg, *tmp;
1187
	nfs4_stateid stateid;
F
Fred Isaman 已提交
1188
	LIST_HEAD(tmp_list);
1189
	bool found = false, layoutreturn = false, roc = false;
F
Fred Isaman 已提交
1190 1191

	spin_lock(&ino->i_lock);
1192
	lo = nfsi->layout;
1193 1194
	if (!lo || !pnfs_layout_is_valid(lo) ||
	    test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
1195 1196
		goto out_noroc;

1197
	/* no roc if we hold a delegation */
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
	if (nfs4_check_delegation(ino, FMODE_READ))
		goto out_noroc;

	list_for_each_entry(ctx, &nfsi->open_files, list) {
		state = ctx->state;
		/* Don't return layout if there is open file state */
		if (state != NULL && state->state != 0)
			goto out_noroc;
	}

1208

1209
	list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list) {
1210
		/* If we are sending layoutreturn, invalidate all valid lsegs */
1211
		if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
F
Fred Isaman 已提交
1212 1213 1214
			mark_lseg_invalid(lseg, &tmp_list);
			found = true;
		}
1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
	}

	/* always send layoutreturn if being marked so */
	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
		layoutreturn = pnfs_prepare_layoutreturn(lo,
				&stateid, NULL);
		if (layoutreturn)
			goto out_noroc;
	}

1225
	/* ROC in two conditions:
1226 1227 1228
	 * 1. there are ROC lsegs
	 * 2. we don't send layoutreturn
	 */
1229
	if (found) {
1230 1231
		/* lo ref dropped in pnfs_roc_release() */
		pnfs_get_layout_hdr(lo);
1232
		roc = true;
1233
	}
F
Fred Isaman 已提交
1234

1235
out_noroc:
F
Fred Isaman 已提交
1236
	spin_unlock(&ino->i_lock);
1237 1238 1239
	pnfs_free_lseg_list(&tmp_list);
	pnfs_layoutcommit_inode(ino, true);
	if (layoutreturn)
1240
		pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1241
	return roc;
F
Fred Isaman 已提交
1242 1243 1244 1245 1246 1247 1248 1249
}

void pnfs_roc_release(struct inode *ino)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1250
	pnfs_clear_layoutreturn_waitbit(lo);
1251 1252 1253 1254 1255 1256
	if (atomic_dec_and_test(&lo->plh_refcount)) {
		pnfs_detach_layout_hdr(lo);
		spin_unlock(&ino->i_lock);
		pnfs_free_layout_hdr(lo);
	} else
		spin_unlock(&ino->i_lock);
F
Fred Isaman 已提交
1257 1258 1259 1260 1261 1262 1263 1264
}

void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1265
	if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
F
Fred Isaman 已提交
1266 1267
		lo->plh_barrier = barrier;
	spin_unlock(&ino->i_lock);
1268
	trace_nfs4_layoutreturn_on_close(ino, 0);
F
Fred Isaman 已提交
1269 1270
}

1271
void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier)
F
Fred Isaman 已提交
1272 1273
{
	struct nfs_inode *nfsi = NFS_I(ino);
1274 1275
	struct pnfs_layout_hdr *lo;
	u32 current_seqid;
F
Fred Isaman 已提交
1276 1277

	spin_lock(&ino->i_lock);
1278 1279
	lo = nfsi->layout;
	current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
F
Fred Isaman 已提交
1280

1281 1282 1283 1284
	/* Since close does not return a layout stateid for use as
	 * a barrier, we choose the worst-case barrier.
	 */
	*barrier = current_seqid + atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
1285 1286 1287
	spin_unlock(&ino->i_lock);
}

1288 1289 1290 1291 1292 1293 1294 1295 1296 1297
bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
{
	struct nfs_inode *nfsi = NFS_I(ino);
        struct pnfs_layout_hdr *lo;
        bool sleep = false;

	/* we might not have grabbed lo reference. so need to check under
	 * i_lock */
        spin_lock(&ino->i_lock);
        lo = nfsi->layout;
1298 1299
        if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
1300
                sleep = true;
1301
	}
1302 1303 1304 1305
        spin_unlock(&ino->i_lock);
        return sleep;
}

1306 1307 1308 1309 1310 1311
/*
 * Compare two layout segments for sorting into layout cache.
 * We want to preferentially return RW over RO layouts, so ensure those
 * are seen first.
 */
static s64
1312
pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1313
	   const struct pnfs_layout_range *l2)
1314
{
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
	s64 d;

	/* high offset > low offset */
	d = l1->offset - l2->offset;
	if (d)
		return d;

	/* short length > long length */
	d = l2->length - l1->length;
	if (d)
		return d;

1327
	/* read > read/write */
1328
	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1329 1330
}

1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352
static bool
pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
		const struct pnfs_layout_range *l2)
{
	return pnfs_lseg_range_cmp(l1, l2) > 0;
}

static bool
pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
		struct pnfs_layout_segment *old)
{
	return false;
}

void
pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   bool (*is_after)(const struct pnfs_layout_range *,
			   const struct pnfs_layout_range *),
		   bool (*do_merge)(struct pnfs_layout_segment *,
			   struct pnfs_layout_segment *),
		   struct list_head *free_me)
1353
{
1354
	struct pnfs_layout_segment *lp, *tmp;
1355

1356 1357
	dprintk("%s:Begin\n", __func__);

1358 1359 1360 1361 1362 1363 1364 1365
	list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
			continue;
		if (do_merge(lseg, lp)) {
			mark_lseg_invalid(lp, free_me);
			continue;
		}
		if (is_after(&lseg->pls_range, &lp->pls_range))
1366
			continue;
1367
		list_add_tail(&lseg->pls_list, &lp->pls_list);
1368 1369 1370
		dprintk("%s: inserted lseg %p "
			"iomode %d offset %llu length %llu before "
			"lp %p iomode %d offset %llu length %llu\n",
1371 1372 1373 1374
			__func__, lseg, lseg->pls_range.iomode,
			lseg->pls_range.offset, lseg->pls_range.length,
			lp, lp->pls_range.iomode, lp->pls_range.offset,
			lp->pls_range.length);
1375
		goto out;
1376
	}
1377 1378 1379 1380 1381 1382
	list_add_tail(&lseg->pls_list, &lo->plh_segs);
	dprintk("%s: inserted lseg %p "
		"iomode %d offset %llu length %llu at tail\n",
		__func__, lseg, lseg->pls_range.iomode,
		lseg->pls_range.offset, lseg->pls_range.length);
out:
1383
	pnfs_get_layout_hdr(lo);
1384 1385

	dprintk("%s:Return\n", __func__);
1386
}
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);

static void
pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   struct list_head *free_me)
{
	struct inode *inode = lo->plh_inode;
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;

	if (ld->add_lseg != NULL)
		ld->add_lseg(lo, lseg, free_me);
	else
		pnfs_generic_layout_insert_lseg(lo, lseg,
				pnfs_lseg_range_is_after,
				pnfs_lseg_no_merge,
				free_me);
}
1405 1406

static struct pnfs_layout_hdr *
1407 1408 1409
alloc_init_layout_hdr(struct inode *ino,
		      struct nfs_open_context *ctx,
		      gfp_t gfp_flags)
1410 1411 1412
{
	struct pnfs_layout_hdr *lo;

1413
	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1414 1415
	if (!lo)
		return NULL;
1416
	atomic_set(&lo->plh_refcount, 1);
1417 1418
	INIT_LIST_HEAD(&lo->plh_layouts);
	INIT_LIST_HEAD(&lo->plh_segs);
1419
	INIT_LIST_HEAD(&lo->plh_return_segs);
1420
	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1421
	lo->plh_inode = ino;
1422
	lo->plh_lc_cred = get_rpccred(ctx->cred);
1423
	lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID;
1424 1425 1426 1427
	return lo;
}

static struct pnfs_layout_hdr *
1428 1429 1430
pnfs_find_alloc_layout(struct inode *ino,
		       struct nfs_open_context *ctx,
		       gfp_t gfp_flags)
1431 1432
	__releases(&ino->i_lock)
	__acquires(&ino->i_lock)
1433 1434 1435 1436 1437 1438
{
	struct nfs_inode *nfsi = NFS_I(ino);
	struct pnfs_layout_hdr *new = NULL;

	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

1439 1440
	if (nfsi->layout != NULL)
		goto out_existing;
1441
	spin_unlock(&ino->i_lock);
1442
	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1443 1444
	spin_lock(&ino->i_lock);

1445
	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1446
		nfsi->layout = new;
1447
		return new;
1448 1449
	} else if (new != NULL)
		pnfs_free_layout_hdr(new);
1450 1451
out_existing:
	pnfs_get_layout_hdr(nfsi->layout);
1452 1453 1454
	return nfsi->layout;
}

1455 1456
/*
 * iomode matching rules:
1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
 * iomode	lseg	strict match
 *                      iomode
 * -----	-----	------ -----
 * ANY		READ	N/A    true
 * ANY		RW	N/A    true
 * RW		READ	N/A    false
 * RW		RW	N/A    true
 * READ		READ	N/A    true
 * READ		RW	true   false
 * READ		RW	false  true
1467
 */
1468
static bool
1469
pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1470 1471
		 const struct pnfs_layout_range *range,
		 bool strict_iomode)
1472
{
1473 1474 1475 1476
	struct pnfs_layout_range range1;

	if ((range->iomode == IOMODE_RW &&
	     ls_range->iomode != IOMODE_RW) ||
1477 1478
	    (range->iomode != ls_range->iomode &&
	     strict_iomode == true) ||
1479
	    !pnfs_lseg_range_intersecting(ls_range, range))
1480 1481 1482 1483 1484
		return 0;

	/* range1 covers only the first byte in the range */
	range1 = *range;
	range1.length = 1;
1485
	return pnfs_lseg_range_contained(ls_range, &range1);
1486 1487 1488 1489 1490
}

/*
 * lookup range in layout
 */
1491
static struct pnfs_layout_segment *
1492
pnfs_find_lseg(struct pnfs_layout_hdr *lo,
1493 1494
		struct pnfs_layout_range *range,
		bool strict_iomode)
1495
{
1496 1497 1498 1499
	struct pnfs_layout_segment *lseg, *ret = NULL;

	dprintk("%s:Begin\n", __func__);

1500
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1501
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1502
		    !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
1503 1504
		    pnfs_lseg_range_match(&lseg->pls_range, range,
					  strict_iomode)) {
1505
			ret = pnfs_get_lseg(lseg);
1506 1507 1508 1509 1510
			break;
		}
	}

	dprintk("%s:Return lseg %p ref %d\n",
1511
		__func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
1512
	return ret;
1513 1514
}

1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589
/*
 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
 * to the MDS or over pNFS
 *
 * The nfs_inode read_io and write_io fields are cumulative counters reset
 * when there are no layout segments. Note that in pnfs_update_layout iomode
 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
 * WRITE request.
 *
 * A return of true means use MDS I/O.
 *
 * From rfc 5661:
 * If a file's size is smaller than the file size threshold, data accesses
 * SHOULD be sent to the metadata server.  If an I/O request has a length that
 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
 * server.  If both file size and I/O size are provided, the client SHOULD
 * reach or exceed  both thresholds before sending its read or write
 * requests to the data server.
 */
static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
				     struct inode *ino, int iomode)
{
	struct nfs4_threshold *t = ctx->mdsthreshold;
	struct nfs_inode *nfsi = NFS_I(ino);
	loff_t fsize = i_size_read(ino);
	bool size = false, size_set = false, io = false, io_set = false, ret = false;

	if (t == NULL)
		return ret;

	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);

	switch (iomode) {
	case IOMODE_READ:
		if (t->bm & THRESHOLD_RD) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->rd_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_RD_IO) {
			dprintk("%s nfsi->read_io %llu\n", __func__,
				nfsi->read_io);
			io_set = true;
			if (nfsi->read_io < t->rd_io_sz)
				io = true;
		}
		break;
	case IOMODE_RW:
		if (t->bm & THRESHOLD_WR) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->wr_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_WR_IO) {
			dprintk("%s nfsi->write_io %llu\n", __func__,
				nfsi->write_io);
			io_set = true;
			if (nfsi->write_io < t->wr_io_sz)
				io = true;
		}
		break;
	}
	if (size_set && io_set) {
		if (size && io)
			ret = true;
	} else if (size || io)
		ret = true;

	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
	return ret;
}

1590 1591 1592 1593 1594 1595 1596 1597
static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
{
	/*
	 * send layoutcommit as it can hold up layoutreturn due to lseg
	 * reference
	 */
	pnfs_layoutcommit_inode(lo->plh_inode, false);
	return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
1598
				   nfs_wait_bit_killable,
1599 1600 1601
				   TASK_UNINTERRUPTIBLE);
}

1602 1603 1604 1605 1606 1607 1608 1609 1610
static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
{
	unsigned long *bitlock = &lo->plh_flags;

	clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
	smp_mb__after_atomic();
	wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
}

1611 1612 1613 1614
/*
 * Layout segment is retreived from the server if not cached.
 * The appropriate layout segment is referenced and returned to the caller.
 */
1615
struct pnfs_layout_segment *
1616 1617
pnfs_update_layout(struct inode *ino,
		   struct nfs_open_context *ctx,
1618 1619
		   loff_t pos,
		   u64 count,
1620
		   enum pnfs_iomode iomode,
1621
		   bool strict_iomode,
1622
		   gfp_t gfp_flags)
1623
{
1624 1625 1626 1627 1628
	struct pnfs_layout_range arg = {
		.iomode = iomode,
		.offset = pos,
		.length = count,
	};
1629
	unsigned pg_offset, seq;
1630 1631
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs_client *clp = server->nfs_client;
1632
	struct pnfs_layout_hdr *lo = NULL;
1633
	struct pnfs_layout_segment *lseg = NULL;
1634 1635
	nfs4_stateid stateid;
	long timeout = 0;
1636
	unsigned long giveup = jiffies + (clp->cl_lease_time << 1);
1637
	bool first;
1638

1639
	if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
1640
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1641
				 PNFS_UPDATE_LAYOUT_NO_PNFS);
1642
		goto out;
1643
	}
1644

1645
	if (iomode == IOMODE_READ && i_size_read(ino) == 0) {
1646
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1647
				 PNFS_UPDATE_LAYOUT_RD_ZEROLEN);
1648
		goto out;
1649
	}
1650

1651
	if (pnfs_within_mdsthreshold(ctx, ino, iomode)) {
1652
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1653
				 PNFS_UPDATE_LAYOUT_MDSTHRESH);
1654
		goto out;
1655
	}
1656

1657
lookup_again:
1658
	nfs4_client_recover_expired_lease(clp);
1659
	first = false;
1660
	spin_lock(&ino->i_lock);
1661
	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
1662 1663
	if (lo == NULL) {
		spin_unlock(&ino->i_lock);
1664
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1665
				 PNFS_UPDATE_LAYOUT_NOMEM);
1666 1667
		goto out;
	}
1668

F
Fred Isaman 已提交
1669
	/* Do we even need to bother with this? */
1670
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
1671
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1672
				 PNFS_UPDATE_LAYOUT_BULK_RECALL);
F
Fred Isaman 已提交
1673
		dprintk("%s matches recall, use MDS\n", __func__);
1674 1675 1676 1677
		goto out_unlock;
	}

	/* if LAYOUTGET already failed once we don't try again */
1678
	if (pnfs_layout_io_test_failed(lo, iomode)) {
1679
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1680
				 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL);
1681
		goto out_unlock;
1682
	}
1683

1684
	lseg = pnfs_find_lseg(lo, &arg, strict_iomode);
1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
	if (lseg) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				PNFS_UPDATE_LAYOUT_FOUND_CACHED);
		goto out_unlock;
	}

	if (!nfs4_valid_open_stateid(ctx->state)) {
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				PNFS_UPDATE_LAYOUT_INVALID_OPEN);
		goto out_unlock;
	}

	/*
	 * Choose a stateid for the LAYOUTGET. If we don't have a layout
	 * stateid, or it has been invalidated, then we must use the open
	 * stateid.
	 */
1702
	if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
1703 1704 1705

		/*
		 * The first layoutget for the file. Need to serialize per
1706 1707 1708 1709 1710 1711 1712 1713
		 * RFC 5661 Errata 3208.
		 */
		if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
				     &lo->plh_flags)) {
			spin_unlock(&ino->i_lock);
			wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
				    TASK_UNINTERRUPTIBLE);
			pnfs_put_layout_hdr(lo);
1714
			dprintk("%s retrying\n", __func__);
1715 1716
			goto lookup_again;
		}
1717 1718 1719 1720 1721 1722

		first = true;
		do {
			seq = read_seqbegin(&ctx->state->seqlock);
			nfs4_stateid_copy(&stateid, &ctx->state->stateid);
		} while (read_seqretry(&ctx->state->seqlock, seq));
1723
	} else {
1724
		nfs4_stateid_copy(&stateid, &lo->plh_stateid);
1725
	}
1726

1727 1728 1729 1730
	/*
	 * Because we free lsegs before sending LAYOUTRETURN, we need to wait
	 * for LAYOUTRETURN even if first is true.
	 */
1731
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
1732 1733 1734
		spin_unlock(&ino->i_lock);
		dprintk("%s wait for layoutreturn\n", __func__);
		if (pnfs_prepare_to_retry_layoutget(lo)) {
1735 1736
			if (first)
				pnfs_clear_first_layoutget(lo);
1737 1738
			pnfs_put_layout_hdr(lo);
			dprintk("%s retrying\n", __func__);
1739 1740
			trace_pnfs_update_layout(ino, pos, count, iomode, lo,
					lseg, PNFS_UPDATE_LAYOUT_RETRY);
1741 1742
			goto lookup_again;
		}
1743
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1744
				PNFS_UPDATE_LAYOUT_RETURN);
1745 1746 1747
		goto out_put_layout_hdr;
	}

1748
	if (pnfs_layoutgets_blocked(lo)) {
1749
		trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1750
				PNFS_UPDATE_LAYOUT_BLOCKED);
1751
		goto out_unlock;
1752
	}
1753
	atomic_inc(&lo->plh_outstanding);
1754
	spin_unlock(&ino->i_lock);
1755

1756
	if (list_empty(&lo->plh_layouts)) {
1757 1758 1759 1760
		/* The lo must be on the clp list if there is any
		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
		 */
		spin_lock(&clp->cl_lock);
1761 1762
		if (list_empty(&lo->plh_layouts))
			list_add_tail(&lo->plh_layouts, &server->layouts);
1763 1764
		spin_unlock(&clp->cl_lock);
	}
1765

1766
	pg_offset = arg.offset & ~PAGE_MASK;
1767 1768 1769 1770
	if (pg_offset) {
		arg.offset -= pg_offset;
		arg.length += pg_offset;
	}
1771
	if (arg.length != NFS4_MAX_UINT64)
1772
		arg.length = PAGE_ALIGN(arg.length);
1773

1774 1775 1776
	lseg = send_layoutget(lo, ctx, &stateid, &arg, &timeout, gfp_flags);
	trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
				 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
1777
	atomic_dec(&lo->plh_outstanding);
1778
	if (IS_ERR(lseg)) {
1779
		switch(PTR_ERR(lseg)) {
1780
		case -EBUSY:
1781 1782
			if (time_after(jiffies, giveup))
				lseg = NULL;
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
			break;
		case -ERECALLCONFLICT:
			/* Huh? We hold no layouts, how is there a recall? */
			if (first) {
				lseg = NULL;
				break;
			}
			/* Destroy the existing layout and start over */
			if (time_after(jiffies, giveup))
				pnfs_destroy_layout(NFS_I(ino));
1793 1794
			/* Fallthrough */
		case -EAGAIN:
1795
			break;
1796 1797 1798 1799 1800
		default:
			if (!nfs_error_is_fatal(PTR_ERR(lseg))) {
				pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
				lseg = NULL;
			}
1801 1802 1803 1804 1805 1806 1807 1808 1809
			goto out_put_layout_hdr;
		}
		if (lseg) {
			if (first)
				pnfs_clear_first_layoutget(lo);
			trace_pnfs_update_layout(ino, pos, count,
				iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY);
			pnfs_put_layout_hdr(lo);
			goto lookup_again;
1810 1811 1812 1813 1814
		}
	} else {
		pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
	}

1815
out_put_layout_hdr:
1816 1817
	if (first)
		pnfs_clear_first_layoutget(lo);
1818
	pnfs_put_layout_hdr(lo);
1819
out:
1820 1821 1822 1823
	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
			"(%s, offset: %llu, length: %llu)\n",
			__func__, ino->i_sb->s_id,
			(unsigned long long)NFS_FILEID(ino),
1824
			IS_ERR_OR_NULL(lseg) ? "not found" : "found",
1825 1826 1827
			iomode==IOMODE_RW ?  "read/write" : "read-only",
			(unsigned long long)pos,
			(unsigned long long)count);
1828 1829 1830
	return lseg;
out_unlock:
	spin_unlock(&ino->i_lock);
1831
	goto out_put_layout_hdr;
1832
}
1833
EXPORT_SYMBOL_GPL(pnfs_update_layout);
1834

1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
static bool
pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
{
	switch (range->iomode) {
	case IOMODE_READ:
	case IOMODE_RW:
		break;
	default:
		return false;
	}
	if (range->offset == NFS4_MAX_UINT64)
		return false;
	if (range->length == 0)
		return false;
	if (range->length != NFS4_MAX_UINT64 &&
	    range->length > NFS4_MAX_UINT64 - range->offset)
		return false;
	return true;
}

1855
struct pnfs_layout_segment *
1856 1857 1858 1859 1860
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
	struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
	struct nfs4_layoutget_res *res = &lgp->res;
	struct pnfs_layout_segment *lseg;
1861
	struct inode *ino = lo->plh_inode;
1862
	LIST_HEAD(free_me);
1863 1864

	if (!pnfs_sanity_check_layout_range(&res->range))
1865
		return ERR_PTR(-EINVAL);
1866 1867

	/* Inject layout blob into I/O device driver */
1868
	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
1869
	if (IS_ERR_OR_NULL(lseg)) {
1870
		if (!lseg)
1871 1872 1873 1874 1875
			lseg = ERR_PTR(-ENOMEM);

		dprintk("%s: Could not allocate layout: error %ld\n",
		       __func__, PTR_ERR(lseg));
		return lseg;
1876 1877
	}

1878
	pnfs_init_lseg(lo, lseg, &res->range, &res->stateid);
1879

1880
	spin_lock(&ino->i_lock);
1881
	if (pnfs_layoutgets_blocked(lo)) {
F
Fred Isaman 已提交
1882
		dprintk("%s forget reply due to state\n", __func__);
1883
		goto out_forget;
F
Fred Isaman 已提交
1884
	}
1885

1886 1887 1888 1889
	if (!pnfs_layout_is_valid(lo)) {
		/* We have a completely new layout */
		pnfs_set_layout_stateid(lo, &res->stateid, true);
	} else if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
1890 1891 1892
		/* existing state ID, make sure the sequence number matches. */
		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
			dprintk("%s forget reply due to sequence\n", __func__);
1893
			goto out_forget;
1894 1895 1896 1897 1898
		}
		pnfs_set_layout_stateid(lo, &res->stateid, false);
	} else {
		/*
		 * We got an entirely new state ID.  Mark all segments for the
1899
		 * inode invalid, and retry the layoutget
1900
		 */
1901
		pnfs_mark_layout_stateid_invalid(lo, &free_me);
1902
		goto out_forget;
1903
	}
1904

1905
	pnfs_get_lseg(lseg);
1906
	pnfs_layout_insert_lseg(lo, lseg, &free_me);
1907

1908

P
Peng Tao 已提交
1909
	if (res->return_on_close)
F
Fred Isaman 已提交
1910 1911
		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);

1912
	spin_unlock(&ino->i_lock);
1913
	pnfs_free_lseg_list(&free_me);
1914
	return lseg;
F
Fred Isaman 已提交
1915

1916
out_forget:
F
Fred Isaman 已提交
1917 1918 1919
	spin_unlock(&ino->i_lock);
	lseg->pls_layout = lo;
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
1920
	return ERR_PTR(-EAGAIN);
1921 1922
}

1923
static void
1924 1925
pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode,
			 u32 seq)
1926
{
1927
	if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode)
1928 1929
		iomode = IOMODE_ANY;
	lo->plh_return_iomode = iomode;
1930
	set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
1931 1932
	if (seq != 0) {
		WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq);
1933
		lo->plh_return_seq = seq;
1934
	}
1935 1936
}

1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
/**
 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments
 * @lo: pointer to layout header
 * @tmp_list: list header to be used with pnfs_free_lseg_list()
 * @return_range: describe layout segment ranges to be returned
 *
 * This function is mainly intended for use by layoutrecall. It attempts
 * to free the layout segment immediately, or else to mark it for return
 * as soon as its reference count drops to zero.
 */
1947
int
1948 1949
pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
				struct list_head *tmp_list,
1950 1951
				const struct pnfs_layout_range *return_range,
				u32 seq)
1952 1953
{
	struct pnfs_layout_segment *lseg, *next;
1954
	int remaining = 0;
1955 1956 1957 1958

	dprintk("%s:Begin lo %p\n", __func__, lo);

	if (list_empty(&lo->plh_segs))
1959
		return 0;
1960

1961
	assert_spin_locked(&lo->plh_inode->i_lock);
1962 1963

	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
1964
		if (pnfs_match_lseg_recall(lseg, return_range, seq)) {
1965 1966 1967 1968 1969
			dprintk("%s: marking lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode,
				lseg->pls_range.offset,
				lseg->pls_range.length);
1970 1971 1972
			if (mark_lseg_invalid(lseg, tmp_list))
				continue;
			remaining++;
1973 1974
			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
		}
1975 1976 1977 1978

	if (remaining)
		pnfs_set_plh_return_info(lo, return_range->iomode, seq);

1979
	return remaining;
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990
}

void pnfs_error_mark_layout_for_return(struct inode *inode,
				       struct pnfs_layout_segment *lseg)
{
	struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
	struct pnfs_layout_range range = {
		.iomode = lseg->pls_range.iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
1991
	bool return_now = false;
1992 1993

	spin_lock(&inode->i_lock);
1994
	pnfs_set_plh_return_info(lo, range.iomode, 0);
1995 1996
	/* Block LAYOUTGET */
	set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
1997 1998 1999 2000 2001
	/*
	 * mark all matching lsegs so that we are sure to have no live
	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
	 * for how it works.
	 */
2002
	if (!pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, &range, 0)) {
2003
		nfs4_stateid stateid;
2004
		enum pnfs_iomode iomode;
2005

2006
		return_now = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
2007 2008 2009 2010 2011 2012 2013
		spin_unlock(&inode->i_lock);
		if (return_now)
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
	} else {
		spin_unlock(&inode->i_lock);
		nfs_commit_inode(inode, 0);
	}
2014 2015 2016
}
EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);

2017 2018 2019
void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
P
Peng Tao 已提交
2020 2021
	u64 rd_size = req->wb_bytes;

2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032
	if (pgio->pg_lseg == NULL) {
		if (pgio->pg_dreq == NULL)
			rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
		else
			rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);

		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   rd_size,
						   IOMODE_READ,
2033
						   false,
2034
						   GFP_KERNEL);
2035 2036 2037 2038 2039
		if (IS_ERR(pgio->pg_lseg)) {
			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
			pgio->pg_lseg = NULL;
			return;
		}
2040
	}
2041 2042
	/* If no lseg, fall back to read through mds */
	if (pgio->pg_lseg == NULL)
2043
		nfs_pageio_reset_read_mds(pgio);
2044

2045 2046 2047 2048
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);

void
2049 2050
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
			   struct nfs_page *req, u64 wb_size)
2051
{
2052
	if (pgio->pg_lseg == NULL) {
2053 2054 2055 2056 2057
		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   wb_size,
						   IOMODE_RW,
2058
						   false,
2059
						   GFP_NOFS);
2060 2061 2062 2063 2064 2065
		if (IS_ERR(pgio->pg_lseg)) {
			pgio->pg_error = PTR_ERR(pgio->pg_lseg);
			pgio->pg_lseg = NULL;
			return;
		}
	}
2066 2067
	/* If no lseg, fall back to write through mds */
	if (pgio->pg_lseg == NULL)
2068
		nfs_pageio_reset_write_mds(pgio);
2069 2070 2071
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);

2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
void
pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
{
	if (desc->pg_lseg) {
		pnfs_put_lseg(desc->pg_lseg);
		desc->pg_lseg = NULL;
	}
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);

2082 2083 2084 2085 2086
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
size_t
2087 2088
pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
		     struct nfs_page *prev, struct nfs_page *req)
2089
{
2090
	unsigned int size;
2091
	u64 seg_end, req_start, seg_left;
2092 2093 2094 2095

	size = nfs_generic_pg_test(pgio, prev, req);
	if (!size)
		return 0;
2096

2097
	/*
2098 2099 2100 2101 2102
	 * 'size' contains the number of bytes left in the current page (up
	 * to the original size asked for in @req->wb_bytes).
	 *
	 * Calculate how many bytes are left in the layout segment
	 * and if there are less bytes than 'size', return that instead.
2103 2104 2105 2106 2107
	 *
	 * Please also note that 'end_offset' is actually the offset of the
	 * first byte that lies outside the pnfs_layout_range. FIXME?
	 *
	 */
2108
	if (pgio->pg_lseg) {
2109
		seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset,
2110 2111
				     pgio->pg_lseg->pls_range.length);
		req_start = req_offset(req);
2112
		WARN_ON_ONCE(req_start >= seg_end);
2113
		/* start of request is past the last byte of this segment */
2114 2115 2116 2117 2118 2119
		if (req_start >= seg_end) {
			/* reference the new lseg */
			if (pgio->pg_ops->pg_cleanup)
				pgio->pg_ops->pg_cleanup(pgio);
			if (pgio->pg_ops->pg_init)
				pgio->pg_ops->pg_init(pgio, req);
2120
			return 0;
2121
		}
2122 2123 2124 2125 2126 2127

		/* adjust 'size' iff there are fewer bytes left in the
		 * segment than what nfs_generic_pg_test returned */
		seg_left = seg_end - req_start;
		if (seg_left < size)
			size = (unsigned int)seg_left;
2128
	}
2129

2130
	return size;
2131
}
2132
EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
2133

2134
int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
2135 2136 2137 2138
{
	struct nfs_pageio_descriptor pgio;

	/* Resend all requests through the MDS */
2139 2140
	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
			      hdr->completion_ops);
2141
	set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags);
2142
	return nfs_pageio_resend(&pgio, hdr);
2143
}
2144
EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
2145

2146
static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
2147
{
2148 2149 2150

	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2151
	    PNFS_LAYOUTRET_ON_ERROR) {
2152
		pnfs_return_layout(hdr->inode);
2153
	}
2154
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2155
		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
2156 2157
}

2158 2159 2160
/*
 * Called by non rpc-based layout drivers
 */
2161
void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
2162
{
2163
	if (likely(!hdr->pnfs_error)) {
2164 2165
		pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
				hdr->mds_offset + hdr->res.count);
2166
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2167 2168 2169
	}
	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2170 2171
		pnfs_ld_handle_write_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2172
}
2173
EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
2174

2175 2176
static void
pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
2177
		struct nfs_pgio_header *hdr)
2178
{
2179
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2180

2181
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2182
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2183
		nfs_pageio_reset_write_mds(desc);
2184
		mirror->pg_recoalesce = 1;
2185
	}
2186
	nfs_pgio_data_destroy(hdr);
2187
	hdr->release(hdr);
2188 2189 2190
}

static enum pnfs_try_status
2191
pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
2192 2193 2194
			const struct rpc_call_ops *call_ops,
			struct pnfs_layout_segment *lseg,
			int how)
2195
{
2196
	struct inode *inode = hdr->inode;
2197 2198 2199
	enum pnfs_try_status trypnfs;
	struct nfs_server *nfss = NFS_SERVER(inode);

2200
	hdr->mds_ops = call_ops;
2201 2202

	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
2203 2204
		inode->i_ino, hdr->args.count, hdr->args.offset, how);
	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
2205
	if (trypnfs != PNFS_NOT_ATTEMPTED)
2206 2207 2208 2209 2210
		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}

2211
static void
2212 2213
pnfs_do_write(struct nfs_pageio_descriptor *desc,
	      struct nfs_pgio_header *hdr, int how)
2214 2215 2216
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2217
	enum pnfs_try_status trypnfs;
2218

2219
	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
2220
	if (trypnfs == PNFS_NOT_ATTEMPTED)
2221
		pnfs_write_through_mds(desc, hdr);
2222 2223
}

2224 2225
static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
2226
	pnfs_put_lseg(hdr->lseg);
2227
	nfs_pgio_header_free(hdr);
2228 2229
}

2230 2231 2232
int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
{
2233
	struct nfs_pgio_header *hdr;
2234 2235
	int ret;

2236 2237
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
P
Peng Tao 已提交
2238 2239
		desc->pg_error = -ENOMEM;
		return desc->pg_error;
2240
	}
2241
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
2242

2243
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2244
	ret = nfs_generic_pgio(desc, hdr);
2245
	if (!ret)
2246
		pnfs_do_write(desc, hdr, desc->pg_ioflags);
2247

2248
	return ret;
2249 2250 2251
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);

2252
int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
2253 2254 2255
{
	struct nfs_pageio_descriptor pgio;

2256
	/* Resend all requests through the MDS */
2257 2258
	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
2259
}
2260
EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
2261

2262
static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
2263
{
2264 2265
	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2266
	    PNFS_LAYOUTRET_ON_ERROR) {
2267
		pnfs_return_layout(hdr->inode);
2268
	}
2269
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2270
		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
2271 2272
}

2273 2274 2275
/*
 * Called by non rpc-based layout drivers
 */
2276
void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
2277
{
2278
	if (likely(!hdr->pnfs_error))
2279
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2280 2281
	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2282 2283
		pnfs_ld_handle_read_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2284 2285 2286
}
EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

2287 2288
static void
pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
2289
		struct nfs_pgio_header *hdr)
2290
{
2291
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2292

2293
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2294
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2295
		nfs_pageio_reset_read_mds(desc);
2296
		mirror->pg_recoalesce = 1;
2297
	}
2298
	nfs_pgio_data_destroy(hdr);
2299
	hdr->release(hdr);
2300 2301
}

A
Andy Adamson 已提交
2302 2303 2304
/*
 * Call the appropriate parallel I/O subsystem read function.
 */
2305
static enum pnfs_try_status
2306
pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
2307 2308
		       const struct rpc_call_ops *call_ops,
		       struct pnfs_layout_segment *lseg)
A
Andy Adamson 已提交
2309
{
2310
	struct inode *inode = hdr->inode;
A
Andy Adamson 已提交
2311 2312 2313
	struct nfs_server *nfss = NFS_SERVER(inode);
	enum pnfs_try_status trypnfs;

2314
	hdr->mds_ops = call_ops;
A
Andy Adamson 已提交
2315 2316

	dprintk("%s: Reading ino:%lu %u@%llu\n",
2317
		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
A
Andy Adamson 已提交
2318

2319
	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
2320
	if (trypnfs != PNFS_NOT_ATTEMPTED)
A
Andy Adamson 已提交
2321 2322 2323 2324
		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}
A
Andy Adamson 已提交
2325

2326
/* Resend all requests through pnfs. */
2327
void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
2328 2329 2330
{
	struct nfs_pageio_descriptor pgio;

2331
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2332 2333 2334 2335
		/* Prevent deadlocks with layoutreturn! */
		pnfs_put_lseg(hdr->lseg);
		hdr->lseg = NULL;

2336 2337 2338 2339
		nfs_pageio_init_read(&pgio, hdr->inode, false,
					hdr->completion_ops);
		hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr);
	}
2340 2341 2342
}
EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);

2343
static void
2344
pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
2345 2346 2347
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2348
	enum pnfs_try_status trypnfs;
2349

2350
	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
2351
	if (trypnfs == PNFS_TRY_AGAIN)
2352 2353
		pnfs_read_resend_pnfs(hdr);
	if (trypnfs == PNFS_NOT_ATTEMPTED || hdr->task.tk_status)
2354
		pnfs_read_through_mds(desc, hdr);
2355 2356
}

2357 2358
static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
2359
	pnfs_put_lseg(hdr->lseg);
2360
	nfs_pgio_header_free(hdr);
2361 2362
}

2363 2364 2365
int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
{
2366
	struct nfs_pgio_header *hdr;
2367 2368
	int ret;

2369 2370
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
P
Peng Tao 已提交
2371 2372
		desc->pg_error = -ENOMEM;
		return desc->pg_error;
2373
	}
2374
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
2375
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2376
	ret = nfs_generic_pgio(desc, hdr);
2377
	if (!ret)
2378
		pnfs_do_read(desc, hdr);
2379
	return ret;
2380 2381 2382
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);

2383 2384 2385 2386 2387
static void pnfs_clear_layoutcommitting(struct inode *inode)
{
	unsigned long *bitlock = &NFS_I(inode)->flags;

	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
2388
	smp_mb__after_atomic();
2389 2390 2391
	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}

A
Andy Adamson 已提交
2392
/*
2393
 * There can be multiple RW segments.
A
Andy Adamson 已提交
2394
 */
2395
static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
A
Andy Adamson 已提交
2396
{
2397
	struct pnfs_layout_segment *lseg;
A
Andy Adamson 已提交
2398

2399 2400
	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
		if (lseg->pls_range.iomode == IOMODE_RW &&
2401
		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
2402 2403
			list_add(&lseg->pls_lc_list, listp);
	}
A
Andy Adamson 已提交
2404 2405
}

2406 2407 2408 2409 2410 2411 2412 2413 2414 2415
static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
{
	struct pnfs_layout_segment *lseg, *tmp;

	/* Matched by references in pnfs_set_layoutcommit */
	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
		list_del_init(&lseg->pls_lc_list);
		pnfs_put_lseg(lseg);
	}

2416
	pnfs_clear_layoutcommitting(inode);
2417 2418
}

P
Peng Tao 已提交
2419 2420
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
2421
	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
P
Peng Tao 已提交
2422 2423 2424
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

A
Andy Adamson 已提交
2425
void
2426 2427
pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
		loff_t end_pos)
A
Andy Adamson 已提交
2428
{
2429
	struct nfs_inode *nfsi = NFS_I(inode);
2430
	bool mark_as_dirty = false;
A
Andy Adamson 已提交
2431

2432
	spin_lock(&inode->i_lock);
A
Andy Adamson 已提交
2433
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
2434
		nfsi->layout->plh_lwb = end_pos;
2435
		mark_as_dirty = true;
A
Andy Adamson 已提交
2436
		dprintk("%s: Set layoutcommit for inode %lu ",
2437
			__func__, inode->i_ino);
2438 2439
	} else if (end_pos > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = end_pos;
2440
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
2441
		/* references matched in nfs4_layoutcommit_release */
2442
		pnfs_get_lseg(lseg);
2443
	}
2444
	spin_unlock(&inode->i_lock);
2445
	dprintk("%s: lseg %p end_pos %llu\n",
2446
		__func__, lseg, nfsi->layout->plh_lwb);
2447 2448 2449 2450

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
2451
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2452 2453 2454
}
EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);

A
Andy Adamson 已提交
2455 2456 2457 2458 2459 2460
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
{
	struct nfs_server *nfss = NFS_SERVER(data->args.inode);

	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
2461
	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
A
Andy Adamson 已提交
2462 2463
}

2464 2465 2466 2467 2468 2469 2470 2471
/*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
 * data to disk to allow the server to recover the data if it crashes.
 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
 * is off, and a COMMIT is sent to a data server, or
 * if WRITEs to a data server return NFS_DATA_SYNC.
 */
A
Andy Adamson 已提交
2472
int
2473
pnfs_layoutcommit_inode(struct inode *inode, bool sync)
A
Andy Adamson 已提交
2474
{
2475
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
A
Andy Adamson 已提交
2476 2477 2478
	struct nfs4_layoutcommit_data *data;
	struct nfs_inode *nfsi = NFS_I(inode);
	loff_t end_pos;
2479
	int status;
A
Andy Adamson 已提交
2480

2481
	if (!pnfs_layoutcommit_outstanding(inode))
2482 2483
		return 0;

2484
	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
P
Peng Tao 已提交
2485

2486
	status = -EAGAIN;
P
Peng Tao 已提交
2487
	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2488 2489
		if (!sync)
			goto out;
2490
		status = wait_on_bit_lock_action(&nfsi->flags,
2491 2492 2493
				NFS_INO_LAYOUTCOMMITTING,
				nfs_wait_bit_killable,
				TASK_KILLABLE);
P
Peng Tao 已提交
2494
		if (status)
2495
			goto out;
P
Peng Tao 已提交
2496 2497
	}

2498 2499 2500 2501 2502 2503 2504
	status = -ENOMEM;
	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (!data)
		goto clear_layoutcommitting;

	status = 0;
2505
	spin_lock(&inode->i_lock);
2506 2507
	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		goto out_unlock;
2508

2509
	INIT_LIST_HEAD(&data->lseg_list);
2510
	pnfs_list_write_lseg(inode, &data->lseg_list);
A
Andy Adamson 已提交
2511

2512
	end_pos = nfsi->layout->plh_lwb;
A
Andy Adamson 已提交
2513

2514
	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
A
Andy Adamson 已提交
2515 2516 2517
	spin_unlock(&inode->i_lock);

	data->args.inode = inode;
2518
	data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
A
Andy Adamson 已提交
2519 2520 2521
	nfs_fattr_init(&data->fattr);
	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
	data->res.fattr = &data->fattr;
2522 2523 2524 2525
	if (end_pos != 0)
		data->args.lastbytewritten = end_pos - 1;
	else
		data->args.lastbytewritten = U64_MAX;
A
Andy Adamson 已提交
2526 2527
	data->res.server = NFS_SERVER(inode);

2528 2529 2530
	if (ld->prepare_layoutcommit) {
		status = ld->prepare_layoutcommit(&data->args);
		if (status) {
2531
			put_rpccred(data->cred);
2532
			spin_lock(&inode->i_lock);
2533 2534
			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
			if (end_pos > nfsi->layout->plh_lwb)
2535
				nfsi->layout->plh_lwb = end_pos;
2536
			goto out_unlock;
2537 2538 2539 2540
		}
	}


A
Andy Adamson 已提交
2541 2542
	status = nfs4_proc_layoutcommit(data, sync);
out:
P
Peng Tao 已提交
2543 2544
	if (status)
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2545 2546
	dprintk("<-- %s status %d\n", __func__, status);
	return status;
2547 2548
out_unlock:
	spin_unlock(&inode->i_lock);
P
Peng Tao 已提交
2549
	kfree(data);
2550 2551
clear_layoutcommitting:
	pnfs_clear_layoutcommitting(inode);
P
Peng Tao 已提交
2552
	goto out;
A
Andy Adamson 已提交
2553
}
2554
EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
2555

2556 2557 2558 2559 2560 2561 2562
int
pnfs_generic_sync(struct inode *inode, bool datasync)
{
	return pnfs_layoutcommit_inode(inode, true);
}
EXPORT_SYMBOL_GPL(pnfs_generic_sync);

2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573
struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
{
	struct nfs4_threshold *thp;

	thp = kzalloc(sizeof(*thp), GFP_NOFS);
	if (!thp) {
		dprintk("%s mdsthreshold allocation failed\n", __func__);
		return NULL;
	}
	return thp;
}
2574

2575
#if IS_ENABLED(CONFIG_NFS_V4_2)
2576
int
2577
pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
2578 2579 2580
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
	struct nfs_server *server = NFS_SERVER(inode);
2581
	struct nfs_inode *nfsi = NFS_I(inode);
2582 2583 2584 2585 2586 2587 2588
	struct nfs42_layoutstat_data *data;
	struct pnfs_layout_hdr *hdr;
	int status = 0;

	if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
		goto out;

2589 2590 2591
	if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
		goto out;

2592 2593 2594
	if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
		goto out;

2595 2596 2597
	spin_lock(&inode->i_lock);
	if (!NFS_I(inode)->layout) {
		spin_unlock(&inode->i_lock);
2598
		goto out_clear_layoutstats;
2599 2600 2601 2602 2603
	}
	hdr = NFS_I(inode)->layout;
	pnfs_get_layout_hdr(hdr);
	spin_unlock(&inode->i_lock);

2604
	data = kzalloc(sizeof(*data), gfp_flags);
2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
	if (!data) {
		status = -ENOMEM;
		goto out_put;
	}

	data->args.fh = NFS_FH(inode);
	data->args.inode = inode;
	status = ld->prepare_layoutstats(&data->args);
	if (status)
		goto out_free;

	status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);

out:
	dprintk("%s returns %d\n", __func__, status);
	return status;

out_free:
	kfree(data);
out_put:
	pnfs_put_layout_hdr(hdr);
2626
out_clear_layoutstats:
2627 2628 2629
	smp_mb__before_atomic();
	clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
	smp_mb__after_atomic();
2630 2631 2632
	goto out;
}
EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
2633
#endif
2634 2635 2636 2637

unsigned int layoutstats_timer;
module_param(layoutstats_timer, uint, 0644);
EXPORT_SYMBOL_GPL(layoutstats_timer);