pnfs.c 64.3 KB
Newer Older
R
Ricardo Labiaga 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 *  pNFS functions to call and manage layout drivers.
 *
 *  Copyright (c) 2002 [year of first publication]
 *  The Regents of the University of Michigan
 *  All Rights Reserved
 *
 *  Dean Hildebrand <dhildebz@umich.edu>
 *
 *  Permission is granted to use, copy, create derivative works, and
 *  redistribute this software and such derivative works for any purpose,
 *  so long as the name of the University of Michigan is not used in
 *  any advertising or publicity pertaining to the use or distribution
 *  of this software without specific, written prior authorization. If
 *  the above copyright notice or any other identification of the
 *  University of Michigan is included in any copy of any portion of
 *  this software, then the disclaimer below must also be included.
 *
 *  This software is provided as is, without representation or warranty
 *  of any kind either express or implied, including without limitation
 *  the implied warranties of merchantability, fitness for a particular
 *  purpose, or noninfringement.  The Regents of the University of
 *  Michigan shall not be liable for any damages, including special,
 *  indirect, incidental, or consequential damages, with respect to any
 *  claim arising out of or in connection with the use of the software,
 *  even if it has been or is hereafter advised of the possibility of
 *  such damages.
 */

#include <linux/nfs_fs.h>
31
#include <linux/nfs_page.h>
32
#include <linux/module.h>
33
#include "internal.h"
R
Ricardo Labiaga 已提交
34
#include "pnfs.h"
A
Andy Adamson 已提交
35
#include "iostat.h"
36
#include "nfs4trace.h"
37
#include "delegation.h"
38
#include "nfs42.h"
R
Ricardo Labiaga 已提交
39 40

#define NFSDBG_FACILITY		NFSDBG_PNFS
41
#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
R
Ricardo Labiaga 已提交
42

43 44 45 46 47 48 49 50 51 52 53 54
/* Locking:
 *
 * pnfs_spinlock:
 *      protects pnfs_modules_tbl.
 */
static DEFINE_SPINLOCK(pnfs_spinlock);

/*
 * pnfs_modules_tbl holds all pnfs modules
 */
static LIST_HEAD(pnfs_modules_tbl);

55
static int
56
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
57
		       enum pnfs_iomode iomode, bool sync);
58

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
/* Return the registered pnfs layout driver module matching given id */
static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)
{
	struct pnfs_layoutdriver_type *local;

	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
		if (local->id == id)
			goto out;
	local = NULL;
out:
	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
	return local;
}

R
Ricardo Labiaga 已提交
74 75 76
static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)
{
77 78 79 80
	struct pnfs_layoutdriver_type *local;

	spin_lock(&pnfs_spinlock);
	local = find_pnfs_driver_locked(id);
81 82 83 84
	if (local != NULL && !try_module_get(local->owner)) {
		dprintk("%s: Could not grab reference on module\n", __func__);
		local = NULL;
	}
85 86
	spin_unlock(&pnfs_spinlock);
	return local;
R
Ricardo Labiaga 已提交
87 88 89 90 91
}

void
unset_pnfs_layoutdriver(struct nfs_server *nfss)
{
92 93 94
	if (nfss->pnfs_curr_ld) {
		if (nfss->pnfs_curr_ld->clear_layoutdriver)
			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
95 96 97
		/* Decrement the MDS count. Purge the deviceid cache if zero */
		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
			nfs4_deviceid_purge_client(nfss->nfs_client);
98
		module_put(nfss->pnfs_curr_ld->owner);
99
	}
R
Ricardo Labiaga 已提交
100 101 102 103 104 105 106 107 108 109
	nfss->pnfs_curr_ld = NULL;
}

/*
 * Try to set the server's pnfs module to the pnfs layout type specified by id.
 * Currently only one pNFS layout driver per filesystem is supported.
 *
 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
 */
void
110 111
set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
		      u32 id)
R
Ricardo Labiaga 已提交
112 113 114 115 116 117 118
{
	struct pnfs_layoutdriver_type *ld_type = NULL;

	if (id == 0)
		goto out_no_driver;
	if (!(server->nfs_client->cl_exchange_flags &
		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
119 120
		printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
			__func__, id, server->nfs_client->cl_exchange_flags);
R
Ricardo Labiaga 已提交
121 122 123 124 125 126 127 128 129 130 131 132 133
		goto out_no_driver;
	}
	ld_type = find_pnfs_driver(id);
	if (!ld_type) {
		request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
		ld_type = find_pnfs_driver(id);
		if (!ld_type) {
			dprintk("%s: No pNFS module found for %u.\n",
				__func__, id);
			goto out_no_driver;
		}
	}
	server->pnfs_curr_ld = ld_type;
134 135
	if (ld_type->set_layoutdriver
	    && ld_type->set_layoutdriver(server, mntfh)) {
136 137
		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
			"driver %u.\n", __func__, id);
138 139 140
		module_put(ld_type->owner);
		goto out_no_driver;
	}
141 142
	/* Bump the MDS count */
	atomic_inc(&server->nfs_client->cl_mds_count);
143

R
Ricardo Labiaga 已提交
144 145 146 147 148 149 150
	dprintk("%s: pNFS module for %u set\n", __func__, id);
	return;

out_no_driver:
	dprintk("%s: Using NFSv4 I/O\n", __func__);
	server->pnfs_curr_ld = NULL;
}
151 152 153 154 155 156 157 158

int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	int status = -EINVAL;
	struct pnfs_layoutdriver_type *tmp;

	if (ld_type->id == 0) {
159
		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
160 161
		return status;
	}
162
	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
163
		printk(KERN_ERR "NFS: %s Layout driver must provide "
164 165 166
		       "alloc_lseg and free_lseg.\n", __func__);
		return status;
	}
167 168 169 170 171 172 173 174 175

	spin_lock(&pnfs_spinlock);
	tmp = find_pnfs_driver_locked(ld_type->id);
	if (!tmp) {
		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
		status = 0;
		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
			ld_type->name);
	} else {
176
		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
			__func__, ld_type->id);
	}
	spin_unlock(&pnfs_spinlock);

	return status;
}
EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);

void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
	spin_lock(&pnfs_spinlock);
	list_del(&ld_type->pnfs_tblid);
	spin_unlock(&pnfs_spinlock);
}
EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
194

195 196 197 198
/*
 * pNFS client layout cache
 */

199
/* Need to hold i_lock if caller does not already hold reference */
F
Fred Isaman 已提交
200
void
201
pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
202
{
203
	atomic_inc(&lo->plh_refcount);
204 205
}

206 207 208 209
static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
210
	return ld->alloc_layout_hdr(ino, gfp_flags);
211 212 213 214 215
}

static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
216 217 218 219 220 221 222 223 224 225
	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;

	if (!list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		spin_lock(&clp->cl_lock);
		list_del_init(&lo->plh_layouts);
		spin_unlock(&clp->cl_lock);
	}
226
	put_rpccred(lo->plh_lc_cred);
227
	return ld->free_layout_hdr(lo);
228 229
}

230
static void
231
pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
232
{
233
	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
234
	dprintk("%s: freeing layout cache %p\n", __func__, lo);
235 236 237 238
	nfsi->layout = NULL;
	/* Reset MDS Threshold I/O counters */
	nfsi->write_io = 0;
	nfsi->read_io = 0;
239 240
}

241
void
242
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
243
{
244 245 246
	struct inode *inode = lo->plh_inode;

	if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
247 248
		if (!list_empty(&lo->plh_segs))
			WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
249
		pnfs_detach_layout_hdr(lo);
250
		spin_unlock(&inode->i_lock);
251
		pnfs_free_layout_hdr(lo);
252
	}
253 254
}

255 256 257 258 259 260 261 262
static int
pnfs_iomode_to_fail_bit(u32 iomode)
{
	return iomode == IOMODE_RW ?
		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
}

static void
263
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
264
{
265
	lo->plh_retry_timestamp = jiffies;
266
	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
267 268 269 270 271 272 273 274 275 276 277 278 279 280
		atomic_inc(&lo->plh_refcount);
}

static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
{
	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
		atomic_dec(&lo->plh_refcount);
}

static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
	struct inode *inode = lo->plh_inode;
281 282 283 284 285 286
	struct pnfs_layout_range range = {
		.iomode = iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(head);
287 288 289

	spin_lock(&inode->i_lock);
	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
290
	pnfs_mark_matching_lsegs_invalid(lo, &head, &range);
291
	spin_unlock(&inode->i_lock);
292
	pnfs_free_lseg_list(&head);
293 294 295 296 297 298 299
	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
			iomode == IOMODE_RW ?  "RW" : "READ");
}

static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
300
	unsigned long start, end;
301 302 303
	int fail_bit = pnfs_iomode_to_fail_bit(iomode);

	if (test_bit(fail_bit, &lo->plh_flags) == 0)
304 305 306 307 308
		return false;
	end = jiffies;
	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
		/* It is time to retry the failed layoutgets */
309
		pnfs_layout_clear_fail_bit(lo, fail_bit);
310 311 312
		return false;
	}
	return true;
313 314
}

315 316 317
static void
init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
{
318
	INIT_LIST_HEAD(&lseg->pls_list);
319
	INIT_LIST_HEAD(&lseg->pls_lc_list);
320 321 322
	atomic_set(&lseg->pls_refcount, 1);
	smp_mb();
	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
323
	lseg->pls_layout = lo;
324 325
}

326
static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
327
{
328
	struct inode *ino = lseg->pls_layout->plh_inode;
329

330
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
331 332
}

F
Fred Isaman 已提交
333
static void
334 335
pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
F
Fred Isaman 已提交
336
{
337
	struct inode *inode = lo->plh_inode;
F
Fred Isaman 已提交
338

339
	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
F
Fred Isaman 已提交
340
	list_del_init(&lseg->pls_list);
341 342
	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
	atomic_dec(&lo->plh_refcount);
343 344
	if (list_empty(&lo->plh_segs))
		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
F
Fred Isaman 已提交
345 346 347
	rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}

348 349 350
/* Return true if layoutreturn is needed */
static bool
pnfs_layout_need_return(struct pnfs_layout_hdr *lo,
351
			struct pnfs_layout_segment *lseg)
352 353 354
{
	struct pnfs_layout_segment *s;

355
	if (!test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
356 357 358
		return false;

	list_for_each_entry(s, &lo->plh_segs, pls_list)
359
		if (s != lseg && test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
360 361 362 363 364
			return false;

	return true;
}

365 366 367 368 369 370 371 372 373 374 375
static bool
pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo)
{
	if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
		return false;
	lo->plh_return_iomode = 0;
	pnfs_get_layout_hdr(lo);
	clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, &lo->plh_flags);
	return true;
}

376 377
static void pnfs_layoutreturn_before_put_lseg(struct pnfs_layout_segment *lseg,
		struct pnfs_layout_hdr *lo, struct inode *inode)
378 379 380 381 382 383 384 385
{
	lo = lseg->pls_layout;
	inode = lo->plh_inode;

	spin_lock(&inode->i_lock);
	if (pnfs_layout_need_return(lo, lseg)) {
		nfs4_stateid stateid;
		enum pnfs_iomode iomode;
386
		bool send;
387

388
		nfs4_stateid_copy(&stateid, &lo->plh_stateid);
389
		iomode = lo->plh_return_iomode;
390
		send = pnfs_prepare_layoutreturn(lo);
391
		spin_unlock(&inode->i_lock);
392 393
		if (send) {
			/* Send an async layoutreturn so we dont deadlock */
394
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
395
		}
396
	} else
397
		spin_unlock(&inode->i_lock);
398 399
}

400
void
401
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
402
{
403
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
404 405 406 407 408
	struct inode *inode;

	if (!lseg)
		return;

409 410 411
	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
412 413 414 415 416

	/* Handle the case where refcount != 1 */
	if (atomic_add_unless(&lseg->pls_refcount, -1, 1))
		return;

417 418
	lo = lseg->pls_layout;
	inode = lo->plh_inode;
419 420 421 422
	/* Do we need a layoutreturn? */
	if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
		pnfs_layoutreturn_before_put_lseg(lseg, lo, inode);

F
Fred Isaman 已提交
423
	if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
424 425 426 427
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
			spin_unlock(&inode->i_lock);
			return;
		}
428
		pnfs_get_layout_hdr(lo);
429 430 431 432
		pnfs_layout_remove_lseg(lo, lseg);
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg(lseg);
		pnfs_put_layout_hdr(lo);
433 434
	}
}
435
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
436

437
static void pnfs_free_lseg_async_work(struct work_struct *work)
438 439
{
	struct pnfs_layout_segment *lseg;
440
	struct pnfs_layout_hdr *lo;
441 442

	lseg = container_of(work, struct pnfs_layout_segment, pls_work);
443
	lo = lseg->pls_layout;
444

445 446
	pnfs_free_lseg(lseg);
	pnfs_put_layout_hdr(lo);
447 448
}

449
static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg)
450
{
451
	INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work);
452 453
	schedule_work(&lseg->pls_work);
}
454 455 456 457 458 459 460 461 462 463 464 465 466 467

void
pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
{
	if (!lseg)
		return;

	assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock);

	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
	if (atomic_dec_and_test(&lseg->pls_refcount)) {
		struct pnfs_layout_hdr *lo = lseg->pls_layout;
468 469
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags))
			return;
470 471 472 473 474 475
		pnfs_get_layout_hdr(lo);
		pnfs_layout_remove_lseg(lo, lseg);
		pnfs_free_lseg_async(lseg);
	}
}
EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
476

477
static u64
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
end_offset(u64 start, u64 len)
{
	u64 end;

	end = start + len;
	return end >= start ? end : NFS4_MAX_UINT64;
}

/*
 * is l2 fully contained in l1?
 *   start1                             end1
 *   [----------------------------------)
 *           start2           end2
 *           [----------------)
 */
493
static bool
494
pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
495
		 const struct pnfs_layout_range *l2)
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (start1 <= start2) && (end1 >= end2);
}

/*
 * is l1 and l2 intersecting?
 *   start1                             end1
 *   [----------------------------------)
 *                              start2           end2
 *                              [----------------)
 */
512
static bool
513
pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
514
		    const struct pnfs_layout_range *l2)
515 516 517 518 519 520 521 522 523 524
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
	       (end2 == NFS4_MAX_UINT64 || end2 > start1);
}

525
static bool
526 527
should_free_lseg(const struct pnfs_layout_range *lseg_range,
		 const struct pnfs_layout_range *recall_range)
528
{
529 530
	return (recall_range->iomode == IOMODE_ANY ||
		lseg_range->iomode == recall_range->iomode) &&
531
	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
532 533
}

534 535 536 537 538 539 540 541 542 543
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		struct list_head *tmp_list)
{
	if (!atomic_dec_and_test(&lseg->pls_refcount))
		return false;
	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
	list_add(&lseg->pls_list, tmp_list);
	return true;
}

544 545 546 547 548 549 550 551 552 553 554
/* Returns 1 if lseg is removed from list, 0 otherwise */
static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
			     struct list_head *tmp_list)
{
	int rv = 0;

	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
		/* Remove the reference keeping the lseg in the
		 * list.  It will now be removed when all
		 * outstanding io is finished.
		 */
F
Fred Isaman 已提交
555 556
		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
			atomic_read(&lseg->pls_refcount));
557
		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
F
Fred Isaman 已提交
558
			rv = 1;
559 560 561 562 563 564 565
	}
	return rv;
}

/* Returns count of number of matching invalid lsegs remaining in list
 * after call.
 */
F
Fred Isaman 已提交
566
int
567
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
568
			    struct list_head *tmp_list,
569
			    struct pnfs_layout_range *recall_range)
570 571
{
	struct pnfs_layout_segment *lseg, *next;
572
	int remaining = 0;
573 574 575

	dprintk("%s:Begin lo %p\n", __func__, lo);

576
	if (list_empty(&lo->plh_segs))
577
		return 0;
578
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
579 580
		if (!recall_range ||
		    should_free_lseg(&lseg->pls_range, recall_range)) {
581 582 583 584
			dprintk("%s: freeing lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode, lseg->pls_range.offset,
				lseg->pls_range.length);
585 586
			if (!mark_lseg_invalid(lseg, tmp_list))
				remaining++;
587
		}
588 589
	dprintk("%s:Return %i\n", __func__, remaining);
	return remaining;
590 591
}

592
/* note free_me must contain lsegs from a single layout_hdr */
F
Fred Isaman 已提交
593
void
594
pnfs_free_lseg_list(struct list_head *free_me)
595
{
596
	struct pnfs_layout_segment *lseg, *tmp;
597 598 599 600

	if (list_empty(free_me))
		return;

601
	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
602
		list_del(&lseg->pls_list);
603
		pnfs_free_lseg(lseg);
604 605 606
	}
}

607 608 609 610
void
pnfs_destroy_layout(struct nfs_inode *nfsi)
{
	struct pnfs_layout_hdr *lo;
611
	LIST_HEAD(tmp_list);
612 613 614 615

	spin_lock(&nfsi->vfs_inode.i_lock);
	lo = nfsi->layout;
	if (lo) {
616
		lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
617
		pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
618 619 620
		pnfs_get_layout_hdr(lo);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
621
		pnfs_clear_retry_layoutget(lo);
622 623 624 625 626
		spin_unlock(&nfsi->vfs_inode.i_lock);
		pnfs_free_lseg_list(&tmp_list);
		pnfs_put_layout_hdr(lo);
	} else
		spin_unlock(&nfsi->vfs_inode.i_lock);
627
}
628
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
629

630 631 632
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
		struct list_head *layout_list)
633 634
{
	struct pnfs_layout_hdr *lo;
635
	bool ret = false;
636

637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
		pnfs_get_layout_hdr(lo);
		list_add(&lo->plh_bulk_destroy, layout_list);
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

/* Caller must hold rcu_read_lock and clp->cl_lock */
static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
		struct nfs_server *server,
		struct list_head *layout_list)
{
	struct pnfs_layout_hdr *lo, *next;
	struct inode *inode;

	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
		inode = igrab(lo->plh_inode);
		if (inode == NULL)
			continue;
		list_del_init(&lo->plh_layouts);
		if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
			continue;
		rcu_read_unlock();
		spin_unlock(&clp->cl_lock);
		iput(inode);
		spin_lock(&clp->cl_lock);
		rcu_read_lock();
		return -EAGAIN;
	}
	return 0;
}

static int
pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
		bool is_bulk_recall)
{
	struct pnfs_layout_hdr *lo;
	struct inode *inode;
	struct pnfs_layout_range range = {
		.iomode = IOMODE_ANY,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(lseg_list);
	int ret = 0;

	while (!list_empty(layout_list)) {
		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
				plh_bulk_destroy);
		dprintk("%s freeing layout for inode %lu\n", __func__,
			lo->plh_inode->i_ino);
		inode = lo->plh_inode;
694 695 696

		pnfs_layoutcommit_inode(inode, false);

697 698 699 700 701 702 703 704 705
		spin_lock(&inode->i_lock);
		list_del_init(&lo->plh_bulk_destroy);
		lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
		if (is_bulk_recall)
			set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
		if (pnfs_mark_matching_lsegs_invalid(lo, &lseg_list, &range))
			ret = -EAGAIN;
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg_list(&lseg_list);
706 707
		/* Free all lsegs that are attached to commit buckets */
		nfs_commit_inode(inode, 0);
708 709 710 711 712 713 714 715 716 717 718 719 720
		pnfs_put_layout_hdr(lo);
		iput(inode);
	}
	return ret;
}

int
pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
		struct nfs_fsid *fsid,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);
721

722
	spin_lock(&clp->cl_lock);
723
	rcu_read_lock();
724
restart:
725
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
726 727 728 729 730 731
		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
			continue;
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
				server,
				&layout_list) != 0)
			goto restart;
732 733
	}
	rcu_read_unlock();
734 735
	spin_unlock(&clp->cl_lock);

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

int
pnfs_destroy_layouts_byclid(struct nfs_client *clp,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);

	spin_lock(&clp->cl_lock);
	rcu_read_lock();
restart:
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
					server,
					&layout_list) != 0)
			goto restart;
756
	}
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
	rcu_read_unlock();
	spin_unlock(&clp->cl_lock);

	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

/*
 * Called by the state manger to remove all layouts established under an
 * expired lease.
 */
void
pnfs_destroy_all_layouts(struct nfs_client *clp)
{
	nfs4_deviceid_mark_client_invalid(clp);
	nfs4_deviceid_purge_client(clp);

	pnfs_destroy_layouts_byclid(clp, false);
776 777
}

778 779 780 781 782 783
/*
 * Compare 2 layout stateid sequence ids, to see which is newer,
 * taking into account wraparound issues.
 */
static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
{
784
	return (s32)(s1 - s2) > 0;
785 786
}

787
/* update lo->plh_stateid with new if is more recent */
F
Fred Isaman 已提交
788 789 790
void
pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
			bool update_barrier)
791
{
792 793
	u32 oldseq, newseq, new_barrier;
	int empty = list_empty(&lo->plh_segs);
794

795 796
	oldseq = be32_to_cpu(lo->plh_stateid.seqid);
	newseq = be32_to_cpu(new->seqid);
797
	if (empty || pnfs_seqid_is_newer(newseq, oldseq)) {
798
		nfs4_stateid_copy(&lo->plh_stateid, new);
F
Fred Isaman 已提交
799
		if (update_barrier) {
800
			new_barrier = be32_to_cpu(new->seqid);
F
Fred Isaman 已提交
801 802
		} else {
			/* Because of wraparound, we want to keep the barrier
803
			 * "close" to the current seqids.
F
Fred Isaman 已提交
804
			 */
805
			new_barrier = newseq - atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
806
		}
807 808
		if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
			lo->plh_barrier = new_barrier;
F
Fred Isaman 已提交
809
	}
810 811
}

812
static bool
813 814
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid)
F
Fred Isaman 已提交
815
{
816
	u32 seqid = be32_to_cpu(stateid->seqid);
817

818 819 820 821 822
	return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
}

/* lget is set to 1 if called from inside send_layoutget call chain */
static bool
823
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
824
{
F
Fred Isaman 已提交
825
	return lo->plh_block_lgets ||
826
		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
827 828
}

829 830
int
pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
831
			      struct pnfs_layout_range *range,
832
			      struct nfs4_state *open_state)
833
{
834
	int status = 0;
835

836
	dprintk("--> %s\n", __func__);
837
	spin_lock(&lo->plh_inode->i_lock);
838
	if (pnfs_layoutgets_blocked(lo)) {
839
		status = -EAGAIN;
840 841
	} else if (!nfs4_valid_open_stateid(open_state)) {
		status = -EBADF;
842 843
	} else if (list_empty(&lo->plh_segs) ||
		   test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
844 845 846 847
		int seq;

		do {
			seq = read_seqbegin(&open_state->seqlock);
848
			nfs4_stateid_copy(dst, &open_state->stateid);
849 850
		} while (read_seqretry(&open_state->seqlock, seq));
	} else
851
		nfs4_stateid_copy(dst, &lo->plh_stateid);
852
	spin_unlock(&lo->plh_inode->i_lock);
853
	dprintk("<-- %s\n", __func__);
854
	return status;
855 856 857 858 859 860 861 862
}

/*
* Get layout from server.
*    for now, assume that whole file layouts are requested.
*    arg->offset: 0
*    arg->length: all ones
*/
863 864 865
static struct pnfs_layout_segment *
send_layoutget(struct pnfs_layout_hdr *lo,
	   struct nfs_open_context *ctx,
866
	   struct pnfs_layout_range *range,
867
	   gfp_t gfp_flags)
868
{
869
	struct inode *ino = lo->plh_inode;
870 871
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs4_layoutget *lgp;
872
	struct pnfs_layout_segment *lseg;
873
	loff_t i_size;
874 875

	dprintk("--> %s\n", __func__);
876

877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904
	/*
	 * Synchronously retrieve layout information from server and
	 * store in lseg. If we race with a concurrent seqid morphing
	 * op, then re-send the LAYOUTGET.
	 */
	do {
		lgp = kzalloc(sizeof(*lgp), gfp_flags);
		if (lgp == NULL)
			return NULL;

		i_size = i_size_read(ino);

		lgp->args.minlength = PAGE_CACHE_SIZE;
		if (lgp->args.minlength > range->length)
			lgp->args.minlength = range->length;
		if (range->iomode == IOMODE_READ) {
			if (range->offset >= i_size)
				lgp->args.minlength = 0;
			else if (i_size - range->offset < lgp->args.minlength)
				lgp->args.minlength = i_size - range->offset;
		}
		lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
		lgp->args.range = *range;
		lgp->args.type = server->pnfs_curr_ld->id;
		lgp->args.inode = ino;
		lgp->args.ctx = get_nfs_open_context(ctx);
		lgp->gfp_flags = gfp_flags;
		lgp->cred = lo->plh_lc_cred;
905

906 907
		lseg = nfs4_proc_layoutget(lgp, gfp_flags);
	} while (lseg == ERR_PTR(-EAGAIN));
908

909 910 911 912 913 914 915
	if (IS_ERR(lseg)) {
		switch (PTR_ERR(lseg)) {
		case -ENOMEM:
		case -ERESTARTSYS:
			break;
		default:
			/* remember that LAYOUTGET failed and suspend trying */
916
			pnfs_layout_io_set_failed(lo, range->iomode);
917 918
		}
		return NULL;
919 920 921
	} else
		pnfs_layout_clear_fail_bit(lo,
				pnfs_iomode_to_fail_bit(range->iomode));
922

923 924 925
	return lseg;
}

926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
static void pnfs_clear_layoutcommit(struct inode *inode,
		struct list_head *head)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct pnfs_layout_segment *lseg, *tmp;

	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		return;
	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
			continue;
		pnfs_lseg_dec_and_remove_zero(lseg, head);
	}
}

941 942 943 944 945
void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
{
	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
	smp_mb__after_atomic();
	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
946
	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
947 948
}

949
static int
950
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
951
		       enum pnfs_iomode iomode, bool sync)
952 953 954 955 956
{
	struct inode *ino = lo->plh_inode;
	struct nfs4_layoutreturn *lrp;
	int status = 0;

957
	lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
958 959 960
	if (unlikely(lrp == NULL)) {
		status = -ENOMEM;
		spin_lock(&ino->i_lock);
961
		pnfs_clear_layoutreturn_waitbit(lo);
962 963 964 965 966
		spin_unlock(&ino->i_lock);
		pnfs_put_layout_hdr(lo);
		goto out;
	}

967
	nfs4_stateid_copy(&lrp->args.stateid, stateid);
968 969
	lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
	lrp->args.inode = ino;
P
Peng Tao 已提交
970 971 972
	lrp->args.range.iomode = iomode;
	lrp->args.range.offset = 0;
	lrp->args.range.length = NFS4_MAX_UINT64;
973 974 975 976
	lrp->args.layout = lo;
	lrp->clp = NFS_SERVER(ino)->nfs_client;
	lrp->cred = lo->plh_lc_cred;

977
	status = nfs4_proc_layoutreturn(lrp, sync);
978 979 980 981 982
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

983 984 985 986 987 988 989 990
/*
 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
 * when the layout segment list is empty.
 *
 * Note that a pnfs_layout_hdr can exist with an empty layout segment
 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
 * deviceid is marked invalid.
 */
B
Benny Halevy 已提交
991 992 993 994 995 996 997
int
_pnfs_return_layout(struct inode *ino)
{
	struct pnfs_layout_hdr *lo = NULL;
	struct nfs_inode *nfsi = NFS_I(ino);
	LIST_HEAD(tmp_list);
	nfs4_stateid stateid;
998
	int status = 0, empty;
999
	bool send;
B
Benny Halevy 已提交
1000

1001
	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
B
Benny Halevy 已提交
1002 1003 1004

	spin_lock(&ino->i_lock);
	lo = nfsi->layout;
1005
	if (!lo) {
B
Benny Halevy 已提交
1006
		spin_unlock(&ino->i_lock);
1007 1008
		dprintk("NFS: %s no layout to return\n", __func__);
		goto out;
B
Benny Halevy 已提交
1009
	}
1010
	nfs4_stateid_copy(&stateid, &nfsi->layout->plh_stateid);
B
Benny Halevy 已提交
1011
	/* Reference matched in nfs4_layoutreturn_release */
1012
	pnfs_get_layout_hdr(lo);
1013
	empty = list_empty(&lo->plh_segs);
1014
	pnfs_clear_layoutcommit(ino, &tmp_list);
1015
	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025

	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
		struct pnfs_layout_range range = {
			.iomode		= IOMODE_ANY,
			.offset		= 0,
			.length		= NFS4_MAX_UINT64,
		};
		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
	}

1026 1027 1028 1029
	/* Don't send a LAYOUTRETURN if list was initially empty */
	if (empty) {
		spin_unlock(&ino->i_lock);
		dprintk("NFS: %s no layout segments to return\n", __func__);
1030
		goto out_put_layout_hdr;
1031
	}
1032 1033

	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
1034
	send = pnfs_prepare_layoutreturn(lo);
B
Benny Halevy 已提交
1035 1036
	spin_unlock(&ino->i_lock);
	pnfs_free_lseg_list(&tmp_list);
1037
	if (send)
1038
		status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1039 1040
out_put_layout_hdr:
	pnfs_put_layout_hdr(lo);
B
Benny Halevy 已提交
1041 1042 1043 1044
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}
1045
EXPORT_SYMBOL_GPL(_pnfs_return_layout);
B
Benny Halevy 已提交
1046

1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
int
pnfs_commit_and_return_layout(struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	int ret;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo == NULL) {
		spin_unlock(&inode->i_lock);
		return 0;
	}
	pnfs_get_layout_hdr(lo);
	/* Block new layoutgets and read/write to ds */
	lo->plh_block_lgets++;
	spin_unlock(&inode->i_lock);
	filemap_fdatawait(inode->i_mapping);
	ret = pnfs_layoutcommit_inode(inode, true);
	if (ret == 0)
		ret = _pnfs_return_layout(inode);
	spin_lock(&inode->i_lock);
	lo->plh_block_lgets--;
	spin_unlock(&inode->i_lock);
	pnfs_put_layout_hdr(lo);
	return ret;
}

F
Fred Isaman 已提交
1074 1075
bool pnfs_roc(struct inode *ino)
{
1076 1077 1078
	struct nfs_inode *nfsi = NFS_I(ino);
	struct nfs_open_context *ctx;
	struct nfs4_state *state;
F
Fred Isaman 已提交
1079 1080
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg, *tmp;
1081
	nfs4_stateid stateid;
F
Fred Isaman 已提交
1082
	LIST_HEAD(tmp_list);
1083
	bool found = false, layoutreturn = false, roc = false;
F
Fred Isaman 已提交
1084 1085

	spin_lock(&ino->i_lock);
1086
	lo = nfsi->layout;
P
Peng Tao 已提交
1087
	if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
1088 1089
		goto out_noroc;

1090
	/* no roc if we hold a delegation */
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
	if (nfs4_check_delegation(ino, FMODE_READ))
		goto out_noroc;

	list_for_each_entry(ctx, &nfsi->open_files, list) {
		state = ctx->state;
		/* Don't return layout if there is open file state */
		if (state != NULL && state->state != 0)
			goto out_noroc;
	}

1101
	nfs4_stateid_copy(&stateid, &lo->plh_stateid);
1102 1103 1104 1105 1106
	/* always send layoutreturn if being marked so */
	if (test_and_clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
				   &lo->plh_flags))
		layoutreturn = pnfs_prepare_layoutreturn(lo);

1107
	pnfs_clear_retry_layoutget(lo);
F
Fred Isaman 已提交
1108
	list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
1109 1110
		/* If we are sending layoutreturn, invalidate all valid lsegs */
		if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
F
Fred Isaman 已提交
1111 1112 1113
			mark_lseg_invalid(lseg, &tmp_list);
			found = true;
		}
1114
	/* ROC in two conditions:
1115 1116 1117
	 * 1. there are ROC lsegs
	 * 2. we don't send layoutreturn
	 */
1118 1119 1120
	if (found && !layoutreturn) {
		/* lo ref dropped in pnfs_roc_release() */
		pnfs_get_layout_hdr(lo);
1121
		roc = true;
1122
	}
F
Fred Isaman 已提交
1123

1124
out_noroc:
F
Fred Isaman 已提交
1125
	spin_unlock(&ino->i_lock);
1126 1127 1128
	pnfs_free_lseg_list(&tmp_list);
	pnfs_layoutcommit_inode(ino, true);
	if (layoutreturn)
1129
		pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1130
	return roc;
F
Fred Isaman 已提交
1131 1132 1133 1134 1135 1136 1137 1138
}

void pnfs_roc_release(struct inode *ino)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1139
	pnfs_clear_layoutreturn_waitbit(lo);
1140 1141 1142 1143 1144 1145
	if (atomic_dec_and_test(&lo->plh_refcount)) {
		pnfs_detach_layout_hdr(lo);
		spin_unlock(&ino->i_lock);
		pnfs_free_layout_hdr(lo);
	} else
		spin_unlock(&ino->i_lock);
F
Fred Isaman 已提交
1146 1147 1148 1149 1150 1151 1152 1153
}

void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1154
	pnfs_mark_layout_returned_if_empty(lo);
1155
	if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
F
Fred Isaman 已提交
1156 1157
		lo->plh_barrier = barrier;
	spin_unlock(&ino->i_lock);
1158
	trace_nfs4_layoutreturn_on_close(ino, 0);
F
Fred Isaman 已提交
1159 1160
}

1161
void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier)
F
Fred Isaman 已提交
1162 1163
{
	struct nfs_inode *nfsi = NFS_I(ino);
1164 1165
	struct pnfs_layout_hdr *lo;
	u32 current_seqid;
F
Fred Isaman 已提交
1166 1167

	spin_lock(&ino->i_lock);
1168 1169
	lo = nfsi->layout;
	current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
F
Fred Isaman 已提交
1170

1171 1172 1173 1174
	/* Since close does not return a layout stateid for use as
	 * a barrier, we choose the worst-case barrier.
	 */
	*barrier = current_seqid + atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
1175 1176 1177
	spin_unlock(&ino->i_lock);
}

1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
{
	struct nfs_inode *nfsi = NFS_I(ino);
        struct pnfs_layout_hdr *lo;
        bool sleep = false;

	/* we might not have grabbed lo reference. so need to check under
	 * i_lock */
        spin_lock(&ino->i_lock);
        lo = nfsi->layout;
        if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
                sleep = true;
        spin_unlock(&ino->i_lock);

        if (sleep)
                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);

        return sleep;
}

1198 1199 1200 1201 1202 1203
/*
 * Compare two layout segments for sorting into layout cache.
 * We want to preferentially return RW over RO layouts, so ensure those
 * are seen first.
 */
static s64
1204
pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1205
	   const struct pnfs_layout_range *l2)
1206
{
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
	s64 d;

	/* high offset > low offset */
	d = l1->offset - l2->offset;
	if (d)
		return d;

	/* short length > long length */
	d = l2->length - l1->length;
	if (d)
		return d;

1219
	/* read > read/write */
1220
	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1221 1222
}

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
static bool
pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
		const struct pnfs_layout_range *l2)
{
	return pnfs_lseg_range_cmp(l1, l2) > 0;
}

static bool
pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
		struct pnfs_layout_segment *old)
{
	return false;
}

void
pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   bool (*is_after)(const struct pnfs_layout_range *,
			   const struct pnfs_layout_range *),
		   bool (*do_merge)(struct pnfs_layout_segment *,
			   struct pnfs_layout_segment *),
		   struct list_head *free_me)
1245
{
1246
	struct pnfs_layout_segment *lp, *tmp;
1247

1248 1249
	dprintk("%s:Begin\n", __func__);

1250 1251 1252 1253 1254 1255 1256 1257
	list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
			continue;
		if (do_merge(lseg, lp)) {
			mark_lseg_invalid(lp, free_me);
			continue;
		}
		if (is_after(&lseg->pls_range, &lp->pls_range))
1258
			continue;
1259
		list_add_tail(&lseg->pls_list, &lp->pls_list);
1260 1261 1262
		dprintk("%s: inserted lseg %p "
			"iomode %d offset %llu length %llu before "
			"lp %p iomode %d offset %llu length %llu\n",
1263 1264 1265 1266
			__func__, lseg, lseg->pls_range.iomode,
			lseg->pls_range.offset, lseg->pls_range.length,
			lp, lp->pls_range.iomode, lp->pls_range.offset,
			lp->pls_range.length);
1267
		goto out;
1268
	}
1269 1270 1271 1272 1273 1274
	list_add_tail(&lseg->pls_list, &lo->plh_segs);
	dprintk("%s: inserted lseg %p "
		"iomode %d offset %llu length %llu at tail\n",
		__func__, lseg, lseg->pls_range.iomode,
		lseg->pls_range.offset, lseg->pls_range.length);
out:
1275
	pnfs_get_layout_hdr(lo);
1276 1277

	dprintk("%s:Return\n", __func__);
1278
}
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);

static void
pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   struct list_head *free_me)
{
	struct inode *inode = lo->plh_inode;
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;

	if (ld->add_lseg != NULL)
		ld->add_lseg(lo, lseg, free_me);
	else
		pnfs_generic_layout_insert_lseg(lo, lseg,
				pnfs_lseg_range_is_after,
				pnfs_lseg_no_merge,
				free_me);
}
1297 1298

static struct pnfs_layout_hdr *
1299 1300 1301
alloc_init_layout_hdr(struct inode *ino,
		      struct nfs_open_context *ctx,
		      gfp_t gfp_flags)
1302 1303 1304
{
	struct pnfs_layout_hdr *lo;

1305
	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1306 1307
	if (!lo)
		return NULL;
1308
	atomic_set(&lo->plh_refcount, 1);
1309 1310
	INIT_LIST_HEAD(&lo->plh_layouts);
	INIT_LIST_HEAD(&lo->plh_segs);
1311
	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1312
	lo->plh_inode = ino;
1313
	lo->plh_lc_cred = get_rpccred(ctx->cred);
1314 1315 1316 1317
	return lo;
}

static struct pnfs_layout_hdr *
1318 1319 1320
pnfs_find_alloc_layout(struct inode *ino,
		       struct nfs_open_context *ctx,
		       gfp_t gfp_flags)
1321 1322 1323 1324 1325 1326
{
	struct nfs_inode *nfsi = NFS_I(ino);
	struct pnfs_layout_hdr *new = NULL;

	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

1327 1328
	if (nfsi->layout != NULL)
		goto out_existing;
1329
	spin_unlock(&ino->i_lock);
1330
	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1331 1332
	spin_lock(&ino->i_lock);

1333
	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1334
		nfsi->layout = new;
1335
		return new;
1336 1337
	} else if (new != NULL)
		pnfs_free_layout_hdr(new);
1338 1339
out_existing:
	pnfs_get_layout_hdr(nfsi->layout);
1340 1341 1342
	return nfsi->layout;
}

1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
/*
 * iomode matching rules:
 * iomode	lseg	match
 * -----	-----	-----
 * ANY		READ	true
 * ANY		RW	true
 * RW		READ	false
 * RW		RW	true
 * READ		READ	true
 * READ		RW	true
 */
1354
static bool
1355
pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1356
		 const struct pnfs_layout_range *range)
1357
{
1358 1359 1360 1361
	struct pnfs_layout_range range1;

	if ((range->iomode == IOMODE_RW &&
	     ls_range->iomode != IOMODE_RW) ||
1362
	    !pnfs_lseg_range_intersecting(ls_range, range))
1363 1364 1365 1366 1367
		return 0;

	/* range1 covers only the first byte in the range */
	range1 = *range;
	range1.length = 1;
1368
	return pnfs_lseg_range_contained(ls_range, &range1);
1369 1370 1371 1372 1373
}

/*
 * lookup range in layout
 */
1374
static struct pnfs_layout_segment *
1375 1376
pnfs_find_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_range *range)
1377
{
1378 1379 1380 1381
	struct pnfs_layout_segment *lseg, *ret = NULL;

	dprintk("%s:Begin\n", __func__);

1382
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1383
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1384
		    !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
1385
		    pnfs_lseg_range_match(&lseg->pls_range, range)) {
1386
			ret = pnfs_get_lseg(lseg);
1387 1388 1389 1390 1391
			break;
		}
	}

	dprintk("%s:Return lseg %p ref %d\n",
1392
		__func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
1393
	return ret;
1394 1395
}

1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
/*
 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
 * to the MDS or over pNFS
 *
 * The nfs_inode read_io and write_io fields are cumulative counters reset
 * when there are no layout segments. Note that in pnfs_update_layout iomode
 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
 * WRITE request.
 *
 * A return of true means use MDS I/O.
 *
 * From rfc 5661:
 * If a file's size is smaller than the file size threshold, data accesses
 * SHOULD be sent to the metadata server.  If an I/O request has a length that
 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
 * server.  If both file size and I/O size are provided, the client SHOULD
 * reach or exceed  both thresholds before sending its read or write
 * requests to the data server.
 */
static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
				     struct inode *ino, int iomode)
{
	struct nfs4_threshold *t = ctx->mdsthreshold;
	struct nfs_inode *nfsi = NFS_I(ino);
	loff_t fsize = i_size_read(ino);
	bool size = false, size_set = false, io = false, io_set = false, ret = false;

	if (t == NULL)
		return ret;

	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);

	switch (iomode) {
	case IOMODE_READ:
		if (t->bm & THRESHOLD_RD) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->rd_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_RD_IO) {
			dprintk("%s nfsi->read_io %llu\n", __func__,
				nfsi->read_io);
			io_set = true;
			if (nfsi->read_io < t->rd_io_sz)
				io = true;
		}
		break;
	case IOMODE_RW:
		if (t->bm & THRESHOLD_WR) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->wr_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_WR_IO) {
			dprintk("%s nfsi->write_io %llu\n", __func__,
				nfsi->write_io);
			io_set = true;
			if (nfsi->write_io < t->wr_io_sz)
				io = true;
		}
		break;
	}
	if (size_set && io_set) {
		if (size && io)
			ret = true;
	} else if (size || io)
		ret = true;

	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
	return ret;
}

1471
/* stop waiting if someone clears NFS_LAYOUT_RETRY_LAYOUTGET bit. */
1472
static int pnfs_layoutget_retry_bit_wait(struct wait_bit_key *key, int mode)
1473 1474 1475
{
	if (!test_bit(NFS_LAYOUT_RETRY_LAYOUTGET, key->flags))
		return 1;
1476
	return nfs_wait_bit_killable(key, mode);
1477 1478 1479 1480
}

static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
{
1481 1482
	if (!pnfs_should_retry_layoutget(lo))
		return false;
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
	/*
	 * send layoutcommit as it can hold up layoutreturn due to lseg
	 * reference
	 */
	pnfs_layoutcommit_inode(lo->plh_inode, false);
	return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
				   pnfs_layoutget_retry_bit_wait,
				   TASK_UNINTERRUPTIBLE);
}

1493 1494 1495 1496 1497 1498 1499 1500 1501
static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
{
	unsigned long *bitlock = &lo->plh_flags;

	clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
	smp_mb__after_atomic();
	wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
}

1502 1503 1504 1505
/*
 * Layout segment is retreived from the server if not cached.
 * The appropriate layout segment is referenced and returned to the caller.
 */
1506
struct pnfs_layout_segment *
1507 1508
pnfs_update_layout(struct inode *ino,
		   struct nfs_open_context *ctx,
1509 1510
		   loff_t pos,
		   u64 count,
1511 1512
		   enum pnfs_iomode iomode,
		   gfp_t gfp_flags)
1513
{
1514 1515 1516 1517 1518
	struct pnfs_layout_range arg = {
		.iomode = iomode,
		.offset = pos,
		.length = count,
	};
1519
	unsigned pg_offset;
1520 1521
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs_client *clp = server->nfs_client;
1522 1523
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg = NULL;
1524
	bool first;
1525

1526
	if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
1527
		trace_pnfs_update_layout(ino, pos, count, iomode, NULL,
1528
				 PNFS_UPDATE_LAYOUT_NO_PNFS);
1529
		goto out;
1530
	}
1531

1532
	if (iomode == IOMODE_READ && i_size_read(ino) == 0) {
1533
		trace_pnfs_update_layout(ino, pos, count, iomode, NULL,
1534
				 PNFS_UPDATE_LAYOUT_RD_ZEROLEN);
1535
		goto out;
1536
	}
1537

1538
	if (pnfs_within_mdsthreshold(ctx, ino, iomode)) {
1539
		trace_pnfs_update_layout(ino, pos, count, iomode, NULL,
1540
				 PNFS_UPDATE_LAYOUT_MDSTHRESH);
1541
		goto out;
1542
	}
1543

1544 1545
lookup_again:
	first = false;
1546
	spin_lock(&ino->i_lock);
1547
	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
1548 1549
	if (lo == NULL) {
		spin_unlock(&ino->i_lock);
1550
		trace_pnfs_update_layout(ino, pos, count, iomode, NULL,
1551
				 PNFS_UPDATE_LAYOUT_NOMEM);
1552 1553
		goto out;
	}
1554

F
Fred Isaman 已提交
1555
	/* Do we even need to bother with this? */
1556
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
1557
		trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1558
				 PNFS_UPDATE_LAYOUT_BULK_RECALL);
F
Fred Isaman 已提交
1559
		dprintk("%s matches recall, use MDS\n", __func__);
1560 1561 1562 1563
		goto out_unlock;
	}

	/* if LAYOUTGET already failed once we don't try again */
1564
	if (pnfs_layout_io_test_failed(lo, iomode) &&
1565
	    !pnfs_should_retry_layoutget(lo)) {
1566
		trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1567
				 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL);
1568
		goto out_unlock;
1569
	}
1570

1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588
	first = list_empty(&lo->plh_segs);
	if (first) {
		/* The first layoutget for the file. Need to serialize per
		 * RFC 5661 Errata 3208.
		 */
		if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
				     &lo->plh_flags)) {
			spin_unlock(&ino->i_lock);
			wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
				    TASK_UNINTERRUPTIBLE);
			pnfs_put_layout_hdr(lo);
			goto lookup_again;
		}
	} else {
		/* Check to see if the layout for the given range
		 * already exists
		 */
		lseg = pnfs_find_lseg(lo, &arg);
1589
		if (lseg) {
1590
			trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1591
					PNFS_UPDATE_LAYOUT_FOUND_CACHED);
1592
			goto out_unlock;
1593
		}
1594
	}
1595

1596 1597 1598 1599
	/*
	 * Because we free lsegs before sending LAYOUTRETURN, we need to wait
	 * for LAYOUTRETURN even if first is true.
	 */
1600
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
1601 1602 1603
		spin_unlock(&ino->i_lock);
		dprintk("%s wait for layoutreturn\n", __func__);
		if (pnfs_prepare_to_retry_layoutget(lo)) {
1604 1605
			if (first)
				pnfs_clear_first_layoutget(lo);
1606 1607 1608 1609
			pnfs_put_layout_hdr(lo);
			dprintk("%s retrying\n", __func__);
			goto lookup_again;
		}
1610
		trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1611
				PNFS_UPDATE_LAYOUT_RETURN);
1612 1613 1614
		goto out_put_layout_hdr;
	}

1615
	if (pnfs_layoutgets_blocked(lo)) {
1616
		trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1617
				PNFS_UPDATE_LAYOUT_BLOCKED);
1618
		goto out_unlock;
1619
	}
1620
	atomic_inc(&lo->plh_outstanding);
1621
	spin_unlock(&ino->i_lock);
1622

1623
	if (list_empty(&lo->plh_layouts)) {
1624 1625 1626 1627
		/* The lo must be on the clp list if there is any
		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
		 */
		spin_lock(&clp->cl_lock);
1628 1629
		if (list_empty(&lo->plh_layouts))
			list_add_tail(&lo->plh_layouts, &server->layouts);
1630 1631
		spin_unlock(&clp->cl_lock);
	}
1632

1633 1634 1635 1636 1637
	pg_offset = arg.offset & ~PAGE_CACHE_MASK;
	if (pg_offset) {
		arg.offset -= pg_offset;
		arg.length += pg_offset;
	}
1638 1639
	if (arg.length != NFS4_MAX_UINT64)
		arg.length = PAGE_CACHE_ALIGN(arg.length);
1640

1641
	lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
1642
	pnfs_clear_retry_layoutget(lo);
1643
	atomic_dec(&lo->plh_outstanding);
1644
	trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1645
				 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
1646
out_put_layout_hdr:
1647 1648
	if (first)
		pnfs_clear_first_layoutget(lo);
1649
	pnfs_put_layout_hdr(lo);
1650
out:
1651 1652 1653 1654 1655 1656 1657 1658
	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
			"(%s, offset: %llu, length: %llu)\n",
			__func__, ino->i_sb->s_id,
			(unsigned long long)NFS_FILEID(ino),
			lseg == NULL ? "not found" : "found",
			iomode==IOMODE_RW ?  "read/write" : "read-only",
			(unsigned long long)pos,
			(unsigned long long)count);
1659 1660 1661
	return lseg;
out_unlock:
	spin_unlock(&ino->i_lock);
1662
	goto out_put_layout_hdr;
1663
}
1664
EXPORT_SYMBOL_GPL(pnfs_update_layout);
1665

1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
static bool
pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
{
	switch (range->iomode) {
	case IOMODE_READ:
	case IOMODE_RW:
		break;
	default:
		return false;
	}
	if (range->offset == NFS4_MAX_UINT64)
		return false;
	if (range->length == 0)
		return false;
	if (range->length != NFS4_MAX_UINT64 &&
	    range->length > NFS4_MAX_UINT64 - range->offset)
		return false;
	return true;
}

1686
struct pnfs_layout_segment *
1687 1688 1689 1690 1691
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
	struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
	struct nfs4_layoutget_res *res = &lgp->res;
	struct pnfs_layout_segment *lseg;
1692
	struct inode *ino = lo->plh_inode;
1693
	LIST_HEAD(free_me);
1694 1695 1696 1697
	int status = -EINVAL;

	if (!pnfs_sanity_check_layout_range(&res->range))
		goto out;
1698 1699

	/* Inject layout blob into I/O device driver */
1700
	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710
	if (!lseg || IS_ERR(lseg)) {
		if (!lseg)
			status = -ENOMEM;
		else
			status = PTR_ERR(lseg);
		dprintk("%s: Could not allocate layout: error %d\n",
		       __func__, status);
		goto out;
	}

1711 1712 1713
	init_lseg(lo, lseg);
	lseg->pls_range = res->range;

1714
	spin_lock(&ino->i_lock);
1715
	if (pnfs_layoutgets_blocked(lo)) {
F
Fred Isaman 已提交
1716 1717 1718
		dprintk("%s forget reply due to state\n", __func__);
		goto out_forget_reply;
	}
1719

1720 1721 1722 1723
	if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
		/* existing state ID, make sure the sequence number matches. */
		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
			dprintk("%s forget reply due to sequence\n", __func__);
1724
			status = -EAGAIN;
1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
			goto out_forget_reply;
		}
		pnfs_set_layout_stateid(lo, &res->stateid, false);
	} else {
		/*
		 * We got an entirely new state ID.  Mark all segments for the
		 * inode invalid, and don't bother validating the stateid
		 * sequence number.
		 */
		pnfs_mark_matching_lsegs_invalid(lo, &free_me, NULL);

		nfs4_stateid_copy(&lo->plh_stateid, &res->stateid);
		lo->plh_barrier = be32_to_cpu(res->stateid.seqid);
	}
1739

1740 1741
	clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);

1742
	pnfs_get_lseg(lseg);
1743
	pnfs_layout_insert_lseg(lo, lseg, &free_me);
1744

P
Peng Tao 已提交
1745
	if (res->return_on_close)
F
Fred Isaman 已提交
1746 1747
		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);

1748
	spin_unlock(&ino->i_lock);
1749
	pnfs_free_lseg_list(&free_me);
1750
	return lseg;
1751
out:
1752
	return ERR_PTR(status);
F
Fred Isaman 已提交
1753 1754 1755 1756 1757 1758

out_forget_reply:
	spin_unlock(&ino->i_lock);
	lseg->pls_layout = lo;
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
	goto out;
1759 1760
}

1761 1762 1763 1764 1765 1766 1767 1768 1769 1770
static void
pnfs_set_plh_return_iomode(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode)
{
	if (lo->plh_return_iomode == iomode)
		return;
	if (lo->plh_return_iomode != 0)
		iomode = IOMODE_ANY;
	lo->plh_return_iomode = iomode;
}

1771
int
1772 1773 1774 1775 1776
pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
				struct list_head *tmp_list,
				struct pnfs_layout_range *return_range)
{
	struct pnfs_layout_segment *lseg, *next;
1777
	int remaining = 0;
1778 1779 1780 1781

	dprintk("%s:Begin lo %p\n", __func__, lo);

	if (list_empty(&lo->plh_segs))
1782
		return 0;
1783

1784 1785
	assert_spin_locked(&lo->plh_inode->i_lock);

1786 1787 1788 1789 1790 1791 1792 1793
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
		if (should_free_lseg(&lseg->pls_range, return_range)) {
			dprintk("%s: marking lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode,
				lseg->pls_range.offset,
				lseg->pls_range.length);
			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
1794
			pnfs_set_plh_return_iomode(lo, return_range->iomode);
1795 1796
			if (!mark_lseg_invalid(lseg, tmp_list))
				remaining++;
1797 1798
			set_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
					&lo->plh_flags);
1799
		}
1800
	return remaining;
1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
}

void pnfs_error_mark_layout_for_return(struct inode *inode,
				       struct pnfs_layout_segment *lseg)
{
	struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
	int iomode = pnfs_iomode_to_fail_bit(lseg->pls_range.iomode);
	struct pnfs_layout_range range = {
		.iomode = lseg->pls_range.iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(free_me);
1814
	bool return_now = false;
1815 1816 1817 1818

	spin_lock(&inode->i_lock);
	/* set failure bit so that pnfs path will be retried later */
	pnfs_layout_set_fail_bit(lo, iomode);
1819
	pnfs_set_plh_return_iomode(lo, range.iomode);
1820 1821 1822 1823 1824
	/*
	 * mark all matching lsegs so that we are sure to have no live
	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
	 * for how it works.
	 */
1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
	if (!pnfs_mark_matching_lsegs_return(lo, &free_me, &range)) {
		nfs4_stateid stateid;
		enum pnfs_iomode iomode = lo->plh_return_iomode;

		nfs4_stateid_copy(&stateid, &lo->plh_stateid);
		return_now = pnfs_prepare_layoutreturn(lo);
		spin_unlock(&inode->i_lock);
		if (return_now)
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
	} else {
		spin_unlock(&inode->i_lock);
		nfs_commit_inode(inode, 0);
	}
1838 1839 1840 1841
	pnfs_free_lseg_list(&free_me);
}
EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);

1842 1843 1844
void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
P
Peng Tao 已提交
1845 1846
	u64 rd_size = req->wb_bytes;

1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
	if (pgio->pg_lseg == NULL) {
		if (pgio->pg_dreq == NULL)
			rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
		else
			rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);

		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   rd_size,
						   IOMODE_READ,
						   GFP_KERNEL);
	}
1860 1861
	/* If no lseg, fall back to read through mds */
	if (pgio->pg_lseg == NULL)
1862
		nfs_pageio_reset_read_mds(pgio);
1863

1864 1865 1866 1867
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);

void
1868 1869
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
			   struct nfs_page *req, u64 wb_size)
1870
{
1871 1872 1873 1874 1875 1876 1877
	if (pgio->pg_lseg == NULL)
		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   wb_size,
						   IOMODE_RW,
						   GFP_NOFS);
1878 1879
	/* If no lseg, fall back to write through mds */
	if (pgio->pg_lseg == NULL)
1880
		nfs_pageio_reset_write_mds(pgio);
1881 1882 1883
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);

1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
void
pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
{
	if (desc->pg_lseg) {
		pnfs_put_lseg(desc->pg_lseg);
		desc->pg_lseg = NULL;
	}
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);

1894 1895 1896 1897 1898
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
size_t
1899 1900
pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
		     struct nfs_page *prev, struct nfs_page *req)
1901
{
1902
	unsigned int size;
1903
	u64 seg_end, req_start, seg_left;
1904 1905 1906 1907

	size = nfs_generic_pg_test(pgio, prev, req);
	if (!size)
		return 0;
1908

1909
	/*
1910 1911 1912 1913 1914
	 * 'size' contains the number of bytes left in the current page (up
	 * to the original size asked for in @req->wb_bytes).
	 *
	 * Calculate how many bytes are left in the layout segment
	 * and if there are less bytes than 'size', return that instead.
1915 1916 1917 1918 1919
	 *
	 * Please also note that 'end_offset' is actually the offset of the
	 * first byte that lies outside the pnfs_layout_range. FIXME?
	 *
	 */
1920
	if (pgio->pg_lseg) {
1921 1922 1923
		seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
				     pgio->pg_lseg->pls_range.length);
		req_start = req_offset(req);
1924
		WARN_ON_ONCE(req_start >= seg_end);
1925
		/* start of request is past the last byte of this segment */
1926 1927 1928 1929 1930 1931
		if (req_start >= seg_end) {
			/* reference the new lseg */
			if (pgio->pg_ops->pg_cleanup)
				pgio->pg_ops->pg_cleanup(pgio);
			if (pgio->pg_ops->pg_init)
				pgio->pg_ops->pg_init(pgio, req);
1932
			return 0;
1933
		}
1934 1935 1936 1937 1938 1939

		/* adjust 'size' iff there are fewer bytes left in the
		 * segment than what nfs_generic_pg_test returned */
		seg_left = seg_end - req_start;
		if (seg_left < size)
			size = (unsigned int)seg_left;
1940
	}
1941

1942
	return size;
1943
}
1944
EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
1945

1946
int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
1947 1948 1949 1950
{
	struct nfs_pageio_descriptor pgio;

	/* Resend all requests through the MDS */
1951 1952
	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
			      hdr->completion_ops);
1953
	set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags);
1954
	return nfs_pageio_resend(&pgio, hdr);
1955
}
1956
EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
1957

1958
static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
1959
{
1960 1961 1962

	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
1963
	    PNFS_LAYOUTRET_ON_ERROR) {
1964
		pnfs_return_layout(hdr->inode);
1965
	}
1966
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1967
		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
1968 1969
}

1970 1971 1972
/*
 * Called by non rpc-based layout drivers
 */
1973
void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
1974
{
1975
	if (likely(!hdr->pnfs_error)) {
1976 1977
		pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
				hdr->mds_offset + hdr->res.count);
1978
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
1979 1980 1981
	}
	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
1982 1983
		pnfs_ld_handle_write_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
1984
}
1985
EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
1986

1987 1988
static void
pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
1989
		struct nfs_pgio_header *hdr)
1990
{
1991
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
1992

1993
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1994
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
1995
		nfs_pageio_reset_write_mds(desc);
1996
		mirror->pg_recoalesce = 1;
1997
	}
1998
	nfs_pgio_data_destroy(hdr);
1999
	hdr->release(hdr);
2000 2001 2002
}

static enum pnfs_try_status
2003
pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
2004 2005 2006
			const struct rpc_call_ops *call_ops,
			struct pnfs_layout_segment *lseg,
			int how)
2007
{
2008
	struct inode *inode = hdr->inode;
2009 2010 2011
	enum pnfs_try_status trypnfs;
	struct nfs_server *nfss = NFS_SERVER(inode);

2012
	hdr->mds_ops = call_ops;
2013 2014

	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
2015 2016
		inode->i_ino, hdr->args.count, hdr->args.offset, how);
	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
2017
	if (trypnfs != PNFS_NOT_ATTEMPTED)
2018 2019 2020 2021 2022
		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}

2023
static void
2024 2025
pnfs_do_write(struct nfs_pageio_descriptor *desc,
	      struct nfs_pgio_header *hdr, int how)
2026 2027 2028
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2029
	enum pnfs_try_status trypnfs;
2030

2031
	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
2032
	if (trypnfs == PNFS_NOT_ATTEMPTED)
2033
		pnfs_write_through_mds(desc, hdr);
2034 2035
}

2036 2037
static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
2038
	pnfs_put_lseg(hdr->lseg);
2039
	nfs_pgio_header_free(hdr);
2040 2041
}

2042 2043 2044
int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
{
2045
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2046

2047
	struct nfs_pgio_header *hdr;
2048 2049
	int ret;

2050 2051
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
2052
		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
2053
		return -ENOMEM;
2054
	}
2055
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
2056

2057
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2058
	ret = nfs_generic_pgio(desc, hdr);
2059
	if (!ret)
2060
		pnfs_do_write(desc, hdr, desc->pg_ioflags);
2061

2062
	return ret;
2063 2064 2065
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);

2066
int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
2067 2068 2069
{
	struct nfs_pageio_descriptor pgio;

2070
	/* Resend all requests through the MDS */
2071 2072
	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
2073
}
2074
EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
2075

2076
static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
2077
{
2078 2079
	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2080
	    PNFS_LAYOUTRET_ON_ERROR) {
2081
		pnfs_return_layout(hdr->inode);
2082
	}
2083
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2084
		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
2085 2086
}

2087 2088 2089
/*
 * Called by non rpc-based layout drivers
 */
2090
void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
2091
{
2092
	if (likely(!hdr->pnfs_error)) {
2093 2094
		__nfs4_read_done_cb(hdr);
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2095 2096 2097
	}
	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2098 2099
		pnfs_ld_handle_read_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2100 2101 2102
}
EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

2103 2104
static void
pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
2105
		struct nfs_pgio_header *hdr)
2106
{
2107
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2108

2109
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2110
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2111
		nfs_pageio_reset_read_mds(desc);
2112
		mirror->pg_recoalesce = 1;
2113
	}
2114
	nfs_pgio_data_destroy(hdr);
2115
	hdr->release(hdr);
2116 2117
}

A
Andy Adamson 已提交
2118 2119 2120
/*
 * Call the appropriate parallel I/O subsystem read function.
 */
2121
static enum pnfs_try_status
2122
pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
2123 2124
		       const struct rpc_call_ops *call_ops,
		       struct pnfs_layout_segment *lseg)
A
Andy Adamson 已提交
2125
{
2126
	struct inode *inode = hdr->inode;
A
Andy Adamson 已提交
2127 2128 2129
	struct nfs_server *nfss = NFS_SERVER(inode);
	enum pnfs_try_status trypnfs;

2130
	hdr->mds_ops = call_ops;
A
Andy Adamson 已提交
2131 2132

	dprintk("%s: Reading ino:%lu %u@%llu\n",
2133
		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
A
Andy Adamson 已提交
2134

2135
	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
2136
	if (trypnfs != PNFS_NOT_ATTEMPTED)
A
Andy Adamson 已提交
2137 2138 2139 2140
		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}
A
Andy Adamson 已提交
2141

2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
/* Resend all requests through pnfs. */
int pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
{
	struct nfs_pageio_descriptor pgio;

	nfs_pageio_init_read(&pgio, hdr->inode, false, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
}
EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);

2152
static void
2153
pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
2154 2155 2156
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2157
	enum pnfs_try_status trypnfs;
2158
	int err = 0;
2159

2160
	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
2161 2162 2163
	if (trypnfs == PNFS_TRY_AGAIN)
		err = pnfs_read_resend_pnfs(hdr);
	if (trypnfs == PNFS_NOT_ATTEMPTED || err)
2164
		pnfs_read_through_mds(desc, hdr);
2165 2166
}

2167 2168
static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
2169
	pnfs_put_lseg(hdr->lseg);
2170
	nfs_pgio_header_free(hdr);
2171 2172
}

2173 2174 2175
int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
{
2176
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2177

2178
	struct nfs_pgio_header *hdr;
2179 2180
	int ret;

2181 2182
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
2183
		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
2184
		return -ENOMEM;
2185
	}
2186
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
2187
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2188
	ret = nfs_generic_pgio(desc, hdr);
2189
	if (!ret)
2190
		pnfs_do_read(desc, hdr);
2191
	return ret;
2192 2193 2194
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);

2195 2196 2197 2198 2199
static void pnfs_clear_layoutcommitting(struct inode *inode)
{
	unsigned long *bitlock = &NFS_I(inode)->flags;

	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
2200
	smp_mb__after_atomic();
2201 2202 2203
	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}

A
Andy Adamson 已提交
2204
/*
2205
 * There can be multiple RW segments.
A
Andy Adamson 已提交
2206
 */
2207
static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
A
Andy Adamson 已提交
2208
{
2209
	struct pnfs_layout_segment *lseg;
A
Andy Adamson 已提交
2210

2211 2212
	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
		if (lseg->pls_range.iomode == IOMODE_RW &&
2213
		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
2214 2215
			list_add(&lseg->pls_lc_list, listp);
	}
A
Andy Adamson 已提交
2216 2217
}

2218 2219 2220 2221 2222 2223 2224 2225 2226 2227
static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
{
	struct pnfs_layout_segment *lseg, *tmp;

	/* Matched by references in pnfs_set_layoutcommit */
	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
		list_del_init(&lseg->pls_lc_list);
		pnfs_put_lseg(lseg);
	}

2228
	pnfs_clear_layoutcommitting(inode);
2229 2230
}

P
Peng Tao 已提交
2231 2232
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
2233
	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
P
Peng Tao 已提交
2234 2235 2236
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

A
Andy Adamson 已提交
2237
void
2238 2239
pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
		loff_t end_pos)
A
Andy Adamson 已提交
2240
{
2241
	struct nfs_inode *nfsi = NFS_I(inode);
2242
	bool mark_as_dirty = false;
A
Andy Adamson 已提交
2243

2244
	spin_lock(&inode->i_lock);
A
Andy Adamson 已提交
2245
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
2246
		nfsi->layout->plh_lwb = end_pos;
2247
		mark_as_dirty = true;
A
Andy Adamson 已提交
2248
		dprintk("%s: Set layoutcommit for inode %lu ",
2249
			__func__, inode->i_ino);
2250 2251
	} else if (end_pos > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = end_pos;
2252
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
2253
		/* references matched in nfs4_layoutcommit_release */
2254
		pnfs_get_lseg(lseg);
2255
	}
2256
	spin_unlock(&inode->i_lock);
2257
	dprintk("%s: lseg %p end_pos %llu\n",
2258
		__func__, lseg, nfsi->layout->plh_lwb);
2259 2260 2261 2262

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
2263
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2264 2265 2266
}
EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);

A
Andy Adamson 已提交
2267 2268 2269 2270 2271 2272
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
{
	struct nfs_server *nfss = NFS_SERVER(data->args.inode);

	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
2273
	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
A
Andy Adamson 已提交
2274 2275
}

2276 2277 2278 2279 2280 2281 2282 2283
/*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
 * data to disk to allow the server to recover the data if it crashes.
 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
 * is off, and a COMMIT is sent to a data server, or
 * if WRITEs to a data server return NFS_DATA_SYNC.
 */
A
Andy Adamson 已提交
2284
int
2285
pnfs_layoutcommit_inode(struct inode *inode, bool sync)
A
Andy Adamson 已提交
2286
{
2287
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
A
Andy Adamson 已提交
2288 2289 2290
	struct nfs4_layoutcommit_data *data;
	struct nfs_inode *nfsi = NFS_I(inode);
	loff_t end_pos;
2291
	int status;
A
Andy Adamson 已提交
2292

2293
	if (!pnfs_layoutcommit_outstanding(inode))
2294 2295
		return 0;

2296
	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
P
Peng Tao 已提交
2297

2298
	status = -EAGAIN;
P
Peng Tao 已提交
2299
	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2300 2301
		if (!sync)
			goto out;
2302
		status = wait_on_bit_lock_action(&nfsi->flags,
2303 2304 2305
				NFS_INO_LAYOUTCOMMITTING,
				nfs_wait_bit_killable,
				TASK_KILLABLE);
P
Peng Tao 已提交
2306
		if (status)
2307
			goto out;
P
Peng Tao 已提交
2308 2309
	}

2310 2311 2312 2313 2314 2315 2316
	status = -ENOMEM;
	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (!data)
		goto clear_layoutcommitting;

	status = 0;
2317
	spin_lock(&inode->i_lock);
2318 2319
	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		goto out_unlock;
2320

2321
	INIT_LIST_HEAD(&data->lseg_list);
2322
	pnfs_list_write_lseg(inode, &data->lseg_list);
A
Andy Adamson 已提交
2323

2324
	end_pos = nfsi->layout->plh_lwb;
A
Andy Adamson 已提交
2325

2326
	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
A
Andy Adamson 已提交
2327 2328 2329
	spin_unlock(&inode->i_lock);

	data->args.inode = inode;
2330
	data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
A
Andy Adamson 已提交
2331 2332 2333 2334 2335 2336
	nfs_fattr_init(&data->fattr);
	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
	data->res.fattr = &data->fattr;
	data->args.lastbytewritten = end_pos - 1;
	data->res.server = NFS_SERVER(inode);

2337 2338 2339
	if (ld->prepare_layoutcommit) {
		status = ld->prepare_layoutcommit(&data->args);
		if (status) {
2340
			put_rpccred(data->cred);
2341
			spin_lock(&inode->i_lock);
2342 2343
			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
			if (end_pos > nfsi->layout->plh_lwb)
2344
				nfsi->layout->plh_lwb = end_pos;
2345
			goto out_unlock;
2346 2347 2348 2349
		}
	}


A
Andy Adamson 已提交
2350 2351
	status = nfs4_proc_layoutcommit(data, sync);
out:
P
Peng Tao 已提交
2352 2353
	if (status)
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2354 2355
	dprintk("<-- %s status %d\n", __func__, status);
	return status;
2356 2357
out_unlock:
	spin_unlock(&inode->i_lock);
P
Peng Tao 已提交
2358
	kfree(data);
2359 2360
clear_layoutcommitting:
	pnfs_clear_layoutcommitting(inode);
P
Peng Tao 已提交
2361
	goto out;
A
Andy Adamson 已提交
2362
}
2363
EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
2364

2365 2366 2367 2368 2369 2370 2371
int
pnfs_generic_sync(struct inode *inode, bool datasync)
{
	return pnfs_layoutcommit_inode(inode, true);
}
EXPORT_SYMBOL_GPL(pnfs_generic_sync);

2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382
struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
{
	struct nfs4_threshold *thp;

	thp = kzalloc(sizeof(*thp), GFP_NOFS);
	if (!thp) {
		dprintk("%s mdsthreshold allocation failed\n", __func__);
		return NULL;
	}
	return thp;
}
2383

2384
#if IS_ENABLED(CONFIG_NFS_V4_2)
2385
int
2386
pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
2387 2388 2389
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
	struct nfs_server *server = NFS_SERVER(inode);
2390
	struct nfs_inode *nfsi = NFS_I(inode);
2391 2392 2393 2394 2395 2396 2397
	struct nfs42_layoutstat_data *data;
	struct pnfs_layout_hdr *hdr;
	int status = 0;

	if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
		goto out;

2398 2399 2400
	if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
		goto out;

2401 2402 2403
	if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
		goto out;

2404 2405 2406 2407 2408 2409 2410 2411 2412
	spin_lock(&inode->i_lock);
	if (!NFS_I(inode)->layout) {
		spin_unlock(&inode->i_lock);
		goto out;
	}
	hdr = NFS_I(inode)->layout;
	pnfs_get_layout_hdr(hdr);
	spin_unlock(&inode->i_lock);

2413
	data = kzalloc(sizeof(*data), gfp_flags);
2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435
	if (!data) {
		status = -ENOMEM;
		goto out_put;
	}

	data->args.fh = NFS_FH(inode);
	data->args.inode = inode;
	nfs4_stateid_copy(&data->args.stateid, &hdr->plh_stateid);
	status = ld->prepare_layoutstats(&data->args);
	if (status)
		goto out_free;

	status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);

out:
	dprintk("%s returns %d\n", __func__, status);
	return status;

out_free:
	kfree(data);
out_put:
	pnfs_put_layout_hdr(hdr);
2436 2437 2438
	smp_mb__before_atomic();
	clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
	smp_mb__after_atomic();
2439 2440 2441
	goto out;
}
EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
2442
#endif
2443 2444 2445 2446

unsigned int layoutstats_timer;
module_param(layoutstats_timer, uint, 0644);
EXPORT_SYMBOL_GPL(layoutstats_timer);