pnfs.c 64.0 KB
Newer Older
R
Ricardo Labiaga 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 *  pNFS functions to call and manage layout drivers.
 *
 *  Copyright (c) 2002 [year of first publication]
 *  The Regents of the University of Michigan
 *  All Rights Reserved
 *
 *  Dean Hildebrand <dhildebz@umich.edu>
 *
 *  Permission is granted to use, copy, create derivative works, and
 *  redistribute this software and such derivative works for any purpose,
 *  so long as the name of the University of Michigan is not used in
 *  any advertising or publicity pertaining to the use or distribution
 *  of this software without specific, written prior authorization. If
 *  the above copyright notice or any other identification of the
 *  University of Michigan is included in any copy of any portion of
 *  this software, then the disclaimer below must also be included.
 *
 *  This software is provided as is, without representation or warranty
 *  of any kind either express or implied, including without limitation
 *  the implied warranties of merchantability, fitness for a particular
 *  purpose, or noninfringement.  The Regents of the University of
 *  Michigan shall not be liable for any damages, including special,
 *  indirect, incidental, or consequential damages, with respect to any
 *  claim arising out of or in connection with the use of the software,
 *  even if it has been or is hereafter advised of the possibility of
 *  such damages.
 */

#include <linux/nfs_fs.h>
31
#include <linux/nfs_page.h>
32
#include <linux/module.h>
33
#include "internal.h"
R
Ricardo Labiaga 已提交
34
#include "pnfs.h"
A
Andy Adamson 已提交
35
#include "iostat.h"
36
#include "nfs4trace.h"
37
#include "delegation.h"
38
#include "nfs42.h"
R
Ricardo Labiaga 已提交
39 40

#define NFSDBG_FACILITY		NFSDBG_PNFS
41
#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
R
Ricardo Labiaga 已提交
42

43 44 45 46 47 48 49 50 51 52 53 54
/* Locking:
 *
 * pnfs_spinlock:
 *      protects pnfs_modules_tbl.
 */
static DEFINE_SPINLOCK(pnfs_spinlock);

/*
 * pnfs_modules_tbl holds all pnfs modules
 */
static LIST_HEAD(pnfs_modules_tbl);

55
static int
56
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
57
		       enum pnfs_iomode iomode, bool sync);
58

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
/* Return the registered pnfs layout driver module matching given id */
static struct pnfs_layoutdriver_type *
find_pnfs_driver_locked(u32 id)
{
	struct pnfs_layoutdriver_type *local;

	list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
		if (local->id == id)
			goto out;
	local = NULL;
out:
	dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
	return local;
}

R
Ricardo Labiaga 已提交
74 75 76
static struct pnfs_layoutdriver_type *
find_pnfs_driver(u32 id)
{
77 78 79 80
	struct pnfs_layoutdriver_type *local;

	spin_lock(&pnfs_spinlock);
	local = find_pnfs_driver_locked(id);
81 82 83 84
	if (local != NULL && !try_module_get(local->owner)) {
		dprintk("%s: Could not grab reference on module\n", __func__);
		local = NULL;
	}
85 86
	spin_unlock(&pnfs_spinlock);
	return local;
R
Ricardo Labiaga 已提交
87 88 89 90 91
}

void
unset_pnfs_layoutdriver(struct nfs_server *nfss)
{
92 93 94
	if (nfss->pnfs_curr_ld) {
		if (nfss->pnfs_curr_ld->clear_layoutdriver)
			nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
95 96 97
		/* Decrement the MDS count. Purge the deviceid cache if zero */
		if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
			nfs4_deviceid_purge_client(nfss->nfs_client);
98
		module_put(nfss->pnfs_curr_ld->owner);
99
	}
R
Ricardo Labiaga 已提交
100 101 102 103 104 105 106 107 108 109
	nfss->pnfs_curr_ld = NULL;
}

/*
 * Try to set the server's pnfs module to the pnfs layout type specified by id.
 * Currently only one pNFS layout driver per filesystem is supported.
 *
 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
 */
void
110 111
set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
		      u32 id)
R
Ricardo Labiaga 已提交
112 113 114 115 116 117 118
{
	struct pnfs_layoutdriver_type *ld_type = NULL;

	if (id == 0)
		goto out_no_driver;
	if (!(server->nfs_client->cl_exchange_flags &
		 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
119 120
		printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
			__func__, id, server->nfs_client->cl_exchange_flags);
R
Ricardo Labiaga 已提交
121 122 123 124 125 126 127 128 129 130 131 132 133
		goto out_no_driver;
	}
	ld_type = find_pnfs_driver(id);
	if (!ld_type) {
		request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
		ld_type = find_pnfs_driver(id);
		if (!ld_type) {
			dprintk("%s: No pNFS module found for %u.\n",
				__func__, id);
			goto out_no_driver;
		}
	}
	server->pnfs_curr_ld = ld_type;
134 135
	if (ld_type->set_layoutdriver
	    && ld_type->set_layoutdriver(server, mntfh)) {
136 137
		printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
			"driver %u.\n", __func__, id);
138 139 140
		module_put(ld_type->owner);
		goto out_no_driver;
	}
141 142
	/* Bump the MDS count */
	atomic_inc(&server->nfs_client->cl_mds_count);
143

R
Ricardo Labiaga 已提交
144 145 146 147 148 149 150
	dprintk("%s: pNFS module for %u set\n", __func__, id);
	return;

out_no_driver:
	dprintk("%s: Using NFSv4 I/O\n", __func__);
	server->pnfs_curr_ld = NULL;
}
151 152 153 154 155 156 157 158

int
pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	int status = -EINVAL;
	struct pnfs_layoutdriver_type *tmp;

	if (ld_type->id == 0) {
159
		printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
160 161
		return status;
	}
162
	if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
163
		printk(KERN_ERR "NFS: %s Layout driver must provide "
164 165 166
		       "alloc_lseg and free_lseg.\n", __func__);
		return status;
	}
167 168 169 170 171 172 173 174 175

	spin_lock(&pnfs_spinlock);
	tmp = find_pnfs_driver_locked(ld_type->id);
	if (!tmp) {
		list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
		status = 0;
		dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
			ld_type->name);
	} else {
176
		printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
			__func__, ld_type->id);
	}
	spin_unlock(&pnfs_spinlock);

	return status;
}
EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);

void
pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
{
	dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
	spin_lock(&pnfs_spinlock);
	list_del(&ld_type->pnfs_tblid);
	spin_unlock(&pnfs_spinlock);
}
EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
194

195 196 197 198
/*
 * pNFS client layout cache
 */

199
/* Need to hold i_lock if caller does not already hold reference */
F
Fred Isaman 已提交
200
void
201
pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
202
{
203
	atomic_inc(&lo->plh_refcount);
204 205
}

206 207 208 209
static struct pnfs_layout_hdr *
pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
210
	return ld->alloc_layout_hdr(ino, gfp_flags);
211 212 213 214 215
}

static void
pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
216 217 218 219 220 221 222 223 224 225
	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
	struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;

	if (!list_empty(&lo->plh_layouts)) {
		struct nfs_client *clp = server->nfs_client;

		spin_lock(&clp->cl_lock);
		list_del_init(&lo->plh_layouts);
		spin_unlock(&clp->cl_lock);
	}
226
	put_rpccred(lo->plh_lc_cred);
227
	return ld->free_layout_hdr(lo);
228 229
}

230
static void
231
pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
232
{
233
	struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
234
	dprintk("%s: freeing layout cache %p\n", __func__, lo);
235 236 237 238
	nfsi->layout = NULL;
	/* Reset MDS Threshold I/O counters */
	nfsi->write_io = 0;
	nfsi->read_io = 0;
239 240
}

241
void
242
pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
243
{
244 245 246
	struct inode *inode = lo->plh_inode;

	if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
247 248
		if (!list_empty(&lo->plh_segs))
			WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
249
		pnfs_detach_layout_hdr(lo);
250
		spin_unlock(&inode->i_lock);
251
		pnfs_free_layout_hdr(lo);
252
	}
253 254
}

255 256 257 258 259 260 261 262
static int
pnfs_iomode_to_fail_bit(u32 iomode)
{
	return iomode == IOMODE_RW ?
		NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
}

static void
263
pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
264
{
265
	lo->plh_retry_timestamp = jiffies;
266
	if (!test_and_set_bit(fail_bit, &lo->plh_flags))
267 268 269 270 271 272 273 274 275 276 277 278 279 280
		atomic_inc(&lo->plh_refcount);
}

static void
pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
{
	if (test_and_clear_bit(fail_bit, &lo->plh_flags))
		atomic_dec(&lo->plh_refcount);
}

static void
pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
	struct inode *inode = lo->plh_inode;
281 282 283 284 285 286
	struct pnfs_layout_range range = {
		.iomode = iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(head);
287 288 289

	spin_lock(&inode->i_lock);
	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
290
	pnfs_mark_matching_lsegs_invalid(lo, &head, &range);
291
	spin_unlock(&inode->i_lock);
292
	pnfs_free_lseg_list(&head);
293 294 295 296 297 298 299
	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
			iomode == IOMODE_RW ?  "RW" : "READ");
}

static bool
pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
{
300
	unsigned long start, end;
301 302 303
	int fail_bit = pnfs_iomode_to_fail_bit(iomode);

	if (test_bit(fail_bit, &lo->plh_flags) == 0)
304 305 306 307 308
		return false;
	end = jiffies;
	start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
	if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
		/* It is time to retry the failed layoutgets */
309
		pnfs_layout_clear_fail_bit(lo, fail_bit);
310 311 312
		return false;
	}
	return true;
313 314
}

315 316 317
static void
init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
{
318
	INIT_LIST_HEAD(&lseg->pls_list);
319
	INIT_LIST_HEAD(&lseg->pls_lc_list);
320 321 322
	atomic_set(&lseg->pls_refcount, 1);
	smp_mb();
	set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
323
	lseg->pls_layout = lo;
324 325
}

326
static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
327
{
328
	struct inode *ino = lseg->pls_layout->plh_inode;
329

330
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
331 332
}

F
Fred Isaman 已提交
333
static void
334 335
pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_segment *lseg)
F
Fred Isaman 已提交
336
{
337
	struct inode *inode = lo->plh_inode;
F
Fred Isaman 已提交
338

339
	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
F
Fred Isaman 已提交
340
	list_del_init(&lseg->pls_list);
341 342
	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
	atomic_dec(&lo->plh_refcount);
343 344
	if (list_empty(&lo->plh_segs))
		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
F
Fred Isaman 已提交
345 346 347
	rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
}

348 349 350
/* Return true if layoutreturn is needed */
static bool
pnfs_layout_need_return(struct pnfs_layout_hdr *lo,
351
			struct pnfs_layout_segment *lseg)
352 353 354
{
	struct pnfs_layout_segment *s;

355
	if (!test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
356 357 358
		return false;

	list_for_each_entry(s, &lo->plh_segs, pls_list)
359
		if (s != lseg && test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
360 361 362 363 364
			return false;

	return true;
}

365 366 367 368 369 370 371 372 373 374 375
static bool
pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo)
{
	if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
		return false;
	lo->plh_return_iomode = 0;
	pnfs_get_layout_hdr(lo);
	clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, &lo->plh_flags);
	return true;
}

376 377
static void pnfs_layoutreturn_before_put_lseg(struct pnfs_layout_segment *lseg,
		struct pnfs_layout_hdr *lo, struct inode *inode)
378 379 380 381 382 383 384 385
{
	lo = lseg->pls_layout;
	inode = lo->plh_inode;

	spin_lock(&inode->i_lock);
	if (pnfs_layout_need_return(lo, lseg)) {
		nfs4_stateid stateid;
		enum pnfs_iomode iomode;
386
		bool send;
387

388
		nfs4_stateid_copy(&stateid, &lo->plh_stateid);
389
		iomode = lo->plh_return_iomode;
390
		send = pnfs_prepare_layoutreturn(lo);
391
		spin_unlock(&inode->i_lock);
392 393
		if (send) {
			/* Send an async layoutreturn so we dont deadlock */
394
			pnfs_send_layoutreturn(lo, &stateid, iomode, false);
395
		}
396
	} else
397
		spin_unlock(&inode->i_lock);
398 399
}

400
void
401
pnfs_put_lseg(struct pnfs_layout_segment *lseg)
402
{
403
	struct pnfs_layout_hdr *lo;
F
Fred Isaman 已提交
404 405 406 407 408
	struct inode *inode;

	if (!lseg)
		return;

409 410 411
	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
412 413 414 415 416

	/* Handle the case where refcount != 1 */
	if (atomic_add_unless(&lseg->pls_refcount, -1, 1))
		return;

417 418
	lo = lseg->pls_layout;
	inode = lo->plh_inode;
419 420 421 422
	/* Do we need a layoutreturn? */
	if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
		pnfs_layoutreturn_before_put_lseg(lseg, lo, inode);

F
Fred Isaman 已提交
423
	if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
424 425 426 427
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
			spin_unlock(&inode->i_lock);
			return;
		}
428
		pnfs_get_layout_hdr(lo);
429 430 431 432
		pnfs_layout_remove_lseg(lo, lseg);
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg(lseg);
		pnfs_put_layout_hdr(lo);
433 434
	}
}
435
EXPORT_SYMBOL_GPL(pnfs_put_lseg);
436

437
static void pnfs_free_lseg_async_work(struct work_struct *work)
438 439
{
	struct pnfs_layout_segment *lseg;
440
	struct pnfs_layout_hdr *lo;
441 442

	lseg = container_of(work, struct pnfs_layout_segment, pls_work);
443
	lo = lseg->pls_layout;
444

445 446
	pnfs_free_lseg(lseg);
	pnfs_put_layout_hdr(lo);
447 448
}

449
static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg)
450
{
451
	INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work);
452 453
	schedule_work(&lseg->pls_work);
}
454 455 456 457 458 459 460 461 462 463 464 465 466 467

void
pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
{
	if (!lseg)
		return;

	assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock);

	dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
		atomic_read(&lseg->pls_refcount),
		test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
	if (atomic_dec_and_test(&lseg->pls_refcount)) {
		struct pnfs_layout_hdr *lo = lseg->pls_layout;
468 469
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags))
			return;
470 471 472 473 474 475
		pnfs_get_layout_hdr(lo);
		pnfs_layout_remove_lseg(lo, lseg);
		pnfs_free_lseg_async(lseg);
	}
}
EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
476

477
static u64
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
end_offset(u64 start, u64 len)
{
	u64 end;

	end = start + len;
	return end >= start ? end : NFS4_MAX_UINT64;
}

/*
 * is l2 fully contained in l1?
 *   start1                             end1
 *   [----------------------------------)
 *           start2           end2
 *           [----------------)
 */
493
static bool
494
pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
495
		 const struct pnfs_layout_range *l2)
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (start1 <= start2) && (end1 >= end2);
}

/*
 * is l1 and l2 intersecting?
 *   start1                             end1
 *   [----------------------------------)
 *                              start2           end2
 *                              [----------------)
 */
512
static bool
513
pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
514
		    const struct pnfs_layout_range *l2)
515 516 517 518 519 520 521 522 523 524
{
	u64 start1 = l1->offset;
	u64 end1 = end_offset(start1, l1->length);
	u64 start2 = l2->offset;
	u64 end2 = end_offset(start2, l2->length);

	return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
	       (end2 == NFS4_MAX_UINT64 || end2 > start1);
}

525
static bool
526 527
should_free_lseg(const struct pnfs_layout_range *lseg_range,
		 const struct pnfs_layout_range *recall_range)
528
{
529 530
	return (recall_range->iomode == IOMODE_ANY ||
		lseg_range->iomode == recall_range->iomode) &&
531
	       pnfs_lseg_range_intersecting(lseg_range, recall_range);
532 533
}

534 535 536 537 538 539 540 541 542 543
static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
		struct list_head *tmp_list)
{
	if (!atomic_dec_and_test(&lseg->pls_refcount))
		return false;
	pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
	list_add(&lseg->pls_list, tmp_list);
	return true;
}

544 545 546 547 548 549 550 551 552 553 554
/* Returns 1 if lseg is removed from list, 0 otherwise */
static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
			     struct list_head *tmp_list)
{
	int rv = 0;

	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
		/* Remove the reference keeping the lseg in the
		 * list.  It will now be removed when all
		 * outstanding io is finished.
		 */
F
Fred Isaman 已提交
555 556
		dprintk("%s: lseg %p ref %d\n", __func__, lseg,
			atomic_read(&lseg->pls_refcount));
557
		if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
F
Fred Isaman 已提交
558
			rv = 1;
559 560 561 562 563 564 565
	}
	return rv;
}

/* Returns count of number of matching invalid lsegs remaining in list
 * after call.
 */
F
Fred Isaman 已提交
566
int
567
pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
568
			    struct list_head *tmp_list,
569
			    struct pnfs_layout_range *recall_range)
570 571
{
	struct pnfs_layout_segment *lseg, *next;
572
	int invalid = 0, removed = 0;
573 574 575

	dprintk("%s:Begin lo %p\n", __func__, lo);

576
	if (list_empty(&lo->plh_segs))
577
		return 0;
578
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
579 580
		if (!recall_range ||
		    should_free_lseg(&lseg->pls_range, recall_range)) {
581 582 583 584 585 586 587 588 589
			dprintk("%s: freeing lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode, lseg->pls_range.offset,
				lseg->pls_range.length);
			invalid++;
			removed += mark_lseg_invalid(lseg, tmp_list);
		}
	dprintk("%s:Return %i\n", __func__, invalid - removed);
	return invalid - removed;
590 591
}

592
/* note free_me must contain lsegs from a single layout_hdr */
F
Fred Isaman 已提交
593
void
594
pnfs_free_lseg_list(struct list_head *free_me)
595
{
596
	struct pnfs_layout_segment *lseg, *tmp;
597 598 599 600

	if (list_empty(free_me))
		return;

601
	list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
602
		list_del(&lseg->pls_list);
603
		pnfs_free_lseg(lseg);
604 605 606
	}
}

607 608 609 610
void
pnfs_destroy_layout(struct nfs_inode *nfsi)
{
	struct pnfs_layout_hdr *lo;
611
	LIST_HEAD(tmp_list);
612 613 614 615

	spin_lock(&nfsi->vfs_inode.i_lock);
	lo = nfsi->layout;
	if (lo) {
616
		lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
617
		pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
618 619 620
		pnfs_get_layout_hdr(lo);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
		pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
621
		pnfs_clear_retry_layoutget(lo);
622 623 624 625 626
		spin_unlock(&nfsi->vfs_inode.i_lock);
		pnfs_free_lseg_list(&tmp_list);
		pnfs_put_layout_hdr(lo);
	} else
		spin_unlock(&nfsi->vfs_inode.i_lock);
627
}
628
EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
629

630 631 632
static bool
pnfs_layout_add_bulk_destroy_list(struct inode *inode,
		struct list_head *layout_list)
633 634
{
	struct pnfs_layout_hdr *lo;
635
	bool ret = false;
636

637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
		pnfs_get_layout_hdr(lo);
		list_add(&lo->plh_bulk_destroy, layout_list);
		ret = true;
	}
	spin_unlock(&inode->i_lock);
	return ret;
}

/* Caller must hold rcu_read_lock and clp->cl_lock */
static int
pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
		struct nfs_server *server,
		struct list_head *layout_list)
{
	struct pnfs_layout_hdr *lo, *next;
	struct inode *inode;

	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
		inode = igrab(lo->plh_inode);
		if (inode == NULL)
			continue;
		list_del_init(&lo->plh_layouts);
		if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
			continue;
		rcu_read_unlock();
		spin_unlock(&clp->cl_lock);
		iput(inode);
		spin_lock(&clp->cl_lock);
		rcu_read_lock();
		return -EAGAIN;
	}
	return 0;
}

static int
pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
		bool is_bulk_recall)
{
	struct pnfs_layout_hdr *lo;
	struct inode *inode;
	struct pnfs_layout_range range = {
		.iomode = IOMODE_ANY,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(lseg_list);
	int ret = 0;

	while (!list_empty(layout_list)) {
		lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
				plh_bulk_destroy);
		dprintk("%s freeing layout for inode %lu\n", __func__,
			lo->plh_inode->i_ino);
		inode = lo->plh_inode;
694 695 696

		pnfs_layoutcommit_inode(inode, false);

697 698 699 700 701 702 703 704 705
		spin_lock(&inode->i_lock);
		list_del_init(&lo->plh_bulk_destroy);
		lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
		if (is_bulk_recall)
			set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
		if (pnfs_mark_matching_lsegs_invalid(lo, &lseg_list, &range))
			ret = -EAGAIN;
		spin_unlock(&inode->i_lock);
		pnfs_free_lseg_list(&lseg_list);
706 707
		/* Free all lsegs that are attached to commit buckets */
		nfs_commit_inode(inode, 0);
708 709 710 711 712 713 714 715 716 717 718 719 720
		pnfs_put_layout_hdr(lo);
		iput(inode);
	}
	return ret;
}

int
pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
		struct nfs_fsid *fsid,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);
721

722
	spin_lock(&clp->cl_lock);
723
	rcu_read_lock();
724
restart:
725
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
726 727 728 729 730 731
		if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
			continue;
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
				server,
				&layout_list) != 0)
			goto restart;
732 733
	}
	rcu_read_unlock();
734 735
	spin_unlock(&clp->cl_lock);

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

int
pnfs_destroy_layouts_byclid(struct nfs_client *clp,
		bool is_recall)
{
	struct nfs_server *server;
	LIST_HEAD(layout_list);

	spin_lock(&clp->cl_lock);
	rcu_read_lock();
restart:
	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
		if (pnfs_layout_bulk_destroy_byserver_locked(clp,
					server,
					&layout_list) != 0)
			goto restart;
756
	}
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
	rcu_read_unlock();
	spin_unlock(&clp->cl_lock);

	if (list_empty(&layout_list))
		return 0;
	return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
}

/*
 * Called by the state manger to remove all layouts established under an
 * expired lease.
 */
void
pnfs_destroy_all_layouts(struct nfs_client *clp)
{
	nfs4_deviceid_mark_client_invalid(clp);
	nfs4_deviceid_purge_client(clp);

	pnfs_destroy_layouts_byclid(clp, false);
776 777
}

778 779 780 781 782 783
/*
 * Compare 2 layout stateid sequence ids, to see which is newer,
 * taking into account wraparound issues.
 */
static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
{
784
	return (s32)(s1 - s2) > 0;
785 786
}

787
/* update lo->plh_stateid with new if is more recent */
F
Fred Isaman 已提交
788 789 790
void
pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
			bool update_barrier)
791
{
792 793
	u32 oldseq, newseq, new_barrier;
	int empty = list_empty(&lo->plh_segs);
794

795 796
	oldseq = be32_to_cpu(lo->plh_stateid.seqid);
	newseq = be32_to_cpu(new->seqid);
797
	if (empty || pnfs_seqid_is_newer(newseq, oldseq)) {
798
		nfs4_stateid_copy(&lo->plh_stateid, new);
F
Fred Isaman 已提交
799
		if (update_barrier) {
800
			new_barrier = be32_to_cpu(new->seqid);
F
Fred Isaman 已提交
801 802
		} else {
			/* Because of wraparound, we want to keep the barrier
803
			 * "close" to the current seqids.
F
Fred Isaman 已提交
804
			 */
805
			new_barrier = newseq - atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
806
		}
807 808
		if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
			lo->plh_barrier = new_barrier;
F
Fred Isaman 已提交
809
	}
810 811
}

812
static bool
813 814
pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
		const nfs4_stateid *stateid)
F
Fred Isaman 已提交
815
{
816
	u32 seqid = be32_to_cpu(stateid->seqid);
817

818 819 820 821 822
	return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
}

/* lget is set to 1 if called from inside send_layoutget call chain */
static bool
823
pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
824
{
F
Fred Isaman 已提交
825
	return lo->plh_block_lgets ||
826
		test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
827 828
}

829 830
int
pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
831
			      struct pnfs_layout_range *range,
832
			      struct nfs4_state *open_state)
833
{
834
	int status = 0;
835

836
	dprintk("--> %s\n", __func__);
837
	spin_lock(&lo->plh_inode->i_lock);
838
	if (pnfs_layoutgets_blocked(lo)) {
839
		status = -EAGAIN;
840 841
	} else if (!nfs4_valid_open_stateid(open_state)) {
		status = -EBADF;
842 843
	} else if (list_empty(&lo->plh_segs) ||
		   test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
844 845 846 847
		int seq;

		do {
			seq = read_seqbegin(&open_state->seqlock);
848
			nfs4_stateid_copy(dst, &open_state->stateid);
849 850
		} while (read_seqretry(&open_state->seqlock, seq));
	} else
851
		nfs4_stateid_copy(dst, &lo->plh_stateid);
852
	spin_unlock(&lo->plh_inode->i_lock);
853
	dprintk("<-- %s\n", __func__);
854
	return status;
855 856 857 858 859 860 861 862
}

/*
* Get layout from server.
*    for now, assume that whole file layouts are requested.
*    arg->offset: 0
*    arg->length: all ones
*/
863 864 865
static struct pnfs_layout_segment *
send_layoutget(struct pnfs_layout_hdr *lo,
	   struct nfs_open_context *ctx,
866
	   struct pnfs_layout_range *range,
867
	   gfp_t gfp_flags)
868
{
869
	struct inode *ino = lo->plh_inode;
870 871
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs4_layoutget *lgp;
872
	struct pnfs_layout_segment *lseg;
873
	loff_t i_size;
874 875

	dprintk("--> %s\n", __func__);
876

877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904
	/*
	 * Synchronously retrieve layout information from server and
	 * store in lseg. If we race with a concurrent seqid morphing
	 * op, then re-send the LAYOUTGET.
	 */
	do {
		lgp = kzalloc(sizeof(*lgp), gfp_flags);
		if (lgp == NULL)
			return NULL;

		i_size = i_size_read(ino);

		lgp->args.minlength = PAGE_CACHE_SIZE;
		if (lgp->args.minlength > range->length)
			lgp->args.minlength = range->length;
		if (range->iomode == IOMODE_READ) {
			if (range->offset >= i_size)
				lgp->args.minlength = 0;
			else if (i_size - range->offset < lgp->args.minlength)
				lgp->args.minlength = i_size - range->offset;
		}
		lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
		lgp->args.range = *range;
		lgp->args.type = server->pnfs_curr_ld->id;
		lgp->args.inode = ino;
		lgp->args.ctx = get_nfs_open_context(ctx);
		lgp->gfp_flags = gfp_flags;
		lgp->cred = lo->plh_lc_cred;
905

906 907
		lseg = nfs4_proc_layoutget(lgp, gfp_flags);
	} while (lseg == ERR_PTR(-EAGAIN));
908

909 910 911 912 913 914 915
	if (IS_ERR(lseg)) {
		switch (PTR_ERR(lseg)) {
		case -ENOMEM:
		case -ERESTARTSYS:
			break;
		default:
			/* remember that LAYOUTGET failed and suspend trying */
916
			pnfs_layout_io_set_failed(lo, range->iomode);
917 918
		}
		return NULL;
919 920 921
	} else
		pnfs_layout_clear_fail_bit(lo,
				pnfs_iomode_to_fail_bit(range->iomode));
922

923 924 925
	return lseg;
}

926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
static void pnfs_clear_layoutcommit(struct inode *inode,
		struct list_head *head)
{
	struct nfs_inode *nfsi = NFS_I(inode);
	struct pnfs_layout_segment *lseg, *tmp;

	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		return;
	list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
		if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
			continue;
		pnfs_lseg_dec_and_remove_zero(lseg, head);
	}
}

941 942 943 944 945
void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
{
	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
	smp_mb__after_atomic();
	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
946
	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
947 948
}

949
static int
950
pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
951
		       enum pnfs_iomode iomode, bool sync)
952 953 954 955 956
{
	struct inode *ino = lo->plh_inode;
	struct nfs4_layoutreturn *lrp;
	int status = 0;

957
	lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
958 959 960
	if (unlikely(lrp == NULL)) {
		status = -ENOMEM;
		spin_lock(&ino->i_lock);
961
		pnfs_clear_layoutreturn_waitbit(lo);
962 963 964 965 966
		spin_unlock(&ino->i_lock);
		pnfs_put_layout_hdr(lo);
		goto out;
	}

967
	nfs4_stateid_copy(&lrp->args.stateid, stateid);
968 969
	lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
	lrp->args.inode = ino;
P
Peng Tao 已提交
970 971 972
	lrp->args.range.iomode = iomode;
	lrp->args.range.offset = 0;
	lrp->args.range.length = NFS4_MAX_UINT64;
973 974 975 976
	lrp->args.layout = lo;
	lrp->clp = NFS_SERVER(ino)->nfs_client;
	lrp->cred = lo->plh_lc_cred;

977
	status = nfs4_proc_layoutreturn(lrp, sync);
978 979 980 981 982
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}

983 984 985 986 987 988 989 990
/*
 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
 * when the layout segment list is empty.
 *
 * Note that a pnfs_layout_hdr can exist with an empty layout segment
 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
 * deviceid is marked invalid.
 */
B
Benny Halevy 已提交
991 992 993 994 995 996 997
int
_pnfs_return_layout(struct inode *ino)
{
	struct pnfs_layout_hdr *lo = NULL;
	struct nfs_inode *nfsi = NFS_I(ino);
	LIST_HEAD(tmp_list);
	nfs4_stateid stateid;
998
	int status = 0, empty;
999
	bool send;
B
Benny Halevy 已提交
1000

1001
	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
B
Benny Halevy 已提交
1002 1003 1004

	spin_lock(&ino->i_lock);
	lo = nfsi->layout;
1005
	if (!lo) {
B
Benny Halevy 已提交
1006
		spin_unlock(&ino->i_lock);
1007 1008
		dprintk("NFS: %s no layout to return\n", __func__);
		goto out;
B
Benny Halevy 已提交
1009
	}
1010
	nfs4_stateid_copy(&stateid, &nfsi->layout->plh_stateid);
B
Benny Halevy 已提交
1011
	/* Reference matched in nfs4_layoutreturn_release */
1012
	pnfs_get_layout_hdr(lo);
1013
	empty = list_empty(&lo->plh_segs);
1014
	pnfs_clear_layoutcommit(ino, &tmp_list);
1015
	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025

	if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
		struct pnfs_layout_range range = {
			.iomode		= IOMODE_ANY,
			.offset		= 0,
			.length		= NFS4_MAX_UINT64,
		};
		NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
	}

1026 1027 1028 1029
	/* Don't send a LAYOUTRETURN if list was initially empty */
	if (empty) {
		spin_unlock(&ino->i_lock);
		dprintk("NFS: %s no layout segments to return\n", __func__);
1030
		goto out_put_layout_hdr;
1031
	}
1032 1033

	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
1034
	send = pnfs_prepare_layoutreturn(lo);
B
Benny Halevy 已提交
1035 1036
	spin_unlock(&ino->i_lock);
	pnfs_free_lseg_list(&tmp_list);
1037
	if (send)
1038
		status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1039 1040
out_put_layout_hdr:
	pnfs_put_layout_hdr(lo);
B
Benny Halevy 已提交
1041 1042 1043 1044
out:
	dprintk("<-- %s status: %d\n", __func__, status);
	return status;
}
1045
EXPORT_SYMBOL_GPL(_pnfs_return_layout);
B
Benny Halevy 已提交
1046

1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
int
pnfs_commit_and_return_layout(struct inode *inode)
{
	struct pnfs_layout_hdr *lo;
	int ret;

	spin_lock(&inode->i_lock);
	lo = NFS_I(inode)->layout;
	if (lo == NULL) {
		spin_unlock(&inode->i_lock);
		return 0;
	}
	pnfs_get_layout_hdr(lo);
	/* Block new layoutgets and read/write to ds */
	lo->plh_block_lgets++;
	spin_unlock(&inode->i_lock);
	filemap_fdatawait(inode->i_mapping);
	ret = pnfs_layoutcommit_inode(inode, true);
	if (ret == 0)
		ret = _pnfs_return_layout(inode);
	spin_lock(&inode->i_lock);
	lo->plh_block_lgets--;
	spin_unlock(&inode->i_lock);
	pnfs_put_layout_hdr(lo);
	return ret;
}

F
Fred Isaman 已提交
1074 1075
bool pnfs_roc(struct inode *ino)
{
1076 1077 1078
	struct nfs_inode *nfsi = NFS_I(ino);
	struct nfs_open_context *ctx;
	struct nfs4_state *state;
F
Fred Isaman 已提交
1079 1080
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg, *tmp;
1081
	nfs4_stateid stateid;
F
Fred Isaman 已提交
1082
	LIST_HEAD(tmp_list);
1083
	bool found = false, layoutreturn = false, roc = false;
F
Fred Isaman 已提交
1084 1085

	spin_lock(&ino->i_lock);
1086
	lo = nfsi->layout;
P
Peng Tao 已提交
1087
	if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
1088 1089
		goto out_noroc;

1090
	/* no roc if we hold a delegation */
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
	if (nfs4_check_delegation(ino, FMODE_READ))
		goto out_noroc;

	list_for_each_entry(ctx, &nfsi->open_files, list) {
		state = ctx->state;
		/* Don't return layout if there is open file state */
		if (state != NULL && state->state != 0)
			goto out_noroc;
	}

1101
	nfs4_stateid_copy(&stateid, &lo->plh_stateid);
1102 1103 1104 1105 1106
	/* always send layoutreturn if being marked so */
	if (test_and_clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
				   &lo->plh_flags))
		layoutreturn = pnfs_prepare_layoutreturn(lo);

1107
	pnfs_clear_retry_layoutget(lo);
F
Fred Isaman 已提交
1108
	list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
1109 1110
		/* If we are sending layoutreturn, invalidate all valid lsegs */
		if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
F
Fred Isaman 已提交
1111 1112 1113
			mark_lseg_invalid(lseg, &tmp_list);
			found = true;
		}
1114
	/* ROC in two conditions:
1115 1116 1117
	 * 1. there are ROC lsegs
	 * 2. we don't send layoutreturn
	 */
1118 1119 1120
	if (found && !layoutreturn) {
		/* lo ref dropped in pnfs_roc_release() */
		pnfs_get_layout_hdr(lo);
1121
		roc = true;
1122
	}
F
Fred Isaman 已提交
1123

1124
out_noroc:
F
Fred Isaman 已提交
1125
	spin_unlock(&ino->i_lock);
1126 1127 1128
	pnfs_free_lseg_list(&tmp_list);
	pnfs_layoutcommit_inode(ino, true);
	if (layoutreturn)
1129
		pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
1130
	return roc;
F
Fred Isaman 已提交
1131 1132 1133 1134 1135 1136 1137 1138
}

void pnfs_roc_release(struct inode *ino)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1139
	pnfs_clear_layoutreturn_waitbit(lo);
1140 1141 1142 1143 1144 1145
	if (atomic_dec_and_test(&lo->plh_refcount)) {
		pnfs_detach_layout_hdr(lo);
		spin_unlock(&ino->i_lock);
		pnfs_free_layout_hdr(lo);
	} else
		spin_unlock(&ino->i_lock);
F
Fred Isaman 已提交
1146 1147 1148 1149 1150 1151 1152 1153
}

void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
{
	struct pnfs_layout_hdr *lo;

	spin_lock(&ino->i_lock);
	lo = NFS_I(ino)->layout;
1154
	pnfs_mark_layout_returned_if_empty(lo);
1155
	if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
F
Fred Isaman 已提交
1156 1157
		lo->plh_barrier = barrier;
	spin_unlock(&ino->i_lock);
1158
	trace_nfs4_layoutreturn_on_close(ino, 0);
F
Fred Isaman 已提交
1159 1160
}

1161
void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier)
F
Fred Isaman 已提交
1162 1163
{
	struct nfs_inode *nfsi = NFS_I(ino);
1164 1165
	struct pnfs_layout_hdr *lo;
	u32 current_seqid;
F
Fred Isaman 已提交
1166 1167

	spin_lock(&ino->i_lock);
1168 1169
	lo = nfsi->layout;
	current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
F
Fred Isaman 已提交
1170

1171 1172 1173 1174
	/* Since close does not return a layout stateid for use as
	 * a barrier, we choose the worst-case barrier.
	 */
	*barrier = current_seqid + atomic_read(&lo->plh_outstanding);
F
Fred Isaman 已提交
1175 1176 1177
	spin_unlock(&ino->i_lock);
}

1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
{
	struct nfs_inode *nfsi = NFS_I(ino);
        struct pnfs_layout_hdr *lo;
        bool sleep = false;

	/* we might not have grabbed lo reference. so need to check under
	 * i_lock */
        spin_lock(&ino->i_lock);
        lo = nfsi->layout;
        if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
                sleep = true;
        spin_unlock(&ino->i_lock);

        if (sleep)
                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);

        return sleep;
}

1198 1199 1200 1201 1202 1203
/*
 * Compare two layout segments for sorting into layout cache.
 * We want to preferentially return RW over RO layouts, so ensure those
 * are seen first.
 */
static s64
1204
pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
1205
	   const struct pnfs_layout_range *l2)
1206
{
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
	s64 d;

	/* high offset > low offset */
	d = l1->offset - l2->offset;
	if (d)
		return d;

	/* short length > long length */
	d = l2->length - l1->length;
	if (d)
		return d;

1219
	/* read > read/write */
1220
	return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
1221 1222
}

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
static bool
pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
		const struct pnfs_layout_range *l2)
{
	return pnfs_lseg_range_cmp(l1, l2) > 0;
}

static bool
pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
		struct pnfs_layout_segment *old)
{
	return false;
}

void
pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   bool (*is_after)(const struct pnfs_layout_range *,
			   const struct pnfs_layout_range *),
		   bool (*do_merge)(struct pnfs_layout_segment *,
			   struct pnfs_layout_segment *),
		   struct list_head *free_me)
1245
{
1246
	struct pnfs_layout_segment *lp, *tmp;
1247

1248 1249
	dprintk("%s:Begin\n", __func__);

1250 1251 1252 1253 1254 1255 1256 1257
	list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
		if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
			continue;
		if (do_merge(lseg, lp)) {
			mark_lseg_invalid(lp, free_me);
			continue;
		}
		if (is_after(&lseg->pls_range, &lp->pls_range))
1258
			continue;
1259
		list_add_tail(&lseg->pls_list, &lp->pls_list);
1260 1261 1262
		dprintk("%s: inserted lseg %p "
			"iomode %d offset %llu length %llu before "
			"lp %p iomode %d offset %llu length %llu\n",
1263 1264 1265 1266
			__func__, lseg, lseg->pls_range.iomode,
			lseg->pls_range.offset, lseg->pls_range.length,
			lp, lp->pls_range.iomode, lp->pls_range.offset,
			lp->pls_range.length);
1267
		goto out;
1268
	}
1269 1270 1271 1272 1273 1274
	list_add_tail(&lseg->pls_list, &lo->plh_segs);
	dprintk("%s: inserted lseg %p "
		"iomode %d offset %llu length %llu at tail\n",
		__func__, lseg, lseg->pls_range.iomode,
		lseg->pls_range.offset, lseg->pls_range.length);
out:
1275
	pnfs_get_layout_hdr(lo);
1276 1277

	dprintk("%s:Return\n", __func__);
1278
}
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);

static void
pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
		   struct pnfs_layout_segment *lseg,
		   struct list_head *free_me)
{
	struct inode *inode = lo->plh_inode;
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;

	if (ld->add_lseg != NULL)
		ld->add_lseg(lo, lseg, free_me);
	else
		pnfs_generic_layout_insert_lseg(lo, lseg,
				pnfs_lseg_range_is_after,
				pnfs_lseg_no_merge,
				free_me);
}
1297 1298

static struct pnfs_layout_hdr *
1299 1300 1301
alloc_init_layout_hdr(struct inode *ino,
		      struct nfs_open_context *ctx,
		      gfp_t gfp_flags)
1302 1303 1304
{
	struct pnfs_layout_hdr *lo;

1305
	lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
1306 1307
	if (!lo)
		return NULL;
1308
	atomic_set(&lo->plh_refcount, 1);
1309 1310
	INIT_LIST_HEAD(&lo->plh_layouts);
	INIT_LIST_HEAD(&lo->plh_segs);
1311
	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
1312
	lo->plh_inode = ino;
1313
	lo->plh_lc_cred = get_rpccred(ctx->cred);
1314 1315 1316 1317
	return lo;
}

static struct pnfs_layout_hdr *
1318 1319 1320
pnfs_find_alloc_layout(struct inode *ino,
		       struct nfs_open_context *ctx,
		       gfp_t gfp_flags)
1321 1322 1323 1324 1325 1326
{
	struct nfs_inode *nfsi = NFS_I(ino);
	struct pnfs_layout_hdr *new = NULL;

	dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);

1327 1328
	if (nfsi->layout != NULL)
		goto out_existing;
1329
	spin_unlock(&ino->i_lock);
1330
	new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
1331 1332
	spin_lock(&ino->i_lock);

1333
	if (likely(nfsi->layout == NULL)) {	/* Won the race? */
1334
		nfsi->layout = new;
1335
		return new;
1336 1337
	} else if (new != NULL)
		pnfs_free_layout_hdr(new);
1338 1339
out_existing:
	pnfs_get_layout_hdr(nfsi->layout);
1340 1341 1342
	return nfsi->layout;
}

1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
/*
 * iomode matching rules:
 * iomode	lseg	match
 * -----	-----	-----
 * ANY		READ	true
 * ANY		RW	true
 * RW		READ	false
 * RW		RW	true
 * READ		READ	true
 * READ		RW	true
 */
1354
static bool
1355
pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
1356
		 const struct pnfs_layout_range *range)
1357
{
1358 1359 1360 1361
	struct pnfs_layout_range range1;

	if ((range->iomode == IOMODE_RW &&
	     ls_range->iomode != IOMODE_RW) ||
1362
	    !pnfs_lseg_range_intersecting(ls_range, range))
1363 1364 1365 1366 1367
		return 0;

	/* range1 covers only the first byte in the range */
	range1 = *range;
	range1.length = 1;
1368
	return pnfs_lseg_range_contained(ls_range, &range1);
1369 1370 1371 1372 1373
}

/*
 * lookup range in layout
 */
1374
static struct pnfs_layout_segment *
1375 1376
pnfs_find_lseg(struct pnfs_layout_hdr *lo,
		struct pnfs_layout_range *range)
1377
{
1378 1379 1380 1381
	struct pnfs_layout_segment *lseg, *ret = NULL;

	dprintk("%s:Begin\n", __func__);

1382
	list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
1383
		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
1384
		    !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
1385
		    pnfs_lseg_range_match(&lseg->pls_range, range)) {
1386
			ret = pnfs_get_lseg(lseg);
1387 1388 1389 1390 1391
			break;
		}
	}

	dprintk("%s:Return lseg %p ref %d\n",
1392
		__func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
1393
	return ret;
1394 1395
}

1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
/*
 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
 * to the MDS or over pNFS
 *
 * The nfs_inode read_io and write_io fields are cumulative counters reset
 * when there are no layout segments. Note that in pnfs_update_layout iomode
 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
 * WRITE request.
 *
 * A return of true means use MDS I/O.
 *
 * From rfc 5661:
 * If a file's size is smaller than the file size threshold, data accesses
 * SHOULD be sent to the metadata server.  If an I/O request has a length that
 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
 * server.  If both file size and I/O size are provided, the client SHOULD
 * reach or exceed  both thresholds before sending its read or write
 * requests to the data server.
 */
static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
				     struct inode *ino, int iomode)
{
	struct nfs4_threshold *t = ctx->mdsthreshold;
	struct nfs_inode *nfsi = NFS_I(ino);
	loff_t fsize = i_size_read(ino);
	bool size = false, size_set = false, io = false, io_set = false, ret = false;

	if (t == NULL)
		return ret;

	dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
		__func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);

	switch (iomode) {
	case IOMODE_READ:
		if (t->bm & THRESHOLD_RD) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->rd_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_RD_IO) {
			dprintk("%s nfsi->read_io %llu\n", __func__,
				nfsi->read_io);
			io_set = true;
			if (nfsi->read_io < t->rd_io_sz)
				io = true;
		}
		break;
	case IOMODE_RW:
		if (t->bm & THRESHOLD_WR) {
			dprintk("%s fsize %llu\n", __func__, fsize);
			size_set = true;
			if (fsize < t->wr_sz)
				size = true;
		}
		if (t->bm & THRESHOLD_WR_IO) {
			dprintk("%s nfsi->write_io %llu\n", __func__,
				nfsi->write_io);
			io_set = true;
			if (nfsi->write_io < t->wr_io_sz)
				io = true;
		}
		break;
	}
	if (size_set && io_set) {
		if (size && io)
			ret = true;
	} else if (size || io)
		ret = true;

	dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
	return ret;
}

1471
/* stop waiting if someone clears NFS_LAYOUT_RETRY_LAYOUTGET bit. */
1472
static int pnfs_layoutget_retry_bit_wait(struct wait_bit_key *key, int mode)
1473 1474 1475
{
	if (!test_bit(NFS_LAYOUT_RETRY_LAYOUTGET, key->flags))
		return 1;
1476
	return nfs_wait_bit_killable(key, mode);
1477 1478 1479 1480
}

static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
{
1481 1482
	if (!pnfs_should_retry_layoutget(lo))
		return false;
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
	/*
	 * send layoutcommit as it can hold up layoutreturn due to lseg
	 * reference
	 */
	pnfs_layoutcommit_inode(lo->plh_inode, false);
	return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
				   pnfs_layoutget_retry_bit_wait,
				   TASK_UNINTERRUPTIBLE);
}

1493 1494 1495 1496 1497 1498 1499 1500 1501
static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
{
	unsigned long *bitlock = &lo->plh_flags;

	clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
	smp_mb__after_atomic();
	wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
}

1502 1503 1504 1505
/*
 * Layout segment is retreived from the server if not cached.
 * The appropriate layout segment is referenced and returned to the caller.
 */
1506
struct pnfs_layout_segment *
1507 1508
pnfs_update_layout(struct inode *ino,
		   struct nfs_open_context *ctx,
1509 1510
		   loff_t pos,
		   u64 count,
1511 1512
		   enum pnfs_iomode iomode,
		   gfp_t gfp_flags)
1513
{
1514 1515 1516 1517 1518
	struct pnfs_layout_range arg = {
		.iomode = iomode,
		.offset = pos,
		.length = count,
	};
1519
	unsigned pg_offset;
1520 1521
	struct nfs_server *server = NFS_SERVER(ino);
	struct nfs_client *clp = server->nfs_client;
1522 1523
	struct pnfs_layout_hdr *lo;
	struct pnfs_layout_segment *lseg = NULL;
1524
	bool first;
1525

1526
	if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
1527
		trace_pnfs_update_layout(ino, pos, count, iomode, NULL,
1528
				 PNFS_UPDATE_LAYOUT_NO_PNFS);
1529
		goto out;
1530
	}
1531

1532
	if (iomode == IOMODE_READ && i_size_read(ino) == 0) {
1533
		trace_pnfs_update_layout(ino, pos, count, iomode, NULL,
1534
				 PNFS_UPDATE_LAYOUT_RD_ZEROLEN);
1535
		goto out;
1536
	}
1537

1538
	if (pnfs_within_mdsthreshold(ctx, ino, iomode)) {
1539
		trace_pnfs_update_layout(ino, pos, count, iomode, NULL,
1540
				 PNFS_UPDATE_LAYOUT_MDSTHRESH);
1541
		goto out;
1542
	}
1543

1544 1545
lookup_again:
	first = false;
1546
	spin_lock(&ino->i_lock);
1547
	lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
1548 1549
	if (lo == NULL) {
		spin_unlock(&ino->i_lock);
1550
		trace_pnfs_update_layout(ino, pos, count, iomode, NULL,
1551
				 PNFS_UPDATE_LAYOUT_NOMEM);
1552 1553
		goto out;
	}
1554

F
Fred Isaman 已提交
1555
	/* Do we even need to bother with this? */
1556
	if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
1557
		trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1558
				 PNFS_UPDATE_LAYOUT_BULK_RECALL);
F
Fred Isaman 已提交
1559
		dprintk("%s matches recall, use MDS\n", __func__);
1560 1561 1562 1563
		goto out_unlock;
	}

	/* if LAYOUTGET already failed once we don't try again */
1564
	if (pnfs_layout_io_test_failed(lo, iomode) &&
1565
	    !pnfs_should_retry_layoutget(lo)) {
1566
		trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1567
				 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL);
1568
		goto out_unlock;
1569
	}
1570

1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588
	first = list_empty(&lo->plh_segs);
	if (first) {
		/* The first layoutget for the file. Need to serialize per
		 * RFC 5661 Errata 3208.
		 */
		if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
				     &lo->plh_flags)) {
			spin_unlock(&ino->i_lock);
			wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
				    TASK_UNINTERRUPTIBLE);
			pnfs_put_layout_hdr(lo);
			goto lookup_again;
		}
	} else {
		/* Check to see if the layout for the given range
		 * already exists
		 */
		lseg = pnfs_find_lseg(lo, &arg);
1589
		if (lseg) {
1590
			trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1591
					PNFS_UPDATE_LAYOUT_FOUND_CACHED);
1592
			goto out_unlock;
1593
		}
1594
	}
1595

1596 1597 1598 1599
	/*
	 * Because we free lsegs before sending LAYOUTRETURN, we need to wait
	 * for LAYOUTRETURN even if first is true.
	 */
1600
	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
1601 1602 1603
		spin_unlock(&ino->i_lock);
		dprintk("%s wait for layoutreturn\n", __func__);
		if (pnfs_prepare_to_retry_layoutget(lo)) {
1604 1605
			if (first)
				pnfs_clear_first_layoutget(lo);
1606 1607 1608 1609
			pnfs_put_layout_hdr(lo);
			dprintk("%s retrying\n", __func__);
			goto lookup_again;
		}
1610
		trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1611
				PNFS_UPDATE_LAYOUT_RETURN);
1612 1613 1614
		goto out_put_layout_hdr;
	}

1615
	if (pnfs_layoutgets_blocked(lo)) {
1616
		trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1617
				PNFS_UPDATE_LAYOUT_BLOCKED);
1618
		goto out_unlock;
1619
	}
1620
	atomic_inc(&lo->plh_outstanding);
1621
	spin_unlock(&ino->i_lock);
1622

1623
	if (list_empty(&lo->plh_layouts)) {
1624 1625 1626 1627
		/* The lo must be on the clp list if there is any
		 * chance of a CB_LAYOUTRECALL(FILE) coming in.
		 */
		spin_lock(&clp->cl_lock);
1628 1629
		if (list_empty(&lo->plh_layouts))
			list_add_tail(&lo->plh_layouts, &server->layouts);
1630 1631
		spin_unlock(&clp->cl_lock);
	}
1632

1633 1634 1635 1636 1637
	pg_offset = arg.offset & ~PAGE_CACHE_MASK;
	if (pg_offset) {
		arg.offset -= pg_offset;
		arg.length += pg_offset;
	}
1638 1639
	if (arg.length != NFS4_MAX_UINT64)
		arg.length = PAGE_CACHE_ALIGN(arg.length);
1640

1641
	lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
1642
	pnfs_clear_retry_layoutget(lo);
1643
	atomic_dec(&lo->plh_outstanding);
1644
	trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1645
				 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
1646
out_put_layout_hdr:
1647 1648
	if (first)
		pnfs_clear_first_layoutget(lo);
1649
	pnfs_put_layout_hdr(lo);
1650
out:
1651 1652 1653 1654 1655 1656 1657 1658
	dprintk("%s: inode %s/%llu pNFS layout segment %s for "
			"(%s, offset: %llu, length: %llu)\n",
			__func__, ino->i_sb->s_id,
			(unsigned long long)NFS_FILEID(ino),
			lseg == NULL ? "not found" : "found",
			iomode==IOMODE_RW ?  "read/write" : "read-only",
			(unsigned long long)pos,
			(unsigned long long)count);
1659 1660 1661
	return lseg;
out_unlock:
	spin_unlock(&ino->i_lock);
1662
	goto out_put_layout_hdr;
1663
}
1664
EXPORT_SYMBOL_GPL(pnfs_update_layout);
1665

1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
static bool
pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
{
	switch (range->iomode) {
	case IOMODE_READ:
	case IOMODE_RW:
		break;
	default:
		return false;
	}
	if (range->offset == NFS4_MAX_UINT64)
		return false;
	if (range->length == 0)
		return false;
	if (range->length != NFS4_MAX_UINT64 &&
	    range->length > NFS4_MAX_UINT64 - range->offset)
		return false;
	return true;
}

1686
struct pnfs_layout_segment *
1687 1688 1689 1690 1691
pnfs_layout_process(struct nfs4_layoutget *lgp)
{
	struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
	struct nfs4_layoutget_res *res = &lgp->res;
	struct pnfs_layout_segment *lseg;
1692
	struct inode *ino = lo->plh_inode;
1693
	LIST_HEAD(free_me);
1694 1695 1696 1697
	int status = -EINVAL;

	if (!pnfs_sanity_check_layout_range(&res->range))
		goto out;
1698 1699

	/* Inject layout blob into I/O device driver */
1700
	lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710
	if (!lseg || IS_ERR(lseg)) {
		if (!lseg)
			status = -ENOMEM;
		else
			status = PTR_ERR(lseg);
		dprintk("%s: Could not allocate layout: error %d\n",
		       __func__, status);
		goto out;
	}

1711 1712 1713
	init_lseg(lo, lseg);
	lseg->pls_range = res->range;

1714
	spin_lock(&ino->i_lock);
1715
	if (pnfs_layoutgets_blocked(lo)) {
F
Fred Isaman 已提交
1716 1717 1718
		dprintk("%s forget reply due to state\n", __func__);
		goto out_forget_reply;
	}
1719

1720 1721 1722 1723
	if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
		/* existing state ID, make sure the sequence number matches. */
		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
			dprintk("%s forget reply due to sequence\n", __func__);
1724
			status = -EAGAIN;
1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738
			goto out_forget_reply;
		}
		pnfs_set_layout_stateid(lo, &res->stateid, false);
	} else {
		/*
		 * We got an entirely new state ID.  Mark all segments for the
		 * inode invalid, and don't bother validating the stateid
		 * sequence number.
		 */
		pnfs_mark_matching_lsegs_invalid(lo, &free_me, NULL);

		nfs4_stateid_copy(&lo->plh_stateid, &res->stateid);
		lo->plh_barrier = be32_to_cpu(res->stateid.seqid);
	}
1739

1740 1741
	clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);

1742
	pnfs_get_lseg(lseg);
1743
	pnfs_layout_insert_lseg(lo, lseg, &free_me);
1744

P
Peng Tao 已提交
1745
	if (res->return_on_close)
F
Fred Isaman 已提交
1746 1747
		set_bit(NFS_LSEG_ROC, &lseg->pls_flags);

1748
	spin_unlock(&ino->i_lock);
1749
	pnfs_free_lseg_list(&free_me);
1750
	return lseg;
1751
out:
1752
	return ERR_PTR(status);
F
Fred Isaman 已提交
1753 1754 1755 1756 1757 1758

out_forget_reply:
	spin_unlock(&ino->i_lock);
	lseg->pls_layout = lo;
	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
	goto out;
1759 1760
}

1761 1762 1763 1764 1765 1766 1767 1768 1769 1770
static void
pnfs_set_plh_return_iomode(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode)
{
	if (lo->plh_return_iomode == iomode)
		return;
	if (lo->plh_return_iomode != 0)
		iomode = IOMODE_ANY;
	lo->plh_return_iomode = iomode;
}

1771
void
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782
pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
				struct list_head *tmp_list,
				struct pnfs_layout_range *return_range)
{
	struct pnfs_layout_segment *lseg, *next;

	dprintk("%s:Begin lo %p\n", __func__, lo);

	if (list_empty(&lo->plh_segs))
		return;

1783 1784
	assert_spin_locked(&lo->plh_inode->i_lock);

1785 1786 1787 1788 1789 1790 1791 1792
	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
		if (should_free_lseg(&lseg->pls_range, return_range)) {
			dprintk("%s: marking lseg %p iomode %d "
				"offset %llu length %llu\n", __func__,
				lseg, lseg->pls_range.iomode,
				lseg->pls_range.offset,
				lseg->pls_range.length);
			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
1793
			pnfs_set_plh_return_iomode(lo, return_range->iomode);
1794
			mark_lseg_invalid(lseg, tmp_list);
1795 1796
			set_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
					&lo->plh_flags);
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814
		}
}

void pnfs_error_mark_layout_for_return(struct inode *inode,
				       struct pnfs_layout_segment *lseg)
{
	struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
	int iomode = pnfs_iomode_to_fail_bit(lseg->pls_range.iomode);
	struct pnfs_layout_range range = {
		.iomode = lseg->pls_range.iomode,
		.offset = 0,
		.length = NFS4_MAX_UINT64,
	};
	LIST_HEAD(free_me);

	spin_lock(&inode->i_lock);
	/* set failure bit so that pnfs path will be retried later */
	pnfs_layout_set_fail_bit(lo, iomode);
1815
	pnfs_set_plh_return_iomode(lo, range.iomode);
1816 1817 1818 1819 1820 1821 1822 1823
	/*
	 * mark all matching lsegs so that we are sure to have no live
	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
	 * for how it works.
	 */
	pnfs_mark_matching_lsegs_return(lo, &free_me, &range);
	spin_unlock(&inode->i_lock);
	pnfs_free_lseg_list(&free_me);
1824
	nfs_commit_inode(inode, 0);
1825 1826 1827
}
EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);

1828 1829 1830
void
pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
P
Peng Tao 已提交
1831 1832
	u64 rd_size = req->wb_bytes;

1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
	if (pgio->pg_lseg == NULL) {
		if (pgio->pg_dreq == NULL)
			rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
		else
			rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);

		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   rd_size,
						   IOMODE_READ,
						   GFP_KERNEL);
	}
1846 1847
	/* If no lseg, fall back to read through mds */
	if (pgio->pg_lseg == NULL)
1848
		nfs_pageio_reset_read_mds(pgio);
1849

1850 1851 1852 1853
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);

void
1854 1855
pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
			   struct nfs_page *req, u64 wb_size)
1856
{
1857 1858 1859 1860 1861 1862 1863
	if (pgio->pg_lseg == NULL)
		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
						   req->wb_context,
						   req_offset(req),
						   wb_size,
						   IOMODE_RW,
						   GFP_NOFS);
1864 1865
	/* If no lseg, fall back to write through mds */
	if (pgio->pg_lseg == NULL)
1866
		nfs_pageio_reset_write_mds(pgio);
1867 1868 1869
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);

1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
void
pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
{
	if (desc->pg_lseg) {
		pnfs_put_lseg(desc->pg_lseg);
		desc->pg_lseg = NULL;
	}
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);

1880 1881 1882 1883 1884
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
size_t
1885 1886
pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
		     struct nfs_page *prev, struct nfs_page *req)
1887
{
1888
	unsigned int size;
1889
	u64 seg_end, req_start, seg_left;
1890 1891 1892 1893

	size = nfs_generic_pg_test(pgio, prev, req);
	if (!size)
		return 0;
1894

1895
	/*
1896 1897 1898 1899 1900
	 * 'size' contains the number of bytes left in the current page (up
	 * to the original size asked for in @req->wb_bytes).
	 *
	 * Calculate how many bytes are left in the layout segment
	 * and if there are less bytes than 'size', return that instead.
1901 1902 1903 1904 1905
	 *
	 * Please also note that 'end_offset' is actually the offset of the
	 * first byte that lies outside the pnfs_layout_range. FIXME?
	 *
	 */
1906
	if (pgio->pg_lseg) {
1907 1908 1909
		seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
				     pgio->pg_lseg->pls_range.length);
		req_start = req_offset(req);
1910
		WARN_ON_ONCE(req_start >= seg_end);
1911
		/* start of request is past the last byte of this segment */
1912 1913 1914 1915 1916 1917
		if (req_start >= seg_end) {
			/* reference the new lseg */
			if (pgio->pg_ops->pg_cleanup)
				pgio->pg_ops->pg_cleanup(pgio);
			if (pgio->pg_ops->pg_init)
				pgio->pg_ops->pg_init(pgio, req);
1918
			return 0;
1919
		}
1920 1921 1922 1923 1924 1925

		/* adjust 'size' iff there are fewer bytes left in the
		 * segment than what nfs_generic_pg_test returned */
		seg_left = seg_end - req_start;
		if (seg_left < size)
			size = (unsigned int)seg_left;
1926
	}
1927

1928
	return size;
1929
}
1930
EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
1931

1932
int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
1933 1934 1935 1936
{
	struct nfs_pageio_descriptor pgio;

	/* Resend all requests through the MDS */
1937 1938
	nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
			      hdr->completion_ops);
1939
	set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags);
1940
	return nfs_pageio_resend(&pgio, hdr);
1941
}
1942
EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
1943

1944
static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
1945
{
1946 1947 1948

	dprintk("pnfs write error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
1949
	    PNFS_LAYOUTRET_ON_ERROR) {
1950
		pnfs_return_layout(hdr->inode);
1951
	}
1952
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1953
		hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
1954 1955
}

1956 1957 1958
/*
 * Called by non rpc-based layout drivers
 */
1959
void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
1960
{
1961
	if (likely(!hdr->pnfs_error)) {
1962 1963
		pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
				hdr->mds_offset + hdr->res.count);
1964
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
1965 1966 1967
	}
	trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
1968 1969
		pnfs_ld_handle_write_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
1970
}
1971
EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
1972

1973 1974
static void
pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
1975
		struct nfs_pgio_header *hdr)
1976
{
1977
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
1978

1979
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1980
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
1981
		nfs_pageio_reset_write_mds(desc);
1982
		mirror->pg_recoalesce = 1;
1983
	}
1984
	nfs_pgio_data_destroy(hdr);
1985
	hdr->release(hdr);
1986 1987 1988
}

static enum pnfs_try_status
1989
pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
1990 1991 1992
			const struct rpc_call_ops *call_ops,
			struct pnfs_layout_segment *lseg,
			int how)
1993
{
1994
	struct inode *inode = hdr->inode;
1995 1996 1997
	enum pnfs_try_status trypnfs;
	struct nfs_server *nfss = NFS_SERVER(inode);

1998
	hdr->mds_ops = call_ops;
1999 2000

	dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
2001 2002
		inode->i_ino, hdr->args.count, hdr->args.offset, how);
	trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
2003
	if (trypnfs != PNFS_NOT_ATTEMPTED)
2004 2005 2006 2007 2008
		nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}

2009
static void
2010 2011
pnfs_do_write(struct nfs_pageio_descriptor *desc,
	      struct nfs_pgio_header *hdr, int how)
2012 2013 2014
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2015
	enum pnfs_try_status trypnfs;
2016

2017
	trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
2018
	if (trypnfs == PNFS_NOT_ATTEMPTED)
2019
		pnfs_write_through_mds(desc, hdr);
2020 2021
}

2022 2023
static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
{
2024
	pnfs_put_lseg(hdr->lseg);
2025
	nfs_pgio_header_free(hdr);
2026 2027
}

2028 2029 2030
int
pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
{
2031
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2032

2033
	struct nfs_pgio_header *hdr;
2034 2035
	int ret;

2036 2037
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
2038
		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
2039
		return -ENOMEM;
2040
	}
2041
	nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
2042

2043
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2044
	ret = nfs_generic_pgio(desc, hdr);
2045
	if (!ret)
2046
		pnfs_do_write(desc, hdr, desc->pg_ioflags);
2047

2048
	return ret;
2049 2050 2051
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);

2052
int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
2053 2054 2055
{
	struct nfs_pageio_descriptor pgio;

2056
	/* Resend all requests through the MDS */
2057 2058
	nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
2059
}
2060
EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
2061

2062
static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
2063
{
2064 2065
	dprintk("pnfs read error = %d\n", hdr->pnfs_error);
	if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
2066
	    PNFS_LAYOUTRET_ON_ERROR) {
2067
		pnfs_return_layout(hdr->inode);
2068
	}
2069
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
2070
		hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
2071 2072
}

2073 2074 2075
/*
 * Called by non rpc-based layout drivers
 */
2076
void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
2077
{
2078
	if (likely(!hdr->pnfs_error)) {
2079 2080
		__nfs4_read_done_cb(hdr);
		hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
2081 2082 2083
	}
	trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
	if (unlikely(hdr->pnfs_error))
2084 2085
		pnfs_ld_handle_read_error(hdr);
	hdr->mds_ops->rpc_release(hdr);
2086 2087 2088
}
EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

2089 2090
static void
pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
2091
		struct nfs_pgio_header *hdr)
2092
{
2093
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2094

2095
	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
2096
		list_splice_tail_init(&hdr->pages, &mirror->pg_list);
2097
		nfs_pageio_reset_read_mds(desc);
2098
		mirror->pg_recoalesce = 1;
2099
	}
2100
	nfs_pgio_data_destroy(hdr);
2101
	hdr->release(hdr);
2102 2103
}

A
Andy Adamson 已提交
2104 2105 2106
/*
 * Call the appropriate parallel I/O subsystem read function.
 */
2107
static enum pnfs_try_status
2108
pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
2109 2110
		       const struct rpc_call_ops *call_ops,
		       struct pnfs_layout_segment *lseg)
A
Andy Adamson 已提交
2111
{
2112
	struct inode *inode = hdr->inode;
A
Andy Adamson 已提交
2113 2114 2115
	struct nfs_server *nfss = NFS_SERVER(inode);
	enum pnfs_try_status trypnfs;

2116
	hdr->mds_ops = call_ops;
A
Andy Adamson 已提交
2117 2118

	dprintk("%s: Reading ino:%lu %u@%llu\n",
2119
		__func__, inode->i_ino, hdr->args.count, hdr->args.offset);
A
Andy Adamson 已提交
2120

2121
	trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
2122
	if (trypnfs != PNFS_NOT_ATTEMPTED)
A
Andy Adamson 已提交
2123 2124 2125 2126
		nfs_inc_stats(inode, NFSIOS_PNFS_READ);
	dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
	return trypnfs;
}
A
Andy Adamson 已提交
2127

2128 2129 2130 2131 2132 2133 2134 2135 2136 2137
/* Resend all requests through pnfs. */
int pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
{
	struct nfs_pageio_descriptor pgio;

	nfs_pageio_init_read(&pgio, hdr->inode, false, hdr->completion_ops);
	return nfs_pageio_resend(&pgio, hdr);
}
EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);

2138
static void
2139
pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
2140 2141 2142
{
	const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
	struct pnfs_layout_segment *lseg = desc->pg_lseg;
2143
	enum pnfs_try_status trypnfs;
2144
	int err = 0;
2145

2146
	trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
2147 2148 2149
	if (trypnfs == PNFS_TRY_AGAIN)
		err = pnfs_read_resend_pnfs(hdr);
	if (trypnfs == PNFS_NOT_ATTEMPTED || err)
2150
		pnfs_read_through_mds(desc, hdr);
2151 2152
}

2153 2154
static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
{
2155
	pnfs_put_lseg(hdr->lseg);
2156
	nfs_pgio_header_free(hdr);
2157 2158
}

2159 2160 2161
int
pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
{
2162
	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
2163

2164
	struct nfs_pgio_header *hdr;
2165 2166
	int ret;

2167 2168
	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
	if (!hdr) {
2169
		desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
2170
		return -ENOMEM;
2171
	}
2172
	nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
2173
	hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
2174
	ret = nfs_generic_pgio(desc, hdr);
2175
	if (!ret)
2176
		pnfs_do_read(desc, hdr);
2177
	return ret;
2178 2179 2180
}
EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);

2181 2182 2183 2184 2185
static void pnfs_clear_layoutcommitting(struct inode *inode)
{
	unsigned long *bitlock = &NFS_I(inode)->flags;

	clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
2186
	smp_mb__after_atomic();
2187 2188 2189
	wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
}

A
Andy Adamson 已提交
2190
/*
2191
 * There can be multiple RW segments.
A
Andy Adamson 已提交
2192
 */
2193
static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
A
Andy Adamson 已提交
2194
{
2195
	struct pnfs_layout_segment *lseg;
A
Andy Adamson 已提交
2196

2197 2198
	list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
		if (lseg->pls_range.iomode == IOMODE_RW &&
2199
		    test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
2200 2201
			list_add(&lseg->pls_lc_list, listp);
	}
A
Andy Adamson 已提交
2202 2203
}

2204 2205 2206 2207 2208 2209 2210 2211 2212 2213
static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
{
	struct pnfs_layout_segment *lseg, *tmp;

	/* Matched by references in pnfs_set_layoutcommit */
	list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
		list_del_init(&lseg->pls_lc_list);
		pnfs_put_lseg(lseg);
	}

2214
	pnfs_clear_layoutcommitting(inode);
2215 2216
}

P
Peng Tao 已提交
2217 2218
void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
{
2219
	pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
P
Peng Tao 已提交
2220 2221 2222
}
EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);

A
Andy Adamson 已提交
2223
void
2224 2225
pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
		loff_t end_pos)
A
Andy Adamson 已提交
2226
{
2227
	struct nfs_inode *nfsi = NFS_I(inode);
2228
	bool mark_as_dirty = false;
A
Andy Adamson 已提交
2229

2230
	spin_lock(&inode->i_lock);
A
Andy Adamson 已提交
2231
	if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
2232
		nfsi->layout->plh_lwb = end_pos;
2233
		mark_as_dirty = true;
A
Andy Adamson 已提交
2234
		dprintk("%s: Set layoutcommit for inode %lu ",
2235
			__func__, inode->i_ino);
2236 2237
	} else if (end_pos > nfsi->layout->plh_lwb)
		nfsi->layout->plh_lwb = end_pos;
2238
	if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
2239
		/* references matched in nfs4_layoutcommit_release */
2240
		pnfs_get_lseg(lseg);
2241
	}
2242
	spin_unlock(&inode->i_lock);
2243
	dprintk("%s: lseg %p end_pos %llu\n",
2244
		__func__, lseg, nfsi->layout->plh_lwb);
2245 2246 2247 2248

	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
	if (mark_as_dirty)
2249
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2250 2251 2252
}
EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);

A
Andy Adamson 已提交
2253 2254 2255 2256 2257 2258
void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
{
	struct nfs_server *nfss = NFS_SERVER(data->args.inode);

	if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
		nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
2259
	pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
A
Andy Adamson 已提交
2260 2261
}

2262 2263 2264 2265 2266 2267 2268 2269
/*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
 * data to disk to allow the server to recover the data if it crashes.
 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
 * is off, and a COMMIT is sent to a data server, or
 * if WRITEs to a data server return NFS_DATA_SYNC.
 */
A
Andy Adamson 已提交
2270
int
2271
pnfs_layoutcommit_inode(struct inode *inode, bool sync)
A
Andy Adamson 已提交
2272
{
2273
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
A
Andy Adamson 已提交
2274 2275 2276
	struct nfs4_layoutcommit_data *data;
	struct nfs_inode *nfsi = NFS_I(inode);
	loff_t end_pos;
2277
	int status;
A
Andy Adamson 已提交
2278

2279
	if (!pnfs_layoutcommit_outstanding(inode))
2280 2281
		return 0;

2282
	dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
P
Peng Tao 已提交
2283

2284
	status = -EAGAIN;
P
Peng Tao 已提交
2285
	if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2286 2287
		if (!sync)
			goto out;
2288
		status = wait_on_bit_lock_action(&nfsi->flags,
2289 2290 2291
				NFS_INO_LAYOUTCOMMITTING,
				nfs_wait_bit_killable,
				TASK_KILLABLE);
P
Peng Tao 已提交
2292
		if (status)
2293
			goto out;
P
Peng Tao 已提交
2294 2295
	}

2296 2297 2298 2299 2300 2301 2302
	status = -ENOMEM;
	/* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
	data = kzalloc(sizeof(*data), GFP_NOFS);
	if (!data)
		goto clear_layoutcommitting;

	status = 0;
2303
	spin_lock(&inode->i_lock);
2304 2305
	if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
		goto out_unlock;
2306

2307
	INIT_LIST_HEAD(&data->lseg_list);
2308
	pnfs_list_write_lseg(inode, &data->lseg_list);
A
Andy Adamson 已提交
2309

2310
	end_pos = nfsi->layout->plh_lwb;
A
Andy Adamson 已提交
2311

2312
	nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
A
Andy Adamson 已提交
2313 2314 2315
	spin_unlock(&inode->i_lock);

	data->args.inode = inode;
2316
	data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
A
Andy Adamson 已提交
2317 2318 2319 2320 2321 2322
	nfs_fattr_init(&data->fattr);
	data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
	data->res.fattr = &data->fattr;
	data->args.lastbytewritten = end_pos - 1;
	data->res.server = NFS_SERVER(inode);

2323 2324 2325
	if (ld->prepare_layoutcommit) {
		status = ld->prepare_layoutcommit(&data->args);
		if (status) {
2326
			put_rpccred(data->cred);
2327
			spin_lock(&inode->i_lock);
2328 2329
			set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
			if (end_pos > nfsi->layout->plh_lwb)
2330
				nfsi->layout->plh_lwb = end_pos;
2331
			goto out_unlock;
2332 2333 2334 2335
		}
	}


A
Andy Adamson 已提交
2336 2337
	status = nfs4_proc_layoutcommit(data, sync);
out:
P
Peng Tao 已提交
2338 2339
	if (status)
		mark_inode_dirty_sync(inode);
A
Andy Adamson 已提交
2340 2341
	dprintk("<-- %s status %d\n", __func__, status);
	return status;
2342 2343
out_unlock:
	spin_unlock(&inode->i_lock);
P
Peng Tao 已提交
2344
	kfree(data);
2345 2346
clear_layoutcommitting:
	pnfs_clear_layoutcommitting(inode);
P
Peng Tao 已提交
2347
	goto out;
A
Andy Adamson 已提交
2348
}
2349
EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
2350

2351 2352 2353 2354 2355 2356 2357
int
pnfs_generic_sync(struct inode *inode, bool datasync)
{
	return pnfs_layoutcommit_inode(inode, true);
}
EXPORT_SYMBOL_GPL(pnfs_generic_sync);

2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368
struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
{
	struct nfs4_threshold *thp;

	thp = kzalloc(sizeof(*thp), GFP_NOFS);
	if (!thp) {
		dprintk("%s mdsthreshold allocation failed\n", __func__);
		return NULL;
	}
	return thp;
}
2369

2370
#if IS_ENABLED(CONFIG_NFS_V4_2)
2371
int
2372
pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
2373 2374 2375
{
	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
	struct nfs_server *server = NFS_SERVER(inode);
2376
	struct nfs_inode *nfsi = NFS_I(inode);
2377 2378 2379 2380 2381 2382 2383
	struct nfs42_layoutstat_data *data;
	struct pnfs_layout_hdr *hdr;
	int status = 0;

	if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
		goto out;

2384 2385 2386
	if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
		goto out;

2387 2388 2389
	if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
		goto out;

2390 2391 2392 2393 2394 2395 2396 2397 2398
	spin_lock(&inode->i_lock);
	if (!NFS_I(inode)->layout) {
		spin_unlock(&inode->i_lock);
		goto out;
	}
	hdr = NFS_I(inode)->layout;
	pnfs_get_layout_hdr(hdr);
	spin_unlock(&inode->i_lock);

2399
	data = kzalloc(sizeof(*data), gfp_flags);
2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421
	if (!data) {
		status = -ENOMEM;
		goto out_put;
	}

	data->args.fh = NFS_FH(inode);
	data->args.inode = inode;
	nfs4_stateid_copy(&data->args.stateid, &hdr->plh_stateid);
	status = ld->prepare_layoutstats(&data->args);
	if (status)
		goto out_free;

	status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);

out:
	dprintk("%s returns %d\n", __func__, status);
	return status;

out_free:
	kfree(data);
out_put:
	pnfs_put_layout_hdr(hdr);
2422 2423 2424
	smp_mb__before_atomic();
	clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
	smp_mb__after_atomic();
2425 2426 2427
	goto out;
}
EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
2428
#endif
2429 2430 2431 2432

unsigned int layoutstats_timer;
module_param(layoutstats_timer, uint, 0644);
EXPORT_SYMBOL_GPL(layoutstats_timer);