blocklayout.c 37.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 *  linux/fs/nfs/blocklayout/blocklayout.c
 *
 *  Module for the NFSv4.1 pNFS block layout driver.
 *
 *  Copyright (c) 2006 The Regents of the University of Michigan.
 *  All rights reserved.
 *
 *  Andy Adamson <andros@citi.umich.edu>
 *  Fred Isaman <iisaman@umich.edu>
 *
 * permission is granted to use, copy, create derivative works and
 * redistribute this software and such derivative works for any purpose,
 * so long as the name of the university of michigan is not used in
 * any advertising or publicity pertaining to the use or distribution
 * of this software without specific, written prior authorization.  if
 * the above copyright notice or any other identification of the
 * university of michigan is included in any copy of any portion of
 * this software, then the disclaimer below must also be included.
 *
 * this software is provided as is, without representation from the
 * university of michigan as to its fitness for any purpose, and without
 * warranty by the university of michigan of any kind, either express
 * or implied, including without limitation the implied warranties of
 * merchantability and fitness for a particular purpose.  the regents
 * of the university of michigan shall not be liable for any damages,
 * including special, indirect, incidental, or consequential damages,
 * with respect to any claim arising out or in connection with the use
 * of the software, even if it has been or is hereafter advised of the
 * possibility of such damages.
 */
F
Fred Isaman 已提交
32

33 34
#include <linux/module.h>
#include <linux/init.h>
J
Jim Rees 已提交
35 36
#include <linux/mount.h>
#include <linux/namei.h>
F
Fred Isaman 已提交
37
#include <linux/bio.h>		/* struct bio */
38
#include <linux/buffer_head.h>	/* various write calls */
39
#include <linux/prefetch.h>
40
#include <linux/pagevec.h>
41

42
#include "../pnfs.h"
43
#include "../nfs4session.h"
44
#include "../internal.h"
45 46 47 48 49 50 51 52
#include "blocklayout.h"

#define NFSDBG_FACILITY	NFSDBG_PNFS_LD

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Andy Adamson <andros@citi.umich.edu>");
MODULE_DESCRIPTION("The NFSv4.1 pNFS Block layout driver");

F
Fred Isaman 已提交
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
static void print_page(struct page *page)
{
	dprintk("PRINTPAGE page %p\n", page);
	dprintk("	PagePrivate %d\n", PagePrivate(page));
	dprintk("	PageUptodate %d\n", PageUptodate(page));
	dprintk("	PageError %d\n", PageError(page));
	dprintk("	PageDirty %d\n", PageDirty(page));
	dprintk("	PageReferenced %d\n", PageReferenced(page));
	dprintk("	PageLocked %d\n", PageLocked(page));
	dprintk("	PageWriteback %d\n", PageWriteback(page));
	dprintk("	PageMappedToDisk %d\n", PageMappedToDisk(page));
	dprintk("\n");
}

/* Given the be associated with isect, determine if page data needs to be
 * initialized.
 */
static int is_hole(struct pnfs_block_extent *be, sector_t isect)
{
	if (be->be_state == PNFS_BLOCK_NONE_DATA)
		return 1;
	else if (be->be_state != PNFS_BLOCK_INVALID_DATA)
		return 0;
	else
		return !bl_is_sector_init(be->be_inval, isect);
}

F
Fred Isaman 已提交
80 81 82 83 84
/* Given the be associated with isect, determine if page data can be
 * written to disk.
 */
static int is_writable(struct pnfs_block_extent *be, sector_t isect)
{
85 86
	return (be->be_state == PNFS_BLOCK_READWRITE_DATA ||
		be->be_state == PNFS_BLOCK_INVALID_DATA);
F
Fred Isaman 已提交
87 88
}

F
Fred Isaman 已提交
89 90 91 92 93
/* The data we are handed might be spread across several bios.  We need
 * to track when the last one is finished.
 */
struct parallel_io {
	struct kref refcnt;
94
	void (*pnfs_callback) (void *data, int num_se);
F
Fred Isaman 已提交
95
	void *data;
96
	int bse_count;
F
Fred Isaman 已提交
97 98 99 100 101 102 103 104 105 106
};

static inline struct parallel_io *alloc_parallel(void *data)
{
	struct parallel_io *rv;

	rv  = kmalloc(sizeof(*rv), GFP_NOFS);
	if (rv) {
		rv->data = data;
		kref_init(&rv->refcnt);
107
		rv->bse_count = 0;
F
Fred Isaman 已提交
108 109 110 111 112 113 114 115 116 117 118 119 120 121
	}
	return rv;
}

static inline void get_parallel(struct parallel_io *p)
{
	kref_get(&p->refcnt);
}

static void destroy_parallel(struct kref *kref)
{
	struct parallel_io *p = container_of(kref, struct parallel_io, refcnt);

	dprintk("%s enter\n", __func__);
122
	p->pnfs_callback(p->data, p->bse_count);
F
Fred Isaman 已提交
123 124 125 126 127 128 129 130 131 132 133 134 135 136
	kfree(p);
}

static inline void put_parallel(struct parallel_io *p)
{
	kref_put(&p->refcnt, destroy_parallel);
}

static struct bio *
bl_submit_bio(int rw, struct bio *bio)
{
	if (bio) {
		get_parallel(bio->bi_private);
		dprintk("%s submitting %s bio %u@%llu\n", __func__,
137 138
			rw == READ ? "read" : "write", bio->bi_iter.bi_size,
			(unsigned long long)bio->bi_iter.bi_sector);
F
Fred Isaman 已提交
139 140 141 142 143 144 145 146 147 148 149 150
		submit_bio(rw, bio);
	}
	return NULL;
}

static struct bio *bl_alloc_init_bio(int npg, sector_t isect,
				     struct pnfs_block_extent *be,
				     void (*end_io)(struct bio *, int err),
				     struct parallel_io *par)
{
	struct bio *bio;

P
Peng Tao 已提交
151
	npg = min(npg, BIO_MAX_PAGES);
F
Fred Isaman 已提交
152
	bio = bio_alloc(GFP_NOIO, npg);
P
Peng Tao 已提交
153 154 155 156
	if (!bio && (current->flags & PF_MEMALLOC)) {
		while (!bio && (npg /= 2))
			bio = bio_alloc(GFP_NOIO, npg);
	}
F
Fred Isaman 已提交
157

P
Peng Tao 已提交
158
	if (bio) {
159 160
		bio->bi_iter.bi_sector = isect - be->be_f_offset +
			be->be_v_offset;
P
Peng Tao 已提交
161 162 163 164
		bio->bi_bdev = be->be_mdev;
		bio->bi_end_io = end_io;
		bio->bi_private = par;
	}
F
Fred Isaman 已提交
165 166 167
	return bio;
}

168
static struct bio *do_add_page_to_bio(struct bio *bio, int npg, int rw,
F
Fred Isaman 已提交
169 170 171
				      sector_t isect, struct page *page,
				      struct pnfs_block_extent *be,
				      void (*end_io)(struct bio *, int err),
172 173
				      struct parallel_io *par,
				      unsigned int offset, int len)
F
Fred Isaman 已提交
174
{
175 176 177
	isect = isect + (offset >> SECTOR_SHIFT);
	dprintk("%s: npg %d rw %d isect %llu offset %u len %d\n", __func__,
		npg, rw, (unsigned long long)isect, offset, len);
F
Fred Isaman 已提交
178 179 180 181 182 183
retry:
	if (!bio) {
		bio = bl_alloc_init_bio(npg, isect, be, end_io, par);
		if (!bio)
			return ERR_PTR(-ENOMEM);
	}
184
	if (bio_add_page(bio, page, len, offset) < len) {
F
Fred Isaman 已提交
185 186 187 188 189 190
		bio = bl_submit_bio(rw, bio);
		goto retry;
	}
	return bio;
}

191 192 193 194 195 196 197 198 199 200
static struct bio *bl_add_page_to_bio(struct bio *bio, int npg, int rw,
				      sector_t isect, struct page *page,
				      struct pnfs_block_extent *be,
				      void (*end_io)(struct bio *, int err),
				      struct parallel_io *par)
{
	return do_add_page_to_bio(bio, npg, rw, isect, page, be,
				  end_io, par, 0, PAGE_CACHE_SIZE);
}

F
Fred Isaman 已提交
201 202 203 204
/* This is basically copied from mpage_end_io_read */
static void bl_end_io_read(struct bio *bio, int err)
{
	struct parallel_io *par = bio->bi_private;
205 206
	struct bio_vec *bvec;
	int i;
F
Fred Isaman 已提交
207

208 209 210
	if (!err)
		bio_for_each_segment_all(bvec, bio, i)
			SetPageUptodate(bvec->bv_page);
F
Fred Isaman 已提交
211

212
	if (err) {
213
		struct nfs_pgio_header *header = par->data;
214 215 216 217

		if (!header->pnfs_error)
			header->pnfs_error = -EIO;
		pnfs_set_lo_fail(header->lseg);
F
Fred Isaman 已提交
218 219 220 221 222 223 224 225
	}
	bio_put(bio);
	put_parallel(par);
}

static void bl_read_cleanup(struct work_struct *work)
{
	struct rpc_task *task;
226
	struct nfs_pgio_header *hdr;
F
Fred Isaman 已提交
227 228
	dprintk("%s enter\n", __func__);
	task = container_of(work, struct rpc_task, u.tk_work);
229 230
	hdr = container_of(task, struct nfs_pgio_header, task);
	pnfs_ld_read_done(hdr);
F
Fred Isaman 已提交
231 232 233
}

static void
234
bl_end_par_io_read(void *data, int unused)
F
Fred Isaman 已提交
235
{
236
	struct nfs_pgio_header *hdr = data;
F
Fred Isaman 已提交
237

238 239 240
	hdr->task.tk_status = hdr->pnfs_error;
	INIT_WORK(&hdr->task.u.tk_work, bl_read_cleanup);
	schedule_work(&hdr->task.u.tk_work);
F
Fred Isaman 已提交
241 242
}

243
static enum pnfs_try_status
244
bl_read_pagelist(struct nfs_pgio_header *hdr)
245
{
246
	struct nfs_pgio_header *header = hdr;
F
Fred Isaman 已提交
247 248 249 250 251
	int i, hole;
	struct bio *bio = NULL;
	struct pnfs_block_extent *be = NULL, *cow_read = NULL;
	sector_t isect, extent_length = 0;
	struct parallel_io *par;
252 253
	loff_t f_offset = hdr->args.offset;
	size_t bytes_left = hdr->args.count;
P
Peng Tao 已提交
254
	unsigned int pg_offset, pg_len;
255 256
	struct page **pages = hdr->args.pages;
	int pg_index = hdr->args.pgbase >> PAGE_CACHE_SHIFT;
P
Peng Tao 已提交
257
	const bool is_dio = (header->dreq != NULL);
F
Fred Isaman 已提交
258

259
	dprintk("%s enter nr_pages %u offset %lld count %u\n", __func__,
260 261
		hdr->page_array.npages, f_offset,
		(unsigned int)hdr->args.count);
F
Fred Isaman 已提交
262

263
	par = alloc_parallel(hdr);
F
Fred Isaman 已提交
264 265 266 267 268 269 270
	if (!par)
		goto use_mds;
	par->pnfs_callback = bl_end_par_io_read;
	/* At this point, we can no longer jump to use_mds */

	isect = (sector_t) (f_offset >> SECTOR_SHIFT);
	/* Code assumes extents are page-aligned */
271
	for (i = pg_index; i < hdr->page_array.npages; i++) {
F
Fred Isaman 已提交
272 273 274 275 276 277
		if (!extent_length) {
			/* We've used up the previous extent */
			bl_put_extent(be);
			bl_put_extent(cow_read);
			bio = bl_submit_bio(READ, bio);
			/* Get the next one */
278
			be = bl_find_get_extent(BLK_LSEG2EXT(header->lseg),
F
Fred Isaman 已提交
279 280
					     isect, &cow_read);
			if (!be) {
281
				header->pnfs_error = -EIO;
F
Fred Isaman 已提交
282 283 284 285 286 287 288 289 290 291
				goto out;
			}
			extent_length = be->be_length -
				(isect - be->be_f_offset);
			if (cow_read) {
				sector_t cow_length = cow_read->be_length -
					(isect - cow_read->be_f_offset);
				extent_length = min(extent_length, cow_length);
			}
		}
P
Peng Tao 已提交
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307

		if (is_dio) {
			pg_offset = f_offset & ~PAGE_CACHE_MASK;
			if (pg_offset + bytes_left > PAGE_CACHE_SIZE)
				pg_len = PAGE_CACHE_SIZE - pg_offset;
			else
				pg_len = bytes_left;

			f_offset += pg_len;
			bytes_left -= pg_len;
			isect += (pg_offset >> SECTOR_SHIFT);
		} else {
			pg_offset = 0;
			pg_len = PAGE_CACHE_SIZE;
		}

F
Fred Isaman 已提交
308 309 310 311 312
		hole = is_hole(be, isect);
		if (hole && !cow_read) {
			bio = bl_submit_bio(READ, bio);
			/* Fill hole w/ zeroes w/o accessing device */
			dprintk("%s Zeroing page for hole\n", __func__);
P
Peng Tao 已提交
313
			zero_user_segment(pages[i], pg_offset, pg_len);
F
Fred Isaman 已提交
314 315 316 317 318 319
			print_page(pages[i]);
			SetPageUptodate(pages[i]);
		} else {
			struct pnfs_block_extent *be_read;

			be_read = (hole && cow_read) ? cow_read : be;
320
			bio = do_add_page_to_bio(bio,
321
						 hdr->page_array.npages - i,
F
Fred Isaman 已提交
322
						 READ,
F
Fred Isaman 已提交
323
						 isect, pages[i], be_read,
P
Peng Tao 已提交
324 325
						 bl_end_io_read, par,
						 pg_offset, pg_len);
F
Fred Isaman 已提交
326
			if (IS_ERR(bio)) {
327
				header->pnfs_error = PTR_ERR(bio);
P
Peng Tao 已提交
328
				bio = NULL;
F
Fred Isaman 已提交
329 330 331
				goto out;
			}
		}
P
Peng Tao 已提交
332
		isect += (pg_len >> SECTOR_SHIFT);
F
Fred Isaman 已提交
333 334
		extent_length -= PAGE_CACHE_SECTORS;
	}
335
	if ((isect << SECTOR_SHIFT) >= header->inode->i_size) {
336 337
		hdr->res.eof = 1;
		hdr->res.count = header->inode->i_size - hdr->args.offset;
F
Fred Isaman 已提交
338
	} else {
339
		hdr->res.count = (isect << SECTOR_SHIFT) - hdr->args.offset;
F
Fred Isaman 已提交
340 341 342 343 344 345 346 347 348 349
	}
out:
	bl_put_extent(be);
	bl_put_extent(cow_read);
	bl_submit_bio(READ, bio);
	put_parallel(par);
	return PNFS_ATTEMPTED;

 use_mds:
	dprintk("Giving up and using normal NFS\n");
350 351 352
	return PNFS_NOT_ATTEMPTED;
}

353 354 355 356 357
static void mark_extents_written(struct pnfs_block_layout *bl,
				 __u64 offset, __u32 count)
{
	sector_t isect, end;
	struct pnfs_block_extent *be;
358
	struct pnfs_block_short_extent *se;
359 360 361 362 363 364 365 366 367 368 369 370

	dprintk("%s(%llu, %u)\n", __func__, offset, count);
	if (count == 0)
		return;
	isect = (offset & (long)(PAGE_CACHE_MASK)) >> SECTOR_SHIFT;
	end = (offset + count + PAGE_CACHE_SIZE - 1) & (long)(PAGE_CACHE_MASK);
	end >>= SECTOR_SHIFT;
	while (isect < end) {
		sector_t len;
		be = bl_find_get_extent(bl, isect, NULL);
		BUG_ON(!be); /* FIXME */
		len = min(end, be->be_f_offset + be->be_length) - isect;
371 372 373 374 375
		if (be->be_state == PNFS_BLOCK_INVALID_DATA) {
			se = bl_pop_one_short_extent(be->be_inval);
			BUG_ON(!se);
			bl_mark_for_commit(be, isect, len, se);
		}
376 377 378 379 380
		isect += len;
		bl_put_extent(be);
	}
}

381 382 383
static void bl_end_io_write_zero(struct bio *bio, int err)
{
	struct parallel_io *par = bio->bi_private;
384 385
	struct bio_vec *bvec;
	int i;
386

387
	bio_for_each_segment_all(bvec, bio, i) {
388
		/* This is the zeroing page we added */
389 390 391
		end_page_writeback(bvec->bv_page);
		page_cache_release(bvec->bv_page);
	}
392

393
	if (unlikely(err)) {
394
		struct nfs_pgio_header *header = par->data;
395 396 397 398

		if (!header->pnfs_error)
			header->pnfs_error = -EIO;
		pnfs_set_lo_fail(header->lseg);
399 400 401 402 403
	}
	bio_put(bio);
	put_parallel(par);
}

F
Fred Isaman 已提交
404 405 406 407
static void bl_end_io_write(struct bio *bio, int err)
{
	struct parallel_io *par = bio->bi_private;
	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
408
	struct nfs_pgio_header *header = par->data;
F
Fred Isaman 已提交
409 410

	if (!uptodate) {
411 412 413
		if (!header->pnfs_error)
			header->pnfs_error = -EIO;
		pnfs_set_lo_fail(header->lseg);
F
Fred Isaman 已提交
414 415 416 417 418 419 420 421 422 423 424
	}
	bio_put(bio);
	put_parallel(par);
}

/* Function scheduled for call during bl_end_par_io_write,
 * it marks sectors as written and extends the commitlist.
 */
static void bl_write_cleanup(struct work_struct *work)
{
	struct rpc_task *task;
425
	struct nfs_pgio_header *hdr;
F
Fred Isaman 已提交
426 427
	dprintk("%s enter\n", __func__);
	task = container_of(work, struct rpc_task, u.tk_work);
428 429
	hdr = container_of(task, struct nfs_pgio_header, task);
	if (likely(!hdr->pnfs_error)) {
430
		/* Marks for LAYOUTCOMMIT */
431 432
		mark_extents_written(BLK_LSEG2EXT(hdr->lseg),
				     hdr->args.offset, hdr->args.count);
433
	}
434
	pnfs_ld_write_done(hdr);
F
Fred Isaman 已提交
435 436 437
}

/* Called when last of bios associated with a bl_write_pagelist call finishes */
438
static void bl_end_par_io_write(void *data, int num_se)
F
Fred Isaman 已提交
439
{
440
	struct nfs_pgio_header *hdr = data;
F
Fred Isaman 已提交
441

442 443
	if (unlikely(hdr->pnfs_error)) {
		bl_free_short_extents(&BLK_LSEG2EXT(hdr->lseg)->bl_inval,
444 445 446
					num_se);
	}

447
	hdr->task.tk_status = hdr->pnfs_error;
448
	hdr->verf.committed = NFS_FILE_SYNC;
449 450
	INIT_WORK(&hdr->task.u.tk_work, bl_write_cleanup);
	schedule_work(&hdr->task.u.tk_work);
F
Fred Isaman 已提交
451 452
}

453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
/* FIXME STUB - mark intersection of layout and page as bad, so is not
 * used again.
 */
static void mark_bad_read(void)
{
	return;
}

/*
 * map_block:  map a requested I/0 block (isect) into an offset in the LVM
 * block_device
 */
static void
map_block(struct buffer_head *bh, sector_t isect, struct pnfs_block_extent *be)
{
	dprintk("%s enter be=%p\n", __func__, be);

	set_buffer_mapped(bh);
	bh->b_bdev = be->be_mdev;
	bh->b_blocknr = (isect - be->be_f_offset + be->be_v_offset) >>
	    (be->be_mdev->bd_inode->i_blkbits - SECTOR_SHIFT);

	dprintk("%s isect %llu, bh->b_blocknr %ld, using bsize %Zd\n",
		__func__, (unsigned long long)isect, (long)bh->b_blocknr,
		bh->b_size);
	return;
}

481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
static void
bl_read_single_end_io(struct bio *bio, int error)
{
	struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
	struct page *page = bvec->bv_page;

	/* Only one page in bvec */
	unlock_page(page);
}

static int
bl_do_readpage_sync(struct page *page, struct pnfs_block_extent *be,
		    unsigned int offset, unsigned int len)
{
	struct bio *bio;
	struct page *shadow_page;
	sector_t isect;
	char *kaddr, *kshadow_addr;
	int ret = 0;

	dprintk("%s: offset %u len %u\n", __func__, offset, len);

	shadow_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
	if (shadow_page == NULL)
		return -ENOMEM;

	bio = bio_alloc(GFP_NOIO, 1);
	if (bio == NULL)
		return -ENOMEM;

	isect = (page->index << PAGE_CACHE_SECTOR_SHIFT) +
		(offset / SECTOR_SIZE);

514
	bio->bi_iter.bi_sector = isect - be->be_f_offset + be->be_v_offset;
515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580
	bio->bi_bdev = be->be_mdev;
	bio->bi_end_io = bl_read_single_end_io;

	lock_page(shadow_page);
	if (bio_add_page(bio, shadow_page,
			 SECTOR_SIZE, round_down(offset, SECTOR_SIZE)) == 0) {
		unlock_page(shadow_page);
		bio_put(bio);
		return -EIO;
	}

	submit_bio(READ, bio);
	wait_on_page_locked(shadow_page);
	if (unlikely(!test_bit(BIO_UPTODATE, &bio->bi_flags))) {
		ret = -EIO;
	} else {
		kaddr = kmap_atomic(page);
		kshadow_addr = kmap_atomic(shadow_page);
		memcpy(kaddr + offset, kshadow_addr + offset, len);
		kunmap_atomic(kshadow_addr);
		kunmap_atomic(kaddr);
	}
	__free_page(shadow_page);
	bio_put(bio);

	return ret;
}

static int
bl_read_partial_page_sync(struct page *page, struct pnfs_block_extent *be,
			  unsigned int dirty_offset, unsigned int dirty_len,
			  bool full_page)
{
	int ret = 0;
	unsigned int start, end;

	if (full_page) {
		start = 0;
		end = PAGE_CACHE_SIZE;
	} else {
		start = round_down(dirty_offset, SECTOR_SIZE);
		end = round_up(dirty_offset + dirty_len, SECTOR_SIZE);
	}

	dprintk("%s: offset %u len %d\n", __func__, dirty_offset, dirty_len);
	if (!be) {
		zero_user_segments(page, start, dirty_offset,
				   dirty_offset + dirty_len, end);
		if (start == 0 && end == PAGE_CACHE_SIZE &&
		    trylock_page(page)) {
			SetPageUptodate(page);
			unlock_page(page);
		}
		return ret;
	}

	if (start != dirty_offset)
		ret = bl_do_readpage_sync(page, be, start, dirty_offset - start);

	if (!ret && (dirty_offset + dirty_len < end))
		ret = bl_do_readpage_sync(page, be, dirty_offset + dirty_len,
					  end - dirty_offset - dirty_len);

	return ret;
}

581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624
/* Given an unmapped page, zero it or read in page for COW, page is locked
 * by caller.
 */
static int
init_page_for_write(struct page *page, struct pnfs_block_extent *cow_read)
{
	struct buffer_head *bh = NULL;
	int ret = 0;
	sector_t isect;

	dprintk("%s enter, %p\n", __func__, page);
	BUG_ON(PageUptodate(page));
	if (!cow_read) {
		zero_user_segment(page, 0, PAGE_SIZE);
		SetPageUptodate(page);
		goto cleanup;
	}

	bh = alloc_page_buffers(page, PAGE_CACHE_SIZE, 0);
	if (!bh) {
		ret = -ENOMEM;
		goto cleanup;
	}

	isect = (sector_t) page->index << PAGE_CACHE_SECTOR_SHIFT;
	map_block(bh, isect, cow_read);
	if (!bh_uptodate_or_lock(bh))
		ret = bh_submit_read(bh);
	if (ret)
		goto cleanup;
	SetPageUptodate(page);

cleanup:
	if (bh)
		free_buffer_head(bh);
	if (ret) {
		/* Need to mark layout with bad read...should now
		 * just use nfs4 for reads and writes.
		 */
		mark_bad_read();
	}
	return ret;
}

625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673
/* Find or create a zeroing page marked being writeback.
 * Return ERR_PTR on error, NULL to indicate skip this page and page itself
 * to indicate write out.
 */
static struct page *
bl_find_get_zeroing_page(struct inode *inode, pgoff_t index,
			struct pnfs_block_extent *cow_read)
{
	struct page *page;
	int locked = 0;
	page = find_get_page(inode->i_mapping, index);
	if (page)
		goto check_page;

	page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
	if (unlikely(!page)) {
		dprintk("%s oom\n", __func__);
		return ERR_PTR(-ENOMEM);
	}
	locked = 1;

check_page:
	/* PageDirty: Other will write this out
	 * PageWriteback: Other is writing this out
	 * PageUptodate: It was read before
	 */
	if (PageDirty(page) || PageWriteback(page)) {
		print_page(page);
		if (locked)
			unlock_page(page);
		page_cache_release(page);
		return NULL;
	}

	if (!locked) {
		lock_page(page);
		locked = 1;
		goto check_page;
	}
	if (!PageUptodate(page)) {
		/* New page, readin or zero it */
		init_page_for_write(page, cow_read);
	}
	set_page_writeback(page);
	unlock_page(page);

	return page;
}

674
static enum pnfs_try_status
675
bl_write_pagelist(struct nfs_pgio_header *header, int sync)
676
{
677
	int i, ret, npg_zero, pg_index, last = 0;
F
Fred Isaman 已提交
678
	struct bio *bio = NULL;
679 680
	struct pnfs_block_extent *be = NULL, *cow_read = NULL;
	sector_t isect, last_isect = 0, extent_length = 0;
P
Peng Tao 已提交
681
	struct parallel_io *par = NULL;
682 683
	loff_t offset = header->args.offset;
	size_t count = header->args.count;
684
	unsigned int pg_offset, pg_len, saved_len;
685
	struct page **pages = header->args.pages;
686 687 688 689
	struct page *page;
	pgoff_t index;
	u64 temp;
	int npg_per_block =
690
	    NFS_SERVER(header->inode)->pnfs_blksize >> PAGE_CACHE_SHIFT;
F
Fred Isaman 已提交
691 692

	dprintk("%s enter, %Zu@%lld\n", __func__, count, offset);
P
Peng Tao 已提交
693 694 695 696 697 698 699

	if (header->dreq != NULL &&
	    (!IS_ALIGNED(offset, NFS_SERVER(header->inode)->pnfs_blksize) ||
	     !IS_ALIGNED(count, NFS_SERVER(header->inode)->pnfs_blksize))) {
		dprintk("pnfsblock nonblock aligned DIO writes. Resend MDS\n");
		goto out_mds;
	}
700
	/* At this point, header->page_aray is a (sequential) list of nfs_pages.
701 702
	 * We want to write each, and if there is an error set pnfs_error
	 * to have it redone using nfs.
F
Fred Isaman 已提交
703
	 */
704
	par = alloc_parallel(header);
F
Fred Isaman 已提交
705
	if (!par)
706
		goto out_mds;
F
Fred Isaman 已提交
707 708 709 710
	par->pnfs_callback = bl_end_par_io_write;
	/* At this point, have to be more careful with error handling */

	isect = (sector_t) ((offset & (long)PAGE_CACHE_MASK) >> SECTOR_SHIFT);
711
	be = bl_find_get_extent(BLK_LSEG2EXT(header->lseg), isect, &cow_read);
712 713
	if (!be || !is_writable(be, isect)) {
		dprintk("%s no matching extents!\n", __func__);
714
		goto out_mds;
715 716 717 718
	}

	/* First page inside INVALID extent */
	if (be->be_state == PNFS_BLOCK_INVALID_DATA) {
719 720 721 722
		if (likely(!bl_push_one_short_extent(be->be_inval)))
			par->bse_count++;
		else
			goto out_mds;
723 724 725 726 727 728 729 730 731
		temp = offset >> PAGE_CACHE_SHIFT;
		npg_zero = do_div(temp, npg_per_block);
		isect = (sector_t) (((offset - npg_zero * PAGE_CACHE_SIZE) &
				     (long)PAGE_CACHE_MASK) >> SECTOR_SHIFT);
		extent_length = be->be_length - (isect - be->be_f_offset);

fill_invalid_ext:
		dprintk("%s need to zero %d pages\n", __func__, npg_zero);
		for (;npg_zero > 0; npg_zero--) {
P
Peng Tao 已提交
732 733 734 735 736
			if (bl_is_sector_init(be->be_inval, isect)) {
				dprintk("isect %llu already init\n",
					(unsigned long long)isect);
				goto next_page;
			}
737 738 739 740 741
			/* page ref released in bl_end_io_write_zero */
			index = isect >> PAGE_CACHE_SECTOR_SHIFT;
			dprintk("%s zero %dth page: index %lu isect %llu\n",
				__func__, npg_zero, index,
				(unsigned long long)isect);
742
			page = bl_find_get_zeroing_page(header->inode, index,
743 744
							cow_read);
			if (unlikely(IS_ERR(page))) {
745
				header->pnfs_error = PTR_ERR(page);
746
				goto out;
747
			} else if (page == NULL)
748 749 750
				goto next_page;

			ret = bl_mark_sectors_init(be->be_inval, isect,
751
						       PAGE_CACHE_SECTORS);
752 753 754 755 756
			if (unlikely(ret)) {
				dprintk("%s bl_mark_sectors_init fail %d\n",
					__func__, ret);
				end_page_writeback(page);
				page_cache_release(page);
757
				header->pnfs_error = ret;
758 759
				goto out;
			}
760 761 762 763 764
			if (likely(!bl_push_one_short_extent(be->be_inval)))
				par->bse_count++;
			else {
				end_page_writeback(page);
				page_cache_release(page);
765
				header->pnfs_error = -ENOMEM;
766 767 768
				goto out;
			}
			/* FIXME: This should be done in bi_end_io */
769
			mark_extents_written(BLK_LSEG2EXT(header->lseg),
770 771 772
					     page->index << PAGE_CACHE_SHIFT,
					     PAGE_CACHE_SIZE);

773 774 775 776
			bio = bl_add_page_to_bio(bio, npg_zero, WRITE,
						 isect, page, be,
						 bl_end_io_write_zero, par);
			if (IS_ERR(bio)) {
777
				header->pnfs_error = PTR_ERR(bio);
P
Peng Tao 已提交
778
				bio = NULL;
779 780 781 782 783 784 785 786 787 788 789 790
				goto out;
			}
next_page:
			isect += PAGE_CACHE_SECTORS;
			extent_length -= PAGE_CACHE_SECTORS;
		}
		if (last)
			goto write_done;
	}
	bio = bl_submit_bio(WRITE, bio);

	/* Middle pages */
791 792
	pg_index = header->args.pgbase >> PAGE_CACHE_SHIFT;
	for (i = pg_index; i < header->page_array.npages; i++) {
F
Fred Isaman 已提交
793 794 795
		if (!extent_length) {
			/* We've used up the previous extent */
			bl_put_extent(be);
796
			bl_put_extent(cow_read);
F
Fred Isaman 已提交
797 798
			bio = bl_submit_bio(WRITE, bio);
			/* Get the next one */
799
			be = bl_find_get_extent(BLK_LSEG2EXT(header->lseg),
800
					     isect, &cow_read);
F
Fred Isaman 已提交
801
			if (!be || !is_writable(be, isect)) {
802
				header->pnfs_error = -EINVAL;
F
Fred Isaman 已提交
803 804
				goto out;
			}
805 806 807 808 809
			if (be->be_state == PNFS_BLOCK_INVALID_DATA) {
				if (likely(!bl_push_one_short_extent(
								be->be_inval)))
					par->bse_count++;
				else {
810
					header->pnfs_error = -ENOMEM;
811 812 813
					goto out;
				}
			}
F
Fred Isaman 已提交
814
			extent_length = be->be_length -
815
			    (isect - be->be_f_offset);
F
Fred Isaman 已提交
816
		}
817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836

		dprintk("%s offset %lld count %Zu\n", __func__, offset, count);
		pg_offset = offset & ~PAGE_CACHE_MASK;
		if (pg_offset + count > PAGE_CACHE_SIZE)
			pg_len = PAGE_CACHE_SIZE - pg_offset;
		else
			pg_len = count;

		saved_len = pg_len;
		if (be->be_state == PNFS_BLOCK_INVALID_DATA &&
		    !bl_is_sector_init(be->be_inval, isect)) {
			ret = bl_read_partial_page_sync(pages[i], cow_read,
							pg_offset, pg_len, true);
			if (ret) {
				dprintk("%s bl_read_partial_page_sync fail %d\n",
					__func__, ret);
				header->pnfs_error = ret;
				goto out;
			}

837
			ret = bl_mark_sectors_init(be->be_inval, isect,
838
						       PAGE_CACHE_SECTORS);
839 840 841
			if (unlikely(ret)) {
				dprintk("%s bl_mark_sectors_init fail %d\n",
					__func__, ret);
842
				header->pnfs_error = ret;
843
				goto out;
F
Fred Isaman 已提交
844
			}
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859

			/* Expand to full page write */
			pg_offset = 0;
			pg_len = PAGE_CACHE_SIZE;
		} else if  ((pg_offset & (SECTOR_SIZE - 1)) ||
			    (pg_len & (SECTOR_SIZE - 1))){
			/* ahh, nasty case. We have to do sync full sector
			 * read-modify-write cycles.
			 */
			unsigned int saved_offset = pg_offset;
			ret = bl_read_partial_page_sync(pages[i], be, pg_offset,
							pg_len, false);
			pg_offset = round_down(pg_offset, SECTOR_SIZE);
			pg_len = round_up(saved_offset + pg_len, SECTOR_SIZE)
				 - pg_offset;
860
		}
861 862


863
		bio = do_add_page_to_bio(bio, header->page_array.npages - i,
864
					 WRITE,
865
					 isect, pages[i], be,
866 867
					 bl_end_io_write, par,
					 pg_offset, pg_len);
868
		if (IS_ERR(bio)) {
869
			header->pnfs_error = PTR_ERR(bio);
P
Peng Tao 已提交
870
			bio = NULL;
871
			goto out;
F
Fred Isaman 已提交
872
		}
873 874
		offset += saved_len;
		count -= saved_len;
F
Fred Isaman 已提交
875
		isect += PAGE_CACHE_SECTORS;
876
		last_isect = isect;
F
Fred Isaman 已提交
877 878
		extent_length -= PAGE_CACHE_SECTORS;
	}
879 880 881 882 883 884 885 886 887 888 889 890 891

	/* Last page inside INVALID extent */
	if (be->be_state == PNFS_BLOCK_INVALID_DATA) {
		bio = bl_submit_bio(WRITE, bio);
		temp = last_isect >> PAGE_CACHE_SECTOR_SHIFT;
		npg_zero = npg_per_block - do_div(temp, npg_per_block);
		if (npg_zero < npg_per_block) {
			last = 1;
			goto fill_invalid_ext;
		}
	}

write_done:
892
	header->res.count = header->args.count;
F
Fred Isaman 已提交
893 894
out:
	bl_put_extent(be);
895
	bl_put_extent(cow_read);
F
Fred Isaman 已提交
896 897 898
	bl_submit_bio(WRITE, bio);
	put_parallel(par);
	return PNFS_ATTEMPTED;
899 900
out_mds:
	bl_put_extent(be);
901
	bl_put_extent(cow_read);
902 903
	kfree(par);
	return PNFS_NOT_ATTEMPTED;
904 905
}

F
Fred Isaman 已提交
906
/* FIXME - range ignored */
907
static void
F
Fred Isaman 已提交
908
release_extents(struct pnfs_block_layout *bl, struct pnfs_layout_range *range)
909
{
F
Fred Isaman 已提交
910 911 912 913 914 915 916 917 918 919 920 921 922 923
	int i;
	struct pnfs_block_extent *be;

	spin_lock(&bl->bl_ext_lock);
	for (i = 0; i < EXTENT_LISTS; i++) {
		while (!list_empty(&bl->bl_extents[i])) {
			be = list_first_entry(&bl->bl_extents[i],
					      struct pnfs_block_extent,
					      be_node);
			list_del(&be->be_node);
			bl_put_extent(be);
		}
	}
	spin_unlock(&bl->bl_ext_lock);
924 925 926 927 928
}

static void
release_inval_marks(struct pnfs_inval_markings *marks)
{
929
	struct pnfs_inval_tracking *pos, *temp;
930
	struct pnfs_block_short_extent *se, *stemp;
931 932 933 934 935

	list_for_each_entry_safe(pos, temp, &marks->im_tree.mtt_stub, it_link) {
		list_del(&pos->it_link);
		kfree(pos);
	}
936 937 938 939 940

	list_for_each_entry_safe(se, stemp, &marks->im_extents, bse_node) {
		list_del(&se->bse_node);
		kfree(se);
	}
941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973
	return;
}

static void bl_free_layout_hdr(struct pnfs_layout_hdr *lo)
{
	struct pnfs_block_layout *bl = BLK_LO2EXT(lo);

	dprintk("%s enter\n", __func__);
	release_extents(bl, NULL);
	release_inval_marks(&bl->bl_inval);
	kfree(bl);
}

static struct pnfs_layout_hdr *bl_alloc_layout_hdr(struct inode *inode,
						   gfp_t gfp_flags)
{
	struct pnfs_block_layout *bl;

	dprintk("%s enter\n", __func__);
	bl = kzalloc(sizeof(*bl), gfp_flags);
	if (!bl)
		return NULL;
	spin_lock_init(&bl->bl_ext_lock);
	INIT_LIST_HEAD(&bl->bl_extents[0]);
	INIT_LIST_HEAD(&bl->bl_extents[1]);
	INIT_LIST_HEAD(&bl->bl_commit);
	INIT_LIST_HEAD(&bl->bl_committing);
	bl->bl_count = 0;
	bl->bl_blocksize = NFS_SERVER(inode)->pnfs_blksize >> SECTOR_SHIFT;
	BL_INIT_INVAL_MARKS(&bl->bl_inval, bl->bl_blocksize);
	return &bl->bl_layout;
}

F
Fred Isaman 已提交
974
static void bl_free_lseg(struct pnfs_layout_segment *lseg)
975
{
F
Fred Isaman 已提交
976 977
	dprintk("%s enter\n", __func__);
	kfree(lseg);
978 979
}

F
Fred Isaman 已提交
980 981 982 983 984 985
/* We pretty much ignore lseg, and store all data layout wide, so we
 * can correctly merge.
 */
static struct pnfs_layout_segment *bl_alloc_lseg(struct pnfs_layout_hdr *lo,
						 struct nfs4_layoutget_res *lgr,
						 gfp_t gfp_flags)
986
{
F
Fred Isaman 已提交
987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
	struct pnfs_layout_segment *lseg;
	int status;

	dprintk("%s enter\n", __func__);
	lseg = kzalloc(sizeof(*lseg), gfp_flags);
	if (!lseg)
		return ERR_PTR(-ENOMEM);
	status = nfs4_blk_process_layoutget(lo, lgr, gfp_flags);
	if (status) {
		/* We don't want to call the full-blown bl_free_lseg,
		 * since on error extents were not touched.
		 */
		kfree(lseg);
		return ERR_PTR(status);
	}
	return lseg;
1003 1004 1005 1006 1007 1008
}

static void
bl_encode_layoutcommit(struct pnfs_layout_hdr *lo, struct xdr_stream *xdr,
		       const struct nfs4_layoutcommit_args *arg)
{
F
Fred Isaman 已提交
1009 1010
	dprintk("%s enter\n", __func__);
	encode_pnfs_block_layoutupdate(BLK_LO2EXT(lo), xdr, arg);
1011 1012 1013 1014 1015
}

static void
bl_cleanup_layoutcommit(struct nfs4_layoutcommit_data *lcdata)
{
F
Fred Isaman 已提交
1016 1017 1018 1019
	struct pnfs_layout_hdr *lo = NFS_I(lcdata->args.inode)->layout;

	dprintk("%s enter\n", __func__);
	clean_pnfs_block_layoutupdate(BLK_LO2EXT(lo), &lcdata->args, lcdata->res.status);
1020 1021
}

1022 1023 1024
static void free_blk_mountid(struct block_mount_id *mid)
{
	if (mid) {
1025 1026 1027 1028
		struct pnfs_block_dev *dev, *tmp;

		/* No need to take bm_lock as we are last user freeing bm_devlist */
		list_for_each_entry_safe(dev, tmp, &mid->bm_devlist, bm_node) {
1029 1030 1031 1032 1033 1034 1035
			list_del(&dev->bm_node);
			bl_free_block_dev(dev);
		}
		kfree(mid);
	}
}

1036
/* This is mostly copied from the filelayout_get_device_info function.
1037 1038 1039 1040 1041 1042 1043
 * It seems much of this should be at the generic pnfs level.
 */
static struct pnfs_block_dev *
nfs4_blk_get_deviceinfo(struct nfs_server *server, const struct nfs_fh *fh,
			struct nfs4_deviceid *d_id)
{
	struct pnfs_device *dev;
J
Jim Rees 已提交
1044
	struct pnfs_block_dev *rv;
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
	u32 max_resp_sz;
	int max_pages;
	struct page **pages = NULL;
	int i, rc;

	/*
	 * Use the session max response size as the basis for setting
	 * GETDEVICEINFO's maxcount
	 */
	max_resp_sz = server->nfs_client->cl_session->fc_attrs.max_resp_sz;
1055
	max_pages = nfs_page_array_len(0, max_resp_sz);
1056 1057 1058 1059 1060 1061
	dprintk("%s max_resp_sz %u max_pages %d\n",
		__func__, max_resp_sz, max_pages);

	dev = kmalloc(sizeof(*dev), GFP_NOFS);
	if (!dev) {
		dprintk("%s kmalloc failed\n", __func__);
J
Jim Rees 已提交
1062
		return ERR_PTR(-ENOMEM);
1063 1064
	}

1065
	pages = kcalloc(max_pages, sizeof(struct page *), GFP_NOFS);
1066 1067
	if (pages == NULL) {
		kfree(dev);
J
Jim Rees 已提交
1068
		return ERR_PTR(-ENOMEM);
1069 1070 1071
	}
	for (i = 0; i < max_pages; i++) {
		pages[i] = alloc_page(GFP_NOFS);
J
Jim Rees 已提交
1072 1073
		if (!pages[i]) {
			rv = ERR_PTR(-ENOMEM);
1074
			goto out_free;
J
Jim Rees 已提交
1075
		}
1076 1077 1078 1079 1080 1081 1082 1083
	}

	memcpy(&dev->dev_id, d_id, sizeof(*d_id));
	dev->layout_type = LAYOUT_BLOCK_VOLUME;
	dev->pages = pages;
	dev->pgbase = 0;
	dev->pglen = PAGE_SIZE * max_pages;
	dev->mincount = 0;
1084
	dev->maxcount = max_resp_sz - nfs41_maxgetdevinfo_overhead;
1085 1086

	dprintk("%s: dev_id: %s\n", __func__, dev->dev_id.data);
1087
	rc = nfs4_proc_getdeviceinfo(server, dev, NULL);
1088
	dprintk("%s getdevice info returns %d\n", __func__, rc);
J
Jim Rees 已提交
1089 1090
	if (rc) {
		rv = ERR_PTR(rc);
1091
		goto out_free;
J
Jim Rees 已提交
1092
	}
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102

	rv = nfs4_blk_decode_device(server, dev);
 out_free:
	for (i = 0; i < max_pages; i++)
		__free_page(pages[i]);
	kfree(pages);
	kfree(dev);
	return rv;
}

1103 1104 1105
static int
bl_set_layoutdriver(struct nfs_server *server, const struct nfs_fh *fh)
{
1106 1107 1108 1109
	struct block_mount_id *b_mt_id = NULL;
	struct pnfs_devicelist *dlist = NULL;
	struct pnfs_block_dev *bdev;
	LIST_HEAD(block_disklist);
J
Jim Rees 已提交
1110
	int status, i;
1111

1112
	dprintk("%s enter\n", __func__);
1113 1114 1115 1116 1117

	if (server->pnfs_blksize == 0) {
		dprintk("%s Server did not return blksize\n", __func__);
		return -EINVAL;
	}
1118 1119 1120 1121 1122 1123
	if (server->pnfs_blksize > PAGE_SIZE) {
		printk(KERN_ERR "%s: pNFS blksize %d not supported.\n",
			__func__, server->pnfs_blksize);
		return -EINVAL;
	}

1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
	b_mt_id = kzalloc(sizeof(struct block_mount_id), GFP_NOFS);
	if (!b_mt_id) {
		status = -ENOMEM;
		goto out_error;
	}
	/* Initialize nfs4 block layout mount id */
	spin_lock_init(&b_mt_id->bm_lock);
	INIT_LIST_HEAD(&b_mt_id->bm_devlist);

	dlist = kmalloc(sizeof(struct pnfs_devicelist), GFP_NOFS);
	if (!dlist) {
		status = -ENOMEM;
		goto out_error;
	}
	dlist->eof = 0;
	while (!dlist->eof) {
		status = nfs4_proc_getdevicelist(server, fh, dlist);
		if (status)
			goto out_error;
		dprintk("%s GETDEVICELIST numdevs=%i, eof=%i\n",
			__func__, dlist->num_devs, dlist->eof);
		for (i = 0; i < dlist->num_devs; i++) {
			bdev = nfs4_blk_get_deviceinfo(server, fh,
						       &dlist->dev_id[i]);
J
Jim Rees 已提交
1148 1149
			if (IS_ERR(bdev)) {
				status = PTR_ERR(bdev);
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
				goto out_error;
			}
			spin_lock(&b_mt_id->bm_lock);
			list_add(&bdev->bm_node, &b_mt_id->bm_devlist);
			spin_unlock(&b_mt_id->bm_lock);
		}
	}
	dprintk("%s SUCCESS\n", __func__);
	server->pnfs_ld_data = b_mt_id;

 out_return:
	kfree(dlist);
	return status;

 out_error:
	free_blk_mountid(b_mt_id);
	goto out_return;
1167 1168 1169 1170 1171
}

static int
bl_clear_layoutdriver(struct nfs_server *server)
{
1172 1173
	struct block_mount_id *b_mt_id = server->pnfs_ld_data;

1174
	dprintk("%s enter\n", __func__);
1175 1176
	free_blk_mountid(b_mt_id);
	dprintk("%s RETURNS\n", __func__);
1177 1178 1179
	return 0;
}

P
Peng Tao 已提交
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
static bool
is_aligned_req(struct nfs_page *req, unsigned int alignment)
{
	return IS_ALIGNED(req->wb_offset, alignment) &&
	       IS_ALIGNED(req->wb_bytes, alignment);
}

static void
bl_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
	if (pgio->pg_dreq != NULL &&
	    !is_aligned_req(req, SECTOR_SIZE))
		nfs_pageio_reset_read_mds(pgio);
	else
		pnfs_generic_pg_init_read(pgio, req);
}

1197 1198 1199 1200 1201
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
static size_t
P
Peng Tao 已提交
1202 1203 1204 1205 1206
bl_pg_test_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
		struct nfs_page *req)
{
	if (pgio->pg_dreq != NULL &&
	    !is_aligned_req(req, SECTOR_SIZE))
1207
		return 0;
P
Peng Tao 已提交
1208 1209 1210 1211

	return pnfs_generic_pg_test(pgio, prev, req);
}

1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
/*
 * Return the number of contiguous bytes for a given inode
 * starting at page frame idx.
 */
static u64 pnfs_num_cont_bytes(struct inode *inode, pgoff_t idx)
{
	struct address_space *mapping = inode->i_mapping;
	pgoff_t end;

	/* Optimize common case that writes from 0 to end of file */
	end = DIV_ROUND_UP(i_size_read(inode), PAGE_CACHE_SIZE);
	if (end != NFS_I(inode)->npages) {
		rcu_read_lock();
1225
		end = page_cache_next_hole(mapping, idx + 1, ULONG_MAX);
1226 1227 1228 1229 1230 1231 1232 1233 1234
		rcu_read_unlock();
	}

	if (!end)
		return i_size_read(inode) - (idx << PAGE_CACHE_SHIFT);
	else
		return (end - idx) << PAGE_CACHE_SHIFT;
}

1235
static void
P
Peng Tao 已提交
1236 1237 1238
bl_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
{
	if (pgio->pg_dreq != NULL &&
1239
	    !is_aligned_req(req, PAGE_CACHE_SIZE)) {
P
Peng Tao 已提交
1240
		nfs_pageio_reset_write_mds(pgio);
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
	} else {
		u64 wb_size;
		if (pgio->pg_dreq == NULL)
			wb_size = pnfs_num_cont_bytes(pgio->pg_inode,
						      req->wb_index);
		else
			wb_size = nfs_dreq_bytes_left(pgio->pg_dreq);

		pnfs_generic_pg_init_write(pgio, req, wb_size);
	}
P
Peng Tao 已提交
1251 1252
}

1253 1254 1255 1256 1257
/*
 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
 * of bytes (maximum @req->wb_bytes) that can be coalesced.
 */
static size_t
P
Peng Tao 已提交
1258 1259 1260 1261 1262
bl_pg_test_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
		 struct nfs_page *req)
{
	if (pgio->pg_dreq != NULL &&
	    !is_aligned_req(req, PAGE_CACHE_SIZE))
1263
		return 0;
P
Peng Tao 已提交
1264 1265 1266 1267

	return pnfs_generic_pg_test(pgio, prev, req);
}

B
Benny Halevy 已提交
1268
static const struct nfs_pageio_ops bl_pg_read_ops = {
P
Peng Tao 已提交
1269 1270
	.pg_init = bl_pg_init_read,
	.pg_test = bl_pg_test_read,
B
Benny Halevy 已提交
1271 1272 1273 1274
	.pg_doio = pnfs_generic_pg_readpages,
};

static const struct nfs_pageio_ops bl_pg_write_ops = {
P
Peng Tao 已提交
1275 1276
	.pg_init = bl_pg_init_write,
	.pg_test = bl_pg_test_write,
B
Benny Halevy 已提交
1277 1278 1279
	.pg_doio = pnfs_generic_pg_writepages,
};

1280 1281 1282
static struct pnfs_layoutdriver_type blocklayout_type = {
	.id				= LAYOUT_BLOCK_VOLUME,
	.name				= "LAYOUT_BLOCK_VOLUME",
1283
	.owner				= THIS_MODULE,
1284 1285 1286 1287 1288 1289 1290 1291 1292 1293
	.read_pagelist			= bl_read_pagelist,
	.write_pagelist			= bl_write_pagelist,
	.alloc_layout_hdr		= bl_alloc_layout_hdr,
	.free_layout_hdr		= bl_free_layout_hdr,
	.alloc_lseg			= bl_alloc_lseg,
	.free_lseg			= bl_free_lseg,
	.encode_layoutcommit		= bl_encode_layoutcommit,
	.cleanup_layoutcommit		= bl_cleanup_layoutcommit,
	.set_layoutdriver		= bl_set_layoutdriver,
	.clear_layoutdriver		= bl_clear_layoutdriver,
B
Benny Halevy 已提交
1294 1295
	.pg_read_ops			= &bl_pg_read_ops,
	.pg_write_ops			= &bl_pg_write_ops,
1296 1297
};

J
Jim Rees 已提交
1298
static const struct rpc_pipe_ops bl_upcall_ops = {
1299
	.upcall		= rpc_pipe_generic_upcall,
J
Jim Rees 已提交
1300 1301 1302 1303
	.downcall	= bl_pipe_downcall,
	.destroy_msg	= bl_pipe_destroy_msg,
};

1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
static struct dentry *nfs4blocklayout_register_sb(struct super_block *sb,
					    struct rpc_pipe *pipe)
{
	struct dentry *dir, *dentry;

	dir = rpc_d_lookup_sb(sb, NFS_PIPE_DIRNAME);
	if (dir == NULL)
		return ERR_PTR(-ENOENT);
	dentry = rpc_mkpipe_dentry(dir, "blocklayout", NULL, pipe);
	dput(dir);
	return dentry;
}

static void nfs4blocklayout_unregister_sb(struct super_block *sb,
					  struct rpc_pipe *pipe)
{
	if (pipe->dentry)
		rpc_unlink(pipe->dentry);
}

1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365
static int rpc_pipefs_event(struct notifier_block *nb, unsigned long event,
			   void *ptr)
{
	struct super_block *sb = ptr;
	struct net *net = sb->s_fs_info;
	struct nfs_net *nn = net_generic(net, nfs_net_id);
	struct dentry *dentry;
	int ret = 0;

	if (!try_module_get(THIS_MODULE))
		return 0;

	if (nn->bl_device_pipe == NULL) {
		module_put(THIS_MODULE);
		return 0;
	}

	switch (event) {
	case RPC_PIPEFS_MOUNT:
		dentry = nfs4blocklayout_register_sb(sb, nn->bl_device_pipe);
		if (IS_ERR(dentry)) {
			ret = PTR_ERR(dentry);
			break;
		}
		nn->bl_device_pipe->dentry = dentry;
		break;
	case RPC_PIPEFS_UMOUNT:
		if (nn->bl_device_pipe->dentry)
			nfs4blocklayout_unregister_sb(sb, nn->bl_device_pipe);
		break;
	default:
		ret = -ENOTSUPP;
		break;
	}
	module_put(THIS_MODULE);
	return ret;
}

static struct notifier_block nfs4blocklayout_block = {
	.notifier_call = rpc_pipefs_event,
};

1366 1367 1368 1369 1370 1371 1372 1373
static struct dentry *nfs4blocklayout_register_net(struct net *net,
						   struct rpc_pipe *pipe)
{
	struct super_block *pipefs_sb;
	struct dentry *dentry;

	pipefs_sb = rpc_get_sb_net(net);
	if (!pipefs_sb)
1374
		return NULL;
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
	dentry = nfs4blocklayout_register_sb(pipefs_sb, pipe);
	rpc_put_sb_net(net);
	return dentry;
}

static void nfs4blocklayout_unregister_net(struct net *net,
					   struct rpc_pipe *pipe)
{
	struct super_block *pipefs_sb;

	pipefs_sb = rpc_get_sb_net(net);
	if (pipefs_sb) {
		nfs4blocklayout_unregister_sb(pipefs_sb, pipe);
		rpc_put_sb_net(net);
	}
}

1392 1393 1394 1395 1396
static int nfs4blocklayout_net_init(struct net *net)
{
	struct nfs_net *nn = net_generic(net, nfs_net_id);
	struct dentry *dentry;

1397
	init_waitqueue_head(&nn->bl_wq);
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
	nn->bl_device_pipe = rpc_mkpipe_data(&bl_upcall_ops, 0);
	if (IS_ERR(nn->bl_device_pipe))
		return PTR_ERR(nn->bl_device_pipe);
	dentry = nfs4blocklayout_register_net(net, nn->bl_device_pipe);
	if (IS_ERR(dentry)) {
		rpc_destroy_pipe_data(nn->bl_device_pipe);
		return PTR_ERR(dentry);
	}
	nn->bl_device_pipe->dentry = dentry;
	return 0;
}

static void nfs4blocklayout_net_exit(struct net *net)
{
	struct nfs_net *nn = net_generic(net, nfs_net_id);

	nfs4blocklayout_unregister_net(net, nn->bl_device_pipe);
	rpc_destroy_pipe_data(nn->bl_device_pipe);
	nn->bl_device_pipe = NULL;
}

static struct pernet_operations nfs4blocklayout_net_ops = {
	.init = nfs4blocklayout_net_init,
	.exit = nfs4blocklayout_net_exit,
};

1424 1425 1426 1427 1428 1429 1430
static int __init nfs4blocklayout_init(void)
{
	int ret;

	dprintk("%s: NFSv4 Block Layout Driver Registering...\n", __func__);

	ret = pnfs_register_layoutdriver(&blocklayout_type);
J
Jim Rees 已提交
1431 1432 1433
	if (ret)
		goto out;

1434
	ret = rpc_pipefs_notifier_register(&nfs4blocklayout_block);
1435 1436
	if (ret)
		goto out_remove;
1437 1438 1439
	ret = register_pernet_subsys(&nfs4blocklayout_net_ops);
	if (ret)
		goto out_notifier;
J
Jim Rees 已提交
1440 1441 1442
out:
	return ret;

1443 1444
out_notifier:
	rpc_pipefs_notifier_unregister(&nfs4blocklayout_block);
J
Jim Rees 已提交
1445 1446
out_remove:
	pnfs_unregister_layoutdriver(&blocklayout_type);
1447 1448 1449 1450 1451 1452 1453 1454
	return ret;
}

static void __exit nfs4blocklayout_exit(void)
{
	dprintk("%s: NFSv4 Block Layout Driver Unregistering...\n",
	       __func__);

1455
	rpc_pipefs_notifier_unregister(&nfs4blocklayout_block);
1456
	unregister_pernet_subsys(&nfs4blocklayout_net_ops);
1457 1458 1459 1460 1461 1462 1463
	pnfs_unregister_layoutdriver(&blocklayout_type);
}

MODULE_ALIAS("nfs-layouttype4-3");

module_init(nfs4blocklayout_init);
module_exit(nfs4blocklayout_exit);