dax.c 34.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * fs/dax.c - Direct Access filesystem code
 * Copyright (c) 2013-2014 Intel Corporation
 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/atomic.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
20
#include <linux/dax.h>
21 22
#include <linux/fs.h>
#include <linux/genhd.h>
23 24 25
#include <linux/highmem.h>
#include <linux/memcontrol.h>
#include <linux/mm.h>
26
#include <linux/mutex.h>
R
Ross Zwisler 已提交
27
#include <linux/pagevec.h>
28
#include <linux/pmem.h>
29
#include <linux/sched.h>
30
#include <linux/uio.h>
31
#include <linux/vmstat.h>
D
Dan Williams 已提交
32
#include <linux/pfn_t.h>
33
#include <linux/sizes.h>
34

35 36 37 38 39 40 41 42 43
/*
 * We use lowest available bit in exceptional entry for locking, other two
 * bits to determine entry type. In total 3 special bits.
 */
#define RADIX_DAX_SHIFT	(RADIX_TREE_EXCEPTIONAL_SHIFT + 3)
#define RADIX_DAX_PTE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
#define RADIX_DAX_TYPE_MASK (RADIX_DAX_PTE | RADIX_DAX_PMD)
#define RADIX_DAX_TYPE(entry) ((unsigned long)entry & RADIX_DAX_TYPE_MASK)
44 45
#define RADIX_DAX_SECTOR(entry) (((unsigned long)entry >> RADIX_DAX_SHIFT))
#define RADIX_DAX_ENTRY(sector, pmd) ((void *)((unsigned long)sector << \
46 47
		RADIX_DAX_SHIFT | (pmd ? RADIX_DAX_PMD : RADIX_DAX_PTE) | \
		RADIX_TREE_EXCEPTIONAL_ENTRY))
48

J
Jan Kara 已提交
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
/* We choose 4096 entries - same as per-zone page wait tables */
#define DAX_WAIT_TABLE_BITS 12
#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)

wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];

static int __init init_dax_wait_table(void)
{
	int i;

	for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
		init_waitqueue_head(wait_table + i);
	return 0;
}
fs_initcall(init_dax_wait_table);

static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
					      pgoff_t index)
{
	unsigned long hash = hash_long((unsigned long)mapping ^ index,
				       DAX_WAIT_TABLE_BITS);
	return wait_table + hash;
}
72

73 74 75 76 77
static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax)
{
	struct request_queue *q = bdev->bd_queue;
	long rc = -EIO;

D
Dan Williams 已提交
78
	dax->addr = ERR_PTR(-EIO);
79 80 81 82 83
	if (blk_queue_enter(q, true) != 0)
		return rc;

	rc = bdev_direct_access(bdev, dax);
	if (rc < 0) {
D
Dan Williams 已提交
84
		dax->addr = ERR_PTR(rc);
85 86 87 88 89 90 91 92 93 94 95 96 97 98
		blk_queue_exit(q);
		return rc;
	}
	return rc;
}

static void dax_unmap_atomic(struct block_device *bdev,
		const struct blk_dax_ctl *dax)
{
	if (IS_ERR(dax->addr))
		return;
	blk_queue_exit(bdev->bd_queue);
}

99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
struct page *read_dax_sector(struct block_device *bdev, sector_t n)
{
	struct page *page = alloc_pages(GFP_KERNEL, 0);
	struct blk_dax_ctl dax = {
		.size = PAGE_SIZE,
		.sector = n & ~((((int) PAGE_SIZE) / 512) - 1),
	};
	long rc;

	if (!page)
		return ERR_PTR(-ENOMEM);

	rc = dax_map_atomic(bdev, &dax);
	if (rc < 0)
		return ERR_PTR(rc);
	memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE);
	dax_unmap_atomic(bdev, &dax);
	return page;
}

119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
static bool buffer_written(struct buffer_head *bh)
{
	return buffer_mapped(bh) && !buffer_unwritten(bh);
}

/*
 * When ext4 encounters a hole, it returns without modifying the buffer_head
 * which means that we can't trust b_size.  To cope with this, we set b_state
 * to 0 before calling get_block and, if any bit is set, we know we can trust
 * b_size.  Unfortunate, really, since ext4 knows precisely how long a hole is
 * and would save us time calling get_block repeatedly.
 */
static bool buffer_size_valid(struct buffer_head *bh)
{
	return bh->b_state != 0;
}

136 137 138 139 140 141 142 143 144

static sector_t to_sector(const struct buffer_head *bh,
		const struct inode *inode)
{
	sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9);

	return sector;
}

O
Omar Sandoval 已提交
145 146 147
static ssize_t dax_io(struct inode *inode, struct iov_iter *iter,
		      loff_t start, loff_t end, get_block_t get_block,
		      struct buffer_head *bh)
148
{
149
	loff_t pos = start, max = start, bh_max = start;
D
Dan Williams 已提交
150
	bool hole = false;
151 152 153 154
	struct block_device *bdev = NULL;
	int rw = iov_iter_rw(iter), rc;
	long map_len = 0;
	struct blk_dax_ctl dax = {
D
Dan Williams 已提交
155
		.addr = ERR_PTR(-EIO),
156
	};
J
Jan Kara 已提交
157 158 159
	unsigned blkbits = inode->i_blkbits;
	sector_t file_blks = (i_size_read(inode) + (1 << blkbits) - 1)
								>> blkbits;
160 161

	if (rw == READ)
162 163 164
		end = min(end, i_size_read(inode));

	while (pos < end) {
165
		size_t len;
166
		if (pos == max) {
167 168
			long page = pos >> PAGE_SHIFT;
			sector_t block = page << (PAGE_SHIFT - blkbits);
169 170 171 172 173 174
			unsigned first = pos - (block << blkbits);
			long size;

			if (pos == bh_max) {
				bh->b_size = PAGE_ALIGN(end - pos);
				bh->b_state = 0;
175 176
				rc = get_block(inode, block, bh, rw == WRITE);
				if (rc)
177 178 179 180
					break;
				if (!buffer_size_valid(bh))
					bh->b_size = 1 << blkbits;
				bh_max = pos - first + bh->b_size;
181
				bdev = bh->b_bdev;
J
Jan Kara 已提交
182 183 184 185 186 187 188
				/*
				 * We allow uninitialized buffers for writes
				 * beyond EOF as those cannot race with faults
				 */
				WARN_ON_ONCE(
					(buffer_new(bh) && block < file_blks) ||
					(rw == WRITE && buffer_unwritten(bh)));
189 190 191 192 193 194 195
			} else {
				unsigned done = bh->b_size -
						(bh_max - (pos - first));
				bh->b_blocknr += done >> blkbits;
				bh->b_size -= done;
			}

196
			hole = rw == READ && !buffer_written(bh);
197 198 199
			if (hole) {
				size = bh->b_size - first;
			} else {
200 201 202 203 204 205
				dax_unmap_atomic(bdev, &dax);
				dax.sector = to_sector(bh, inode);
				dax.size = bh->b_size;
				map_len = dax_map_atomic(bdev, &dax);
				if (map_len < 0) {
					rc = map_len;
206
					break;
207 208 209
				}
				dax.addr += first;
				size = map_len - first;
210
			}
211 212 213 214 215 216
			/*
			 * pos + size is one past the last offset for IO,
			 * so pos + size can overflow loff_t at extreme offsets.
			 * Cast to u64 to catch this and get the true minimum.
			 */
			max = min_t(u64, pos + size, end);
217 218
		}

219
		if (iov_iter_rw(iter) == WRITE) {
220
			len = copy_from_iter_pmem(dax.addr, max - pos, iter);
221
		} else if (!hole)
222
			len = copy_to_iter((void __force *) dax.addr, max - pos,
223
					iter);
224 225 226
		else
			len = iov_iter_zero(max - pos, iter);

227
		if (!len) {
228
			rc = -EFAULT;
229
			break;
230
		}
231 232

		pos += len;
233 234
		if (!IS_ERR(dax.addr))
			dax.addr += len;
235 236
	}

237
	dax_unmap_atomic(bdev, &dax);
238

239
	return (pos == start) ? rc : pos - start;
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
}

/**
 * dax_do_io - Perform I/O to a DAX file
 * @iocb: The control block for this I/O
 * @inode: The file which the I/O is directed at
 * @iter: The addresses to do I/O from or to
 * @get_block: The filesystem method used to translate file offsets to blocks
 * @end_io: A filesystem callback for I/O completion
 * @flags: See below
 *
 * This function uses the same locking scheme as do_blockdev_direct_IO:
 * If @flags has DIO_LOCKING set, we assume that the i_mutex is held by the
 * caller for writes.  For reads, we take and release the i_mutex ourselves.
 * If DIO_LOCKING is not set, the filesystem takes care of its own locking.
 * As with do_blockdev_direct_IO(), we increment i_dio_count while the I/O
 * is in progress.
 */
O
Omar Sandoval 已提交
258
ssize_t dax_do_io(struct kiocb *iocb, struct inode *inode,
259
		  struct iov_iter *iter, get_block_t get_block,
O
Omar Sandoval 已提交
260
		  dio_iodone_t end_io, int flags)
261 262 263
{
	struct buffer_head bh;
	ssize_t retval = -EINVAL;
264
	loff_t pos = iocb->ki_pos;
265 266 267
	loff_t end = pos + iov_iter_count(iter);

	memset(&bh, 0, sizeof(bh));
268
	bh.b_bdev = inode->i_sb->s_bdev;
269

270
	if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ)
A
Al Viro 已提交
271
		inode_lock(inode);
272 273

	/* Protects against truncate */
274 275
	if (!(flags & DIO_SKIP_DIO_COUNT))
		inode_dio_begin(inode);
276

O
Omar Sandoval 已提交
277
	retval = dax_io(inode, iter, pos, end, get_block, &bh);
278

O
Omar Sandoval 已提交
279
	if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ)
A
Al Viro 已提交
280
		inode_unlock(inode);
281

282 283 284 285 286 287 288
	if (end_io) {
		int err;

		err = end_io(iocb, pos, retval, bh.b_private);
		if (err)
			retval = err;
	}
289

290 291
	if (!(flags & DIO_SKIP_DIO_COUNT))
		inode_dio_end(inode);
292 293 294
	return retval;
}
EXPORT_SYMBOL_GPL(dax_do_io);
295

J
Jan Kara 已提交
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
/*
 * DAX radix tree locking
 */
struct exceptional_entry_key {
	struct address_space *mapping;
	unsigned long index;
};

struct wait_exceptional_entry_queue {
	wait_queue_t wait;
	struct exceptional_entry_key key;
};

static int wake_exceptional_entry_func(wait_queue_t *wait, unsigned int mode,
				       int sync, void *keyp)
{
	struct exceptional_entry_key *key = keyp;
	struct wait_exceptional_entry_queue *ewait =
		container_of(wait, struct wait_exceptional_entry_queue, wait);

	if (key->mapping != ewait->key.mapping ||
	    key->index != ewait->key.index)
		return 0;
	return autoremove_wake_function(wait, mode, sync, NULL);
}

/*
 * Check whether the given slot is locked. The function must be called with
 * mapping->tree_lock held
 */
static inline int slot_locked(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
	return entry & RADIX_DAX_ENTRY_LOCK;
}

/*
 * Mark the given slot is locked. The function must be called with
 * mapping->tree_lock held
 */
static inline void *lock_slot(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

	entry |= RADIX_DAX_ENTRY_LOCK;
	radix_tree_replace_slot(slot, (void *)entry);
	return (void *)entry;
}

/*
 * Mark the given slot is unlocked. The function must be called with
 * mapping->tree_lock held
 */
static inline void *unlock_slot(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

	entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
	radix_tree_replace_slot(slot, (void *)entry);
	return (void *)entry;
}

/*
 * Lookup entry in radix tree, wait for it to become unlocked if it is
 * exceptional entry and return it. The caller must call
 * put_unlocked_mapping_entry() when he decided not to lock the entry or
 * put_locked_mapping_entry() when he locked the entry and now wants to
 * unlock it.
 *
 * The function must be called with mapping->tree_lock held.
 */
static void *get_unlocked_mapping_entry(struct address_space *mapping,
					pgoff_t index, void ***slotp)
{
	void *ret, **slot;
	struct wait_exceptional_entry_queue ewait;
	wait_queue_head_t *wq = dax_entry_waitqueue(mapping, index);

	init_wait(&ewait.wait);
	ewait.wait.func = wake_exceptional_entry_func;
	ewait.key.mapping = mapping;
	ewait.key.index = index;

	for (;;) {
		ret = __radix_tree_lookup(&mapping->page_tree, index, NULL,
					  &slot);
		if (!ret || !radix_tree_exceptional_entry(ret) ||
		    !slot_locked(mapping, slot)) {
			if (slotp)
				*slotp = slot;
			return ret;
		}
		prepare_to_wait_exclusive(wq, &ewait.wait,
					  TASK_UNINTERRUPTIBLE);
		spin_unlock_irq(&mapping->tree_lock);
		schedule();
		finish_wait(wq, &ewait.wait);
		spin_lock_irq(&mapping->tree_lock);
	}
}

/*
 * Find radix tree entry at given index. If it points to a page, return with
 * the page locked. If it points to the exceptional entry, return with the
 * radix tree entry locked. If the radix tree doesn't contain given index,
 * create empty exceptional entry for the index and return with it locked.
 *
 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
 * persistent memory the benefit is doubtful. We can add that later if we can
 * show it helps.
 */
static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index)
{
	void *ret, **slot;

restart:
	spin_lock_irq(&mapping->tree_lock);
	ret = get_unlocked_mapping_entry(mapping, index, &slot);
	/* No entry for given index? Make sure radix tree is big enough. */
	if (!ret) {
		int err;

		spin_unlock_irq(&mapping->tree_lock);
		err = radix_tree_preload(
				mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
		if (err)
			return ERR_PTR(err);
		ret = (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
			       RADIX_DAX_ENTRY_LOCK);
		spin_lock_irq(&mapping->tree_lock);
		err = radix_tree_insert(&mapping->page_tree, index, ret);
		radix_tree_preload_end();
		if (err) {
			spin_unlock_irq(&mapping->tree_lock);
			/* Someone already created the entry? */
			if (err == -EEXIST)
				goto restart;
			return ERR_PTR(err);
		}
		/* Good, we have inserted empty locked entry into the tree. */
		mapping->nrexceptional++;
		spin_unlock_irq(&mapping->tree_lock);
		return ret;
	}
	/* Normal page in radix tree? */
	if (!radix_tree_exceptional_entry(ret)) {
		struct page *page = ret;

		get_page(page);
		spin_unlock_irq(&mapping->tree_lock);
		lock_page(page);
		/* Page got truncated? Retry... */
		if (unlikely(page->mapping != mapping)) {
			unlock_page(page);
			put_page(page);
			goto restart;
		}
		return page;
	}
	ret = lock_slot(mapping, slot);
	spin_unlock_irq(&mapping->tree_lock);
	return ret;
}

void dax_wake_mapping_entry_waiter(struct address_space *mapping,
				   pgoff_t index, bool wake_all)
{
	wait_queue_head_t *wq = dax_entry_waitqueue(mapping, index);

	/*
	 * Checking for locked entry and prepare_to_wait_exclusive() happens
	 * under mapping->tree_lock, ditto for entry handling in our callers.
	 * So at this point all tasks that could have seen our entry locked
	 * must be in the waitqueue and the following check will see them.
	 */
	if (waitqueue_active(wq)) {
		struct exceptional_entry_key key;

		key.mapping = mapping;
		key.index = index;
		__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
	}
}

483
void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index)
J
Jan Kara 已提交
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
{
	void *ret, **slot;

	spin_lock_irq(&mapping->tree_lock);
	ret = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
	if (WARN_ON_ONCE(!ret || !radix_tree_exceptional_entry(ret) ||
			 !slot_locked(mapping, slot))) {
		spin_unlock_irq(&mapping->tree_lock);
		return;
	}
	unlock_slot(mapping, slot);
	spin_unlock_irq(&mapping->tree_lock);
	dax_wake_mapping_entry_waiter(mapping, index, false);
}

static void put_locked_mapping_entry(struct address_space *mapping,
				     pgoff_t index, void *entry)
{
	if (!radix_tree_exceptional_entry(entry)) {
		unlock_page(entry);
		put_page(entry);
	} else {
506
		dax_unlock_mapping_entry(mapping, index);
J
Jan Kara 已提交
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
	}
}

/*
 * Called when we are done with radix tree entry we looked up via
 * get_unlocked_mapping_entry() and which we didn't lock in the end.
 */
static void put_unlocked_mapping_entry(struct address_space *mapping,
				       pgoff_t index, void *entry)
{
	if (!radix_tree_exceptional_entry(entry))
		return;

	/* We have to wake up next waiter for the radix tree entry lock */
	dax_wake_mapping_entry_waiter(mapping, index, false);
}

/*
 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
 * entry to get unlocked before deleting it.
 */
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
{
	void *entry;

	spin_lock_irq(&mapping->tree_lock);
	entry = get_unlocked_mapping_entry(mapping, index, NULL);
	/*
	 * This gets called from truncate / punch_hole path. As such, the caller
	 * must hold locks protecting against concurrent modifications of the
	 * radix tree (usually fs-private i_mmap_sem for writing). Since the
	 * caller has seen exceptional entry for this index, we better find it
	 * at that index as well...
	 */
	if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry))) {
		spin_unlock_irq(&mapping->tree_lock);
		return 0;
	}
	radix_tree_delete(&mapping->page_tree, index);
	mapping->nrexceptional--;
	spin_unlock_irq(&mapping->tree_lock);
	dax_wake_mapping_entry_waiter(mapping, index, true);

	return 1;
}

553 554 555 556 557 558 559 560
/*
 * The user has performed a load from a hole in the file.  Allocating
 * a new page in the file would cause excessive storage usage for
 * workloads with sparse files.  We allocate a page cache page instead.
 * We'll kick it out of the page cache if it's ever written to,
 * otherwise it will simply fall out of the page cache under memory
 * pressure without ever having been dirtied.
 */
J
Jan Kara 已提交
561 562
static int dax_load_hole(struct address_space *mapping, void *entry,
			 struct vm_fault *vmf)
563
{
J
Jan Kara 已提交
564
	struct page *page;
565

J
Jan Kara 已提交
566 567 568 569 570
	/* Hole page already exists? Return it...  */
	if (!radix_tree_exceptional_entry(entry)) {
		vmf->page = entry;
		return VM_FAULT_LOCKED;
	}
571

J
Jan Kara 已提交
572 573 574 575 576 577 578
	/* This will replace locked radix tree entry with a hole page */
	page = find_or_create_page(mapping, vmf->pgoff,
				   vmf->gfp_mask | __GFP_ZERO);
	if (!page) {
		put_locked_mapping_entry(mapping, vmf->pgoff, entry);
		return VM_FAULT_OOM;
	}
579 580 581 582
	vmf->page = page;
	return VM_FAULT_LOCKED;
}

583 584
static int copy_user_bh(struct page *to, struct inode *inode,
		struct buffer_head *bh, unsigned long vaddr)
585
{
586 587 588 589 590
	struct blk_dax_ctl dax = {
		.sector = to_sector(bh, inode),
		.size = bh->b_size,
	};
	struct block_device *bdev = bh->b_bdev;
591 592
	void *vto;

593 594
	if (dax_map_atomic(bdev, &dax) < 0)
		return PTR_ERR(dax.addr);
595
	vto = kmap_atomic(to);
596
	copy_user_page(vto, (void __force *)dax.addr, vaddr, to);
597
	kunmap_atomic(vto);
598
	dax_unmap_atomic(bdev, &dax);
599 600 601
	return 0;
}

602
#define DAX_PMD_INDEX(page_index) (page_index & (PMD_MASK >> PAGE_SHIFT))
R
Ross Zwisler 已提交
603

J
Jan Kara 已提交
604 605 606
static void *dax_insert_mapping_entry(struct address_space *mapping,
				      struct vm_fault *vmf,
				      void *entry, sector_t sector)
R
Ross Zwisler 已提交
607 608
{
	struct radix_tree_root *page_tree = &mapping->page_tree;
J
Jan Kara 已提交
609 610 611 612
	int error = 0;
	bool hole_fill = false;
	void *new_entry;
	pgoff_t index = vmf->pgoff;
R
Ross Zwisler 已提交
613

J
Jan Kara 已提交
614
	if (vmf->flags & FAULT_FLAG_WRITE)
615
		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
R
Ross Zwisler 已提交
616

J
Jan Kara 已提交
617 618 619 620 621 622 623 624 625 626 627 628
	/* Replacing hole page with block mapping? */
	if (!radix_tree_exceptional_entry(entry)) {
		hole_fill = true;
		/*
		 * Unmap the page now before we remove it from page cache below.
		 * The page is locked so it cannot be faulted in again.
		 */
		unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
				    PAGE_SIZE, 0);
		error = radix_tree_preload(vmf->gfp_mask & ~__GFP_HIGHMEM);
		if (error)
			return ERR_PTR(error);
R
Ross Zwisler 已提交
629 630
	}

J
Jan Kara 已提交
631 632 633 634 635 636 637 638 639 640
	spin_lock_irq(&mapping->tree_lock);
	new_entry = (void *)((unsigned long)RADIX_DAX_ENTRY(sector, false) |
		       RADIX_DAX_ENTRY_LOCK);
	if (hole_fill) {
		__delete_from_page_cache(entry, NULL);
		/* Drop pagecache reference */
		put_page(entry);
		error = radix_tree_insert(page_tree, index, new_entry);
		if (error) {
			new_entry = ERR_PTR(error);
R
Ross Zwisler 已提交
641 642
			goto unlock;
		}
J
Jan Kara 已提交
643 644 645 646
		mapping->nrexceptional++;
	} else {
		void **slot;
		void *ret;
R
Ross Zwisler 已提交
647

J
Jan Kara 已提交
648 649 650
		ret = __radix_tree_lookup(page_tree, index, NULL, &slot);
		WARN_ON_ONCE(ret != entry);
		radix_tree_replace_slot(slot, new_entry);
R
Ross Zwisler 已提交
651
	}
J
Jan Kara 已提交
652
	if (vmf->flags & FAULT_FLAG_WRITE)
R
Ross Zwisler 已提交
653 654 655
		radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
 unlock:
	spin_unlock_irq(&mapping->tree_lock);
J
Jan Kara 已提交
656 657 658 659 660 661 662 663 664 665 666 667
	if (hole_fill) {
		radix_tree_preload_end();
		/*
		 * We don't need hole page anymore, it has been replaced with
		 * locked radix tree entry now.
		 */
		if (mapping->a_ops->freepage)
			mapping->a_ops->freepage(entry);
		unlock_page(entry);
		put_page(entry);
	}
	return new_entry;
R
Ross Zwisler 已提交
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735
}

static int dax_writeback_one(struct block_device *bdev,
		struct address_space *mapping, pgoff_t index, void *entry)
{
	struct radix_tree_root *page_tree = &mapping->page_tree;
	int type = RADIX_DAX_TYPE(entry);
	struct radix_tree_node *node;
	struct blk_dax_ctl dax;
	void **slot;
	int ret = 0;

	spin_lock_irq(&mapping->tree_lock);
	/*
	 * Regular page slots are stabilized by the page lock even
	 * without the tree itself locked.  These unlocked entries
	 * need verification under the tree lock.
	 */
	if (!__radix_tree_lookup(page_tree, index, &node, &slot))
		goto unlock;
	if (*slot != entry)
		goto unlock;

	/* another fsync thread may have already written back this entry */
	if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
		goto unlock;

	if (WARN_ON_ONCE(type != RADIX_DAX_PTE && type != RADIX_DAX_PMD)) {
		ret = -EIO;
		goto unlock;
	}

	dax.sector = RADIX_DAX_SECTOR(entry);
	dax.size = (type == RADIX_DAX_PMD ? PMD_SIZE : PAGE_SIZE);
	spin_unlock_irq(&mapping->tree_lock);

	/*
	 * We cannot hold tree_lock while calling dax_map_atomic() because it
	 * eventually calls cond_resched().
	 */
	ret = dax_map_atomic(bdev, &dax);
	if (ret < 0)
		return ret;

	if (WARN_ON_ONCE(ret < dax.size)) {
		ret = -EIO;
		goto unmap;
	}

	wb_cache_pmem(dax.addr, dax.size);

	spin_lock_irq(&mapping->tree_lock);
	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
	spin_unlock_irq(&mapping->tree_lock);
 unmap:
	dax_unmap_atomic(bdev, &dax);
	return ret;

 unlock:
	spin_unlock_irq(&mapping->tree_lock);
	return ret;
}

/*
 * Flush the mapping to the persistent domain within the byte range of [start,
 * end]. This is required by data integrity operations to ensure file data is
 * on persistent storage prior to completion of the operation.
 */
736 737
int dax_writeback_mapping_range(struct address_space *mapping,
		struct block_device *bdev, struct writeback_control *wbc)
R
Ross Zwisler 已提交
738 739 740 741 742 743 744 745 746 747 748 749
{
	struct inode *inode = mapping->host;
	pgoff_t start_index, end_index, pmd_index;
	pgoff_t indices[PAGEVEC_SIZE];
	struct pagevec pvec;
	bool done = false;
	int i, ret = 0;
	void *entry;

	if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
		return -EIO;

750 751 752
	if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
		return 0;

753 754
	start_index = wbc->range_start >> PAGE_SHIFT;
	end_index = wbc->range_end >> PAGE_SHIFT;
R
Ross Zwisler 已提交
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
	pmd_index = DAX_PMD_INDEX(start_index);

	rcu_read_lock();
	entry = radix_tree_lookup(&mapping->page_tree, pmd_index);
	rcu_read_unlock();

	/* see if the start of our range is covered by a PMD entry */
	if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD)
		start_index = pmd_index;

	tag_pages_for_writeback(mapping, start_index, end_index);

	pagevec_init(&pvec, 0);
	while (!done) {
		pvec.nr = find_get_entries_tag(mapping, start_index,
				PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
				pvec.pages, indices);

		if (pvec.nr == 0)
			break;

		for (i = 0; i < pvec.nr; i++) {
			if (indices[i] > end_index) {
				done = true;
				break;
			}

			ret = dax_writeback_one(bdev, mapping, indices[i],
					pvec.pages[i]);
			if (ret < 0)
				return ret;
		}
	}
	return 0;
}
EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);

J
Jan Kara 已提交
792 793
static int dax_insert_mapping(struct address_space *mapping,
			struct buffer_head *bh, void **entryp,
794 795 796
			struct vm_area_struct *vma, struct vm_fault *vmf)
{
	unsigned long vaddr = (unsigned long)vmf->virtual_address;
797 798
	struct block_device *bdev = bh->b_bdev;
	struct blk_dax_ctl dax = {
J
Jan Kara 已提交
799
		.sector = to_sector(bh, mapping->host),
800 801
		.size = bh->b_size,
	};
J
Jan Kara 已提交
802 803
	void *ret;
	void *entry = *entryp;
804

J
Jan Kara 已提交
805 806
	if (dax_map_atomic(bdev, &dax) < 0)
		return PTR_ERR(dax.addr);
807
	dax_unmap_atomic(bdev, &dax);
808

J
Jan Kara 已提交
809
	ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector);
J
Jan Kara 已提交
810 811
	if (IS_ERR(ret))
		return PTR_ERR(ret);
J
Jan Kara 已提交
812
	*entryp = ret;
R
Ross Zwisler 已提交
813

J
Jan Kara 已提交
814
	return vm_insert_mixed(vma, vaddr, dax.pfn);
815 816
}

817
/**
R
Ross Zwisler 已提交
818
 * dax_fault - handle a page fault on a DAX file
819 820 821 822 823
 * @vma: The virtual memory area where the fault occurred
 * @vmf: The description of the fault
 * @get_block: The filesystem method used to translate file offsets to blocks
 *
 * When a page fault occurs, filesystems may call this helper in their
R
Ross Zwisler 已提交
824
 * fault handler for DAX files. dax_fault() assumes the caller has done all
825 826
 * the necessary locking for the page fault to proceed successfully.
 */
R
Ross Zwisler 已提交
827
int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
828
			get_block_t get_block)
829 830 831 832
{
	struct file *file = vma->vm_file;
	struct address_space *mapping = file->f_mapping;
	struct inode *inode = mapping->host;
J
Jan Kara 已提交
833
	void *entry;
834 835 836 837 838 839 840 841
	struct buffer_head bh;
	unsigned long vaddr = (unsigned long)vmf->virtual_address;
	unsigned blkbits = inode->i_blkbits;
	sector_t block;
	pgoff_t size;
	int error;
	int major = 0;

J
Jan Kara 已提交
842 843 844 845 846
	/*
	 * Check whether offset isn't beyond end of file now. Caller is supposed
	 * to hold locks serializing us with truncate / punch hole so this is
	 * a reliable test.
	 */
847 848 849 850 851 852
	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
	if (vmf->pgoff >= size)
		return VM_FAULT_SIGBUS;

	memset(&bh, 0, sizeof(bh));
	block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits);
853
	bh.b_bdev = inode->i_sb->s_bdev;
854 855
	bh.b_size = PAGE_SIZE;

J
Jan Kara 已提交
856 857 858 859
	entry = grab_mapping_entry(mapping, vmf->pgoff);
	if (IS_ERR(entry)) {
		error = PTR_ERR(entry);
		goto out;
860 861 862 863 864 865
	}

	error = get_block(inode, block, &bh, 0);
	if (!error && (bh.b_size < PAGE_SIZE))
		error = -EIO;		/* fs corruption? */
	if (error)
J
Jan Kara 已提交
866
		goto unlock_entry;
867 868 869 870

	if (vmf->cow_page) {
		struct page *new_page = vmf->cow_page;
		if (buffer_written(&bh))
871
			error = copy_user_bh(new_page, inode, &bh, vaddr);
872 873 874
		else
			clear_user_highpage(new_page, vaddr);
		if (error)
J
Jan Kara 已提交
875 876 877
			goto unlock_entry;
		if (!radix_tree_exceptional_entry(entry)) {
			vmf->page = entry;
878
			return VM_FAULT_LOCKED;
J
Jan Kara 已提交
879
		}
880 881
		vmf->entry = entry;
		return VM_FAULT_DAX_LOCKED;
882 883
	}

J
Jan Kara 已提交
884
	if (!buffer_mapped(&bh)) {
885 886 887 888 889 890 891 892
		if (vmf->flags & FAULT_FLAG_WRITE) {
			error = get_block(inode, block, &bh, 1);
			count_vm_event(PGMAJFAULT);
			mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
			major = VM_FAULT_MAJOR;
			if (!error && (bh.b_size < PAGE_SIZE))
				error = -EIO;
			if (error)
J
Jan Kara 已提交
893
				goto unlock_entry;
894
		} else {
J
Jan Kara 已提交
895
			return dax_load_hole(mapping, entry, vmf);
896 897 898
		}
	}

899
	/* Filesystem should not return unwritten buffers to us! */
900
	WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh));
J
Jan Kara 已提交
901 902 903
	error = dax_insert_mapping(mapping, &bh, &entry, vma, vmf);
 unlock_entry:
	put_locked_mapping_entry(mapping, vmf->pgoff, entry);
904 905 906 907 908 909 910 911 912
 out:
	if (error == -ENOMEM)
		return VM_FAULT_OOM | major;
	/* -EBUSY is fine, somebody else faulted on the same PTE */
	if ((error < 0) && (error != -EBUSY))
		return VM_FAULT_SIGBUS | major;
	return VM_FAULT_NOPAGE | major;
}
EXPORT_SYMBOL_GPL(dax_fault);
913

914
#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
915 916 917 918 919 920
/*
 * The 'colour' (ie low bits) within a PMD of a page offset.  This comes up
 * more often than one might expect in the below function.
 */
#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)

921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938
static void __dax_dbg(struct buffer_head *bh, unsigned long address,
		const char *reason, const char *fn)
{
	if (bh) {
		char bname[BDEVNAME_SIZE];
		bdevname(bh->b_bdev, bname);
		pr_debug("%s: %s addr: %lx dev %s state %lx start %lld "
			"length %zd fallback: %s\n", fn, current->comm,
			address, bname, bh->b_state, (u64)bh->b_blocknr,
			bh->b_size, reason);
	} else {
		pr_debug("%s: %s addr: %lx fallback: %s\n", fn,
			current->comm, address, reason);
	}
}

#define dax_pmd_dbg(bh, address, reason)	__dax_dbg(bh, address, reason, "dax_pmd")

R
Ross Zwisler 已提交
939 940 941 942 943 944 945 946 947 948
/**
 * dax_pmd_fault - handle a PMD fault on a DAX file
 * @vma: The virtual memory area where the fault occurred
 * @vmf: The description of the fault
 * @get_block: The filesystem method used to translate file offsets to blocks
 *
 * When a page fault occurs, filesystems may call this helper in their
 * pmd_fault handler for DAX files.
 */
int dax_pmd_fault(struct vm_area_struct *vma, unsigned long address,
949
		pmd_t *pmd, unsigned int flags, get_block_t get_block)
950 951 952 953 954 955 956 957
{
	struct file *file = vma->vm_file;
	struct address_space *mapping = file->f_mapping;
	struct inode *inode = mapping->host;
	struct buffer_head bh;
	unsigned blkbits = inode->i_blkbits;
	unsigned long pmd_addr = address & PMD_MASK;
	bool write = flags & FAULT_FLAG_WRITE;
958
	struct block_device *bdev;
959
	pgoff_t size, pgoff;
960
	sector_t block;
J
Jan Kara 已提交
961
	int result = 0;
R
Ross Zwisler 已提交
962
	bool alloc = false;
963

D
Dan Williams 已提交
964
	/* dax pmd mappings require pfn_t_devmap() */
D
Dan Williams 已提交
965 966 967
	if (!IS_ENABLED(CONFIG_FS_DAX_PMD))
		return VM_FAULT_FALLBACK;

968
	/* Fall back to PTEs if we're going to COW */
969 970
	if (write && !(vma->vm_flags & VM_SHARED)) {
		split_huge_pmd(vma, pmd, address);
971
		dax_pmd_dbg(NULL, address, "cow write");
972
		return VM_FAULT_FALLBACK;
973
	}
974
	/* If the PMD would extend outside the VMA */
975 976
	if (pmd_addr < vma->vm_start) {
		dax_pmd_dbg(NULL, address, "vma start unaligned");
977
		return VM_FAULT_FALLBACK;
978 979 980
	}
	if ((pmd_addr + PMD_SIZE) > vma->vm_end) {
		dax_pmd_dbg(NULL, address, "vma end unaligned");
981
		return VM_FAULT_FALLBACK;
982
	}
983

M
Matthew Wilcox 已提交
984
	pgoff = linear_page_index(vma, pmd_addr);
985 986 987 988
	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
	if (pgoff >= size)
		return VM_FAULT_SIGBUS;
	/* If the PMD would cover blocks out of the file */
989 990 991
	if ((pgoff | PG_PMD_COLOUR) >= size) {
		dax_pmd_dbg(NULL, address,
				"offset + huge page size > file size");
992
		return VM_FAULT_FALLBACK;
993
	}
994 995

	memset(&bh, 0, sizeof(bh));
996
	bh.b_bdev = inode->i_sb->s_bdev;
997 998 999
	block = (sector_t)pgoff << (PAGE_SHIFT - blkbits);

	bh.b_size = PMD_SIZE;
R
Ross Zwisler 已提交
1000 1001

	if (get_block(inode, block, &bh, 0) != 0)
1002
		return VM_FAULT_SIGBUS;
R
Ross Zwisler 已提交
1003 1004 1005 1006 1007

	if (!buffer_mapped(&bh) && write) {
		if (get_block(inode, block, &bh, 1) != 0)
			return VM_FAULT_SIGBUS;
		alloc = true;
1008
		WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh));
R
Ross Zwisler 已提交
1009 1010
	}

1011
	bdev = bh.b_bdev;
1012 1013 1014 1015 1016 1017

	/*
	 * If the filesystem isn't willing to tell us the length of a hole,
	 * just fall back to PTEs.  Calling get_block 512 times in a loop
	 * would be silly.
	 */
1018 1019
	if (!buffer_size_valid(&bh) || bh.b_size < PMD_SIZE) {
		dax_pmd_dbg(&bh, address, "allocated block too small");
R
Ross Zwisler 已提交
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
		return VM_FAULT_FALLBACK;
	}

	/*
	 * If we allocated new storage, make sure no process has any
	 * zero pages covering this hole
	 */
	if (alloc) {
		loff_t lstart = pgoff << PAGE_SHIFT;
		loff_t lend = lstart + PMD_SIZE - 1; /* inclusive */

		truncate_pagecache_range(inode, lstart, lend);
1032
	}
1033

1034
	if (!write && !buffer_mapped(&bh)) {
1035
		spinlock_t *ptl;
1036
		pmd_t entry;
1037
		struct page *zero_page = get_huge_zero_page();
1038

1039 1040
		if (unlikely(!zero_page)) {
			dax_pmd_dbg(&bh, address, "no zero page");
1041
			goto fallback;
1042
		}
1043

1044 1045 1046
		ptl = pmd_lock(vma->vm_mm, pmd);
		if (!pmd_none(*pmd)) {
			spin_unlock(ptl);
1047
			dax_pmd_dbg(&bh, address, "pmd already present");
1048 1049 1050
			goto fallback;
		}

1051 1052 1053 1054 1055
		dev_dbg(part_to_dev(bdev->bd_part),
				"%s: %s addr: %lx pfn: <zero> sect: %llx\n",
				__func__, current->comm, address,
				(unsigned long long) to_sector(&bh, inode));

1056 1057 1058
		entry = mk_pmd(zero_page, vma->vm_page_prot);
		entry = pmd_mkhuge(entry);
		set_pmd_at(vma->vm_mm, pmd_addr, pmd, entry);
1059
		result = VM_FAULT_NOPAGE;
1060
		spin_unlock(ptl);
1061
	} else {
1062 1063 1064 1065 1066 1067
		struct blk_dax_ctl dax = {
			.sector = to_sector(&bh, inode),
			.size = PMD_SIZE,
		};
		long length = dax_map_atomic(bdev, &dax);

1068
		if (length < 0) {
1069 1070
			dax_pmd_dbg(&bh, address, "dax-error fallback");
			goto fallback;
1071
		}
1072 1073 1074 1075 1076 1077 1078
		if (length < PMD_SIZE) {
			dax_pmd_dbg(&bh, address, "dax-length too small");
			dax_unmap_atomic(bdev, &dax);
			goto fallback;
		}
		if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR) {
			dax_pmd_dbg(&bh, address, "pfn unaligned");
1079
			dax_unmap_atomic(bdev, &dax);
1080
			goto fallback;
1081
		}
1082

D
Dan Williams 已提交
1083
		if (!pfn_t_devmap(dax.pfn)) {
1084
			dax_unmap_atomic(bdev, &dax);
1085
			dax_pmd_dbg(&bh, address, "pfn not in memmap");
D
Dan Williams 已提交
1086
			goto fallback;
1087 1088
		}
		dax_unmap_atomic(bdev, &dax);
R
Ross Zwisler 已提交
1089

R
Ross Zwisler 已提交
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
		/*
		 * For PTE faults we insert a radix tree entry for reads, and
		 * leave it clean.  Then on the first write we dirty the radix
		 * tree entry via the dax_pfn_mkwrite() path.  This sequence
		 * allows the dax_pfn_mkwrite() call to be simpler and avoid a
		 * call into get_block() to translate the pgoff to a sector in
		 * order to be able to create a new radix tree entry.
		 *
		 * The PMD path doesn't have an equivalent to
		 * dax_pfn_mkwrite(), though, so for a read followed by a
R
Ross Zwisler 已提交
1100
		 * write we traverse all the way through dax_pmd_fault()
R
Ross Zwisler 已提交
1101 1102 1103 1104 1105
		 * twice.  This means we can just skip inserting a radix tree
		 * entry completely on the initial read and just wait until
		 * the write to insert a dirty entry.
		 */
		if (write) {
J
Jan Kara 已提交
1106 1107 1108 1109
			/*
			 * We should insert radix-tree entry and dirty it here.
			 * For now this is broken...
			 */
R
Ross Zwisler 已提交
1110 1111
		}

1112 1113 1114 1115 1116
		dev_dbg(part_to_dev(bdev->bd_part),
				"%s: %s addr: %lx pfn: %lx sect: %llx\n",
				__func__, current->comm, address,
				pfn_t_to_pfn(dax.pfn),
				(unsigned long long) dax.sector);
D
Dan Williams 已提交
1117
		result |= vmf_insert_pfn_pmd(vma, address, pmd,
1118
				dax.pfn, write);
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	}

 out:
	return result;

 fallback:
	count_vm_event(THP_FAULT_FALLBACK);
	result = VM_FAULT_FALLBACK;
	goto out;
}
EXPORT_SYMBOL_GPL(dax_pmd_fault);
1130
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1131

1132 1133 1134 1135 1136 1137 1138
/**
 * dax_pfn_mkwrite - handle first write to DAX page
 * @vma: The virtual memory area where the fault occurred
 * @vmf: The description of the fault
 */
int dax_pfn_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
{
R
Ross Zwisler 已提交
1139
	struct file *file = vma->vm_file;
J
Jan Kara 已提交
1140 1141 1142
	struct address_space *mapping = file->f_mapping;
	void *entry;
	pgoff_t index = vmf->pgoff;
1143

J
Jan Kara 已提交
1144 1145 1146 1147 1148 1149 1150 1151
	spin_lock_irq(&mapping->tree_lock);
	entry = get_unlocked_mapping_entry(mapping, index, NULL);
	if (!entry || !radix_tree_exceptional_entry(entry))
		goto out;
	radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY);
	put_unlocked_mapping_entry(mapping, index, entry);
out:
	spin_unlock_irq(&mapping->tree_lock);
1152 1153 1154 1155
	return VM_FAULT_NOPAGE;
}
EXPORT_SYMBOL_GPL(dax_pfn_mkwrite);

1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
static bool dax_range_is_aligned(struct block_device *bdev,
				 unsigned int offset, unsigned int length)
{
	unsigned short sector_size = bdev_logical_block_size(bdev);

	if (!IS_ALIGNED(offset, sector_size))
		return false;
	if (!IS_ALIGNED(length, sector_size))
		return false;

	return true;
}

1169 1170 1171 1172 1173 1174 1175 1176
int __dax_zero_page_range(struct block_device *bdev, sector_t sector,
		unsigned int offset, unsigned int length)
{
	struct blk_dax_ctl dax = {
		.sector		= sector,
		.size		= PAGE_SIZE,
	};

1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187
	if (dax_range_is_aligned(bdev, offset, length)) {
		sector_t start_sector = dax.sector + (offset >> 9);

		return blkdev_issue_zeroout(bdev, start_sector,
				length >> 9, GFP_NOFS, true);
	} else {
		if (dax_map_atomic(bdev, &dax) < 0)
			return PTR_ERR(dax.addr);
		clear_pmem(dax.addr + offset, length);
		dax_unmap_atomic(bdev, &dax);
	}
1188 1189 1190 1191
	return 0;
}
EXPORT_SYMBOL_GPL(__dax_zero_page_range);

1192
/**
M
Matthew Wilcox 已提交
1193
 * dax_zero_page_range - zero a range within a page of a DAX file
1194 1195
 * @inode: The file being truncated
 * @from: The file offset that is being truncated to
M
Matthew Wilcox 已提交
1196
 * @length: The number of bytes to zero
1197 1198
 * @get_block: The filesystem method used to translate file offsets to blocks
 *
M
Matthew Wilcox 已提交
1199 1200 1201 1202
 * This function can be called by a filesystem when it is zeroing part of a
 * page in a DAX file.  This is intended for hole-punch operations.  If
 * you are truncating a file, the helper function dax_truncate_page() may be
 * more convenient.
1203
 */
M
Matthew Wilcox 已提交
1204 1205
int dax_zero_page_range(struct inode *inode, loff_t from, unsigned length,
							get_block_t get_block)
1206 1207
{
	struct buffer_head bh;
1208 1209
	pgoff_t index = from >> PAGE_SHIFT;
	unsigned offset = from & (PAGE_SIZE-1);
1210 1211 1212 1213 1214
	int err;

	/* Block boundary? Nothing to do */
	if (!length)
		return 0;
1215
	BUG_ON((offset + length) > PAGE_SIZE);
1216 1217

	memset(&bh, 0, sizeof(bh));
1218
	bh.b_bdev = inode->i_sb->s_bdev;
1219
	bh.b_size = PAGE_SIZE;
1220
	err = get_block(inode, index, &bh, 0);
1221
	if (err < 0 || !buffer_written(&bh))
1222 1223
		return err;

1224 1225
	return __dax_zero_page_range(bh.b_bdev, to_sector(&bh, inode),
			offset, length);
1226
}
M
Matthew Wilcox 已提交
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
EXPORT_SYMBOL_GPL(dax_zero_page_range);

/**
 * dax_truncate_page - handle a partial page being truncated in a DAX file
 * @inode: The file being truncated
 * @from: The file offset that is being truncated to
 * @get_block: The filesystem method used to translate file offsets to blocks
 *
 * Similar to block_truncate_page(), this function can be called by a
 * filesystem when it is truncating a DAX file to handle the partial page.
 */
int dax_truncate_page(struct inode *inode, loff_t from, get_block_t get_block)
{
1240
	unsigned length = PAGE_ALIGN(from) - from;
M
Matthew Wilcox 已提交
1241 1242
	return dax_zero_page_range(inode, from, length, get_block);
}
1243
EXPORT_SYMBOL_GPL(dax_truncate_page);