dax.c 43.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * fs/dax.c - Direct Access filesystem code
 * Copyright (c) 2013-2014 Intel Corporation
 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/atomic.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
20
#include <linux/dax.h>
21 22
#include <linux/fs.h>
#include <linux/genhd.h>
23 24 25
#include <linux/highmem.h>
#include <linux/memcontrol.h>
#include <linux/mm.h>
26
#include <linux/mutex.h>
R
Ross Zwisler 已提交
27
#include <linux/pagevec.h>
28
#include <linux/sched.h>
29
#include <linux/sched/signal.h>
30
#include <linux/uio.h>
31
#include <linux/vmstat.h>
D
Dan Williams 已提交
32
#include <linux/pfn_t.h>
33
#include <linux/sizes.h>
34
#include <linux/mmu_notifier.h>
35 36
#include <linux/iomap.h>
#include "internal.h"
37

38 39 40
#define CREATE_TRACE_POINTS
#include <trace/events/fs_dax.h>

J
Jan Kara 已提交
41 42 43 44
/* We choose 4096 entries - same as per-zone page wait tables */
#define DAX_WAIT_TABLE_BITS 12
#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)

45 46 47
/* The 'colour' (ie low bits) within a PMD of a page offset.  */
#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)

48
static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
J
Jan Kara 已提交
49 50 51 52 53 54 55 56 57 58 59

static int __init init_dax_wait_table(void)
{
	int i;

	for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
		init_waitqueue_head(wait_table + i);
	return 0;
}
fs_initcall(init_dax_wait_table);

60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
/*
 * We use lowest available bit in exceptional entry for locking, one bit for
 * the entry size (PMD) and two more to tell us if the entry is a zero page or
 * an empty entry that is just used for locking.  In total four special bits.
 *
 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
 * block allocation.
 */
#define RADIX_DAX_SHIFT		(RADIX_TREE_EXCEPTIONAL_SHIFT + 4)
#define RADIX_DAX_ENTRY_LOCK	(1 << RADIX_TREE_EXCEPTIONAL_SHIFT)
#define RADIX_DAX_PMD		(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
#define RADIX_DAX_ZERO_PAGE	(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
#define RADIX_DAX_EMPTY		(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))

static unsigned long dax_radix_sector(void *entry)
{
	return (unsigned long)entry >> RADIX_DAX_SHIFT;
}

static void *dax_radix_locked_entry(sector_t sector, unsigned long flags)
{
	return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags |
			((unsigned long)sector << RADIX_DAX_SHIFT) |
			RADIX_DAX_ENTRY_LOCK);
}

static unsigned int dax_radix_order(void *entry)
{
	if ((unsigned long)entry & RADIX_DAX_PMD)
		return PMD_SHIFT - PAGE_SHIFT;
	return 0;
}

94
static int dax_is_pmd_entry(void *entry)
95
{
96
	return (unsigned long)entry & RADIX_DAX_PMD;
97 98
}

99
static int dax_is_pte_entry(void *entry)
100
{
101
	return !((unsigned long)entry & RADIX_DAX_PMD);
102 103
}

104
static int dax_is_zero_entry(void *entry)
105
{
106
	return (unsigned long)entry & RADIX_DAX_ZERO_PAGE;
107 108
}

109
static int dax_is_empty_entry(void *entry)
110
{
111
	return (unsigned long)entry & RADIX_DAX_EMPTY;
112 113
}

J
Jan Kara 已提交
114 115 116 117 118
/*
 * DAX radix tree locking
 */
struct exceptional_entry_key {
	struct address_space *mapping;
119
	pgoff_t entry_start;
J
Jan Kara 已提交
120 121 122
};

struct wait_exceptional_entry_queue {
123
	wait_queue_entry_t wait;
J
Jan Kara 已提交
124 125 126
	struct exceptional_entry_key key;
};

127 128 129 130 131 132 133 134 135 136
static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
		pgoff_t index, void *entry, struct exceptional_entry_key *key)
{
	unsigned long hash;

	/*
	 * If 'entry' is a PMD, align the 'index' that we use for the wait
	 * queue to the start of that PMD.  This ensures that all offsets in
	 * the range covered by the PMD map to the same bit lock.
	 */
137
	if (dax_is_pmd_entry(entry))
138
		index &= ~PG_PMD_COLOUR;
139 140 141 142 143 144 145 146

	key->mapping = mapping;
	key->entry_start = index;

	hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
	return wait_table + hash;
}

147
static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode,
J
Jan Kara 已提交
148 149 150 151 152 153 154
				       int sync, void *keyp)
{
	struct exceptional_entry_key *key = keyp;
	struct wait_exceptional_entry_queue *ewait =
		container_of(wait, struct wait_exceptional_entry_queue, wait);

	if (key->mapping != ewait->key.mapping ||
155
	    key->entry_start != ewait->key.entry_start)
J
Jan Kara 已提交
156 157 158 159
		return 0;
	return autoremove_wake_function(wait, mode, sync, NULL);
}

R
Ross Zwisler 已提交
160 161 162 163 164 165 166
/*
 * We do not necessarily hold the mapping->tree_lock when we call this
 * function so it is possible that 'entry' is no longer a valid item in the
 * radix tree.  This is okay because all we really need to do is to find the
 * correct waitqueue where tasks might be waiting for that old 'entry' and
 * wake them.
 */
167
static void dax_wake_mapping_entry_waiter(struct address_space *mapping,
R
Ross Zwisler 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
		pgoff_t index, void *entry, bool wake_all)
{
	struct exceptional_entry_key key;
	wait_queue_head_t *wq;

	wq = dax_entry_waitqueue(mapping, index, entry, &key);

	/*
	 * Checking for locked entry and prepare_to_wait_exclusive() happens
	 * under mapping->tree_lock, ditto for entry handling in our callers.
	 * So at this point all tasks that could have seen our entry locked
	 * must be in the waitqueue and the following check will see them.
	 */
	if (waitqueue_active(wq))
		__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
}

J
Jan Kara 已提交
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
/*
 * Check whether the given slot is locked. The function must be called with
 * mapping->tree_lock held
 */
static inline int slot_locked(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
	return entry & RADIX_DAX_ENTRY_LOCK;
}

/*
 * Mark the given slot is locked. The function must be called with
 * mapping->tree_lock held
 */
static inline void *lock_slot(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

	entry |= RADIX_DAX_ENTRY_LOCK;
206
	radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
J
Jan Kara 已提交
207 208 209 210 211 212 213 214 215 216 217 218 219
	return (void *)entry;
}

/*
 * Mark the given slot is unlocked. The function must be called with
 * mapping->tree_lock held
 */
static inline void *unlock_slot(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

	entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
220
	radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
J
Jan Kara 已提交
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
	return (void *)entry;
}

/*
 * Lookup entry in radix tree, wait for it to become unlocked if it is
 * exceptional entry and return it. The caller must call
 * put_unlocked_mapping_entry() when he decided not to lock the entry or
 * put_locked_mapping_entry() when he locked the entry and now wants to
 * unlock it.
 *
 * The function must be called with mapping->tree_lock held.
 */
static void *get_unlocked_mapping_entry(struct address_space *mapping,
					pgoff_t index, void ***slotp)
{
236
	void *entry, **slot;
J
Jan Kara 已提交
237
	struct wait_exceptional_entry_queue ewait;
238
	wait_queue_head_t *wq;
J
Jan Kara 已提交
239 240 241 242 243

	init_wait(&ewait.wait);
	ewait.wait.func = wake_exceptional_entry_func;

	for (;;) {
244
		entry = __radix_tree_lookup(&mapping->page_tree, index, NULL,
J
Jan Kara 已提交
245
					  &slot);
246 247
		if (!entry ||
		    WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) ||
J
Jan Kara 已提交
248 249 250
		    !slot_locked(mapping, slot)) {
			if (slotp)
				*slotp = slot;
251
			return entry;
J
Jan Kara 已提交
252
		}
253 254

		wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
J
Jan Kara 已提交
255 256 257 258 259 260 261 262 263
		prepare_to_wait_exclusive(wq, &ewait.wait,
					  TASK_UNINTERRUPTIBLE);
		spin_unlock_irq(&mapping->tree_lock);
		schedule();
		finish_wait(wq, &ewait.wait);
		spin_lock_irq(&mapping->tree_lock);
	}
}

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
static void dax_unlock_mapping_entry(struct address_space *mapping,
				     pgoff_t index)
{
	void *entry, **slot;

	spin_lock_irq(&mapping->tree_lock);
	entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
	if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
			 !slot_locked(mapping, slot))) {
		spin_unlock_irq(&mapping->tree_lock);
		return;
	}
	unlock_slot(mapping, slot);
	spin_unlock_irq(&mapping->tree_lock);
	dax_wake_mapping_entry_waiter(mapping, index, entry, false);
}

281
static void put_locked_mapping_entry(struct address_space *mapping,
282
		pgoff_t index)
283
{
284
	dax_unlock_mapping_entry(mapping, index);
285 286 287 288 289 290 291 292 293
}

/*
 * Called when we are done with radix tree entry we looked up via
 * get_unlocked_mapping_entry() and which we didn't lock in the end.
 */
static void put_unlocked_mapping_entry(struct address_space *mapping,
				       pgoff_t index, void *entry)
{
294
	if (!entry)
295 296 297 298 299 300
		return;

	/* We have to wake up next waiter for the radix tree entry lock */
	dax_wake_mapping_entry_waiter(mapping, index, entry, false);
}

J
Jan Kara 已提交
301
/*
302 303 304 305
 * Find radix tree entry at given index. If it points to an exceptional entry,
 * return it with the radix tree entry locked. If the radix tree doesn't
 * contain given index, create an empty exceptional entry for the index and
 * return with it locked.
J
Jan Kara 已提交
306
 *
307 308
 * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
 * either return that locked entry or will return an error.  This error will
309 310
 * happen if there are any 4k entries within the 2MiB range that we are
 * requesting.
311 312 313 314 315 316 317 318 319 320 321 322
 *
 * We always favor 4k entries over 2MiB entries. There isn't a flow where we
 * evict 4k entries in order to 'upgrade' them to a 2MiB entry.  A 2MiB
 * insertion will fail if it finds any 4k entries already in the tree, and a
 * 4k insertion will cause an existing 2MiB entry to be unmapped and
 * downgraded to 4k entries.  This happens for both 2MiB huge zero pages as
 * well as 2MiB empty entries.
 *
 * The exception to this downgrade path is for 2MiB DAX PMD entries that have
 * real storage backing them.  We will leave these real 2MiB DAX entries in
 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
 *
J
Jan Kara 已提交
323 324 325 326
 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
 * persistent memory the benefit is doubtful. We can add that later if we can
 * show it helps.
 */
327 328
static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
		unsigned long size_flag)
J
Jan Kara 已提交
329
{
330
	bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
331
	void *entry, **slot;
J
Jan Kara 已提交
332 333 334

restart:
	spin_lock_irq(&mapping->tree_lock);
335
	entry = get_unlocked_mapping_entry(mapping, index, &slot);
336

337 338 339 340 341
	if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) {
		entry = ERR_PTR(-EIO);
		goto out_unlock;
	}

342 343
	if (entry) {
		if (size_flag & RADIX_DAX_PMD) {
344
			if (dax_is_pte_entry(entry)) {
345 346 347 348 349 350
				put_unlocked_mapping_entry(mapping, index,
						entry);
				entry = ERR_PTR(-EEXIST);
				goto out_unlock;
			}
		} else { /* trying to grab a PTE entry */
351
			if (dax_is_pmd_entry(entry) &&
352 353 354 355 356 357 358
			    (dax_is_zero_entry(entry) ||
			     dax_is_empty_entry(entry))) {
				pmd_downgrade = true;
			}
		}
	}

J
Jan Kara 已提交
359
	/* No entry for given index? Make sure radix tree is big enough. */
360
	if (!entry || pmd_downgrade) {
J
Jan Kara 已提交
361 362
		int err;

363 364 365 366 367 368 369 370
		if (pmd_downgrade) {
			/*
			 * Make sure 'entry' remains valid while we drop
			 * mapping->tree_lock.
			 */
			entry = lock_slot(mapping, slot);
		}

J
Jan Kara 已提交
371
		spin_unlock_irq(&mapping->tree_lock);
372 373 374 375 376 377 378 379 380
		/*
		 * Besides huge zero pages the only other thing that gets
		 * downgraded are empty entries which don't need to be
		 * unmapped.
		 */
		if (pmd_downgrade && dax_is_zero_entry(entry))
			unmap_mapping_range(mapping,
				(index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);

J
Jan Kara 已提交
381 382
		err = radix_tree_preload(
				mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
383 384
		if (err) {
			if (pmd_downgrade)
385
				put_locked_mapping_entry(mapping, index);
J
Jan Kara 已提交
386
			return ERR_PTR(err);
387
		}
J
Jan Kara 已提交
388
		spin_lock_irq(&mapping->tree_lock);
389

390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
		if (!entry) {
			/*
			 * We needed to drop the page_tree lock while calling
			 * radix_tree_preload() and we didn't have an entry to
			 * lock.  See if another thread inserted an entry at
			 * our index during this time.
			 */
			entry = __radix_tree_lookup(&mapping->page_tree, index,
					NULL, &slot);
			if (entry) {
				radix_tree_preload_end();
				spin_unlock_irq(&mapping->tree_lock);
				goto restart;
			}
		}

406 407 408 409 410 411 412 413 414 415 416
		if (pmd_downgrade) {
			radix_tree_delete(&mapping->page_tree, index);
			mapping->nrexceptional--;
			dax_wake_mapping_entry_waiter(mapping, index, entry,
					true);
		}

		entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);

		err = __radix_tree_insert(&mapping->page_tree, index,
				dax_radix_order(entry), entry);
J
Jan Kara 已提交
417 418 419
		radix_tree_preload_end();
		if (err) {
			spin_unlock_irq(&mapping->tree_lock);
420
			/*
421 422 423 424 425 426
			 * Our insertion of a DAX entry failed, most likely
			 * because we were inserting a PMD entry and it
			 * collided with a PTE sized entry at a different
			 * index in the PMD range.  We haven't inserted
			 * anything into the radix tree and have no waiters to
			 * wake.
427
			 */
J
Jan Kara 已提交
428 429 430 431 432
			return ERR_PTR(err);
		}
		/* Good, we have inserted empty locked entry into the tree. */
		mapping->nrexceptional++;
		spin_unlock_irq(&mapping->tree_lock);
433
		return entry;
J
Jan Kara 已提交
434
	}
435
	entry = lock_slot(mapping, slot);
436
 out_unlock:
J
Jan Kara 已提交
437
	spin_unlock_irq(&mapping->tree_lock);
438
	return entry;
J
Jan Kara 已提交
439 440
}

441 442 443 444 445 446 447 448 449
static int __dax_invalidate_mapping_entry(struct address_space *mapping,
					  pgoff_t index, bool trunc)
{
	int ret = 0;
	void *entry;
	struct radix_tree_root *page_tree = &mapping->page_tree;

	spin_lock_irq(&mapping->tree_lock);
	entry = get_unlocked_mapping_entry(mapping, index, NULL);
450
	if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)))
451 452 453 454 455 456 457 458 459 460 461 462 463
		goto out;
	if (!trunc &&
	    (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||
	     radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)))
		goto out;
	radix_tree_delete(page_tree, index);
	mapping->nrexceptional--;
	ret = 1;
out:
	put_unlocked_mapping_entry(mapping, index, entry);
	spin_unlock_irq(&mapping->tree_lock);
	return ret;
}
J
Jan Kara 已提交
464 465 466 467 468 469
/*
 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
 * entry to get unlocked before deleting it.
 */
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
{
470
	int ret = __dax_invalidate_mapping_entry(mapping, index, true);
J
Jan Kara 已提交
471 472 473 474 475 476 477 478

	/*
	 * This gets called from truncate / punch_hole path. As such, the caller
	 * must hold locks protecting against concurrent modifications of the
	 * radix tree (usually fs-private i_mmap_sem for writing). Since the
	 * caller has seen exceptional entry for this index, we better find it
	 * at that index as well...
	 */
479 480 481 482 483 484 485 486 487 488 489
	WARN_ON_ONCE(!ret);
	return ret;
}

/*
 * Invalidate exceptional DAX entry if it is clean.
 */
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
				      pgoff_t index)
{
	return __dax_invalidate_mapping_entry(mapping, index, false);
J
Jan Kara 已提交
490 491
}

492 493 494
static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
		sector_t sector, size_t size, struct page *to,
		unsigned long vaddr)
495
{
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
	void *vto, *kaddr;
	pgoff_t pgoff;
	pfn_t pfn;
	long rc;
	int id;

	rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
	if (rc)
		return rc;

	id = dax_read_lock();
	rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
	if (rc < 0) {
		dax_read_unlock(id);
		return rc;
	}
512
	vto = kmap_atomic(to);
513
	copy_user_page(vto, (void __force *)kaddr, vaddr, to);
514
	kunmap_atomic(vto);
515
	dax_read_unlock(id);
516 517 518
	return 0;
}

519 520 521 522 523 524 525
/*
 * By this point grab_mapping_entry() has ensured that we have a locked entry
 * of the appropriate size so we don't have to worry about downgrading PMDs to
 * PTEs.  If we happen to be trying to insert a PTE and there is a PMD
 * already in the tree, we will skip the insertion and just dirty the PMD as
 * appropriate.
 */
J
Jan Kara 已提交
526 527
static void *dax_insert_mapping_entry(struct address_space *mapping,
				      struct vm_fault *vmf,
528
				      void *entry, sector_t sector,
529
				      unsigned long flags, bool dirty)
R
Ross Zwisler 已提交
530 531
{
	struct radix_tree_root *page_tree = &mapping->page_tree;
J
Jan Kara 已提交
532 533
	void *new_entry;
	pgoff_t index = vmf->pgoff;
R
Ross Zwisler 已提交
534

535
	if (dirty)
536
		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
R
Ross Zwisler 已提交
537

538 539 540 541 542 543 544 545 546
	if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) {
		/* we are replacing a zero page with block mapping */
		if (dax_is_pmd_entry(entry))
			unmap_mapping_range(mapping,
					(vmf->pgoff << PAGE_SHIFT) & PMD_MASK,
					PMD_SIZE, 0);
		else /* pte entry */
			unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
					PAGE_SIZE, 0);
R
Ross Zwisler 已提交
547 548
	}

J
Jan Kara 已提交
549
	spin_lock_irq(&mapping->tree_lock);
550 551
	new_entry = dax_radix_locked_entry(sector, flags);

552
	if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
553 554 555 556 557 558 559 560
		/*
		 * Only swap our new entry into the radix tree if the current
		 * entry is a zero page or an empty entry.  If a normal PTE or
		 * PMD entry is already in the tree, we leave it alone.  This
		 * means that if we are trying to insert a PTE and the
		 * existing entry is a PMD, we will just leave the PMD in the
		 * tree and dirty it if necessary.
		 */
561
		struct radix_tree_node *node;
J
Jan Kara 已提交
562 563
		void **slot;
		void *ret;
R
Ross Zwisler 已提交
564

565
		ret = __radix_tree_lookup(page_tree, index, &node, &slot);
J
Jan Kara 已提交
566
		WARN_ON_ONCE(ret != entry);
567
		__radix_tree_replace(page_tree, node, slot,
568
				     new_entry, NULL);
569
		entry = new_entry;
R
Ross Zwisler 已提交
570
	}
571

572
	if (dirty)
R
Ross Zwisler 已提交
573
		radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
574

R
Ross Zwisler 已提交
575
	spin_unlock_irq(&mapping->tree_lock);
576
	return entry;
R
Ross Zwisler 已提交
577 578
}

579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
static inline unsigned long
pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
{
	unsigned long address;

	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
	VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
	return address;
}

/* Walk all mappings of a given index of a file and writeprotect them */
static void dax_mapping_entry_mkclean(struct address_space *mapping,
				      pgoff_t index, unsigned long pfn)
{
	struct vm_area_struct *vma;
594 595
	pte_t pte, *ptep = NULL;
	pmd_t *pmdp = NULL;
596 597 598 599
	spinlock_t *ptl;

	i_mmap_lock_read(mapping);
	vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
600
		unsigned long address, start, end;
601 602 603 604 605 606 607

		cond_resched();

		if (!(vma->vm_flags & VM_SHARED))
			continue;

		address = pgoff_address(index, vma);
608 609 610 611 612 613 614

		/*
		 * Note because we provide start/end to follow_pte_pmd it will
		 * call mmu_notifier_invalidate_range_start() on our behalf
		 * before taking any lock.
		 */
		if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))
615 616
			continue;

617 618 619 620 621 622 623
		/*
		 * No need to call mmu_notifier_invalidate_range() as we are
		 * downgrading page table protection not changing it to point
		 * to a new page.
		 *
		 * See Documentation/vm/mmu_notifier.txt
		 */
624 625 626 627 628 629
		if (pmdp) {
#ifdef CONFIG_FS_DAX_PMD
			pmd_t pmd;

			if (pfn != pmd_pfn(*pmdp))
				goto unlock_pmd;
630
			if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
				goto unlock_pmd;

			flush_cache_page(vma, address, pfn);
			pmd = pmdp_huge_clear_flush(vma, address, pmdp);
			pmd = pmd_wrprotect(pmd);
			pmd = pmd_mkclean(pmd);
			set_pmd_at(vma->vm_mm, address, pmdp, pmd);
unlock_pmd:
			spin_unlock(ptl);
#endif
		} else {
			if (pfn != pte_pfn(*ptep))
				goto unlock_pte;
			if (!pte_dirty(*ptep) && !pte_write(*ptep))
				goto unlock_pte;

			flush_cache_page(vma, address, pfn);
			pte = ptep_clear_flush(vma, address, ptep);
			pte = pte_wrprotect(pte);
			pte = pte_mkclean(pte);
			set_pte_at(vma->vm_mm, address, ptep, pte);
unlock_pte:
			pte_unmap_unlock(ptep, ptl);
		}
655

656
		mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
657 658 659 660
	}
	i_mmap_unlock_read(mapping);
}

R
Ross Zwisler 已提交
661
static int dax_writeback_one(struct block_device *bdev,
662 663
		struct dax_device *dax_dev, struct address_space *mapping,
		pgoff_t index, void *entry)
R
Ross Zwisler 已提交
664 665
{
	struct radix_tree_root *page_tree = &mapping->page_tree;
666 667 668 669 670 671
	void *entry2, **slot, *kaddr;
	long ret = 0, id;
	sector_t sector;
	pgoff_t pgoff;
	size_t size;
	pfn_t pfn;
R
Ross Zwisler 已提交
672 673

	/*
674 675
	 * A page got tagged dirty in DAX mapping? Something is seriously
	 * wrong.
R
Ross Zwisler 已提交
676
	 */
677 678
	if (WARN_ON(!radix_tree_exceptional_entry(entry)))
		return -EIO;
R
Ross Zwisler 已提交
679

680 681 682
	spin_lock_irq(&mapping->tree_lock);
	entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
	/* Entry got punched out / reallocated? */
683
	if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2)))
684 685 686 687 688 689 690 691
		goto put_unlocked;
	/*
	 * Entry got reallocated elsewhere? No need to writeback. We have to
	 * compare sectors as we must not bail out due to difference in lockbit
	 * or entry type.
	 */
	if (dax_radix_sector(entry2) != dax_radix_sector(entry))
		goto put_unlocked;
692 693
	if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
				dax_is_zero_entry(entry))) {
R
Ross Zwisler 已提交
694
		ret = -EIO;
695
		goto put_unlocked;
R
Ross Zwisler 已提交
696 697
	}

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
	/* Another fsync thread may have already written back this entry */
	if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
		goto put_unlocked;
	/* Lock the entry to serialize with page faults */
	entry = lock_slot(mapping, slot);
	/*
	 * We can clear the tag now but we have to be careful so that concurrent
	 * dax_writeback_one() calls for the same index cannot finish before we
	 * actually flush the caches. This is achieved as the calls will look
	 * at the entry only under tree_lock and once they do that they will
	 * see the entry locked and wait for it to unlock.
	 */
	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
	spin_unlock_irq(&mapping->tree_lock);

713 714 715 716 717 718 719
	/*
	 * Even if dax_writeback_mapping_range() was given a wbc->range_start
	 * in the middle of a PMD, the 'index' we are given will be aligned to
	 * the start index of the PMD, as will the sector we pull from
	 * 'entry'.  This allows us to flush for PMD_SIZE and not have to
	 * worry about partial PMD writebacks.
	 */
720 721 722 723 724 725 726
	sector = dax_radix_sector(entry);
	size = PAGE_SIZE << dax_radix_order(entry);

	id = dax_read_lock();
	ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
	if (ret)
		goto dax_unlock;
R
Ross Zwisler 已提交
727 728

	/*
729 730
	 * dax_direct_access() may sleep, so cannot hold tree_lock over
	 * its invocation.
R
Ross Zwisler 已提交
731
	 */
732 733 734
	ret = dax_direct_access(dax_dev, pgoff, size / PAGE_SIZE, &kaddr, &pfn);
	if (ret < 0)
		goto dax_unlock;
R
Ross Zwisler 已提交
735

736
	if (WARN_ON_ONCE(ret < size / PAGE_SIZE)) {
R
Ross Zwisler 已提交
737
		ret = -EIO;
738
		goto dax_unlock;
R
Ross Zwisler 已提交
739 740
	}

741
	dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(pfn));
742
	dax_flush(dax_dev, kaddr, size);
743 744 745 746 747 748 749 750 751
	/*
	 * After we have flushed the cache, we can clear the dirty tag. There
	 * cannot be new dirty data in the pfn after the flush has completed as
	 * the pfn mappings are writeprotected and fault waits for mapping
	 * entry lock.
	 */
	spin_lock_irq(&mapping->tree_lock);
	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);
	spin_unlock_irq(&mapping->tree_lock);
752
	trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT);
753 754
 dax_unlock:
	dax_read_unlock(id);
755
	put_locked_mapping_entry(mapping, index);
R
Ross Zwisler 已提交
756 757
	return ret;

758 759
 put_unlocked:
	put_unlocked_mapping_entry(mapping, index, entry2);
R
Ross Zwisler 已提交
760 761 762 763 764 765 766 767 768
	spin_unlock_irq(&mapping->tree_lock);
	return ret;
}

/*
 * Flush the mapping to the persistent domain within the byte range of [start,
 * end]. This is required by data integrity operations to ensure file data is
 * on persistent storage prior to completion of the operation.
 */
769 770
int dax_writeback_mapping_range(struct address_space *mapping,
		struct block_device *bdev, struct writeback_control *wbc)
R
Ross Zwisler 已提交
771 772
{
	struct inode *inode = mapping->host;
773
	pgoff_t start_index, end_index;
R
Ross Zwisler 已提交
774
	pgoff_t indices[PAGEVEC_SIZE];
775
	struct dax_device *dax_dev;
R
Ross Zwisler 已提交
776 777 778 779 780 781 782
	struct pagevec pvec;
	bool done = false;
	int i, ret = 0;

	if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
		return -EIO;

783 784 785
	if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
		return 0;

786 787 788 789
	dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
	if (!dax_dev)
		return -EIO;

790 791
	start_index = wbc->range_start >> PAGE_SHIFT;
	end_index = wbc->range_end >> PAGE_SHIFT;
R
Ross Zwisler 已提交
792

793 794
	trace_dax_writeback_range(inode, start_index, end_index);

R
Ross Zwisler 已提交
795 796
	tag_pages_for_writeback(mapping, start_index, end_index);

797
	pagevec_init(&pvec);
R
Ross Zwisler 已提交
798 799 800 801 802 803 804 805 806 807 808 809 810 811
	while (!done) {
		pvec.nr = find_get_entries_tag(mapping, start_index,
				PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
				pvec.pages, indices);

		if (pvec.nr == 0)
			break;

		for (i = 0; i < pvec.nr; i++) {
			if (indices[i] > end_index) {
				done = true;
				break;
			}

812 813
			ret = dax_writeback_one(bdev, dax_dev, mapping,
					indices[i], pvec.pages[i]);
814 815
			if (ret < 0) {
				mapping_set_error(mapping, ret);
816
				goto out;
817
			}
R
Ross Zwisler 已提交
818
		}
819
		start_index = indices[pvec.nr - 1] + 1;
R
Ross Zwisler 已提交
820
	}
821
out:
822
	put_dax(dax_dev);
823 824
	trace_dax_writeback_range_done(inode, start_index, end_index);
	return (ret < 0 ? ret : 0);
R
Ross Zwisler 已提交
825 826 827
}
EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);

828
static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
829
{
830
	return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
831 832
}

833 834
static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
			 pfn_t *pfnp)
835
{
836
	const sector_t sector = dax_iomap_sector(iomap, pos);
837
	pgoff_t pgoff;
838
	void *kaddr;
839
	int id, rc;
840
	long length;
841

842
	rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
843 844 845
	if (rc)
		return rc;
	id = dax_read_lock();
846 847 848 849 850
	length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
				   &kaddr, pfnp);
	if (length < 0) {
		rc = length;
		goto out;
851
	}
852 853 854 855 856 857 858 859 860 861
	rc = -EINVAL;
	if (PFN_PHYS(length) < size)
		goto out;
	if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
		goto out;
	/* For larger pages we need devmap */
	if (length > 1 && !pfn_t_devmap(*pfnp))
		goto out;
	rc = 0;
out:
862
	dax_read_unlock(id);
863
	return rc;
864 865
}

R
Ross Zwisler 已提交
866
/*
867 868 869 870 871
 * The user has performed a load from a hole in the file.  Allocating a new
 * page in the file would cause excessive storage usage for workloads with
 * sparse files.  Instead we insert a read-only mapping of the 4k zero page.
 * If this page is ever written to we will re-fault and change the mapping to
 * point to real DAX storage instead.
R
Ross Zwisler 已提交
872
 */
873
static int dax_load_hole(struct address_space *mapping, void *entry,
R
Ross Zwisler 已提交
874 875 876
			 struct vm_fault *vmf)
{
	struct inode *inode = mapping->host;
877 878 879 880
	unsigned long vaddr = vmf->address;
	int ret = VM_FAULT_NOPAGE;
	struct page *zero_page;
	void *entry2;
R
Ross Zwisler 已提交
881

882 883
	zero_page = ZERO_PAGE(0);
	if (unlikely(!zero_page)) {
R
Ross Zwisler 已提交
884 885 886 887
		ret = VM_FAULT_OOM;
		goto out;
	}

888
	entry2 = dax_insert_mapping_entry(mapping, vmf, entry, 0,
889
			RADIX_DAX_ZERO_PAGE, false);
890 891 892
	if (IS_ERR(entry2)) {
		ret = VM_FAULT_SIGBUS;
		goto out;
R
Ross Zwisler 已提交
893
	}
894 895

	vm_insert_mixed(vmf->vma, vaddr, page_to_pfn_t(zero_page));
R
Ross Zwisler 已提交
896 897 898 899 900
out:
	trace_dax_load_hole(inode, vmf, ret);
	return ret;
}

901 902 903 904 905 906 907 908 909 910 911 912 913
static bool dax_range_is_aligned(struct block_device *bdev,
				 unsigned int offset, unsigned int length)
{
	unsigned short sector_size = bdev_logical_block_size(bdev);

	if (!IS_ALIGNED(offset, sector_size))
		return false;
	if (!IS_ALIGNED(length, sector_size))
		return false;

	return true;
}

914 915 916
int __dax_zero_page_range(struct block_device *bdev,
		struct dax_device *dax_dev, sector_t sector,
		unsigned int offset, unsigned int size)
917
{
918 919
	if (dax_range_is_aligned(bdev, offset, size)) {
		sector_t start_sector = sector + (offset >> 9);
920 921

		return blkdev_issue_zeroout(bdev, start_sector,
922
				size >> 9, GFP_NOFS, 0);
923
	} else {
924 925 926 927 928
		pgoff_t pgoff;
		long rc, id;
		void *kaddr;
		pfn_t pfn;

929
		rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
930 931 932 933
		if (rc)
			return rc;

		id = dax_read_lock();
934
		rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr,
935 936 937 938 939
				&pfn);
		if (rc < 0) {
			dax_read_unlock(id);
			return rc;
		}
940
		memset(kaddr + offset, 0, size);
941
		dax_flush(dax_dev, kaddr + offset, size);
942
		dax_read_unlock(id);
943
	}
944 945 946 947
	return 0;
}
EXPORT_SYMBOL_GPL(__dax_zero_page_range);

948
static loff_t
949
dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
950 951
		struct iomap *iomap)
{
952 953
	struct block_device *bdev = iomap->bdev;
	struct dax_device *dax_dev = iomap->dax_dev;
954 955 956
	struct iov_iter *iter = data;
	loff_t end = pos + length, done = 0;
	ssize_t ret = 0;
957
	int id;
958 959 960 961 962 963 964 965 966 967 968 969 970

	if (iov_iter_rw(iter) == READ) {
		end = min(end, i_size_read(inode));
		if (pos >= end)
			return 0;

		if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
			return iov_iter_zero(min(length, end - pos), iter);
	}

	if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
		return -EIO;

971 972 973 974 975
	/*
	 * Write can allocate block for an area which has a hole page mapped
	 * into page tables. We have to tear down these mappings so that data
	 * written by write(2) is visible in mmap.
	 */
976
	if (iomap->flags & IOMAP_F_NEW) {
977 978 979 980 981
		invalidate_inode_pages2_range(inode->i_mapping,
					      pos >> PAGE_SHIFT,
					      (end - 1) >> PAGE_SHIFT);
	}

982
	id = dax_read_lock();
983 984
	while (pos < end) {
		unsigned offset = pos & (PAGE_SIZE - 1);
985 986
		const size_t size = ALIGN(length + offset, PAGE_SIZE);
		const sector_t sector = dax_iomap_sector(iomap, pos);
987
		ssize_t map_len;
988 989 990
		pgoff_t pgoff;
		void *kaddr;
		pfn_t pfn;
991

992 993 994 995 996
		if (fatal_signal_pending(current)) {
			ret = -EINTR;
			break;
		}

997 998 999 1000 1001 1002
		ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
		if (ret)
			break;

		map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
				&kaddr, &pfn);
1003 1004 1005 1006 1007
		if (map_len < 0) {
			ret = map_len;
			break;
		}

1008 1009
		map_len = PFN_PHYS(map_len);
		kaddr += offset;
1010 1011 1012 1013
		map_len -= offset;
		if (map_len > end - pos)
			map_len = end - pos;

1014 1015 1016 1017 1018
		/*
		 * The userspace address for the memory copy has already been
		 * validated via access_ok() in either vfs_read() or
		 * vfs_write(), depending on which operation we are doing.
		 */
1019
		if (iov_iter_rw(iter) == WRITE)
1020 1021
			map_len = dax_copy_from_iter(dax_dev, pgoff, kaddr,
					map_len, iter);
1022
		else
1023
			map_len = copy_to_iter(kaddr, map_len, iter);
1024 1025 1026 1027 1028 1029 1030 1031 1032
		if (map_len <= 0) {
			ret = map_len ? map_len : -EFAULT;
			break;
		}

		pos += map_len;
		length -= map_len;
		done += map_len;
	}
1033
	dax_read_unlock(id);
1034 1035 1036 1037 1038

	return done ? done : ret;
}

/**
1039
 * dax_iomap_rw - Perform I/O to a DAX file
1040 1041 1042 1043 1044 1045 1046 1047 1048
 * @iocb:	The control block for this I/O
 * @iter:	The addresses to do I/O from or to
 * @ops:	iomap ops passed from the file system
 *
 * This function performs read and write operations to directly mapped
 * persistent memory.  The callers needs to take care of read/write exclusion
 * and evicting any page cache pages in the region under I/O.
 */
ssize_t
1049
dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
1050
		const struct iomap_ops *ops)
1051 1052 1053 1054 1055 1056
{
	struct address_space *mapping = iocb->ki_filp->f_mapping;
	struct inode *inode = mapping->host;
	loff_t pos = iocb->ki_pos, ret = 0, done = 0;
	unsigned flags = 0;

1057 1058
	if (iov_iter_rw(iter) == WRITE) {
		lockdep_assert_held_exclusive(&inode->i_rwsem);
1059
		flags |= IOMAP_WRITE;
1060 1061 1062
	} else {
		lockdep_assert_held(&inode->i_rwsem);
	}
1063 1064 1065

	while (iov_iter_count(iter)) {
		ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
1066
				iter, dax_iomap_actor);
1067 1068 1069 1070 1071 1072 1073 1074 1075
		if (ret <= 0)
			break;
		pos += ret;
		done += ret;
	}

	iocb->ki_pos += done;
	return done ? done : ret;
}
1076
EXPORT_SYMBOL_GPL(dax_iomap_rw);
1077

1078 1079 1080 1081 1082 1083 1084 1085 1086
static int dax_fault_return(int error)
{
	if (error == 0)
		return VM_FAULT_NOPAGE;
	if (error == -ENOMEM)
		return VM_FAULT_OOM;
	return VM_FAULT_SIGBUS;
}

1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
/*
 * MAP_SYNC on a dax mapping guarantees dirty metadata is
 * flushed on write-faults (non-cow), but not read-faults.
 */
static bool dax_fault_is_synchronous(unsigned long flags,
		struct vm_area_struct *vma, struct iomap *iomap)
{
	return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
		&& (iomap->flags & IOMAP_F_DIRTY);
}

1098
static int dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
1099
			       const struct iomap_ops *ops)
1100
{
1101 1102
	struct vm_area_struct *vma = vmf->vma;
	struct address_space *mapping = vma->vm_file->f_mapping;
1103
	struct inode *inode = mapping->host;
1104
	unsigned long vaddr = vmf->address;
1105 1106
	loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
	struct iomap iomap = { 0 };
J
Jan Kara 已提交
1107
	unsigned flags = IOMAP_FAULT;
1108
	int error, major = 0;
1109
	bool write = vmf->flags & FAULT_FLAG_WRITE;
1110
	bool sync;
1111
	int vmf_ret = 0;
1112
	void *entry;
1113
	pfn_t pfn;
1114

1115
	trace_dax_pte_fault(inode, vmf, vmf_ret);
1116 1117 1118 1119 1120
	/*
	 * Check whether offset isn't beyond end of file now. Caller is supposed
	 * to hold locks serializing us with truncate / punch hole so this is
	 * a reliable test.
	 */
1121 1122 1123 1124
	if (pos >= i_size_read(inode)) {
		vmf_ret = VM_FAULT_SIGBUS;
		goto out;
	}
1125

1126
	if (write && !vmf->cow_page)
1127 1128
		flags |= IOMAP_WRITE;

1129 1130 1131 1132 1133 1134
	entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
	if (IS_ERR(entry)) {
		vmf_ret = dax_fault_return(PTR_ERR(entry));
		goto out;
	}

1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145
	/*
	 * It is possible, particularly with mixed reads & writes to private
	 * mappings, that we have raced with a PMD fault that overlaps with
	 * the PTE we need to set up.  If so just return and the fault will be
	 * retried.
	 */
	if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
		vmf_ret = VM_FAULT_NOPAGE;
		goto unlock_entry;
	}

1146 1147 1148 1149 1150 1151
	/*
	 * Note that we don't bother to use iomap_apply here: DAX required
	 * the file system block size to be equal the page size, which means
	 * that we never have to deal with more than a single extent here.
	 */
	error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1152 1153
	if (error) {
		vmf_ret = dax_fault_return(error);
1154
		goto unlock_entry;
1155
	}
1156
	if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
1157 1158
		error = -EIO;	/* fs corruption? */
		goto error_finish_iomap;
1159 1160 1161
	}

	if (vmf->cow_page) {
1162 1163
		sector_t sector = dax_iomap_sector(&iomap, pos);

1164 1165 1166 1167 1168 1169
		switch (iomap.type) {
		case IOMAP_HOLE:
		case IOMAP_UNWRITTEN:
			clear_user_highpage(vmf->cow_page, vaddr);
			break;
		case IOMAP_MAPPED:
1170 1171
			error = copy_user_dax(iomap.bdev, iomap.dax_dev,
					sector, PAGE_SIZE, vmf->cow_page, vaddr);
1172 1173 1174 1175 1176 1177 1178 1179
			break;
		default:
			WARN_ON_ONCE(1);
			error = -EIO;
			break;
		}

		if (error)
1180
			goto error_finish_iomap;
1181 1182 1183 1184 1185

		__SetPageUptodate(vmf->cow_page);
		vmf_ret = finish_fault(vmf);
		if (!vmf_ret)
			vmf_ret = VM_FAULT_DONE_COW;
1186
		goto finish_iomap;
1187 1188
	}

1189
	sync = dax_fault_is_synchronous(flags, vma, &iomap);
1190

1191 1192 1193 1194
	switch (iomap.type) {
	case IOMAP_MAPPED:
		if (iomap.flags & IOMAP_F_NEW) {
			count_vm_event(PGMAJFAULT);
1195
			count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
1196 1197
			major = VM_FAULT_MAJOR;
		}
1198 1199 1200 1201 1202 1203
		error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
		if (error < 0)
			goto error_finish_iomap;

		entry = dax_insert_mapping_entry(mapping, vmf, entry,
						 dax_iomap_sector(&iomap, pos),
1204
						 0, write && !sync);
1205 1206 1207 1208 1209
		if (IS_ERR(entry)) {
			error = PTR_ERR(entry);
			goto error_finish_iomap;
		}

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
		/*
		 * If we are doing synchronous page fault and inode needs fsync,
		 * we can insert PTE into page tables only after that happens.
		 * Skip insertion for now and return the pfn so that caller can
		 * insert it after fsync is done.
		 */
		if (sync) {
			if (WARN_ON_ONCE(!pfnp)) {
				error = -EIO;
				goto error_finish_iomap;
			}
			*pfnp = pfn;
			vmf_ret = VM_FAULT_NEEDDSYNC | major;
			goto finish_iomap;
		}
1225 1226 1227 1228 1229 1230
		trace_dax_insert_mapping(inode, vmf, entry);
		if (write)
			error = vm_insert_mixed_mkwrite(vma, vaddr, pfn);
		else
			error = vm_insert_mixed(vma, vaddr, pfn);

1231 1232 1233
		/* -EBUSY is fine, somebody else faulted on the same PTE */
		if (error == -EBUSY)
			error = 0;
1234 1235 1236
		break;
	case IOMAP_UNWRITTEN:
	case IOMAP_HOLE:
1237
		if (!write) {
1238
			vmf_ret = dax_load_hole(mapping, entry, vmf);
1239
			goto finish_iomap;
1240
		}
1241 1242 1243 1244 1245 1246 1247
		/*FALLTHRU*/
	default:
		WARN_ON_ONCE(1);
		error = -EIO;
		break;
	}

1248
 error_finish_iomap:
1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
	vmf_ret = dax_fault_return(error) | major;
 finish_iomap:
	if (ops->iomap_end) {
		int copied = PAGE_SIZE;

		if (vmf_ret & VM_FAULT_ERROR)
			copied = 0;
		/*
		 * The fault is done by now and there's no way back (other
		 * thread may be already happily using PTE we have installed).
		 * Just ignore error from ->iomap_end since we cannot do much
		 * with it.
		 */
		ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
1263
	}
1264
 unlock_entry:
1265
	put_locked_mapping_entry(mapping, vmf->pgoff);
1266
 out:
1267
	trace_dax_pte_fault_done(inode, vmf, vmf_ret);
1268
	return vmf_ret;
1269
}
1270 1271

#ifdef CONFIG_FS_DAX_PMD
1272 1273 1274 1275 1276
/*
 * The 'colour' (ie low bits) within a PMD of a page offset.  This comes up
 * more often than one might expect in the below functions.
 */
#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)
1277

1278
static int dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap,
1279
		void *entry)
1280
{
1281 1282
	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
	unsigned long pmd_addr = vmf->address & PMD_MASK;
1283
	struct inode *inode = mapping->host;
1284
	struct page *zero_page;
1285
	void *ret = NULL;
1286 1287 1288
	spinlock_t *ptl;
	pmd_t pmd_entry;

1289
	zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
1290 1291

	if (unlikely(!zero_page))
1292
		goto fallback;
1293

1294
	ret = dax_insert_mapping_entry(mapping, vmf, entry, 0,
1295
			RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE, false);
1296
	if (IS_ERR(ret))
1297
		goto fallback;
1298

1299 1300
	ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
	if (!pmd_none(*(vmf->pmd))) {
1301
		spin_unlock(ptl);
1302
		goto fallback;
1303 1304
	}

1305
	pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
1306
	pmd_entry = pmd_mkhuge(pmd_entry);
1307
	set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
1308
	spin_unlock(ptl);
1309
	trace_dax_pmd_load_hole(inode, vmf, zero_page, ret);
1310
	return VM_FAULT_NOPAGE;
1311 1312

fallback:
1313
	trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret);
1314
	return VM_FAULT_FALLBACK;
1315 1316
}

1317
static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
1318
			       const struct iomap_ops *ops)
1319
{
1320
	struct vm_area_struct *vma = vmf->vma;
1321
	struct address_space *mapping = vma->vm_file->f_mapping;
1322 1323
	unsigned long pmd_addr = vmf->address & PMD_MASK;
	bool write = vmf->flags & FAULT_FLAG_WRITE;
1324
	bool sync;
J
Jan Kara 已提交
1325
	unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
1326 1327 1328 1329 1330 1331 1332
	struct inode *inode = mapping->host;
	int result = VM_FAULT_FALLBACK;
	struct iomap iomap = { 0 };
	pgoff_t max_pgoff, pgoff;
	void *entry;
	loff_t pos;
	int error;
1333
	pfn_t pfn;
1334

1335 1336 1337 1338 1339 1340
	/*
	 * Check whether offset isn't beyond end of file now. Caller is
	 * supposed to hold locks serializing us with truncate / punch hole so
	 * this is a reliable test.
	 */
	pgoff = linear_page_index(vma, pmd_addr);
1341
	max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
1342

1343
	trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
1344

1345 1346 1347 1348 1349 1350 1351 1352 1353 1354
	/*
	 * Make sure that the faulting address's PMD offset (color) matches
	 * the PMD offset from the start of the file.  This is necessary so
	 * that a PMD range in the page table overlaps exactly with a PMD
	 * range in the radix tree.
	 */
	if ((vmf->pgoff & PG_PMD_COLOUR) !=
	    ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
		goto fallback;

1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
	/* Fall back to PTEs if we're going to COW */
	if (write && !(vma->vm_flags & VM_SHARED))
		goto fallback;

	/* If the PMD would extend outside the VMA */
	if (pmd_addr < vma->vm_start)
		goto fallback;
	if ((pmd_addr + PMD_SIZE) > vma->vm_end)
		goto fallback;

1365
	if (pgoff >= max_pgoff) {
1366 1367 1368
		result = VM_FAULT_SIGBUS;
		goto out;
	}
1369 1370

	/* If the PMD would extend beyond the file size */
1371
	if ((pgoff | PG_PMD_COLOUR) >= max_pgoff)
1372 1373
		goto fallback;

1374
	/*
1375 1376 1377 1378
	 * grab_mapping_entry() will make sure we get a 2MiB empty entry, a
	 * 2MiB zero page entry or a DAX PMD.  If it can't (because a 4k page
	 * is already in the tree, for instance), it will return -EEXIST and
	 * we just fall back to 4k entries.
1379 1380 1381 1382 1383
	 */
	entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
	if (IS_ERR(entry))
		goto fallback;

1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
	/*
	 * It is possible, particularly with mixed reads & writes to private
	 * mappings, that we have raced with a PTE fault that overlaps with
	 * the PMD we need to set up.  If so just return and the fault will be
	 * retried.
	 */
	if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
			!pmd_devmap(*vmf->pmd)) {
		result = 0;
		goto unlock_entry;
	}

1396 1397 1398 1399 1400 1401 1402 1403
	/*
	 * Note that we don't use iomap_apply here.  We aren't doing I/O, only
	 * setting up a mapping, so really we're using iomap_begin() as a way
	 * to look up our filesystem block.
	 */
	pos = (loff_t)pgoff << PAGE_SHIFT;
	error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
	if (error)
1404
		goto unlock_entry;
1405

1406 1407 1408
	if (iomap.offset + iomap.length < pos + PMD_SIZE)
		goto finish_iomap;

1409
	sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);
1410

1411 1412
	switch (iomap.type) {
	case IOMAP_MAPPED:
1413 1414 1415 1416 1417 1418
		error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);
		if (error < 0)
			goto finish_iomap;

		entry = dax_insert_mapping_entry(mapping, vmf, entry,
						dax_iomap_sector(&iomap, pos),
1419
						RADIX_DAX_PMD, write && !sync);
1420 1421 1422
		if (IS_ERR(entry))
			goto finish_iomap;

1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
		/*
		 * If we are doing synchronous page fault and inode needs fsync,
		 * we can insert PMD into page tables only after that happens.
		 * Skip insertion for now and return the pfn so that caller can
		 * insert it after fsync is done.
		 */
		if (sync) {
			if (WARN_ON_ONCE(!pfnp))
				goto finish_iomap;
			*pfnp = pfn;
			result = VM_FAULT_NEEDDSYNC;
			goto finish_iomap;
		}

1437 1438 1439
		trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
		result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn,
					    write);
1440 1441 1442 1443
		break;
	case IOMAP_UNWRITTEN:
	case IOMAP_HOLE:
		if (WARN_ON_ONCE(write))
1444
			break;
1445
		result = dax_pmd_load_hole(vmf, &iomap, entry);
1446 1447 1448 1449 1450 1451 1452 1453
		break;
	default:
		WARN_ON_ONCE(1);
		break;
	}

 finish_iomap:
	if (ops->iomap_end) {
1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465
		int copied = PMD_SIZE;

		if (result == VM_FAULT_FALLBACK)
			copied = 0;
		/*
		 * The fault is done by now and there's no way back (other
		 * thread may be already happily using PMD we have installed).
		 * Just ignore error from ->iomap_end since we cannot do much
		 * with it.
		 */
		ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
				&iomap);
1466
	}
1467
 unlock_entry:
1468
	put_locked_mapping_entry(mapping, pgoff);
1469 1470
 fallback:
	if (result == VM_FAULT_FALLBACK) {
1471
		split_huge_pmd(vma, vmf->pmd, vmf->address);
1472 1473
		count_vm_event(THP_FAULT_FALLBACK);
	}
1474
out:
1475
	trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
1476 1477
	return result;
}
1478
#else
1479
static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
1480
			       const struct iomap_ops *ops)
1481 1482 1483
{
	return VM_FAULT_FALLBACK;
}
1484
#endif /* CONFIG_FS_DAX_PMD */
1485 1486 1487 1488

/**
 * dax_iomap_fault - handle a page fault on a DAX file
 * @vmf: The description of the fault
1489
 * @pe_size: Size of the page to fault in
1490
 * @pfnp: PFN to insert for synchronous faults if fsync is required
1491
 * @ops: Iomap ops passed from the file system
1492 1493 1494 1495 1496 1497
 *
 * When a page fault occurs, filesystems may call this helper in
 * their fault handler for DAX files. dax_iomap_fault() assumes the caller
 * has done all the necessary locking for page fault to proceed
 * successfully.
 */
1498
int dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
1499
		    pfn_t *pfnp, const struct iomap_ops *ops)
1500
{
1501 1502
	switch (pe_size) {
	case PE_SIZE_PTE:
1503
		return dax_iomap_pte_fault(vmf, pfnp, ops);
1504
	case PE_SIZE_PMD:
1505
		return dax_iomap_pmd_fault(vmf, pfnp, ops);
1506 1507 1508 1509 1510
	default:
		return VM_FAULT_FALLBACK;
	}
}
EXPORT_SYMBOL_GPL(dax_iomap_fault);
J
Jan Kara 已提交
1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593

/**
 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
 * @vmf: The description of the fault
 * @pe_size: Size of entry to be inserted
 * @pfn: PFN to insert
 *
 * This function inserts writeable PTE or PMD entry into page tables for mmaped
 * DAX file.  It takes care of marking corresponding radix tree entry as dirty
 * as well.
 */
static int dax_insert_pfn_mkwrite(struct vm_fault *vmf,
				  enum page_entry_size pe_size,
				  pfn_t pfn)
{
	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
	void *entry, **slot;
	pgoff_t index = vmf->pgoff;
	int vmf_ret, error;

	spin_lock_irq(&mapping->tree_lock);
	entry = get_unlocked_mapping_entry(mapping, index, &slot);
	/* Did we race with someone splitting entry or so? */
	if (!entry ||
	    (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) ||
	    (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) {
		put_unlocked_mapping_entry(mapping, index, entry);
		spin_unlock_irq(&mapping->tree_lock);
		trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
						      VM_FAULT_NOPAGE);
		return VM_FAULT_NOPAGE;
	}
	radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY);
	entry = lock_slot(mapping, slot);
	spin_unlock_irq(&mapping->tree_lock);
	switch (pe_size) {
	case PE_SIZE_PTE:
		error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
		vmf_ret = dax_fault_return(error);
		break;
#ifdef CONFIG_FS_DAX_PMD
	case PE_SIZE_PMD:
		vmf_ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
			pfn, true);
		break;
#endif
	default:
		vmf_ret = VM_FAULT_FALLBACK;
	}
	put_locked_mapping_entry(mapping, index);
	trace_dax_insert_pfn_mkwrite(mapping->host, vmf, vmf_ret);
	return vmf_ret;
}

/**
 * dax_finish_sync_fault - finish synchronous page fault
 * @vmf: The description of the fault
 * @pe_size: Size of entry to be inserted
 * @pfn: PFN to insert
 *
 * This function ensures that the file range touched by the page fault is
 * stored persistently on the media and handles inserting of appropriate page
 * table entry.
 */
int dax_finish_sync_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
			  pfn_t pfn)
{
	int err;
	loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
	size_t len = 0;

	if (pe_size == PE_SIZE_PTE)
		len = PAGE_SIZE;
	else if (pe_size == PE_SIZE_PMD)
		len = PMD_SIZE;
	else
		WARN_ON_ONCE(1);
	err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
	if (err)
		return VM_FAULT_SIGBUS;
	return dax_insert_pfn_mkwrite(vmf, pe_size, pfn);
}
EXPORT_SYMBOL_GPL(dax_finish_sync_fault);