dax.c 43.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * fs/dax.c - Direct Access filesystem code
 * Copyright (c) 2013-2014 Intel Corporation
 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/atomic.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
20
#include <linux/dax.h>
21 22
#include <linux/fs.h>
#include <linux/genhd.h>
23 24 25
#include <linux/highmem.h>
#include <linux/memcontrol.h>
#include <linux/mm.h>
26
#include <linux/mutex.h>
R
Ross Zwisler 已提交
27
#include <linux/pagevec.h>
28
#include <linux/sched.h>
29
#include <linux/sched/signal.h>
30
#include <linux/uio.h>
31
#include <linux/vmstat.h>
D
Dan Williams 已提交
32
#include <linux/pfn_t.h>
33
#include <linux/sizes.h>
34
#include <linux/mmu_notifier.h>
35 36
#include <linux/iomap.h>
#include "internal.h"
37

38 39 40
#define CREATE_TRACE_POINTS
#include <trace/events/fs_dax.h>

J
Jan Kara 已提交
41 42 43 44
/* We choose 4096 entries - same as per-zone page wait tables */
#define DAX_WAIT_TABLE_BITS 12
#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)

45 46
/* The 'colour' (ie low bits) within a PMD of a page offset.  */
#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)
M
Matthew Wilcox 已提交
47
#define PG_PMD_NR	(PMD_SIZE >> PAGE_SHIFT)
48

49
static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
J
Jan Kara 已提交
50 51 52 53 54 55 56 57 58 59 60

static int __init init_dax_wait_table(void)
{
	int i;

	for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
		init_waitqueue_head(wait_table + i);
	return 0;
}
fs_initcall(init_dax_wait_table);

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
/*
 * We use lowest available bit in exceptional entry for locking, one bit for
 * the entry size (PMD) and two more to tell us if the entry is a zero page or
 * an empty entry that is just used for locking.  In total four special bits.
 *
 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
 * block allocation.
 */
#define RADIX_DAX_SHIFT		(RADIX_TREE_EXCEPTIONAL_SHIFT + 4)
#define RADIX_DAX_ENTRY_LOCK	(1 << RADIX_TREE_EXCEPTIONAL_SHIFT)
#define RADIX_DAX_PMD		(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
#define RADIX_DAX_ZERO_PAGE	(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
#define RADIX_DAX_EMPTY		(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))

D
Dan Williams 已提交
76
static unsigned long dax_radix_pfn(void *entry)
77 78 79 80
{
	return (unsigned long)entry >> RADIX_DAX_SHIFT;
}

D
Dan Williams 已提交
81
static void *dax_radix_locked_entry(unsigned long pfn, unsigned long flags)
82 83
{
	return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags |
D
Dan Williams 已提交
84
			(pfn << RADIX_DAX_SHIFT) | RADIX_DAX_ENTRY_LOCK);
85 86 87 88 89 90 91 92 93
}

static unsigned int dax_radix_order(void *entry)
{
	if ((unsigned long)entry & RADIX_DAX_PMD)
		return PMD_SHIFT - PAGE_SHIFT;
	return 0;
}

94
static int dax_is_pmd_entry(void *entry)
95
{
96
	return (unsigned long)entry & RADIX_DAX_PMD;
97 98
}

99
static int dax_is_pte_entry(void *entry)
100
{
101
	return !((unsigned long)entry & RADIX_DAX_PMD);
102 103
}

104
static int dax_is_zero_entry(void *entry)
105
{
106
	return (unsigned long)entry & RADIX_DAX_ZERO_PAGE;
107 108
}

109
static int dax_is_empty_entry(void *entry)
110
{
111
	return (unsigned long)entry & RADIX_DAX_EMPTY;
112 113
}

J
Jan Kara 已提交
114 115 116 117 118
/*
 * DAX radix tree locking
 */
struct exceptional_entry_key {
	struct address_space *mapping;
119
	pgoff_t entry_start;
J
Jan Kara 已提交
120 121 122
};

struct wait_exceptional_entry_queue {
123
	wait_queue_entry_t wait;
J
Jan Kara 已提交
124 125 126
	struct exceptional_entry_key key;
};

127 128 129 130 131 132 133 134 135 136
static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
		pgoff_t index, void *entry, struct exceptional_entry_key *key)
{
	unsigned long hash;

	/*
	 * If 'entry' is a PMD, align the 'index' that we use for the wait
	 * queue to the start of that PMD.  This ensures that all offsets in
	 * the range covered by the PMD map to the same bit lock.
	 */
137
	if (dax_is_pmd_entry(entry))
138
		index &= ~PG_PMD_COLOUR;
139 140 141 142 143 144 145 146

	key->mapping = mapping;
	key->entry_start = index;

	hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
	return wait_table + hash;
}

147
static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode,
J
Jan Kara 已提交
148 149 150 151 152 153 154
				       int sync, void *keyp)
{
	struct exceptional_entry_key *key = keyp;
	struct wait_exceptional_entry_queue *ewait =
		container_of(wait, struct wait_exceptional_entry_queue, wait);

	if (key->mapping != ewait->key.mapping ||
155
	    key->entry_start != ewait->key.entry_start)
J
Jan Kara 已提交
156 157 158 159
		return 0;
	return autoremove_wake_function(wait, mode, sync, NULL);
}

R
Ross Zwisler 已提交
160 161 162 163 164 165 166
/*
 * We do not necessarily hold the mapping->tree_lock when we call this
 * function so it is possible that 'entry' is no longer a valid item in the
 * radix tree.  This is okay because all we really need to do is to find the
 * correct waitqueue where tasks might be waiting for that old 'entry' and
 * wake them.
 */
167
static void dax_wake_mapping_entry_waiter(struct address_space *mapping,
R
Ross Zwisler 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
		pgoff_t index, void *entry, bool wake_all)
{
	struct exceptional_entry_key key;
	wait_queue_head_t *wq;

	wq = dax_entry_waitqueue(mapping, index, entry, &key);

	/*
	 * Checking for locked entry and prepare_to_wait_exclusive() happens
	 * under mapping->tree_lock, ditto for entry handling in our callers.
	 * So at this point all tasks that could have seen our entry locked
	 * must be in the waitqueue and the following check will see them.
	 */
	if (waitqueue_active(wq))
		__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
}

J
Jan Kara 已提交
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
/*
 * Check whether the given slot is locked. The function must be called with
 * mapping->tree_lock held
 */
static inline int slot_locked(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
	return entry & RADIX_DAX_ENTRY_LOCK;
}

/*
 * Mark the given slot is locked. The function must be called with
 * mapping->tree_lock held
 */
static inline void *lock_slot(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

	entry |= RADIX_DAX_ENTRY_LOCK;
206
	radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
J
Jan Kara 已提交
207 208 209 210 211 212 213 214 215 216 217 218 219
	return (void *)entry;
}

/*
 * Mark the given slot is unlocked. The function must be called with
 * mapping->tree_lock held
 */
static inline void *unlock_slot(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

	entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
220
	radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
J
Jan Kara 已提交
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
	return (void *)entry;
}

/*
 * Lookup entry in radix tree, wait for it to become unlocked if it is
 * exceptional entry and return it. The caller must call
 * put_unlocked_mapping_entry() when he decided not to lock the entry or
 * put_locked_mapping_entry() when he locked the entry and now wants to
 * unlock it.
 *
 * The function must be called with mapping->tree_lock held.
 */
static void *get_unlocked_mapping_entry(struct address_space *mapping,
					pgoff_t index, void ***slotp)
{
236
	void *entry, **slot;
J
Jan Kara 已提交
237
	struct wait_exceptional_entry_queue ewait;
238
	wait_queue_head_t *wq;
J
Jan Kara 已提交
239 240 241 242 243

	init_wait(&ewait.wait);
	ewait.wait.func = wake_exceptional_entry_func;

	for (;;) {
244
		entry = __radix_tree_lookup(&mapping->page_tree, index, NULL,
J
Jan Kara 已提交
245
					  &slot);
246 247
		if (!entry ||
		    WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) ||
J
Jan Kara 已提交
248 249 250
		    !slot_locked(mapping, slot)) {
			if (slotp)
				*slotp = slot;
251
			return entry;
J
Jan Kara 已提交
252
		}
253 254

		wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
J
Jan Kara 已提交
255 256 257 258 259 260 261 262 263
		prepare_to_wait_exclusive(wq, &ewait.wait,
					  TASK_UNINTERRUPTIBLE);
		spin_unlock_irq(&mapping->tree_lock);
		schedule();
		finish_wait(wq, &ewait.wait);
		spin_lock_irq(&mapping->tree_lock);
	}
}

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
static void dax_unlock_mapping_entry(struct address_space *mapping,
				     pgoff_t index)
{
	void *entry, **slot;

	spin_lock_irq(&mapping->tree_lock);
	entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
	if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
			 !slot_locked(mapping, slot))) {
		spin_unlock_irq(&mapping->tree_lock);
		return;
	}
	unlock_slot(mapping, slot);
	spin_unlock_irq(&mapping->tree_lock);
	dax_wake_mapping_entry_waiter(mapping, index, entry, false);
}

281
static void put_locked_mapping_entry(struct address_space *mapping,
282
		pgoff_t index)
283
{
284
	dax_unlock_mapping_entry(mapping, index);
285 286 287 288 289 290 291 292 293
}

/*
 * Called when we are done with radix tree entry we looked up via
 * get_unlocked_mapping_entry() and which we didn't lock in the end.
 */
static void put_unlocked_mapping_entry(struct address_space *mapping,
				       pgoff_t index, void *entry)
{
294
	if (!entry)
295 296 297 298 299 300
		return;

	/* We have to wake up next waiter for the radix tree entry lock */
	dax_wake_mapping_entry_waiter(mapping, index, entry, false);
}

J
Jan Kara 已提交
301
/*
302 303 304 305
 * Find radix tree entry at given index. If it points to an exceptional entry,
 * return it with the radix tree entry locked. If the radix tree doesn't
 * contain given index, create an empty exceptional entry for the index and
 * return with it locked.
J
Jan Kara 已提交
306
 *
307 308
 * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
 * either return that locked entry or will return an error.  This error will
309 310
 * happen if there are any 4k entries within the 2MiB range that we are
 * requesting.
311 312 313 314 315 316 317 318 319 320 321 322
 *
 * We always favor 4k entries over 2MiB entries. There isn't a flow where we
 * evict 4k entries in order to 'upgrade' them to a 2MiB entry.  A 2MiB
 * insertion will fail if it finds any 4k entries already in the tree, and a
 * 4k insertion will cause an existing 2MiB entry to be unmapped and
 * downgraded to 4k entries.  This happens for both 2MiB huge zero pages as
 * well as 2MiB empty entries.
 *
 * The exception to this downgrade path is for 2MiB DAX PMD entries that have
 * real storage backing them.  We will leave these real 2MiB DAX entries in
 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
 *
J
Jan Kara 已提交
323 324 325 326
 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
 * persistent memory the benefit is doubtful. We can add that later if we can
 * show it helps.
 */
327 328
static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
		unsigned long size_flag)
J
Jan Kara 已提交
329
{
330
	bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
331
	void *entry, **slot;
J
Jan Kara 已提交
332 333 334

restart:
	spin_lock_irq(&mapping->tree_lock);
335
	entry = get_unlocked_mapping_entry(mapping, index, &slot);
336

337 338 339 340 341
	if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) {
		entry = ERR_PTR(-EIO);
		goto out_unlock;
	}

342 343
	if (entry) {
		if (size_flag & RADIX_DAX_PMD) {
344
			if (dax_is_pte_entry(entry)) {
345 346 347 348 349 350
				put_unlocked_mapping_entry(mapping, index,
						entry);
				entry = ERR_PTR(-EEXIST);
				goto out_unlock;
			}
		} else { /* trying to grab a PTE entry */
351
			if (dax_is_pmd_entry(entry) &&
352 353 354 355 356 357 358
			    (dax_is_zero_entry(entry) ||
			     dax_is_empty_entry(entry))) {
				pmd_downgrade = true;
			}
		}
	}

J
Jan Kara 已提交
359
	/* No entry for given index? Make sure radix tree is big enough. */
360
	if (!entry || pmd_downgrade) {
J
Jan Kara 已提交
361 362
		int err;

363 364 365 366 367 368 369 370
		if (pmd_downgrade) {
			/*
			 * Make sure 'entry' remains valid while we drop
			 * mapping->tree_lock.
			 */
			entry = lock_slot(mapping, slot);
		}

J
Jan Kara 已提交
371
		spin_unlock_irq(&mapping->tree_lock);
372 373 374 375 376 377
		/*
		 * Besides huge zero pages the only other thing that gets
		 * downgraded are empty entries which don't need to be
		 * unmapped.
		 */
		if (pmd_downgrade && dax_is_zero_entry(entry))
M
Matthew Wilcox 已提交
378 379
			unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
							PG_PMD_NR, false);
380

J
Jan Kara 已提交
381 382
		err = radix_tree_preload(
				mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
383 384
		if (err) {
			if (pmd_downgrade)
385
				put_locked_mapping_entry(mapping, index);
J
Jan Kara 已提交
386
			return ERR_PTR(err);
387
		}
J
Jan Kara 已提交
388
		spin_lock_irq(&mapping->tree_lock);
389

390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
		if (!entry) {
			/*
			 * We needed to drop the page_tree lock while calling
			 * radix_tree_preload() and we didn't have an entry to
			 * lock.  See if another thread inserted an entry at
			 * our index during this time.
			 */
			entry = __radix_tree_lookup(&mapping->page_tree, index,
					NULL, &slot);
			if (entry) {
				radix_tree_preload_end();
				spin_unlock_irq(&mapping->tree_lock);
				goto restart;
			}
		}

406 407 408 409 410 411 412 413 414 415 416
		if (pmd_downgrade) {
			radix_tree_delete(&mapping->page_tree, index);
			mapping->nrexceptional--;
			dax_wake_mapping_entry_waiter(mapping, index, entry,
					true);
		}

		entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);

		err = __radix_tree_insert(&mapping->page_tree, index,
				dax_radix_order(entry), entry);
J
Jan Kara 已提交
417 418 419
		radix_tree_preload_end();
		if (err) {
			spin_unlock_irq(&mapping->tree_lock);
420
			/*
421 422 423 424 425 426
			 * Our insertion of a DAX entry failed, most likely
			 * because we were inserting a PMD entry and it
			 * collided with a PTE sized entry at a different
			 * index in the PMD range.  We haven't inserted
			 * anything into the radix tree and have no waiters to
			 * wake.
427
			 */
J
Jan Kara 已提交
428 429 430 431 432
			return ERR_PTR(err);
		}
		/* Good, we have inserted empty locked entry into the tree. */
		mapping->nrexceptional++;
		spin_unlock_irq(&mapping->tree_lock);
433
		return entry;
J
Jan Kara 已提交
434
	}
435
	entry = lock_slot(mapping, slot);
436
 out_unlock:
J
Jan Kara 已提交
437
	spin_unlock_irq(&mapping->tree_lock);
438
	return entry;
J
Jan Kara 已提交
439 440
}

441 442 443 444 445 446 447 448 449
static int __dax_invalidate_mapping_entry(struct address_space *mapping,
					  pgoff_t index, bool trunc)
{
	int ret = 0;
	void *entry;
	struct radix_tree_root *page_tree = &mapping->page_tree;

	spin_lock_irq(&mapping->tree_lock);
	entry = get_unlocked_mapping_entry(mapping, index, NULL);
450
	if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)))
451 452 453 454 455 456 457 458 459 460 461 462 463
		goto out;
	if (!trunc &&
	    (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||
	     radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)))
		goto out;
	radix_tree_delete(page_tree, index);
	mapping->nrexceptional--;
	ret = 1;
out:
	put_unlocked_mapping_entry(mapping, index, entry);
	spin_unlock_irq(&mapping->tree_lock);
	return ret;
}
J
Jan Kara 已提交
464 465 466 467 468 469
/*
 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
 * entry to get unlocked before deleting it.
 */
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
{
470
	int ret = __dax_invalidate_mapping_entry(mapping, index, true);
J
Jan Kara 已提交
471 472 473 474 475 476 477 478

	/*
	 * This gets called from truncate / punch_hole path. As such, the caller
	 * must hold locks protecting against concurrent modifications of the
	 * radix tree (usually fs-private i_mmap_sem for writing). Since the
	 * caller has seen exceptional entry for this index, we better find it
	 * at that index as well...
	 */
479 480 481 482 483 484 485 486 487 488 489
	WARN_ON_ONCE(!ret);
	return ret;
}

/*
 * Invalidate exceptional DAX entry if it is clean.
 */
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
				      pgoff_t index)
{
	return __dax_invalidate_mapping_entry(mapping, index, false);
J
Jan Kara 已提交
490 491
}

492 493 494
static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
		sector_t sector, size_t size, struct page *to,
		unsigned long vaddr)
495
{
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
	void *vto, *kaddr;
	pgoff_t pgoff;
	pfn_t pfn;
	long rc;
	int id;

	rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
	if (rc)
		return rc;

	id = dax_read_lock();
	rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
	if (rc < 0) {
		dax_read_unlock(id);
		return rc;
	}
512
	vto = kmap_atomic(to);
513
	copy_user_page(vto, (void __force *)kaddr, vaddr, to);
514
	kunmap_atomic(vto);
515
	dax_read_unlock(id);
516 517 518
	return 0;
}

519 520 521 522 523 524 525
/*
 * By this point grab_mapping_entry() has ensured that we have a locked entry
 * of the appropriate size so we don't have to worry about downgrading PMDs to
 * PTEs.  If we happen to be trying to insert a PTE and there is a PMD
 * already in the tree, we will skip the insertion and just dirty the PMD as
 * appropriate.
 */
J
Jan Kara 已提交
526 527
static void *dax_insert_mapping_entry(struct address_space *mapping,
				      struct vm_fault *vmf,
D
Dan Williams 已提交
528
				      void *entry, pfn_t pfn_t,
529
				      unsigned long flags, bool dirty)
R
Ross Zwisler 已提交
530 531
{
	struct radix_tree_root *page_tree = &mapping->page_tree;
D
Dan Williams 已提交
532
	unsigned long pfn = pfn_t_to_pfn(pfn_t);
J
Jan Kara 已提交
533
	pgoff_t index = vmf->pgoff;
D
Dan Williams 已提交
534
	void *new_entry;
R
Ross Zwisler 已提交
535

536
	if (dirty)
537
		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
R
Ross Zwisler 已提交
538

539 540 541
	if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) {
		/* we are replacing a zero page with block mapping */
		if (dax_is_pmd_entry(entry))
M
Matthew Wilcox 已提交
542 543
			unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
							PG_PMD_NR, false);
544
		else /* pte entry */
M
Matthew Wilcox 已提交
545
			unmap_mapping_pages(mapping, vmf->pgoff, 1, false);
R
Ross Zwisler 已提交
546 547
	}

J
Jan Kara 已提交
548
	spin_lock_irq(&mapping->tree_lock);
D
Dan Williams 已提交
549
	new_entry = dax_radix_locked_entry(pfn, flags);
550

551
	if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
552 553 554 555 556 557 558 559
		/*
		 * Only swap our new entry into the radix tree if the current
		 * entry is a zero page or an empty entry.  If a normal PTE or
		 * PMD entry is already in the tree, we leave it alone.  This
		 * means that if we are trying to insert a PTE and the
		 * existing entry is a PMD, we will just leave the PMD in the
		 * tree and dirty it if necessary.
		 */
560
		struct radix_tree_node *node;
J
Jan Kara 已提交
561 562
		void **slot;
		void *ret;
R
Ross Zwisler 已提交
563

564
		ret = __radix_tree_lookup(page_tree, index, &node, &slot);
J
Jan Kara 已提交
565
		WARN_ON_ONCE(ret != entry);
566
		__radix_tree_replace(page_tree, node, slot,
567
				     new_entry, NULL);
568
		entry = new_entry;
R
Ross Zwisler 已提交
569
	}
570

571
	if (dirty)
R
Ross Zwisler 已提交
572
		radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
573

R
Ross Zwisler 已提交
574
	spin_unlock_irq(&mapping->tree_lock);
575
	return entry;
R
Ross Zwisler 已提交
576 577
}

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
static inline unsigned long
pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
{
	unsigned long address;

	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
	VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
	return address;
}

/* Walk all mappings of a given index of a file and writeprotect them */
static void dax_mapping_entry_mkclean(struct address_space *mapping,
				      pgoff_t index, unsigned long pfn)
{
	struct vm_area_struct *vma;
593 594
	pte_t pte, *ptep = NULL;
	pmd_t *pmdp = NULL;
595 596 597 598
	spinlock_t *ptl;

	i_mmap_lock_read(mapping);
	vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
599
		unsigned long address, start, end;
600 601 602 603 604 605 606

		cond_resched();

		if (!(vma->vm_flags & VM_SHARED))
			continue;

		address = pgoff_address(index, vma);
607 608 609 610 611 612 613

		/*
		 * Note because we provide start/end to follow_pte_pmd it will
		 * call mmu_notifier_invalidate_range_start() on our behalf
		 * before taking any lock.
		 */
		if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))
614 615
			continue;

616 617 618 619 620 621 622
		/*
		 * No need to call mmu_notifier_invalidate_range() as we are
		 * downgrading page table protection not changing it to point
		 * to a new page.
		 *
		 * See Documentation/vm/mmu_notifier.txt
		 */
623 624 625 626 627 628
		if (pmdp) {
#ifdef CONFIG_FS_DAX_PMD
			pmd_t pmd;

			if (pfn != pmd_pfn(*pmdp))
				goto unlock_pmd;
629
			if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
630 631 632 633 634 635 636 637 638
				goto unlock_pmd;

			flush_cache_page(vma, address, pfn);
			pmd = pmdp_huge_clear_flush(vma, address, pmdp);
			pmd = pmd_wrprotect(pmd);
			pmd = pmd_mkclean(pmd);
			set_pmd_at(vma->vm_mm, address, pmdp, pmd);
unlock_pmd:
#endif
639
			spin_unlock(ptl);
640 641 642 643 644 645 646 647 648 649 650 651 652 653
		} else {
			if (pfn != pte_pfn(*ptep))
				goto unlock_pte;
			if (!pte_dirty(*ptep) && !pte_write(*ptep))
				goto unlock_pte;

			flush_cache_page(vma, address, pfn);
			pte = ptep_clear_flush(vma, address, ptep);
			pte = pte_wrprotect(pte);
			pte = pte_mkclean(pte);
			set_pte_at(vma->vm_mm, address, ptep, pte);
unlock_pte:
			pte_unmap_unlock(ptep, ptl);
		}
654

655
		mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
656 657 658 659
	}
	i_mmap_unlock_read(mapping);
}

D
Dan Williams 已提交
660 661
static int dax_writeback_one(struct dax_device *dax_dev,
		struct address_space *mapping, pgoff_t index, void *entry)
R
Ross Zwisler 已提交
662 663
{
	struct radix_tree_root *page_tree = &mapping->page_tree;
D
Dan Williams 已提交
664 665 666
	void *entry2, **slot;
	unsigned long pfn;
	long ret = 0;
667
	size_t size;
R
Ross Zwisler 已提交
668 669

	/*
670 671
	 * A page got tagged dirty in DAX mapping? Something is seriously
	 * wrong.
R
Ross Zwisler 已提交
672
	 */
673 674
	if (WARN_ON(!radix_tree_exceptional_entry(entry)))
		return -EIO;
R
Ross Zwisler 已提交
675

676 677 678
	spin_lock_irq(&mapping->tree_lock);
	entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
	/* Entry got punched out / reallocated? */
679
	if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2)))
680 681 682
		goto put_unlocked;
	/*
	 * Entry got reallocated elsewhere? No need to writeback. We have to
D
Dan Williams 已提交
683
	 * compare pfns as we must not bail out due to difference in lockbit
684 685
	 * or entry type.
	 */
D
Dan Williams 已提交
686
	if (dax_radix_pfn(entry2) != dax_radix_pfn(entry))
687
		goto put_unlocked;
688 689
	if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
				dax_is_zero_entry(entry))) {
R
Ross Zwisler 已提交
690
		ret = -EIO;
691
		goto put_unlocked;
R
Ross Zwisler 已提交
692 693
	}

694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
	/* Another fsync thread may have already written back this entry */
	if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
		goto put_unlocked;
	/* Lock the entry to serialize with page faults */
	entry = lock_slot(mapping, slot);
	/*
	 * We can clear the tag now but we have to be careful so that concurrent
	 * dax_writeback_one() calls for the same index cannot finish before we
	 * actually flush the caches. This is achieved as the calls will look
	 * at the entry only under tree_lock and once they do that they will
	 * see the entry locked and wait for it to unlock.
	 */
	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
	spin_unlock_irq(&mapping->tree_lock);

709 710 711
	/*
	 * Even if dax_writeback_mapping_range() was given a wbc->range_start
	 * in the middle of a PMD, the 'index' we are given will be aligned to
D
Dan Williams 已提交
712 713 714
	 * the start index of the PMD, as will the pfn we pull from 'entry'.
	 * This allows us to flush for PMD_SIZE and not have to worry about
	 * partial PMD writebacks.
715
	 */
D
Dan Williams 已提交
716
	pfn = dax_radix_pfn(entry);
717 718
	size = PAGE_SIZE << dax_radix_order(entry);

D
Dan Williams 已提交
719 720
	dax_mapping_entry_mkclean(mapping, index, pfn);
	dax_flush(dax_dev, page_address(pfn_to_page(pfn)), size);
721 722 723 724 725 726 727 728 729
	/*
	 * After we have flushed the cache, we can clear the dirty tag. There
	 * cannot be new dirty data in the pfn after the flush has completed as
	 * the pfn mappings are writeprotected and fault waits for mapping
	 * entry lock.
	 */
	spin_lock_irq(&mapping->tree_lock);
	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);
	spin_unlock_irq(&mapping->tree_lock);
730
	trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT);
731
	put_locked_mapping_entry(mapping, index);
R
Ross Zwisler 已提交
732 733
	return ret;

734 735
 put_unlocked:
	put_unlocked_mapping_entry(mapping, index, entry2);
R
Ross Zwisler 已提交
736 737 738 739 740 741 742 743 744
	spin_unlock_irq(&mapping->tree_lock);
	return ret;
}

/*
 * Flush the mapping to the persistent domain within the byte range of [start,
 * end]. This is required by data integrity operations to ensure file data is
 * on persistent storage prior to completion of the operation.
 */
745 746
int dax_writeback_mapping_range(struct address_space *mapping,
		struct block_device *bdev, struct writeback_control *wbc)
R
Ross Zwisler 已提交
747 748
{
	struct inode *inode = mapping->host;
749
	pgoff_t start_index, end_index;
R
Ross Zwisler 已提交
750
	pgoff_t indices[PAGEVEC_SIZE];
751
	struct dax_device *dax_dev;
R
Ross Zwisler 已提交
752 753 754 755 756 757 758
	struct pagevec pvec;
	bool done = false;
	int i, ret = 0;

	if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
		return -EIO;

759 760 761
	if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
		return 0;

762 763 764 765
	dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
	if (!dax_dev)
		return -EIO;

766 767
	start_index = wbc->range_start >> PAGE_SHIFT;
	end_index = wbc->range_end >> PAGE_SHIFT;
R
Ross Zwisler 已提交
768

769 770
	trace_dax_writeback_range(inode, start_index, end_index);

R
Ross Zwisler 已提交
771 772
	tag_pages_for_writeback(mapping, start_index, end_index);

773
	pagevec_init(&pvec);
R
Ross Zwisler 已提交
774 775 776 777 778 779 780 781 782 783 784 785 786 787
	while (!done) {
		pvec.nr = find_get_entries_tag(mapping, start_index,
				PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
				pvec.pages, indices);

		if (pvec.nr == 0)
			break;

		for (i = 0; i < pvec.nr; i++) {
			if (indices[i] > end_index) {
				done = true;
				break;
			}

D
Dan Williams 已提交
788 789
			ret = dax_writeback_one(dax_dev, mapping, indices[i],
					pvec.pages[i]);
790 791
			if (ret < 0) {
				mapping_set_error(mapping, ret);
792
				goto out;
793
			}
R
Ross Zwisler 已提交
794
		}
795
		start_index = indices[pvec.nr - 1] + 1;
R
Ross Zwisler 已提交
796
	}
797
out:
798
	put_dax(dax_dev);
799 800
	trace_dax_writeback_range_done(inode, start_index, end_index);
	return (ret < 0 ? ret : 0);
R
Ross Zwisler 已提交
801 802 803
}
EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);

804
static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
805
{
806
	return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
807 808
}

809 810
static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
			 pfn_t *pfnp)
811
{
812
	const sector_t sector = dax_iomap_sector(iomap, pos);
813
	pgoff_t pgoff;
814
	void *kaddr;
815
	int id, rc;
816
	long length;
817

818
	rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
819 820 821
	if (rc)
		return rc;
	id = dax_read_lock();
822 823 824 825 826
	length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
				   &kaddr, pfnp);
	if (length < 0) {
		rc = length;
		goto out;
827
	}
828 829 830 831 832 833 834 835 836 837
	rc = -EINVAL;
	if (PFN_PHYS(length) < size)
		goto out;
	if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
		goto out;
	/* For larger pages we need devmap */
	if (length > 1 && !pfn_t_devmap(*pfnp))
		goto out;
	rc = 0;
out:
838
	dax_read_unlock(id);
839
	return rc;
840 841
}

R
Ross Zwisler 已提交
842
/*
843 844 845 846 847
 * The user has performed a load from a hole in the file.  Allocating a new
 * page in the file would cause excessive storage usage for workloads with
 * sparse files.  Instead we insert a read-only mapping of the 4k zero page.
 * If this page is ever written to we will re-fault and change the mapping to
 * point to real DAX storage instead.
R
Ross Zwisler 已提交
848
 */
849
static int dax_load_hole(struct address_space *mapping, void *entry,
R
Ross Zwisler 已提交
850 851 852
			 struct vm_fault *vmf)
{
	struct inode *inode = mapping->host;
853 854 855 856
	unsigned long vaddr = vmf->address;
	int ret = VM_FAULT_NOPAGE;
	struct page *zero_page;
	void *entry2;
D
Dan Williams 已提交
857
	pfn_t pfn;
R
Ross Zwisler 已提交
858

859 860
	zero_page = ZERO_PAGE(0);
	if (unlikely(!zero_page)) {
R
Ross Zwisler 已提交
861 862 863 864
		ret = VM_FAULT_OOM;
		goto out;
	}

D
Dan Williams 已提交
865 866
	pfn = page_to_pfn_t(zero_page);
	entry2 = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
867
			RADIX_DAX_ZERO_PAGE, false);
868 869 870
	if (IS_ERR(entry2)) {
		ret = VM_FAULT_SIGBUS;
		goto out;
R
Ross Zwisler 已提交
871
	}
872

D
Dan Williams 已提交
873
	vm_insert_mixed(vmf->vma, vaddr, pfn);
R
Ross Zwisler 已提交
874 875 876 877 878
out:
	trace_dax_load_hole(inode, vmf, ret);
	return ret;
}

879 880 881 882 883 884 885 886 887 888 889 890 891
static bool dax_range_is_aligned(struct block_device *bdev,
				 unsigned int offset, unsigned int length)
{
	unsigned short sector_size = bdev_logical_block_size(bdev);

	if (!IS_ALIGNED(offset, sector_size))
		return false;
	if (!IS_ALIGNED(length, sector_size))
		return false;

	return true;
}

892 893 894
int __dax_zero_page_range(struct block_device *bdev,
		struct dax_device *dax_dev, sector_t sector,
		unsigned int offset, unsigned int size)
895
{
896 897
	if (dax_range_is_aligned(bdev, offset, size)) {
		sector_t start_sector = sector + (offset >> 9);
898 899

		return blkdev_issue_zeroout(bdev, start_sector,
900
				size >> 9, GFP_NOFS, 0);
901
	} else {
902 903 904 905 906
		pgoff_t pgoff;
		long rc, id;
		void *kaddr;
		pfn_t pfn;

907
		rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
908 909 910 911
		if (rc)
			return rc;

		id = dax_read_lock();
912
		rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr,
913 914 915 916 917
				&pfn);
		if (rc < 0) {
			dax_read_unlock(id);
			return rc;
		}
918
		memset(kaddr + offset, 0, size);
919
		dax_flush(dax_dev, kaddr + offset, size);
920
		dax_read_unlock(id);
921
	}
922 923 924 925
	return 0;
}
EXPORT_SYMBOL_GPL(__dax_zero_page_range);

926
static loff_t
927
dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
928 929
		struct iomap *iomap)
{
930 931
	struct block_device *bdev = iomap->bdev;
	struct dax_device *dax_dev = iomap->dax_dev;
932 933 934
	struct iov_iter *iter = data;
	loff_t end = pos + length, done = 0;
	ssize_t ret = 0;
935
	int id;
936 937 938 939 940 941 942 943 944 945 946 947 948

	if (iov_iter_rw(iter) == READ) {
		end = min(end, i_size_read(inode));
		if (pos >= end)
			return 0;

		if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
			return iov_iter_zero(min(length, end - pos), iter);
	}

	if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
		return -EIO;

949 950 951 952 953
	/*
	 * Write can allocate block for an area which has a hole page mapped
	 * into page tables. We have to tear down these mappings so that data
	 * written by write(2) is visible in mmap.
	 */
954
	if (iomap->flags & IOMAP_F_NEW) {
955 956 957 958 959
		invalidate_inode_pages2_range(inode->i_mapping,
					      pos >> PAGE_SHIFT,
					      (end - 1) >> PAGE_SHIFT);
	}

960
	id = dax_read_lock();
961 962
	while (pos < end) {
		unsigned offset = pos & (PAGE_SIZE - 1);
963 964
		const size_t size = ALIGN(length + offset, PAGE_SIZE);
		const sector_t sector = dax_iomap_sector(iomap, pos);
965
		ssize_t map_len;
966 967 968
		pgoff_t pgoff;
		void *kaddr;
		pfn_t pfn;
969

970 971 972 973 974
		if (fatal_signal_pending(current)) {
			ret = -EINTR;
			break;
		}

975 976 977 978 979 980
		ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
		if (ret)
			break;

		map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
				&kaddr, &pfn);
981 982 983 984 985
		if (map_len < 0) {
			ret = map_len;
			break;
		}

986 987
		map_len = PFN_PHYS(map_len);
		kaddr += offset;
988 989 990 991
		map_len -= offset;
		if (map_len > end - pos)
			map_len = end - pos;

992 993 994 995 996
		/*
		 * The userspace address for the memory copy has already been
		 * validated via access_ok() in either vfs_read() or
		 * vfs_write(), depending on which operation we are doing.
		 */
997
		if (iov_iter_rw(iter) == WRITE)
998 999
			map_len = dax_copy_from_iter(dax_dev, pgoff, kaddr,
					map_len, iter);
1000
		else
1001
			map_len = copy_to_iter(kaddr, map_len, iter);
1002 1003 1004 1005 1006 1007 1008 1009 1010
		if (map_len <= 0) {
			ret = map_len ? map_len : -EFAULT;
			break;
		}

		pos += map_len;
		length -= map_len;
		done += map_len;
	}
1011
	dax_read_unlock(id);
1012 1013 1014 1015 1016

	return done ? done : ret;
}

/**
1017
 * dax_iomap_rw - Perform I/O to a DAX file
1018 1019 1020 1021 1022 1023 1024 1025 1026
 * @iocb:	The control block for this I/O
 * @iter:	The addresses to do I/O from or to
 * @ops:	iomap ops passed from the file system
 *
 * This function performs read and write operations to directly mapped
 * persistent memory.  The callers needs to take care of read/write exclusion
 * and evicting any page cache pages in the region under I/O.
 */
ssize_t
1027
dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
1028
		const struct iomap_ops *ops)
1029 1030 1031 1032 1033 1034
{
	struct address_space *mapping = iocb->ki_filp->f_mapping;
	struct inode *inode = mapping->host;
	loff_t pos = iocb->ki_pos, ret = 0, done = 0;
	unsigned flags = 0;

1035 1036
	if (iov_iter_rw(iter) == WRITE) {
		lockdep_assert_held_exclusive(&inode->i_rwsem);
1037
		flags |= IOMAP_WRITE;
1038 1039 1040
	} else {
		lockdep_assert_held(&inode->i_rwsem);
	}
1041 1042 1043

	while (iov_iter_count(iter)) {
		ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
1044
				iter, dax_iomap_actor);
1045 1046 1047 1048 1049 1050 1051 1052 1053
		if (ret <= 0)
			break;
		pos += ret;
		done += ret;
	}

	iocb->ki_pos += done;
	return done ? done : ret;
}
1054
EXPORT_SYMBOL_GPL(dax_iomap_rw);
1055

1056 1057 1058 1059 1060 1061 1062 1063 1064
static int dax_fault_return(int error)
{
	if (error == 0)
		return VM_FAULT_NOPAGE;
	if (error == -ENOMEM)
		return VM_FAULT_OOM;
	return VM_FAULT_SIGBUS;
}

1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
/*
 * MAP_SYNC on a dax mapping guarantees dirty metadata is
 * flushed on write-faults (non-cow), but not read-faults.
 */
static bool dax_fault_is_synchronous(unsigned long flags,
		struct vm_area_struct *vma, struct iomap *iomap)
{
	return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
		&& (iomap->flags & IOMAP_F_DIRTY);
}

1076
static int dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
1077
			       int *iomap_errp, const struct iomap_ops *ops)
1078
{
1079 1080
	struct vm_area_struct *vma = vmf->vma;
	struct address_space *mapping = vma->vm_file->f_mapping;
1081
	struct inode *inode = mapping->host;
1082
	unsigned long vaddr = vmf->address;
1083 1084
	loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
	struct iomap iomap = { 0 };
J
Jan Kara 已提交
1085
	unsigned flags = IOMAP_FAULT;
1086
	int error, major = 0;
1087
	bool write = vmf->flags & FAULT_FLAG_WRITE;
1088
	bool sync;
1089
	int vmf_ret = 0;
1090
	void *entry;
1091
	pfn_t pfn;
1092

1093
	trace_dax_pte_fault(inode, vmf, vmf_ret);
1094 1095 1096 1097 1098
	/*
	 * Check whether offset isn't beyond end of file now. Caller is supposed
	 * to hold locks serializing us with truncate / punch hole so this is
	 * a reliable test.
	 */
1099 1100 1101 1102
	if (pos >= i_size_read(inode)) {
		vmf_ret = VM_FAULT_SIGBUS;
		goto out;
	}
1103

1104
	if (write && !vmf->cow_page)
1105 1106
		flags |= IOMAP_WRITE;

1107 1108 1109 1110 1111 1112
	entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
	if (IS_ERR(entry)) {
		vmf_ret = dax_fault_return(PTR_ERR(entry));
		goto out;
	}

1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
	/*
	 * It is possible, particularly with mixed reads & writes to private
	 * mappings, that we have raced with a PMD fault that overlaps with
	 * the PTE we need to set up.  If so just return and the fault will be
	 * retried.
	 */
	if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
		vmf_ret = VM_FAULT_NOPAGE;
		goto unlock_entry;
	}

1124 1125 1126 1127 1128 1129
	/*
	 * Note that we don't bother to use iomap_apply here: DAX required
	 * the file system block size to be equal the page size, which means
	 * that we never have to deal with more than a single extent here.
	 */
	error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1130 1131
	if (iomap_errp)
		*iomap_errp = error;
1132 1133
	if (error) {
		vmf_ret = dax_fault_return(error);
1134
		goto unlock_entry;
1135
	}
1136
	if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
1137 1138
		error = -EIO;	/* fs corruption? */
		goto error_finish_iomap;
1139 1140 1141
	}

	if (vmf->cow_page) {
1142 1143
		sector_t sector = dax_iomap_sector(&iomap, pos);

1144 1145 1146 1147 1148 1149
		switch (iomap.type) {
		case IOMAP_HOLE:
		case IOMAP_UNWRITTEN:
			clear_user_highpage(vmf->cow_page, vaddr);
			break;
		case IOMAP_MAPPED:
1150 1151
			error = copy_user_dax(iomap.bdev, iomap.dax_dev,
					sector, PAGE_SIZE, vmf->cow_page, vaddr);
1152 1153 1154 1155 1156 1157 1158 1159
			break;
		default:
			WARN_ON_ONCE(1);
			error = -EIO;
			break;
		}

		if (error)
1160
			goto error_finish_iomap;
1161 1162 1163 1164 1165

		__SetPageUptodate(vmf->cow_page);
		vmf_ret = finish_fault(vmf);
		if (!vmf_ret)
			vmf_ret = VM_FAULT_DONE_COW;
1166
		goto finish_iomap;
1167 1168
	}

1169
	sync = dax_fault_is_synchronous(flags, vma, &iomap);
1170

1171 1172 1173 1174
	switch (iomap.type) {
	case IOMAP_MAPPED:
		if (iomap.flags & IOMAP_F_NEW) {
			count_vm_event(PGMAJFAULT);
1175
			count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
1176 1177
			major = VM_FAULT_MAJOR;
		}
1178 1179 1180 1181
		error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
		if (error < 0)
			goto error_finish_iomap;

D
Dan Williams 已提交
1182
		entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
1183
						 0, write && !sync);
1184 1185 1186 1187 1188
		if (IS_ERR(entry)) {
			error = PTR_ERR(entry);
			goto error_finish_iomap;
		}

1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
		/*
		 * If we are doing synchronous page fault and inode needs fsync,
		 * we can insert PTE into page tables only after that happens.
		 * Skip insertion for now and return the pfn so that caller can
		 * insert it after fsync is done.
		 */
		if (sync) {
			if (WARN_ON_ONCE(!pfnp)) {
				error = -EIO;
				goto error_finish_iomap;
			}
			*pfnp = pfn;
			vmf_ret = VM_FAULT_NEEDDSYNC | major;
			goto finish_iomap;
		}
1204 1205 1206 1207 1208 1209
		trace_dax_insert_mapping(inode, vmf, entry);
		if (write)
			error = vm_insert_mixed_mkwrite(vma, vaddr, pfn);
		else
			error = vm_insert_mixed(vma, vaddr, pfn);

1210 1211 1212
		/* -EBUSY is fine, somebody else faulted on the same PTE */
		if (error == -EBUSY)
			error = 0;
1213 1214 1215
		break;
	case IOMAP_UNWRITTEN:
	case IOMAP_HOLE:
1216
		if (!write) {
1217
			vmf_ret = dax_load_hole(mapping, entry, vmf);
1218
			goto finish_iomap;
1219
		}
1220 1221 1222 1223 1224 1225 1226
		/*FALLTHRU*/
	default:
		WARN_ON_ONCE(1);
		error = -EIO;
		break;
	}

1227
 error_finish_iomap:
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
	vmf_ret = dax_fault_return(error) | major;
 finish_iomap:
	if (ops->iomap_end) {
		int copied = PAGE_SIZE;

		if (vmf_ret & VM_FAULT_ERROR)
			copied = 0;
		/*
		 * The fault is done by now and there's no way back (other
		 * thread may be already happily using PTE we have installed).
		 * Just ignore error from ->iomap_end since we cannot do much
		 * with it.
		 */
		ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
1242
	}
1243
 unlock_entry:
1244
	put_locked_mapping_entry(mapping, vmf->pgoff);
1245
 out:
1246
	trace_dax_pte_fault_done(inode, vmf, vmf_ret);
1247
	return vmf_ret;
1248
}
1249 1250

#ifdef CONFIG_FS_DAX_PMD
1251
static int dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap,
1252
		void *entry)
1253
{
1254 1255
	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
	unsigned long pmd_addr = vmf->address & PMD_MASK;
1256
	struct inode *inode = mapping->host;
1257
	struct page *zero_page;
1258
	void *ret = NULL;
1259 1260
	spinlock_t *ptl;
	pmd_t pmd_entry;
D
Dan Williams 已提交
1261
	pfn_t pfn;
1262

1263
	zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
1264 1265

	if (unlikely(!zero_page))
1266
		goto fallback;
1267

D
Dan Williams 已提交
1268 1269
	pfn = page_to_pfn_t(zero_page);
	ret = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
1270
			RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE, false);
1271
	if (IS_ERR(ret))
1272
		goto fallback;
1273

1274 1275
	ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
	if (!pmd_none(*(vmf->pmd))) {
1276
		spin_unlock(ptl);
1277
		goto fallback;
1278 1279
	}

1280
	pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
1281
	pmd_entry = pmd_mkhuge(pmd_entry);
1282
	set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
1283
	spin_unlock(ptl);
1284
	trace_dax_pmd_load_hole(inode, vmf, zero_page, ret);
1285
	return VM_FAULT_NOPAGE;
1286 1287

fallback:
1288
	trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret);
1289
	return VM_FAULT_FALLBACK;
1290 1291
}

1292
static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
1293
			       const struct iomap_ops *ops)
1294
{
1295
	struct vm_area_struct *vma = vmf->vma;
1296
	struct address_space *mapping = vma->vm_file->f_mapping;
1297 1298
	unsigned long pmd_addr = vmf->address & PMD_MASK;
	bool write = vmf->flags & FAULT_FLAG_WRITE;
1299
	bool sync;
J
Jan Kara 已提交
1300
	unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
1301 1302 1303 1304 1305 1306 1307
	struct inode *inode = mapping->host;
	int result = VM_FAULT_FALLBACK;
	struct iomap iomap = { 0 };
	pgoff_t max_pgoff, pgoff;
	void *entry;
	loff_t pos;
	int error;
1308
	pfn_t pfn;
1309

1310 1311 1312 1313 1314 1315
	/*
	 * Check whether offset isn't beyond end of file now. Caller is
	 * supposed to hold locks serializing us with truncate / punch hole so
	 * this is a reliable test.
	 */
	pgoff = linear_page_index(vma, pmd_addr);
1316
	max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
1317

1318
	trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
1319

1320 1321 1322 1323 1324 1325 1326 1327 1328 1329
	/*
	 * Make sure that the faulting address's PMD offset (color) matches
	 * the PMD offset from the start of the file.  This is necessary so
	 * that a PMD range in the page table overlaps exactly with a PMD
	 * range in the radix tree.
	 */
	if ((vmf->pgoff & PG_PMD_COLOUR) !=
	    ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
		goto fallback;

1330 1331 1332 1333 1334 1335 1336 1337 1338 1339
	/* Fall back to PTEs if we're going to COW */
	if (write && !(vma->vm_flags & VM_SHARED))
		goto fallback;

	/* If the PMD would extend outside the VMA */
	if (pmd_addr < vma->vm_start)
		goto fallback;
	if ((pmd_addr + PMD_SIZE) > vma->vm_end)
		goto fallback;

1340
	if (pgoff >= max_pgoff) {
1341 1342 1343
		result = VM_FAULT_SIGBUS;
		goto out;
	}
1344 1345

	/* If the PMD would extend beyond the file size */
1346
	if ((pgoff | PG_PMD_COLOUR) >= max_pgoff)
1347 1348
		goto fallback;

1349
	/*
1350 1351 1352 1353
	 * grab_mapping_entry() will make sure we get a 2MiB empty entry, a
	 * 2MiB zero page entry or a DAX PMD.  If it can't (because a 4k page
	 * is already in the tree, for instance), it will return -EEXIST and
	 * we just fall back to 4k entries.
1354 1355 1356 1357 1358
	 */
	entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
	if (IS_ERR(entry))
		goto fallback;

1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
	/*
	 * It is possible, particularly with mixed reads & writes to private
	 * mappings, that we have raced with a PTE fault that overlaps with
	 * the PMD we need to set up.  If so just return and the fault will be
	 * retried.
	 */
	if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
			!pmd_devmap(*vmf->pmd)) {
		result = 0;
		goto unlock_entry;
	}

1371 1372 1373 1374 1375 1376 1377 1378
	/*
	 * Note that we don't use iomap_apply here.  We aren't doing I/O, only
	 * setting up a mapping, so really we're using iomap_begin() as a way
	 * to look up our filesystem block.
	 */
	pos = (loff_t)pgoff << PAGE_SHIFT;
	error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
	if (error)
1379
		goto unlock_entry;
1380

1381 1382 1383
	if (iomap.offset + iomap.length < pos + PMD_SIZE)
		goto finish_iomap;

1384
	sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);
1385

1386 1387
	switch (iomap.type) {
	case IOMAP_MAPPED:
1388 1389 1390 1391
		error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);
		if (error < 0)
			goto finish_iomap;

D
Dan Williams 已提交
1392
		entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
1393
						RADIX_DAX_PMD, write && !sync);
1394 1395 1396
		if (IS_ERR(entry))
			goto finish_iomap;

1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410
		/*
		 * If we are doing synchronous page fault and inode needs fsync,
		 * we can insert PMD into page tables only after that happens.
		 * Skip insertion for now and return the pfn so that caller can
		 * insert it after fsync is done.
		 */
		if (sync) {
			if (WARN_ON_ONCE(!pfnp))
				goto finish_iomap;
			*pfnp = pfn;
			result = VM_FAULT_NEEDDSYNC;
			goto finish_iomap;
		}

1411 1412 1413
		trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
		result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn,
					    write);
1414 1415 1416 1417
		break;
	case IOMAP_UNWRITTEN:
	case IOMAP_HOLE:
		if (WARN_ON_ONCE(write))
1418
			break;
1419
		result = dax_pmd_load_hole(vmf, &iomap, entry);
1420 1421 1422 1423 1424 1425 1426 1427
		break;
	default:
		WARN_ON_ONCE(1);
		break;
	}

 finish_iomap:
	if (ops->iomap_end) {
1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439
		int copied = PMD_SIZE;

		if (result == VM_FAULT_FALLBACK)
			copied = 0;
		/*
		 * The fault is done by now and there's no way back (other
		 * thread may be already happily using PMD we have installed).
		 * Just ignore error from ->iomap_end since we cannot do much
		 * with it.
		 */
		ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
				&iomap);
1440
	}
1441
 unlock_entry:
1442
	put_locked_mapping_entry(mapping, pgoff);
1443 1444
 fallback:
	if (result == VM_FAULT_FALLBACK) {
1445
		split_huge_pmd(vma, vmf->pmd, vmf->address);
1446 1447
		count_vm_event(THP_FAULT_FALLBACK);
	}
1448
out:
1449
	trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
1450 1451
	return result;
}
1452
#else
1453
static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
1454
			       const struct iomap_ops *ops)
1455 1456 1457
{
	return VM_FAULT_FALLBACK;
}
1458
#endif /* CONFIG_FS_DAX_PMD */
1459 1460 1461 1462

/**
 * dax_iomap_fault - handle a page fault on a DAX file
 * @vmf: The description of the fault
1463
 * @pe_size: Size of the page to fault in
1464
 * @pfnp: PFN to insert for synchronous faults if fsync is required
1465
 * @iomap_errp: Storage for detailed error code in case of error
1466
 * @ops: Iomap ops passed from the file system
1467 1468 1469 1470 1471 1472
 *
 * When a page fault occurs, filesystems may call this helper in
 * their fault handler for DAX files. dax_iomap_fault() assumes the caller
 * has done all the necessary locking for page fault to proceed
 * successfully.
 */
1473
int dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
1474
		    pfn_t *pfnp, int *iomap_errp, const struct iomap_ops *ops)
1475
{
1476 1477
	switch (pe_size) {
	case PE_SIZE_PTE:
1478
		return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);
1479
	case PE_SIZE_PMD:
1480
		return dax_iomap_pmd_fault(vmf, pfnp, ops);
1481 1482 1483 1484 1485
	default:
		return VM_FAULT_FALLBACK;
	}
}
EXPORT_SYMBOL_GPL(dax_iomap_fault);
J
Jan Kara 已提交
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568

/**
 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
 * @vmf: The description of the fault
 * @pe_size: Size of entry to be inserted
 * @pfn: PFN to insert
 *
 * This function inserts writeable PTE or PMD entry into page tables for mmaped
 * DAX file.  It takes care of marking corresponding radix tree entry as dirty
 * as well.
 */
static int dax_insert_pfn_mkwrite(struct vm_fault *vmf,
				  enum page_entry_size pe_size,
				  pfn_t pfn)
{
	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
	void *entry, **slot;
	pgoff_t index = vmf->pgoff;
	int vmf_ret, error;

	spin_lock_irq(&mapping->tree_lock);
	entry = get_unlocked_mapping_entry(mapping, index, &slot);
	/* Did we race with someone splitting entry or so? */
	if (!entry ||
	    (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) ||
	    (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) {
		put_unlocked_mapping_entry(mapping, index, entry);
		spin_unlock_irq(&mapping->tree_lock);
		trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
						      VM_FAULT_NOPAGE);
		return VM_FAULT_NOPAGE;
	}
	radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY);
	entry = lock_slot(mapping, slot);
	spin_unlock_irq(&mapping->tree_lock);
	switch (pe_size) {
	case PE_SIZE_PTE:
		error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
		vmf_ret = dax_fault_return(error);
		break;
#ifdef CONFIG_FS_DAX_PMD
	case PE_SIZE_PMD:
		vmf_ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
			pfn, true);
		break;
#endif
	default:
		vmf_ret = VM_FAULT_FALLBACK;
	}
	put_locked_mapping_entry(mapping, index);
	trace_dax_insert_pfn_mkwrite(mapping->host, vmf, vmf_ret);
	return vmf_ret;
}

/**
 * dax_finish_sync_fault - finish synchronous page fault
 * @vmf: The description of the fault
 * @pe_size: Size of entry to be inserted
 * @pfn: PFN to insert
 *
 * This function ensures that the file range touched by the page fault is
 * stored persistently on the media and handles inserting of appropriate page
 * table entry.
 */
int dax_finish_sync_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
			  pfn_t pfn)
{
	int err;
	loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
	size_t len = 0;

	if (pe_size == PE_SIZE_PTE)
		len = PAGE_SIZE;
	else if (pe_size == PE_SIZE_PMD)
		len = PMD_SIZE;
	else
		WARN_ON_ONCE(1);
	err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
	if (err)
		return VM_FAULT_SIGBUS;
	return dax_insert_pfn_mkwrite(vmf, pe_size, pfn);
}
EXPORT_SYMBOL_GPL(dax_finish_sync_fault);