dax.c 40.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * fs/dax.c - Direct Access filesystem code
 * Copyright (c) 2013-2014 Intel Corporation
 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/atomic.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
20
#include <linux/dax.h>
21 22
#include <linux/fs.h>
#include <linux/genhd.h>
23 24 25
#include <linux/highmem.h>
#include <linux/memcontrol.h>
#include <linux/mm.h>
26
#include <linux/mutex.h>
R
Ross Zwisler 已提交
27
#include <linux/pagevec.h>
28
#include <linux/sched.h>
29
#include <linux/sched/signal.h>
30
#include <linux/uio.h>
31
#include <linux/vmstat.h>
D
Dan Williams 已提交
32
#include <linux/pfn_t.h>
33
#include <linux/sizes.h>
34
#include <linux/mmu_notifier.h>
35 36
#include <linux/iomap.h>
#include "internal.h"
37

38 39 40
#define CREATE_TRACE_POINTS
#include <trace/events/fs_dax.h>

J
Jan Kara 已提交
41 42 43 44
/* We choose 4096 entries - same as per-zone page wait tables */
#define DAX_WAIT_TABLE_BITS 12
#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)

45 46 47
/* The 'colour' (ie low bits) within a PMD of a page offset.  */
#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)

48
static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
J
Jan Kara 已提交
49 50 51 52 53 54 55 56 57 58 59

static int __init init_dax_wait_table(void)
{
	int i;

	for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
		init_waitqueue_head(wait_table + i);
	return 0;
}
fs_initcall(init_dax_wait_table);

60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
/*
 * We use lowest available bit in exceptional entry for locking, one bit for
 * the entry size (PMD) and two more to tell us if the entry is a zero page or
 * an empty entry that is just used for locking.  In total four special bits.
 *
 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
 * block allocation.
 */
#define RADIX_DAX_SHIFT		(RADIX_TREE_EXCEPTIONAL_SHIFT + 4)
#define RADIX_DAX_ENTRY_LOCK	(1 << RADIX_TREE_EXCEPTIONAL_SHIFT)
#define RADIX_DAX_PMD		(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
#define RADIX_DAX_ZERO_PAGE	(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
#define RADIX_DAX_EMPTY		(1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))

static unsigned long dax_radix_sector(void *entry)
{
	return (unsigned long)entry >> RADIX_DAX_SHIFT;
}

static void *dax_radix_locked_entry(sector_t sector, unsigned long flags)
{
	return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags |
			((unsigned long)sector << RADIX_DAX_SHIFT) |
			RADIX_DAX_ENTRY_LOCK);
}

static unsigned int dax_radix_order(void *entry)
{
	if ((unsigned long)entry & RADIX_DAX_PMD)
		return PMD_SHIFT - PAGE_SHIFT;
	return 0;
}

94
static int dax_is_pmd_entry(void *entry)
95
{
96
	return (unsigned long)entry & RADIX_DAX_PMD;
97 98
}

99
static int dax_is_pte_entry(void *entry)
100
{
101
	return !((unsigned long)entry & RADIX_DAX_PMD);
102 103
}

104
static int dax_is_zero_entry(void *entry)
105
{
106
	return (unsigned long)entry & RADIX_DAX_ZERO_PAGE;
107 108
}

109
static int dax_is_empty_entry(void *entry)
110
{
111
	return (unsigned long)entry & RADIX_DAX_EMPTY;
112 113
}

J
Jan Kara 已提交
114 115 116 117 118
/*
 * DAX radix tree locking
 */
struct exceptional_entry_key {
	struct address_space *mapping;
119
	pgoff_t entry_start;
J
Jan Kara 已提交
120 121 122
};

struct wait_exceptional_entry_queue {
123
	wait_queue_entry_t wait;
J
Jan Kara 已提交
124 125 126
	struct exceptional_entry_key key;
};

127 128 129 130 131 132 133 134 135 136
static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
		pgoff_t index, void *entry, struct exceptional_entry_key *key)
{
	unsigned long hash;

	/*
	 * If 'entry' is a PMD, align the 'index' that we use for the wait
	 * queue to the start of that PMD.  This ensures that all offsets in
	 * the range covered by the PMD map to the same bit lock.
	 */
137
	if (dax_is_pmd_entry(entry))
138
		index &= ~PG_PMD_COLOUR;
139 140 141 142 143 144 145 146

	key->mapping = mapping;
	key->entry_start = index;

	hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
	return wait_table + hash;
}

147
static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode,
J
Jan Kara 已提交
148 149 150 151 152 153 154
				       int sync, void *keyp)
{
	struct exceptional_entry_key *key = keyp;
	struct wait_exceptional_entry_queue *ewait =
		container_of(wait, struct wait_exceptional_entry_queue, wait);

	if (key->mapping != ewait->key.mapping ||
155
	    key->entry_start != ewait->key.entry_start)
J
Jan Kara 已提交
156 157 158 159
		return 0;
	return autoremove_wake_function(wait, mode, sync, NULL);
}

R
Ross Zwisler 已提交
160 161 162 163 164 165 166
/*
 * We do not necessarily hold the mapping->tree_lock when we call this
 * function so it is possible that 'entry' is no longer a valid item in the
 * radix tree.  This is okay because all we really need to do is to find the
 * correct waitqueue where tasks might be waiting for that old 'entry' and
 * wake them.
 */
167
static void dax_wake_mapping_entry_waiter(struct address_space *mapping,
R
Ross Zwisler 已提交
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
		pgoff_t index, void *entry, bool wake_all)
{
	struct exceptional_entry_key key;
	wait_queue_head_t *wq;

	wq = dax_entry_waitqueue(mapping, index, entry, &key);

	/*
	 * Checking for locked entry and prepare_to_wait_exclusive() happens
	 * under mapping->tree_lock, ditto for entry handling in our callers.
	 * So at this point all tasks that could have seen our entry locked
	 * must be in the waitqueue and the following check will see them.
	 */
	if (waitqueue_active(wq))
		__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
}

J
Jan Kara 已提交
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
/*
 * Check whether the given slot is locked. The function must be called with
 * mapping->tree_lock held
 */
static inline int slot_locked(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
	return entry & RADIX_DAX_ENTRY_LOCK;
}

/*
 * Mark the given slot is locked. The function must be called with
 * mapping->tree_lock held
 */
static inline void *lock_slot(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

	entry |= RADIX_DAX_ENTRY_LOCK;
206
	radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
J
Jan Kara 已提交
207 208 209 210 211 212 213 214 215 216 217 218 219
	return (void *)entry;
}

/*
 * Mark the given slot is unlocked. The function must be called with
 * mapping->tree_lock held
 */
static inline void *unlock_slot(struct address_space *mapping, void **slot)
{
	unsigned long entry = (unsigned long)
		radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

	entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
220
	radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
J
Jan Kara 已提交
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
	return (void *)entry;
}

/*
 * Lookup entry in radix tree, wait for it to become unlocked if it is
 * exceptional entry and return it. The caller must call
 * put_unlocked_mapping_entry() when he decided not to lock the entry or
 * put_locked_mapping_entry() when he locked the entry and now wants to
 * unlock it.
 *
 * The function must be called with mapping->tree_lock held.
 */
static void *get_unlocked_mapping_entry(struct address_space *mapping,
					pgoff_t index, void ***slotp)
{
236
	void *entry, **slot;
J
Jan Kara 已提交
237
	struct wait_exceptional_entry_queue ewait;
238
	wait_queue_head_t *wq;
J
Jan Kara 已提交
239 240 241 242 243

	init_wait(&ewait.wait);
	ewait.wait.func = wake_exceptional_entry_func;

	for (;;) {
244
		entry = __radix_tree_lookup(&mapping->page_tree, index, NULL,
J
Jan Kara 已提交
245
					  &slot);
246 247
		if (!entry ||
		    WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) ||
J
Jan Kara 已提交
248 249 250
		    !slot_locked(mapping, slot)) {
			if (slotp)
				*slotp = slot;
251
			return entry;
J
Jan Kara 已提交
252
		}
253 254

		wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
J
Jan Kara 已提交
255 256 257 258 259 260 261 262 263
		prepare_to_wait_exclusive(wq, &ewait.wait,
					  TASK_UNINTERRUPTIBLE);
		spin_unlock_irq(&mapping->tree_lock);
		schedule();
		finish_wait(wq, &ewait.wait);
		spin_lock_irq(&mapping->tree_lock);
	}
}

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
static void dax_unlock_mapping_entry(struct address_space *mapping,
				     pgoff_t index)
{
	void *entry, **slot;

	spin_lock_irq(&mapping->tree_lock);
	entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
	if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
			 !slot_locked(mapping, slot))) {
		spin_unlock_irq(&mapping->tree_lock);
		return;
	}
	unlock_slot(mapping, slot);
	spin_unlock_irq(&mapping->tree_lock);
	dax_wake_mapping_entry_waiter(mapping, index, entry, false);
}

281
static void put_locked_mapping_entry(struct address_space *mapping,
282
		pgoff_t index)
283
{
284
	dax_unlock_mapping_entry(mapping, index);
285 286 287 288 289 290 291 292 293
}

/*
 * Called when we are done with radix tree entry we looked up via
 * get_unlocked_mapping_entry() and which we didn't lock in the end.
 */
static void put_unlocked_mapping_entry(struct address_space *mapping,
				       pgoff_t index, void *entry)
{
294
	if (!entry)
295 296 297 298 299 300
		return;

	/* We have to wake up next waiter for the radix tree entry lock */
	dax_wake_mapping_entry_waiter(mapping, index, entry, false);
}

J
Jan Kara 已提交
301
/*
302 303 304 305
 * Find radix tree entry at given index. If it points to an exceptional entry,
 * return it with the radix tree entry locked. If the radix tree doesn't
 * contain given index, create an empty exceptional entry for the index and
 * return with it locked.
J
Jan Kara 已提交
306
 *
307 308
 * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
 * either return that locked entry or will return an error.  This error will
309 310
 * happen if there are any 4k entries within the 2MiB range that we are
 * requesting.
311 312 313 314 315 316 317 318 319 320 321 322
 *
 * We always favor 4k entries over 2MiB entries. There isn't a flow where we
 * evict 4k entries in order to 'upgrade' them to a 2MiB entry.  A 2MiB
 * insertion will fail if it finds any 4k entries already in the tree, and a
 * 4k insertion will cause an existing 2MiB entry to be unmapped and
 * downgraded to 4k entries.  This happens for both 2MiB huge zero pages as
 * well as 2MiB empty entries.
 *
 * The exception to this downgrade path is for 2MiB DAX PMD entries that have
 * real storage backing them.  We will leave these real 2MiB DAX entries in
 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
 *
J
Jan Kara 已提交
323 324 325 326
 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
 * persistent memory the benefit is doubtful. We can add that later if we can
 * show it helps.
 */
327 328
static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
		unsigned long size_flag)
J
Jan Kara 已提交
329
{
330
	bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
331
	void *entry, **slot;
J
Jan Kara 已提交
332 333 334

restart:
	spin_lock_irq(&mapping->tree_lock);
335
	entry = get_unlocked_mapping_entry(mapping, index, &slot);
336

337 338 339 340 341
	if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) {
		entry = ERR_PTR(-EIO);
		goto out_unlock;
	}

342 343
	if (entry) {
		if (size_flag & RADIX_DAX_PMD) {
344
			if (dax_is_pte_entry(entry)) {
345 346 347 348 349 350
				put_unlocked_mapping_entry(mapping, index,
						entry);
				entry = ERR_PTR(-EEXIST);
				goto out_unlock;
			}
		} else { /* trying to grab a PTE entry */
351
			if (dax_is_pmd_entry(entry) &&
352 353 354 355 356 357 358
			    (dax_is_zero_entry(entry) ||
			     dax_is_empty_entry(entry))) {
				pmd_downgrade = true;
			}
		}
	}

J
Jan Kara 已提交
359
	/* No entry for given index? Make sure radix tree is big enough. */
360
	if (!entry || pmd_downgrade) {
J
Jan Kara 已提交
361 362
		int err;

363 364 365 366 367 368 369 370
		if (pmd_downgrade) {
			/*
			 * Make sure 'entry' remains valid while we drop
			 * mapping->tree_lock.
			 */
			entry = lock_slot(mapping, slot);
		}

J
Jan Kara 已提交
371
		spin_unlock_irq(&mapping->tree_lock);
372 373 374 375 376 377 378 379 380
		/*
		 * Besides huge zero pages the only other thing that gets
		 * downgraded are empty entries which don't need to be
		 * unmapped.
		 */
		if (pmd_downgrade && dax_is_zero_entry(entry))
			unmap_mapping_range(mapping,
				(index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);

J
Jan Kara 已提交
381 382
		err = radix_tree_preload(
				mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
383 384
		if (err) {
			if (pmd_downgrade)
385
				put_locked_mapping_entry(mapping, index);
J
Jan Kara 已提交
386
			return ERR_PTR(err);
387
		}
J
Jan Kara 已提交
388
		spin_lock_irq(&mapping->tree_lock);
389

390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
		if (!entry) {
			/*
			 * We needed to drop the page_tree lock while calling
			 * radix_tree_preload() and we didn't have an entry to
			 * lock.  See if another thread inserted an entry at
			 * our index during this time.
			 */
			entry = __radix_tree_lookup(&mapping->page_tree, index,
					NULL, &slot);
			if (entry) {
				radix_tree_preload_end();
				spin_unlock_irq(&mapping->tree_lock);
				goto restart;
			}
		}

406 407 408 409 410 411 412 413 414 415 416
		if (pmd_downgrade) {
			radix_tree_delete(&mapping->page_tree, index);
			mapping->nrexceptional--;
			dax_wake_mapping_entry_waiter(mapping, index, entry,
					true);
		}

		entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);

		err = __radix_tree_insert(&mapping->page_tree, index,
				dax_radix_order(entry), entry);
J
Jan Kara 已提交
417 418 419
		radix_tree_preload_end();
		if (err) {
			spin_unlock_irq(&mapping->tree_lock);
420
			/*
421 422 423 424 425 426
			 * Our insertion of a DAX entry failed, most likely
			 * because we were inserting a PMD entry and it
			 * collided with a PTE sized entry at a different
			 * index in the PMD range.  We haven't inserted
			 * anything into the radix tree and have no waiters to
			 * wake.
427
			 */
J
Jan Kara 已提交
428 429 430 431 432
			return ERR_PTR(err);
		}
		/* Good, we have inserted empty locked entry into the tree. */
		mapping->nrexceptional++;
		spin_unlock_irq(&mapping->tree_lock);
433
		return entry;
J
Jan Kara 已提交
434
	}
435
	entry = lock_slot(mapping, slot);
436
 out_unlock:
J
Jan Kara 已提交
437
	spin_unlock_irq(&mapping->tree_lock);
438
	return entry;
J
Jan Kara 已提交
439 440
}

441 442 443 444 445 446 447 448 449
static int __dax_invalidate_mapping_entry(struct address_space *mapping,
					  pgoff_t index, bool trunc)
{
	int ret = 0;
	void *entry;
	struct radix_tree_root *page_tree = &mapping->page_tree;

	spin_lock_irq(&mapping->tree_lock);
	entry = get_unlocked_mapping_entry(mapping, index, NULL);
450
	if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)))
451 452 453 454 455 456 457 458 459 460 461 462 463
		goto out;
	if (!trunc &&
	    (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||
	     radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)))
		goto out;
	radix_tree_delete(page_tree, index);
	mapping->nrexceptional--;
	ret = 1;
out:
	put_unlocked_mapping_entry(mapping, index, entry);
	spin_unlock_irq(&mapping->tree_lock);
	return ret;
}
J
Jan Kara 已提交
464 465 466 467 468 469
/*
 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
 * entry to get unlocked before deleting it.
 */
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
{
470
	int ret = __dax_invalidate_mapping_entry(mapping, index, true);
J
Jan Kara 已提交
471 472 473 474 475 476 477 478

	/*
	 * This gets called from truncate / punch_hole path. As such, the caller
	 * must hold locks protecting against concurrent modifications of the
	 * radix tree (usually fs-private i_mmap_sem for writing). Since the
	 * caller has seen exceptional entry for this index, we better find it
	 * at that index as well...
	 */
479 480 481 482 483 484 485 486 487 488 489
	WARN_ON_ONCE(!ret);
	return ret;
}

/*
 * Invalidate exceptional DAX entry if it is clean.
 */
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
				      pgoff_t index)
{
	return __dax_invalidate_mapping_entry(mapping, index, false);
J
Jan Kara 已提交
490 491
}

492 493 494
static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
		sector_t sector, size_t size, struct page *to,
		unsigned long vaddr)
495
{
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
	void *vto, *kaddr;
	pgoff_t pgoff;
	pfn_t pfn;
	long rc;
	int id;

	rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
	if (rc)
		return rc;

	id = dax_read_lock();
	rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
	if (rc < 0) {
		dax_read_unlock(id);
		return rc;
	}
512
	vto = kmap_atomic(to);
513
	copy_user_page(vto, (void __force *)kaddr, vaddr, to);
514
	kunmap_atomic(vto);
515
	dax_read_unlock(id);
516 517 518
	return 0;
}

519 520 521 522 523 524 525
/*
 * By this point grab_mapping_entry() has ensured that we have a locked entry
 * of the appropriate size so we don't have to worry about downgrading PMDs to
 * PTEs.  If we happen to be trying to insert a PTE and there is a PMD
 * already in the tree, we will skip the insertion and just dirty the PMD as
 * appropriate.
 */
J
Jan Kara 已提交
526 527
static void *dax_insert_mapping_entry(struct address_space *mapping,
				      struct vm_fault *vmf,
528 529
				      void *entry, sector_t sector,
				      unsigned long flags)
R
Ross Zwisler 已提交
530 531
{
	struct radix_tree_root *page_tree = &mapping->page_tree;
J
Jan Kara 已提交
532 533
	void *new_entry;
	pgoff_t index = vmf->pgoff;
R
Ross Zwisler 已提交
534

J
Jan Kara 已提交
535
	if (vmf->flags & FAULT_FLAG_WRITE)
536
		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
R
Ross Zwisler 已提交
537

538 539 540 541 542 543 544 545 546
	if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) {
		/* we are replacing a zero page with block mapping */
		if (dax_is_pmd_entry(entry))
			unmap_mapping_range(mapping,
					(vmf->pgoff << PAGE_SHIFT) & PMD_MASK,
					PMD_SIZE, 0);
		else /* pte entry */
			unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
					PAGE_SIZE, 0);
R
Ross Zwisler 已提交
547 548
	}

J
Jan Kara 已提交
549
	spin_lock_irq(&mapping->tree_lock);
550 551
	new_entry = dax_radix_locked_entry(sector, flags);

552
	if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
553 554 555 556 557 558 559 560
		/*
		 * Only swap our new entry into the radix tree if the current
		 * entry is a zero page or an empty entry.  If a normal PTE or
		 * PMD entry is already in the tree, we leave it alone.  This
		 * means that if we are trying to insert a PTE and the
		 * existing entry is a PMD, we will just leave the PMD in the
		 * tree and dirty it if necessary.
		 */
561
		struct radix_tree_node *node;
J
Jan Kara 已提交
562 563
		void **slot;
		void *ret;
R
Ross Zwisler 已提交
564

565
		ret = __radix_tree_lookup(page_tree, index, &node, &slot);
J
Jan Kara 已提交
566
		WARN_ON_ONCE(ret != entry);
567
		__radix_tree_replace(page_tree, node, slot,
568
				     new_entry, NULL);
569
		entry = new_entry;
R
Ross Zwisler 已提交
570
	}
571

J
Jan Kara 已提交
572
	if (vmf->flags & FAULT_FLAG_WRITE)
R
Ross Zwisler 已提交
573
		radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
574

R
Ross Zwisler 已提交
575
	spin_unlock_irq(&mapping->tree_lock);
576
	return entry;
R
Ross Zwisler 已提交
577 578
}

579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
static inline unsigned long
pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
{
	unsigned long address;

	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
	VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
	return address;
}

/* Walk all mappings of a given index of a file and writeprotect them */
static void dax_mapping_entry_mkclean(struct address_space *mapping,
				      pgoff_t index, unsigned long pfn)
{
	struct vm_area_struct *vma;
594 595
	pte_t pte, *ptep = NULL;
	pmd_t *pmdp = NULL;
596 597 598 599
	spinlock_t *ptl;

	i_mmap_lock_read(mapping);
	vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
600
		unsigned long address, start, end;
601 602 603 604 605 606 607

		cond_resched();

		if (!(vma->vm_flags & VM_SHARED))
			continue;

		address = pgoff_address(index, vma);
608 609 610 611 612 613 614

		/*
		 * Note because we provide start/end to follow_pte_pmd it will
		 * call mmu_notifier_invalidate_range_start() on our behalf
		 * before taking any lock.
		 */
		if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))
615 616
			continue;

617 618 619 620 621 622 623
		/*
		 * No need to call mmu_notifier_invalidate_range() as we are
		 * downgrading page table protection not changing it to point
		 * to a new page.
		 *
		 * See Documentation/vm/mmu_notifier.txt
		 */
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
		if (pmdp) {
#ifdef CONFIG_FS_DAX_PMD
			pmd_t pmd;

			if (pfn != pmd_pfn(*pmdp))
				goto unlock_pmd;
			if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
				goto unlock_pmd;

			flush_cache_page(vma, address, pfn);
			pmd = pmdp_huge_clear_flush(vma, address, pmdp);
			pmd = pmd_wrprotect(pmd);
			pmd = pmd_mkclean(pmd);
			set_pmd_at(vma->vm_mm, address, pmdp, pmd);
unlock_pmd:
			spin_unlock(ptl);
#endif
		} else {
			if (pfn != pte_pfn(*ptep))
				goto unlock_pte;
			if (!pte_dirty(*ptep) && !pte_write(*ptep))
				goto unlock_pte;

			flush_cache_page(vma, address, pfn);
			pte = ptep_clear_flush(vma, address, ptep);
			pte = pte_wrprotect(pte);
			pte = pte_mkclean(pte);
			set_pte_at(vma->vm_mm, address, ptep, pte);
unlock_pte:
			pte_unmap_unlock(ptep, ptl);
		}
655

656
		mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
657 658 659 660
	}
	i_mmap_unlock_read(mapping);
}

R
Ross Zwisler 已提交
661
static int dax_writeback_one(struct block_device *bdev,
662 663
		struct dax_device *dax_dev, struct address_space *mapping,
		pgoff_t index, void *entry)
R
Ross Zwisler 已提交
664 665
{
	struct radix_tree_root *page_tree = &mapping->page_tree;
666 667 668 669 670 671
	void *entry2, **slot, *kaddr;
	long ret = 0, id;
	sector_t sector;
	pgoff_t pgoff;
	size_t size;
	pfn_t pfn;
R
Ross Zwisler 已提交
672 673

	/*
674 675
	 * A page got tagged dirty in DAX mapping? Something is seriously
	 * wrong.
R
Ross Zwisler 已提交
676
	 */
677 678
	if (WARN_ON(!radix_tree_exceptional_entry(entry)))
		return -EIO;
R
Ross Zwisler 已提交
679

680 681 682
	spin_lock_irq(&mapping->tree_lock);
	entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
	/* Entry got punched out / reallocated? */
683
	if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2)))
684 685 686 687 688 689 690 691
		goto put_unlocked;
	/*
	 * Entry got reallocated elsewhere? No need to writeback. We have to
	 * compare sectors as we must not bail out due to difference in lockbit
	 * or entry type.
	 */
	if (dax_radix_sector(entry2) != dax_radix_sector(entry))
		goto put_unlocked;
692 693
	if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
				dax_is_zero_entry(entry))) {
R
Ross Zwisler 已提交
694
		ret = -EIO;
695
		goto put_unlocked;
R
Ross Zwisler 已提交
696 697
	}

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
	/* Another fsync thread may have already written back this entry */
	if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
		goto put_unlocked;
	/* Lock the entry to serialize with page faults */
	entry = lock_slot(mapping, slot);
	/*
	 * We can clear the tag now but we have to be careful so that concurrent
	 * dax_writeback_one() calls for the same index cannot finish before we
	 * actually flush the caches. This is achieved as the calls will look
	 * at the entry only under tree_lock and once they do that they will
	 * see the entry locked and wait for it to unlock.
	 */
	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
	spin_unlock_irq(&mapping->tree_lock);

713 714 715 716 717 718 719
	/*
	 * Even if dax_writeback_mapping_range() was given a wbc->range_start
	 * in the middle of a PMD, the 'index' we are given will be aligned to
	 * the start index of the PMD, as will the sector we pull from
	 * 'entry'.  This allows us to flush for PMD_SIZE and not have to
	 * worry about partial PMD writebacks.
	 */
720 721 722 723 724 725 726
	sector = dax_radix_sector(entry);
	size = PAGE_SIZE << dax_radix_order(entry);

	id = dax_read_lock();
	ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
	if (ret)
		goto dax_unlock;
R
Ross Zwisler 已提交
727 728

	/*
729 730
	 * dax_direct_access() may sleep, so cannot hold tree_lock over
	 * its invocation.
R
Ross Zwisler 已提交
731
	 */
732 733 734
	ret = dax_direct_access(dax_dev, pgoff, size / PAGE_SIZE, &kaddr, &pfn);
	if (ret < 0)
		goto dax_unlock;
R
Ross Zwisler 已提交
735

736
	if (WARN_ON_ONCE(ret < size / PAGE_SIZE)) {
R
Ross Zwisler 已提交
737
		ret = -EIO;
738
		goto dax_unlock;
R
Ross Zwisler 已提交
739 740
	}

741
	dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(pfn));
742
	dax_flush(dax_dev, kaddr, size);
743 744 745 746 747 748 749 750 751
	/*
	 * After we have flushed the cache, we can clear the dirty tag. There
	 * cannot be new dirty data in the pfn after the flush has completed as
	 * the pfn mappings are writeprotected and fault waits for mapping
	 * entry lock.
	 */
	spin_lock_irq(&mapping->tree_lock);
	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);
	spin_unlock_irq(&mapping->tree_lock);
752
	trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT);
753 754
 dax_unlock:
	dax_read_unlock(id);
755
	put_locked_mapping_entry(mapping, index);
R
Ross Zwisler 已提交
756 757
	return ret;

758 759
 put_unlocked:
	put_unlocked_mapping_entry(mapping, index, entry2);
R
Ross Zwisler 已提交
760 761 762 763 764 765 766 767 768
	spin_unlock_irq(&mapping->tree_lock);
	return ret;
}

/*
 * Flush the mapping to the persistent domain within the byte range of [start,
 * end]. This is required by data integrity operations to ensure file data is
 * on persistent storage prior to completion of the operation.
 */
769 770
int dax_writeback_mapping_range(struct address_space *mapping,
		struct block_device *bdev, struct writeback_control *wbc)
R
Ross Zwisler 已提交
771 772
{
	struct inode *inode = mapping->host;
773
	pgoff_t start_index, end_index;
R
Ross Zwisler 已提交
774
	pgoff_t indices[PAGEVEC_SIZE];
775
	struct dax_device *dax_dev;
R
Ross Zwisler 已提交
776 777 778 779 780 781 782
	struct pagevec pvec;
	bool done = false;
	int i, ret = 0;

	if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
		return -EIO;

783 784 785
	if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
		return 0;

786 787 788 789
	dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
	if (!dax_dev)
		return -EIO;

790 791
	start_index = wbc->range_start >> PAGE_SHIFT;
	end_index = wbc->range_end >> PAGE_SHIFT;
R
Ross Zwisler 已提交
792

793 794
	trace_dax_writeback_range(inode, start_index, end_index);

R
Ross Zwisler 已提交
795 796
	tag_pages_for_writeback(mapping, start_index, end_index);

797
	pagevec_init(&pvec);
R
Ross Zwisler 已提交
798 799 800 801 802 803 804 805 806 807 808 809 810 811
	while (!done) {
		pvec.nr = find_get_entries_tag(mapping, start_index,
				PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
				pvec.pages, indices);

		if (pvec.nr == 0)
			break;

		for (i = 0; i < pvec.nr; i++) {
			if (indices[i] > end_index) {
				done = true;
				break;
			}

812 813
			ret = dax_writeback_one(bdev, dax_dev, mapping,
					indices[i], pvec.pages[i]);
814 815
			if (ret < 0) {
				mapping_set_error(mapping, ret);
816
				goto out;
817
			}
R
Ross Zwisler 已提交
818
		}
819
		start_index = indices[pvec.nr - 1] + 1;
R
Ross Zwisler 已提交
820
	}
821
out:
822
	put_dax(dax_dev);
823 824
	trace_dax_writeback_range_done(inode, start_index, end_index);
	return (ret < 0 ? ret : 0);
R
Ross Zwisler 已提交
825 826 827
}
EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);

J
Jan Kara 已提交
828
static int dax_insert_mapping(struct address_space *mapping,
829
		struct block_device *bdev, struct dax_device *dax_dev,
830
		sector_t sector, size_t size, void *entry,
831
		struct vm_area_struct *vma, struct vm_fault *vmf)
832
{
833
	unsigned long vaddr = vmf->address;
834 835 836 837
	void *ret, *kaddr;
	pgoff_t pgoff;
	int id, rc;
	pfn_t pfn;
838

839 840 841
	rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
	if (rc)
		return rc;
842

843 844 845 846 847 848 849 850 851
	id = dax_read_lock();
	rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
	if (rc < 0) {
		dax_read_unlock(id);
		return rc;
	}
	dax_read_unlock(id);

	ret = dax_insert_mapping_entry(mapping, vmf, entry, sector, 0);
J
Jan Kara 已提交
852 853
	if (IS_ERR(ret))
		return PTR_ERR(ret);
R
Ross Zwisler 已提交
854

855
	trace_dax_insert_mapping(mapping->host, vmf, ret);
856 857 858 859
	if (vmf->flags & FAULT_FLAG_WRITE)
		return vm_insert_mixed_mkwrite(vma, vaddr, pfn);
	else
		return vm_insert_mixed(vma, vaddr, pfn);
860 861
}

R
Ross Zwisler 已提交
862
/*
863 864 865 866 867
 * The user has performed a load from a hole in the file.  Allocating a new
 * page in the file would cause excessive storage usage for workloads with
 * sparse files.  Instead we insert a read-only mapping of the 4k zero page.
 * If this page is ever written to we will re-fault and change the mapping to
 * point to real DAX storage instead.
R
Ross Zwisler 已提交
868
 */
869
static int dax_load_hole(struct address_space *mapping, void *entry,
R
Ross Zwisler 已提交
870 871 872
			 struct vm_fault *vmf)
{
	struct inode *inode = mapping->host;
873 874 875 876
	unsigned long vaddr = vmf->address;
	int ret = VM_FAULT_NOPAGE;
	struct page *zero_page;
	void *entry2;
R
Ross Zwisler 已提交
877

878 879
	zero_page = ZERO_PAGE(0);
	if (unlikely(!zero_page)) {
R
Ross Zwisler 已提交
880 881 882 883
		ret = VM_FAULT_OOM;
		goto out;
	}

884 885 886 887 888
	entry2 = dax_insert_mapping_entry(mapping, vmf, entry, 0,
			RADIX_DAX_ZERO_PAGE);
	if (IS_ERR(entry2)) {
		ret = VM_FAULT_SIGBUS;
		goto out;
R
Ross Zwisler 已提交
889
	}
890 891

	vm_insert_mixed(vmf->vma, vaddr, page_to_pfn_t(zero_page));
R
Ross Zwisler 已提交
892 893 894 895 896
out:
	trace_dax_load_hole(inode, vmf, ret);
	return ret;
}

897 898 899 900 901 902 903 904 905 906 907 908 909
static bool dax_range_is_aligned(struct block_device *bdev,
				 unsigned int offset, unsigned int length)
{
	unsigned short sector_size = bdev_logical_block_size(bdev);

	if (!IS_ALIGNED(offset, sector_size))
		return false;
	if (!IS_ALIGNED(length, sector_size))
		return false;

	return true;
}

910 911 912
int __dax_zero_page_range(struct block_device *bdev,
		struct dax_device *dax_dev, sector_t sector,
		unsigned int offset, unsigned int size)
913
{
914 915
	if (dax_range_is_aligned(bdev, offset, size)) {
		sector_t start_sector = sector + (offset >> 9);
916 917

		return blkdev_issue_zeroout(bdev, start_sector,
918
				size >> 9, GFP_NOFS, 0);
919
	} else {
920 921 922 923 924
		pgoff_t pgoff;
		long rc, id;
		void *kaddr;
		pfn_t pfn;

925
		rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
926 927 928 929
		if (rc)
			return rc;

		id = dax_read_lock();
930
		rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr,
931 932 933 934 935
				&pfn);
		if (rc < 0) {
			dax_read_unlock(id);
			return rc;
		}
936
		memset(kaddr + offset, 0, size);
937
		dax_flush(dax_dev, kaddr + offset, size);
938
		dax_read_unlock(id);
939
	}
940 941 942 943
	return 0;
}
EXPORT_SYMBOL_GPL(__dax_zero_page_range);

944
static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
M
Matthew Wilcox 已提交
945
{
946
	return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
M
Matthew Wilcox 已提交
947
}
948 949

static loff_t
950
dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
951 952
		struct iomap *iomap)
{
953 954
	struct block_device *bdev = iomap->bdev;
	struct dax_device *dax_dev = iomap->dax_dev;
955 956 957
	struct iov_iter *iter = data;
	loff_t end = pos + length, done = 0;
	ssize_t ret = 0;
958
	int id;
959 960 961 962 963 964 965 966 967 968 969 970 971

	if (iov_iter_rw(iter) == READ) {
		end = min(end, i_size_read(inode));
		if (pos >= end)
			return 0;

		if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
			return iov_iter_zero(min(length, end - pos), iter);
	}

	if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
		return -EIO;

972 973 974 975 976
	/*
	 * Write can allocate block for an area which has a hole page mapped
	 * into page tables. We have to tear down these mappings so that data
	 * written by write(2) is visible in mmap.
	 */
977
	if (iomap->flags & IOMAP_F_NEW) {
978 979 980 981 982
		invalidate_inode_pages2_range(inode->i_mapping,
					      pos >> PAGE_SHIFT,
					      (end - 1) >> PAGE_SHIFT);
	}

983
	id = dax_read_lock();
984 985
	while (pos < end) {
		unsigned offset = pos & (PAGE_SIZE - 1);
986 987
		const size_t size = ALIGN(length + offset, PAGE_SIZE);
		const sector_t sector = dax_iomap_sector(iomap, pos);
988
		ssize_t map_len;
989 990 991
		pgoff_t pgoff;
		void *kaddr;
		pfn_t pfn;
992

993 994 995 996 997
		if (fatal_signal_pending(current)) {
			ret = -EINTR;
			break;
		}

998 999 1000 1001 1002 1003
		ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
		if (ret)
			break;

		map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
				&kaddr, &pfn);
1004 1005 1006 1007 1008
		if (map_len < 0) {
			ret = map_len;
			break;
		}

1009 1010
		map_len = PFN_PHYS(map_len);
		kaddr += offset;
1011 1012 1013 1014
		map_len -= offset;
		if (map_len > end - pos)
			map_len = end - pos;

1015 1016 1017 1018 1019
		/*
		 * The userspace address for the memory copy has already been
		 * validated via access_ok() in either vfs_read() or
		 * vfs_write(), depending on which operation we are doing.
		 */
1020
		if (iov_iter_rw(iter) == WRITE)
1021 1022
			map_len = dax_copy_from_iter(dax_dev, pgoff, kaddr,
					map_len, iter);
1023
		else
1024
			map_len = copy_to_iter(kaddr, map_len, iter);
1025 1026 1027 1028 1029 1030 1031 1032 1033
		if (map_len <= 0) {
			ret = map_len ? map_len : -EFAULT;
			break;
		}

		pos += map_len;
		length -= map_len;
		done += map_len;
	}
1034
	dax_read_unlock(id);
1035 1036 1037 1038 1039

	return done ? done : ret;
}

/**
1040
 * dax_iomap_rw - Perform I/O to a DAX file
1041 1042 1043 1044 1045 1046 1047 1048 1049
 * @iocb:	The control block for this I/O
 * @iter:	The addresses to do I/O from or to
 * @ops:	iomap ops passed from the file system
 *
 * This function performs read and write operations to directly mapped
 * persistent memory.  The callers needs to take care of read/write exclusion
 * and evicting any page cache pages in the region under I/O.
 */
ssize_t
1050
dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
1051
		const struct iomap_ops *ops)
1052 1053 1054 1055 1056 1057
{
	struct address_space *mapping = iocb->ki_filp->f_mapping;
	struct inode *inode = mapping->host;
	loff_t pos = iocb->ki_pos, ret = 0, done = 0;
	unsigned flags = 0;

1058 1059
	if (iov_iter_rw(iter) == WRITE) {
		lockdep_assert_held_exclusive(&inode->i_rwsem);
1060
		flags |= IOMAP_WRITE;
1061 1062 1063
	} else {
		lockdep_assert_held(&inode->i_rwsem);
	}
1064 1065 1066

	while (iov_iter_count(iter)) {
		ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
1067
				iter, dax_iomap_actor);
1068 1069 1070 1071 1072 1073 1074 1075 1076
		if (ret <= 0)
			break;
		pos += ret;
		done += ret;
	}

	iocb->ki_pos += done;
	return done ? done : ret;
}
1077
EXPORT_SYMBOL_GPL(dax_iomap_rw);
1078

1079 1080 1081 1082 1083 1084 1085 1086 1087
static int dax_fault_return(int error)
{
	if (error == 0)
		return VM_FAULT_NOPAGE;
	if (error == -ENOMEM)
		return VM_FAULT_OOM;
	return VM_FAULT_SIGBUS;
}

1088 1089
static int dax_iomap_pte_fault(struct vm_fault *vmf,
			       const struct iomap_ops *ops)
1090
{
1091
	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1092
	struct inode *inode = mapping->host;
1093
	unsigned long vaddr = vmf->address;
1094 1095 1096
	loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
	sector_t sector;
	struct iomap iomap = { 0 };
J
Jan Kara 已提交
1097
	unsigned flags = IOMAP_FAULT;
1098
	int error, major = 0;
1099
	int vmf_ret = 0;
1100 1101
	void *entry;

1102
	trace_dax_pte_fault(inode, vmf, vmf_ret);
1103 1104 1105 1106 1107
	/*
	 * Check whether offset isn't beyond end of file now. Caller is supposed
	 * to hold locks serializing us with truncate / punch hole so this is
	 * a reliable test.
	 */
1108 1109 1110 1111
	if (pos >= i_size_read(inode)) {
		vmf_ret = VM_FAULT_SIGBUS;
		goto out;
	}
1112 1113 1114 1115

	if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page)
		flags |= IOMAP_WRITE;

1116 1117 1118 1119 1120 1121
	entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
	if (IS_ERR(entry)) {
		vmf_ret = dax_fault_return(PTR_ERR(entry));
		goto out;
	}

1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
	/*
	 * It is possible, particularly with mixed reads & writes to private
	 * mappings, that we have raced with a PMD fault that overlaps with
	 * the PTE we need to set up.  If so just return and the fault will be
	 * retried.
	 */
	if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
		vmf_ret = VM_FAULT_NOPAGE;
		goto unlock_entry;
	}

1133 1134 1135 1136 1137 1138
	/*
	 * Note that we don't bother to use iomap_apply here: DAX required
	 * the file system block size to be equal the page size, which means
	 * that we never have to deal with more than a single extent here.
	 */
	error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1139 1140
	if (error) {
		vmf_ret = dax_fault_return(error);
1141
		goto unlock_entry;
1142
	}
1143
	if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
1144 1145
		error = -EIO;	/* fs corruption? */
		goto error_finish_iomap;
1146 1147
	}

1148
	sector = dax_iomap_sector(&iomap, pos);
1149 1150 1151 1152 1153 1154 1155 1156

	if (vmf->cow_page) {
		switch (iomap.type) {
		case IOMAP_HOLE:
		case IOMAP_UNWRITTEN:
			clear_user_highpage(vmf->cow_page, vaddr);
			break;
		case IOMAP_MAPPED:
1157 1158
			error = copy_user_dax(iomap.bdev, iomap.dax_dev,
					sector, PAGE_SIZE, vmf->cow_page, vaddr);
1159 1160 1161 1162 1163 1164 1165 1166
			break;
		default:
			WARN_ON_ONCE(1);
			error = -EIO;
			break;
		}

		if (error)
1167
			goto error_finish_iomap;
1168 1169 1170 1171 1172

		__SetPageUptodate(vmf->cow_page);
		vmf_ret = finish_fault(vmf);
		if (!vmf_ret)
			vmf_ret = VM_FAULT_DONE_COW;
1173
		goto finish_iomap;
1174 1175 1176 1177 1178 1179
	}

	switch (iomap.type) {
	case IOMAP_MAPPED:
		if (iomap.flags & IOMAP_F_NEW) {
			count_vm_event(PGMAJFAULT);
1180
			count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
1181 1182
			major = VM_FAULT_MAJOR;
		}
1183
		error = dax_insert_mapping(mapping, iomap.bdev, iomap.dax_dev,
1184
				sector, PAGE_SIZE, entry, vmf->vma, vmf);
1185 1186 1187
		/* -EBUSY is fine, somebody else faulted on the same PTE */
		if (error == -EBUSY)
			error = 0;
1188 1189 1190
		break;
	case IOMAP_UNWRITTEN:
	case IOMAP_HOLE:
1191
		if (!(vmf->flags & FAULT_FLAG_WRITE)) {
1192
			vmf_ret = dax_load_hole(mapping, entry, vmf);
1193
			goto finish_iomap;
1194
		}
1195 1196 1197 1198 1199 1200 1201
		/*FALLTHRU*/
	default:
		WARN_ON_ONCE(1);
		error = -EIO;
		break;
	}

1202
 error_finish_iomap:
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
	vmf_ret = dax_fault_return(error) | major;
 finish_iomap:
	if (ops->iomap_end) {
		int copied = PAGE_SIZE;

		if (vmf_ret & VM_FAULT_ERROR)
			copied = 0;
		/*
		 * The fault is done by now and there's no way back (other
		 * thread may be already happily using PTE we have installed).
		 * Just ignore error from ->iomap_end since we cannot do much
		 * with it.
		 */
		ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
1217
	}
1218
 unlock_entry:
1219
	put_locked_mapping_entry(mapping, vmf->pgoff);
1220
 out:
1221
	trace_dax_pte_fault_done(inode, vmf, vmf_ret);
1222
	return vmf_ret;
1223
}
1224 1225

#ifdef CONFIG_FS_DAX_PMD
1226
static int dax_pmd_insert_mapping(struct vm_fault *vmf, struct iomap *iomap,
1227
		loff_t pos, void *entry)
1228
{
1229
	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1230 1231
	const sector_t sector = dax_iomap_sector(iomap, pos);
	struct dax_device *dax_dev = iomap->dax_dev;
1232
	struct block_device *bdev = iomap->bdev;
1233
	struct inode *inode = mapping->host;
1234 1235 1236 1237
	const size_t size = PMD_SIZE;
	void *ret = NULL, *kaddr;
	long length = 0;
	pgoff_t pgoff;
1238
	pfn_t pfn = {};
1239 1240 1241
	int id;

	if (bdev_dax_pgoff(bdev, sector, size, &pgoff) != 0)
1242
		goto fallback;
1243

1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257
	id = dax_read_lock();
	length = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
	if (length < 0)
		goto unlock_fallback;
	length = PFN_PHYS(length);

	if (length < size)
		goto unlock_fallback;
	if (pfn_t_to_pfn(pfn) & PG_PMD_COLOUR)
		goto unlock_fallback;
	if (!pfn_t_devmap(pfn))
		goto unlock_fallback;
	dax_read_unlock(id);

1258
	ret = dax_insert_mapping_entry(mapping, vmf, entry, sector,
1259 1260
			RADIX_DAX_PMD);
	if (IS_ERR(ret))
1261
		goto fallback;
1262

1263
	trace_dax_pmd_insert_mapping(inode, vmf, length, pfn, ret);
1264
	return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
1265
			pfn, vmf->flags & FAULT_FLAG_WRITE);
1266

1267 1268
unlock_fallback:
	dax_read_unlock(id);
1269
fallback:
1270
	trace_dax_pmd_insert_mapping_fallback(inode, vmf, length, pfn, ret);
1271 1272 1273
	return VM_FAULT_FALLBACK;
}

1274
static int dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap,
1275
		void *entry)
1276
{
1277 1278
	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
	unsigned long pmd_addr = vmf->address & PMD_MASK;
1279
	struct inode *inode = mapping->host;
1280
	struct page *zero_page;
1281
	void *ret = NULL;
1282 1283 1284
	spinlock_t *ptl;
	pmd_t pmd_entry;

1285
	zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
1286 1287

	if (unlikely(!zero_page))
1288
		goto fallback;
1289

1290 1291
	ret = dax_insert_mapping_entry(mapping, vmf, entry, 0,
			RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE);
1292
	if (IS_ERR(ret))
1293
		goto fallback;
1294

1295 1296
	ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
	if (!pmd_none(*(vmf->pmd))) {
1297
		spin_unlock(ptl);
1298
		goto fallback;
1299 1300
	}

1301
	pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
1302
	pmd_entry = pmd_mkhuge(pmd_entry);
1303
	set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
1304
	spin_unlock(ptl);
1305
	trace_dax_pmd_load_hole(inode, vmf, zero_page, ret);
1306
	return VM_FAULT_NOPAGE;
1307 1308

fallback:
1309
	trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret);
1310
	return VM_FAULT_FALLBACK;
1311 1312
}

1313 1314
static int dax_iomap_pmd_fault(struct vm_fault *vmf,
			       const struct iomap_ops *ops)
1315
{
1316
	struct vm_area_struct *vma = vmf->vma;
1317
	struct address_space *mapping = vma->vm_file->f_mapping;
1318 1319
	unsigned long pmd_addr = vmf->address & PMD_MASK;
	bool write = vmf->flags & FAULT_FLAG_WRITE;
J
Jan Kara 已提交
1320
	unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
1321 1322 1323 1324 1325 1326 1327 1328
	struct inode *inode = mapping->host;
	int result = VM_FAULT_FALLBACK;
	struct iomap iomap = { 0 };
	pgoff_t max_pgoff, pgoff;
	void *entry;
	loff_t pos;
	int error;

1329 1330 1331 1332 1333 1334 1335 1336
	/*
	 * Check whether offset isn't beyond end of file now. Caller is
	 * supposed to hold locks serializing us with truncate / punch hole so
	 * this is a reliable test.
	 */
	pgoff = linear_page_index(vma, pmd_addr);
	max_pgoff = (i_size_read(inode) - 1) >> PAGE_SHIFT;

1337
	trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
1338

1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
	/*
	 * Make sure that the faulting address's PMD offset (color) matches
	 * the PMD offset from the start of the file.  This is necessary so
	 * that a PMD range in the page table overlaps exactly with a PMD
	 * range in the radix tree.
	 */
	if ((vmf->pgoff & PG_PMD_COLOUR) !=
	    ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
		goto fallback;

1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
	/* Fall back to PTEs if we're going to COW */
	if (write && !(vma->vm_flags & VM_SHARED))
		goto fallback;

	/* If the PMD would extend outside the VMA */
	if (pmd_addr < vma->vm_start)
		goto fallback;
	if ((pmd_addr + PMD_SIZE) > vma->vm_end)
		goto fallback;

1359 1360 1361 1362
	if (pgoff > max_pgoff) {
		result = VM_FAULT_SIGBUS;
		goto out;
	}
1363 1364 1365 1366 1367

	/* If the PMD would extend beyond the file size */
	if ((pgoff | PG_PMD_COLOUR) > max_pgoff)
		goto fallback;

1368
	/*
1369 1370 1371 1372
	 * grab_mapping_entry() will make sure we get a 2MiB empty entry, a
	 * 2MiB zero page entry or a DAX PMD.  If it can't (because a 4k page
	 * is already in the tree, for instance), it will return -EEXIST and
	 * we just fall back to 4k entries.
1373 1374 1375 1376 1377
	 */
	entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
	if (IS_ERR(entry))
		goto fallback;

1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
	/*
	 * It is possible, particularly with mixed reads & writes to private
	 * mappings, that we have raced with a PTE fault that overlaps with
	 * the PMD we need to set up.  If so just return and the fault will be
	 * retried.
	 */
	if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
			!pmd_devmap(*vmf->pmd)) {
		result = 0;
		goto unlock_entry;
	}

1390 1391 1392 1393 1394 1395 1396 1397
	/*
	 * Note that we don't use iomap_apply here.  We aren't doing I/O, only
	 * setting up a mapping, so really we're using iomap_begin() as a way
	 * to look up our filesystem block.
	 */
	pos = (loff_t)pgoff << PAGE_SHIFT;
	error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
	if (error)
1398
		goto unlock_entry;
1399

1400 1401 1402 1403 1404
	if (iomap.offset + iomap.length < pos + PMD_SIZE)
		goto finish_iomap;

	switch (iomap.type) {
	case IOMAP_MAPPED:
1405
		result = dax_pmd_insert_mapping(vmf, &iomap, pos, entry);
1406 1407 1408 1409
		break;
	case IOMAP_UNWRITTEN:
	case IOMAP_HOLE:
		if (WARN_ON_ONCE(write))
1410
			break;
1411
		result = dax_pmd_load_hole(vmf, &iomap, entry);
1412 1413 1414 1415 1416 1417 1418 1419
		break;
	default:
		WARN_ON_ONCE(1);
		break;
	}

 finish_iomap:
	if (ops->iomap_end) {
1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
		int copied = PMD_SIZE;

		if (result == VM_FAULT_FALLBACK)
			copied = 0;
		/*
		 * The fault is done by now and there's no way back (other
		 * thread may be already happily using PMD we have installed).
		 * Just ignore error from ->iomap_end since we cannot do much
		 * with it.
		 */
		ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
				&iomap);
1432
	}
1433
 unlock_entry:
1434
	put_locked_mapping_entry(mapping, pgoff);
1435 1436
 fallback:
	if (result == VM_FAULT_FALLBACK) {
1437
		split_huge_pmd(vma, vmf->pmd, vmf->address);
1438 1439
		count_vm_event(THP_FAULT_FALLBACK);
	}
1440
out:
1441
	trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
1442 1443
	return result;
}
1444
#else
1445 1446
static int dax_iomap_pmd_fault(struct vm_fault *vmf,
			       const struct iomap_ops *ops)
1447 1448 1449
{
	return VM_FAULT_FALLBACK;
}
1450
#endif /* CONFIG_FS_DAX_PMD */
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461

/**
 * dax_iomap_fault - handle a page fault on a DAX file
 * @vmf: The description of the fault
 * @ops: iomap ops passed from the file system
 *
 * When a page fault occurs, filesystems may call this helper in
 * their fault handler for DAX files. dax_iomap_fault() assumes the caller
 * has done all the necessary locking for page fault to proceed
 * successfully.
 */
1462 1463
int dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
		    const struct iomap_ops *ops)
1464
{
1465 1466
	switch (pe_size) {
	case PE_SIZE_PTE:
1467
		return dax_iomap_pte_fault(vmf, ops);
1468
	case PE_SIZE_PMD:
1469 1470 1471 1472 1473 1474
		return dax_iomap_pmd_fault(vmf, ops);
	default:
		return VM_FAULT_FALLBACK;
	}
}
EXPORT_SYMBOL_GPL(dax_iomap_fault);