huge_memory.c 79.5 KB
Newer Older
1 2 3 4 5 6 7
/*
 *  Copyright (C) 2009  Red Hat, Inc.
 *
 *  This work is licensed under the terms of the GNU GPL, version 2. See
 *  the COPYING file in the top-level directory.
 */

8 9
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

10 11
#include <linux/mm.h>
#include <linux/sched.h>
12
#include <linux/sched/coredump.h>
13
#include <linux/sched/numa_balancing.h>
14 15 16 17 18
#include <linux/highmem.h>
#include <linux/hugetlb.h>
#include <linux/mmu_notifier.h>
#include <linux/rmap.h>
#include <linux/swap.h>
19
#include <linux/shrinker.h>
A
Andrea Arcangeli 已提交
20
#include <linux/mm_inline.h>
21
#include <linux/swapops.h>
M
Matthew Wilcox 已提交
22
#include <linux/dax.h>
A
Andrea Arcangeli 已提交
23
#include <linux/khugepaged.h>
24
#include <linux/freezer.h>
25
#include <linux/pfn_t.h>
A
Andrea Arcangeli 已提交
26
#include <linux/mman.h>
27
#include <linux/memremap.h>
R
Ralf Baechle 已提交
28
#include <linux/pagemap.h>
29
#include <linux/debugfs.h>
30
#include <linux/migrate.h>
31
#include <linux/hashtable.h>
32
#include <linux/userfaultfd_k.h>
33
#include <linux/page_idle.h>
34
#include <linux/shmem_fs.h>
35
#include <linux/oom.h>
36

37 38 39 40
#include <asm/tlb.h>
#include <asm/pgalloc.h>
#include "internal.h"

A
Andrea Arcangeli 已提交
41
/*
42 43 44 45 46 47
 * By default transparent hugepage support is disabled in order that avoid
 * to risk increase the memory footprint of applications without a guaranteed
 * benefit. When transparent hugepage support is enabled, is for all mappings,
 * and khugepaged scans all mappings.
 * Defrag is invoked by khugepaged hugepage allocations and by page faults
 * for all hugepage allocations.
A
Andrea Arcangeli 已提交
48
 */
49
unsigned long transparent_hugepage_flags __read_mostly =
50
#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
A
Andrea Arcangeli 已提交
51
	(1<<TRANSPARENT_HUGEPAGE_FLAG)|
52 53 54 55
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
#endif
56
	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
57 58
	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
A
Andrea Arcangeli 已提交
59

60
static struct shrinker deferred_split_shrinker;
61

62
static atomic_t huge_zero_refcount;
63
struct page *huge_zero_page __read_mostly;
64

65
static struct page *get_huge_zero_page(void)
66 67 68 69
{
	struct page *zero_page;
retry:
	if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
70
		return READ_ONCE(huge_zero_page);
71 72

	zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
73
			HPAGE_PMD_ORDER);
74 75
	if (!zero_page) {
		count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
76
		return NULL;
77 78
	}
	count_vm_event(THP_ZERO_PAGE_ALLOC);
79
	preempt_disable();
80
	if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
81
		preempt_enable();
82
		__free_pages(zero_page, compound_order(zero_page));
83 84 85 86 87 88
		goto retry;
	}

	/* We take additional reference here. It will be put back by shrinker */
	atomic_set(&huge_zero_refcount, 2);
	preempt_enable();
89
	return READ_ONCE(huge_zero_page);
90 91
}

92
static void put_huge_zero_page(void)
93
{
94 95 96 97 98
	/*
	 * Counter should never go to zero here. Only shrinker can put
	 * last reference.
	 */
	BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
99 100
}

101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
struct page *mm_get_huge_zero_page(struct mm_struct *mm)
{
	if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
		return READ_ONCE(huge_zero_page);

	if (!get_huge_zero_page())
		return NULL;

	if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
		put_huge_zero_page();

	return READ_ONCE(huge_zero_page);
}

void mm_put_huge_zero_page(struct mm_struct *mm)
{
	if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
		put_huge_zero_page();
}

121 122
static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
					struct shrink_control *sc)
123
{
124 125 126
	/* we can free zero page only if last reference remains */
	return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
}
127

128 129 130
static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
				       struct shrink_control *sc)
{
131
	if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
132 133
		struct page *zero_page = xchg(&huge_zero_page, NULL);
		BUG_ON(zero_page == NULL);
134
		__free_pages(zero_page, compound_order(zero_page));
135
		return HPAGE_PMD_NR;
136 137 138
	}

	return 0;
139 140
}

141
static struct shrinker huge_zero_page_shrinker = {
142 143
	.count_objects = shrink_huge_zero_page_count,
	.scan_objects = shrink_huge_zero_page_scan,
144 145 146
	.seeks = DEFAULT_SEEKS,
};

147 148 149 150
#ifdef CONFIG_SYSFS
static ssize_t enabled_show(struct kobject *kobj,
			    struct kobj_attribute *attr, char *buf)
{
151 152 153 154 155 156
	if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
		return sprintf(buf, "[always] madvise never\n");
	else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
		return sprintf(buf, "always [madvise] never\n");
	else
		return sprintf(buf, "always madvise [never]\n");
157
}
158

159 160 161 162
static ssize_t enabled_store(struct kobject *kobj,
			     struct kobj_attribute *attr,
			     const char *buf, size_t count)
{
163
	ssize_t ret = count;
A
Andrea Arcangeli 已提交
164

165 166 167 168 169 170 171 172 173 174 175 176 177 178
	if (!memcmp("always", buf,
		    min(sizeof("always")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
	} else if (!memcmp("madvise", buf,
			   min(sizeof("madvise")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
	} else if (!memcmp("never", buf,
			   min(sizeof("never")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
	} else
		ret = -EINVAL;
A
Andrea Arcangeli 已提交
179 180

	if (ret > 0) {
181
		int err = start_stop_khugepaged();
A
Andrea Arcangeli 已提交
182 183 184 185
		if (err)
			ret = err;
	}
	return ret;
186 187 188 189
}
static struct kobj_attribute enabled_attr =
	__ATTR(enabled, 0644, enabled_show, enabled_store);

190
ssize_t single_hugepage_flag_show(struct kobject *kobj,
191 192 193
				struct kobj_attribute *attr, char *buf,
				enum transparent_hugepage_flag flag)
{
194 195
	return sprintf(buf, "%d\n",
		       !!test_bit(flag, &transparent_hugepage_flags));
196
}
197

198
ssize_t single_hugepage_flag_store(struct kobject *kobj,
199 200 201 202
				 struct kobj_attribute *attr,
				 const char *buf, size_t count,
				 enum transparent_hugepage_flag flag)
{
203 204 205 206 207 208 209 210 211 212
	unsigned long value;
	int ret;

	ret = kstrtoul(buf, 10, &value);
	if (ret < 0)
		return ret;
	if (value > 1)
		return -EINVAL;

	if (value)
213
		set_bit(flag, &transparent_hugepage_flags);
214
	else
215 216 217 218 219 220 221 222
		clear_bit(flag, &transparent_hugepage_flags);

	return count;
}

static ssize_t defrag_show(struct kobject *kobj,
			   struct kobj_attribute *attr, char *buf)
{
223
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
224
		return sprintf(buf, "[always] defer defer+madvise madvise never\n");
225
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
226 227 228 229 230 231
		return sprintf(buf, "always [defer] defer+madvise madvise never\n");
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
		return sprintf(buf, "always defer [defer+madvise] madvise never\n");
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
		return sprintf(buf, "always defer defer+madvise [madvise] never\n");
	return sprintf(buf, "always defer defer+madvise madvise [never]\n");
232
}
233

234 235 236 237
static ssize_t defrag_store(struct kobject *kobj,
			    struct kobj_attribute *attr,
			    const char *buf, size_t count)
{
238 239 240 241 242 243 244 245 246 247 248 249
	if (!memcmp("always", buf,
		    min(sizeof("always")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
	} else if (!memcmp("defer+madvise", buf,
		    min(sizeof("defer+madvise")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
250 251 252 253 254 255
	} else if (!memcmp("defer", buf,
		    min(sizeof("defer")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
	} else if (!memcmp("madvise", buf,
			   min(sizeof("madvise")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
	} else if (!memcmp("never", buf,
			   min(sizeof("never")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
	} else
		return -EINVAL;

	return count;
272 273 274 275
}
static struct kobj_attribute defrag_attr =
	__ATTR(defrag, 0644, defrag_show, defrag_store);

276 277 278
static ssize_t use_zero_page_show(struct kobject *kobj,
		struct kobj_attribute *attr, char *buf)
{
279
	return single_hugepage_flag_show(kobj, attr, buf,
280 281 282 283 284
				TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
}
static ssize_t use_zero_page_store(struct kobject *kobj,
		struct kobj_attribute *attr, const char *buf, size_t count)
{
285
	return single_hugepage_flag_store(kobj, attr, buf, count,
286 287 288 289
				 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
}
static struct kobj_attribute use_zero_page_attr =
	__ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
290 291 292 293 294 295 296 297 298

static ssize_t hpage_pmd_size_show(struct kobject *kobj,
		struct kobj_attribute *attr, char *buf)
{
	return sprintf(buf, "%lu\n", HPAGE_PMD_SIZE);
}
static struct kobj_attribute hpage_pmd_size_attr =
	__ATTR_RO(hpage_pmd_size);

299 300 301 302
#ifdef CONFIG_DEBUG_VM
static ssize_t debug_cow_show(struct kobject *kobj,
				struct kobj_attribute *attr, char *buf)
{
303
	return single_hugepage_flag_show(kobj, attr, buf,
304 305 306 307 308 309
				TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
}
static ssize_t debug_cow_store(struct kobject *kobj,
			       struct kobj_attribute *attr,
			       const char *buf, size_t count)
{
310
	return single_hugepage_flag_store(kobj, attr, buf, count,
311 312 313 314 315 316 317 318 319
				 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
}
static struct kobj_attribute debug_cow_attr =
	__ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
#endif /* CONFIG_DEBUG_VM */

static struct attribute *hugepage_attr[] = {
	&enabled_attr.attr,
	&defrag_attr.attr,
320
	&use_zero_page_attr.attr,
321
	&hpage_pmd_size_attr.attr,
322
#if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
323 324
	&shmem_enabled_attr.attr,
#endif
325 326 327 328 329 330
#ifdef CONFIG_DEBUG_VM
	&debug_cow_attr.attr,
#endif
	NULL,
};

331
static const struct attribute_group hugepage_attr_group = {
332
	.attrs = hugepage_attr,
A
Andrea Arcangeli 已提交
333 334
};

S
Shaohua Li 已提交
335
static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
336 337 338
{
	int err;

S
Shaohua Li 已提交
339 340
	*hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
	if (unlikely(!*hugepage_kobj)) {
341
		pr_err("failed to create transparent hugepage kobject\n");
S
Shaohua Li 已提交
342
		return -ENOMEM;
A
Andrea Arcangeli 已提交
343 344
	}

S
Shaohua Li 已提交
345
	err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
A
Andrea Arcangeli 已提交
346
	if (err) {
347
		pr_err("failed to register transparent hugepage group\n");
S
Shaohua Li 已提交
348
		goto delete_obj;
A
Andrea Arcangeli 已提交
349 350
	}

S
Shaohua Li 已提交
351
	err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
A
Andrea Arcangeli 已提交
352
	if (err) {
353
		pr_err("failed to register transparent hugepage group\n");
S
Shaohua Li 已提交
354
		goto remove_hp_group;
A
Andrea Arcangeli 已提交
355
	}
S
Shaohua Li 已提交
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392

	return 0;

remove_hp_group:
	sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
delete_obj:
	kobject_put(*hugepage_kobj);
	return err;
}

static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
{
	sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
	sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
	kobject_put(hugepage_kobj);
}
#else
static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
{
	return 0;
}

static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
{
}
#endif /* CONFIG_SYSFS */

static int __init hugepage_init(void)
{
	int err;
	struct kobject *hugepage_kobj;

	if (!has_transparent_hugepage()) {
		transparent_hugepage_flags = 0;
		return -EINVAL;
	}

393 394 395 396 397 398 399 400 401 402
	/*
	 * hugepages can't be allocated by the buddy allocator
	 */
	MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER);
	/*
	 * we use page->mapping and page->index in second tail page
	 * as list_head: assuming THP order >= 2
	 */
	MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);

S
Shaohua Li 已提交
403 404
	err = hugepage_init_sysfs(&hugepage_kobj);
	if (err)
405
		goto err_sysfs;
A
Andrea Arcangeli 已提交
406

407
	err = khugepaged_init();
A
Andrea Arcangeli 已提交
408
	if (err)
409
		goto err_slab;
A
Andrea Arcangeli 已提交
410

411 412 413
	err = register_shrinker(&huge_zero_page_shrinker);
	if (err)
		goto err_hzp_shrinker;
414 415 416
	err = register_shrinker(&deferred_split_shrinker);
	if (err)
		goto err_split_shrinker;
417

418 419 420 421 422
	/*
	 * By default disable transparent hugepages on smaller systems,
	 * where the extra memory used could hurt more than TLB overhead
	 * is likely to save.  The admin can still enable it through /sys.
	 */
423
	if (totalram_pages < (512 << (20 - PAGE_SHIFT))) {
424
		transparent_hugepage_flags = 0;
425 426
		return 0;
	}
427

428
	err = start_stop_khugepaged();
429 430
	if (err)
		goto err_khugepaged;
A
Andrea Arcangeli 已提交
431

S
Shaohua Li 已提交
432
	return 0;
433
err_khugepaged:
434 435
	unregister_shrinker(&deferred_split_shrinker);
err_split_shrinker:
436 437
	unregister_shrinker(&huge_zero_page_shrinker);
err_hzp_shrinker:
438
	khugepaged_destroy();
439
err_slab:
S
Shaohua Li 已提交
440
	hugepage_exit_sysfs(hugepage_kobj);
441
err_sysfs:
A
Andrea Arcangeli 已提交
442
	return err;
443
}
444
subsys_initcall(hugepage_init);
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471

static int __init setup_transparent_hugepage(char *str)
{
	int ret = 0;
	if (!str)
		goto out;
	if (!strcmp(str, "always")) {
		set_bit(TRANSPARENT_HUGEPAGE_FLAG,
			&transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
			  &transparent_hugepage_flags);
		ret = 1;
	} else if (!strcmp(str, "madvise")) {
		clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
			  &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
			&transparent_hugepage_flags);
		ret = 1;
	} else if (!strcmp(str, "never")) {
		clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
			  &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
			  &transparent_hugepage_flags);
		ret = 1;
	}
out:
	if (!ret)
472
		pr_warn("transparent_hugepage= cannot parse, ignored\n");
473 474 475 476
	return ret;
}
__setup("transparent_hugepage=", setup_transparent_hugepage);

477
pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
478 479 480 481 482 483
{
	if (likely(vma->vm_flags & VM_WRITE))
		pmd = pmd_mkwrite(pmd);
	return pmd;
}

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
static inline struct list_head *page_deferred_list(struct page *page)
{
	/*
	 * ->lru in the tail pages is occupied by compound_head.
	 * Let's use ->mapping + ->index in the second tail page as list_head.
	 */
	return (struct list_head *)&page[2].mapping;
}

void prep_transhuge_page(struct page *page)
{
	/*
	 * we use page->mapping and page->indexlru in second tail page
	 * as list_head: assuming THP order >= 2
	 */

	INIT_LIST_HEAD(page_deferred_list(page));
	set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
}

504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long len,
		loff_t off, unsigned long flags, unsigned long size)
{
	unsigned long addr;
	loff_t off_end = off + len;
	loff_t off_align = round_up(off, size);
	unsigned long len_pad;

	if (off_end <= off_align || (off_end - off_align) < size)
		return 0;

	len_pad = len + size;
	if (len_pad < len || (off + len_pad) < off)
		return 0;

	addr = current->mm->get_unmapped_area(filp, 0, len_pad,
					      off >> PAGE_SHIFT, flags);
	if (IS_ERR_VALUE(addr))
		return 0;

	addr += (off - addr) & (size - 1);
	return addr;
}

unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	loff_t off = (loff_t)pgoff << PAGE_SHIFT;

	if (addr)
		goto out;
	if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD))
		goto out;

	addr = __thp_get_unmapped_area(filp, len, off, flags, PMD_SIZE);
	if (addr)
		return addr;

 out:
	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
}
EXPORT_SYMBOL_GPL(thp_get_unmapped_area);

J
Jan Kara 已提交
547
static int __do_huge_pmd_anonymous_page(struct vm_fault *vmf, struct page *page,
K
Kirill A. Shutemov 已提交
548
		gfp_t gfp)
549
{
J
Jan Kara 已提交
550
	struct vm_area_struct *vma = vmf->vma;
551
	struct mem_cgroup *memcg;
552
	pgtable_t pgtable;
J
Jan Kara 已提交
553
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
554
	int ret = 0;
555

556
	VM_BUG_ON_PAGE(!PageCompound(page), page);
557

K
Kirill A. Shutemov 已提交
558
	if (mem_cgroup_try_charge(page, vma->vm_mm, gfp, &memcg, true)) {
559 560 561 562
		put_page(page);
		count_vm_event(THP_FAULT_FALLBACK);
		return VM_FAULT_FALLBACK;
	}
563

K
Kirill A. Shutemov 已提交
564
	pgtable = pte_alloc_one(vma->vm_mm, haddr);
565
	if (unlikely(!pgtable)) {
566 567
		ret = VM_FAULT_OOM;
		goto release;
568
	}
569

570
	clear_huge_page(page, vmf->address, HPAGE_PMD_NR);
571 572 573 574 575
	/*
	 * The memory barrier inside __SetPageUptodate makes sure that
	 * clear_huge_page writes become visible before the set_pmd_at()
	 * write.
	 */
576 577
	__SetPageUptodate(page);

J
Jan Kara 已提交
578 579
	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_none(*vmf->pmd))) {
580
		goto unlock_release;
581 582
	} else {
		pmd_t entry;
583

584 585 586 587
		ret = check_stable_address_space(vma->vm_mm);
		if (ret)
			goto unlock_release;

588 589 590 591
		/* Deliver the page fault to userland */
		if (userfaultfd_missing(vma)) {
			int ret;

J
Jan Kara 已提交
592
			spin_unlock(vmf->ptl);
593
			mem_cgroup_cancel_charge(page, memcg, true);
594
			put_page(page);
K
Kirill A. Shutemov 已提交
595
			pte_free(vma->vm_mm, pgtable);
J
Jan Kara 已提交
596
			ret = handle_userfault(vmf, VM_UFFD_MISSING);
597 598 599 600
			VM_BUG_ON(ret & VM_FAULT_FALLBACK);
			return ret;
		}

601 602
		entry = mk_huge_pmd(page, vma->vm_page_prot);
		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
603
		page_add_new_anon_rmap(page, vma, haddr, true);
604
		mem_cgroup_commit_charge(page, memcg, false, true);
605
		lru_cache_add_active_or_unevictable(page, vma);
J
Jan Kara 已提交
606 607
		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
K
Kirill A. Shutemov 已提交
608 609
		add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
		atomic_long_inc(&vma->vm_mm->nr_ptes);
J
Jan Kara 已提交
610
		spin_unlock(vmf->ptl);
611
		count_vm_event(THP_FAULT_ALLOC);
612 613
	}

614
	return 0;
615 616 617 618 619 620 621 622 623
unlock_release:
	spin_unlock(vmf->ptl);
release:
	if (pgtable)
		pte_free(vma->vm_mm, pgtable);
	mem_cgroup_cancel_charge(page, memcg, true);
	put_page(page);
	return ret;

624 625
}

626
/*
627 628 629 630 631 632 633
 * always: directly stall for all thp allocations
 * defer: wake kswapd and fail if not immediately available
 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
 *		  fail if not immediately available
 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
 *	    available
 * never: never stall for any thp allocation
634 635 636
 */
static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
{
637
	const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
638

639
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
640
		return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
641 642 643 644 645 646 647 648
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
		return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
		return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :
							     __GFP_KSWAPD_RECLAIM);
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
		return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :
							     0);
649
	return GFP_TRANSHUGE_LIGHT;
650 651
}

652
/* Caller must hold page table lock. */
653
static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
654
		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
655
		struct page *zero_page)
656 657
{
	pmd_t entry;
A
Andrew Morton 已提交
658 659
	if (!pmd_none(*pmd))
		return false;
660
	entry = mk_pmd(zero_page, vma->vm_page_prot);
661
	entry = pmd_mkhuge(entry);
662 663
	if (pgtable)
		pgtable_trans_huge_deposit(mm, pmd, pgtable);
664
	set_pmd_at(mm, haddr, pmd, entry);
665
	atomic_long_inc(&mm->nr_ptes);
A
Andrew Morton 已提交
666
	return true;
667 668
}

J
Jan Kara 已提交
669
int do_huge_pmd_anonymous_page(struct vm_fault *vmf)
670
{
J
Jan Kara 已提交
671
	struct vm_area_struct *vma = vmf->vma;
672
	gfp_t gfp;
673
	struct page *page;
J
Jan Kara 已提交
674
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
675

676
	if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
677
		return VM_FAULT_FALLBACK;
678 679
	if (unlikely(anon_vma_prepare(vma)))
		return VM_FAULT_OOM;
680
	if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
681
		return VM_FAULT_OOM;
J
Jan Kara 已提交
682
	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
K
Kirill A. Shutemov 已提交
683
			!mm_forbids_zeropage(vma->vm_mm) &&
684 685 686 687
			transparent_hugepage_use_zero_page()) {
		pgtable_t pgtable;
		struct page *zero_page;
		bool set;
688
		int ret;
K
Kirill A. Shutemov 已提交
689
		pgtable = pte_alloc_one(vma->vm_mm, haddr);
690
		if (unlikely(!pgtable))
A
Andrea Arcangeli 已提交
691
			return VM_FAULT_OOM;
692
		zero_page = mm_get_huge_zero_page(vma->vm_mm);
693
		if (unlikely(!zero_page)) {
K
Kirill A. Shutemov 已提交
694
			pte_free(vma->vm_mm, pgtable);
695
			count_vm_event(THP_FAULT_FALLBACK);
696
			return VM_FAULT_FALLBACK;
A
Andrea Arcangeli 已提交
697
		}
J
Jan Kara 已提交
698
		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
699 700
		ret = 0;
		set = false;
J
Jan Kara 已提交
701
		if (pmd_none(*vmf->pmd)) {
702 703 704 705
			ret = check_stable_address_space(vma->vm_mm);
			if (ret) {
				spin_unlock(vmf->ptl);
			} else if (userfaultfd_missing(vma)) {
J
Jan Kara 已提交
706 707
				spin_unlock(vmf->ptl);
				ret = handle_userfault(vmf, VM_UFFD_MISSING);
708 709
				VM_BUG_ON(ret & VM_FAULT_FALLBACK);
			} else {
K
Kirill A. Shutemov 已提交
710
				set_huge_zero_page(pgtable, vma->vm_mm, vma,
J
Jan Kara 已提交
711 712
						   haddr, vmf->pmd, zero_page);
				spin_unlock(vmf->ptl);
713 714 715
				set = true;
			}
		} else
J
Jan Kara 已提交
716
			spin_unlock(vmf->ptl);
717
		if (!set)
K
Kirill A. Shutemov 已提交
718
			pte_free(vma->vm_mm, pgtable);
719
		return ret;
720
	}
721
	gfp = alloc_hugepage_direct_gfpmask(vma);
722
	page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
723 724
	if (unlikely(!page)) {
		count_vm_event(THP_FAULT_FALLBACK);
725
		return VM_FAULT_FALLBACK;
726
	}
727
	prep_transhuge_page(page);
J
Jan Kara 已提交
728
	return __do_huge_pmd_anonymous_page(vmf, page, gfp);
729 730
}

731
static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
732 733
		pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,
		pgtable_t pgtable)
M
Matthew Wilcox 已提交
734 735 736 737 738 739
{
	struct mm_struct *mm = vma->vm_mm;
	pmd_t entry;
	spinlock_t *ptl;

	ptl = pmd_lock(mm, pmd);
740 741 742
	entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
	if (pfn_t_devmap(pfn))
		entry = pmd_mkdevmap(entry);
743 744 745
	if (write) {
		entry = pmd_mkyoung(pmd_mkdirty(entry));
		entry = maybe_pmd_mkwrite(entry, vma);
M
Matthew Wilcox 已提交
746
	}
747 748 749 750 751 752

	if (pgtable) {
		pgtable_trans_huge_deposit(mm, pmd, pgtable);
		atomic_long_inc(&mm->nr_ptes);
	}

753 754
	set_pmd_at(mm, addr, pmd, entry);
	update_mmu_cache_pmd(vma, addr, pmd);
M
Matthew Wilcox 已提交
755 756 757 758
	spin_unlock(ptl);
}

int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
759
			pmd_t *pmd, pfn_t pfn, bool write)
M
Matthew Wilcox 已提交
760 761
{
	pgprot_t pgprot = vma->vm_page_prot;
762
	pgtable_t pgtable = NULL;
M
Matthew Wilcox 已提交
763 764 765 766 767 768 769 770 771
	/*
	 * If we had pmd_special, we could avoid all these restrictions,
	 * but we need to be consistent with PTEs and architectures that
	 * can't support a 'special' bit.
	 */
	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
						(VM_PFNMAP|VM_MIXEDMAP));
	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
772
	BUG_ON(!pfn_t_devmap(pfn));
M
Matthew Wilcox 已提交
773 774 775

	if (addr < vma->vm_start || addr >= vma->vm_end)
		return VM_FAULT_SIGBUS;
776

777 778 779 780 781 782
	if (arch_needs_pgtable_deposit()) {
		pgtable = pte_alloc_one(vma->vm_mm, addr);
		if (!pgtable)
			return VM_FAULT_OOM;
	}

783 784
	track_pfn_insert(vma, &pgprot, pfn);

785
	insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write, pgtable);
786
	return VM_FAULT_NOPAGE;
M
Matthew Wilcox 已提交
787
}
788
EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
M
Matthew Wilcox 已提交
789

790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
{
	if (likely(vma->vm_flags & VM_WRITE))
		pud = pud_mkwrite(pud);
	return pud;
}

static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
		pud_t *pud, pfn_t pfn, pgprot_t prot, bool write)
{
	struct mm_struct *mm = vma->vm_mm;
	pud_t entry;
	spinlock_t *ptl;

	ptl = pud_lock(mm, pud);
	entry = pud_mkhuge(pfn_t_pud(pfn, prot));
	if (pfn_t_devmap(pfn))
		entry = pud_mkdevmap(entry);
	if (write) {
		entry = pud_mkyoung(pud_mkdirty(entry));
		entry = maybe_pud_mkwrite(entry, vma);
	}
	set_pud_at(mm, addr, pud, entry);
	update_mmu_cache_pud(vma, addr, pud);
	spin_unlock(ptl);
}

int vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
			pud_t *pud, pfn_t pfn, bool write)
{
	pgprot_t pgprot = vma->vm_page_prot;
	/*
	 * If we had pud_special, we could avoid all these restrictions,
	 * but we need to be consistent with PTEs and architectures that
	 * can't support a 'special' bit.
	 */
	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
						(VM_PFNMAP|VM_MIXEDMAP));
	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
	BUG_ON(!pfn_t_devmap(pfn));

	if (addr < vma->vm_start || addr >= vma->vm_end)
		return VM_FAULT_SIGBUS;

	track_pfn_insert(vma, &pgprot, pfn);

	insert_pfn_pud(vma, addr, pud, pfn, pgprot, write);
	return VM_FAULT_NOPAGE;
}
EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
		pmd_t *pmd)
{
	pmd_t _pmd;

	/*
	 * We should set the dirty bit only for FOLL_WRITE but for now
	 * the dirty bit in the pmd is meaningless.  And if the dirty
	 * bit will become meaningful and we'll only set it with
	 * FOLL_WRITE, an atomic set_bit will be required on the pmd to
	 * set the young bit, instead of the current set_pmd_at.
	 */
	_pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
	if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
				pmd, _pmd,  1))
		update_mmu_cache_pmd(vma, addr, pmd);
}

struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
		pmd_t *pmd, int flags)
{
	unsigned long pfn = pmd_pfn(*pmd);
	struct mm_struct *mm = vma->vm_mm;
	struct dev_pagemap *pgmap;
	struct page *page;

	assert_spin_locked(pmd_lockptr(mm, pmd));

872 873 874 875 876 877
	/*
	 * When we COW a devmap PMD entry, we split it into PTEs, so we should
	 * not be in this function with `flags & FOLL_COW` set.
	 */
	WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set");

878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
	if (flags & FOLL_WRITE && !pmd_write(*pmd))
		return NULL;

	if (pmd_present(*pmd) && pmd_devmap(*pmd))
		/* pass */;
	else
		return NULL;

	if (flags & FOLL_TOUCH)
		touch_pmd(vma, addr, pmd);

	/*
	 * device mapped pages can only be returned if the
	 * caller will manage the page reference count.
	 */
	if (!(flags & FOLL_GET))
		return ERR_PTR(-EEXIST);

	pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
	pgmap = get_dev_pagemap(pfn, NULL);
	if (!pgmap)
		return ERR_PTR(-EFAULT);
	page = pfn_to_page(pfn);
	get_page(page);
	put_dev_pagemap(pgmap);

	return page;
}

907 908 909 910
int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
		  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
		  struct vm_area_struct *vma)
{
911
	spinlock_t *dst_ptl, *src_ptl;
912 913
	struct page *src_page;
	pmd_t pmd;
914
	pgtable_t pgtable = NULL;
915
	int ret = -ENOMEM;
916

917 918 919 920 921 922 923
	/* Skip if can be re-fill on fault */
	if (!vma_is_anonymous(vma))
		return 0;

	pgtable = pte_alloc_one(dst_mm, addr);
	if (unlikely(!pgtable))
		goto out;
924

925 926 927
	dst_ptl = pmd_lock(dst_mm, dst_pmd);
	src_ptl = pmd_lockptr(src_mm, src_pmd);
	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
928 929 930

	ret = -EAGAIN;
	pmd = *src_pmd;
931 932 933 934 935 936 937 938 939

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
	if (unlikely(is_swap_pmd(pmd))) {
		swp_entry_t entry = pmd_to_swp_entry(pmd);

		VM_BUG_ON(!is_pmd_migration_entry(pmd));
		if (is_write_migration_entry(entry)) {
			make_migration_entry_read(&entry);
			pmd = swp_entry_to_pmd(entry);
940 941
			if (pmd_swp_soft_dirty(*src_pmd))
				pmd = pmd_swp_mksoft_dirty(pmd);
942 943 944 945 946 947 948 949
			set_pmd_at(src_mm, addr, src_pmd, pmd);
		}
		set_pmd_at(dst_mm, addr, dst_pmd, pmd);
		ret = 0;
		goto out_unlock;
	}
#endif

950
	if (unlikely(!pmd_trans_huge(pmd))) {
951 952 953
		pte_free(dst_mm, pgtable);
		goto out_unlock;
	}
954
	/*
955
	 * When page table lock is held, the huge zero pmd should not be
956 957 958 959
	 * under splitting since we don't split the page itself, only pmd to
	 * a page table.
	 */
	if (is_huge_zero_pmd(pmd)) {
960
		struct page *zero_page;
961 962 963 964 965
		/*
		 * get_huge_zero_page() will never allocate a new page here,
		 * since we already have a zero page to copy. It just takes a
		 * reference.
		 */
966
		zero_page = mm_get_huge_zero_page(dst_mm);
967
		set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
968
				zero_page);
969 970 971
		ret = 0;
		goto out_unlock;
	}
972

973 974 975 976 977 978 979
	src_page = pmd_page(pmd);
	VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
	get_page(src_page);
	page_dup_rmap(src_page, true);
	add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
	atomic_long_inc(&dst_mm->nr_ptes);
	pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
980 981 982 983 984 985 986

	pmdp_set_wrprotect(src_mm, addr, src_pmd);
	pmd = pmd_mkold(pmd_wrprotect(pmd));
	set_pmd_at(dst_mm, addr, dst_pmd, pmd);

	ret = 0;
out_unlock:
987 988
	spin_unlock(src_ptl);
	spin_unlock(dst_ptl);
989 990 991 992
out:
	return ret;
}

993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static void touch_pud(struct vm_area_struct *vma, unsigned long addr,
		pud_t *pud)
{
	pud_t _pud;

	/*
	 * We should set the dirty bit only for FOLL_WRITE but for now
	 * the dirty bit in the pud is meaningless.  And if the dirty
	 * bit will become meaningful and we'll only set it with
	 * FOLL_WRITE, an atomic set_bit will be required on the pud to
	 * set the young bit, instead of the current set_pud_at.
	 */
	_pud = pud_mkyoung(pud_mkdirty(*pud));
	if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
				pud, _pud,  1))
		update_mmu_cache_pud(vma, addr, pud);
}

struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
		pud_t *pud, int flags)
{
	unsigned long pfn = pud_pfn(*pud);
	struct mm_struct *mm = vma->vm_mm;
	struct dev_pagemap *pgmap;
	struct page *page;

	assert_spin_locked(pud_lockptr(mm, pud));

	if (flags & FOLL_WRITE && !pud_write(*pud))
		return NULL;

	if (pud_present(*pud) && pud_devmap(*pud))
		/* pass */;
	else
		return NULL;

	if (flags & FOLL_TOUCH)
		touch_pud(vma, addr, pud);

	/*
	 * device mapped pages can only be returned if the
	 * caller will manage the page reference count.
	 */
	if (!(flags & FOLL_GET))
		return ERR_PTR(-EEXIST);

	pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
	pgmap = get_dev_pagemap(pfn, NULL);
	if (!pgmap)
		return ERR_PTR(-EFAULT);
	page = pfn_to_page(pfn);
	get_page(page);
	put_dev_pagemap(pgmap);

	return page;
}

int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
		  pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
		  struct vm_area_struct *vma)
{
	spinlock_t *dst_ptl, *src_ptl;
	pud_t pud;
	int ret;

	dst_ptl = pud_lock(dst_mm, dst_pud);
	src_ptl = pud_lockptr(src_mm, src_pud);
	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);

	ret = -EAGAIN;
	pud = *src_pud;
	if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud)))
		goto out_unlock;

	/*
	 * When page table lock is held, the huge zero pud should not be
	 * under splitting since we don't split the page itself, only pud to
	 * a page table.
	 */
	if (is_huge_zero_pud(pud)) {
		/* No huge zero pud yet */
	}

	pudp_set_wrprotect(src_mm, addr, src_pud);
	pud = pud_mkold(pud_wrprotect(pud));
	set_pud_at(dst_mm, addr, dst_pud, pud);

	ret = 0;
out_unlock:
	spin_unlock(src_ptl);
	spin_unlock(dst_ptl);
	return ret;
}

void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
{
	pud_t entry;
	unsigned long haddr;
	bool write = vmf->flags & FAULT_FLAG_WRITE;

	vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);
	if (unlikely(!pud_same(*vmf->pud, orig_pud)))
		goto unlock;

	entry = pud_mkyoung(orig_pud);
	if (write)
		entry = pud_mkdirty(entry);
	haddr = vmf->address & HPAGE_PUD_MASK;
	if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write))
		update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud);

unlock:
	spin_unlock(vmf->ptl);
}
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

J
Jan Kara 已提交
1110
void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd)
1111 1112 1113
{
	pmd_t entry;
	unsigned long haddr;
1114
	bool write = vmf->flags & FAULT_FLAG_WRITE;
1115

J
Jan Kara 已提交
1116 1117
	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1118 1119 1120
		goto unlock;

	entry = pmd_mkyoung(orig_pmd);
1121 1122
	if (write)
		entry = pmd_mkdirty(entry);
J
Jan Kara 已提交
1123
	haddr = vmf->address & HPAGE_PMD_MASK;
1124
	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))
J
Jan Kara 已提交
1125
		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);
1126 1127

unlock:
J
Jan Kara 已提交
1128
	spin_unlock(vmf->ptl);
1129 1130
}

J
Jan Kara 已提交
1131
static int do_huge_pmd_wp_page_fallback(struct vm_fault *vmf, pmd_t orig_pmd,
K
Kirill A. Shutemov 已提交
1132
		struct page *page)
1133
{
J
Jan Kara 已提交
1134 1135
	struct vm_area_struct *vma = vmf->vma;
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1136
	struct mem_cgroup *memcg;
1137 1138 1139 1140
	pgtable_t pgtable;
	pmd_t _pmd;
	int ret = 0, i;
	struct page **pages;
1141 1142
	unsigned long mmun_start;	/* For mmu_notifiers */
	unsigned long mmun_end;		/* For mmu_notifiers */
1143 1144 1145 1146 1147 1148 1149 1150 1151

	pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
			GFP_KERNEL);
	if (unlikely(!pages)) {
		ret |= VM_FAULT_OOM;
		goto out;
	}

	for (i = 0; i < HPAGE_PMD_NR; i++) {
M
Michal Hocko 已提交
1152
		pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE, vma,
J
Jan Kara 已提交
1153
					       vmf->address, page_to_nid(page));
A
Andrea Arcangeli 已提交
1154
		if (unlikely(!pages[i] ||
K
Kirill A. Shutemov 已提交
1155 1156
			     mem_cgroup_try_charge(pages[i], vma->vm_mm,
				     GFP_KERNEL, &memcg, false))) {
A
Andrea Arcangeli 已提交
1157
			if (pages[i])
1158
				put_page(pages[i]);
A
Andrea Arcangeli 已提交
1159
			while (--i >= 0) {
1160 1161
				memcg = (void *)page_private(pages[i]);
				set_page_private(pages[i], 0);
1162 1163
				mem_cgroup_cancel_charge(pages[i], memcg,
						false);
A
Andrea Arcangeli 已提交
1164 1165
				put_page(pages[i]);
			}
1166 1167 1168 1169
			kfree(pages);
			ret |= VM_FAULT_OOM;
			goto out;
		}
1170
		set_page_private(pages[i], (unsigned long)memcg);
1171 1172 1173 1174
	}

	for (i = 0; i < HPAGE_PMD_NR; i++) {
		copy_user_highpage(pages[i], page + i,
1175
				   haddr + PAGE_SIZE * i, vma);
1176 1177 1178 1179
		__SetPageUptodate(pages[i]);
		cond_resched();
	}

1180 1181
	mmun_start = haddr;
	mmun_end   = haddr + HPAGE_PMD_SIZE;
K
Kirill A. Shutemov 已提交
1182
	mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
1183

J
Jan Kara 已提交
1184 1185
	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1186
		goto out_free_pages;
1187
	VM_BUG_ON_PAGE(!PageHead(page), page);
1188

J
Jan Kara 已提交
1189
	pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
1190 1191
	/* leave pmd empty until pte is filled */

J
Jan Kara 已提交
1192
	pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);
K
Kirill A. Shutemov 已提交
1193
	pmd_populate(vma->vm_mm, &_pmd, pgtable);
1194 1195

	for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
K
Kirill A. Shutemov 已提交
1196
		pte_t entry;
1197 1198
		entry = mk_pte(pages[i], vma->vm_page_prot);
		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1199 1200
		memcg = (void *)page_private(pages[i]);
		set_page_private(pages[i], 0);
J
Jan Kara 已提交
1201
		page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false);
1202
		mem_cgroup_commit_charge(pages[i], memcg, false, false);
1203
		lru_cache_add_active_or_unevictable(pages[i], vma);
J
Jan Kara 已提交
1204 1205 1206 1207
		vmf->pte = pte_offset_map(&_pmd, haddr);
		VM_BUG_ON(!pte_none(*vmf->pte));
		set_pte_at(vma->vm_mm, haddr, vmf->pte, entry);
		pte_unmap(vmf->pte);
1208 1209 1210 1211
	}
	kfree(pages);

	smp_wmb(); /* make pte visible before pmd */
J
Jan Kara 已提交
1212
	pmd_populate(vma->vm_mm, vmf->pmd, pgtable);
1213
	page_remove_rmap(page, true);
J
Jan Kara 已提交
1214
	spin_unlock(vmf->ptl);
1215

K
Kirill A. Shutemov 已提交
1216
	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
1217

1218 1219 1220 1221 1222 1223 1224
	ret |= VM_FAULT_WRITE;
	put_page(page);

out:
	return ret;

out_free_pages:
J
Jan Kara 已提交
1225
	spin_unlock(vmf->ptl);
K
Kirill A. Shutemov 已提交
1226
	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
A
Andrea Arcangeli 已提交
1227
	for (i = 0; i < HPAGE_PMD_NR; i++) {
1228 1229
		memcg = (void *)page_private(pages[i]);
		set_page_private(pages[i], 0);
1230
		mem_cgroup_cancel_charge(pages[i], memcg, false);
1231
		put_page(pages[i]);
A
Andrea Arcangeli 已提交
1232
	}
1233 1234 1235 1236
	kfree(pages);
	goto out;
}

J
Jan Kara 已提交
1237
int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
1238
{
J
Jan Kara 已提交
1239
	struct vm_area_struct *vma = vmf->vma;
1240
	struct page *page = NULL, *new_page;
1241
	struct mem_cgroup *memcg;
J
Jan Kara 已提交
1242
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1243 1244
	unsigned long mmun_start;	/* For mmu_notifiers */
	unsigned long mmun_end;		/* For mmu_notifiers */
1245
	gfp_t huge_gfp;			/* for allocation and charge */
K
Kirill A. Shutemov 已提交
1246
	int ret = 0;
1247

J
Jan Kara 已提交
1248
	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
1249
	VM_BUG_ON_VMA(!vma->anon_vma, vma);
1250 1251
	if (is_huge_zero_pmd(orig_pmd))
		goto alloc;
J
Jan Kara 已提交
1252 1253
	spin_lock(vmf->ptl);
	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1254 1255 1256
		goto out_unlock;

	page = pmd_page(orig_pmd);
1257
	VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
1258 1259
	/*
	 * We can only reuse the page if nobody else maps the huge page or it's
1260
	 * part.
1261
	 */
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
	if (!trylock_page(page)) {
		get_page(page);
		spin_unlock(vmf->ptl);
		lock_page(page);
		spin_lock(vmf->ptl);
		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
			unlock_page(page);
			put_page(page);
			goto out_unlock;
		}
		put_page(page);
	}
	if (reuse_swap_page(page, NULL)) {
1275 1276 1277
		pmd_t entry;
		entry = pmd_mkyoung(orig_pmd);
		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
J
Jan Kara 已提交
1278 1279
		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))
			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1280
		ret |= VM_FAULT_WRITE;
1281
		unlock_page(page);
1282 1283
		goto out_unlock;
	}
1284
	unlock_page(page);
1285
	get_page(page);
J
Jan Kara 已提交
1286
	spin_unlock(vmf->ptl);
1287
alloc:
1288
	if (transparent_hugepage_enabled(vma) &&
1289
	    !transparent_hugepage_debug_cow()) {
1290
		huge_gfp = alloc_hugepage_direct_gfpmask(vma);
1291
		new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1292
	} else
1293 1294
		new_page = NULL;

1295 1296 1297
	if (likely(new_page)) {
		prep_transhuge_page(new_page);
	} else {
1298
		if (!page) {
J
Jan Kara 已提交
1299
			split_huge_pmd(vma, vmf->pmd, vmf->address);
1300
			ret |= VM_FAULT_FALLBACK;
1301
		} else {
J
Jan Kara 已提交
1302
			ret = do_huge_pmd_wp_page_fallback(vmf, orig_pmd, page);
1303
			if (ret & VM_FAULT_OOM) {
J
Jan Kara 已提交
1304
				split_huge_pmd(vma, vmf->pmd, vmf->address);
1305 1306
				ret |= VM_FAULT_FALLBACK;
			}
1307
			put_page(page);
1308
		}
1309
		count_vm_event(THP_FAULT_FALLBACK);
1310 1311 1312
		goto out;
	}

K
Kirill A. Shutemov 已提交
1313 1314
	if (unlikely(mem_cgroup_try_charge(new_page, vma->vm_mm,
					huge_gfp, &memcg, true))) {
A
Andrea Arcangeli 已提交
1315
		put_page(new_page);
J
Jan Kara 已提交
1316
		split_huge_pmd(vma, vmf->pmd, vmf->address);
K
Kirill A. Shutemov 已提交
1317
		if (page)
1318
			put_page(page);
1319
		ret |= VM_FAULT_FALLBACK;
1320
		count_vm_event(THP_FAULT_FALLBACK);
A
Andrea Arcangeli 已提交
1321 1322 1323
		goto out;
	}

1324 1325
	count_vm_event(THP_FAULT_ALLOC);

1326
	if (!page)
1327
		clear_huge_page(new_page, vmf->address, HPAGE_PMD_NR);
1328 1329
	else
		copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
1330 1331
	__SetPageUptodate(new_page);

1332 1333
	mmun_start = haddr;
	mmun_end   = haddr + HPAGE_PMD_SIZE;
K
Kirill A. Shutemov 已提交
1334
	mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
1335

J
Jan Kara 已提交
1336
	spin_lock(vmf->ptl);
1337
	if (page)
1338
		put_page(page);
J
Jan Kara 已提交
1339 1340
	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
		spin_unlock(vmf->ptl);
1341
		mem_cgroup_cancel_charge(new_page, memcg, true);
1342
		put_page(new_page);
1343
		goto out_mn;
A
Andrea Arcangeli 已提交
1344
	} else {
1345
		pmd_t entry;
1346 1347
		entry = mk_huge_pmd(new_page, vma->vm_page_prot);
		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
J
Jan Kara 已提交
1348
		pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
1349
		page_add_new_anon_rmap(new_page, vma, haddr, true);
1350
		mem_cgroup_commit_charge(new_page, memcg, false, true);
1351
		lru_cache_add_active_or_unevictable(new_page, vma);
J
Jan Kara 已提交
1352 1353
		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1354
		if (!page) {
K
Kirill A. Shutemov 已提交
1355
			add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1356
		} else {
1357
			VM_BUG_ON_PAGE(!PageHead(page), page);
1358
			page_remove_rmap(page, true);
1359 1360
			put_page(page);
		}
1361 1362
		ret |= VM_FAULT_WRITE;
	}
J
Jan Kara 已提交
1363
	spin_unlock(vmf->ptl);
1364
out_mn:
K
Kirill A. Shutemov 已提交
1365
	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
1366 1367
out:
	return ret;
1368
out_unlock:
J
Jan Kara 已提交
1369
	spin_unlock(vmf->ptl);
1370
	return ret;
1371 1372
}

1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
/*
 * FOLL_FORCE can write to even unwritable pmd's, but only
 * after we've gone through a COW cycle and they are dirty.
 */
static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)
{
	return pmd_write(pmd) ||
	       ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));
}

1383
struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1384 1385 1386 1387
				   unsigned long addr,
				   pmd_t *pmd,
				   unsigned int flags)
{
1388
	struct mm_struct *mm = vma->vm_mm;
1389 1390
	struct page *page = NULL;

1391
	assert_spin_locked(pmd_lockptr(mm, pmd));
1392

1393
	if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))
1394 1395
		goto out;

1396 1397 1398 1399
	/* Avoid dumping huge zero page */
	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
		return ERR_PTR(-EFAULT);

1400
	/* Full NUMA hinting faults to serialise migration in fault paths */
1401
	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
1402 1403
		goto out;

1404
	page = pmd_page(*pmd);
1405
	VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
1406 1407
	if (flags & FOLL_TOUCH)
		touch_pmd(vma, addr, pmd);
E
Eric B Munson 已提交
1408
	if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
1409 1410 1411 1412
		/*
		 * We don't mlock() pte-mapped THPs. This way we can avoid
		 * leaking mlocked pages into non-VM_LOCKED VMAs.
		 *
1413 1414
		 * For anon THP:
		 *
1415 1416 1417 1418 1419 1420 1421
		 * In most cases the pmd is the only mapping of the page as we
		 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
		 * writable private mappings in populate_vma_page_range().
		 *
		 * The only scenario when we have the page shared here is if we
		 * mlocking read-only mapping shared over fork(). We skip
		 * mlocking such pages.
1422 1423 1424 1425 1426 1427
		 *
		 * For file THP:
		 *
		 * We can expect PageDoubleMap() to be stable under page lock:
		 * for file pages we set it in page_add_file_rmap(), which
		 * requires page to be locked.
1428
		 */
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439

		if (PageAnon(page) && compound_mapcount(page) != 1)
			goto skip_mlock;
		if (PageDoubleMap(page) || !page->mapping)
			goto skip_mlock;
		if (!trylock_page(page))
			goto skip_mlock;
		lru_add_drain();
		if (page->mapping && !PageDoubleMap(page))
			mlock_vma_page(page);
		unlock_page(page);
1440
	}
1441
skip_mlock:
1442
	page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1443
	VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
1444
	if (flags & FOLL_GET)
1445
		get_page(page);
1446 1447 1448 1449 1450

out:
	return page;
}

1451
/* NUMA hinting page fault entry point for trans huge pmds */
J
Jan Kara 已提交
1452
int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
1453
{
J
Jan Kara 已提交
1454
	struct vm_area_struct *vma = vmf->vma;
1455
	struct anon_vma *anon_vma = NULL;
1456
	struct page *page;
J
Jan Kara 已提交
1457
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1458
	int page_nid = -1, this_nid = numa_node_id();
1459
	int target_nid, last_cpupid = -1;
1460 1461
	bool page_locked;
	bool migrated = false;
1462
	bool was_writable;
1463
	int flags = 0;
1464

J
Jan Kara 已提交
1465 1466
	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_same(pmd, *vmf->pmd)))
1467 1468
		goto out_unlock;

1469 1470 1471 1472 1473
	/*
	 * If there are potential migrations, wait for completion and retry
	 * without disrupting NUMA hinting information. Do not relock and
	 * check_same as the page may no longer be mapped.
	 */
J
Jan Kara 已提交
1474 1475
	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {
		page = pmd_page(*vmf->pmd);
1476 1477
		if (!get_page_unless_zero(page))
			goto out_unlock;
J
Jan Kara 已提交
1478
		spin_unlock(vmf->ptl);
1479
		wait_on_page_locked(page);
1480
		put_page(page);
1481 1482 1483
		goto out;
	}

1484
	page = pmd_page(pmd);
1485
	BUG_ON(is_huge_zero_page(page));
1486
	page_nid = page_to_nid(page);
1487
	last_cpupid = page_cpupid_last(page);
1488
	count_vm_numa_event(NUMA_HINT_FAULTS);
1489
	if (page_nid == this_nid) {
1490
		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
1491 1492
		flags |= TNF_FAULT_LOCAL;
	}
1493

1494
	/* See similar comment in do_numa_page for explanation */
1495
	if (!pmd_savedwrite(pmd))
1496 1497
		flags |= TNF_NO_GROUP;

1498 1499 1500 1501
	/*
	 * Acquire the page lock to serialise THP migrations but avoid dropping
	 * page_table_lock if at all possible
	 */
1502 1503 1504 1505
	page_locked = trylock_page(page);
	target_nid = mpol_misplaced(page, vma, haddr);
	if (target_nid == -1) {
		/* If the page was locked, there are no parallel migrations */
1506
		if (page_locked)
1507
			goto clear_pmdnuma;
1508
	}
1509

1510
	/* Migration could have started since the pmd_trans_migrating check */
1511
	if (!page_locked) {
1512 1513 1514
		page_nid = -1;
		if (!get_page_unless_zero(page))
			goto out_unlock;
J
Jan Kara 已提交
1515
		spin_unlock(vmf->ptl);
1516
		wait_on_page_locked(page);
1517
		put_page(page);
1518 1519 1520
		goto out;
	}

1521 1522 1523 1524
	/*
	 * Page is misplaced. Page lock serialises migrations. Acquire anon_vma
	 * to serialises splits
	 */
1525
	get_page(page);
J
Jan Kara 已提交
1526
	spin_unlock(vmf->ptl);
1527
	anon_vma = page_lock_anon_vma_read(page);
1528

P
Peter Zijlstra 已提交
1529
	/* Confirm the PMD did not change while page_table_lock was released */
J
Jan Kara 已提交
1530 1531
	spin_lock(vmf->ptl);
	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {
1532 1533
		unlock_page(page);
		put_page(page);
1534
		page_nid = -1;
1535
		goto out_unlock;
1536
	}
1537

1538 1539 1540 1541 1542 1543 1544
	/* Bail if we fail to protect against THP splits for any reason */
	if (unlikely(!anon_vma)) {
		put_page(page);
		page_nid = -1;
		goto clear_pmdnuma;
	}

1545 1546 1547 1548 1549 1550
	/*
	 * Since we took the NUMA fault, we must have observed the !accessible
	 * bit. Make sure all other CPUs agree with that, to avoid them
	 * modifying the page we're about to migrate.
	 *
	 * Must be done under PTL such that we'll observe the relevant
1551 1552 1553 1554
	 * inc_tlb_flush_pending().
	 *
	 * We are not sure a pending tlb flush here is for a huge page
	 * mapping or not. Hence use the tlb range variant
1555 1556
	 */
	if (mm_tlb_flush_pending(vma->vm_mm))
1557
		flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
1558

1559 1560
	/*
	 * Migrate the THP to the requested node, returns with page unlocked
1561
	 * and access rights restored.
1562
	 */
J
Jan Kara 已提交
1563
	spin_unlock(vmf->ptl);
1564

K
Kirill A. Shutemov 已提交
1565
	migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma,
J
Jan Kara 已提交
1566
				vmf->pmd, pmd, vmf->address, page, target_nid);
1567 1568
	if (migrated) {
		flags |= TNF_MIGRATED;
1569
		page_nid = target_nid;
1570 1571
	} else
		flags |= TNF_MIGRATE_FAIL;
1572

1573
	goto out;
1574
clear_pmdnuma:
1575
	BUG_ON(!PageLocked(page));
1576
	was_writable = pmd_savedwrite(pmd);
1577
	pmd = pmd_modify(pmd, vma->vm_page_prot);
1578
	pmd = pmd_mkyoung(pmd);
1579 1580
	if (was_writable)
		pmd = pmd_mkwrite(pmd);
J
Jan Kara 已提交
1581 1582
	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1583
	unlock_page(page);
1584
out_unlock:
J
Jan Kara 已提交
1585
	spin_unlock(vmf->ptl);
1586 1587 1588 1589 1590

out:
	if (anon_vma)
		page_unlock_anon_vma_read(anon_vma);

1591
	if (page_nid != -1)
J
Jan Kara 已提交
1592
		task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR,
1593
				flags);
1594

1595 1596 1597
	return 0;
}

1598 1599 1600 1601 1602
/*
 * Return true if we do MADV_FREE successfully on entire pmd page.
 * Otherwise, return false.
 */
bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1603 1604 1605 1606 1607 1608
		pmd_t *pmd, unsigned long addr, unsigned long next)
{
	spinlock_t *ptl;
	pmd_t orig_pmd;
	struct page *page;
	struct mm_struct *mm = tlb->mm;
1609
	bool ret = false;
1610

1611 1612
	tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);

1613 1614
	ptl = pmd_trans_huge_lock(pmd, vma);
	if (!ptl)
1615
		goto out_unlocked;
1616 1617

	orig_pmd = *pmd;
1618
	if (is_huge_zero_pmd(orig_pmd))
1619 1620
		goto out;

1621 1622 1623 1624 1625 1626
	if (unlikely(!pmd_present(orig_pmd))) {
		VM_BUG_ON(thp_migration_supported() &&
				  !is_pmd_migration_entry(orig_pmd));
		goto out;
	}

1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644
	page = pmd_page(orig_pmd);
	/*
	 * If other processes are mapping this page, we couldn't discard
	 * the page unless they all do MADV_FREE so let's skip the page.
	 */
	if (page_mapcount(page) != 1)
		goto out;

	if (!trylock_page(page))
		goto out;

	/*
	 * If user want to discard part-pages of THP, split it so MADV_FREE
	 * will deactivate only them.
	 */
	if (next - addr != HPAGE_PMD_SIZE) {
		get_page(page);
		spin_unlock(ptl);
1645
		split_huge_page(page);
1646
		unlock_page(page);
1647
		put_page(page);
1648 1649 1650 1651 1652 1653 1654 1655
		goto out_unlocked;
	}

	if (PageDirty(page))
		ClearPageDirty(page);
	unlock_page(page);

	if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1656
		pmdp_invalidate(vma, addr, pmd);
1657 1658 1659 1660 1661 1662
		orig_pmd = pmd_mkold(orig_pmd);
		orig_pmd = pmd_mkclean(orig_pmd);

		set_pmd_at(mm, addr, pmd, orig_pmd);
		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
	}
S
Shaohua Li 已提交
1663 1664

	mark_page_lazyfree(page);
1665
	ret = true;
1666 1667 1668 1669 1670 1671
out:
	spin_unlock(ptl);
out_unlocked:
	return ret;
}

1672 1673 1674 1675 1676 1677 1678 1679 1680
static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
{
	pgtable_t pgtable;

	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
	pte_free(mm, pgtable);
	atomic_long_dec(&mm->nr_ptes);
}

1681
int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
S
Shaohua Li 已提交
1682
		 pmd_t *pmd, unsigned long addr)
1683
{
1684
	pmd_t orig_pmd;
1685
	spinlock_t *ptl;
1686

1687 1688
	tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);

1689 1690
	ptl = __pmd_trans_huge_lock(pmd, vma);
	if (!ptl)
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
		return 0;
	/*
	 * For architectures like ppc64 we look at deposited pgtable
	 * when calling pmdp_huge_get_and_clear. So do the
	 * pgtable_trans_huge_withdraw after finishing pmdp related
	 * operations.
	 */
	orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
			tlb->fullmm);
	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
	if (vma_is_dax(vma)) {
1702 1703
		if (arch_needs_pgtable_deposit())
			zap_deposited_table(tlb->mm, pmd);
1704 1705
		spin_unlock(ptl);
		if (is_huge_zero_pmd(orig_pmd))
1706
			tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE);
1707
	} else if (is_huge_zero_pmd(orig_pmd)) {
1708
		zap_deposited_table(tlb->mm, pmd);
1709
		spin_unlock(ptl);
1710
		tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE);
1711
	} else {
1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729
		struct page *page = NULL;
		int flush_needed = 1;

		if (pmd_present(orig_pmd)) {
			page = pmd_page(orig_pmd);
			page_remove_rmap(page, true);
			VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
			VM_BUG_ON_PAGE(!PageHead(page), page);
		} else if (thp_migration_supported()) {
			swp_entry_t entry;

			VM_BUG_ON(!is_pmd_migration_entry(orig_pmd));
			entry = pmd_to_swp_entry(orig_pmd);
			page = pfn_to_page(swp_offset(entry));
			flush_needed = 0;
		} else
			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");

1730
		if (PageAnon(page)) {
1731
			zap_deposited_table(tlb->mm, pmd);
1732 1733
			add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
		} else {
1734 1735
			if (arch_needs_pgtable_deposit())
				zap_deposited_table(tlb->mm, pmd);
1736 1737
			add_mm_counter(tlb->mm, MM_FILEPAGES, -HPAGE_PMD_NR);
		}
1738

1739
		spin_unlock(ptl);
1740 1741
		if (flush_needed)
			tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
1742
	}
1743
	return 1;
1744 1745
}

1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
#ifndef pmd_move_must_withdraw
static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
					 spinlock_t *old_pmd_ptl,
					 struct vm_area_struct *vma)
{
	/*
	 * With split pmd lock we also need to move preallocated
	 * PTE page table if new_pmd is on different PMD page table.
	 *
	 * We also don't deposit and withdraw tables for file pages.
	 */
	return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);
}
#endif

1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771
static pmd_t move_soft_dirty_pmd(pmd_t pmd)
{
#ifdef CONFIG_MEM_SOFT_DIRTY
	if (unlikely(is_pmd_migration_entry(pmd)))
		pmd = pmd_swp_mksoft_dirty(pmd);
	else if (pmd_present(pmd))
		pmd = pmd_mksoft_dirty(pmd);
#endif
	return pmd;
}

1772
bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
1773
		  unsigned long new_addr, unsigned long old_end,
1774
		  pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush)
1775
{
1776
	spinlock_t *old_ptl, *new_ptl;
1777 1778
	pmd_t pmd;
	struct mm_struct *mm = vma->vm_mm;
1779
	bool force_flush = false;
1780 1781 1782

	if ((old_addr & ~HPAGE_PMD_MASK) ||
	    (new_addr & ~HPAGE_PMD_MASK) ||
1783
	    old_end - old_addr < HPAGE_PMD_SIZE)
1784
		return false;
1785 1786 1787 1788 1789 1790 1791

	/*
	 * The destination pmd shouldn't be established, free_pgtables()
	 * should have release it.
	 */
	if (WARN_ON(!pmd_none(*new_pmd))) {
		VM_BUG_ON(pmd_trans_huge(*new_pmd));
1792
		return false;
1793 1794
	}

1795 1796 1797 1798
	/*
	 * We don't have to worry about the ordering of src and dst
	 * ptlocks because exclusive mmap_sem prevents deadlock.
	 */
1799 1800
	old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
	if (old_ptl) {
1801 1802 1803
		new_ptl = pmd_lockptr(mm, new_pmd);
		if (new_ptl != old_ptl)
			spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
1804
		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
1805 1806
		if (pmd_present(pmd) && pmd_dirty(pmd))
			force_flush = true;
1807
		VM_BUG_ON(!pmd_none(*new_pmd));
1808

1809
		if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
1810
			pgtable_t pgtable;
1811 1812 1813
			pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
			pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
		}
1814 1815
		pmd = move_soft_dirty_pmd(pmd);
		set_pmd_at(mm, new_addr, new_pmd, pmd);
1816 1817
		if (new_ptl != old_ptl)
			spin_unlock(new_ptl);
1818 1819 1820 1821
		if (force_flush)
			flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
		else
			*need_flush = true;
1822
		spin_unlock(old_ptl);
1823
		return true;
1824
	}
1825
	return false;
1826 1827
}

1828 1829 1830 1831 1832 1833
/*
 * Returns
 *  - 0 if PMD could not be locked
 *  - 1 if PMD was locked but protections unchange and TLB flush unnecessary
 *  - HPAGE_PMD_NR is protections changed and TLB flush necessary
 */
1834
int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
1835
		unsigned long addr, pgprot_t newprot, int prot_numa)
1836 1837
{
	struct mm_struct *mm = vma->vm_mm;
1838
	spinlock_t *ptl;
1839 1840 1841
	pmd_t entry;
	bool preserve_write;
	int ret;
1842

1843
	ptl = __pmd_trans_huge_lock(pmd, vma);
1844 1845
	if (!ptl)
		return 0;
1846

1847 1848
	preserve_write = prot_numa && pmd_write(*pmd);
	ret = 1;
1849

1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
	if (is_swap_pmd(*pmd)) {
		swp_entry_t entry = pmd_to_swp_entry(*pmd);

		VM_BUG_ON(!is_pmd_migration_entry(*pmd));
		if (is_write_migration_entry(entry)) {
			pmd_t newpmd;
			/*
			 * A protection check is difficult so
			 * just be safe and disable write
			 */
			make_migration_entry_read(&entry);
			newpmd = swp_entry_to_pmd(entry);
1863 1864
			if (pmd_swp_soft_dirty(*pmd))
				newpmd = pmd_swp_mksoft_dirty(newpmd);
1865 1866 1867 1868 1869 1870
			set_pmd_at(mm, addr, pmd, newpmd);
		}
		goto unlock;
	}
#endif

1871 1872 1873 1874 1875 1876 1877
	/*
	 * Avoid trapping faults against the zero page. The read-only
	 * data is likely to be read-cached on the local CPU and
	 * local/remote hits to the zero page are not interesting.
	 */
	if (prot_numa && is_huge_zero_pmd(*pmd))
		goto unlock;
1878

1879 1880 1881
	if (prot_numa && pmd_protnone(*pmd))
		goto unlock;

1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
	/*
	 * In case prot_numa, we are under down_read(mmap_sem). It's critical
	 * to not clear pmd intermittently to avoid race with MADV_DONTNEED
	 * which is also under down_read(mmap_sem):
	 *
	 *	CPU0:				CPU1:
	 *				change_huge_pmd(prot_numa=1)
	 *				 pmdp_huge_get_and_clear_notify()
	 * madvise_dontneed()
	 *  zap_pmd_range()
	 *   pmd_trans_huge(*pmd) == 0 (without ptl)
	 *   // skip the pmd
	 *				 set_pmd_at();
	 *				 // pmd is re-established
	 *
	 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it
	 * which may break userspace.
	 *
	 * pmdp_invalidate() is required to make sure we don't miss
	 * dirty/young flags set by hardware.
	 */
	entry = *pmd;
	pmdp_invalidate(vma, addr, pmd);

	/*
	 * Recover dirty/young flags.  It relies on pmdp_invalidate to not
	 * corrupt them.
	 */
	if (pmd_dirty(*pmd))
		entry = pmd_mkdirty(entry);
	if (pmd_young(*pmd))
		entry = pmd_mkyoung(entry);

1915 1916 1917 1918 1919 1920 1921 1922
	entry = pmd_modify(entry, newprot);
	if (preserve_write)
		entry = pmd_mk_savedwrite(entry);
	ret = HPAGE_PMD_NR;
	set_pmd_at(mm, addr, pmd, entry);
	BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry));
unlock:
	spin_unlock(ptl);
1923 1924 1925 1926
	return ret;
}

/*
1927
 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
1928
 *
1929 1930
 * Note that if it returns page table lock pointer, this routine returns without
 * unlocking page table lock. So callers must unlock it.
1931
 */
1932
spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
1933
{
1934 1935
	spinlock_t *ptl;
	ptl = pmd_lock(vma->vm_mm, pmd);
1936 1937
	if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||
			pmd_devmap(*pmd)))
1938 1939 1940
		return ptl;
	spin_unlock(ptl);
	return NULL;
1941 1942
}

1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996
/*
 * Returns true if a given pud maps a thp, false otherwise.
 *
 * Note that if it returns true, this routine returns without unlocking page
 * table lock. So callers must unlock it.
 */
spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
{
	spinlock_t *ptl;

	ptl = pud_lock(vma->vm_mm, pud);
	if (likely(pud_trans_huge(*pud) || pud_devmap(*pud)))
		return ptl;
	spin_unlock(ptl);
	return NULL;
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
		 pud_t *pud, unsigned long addr)
{
	pud_t orig_pud;
	spinlock_t *ptl;

	ptl = __pud_trans_huge_lock(pud, vma);
	if (!ptl)
		return 0;
	/*
	 * For architectures like ppc64 we look at deposited pgtable
	 * when calling pudp_huge_get_and_clear. So do the
	 * pgtable_trans_huge_withdraw after finishing pudp related
	 * operations.
	 */
	orig_pud = pudp_huge_get_and_clear_full(tlb->mm, addr, pud,
			tlb->fullmm);
	tlb_remove_pud_tlb_entry(tlb, pud, addr);
	if (vma_is_dax(vma)) {
		spin_unlock(ptl);
		/* No zero page support yet */
	} else {
		/* No support for anonymous PUD pages yet */
		BUG();
	}
	return 1;
}

static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
		unsigned long haddr)
{
	VM_BUG_ON(haddr & ~HPAGE_PUD_MASK);
	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
	VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud));

1997
	count_vm_event(THP_SPLIT_PUD);
1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020

	pudp_huge_clear_flush_notify(vma, haddr, pud);
}

void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
		unsigned long address)
{
	spinlock_t *ptl;
	struct mm_struct *mm = vma->vm_mm;
	unsigned long haddr = address & HPAGE_PUD_MASK;

	mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PUD_SIZE);
	ptl = pud_lock(mm, pud);
	if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud)))
		goto out;
	__split_huge_pud_locked(vma, pud, haddr);

out:
	spin_unlock(ptl);
	mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PUD_SIZE);
}
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
		unsigned long haddr, pmd_t *pmd)
{
	struct mm_struct *mm = vma->vm_mm;
	pgtable_t pgtable;
	pmd_t _pmd;
	int i;

	/* leave pmd empty until pte is filled */
	pmdp_huge_clear_flush_notify(vma, haddr, pmd);

	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
	pmd_populate(mm, &_pmd, pgtable);

	for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
		pte_t *pte, entry;
		entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
		entry = pte_mkspecial(entry);
		pte = pte_offset_map(&_pmd, haddr);
		VM_BUG_ON(!pte_none(*pte));
		set_pte_at(mm, haddr, pte, entry);
		pte_unmap(pte);
	}
	smp_wmb(); /* make pte visible before pmd */
	pmd_populate(mm, pmd, pgtable);
}

static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
2049
		unsigned long haddr, bool freeze)
2050 2051 2052 2053 2054
{
	struct mm_struct *mm = vma->vm_mm;
	struct page *page;
	pgtable_t pgtable;
	pmd_t _pmd;
2055
	bool young, write, dirty, soft_dirty, pmd_migration = false;
2056
	unsigned long addr;
2057 2058 2059 2060 2061
	int i;

	VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
2062 2063
	VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)
				&& !pmd_devmap(*pmd));
2064 2065 2066

	count_vm_event(THP_SPLIT_PMD);

2067 2068
	if (!vma_is_anonymous(vma)) {
		_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2069 2070 2071 2072 2073 2074
		/*
		 * We are going to unmap this huge page. So
		 * just go ahead and zap it
		 */
		if (arch_needs_pgtable_deposit())
			zap_deposited_table(mm, pmd);
2075 2076 2077 2078 2079 2080 2081 2082
		if (vma_is_dax(vma))
			return;
		page = pmd_page(_pmd);
		if (!PageReferenced(page) && pmd_young(_pmd))
			SetPageReferenced(page);
		page_remove_rmap(page, true);
		put_page(page);
		add_mm_counter(mm, MM_FILEPAGES, -HPAGE_PMD_NR);
2083 2084 2085 2086 2087
		return;
	} else if (is_huge_zero_pmd(*pmd)) {
		return __split_huge_zero_page_pmd(vma, haddr, pmd);
	}

2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
	pmd_migration = is_pmd_migration_entry(*pmd);
	if (pmd_migration) {
		swp_entry_t entry;

		entry = pmd_to_swp_entry(*pmd);
		page = pfn_to_page(swp_offset(entry));
	} else
#endif
		page = pmd_page(*pmd);
2098
	VM_BUG_ON_PAGE(!page_count(page), page);
2099
	page_ref_add(page, HPAGE_PMD_NR - 1);
2100 2101
	write = pmd_write(*pmd);
	young = pmd_young(*pmd);
2102
	dirty = pmd_dirty(*pmd);
2103
	soft_dirty = pmd_soft_dirty(*pmd);
2104

2105
	pmdp_huge_split_prepare(vma, haddr, pmd);
2106 2107 2108
	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
	pmd_populate(mm, &_pmd, pgtable);

2109
	for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
2110 2111 2112 2113 2114 2115
		pte_t entry, *pte;
		/*
		 * Note that NUMA hinting access restrictions are not
		 * transferred to avoid any possibility of altering
		 * permissions across VMAs.
		 */
2116
		if (freeze || pmd_migration) {
2117 2118 2119
			swp_entry_t swp_entry;
			swp_entry = make_migration_entry(page + i, write);
			entry = swp_entry_to_pte(swp_entry);
2120 2121
			if (soft_dirty)
				entry = pte_swp_mksoft_dirty(entry);
2122
		} else {
2123
			entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
2124
			entry = maybe_mkwrite(entry, vma);
2125 2126 2127 2128
			if (!write)
				entry = pte_wrprotect(entry);
			if (!young)
				entry = pte_mkold(entry);
2129 2130
			if (soft_dirty)
				entry = pte_mksoft_dirty(entry);
2131
		}
2132 2133
		if (dirty)
			SetPageDirty(page + i);
2134
		pte = pte_offset_map(&_pmd, addr);
2135
		BUG_ON(!pte_none(*pte));
2136
		set_pte_at(mm, addr, pte, entry);
2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
		atomic_inc(&page[i]._mapcount);
		pte_unmap(pte);
	}

	/*
	 * Set PG_double_map before dropping compound_mapcount to avoid
	 * false-negative page_mapped().
	 */
	if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) {
		for (i = 0; i < HPAGE_PMD_NR; i++)
			atomic_inc(&page[i]._mapcount);
	}

	if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
		/* Last compound_mapcount is gone. */
2152
		__dec_node_page_state(page, NR_ANON_THPS);
2153 2154 2155 2156 2157 2158 2159 2160
		if (TestClearPageDoubleMap(page)) {
			/* No need in mapcount reference anymore */
			for (i = 0; i < HPAGE_PMD_NR; i++)
				atomic_dec(&page[i]._mapcount);
		}
	}

	smp_wmb(); /* make pte visible before pmd */
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182
	/*
	 * Up to this point the pmd is present and huge and userland has the
	 * whole access to the hugepage during the split (which happens in
	 * place). If we overwrite the pmd with the not-huge version pointing
	 * to the pte here (which of course we could if all CPUs were bug
	 * free), userland could trigger a small page size TLB miss on the
	 * small sized TLB while the hugepage TLB entry is still established in
	 * the huge TLB. Some CPU doesn't like that.
	 * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum
	 * 383 on page 93. Intel should be safe but is also warns that it's
	 * only safe if the permission and cache attributes of the two entries
	 * loaded in the two TLB is identical (which should be the case here).
	 * But it is generally safer to never allow small and huge TLB entries
	 * for the same virtual address to be loaded simultaneously. So instead
	 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
	 * current pmd notpresent (atomically because here the pmd_trans_huge
	 * and pmd_trans_splitting must remain set at all times on the pmd
	 * until the split is complete for this pmd), then we flush the SMP TLB
	 * and finally we write the non-huge version of the pmd entry with
	 * pmd_populate.
	 */
	pmdp_invalidate(vma, haddr, pmd);
2183
	pmd_populate(mm, pmd, pgtable);
2184 2185

	if (freeze) {
2186
		for (i = 0; i < HPAGE_PMD_NR; i++) {
2187 2188 2189 2190
			page_remove_rmap(page + i, false);
			put_page(page + i);
		}
	}
2191 2192 2193
}

void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2194
		unsigned long address, bool freeze, struct page *page)
2195 2196 2197 2198 2199 2200 2201
{
	spinlock_t *ptl;
	struct mm_struct *mm = vma->vm_mm;
	unsigned long haddr = address & HPAGE_PMD_MASK;

	mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
	ptl = pmd_lock(mm, pmd);
2202 2203 2204 2205 2206 2207 2208 2209 2210

	/*
	 * If caller asks to setup a migration entries, we need a page to check
	 * pmd against. Otherwise we can end up replacing wrong page.
	 */
	VM_BUG_ON(freeze && !page);
	if (page && page != pmd_page(*pmd))
	        goto out;

2211
	if (pmd_trans_huge(*pmd)) {
2212
		page = pmd_page(*pmd);
2213
		if (PageMlocked(page))
2214
			clear_page_mlock(page);
2215
	} else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))
2216
		goto out;
2217
	__split_huge_pmd_locked(vma, pmd, haddr, freeze);
2218
out:
2219 2220 2221 2222
	spin_unlock(ptl);
	mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
}

2223 2224
void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
		bool freeze, struct page *page)
2225
{
2226
	pgd_t *pgd;
2227
	p4d_t *p4d;
2228
	pud_t *pud;
2229 2230
	pmd_t *pmd;

2231
	pgd = pgd_offset(vma->vm_mm, address);
2232 2233 2234
	if (!pgd_present(*pgd))
		return;

2235 2236 2237 2238 2239
	p4d = p4d_offset(pgd, address);
	if (!p4d_present(*p4d))
		return;

	pud = pud_offset(p4d, address);
2240 2241 2242 2243
	if (!pud_present(*pud))
		return;

	pmd = pmd_offset(pud, address);
2244

2245
	__split_huge_pmd(vma, pmd, address, freeze, page);
2246 2247
}

2248
void vma_adjust_trans_huge(struct vm_area_struct *vma,
2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
			     unsigned long start,
			     unsigned long end,
			     long adjust_next)
{
	/*
	 * If the new start address isn't hpage aligned and it could
	 * previously contain an hugepage: check if we need to split
	 * an huge pmd.
	 */
	if (start & ~HPAGE_PMD_MASK &&
	    (start & HPAGE_PMD_MASK) >= vma->vm_start &&
	    (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2261
		split_huge_pmd_address(vma, start, false, NULL);
2262 2263 2264 2265 2266 2267 2268 2269 2270

	/*
	 * If the new end address isn't hpage aligned and it could
	 * previously contain an hugepage: check if we need to split
	 * an huge pmd.
	 */
	if (end & ~HPAGE_PMD_MASK &&
	    (end & HPAGE_PMD_MASK) >= vma->vm_start &&
	    (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2271
		split_huge_pmd_address(vma, end, false, NULL);
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284

	/*
	 * If we're also updating the vma->vm_next->vm_start, if the new
	 * vm_next->vm_start isn't page aligned and it could previously
	 * contain an hugepage: check if we need to split an huge pmd.
	 */
	if (adjust_next > 0) {
		struct vm_area_struct *next = vma->vm_next;
		unsigned long nstart = next->vm_start;
		nstart += adjust_next << PAGE_SHIFT;
		if (nstart & ~HPAGE_PMD_MASK &&
		    (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
		    (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
2285
			split_huge_pmd_address(next, nstart, false, NULL);
2286 2287
	}
}
2288

2289
static void freeze_page(struct page *page)
2290
{
2291
	enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS |
2292
		TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD;
M
Minchan Kim 已提交
2293
	bool unmap_success;
2294 2295 2296

	VM_BUG_ON_PAGE(!PageHead(page), page);

2297
	if (PageAnon(page))
2298
		ttu_flags |= TTU_SPLIT_FREEZE;
2299

M
Minchan Kim 已提交
2300 2301
	unmap_success = try_to_unmap(page, ttu_flags);
	VM_BUG_ON_PAGE(!unmap_success, page);
2302 2303
}

2304
static void unfreeze_page(struct page *page)
2305
{
2306
	int i;
2307 2308 2309 2310 2311 2312
	if (PageTransHuge(page)) {
		remove_migration_ptes(page, page, true);
	} else {
		for (i = 0; i < HPAGE_PMD_NR; i++)
			remove_migration_ptes(page + i, page + i, true);
	}
2313 2314
}

2315
static void __split_huge_page_tail(struct page *head, int tail,
2316 2317 2318 2319
		struct lruvec *lruvec, struct list_head *list)
{
	struct page *page_tail = head + tail;

2320
	VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
2321
	VM_BUG_ON_PAGE(page_ref_count(page_tail) != 0, page_tail);
2322 2323

	/*
2324
	 * tail_page->_refcount is zero and not changing from under us. But
2325
	 * get_page_unless_zero() may be running from under us on the
2326 2327
	 * tail_page. If we used atomic_set() below instead of atomic_inc() or
	 * atomic_add(), we would then run atomic_set() concurrently with
2328 2329 2330 2331
	 * get_page_unless_zero(), and atomic_set() is implemented in C not
	 * using locked ops. spin_unlock on x86 sometime uses locked ops
	 * because of PPro errata 66, 92, so unless somebody can guarantee
	 * atomic_set() here would be safe on all archs (and not only on x86),
2332
	 * it's safer to use atomic_inc()/atomic_add().
2333
	 */
2334
	if (PageAnon(head) && !PageSwapCache(head)) {
2335 2336 2337 2338 2339
		page_ref_inc(page_tail);
	} else {
		/* Additional pin to radix tree */
		page_ref_add(page_tail, 2);
	}
2340 2341 2342 2343 2344

	page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
	page_tail->flags |= (head->flags &
			((1L << PG_referenced) |
			 (1L << PG_swapbacked) |
2345
			 (1L << PG_swapcache) |
2346 2347 2348 2349
			 (1L << PG_mlocked) |
			 (1L << PG_uptodate) |
			 (1L << PG_active) |
			 (1L << PG_locked) |
2350 2351
			 (1L << PG_unevictable) |
			 (1L << PG_dirty)));
2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366

	/*
	 * After clearing PageTail the gup refcount can be released.
	 * Page flags also must be visible before we make the page non-compound.
	 */
	smp_wmb();

	clear_compound_head(page_tail);

	if (page_is_young(head))
		set_page_young(page_tail);
	if (page_is_idle(head))
		set_page_idle(page_tail);

	/* ->mapping in first tail page is compound_mapcount */
2367
	VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
2368 2369 2370 2371 2372 2373 2374 2375
			page_tail);
	page_tail->mapping = head->mapping;

	page_tail->index = head->index + tail;
	page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
	lru_add_page_tail(head, page_tail, lruvec, list);
}

2376 2377
static void __split_huge_page(struct page *page, struct list_head *list,
		unsigned long flags)
2378 2379 2380 2381
{
	struct page *head = compound_head(page);
	struct zone *zone = page_zone(head);
	struct lruvec *lruvec;
2382
	pgoff_t end = -1;
2383
	int i;
2384

M
Mel Gorman 已提交
2385
	lruvec = mem_cgroup_page_lruvec(head, zone->zone_pgdat);
2386 2387 2388 2389

	/* complete memcg works before add pages to LRU */
	mem_cgroup_split_huge_fixup(head);

2390 2391 2392 2393
	if (!PageAnon(page))
		end = DIV_ROUND_UP(i_size_read(head->mapping->host), PAGE_SIZE);

	for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
2394
		__split_huge_page_tail(head, i, lruvec, list);
2395 2396 2397 2398
		/* Some pages can be beyond i_size: drop them from page cache */
		if (head[i].index >= end) {
			__ClearPageDirty(head + i);
			__delete_from_page_cache(head + i, NULL);
2399 2400
			if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head))
				shmem_uncharge(head->mapping->host, 1);
2401 2402 2403
			put_page(head + i);
		}
	}
2404 2405

	ClearPageCompound(head);
2406 2407
	/* See comment in __split_huge_page_tail() */
	if (PageAnon(head)) {
2408 2409 2410 2411 2412
		/* Additional pin to radix tree of swap cache */
		if (PageSwapCache(head))
			page_ref_add(head, 2);
		else
			page_ref_inc(head);
2413 2414 2415 2416 2417 2418
	} else {
		/* Additional pin to radix tree */
		page_ref_add(head, 2);
		spin_unlock(&head->mapping->tree_lock);
	}

2419
	spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2420

2421
	unfreeze_page(head);
2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439

	for (i = 0; i < HPAGE_PMD_NR; i++) {
		struct page *subpage = head + i;
		if (subpage == page)
			continue;
		unlock_page(subpage);

		/*
		 * Subpages may be freed if there wasn't any mapping
		 * like if add_to_swap() is running on a lru page that
		 * had its mapping zapped. And freeing these pages
		 * requires taking the lru_lock so we do the put_page
		 * of the tail pages after the split is complete.
		 */
		put_page(subpage);
	}
}

2440 2441
int total_mapcount(struct page *page)
{
K
Kirill A. Shutemov 已提交
2442
	int i, compound, ret;
2443 2444 2445 2446 2447 2448

	VM_BUG_ON_PAGE(PageTail(page), page);

	if (likely(!PageCompound(page)))
		return atomic_read(&page->_mapcount) + 1;

K
Kirill A. Shutemov 已提交
2449
	compound = compound_mapcount(page);
2450
	if (PageHuge(page))
K
Kirill A. Shutemov 已提交
2451 2452
		return compound;
	ret = compound;
2453 2454
	for (i = 0; i < HPAGE_PMD_NR; i++)
		ret += atomic_read(&page[i]._mapcount) + 1;
K
Kirill A. Shutemov 已提交
2455 2456 2457
	/* File pages has compound_mapcount included in _mapcount */
	if (!PageAnon(page))
		return ret - compound * HPAGE_PMD_NR;
2458 2459 2460 2461 2462
	if (PageDoubleMap(page))
		ret -= HPAGE_PMD_NR;
	return ret;
}

2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520
/*
 * This calculates accurately how many mappings a transparent hugepage
 * has (unlike page_mapcount() which isn't fully accurate). This full
 * accuracy is primarily needed to know if copy-on-write faults can
 * reuse the page and change the mapping to read-write instead of
 * copying them. At the same time this returns the total_mapcount too.
 *
 * The function returns the highest mapcount any one of the subpages
 * has. If the return value is one, even if different processes are
 * mapping different subpages of the transparent hugepage, they can
 * all reuse it, because each process is reusing a different subpage.
 *
 * The total_mapcount is instead counting all virtual mappings of the
 * subpages. If the total_mapcount is equal to "one", it tells the
 * caller all mappings belong to the same "mm" and in turn the
 * anon_vma of the transparent hugepage can become the vma->anon_vma
 * local one as no other process may be mapping any of the subpages.
 *
 * It would be more accurate to replace page_mapcount() with
 * page_trans_huge_mapcount(), however we only use
 * page_trans_huge_mapcount() in the copy-on-write faults where we
 * need full accuracy to avoid breaking page pinning, because
 * page_trans_huge_mapcount() is slower than page_mapcount().
 */
int page_trans_huge_mapcount(struct page *page, int *total_mapcount)
{
	int i, ret, _total_mapcount, mapcount;

	/* hugetlbfs shouldn't call it */
	VM_BUG_ON_PAGE(PageHuge(page), page);

	if (likely(!PageTransCompound(page))) {
		mapcount = atomic_read(&page->_mapcount) + 1;
		if (total_mapcount)
			*total_mapcount = mapcount;
		return mapcount;
	}

	page = compound_head(page);

	_total_mapcount = ret = 0;
	for (i = 0; i < HPAGE_PMD_NR; i++) {
		mapcount = atomic_read(&page[i]._mapcount) + 1;
		ret = max(ret, mapcount);
		_total_mapcount += mapcount;
	}
	if (PageDoubleMap(page)) {
		ret -= 1;
		_total_mapcount -= HPAGE_PMD_NR;
	}
	mapcount = compound_mapcount(page);
	ret += mapcount;
	_total_mapcount += mapcount;
	if (total_mapcount)
		*total_mapcount = _total_mapcount;
	return ret;
}

2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535
/* Racy check whether the huge page can be split */
bool can_split_huge_page(struct page *page, int *pextra_pins)
{
	int extra_pins;

	/* Additional pins from radix tree */
	if (PageAnon(page))
		extra_pins = PageSwapCache(page) ? HPAGE_PMD_NR : 0;
	else
		extra_pins = HPAGE_PMD_NR;
	if (pextra_pins)
		*pextra_pins = extra_pins;
	return total_mapcount(page) == page_count(page) - extra_pins - 1;
}

2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557
/*
 * This function splits huge page into normal pages. @page can point to any
 * subpage of huge page to split. Split doesn't change the position of @page.
 *
 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
 * The huge page must be locked.
 *
 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
 *
 * Both head page and tail pages will inherit mapping, flags, and so on from
 * the hugepage.
 *
 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
 * they are not mapped.
 *
 * Returns 0 if the hugepage is split successfully.
 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
 * us.
 */
int split_huge_page_to_list(struct page *page, struct list_head *list)
{
	struct page *head = compound_head(page);
2558
	struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
2559 2560 2561
	struct anon_vma *anon_vma = NULL;
	struct address_space *mapping = NULL;
	int count, mapcount, extra_pins, ret;
2562
	bool mlocked;
2563
	unsigned long flags;
2564 2565 2566 2567 2568

	VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(!PageCompound(page), page);

2569 2570 2571
	if (PageWriteback(page))
		return -EBUSY;

2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
	if (PageAnon(head)) {
		/*
		 * The caller does not necessarily hold an mmap_sem that would
		 * prevent the anon_vma disappearing so we first we take a
		 * reference to it and then lock the anon_vma for write. This
		 * is similar to page_lock_anon_vma_read except the write lock
		 * is taken to serialise against parallel split or collapse
		 * operations.
		 */
		anon_vma = page_get_anon_vma(head);
		if (!anon_vma) {
			ret = -EBUSY;
			goto out;
		}
		mapping = NULL;
		anon_vma_lock_write(anon_vma);
	} else {
		mapping = head->mapping;

		/* Truncated ? */
		if (!mapping) {
			ret = -EBUSY;
			goto out;
		}

		anon_vma = NULL;
		i_mmap_lock_read(mapping);
2599 2600 2601 2602 2603 2604
	}

	/*
	 * Racy check if we can split the page, before freeze_page() will
	 * split PMDs
	 */
2605
	if (!can_split_huge_page(head, &extra_pins)) {
2606 2607 2608 2609
		ret = -EBUSY;
		goto out_unlock;
	}

2610
	mlocked = PageMlocked(page);
2611
	freeze_page(head);
2612 2613
	VM_BUG_ON_PAGE(compound_mapcount(head), head);

2614 2615 2616 2617
	/* Make sure the page is not on per-CPU pagevec as it takes pin */
	if (mlocked)
		lru_add_drain();

2618
	/* prevent PageLRU to go away from under us, and freeze lru stats */
2619
	spin_lock_irqsave(zone_lru_lock(page_zone(head)), flags);
2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635

	if (mapping) {
		void **pslot;

		spin_lock(&mapping->tree_lock);
		pslot = radix_tree_lookup_slot(&mapping->page_tree,
				page_index(head));
		/*
		 * Check if the head page is present in radix tree.
		 * We assume all tail are present too, if head is there.
		 */
		if (radix_tree_deref_slot_protected(pslot,
					&mapping->tree_lock) != head)
			goto fail;
	}

2636
	/* Prevent deferred_split_scan() touching ->_refcount */
2637
	spin_lock(&pgdata->split_queue_lock);
2638 2639
	count = page_count(head);
	mapcount = total_mapcount(head);
2640
	if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) {
2641
		if (!list_empty(page_deferred_list(head))) {
2642
			pgdata->split_queue_len--;
2643 2644
			list_del(page_deferred_list(head));
		}
2645
		if (mapping)
2646
			__dec_node_page_state(page, NR_SHMEM_THPS);
2647 2648
		spin_unlock(&pgdata->split_queue_lock);
		__split_huge_page(page, list, flags);
2649 2650 2651 2652 2653 2654
		if (PageSwapCache(head)) {
			swp_entry_t entry = { .val = page_private(head) };

			ret = split_swap_cluster(entry);
		} else
			ret = 0;
2655
	} else {
2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666
		if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) {
			pr_alert("total_mapcount: %u, page_count(): %u\n",
					mapcount, count);
			if (PageTail(page))
				dump_page(head, NULL);
			dump_page(page, "total_mapcount(head) > 0");
			BUG();
		}
		spin_unlock(&pgdata->split_queue_lock);
fail:		if (mapping)
			spin_unlock(&mapping->tree_lock);
2667
		spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2668
		unfreeze_page(head);
2669 2670 2671 2672
		ret = -EBUSY;
	}

out_unlock:
2673 2674 2675 2676 2677 2678
	if (anon_vma) {
		anon_vma_unlock_write(anon_vma);
		put_anon_vma(anon_vma);
	}
	if (mapping)
		i_mmap_unlock_read(mapping);
2679 2680 2681 2682
out:
	count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
	return ret;
}
2683 2684 2685

void free_transhuge_page(struct page *page)
{
2686
	struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2687 2688
	unsigned long flags;

2689
	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2690
	if (!list_empty(page_deferred_list(page))) {
2691
		pgdata->split_queue_len--;
2692 2693
		list_del(page_deferred_list(page));
	}
2694
	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2695 2696 2697 2698 2699
	free_compound_page(page);
}

void deferred_split_huge_page(struct page *page)
{
2700
	struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2701 2702 2703 2704
	unsigned long flags;

	VM_BUG_ON_PAGE(!PageTransHuge(page), page);

2705
	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2706
	if (list_empty(page_deferred_list(page))) {
2707
		count_vm_event(THP_DEFERRED_SPLIT_PAGE);
2708 2709
		list_add_tail(page_deferred_list(page), &pgdata->split_queue);
		pgdata->split_queue_len++;
2710
	}
2711
	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2712 2713 2714 2715 2716
}

static unsigned long deferred_split_count(struct shrinker *shrink,
		struct shrink_control *sc)
{
2717
	struct pglist_data *pgdata = NODE_DATA(sc->nid);
2718
	return ACCESS_ONCE(pgdata->split_queue_len);
2719 2720 2721 2722 2723
}

static unsigned long deferred_split_scan(struct shrinker *shrink,
		struct shrink_control *sc)
{
2724
	struct pglist_data *pgdata = NODE_DATA(sc->nid);
2725 2726 2727 2728 2729
	unsigned long flags;
	LIST_HEAD(list), *pos, *next;
	struct page *page;
	int split = 0;

2730
	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2731
	/* Take pin on all head pages to avoid freeing them under us */
2732
	list_for_each_safe(pos, next, &pgdata->split_queue) {
2733 2734
		page = list_entry((void *)pos, struct page, mapping);
		page = compound_head(page);
2735 2736 2737 2738
		if (get_page_unless_zero(page)) {
			list_move(page_deferred_list(page), &list);
		} else {
			/* We lost race with put_compound_page() */
2739
			list_del_init(page_deferred_list(page));
2740
			pgdata->split_queue_len--;
2741
		}
2742 2743
		if (!--sc->nr_to_scan)
			break;
2744
	}
2745
	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756

	list_for_each_safe(pos, next, &list) {
		page = list_entry((void *)pos, struct page, mapping);
		lock_page(page);
		/* split_huge_page() removes page from list on success */
		if (!split_huge_page(page))
			split++;
		unlock_page(page);
		put_page(page);
	}

2757 2758 2759
	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
	list_splice_tail(&list, &pgdata->split_queue);
	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2760

2761 2762 2763 2764 2765 2766 2767
	/*
	 * Stop shrinker if we didn't split any page, but the queue is empty.
	 * This can happen if pages were freed under us.
	 */
	if (!split && list_empty(&pgdata->split_queue))
		return SHRINK_STOP;
	return split;
2768 2769 2770 2771 2772 2773
}

static struct shrinker deferred_split_shrinker = {
	.count_objects = deferred_split_count,
	.scan_objects = deferred_split_scan,
	.seeks = DEFAULT_SEEKS,
2774
	.flags = SHRINKER_NUMA_AWARE,
2775
};
2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800

#ifdef CONFIG_DEBUG_FS
static int split_huge_pages_set(void *data, u64 val)
{
	struct zone *zone;
	struct page *page;
	unsigned long pfn, max_zone_pfn;
	unsigned long total = 0, split = 0;

	if (val != 1)
		return -EINVAL;

	for_each_populated_zone(zone) {
		max_zone_pfn = zone_end_pfn(zone);
		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
			if (!pfn_valid(pfn))
				continue;

			page = pfn_to_page(pfn);
			if (!get_page_unless_zero(page))
				continue;

			if (zone != page_zone(page))
				goto next;

2801
			if (!PageHead(page) || PageHuge(page) || !PageLRU(page))
2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813
				goto next;

			total++;
			lock_page(page);
			if (!split_huge_page(page))
				split++;
			unlock_page(page);
next:
			put_page(page);
		}
	}

2814
	pr_info("%lu of %lu THP split\n", split, total);
2815 2816 2817 2818 2819 2820 2821 2822 2823 2824

	return 0;
}
DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
		"%llu\n");

static int __init split_huge_pages_debugfs(void)
{
	void *ret;

2825
	ret = debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
2826 2827 2828 2829 2830 2831 2832
			&split_huge_pages_fops);
	if (!ret)
		pr_warn("Failed to create split_huge_pages in debugfs");
	return 0;
}
late_initcall(split_huge_pages_debugfs);
#endif
2833 2834 2835 2836 2837 2838 2839 2840 2841 2842

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
		struct page *page)
{
	struct vm_area_struct *vma = pvmw->vma;
	struct mm_struct *mm = vma->vm_mm;
	unsigned long address = pvmw->address;
	pmd_t pmdval;
	swp_entry_t entry;
2843
	pmd_t pmdswp;
2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856

	if (!(pvmw->pmd && !pvmw->pte))
		return;

	mmu_notifier_invalidate_range_start(mm, address,
			address + HPAGE_PMD_SIZE);

	flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
	pmdval = *pvmw->pmd;
	pmdp_invalidate(vma, address, pvmw->pmd);
	if (pmd_dirty(pmdval))
		set_page_dirty(page);
	entry = make_migration_entry(page, pmd_write(pmdval));
2857 2858 2859 2860
	pmdswp = swp_entry_to_pmd(entry);
	if (pmd_soft_dirty(pmdval))
		pmdswp = pmd_swp_mksoft_dirty(pmdswp);
	set_pmd_at(mm, address, pvmw->pmd, pmdswp);
2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882
	page_remove_rmap(page, true);
	put_page(page);

	mmu_notifier_invalidate_range_end(mm, address,
			address + HPAGE_PMD_SIZE);
}

void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
{
	struct vm_area_struct *vma = pvmw->vma;
	struct mm_struct *mm = vma->vm_mm;
	unsigned long address = pvmw->address;
	unsigned long mmun_start = address & HPAGE_PMD_MASK;
	pmd_t pmde;
	swp_entry_t entry;

	if (!(pvmw->pmd && !pvmw->pte))
		return;

	entry = pmd_to_swp_entry(*pvmw->pmd);
	get_page(new);
	pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot));
2883 2884
	if (pmd_swp_soft_dirty(*pvmw->pmd))
		pmde = pmd_mksoft_dirty(pmde);
2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
	if (is_write_migration_entry(entry))
		pmde = maybe_pmd_mkwrite(pmde, vma);

	flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE);
	page_add_anon_rmap(new, vma, mmun_start, true);
	set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);
	if (vma->vm_flags & VM_LOCKED)
		mlock_vma_page(new);
	update_mmu_cache_pmd(vma, address, pvmw->pmd);
}
#endif