huge_memory.c 74.5 KB
Newer Older
1 2 3 4 5 6 7
/*
 *  Copyright (C) 2009  Red Hat, Inc.
 *
 *  This work is licensed under the terms of the GNU GPL, version 2. See
 *  the COPYING file in the top-level directory.
 */

8 9
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

10 11
#include <linux/mm.h>
#include <linux/sched.h>
12
#include <linux/sched/coredump.h>
13
#include <linux/sched/numa_balancing.h>
14 15 16 17 18
#include <linux/highmem.h>
#include <linux/hugetlb.h>
#include <linux/mmu_notifier.h>
#include <linux/rmap.h>
#include <linux/swap.h>
19
#include <linux/shrinker.h>
A
Andrea Arcangeli 已提交
20
#include <linux/mm_inline.h>
21
#include <linux/swapops.h>
M
Matthew Wilcox 已提交
22
#include <linux/dax.h>
A
Andrea Arcangeli 已提交
23
#include <linux/khugepaged.h>
24
#include <linux/freezer.h>
25
#include <linux/pfn_t.h>
A
Andrea Arcangeli 已提交
26
#include <linux/mman.h>
27
#include <linux/memremap.h>
R
Ralf Baechle 已提交
28
#include <linux/pagemap.h>
29
#include <linux/debugfs.h>
30
#include <linux/migrate.h>
31
#include <linux/hashtable.h>
32
#include <linux/userfaultfd_k.h>
33
#include <linux/page_idle.h>
34
#include <linux/shmem_fs.h>
35

36 37 38 39
#include <asm/tlb.h>
#include <asm/pgalloc.h>
#include "internal.h"

A
Andrea Arcangeli 已提交
40
/*
41 42 43 44 45 46
 * By default transparent hugepage support is disabled in order that avoid
 * to risk increase the memory footprint of applications without a guaranteed
 * benefit. When transparent hugepage support is enabled, is for all mappings,
 * and khugepaged scans all mappings.
 * Defrag is invoked by khugepaged hugepage allocations and by page faults
 * for all hugepage allocations.
A
Andrea Arcangeli 已提交
47
 */
48
unsigned long transparent_hugepage_flags __read_mostly =
49
#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
A
Andrea Arcangeli 已提交
50
	(1<<TRANSPARENT_HUGEPAGE_FLAG)|
51 52 53 54
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
#endif
55
	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
56 57
	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
A
Andrea Arcangeli 已提交
58

59
static struct shrinker deferred_split_shrinker;
60

61
static atomic_t huge_zero_refcount;
62
struct page *huge_zero_page __read_mostly;
63

64
static struct page *get_huge_zero_page(void)
65 66 67 68
{
	struct page *zero_page;
retry:
	if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
69
		return READ_ONCE(huge_zero_page);
70 71

	zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
72
			HPAGE_PMD_ORDER);
73 74
	if (!zero_page) {
		count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
75
		return NULL;
76 77
	}
	count_vm_event(THP_ZERO_PAGE_ALLOC);
78
	preempt_disable();
79
	if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
80
		preempt_enable();
81
		__free_pages(zero_page, compound_order(zero_page));
82 83 84 85 86 87
		goto retry;
	}

	/* We take additional reference here. It will be put back by shrinker */
	atomic_set(&huge_zero_refcount, 2);
	preempt_enable();
88
	return READ_ONCE(huge_zero_page);
89 90
}

91
static void put_huge_zero_page(void)
92
{
93 94 95 96 97
	/*
	 * Counter should never go to zero here. Only shrinker can put
	 * last reference.
	 */
	BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
98 99
}

100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
struct page *mm_get_huge_zero_page(struct mm_struct *mm)
{
	if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
		return READ_ONCE(huge_zero_page);

	if (!get_huge_zero_page())
		return NULL;

	if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
		put_huge_zero_page();

	return READ_ONCE(huge_zero_page);
}

void mm_put_huge_zero_page(struct mm_struct *mm)
{
	if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
		put_huge_zero_page();
}

120 121
static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
					struct shrink_control *sc)
122
{
123 124 125
	/* we can free zero page only if last reference remains */
	return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
}
126

127 128 129
static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
				       struct shrink_control *sc)
{
130
	if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
131 132
		struct page *zero_page = xchg(&huge_zero_page, NULL);
		BUG_ON(zero_page == NULL);
133
		__free_pages(zero_page, compound_order(zero_page));
134
		return HPAGE_PMD_NR;
135 136 137
	}

	return 0;
138 139
}

140
static struct shrinker huge_zero_page_shrinker = {
141 142
	.count_objects = shrink_huge_zero_page_count,
	.scan_objects = shrink_huge_zero_page_scan,
143 144 145
	.seeks = DEFAULT_SEEKS,
};

146 147 148 149
#ifdef CONFIG_SYSFS
static ssize_t enabled_show(struct kobject *kobj,
			    struct kobj_attribute *attr, char *buf)
{
150 151 152 153 154 155
	if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
		return sprintf(buf, "[always] madvise never\n");
	else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
		return sprintf(buf, "always [madvise] never\n");
	else
		return sprintf(buf, "always madvise [never]\n");
156
}
157

158 159 160 161
static ssize_t enabled_store(struct kobject *kobj,
			     struct kobj_attribute *attr,
			     const char *buf, size_t count)
{
162
	ssize_t ret = count;
A
Andrea Arcangeli 已提交
163

164 165 166 167 168 169 170 171 172 173 174 175 176 177
	if (!memcmp("always", buf,
		    min(sizeof("always")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
	} else if (!memcmp("madvise", buf,
			   min(sizeof("madvise")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
	} else if (!memcmp("never", buf,
			   min(sizeof("never")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
	} else
		ret = -EINVAL;
A
Andrea Arcangeli 已提交
178 179

	if (ret > 0) {
180
		int err = start_stop_khugepaged();
A
Andrea Arcangeli 已提交
181 182 183 184
		if (err)
			ret = err;
	}
	return ret;
185 186 187 188
}
static struct kobj_attribute enabled_attr =
	__ATTR(enabled, 0644, enabled_show, enabled_store);

189
ssize_t single_hugepage_flag_show(struct kobject *kobj,
190 191 192
				struct kobj_attribute *attr, char *buf,
				enum transparent_hugepage_flag flag)
{
193 194
	return sprintf(buf, "%d\n",
		       !!test_bit(flag, &transparent_hugepage_flags));
195
}
196

197
ssize_t single_hugepage_flag_store(struct kobject *kobj,
198 199 200 201
				 struct kobj_attribute *attr,
				 const char *buf, size_t count,
				 enum transparent_hugepage_flag flag)
{
202 203 204 205 206 207 208 209 210 211
	unsigned long value;
	int ret;

	ret = kstrtoul(buf, 10, &value);
	if (ret < 0)
		return ret;
	if (value > 1)
		return -EINVAL;

	if (value)
212
		set_bit(flag, &transparent_hugepage_flags);
213
	else
214 215 216 217 218 219 220 221
		clear_bit(flag, &transparent_hugepage_flags);

	return count;
}

static ssize_t defrag_show(struct kobject *kobj,
			   struct kobj_attribute *attr, char *buf)
{
222
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
223
		return sprintf(buf, "[always] defer defer+madvise madvise never\n");
224
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
225 226 227 228 229 230
		return sprintf(buf, "always [defer] defer+madvise madvise never\n");
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
		return sprintf(buf, "always defer [defer+madvise] madvise never\n");
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
		return sprintf(buf, "always defer defer+madvise [madvise] never\n");
	return sprintf(buf, "always defer defer+madvise madvise [never]\n");
231
}
232

233 234 235 236
static ssize_t defrag_store(struct kobject *kobj,
			    struct kobj_attribute *attr,
			    const char *buf, size_t count)
{
237 238 239 240 241 242 243 244 245 246 247 248
	if (!memcmp("always", buf,
		    min(sizeof("always")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
	} else if (!memcmp("defer+madvise", buf,
		    min(sizeof("defer+madvise")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
249 250 251 252 253 254
	} else if (!memcmp("defer", buf,
		    min(sizeof("defer")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
	} else if (!memcmp("madvise", buf,
			   min(sizeof("madvise")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
	} else if (!memcmp("never", buf,
			   min(sizeof("never")-1, count))) {
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
	} else
		return -EINVAL;

	return count;
271 272 273 274
}
static struct kobj_attribute defrag_attr =
	__ATTR(defrag, 0644, defrag_show, defrag_store);

275 276 277
static ssize_t use_zero_page_show(struct kobject *kobj,
		struct kobj_attribute *attr, char *buf)
{
278
	return single_hugepage_flag_show(kobj, attr, buf,
279 280 281 282 283
				TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
}
static ssize_t use_zero_page_store(struct kobject *kobj,
		struct kobj_attribute *attr, const char *buf, size_t count)
{
284
	return single_hugepage_flag_store(kobj, attr, buf, count,
285 286 287 288
				 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
}
static struct kobj_attribute use_zero_page_attr =
	__ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
289 290 291 292 293 294 295 296 297

static ssize_t hpage_pmd_size_show(struct kobject *kobj,
		struct kobj_attribute *attr, char *buf)
{
	return sprintf(buf, "%lu\n", HPAGE_PMD_SIZE);
}
static struct kobj_attribute hpage_pmd_size_attr =
	__ATTR_RO(hpage_pmd_size);

298 299 300 301
#ifdef CONFIG_DEBUG_VM
static ssize_t debug_cow_show(struct kobject *kobj,
				struct kobj_attribute *attr, char *buf)
{
302
	return single_hugepage_flag_show(kobj, attr, buf,
303 304 305 306 307 308
				TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
}
static ssize_t debug_cow_store(struct kobject *kobj,
			       struct kobj_attribute *attr,
			       const char *buf, size_t count)
{
309
	return single_hugepage_flag_store(kobj, attr, buf, count,
310 311 312 313 314 315 316 317 318
				 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
}
static struct kobj_attribute debug_cow_attr =
	__ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
#endif /* CONFIG_DEBUG_VM */

static struct attribute *hugepage_attr[] = {
	&enabled_attr.attr,
	&defrag_attr.attr,
319
	&use_zero_page_attr.attr,
320
	&hpage_pmd_size_attr.attr,
321
#if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
322 323
	&shmem_enabled_attr.attr,
#endif
324 325 326 327 328 329 330 331
#ifdef CONFIG_DEBUG_VM
	&debug_cow_attr.attr,
#endif
	NULL,
};

static struct attribute_group hugepage_attr_group = {
	.attrs = hugepage_attr,
A
Andrea Arcangeli 已提交
332 333
};

S
Shaohua Li 已提交
334
static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
335 336 337
{
	int err;

S
Shaohua Li 已提交
338 339
	*hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
	if (unlikely(!*hugepage_kobj)) {
340
		pr_err("failed to create transparent hugepage kobject\n");
S
Shaohua Li 已提交
341
		return -ENOMEM;
A
Andrea Arcangeli 已提交
342 343
	}

S
Shaohua Li 已提交
344
	err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
A
Andrea Arcangeli 已提交
345
	if (err) {
346
		pr_err("failed to register transparent hugepage group\n");
S
Shaohua Li 已提交
347
		goto delete_obj;
A
Andrea Arcangeli 已提交
348 349
	}

S
Shaohua Li 已提交
350
	err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
A
Andrea Arcangeli 已提交
351
	if (err) {
352
		pr_err("failed to register transparent hugepage group\n");
S
Shaohua Li 已提交
353
		goto remove_hp_group;
A
Andrea Arcangeli 已提交
354
	}
S
Shaohua Li 已提交
355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391

	return 0;

remove_hp_group:
	sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
delete_obj:
	kobject_put(*hugepage_kobj);
	return err;
}

static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
{
	sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
	sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
	kobject_put(hugepage_kobj);
}
#else
static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
{
	return 0;
}

static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
{
}
#endif /* CONFIG_SYSFS */

static int __init hugepage_init(void)
{
	int err;
	struct kobject *hugepage_kobj;

	if (!has_transparent_hugepage()) {
		transparent_hugepage_flags = 0;
		return -EINVAL;
	}

392 393 394 395 396 397 398 399 400 401
	/*
	 * hugepages can't be allocated by the buddy allocator
	 */
	MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER);
	/*
	 * we use page->mapping and page->index in second tail page
	 * as list_head: assuming THP order >= 2
	 */
	MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);

S
Shaohua Li 已提交
402 403
	err = hugepage_init_sysfs(&hugepage_kobj);
	if (err)
404
		goto err_sysfs;
A
Andrea Arcangeli 已提交
405

406
	err = khugepaged_init();
A
Andrea Arcangeli 已提交
407
	if (err)
408
		goto err_slab;
A
Andrea Arcangeli 已提交
409

410 411 412
	err = register_shrinker(&huge_zero_page_shrinker);
	if (err)
		goto err_hzp_shrinker;
413 414 415
	err = register_shrinker(&deferred_split_shrinker);
	if (err)
		goto err_split_shrinker;
416

417 418 419 420 421
	/*
	 * By default disable transparent hugepages on smaller systems,
	 * where the extra memory used could hurt more than TLB overhead
	 * is likely to save.  The admin can still enable it through /sys.
	 */
422
	if (totalram_pages < (512 << (20 - PAGE_SHIFT))) {
423
		transparent_hugepage_flags = 0;
424 425
		return 0;
	}
426

427
	err = start_stop_khugepaged();
428 429
	if (err)
		goto err_khugepaged;
A
Andrea Arcangeli 已提交
430

S
Shaohua Li 已提交
431
	return 0;
432
err_khugepaged:
433 434
	unregister_shrinker(&deferred_split_shrinker);
err_split_shrinker:
435 436
	unregister_shrinker(&huge_zero_page_shrinker);
err_hzp_shrinker:
437
	khugepaged_destroy();
438
err_slab:
S
Shaohua Li 已提交
439
	hugepage_exit_sysfs(hugepage_kobj);
440
err_sysfs:
A
Andrea Arcangeli 已提交
441
	return err;
442
}
443
subsys_initcall(hugepage_init);
444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470

static int __init setup_transparent_hugepage(char *str)
{
	int ret = 0;
	if (!str)
		goto out;
	if (!strcmp(str, "always")) {
		set_bit(TRANSPARENT_HUGEPAGE_FLAG,
			&transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
			  &transparent_hugepage_flags);
		ret = 1;
	} else if (!strcmp(str, "madvise")) {
		clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
			  &transparent_hugepage_flags);
		set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
			&transparent_hugepage_flags);
		ret = 1;
	} else if (!strcmp(str, "never")) {
		clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
			  &transparent_hugepage_flags);
		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
			  &transparent_hugepage_flags);
		ret = 1;
	}
out:
	if (!ret)
471
		pr_warn("transparent_hugepage= cannot parse, ignored\n");
472 473 474 475
	return ret;
}
__setup("transparent_hugepage=", setup_transparent_hugepage);

476
pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
477 478 479 480 481 482
{
	if (likely(vma->vm_flags & VM_WRITE))
		pmd = pmd_mkwrite(pmd);
	return pmd;
}

483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
static inline struct list_head *page_deferred_list(struct page *page)
{
	/*
	 * ->lru in the tail pages is occupied by compound_head.
	 * Let's use ->mapping + ->index in the second tail page as list_head.
	 */
	return (struct list_head *)&page[2].mapping;
}

void prep_transhuge_page(struct page *page)
{
	/*
	 * we use page->mapping and page->indexlru in second tail page
	 * as list_head: assuming THP order >= 2
	 */

	INIT_LIST_HEAD(page_deferred_list(page));
	set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
}

503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long len,
		loff_t off, unsigned long flags, unsigned long size)
{
	unsigned long addr;
	loff_t off_end = off + len;
	loff_t off_align = round_up(off, size);
	unsigned long len_pad;

	if (off_end <= off_align || (off_end - off_align) < size)
		return 0;

	len_pad = len + size;
	if (len_pad < len || (off + len_pad) < off)
		return 0;

	addr = current->mm->get_unmapped_area(filp, 0, len_pad,
					      off >> PAGE_SHIFT, flags);
	if (IS_ERR_VALUE(addr))
		return 0;

	addr += (off - addr) & (size - 1);
	return addr;
}

unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	loff_t off = (loff_t)pgoff << PAGE_SHIFT;

	if (addr)
		goto out;
	if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD))
		goto out;

	addr = __thp_get_unmapped_area(filp, len, off, flags, PMD_SIZE);
	if (addr)
		return addr;

 out:
	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
}
EXPORT_SYMBOL_GPL(thp_get_unmapped_area);

J
Jan Kara 已提交
546
static int __do_huge_pmd_anonymous_page(struct vm_fault *vmf, struct page *page,
K
Kirill A. Shutemov 已提交
547
		gfp_t gfp)
548
{
J
Jan Kara 已提交
549
	struct vm_area_struct *vma = vmf->vma;
550
	struct mem_cgroup *memcg;
551
	pgtable_t pgtable;
J
Jan Kara 已提交
552
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
553

554
	VM_BUG_ON_PAGE(!PageCompound(page), page);
555

K
Kirill A. Shutemov 已提交
556
	if (mem_cgroup_try_charge(page, vma->vm_mm, gfp, &memcg, true)) {
557 558 559 560
		put_page(page);
		count_vm_event(THP_FAULT_FALLBACK);
		return VM_FAULT_FALLBACK;
	}
561

K
Kirill A. Shutemov 已提交
562
	pgtable = pte_alloc_one(vma->vm_mm, haddr);
563
	if (unlikely(!pgtable)) {
564
		mem_cgroup_cancel_charge(page, memcg, true);
565
		put_page(page);
566
		return VM_FAULT_OOM;
567
	}
568 569

	clear_huge_page(page, haddr, HPAGE_PMD_NR);
570 571 572 573 574
	/*
	 * The memory barrier inside __SetPageUptodate makes sure that
	 * clear_huge_page writes become visible before the set_pmd_at()
	 * write.
	 */
575 576
	__SetPageUptodate(page);

J
Jan Kara 已提交
577 578 579
	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_none(*vmf->pmd))) {
		spin_unlock(vmf->ptl);
580
		mem_cgroup_cancel_charge(page, memcg, true);
581
		put_page(page);
K
Kirill A. Shutemov 已提交
582
		pte_free(vma->vm_mm, pgtable);
583 584
	} else {
		pmd_t entry;
585 586 587 588 589

		/* Deliver the page fault to userland */
		if (userfaultfd_missing(vma)) {
			int ret;

J
Jan Kara 已提交
590
			spin_unlock(vmf->ptl);
591
			mem_cgroup_cancel_charge(page, memcg, true);
592
			put_page(page);
K
Kirill A. Shutemov 已提交
593
			pte_free(vma->vm_mm, pgtable);
J
Jan Kara 已提交
594
			ret = handle_userfault(vmf, VM_UFFD_MISSING);
595 596 597 598
			VM_BUG_ON(ret & VM_FAULT_FALLBACK);
			return ret;
		}

599 600
		entry = mk_huge_pmd(page, vma->vm_page_prot);
		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
601
		page_add_new_anon_rmap(page, vma, haddr, true);
602
		mem_cgroup_commit_charge(page, memcg, false, true);
603
		lru_cache_add_active_or_unevictable(page, vma);
J
Jan Kara 已提交
604 605
		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
K
Kirill A. Shutemov 已提交
606 607
		add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
		atomic_long_inc(&vma->vm_mm->nr_ptes);
J
Jan Kara 已提交
608
		spin_unlock(vmf->ptl);
609
		count_vm_event(THP_FAULT_ALLOC);
610 611
	}

612
	return 0;
613 614
}

615
/*
616 617 618 619 620 621 622
 * always: directly stall for all thp allocations
 * defer: wake kswapd and fail if not immediately available
 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
 *		  fail if not immediately available
 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
 *	    available
 * never: never stall for any thp allocation
623 624 625
 */
static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
{
626
	const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
627

628
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
629
		return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
630 631 632 633 634 635 636 637
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
		return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
		return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :
							     __GFP_KSWAPD_RECLAIM);
	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
		return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :
							     0);
638
	return GFP_TRANSHUGE_LIGHT;
639 640
}

641
/* Caller must hold page table lock. */
642
static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
643
		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
644
		struct page *zero_page)
645 646
{
	pmd_t entry;
A
Andrew Morton 已提交
647 648
	if (!pmd_none(*pmd))
		return false;
649
	entry = mk_pmd(zero_page, vma->vm_page_prot);
650
	entry = pmd_mkhuge(entry);
651 652
	if (pgtable)
		pgtable_trans_huge_deposit(mm, pmd, pgtable);
653
	set_pmd_at(mm, haddr, pmd, entry);
654
	atomic_long_inc(&mm->nr_ptes);
A
Andrew Morton 已提交
655
	return true;
656 657
}

J
Jan Kara 已提交
658
int do_huge_pmd_anonymous_page(struct vm_fault *vmf)
659
{
J
Jan Kara 已提交
660
	struct vm_area_struct *vma = vmf->vma;
661
	gfp_t gfp;
662
	struct page *page;
J
Jan Kara 已提交
663
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
664

665
	if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
666
		return VM_FAULT_FALLBACK;
667 668
	if (unlikely(anon_vma_prepare(vma)))
		return VM_FAULT_OOM;
669
	if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
670
		return VM_FAULT_OOM;
J
Jan Kara 已提交
671
	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
K
Kirill A. Shutemov 已提交
672
			!mm_forbids_zeropage(vma->vm_mm) &&
673 674 675 676
			transparent_hugepage_use_zero_page()) {
		pgtable_t pgtable;
		struct page *zero_page;
		bool set;
677
		int ret;
K
Kirill A. Shutemov 已提交
678
		pgtable = pte_alloc_one(vma->vm_mm, haddr);
679
		if (unlikely(!pgtable))
A
Andrea Arcangeli 已提交
680
			return VM_FAULT_OOM;
681
		zero_page = mm_get_huge_zero_page(vma->vm_mm);
682
		if (unlikely(!zero_page)) {
K
Kirill A. Shutemov 已提交
683
			pte_free(vma->vm_mm, pgtable);
684
			count_vm_event(THP_FAULT_FALLBACK);
685
			return VM_FAULT_FALLBACK;
A
Andrea Arcangeli 已提交
686
		}
J
Jan Kara 已提交
687
		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
688 689
		ret = 0;
		set = false;
J
Jan Kara 已提交
690
		if (pmd_none(*vmf->pmd)) {
691
			if (userfaultfd_missing(vma)) {
J
Jan Kara 已提交
692 693
				spin_unlock(vmf->ptl);
				ret = handle_userfault(vmf, VM_UFFD_MISSING);
694 695
				VM_BUG_ON(ret & VM_FAULT_FALLBACK);
			} else {
K
Kirill A. Shutemov 已提交
696
				set_huge_zero_page(pgtable, vma->vm_mm, vma,
J
Jan Kara 已提交
697 698
						   haddr, vmf->pmd, zero_page);
				spin_unlock(vmf->ptl);
699 700 701
				set = true;
			}
		} else
J
Jan Kara 已提交
702
			spin_unlock(vmf->ptl);
703
		if (!set)
K
Kirill A. Shutemov 已提交
704
			pte_free(vma->vm_mm, pgtable);
705
		return ret;
706
	}
707
	gfp = alloc_hugepage_direct_gfpmask(vma);
708
	page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
709 710
	if (unlikely(!page)) {
		count_vm_event(THP_FAULT_FALLBACK);
711
		return VM_FAULT_FALLBACK;
712
	}
713
	prep_transhuge_page(page);
J
Jan Kara 已提交
714
	return __do_huge_pmd_anonymous_page(vmf, page, gfp);
715 716
}

717
static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
718 719
		pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,
		pgtable_t pgtable)
M
Matthew Wilcox 已提交
720 721 722 723 724 725
{
	struct mm_struct *mm = vma->vm_mm;
	pmd_t entry;
	spinlock_t *ptl;

	ptl = pmd_lock(mm, pmd);
726 727 728
	entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
	if (pfn_t_devmap(pfn))
		entry = pmd_mkdevmap(entry);
729 730 731
	if (write) {
		entry = pmd_mkyoung(pmd_mkdirty(entry));
		entry = maybe_pmd_mkwrite(entry, vma);
M
Matthew Wilcox 已提交
732
	}
733 734 735 736 737 738

	if (pgtable) {
		pgtable_trans_huge_deposit(mm, pmd, pgtable);
		atomic_long_inc(&mm->nr_ptes);
	}

739 740
	set_pmd_at(mm, addr, pmd, entry);
	update_mmu_cache_pmd(vma, addr, pmd);
M
Matthew Wilcox 已提交
741 742 743 744
	spin_unlock(ptl);
}

int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
745
			pmd_t *pmd, pfn_t pfn, bool write)
M
Matthew Wilcox 已提交
746 747
{
	pgprot_t pgprot = vma->vm_page_prot;
748
	pgtable_t pgtable = NULL;
M
Matthew Wilcox 已提交
749 750 751 752 753 754 755 756 757
	/*
	 * If we had pmd_special, we could avoid all these restrictions,
	 * but we need to be consistent with PTEs and architectures that
	 * can't support a 'special' bit.
	 */
	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
						(VM_PFNMAP|VM_MIXEDMAP));
	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
758
	BUG_ON(!pfn_t_devmap(pfn));
M
Matthew Wilcox 已提交
759 760 761

	if (addr < vma->vm_start || addr >= vma->vm_end)
		return VM_FAULT_SIGBUS;
762

763 764 765 766 767 768
	if (arch_needs_pgtable_deposit()) {
		pgtable = pte_alloc_one(vma->vm_mm, addr);
		if (!pgtable)
			return VM_FAULT_OOM;
	}

769 770
	track_pfn_insert(vma, &pgprot, pfn);

771
	insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write, pgtable);
772
	return VM_FAULT_NOPAGE;
M
Matthew Wilcox 已提交
773
}
774
EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
M
Matthew Wilcox 已提交
775

776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
{
	if (likely(vma->vm_flags & VM_WRITE))
		pud = pud_mkwrite(pud);
	return pud;
}

static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
		pud_t *pud, pfn_t pfn, pgprot_t prot, bool write)
{
	struct mm_struct *mm = vma->vm_mm;
	pud_t entry;
	spinlock_t *ptl;

	ptl = pud_lock(mm, pud);
	entry = pud_mkhuge(pfn_t_pud(pfn, prot));
	if (pfn_t_devmap(pfn))
		entry = pud_mkdevmap(entry);
	if (write) {
		entry = pud_mkyoung(pud_mkdirty(entry));
		entry = maybe_pud_mkwrite(entry, vma);
	}
	set_pud_at(mm, addr, pud, entry);
	update_mmu_cache_pud(vma, addr, pud);
	spin_unlock(ptl);
}

int vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
			pud_t *pud, pfn_t pfn, bool write)
{
	pgprot_t pgprot = vma->vm_page_prot;
	/*
	 * If we had pud_special, we could avoid all these restrictions,
	 * but we need to be consistent with PTEs and architectures that
	 * can't support a 'special' bit.
	 */
	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
						(VM_PFNMAP|VM_MIXEDMAP));
	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
	BUG_ON(!pfn_t_devmap(pfn));

	if (addr < vma->vm_start || addr >= vma->vm_end)
		return VM_FAULT_SIGBUS;

	track_pfn_insert(vma, &pgprot, pfn);

	insert_pfn_pud(vma, addr, pud, pfn, pgprot, write);
	return VM_FAULT_NOPAGE;
}
EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857
static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
		pmd_t *pmd)
{
	pmd_t _pmd;

	/*
	 * We should set the dirty bit only for FOLL_WRITE but for now
	 * the dirty bit in the pmd is meaningless.  And if the dirty
	 * bit will become meaningful and we'll only set it with
	 * FOLL_WRITE, an atomic set_bit will be required on the pmd to
	 * set the young bit, instead of the current set_pmd_at.
	 */
	_pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
	if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
				pmd, _pmd,  1))
		update_mmu_cache_pmd(vma, addr, pmd);
}

struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
		pmd_t *pmd, int flags)
{
	unsigned long pfn = pmd_pfn(*pmd);
	struct mm_struct *mm = vma->vm_mm;
	struct dev_pagemap *pgmap;
	struct page *page;

	assert_spin_locked(pmd_lockptr(mm, pmd));

858 859 860 861 862 863
	/*
	 * When we COW a devmap PMD entry, we split it into PTEs, so we should
	 * not be in this function with `flags & FOLL_COW` set.
	 */
	WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set");

864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892
	if (flags & FOLL_WRITE && !pmd_write(*pmd))
		return NULL;

	if (pmd_present(*pmd) && pmd_devmap(*pmd))
		/* pass */;
	else
		return NULL;

	if (flags & FOLL_TOUCH)
		touch_pmd(vma, addr, pmd);

	/*
	 * device mapped pages can only be returned if the
	 * caller will manage the page reference count.
	 */
	if (!(flags & FOLL_GET))
		return ERR_PTR(-EEXIST);

	pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
	pgmap = get_dev_pagemap(pfn, NULL);
	if (!pgmap)
		return ERR_PTR(-EFAULT);
	page = pfn_to_page(pfn);
	get_page(page);
	put_dev_pagemap(pgmap);

	return page;
}

893 894 895 896
int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
		  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
		  struct vm_area_struct *vma)
{
897
	spinlock_t *dst_ptl, *src_ptl;
898 899
	struct page *src_page;
	pmd_t pmd;
900
	pgtable_t pgtable = NULL;
901
	int ret = -ENOMEM;
902

903 904 905 906 907 908 909
	/* Skip if can be re-fill on fault */
	if (!vma_is_anonymous(vma))
		return 0;

	pgtable = pte_alloc_one(dst_mm, addr);
	if (unlikely(!pgtable))
		goto out;
910

911 912 913
	dst_ptl = pmd_lock(dst_mm, dst_pmd);
	src_ptl = pmd_lockptr(src_mm, src_pmd);
	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
914 915 916

	ret = -EAGAIN;
	pmd = *src_pmd;
917
	if (unlikely(!pmd_trans_huge(pmd))) {
918 919 920
		pte_free(dst_mm, pgtable);
		goto out_unlock;
	}
921
	/*
922
	 * When page table lock is held, the huge zero pmd should not be
923 924 925 926
	 * under splitting since we don't split the page itself, only pmd to
	 * a page table.
	 */
	if (is_huge_zero_pmd(pmd)) {
927
		struct page *zero_page;
928 929 930 931 932
		/*
		 * get_huge_zero_page() will never allocate a new page here,
		 * since we already have a zero page to copy. It just takes a
		 * reference.
		 */
933
		zero_page = mm_get_huge_zero_page(dst_mm);
934
		set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
935
				zero_page);
936 937 938
		ret = 0;
		goto out_unlock;
	}
939

940 941 942 943 944 945 946
	src_page = pmd_page(pmd);
	VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
	get_page(src_page);
	page_dup_rmap(src_page, true);
	add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
	atomic_long_inc(&dst_mm->nr_ptes);
	pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
947 948 949 950 951 952 953

	pmdp_set_wrprotect(src_mm, addr, src_pmd);
	pmd = pmd_mkold(pmd_wrprotect(pmd));
	set_pmd_at(dst_mm, addr, dst_pmd, pmd);

	ret = 0;
out_unlock:
954 955
	spin_unlock(src_ptl);
	spin_unlock(dst_ptl);
956 957 958 959
out:
	return ret;
}

960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static void touch_pud(struct vm_area_struct *vma, unsigned long addr,
		pud_t *pud)
{
	pud_t _pud;

	/*
	 * We should set the dirty bit only for FOLL_WRITE but for now
	 * the dirty bit in the pud is meaningless.  And if the dirty
	 * bit will become meaningful and we'll only set it with
	 * FOLL_WRITE, an atomic set_bit will be required on the pud to
	 * set the young bit, instead of the current set_pud_at.
	 */
	_pud = pud_mkyoung(pud_mkdirty(*pud));
	if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
				pud, _pud,  1))
		update_mmu_cache_pud(vma, addr, pud);
}

struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
		pud_t *pud, int flags)
{
	unsigned long pfn = pud_pfn(*pud);
	struct mm_struct *mm = vma->vm_mm;
	struct dev_pagemap *pgmap;
	struct page *page;

	assert_spin_locked(pud_lockptr(mm, pud));

	if (flags & FOLL_WRITE && !pud_write(*pud))
		return NULL;

	if (pud_present(*pud) && pud_devmap(*pud))
		/* pass */;
	else
		return NULL;

	if (flags & FOLL_TOUCH)
		touch_pud(vma, addr, pud);

	/*
	 * device mapped pages can only be returned if the
	 * caller will manage the page reference count.
	 */
	if (!(flags & FOLL_GET))
		return ERR_PTR(-EEXIST);

	pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
	pgmap = get_dev_pagemap(pfn, NULL);
	if (!pgmap)
		return ERR_PTR(-EFAULT);
	page = pfn_to_page(pfn);
	get_page(page);
	put_dev_pagemap(pgmap);

	return page;
}

int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
		  pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
		  struct vm_area_struct *vma)
{
	spinlock_t *dst_ptl, *src_ptl;
	pud_t pud;
	int ret;

	dst_ptl = pud_lock(dst_mm, dst_pud);
	src_ptl = pud_lockptr(src_mm, src_pud);
	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);

	ret = -EAGAIN;
	pud = *src_pud;
	if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud)))
		goto out_unlock;

	/*
	 * When page table lock is held, the huge zero pud should not be
	 * under splitting since we don't split the page itself, only pud to
	 * a page table.
	 */
	if (is_huge_zero_pud(pud)) {
		/* No huge zero pud yet */
	}

	pudp_set_wrprotect(src_mm, addr, src_pud);
	pud = pud_mkold(pud_wrprotect(pud));
	set_pud_at(dst_mm, addr, dst_pud, pud);

	ret = 0;
out_unlock:
	spin_unlock(src_ptl);
	spin_unlock(dst_ptl);
	return ret;
}

void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
{
	pud_t entry;
	unsigned long haddr;
	bool write = vmf->flags & FAULT_FLAG_WRITE;

	vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);
	if (unlikely(!pud_same(*vmf->pud, orig_pud)))
		goto unlock;

	entry = pud_mkyoung(orig_pud);
	if (write)
		entry = pud_mkdirty(entry);
	haddr = vmf->address & HPAGE_PUD_MASK;
	if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write))
		update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud);

unlock:
	spin_unlock(vmf->ptl);
}
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

J
Jan Kara 已提交
1077
void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd)
1078 1079 1080
{
	pmd_t entry;
	unsigned long haddr;
1081
	bool write = vmf->flags & FAULT_FLAG_WRITE;
1082

J
Jan Kara 已提交
1083 1084
	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1085 1086 1087
		goto unlock;

	entry = pmd_mkyoung(orig_pmd);
1088 1089
	if (write)
		entry = pmd_mkdirty(entry);
J
Jan Kara 已提交
1090
	haddr = vmf->address & HPAGE_PMD_MASK;
1091
	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))
J
Jan Kara 已提交
1092
		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);
1093 1094

unlock:
J
Jan Kara 已提交
1095
	spin_unlock(vmf->ptl);
1096 1097
}

J
Jan Kara 已提交
1098
static int do_huge_pmd_wp_page_fallback(struct vm_fault *vmf, pmd_t orig_pmd,
K
Kirill A. Shutemov 已提交
1099
		struct page *page)
1100
{
J
Jan Kara 已提交
1101 1102
	struct vm_area_struct *vma = vmf->vma;
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1103
	struct mem_cgroup *memcg;
1104 1105 1106 1107
	pgtable_t pgtable;
	pmd_t _pmd;
	int ret = 0, i;
	struct page **pages;
1108 1109
	unsigned long mmun_start;	/* For mmu_notifiers */
	unsigned long mmun_end;		/* For mmu_notifiers */
1110 1111 1112 1113 1114 1115 1116 1117 1118

	pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
			GFP_KERNEL);
	if (unlikely(!pages)) {
		ret |= VM_FAULT_OOM;
		goto out;
	}

	for (i = 0; i < HPAGE_PMD_NR; i++) {
M
Michal Hocko 已提交
1119
		pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE, vma,
J
Jan Kara 已提交
1120
					       vmf->address, page_to_nid(page));
A
Andrea Arcangeli 已提交
1121
		if (unlikely(!pages[i] ||
K
Kirill A. Shutemov 已提交
1122 1123
			     mem_cgroup_try_charge(pages[i], vma->vm_mm,
				     GFP_KERNEL, &memcg, false))) {
A
Andrea Arcangeli 已提交
1124
			if (pages[i])
1125
				put_page(pages[i]);
A
Andrea Arcangeli 已提交
1126
			while (--i >= 0) {
1127 1128
				memcg = (void *)page_private(pages[i]);
				set_page_private(pages[i], 0);
1129 1130
				mem_cgroup_cancel_charge(pages[i], memcg,
						false);
A
Andrea Arcangeli 已提交
1131 1132
				put_page(pages[i]);
			}
1133 1134 1135 1136
			kfree(pages);
			ret |= VM_FAULT_OOM;
			goto out;
		}
1137
		set_page_private(pages[i], (unsigned long)memcg);
1138 1139 1140 1141
	}

	for (i = 0; i < HPAGE_PMD_NR; i++) {
		copy_user_highpage(pages[i], page + i,
1142
				   haddr + PAGE_SIZE * i, vma);
1143 1144 1145 1146
		__SetPageUptodate(pages[i]);
		cond_resched();
	}

1147 1148
	mmun_start = haddr;
	mmun_end   = haddr + HPAGE_PMD_SIZE;
K
Kirill A. Shutemov 已提交
1149
	mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
1150

J
Jan Kara 已提交
1151 1152
	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1153
		goto out_free_pages;
1154
	VM_BUG_ON_PAGE(!PageHead(page), page);
1155

J
Jan Kara 已提交
1156
	pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
1157 1158
	/* leave pmd empty until pte is filled */

J
Jan Kara 已提交
1159
	pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);
K
Kirill A. Shutemov 已提交
1160
	pmd_populate(vma->vm_mm, &_pmd, pgtable);
1161 1162

	for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
K
Kirill A. Shutemov 已提交
1163
		pte_t entry;
1164 1165
		entry = mk_pte(pages[i], vma->vm_page_prot);
		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1166 1167
		memcg = (void *)page_private(pages[i]);
		set_page_private(pages[i], 0);
J
Jan Kara 已提交
1168
		page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false);
1169
		mem_cgroup_commit_charge(pages[i], memcg, false, false);
1170
		lru_cache_add_active_or_unevictable(pages[i], vma);
J
Jan Kara 已提交
1171 1172 1173 1174
		vmf->pte = pte_offset_map(&_pmd, haddr);
		VM_BUG_ON(!pte_none(*vmf->pte));
		set_pte_at(vma->vm_mm, haddr, vmf->pte, entry);
		pte_unmap(vmf->pte);
1175 1176 1177 1178
	}
	kfree(pages);

	smp_wmb(); /* make pte visible before pmd */
J
Jan Kara 已提交
1179
	pmd_populate(vma->vm_mm, vmf->pmd, pgtable);
1180
	page_remove_rmap(page, true);
J
Jan Kara 已提交
1181
	spin_unlock(vmf->ptl);
1182

K
Kirill A. Shutemov 已提交
1183
	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
1184

1185 1186 1187 1188 1189 1190 1191
	ret |= VM_FAULT_WRITE;
	put_page(page);

out:
	return ret;

out_free_pages:
J
Jan Kara 已提交
1192
	spin_unlock(vmf->ptl);
K
Kirill A. Shutemov 已提交
1193
	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
A
Andrea Arcangeli 已提交
1194
	for (i = 0; i < HPAGE_PMD_NR; i++) {
1195 1196
		memcg = (void *)page_private(pages[i]);
		set_page_private(pages[i], 0);
1197
		mem_cgroup_cancel_charge(pages[i], memcg, false);
1198
		put_page(pages[i]);
A
Andrea Arcangeli 已提交
1199
	}
1200 1201 1202 1203
	kfree(pages);
	goto out;
}

J
Jan Kara 已提交
1204
int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
1205
{
J
Jan Kara 已提交
1206
	struct vm_area_struct *vma = vmf->vma;
1207
	struct page *page = NULL, *new_page;
1208
	struct mem_cgroup *memcg;
J
Jan Kara 已提交
1209
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1210 1211
	unsigned long mmun_start;	/* For mmu_notifiers */
	unsigned long mmun_end;		/* For mmu_notifiers */
1212
	gfp_t huge_gfp;			/* for allocation and charge */
K
Kirill A. Shutemov 已提交
1213
	int ret = 0;
1214

J
Jan Kara 已提交
1215
	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
1216
	VM_BUG_ON_VMA(!vma->anon_vma, vma);
1217 1218
	if (is_huge_zero_pmd(orig_pmd))
		goto alloc;
J
Jan Kara 已提交
1219 1220
	spin_lock(vmf->ptl);
	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1221 1222 1223
		goto out_unlock;

	page = pmd_page(orig_pmd);
1224
	VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
1225 1226
	/*
	 * We can only reuse the page if nobody else maps the huge page or it's
1227
	 * part.
1228
	 */
1229
	if (page_trans_huge_mapcount(page, NULL) == 1) {
1230 1231 1232
		pmd_t entry;
		entry = pmd_mkyoung(orig_pmd);
		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
J
Jan Kara 已提交
1233 1234
		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))
			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1235 1236 1237
		ret |= VM_FAULT_WRITE;
		goto out_unlock;
	}
1238
	get_page(page);
J
Jan Kara 已提交
1239
	spin_unlock(vmf->ptl);
1240
alloc:
1241
	if (transparent_hugepage_enabled(vma) &&
1242
	    !transparent_hugepage_debug_cow()) {
1243
		huge_gfp = alloc_hugepage_direct_gfpmask(vma);
1244
		new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1245
	} else
1246 1247
		new_page = NULL;

1248 1249 1250
	if (likely(new_page)) {
		prep_transhuge_page(new_page);
	} else {
1251
		if (!page) {
J
Jan Kara 已提交
1252
			split_huge_pmd(vma, vmf->pmd, vmf->address);
1253
			ret |= VM_FAULT_FALLBACK;
1254
		} else {
J
Jan Kara 已提交
1255
			ret = do_huge_pmd_wp_page_fallback(vmf, orig_pmd, page);
1256
			if (ret & VM_FAULT_OOM) {
J
Jan Kara 已提交
1257
				split_huge_pmd(vma, vmf->pmd, vmf->address);
1258 1259
				ret |= VM_FAULT_FALLBACK;
			}
1260
			put_page(page);
1261
		}
1262
		count_vm_event(THP_FAULT_FALLBACK);
1263 1264 1265
		goto out;
	}

K
Kirill A. Shutemov 已提交
1266 1267
	if (unlikely(mem_cgroup_try_charge(new_page, vma->vm_mm,
					huge_gfp, &memcg, true))) {
A
Andrea Arcangeli 已提交
1268
		put_page(new_page);
J
Jan Kara 已提交
1269
		split_huge_pmd(vma, vmf->pmd, vmf->address);
K
Kirill A. Shutemov 已提交
1270
		if (page)
1271
			put_page(page);
1272
		ret |= VM_FAULT_FALLBACK;
1273
		count_vm_event(THP_FAULT_FALLBACK);
A
Andrea Arcangeli 已提交
1274 1275 1276
		goto out;
	}

1277 1278
	count_vm_event(THP_FAULT_ALLOC);

1279
	if (!page)
1280 1281 1282
		clear_huge_page(new_page, haddr, HPAGE_PMD_NR);
	else
		copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
1283 1284
	__SetPageUptodate(new_page);

1285 1286
	mmun_start = haddr;
	mmun_end   = haddr + HPAGE_PMD_SIZE;
K
Kirill A. Shutemov 已提交
1287
	mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
1288

J
Jan Kara 已提交
1289
	spin_lock(vmf->ptl);
1290
	if (page)
1291
		put_page(page);
J
Jan Kara 已提交
1292 1293
	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
		spin_unlock(vmf->ptl);
1294
		mem_cgroup_cancel_charge(new_page, memcg, true);
1295
		put_page(new_page);
1296
		goto out_mn;
A
Andrea Arcangeli 已提交
1297
	} else {
1298
		pmd_t entry;
1299 1300
		entry = mk_huge_pmd(new_page, vma->vm_page_prot);
		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
J
Jan Kara 已提交
1301
		pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
1302
		page_add_new_anon_rmap(new_page, vma, haddr, true);
1303
		mem_cgroup_commit_charge(new_page, memcg, false, true);
1304
		lru_cache_add_active_or_unevictable(new_page, vma);
J
Jan Kara 已提交
1305 1306
		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1307
		if (!page) {
K
Kirill A. Shutemov 已提交
1308
			add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1309
		} else {
1310
			VM_BUG_ON_PAGE(!PageHead(page), page);
1311
			page_remove_rmap(page, true);
1312 1313
			put_page(page);
		}
1314 1315
		ret |= VM_FAULT_WRITE;
	}
J
Jan Kara 已提交
1316
	spin_unlock(vmf->ptl);
1317
out_mn:
K
Kirill A. Shutemov 已提交
1318
	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
1319 1320
out:
	return ret;
1321
out_unlock:
J
Jan Kara 已提交
1322
	spin_unlock(vmf->ptl);
1323
	return ret;
1324 1325
}

1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
/*
 * FOLL_FORCE can write to even unwritable pmd's, but only
 * after we've gone through a COW cycle and they are dirty.
 */
static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)
{
	return pmd_write(pmd) ||
	       ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));
}

1336
struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1337 1338 1339 1340
				   unsigned long addr,
				   pmd_t *pmd,
				   unsigned int flags)
{
1341
	struct mm_struct *mm = vma->vm_mm;
1342 1343
	struct page *page = NULL;

1344
	assert_spin_locked(pmd_lockptr(mm, pmd));
1345

1346
	if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))
1347 1348
		goto out;

1349 1350 1351 1352
	/* Avoid dumping huge zero page */
	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
		return ERR_PTR(-EFAULT);

1353
	/* Full NUMA hinting faults to serialise migration in fault paths */
1354
	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
1355 1356
		goto out;

1357
	page = pmd_page(*pmd);
1358
	VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
1359 1360
	if (flags & FOLL_TOUCH)
		touch_pmd(vma, addr, pmd);
E
Eric B Munson 已提交
1361
	if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
1362 1363 1364 1365
		/*
		 * We don't mlock() pte-mapped THPs. This way we can avoid
		 * leaking mlocked pages into non-VM_LOCKED VMAs.
		 *
1366 1367
		 * For anon THP:
		 *
1368 1369 1370 1371 1372 1373 1374
		 * In most cases the pmd is the only mapping of the page as we
		 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
		 * writable private mappings in populate_vma_page_range().
		 *
		 * The only scenario when we have the page shared here is if we
		 * mlocking read-only mapping shared over fork(). We skip
		 * mlocking such pages.
1375 1376 1377 1378 1379 1380
		 *
		 * For file THP:
		 *
		 * We can expect PageDoubleMap() to be stable under page lock:
		 * for file pages we set it in page_add_file_rmap(), which
		 * requires page to be locked.
1381
		 */
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392

		if (PageAnon(page) && compound_mapcount(page) != 1)
			goto skip_mlock;
		if (PageDoubleMap(page) || !page->mapping)
			goto skip_mlock;
		if (!trylock_page(page))
			goto skip_mlock;
		lru_add_drain();
		if (page->mapping && !PageDoubleMap(page))
			mlock_vma_page(page);
		unlock_page(page);
1393
	}
1394
skip_mlock:
1395
	page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1396
	VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
1397
	if (flags & FOLL_GET)
1398
		get_page(page);
1399 1400 1401 1402 1403

out:
	return page;
}

1404
/* NUMA hinting page fault entry point for trans huge pmds */
J
Jan Kara 已提交
1405
int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
1406
{
J
Jan Kara 已提交
1407
	struct vm_area_struct *vma = vmf->vma;
1408
	struct anon_vma *anon_vma = NULL;
1409
	struct page *page;
J
Jan Kara 已提交
1410
	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1411
	int page_nid = -1, this_nid = numa_node_id();
1412
	int target_nid, last_cpupid = -1;
1413 1414
	bool page_locked;
	bool migrated = false;
1415
	bool was_writable;
1416
	int flags = 0;
1417

J
Jan Kara 已提交
1418 1419
	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
	if (unlikely(!pmd_same(pmd, *vmf->pmd)))
1420 1421
		goto out_unlock;

1422 1423 1424 1425 1426
	/*
	 * If there are potential migrations, wait for completion and retry
	 * without disrupting NUMA hinting information. Do not relock and
	 * check_same as the page may no longer be mapped.
	 */
J
Jan Kara 已提交
1427 1428
	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {
		page = pmd_page(*vmf->pmd);
1429 1430
		if (!get_page_unless_zero(page))
			goto out_unlock;
J
Jan Kara 已提交
1431
		spin_unlock(vmf->ptl);
1432
		wait_on_page_locked(page);
1433
		put_page(page);
1434 1435 1436
		goto out;
	}

1437
	page = pmd_page(pmd);
1438
	BUG_ON(is_huge_zero_page(page));
1439
	page_nid = page_to_nid(page);
1440
	last_cpupid = page_cpupid_last(page);
1441
	count_vm_numa_event(NUMA_HINT_FAULTS);
1442
	if (page_nid == this_nid) {
1443
		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
1444 1445
		flags |= TNF_FAULT_LOCAL;
	}
1446

1447
	/* See similar comment in do_numa_page for explanation */
1448
	if (!pmd_savedwrite(pmd))
1449 1450
		flags |= TNF_NO_GROUP;

1451 1452 1453 1454
	/*
	 * Acquire the page lock to serialise THP migrations but avoid dropping
	 * page_table_lock if at all possible
	 */
1455 1456 1457 1458
	page_locked = trylock_page(page);
	target_nid = mpol_misplaced(page, vma, haddr);
	if (target_nid == -1) {
		/* If the page was locked, there are no parallel migrations */
1459
		if (page_locked)
1460
			goto clear_pmdnuma;
1461
	}
1462

1463
	/* Migration could have started since the pmd_trans_migrating check */
1464
	if (!page_locked) {
1465 1466 1467
		page_nid = -1;
		if (!get_page_unless_zero(page))
			goto out_unlock;
J
Jan Kara 已提交
1468
		spin_unlock(vmf->ptl);
1469
		wait_on_page_locked(page);
1470
		put_page(page);
1471 1472 1473
		goto out;
	}

1474 1475 1476 1477
	/*
	 * Page is misplaced. Page lock serialises migrations. Acquire anon_vma
	 * to serialises splits
	 */
1478
	get_page(page);
J
Jan Kara 已提交
1479
	spin_unlock(vmf->ptl);
1480
	anon_vma = page_lock_anon_vma_read(page);
1481

P
Peter Zijlstra 已提交
1482
	/* Confirm the PMD did not change while page_table_lock was released */
J
Jan Kara 已提交
1483 1484
	spin_lock(vmf->ptl);
	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {
1485 1486
		unlock_page(page);
		put_page(page);
1487
		page_nid = -1;
1488
		goto out_unlock;
1489
	}
1490

1491 1492 1493 1494 1495 1496 1497
	/* Bail if we fail to protect against THP splits for any reason */
	if (unlikely(!anon_vma)) {
		put_page(page);
		page_nid = -1;
		goto clear_pmdnuma;
	}

1498 1499
	/*
	 * Migrate the THP to the requested node, returns with page unlocked
1500
	 * and access rights restored.
1501
	 */
J
Jan Kara 已提交
1502
	spin_unlock(vmf->ptl);
K
Kirill A. Shutemov 已提交
1503
	migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma,
J
Jan Kara 已提交
1504
				vmf->pmd, pmd, vmf->address, page, target_nid);
1505 1506
	if (migrated) {
		flags |= TNF_MIGRATED;
1507
		page_nid = target_nid;
1508 1509
	} else
		flags |= TNF_MIGRATE_FAIL;
1510

1511
	goto out;
1512
clear_pmdnuma:
1513
	BUG_ON(!PageLocked(page));
1514
	was_writable = pmd_savedwrite(pmd);
1515
	pmd = pmd_modify(pmd, vma->vm_page_prot);
1516
	pmd = pmd_mkyoung(pmd);
1517 1518
	if (was_writable)
		pmd = pmd_mkwrite(pmd);
J
Jan Kara 已提交
1519 1520
	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1521
	unlock_page(page);
1522
out_unlock:
J
Jan Kara 已提交
1523
	spin_unlock(vmf->ptl);
1524 1525 1526 1527 1528

out:
	if (anon_vma)
		page_unlock_anon_vma_read(anon_vma);

1529
	if (page_nid != -1)
J
Jan Kara 已提交
1530
		task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR,
1531
				flags);
1532

1533 1534 1535
	return 0;
}

1536 1537 1538 1539 1540
/*
 * Return true if we do MADV_FREE successfully on entire pmd page.
 * Otherwise, return false.
 */
bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1541 1542 1543 1544 1545 1546
		pmd_t *pmd, unsigned long addr, unsigned long next)
{
	spinlock_t *ptl;
	pmd_t orig_pmd;
	struct page *page;
	struct mm_struct *mm = tlb->mm;
1547
	bool ret = false;
1548

1549 1550
	tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);

1551 1552
	ptl = pmd_trans_huge_lock(pmd, vma);
	if (!ptl)
1553
		goto out_unlocked;
1554 1555

	orig_pmd = *pmd;
1556
	if (is_huge_zero_pmd(orig_pmd))
1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576
		goto out;

	page = pmd_page(orig_pmd);
	/*
	 * If other processes are mapping this page, we couldn't discard
	 * the page unless they all do MADV_FREE so let's skip the page.
	 */
	if (page_mapcount(page) != 1)
		goto out;

	if (!trylock_page(page))
		goto out;

	/*
	 * If user want to discard part-pages of THP, split it so MADV_FREE
	 * will deactivate only them.
	 */
	if (next - addr != HPAGE_PMD_SIZE) {
		get_page(page);
		spin_unlock(ptl);
1577
		split_huge_page(page);
1578
		unlock_page(page);
1579
		put_page(page);
1580 1581 1582 1583 1584 1585 1586 1587
		goto out_unlocked;
	}

	if (PageDirty(page))
		ClearPageDirty(page);
	unlock_page(page);

	if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1588
		pmdp_invalidate(vma, addr, pmd);
1589 1590 1591 1592 1593 1594
		orig_pmd = pmd_mkold(orig_pmd);
		orig_pmd = pmd_mkclean(orig_pmd);

		set_pmd_at(mm, addr, pmd, orig_pmd);
		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
	}
S
Shaohua Li 已提交
1595 1596

	mark_page_lazyfree(page);
1597
	ret = true;
1598 1599 1600 1601 1602 1603
out:
	spin_unlock(ptl);
out_unlocked:
	return ret;
}

1604 1605 1606 1607 1608 1609 1610 1611 1612
static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
{
	pgtable_t pgtable;

	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
	pte_free(mm, pgtable);
	atomic_long_dec(&mm->nr_ptes);
}

1613
int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
S
Shaohua Li 已提交
1614
		 pmd_t *pmd, unsigned long addr)
1615
{
1616
	pmd_t orig_pmd;
1617
	spinlock_t *ptl;
1618

1619 1620
	tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);

1621 1622
	ptl = __pmd_trans_huge_lock(pmd, vma);
	if (!ptl)
1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633
		return 0;
	/*
	 * For architectures like ppc64 we look at deposited pgtable
	 * when calling pmdp_huge_get_and_clear. So do the
	 * pgtable_trans_huge_withdraw after finishing pmdp related
	 * operations.
	 */
	orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
			tlb->fullmm);
	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
	if (vma_is_dax(vma)) {
1634 1635
		if (arch_needs_pgtable_deposit())
			zap_deposited_table(tlb->mm, pmd);
1636 1637
		spin_unlock(ptl);
		if (is_huge_zero_pmd(orig_pmd))
1638
			tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE);
1639
	} else if (is_huge_zero_pmd(orig_pmd)) {
1640
		zap_deposited_table(tlb->mm, pmd);
1641
		spin_unlock(ptl);
1642
		tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE);
1643 1644
	} else {
		struct page *page = pmd_page(orig_pmd);
1645
		page_remove_rmap(page, true);
1646 1647
		VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
		VM_BUG_ON_PAGE(!PageHead(page), page);
1648
		if (PageAnon(page)) {
1649
			zap_deposited_table(tlb->mm, pmd);
1650 1651
			add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
		} else {
1652 1653
			if (arch_needs_pgtable_deposit())
				zap_deposited_table(tlb->mm, pmd);
1654 1655
			add_mm_counter(tlb->mm, MM_FILEPAGES, -HPAGE_PMD_NR);
		}
1656
		spin_unlock(ptl);
1657
		tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
1658
	}
1659
	return 1;
1660 1661
}

1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676
#ifndef pmd_move_must_withdraw
static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
					 spinlock_t *old_pmd_ptl,
					 struct vm_area_struct *vma)
{
	/*
	 * With split pmd lock we also need to move preallocated
	 * PTE page table if new_pmd is on different PMD page table.
	 *
	 * We also don't deposit and withdraw tables for file pages.
	 */
	return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);
}
#endif

1677
bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
1678
		  unsigned long new_addr, unsigned long old_end,
1679
		  pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush)
1680
{
1681
	spinlock_t *old_ptl, *new_ptl;
1682 1683
	pmd_t pmd;
	struct mm_struct *mm = vma->vm_mm;
1684
	bool force_flush = false;
1685 1686 1687

	if ((old_addr & ~HPAGE_PMD_MASK) ||
	    (new_addr & ~HPAGE_PMD_MASK) ||
1688
	    old_end - old_addr < HPAGE_PMD_SIZE)
1689
		return false;
1690 1691 1692 1693 1694 1695 1696

	/*
	 * The destination pmd shouldn't be established, free_pgtables()
	 * should have release it.
	 */
	if (WARN_ON(!pmd_none(*new_pmd))) {
		VM_BUG_ON(pmd_trans_huge(*new_pmd));
1697
		return false;
1698 1699
	}

1700 1701 1702 1703
	/*
	 * We don't have to worry about the ordering of src and dst
	 * ptlocks because exclusive mmap_sem prevents deadlock.
	 */
1704 1705
	old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
	if (old_ptl) {
1706 1707 1708
		new_ptl = pmd_lockptr(mm, new_pmd);
		if (new_ptl != old_ptl)
			spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
1709
		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
1710 1711
		if (pmd_present(pmd) && pmd_dirty(pmd))
			force_flush = true;
1712
		VM_BUG_ON(!pmd_none(*new_pmd));
1713

1714
		if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
1715
			pgtable_t pgtable;
1716 1717 1718
			pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
			pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
		}
1719 1720 1721
		set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));
		if (new_ptl != old_ptl)
			spin_unlock(new_ptl);
1722 1723 1724 1725
		if (force_flush)
			flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
		else
			*need_flush = true;
1726
		spin_unlock(old_ptl);
1727
		return true;
1728
	}
1729
	return false;
1730 1731
}

1732 1733 1734 1735 1736 1737
/*
 * Returns
 *  - 0 if PMD could not be locked
 *  - 1 if PMD was locked but protections unchange and TLB flush unnecessary
 *  - HPAGE_PMD_NR is protections changed and TLB flush necessary
 */
1738
int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
1739
		unsigned long addr, pgprot_t newprot, int prot_numa)
1740 1741
{
	struct mm_struct *mm = vma->vm_mm;
1742
	spinlock_t *ptl;
1743 1744 1745
	pmd_t entry;
	bool preserve_write;
	int ret;
1746

1747
	ptl = __pmd_trans_huge_lock(pmd, vma);
1748 1749
	if (!ptl)
		return 0;
1750

1751 1752
	preserve_write = prot_numa && pmd_write(*pmd);
	ret = 1;
1753

1754 1755 1756 1757 1758 1759 1760
	/*
	 * Avoid trapping faults against the zero page. The read-only
	 * data is likely to be read-cached on the local CPU and
	 * local/remote hits to the zero page are not interesting.
	 */
	if (prot_numa && is_huge_zero_pmd(*pmd))
		goto unlock;
1761

1762 1763 1764
	if (prot_numa && pmd_protnone(*pmd))
		goto unlock;

1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
	/*
	 * In case prot_numa, we are under down_read(mmap_sem). It's critical
	 * to not clear pmd intermittently to avoid race with MADV_DONTNEED
	 * which is also under down_read(mmap_sem):
	 *
	 *	CPU0:				CPU1:
	 *				change_huge_pmd(prot_numa=1)
	 *				 pmdp_huge_get_and_clear_notify()
	 * madvise_dontneed()
	 *  zap_pmd_range()
	 *   pmd_trans_huge(*pmd) == 0 (without ptl)
	 *   // skip the pmd
	 *				 set_pmd_at();
	 *				 // pmd is re-established
	 *
	 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it
	 * which may break userspace.
	 *
	 * pmdp_invalidate() is required to make sure we don't miss
	 * dirty/young flags set by hardware.
	 */
	entry = *pmd;
	pmdp_invalidate(vma, addr, pmd);

	/*
	 * Recover dirty/young flags.  It relies on pmdp_invalidate to not
	 * corrupt them.
	 */
	if (pmd_dirty(*pmd))
		entry = pmd_mkdirty(entry);
	if (pmd_young(*pmd))
		entry = pmd_mkyoung(entry);

1798 1799 1800 1801 1802 1803 1804 1805
	entry = pmd_modify(entry, newprot);
	if (preserve_write)
		entry = pmd_mk_savedwrite(entry);
	ret = HPAGE_PMD_NR;
	set_pmd_at(mm, addr, pmd, entry);
	BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry));
unlock:
	spin_unlock(ptl);
1806 1807 1808 1809
	return ret;
}

/*
1810
 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
1811
 *
1812 1813
 * Note that if it returns page table lock pointer, this routine returns without
 * unlocking page table lock. So callers must unlock it.
1814
 */
1815
spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
1816
{
1817 1818
	spinlock_t *ptl;
	ptl = pmd_lock(vma->vm_mm, pmd);
1819
	if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd)))
1820 1821 1822
		return ptl;
	spin_unlock(ptl);
	return NULL;
1823 1824
}

1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
/*
 * Returns true if a given pud maps a thp, false otherwise.
 *
 * Note that if it returns true, this routine returns without unlocking page
 * table lock. So callers must unlock it.
 */
spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
{
	spinlock_t *ptl;

	ptl = pud_lock(vma->vm_mm, pud);
	if (likely(pud_trans_huge(*pud) || pud_devmap(*pud)))
		return ptl;
	spin_unlock(ptl);
	return NULL;
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
		 pud_t *pud, unsigned long addr)
{
	pud_t orig_pud;
	spinlock_t *ptl;

	ptl = __pud_trans_huge_lock(pud, vma);
	if (!ptl)
		return 0;
	/*
	 * For architectures like ppc64 we look at deposited pgtable
	 * when calling pudp_huge_get_and_clear. So do the
	 * pgtable_trans_huge_withdraw after finishing pudp related
	 * operations.
	 */
	orig_pud = pudp_huge_get_and_clear_full(tlb->mm, addr, pud,
			tlb->fullmm);
	tlb_remove_pud_tlb_entry(tlb, pud, addr);
	if (vma_is_dax(vma)) {
		spin_unlock(ptl);
		/* No zero page support yet */
	} else {
		/* No support for anonymous PUD pages yet */
		BUG();
	}
	return 1;
}

static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
		unsigned long haddr)
{
	VM_BUG_ON(haddr & ~HPAGE_PUD_MASK);
	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
	VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud));

1879
	count_vm_event(THP_SPLIT_PUD);
1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902

	pudp_huge_clear_flush_notify(vma, haddr, pud);
}

void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
		unsigned long address)
{
	spinlock_t *ptl;
	struct mm_struct *mm = vma->vm_mm;
	unsigned long haddr = address & HPAGE_PUD_MASK;

	mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PUD_SIZE);
	ptl = pud_lock(mm, pud);
	if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud)))
		goto out;
	__split_huge_pud_locked(vma, pud, haddr);

out:
	spin_unlock(ptl);
	mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PUD_SIZE);
}
#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
		unsigned long haddr, pmd_t *pmd)
{
	struct mm_struct *mm = vma->vm_mm;
	pgtable_t pgtable;
	pmd_t _pmd;
	int i;

	/* leave pmd empty until pte is filled */
	pmdp_huge_clear_flush_notify(vma, haddr, pmd);

	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
	pmd_populate(mm, &_pmd, pgtable);

	for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
		pte_t *pte, entry;
		entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
		entry = pte_mkspecial(entry);
		pte = pte_offset_map(&_pmd, haddr);
		VM_BUG_ON(!pte_none(*pte));
		set_pte_at(mm, haddr, pte, entry);
		pte_unmap(pte);
	}
	smp_wmb(); /* make pte visible before pmd */
	pmd_populate(mm, pmd, pgtable);
}

static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
1931
		unsigned long haddr, bool freeze)
1932 1933 1934 1935 1936
{
	struct mm_struct *mm = vma->vm_mm;
	struct page *page;
	pgtable_t pgtable;
	pmd_t _pmd;
1937
	bool young, write, dirty, soft_dirty;
1938
	unsigned long addr;
1939 1940 1941 1942 1943
	int i;

	VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
1944
	VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd));
1945 1946 1947

	count_vm_event(THP_SPLIT_PMD);

1948 1949
	if (!vma_is_anonymous(vma)) {
		_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
1950 1951 1952 1953 1954 1955
		/*
		 * We are going to unmap this huge page. So
		 * just go ahead and zap it
		 */
		if (arch_needs_pgtable_deposit())
			zap_deposited_table(mm, pmd);
1956 1957 1958 1959 1960 1961 1962 1963
		if (vma_is_dax(vma))
			return;
		page = pmd_page(_pmd);
		if (!PageReferenced(page) && pmd_young(_pmd))
			SetPageReferenced(page);
		page_remove_rmap(page, true);
		put_page(page);
		add_mm_counter(mm, MM_FILEPAGES, -HPAGE_PMD_NR);
1964 1965 1966 1967 1968 1969 1970
		return;
	} else if (is_huge_zero_pmd(*pmd)) {
		return __split_huge_zero_page_pmd(vma, haddr, pmd);
	}

	page = pmd_page(*pmd);
	VM_BUG_ON_PAGE(!page_count(page), page);
1971
	page_ref_add(page, HPAGE_PMD_NR - 1);
1972 1973
	write = pmd_write(*pmd);
	young = pmd_young(*pmd);
1974
	dirty = pmd_dirty(*pmd);
1975
	soft_dirty = pmd_soft_dirty(*pmd);
1976

1977
	pmdp_huge_split_prepare(vma, haddr, pmd);
1978 1979 1980
	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
	pmd_populate(mm, &_pmd, pgtable);

1981
	for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
1982 1983 1984 1985 1986 1987
		pte_t entry, *pte;
		/*
		 * Note that NUMA hinting access restrictions are not
		 * transferred to avoid any possibility of altering
		 * permissions across VMAs.
		 */
1988 1989 1990 1991
		if (freeze) {
			swp_entry_t swp_entry;
			swp_entry = make_migration_entry(page + i, write);
			entry = swp_entry_to_pte(swp_entry);
1992 1993
			if (soft_dirty)
				entry = pte_swp_mksoft_dirty(entry);
1994
		} else {
1995
			entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
1996
			entry = maybe_mkwrite(entry, vma);
1997 1998 1999 2000
			if (!write)
				entry = pte_wrprotect(entry);
			if (!young)
				entry = pte_mkold(entry);
2001 2002
			if (soft_dirty)
				entry = pte_mksoft_dirty(entry);
2003
		}
2004 2005
		if (dirty)
			SetPageDirty(page + i);
2006
		pte = pte_offset_map(&_pmd, addr);
2007
		BUG_ON(!pte_none(*pte));
2008
		set_pte_at(mm, addr, pte, entry);
2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023
		atomic_inc(&page[i]._mapcount);
		pte_unmap(pte);
	}

	/*
	 * Set PG_double_map before dropping compound_mapcount to avoid
	 * false-negative page_mapped().
	 */
	if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) {
		for (i = 0; i < HPAGE_PMD_NR; i++)
			atomic_inc(&page[i]._mapcount);
	}

	if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
		/* Last compound_mapcount is gone. */
2024
		__dec_node_page_state(page, NR_ANON_THPS);
2025 2026 2027 2028 2029 2030 2031 2032
		if (TestClearPageDoubleMap(page)) {
			/* No need in mapcount reference anymore */
			for (i = 0; i < HPAGE_PMD_NR; i++)
				atomic_dec(&page[i]._mapcount);
		}
	}

	smp_wmb(); /* make pte visible before pmd */
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054
	/*
	 * Up to this point the pmd is present and huge and userland has the
	 * whole access to the hugepage during the split (which happens in
	 * place). If we overwrite the pmd with the not-huge version pointing
	 * to the pte here (which of course we could if all CPUs were bug
	 * free), userland could trigger a small page size TLB miss on the
	 * small sized TLB while the hugepage TLB entry is still established in
	 * the huge TLB. Some CPU doesn't like that.
	 * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum
	 * 383 on page 93. Intel should be safe but is also warns that it's
	 * only safe if the permission and cache attributes of the two entries
	 * loaded in the two TLB is identical (which should be the case here).
	 * But it is generally safer to never allow small and huge TLB entries
	 * for the same virtual address to be loaded simultaneously. So instead
	 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
	 * current pmd notpresent (atomically because here the pmd_trans_huge
	 * and pmd_trans_splitting must remain set at all times on the pmd
	 * until the split is complete for this pmd), then we flush the SMP TLB
	 * and finally we write the non-huge version of the pmd entry with
	 * pmd_populate.
	 */
	pmdp_invalidate(vma, haddr, pmd);
2055
	pmd_populate(mm, pmd, pgtable);
2056 2057

	if (freeze) {
2058
		for (i = 0; i < HPAGE_PMD_NR; i++) {
2059 2060 2061 2062
			page_remove_rmap(page + i, false);
			put_page(page + i);
		}
	}
2063 2064 2065
}

void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2066
		unsigned long address, bool freeze, struct page *page)
2067 2068 2069 2070 2071 2072 2073
{
	spinlock_t *ptl;
	struct mm_struct *mm = vma->vm_mm;
	unsigned long haddr = address & HPAGE_PMD_MASK;

	mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
	ptl = pmd_lock(mm, pmd);
2074 2075 2076 2077 2078 2079 2080 2081 2082

	/*
	 * If caller asks to setup a migration entries, we need a page to check
	 * pmd against. Otherwise we can end up replacing wrong page.
	 */
	VM_BUG_ON(freeze && !page);
	if (page && page != pmd_page(*pmd))
	        goto out;

2083
	if (pmd_trans_huge(*pmd)) {
2084
		page = pmd_page(*pmd);
2085
		if (PageMlocked(page))
2086
			clear_page_mlock(page);
2087
	} else if (!pmd_devmap(*pmd))
2088
		goto out;
2089
	__split_huge_pmd_locked(vma, pmd, haddr, freeze);
2090
out:
2091 2092 2093 2094
	spin_unlock(ptl);
	mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
}

2095 2096
void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
		bool freeze, struct page *page)
2097
{
2098
	pgd_t *pgd;
2099
	p4d_t *p4d;
2100
	pud_t *pud;
2101 2102
	pmd_t *pmd;

2103
	pgd = pgd_offset(vma->vm_mm, address);
2104 2105 2106
	if (!pgd_present(*pgd))
		return;

2107 2108 2109 2110 2111
	p4d = p4d_offset(pgd, address);
	if (!p4d_present(*p4d))
		return;

	pud = pud_offset(p4d, address);
2112 2113 2114 2115
	if (!pud_present(*pud))
		return;

	pmd = pmd_offset(pud, address);
2116

2117
	__split_huge_pmd(vma, pmd, address, freeze, page);
2118 2119
}

2120
void vma_adjust_trans_huge(struct vm_area_struct *vma,
2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
			     unsigned long start,
			     unsigned long end,
			     long adjust_next)
{
	/*
	 * If the new start address isn't hpage aligned and it could
	 * previously contain an hugepage: check if we need to split
	 * an huge pmd.
	 */
	if (start & ~HPAGE_PMD_MASK &&
	    (start & HPAGE_PMD_MASK) >= vma->vm_start &&
	    (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2133
		split_huge_pmd_address(vma, start, false, NULL);
2134 2135 2136 2137 2138 2139 2140 2141 2142

	/*
	 * If the new end address isn't hpage aligned and it could
	 * previously contain an hugepage: check if we need to split
	 * an huge pmd.
	 */
	if (end & ~HPAGE_PMD_MASK &&
	    (end & HPAGE_PMD_MASK) >= vma->vm_start &&
	    (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2143
		split_huge_pmd_address(vma, end, false, NULL);
2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156

	/*
	 * If we're also updating the vma->vm_next->vm_start, if the new
	 * vm_next->vm_start isn't page aligned and it could previously
	 * contain an hugepage: check if we need to split an huge pmd.
	 */
	if (adjust_next > 0) {
		struct vm_area_struct *next = vma->vm_next;
		unsigned long nstart = next->vm_start;
		nstart += adjust_next << PAGE_SHIFT;
		if (nstart & ~HPAGE_PMD_MASK &&
		    (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
		    (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
2157
			split_huge_pmd_address(next, nstart, false, NULL);
2158 2159
	}
}
2160

2161
static void freeze_page(struct page *page)
2162
{
2163
	enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS |
2164
		TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD;
M
Minchan Kim 已提交
2165
	bool unmap_success;
2166 2167 2168

	VM_BUG_ON_PAGE(!PageHead(page), page);

2169 2170 2171
	if (PageAnon(page))
		ttu_flags |= TTU_MIGRATION;

M
Minchan Kim 已提交
2172 2173
	unmap_success = try_to_unmap(page, ttu_flags);
	VM_BUG_ON_PAGE(!unmap_success, page);
2174 2175
}

2176
static void unfreeze_page(struct page *page)
2177
{
2178
	int i;
2179 2180 2181 2182 2183 2184
	if (PageTransHuge(page)) {
		remove_migration_ptes(page, page, true);
	} else {
		for (i = 0; i < HPAGE_PMD_NR; i++)
			remove_migration_ptes(page + i, page + i, true);
	}
2185 2186
}

2187
static void __split_huge_page_tail(struct page *head, int tail,
2188 2189 2190 2191
		struct lruvec *lruvec, struct list_head *list)
{
	struct page *page_tail = head + tail;

2192
	VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
2193
	VM_BUG_ON_PAGE(page_ref_count(page_tail) != 0, page_tail);
2194 2195

	/*
2196
	 * tail_page->_refcount is zero and not changing from under us. But
2197
	 * get_page_unless_zero() may be running from under us on the
2198 2199
	 * tail_page. If we used atomic_set() below instead of atomic_inc() or
	 * atomic_add(), we would then run atomic_set() concurrently with
2200 2201 2202 2203
	 * get_page_unless_zero(), and atomic_set() is implemented in C not
	 * using locked ops. spin_unlock on x86 sometime uses locked ops
	 * because of PPro errata 66, 92, so unless somebody can guarantee
	 * atomic_set() here would be safe on all archs (and not only on x86),
2204
	 * it's safer to use atomic_inc()/atomic_add().
2205
	 */
2206
	if (PageAnon(head) && !PageSwapCache(head)) {
2207 2208 2209 2210 2211
		page_ref_inc(page_tail);
	} else {
		/* Additional pin to radix tree */
		page_ref_add(page_tail, 2);
	}
2212 2213 2214 2215 2216

	page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
	page_tail->flags |= (head->flags &
			((1L << PG_referenced) |
			 (1L << PG_swapbacked) |
2217
			 (1L << PG_swapcache) |
2218 2219 2220 2221
			 (1L << PG_mlocked) |
			 (1L << PG_uptodate) |
			 (1L << PG_active) |
			 (1L << PG_locked) |
2222 2223
			 (1L << PG_unevictable) |
			 (1L << PG_dirty)));
2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238

	/*
	 * After clearing PageTail the gup refcount can be released.
	 * Page flags also must be visible before we make the page non-compound.
	 */
	smp_wmb();

	clear_compound_head(page_tail);

	if (page_is_young(head))
		set_page_young(page_tail);
	if (page_is_idle(head))
		set_page_idle(page_tail);

	/* ->mapping in first tail page is compound_mapcount */
2239
	VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
2240 2241 2242 2243 2244 2245 2246 2247
			page_tail);
	page_tail->mapping = head->mapping;

	page_tail->index = head->index + tail;
	page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
	lru_add_page_tail(head, page_tail, lruvec, list);
}

2248 2249
static void __split_huge_page(struct page *page, struct list_head *list,
		unsigned long flags)
2250 2251 2252 2253
{
	struct page *head = compound_head(page);
	struct zone *zone = page_zone(head);
	struct lruvec *lruvec;
2254
	pgoff_t end = -1;
2255
	int i;
2256

M
Mel Gorman 已提交
2257
	lruvec = mem_cgroup_page_lruvec(head, zone->zone_pgdat);
2258 2259 2260 2261

	/* complete memcg works before add pages to LRU */
	mem_cgroup_split_huge_fixup(head);

2262 2263 2264 2265
	if (!PageAnon(page))
		end = DIV_ROUND_UP(i_size_read(head->mapping->host), PAGE_SIZE);

	for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
2266
		__split_huge_page_tail(head, i, lruvec, list);
2267 2268 2269 2270
		/* Some pages can be beyond i_size: drop them from page cache */
		if (head[i].index >= end) {
			__ClearPageDirty(head + i);
			__delete_from_page_cache(head + i, NULL);
2271 2272
			if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head))
				shmem_uncharge(head->mapping->host, 1);
2273 2274 2275
			put_page(head + i);
		}
	}
2276 2277

	ClearPageCompound(head);
2278 2279
	/* See comment in __split_huge_page_tail() */
	if (PageAnon(head)) {
2280 2281 2282 2283 2284
		/* Additional pin to radix tree of swap cache */
		if (PageSwapCache(head))
			page_ref_add(head, 2);
		else
			page_ref_inc(head);
2285 2286 2287 2288 2289 2290
	} else {
		/* Additional pin to radix tree */
		page_ref_add(head, 2);
		spin_unlock(&head->mapping->tree_lock);
	}

2291
	spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2292

2293
	unfreeze_page(head);
2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311

	for (i = 0; i < HPAGE_PMD_NR; i++) {
		struct page *subpage = head + i;
		if (subpage == page)
			continue;
		unlock_page(subpage);

		/*
		 * Subpages may be freed if there wasn't any mapping
		 * like if add_to_swap() is running on a lru page that
		 * had its mapping zapped. And freeing these pages
		 * requires taking the lru_lock so we do the put_page
		 * of the tail pages after the split is complete.
		 */
		put_page(subpage);
	}
}

2312 2313
int total_mapcount(struct page *page)
{
K
Kirill A. Shutemov 已提交
2314
	int i, compound, ret;
2315 2316 2317 2318 2319 2320

	VM_BUG_ON_PAGE(PageTail(page), page);

	if (likely(!PageCompound(page)))
		return atomic_read(&page->_mapcount) + 1;

K
Kirill A. Shutemov 已提交
2321
	compound = compound_mapcount(page);
2322
	if (PageHuge(page))
K
Kirill A. Shutemov 已提交
2323 2324
		return compound;
	ret = compound;
2325 2326
	for (i = 0; i < HPAGE_PMD_NR; i++)
		ret += atomic_read(&page[i]._mapcount) + 1;
K
Kirill A. Shutemov 已提交
2327 2328 2329
	/* File pages has compound_mapcount included in _mapcount */
	if (!PageAnon(page))
		return ret - compound * HPAGE_PMD_NR;
2330 2331 2332 2333 2334
	if (PageDoubleMap(page))
		ret -= HPAGE_PMD_NR;
	return ret;
}

2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392
/*
 * This calculates accurately how many mappings a transparent hugepage
 * has (unlike page_mapcount() which isn't fully accurate). This full
 * accuracy is primarily needed to know if copy-on-write faults can
 * reuse the page and change the mapping to read-write instead of
 * copying them. At the same time this returns the total_mapcount too.
 *
 * The function returns the highest mapcount any one of the subpages
 * has. If the return value is one, even if different processes are
 * mapping different subpages of the transparent hugepage, they can
 * all reuse it, because each process is reusing a different subpage.
 *
 * The total_mapcount is instead counting all virtual mappings of the
 * subpages. If the total_mapcount is equal to "one", it tells the
 * caller all mappings belong to the same "mm" and in turn the
 * anon_vma of the transparent hugepage can become the vma->anon_vma
 * local one as no other process may be mapping any of the subpages.
 *
 * It would be more accurate to replace page_mapcount() with
 * page_trans_huge_mapcount(), however we only use
 * page_trans_huge_mapcount() in the copy-on-write faults where we
 * need full accuracy to avoid breaking page pinning, because
 * page_trans_huge_mapcount() is slower than page_mapcount().
 */
int page_trans_huge_mapcount(struct page *page, int *total_mapcount)
{
	int i, ret, _total_mapcount, mapcount;

	/* hugetlbfs shouldn't call it */
	VM_BUG_ON_PAGE(PageHuge(page), page);

	if (likely(!PageTransCompound(page))) {
		mapcount = atomic_read(&page->_mapcount) + 1;
		if (total_mapcount)
			*total_mapcount = mapcount;
		return mapcount;
	}

	page = compound_head(page);

	_total_mapcount = ret = 0;
	for (i = 0; i < HPAGE_PMD_NR; i++) {
		mapcount = atomic_read(&page[i]._mapcount) + 1;
		ret = max(ret, mapcount);
		_total_mapcount += mapcount;
	}
	if (PageDoubleMap(page)) {
		ret -= 1;
		_total_mapcount -= HPAGE_PMD_NR;
	}
	mapcount = compound_mapcount(page);
	ret += mapcount;
	_total_mapcount += mapcount;
	if (total_mapcount)
		*total_mapcount = _total_mapcount;
	return ret;
}

2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407
/* Racy check whether the huge page can be split */
bool can_split_huge_page(struct page *page, int *pextra_pins)
{
	int extra_pins;

	/* Additional pins from radix tree */
	if (PageAnon(page))
		extra_pins = PageSwapCache(page) ? HPAGE_PMD_NR : 0;
	else
		extra_pins = HPAGE_PMD_NR;
	if (pextra_pins)
		*pextra_pins = extra_pins;
	return total_mapcount(page) == page_count(page) - extra_pins - 1;
}

2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429
/*
 * This function splits huge page into normal pages. @page can point to any
 * subpage of huge page to split. Split doesn't change the position of @page.
 *
 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
 * The huge page must be locked.
 *
 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
 *
 * Both head page and tail pages will inherit mapping, flags, and so on from
 * the hugepage.
 *
 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
 * they are not mapped.
 *
 * Returns 0 if the hugepage is split successfully.
 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
 * us.
 */
int split_huge_page_to_list(struct page *page, struct list_head *list)
{
	struct page *head = compound_head(page);
2430
	struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
2431 2432 2433
	struct anon_vma *anon_vma = NULL;
	struct address_space *mapping = NULL;
	int count, mapcount, extra_pins, ret;
2434
	bool mlocked;
2435
	unsigned long flags;
2436 2437 2438 2439 2440

	VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(!PageCompound(page), page);

2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467
	if (PageAnon(head)) {
		/*
		 * The caller does not necessarily hold an mmap_sem that would
		 * prevent the anon_vma disappearing so we first we take a
		 * reference to it and then lock the anon_vma for write. This
		 * is similar to page_lock_anon_vma_read except the write lock
		 * is taken to serialise against parallel split or collapse
		 * operations.
		 */
		anon_vma = page_get_anon_vma(head);
		if (!anon_vma) {
			ret = -EBUSY;
			goto out;
		}
		mapping = NULL;
		anon_vma_lock_write(anon_vma);
	} else {
		mapping = head->mapping;

		/* Truncated ? */
		if (!mapping) {
			ret = -EBUSY;
			goto out;
		}

		anon_vma = NULL;
		i_mmap_lock_read(mapping);
2468 2469 2470 2471 2472 2473
	}

	/*
	 * Racy check if we can split the page, before freeze_page() will
	 * split PMDs
	 */
2474
	if (!can_split_huge_page(head, &extra_pins)) {
2475 2476 2477 2478
		ret = -EBUSY;
		goto out_unlock;
	}

2479
	mlocked = PageMlocked(page);
2480
	freeze_page(head);
2481 2482
	VM_BUG_ON_PAGE(compound_mapcount(head), head);

2483 2484 2485 2486
	/* Make sure the page is not on per-CPU pagevec as it takes pin */
	if (mlocked)
		lru_add_drain();

2487
	/* prevent PageLRU to go away from under us, and freeze lru stats */
2488
	spin_lock_irqsave(zone_lru_lock(page_zone(head)), flags);
2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504

	if (mapping) {
		void **pslot;

		spin_lock(&mapping->tree_lock);
		pslot = radix_tree_lookup_slot(&mapping->page_tree,
				page_index(head));
		/*
		 * Check if the head page is present in radix tree.
		 * We assume all tail are present too, if head is there.
		 */
		if (radix_tree_deref_slot_protected(pslot,
					&mapping->tree_lock) != head)
			goto fail;
	}

2505
	/* Prevent deferred_split_scan() touching ->_refcount */
2506
	spin_lock(&pgdata->split_queue_lock);
2507 2508
	count = page_count(head);
	mapcount = total_mapcount(head);
2509
	if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) {
2510
		if (!list_empty(page_deferred_list(head))) {
2511
			pgdata->split_queue_len--;
2512 2513
			list_del(page_deferred_list(head));
		}
2514
		if (mapping)
2515
			__dec_node_page_state(page, NR_SHMEM_THPS);
2516 2517
		spin_unlock(&pgdata->split_queue_lock);
		__split_huge_page(page, list, flags);
2518 2519
		ret = 0;
	} else {
2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
		if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) {
			pr_alert("total_mapcount: %u, page_count(): %u\n",
					mapcount, count);
			if (PageTail(page))
				dump_page(head, NULL);
			dump_page(page, "total_mapcount(head) > 0");
			BUG();
		}
		spin_unlock(&pgdata->split_queue_lock);
fail:		if (mapping)
			spin_unlock(&mapping->tree_lock);
2531
		spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2532
		unfreeze_page(head);
2533 2534 2535 2536
		ret = -EBUSY;
	}

out_unlock:
2537 2538 2539 2540 2541 2542
	if (anon_vma) {
		anon_vma_unlock_write(anon_vma);
		put_anon_vma(anon_vma);
	}
	if (mapping)
		i_mmap_unlock_read(mapping);
2543 2544 2545 2546
out:
	count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
	return ret;
}
2547 2548 2549

void free_transhuge_page(struct page *page)
{
2550
	struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2551 2552
	unsigned long flags;

2553
	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2554
	if (!list_empty(page_deferred_list(page))) {
2555
		pgdata->split_queue_len--;
2556 2557
		list_del(page_deferred_list(page));
	}
2558
	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2559 2560 2561 2562 2563
	free_compound_page(page);
}

void deferred_split_huge_page(struct page *page)
{
2564
	struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2565 2566 2567 2568
	unsigned long flags;

	VM_BUG_ON_PAGE(!PageTransHuge(page), page);

2569
	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2570
	if (list_empty(page_deferred_list(page))) {
2571
		count_vm_event(THP_DEFERRED_SPLIT_PAGE);
2572 2573
		list_add_tail(page_deferred_list(page), &pgdata->split_queue);
		pgdata->split_queue_len++;
2574
	}
2575
	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2576 2577 2578 2579 2580
}

static unsigned long deferred_split_count(struct shrinker *shrink,
		struct shrink_control *sc)
{
2581
	struct pglist_data *pgdata = NODE_DATA(sc->nid);
2582
	return ACCESS_ONCE(pgdata->split_queue_len);
2583 2584 2585 2586 2587
}

static unsigned long deferred_split_scan(struct shrinker *shrink,
		struct shrink_control *sc)
{
2588
	struct pglist_data *pgdata = NODE_DATA(sc->nid);
2589 2590 2591 2592 2593
	unsigned long flags;
	LIST_HEAD(list), *pos, *next;
	struct page *page;
	int split = 0;

2594
	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2595
	/* Take pin on all head pages to avoid freeing them under us */
2596
	list_for_each_safe(pos, next, &pgdata->split_queue) {
2597 2598
		page = list_entry((void *)pos, struct page, mapping);
		page = compound_head(page);
2599 2600 2601 2602
		if (get_page_unless_zero(page)) {
			list_move(page_deferred_list(page), &list);
		} else {
			/* We lost race with put_compound_page() */
2603
			list_del_init(page_deferred_list(page));
2604
			pgdata->split_queue_len--;
2605
		}
2606 2607
		if (!--sc->nr_to_scan)
			break;
2608
	}
2609
	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620

	list_for_each_safe(pos, next, &list) {
		page = list_entry((void *)pos, struct page, mapping);
		lock_page(page);
		/* split_huge_page() removes page from list on success */
		if (!split_huge_page(page))
			split++;
		unlock_page(page);
		put_page(page);
	}

2621 2622 2623
	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
	list_splice_tail(&list, &pgdata->split_queue);
	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2624

2625 2626 2627 2628 2629 2630 2631
	/*
	 * Stop shrinker if we didn't split any page, but the queue is empty.
	 * This can happen if pages were freed under us.
	 */
	if (!split && list_empty(&pgdata->split_queue))
		return SHRINK_STOP;
	return split;
2632 2633 2634 2635 2636 2637
}

static struct shrinker deferred_split_shrinker = {
	.count_objects = deferred_split_count,
	.scan_objects = deferred_split_scan,
	.seeks = DEFAULT_SEEKS,
2638
	.flags = SHRINKER_NUMA_AWARE,
2639
};
2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664

#ifdef CONFIG_DEBUG_FS
static int split_huge_pages_set(void *data, u64 val)
{
	struct zone *zone;
	struct page *page;
	unsigned long pfn, max_zone_pfn;
	unsigned long total = 0, split = 0;

	if (val != 1)
		return -EINVAL;

	for_each_populated_zone(zone) {
		max_zone_pfn = zone_end_pfn(zone);
		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
			if (!pfn_valid(pfn))
				continue;

			page = pfn_to_page(pfn);
			if (!get_page_unless_zero(page))
				continue;

			if (zone != page_zone(page))
				goto next;

2665
			if (!PageHead(page) || PageHuge(page) || !PageLRU(page))
2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677
				goto next;

			total++;
			lock_page(page);
			if (!split_huge_page(page))
				split++;
			unlock_page(page);
next:
			put_page(page);
		}
	}

2678
	pr_info("%lu of %lu THP split\n", split, total);
2679 2680 2681 2682 2683 2684 2685 2686 2687 2688

	return 0;
}
DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
		"%llu\n");

static int __init split_huge_pages_debugfs(void)
{
	void *ret;

2689
	ret = debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
2690 2691 2692 2693 2694 2695 2696
			&split_huge_pages_fops);
	if (!ret)
		pr_warn("Failed to create split_huge_pages in debugfs");
	return 0;
}
late_initcall(split_huge_pages_debugfs);
#endif