x86: hugepage: use nt copy hugepage to AEP in x86

euleros inclusion
category: feature
feature: etmem
bugzilla: 48246

-------------------------------------------------

Add proc/sys/vm/hugepage_nocache_copy switch. Set 1 to copy hugepage
with movnt SSE instructoin if cpu support it. Set 0 to copy hugepage
as usual.
Signed-off-by: NKemeng Shi <shikemeng@huawei.com>
Reviewed-by: Nlouhongxiang <louhongxiang@huawei.com>
Signed-off-by: NZheng Zengkai <zhengzengkai@huawei.com>

arch/x86/include/asm/page_64.h

@@ -56,6 +56,16 @@ static inline void clear_page(void *page)
 
 void copy_page(void *to, void *from);
 
+void copy_page_nocache(void *to, void *from);
+void copy_page_nocache_barrir(void);
+
+struct page;
+struct ctl_table;
+# define __HAVE_ARCH_COPY_HUGEPAGES 1
+void copy_highpages(struct page *to, struct page *from, int nr_pages);
+int sysctl_hugepage_nocache_copy(struct ctl_table *table, int write,
+		void __user *buffer, size_t *lenp, loff_t *ppos);
+
 #endif	/* !__ASSEMBLY__ */
 
 #ifdef CONFIG_X86_VSYSCALL_EMULATION

arch/x86/lib/Makefile

@@ -70,4 +70,5 @@ else
         lib-y += memmove_64.o memset_64.o
         lib-y += copy_user_64.o
 	lib-y += cmpxchg16b_emu.o
+	lib-y += copy_highpages.o
 endif

arch/x86/lib/copy_highpages.c

+// SPDX-License-Identifier: GPL-2.0
+/*
+ * accelerate copying page to pmem with non-temproal stroes
+ */
+#include <linux/sched.h>
+#include <linux/mmzone.h>
+#include <linux/highmem.h>
+#include <linux/sysctl.h>
+
+DEFINE_STATIC_KEY_FALSE(hugepage_nocache_copy);
+
+static void set_hugepage_nocache_copy(bool enabled)
+{
+	if (enabled)
+		static_branch_enable(&hugepage_nocache_copy);
+	else
+		static_branch_disable(&hugepage_nocache_copy);
+}
+
+int sysctl_hugepage_nocache_copy(struct ctl_table *table, int write,
+		void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+	struct ctl_table t;
+	int err;
+	int state = static_branch_unlikely(&hugepage_nocache_copy);
+
+	if (write && !capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	t = *table;
+	t.data = &state;
+	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
+	if (err < 0)
+		return err;
+	if (write)
+		set_hugepage_nocache_copy(state);
+	return err;
+}
+
+static void copy_highpages_nocache(struct page *to, struct page *from, int nr_pages)
+{
+	char *vfrom, *vto;
+	int i;
+
+	for (i = 0; i < nr_pages; i++) {
+		cond_resched();
+		vfrom = kmap_atomic(from);
+		vto = kmap_atomic(to);
+		copy_page_nocache(vto, vfrom);
+		kunmap_atomic(vto);
+		kunmap_atomic(vfrom);
+		to++;
+		from++;
+	}
+	copy_page_nocache_barrir();
+}
+
+static void copy_highpages_cache(struct page *to, struct page *from, int nr_pages)
+{
+	int i;
+
+	for (i = 0; i < nr_pages; i++) {
+		cond_resched();
+		copy_highpage(to + i, from + i);
+	}
+}
+
+void copy_highpages(struct page *to, struct page *from, int nr_pages)
+{
+	if (static_branch_unlikely(&hugepage_nocache_copy) && is_node_pmem(page_to_nid(to)))
+		return copy_highpages_nocache(to, from, nr_pages);
+
+	return copy_highpages_cache(to, from, nr_pages);
+}

arch/x86/lib/copy_page_64.S

@@ -87,3 +87,76 @@ SYM_FUNC_START_LOCAL(copy_page_regs)
 	addq	$2*8, %rsp
 	ret
 SYM_FUNC_END(copy_page_regs)
+
+SYM_FUNC_START(copy_page_nocache)
+	ALTERNATIVE "jmp copy_page", "", X86_FEATURE_XMM2
+	subq	$2*8,	%rsp
+	movq	%rbx,	(%rsp)
+	movq	%r12,	1*8(%rsp)
+
+	movl	$(4096/64)-5,	%ecx
+	.p2align 4
+.LoopNT64:
+	dec	%rcx
+	movq	0x8*0(%rsi), %rax
+	movq	0x8*1(%rsi), %rbx
+	movq	0x8*2(%rsi), %rdx
+	movq	0x8*3(%rsi), %r8
+	movq	0x8*4(%rsi), %r9
+	movq	0x8*5(%rsi), %r10
+	movq	0x8*6(%rsi), %r11
+	movq	0x8*7(%rsi), %r12
+
+	prefetcht0 5*64(%rsi)
+
+	movnti	%rax, 0x8*0(%rdi)
+	movnti	%rbx, 0x8*1(%rdi)
+	movnti	%rdx, 0x8*2(%rdi)
+	movnti	%r8,  0x8*3(%rdi)
+	movnti	%r9,  0x8*4(%rdi)
+	movnti	%r10, 0x8*5(%rdi)
+	movnti	%r11, 0x8*6(%rdi)
+	movnti	%r12, 0x8*7(%rdi)
+
+	leaq	64 (%rsi), %rsi
+	leaq	64 (%rdi), %rdi
+
+	jnz	.LoopNT64
+
+	movl	$5, %ecx
+	.p2align 4
+.LoopNT2:
+	decl	%ecx
+
+	movq	0x8*0(%rsi), %rax
+	movq	0x8*1(%rsi), %rbx
+	movq	0x8*2(%rsi), %rdx
+	movq	0x8*3(%rsi), %r8
+	movq	0x8*4(%rsi), %r9
+	movq	0x8*5(%rsi), %r10
+	movq	0x8*6(%rsi), %r11
+	movq	0x8*7(%rsi), %r12
+
+	movnti	%rax, 0x8*0(%rdi)
+	movnti	%rbx, 0x8*1(%rdi)
+	movnti	%rdx, 0x8*2(%rdi)
+	movnti	%r8,  0x8*3(%rdi)
+	movnti	%r9,  0x8*4(%rdi)
+	movnti	%r10, 0x8*5(%rdi)
+	movnti	%r11, 0x8*6(%rdi)
+	movnti	%r12, 0x8*7(%rdi)
+
+	leaq	64(%rdi), %rdi
+	leaq	64(%rsi), %rsi
+	jnz	.LoopNT2
+
+	movq	(%rsp), %rbx
+	movq	1*8(%rsp), %r12
+	addq	$2*8, %rsp
+	ret
+SYM_FUNC_END(copy_page_nocache)
+
+SYM_FUNC_START(copy_page_nocache_barrir)
+	ALTERNATIVE "", "sfence", X86_FEATURE_XMM2
+	ret
+SYM_FUNC_END(copy_page_nocache_barrir)

include/linux/highmem.h

@@ -345,4 +345,18 @@ static inline void copy_highpage(struct page *to, struct page *from)
 
 #endif
 
+#ifndef __HAVE_ARCH_COPY_HUGEPAGES
+
+static inline void copy_highpages(struct page *to, struct page *from, int nr_pages)
+{
+	int i;
+
+	for (i = 0; i < nr_pages; i++) {
+		cond_resched();
+		copy_highpage(to + i, from + i);
+	}
+}
+
+#endif /* __HAVE_ARCH_COPY_HUGEPAGES */
+
 #endif /* _LINUX_HIGHMEM_H */

kernel/sysctl.c

@@ -3147,6 +3147,17 @@ static struct ctl_table vm_table[] = {
 		.extra1         = SYSCTL_ZERO,
 		.extra2         = SYSCTL_ONE,
 	},
+#if defined(CONFIG_X86_64)
+	{
+		.procname       = "hugepage_nocache_copy",
+		.data           = NULL, /* filled in by handler */
+		.maxlen         = sizeof(unsigned int),
+		.mode           = 0600,
+		.proc_handler   = sysctl_hugepage_nocache_copy,
+		.extra1         = SYSCTL_ZERO,
+		.extra2         = SYSCTL_ONE,
+	},
+#endif
 	{ }
 };
 

mm/migrate.c

@@ -577,7 +577,6 @@ static void __copy_gigantic_page(struct page *dst, struct page *src,
 
 static void copy_huge_page(struct page *dst, struct page *src)
 {
-	int i;
 	int nr_pages;
 
 	if (PageHuge(src)) {
@@ -595,10 +594,7 @@ static void copy_huge_page(struct page *dst, struct page *src)
 		nr_pages = thp_nr_pages(src);
 	}
 
-	for (i = 0; i < nr_pages; i++) {
-		cond_resched();
-		copy_highpage(dst + i, src + i);
-	}
+	copy_highpages(dst, src, nr_pages);
 }
 
 /*