From 6ab4e4d2789de8127e39aff13ae26e9c082c4926 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 12 Jul 2021 17:26:41 +0800 Subject: [PATCH] powerpc/64s/radix: Enable huge vmalloc mappings mainline inclusion from mainline-5.13-rc1 commit 8abddd968a303db75e4debe77a3df484164f1f33 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I3ZGKZ CVE: NA ------------------------------------------------- This reduces TLB misses by nearly 30x on a `git diff` workload on a 2-node POWER9 (59,800 -> 2,100) and reduces CPU cycles by 0.54%, due to vfs hashes being allocated with 2MB pages. Signed-off-by: Nicholas Piggin Reviewed-by: Christophe Leroy Acked-by: Michael Ellerman Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20210503091755.613393-1-npiggin@gmail.com Conflicts: arch/powerpc/kernel/module.c Signed-off-by: Chen Wandun Reviewed-by: Tong Tiangen Signed-off-by: Zheng Zengkai --- .../admin-guide/kernel-parameters.txt | 2 ++ arch/powerpc/Kconfig | 1 + arch/powerpc/kernel/module.c | 34 ++++++++++++++++--- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ea6b387813b0..64242963ce27 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3274,6 +3274,8 @@ nohugeiomap [KNL,X86,PPC,ARM64] Disable kernel huge I/O mappings. + nohugevmalloc [PPC] Disable kernel huge vmalloc mappings. + nosmt [KNL,S390] Disable symmetric multithreading (SMT). Equivalent to smt=1. diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index cd2760a85553..2288b8d52af8 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -179,6 +179,7 @@ config PPC select GENERIC_TIME_VSYSCALL select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_HUGE_VMAP if PPC_BOOK3S_64 && PPC_RADIX_MMU + select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KASAN if PPC32 && PPC_PAGE_SHIFT <= 14 select HAVE_ARCH_KASAN_VMALLOC if PPC32 && PPC_PAGE_SHIFT <= 14 diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c index a211b0253cdb..0beeebcac967 100644 --- a/arch/powerpc/kernel/module.c +++ b/arch/powerpc/kernel/module.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -87,13 +88,36 @@ int module_finalize(const Elf_Ehdr *hdr, return 0; } -#ifdef MODULES_VADDR +static __always_inline void * +__module_alloc(unsigned long size, unsigned long start, unsigned long end) +{ + /* + * Don't do huge page allocations for modules yet until more testing + * is done. STRICT_MODULE_RWX may require extra work to support this + * too. + */ + return __vmalloc_node_range(size, 1, start, end, GFP_KERNEL, PAGE_KERNEL_EXEC, + VM_FLUSH_RESET_PERMS | VM_NO_HUGE_VMAP, + NUMA_NO_NODE, __builtin_return_address(0)); +} + void *module_alloc(unsigned long size) { +#ifdef MODULES_VADDR + unsigned long limit = (unsigned long)_etext - SZ_32M; + void *ptr = NULL; + BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, GFP_KERNEL, - PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, - __builtin_return_address(0)); -} + /* First try within 32M limit from _etext to avoid branch trampolines */ + if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) + ptr = __module_alloc(size, limit, MODULES_END); + + if (!ptr) + ptr = __module_alloc(size, MODULES_VADDR, MODULES_END); + + return ptr; +#else + return __module_alloc(size, VMALLOC_START, VMALLOC_END); #endif +} -- GitLab