diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index e7d2bdc92acf1b7faf4705024ff168594189a1a7..47f8fae7ecdb00f4a2d4ebb5075f3593e72ca35d 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -68,7 +68,8 @@ #define ARM64_WORKAROUND_1508412 58 #define ARM64_HAS_MPAM 59 #define ARM64_WORKAROUND_HISI_HIP08_RU_PREFETCH 60 +#define ARM64_CLEARPAGE_STNP 61 -#define ARM64_NCAPS 61 +#define ARM64_NCAPS 62 #endif /* __ASM_CPUCAPS_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index d8e1bdb5abedf2c98d1e54002528a7e1132f4177..15b49edfe2c82d5db3d58d8351e7c95615bf4b6c 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1736,6 +1736,34 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap) return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT); } +static bool use_clearpage_stnp; + +static int __init early_use_clearpage_stnp(char *p) +{ + return strtobool(p, &use_clearpage_stnp); +} +early_param("mm.use_clearpage_stnp", early_use_clearpage_stnp); + +static bool has_mor_nontemporal(const struct arm64_cpu_capabilities *entry) +{ + /* + * List of CPUs which have memory ordering ruled non-temporal + * load and store. + */ + static const struct midr_range cpus[] = { + MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), + {}, + }; + + return is_midr_in_range_list(read_cpuid_id(), cpus); +} + +static bool can_clearpage_use_stnp(const struct arm64_cpu_capabilities *entry, + int scope) +{ + return use_clearpage_stnp && has_mor_nontemporal(entry); +} + static const struct arm64_cpu_capabilities arm64_features[] = { { .desc = "GIC system register CPU interface", @@ -2158,6 +2186,12 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_mte, }, #endif /* CONFIG_ARM64_MTE */ + { + .desc = "Clear Page by STNP", + .capability = ARM64_CLEARPAGE_STNP, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = can_clearpage_use_stnp, + }, {}, }; diff --git a/arch/arm64/lib/clear_page.S b/arch/arm64/lib/clear_page.S index 073acbf02a7c842520eeb42df0df4f2dd3a15480..39ad3ee1860e07a5eb5ddd1300fee4c53b792bfe 100644 --- a/arch/arm64/lib/clear_page.S +++ b/arch/arm64/lib/clear_page.S @@ -7,6 +7,25 @@ #include #include #include +#include + +/* + * Clear page @dest + * + * Parameters: + * x0 - dest + */ +SYM_FUNC_START(clear_page_stnp) + .align 6 +1: stnp xzr, xzr, [x0] + stnp xzr, xzr, [x0, #0x10] + stnp xzr, xzr, [x0, #0x20] + stnp xzr, xzr, [x0, #0x30] + add x0, x0, #0x40 + tst x0, #(PAGE_SIZE - 1) + b.ne 1b + ret +SYM_FUNC_END(clear_page_stnp) /* * Clear page @dest @@ -15,6 +34,8 @@ * x0 - dest */ SYM_FUNC_START(clear_page) + ALTERNATIVE("nop", "b clear_page_stnp", ARM64_CLEARPAGE_STNP) + mrs x1, dczid_el0 and w1, w1, #0xf mov x2, #4