arm64: introduce copy_mc_to_kernel() implementation

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I5GB28
CVE: NA

-------------------------------

The copy_mc_to_kernel() helper is memory copy implementation that handles
source exceptions. It can be used in memory copy scenarios that tolerate
hardware memory errors(e.g: pmem_read/dax_copy_to_iter).

Currnently, only x86 and ppc suuport this helper, after arm64 support
machine check safe framework, we introduce copy_mc_to_kernel()
implementation.
Signed-off-by: NTong Tiangen <tongtiangen@huawei.com>

arch/arm64/include/asm/string.h

@@ -35,6 +35,10 @@ extern void *memchr(const void *, int, __kernel_size_t);
 extern void *memcpy(void *, const void *, __kernel_size_t);
 extern void *__memcpy(void *, const void *, __kernel_size_t);
 
+#define __HAVE_ARCH_MEMCPY_MC
+extern unsigned long *memcpy_mcs(void *, const void *, __kernel_size_t);
+extern unsigned long *__memcpy_mcs(void *, const void *, __kernel_size_t);
+
 #define __HAVE_ARCH_MEMMOVE
 extern void *memmove(void *, const void *, __kernel_size_t);
 extern void *__memmove(void *, const void *, __kernel_size_t);
@@ -56,6 +60,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt);
  */
 
 #define memcpy(dst, src, len) __memcpy(dst, src, len)
+#define memcpy_mcs(dst, src, len) __memcpy_mcs(dst, src, len)
 #define memmove(dst, src, len) __memmove(dst, src, len)
 #define memset(s, c, n) __memset(s, c, n)
 

arch/arm64/include/asm/uaccess.h

@@ -432,4 +432,23 @@ static inline int __copy_from_user_flushcache(void *dst, const void __user *src,
 }
 #endif
 
+#ifdef CONFIG_ARCH_HAS_COPY_MC
+/**
+ * copy_mc_to_kernel - memory copy that handles source exceptions
+ *
+ * @dst:	destination address
+ * @src:	source address
+ * @len:	number of bytes to copy
+ *
+ * Return 0 for success, or number of bytes not copied if there was an
+ * exception.
+ */
+static inline unsigned long __must_check
+copy_mc_to_kernel(void *to, const void *from, unsigned long size)
+{
+	return (unsigned long)memcpy_mcs(to, from, size);
+}
+#define copy_mc_to_kernel copy_mc_to_kernel
+#endif
+
 #endif /* __ASM_UACCESS_H */

arch/arm64/lib/Makefile

 # SPDX-License-Identifier: GPL-2.0
-lib-y		:= clear_user.o delay.o copy_from_user.o		\
-		   copy_to_user.o copy_in_user.o copy_page.o		\
-		   clear_page.o csum.o memchr.o memcpy.o memmove.o	\
-		   memset.o memcmp.o strcmp.o strncmp.o strlen.o	\
+lib-y		:= clear_user.o delay.o copy_from_user.o			\
+		   copy_to_user.o copy_in_user.o copy_page.o			\
+		   clear_page.o csum.o memchr.o memcpy.o memcpy_mc.o memmove.o	\
+		   memset.o memcmp.o strcmp.o strncmp.o strlen.o		\
 		   strnlen.o strchr.o strrchr.o tishift.o
 
 ifeq ($(CONFIG_KERNEL_MODE_NEON), y)

arch/arm64/lib/memcpy_mc.S

+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/cache.h>
+
+/*
+ * Copy a buffer from src to dest (alignment handled by the hardware)
+ * with machine check safe.
+ *
+ * Parameters:
+ *	x0 - dest
+ *	x1 - src
+ *	x2 - n
+ * Returns:
+ *	x0 - bytes not copied
+ */
+	.macro ldrb1 reg, ptr, val
+	CPY_MC(9998f, ldrb  \reg, [\ptr], \val)
+	.endm
+
+	.macro strb1 reg, ptr, val
+	CPY_MC(9998f, strb \reg, [\ptr], \val)
+	.endm
+
+	.macro ldrh1 reg, ptr, val
+	CPY_MC(9998f, ldrh  \reg, [\ptr], \val)
+	.endm
+
+	.macro strh1 reg, ptr, val
+	CPY_MC(9998f, strh \reg, [\ptr], \val)
+	.endm
+
+	.macro ldr1 reg, ptr, val
+	CPY_MC(9998f, ldr \reg, [\ptr], \val)
+	.endm
+
+	.macro str1 reg, ptr, val
+	CPY_MC(9998f, str \reg, [\ptr], \val)
+	.endm
+
+	.macro ldp1 reg1, reg2, ptr, val
+	CPY_MC(9998f, ldp \reg1, \reg2, [\ptr], \val)
+	.endm
+
+	.macro stp1 reg1, reg2, ptr, val
+	CPY_MC(9998f, stp \reg1, \reg2, [\ptr], \val)
+	.endm
+
+end	.req	x5
+SYM_FUNC_START_ALIAS(__memcpy_mcs)
+SYM_FUNC_START_WEAK_PI(memcpy_mcs)
+	add end, x0, x2
+#include "copy_template.S"
+	mov x0, #0
+	ret
+
+9998:	sub x0, end, dst
+	ret
+SYM_FUNC_END_PI(memcpy_mcs)
+EXPORT_SYMBOL(memcpy_mcs)
+SYM_FUNC_END_ALIAS(__memcpy_mcs)
+EXPORT_SYMBOL(__memcpy_mcs)