ARM: kernel: switch to relative exception tables

maillist inclusion
commit ccb456783dd71f474e5783a81d7f18c2cd4dda81
category: feature
feature: ARM kaslr support
bugzilla: 47952
CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/ardb/linux.git/commit/?h=arm-kaslr-latest&id=ccb456783dd71f474e5783a81d7f18c2cd4dda81

-------------------------------------------------

To avoid having to relocate the contents of extable entries at
runtime when running with KASLR enabled, wire up the existing
support for emitting them as relative references. This ensures
these quantities are invariant under runtime relocation.

Cc: Russell King <linux@armlinux.org.uk>
Signed-off-by: NArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: NCui GaoSheng <cuigaosheng1@huawei.com>
Reviewed-by: NXiu Jianfeng <xiujianfeng@huawei.com>
Signed-off-by: NZheng Zengkai <zhengzengkai@huawei.com>

arch/arm/include/asm/Kbuild

 # SPDX-License-Identifier: GPL-2.0
 generic-y += early_ioremap.h
-generic-y += extable.h
 generic-y += flat.h
 generic-y += parport.h
 generic-y += seccomp.h

arch/arm/include/asm/assembler.h

@@ -18,6 +18,7 @@
 #endif
 
 #include <asm/ptrace.h>
+#include <asm/extable.h>
 #include <asm/opcodes-virt.h>
 #include <asm/asm-offsets.h>
 #include <asm/page.h>
@@ -242,10 +243,7 @@
 
 #define USERL(l, x...)				\
 9999:	x;					\
-	.pushsection __ex_table,"a";		\
-	.align	3;				\
-	.long	9999b,l;			\
-	.popsection
+	ex_entry	9999b,l;
 
 #define USER(x...)	USERL(9001f, x)
 
@@ -379,10 +377,7 @@ THUMB(	orr	\reg , \reg , #PSR_T_BIT	)
 	.error	"Unsupported inc macro argument"
 	.endif
 
-	.pushsection __ex_table,"a"
-	.align	3
-	.long	9999b, \abort
-	.popsection
+	ex_entry	9999b, \abort
 	.endm
 
 	.macro	usracc, instr, reg, ptr, inc, cond, rept, abort
@@ -420,10 +415,7 @@ THUMB(	orr	\reg , \reg , #PSR_T_BIT	)
 	.error	"Unsupported inc macro argument"
 	.endif
 
-	.pushsection __ex_table,"a"
-	.align	3
-	.long	9999b, \abort
-	.popsection
+	ex_entry	9999b, \abort
 	.endr
 	.endm
 

arch/arm/include/asm/extable.h

+#ifndef __ASM_EXTABLE_H
+#define __ASM_EXTABLE_H
+
+#ifndef __ASSEMBLY__
+
+/*
+ * The exception table consists of pairs of relative offsets: the first
+ * is the relative offset to an instruction that is allowed to fault,
+ * and the second is the relative offset at which the program should
+ * continue. No registers are modified, so it is entirely up to the
+ * continuation code to figure out what to do.
+ */
+
+struct exception_table_entry {
+	int insn, fixup;
+};
+
+#define ARCH_HAS_RELATIVE_EXTABLE
+
+extern int fixup_exception(struct pt_regs *regs);
+
+	/*
+	 * ex_entry - place-relative extable entry
+	 */
+asm(	".macro		ex_entry, insn, fixup		\n"
+	".pushsection	__ex_table, \"a\", %progbits	\n"
+	".align		3				\n"
+	".long		\\insn - .			\n"
+	".long		\\fixup - .			\n"
+	".popsection					\n"
+	".endm						\n");
+
+#else
+
+	/*
+	 * ex_entry - place-relative extable entry
+	 */
+	.macro		ex_entry, insn, fixup
+	.pushsection	__ex_table, "a", %progbits
+	.align		3
+	.long		\insn - .
+	.long		\fixup - .
+	.popsection
+	.endm
+
+#endif
+#endif

arch/arm/include/asm/futex.h

@@ -10,10 +10,8 @@
 
 #define __futex_atomic_ex_table(err_reg)			\
 	"3:\n"							\
-	"	.pushsection __ex_table,\"a\"\n"		\
-	"	.align	3\n"					\
-	"	.long	1b, 4f, 2b, 4f\n"			\
-	"	.popsection\n"					\
+	"	ex_entry	1b, 4f\n"			\
+	"	ex_entry	2b, 4f\n"			\
 	"	.pushsection .text.fixup,\"ax\"\n"		\
 	"	.align	2\n"					\
 	"4:	mov	%0, " err_reg "\n"			\

arch/arm/include/asm/uaccess.h

@@ -340,10 +340,7 @@ do {									\
 	"	mov	%1, #0\n"				\
 	"	b	2b\n"					\
 	"	.popsection\n"					\
-	"	.pushsection __ex_table,\"a\"\n"		\
-	"	.align	3\n"					\
-	"	.long	1b, 3b\n"				\
-	"	.popsection"					\
+	"	ex_entry	1b, 3b\n"			\
 	: "+r" (err), "=&r" (x)					\
 	: "r" (addr), "i" (-EFAULT)				\
 	: "cc")
@@ -442,10 +439,7 @@ do {									\
 	"3:	mov	%0, %3\n"				\
 	"	b	2b\n"					\
 	"	.popsection\n"					\
-	"	.pushsection __ex_table,\"a\"\n"		\
-	"	.align	3\n"					\
-	"	.long	1b, 3b\n"				\
-	"	.popsection"					\
+	"	ex_entry	1b, 3b\n"			\
 	: "+r" (err)						\
 	: "r" (x), "r" (__pu_addr), "i" (-EFAULT)		\
 	: "cc")
@@ -501,11 +495,8 @@ do {									\
 	"4:	mov	%0, %3\n"				\
 	"	b	3b\n"					\
 	"	.popsection\n"					\
-	"	.pushsection __ex_table,\"a\"\n"		\
-	"	.align	3\n"					\
-	"	.long	1b, 4b\n"				\
-	"	.long	2b, 4b\n"				\
-	"	.popsection"					\
+	"	ex_entry	1b, 4b\n"			\
+	"	ex_entry	2b, 4b\n"			\
 	: "+r" (err), "+r" (__pu_addr)				\
 	: "r" (x), "i" (-EFAULT)				\
 	: "cc")

arch/arm/include/asm/word-at-a-time.h

@@ -9,6 +9,7 @@
  * Heavily based on the x86 algorithm.
  */
 #include <linux/kernel.h>
+#include <asm/extable.h>
 
 struct word_at_a_time {
 	const unsigned long one_bits, high_bits;
@@ -85,10 +86,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
 #endif
 	"	b	2b\n"
 	"	.popsection\n"
-	"	.pushsection __ex_table,\"a\"\n"
-	"	.align	3\n"
-	"	.long	1b, 3b\n"
-	"	.popsection"
+	"	ex_entry	1b, 3b\n"
 	: "=&r" (ret), "=&r" (offset)
 	: "r" (addr), "Qo" (*(unsigned long *)addr));
 

arch/arm/kernel/entry-armv.S

@@ -15,6 +15,7 @@
 #include <linux/init.h>
 
 #include <asm/assembler.h>
+#include <asm/extable.h>
 #include <asm/memory.h>
 #include <asm/glue-df.h>
 #include <asm/glue-pf.h>
@@ -535,13 +536,11 @@ ENDPROC(__und_usr)
 4:	str     r4, [sp, #S_PC]			@ retry current instruction
 	ret	r9
 	.popsection
-	.pushsection __ex_table,"a"
-	.long	1b, 4b
+	ex_entry	1b, 4b
 #if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
-	.long	2b, 4b
-	.long	3b, 4b
+	ex_entry	2b, 4b
+	ex_entry	3b, 4b
 #endif
-	.popsection
 
 /*
  * Check whether the instruction is a co-processor instruction.

arch/arm/kernel/swp_emulate.c

@@ -24,6 +24,7 @@
 #include <linux/syscalls.h>
 #include <linux/perf_event.h>
 
+#include <asm/extable.h>
 #include <asm/opcodes.h>
 #include <asm/system_info.h>
 #include <asm/traps.h>
@@ -45,11 +46,8 @@
 	"3:	mov		%0, %5\n"			\
 	"	b		2b\n"				\
 	"	.previous\n"					\
-	"	.section	 __ex_table,\"a\"\n"		\
-	"	.align		3\n"				\
-	"	.long		0b, 3b\n"			\
-	"	.long		1b, 3b\n"			\
-	"	.previous"					\
+	"	ex_entry	0b, 3b\n"			\
+	"	ex_entry	1b, 3b\n"			\
 	: "=&r" (res), "+r" (data), "=&r" (temp)		\
 	: "r" (addr), "i" (-EAGAIN), "i" (-EFAULT)		\
 	: "cc", "memory")

arch/arm/lib/backtrace.S

@@ -106,14 +106,11 @@ for_each_frame:	tst	frame, mask		@ Check for address exceptions
 		bl	printk
 no_frame:	ldmfd	sp!, {r4 - r9, pc}
 ENDPROC(c_backtrace)
-		
-		.pushsection __ex_table,"a"
-		.align	3
-		.long	1001b, 1006b
-		.long	1002b, 1006b
-		.long	1003b, 1006b
-		.long	1004b, 1006b
-		.popsection
+
+		ex_entry	1001b, 1006b
+		ex_entry	1002b, 1006b
+		ex_entry	1003b, 1006b
+		ex_entry	1004b, 1006b
 
 .Lbad:		.asciz	"%sBacktrace aborted due to bad frame pointer <%p>\n"
 		.align

arch/arm/lib/getuser.S

@@ -27,6 +27,7 @@
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/errno.h>
+#include <asm/extable.h>
 #include <asm/domain.h>
 
 ENTRY(__get_user_1)
@@ -149,19 +150,18 @@ _ASM_NOKPROBE(__get_user_bad)
 _ASM_NOKPROBE(__get_user_bad8)
 
 .pushsection __ex_table, "a"
-	.long	1b, __get_user_bad
-	.long	2b, __get_user_bad
+	ex_entry	1b, __get_user_bad
+	ex_entry	2b, __get_user_bad
 #if __LINUX_ARM_ARCH__ < 6
-	.long	3b, __get_user_bad
+	ex_entry	3b, __get_user_bad
 #endif
-	.long	4b, __get_user_bad
-	.long	5b, __get_user_bad8
-	.long	6b, __get_user_bad8
+	ex_entry	4b, __get_user_bad
+	ex_entry	5b, __get_user_bad8
+	ex_entry	6b, __get_user_bad8
 #ifdef __ARMEB__
-	.long   7b, __get_user_bad
-	.long	8b, __get_user_bad8
-	.long	9b, __get_user_bad8
-	.long	10b, __get_user_bad8
-	.long	11b, __get_user_bad8
+	ex_entry	7b, __get_user_bad
+	ex_entry	8b, __get_user_bad8
+	ex_entry	9b, __get_user_bad8
+	ex_entry	10b, __get_user_bad8
+	ex_entry	11b, __get_user_bad8
 #endif
-.popsection

arch/arm/lib/putuser.S

@@ -27,6 +27,7 @@
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/errno.h>
+#include <asm/extable.h>
 #include <asm/domain.h>
 
 ENTRY(__put_user_1)
@@ -83,13 +84,11 @@ __put_user_bad:
 	ret	lr
 ENDPROC(__put_user_bad)
 
-.pushsection __ex_table, "a"
-	.long	1b, __put_user_bad
-	.long	2b, __put_user_bad
+	ex_entry	1b, __put_user_bad
+	ex_entry	2b, __put_user_bad
 #if __LINUX_ARM_ARCH__ < 6
-	.long	3b, __put_user_bad
+	ex_entry	3b, __put_user_bad
 #endif
-	.long	4b, __put_user_bad
-	.long	5b, __put_user_bad
-	.long	6b, __put_user_bad
-.popsection
+	ex_entry	4b, __put_user_bad
+	ex_entry	5b, __put_user_bad
+	ex_entry	6b, __put_user_bad

arch/arm/mm/alignment.c

@@ -21,6 +21,7 @@
 #include <linux/uaccess.h>
 
 #include <asm/cp15.h>
+#include <asm/extable.h>
 #include <asm/system_info.h>
 #include <asm/unaligned.h>
 #include <asm/opcodes.h>
@@ -204,10 +205,7 @@ union offset_union {
 	"3:	mov	%0, #1\n"			\
 	"	b	2b\n"				\
 	"	.popsection\n"				\
-	"	.pushsection __ex_table,\"a\"\n"	\
-	"	.align	3\n"				\
-	"	.long	1b, 3b\n"			\
-	"	.popsection\n"				\
+	"	ex_entry	1b, 3b\n"		\
 	: "=r" (err), "=&r" (val), "=r" (addr)		\
 	: "0" (err), "2" (addr))
 
@@ -264,11 +262,8 @@ union offset_union {
 		"4:	mov	%0, #1\n"			\
 		"	b	3b\n"				\
 		"	.popsection\n"				\
-		"	.pushsection __ex_table,\"a\"\n"	\
-		"	.align	3\n"				\
-		"	.long	1b, 4b\n"			\
-		"	.long	2b, 4b\n"			\
-		"	.popsection\n"				\
+		"	ex_entry	1b, 4b\n"		\
+		"	ex_entry	2b, 4b\n"		\
 		: "=r" (err), "=&r" (v), "=&r" (a)		\
 		: "0" (err), "1" (v), "2" (a));			\
 		if (err)					\
@@ -304,13 +299,10 @@ union offset_union {
 		"6:	mov	%0, #1\n"			\
 		"	b	5b\n"				\
 		"	.popsection\n"				\
-		"	.pushsection __ex_table,\"a\"\n"	\
-		"	.align	3\n"				\
-		"	.long	1b, 6b\n"			\
-		"	.long	2b, 6b\n"			\
-		"	.long	3b, 6b\n"			\
-		"	.long	4b, 6b\n"			\
-		"	.popsection\n"				\
+		"	ex_entry	1b, 6b\n"		\
+		"	ex_entry	2b, 6b\n"		\
+		"	ex_entry	3b, 6b\n"		\
+		"	ex_entry	4b, 6b\n"		\
 		: "=r" (err), "=&r" (v), "=&r" (a)		\
 		: "0" (err), "1" (v), "2" (a));			\
 		if (err)					\

arch/arm/mm/extable.c

@@ -11,7 +11,7 @@ int fixup_exception(struct pt_regs *regs)
 
 	fixup = search_exception_tables(instruction_pointer(regs));
 	if (fixup) {
-		regs->ARM_pc = fixup->fixup;
+		regs->ARM_pc = (unsigned long)&fixup->fixup + fixup->fixup;
 #ifdef CONFIG_THUMB2_KERNEL
 		/* Clear the IT state to avoid nasty surprises in the fixup */
 		regs->ARM_cpsr &= ~PSR_IT_MASK;

arch/arm/nwfpe/entry.S

@@ -8,6 +8,7 @@
 
 */
 #include <asm/assembler.h>
+#include <asm/extable.h>
 #include <asm/opcodes.h>
 
 /* This is the kernel's entry point into the floating point emulator.
@@ -107,7 +108,4 @@ next:
 .Lfix:	ret	r9			@ let the user eat segfaults
 	.popsection
 
-	.pushsection __ex_table,"a"
-	.align	3
-	.long	.Lx1, .Lfix
-	.popsection
+	ex_entry	.Lx1, .Lfix

scripts/sorttable.c

@@ -341,12 +341,12 @@ static int do_file(char const *const fname, void *addr)
 	case EM_AARCH64:
 	case EM_PARISC:
 	case EM_PPC:
+	case EM_ARM:
 	case EM_PPC64:
 		custom_sort = sort_relative_table;
 		break;
 	case EM_ARCOMPACT:
 	case EM_ARCV2:
-	case EM_ARM:
 	case EM_MICROBLAZE:
 	case EM_MIPS:
 	case EM_XTENSA: