arm64: extable: add new extable type "__mc_ex_table"

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I5GB28
CVE: NA

-------------------------------

A new type of extable is added, which is specially used fixup for machine
check safe.
In order to keep kabi consistency, we cannot add type and data members to
struct exception_table_entry(d6e2cc56 arm64: extable: add `type` and
`data` fields), so we put the fixup entry to new extable separately.
Signed-off-by: NTong Tiangen <tongtiangen@huawei.com>

arch/arm64/Kconfig

@@ -1170,6 +1170,9 @@ config ARCH_LLC_128_LINE_SIZE
 config ARCH_HAS_FILTER_PGPROT
 	def_bool y
 
+config ARCH_HAS_MC_EXTABLE
+	bool
+
 config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	def_bool y if PGTABLE_LEVELS > 2
 

arch/arm64/include/asm/assembler.h

@@ -145,6 +145,21 @@ alternative_endif
 	.popsection
 	.endm
 
+/*
+ * Emit an entry into the machine check exception table
+ */
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+	.macro		_asm_mc_extable, from, to
+	.pushsection	__mc_ex_table, "a"
+	.align		3
+	.long		(\from - .), (\to - .)
+	.popsection
+	.endm
+#else
+	.macro		_asm_mc_extable, from, to
+	.endm
+#endif
+
 #define USER(l, x...)				\
 9999:	x;					\
 	_asm_extable	9999b, l

arch/arm64/include/asm/uaccess.h

@@ -75,6 +75,16 @@ static inline unsigned long __range_ok(const void __user *addr, unsigned long si
 	"	.long		(" #from " - .), (" #to " - .)\n"	\
 	"	.popsection\n"
 
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+#define _ASM_MC_EXTABLE(from, to)					\
+	"	.pushsection	__mc_ex_table, \"a\"\n"			\
+	"	.align		3\n"					\
+	"	.long		(" #from " - .), (" #to " - .)\n"	\
+	"	.popsection\n"
+#else
+#define _ASM_MC_EXTABLE(from, to)
+#endif
+
 /*
  * User access enabling/disabling.
  */

include/asm-generic/vmlinux.lds.h

@@ -76,7 +76,9 @@
  * alignment.
  */
 #ifdef RO_EXCEPTION_TABLE_ALIGN
-#define RO_EXCEPTION_TABLE	EXCEPTION_TABLE(RO_EXCEPTION_TABLE_ALIGN)
+#define RO_EXCEPTION_TABLE					\
+	EXCEPTION_TABLE(RO_EXCEPTION_TABLE_ALIGN)		\
+	MC_EXCEPTION_TABLE(RO_EXCEPTION_TABLE_ALIGN)
 #else
 #define RO_EXCEPTION_TABLE
 #endif
@@ -675,6 +677,21 @@
 		__stop___ex_table = .;					\
 	}
 
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+/*
+ * Machine Check Exception table
+ */
+#define MC_EXCEPTION_TABLE(align)					\
+	. = ALIGN(align);						\
+	__mc_ex_table : AT(ADDR(__mc_ex_table) - LOAD_OFFSET) {		\
+		__start___mc_ex_table = .;				\
+		KEEP(*(__mc_ex_table))					\
+		__stop___mc_ex_table = .;				\
+	}
+#else
+#define MC_EXCEPTION_TABLE(align)
+#endif
+
 /*
  * .BTF
  */

include/linux/extable.h

@@ -19,12 +19,18 @@ void trim_init_extable(struct module *m);
 
 /* Given an address, look for it in the exception tables */
 const struct exception_table_entry *search_exception_tables(unsigned long add);
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+const struct exception_table_entry *search_mc_exception_tables(unsigned long add);
+#endif
 const struct exception_table_entry *
 search_kernel_exception_table(unsigned long addr);
 
 #ifdef CONFIG_MODULES
 /* For extable.c to search modules' exception tables. */
 const struct exception_table_entry *search_module_extables(unsigned long addr);
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+const struct exception_table_entry *search_module_mc_extables(unsigned long addr);
+#endif
 #else
 static inline const struct exception_table_entry *
 search_module_extables(unsigned long addr)

include/linux/module.h

@@ -429,6 +429,11 @@ struct module {
 	/* Startup function. */
 	int (*init)(void);
 
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+	/* there is 8-byte hole on all platforms */
+	KABI_FILL_HOLE(unsigned int num_mc_exentries)
+#endif
+
 	/* Core layout: rbtree is accessed frequently, so keep together. */
 	struct module_layout core_layout __module_layout_align;
 	struct module_layout init_layout;
@@ -553,7 +558,13 @@ struct module {
 	struct error_injection_entry *ei_funcs;
 	unsigned int num_ei_funcs;
 #endif
+
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+	KABI_USE(1, struct exception_table_entry *mc_extable)
+#else
 	KABI_RESERVE(1)
+#endif
+
 	KABI_RESERVE(2)
 	KABI_RESERVE(3)
 	KABI_RESERVE(4)

kernel/extable.c

@@ -28,6 +28,11 @@ DEFINE_MUTEX(text_mutex);
 extern struct exception_table_entry __start___ex_table[];
 extern struct exception_table_entry __stop___ex_table[];
 
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+extern struct exception_table_entry __start___mc_ex_table[];
+extern struct exception_table_entry __stop___mc_ex_table[];
+#endif
+
 /* Cleared by build time tools if the table is already sorted. */
 u32 __initdata __visible main_extable_sort_needed = 1;
 
@@ -39,6 +44,14 @@ void __init sort_main_extable(void)
 		pr_notice("Sorting __ex_table...\n");
 		sort_extable(__start___ex_table, __stop___ex_table);
 	}
+
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+	if (main_extable_sort_needed &&
+	    &__stop___mc_ex_table > &__start___mc_ex_table) {
+		pr_notice("Sorting __mc_ex_table...\n");
+		sort_extable(__start___mc_ex_table, __stop___mc_ex_table);
+	}
+#endif
 }
 
 /* Given an address, look for it in the kernel exception table */
@@ -62,6 +75,22 @@ const struct exception_table_entry *search_exception_tables(unsigned long addr)
 	return e;
 }
 
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+/* Given an address, look for it in the machine check exception table */
+const
+struct exception_table_entry *search_mc_exception_tables(unsigned long addr)
+{
+	const struct exception_table_entry *e;
+
+	e = search_extable(__start___mc_ex_table,
+			   __stop___mc_ex_table - __start___mc_ex_table, addr);
+	if (!e)
+		e = search_module_mc_extables(addr);
+
+	return e;
+}
+#endif
+
 int init_kernel_text(unsigned long addr)
 {
 	if (addr >= (unsigned long)_sinittext &&

kernel/module.c

@@ -3423,6 +3423,11 @@ static int find_module_sections(struct module *mod, struct load_info *info)
 	mod->extable = section_objs(info, "__ex_table",
 				    sizeof(*mod->extable), &mod->num_exentries);
 
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+	mod->mc_extable = section_objs(info, "__mc_ex_table",
+				    sizeof(*mod->mc_extable), &mod->num_mc_exentries);
+#endif
+
 	if (section_addr(info, "__obsparm"))
 		pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
 
@@ -3660,6 +3665,10 @@ static int post_relocation(struct module *mod, const struct load_info *info)
 	/* Sort exception table now relocations are done. */
 	sort_extable(mod->extable, mod->extable + mod->num_exentries);
 
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+	sort_extable(mod->mc_extable, mod->mc_extable + mod->num_mc_exentries);
+#endif
+
 	/* Copy relocated percpu area over. */
 	percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
 		       info->sechdrs[info->index.pcpu].sh_size);
@@ -4631,6 +4640,35 @@ const struct exception_table_entry *search_module_extables(unsigned long addr)
 	return e;
 }
 
+#ifdef CONFIG_ARCH_HAS_MC_EXTABLE
+/* Given an address, look for it in the module machine check safe exception tables. */
+const struct exception_table_entry *search_module_mc_extables(unsigned long addr)
+{
+	const struct exception_table_entry *e = NULL;
+	struct module *mod;
+
+	preempt_disable();
+	mod = __module_address(addr);
+	if (!mod)
+		goto out;
+
+	if (!mod->num_exentries)
+		goto out;
+
+	e = search_extable(mod->mc_extable,
+			   mod->num_mc_exentries,
+			   addr);
+out:
+	preempt_enable();
+
+	/*
+	 * Now, if we found one, we are running inside it now, hence
+	 * we cannot unload the module, hence no refcnt needed.
+	 */
+	return e;
+}
+#endif
+
 /*
  * is_module_address - is this address inside a module?
  * @addr: the address to check.

scripts/sorttable.h

@@ -223,6 +223,12 @@ static int do_sort(Elf_Ehdr *ehdr,
 	unsigned int orc_num_entries = 0;
 #endif
 
+	Elf_Shdr *mc_extab_sec = NULL;
+	Elf_Rel *mc_relocs = NULL;
+	int mc_relocs_size = 0;
+	char *mc_extab_image = NULL;
+	int mc_extab_index = 0;
+
 	shstrndx = r2(&ehdr->e_shstrndx);
 	if (shstrndx == SHN_XINDEX)
 		shstrndx = r(&shdr[0].sh_link);
@@ -238,6 +244,12 @@ static int do_sort(Elf_Ehdr *ehdr,
 			extab_sec = s;
 			extab_index = i;
 		}
+
+		if (!strcmp(secstrings + idx, "__mc_ex_table")) {
+			mc_extab_sec = s;
+			mc_extab_index = i;
+		}
+
 		if (!strcmp(secstrings + idx, ".symtab"))
 			symtab_sec = s;
 		if (!strcmp(secstrings + idx, ".strtab"))
@@ -249,6 +261,14 @@ static int do_sort(Elf_Ehdr *ehdr,
 			relocs = (void *)ehdr + _r(&s->sh_offset);
 			relocs_size = _r(&s->sh_size);
 		}
+
+		if ((r(&s->sh_type) == SHT_REL ||
+		     r(&s->sh_type) == SHT_RELA) &&
+		    r(&s->sh_info) == mc_extab_index) {
+			mc_relocs = (void *)ehdr + _r(&s->sh_offset);
+			mc_relocs_size = _r(&s->sh_size);
+		}
+
 		if (r(&s->sh_type) == SHT_SYMTAB_SHNDX)
 			symtab_shndx = (Elf32_Word *)((const char *)ehdr +
 						      _r(&s->sh_offset));
@@ -310,12 +330,18 @@ static int do_sort(Elf_Ehdr *ehdr,
 	}
 
 	extab_image = (void *)ehdr + _r(&extab_sec->sh_offset);
+
+	if (mc_extab_sec)
+		mc_extab_image = (void *)ehdr + _r(&mc_extab_sec->sh_offset);
+
 	strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset);
 	symtab = (const Elf_Sym *)((const char *)ehdr +
 						  _r(&symtab_sec->sh_offset));
 
 	if (custom_sort) {
 		custom_sort(extab_image, _r(&extab_sec->sh_size));
+		if (mc_extab_image)
+			custom_sort(mc_extab_image, _r(&mc_extab_sec->sh_size));
 	} else {
 		int num_entries = _r(&extab_sec->sh_size) / extable_ent_size;
 		qsort(extab_image, num_entries,
@@ -326,6 +352,9 @@ static int do_sort(Elf_Ehdr *ehdr,
 	if (relocs)
 		memset(relocs, 0, relocs_size);
 
+	if (mc_relocs)
+		memset(mc_relocs, 0, mc_relocs_size);
+
 	/* find the flag main_extable_sort_needed */
 	for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset);
 	     sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym);