livepatch/core: support jump_label

hulk inclusion
category: feature
bugzilla: 5391/28338/24634
CVE: NA

-----------------------------------------------

The kpatch-build processes the __jump_table special section,
and only the jump_lable used by the changed functions will be
included in __jump_table section, and the livepatch should
process the tracepoint again after the dynamic relocation.

NOTE: adding new tracepoints definition is not supported.
Signed-off-by: NCheng Jian <cj.chengjian@huawei.com>
Reviewed-by: NXie XiuQi <xiexiuqi@huawei.com>
Signed-off-by: NYang Yingliang <yangyingliang@huawei.com>

include/linux/jump_label.h

@@ -165,6 +165,7 @@ extern void static_key_enable(struct static_key *key);
 extern void static_key_disable(struct static_key *key);
 extern void static_key_enable_cpuslocked(struct static_key *key);
 extern void static_key_disable_cpuslocked(struct static_key *key);
+extern int jump_label_register(struct module *mod);
 
 /*
  * We should be using ATOMIC_INIT() for initializing .enabled, but

include/linux/module.h

@@ -320,6 +320,12 @@ struct mod_kallsyms {
 };
 
 #ifdef CONFIG_LIVEPATCH
+enum MODULE_KLP_REL_STATE {
+	MODULE_KLP_REL_NONE = 0,
+	MODULE_KLP_REL_UNDO,
+	MODULE_KLP_REL_DONE,
+};
+
 struct klp_modinfo {
 	Elf_Ehdr hdr;
 	Elf_Shdr *sechdrs;
@@ -463,6 +469,19 @@ struct module {
 
 	/* Elf information */
 	struct klp_modinfo *klp_info;
+	/*
+	 * livepatch should relocate the key of jump_label by
+	 * using klp_write_module_reloc. So it's necessary to
+	 * do jump_label_apply_nops() and jump_label_add_module()
+	 * later after livepatch relocation finised.
+	 *
+	 * for normal module :
+	 *	always MODULE_KLP_REL_DONE.
+	 * for livepatch module :
+	 *	init as MODULE_KLP_REL_UNDO,
+	 *	set to MODULE_KLP_REL_DONE when relocate completed.
+	 */
+	enum MODULE_KLP_REL_STATE klp_rel_state;
 #endif
 
 #ifdef CONFIG_MODULE_UNLOAD
@@ -652,11 +671,28 @@ static inline bool is_livepatch_module(struct module *mod)
 {
 	return mod->klp;
 }
+
+static inline void set_mod_klp_rel_state(struct module *mod,
+			enum MODULE_KLP_REL_STATE state)
+{
+	mod->klp_rel_state = state;
+}
+
+static inline bool mod_klp_rel_completed(struct module *mod)
+{
+	return mod->klp_rel_state == MODULE_KLP_REL_NONE ||
+		mod->klp_rel_state == MODULE_KLP_REL_DONE;
+}
 #else /* !CONFIG_LIVEPATCH */
 static inline bool is_livepatch_module(struct module *mod)
 {
 	return false;
 }
+
+static inline bool mod_klp_rel_completed(struct module *mod)
+{
+	return true;
+}
 #endif /* CONFIG_LIVEPATCH */
 
 bool is_module_sig_enforced(void);

kernel/jump_label.c

@@ -530,6 +530,9 @@ void jump_label_apply_nops(struct module *mod)
 	struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
 	struct jump_entry *iter;
 
+	if (unlikely(!mod_klp_rel_completed(mod)))
+		return;
+
 	/* if the module doesn't have jump label entries, just return */
 	if (iter_start == iter_stop)
 		return;
@@ -549,6 +552,9 @@ static int jump_label_add_module(struct module *mod)
 	struct static_key *key = NULL;
 	struct static_key_mod *jlm, *jlm2;
 
+	if (unlikely(!mod_klp_rel_completed(mod)))
+		return 0;
+
 	/* if the module doesn't have jump label entries, just return */
 	if (iter_start == iter_stop)
 		return 0;
@@ -699,6 +705,16 @@ static struct notifier_block jump_label_module_nb = {
 	.priority = 1, /* higher than tracepoints */
 };
 
+int jump_label_register(struct module *mod)
+{
+	int ret;
+
+	ret = jump_label_module_notify(&jump_label_module_nb,
+			MODULE_STATE_COMING, mod);
+
+	return notifier_to_errno(ret);
+}
+
 static __init int jump_label_init_module(void)
 {
 	return register_module_notifier(&jump_label_module_nb);

kernel/livepatch/core.c

@@ -1074,10 +1074,17 @@ static int klp_init_object_loaded(struct klp_patch *patch,
 	}
 
 	arch_klp_init_object_loaded(patch, obj);
+
+	set_mod_klp_rel_state(patch->mod, MODULE_KLP_REL_DONE);
+	jump_label_apply_nops(patch->mod);
 	module_enable_ro(patch->mod, true);
 
 	mutex_unlock(&text_mutex);
 
+	ret = jump_label_register(patch->mod);
+	if (ret)
+		return ret;
+
 	klp_for_each_func(obj, func) {
 		ret = klp_find_object_symbol(obj->name, func->old_name,
 					     func->old_sympos,

kernel/module.c

@@ -2873,7 +2873,10 @@ static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
 		add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
 		pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n",
 			       mod->name);
-	}
+
+		set_mod_klp_rel_state(mod, MODULE_KLP_REL_UNDO);
+	} else
+		set_mod_klp_rel_state(mod, MODULE_KLP_REL_NONE);
 
 	return 0;
 }