提交 1b1d9258 编写于 作者: J Jan Beulich 提交者: Ingo Molnar

x86-64: Modify copy_user_generic() alternatives mechanism

In order to avoid unnecessary chains of branches, rather than
implementing copy_user_generic() as a function consisting of
just a single (possibly patched) branch, instead properly deal
with patching call instructions in the alternative instructions
framework, and move the patching into the callers.

As a follow-on, one could also introduce something like
__EXPORT_SYMBOL_ALT() to avoid patching call sites in modules.
Signed-off-by: NJan Beulich <jbeulich@novell.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
LKML-Reference: <4B2BB8180200007800026AE7@vpn.id2.novell.com>
Signed-off-by: NIngo Molnar <mingo@elte.hu>
上级 499a5f1e
...@@ -125,11 +125,16 @@ static inline void alternatives_smp_switch(int smp) {} ...@@ -125,11 +125,16 @@ static inline void alternatives_smp_switch(int smp) {}
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
: output : "i" (0), ## input) : output : "i" (0), ## input)
/* Like alternative_io, but for replacing a direct call with another one. */
#define alternative_call(oldfunc, newfunc, feature, output, input...) \
asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
/* /*
* use this macro(s) if you need more than one output parameter * use this macro(s) if you need more than one output parameter
* in alternative_io * in alternative_io
*/ */
#define ASM_OUTPUT2(a, b) a, b #define ASM_OUTPUT2(a...) a
struct paravirt_patch_site; struct paravirt_patch_site;
#ifdef CONFIG_PARAVIRT #ifdef CONFIG_PARAVIRT
......
...@@ -8,6 +8,8 @@ ...@@ -8,6 +8,8 @@
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/prefetch.h> #include <linux/prefetch.h>
#include <linux/lockdep.h> #include <linux/lockdep.h>
#include <asm/alternative.h>
#include <asm/cpufeature.h>
#include <asm/page.h> #include <asm/page.h>
/* /*
...@@ -16,7 +18,24 @@ ...@@ -16,7 +18,24 @@
/* Handles exceptions in both to and from, but doesn't do access_ok */ /* Handles exceptions in both to and from, but doesn't do access_ok */
__must_check unsigned long __must_check unsigned long
copy_user_generic(void *to, const void *from, unsigned len); copy_user_generic_string(void *to, const void *from, unsigned len);
__must_check unsigned long
copy_user_generic_unrolled(void *to, const void *from, unsigned len);
static __always_inline __must_check unsigned long
copy_user_generic(void *to, const void *from, unsigned len)
{
unsigned ret;
alternative_call(copy_user_generic_unrolled,
copy_user_generic_string,
X86_FEATURE_REP_GOOD,
ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
"=d" (len)),
"1" (to), "2" (from), "3" (len)
: "memory", "rcx", "r8", "r9", "r10", "r11");
return ret;
}
__must_check unsigned long __must_check unsigned long
_copy_to_user(void __user *to, const void *from, unsigned len); _copy_to_user(void __user *to, const void *from, unsigned len);
......
...@@ -205,7 +205,7 @@ void __init_or_module apply_alternatives(struct alt_instr *start, ...@@ -205,7 +205,7 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
struct alt_instr *end) struct alt_instr *end)
{ {
struct alt_instr *a; struct alt_instr *a;
char insnbuf[MAX_PATCH_LEN]; u8 insnbuf[MAX_PATCH_LEN];
DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
for (a = start; a < end; a++) { for (a = start; a < end; a++) {
...@@ -223,6 +223,8 @@ void __init_or_module apply_alternatives(struct alt_instr *start, ...@@ -223,6 +223,8 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
} }
#endif #endif
memcpy(insnbuf, a->replacement, a->replacementlen); memcpy(insnbuf, a->replacement, a->replacementlen);
if (*insnbuf == 0xe8 && a->replacementlen == 5)
*(s32 *)(insnbuf + 1) += a->replacement - a->instr;
add_nops(insnbuf + a->replacementlen, add_nops(insnbuf + a->replacementlen,
a->instrlen - a->replacementlen); a->instrlen - a->replacementlen);
text_poke_early(instr, insnbuf, a->instrlen); text_poke_early(instr, insnbuf, a->instrlen);
......
...@@ -26,7 +26,8 @@ EXPORT_SYMBOL(__put_user_2); ...@@ -26,7 +26,8 @@ EXPORT_SYMBOL(__put_user_2);
EXPORT_SYMBOL(__put_user_4); EXPORT_SYMBOL(__put_user_4);
EXPORT_SYMBOL(__put_user_8); EXPORT_SYMBOL(__put_user_8);
EXPORT_SYMBOL(copy_user_generic); EXPORT_SYMBOL(copy_user_generic_string);
EXPORT_SYMBOL(copy_user_generic_unrolled);
EXPORT_SYMBOL(__copy_user_nocache); EXPORT_SYMBOL(__copy_user_nocache);
EXPORT_SYMBOL(_copy_from_user); EXPORT_SYMBOL(_copy_from_user);
EXPORT_SYMBOL(_copy_to_user); EXPORT_SYMBOL(_copy_to_user);
......
...@@ -90,12 +90,6 @@ ENTRY(_copy_from_user) ...@@ -90,12 +90,6 @@ ENTRY(_copy_from_user)
CFI_ENDPROC CFI_ENDPROC
ENDPROC(_copy_from_user) ENDPROC(_copy_from_user)
ENTRY(copy_user_generic)
CFI_STARTPROC
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
CFI_ENDPROC
ENDPROC(copy_user_generic)
.section .fixup,"ax" .section .fixup,"ax"
/* must zero dest */ /* must zero dest */
ENTRY(bad_from_user) ENTRY(bad_from_user)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册