diff --git a/arch/x86/kernel/kprobes_32.c b/arch/x86/kernel/kprobes_32.c index e7d0d3c2ef6473faeb460e7e77e20e579eedeb23..06b86e5617f62c739bdc4b448985d46462078273 100644 --- a/arch/x86/kernel/kprobes_32.c +++ b/arch/x86/kernel/kprobes_32.c @@ -584,7 +584,7 @@ static int __kprobes post_kprobe_handler(struct pt_regs *regs) return 1; } -static int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) +int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) { struct kprobe *cur = kprobe_running(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); @@ -666,7 +666,6 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, ret = NOTIFY_STOP; break; case DIE_GPF: - case DIE_PAGE_FAULT: /* kprobe_running() needs smp_processor_id() */ preempt_disable(); if (kprobe_running() && diff --git a/arch/x86/kernel/kprobes_64.c b/arch/x86/kernel/kprobes_64.c index 62e28e52d784c8da494ebb6d2e045f5d8a9cb2b6..7c16506d681fc56f985c65f2c4b22dea7a5019ea 100644 --- a/arch/x86/kernel/kprobes_64.c +++ b/arch/x86/kernel/kprobes_64.c @@ -657,7 +657,6 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, ret = NOTIFY_STOP; break; case DIE_GPF: - case DIE_PAGE_FAULT: /* kprobe_running() needs smp_processor_id() */ preempt_disable(); if (kprobe_running() && diff --git a/arch/x86/mm/fault_32.c b/arch/x86/mm/fault_32.c index fcb38e7f35434113896abc3caad887c9aad665d1..be72c2a5b03b4a1e4fea65b3431faf21d1491d3a 100644 --- a/arch/x86/mm/fault_32.c +++ b/arch/x86/mm/fault_32.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -32,33 +33,27 @@ extern void die(const char *,struct pt_regs *,long); -static ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain); - -int register_page_fault_notifier(struct notifier_block *nb) +#ifdef CONFIG_KPROBES +static inline int notify_page_fault(struct pt_regs *regs) { - vmalloc_sync_all(); - return atomic_notifier_chain_register(¬ify_page_fault_chain, nb); -} -EXPORT_SYMBOL_GPL(register_page_fault_notifier); + int ret = 0; + + /* kprobe_running() needs smp_processor_id() */ + if (!user_mode_vm(regs)) { + preempt_disable(); + if (kprobe_running() && kprobe_fault_handler(regs, 14)) + ret = 1; + preempt_enable(); + } -int unregister_page_fault_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_unregister(¬ify_page_fault_chain, nb); + return ret; } -EXPORT_SYMBOL_GPL(unregister_page_fault_notifier); - -static inline int notify_page_fault(struct pt_regs *regs, long err) +#else +static inline int notify_page_fault(struct pt_regs *regs) { - struct die_args args = { - .regs = regs, - .str = "page fault", - .err = err, - .trapnr = 14, - .signr = SIGSEGV - }; - return atomic_notifier_call_chain(¬ify_page_fault_chain, - DIE_PAGE_FAULT, &args); + return 0; } +#endif /* * Return EIP plus the CS segment base. The segment limit is also @@ -331,7 +326,7 @@ fastcall void __kprobes do_page_fault(struct pt_regs *regs, if (unlikely(address >= TASK_SIZE)) { if (!(error_code & 0x0000000d) && vmalloc_fault(address) >= 0) return; - if (notify_page_fault(regs, error_code) == NOTIFY_STOP) + if (notify_page_fault(regs)) return; /* * Don't take the mm semaphore here. If we fixup a prefetch @@ -340,7 +335,7 @@ fastcall void __kprobes do_page_fault(struct pt_regs *regs, goto bad_area_nosemaphore; } - if (notify_page_fault(regs, error_code) == NOTIFY_STOP) + if (notify_page_fault(regs)) return; /* It's safe to allow irq's after cr2 has been saved and the vmalloc diff --git a/arch/x86/mm/fault_64.c b/arch/x86/mm/fault_64.c index 54816adb8e933c8c09e41afcf3008331afb5adc6..5e0e54906c488fe5b7266d013c072672a606ba29 100644 --- a/arch/x86/mm/fault_64.c +++ b/arch/x86/mm/fault_64.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -40,34 +41,27 @@ #define PF_RSVD (1<<3) #define PF_INSTR (1<<4) -static ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain); - -/* Hook to register for page fault notifications */ -int register_page_fault_notifier(struct notifier_block *nb) +#ifdef CONFIG_KPROBES +static inline int notify_page_fault(struct pt_regs *regs) { - vmalloc_sync_all(); - return atomic_notifier_chain_register(¬ify_page_fault_chain, nb); -} -EXPORT_SYMBOL_GPL(register_page_fault_notifier); + int ret = 0; + + /* kprobe_running() needs smp_processor_id() */ + if (!user_mode(regs)) { + preempt_disable(); + if (kprobe_running() && kprobe_fault_handler(regs, 14)) + ret = 1; + preempt_enable(); + } -int unregister_page_fault_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_unregister(¬ify_page_fault_chain, nb); + return ret; } -EXPORT_SYMBOL_GPL(unregister_page_fault_notifier); - -static inline int notify_page_fault(struct pt_regs *regs, long err) +#else +static inline int notify_page_fault(struct pt_regs *regs) { - struct die_args args = { - .regs = regs, - .str = "page fault", - .err = err, - .trapnr = 14, - .signr = SIGSEGV - }; - return atomic_notifier_call_chain(¬ify_page_fault_chain, - DIE_PAGE_FAULT, &args); + return 0; } +#endif /* Sometimes the CPU reports invalid exceptions on prefetch. Check that here and ignore. @@ -345,7 +339,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, if (vmalloc_fault(address) >= 0) return; } - if (notify_page_fault(regs, error_code) == NOTIFY_STOP) + if (notify_page_fault(regs)) return; /* * Don't take the mm semaphore here. If we fixup a prefetch @@ -354,7 +348,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, goto bad_area_nosemaphore; } - if (notify_page_fault(regs, error_code) == NOTIFY_STOP) + if (notify_page_fault(regs)) return; if (likely(regs->eflags & X86_EFLAGS_IF)) diff --git a/include/asm-avr32/kdebug.h b/include/asm-avr32/kdebug.h index 7f54e2b15d136ab5e2ee2234ef49fd0700ab2473..fd7e99046b2f366be239078692c957381c223bc6 100644 --- a/include/asm-avr32/kdebug.h +++ b/include/asm-avr32/kdebug.h @@ -1,26 +1,10 @@ #ifndef __ASM_AVR32_KDEBUG_H #define __ASM_AVR32_KDEBUG_H -#include - /* Grossly misnamed. */ enum die_val { DIE_BREAKPOINT, DIE_SSTEP, }; -/* - * These are only here because kprobes.c wants them to implement a - * blatant layering violation. Will hopefully go away soon once all - * architectures are updated. - */ -static inline int register_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} -static inline int unregister_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} - #endif /* __ASM_AVR32_KDEBUG_H */ diff --git a/include/asm-avr32/kprobes.h b/include/asm-avr32/kprobes.h index 190a6377c80969f7b5ef5313457b42fec539a99d..0f3e636e6e4dda8d36c89b50f697777f59bdf88d 100644 --- a/include/asm-avr32/kprobes.h +++ b/include/asm-avr32/kprobes.h @@ -17,8 +17,6 @@ typedef u16 kprobe_opcode_t; #define BREAKPOINT_INSTRUCTION 0xd673 /* breakpoint */ #define MAX_INSN_SIZE 2 -#define ARCH_INACTIVE_KPROBE_COUNT 1 - #define arch_remove_kprobe(p) do { } while (0) /* Architecture specific copy of original instruction */ diff --git a/include/asm-ia64/kdebug.h b/include/asm-ia64/kdebug.h index 320cd8e754ea44a9841e874251f2e7113721e84b..35e49407d06c6bdada5952abca95a9a45786c23d 100644 --- a/include/asm-ia64/kdebug.h +++ b/include/asm-ia64/kdebug.h @@ -26,21 +26,6 @@ * 2005-Oct Keith Owens . Expand notify_die to cover more * events. */ -#include - -/* - * These are only here because kprobes.c wants them to implement a - * blatant layering violation. Will hopefully go away soon once all - * architectures are updated. - */ -static inline int register_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} -static inline int unregister_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} enum die_val { DIE_BREAK = 1, diff --git a/include/asm-ia64/kprobes.h b/include/asm-ia64/kprobes.h index 067d9dea68f9d627f41ba634e7169185bafcf5e4..6c79edf24d73adc1572b885417394bcb6c02ae13 100644 --- a/include/asm-ia64/kprobes.h +++ b/include/asm-ia64/kprobes.h @@ -83,7 +83,6 @@ struct kprobe_ctlblk { }; #define ARCH_SUPPORTS_KRETPROBES -#define ARCH_INACTIVE_KPROBE_COUNT 1 #define SLOT0_OPCODE_SHIFT (37) #define SLOT1_p1_OPCODE_SHIFT (37 - (64-46)) diff --git a/include/asm-powerpc/kdebug.h b/include/asm-powerpc/kdebug.h index 295f0162c60827a750664b9ece653d134f1a277f..ae6d206728afc476d015b15ee2a9f5e2588f467e 100644 --- a/include/asm-powerpc/kdebug.h +++ b/include/asm-powerpc/kdebug.h @@ -2,25 +2,6 @@ #define _ASM_POWERPC_KDEBUG_H #ifdef __KERNEL__ -/* nearly identical to x86_64/i386 code */ - -#include - -/* - * These are only here because kprobes.c wants them to implement a - * blatant layering violation. Will hopefully go away soon once all - * architectures are updated. - */ -static inline int register_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} -static inline int unregister_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} -extern struct atomic_notifier_head powerpc_die_chain; - /* Grossly misnamed. */ enum die_val { DIE_OOPS = 1, diff --git a/include/asm-powerpc/kprobes.h b/include/asm-powerpc/kprobes.h index 8b08b447d6f31e2e300e39b3813dd090b7c2233f..c16973d5de6287dc4c14d1572065d747c719416e 100644 --- a/include/asm-powerpc/kprobes.h +++ b/include/asm-powerpc/kprobes.h @@ -81,7 +81,6 @@ typedef unsigned int kprobe_opcode_t; #endif #define ARCH_SUPPORTS_KRETPROBES -#define ARCH_INACTIVE_KPROBE_COUNT 1 #define flush_insn_slot(p) do { } while (0) void kretprobe_trampoline(void); diff --git a/include/asm-s390/kdebug.h b/include/asm-s390/kdebug.h index 04418af08f8511a6eea7c1d0ad09e04b49979e5e..40db27cd6e608df9c54285c6e1b52c6d572277f4 100644 --- a/include/asm-s390/kdebug.h +++ b/include/asm-s390/kdebug.h @@ -4,24 +4,9 @@ /* * Feb 2006 Ported to s390 */ -#include struct pt_regs; -/* - * These are only here because kprobes.c wants them to implement a - * blatant layering violation. Will hopefully go away soon once all - * architectures are updated. - */ -static inline int register_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} -static inline int unregister_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} - enum die_val { DIE_OOPS = 1, DIE_BPT, diff --git a/include/asm-s390/kprobes.h b/include/asm-s390/kprobes.h index 340ba10446ea46d2aeca453903532bc537627610..8bc67cc9ffd2972a64f9da56564892e54d48d7c5 100644 --- a/include/asm-s390/kprobes.h +++ b/include/asm-s390/kprobes.h @@ -47,7 +47,6 @@ typedef u16 kprobe_opcode_t; : (((unsigned long)current_thread_info()) + THREAD_SIZE - (ADDR))) #define ARCH_SUPPORTS_KRETPROBES -#define ARCH_INACTIVE_KPROBE_COUNT 0 #define KPROBE_SWAP_INST 0x10 diff --git a/include/asm-sh/kdebug.h b/include/asm-sh/kdebug.h index 382cfc7deb732b233170b3e1ec69edb6d14abb17..49cd69051a8896392fa1c9335c1770c802d1aa15 100644 --- a/include/asm-sh/kdebug.h +++ b/include/asm-sh/kdebug.h @@ -1,8 +1,6 @@ #ifndef __ASM_SH_KDEBUG_H #define __ASM_SH_KDEBUG_H -#include - /* Grossly misnamed. */ enum die_val { DIE_TRAP, diff --git a/include/asm-sparc64/kdebug.h b/include/asm-sparc64/kdebug.h index 9974c7b0aebc18563568cd50a5b5c2c457e5b908..f905b773235a2f977db3383559629d4fbd3e4fda 100644 --- a/include/asm-sparc64/kdebug.h +++ b/include/asm-sparc64/kdebug.h @@ -1,26 +1,8 @@ #ifndef _SPARC64_KDEBUG_H #define _SPARC64_KDEBUG_H -/* Nearly identical to x86_64/i386 code. */ - -#include - struct pt_regs; -/* - * These are only here because kprobes.c wants them to implement a - * blatant layering violation. Will hopefully go away soon once all - * architectures are updated. - */ -static inline int register_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} -static inline int unregister_page_fault_notifier(struct notifier_block *nb) -{ - return 0; -} - extern void bad_trap(struct pt_regs *, long); /* Grossly misnamed. */ diff --git a/include/asm-sparc64/kprobes.h b/include/asm-sparc64/kprobes.h index 7f6774dca5f49f778c434c055ab7355e3711a4c4..a04145b77f9635e8a7a30504bef7f1b2cfdcc1e1 100644 --- a/include/asm-sparc64/kprobes.h +++ b/include/asm-sparc64/kprobes.h @@ -11,7 +11,6 @@ typedef u32 kprobe_opcode_t; #define MAX_INSN_SIZE 2 #define arch_remove_kprobe(p) do {} while (0) -#define ARCH_INACTIVE_KPROBE_COUNT 0 #define flush_insn_slot(p) \ do { flushi(&(p)->ainsn.insn[0]); \ diff --git a/include/asm-x86/kdebug_32.h b/include/asm-x86/kdebug_32.h index a185b5f73e7ff466322cbaa8c8b9743925cce11d..181d437eef4bd059e320155d1a39dab15d15c649 100644 --- a/include/asm-x86/kdebug_32.h +++ b/include/asm-x86/kdebug_32.h @@ -5,14 +5,9 @@ * Aug-05 2004 Ported by Prasanna S Panchamukhi * from x86_64 architecture. */ -#include struct pt_regs; -extern int register_page_fault_notifier(struct notifier_block *); -extern int unregister_page_fault_notifier(struct notifier_block *); - - /* Grossly misnamed. */ enum die_val { DIE_OOPS = 1, @@ -27,7 +22,6 @@ enum die_val { DIE_GPF, DIE_CALL, DIE_NMI_IPI, - DIE_PAGE_FAULT, }; #endif diff --git a/include/asm-x86/kdebug_64.h b/include/asm-x86/kdebug_64.h index d7e2bcf49e4f0ee5ab826b8a2108c2f03f0d0b43..df413e05375e7f6b927c6128c38442d8c3474c56 100644 --- a/include/asm-x86/kdebug_64.h +++ b/include/asm-x86/kdebug_64.h @@ -1,13 +1,10 @@ #ifndef _X86_64_KDEBUG_H #define _X86_64_KDEBUG_H 1 -#include +#include struct pt_regs; -extern int register_page_fault_notifier(struct notifier_block *); -extern int unregister_page_fault_notifier(struct notifier_block *); - /* Grossly misnamed. */ enum die_val { DIE_OOPS = 1, @@ -22,7 +19,6 @@ enum die_val { DIE_GPF, DIE_CALL, DIE_NMI_IPI, - DIE_PAGE_FAULT, }; extern void printk_address(unsigned long address); diff --git a/include/asm-x86/kprobes_32.h b/include/asm-x86/kprobes_32.h index 06f7303c30ca15d111061cfcace36ec043b9362d..f2489d07ce88cdef0dec6d5d5d605a0d84a3c4f0 100644 --- a/include/asm-x86/kprobes_32.h +++ b/include/asm-x86/kprobes_32.h @@ -43,7 +43,6 @@ typedef u8 kprobe_opcode_t; : (((unsigned long)current_thread_info()) + THREAD_SIZE - (ADDR))) #define ARCH_SUPPORTS_KRETPROBES -#define ARCH_INACTIVE_KPROBE_COUNT 0 #define flush_insn_slot(p) do { } while (0) void arch_remove_kprobe(struct kprobe *p); @@ -89,4 +88,5 @@ static inline void restore_interrupts(struct pt_regs *regs) extern int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data); +extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr); #endif /* _ASM_KPROBES_H */ diff --git a/include/asm-x86/kprobes_64.h b/include/asm-x86/kprobes_64.h index 7db825403e013129655fab261035b1391944c888..3f495e5308b13499f4d7122e2bb90510bc293417 100644 --- a/include/asm-x86/kprobes_64.h +++ b/include/asm-x86/kprobes_64.h @@ -42,7 +42,6 @@ typedef u8 kprobe_opcode_t; : (((unsigned long)current_thread_info()) + THREAD_SIZE - (ADDR))) #define ARCH_SUPPORTS_KRETPROBES -#define ARCH_INACTIVE_KPROBE_COUNT 1 void kretprobe_trampoline(void); extern void arch_remove_kprobe(struct kprobe *p); diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 4b8a4493c541873a946e9d62161459fa4f94c35c..f9798ff7899f4a610823fd641fe5e2dde0e746f7 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -64,7 +64,6 @@ static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; -static atomic_t kprobe_count; /* NOTE: change this value only with kprobe_mutex held */ static bool kprobe_enabled; @@ -73,11 +72,6 @@ DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */ static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; -static struct notifier_block kprobe_page_fault_nb = { - .notifier_call = kprobe_exceptions_notify, - .priority = 0x7fffffff /* we need to notified first */ -}; - #ifdef __ARCH_WANT_KPROBES_INSN_SLOT /* * kprobe->ainsn.insn points to the copy of the instruction to be @@ -556,8 +550,6 @@ static int __kprobes __register_kprobe(struct kprobe *p, old_p = get_kprobe(p->addr); if (old_p) { ret = register_aggr_kprobe(old_p, p); - if (!ret) - atomic_inc(&kprobe_count); goto out; } @@ -569,13 +561,9 @@ static int __kprobes __register_kprobe(struct kprobe *p, hlist_add_head_rcu(&p->hlist, &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); - if (kprobe_enabled) { - if (atomic_add_return(1, &kprobe_count) == \ - (ARCH_INACTIVE_KPROBE_COUNT + 1)) - register_page_fault_notifier(&kprobe_page_fault_nb); - + if (kprobe_enabled) arch_arm_kprobe(p); - } + out: mutex_unlock(&kprobe_mutex); @@ -658,16 +646,6 @@ void __kprobes unregister_kprobe(struct kprobe *p) } mutex_unlock(&kprobe_mutex); } - - /* Call unregister_page_fault_notifier() - * if no probes are active - */ - mutex_lock(&kprobe_mutex); - if (atomic_add_return(-1, &kprobe_count) == \ - ARCH_INACTIVE_KPROBE_COUNT) - unregister_page_fault_notifier(&kprobe_page_fault_nb); - mutex_unlock(&kprobe_mutex); - return; } static struct notifier_block kprobe_exceptions_nb = { @@ -815,7 +793,6 @@ static int __init init_kprobes(void) INIT_HLIST_HEAD(&kprobe_table[i]); INIT_HLIST_HEAD(&kretprobe_inst_table[i]); } - atomic_set(&kprobe_count, 0); /* By default, kprobes are enabled */ kprobe_enabled = true; @@ -921,13 +898,6 @@ static void __kprobes enable_all_kprobes(void) if (kprobe_enabled) goto already_enabled; - /* - * Re-register the page fault notifier only if there are any - * active probes at the time of enabling kprobes globally - */ - if (atomic_read(&kprobe_count) > ARCH_INACTIVE_KPROBE_COUNT) - register_page_fault_notifier(&kprobe_page_fault_nb); - for (i = 0; i < KPROBE_TABLE_SIZE; i++) { head = &kprobe_table[i]; hlist_for_each_entry_rcu(p, node, head, hlist) @@ -968,10 +938,7 @@ static void __kprobes disable_all_kprobes(void) mutex_unlock(&kprobe_mutex); /* Allow all currently running kprobes to complete */ synchronize_sched(); - - mutex_lock(&kprobe_mutex); - /* Unconditionally unregister the page_fault notifier */ - unregister_page_fault_notifier(&kprobe_page_fault_nb); + return; already_disabled: mutex_unlock(&kprobe_mutex);