kprobes.c 32.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
/*
 *  Kernel Probes (KProbes)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) IBM Corporation, 2002, 2004
 *
 * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
 *		Probes initial implementation ( includes contributions from
 *		Rusty Russell).
 * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
 *		interface to access function arguments.
25 26
 * 2004-Oct	Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
 *		<prasanna@in.ibm.com> adapted for x86_64 from i386.
L
Linus Torvalds 已提交
27 28
 * 2005-Mar	Roland McGrath <roland@redhat.com>
 *		Fixed to handle %rip-relative addressing mode correctly.
29 30 31 32
 * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
 *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
 *		<prasanna@in.ibm.com> added function-return probes.
 * 2005-May	Rusty Lynch <rusty.lynch@intel.com>
33
 *		Added function return probes functionality
34
 * 2006-Feb	Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added
35
 *		kprobe-booster and kretprobe-booster for i386.
36
 * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
37
 *		and kretprobe-booster for x86-64
38
 * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven
39 40
 *		<arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com>
 *		unified x86 kprobes code.
L
Linus Torvalds 已提交
41 42 43 44 45
 */
#include <linux/kprobes.h>
#include <linux/ptrace.h>
#include <linux/string.h>
#include <linux/slab.h>
46
#include <linux/hardirq.h>
L
Linus Torvalds 已提交
47
#include <linux/preempt.h>
48
#include <linux/module.h>
49
#include <linux/kdebug.h>
50
#include <linux/kallsyms.h>
51
#include <linux/ftrace.h>
52

53 54
#include <asm/cacheflush.h>
#include <asm/desc.h>
L
Linus Torvalds 已提交
55
#include <asm/pgtable.h>
56
#include <asm/uaccess.h>
57
#include <asm/alternative.h>
58
#include <asm/insn.h>
59
#include <asm/debugreg.h>
L
Linus Torvalds 已提交
60

61 62
#include "kprobes-common.h"

L
Linus Torvalds 已提交
63 64
void jprobe_return_end(void);

65 66
DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
L
Linus Torvalds 已提交
67

68
#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
69 70 71 72 73 74 75 76 77 78

#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
	  (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) |   \
	  (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) |   \
	  (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf))    \
	 << (row % 32))
	/*
	 * Undefined/reserved opcodes, conditional jump, Opcode Extension
	 * Groups, and some special opcodes can not boost.
79 80 81
	 * This is non-const and volatile to keep gcc from statically
	 * optimizing it out, as variable_test_bit makes gcc think only
	 * *(unsigned long*) is used. 
82
	 */
83
static volatile u32 twobyte_is_boostable[256 / 32] = {
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
	/*      ----------------------------------------------          */
	W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */
	W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 10 */
	W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */
	W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
	W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
	W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */
	W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
	W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */
	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
	W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */
	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */
	W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
	W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */
	W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */
	W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0)   /* f0 */
	/*      -----------------------------------------------         */
	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
};
#undef W

107 108 109 110 111
struct kretprobe_blackpoint kretprobe_blacklist[] = {
	{"__switch_to", }, /* This function switches only current task, but
			      doesn't switch kernel stack.*/
	{NULL, NULL}	/* Terminator */
};
112

113 114
const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);

115
static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
116
{
117 118
	struct __arch_relative_insn {
		u8 op;
119
		s32 raddr;
120 121 122 123 124 125 126 127
	} __attribute__((packed)) *insn;

	insn = (struct __arch_relative_insn *)from;
	insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
	insn->op = op;
}

/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
128
void __kprobes synthesize_reljump(void *from, void *to)
129 130
{
	__synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE);
131 132
}

133 134 135 136 137 138
/* Insert a call instruction at address 'from', which calls address 'to'.*/
void __kprobes synthesize_relcall(void *from, void *to)
{
	__synthesize_relative_insn(from, to, RELATIVECALL_OPCODE);
}

139
/*
140
 * Skip the prefixes of the instruction.
141
 */
142
static kprobe_opcode_t *__kprobes skip_prefixes(kprobe_opcode_t *insn)
143
{
144 145 146 147 148 149 150
	insn_attr_t attr;

	attr = inat_get_opcode_attribute((insn_byte_t)*insn);
	while (inat_is_legacy_prefix(attr)) {
		insn++;
		attr = inat_get_opcode_attribute((insn_byte_t)*insn);
	}
151
#ifdef CONFIG_X86_64
152 153
	if (inat_is_rex_prefix(attr))
		insn++;
154
#endif
155
	return insn;
156 157
}

158
/*
159 160
 * Returns non-zero if opcode is boostable.
 * RIP relative instructions are adjusted at copying time in 64 bits mode
161
 */
162
int __kprobes can_boost(kprobe_opcode_t *opcodes)
163 164 165 166
{
	kprobe_opcode_t opcode;
	kprobe_opcode_t *orig_opcodes = opcodes;

167
	if (search_exception_tables((unsigned long)opcodes))
168 169
		return 0;	/* Page fault may occur on this address. */

170 171 172 173 174 175 176 177 178
retry:
	if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
		return 0;
	opcode = *(opcodes++);

	/* 2nd-byte opcode */
	if (opcode == 0x0f) {
		if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
			return 0;
179 180
		return test_bit(*opcodes,
				(unsigned long *)twobyte_is_boostable);
181 182 183
	}

	switch (opcode & 0xf0) {
184
#ifdef CONFIG_X86_64
185 186
	case 0x40:
		goto retry; /* REX prefix is boostable */
187
#endif
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
	case 0x60:
		if (0x63 < opcode && opcode < 0x67)
			goto retry; /* prefixes */
		/* can't boost Address-size override and bound */
		return (opcode != 0x62 && opcode != 0x67);
	case 0x70:
		return 0; /* can't boost conditional jump */
	case 0xc0:
		/* can't boost software-interruptions */
		return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
	case 0xd0:
		/* can boost AA* and XLAT */
		return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
	case 0xe0:
		/* can boost in/out and absolute jmps */
		return ((opcode & 0x04) || opcode == 0xea);
	case 0xf0:
		if ((opcode & 0x0c) == 0 && opcode != 0xf1)
			goto retry; /* lock/rep(ne) prefix */
		/* clear and set flags are boostable */
		return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
	default:
		/* segment override prefixes are boostable */
		if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
			goto retry; /* prefixes */
		/* CS override prefix and call are not boostable */
		return (opcode != 0x2e && opcode != 0x9a);
	}
}

218 219
static unsigned long
__recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
220 221
{
	struct kprobe *kp;
222

223
	kp = get_kprobe((void *)addr);
224
	/* There is no probe, return original address */
225
	if (!kp)
226
		return addr;
227 228 229 230

	/*
	 *  Basically, kp->ainsn.insn has an original instruction.
	 *  However, RIP-relative instruction can not do single-stepping
231
	 *  at different place, __copy_instruction() tweaks the displacement of
232 233 234 235 236 237 238 239 240 241 242
	 *  that instruction. In that case, we can't recover the instruction
	 *  from the kp->ainsn.insn.
	 *
	 *  On the other hand, kp->opcode has a copy of the first byte of
	 *  the probed instruction, which is overwritten by int3. And
	 *  the instruction at kp->addr is not modified by kprobes except
	 *  for the first byte, we can recover the original instruction
	 *  from it and kp->opcode.
	 */
	memcpy(buf, kp->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
	buf[0] = kp->opcode;
243 244 245 246 247 248 249 250
	return (unsigned long)buf;
}

/*
 * Recover the probed instruction at addr for further analysis.
 * Caller must lock kprobes by kprobe_mutex, or disable preemption
 * for preventing to release referencing kprobes.
 */
251
unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr)
252 253 254 255 256 257 258 259
{
	unsigned long __addr;

	__addr = __recover_optprobed_insn(buf, addr);
	if (__addr != addr)
		return __addr;

	return __recover_probed_insn(buf, addr);
260 261 262 263 264
}

/* Check if paddr is at an instruction boundary */
static int __kprobes can_probe(unsigned long paddr)
{
265
	unsigned long addr, __addr, offset = 0;
266 267 268
	struct insn insn;
	kprobe_opcode_t buf[MAX_INSN_SIZE];

N
Namhyung Kim 已提交
269
	if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
270 271 272 273 274 275 276 277 278
		return 0;

	/* Decode instructions */
	addr = paddr - offset;
	while (addr < paddr) {
		/*
		 * Check if the instruction has been modified by another
		 * kprobe, in which case we replace the breakpoint by the
		 * original instruction in our buffer.
279 280 281
		 * Also, jump optimization will change the breakpoint to
		 * relative-jump. Since the relative-jump itself is
		 * normally used, we just go through if there is no kprobe.
282
		 */
283 284
		__addr = recover_probed_instruction(buf, addr);
		kernel_insn_init(&insn, (void *)__addr);
285
		insn_get_length(&insn);
286 287 288 289 290 291 292

		/*
		 * Another debugging subsystem might insert this breakpoint.
		 * In that case, we can't recover it.
		 */
		if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
			return 0;
293 294 295 296 297 298
		addr += insn.length;
	}

	return (addr == paddr);
}

L
Linus Torvalds 已提交
299
/*
300
 * Returns non-zero if opcode modifies the interrupt flag.
L
Linus Torvalds 已提交
301
 */
302
static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)
L
Linus Torvalds 已提交
303
{
304 305 306
	/* Skip prefixes */
	insn = skip_prefixes(insn);

L
Linus Torvalds 已提交
307 308 309 310 311 312 313
	switch (*insn) {
	case 0xfa:		/* cli */
	case 0xfb:		/* sti */
	case 0xcf:		/* iret/iretd */
	case 0x9d:		/* popf/popfd */
		return 1;
	}
314

L
Linus Torvalds 已提交
315 316 317 318
	return 0;
}

/*
319 320
 * Copy an instruction and adjust the displacement if the instruction
 * uses the %rip-relative addressing mode.
321
 * If it does, Return the address of the 32-bit displacement word.
L
Linus Torvalds 已提交
322
 * If not, return null.
323
 * Only applicable to 64-bit x86.
L
Linus Torvalds 已提交
324
 */
325
int __kprobes __copy_instruction(u8 *dest, u8 *src)
L
Linus Torvalds 已提交
326
{
327
	struct insn insn;
328
	kprobe_opcode_t buf[MAX_INSN_SIZE];
329

330
	kernel_insn_init(&insn, (void *)recover_probed_instruction(buf, (unsigned long)src));
331
	insn_get_length(&insn);
332
	/* Another subsystem puts a breakpoint, failed to recover */
333
	if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
334
		return 0;
335 336 337
	memcpy(dest, insn.kaddr, insn.length);

#ifdef CONFIG_X86_64
338 339 340
	if (insn_rip_relative(&insn)) {
		s64 newdisp;
		u8 *disp;
341
		kernel_insn_init(&insn, dest);
342 343 344 345 346 347 348 349 350 351 352 353 354
		insn_get_displacement(&insn);
		/*
		 * The copied instruction uses the %rip-relative addressing
		 * mode.  Adjust the displacement for the difference between
		 * the original location of this instruction and the location
		 * of the copy that will actually be run.  The tricky bit here
		 * is making sure that the sign extension happens correctly in
		 * this calculation, since we need a signed 32-bit result to
		 * be sign-extended to 64 bits when it's added to the %rip
		 * value and yield the same 64-bit result that the sign-
		 * extension of the original signed 32-bit displacement would
		 * have given.
		 */
355
		newdisp = (u8 *) src + (s64) insn.displacement.value - (u8 *) dest;
356
		BUG_ON((s64) (s32) newdisp != newdisp); /* Sanity check.  */
357
		disp = (u8 *) dest + insn_offset_displacement(&insn);
358
		*(s32 *) disp = (s32) newdisp;
L
Linus Torvalds 已提交
359
	}
360
#endif
361
	return insn.length;
362
}
L
Linus Torvalds 已提交
363

364
static void __kprobes arch_copy_kprobe(struct kprobe *p)
L
Linus Torvalds 已提交
365
{
366 367 368
	/* Copy an instruction with recovering if other optprobe modifies it.*/
	__copy_instruction(p->ainsn.insn, p->addr);

369
	/*
370 371
	 * __copy_instruction can modify the displacement of the instruction,
	 * but it doesn't affect boostable check.
372
	 */
373
	if (can_boost(p->ainsn.insn))
374
		p->ainsn.boostable = 0;
375
	else
376
		p->ainsn.boostable = -1;
377

378 379
	/* Also, displacement change doesn't affect the first byte */
	p->opcode = p->ainsn.insn[0];
L
Linus Torvalds 已提交
380 381
}

382 383
int __kprobes arch_prepare_kprobe(struct kprobe *p)
{
384 385 386
	if (alternatives_text_reserved(p->addr, p->addr))
		return -EINVAL;

387 388
	if (!can_probe((unsigned long)p->addr))
		return -EILSEQ;
389 390 391 392 393 394 395 396
	/* insn: must be on special executable page on x86. */
	p->ainsn.insn = get_insn_slot();
	if (!p->ainsn.insn)
		return -ENOMEM;
	arch_copy_kprobe(p);
	return 0;
}

397
void __kprobes arch_arm_kprobe(struct kprobe *p)
L
Linus Torvalds 已提交
398
{
399
	text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
L
Linus Torvalds 已提交
400 401
}

402
void __kprobes arch_disarm_kprobe(struct kprobe *p)
L
Linus Torvalds 已提交
403
{
404
	text_poke(p->addr, &p->opcode, 1);
405 406
}

407
void __kprobes arch_remove_kprobe(struct kprobe *p)
408
{
409 410 411 412
	if (p->ainsn.insn) {
		free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
		p->ainsn.insn = NULL;
	}
L
Linus Torvalds 已提交
413 414
}

415
static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
416
{
417 418
	kcb->prev_kprobe.kp = kprobe_running();
	kcb->prev_kprobe.status = kcb->kprobe_status;
419 420
	kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;
	kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
421 422
}

423
static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
424
{
C
Christoph Lameter 已提交
425
	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
426
	kcb->kprobe_status = kcb->prev_kprobe.status;
427 428
	kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;
	kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
429 430
}

431
static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
432
				struct kprobe_ctlblk *kcb)
433
{
C
Christoph Lameter 已提交
434
	__this_cpu_write(current_kprobe, p);
435
	kcb->kprobe_saved_flags = kcb->kprobe_old_flags
436
		= (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
437
	if (is_IF_modifier(p->ainsn.insn))
438
		kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
439 440
}

H
Harvey Harrison 已提交
441
static void __kprobes clear_btf(void)
R
Roland McGrath 已提交
442
{
P
Peter Zijlstra 已提交
443 444 445 446 447 448
	if (test_thread_flag(TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl &= ~DEBUGCTLMSR_BTF;
		update_debugctlmsr(debugctl);
	}
R
Roland McGrath 已提交
449 450
}

H
Harvey Harrison 已提交
451
static void __kprobes restore_btf(void)
R
Roland McGrath 已提交
452
{
P
Peter Zijlstra 已提交
453 454 455 456 457 458
	if (test_thread_flag(TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl |= DEBUGCTLMSR_BTF;
		update_debugctlmsr(debugctl);
	}
R
Roland McGrath 已提交
459 460
}

461 462
void __kprobes
arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
463
{
464
	unsigned long *sara = stack_addr(regs);
465

466
	ri->ret_addr = (kprobe_opcode_t *) *sara;
467

468 469
	/* Replace the return addr with trampoline addr */
	*sara = (unsigned long) &kretprobe_trampoline;
470
}
471

472 473
static void __kprobes
setup_singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb, int reenter)
474
{
475 476 477
	if (setup_detour_execution(p, regs, reenter))
		return;

478
#if !defined(CONFIG_PREEMPT)
479 480
	if (p->ainsn.boostable == 1 && !p->post_handler) {
		/* Boost up -- we can execute copied instructions directly */
481 482 483 484 485 486 487
		if (!reenter)
			reset_current_kprobe();
		/*
		 * Reentering boosted probe doesn't reset current_kprobe,
		 * nor set current_kprobe, because it doesn't use single
		 * stepping.
		 */
488 489 490 491 492
		regs->ip = (unsigned long)p->ainsn.insn;
		preempt_enable_no_resched();
		return;
	}
#endif
493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
	if (reenter) {
		save_previous_kprobe(kcb);
		set_current_kprobe(p, regs, kcb);
		kcb->kprobe_status = KPROBE_REENTER;
	} else
		kcb->kprobe_status = KPROBE_HIT_SS;
	/* Prepare real single stepping */
	clear_btf();
	regs->flags |= X86_EFLAGS_TF;
	regs->flags &= ~X86_EFLAGS_IF;
	/* single step inline if the instruction is an int3 */
	if (p->opcode == BREAKPOINT_INSTRUCTION)
		regs->ip = (unsigned long)p->addr;
	else
		regs->ip = (unsigned long)p->ainsn.insn;
508 509
}

H
Harvey Harrison 已提交
510 511 512 513 514
/*
 * We have reentered the kprobe_handler(), since another probe was hit while
 * within the handler. We save the original kprobes variables and just single
 * step on the instruction of the new probe without calling any user handlers.
 */
515 516
static int __kprobes
reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
H
Harvey Harrison 已提交
517
{
518 519 520
	switch (kcb->kprobe_status) {
	case KPROBE_HIT_SSDONE:
	case KPROBE_HIT_ACTIVE:
521
		kprobes_inc_nmissed_count(p);
522
		setup_singlestep(p, regs, kcb, 1);
523 524
		break;
	case KPROBE_HIT_SS:
525 526 527 528 529 530 531 532 533 534
		/* A probe has been hit in the codepath leading up to, or just
		 * after, single-stepping of a probed instruction. This entire
		 * codepath should strictly reside in .kprobes.text section.
		 * Raise a BUG or we'll continue in an endless reentering loop
		 * and eventually a stack overflow.
		 */
		printk(KERN_WARNING "Unrecoverable kprobe detected at %p.\n",
		       p->addr);
		dump_kprobe(p);
		BUG();
535 536 537
	default:
		/* impossible cases */
		WARN_ON(1);
538
		return 0;
539
	}
540

541
	return 1;
H
Harvey Harrison 已提交
542
}
543

544 545
/*
 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
546
 * remain disabled throughout this function.
547 548
 */
static int __kprobes kprobe_handler(struct pt_regs *regs)
L
Linus Torvalds 已提交
549
{
550
	kprobe_opcode_t *addr;
551
	struct kprobe *p;
552 553
	struct kprobe_ctlblk *kcb;

554
	addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
555 556
	/*
	 * We don't want to be preempted for the entire
557 558 559
	 * duration of kprobe processing. We conditionally
	 * re-enable preemption at the end of this function,
	 * and also in reenter_kprobe() and setup_singlestep().
560 561
	 */
	preempt_disable();
L
Linus Torvalds 已提交
562

563
	kcb = get_kprobe_ctlblk();
564
	p = get_kprobe(addr);
565

566 567
	if (p) {
		if (kprobe_running()) {
568 569
			if (reenter_kprobe(p, regs, kcb))
				return 1;
L
Linus Torvalds 已提交
570
		} else {
571 572
			set_current_kprobe(p, regs, kcb);
			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
573

L
Linus Torvalds 已提交
574
			/*
575 576 577 578 579 580
			 * If we have no pre-handler or it returned 0, we
			 * continue with normal processing.  If we have a
			 * pre-handler and it returned non-zero, it prepped
			 * for calling the break_handler below on re-entry
			 * for jprobe processing, so get out doing nothing
			 * more here.
L
Linus Torvalds 已提交
581
			 */
582
			if (!p->pre_handler || !p->pre_handler(p, regs))
583
				setup_singlestep(p, regs, kcb, 0);
584
			return 1;
585
		}
586 587 588 589 590 591 592 593 594 595 596 597 598
	} else if (*addr != BREAKPOINT_INSTRUCTION) {
		/*
		 * The breakpoint instruction was removed right
		 * after we hit it.  Another cpu has removed
		 * either a probepoint or a debugger breakpoint
		 * at this address.  In either case, no further
		 * handling of this interrupt is appropriate.
		 * Back up over the (now missing) int3 and run
		 * the original instruction.
		 */
		regs->ip = (unsigned long)addr;
		preempt_enable_no_resched();
		return 1;
599
	} else if (kprobe_running()) {
C
Christoph Lameter 已提交
600
		p = __this_cpu_read(current_kprobe);
601
		if (p->break_handler && p->break_handler(p, regs)) {
602
			setup_singlestep(p, regs, kcb, 0);
603
			return 1;
L
Linus Torvalds 已提交
604
		}
605
	} /* else: not a kprobe fault; let the kernel handle it */
L
Linus Torvalds 已提交
606

607
	preempt_enable_no_resched();
608
	return 0;
L
Linus Torvalds 已提交
609 610
}

611
/*
612 613
 * When a retprobed function returns, this code saves registers and
 * calls trampoline_handler() runs, which calls the kretprobe's handler.
614
 */
615
static void __used __kprobes kretprobe_trampoline_holder(void)
616
{
617 618
	asm volatile (
			".global kretprobe_trampoline\n"
619
			"kretprobe_trampoline: \n"
620
#ifdef CONFIG_X86_64
621 622 623
			/* We don't bother saving the ss register */
			"	pushq %rsp\n"
			"	pushfq\n"
624
			SAVE_REGS_STRING
625 626 627 628
			"	movq %rsp, %rdi\n"
			"	call trampoline_handler\n"
			/* Replace saved sp with true return address. */
			"	movq %rax, 152(%rsp)\n"
629
			RESTORE_REGS_STRING
630
			"	popfq\n"
631 632
#else
			"	pushf\n"
633
			SAVE_REGS_STRING
634 635 636
			"	movl %esp, %eax\n"
			"	call trampoline_handler\n"
			/* Move flags to cs */
637 638
			"	movl 56(%esp), %edx\n"
			"	movl %edx, 52(%esp)\n"
639
			/* Replace saved flags with true return address. */
640
			"	movl %eax, 56(%esp)\n"
641
			RESTORE_REGS_STRING
642 643
			"	popf\n"
#endif
644
			"	ret\n");
645
}
646 647

/*
648
 * Called from kretprobe_trampoline
649
 */
650
static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
651
{
B
bibo,mao 已提交
652
	struct kretprobe_instance *ri = NULL;
653
	struct hlist_head *head, empty_rp;
B
bibo,mao 已提交
654
	struct hlist_node *node, *tmp;
655
	unsigned long flags, orig_ret_address = 0;
656
	unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
657
	kprobe_opcode_t *correct_ret_addr = NULL;
658

659
	INIT_HLIST_HEAD(&empty_rp);
660
	kretprobe_hash_lock(current, &head, &flags);
661
	/* fixup registers */
662
#ifdef CONFIG_X86_64
663
	regs->cs = __KERNEL_CS;
664 665
#else
	regs->cs = __KERNEL_CS | get_kernel_rpl();
666
	regs->gs = 0;
667
#endif
668
	regs->ip = trampoline_address;
669
	regs->orig_ax = ~0UL;
670

671 672
	/*
	 * It is possible to have multiple instances associated with a given
673
	 * task either because multiple functions in the call path have
674
	 * return probes installed on them, and/or more than one
675 676 677
	 * return probe was registered for a target function.
	 *
	 * We can handle this because:
678
	 *     - instances are always pushed into the head of the list
679
	 *     - when multiple return probes are registered for the same
680 681 682
	 *	 function, the (chronologically) first instance's ret_addr
	 *	 will be the real return address, and all the rest will
	 *	 point to kretprobe_trampoline.
683 684
	 */
	hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
B
bibo,mao 已提交
685
		if (ri->task != current)
686
			/* another task is sharing our hash bucket */
B
bibo,mao 已提交
687
			continue;
688

689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
		orig_ret_address = (unsigned long)ri->ret_addr;

		if (orig_ret_address != trampoline_address)
			/*
			 * This is the real return address. Any other
			 * instances associated with this task are for
			 * other calls deeper on the call stack
			 */
			break;
	}

	kretprobe_assert(ri, orig_ret_address, trampoline_address);

	correct_ret_addr = ri->ret_addr;
	hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
		if (ri->task != current)
			/* another task is sharing our hash bucket */
			continue;

		orig_ret_address = (unsigned long)ri->ret_addr;
709
		if (ri->rp && ri->rp->handler) {
C
Christoph Lameter 已提交
710
			__this_cpu_write(current_kprobe, &ri->rp->kp);
711
			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
712
			ri->ret_addr = correct_ret_addr;
713
			ri->rp->handler(ri, regs);
C
Christoph Lameter 已提交
714
			__this_cpu_write(current_kprobe, NULL);
715
		}
716

717
		recycle_rp_inst(ri, &empty_rp);
718 719 720 721 722 723 724 725

		if (orig_ret_address != trampoline_address)
			/*
			 * This is the real return address. Any other
			 * instances associated with this task are for
			 * other calls deeper on the call stack
			 */
			break;
726
	}
727

728
	kretprobe_hash_unlock(current, &flags);
729

730 731 732 733
	hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
		hlist_del(&ri->hlist);
		kfree(ri);
	}
734
	return (void *)orig_ret_address;
735 736
}

L
Linus Torvalds 已提交
737 738 739 740 741 742 743 744 745 746 747 748
/*
 * Called after single-stepping.  p->addr is the address of the
 * instruction whose first byte has been replaced by the "int 3"
 * instruction.  To avoid the SMP problems that can occur when we
 * temporarily put back the original opcode to single-step, we
 * single-stepped a copy of the instruction.  The address of this
 * copy is p->ainsn.insn.
 *
 * This function prepares to return from the post-single-step
 * interrupt.  We have to fix up the stack as follows:
 *
 * 0) Except in the case of absolute or indirect jump or call instructions,
749
 * the new ip is relative to the copied instruction.  We need to make
L
Linus Torvalds 已提交
750 751 752
 * it relative to the original instruction.
 *
 * 1) If the single-stepped instruction was pushfl, then the TF and IF
753
 * flags are set in the just-pushed flags, and may need to be cleared.
L
Linus Torvalds 已提交
754 755 756 757
 *
 * 2) If the single-stepped instruction was a call, the return address
 * that is atop the stack is the address following the copied instruction.
 * We need to make it the address following the original instruction.
758 759 760 761 762
 *
 * If this is the first time we've single-stepped the instruction at
 * this probepoint, and the instruction is boostable, boost it: add a
 * jump instruction after the copied instruction, that jumps to the next
 * instruction after the probepoint.
L
Linus Torvalds 已提交
763
 */
764 765
static void __kprobes
resume_execution(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
L
Linus Torvalds 已提交
766
{
767 768 769
	unsigned long *tos = stack_addr(regs);
	unsigned long copy_ip = (unsigned long)p->ainsn.insn;
	unsigned long orig_ip = (unsigned long)p->addr;
L
Linus Torvalds 已提交
770 771
	kprobe_opcode_t *insn = p->ainsn.insn;

772 773
	/* Skip prefixes */
	insn = skip_prefixes(insn);
L
Linus Torvalds 已提交
774

775
	regs->flags &= ~X86_EFLAGS_TF;
L
Linus Torvalds 已提交
776
	switch (*insn) {
M
Masami Hiramatsu 已提交
777
	case 0x9c:	/* pushfl */
778
		*tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF);
779
		*tos |= kcb->kprobe_old_flags;
L
Linus Torvalds 已提交
780
		break;
M
Masami Hiramatsu 已提交
781 782
	case 0xc2:	/* iret/ret/lret */
	case 0xc3:
783
	case 0xca:
M
Masami Hiramatsu 已提交
784 785 786 787
	case 0xcb:
	case 0xcf:
	case 0xea:	/* jmp absolute -- ip is correct */
		/* ip is already adjusted, no more changes required */
788
		p->ainsn.boostable = 1;
M
Masami Hiramatsu 已提交
789 790
		goto no_change;
	case 0xe8:	/* call relative - Fix return addr */
791
		*tos = orig_ip + (*tos - copy_ip);
L
Linus Torvalds 已提交
792
		break;
H
Harvey Harrison 已提交
793
#ifdef CONFIG_X86_32
794 795 796 797
	case 0x9a:	/* call absolute -- same as call absolute, indirect */
		*tos = orig_ip + (*tos - copy_ip);
		goto no_change;
#endif
L
Linus Torvalds 已提交
798
	case 0xff:
799
		if ((insn[1] & 0x30) == 0x10) {
800 801 802 803 804 805
			/*
			 * call absolute, indirect
			 * Fix return addr; ip is correct.
			 * But this is not boostable
			 */
			*tos = orig_ip + (*tos - copy_ip);
M
Masami Hiramatsu 已提交
806
			goto no_change;
807 808 809 810 811 812
		} else if (((insn[1] & 0x31) == 0x20) ||
			   ((insn[1] & 0x31) == 0x21)) {
			/*
			 * jmp near and far, absolute indirect
			 * ip is correct. And this is boostable
			 */
813
			p->ainsn.boostable = 1;
M
Masami Hiramatsu 已提交
814
			goto no_change;
L
Linus Torvalds 已提交
815 816 817 818 819
		}
	default:
		break;
	}

820
	if (p->ainsn.boostable == 0) {
821 822
		if ((regs->ip > copy_ip) &&
		    (regs->ip - copy_ip) + 5 < MAX_INSN_SIZE) {
823 824 825 826
			/*
			 * These instructions can be executed directly if it
			 * jumps back to correct address.
			 */
827 828
			synthesize_reljump((void *)regs->ip,
				(void *)orig_ip + (regs->ip - copy_ip));
829 830 831 832 833 834
			p->ainsn.boostable = 1;
		} else {
			p->ainsn.boostable = -1;
		}
	}

835
	regs->ip += orig_ip - copy_ip;
836

M
Masami Hiramatsu 已提交
837
no_change:
R
Roland McGrath 已提交
838
	restore_btf();
L
Linus Torvalds 已提交
839 840
}

841 842
/*
 * Interrupts are disabled on entry as trap1 is an interrupt gate and they
843
 * remain disabled throughout this function.
844 845
 */
static int __kprobes post_kprobe_handler(struct pt_regs *regs)
L
Linus Torvalds 已提交
846
{
847 848 849 850
	struct kprobe *cur = kprobe_running();
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

	if (!cur)
L
Linus Torvalds 已提交
851 852
		return 0;

853 854 855
	resume_execution(cur, regs, kcb);
	regs->flags |= kcb->kprobe_saved_flags;

856 857 858
	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
		kcb->kprobe_status = KPROBE_HIT_SSDONE;
		cur->post_handler(cur, regs, 0);
859
	}
L
Linus Torvalds 已提交
860

861
	/* Restore back the original saved kprobes variables and continue. */
862 863
	if (kcb->kprobe_status == KPROBE_REENTER) {
		restore_previous_kprobe(kcb);
864 865
		goto out;
	}
866
	reset_current_kprobe();
867
out:
L
Linus Torvalds 已提交
868 869 870
	preempt_enable_no_resched();

	/*
871
	 * if somebody else is singlestepping across a probe point, flags
L
Linus Torvalds 已提交
872 873 874
	 * will have TF set, in which case, continue the remaining processing
	 * of do_debug, as if this is not a probe hit.
	 */
875
	if (regs->flags & X86_EFLAGS_TF)
L
Linus Torvalds 已提交
876 877 878 879 880
		return 0;

	return 1;
}

881
int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
L
Linus Torvalds 已提交
882
{
883 884 885
	struct kprobe *cur = kprobe_running();
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

886
	switch (kcb->kprobe_status) {
887 888 889 890 891
	case KPROBE_HIT_SS:
	case KPROBE_REENTER:
		/*
		 * We are here because the instruction being single
		 * stepped caused a page fault. We reset the current
892
		 * kprobe and the ip points back to the probe address
893 894 895
		 * and allow the page fault handler to continue as a
		 * normal page fault.
		 */
896
		regs->ip = (unsigned long)cur->addr;
897
		regs->flags |= kcb->kprobe_old_flags;
898 899 900 901
		if (kcb->kprobe_status == KPROBE_REENTER)
			restore_previous_kprobe(kcb);
		else
			reset_current_kprobe();
L
Linus Torvalds 已提交
902
		preempt_enable_no_resched();
903 904 905 906 907
		break;
	case KPROBE_HIT_ACTIVE:
	case KPROBE_HIT_SSDONE:
		/*
		 * We increment the nmissed count for accounting,
908
		 * we can also use npre/npostfault count for accounting
909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
		 * these specific fault cases.
		 */
		kprobes_inc_nmissed_count(cur);

		/*
		 * We come here because instructions in the pre/post
		 * handler caused the page_fault, this could happen
		 * if handler tries to access user space by
		 * copy_from_user(), get_user() etc. Let the
		 * user-specified handler try to fix it first.
		 */
		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
			return 1;

		/*
		 * In case the user-specified fault handler returned
		 * zero, try to fix up.
		 */
927 928
		if (fixup_exception(regs))
			return 1;
H
Harvey Harrison 已提交
929

930
		/*
931
		 * fixup routine could not handle it,
932 933 934 935 936
		 * Let do_page_fault() fix it.
		 */
		break;
	default:
		break;
L
Linus Torvalds 已提交
937 938 939 940 941 942 943
	}
	return 0;
}

/*
 * Wrapper routine for handling exceptions.
 */
944 945
int __kprobes
kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data)
L
Linus Torvalds 已提交
946
{
J
Jan Engelhardt 已提交
947
	struct die_args *args = data;
948 949
	int ret = NOTIFY_DONE;

950
	if (args->regs && user_mode_vm(args->regs))
951 952
		return ret;

L
Linus Torvalds 已提交
953 954 955
	switch (val) {
	case DIE_INT3:
		if (kprobe_handler(args->regs))
956
			ret = NOTIFY_STOP;
L
Linus Torvalds 已提交
957 958
		break;
	case DIE_DEBUG:
959 960 961 962 963 964
		if (post_kprobe_handler(args->regs)) {
			/*
			 * Reset the BS bit in dr6 (pointed by args->err) to
			 * denote completion of processing
			 */
			(*(unsigned long *)ERR_PTR(args->err)) &= ~DR_STEP;
965
			ret = NOTIFY_STOP;
966
		}
L
Linus Torvalds 已提交
967 968
		break;
	case DIE_GPF:
969 970 971 972 973 974
		/*
		 * To be potentially processing a kprobe fault and to
		 * trust the result from kprobe_running(), we have
		 * be non-preemptible.
		 */
		if (!preemptible() && kprobe_running() &&
L
Linus Torvalds 已提交
975
		    kprobe_fault_handler(args->regs, args->trapnr))
976
			ret = NOTIFY_STOP;
L
Linus Torvalds 已提交
977 978 979 980
		break;
	default:
		break;
	}
981
	return ret;
L
Linus Torvalds 已提交
982 983
}

984
int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
L
Linus Torvalds 已提交
985 986 987
{
	struct jprobe *jp = container_of(p, struct jprobe, kp);
	unsigned long addr;
988
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
L
Linus Torvalds 已提交
989

990
	kcb->jprobe_saved_regs = *regs;
991 992 993
	kcb->jprobe_saved_sp = stack_addr(regs);
	addr = (unsigned long)(kcb->jprobe_saved_sp);

L
Linus Torvalds 已提交
994 995 996 997 998 999 1000
	/*
	 * As Linus pointed out, gcc assumes that the callee
	 * owns the argument space and could overwrite it, e.g.
	 * tailcall optimization. So, to be absolutely safe
	 * we also save and restore enough stack bytes to cover
	 * the argument area.
	 */
1001
	memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
1002
	       MIN_STACK_SIZE(addr));
1003
	regs->flags &= ~X86_EFLAGS_IF;
1004
	trace_hardirqs_off();
1005
	regs->ip = (unsigned long)(jp->entry);
L
Linus Torvalds 已提交
1006 1007 1008
	return 1;
}

1009
void __kprobes jprobe_return(void)
L
Linus Torvalds 已提交
1010
{
1011 1012
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
	asm volatile (
#ifdef CONFIG_X86_64
			"       xchg   %%rbx,%%rsp	\n"
#else
			"       xchgl   %%ebx,%%esp	\n"
#endif
			"       int3			\n"
			"       .globl jprobe_return_end\n"
			"       jprobe_return_end:	\n"
			"       nop			\n"::"b"
			(kcb->jprobe_saved_sp):"memory");
L
Linus Torvalds 已提交
1024 1025
}

1026
int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
L
Linus Torvalds 已提交
1027
{
1028
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1029
	u8 *addr = (u8 *) (regs->ip - 1);
L
Linus Torvalds 已提交
1030 1031
	struct jprobe *jp = container_of(p, struct jprobe, kp);

1032 1033
	if ((addr > (u8 *) jprobe_return) &&
	    (addr < (u8 *) jprobe_return_end)) {
1034
		if (stack_addr(regs) != kcb->jprobe_saved_sp) {
M
Masami Hiramatsu 已提交
1035
			struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;
1036 1037
			printk(KERN_ERR
			       "current sp %p does not match saved sp %p\n",
1038
			       stack_addr(regs), kcb->jprobe_saved_sp);
1039
			printk(KERN_ERR "Saved registers for jprobe %p\n", jp);
1040
			show_regs(saved_regs);
1041
			printk(KERN_ERR "Current registers\n");
1042
			show_regs(regs);
L
Linus Torvalds 已提交
1043 1044
			BUG();
		}
1045
		*regs = kcb->jprobe_saved_regs;
1046 1047 1048
		memcpy((kprobe_opcode_t *)(kcb->jprobe_saved_sp),
		       kcb->jprobes_stack,
		       MIN_STACK_SIZE(kcb->jprobe_saved_sp));
1049
		preempt_enable_no_resched();
L
Linus Torvalds 已提交
1050 1051 1052 1053
		return 1;
	}
	return 0;
}
1054

1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
#ifdef KPROBES_CAN_USE_FTRACE
/* Ftrace callback handler for kprobes */
void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
				     struct ftrace_ops *ops, struct pt_regs *regs)
{
	struct kprobe *p;
	struct kprobe_ctlblk *kcb;
	unsigned long flags;

	/* Disable irq for emulating a breakpoint and avoiding preempt */
	local_irq_save(flags);

	p = get_kprobe((kprobe_opcode_t *)ip);
	if (unlikely(!p) || kprobe_disabled(p))
		goto end;

	kcb = get_kprobe_ctlblk();
	if (kprobe_running()) {
		kprobes_inc_nmissed_count(p);
	} else {
1075 1076
		/* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */
		regs->ip = ip + sizeof(kprobe_opcode_t);
1077 1078 1079 1080 1081 1082 1083

		__this_cpu_write(current_kprobe, p);
		kcb->kprobe_status = KPROBE_HIT_ACTIVE;
		if (p->pre_handler)
			p->pre_handler(p, regs);

		if (unlikely(p->post_handler)) {
1084 1085 1086 1087
			/*
			 * Emulate singlestep (and also recover regs->ip)
			 * as if there is a 5byte nop
			 */
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
			regs->ip = ip + MCOUNT_INSN_SIZE;
			kcb->kprobe_status = KPROBE_HIT_SSDONE;
			p->post_handler(p, regs, 0);
		}
		__this_cpu_write(current_kprobe, NULL);
	}
end:
	local_irq_restore(flags);
}

int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p)
{
	p->ainsn.insn = NULL;
	p->ainsn.boostable = -1;
	return 0;
}
#endif

1106
int __init arch_init_kprobes(void)
1107
{
1108
	return arch_init_optprobes();
1109
}
1110 1111 1112 1113 1114

int __kprobes arch_trampoline_kprobe(struct kprobe *p)
{
	return 0;
}