core.c 33.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
/*
 *  Kernel Probes (KProbes)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) IBM Corporation, 2002, 2004
 *
 * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
 *		Probes initial implementation ( includes contributions from
 *		Rusty Russell).
 * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
 *		interface to access function arguments.
25 26
 * 2004-Oct	Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
 *		<prasanna@in.ibm.com> adapted for x86_64 from i386.
L
Linus Torvalds 已提交
27 28
 * 2005-Mar	Roland McGrath <roland@redhat.com>
 *		Fixed to handle %rip-relative addressing mode correctly.
29 30 31 32
 * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
 *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
 *		<prasanna@in.ibm.com> added function-return probes.
 * 2005-May	Rusty Lynch <rusty.lynch@intel.com>
33
 *		Added function return probes functionality
34
 * 2006-Feb	Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added
35
 *		kprobe-booster and kretprobe-booster for i386.
36
 * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
37
 *		and kretprobe-booster for x86-64
38
 * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven
39 40
 *		<arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com>
 *		unified x86 kprobes code.
L
Linus Torvalds 已提交
41 42 43 44 45
 */
#include <linux/kprobes.h>
#include <linux/ptrace.h>
#include <linux/string.h>
#include <linux/slab.h>
46
#include <linux/hardirq.h>
L
Linus Torvalds 已提交
47
#include <linux/preempt.h>
48
#include <linux/module.h>
49
#include <linux/kdebug.h>
50
#include <linux/kallsyms.h>
51
#include <linux/ftrace.h>
52
#include <linux/frame.h>
53

54
#include <asm/text-patching.h>
55 56
#include <asm/cacheflush.h>
#include <asm/desc.h>
L
Linus Torvalds 已提交
57
#include <asm/pgtable.h>
58
#include <asm/uaccess.h>
59
#include <asm/alternative.h>
60
#include <asm/insn.h>
61
#include <asm/debugreg.h>
L
Linus Torvalds 已提交
62

63
#include "common.h"
64

L
Linus Torvalds 已提交
65 66
void jprobe_return_end(void);

67 68
DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
L
Linus Torvalds 已提交
69

70
#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
71 72 73 74 75 76 77 78 79 80

#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
	  (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) |   \
	  (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) |   \
	  (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf))    \
	 << (row % 32))
	/*
	 * Undefined/reserved opcodes, conditional jump, Opcode Extension
	 * Groups, and some special opcodes can not boost.
81 82
	 * This is non-const and volatile to keep gcc from statically
	 * optimizing it out, as variable_test_bit makes gcc think only
83
	 * *(unsigned long*) is used.
84
	 */
85
static volatile u32 twobyte_is_boostable[256 / 32] = {
86 87 88
	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
	/*      ----------------------------------------------          */
	W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */
89
	W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1) , /* 10 */
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
	W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */
	W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
	W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
	W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */
	W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
	W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */
	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
	W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */
	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */
	W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
	W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */
	W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */
	W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0)   /* f0 */
	/*      -----------------------------------------------         */
	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
};
#undef W

109 110 111 112 113
struct kretprobe_blackpoint kretprobe_blacklist[] = {
	{"__switch_to", }, /* This function switches only current task, but
			      doesn't switch kernel stack.*/
	{NULL, NULL}	/* Terminator */
};
114

115 116
const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);

117 118
static nokprobe_inline void
__synthesize_relative_insn(void *from, void *to, u8 op)
119
{
120 121
	struct __arch_relative_insn {
		u8 op;
122
		s32 raddr;
123
	} __packed *insn;
124 125 126 127 128 129 130

	insn = (struct __arch_relative_insn *)from;
	insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
	insn->op = op;
}

/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
131
void synthesize_reljump(void *from, void *to)
132 133
{
	__synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE);
134
}
135
NOKPROBE_SYMBOL(synthesize_reljump);
136

137
/* Insert a call instruction at address 'from', which calls address 'to'.*/
138
void synthesize_relcall(void *from, void *to)
139 140 141
{
	__synthesize_relative_insn(from, to, RELATIVECALL_OPCODE);
}
142
NOKPROBE_SYMBOL(synthesize_relcall);
143

144
/*
145
 * Skip the prefixes of the instruction.
146
 */
147
static kprobe_opcode_t *skip_prefixes(kprobe_opcode_t *insn)
148
{
149 150 151 152 153 154 155
	insn_attr_t attr;

	attr = inat_get_opcode_attribute((insn_byte_t)*insn);
	while (inat_is_legacy_prefix(attr)) {
		insn++;
		attr = inat_get_opcode_attribute((insn_byte_t)*insn);
	}
156
#ifdef CONFIG_X86_64
157 158
	if (inat_is_rex_prefix(attr))
		insn++;
159
#endif
160
	return insn;
161
}
162
NOKPROBE_SYMBOL(skip_prefixes);
163

164
/*
165 166
 * Returns non-zero if opcode is boostable.
 * RIP relative instructions are adjusted at copying time in 64 bits mode
167
 */
168
int can_boost(kprobe_opcode_t *opcodes)
169 170 171 172
{
	kprobe_opcode_t opcode;
	kprobe_opcode_t *orig_opcodes = opcodes;

173
	if (search_exception_tables((unsigned long)opcodes))
174 175
		return 0;	/* Page fault may occur on this address. */

176 177 178 179 180 181 182 183 184
retry:
	if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
		return 0;
	opcode = *(opcodes++);

	/* 2nd-byte opcode */
	if (opcode == 0x0f) {
		if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
			return 0;
185 186
		return test_bit(*opcodes,
				(unsigned long *)twobyte_is_boostable);
187 188 189
	}

	switch (opcode & 0xf0) {
190
#ifdef CONFIG_X86_64
191 192
	case 0x40:
		goto retry; /* REX prefix is boostable */
193
#endif
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
	case 0x60:
		if (0x63 < opcode && opcode < 0x67)
			goto retry; /* prefixes */
		/* can't boost Address-size override and bound */
		return (opcode != 0x62 && opcode != 0x67);
	case 0x70:
		return 0; /* can't boost conditional jump */
	case 0xc0:
		/* can't boost software-interruptions */
		return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
	case 0xd0:
		/* can boost AA* and XLAT */
		return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
	case 0xe0:
		/* can boost in/out and absolute jmps */
		return ((opcode & 0x04) || opcode == 0xea);
	case 0xf0:
		if ((opcode & 0x0c) == 0 && opcode != 0xf1)
			goto retry; /* lock/rep(ne) prefix */
		/* clear and set flags are boostable */
		return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
	default:
		/* segment override prefixes are boostable */
		if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
			goto retry; /* prefixes */
		/* CS override prefix and call are not boostable */
		return (opcode != 0x2e && opcode != 0x9a);
	}
}

224 225
static unsigned long
__recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
226 227
{
	struct kprobe *kp;
228
	unsigned long faddr;
229

230
	kp = get_kprobe((void *)addr);
231
	faddr = ftrace_location(addr);
232 233 234 235 236 237 238
	/*
	 * Addresses inside the ftrace location are refused by
	 * arch_check_ftrace_location(). Something went terribly wrong
	 * if such an address is checked here.
	 */
	if (WARN_ON(faddr && faddr != addr))
		return 0UL;
239 240 241 242 243
	/*
	 * Use the current code if it is not modified by Kprobe
	 * and it cannot be modified by ftrace.
	 */
	if (!kp && !faddr)
244
		return addr;
245 246

	/*
247 248 249 250 251 252 253 254 255 256 257
	 * Basically, kp->ainsn.insn has an original instruction.
	 * However, RIP-relative instruction can not do single-stepping
	 * at different place, __copy_instruction() tweaks the displacement of
	 * that instruction. In that case, we can't recover the instruction
	 * from the kp->ainsn.insn.
	 *
	 * On the other hand, in case on normal Kprobe, kp->opcode has a copy
	 * of the first byte of the probed instruction, which is overwritten
	 * by int3. And the instruction at kp->addr is not modified by kprobes
	 * except for the first byte, we can recover the original instruction
	 * from it and kp->opcode.
258
	 *
259 260 261 262 263
	 * In case of Kprobes using ftrace, we do not have a copy of
	 * the original instruction. In fact, the ftrace location might
	 * be modified at anytime and even could be in an inconsistent state.
	 * Fortunately, we know that the original code is the ideal 5-byte
	 * long NOP.
264
	 */
265 266 267 268 269
	memcpy(buf, (void *)addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
	if (faddr)
		memcpy(buf, ideal_nops[NOP_ATOMIC5], 5);
	else
		buf[0] = kp->opcode;
270 271 272 273 274 275 276
	return (unsigned long)buf;
}

/*
 * Recover the probed instruction at addr for further analysis.
 * Caller must lock kprobes by kprobe_mutex, or disable preemption
 * for preventing to release referencing kprobes.
277
 * Returns zero if the instruction can not get recovered.
278
 */
279
unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr)
280 281 282 283 284 285 286 287
{
	unsigned long __addr;

	__addr = __recover_optprobed_insn(buf, addr);
	if (__addr != addr)
		return __addr;

	return __recover_probed_insn(buf, addr);
288 289 290
}

/* Check if paddr is at an instruction boundary */
291
static int can_probe(unsigned long paddr)
292
{
293
	unsigned long addr, __addr, offset = 0;
294 295 296
	struct insn insn;
	kprobe_opcode_t buf[MAX_INSN_SIZE];

N
Namhyung Kim 已提交
297
	if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
298 299 300 301 302 303 304 305 306
		return 0;

	/* Decode instructions */
	addr = paddr - offset;
	while (addr < paddr) {
		/*
		 * Check if the instruction has been modified by another
		 * kprobe, in which case we replace the breakpoint by the
		 * original instruction in our buffer.
307 308 309
		 * Also, jump optimization will change the breakpoint to
		 * relative-jump. Since the relative-jump itself is
		 * normally used, we just go through if there is no kprobe.
310
		 */
311
		__addr = recover_probed_instruction(buf, addr);
312 313
		if (!__addr)
			return 0;
314
		kernel_insn_init(&insn, (void *)__addr, MAX_INSN_SIZE);
315
		insn_get_length(&insn);
316 317 318 319 320 321 322

		/*
		 * Another debugging subsystem might insert this breakpoint.
		 * In that case, we can't recover it.
		 */
		if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
			return 0;
323 324 325 326 327 328
		addr += insn.length;
	}

	return (addr == paddr);
}

L
Linus Torvalds 已提交
329
/*
330
 * Returns non-zero if opcode modifies the interrupt flag.
L
Linus Torvalds 已提交
331
 */
332
static int is_IF_modifier(kprobe_opcode_t *insn)
L
Linus Torvalds 已提交
333
{
334 335 336
	/* Skip prefixes */
	insn = skip_prefixes(insn);

L
Linus Torvalds 已提交
337 338 339 340 341 342 343
	switch (*insn) {
	case 0xfa:		/* cli */
	case 0xfb:		/* sti */
	case 0xcf:		/* iret/iretd */
	case 0x9d:		/* popf/popfd */
		return 1;
	}
344

L
Linus Torvalds 已提交
345 346 347 348
	return 0;
}

/*
349 350
 * Copy an instruction and adjust the displacement if the instruction
 * uses the %rip-relative addressing mode.
351
 * If it does, Return the address of the 32-bit displacement word.
L
Linus Torvalds 已提交
352
 * If not, return null.
353
 * Only applicable to 64-bit x86.
L
Linus Torvalds 已提交
354
 */
355
int __copy_instruction(u8 *dest, u8 *src)
L
Linus Torvalds 已提交
356
{
357
	struct insn insn;
358
	kprobe_opcode_t buf[MAX_INSN_SIZE];
359
	int length;
360 361
	unsigned long recovered_insn =
		recover_probed_instruction(buf, (unsigned long)src);
362

363 364
	if (!recovered_insn)
		return 0;
365
	kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE);
366
	insn_get_length(&insn);
367 368
	length = insn.length;

369
	/* Another subsystem puts a breakpoint, failed to recover */
370
	if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
371
		return 0;
372
	memcpy(dest, insn.kaddr, length);
373 374

#ifdef CONFIG_X86_64
375 376 377
	if (insn_rip_relative(&insn)) {
		s64 newdisp;
		u8 *disp;
378
		kernel_insn_init(&insn, dest, length);
379 380 381 382 383 384 385 386 387 388 389 390 391
		insn_get_displacement(&insn);
		/*
		 * The copied instruction uses the %rip-relative addressing
		 * mode.  Adjust the displacement for the difference between
		 * the original location of this instruction and the location
		 * of the copy that will actually be run.  The tricky bit here
		 * is making sure that the sign extension happens correctly in
		 * this calculation, since we need a signed 32-bit result to
		 * be sign-extended to 64 bits when it's added to the %rip
		 * value and yield the same 64-bit result that the sign-
		 * extension of the original signed 32-bit displacement would
		 * have given.
		 */
392
		newdisp = (u8 *) src + (s64) insn.displacement.value - (u8 *) dest;
393 394 395 396 397
		if ((s64) (s32) newdisp != newdisp) {
			pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp);
			pr_err("\tSrc: %p, Dest: %p, old disp: %x\n", src, dest, insn.displacement.value);
			return 0;
		}
398
		disp = (u8 *) dest + insn_offset_displacement(&insn);
399
		*(s32 *) disp = (s32) newdisp;
L
Linus Torvalds 已提交
400
	}
401
#endif
402
	return length;
403
}
L
Linus Torvalds 已提交
404

405
static int arch_copy_kprobe(struct kprobe *p)
L
Linus Torvalds 已提交
406
{
407 408
	int ret;

409
	/* Copy an instruction with recovering if other optprobe modifies it.*/
410 411 412
	ret = __copy_instruction(p->ainsn.insn, p->addr);
	if (!ret)
		return -EINVAL;
413

414
	/*
415 416
	 * __copy_instruction can modify the displacement of the instruction,
	 * but it doesn't affect boostable check.
417
	 */
418
	if (can_boost(p->ainsn.insn))
419
		p->ainsn.boostable = 0;
420
	else
421
		p->ainsn.boostable = -1;
422

423 424 425
	/* Check whether the instruction modifies Interrupt Flag or not */
	p->ainsn.if_modifier = is_IF_modifier(p->ainsn.insn);

426 427
	/* Also, displacement change doesn't affect the first byte */
	p->opcode = p->ainsn.insn[0];
428 429

	return 0;
L
Linus Torvalds 已提交
430 431
}

432
int arch_prepare_kprobe(struct kprobe *p)
433
{
434 435 436
	if (alternatives_text_reserved(p->addr, p->addr))
		return -EINVAL;

437 438
	if (!can_probe((unsigned long)p->addr))
		return -EILSEQ;
439 440 441 442
	/* insn: must be on special executable page on x86. */
	p->ainsn.insn = get_insn_slot();
	if (!p->ainsn.insn)
		return -ENOMEM;
443 444

	return arch_copy_kprobe(p);
445 446
}

447
void arch_arm_kprobe(struct kprobe *p)
L
Linus Torvalds 已提交
448
{
449
	text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
L
Linus Torvalds 已提交
450 451
}

452
void arch_disarm_kprobe(struct kprobe *p)
L
Linus Torvalds 已提交
453
{
454
	text_poke(p->addr, &p->opcode, 1);
455 456
}

457
void arch_remove_kprobe(struct kprobe *p)
458
{
459 460 461 462
	if (p->ainsn.insn) {
		free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
		p->ainsn.insn = NULL;
	}
L
Linus Torvalds 已提交
463 464
}

465 466
static nokprobe_inline void
save_previous_kprobe(struct kprobe_ctlblk *kcb)
467
{
468 469
	kcb->prev_kprobe.kp = kprobe_running();
	kcb->prev_kprobe.status = kcb->kprobe_status;
470 471
	kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;
	kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
472 473
}

474 475
static nokprobe_inline void
restore_previous_kprobe(struct kprobe_ctlblk *kcb)
476
{
C
Christoph Lameter 已提交
477
	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
478
	kcb->kprobe_status = kcb->prev_kprobe.status;
479 480
	kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;
	kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
481 482
}

483 484 485
static nokprobe_inline void
set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
		   struct kprobe_ctlblk *kcb)
486
{
C
Christoph Lameter 已提交
487
	__this_cpu_write(current_kprobe, p);
488
	kcb->kprobe_saved_flags = kcb->kprobe_old_flags
489
		= (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
490
	if (p->ainsn.if_modifier)
491
		kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
492 493
}

494
static nokprobe_inline void clear_btf(void)
R
Roland McGrath 已提交
495
{
P
Peter Zijlstra 已提交
496 497 498 499 500 501
	if (test_thread_flag(TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl &= ~DEBUGCTLMSR_BTF;
		update_debugctlmsr(debugctl);
	}
R
Roland McGrath 已提交
502 503
}

504
static nokprobe_inline void restore_btf(void)
R
Roland McGrath 已提交
505
{
P
Peter Zijlstra 已提交
506 507 508 509 510 511
	if (test_thread_flag(TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl |= DEBUGCTLMSR_BTF;
		update_debugctlmsr(debugctl);
	}
R
Roland McGrath 已提交
512 513
}

514
void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
515
{
516
	unsigned long *sara = stack_addr(regs);
517

518
	ri->ret_addr = (kprobe_opcode_t *) *sara;
519

520 521
	/* Replace the return addr with trampoline addr */
	*sara = (unsigned long) &kretprobe_trampoline;
522
}
523
NOKPROBE_SYMBOL(arch_prepare_kretprobe);
524

525 526
static void setup_singlestep(struct kprobe *p, struct pt_regs *regs,
			     struct kprobe_ctlblk *kcb, int reenter)
527
{
528 529 530
	if (setup_detour_execution(p, regs, reenter))
		return;

531
#if !defined(CONFIG_PREEMPT)
532 533
	if (p->ainsn.boostable == 1 && !p->post_handler) {
		/* Boost up -- we can execute copied instructions directly */
534 535 536 537 538 539 540
		if (!reenter)
			reset_current_kprobe();
		/*
		 * Reentering boosted probe doesn't reset current_kprobe,
		 * nor set current_kprobe, because it doesn't use single
		 * stepping.
		 */
541 542 543 544 545
		regs->ip = (unsigned long)p->ainsn.insn;
		preempt_enable_no_resched();
		return;
	}
#endif
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
	if (reenter) {
		save_previous_kprobe(kcb);
		set_current_kprobe(p, regs, kcb);
		kcb->kprobe_status = KPROBE_REENTER;
	} else
		kcb->kprobe_status = KPROBE_HIT_SS;
	/* Prepare real single stepping */
	clear_btf();
	regs->flags |= X86_EFLAGS_TF;
	regs->flags &= ~X86_EFLAGS_IF;
	/* single step inline if the instruction is an int3 */
	if (p->opcode == BREAKPOINT_INSTRUCTION)
		regs->ip = (unsigned long)p->addr;
	else
		regs->ip = (unsigned long)p->ainsn.insn;
561
}
562
NOKPROBE_SYMBOL(setup_singlestep);
563

H
Harvey Harrison 已提交
564 565 566 567 568
/*
 * We have reentered the kprobe_handler(), since another probe was hit while
 * within the handler. We save the original kprobes variables and just single
 * step on the instruction of the new probe without calling any user handlers.
 */
569 570
static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
			  struct kprobe_ctlblk *kcb)
H
Harvey Harrison 已提交
571
{
572 573 574
	switch (kcb->kprobe_status) {
	case KPROBE_HIT_SSDONE:
	case KPROBE_HIT_ACTIVE:
575
	case KPROBE_HIT_SS:
576
		kprobes_inc_nmissed_count(p);
577
		setup_singlestep(p, regs, kcb, 1);
578
		break;
579
	case KPROBE_REENTER:
580 581 582 583 584 585 586 587 588 589
		/* A probe has been hit in the codepath leading up to, or just
		 * after, single-stepping of a probed instruction. This entire
		 * codepath should strictly reside in .kprobes.text section.
		 * Raise a BUG or we'll continue in an endless reentering loop
		 * and eventually a stack overflow.
		 */
		printk(KERN_WARNING "Unrecoverable kprobe detected at %p.\n",
		       p->addr);
		dump_kprobe(p);
		BUG();
590 591 592
	default:
		/* impossible cases */
		WARN_ON(1);
593
		return 0;
594
	}
595

596
	return 1;
H
Harvey Harrison 已提交
597
}
598
NOKPROBE_SYMBOL(reenter_kprobe);
599

600 601
/*
 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
602
 * remain disabled throughout this function.
603
 */
604
int kprobe_int3_handler(struct pt_regs *regs)
L
Linus Torvalds 已提交
605
{
606
	kprobe_opcode_t *addr;
607
	struct kprobe *p;
608 609
	struct kprobe_ctlblk *kcb;

610
	if (user_mode(regs))
611 612
		return 0;

613
	addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
614 615
	/*
	 * We don't want to be preempted for the entire
616 617 618
	 * duration of kprobe processing. We conditionally
	 * re-enable preemption at the end of this function,
	 * and also in reenter_kprobe() and setup_singlestep().
619 620
	 */
	preempt_disable();
L
Linus Torvalds 已提交
621

622
	kcb = get_kprobe_ctlblk();
623
	p = get_kprobe(addr);
624

625 626
	if (p) {
		if (kprobe_running()) {
627 628
			if (reenter_kprobe(p, regs, kcb))
				return 1;
L
Linus Torvalds 已提交
629
		} else {
630 631
			set_current_kprobe(p, regs, kcb);
			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
632

L
Linus Torvalds 已提交
633
			/*
634 635 636 637 638 639
			 * If we have no pre-handler or it returned 0, we
			 * continue with normal processing.  If we have a
			 * pre-handler and it returned non-zero, it prepped
			 * for calling the break_handler below on re-entry
			 * for jprobe processing, so get out doing nothing
			 * more here.
L
Linus Torvalds 已提交
640
			 */
641
			if (!p->pre_handler || !p->pre_handler(p, regs))
642
				setup_singlestep(p, regs, kcb, 0);
643
			return 1;
644
		}
645 646 647 648 649 650 651 652 653 654 655 656 657
	} else if (*addr != BREAKPOINT_INSTRUCTION) {
		/*
		 * The breakpoint instruction was removed right
		 * after we hit it.  Another cpu has removed
		 * either a probepoint or a debugger breakpoint
		 * at this address.  In either case, no further
		 * handling of this interrupt is appropriate.
		 * Back up over the (now missing) int3 and run
		 * the original instruction.
		 */
		regs->ip = (unsigned long)addr;
		preempt_enable_no_resched();
		return 1;
658
	} else if (kprobe_running()) {
C
Christoph Lameter 已提交
659
		p = __this_cpu_read(current_kprobe);
660
		if (p->break_handler && p->break_handler(p, regs)) {
661 662
			if (!skip_singlestep(p, regs, kcb))
				setup_singlestep(p, regs, kcb, 0);
663
			return 1;
L
Linus Torvalds 已提交
664
		}
665
	} /* else: not a kprobe fault; let the kernel handle it */
L
Linus Torvalds 已提交
666

667
	preempt_enable_no_resched();
668
	return 0;
L
Linus Torvalds 已提交
669
}
670
NOKPROBE_SYMBOL(kprobe_int3_handler);
L
Linus Torvalds 已提交
671

672
/*
673 674
 * When a retprobed function returns, this code saves registers and
 * calls trampoline_handler() runs, which calls the kretprobe's handler.
675
 */
676 677 678 679
asm(
	".global kretprobe_trampoline\n"
	".type kretprobe_trampoline, @function\n"
	"kretprobe_trampoline:\n"
680
#ifdef CONFIG_X86_64
681 682 683 684 685 686 687 688 689 690
	/* We don't bother saving the ss register */
	"	pushq %rsp\n"
	"	pushfq\n"
	SAVE_REGS_STRING
	"	movq %rsp, %rdi\n"
	"	call trampoline_handler\n"
	/* Replace saved sp with true return address. */
	"	movq %rax, 152(%rsp)\n"
	RESTORE_REGS_STRING
	"	popfq\n"
691
#else
692 693 694 695 696 697 698 699 700 701 702
	"	pushf\n"
	SAVE_REGS_STRING
	"	movl %esp, %eax\n"
	"	call trampoline_handler\n"
	/* Move flags to cs */
	"	movl 56(%esp), %edx\n"
	"	movl %edx, 52(%esp)\n"
	/* Replace saved flags with true return address. */
	"	movl %eax, 56(%esp)\n"
	RESTORE_REGS_STRING
	"	popf\n"
703
#endif
704 705 706
	"	ret\n"
	".size kretprobe_trampoline, .-kretprobe_trampoline\n"
);
707
NOKPROBE_SYMBOL(kretprobe_trampoline);
708
STACK_FRAME_NON_STANDARD(kretprobe_trampoline);
709 710

/*
711
 * Called from kretprobe_trampoline
712
 */
713
__visible __used void *trampoline_handler(struct pt_regs *regs)
714
{
B
bibo,mao 已提交
715
	struct kretprobe_instance *ri = NULL;
716
	struct hlist_head *head, empty_rp;
717
	struct hlist_node *tmp;
718
	unsigned long flags, orig_ret_address = 0;
719
	unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
720
	kprobe_opcode_t *correct_ret_addr = NULL;
721

722
	INIT_HLIST_HEAD(&empty_rp);
723
	kretprobe_hash_lock(current, &head, &flags);
724
	/* fixup registers */
725
#ifdef CONFIG_X86_64
726
	regs->cs = __KERNEL_CS;
727 728
#else
	regs->cs = __KERNEL_CS | get_kernel_rpl();
729
	regs->gs = 0;
730
#endif
731
	regs->ip = trampoline_address;
732
	regs->orig_ax = ~0UL;
733

734 735
	/*
	 * It is possible to have multiple instances associated with a given
736
	 * task either because multiple functions in the call path have
737
	 * return probes installed on them, and/or more than one
738 739 740
	 * return probe was registered for a target function.
	 *
	 * We can handle this because:
741
	 *     - instances are always pushed into the head of the list
742
	 *     - when multiple return probes are registered for the same
743 744 745
	 *	 function, the (chronologically) first instance's ret_addr
	 *	 will be the real return address, and all the rest will
	 *	 point to kretprobe_trampoline.
746
	 */
747
	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
B
bibo,mao 已提交
748
		if (ri->task != current)
749
			/* another task is sharing our hash bucket */
B
bibo,mao 已提交
750
			continue;
751

752 753 754 755 756 757 758 759 760 761 762 763 764 765
		orig_ret_address = (unsigned long)ri->ret_addr;

		if (orig_ret_address != trampoline_address)
			/*
			 * This is the real return address. Any other
			 * instances associated with this task are for
			 * other calls deeper on the call stack
			 */
			break;
	}

	kretprobe_assert(ri, orig_ret_address, trampoline_address);

	correct_ret_addr = ri->ret_addr;
766
	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
767 768 769 770 771
		if (ri->task != current)
			/* another task is sharing our hash bucket */
			continue;

		orig_ret_address = (unsigned long)ri->ret_addr;
772
		if (ri->rp && ri->rp->handler) {
C
Christoph Lameter 已提交
773
			__this_cpu_write(current_kprobe, &ri->rp->kp);
774
			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
775
			ri->ret_addr = correct_ret_addr;
776
			ri->rp->handler(ri, regs);
C
Christoph Lameter 已提交
777
			__this_cpu_write(current_kprobe, NULL);
778
		}
779

780
		recycle_rp_inst(ri, &empty_rp);
781 782 783 784 785 786 787 788

		if (orig_ret_address != trampoline_address)
			/*
			 * This is the real return address. Any other
			 * instances associated with this task are for
			 * other calls deeper on the call stack
			 */
			break;
789
	}
790

791
	kretprobe_hash_unlock(current, &flags);
792

793
	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
794 795 796
		hlist_del(&ri->hlist);
		kfree(ri);
	}
797
	return (void *)orig_ret_address;
798
}
799
NOKPROBE_SYMBOL(trampoline_handler);
800

L
Linus Torvalds 已提交
801 802 803 804 805 806 807 808 809 810 811 812
/*
 * Called after single-stepping.  p->addr is the address of the
 * instruction whose first byte has been replaced by the "int 3"
 * instruction.  To avoid the SMP problems that can occur when we
 * temporarily put back the original opcode to single-step, we
 * single-stepped a copy of the instruction.  The address of this
 * copy is p->ainsn.insn.
 *
 * This function prepares to return from the post-single-step
 * interrupt.  We have to fix up the stack as follows:
 *
 * 0) Except in the case of absolute or indirect jump or call instructions,
813
 * the new ip is relative to the copied instruction.  We need to make
L
Linus Torvalds 已提交
814 815 816
 * it relative to the original instruction.
 *
 * 1) If the single-stepped instruction was pushfl, then the TF and IF
817
 * flags are set in the just-pushed flags, and may need to be cleared.
L
Linus Torvalds 已提交
818 819 820 821
 *
 * 2) If the single-stepped instruction was a call, the return address
 * that is atop the stack is the address following the copied instruction.
 * We need to make it the address following the original instruction.
822 823 824 825 826
 *
 * If this is the first time we've single-stepped the instruction at
 * this probepoint, and the instruction is boostable, boost it: add a
 * jump instruction after the copied instruction, that jumps to the next
 * instruction after the probepoint.
L
Linus Torvalds 已提交
827
 */
828 829
static void resume_execution(struct kprobe *p, struct pt_regs *regs,
			     struct kprobe_ctlblk *kcb)
L
Linus Torvalds 已提交
830
{
831 832 833
	unsigned long *tos = stack_addr(regs);
	unsigned long copy_ip = (unsigned long)p->ainsn.insn;
	unsigned long orig_ip = (unsigned long)p->addr;
L
Linus Torvalds 已提交
834 835
	kprobe_opcode_t *insn = p->ainsn.insn;

836 837
	/* Skip prefixes */
	insn = skip_prefixes(insn);
L
Linus Torvalds 已提交
838

839
	regs->flags &= ~X86_EFLAGS_TF;
L
Linus Torvalds 已提交
840
	switch (*insn) {
M
Masami Hiramatsu 已提交
841
	case 0x9c:	/* pushfl */
842
		*tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF);
843
		*tos |= kcb->kprobe_old_flags;
L
Linus Torvalds 已提交
844
		break;
M
Masami Hiramatsu 已提交
845 846
	case 0xc2:	/* iret/ret/lret */
	case 0xc3:
847
	case 0xca:
M
Masami Hiramatsu 已提交
848 849 850 851
	case 0xcb:
	case 0xcf:
	case 0xea:	/* jmp absolute -- ip is correct */
		/* ip is already adjusted, no more changes required */
852
		p->ainsn.boostable = 1;
M
Masami Hiramatsu 已提交
853 854
		goto no_change;
	case 0xe8:	/* call relative - Fix return addr */
855
		*tos = orig_ip + (*tos - copy_ip);
L
Linus Torvalds 已提交
856
		break;
H
Harvey Harrison 已提交
857
#ifdef CONFIG_X86_32
858 859 860 861
	case 0x9a:	/* call absolute -- same as call absolute, indirect */
		*tos = orig_ip + (*tos - copy_ip);
		goto no_change;
#endif
L
Linus Torvalds 已提交
862
	case 0xff:
863
		if ((insn[1] & 0x30) == 0x10) {
864 865 866 867 868 869
			/*
			 * call absolute, indirect
			 * Fix return addr; ip is correct.
			 * But this is not boostable
			 */
			*tos = orig_ip + (*tos - copy_ip);
M
Masami Hiramatsu 已提交
870
			goto no_change;
871 872 873 874 875 876
		} else if (((insn[1] & 0x31) == 0x20) ||
			   ((insn[1] & 0x31) == 0x21)) {
			/*
			 * jmp near and far, absolute indirect
			 * ip is correct. And this is boostable
			 */
877
			p->ainsn.boostable = 1;
M
Masami Hiramatsu 已提交
878
			goto no_change;
L
Linus Torvalds 已提交
879 880 881 882 883
		}
	default:
		break;
	}

884
	if (p->ainsn.boostable == 0) {
885 886
		if ((regs->ip > copy_ip) &&
		    (regs->ip - copy_ip) + 5 < MAX_INSN_SIZE) {
887 888 889 890
			/*
			 * These instructions can be executed directly if it
			 * jumps back to correct address.
			 */
891 892
			synthesize_reljump((void *)regs->ip,
				(void *)orig_ip + (regs->ip - copy_ip));
893 894 895 896 897 898
			p->ainsn.boostable = 1;
		} else {
			p->ainsn.boostable = -1;
		}
	}

899
	regs->ip += orig_ip - copy_ip;
900

M
Masami Hiramatsu 已提交
901
no_change:
R
Roland McGrath 已提交
902
	restore_btf();
L
Linus Torvalds 已提交
903
}
904
NOKPROBE_SYMBOL(resume_execution);
L
Linus Torvalds 已提交
905

906 907
/*
 * Interrupts are disabled on entry as trap1 is an interrupt gate and they
908
 * remain disabled throughout this function.
909
 */
910
int kprobe_debug_handler(struct pt_regs *regs)
L
Linus Torvalds 已提交
911
{
912 913 914 915
	struct kprobe *cur = kprobe_running();
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

	if (!cur)
L
Linus Torvalds 已提交
916 917
		return 0;

918 919 920
	resume_execution(cur, regs, kcb);
	regs->flags |= kcb->kprobe_saved_flags;

921 922 923
	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
		kcb->kprobe_status = KPROBE_HIT_SSDONE;
		cur->post_handler(cur, regs, 0);
924
	}
L
Linus Torvalds 已提交
925

926
	/* Restore back the original saved kprobes variables and continue. */
927 928
	if (kcb->kprobe_status == KPROBE_REENTER) {
		restore_previous_kprobe(kcb);
929 930
		goto out;
	}
931
	reset_current_kprobe();
932
out:
L
Linus Torvalds 已提交
933 934 935
	preempt_enable_no_resched();

	/*
936
	 * if somebody else is singlestepping across a probe point, flags
L
Linus Torvalds 已提交
937 938 939
	 * will have TF set, in which case, continue the remaining processing
	 * of do_debug, as if this is not a probe hit.
	 */
940
	if (regs->flags & X86_EFLAGS_TF)
L
Linus Torvalds 已提交
941 942 943 944
		return 0;

	return 1;
}
945
NOKPROBE_SYMBOL(kprobe_debug_handler);
L
Linus Torvalds 已提交
946

947
int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
L
Linus Torvalds 已提交
948
{
949 950 951
	struct kprobe *cur = kprobe_running();
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

952 953 954 955
	if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) {
		/* This must happen on single-stepping */
		WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS &&
			kcb->kprobe_status != KPROBE_REENTER);
956 957 958
		/*
		 * We are here because the instruction being single
		 * stepped caused a page fault. We reset the current
959
		 * kprobe and the ip points back to the probe address
960 961 962
		 * and allow the page fault handler to continue as a
		 * normal page fault.
		 */
963
		regs->ip = (unsigned long)cur->addr;
964 965 966 967 968 969 970 971 972 973 974
		/*
		 * Trap flag (TF) has been set here because this fault
		 * happened where the single stepping will be done.
		 * So clear it by resetting the current kprobe:
		 */
		regs->flags &= ~X86_EFLAGS_TF;

		/*
		 * If the TF flag was set before the kprobe hit,
		 * don't touch it:
		 */
975
		regs->flags |= kcb->kprobe_old_flags;
976

977 978 979 980
		if (kcb->kprobe_status == KPROBE_REENTER)
			restore_previous_kprobe(kcb);
		else
			reset_current_kprobe();
L
Linus Torvalds 已提交
981
		preempt_enable_no_resched();
982 983
	} else if (kcb->kprobe_status == KPROBE_HIT_ACTIVE ||
		   kcb->kprobe_status == KPROBE_HIT_SSDONE) {
984 985
		/*
		 * We increment the nmissed count for accounting,
986
		 * we can also use npre/npostfault count for accounting
987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004
		 * these specific fault cases.
		 */
		kprobes_inc_nmissed_count(cur);

		/*
		 * We come here because instructions in the pre/post
		 * handler caused the page_fault, this could happen
		 * if handler tries to access user space by
		 * copy_from_user(), get_user() etc. Let the
		 * user-specified handler try to fix it first.
		 */
		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
			return 1;

		/*
		 * In case the user-specified fault handler returned
		 * zero, try to fix up.
		 */
1005
		if (fixup_exception(regs, trapnr))
1006
			return 1;
H
Harvey Harrison 已提交
1007

1008
		/*
1009
		 * fixup routine could not handle it,
1010 1011
		 * Let do_page_fault() fix it.
		 */
L
Linus Torvalds 已提交
1012
	}
1013

L
Linus Torvalds 已提交
1014 1015
	return 0;
}
1016
NOKPROBE_SYMBOL(kprobe_fault_handler);
L
Linus Torvalds 已提交
1017 1018 1019 1020

/*
 * Wrapper routine for handling exceptions.
 */
1021 1022
int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val,
			     void *data)
L
Linus Torvalds 已提交
1023
{
J
Jan Engelhardt 已提交
1024
	struct die_args *args = data;
1025 1026
	int ret = NOTIFY_DONE;

1027
	if (args->regs && user_mode(args->regs))
1028 1029
		return ret;

1030
	if (val == DIE_GPF) {
1031 1032 1033 1034 1035 1036
		/*
		 * To be potentially processing a kprobe fault and to
		 * trust the result from kprobe_running(), we have
		 * be non-preemptible.
		 */
		if (!preemptible() && kprobe_running() &&
L
Linus Torvalds 已提交
1037
		    kprobe_fault_handler(args->regs, args->trapnr))
1038
			ret = NOTIFY_STOP;
L
Linus Torvalds 已提交
1039
	}
1040
	return ret;
L
Linus Torvalds 已提交
1041
}
1042
NOKPROBE_SYMBOL(kprobe_exceptions_notify);
L
Linus Torvalds 已提交
1043

1044
int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
L
Linus Torvalds 已提交
1045 1046 1047
{
	struct jprobe *jp = container_of(p, struct jprobe, kp);
	unsigned long addr;
1048
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
L
Linus Torvalds 已提交
1049

1050
	kcb->jprobe_saved_regs = *regs;
1051 1052 1053
	kcb->jprobe_saved_sp = stack_addr(regs);
	addr = (unsigned long)(kcb->jprobe_saved_sp);

L
Linus Torvalds 已提交
1054 1055 1056 1057 1058 1059 1060
	/*
	 * As Linus pointed out, gcc assumes that the callee
	 * owns the argument space and could overwrite it, e.g.
	 * tailcall optimization. So, to be absolutely safe
	 * we also save and restore enough stack bytes to cover
	 * the argument area.
	 */
1061
	memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
1062
	       MIN_STACK_SIZE(addr));
1063
	regs->flags &= ~X86_EFLAGS_IF;
1064
	trace_hardirqs_off();
1065
	regs->ip = (unsigned long)(jp->entry);
1066 1067 1068 1069 1070 1071 1072 1073 1074

	/*
	 * jprobes use jprobe_return() which skips the normal return
	 * path of the function, and this messes up the accounting of the
	 * function graph tracer to get messed up.
	 *
	 * Pause function graph tracing while performing the jprobe function.
	 */
	pause_graph_tracing();
L
Linus Torvalds 已提交
1075 1076
	return 1;
}
1077
NOKPROBE_SYMBOL(setjmp_pre_handler);
L
Linus Torvalds 已提交
1078

1079
void jprobe_return(void)
L
Linus Torvalds 已提交
1080
{
1081 1082
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
	asm volatile (
#ifdef CONFIG_X86_64
			"       xchg   %%rbx,%%rsp	\n"
#else
			"       xchgl   %%ebx,%%esp	\n"
#endif
			"       int3			\n"
			"       .globl jprobe_return_end\n"
			"       jprobe_return_end:	\n"
			"       nop			\n"::"b"
			(kcb->jprobe_saved_sp):"memory");
L
Linus Torvalds 已提交
1094
}
1095 1096
NOKPROBE_SYMBOL(jprobe_return);
NOKPROBE_SYMBOL(jprobe_return_end);
L
Linus Torvalds 已提交
1097

1098
int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
L
Linus Torvalds 已提交
1099
{
1100
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1101
	u8 *addr = (u8 *) (regs->ip - 1);
L
Linus Torvalds 已提交
1102
	struct jprobe *jp = container_of(p, struct jprobe, kp);
1103
	void *saved_sp = kcb->jprobe_saved_sp;
L
Linus Torvalds 已提交
1104

1105 1106
	if ((addr > (u8 *) jprobe_return) &&
	    (addr < (u8 *) jprobe_return_end)) {
1107
		if (stack_addr(regs) != saved_sp) {
M
Masami Hiramatsu 已提交
1108
			struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;
1109 1110
			printk(KERN_ERR
			       "current sp %p does not match saved sp %p\n",
1111
			       stack_addr(regs), saved_sp);
1112
			printk(KERN_ERR "Saved registers for jprobe %p\n", jp);
1113
			show_regs(saved_regs);
1114
			printk(KERN_ERR "Current registers\n");
1115
			show_regs(regs);
L
Linus Torvalds 已提交
1116 1117
			BUG();
		}
1118 1119
		/* It's OK to start function graph tracing again */
		unpause_graph_tracing();
1120
		*regs = kcb->jprobe_saved_regs;
1121
		memcpy(saved_sp, kcb->jprobes_stack, MIN_STACK_SIZE(saved_sp));
1122
		preempt_enable_no_resched();
L
Linus Torvalds 已提交
1123 1124 1125 1126
		return 1;
	}
	return 0;
}
1127
NOKPROBE_SYMBOL(longjmp_break_handler);
1128

1129 1130 1131 1132 1133 1134 1135 1136
bool arch_within_kprobe_blacklist(unsigned long addr)
{
	return  (addr >= (unsigned long)__kprobes_text_start &&
		 addr < (unsigned long)__kprobes_text_end) ||
		(addr >= (unsigned long)__entry_text_start &&
		 addr < (unsigned long)__entry_text_end);
}

1137
int __init arch_init_kprobes(void)
1138
{
1139
	return 0;
1140
}
1141

1142
int arch_trampoline_kprobe(struct kprobe *p)
1143 1144 1145
{
	return 0;
}