ftrace.c 24.6 KB
Newer Older
1
/*
2
 * Dynamic function tracing support.
3 4 5 6 7 8 9 10 11
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 *
 * Thanks goes to Ingo Molnar, for suggesting the idea.
 * Mathieu Desnoyers, for suggesting postponing the modifications.
 * Arjan van de Ven, for keeping me straight, and explaining to me
 * the dangers of modifying code on the run.
 */

12 13
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

14 15
#include <linux/spinlock.h>
#include <linux/hardirq.h>
16
#include <linux/uaccess.h>
17 18
#include <linux/ftrace.h>
#include <linux/percpu.h>
19
#include <linux/sched.h>
20
#include <linux/slab.h>
21 22
#include <linux/init.h>
#include <linux/list.h>
23
#include <linux/module.h>
24

25 26
#include <trace/syscall.h>

L
Laura Abbott 已提交
27
#include <asm/set_memory.h>
28
#include <asm/kprobes.h>
29
#include <asm/ftrace.h>
30
#include <asm/nops.h>
31

32
#ifdef CONFIG_DYNAMIC_FTRACE
33

34 35 36
int ftrace_arch_code_modify_prepare(void)
{
	set_kernel_text_rw();
37
	set_all_modules_text_rw();
38 39 40 41 42
	return 0;
}

int ftrace_arch_code_modify_post_process(void)
{
43
	set_all_modules_text_ro();
44 45 46 47
	set_kernel_text_ro();
	return 0;
}

48
union ftrace_code_union {
49
	char code[MCOUNT_INSN_SIZE];
50
	struct {
51
		unsigned char e8;
52 53 54 55
		int offset;
	} __attribute__((packed));
};

56
static int ftrace_calc_offset(long ip, long addr)
57 58 59
{
	return (int)(addr - ip);
}
60

61
static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
62 63
{
	static union ftrace_code_union calc;
64

65
	calc.e8		= 0xe8;
66
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
67 68 69 70 71 72

	/*
	 * No locking needed, this must be called via kstop_machine
	 * which in essence is like running on a uniprocessor machine.
	 */
	return calc.code;
73 74
}

75 76 77 78 79 80
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
	return addr >= start && addr < end;
}

81
static unsigned long text_ip_addr(unsigned long ip)
82
{
83
	/*
84 85 86
	 * On x86_64, kernel text mappings are mapped read-only, so we use
	 * the kernel identity mapping instead of the kernel text mapping
	 * to modify the kernel text.
87 88 89 90 91
	 *
	 * For 32bit kernels, these mappings are same and we can use
	 * kernel identity mapping to modify code.
	 */
	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
92
		ip = (unsigned long)__va(__pa_symbol(ip));
93

94
	return ip;
95 96
}

97
static const unsigned char *ftrace_nop_replace(void)
98
{
99
	return ideal_nops[NOP_ATOMIC5];
100 101
}

102
static int
103
ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
104
		   unsigned const char *new_code)
105
{
106
	unsigned char replaced[MCOUNT_INSN_SIZE];
107

108 109
	ftrace_expected = old_code;

110
	/*
111 112 113 114 115
	 * Note:
	 * We are paranoid about modifying text, as if a bug was to happen, it
	 * could cause us to read or write to someplace that could cause harm.
	 * Carefully read and modify the code with probe_kernel_*(), and make
	 * sure what we read is what we expected it to be before modifying it.
116
	 */
S
Steven Rostedt 已提交
117 118

	/* read the text we want to modify */
S
Steven Rostedt 已提交
119
	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
120
		return -EFAULT;
121

S
Steven Rostedt 已提交
122
	/* Make sure it is what we expect it to be */
123
	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
124
		return -EINVAL;
125

126 127
	ip = text_ip_addr(ip);

S
Steven Rostedt 已提交
128
	/* replace the text with the new text */
129
	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
130
		return -EPERM;
131 132

	sync_core();
133

134
	return 0;
135 136
}

137 138 139
int ftrace_make_nop(struct module *mod,
		    struct dyn_ftrace *rec, unsigned long addr)
{
140
	unsigned const char *new, *old;
141 142 143 144 145
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);
	new = ftrace_nop_replace();

146 147 148 149 150 151 152 153 154 155 156
	/*
	 * On boot up, and when modules are loaded, the MCOUNT_ADDR
	 * is converted to a nop, and will never become MCOUNT_ADDR
	 * again. This code is either running before SMP (on boot up)
	 * or before the code will ever be executed (module load).
	 * We do not want to use the breakpoint version in this case,
	 * just modify the code directly.
	 */
	if (addr == MCOUNT_ADDR)
		return ftrace_modify_code_direct(rec->ip, old, new);

157 158
	ftrace_expected = NULL;

159 160 161
	/* Normal cases use add_brk_on_nop */
	WARN_ONCE(1, "invalid use of ftrace_make_nop");
	return -EINVAL;
162 163 164 165
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
166
	unsigned const char *new, *old;
167 168 169 170 171
	unsigned long ip = rec->ip;

	old = ftrace_nop_replace();
	new = ftrace_call_replace(ip, addr);

172 173
	/* Should only be called when module is loaded */
	return ftrace_modify_code_direct(rec->ip, old, new);
174 175
}

176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
/*
 * The modifying_ftrace_code is used to tell the breakpoint
 * handler to call ftrace_int3_handler(). If it fails to
 * call this handler for a breakpoint added by ftrace, then
 * the kernel may crash.
 *
 * As atomic_writes on x86 do not need a barrier, we do not
 * need to add smp_mb()s for this to work. It is also considered
 * that we can not read the modifying_ftrace_code before
 * executing the breakpoint. That would be quite remarkable if
 * it could do that. Here's the flow that is required:
 *
 *   CPU-0                          CPU-1
 *
 * atomic_inc(mfc);
 * write int3s
 *				<trap-int3> // implicit (r)mb
 *				if (atomic_read(mfc))
 *					call ftrace_int3_handler()
 *
 * Then when we are finished:
 *
 * atomic_dec(mfc);
 *
 * If we hit a breakpoint that was not set by ftrace, it does not
 * matter if ftrace_int3_handler() is called or not. It will
 * simply be ignored. But it is crucial that a ftrace nop/caller
 * breakpoint is handled. No other user should ever place a
 * breakpoint on an ftrace nop/caller location. It must only
 * be done by this code.
 */
atomic_t modifying_ftrace_code __read_mostly;
208

209 210 211 212
static int
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
		   unsigned const char *new_code);

213 214 215 216 217 218 219 220 221 222 223 224
/*
 * Should never be called:
 *  As it is only called by __ftrace_replace_code() which is called by
 *  ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
 *  which is called to turn mcount into nops or nops into function calls
 *  but not to convert a function from not using regs to one that uses
 *  regs, which ftrace_modify_call() is for.
 */
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
				 unsigned long addr)
{
	WARN_ON(1);
225
	ftrace_expected = NULL;
226 227 228
	return -EINVAL;
}

229 230 231
static unsigned long ftrace_update_func;

static int update_ftrace_func(unsigned long ip, void *new)
232
{
233
	unsigned char old[MCOUNT_INSN_SIZE];
234 235
	int ret;

236 237 238 239 240
	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);

	ftrace_update_func = ip;
	/* Make sure the breakpoints see the ftrace_update_func update */
	smp_wmb();
241 242 243 244 245 246

	/* See comment above by declaration of modifying_ftrace_code */
	atomic_inc(&modifying_ftrace_code);

	ret = ftrace_modify_code(ip, old, new);

247 248 249 250 251 252 253 254 255 256 257 258 259 260
	atomic_dec(&modifying_ftrace_code);

	return ret;
}

int ftrace_update_ftrace_func(ftrace_func_t func)
{
	unsigned long ip = (unsigned long)(&ftrace_call);
	unsigned char *new;
	int ret;

	new = ftrace_call_replace(ip, (unsigned long)func);
	ret = update_ftrace_func(ip, new);

261 262 263 264
	/* Also update the regs callback function */
	if (!ret) {
		ip = (unsigned long)(&ftrace_regs_call);
		new = ftrace_call_replace(ip, (unsigned long)func);
265
		ret = update_ftrace_func(ip, new);
266 267
	}

268 269 270
	return ret;
}

271 272
static int is_ftrace_caller(unsigned long ip)
{
273
	if (ip == ftrace_update_func)
274 275 276 277 278
		return 1;

	return 0;
}

279 280 281 282 283 284 285 286 287
/*
 * A breakpoint was added to the code address we are about to
 * modify, and this is the handle that will just skip over it.
 * We are either changing a nop into a trace call, or a trace
 * call to a nop. While the change is taking place, we treat
 * it just like it was a nop.
 */
int ftrace_int3_handler(struct pt_regs *regs)
{
288 289
	unsigned long ip;

290 291 292
	if (WARN_ON_ONCE(!regs))
		return 0;

293 294
	ip = regs->ip - 1;
	if (!ftrace_location(ip) && !is_ftrace_caller(ip))
295 296 297 298 299 300 301 302 303
		return 0;

	regs->ip += MCOUNT_INSN_SIZE - 1;

	return 1;
}

static int ftrace_write(unsigned long ip, const char *val, int size)
{
304
	ip = text_ip_addr(ip);
305

306 307 308 309
	if (probe_kernel_write((void *)ip, val, size))
		return -EPERM;

	return 0;
310 311 312 313 314 315 316 317 318 319
}

static int add_break(unsigned long ip, const char *old)
{
	unsigned char replaced[MCOUNT_INSN_SIZE];
	unsigned char brk = BREAKPOINT_INSTRUCTION;

	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

320 321
	ftrace_expected = old;

322 323 324 325
	/* Make sure it is what we expect it to be */
	if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
		return -EINVAL;

326
	return ftrace_write(ip, &brk, 1);
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
}

static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned const char *old;
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);

	return add_break(rec->ip, old);
}


static int add_brk_on_nop(struct dyn_ftrace *rec)
{
	unsigned const char *old;

	old = ftrace_nop_replace();

	return add_break(rec->ip, old);
}

static int add_breakpoints(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

354
	ftrace_addr = ftrace_get_addr_curr(rec);
355

356
	ret = ftrace_test_record(rec, enable);
357 358 359 360 361 362 363 364 365

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return add_brk_on_nop(rec);

366
	case FTRACE_UPDATE_MODIFY_CALL:
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return add_brk_on_call(rec, ftrace_addr);
	}
	return 0;
}

/*
 * On error, we need to remove breakpoints. This needs to
 * be done caefully. If the address does not currently have a
 * breakpoint, we know we are done. Otherwise, we look at the
 * remaining 4 bytes of the instruction. If it matches a nop
 * we replace the breakpoint with the nop. Otherwise we replace
 * it with the call instruction.
 */
static int remove_breakpoint(struct dyn_ftrace *rec)
{
	unsigned char ins[MCOUNT_INSN_SIZE];
	unsigned char brk = BREAKPOINT_INSTRUCTION;
	const unsigned char *nop;
	unsigned long ftrace_addr;
	unsigned long ip = rec->ip;

	/* If we fail the read, just give up */
	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	/* If this does not have a breakpoint, we are done */
	if (ins[0] != brk)
396
		return 0;
397 398 399 400 401 402 403 404 405 406 407 408 409 410

	nop = ftrace_nop_replace();

	/*
	 * If the last 4 bytes of the instruction do not match
	 * a nop, then we assume that this is a call to ftrace_addr.
	 */
	if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
		/*
		 * For extra paranoidism, we check if the breakpoint is on
		 * a call that would actually jump to the ftrace_addr.
		 * If not, don't touch the breakpoint, we make just create
		 * a disaster.
		 */
411
		ftrace_addr = ftrace_get_addr_new(rec);
412 413 414 415 416 417
		nop = ftrace_call_replace(ip, ftrace_addr);

		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
			goto update;

		/* Check both ftrace_addr and ftrace_old_addr */
418
		ftrace_addr = ftrace_get_addr_curr(rec);
419 420
		nop = ftrace_call_replace(ip, ftrace_addr);

421 422
		ftrace_expected = nop;

423 424 425 426
		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
			return -EINVAL;
	}

427
 update:
428
	return ftrace_write(ip, nop, 1);
429 430 431 432 433 434 435
}

static int add_update_code(unsigned long ip, unsigned const char *new)
{
	/* skip breakpoint */
	ip++;
	new++;
436
	return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
}

static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_call_replace(ip, addr);
	return add_update_code(ip, new);
}

static int add_update_nop(struct dyn_ftrace *rec)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_nop_replace();
	return add_update_code(ip, new);
}

static int add_update(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_test_record(rec, enable);

464
	ftrace_addr  = ftrace_get_addr_new(rec);
465 466 467 468 469

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

470
	case FTRACE_UPDATE_MODIFY_CALL:
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return add_update_call(rec, ftrace_addr);

	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return add_update_nop(rec);
	}

	return 0;
}

static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_call_replace(ip, addr);

490
	return ftrace_write(ip, new, 1);
491 492 493 494 495 496 497 498 499
}

static int finish_update_nop(struct dyn_ftrace *rec)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_nop_replace();

500
	return ftrace_write(ip, new, 1);
501 502 503 504 505 506 507 508 509
}

static int finish_update(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_update_record(rec, enable);

510
	ftrace_addr = ftrace_get_addr_new(rec);
511 512 513 514 515

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

516
	case FTRACE_UPDATE_MODIFY_CALL:
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535
	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return finish_update_call(rec, ftrace_addr);

	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return finish_update_nop(rec);
	}

	return 0;
}

static void do_sync_core(void *data)
{
	sync_core();
}

static void run_sync(void)
{
536 537 538 539 540 541 542
	int enable_irqs;

	/* No need to sync if there's only one CPU */
	if (num_online_cpus() == 1)
		return;

	enable_irqs = irqs_disabled();
543

544
	/* We may be called with interrupts disabled (on bootup). */
545 546 547 548 549 550 551
	if (enable_irqs)
		local_irq_enable();
	on_each_cpu(do_sync_core, NULL, 1);
	if (enable_irqs)
		local_irq_disable();
}

552
void ftrace_replace_code(int enable)
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
{
	struct ftrace_rec_iter *iter;
	struct dyn_ftrace *rec;
	const char *report = "adding breakpoints";
	int count = 0;
	int ret;

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = add_breakpoints(rec, enable);
		if (ret)
			goto remove_breakpoints;
		count++;
	}

	run_sync();

	report = "updating code";
572
	count = 0;
573 574 575 576 577 578 579

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = add_update(rec, enable);
		if (ret)
			goto remove_breakpoints;
580
		count++;
581 582 583 584 585
	}

	run_sync();

	report = "removing breakpoints";
586
	count = 0;
587 588 589 590 591 592 593

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = finish_update(rec, enable);
		if (ret)
			goto remove_breakpoints;
594
		count++;
595 596 597 598 599 600 601
	}

	run_sync();

	return;

 remove_breakpoints:
602
	pr_warn("Failed on %s (%d):\n", report, count);
603
	ftrace_bug(ret, rec);
604 605
	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);
606 607 608 609 610 611
		/*
		 * Breakpoints are handled only when this function is in
		 * progress. The system could not work with them.
		 */
		if (remove_breakpoint(rec))
			BUG();
612
	}
613
	run_sync();
614 615
}

616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
static int
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
		   unsigned const char *new_code)
{
	int ret;

	ret = add_break(ip, old_code);
	if (ret)
		goto out;

	run_sync();

	ret = add_update_code(ip, new_code);
	if (ret)
		goto fail_update;

	run_sync();

	ret = ftrace_write(ip, new_code, 1);
635 636 637 638 639
	/*
	 * The breakpoint is handled only when this function is in progress.
	 * The system could not work if we could not remove it.
	 */
	BUG_ON(ret);
640
 out:
641
	run_sync();
642 643 644
	return ret;

 fail_update:
645 646 647
	/* Also here the system could not work with the breakpoint */
	if (ftrace_write(ip, old_code, 1))
		BUG();
648 649 650
	goto out;
}

651 652
void arch_ftrace_update_code(int command)
{
653 654
	/* See comment above by declaration of modifying_ftrace_code */
	atomic_inc(&modifying_ftrace_code);
655

656
	ftrace_modify_all_code(command);
657

658
	atomic_dec(&modifying_ftrace_code);
659 660
}

661
int __init ftrace_dyn_arch_init(void)
662 663 664
{
	return 0;
}
665

666
#if defined(CONFIG_X86_64) || defined(CONFIG_FUNCTION_GRAPH_TRACER)
667
static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
668
{
669
	static union ftrace_code_union calc;
670

671 672 673
	/* Jmp not a call (ignore the .e8) */
	calc.e8		= 0xe9;
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
674

675 676 677 678 679
	/*
	 * ftrace external locks synchronize the access to the static variable.
	 */
	return calc.code;
}
680 681 682 683 684 685 686 687 688 689 690 691 692 693
#endif

/* Currently only x86_64 supports dynamic trampolines */
#ifdef CONFIG_X86_64

#ifdef CONFIG_MODULES
#include <linux/moduleloader.h>
/* Module allocation simplifies allocating memory for code */
static inline void *alloc_tramp(unsigned long size)
{
	return module_alloc(size);
}
static inline void tramp_free(void *tramp)
{
694
	module_memfree(tramp);
695 696 697 698 699 700 701 702 703 704 705 706
}
#else
/* Trampolines can only be created if modules are supported */
static inline void *alloc_tramp(unsigned long size)
{
	return NULL;
}
static inline void tramp_free(void *tramp) { }
#endif

/* Defined as markers to the end of the ftrace default trampolines */
extern void ftrace_regs_caller_end(void);
707
extern void ftrace_epilogue(void);
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730
extern void ftrace_caller_op_ptr(void);
extern void ftrace_regs_caller_op_ptr(void);

/* movq function_trace_op(%rip), %rdx */
/* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
#define OP_REF_SIZE	7

/*
 * The ftrace_ops is passed to the function callback. Since the
 * trampoline only services a single ftrace_ops, we can pass in
 * that ops directly.
 *
 * The ftrace_op_code_union is used to create a pointer to the
 * ftrace_ops that will be passed to the callback function.
 */
union ftrace_op_code_union {
	char code[OP_REF_SIZE];
	struct {
		char op[3];
		int offset;
	} __attribute__((packed));
};

731 732
static unsigned long
create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
{
	unsigned const char *jmp;
	unsigned long start_offset;
	unsigned long end_offset;
	unsigned long op_offset;
	unsigned long offset;
	unsigned long size;
	unsigned long ip;
	unsigned long *ptr;
	void *trampoline;
	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
	union ftrace_op_code_union op_ptr;
	int ret;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
		start_offset = (unsigned long)ftrace_regs_caller;
		end_offset = (unsigned long)ftrace_regs_caller_end;
		op_offset = (unsigned long)ftrace_regs_caller_op_ptr;
	} else {
		start_offset = (unsigned long)ftrace_caller;
754
		end_offset = (unsigned long)ftrace_epilogue;
755 756 757 758 759 760 761
		op_offset = (unsigned long)ftrace_caller_op_ptr;
	}

	size = end_offset - start_offset;

	/*
	 * Allocate enough size to store the ftrace_caller code,
762
	 * the jmp to ftrace_epilogue, as well as the address of
763 764 765 766 767 768
	 * the ftrace_ops this trampoline is used for.
	 */
	trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));
	if (!trampoline)
		return 0;

769 770
	*tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *);

771 772 773 774 775 776 777 778 779
	/* Copy ftrace_caller onto the trampoline memory */
	ret = probe_kernel_read(trampoline, (void *)start_offset, size);
	if (WARN_ON(ret < 0)) {
		tramp_free(trampoline);
		return 0;
	}

	ip = (unsigned long)trampoline + size;

780 781
	/* The trampoline ends with a jmp to ftrace_epilogue */
	jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue);
782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
	memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);

	/*
	 * The address of the ftrace_ops that is used for this trampoline
	 * is stored at the end of the trampoline. This will be used to
	 * load the third parameter for the callback. Basically, that
	 * location at the end of the trampoline takes the place of
	 * the global function_trace_op variable.
	 */

	ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE);
	*ptr = (unsigned long)ops;

	op_offset -= start_offset;
	memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE);

	/* Are we pointing to the reference? */
	if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0)) {
		tramp_free(trampoline);
		return 0;
	}

	/* Load the contents of ptr into the callback parameter */
	offset = (unsigned long)ptr;
	offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;

	op_ptr.offset = offset;

	/* put in the new offset to the ftrace_ops */
	memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);

	/* ALLOC_TRAMP flags lets us know we created it */
	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;

	return (unsigned long)trampoline;
}

819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834
static unsigned long calc_trampoline_call_offset(bool save_regs)
{
	unsigned long start_offset;
	unsigned long call_offset;

	if (save_regs) {
		start_offset = (unsigned long)ftrace_regs_caller;
		call_offset = (unsigned long)ftrace_regs_call;
	} else {
		start_offset = (unsigned long)ftrace_caller;
		call_offset = (unsigned long)ftrace_call;
	}

	return call_offset - start_offset;
}

835 836 837 838 839 840
void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
	ftrace_func_t func;
	unsigned char *new;
	unsigned long offset;
	unsigned long ip;
841
	unsigned int size;
842 843 844 845 846 847 848 849 850 851
	int ret;

	if (ops->trampoline) {
		/*
		 * The ftrace_ops caller may set up its own trampoline.
		 * In such a case, this code must not modify it.
		 */
		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
			return;
	} else {
852
		ops->trampoline = create_trampoline(ops, &size);
853 854
		if (!ops->trampoline)
			return;
855
		ops->trampoline_size = size;
856 857
	}

858
	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
859 860 861 862 863 864 865 866 867 868 869
	ip = ops->trampoline + offset;

	func = ftrace_ops_get_func(ops);

	/* Do a safe modify in case the trampoline is executing */
	new = ftrace_call_replace(ip, (unsigned long)func);
	ret = update_ftrace_func(ip, new);

	/* The update should never fail */
	WARN_ON(ret);
}
870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889

/* Return the address of the function the trampoline calls */
static void *addr_from_call(void *ptr)
{
	union ftrace_code_union calc;
	int ret;

	ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
	if (WARN_ON_ONCE(ret < 0))
		return NULL;

	/* Make sure this is a call */
	if (WARN_ON_ONCE(calc.e8 != 0xe8)) {
		pr_warn("Expected e8, got %x\n", calc.e8);
		return NULL;
	}

	return ptr + MCOUNT_INSN_SIZE + calc.offset;
}

890
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936
			   unsigned long frame_pointer);

/*
 * If the ops->trampoline was not allocated, then it probably
 * has a static trampoline func, or is the ftrace caller itself.
 */
static void *static_tramp_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	unsigned long offset;
	bool save_regs = rec->flags & FTRACE_FL_REGS_EN;
	void *ptr;

	if (ops && ops->trampoline) {
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
		/*
		 * We only know about function graph tracer setting as static
		 * trampoline.
		 */
		if (ops->trampoline == FTRACE_GRAPH_ADDR)
			return (void *)prepare_ftrace_return;
#endif
		return NULL;
	}

	offset = calc_trampoline_call_offset(save_regs);

	if (save_regs)
		ptr = (void *)FTRACE_REGS_ADDR + offset;
	else
		ptr = (void *)FTRACE_ADDR + offset;

	return addr_from_call(ptr);
}

void *arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	unsigned long offset;

	/* If we didn't allocate this trampoline, consider it static */
	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
		return static_tramp_func(ops, rec);

	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
	return addr_from_call((void *)ops->trampoline + offset);
}

937 938 939 940 941 942 943 944
void arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
		return;

	tramp_free((void *)ops->trampoline);
	ops->trampoline = 0;
}
945

946 947 948 949 950 951 952
#endif /* CONFIG_X86_64 */
#endif /* CONFIG_DYNAMIC_FTRACE */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER

#ifdef CONFIG_DYNAMIC_FTRACE
extern void ftrace_graph_call(void);
953

954 955 956
static int ftrace_mod_jmp(unsigned long ip, void *func)
{
	unsigned char *new;
957

958
	new = ftrace_jmp_replace(ip, (unsigned long)func);
959

960
	return update_ftrace_func(ip, new);
961 962 963 964 965 966
}

int ftrace_enable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);

967
	return ftrace_mod_jmp(ip, &ftrace_graph_caller);
968 969 970 971 972 973
}

int ftrace_disable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);

974
	return ftrace_mod_jmp(ip, &ftrace_stub);
975 976
}

977 978 979 980 981 982
#endif /* !CONFIG_DYNAMIC_FTRACE */

/*
 * Hook the return address and push it in the stack of return addrs
 * in current thread info.
 */
983
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
984
			   unsigned long frame_pointer)
985 986 987
{
	unsigned long old;
	int faulted;
988
	struct ftrace_graph_ent trace;
989 990 991
	unsigned long return_hooker = (unsigned long)
				&return_to_handler;

992 993 994 995 996 997 998 999 1000 1001 1002 1003
	/*
	 * When resuming from suspend-to-ram, this function can be indirectly
	 * called from early CPU startup code while the CPU is in real mode,
	 * which would fail miserably.  Make sure the stack pointer is a
	 * virtual address.
	 *
	 * This check isn't as accurate as virt_addr_valid(), but it should be
	 * good enough for this purpose, and it's fast.
	 */
	if (unlikely((long)__builtin_frame_address(0) >= 0))
		return;

1004 1005 1006
	if (unlikely(ftrace_graph_is_dead()))
		return;

1007
	if (unlikely(atomic_read(&current->tracing_graph_pause)))
1008 1009 1010 1011 1012 1013 1014 1015
		return;

	/*
	 * Protect against fault, even if it shouldn't
	 * happen. This tool is too much intrusive to
	 * ignore such a protection.
	 */
	asm volatile(
1016 1017
		"1: " _ASM_MOV " (%[parent]), %[old]\n"
		"2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
1018
		"   movl $0, %[faulted]\n"
1019
		"3:\n"
1020 1021

		".section .fixup, \"ax\"\n"
1022 1023
		"4: movl $1, %[faulted]\n"
		"   jmp 3b\n"
1024 1025
		".previous\n"

1026 1027
		_ASM_EXTABLE(1b, 4b)
		_ASM_EXTABLE(2b, 4b)
1028

1029
		: [old] "=&r" (old), [faulted] "=r" (faulted)
1030
		: [parent] "r" (parent), [return_hooker] "r" (return_hooker)
1031 1032 1033
		: "memory"
	);

S
Steven Rostedt 已提交
1034 1035 1036
	if (unlikely(faulted)) {
		ftrace_graph_stop();
		WARN_ON(1);
1037 1038 1039
		return;
	}

1040
	trace.func = self_addr;
1041
	trace.depth = current->curr_ret_stack + 1;
1042

1043 1044 1045
	/* Only trace if the calling function expects to */
	if (!ftrace_graph_entry(&trace)) {
		*parent = old;
1046 1047 1048 1049
		return;
	}

	if (ftrace_push_return_trace(old, self_addr, &trace.depth,
1050
				     frame_pointer, parent) == -EBUSY) {
1051 1052
		*parent = old;
		return;
1053
	}
1054
}
1055
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */