ftrace.c 23.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * Code for replacing ftrace calls with jumps.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 *
 * Thanks goes to Ingo Molnar, for suggesting the idea.
 * Mathieu Desnoyers, for suggesting postponing the modifications.
 * Arjan van de Ven, for keeping me straight, and explaining to me
 * the dangers of modifying code on the run.
 */

12 13
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

14 15
#include <linux/spinlock.h>
#include <linux/hardirq.h>
16
#include <linux/uaccess.h>
17 18
#include <linux/ftrace.h>
#include <linux/percpu.h>
19
#include <linux/sched.h>
20
#include <linux/slab.h>
21 22
#include <linux/init.h>
#include <linux/list.h>
23
#include <linux/module.h>
24

25 26
#include <trace/syscall.h>

27
#include <asm/cacheflush.h>
28
#include <asm/kprobes.h>
29
#include <asm/ftrace.h>
30
#include <asm/nops.h>
31

32
#ifdef CONFIG_DYNAMIC_FTRACE
33

34 35 36
int ftrace_arch_code_modify_prepare(void)
{
	set_kernel_text_rw();
37
	set_all_modules_text_rw();
38 39 40 41 42
	return 0;
}

int ftrace_arch_code_modify_post_process(void)
{
43
	set_all_modules_text_ro();
44 45 46 47
	set_kernel_text_ro();
	return 0;
}

48
union ftrace_code_union {
49
	char code[MCOUNT_INSN_SIZE];
50
	struct {
51
		unsigned char e8;
52 53 54 55
		int offset;
	} __attribute__((packed));
};

56
static int ftrace_calc_offset(long ip, long addr)
57 58 59
{
	return (int)(addr - ip);
}
60

61
static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
62 63
{
	static union ftrace_code_union calc;
64

65
	calc.e8		= 0xe8;
66
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
67 68 69 70 71 72

	/*
	 * No locking needed, this must be called via kstop_machine
	 * which in essence is like running on a uniprocessor machine.
	 */
	return calc.code;
73 74
}

75 76 77 78 79 80
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
	return addr >= start && addr < end;
}

81
static unsigned long text_ip_addr(unsigned long ip)
82
{
83 84 85 86 87 88 89 90 91
	/*
	 * On x86_64, kernel text mappings are mapped read-only with
	 * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
	 * of the kernel text mapping to modify the kernel text.
	 *
	 * For 32bit kernels, these mappings are same and we can use
	 * kernel identity mapping to modify code.
	 */
	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
92
		ip = (unsigned long)__va(__pa_symbol(ip));
93

94
	return ip;
95 96
}

97
static const unsigned char *ftrace_nop_replace(void)
98
{
99
	return ideal_nops[NOP_ATOMIC5];
100 101
}

102
static int
103
ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
104
		   unsigned const char *new_code)
105
{
106
	unsigned char replaced[MCOUNT_INSN_SIZE];
107 108 109 110

	/*
	 * Note: Due to modules and __init, code can
	 *  disappear and change, we need to protect against faulting
S
Steven Rostedt 已提交
111
	 *  as well as code changing. We do this by using the
S
Steven Rostedt 已提交
112
	 *  probe_kernel_* functions.
113 114
	 *
	 * No real locking needed, this code is run through
115
	 * kstop_machine, or before SMP starts.
116
	 */
S
Steven Rostedt 已提交
117 118

	/* read the text we want to modify */
S
Steven Rostedt 已提交
119
	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
120
		return -EFAULT;
121

S
Steven Rostedt 已提交
122
	/* Make sure it is what we expect it to be */
123
	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
124
		return -EINVAL;
125

126 127
	ip = text_ip_addr(ip);

S
Steven Rostedt 已提交
128
	/* replace the text with the new text */
129
	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
130
		return -EPERM;
131 132

	sync_core();
133

134
	return 0;
135 136
}

137 138 139
int ftrace_make_nop(struct module *mod,
		    struct dyn_ftrace *rec, unsigned long addr)
{
140
	unsigned const char *new, *old;
141 142 143 144 145
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);
	new = ftrace_nop_replace();

146 147 148 149 150 151 152 153 154 155 156 157 158 159
	/*
	 * On boot up, and when modules are loaded, the MCOUNT_ADDR
	 * is converted to a nop, and will never become MCOUNT_ADDR
	 * again. This code is either running before SMP (on boot up)
	 * or before the code will ever be executed (module load).
	 * We do not want to use the breakpoint version in this case,
	 * just modify the code directly.
	 */
	if (addr == MCOUNT_ADDR)
		return ftrace_modify_code_direct(rec->ip, old, new);

	/* Normal cases use add_brk_on_nop */
	WARN_ONCE(1, "invalid use of ftrace_make_nop");
	return -EINVAL;
160 161 162 163
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
164
	unsigned const char *new, *old;
165 166 167 168 169
	unsigned long ip = rec->ip;

	old = ftrace_nop_replace();
	new = ftrace_call_replace(ip, addr);

170 171
	/* Should only be called when module is loaded */
	return ftrace_modify_code_direct(rec->ip, old, new);
172 173
}

174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
/*
 * The modifying_ftrace_code is used to tell the breakpoint
 * handler to call ftrace_int3_handler(). If it fails to
 * call this handler for a breakpoint added by ftrace, then
 * the kernel may crash.
 *
 * As atomic_writes on x86 do not need a barrier, we do not
 * need to add smp_mb()s for this to work. It is also considered
 * that we can not read the modifying_ftrace_code before
 * executing the breakpoint. That would be quite remarkable if
 * it could do that. Here's the flow that is required:
 *
 *   CPU-0                          CPU-1
 *
 * atomic_inc(mfc);
 * write int3s
 *				<trap-int3> // implicit (r)mb
 *				if (atomic_read(mfc))
 *					call ftrace_int3_handler()
 *
 * Then when we are finished:
 *
 * atomic_dec(mfc);
 *
 * If we hit a breakpoint that was not set by ftrace, it does not
 * matter if ftrace_int3_handler() is called or not. It will
 * simply be ignored. But it is crucial that a ftrace nop/caller
 * breakpoint is handled. No other user should ever place a
 * breakpoint on an ftrace nop/caller location. It must only
 * be done by this code.
 */
atomic_t modifying_ftrace_code __read_mostly;
206

207 208 209 210
static int
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
		   unsigned const char *new_code);

211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
/*
 * Should never be called:
 *  As it is only called by __ftrace_replace_code() which is called by
 *  ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
 *  which is called to turn mcount into nops or nops into function calls
 *  but not to convert a function from not using regs to one that uses
 *  regs, which ftrace_modify_call() is for.
 */
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
				 unsigned long addr)
{
	WARN_ON(1);
	return -EINVAL;
}

226 227 228
static unsigned long ftrace_update_func;

static int update_ftrace_func(unsigned long ip, void *new)
229
{
230
	unsigned char old[MCOUNT_INSN_SIZE];
231 232
	int ret;

233 234 235 236 237
	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);

	ftrace_update_func = ip;
	/* Make sure the breakpoints see the ftrace_update_func update */
	smp_wmb();
238 239 240 241 242 243

	/* See comment above by declaration of modifying_ftrace_code */
	atomic_inc(&modifying_ftrace_code);

	ret = ftrace_modify_code(ip, old, new);

244 245 246 247 248 249 250 251 252 253 254 255 256 257
	atomic_dec(&modifying_ftrace_code);

	return ret;
}

int ftrace_update_ftrace_func(ftrace_func_t func)
{
	unsigned long ip = (unsigned long)(&ftrace_call);
	unsigned char *new;
	int ret;

	new = ftrace_call_replace(ip, (unsigned long)func);
	ret = update_ftrace_func(ip, new);

258 259 260 261
	/* Also update the regs callback function */
	if (!ret) {
		ip = (unsigned long)(&ftrace_regs_call);
		new = ftrace_call_replace(ip, (unsigned long)func);
262
		ret = update_ftrace_func(ip, new);
263 264
	}

265 266 267
	return ret;
}

268 269
static int is_ftrace_caller(unsigned long ip)
{
270
	if (ip == ftrace_update_func)
271 272 273 274 275
		return 1;

	return 0;
}

276 277 278 279 280 281 282 283 284
/*
 * A breakpoint was added to the code address we are about to
 * modify, and this is the handle that will just skip over it.
 * We are either changing a nop into a trace call, or a trace
 * call to a nop. While the change is taking place, we treat
 * it just like it was a nop.
 */
int ftrace_int3_handler(struct pt_regs *regs)
{
285 286
	unsigned long ip;

287 288 289
	if (WARN_ON_ONCE(!regs))
		return 0;

290 291
	ip = regs->ip - 1;
	if (!ftrace_location(ip) && !is_ftrace_caller(ip))
292 293 294 295 296 297 298 299 300
		return 0;

	regs->ip += MCOUNT_INSN_SIZE - 1;

	return 1;
}

static int ftrace_write(unsigned long ip, const char *val, int size)
{
301
	ip = text_ip_addr(ip);
302

303 304 305 306
	if (probe_kernel_write((void *)ip, val, size))
		return -EPERM;

	return 0;
307 308 309 310 311 312 313 314 315 316 317 318 319 320
}

static int add_break(unsigned long ip, const char *old)
{
	unsigned char replaced[MCOUNT_INSN_SIZE];
	unsigned char brk = BREAKPOINT_INSTRUCTION;

	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	/* Make sure it is what we expect it to be */
	if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
		return -EINVAL;

321
	return ftrace_write(ip, &brk, 1);
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
}

static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned const char *old;
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);

	return add_break(rec->ip, old);
}


static int add_brk_on_nop(struct dyn_ftrace *rec)
{
	unsigned const char *old;

	old = ftrace_nop_replace();

	return add_break(rec->ip, old);
}

static int add_breakpoints(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

349
	ftrace_addr = ftrace_get_addr_curr(rec);
350

351
	ret = ftrace_test_record(rec, enable);
352 353 354 355 356 357 358 359 360

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return add_brk_on_nop(rec);

361
	case FTRACE_UPDATE_MODIFY_CALL:
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return add_brk_on_call(rec, ftrace_addr);
	}
	return 0;
}

/*
 * On error, we need to remove breakpoints. This needs to
 * be done caefully. If the address does not currently have a
 * breakpoint, we know we are done. Otherwise, we look at the
 * remaining 4 bytes of the instruction. If it matches a nop
 * we replace the breakpoint with the nop. Otherwise we replace
 * it with the call instruction.
 */
static int remove_breakpoint(struct dyn_ftrace *rec)
{
	unsigned char ins[MCOUNT_INSN_SIZE];
	unsigned char brk = BREAKPOINT_INSTRUCTION;
	const unsigned char *nop;
	unsigned long ftrace_addr;
	unsigned long ip = rec->ip;

	/* If we fail the read, just give up */
	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	/* If this does not have a breakpoint, we are done */
	if (ins[0] != brk)
391
		return 0;
392 393 394 395 396 397 398 399 400 401 402 403 404 405

	nop = ftrace_nop_replace();

	/*
	 * If the last 4 bytes of the instruction do not match
	 * a nop, then we assume that this is a call to ftrace_addr.
	 */
	if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
		/*
		 * For extra paranoidism, we check if the breakpoint is on
		 * a call that would actually jump to the ftrace_addr.
		 * If not, don't touch the breakpoint, we make just create
		 * a disaster.
		 */
406
		ftrace_addr = ftrace_get_addr_new(rec);
407 408 409 410 411 412
		nop = ftrace_call_replace(ip, ftrace_addr);

		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
			goto update;

		/* Check both ftrace_addr and ftrace_old_addr */
413
		ftrace_addr = ftrace_get_addr_curr(rec);
414 415 416 417 418 419
		nop = ftrace_call_replace(ip, ftrace_addr);

		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
			return -EINVAL;
	}

420
 update:
421
	return ftrace_write(ip, nop, 1);
422 423 424 425 426 427 428
}

static int add_update_code(unsigned long ip, unsigned const char *new)
{
	/* skip breakpoint */
	ip++;
	new++;
429
	return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
}

static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_call_replace(ip, addr);
	return add_update_code(ip, new);
}

static int add_update_nop(struct dyn_ftrace *rec)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_nop_replace();
	return add_update_code(ip, new);
}

static int add_update(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_test_record(rec, enable);

457
	ftrace_addr  = ftrace_get_addr_new(rec);
458 459 460 461 462

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

463
	case FTRACE_UPDATE_MODIFY_CALL:
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return add_update_call(rec, ftrace_addr);

	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return add_update_nop(rec);
	}

	return 0;
}

static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_call_replace(ip, addr);

483
	return ftrace_write(ip, new, 1);
484 485 486 487 488 489 490 491 492
}

static int finish_update_nop(struct dyn_ftrace *rec)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_nop_replace();

493
	return ftrace_write(ip, new, 1);
494 495 496 497 498 499 500 501 502
}

static int finish_update(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_update_record(rec, enable);

503
	ftrace_addr = ftrace_get_addr_new(rec);
504 505 506 507 508

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

509
	case FTRACE_UPDATE_MODIFY_CALL:
510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return finish_update_call(rec, ftrace_addr);

	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return finish_update_nop(rec);
	}

	return 0;
}

static void do_sync_core(void *data)
{
	sync_core();
}

static void run_sync(void)
{
	int enable_irqs = irqs_disabled();

	/* We may be called with interrupts disbled (on bootup). */
	if (enable_irqs)
		local_irq_enable();
	on_each_cpu(do_sync_core, NULL, 1);
	if (enable_irqs)
		local_irq_disable();
}

539
void ftrace_replace_code(int enable)
540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
{
	struct ftrace_rec_iter *iter;
	struct dyn_ftrace *rec;
	const char *report = "adding breakpoints";
	int count = 0;
	int ret;

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = add_breakpoints(rec, enable);
		if (ret)
			goto remove_breakpoints;
		count++;
	}

	run_sync();

	report = "updating code";

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = add_update(rec, enable);
		if (ret)
			goto remove_breakpoints;
	}

	run_sync();

	report = "removing breakpoints";

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = finish_update(rec, enable);
		if (ret)
			goto remove_breakpoints;
	}

	run_sync();

	return;

 remove_breakpoints:
585
	pr_warn("Failed on %s (%d):\n", report, count);
586 587 588
	ftrace_bug(ret, rec ? rec->ip : 0);
	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);
589 590 591 592 593 594
		/*
		 * Breakpoints are handled only when this function is in
		 * progress. The system could not work with them.
		 */
		if (remove_breakpoint(rec))
			BUG();
595
	}
596
	run_sync();
597 598
}

599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
static int
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
		   unsigned const char *new_code)
{
	int ret;

	ret = add_break(ip, old_code);
	if (ret)
		goto out;

	run_sync();

	ret = add_update_code(ip, new_code);
	if (ret)
		goto fail_update;

	run_sync();

	ret = ftrace_write(ip, new_code, 1);
618 619 620 621 622
	/*
	 * The breakpoint is handled only when this function is in progress.
	 * The system could not work if we could not remove it.
	 */
	BUG_ON(ret);
623
 out:
624
	run_sync();
625 626 627
	return ret;

 fail_update:
628 629 630
	/* Also here the system could not work with the breakpoint */
	if (ftrace_write(ip, old_code, 1))
		BUG();
631 632 633
	goto out;
}

634 635
void arch_ftrace_update_code(int command)
{
636 637
	/* See comment above by declaration of modifying_ftrace_code */
	atomic_inc(&modifying_ftrace_code);
638

639
	ftrace_modify_all_code(command);
640

641
	atomic_dec(&modifying_ftrace_code);
642 643
}

644
int __init ftrace_dyn_arch_init(void)
645 646 647
{
	return 0;
}
648

649
#if defined(CONFIG_X86_64) || defined(CONFIG_FUNCTION_GRAPH_TRACER)
650
static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
651
{
652
	static union ftrace_code_union calc;
653

654 655 656
	/* Jmp not a call (ignore the .e8) */
	calc.e8		= 0xe9;
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
657

658 659 660 661 662
	/*
	 * ftrace external locks synchronize the access to the static variable.
	 */
	return calc.code;
}
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
#endif

/* Currently only x86_64 supports dynamic trampolines */
#ifdef CONFIG_X86_64

#ifdef CONFIG_MODULES
#include <linux/moduleloader.h>
/* Module allocation simplifies allocating memory for code */
static inline void *alloc_tramp(unsigned long size)
{
	return module_alloc(size);
}
static inline void tramp_free(void *tramp)
{
	module_free(NULL, tramp);
}
#else
/* Trampolines can only be created if modules are supported */
static inline void *alloc_tramp(unsigned long size)
{
	return NULL;
}
static inline void tramp_free(void *tramp) { }
#endif

/* Defined as markers to the end of the ftrace default trampolines */
extern void ftrace_caller_end(void);
extern void ftrace_regs_caller_end(void);
extern void ftrace_return(void);
extern void ftrace_caller_op_ptr(void);
extern void ftrace_regs_caller_op_ptr(void);

/* movq function_trace_op(%rip), %rdx */
/* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
#define OP_REF_SIZE	7

/*
 * The ftrace_ops is passed to the function callback. Since the
 * trampoline only services a single ftrace_ops, we can pass in
 * that ops directly.
 *
 * The ftrace_op_code_union is used to create a pointer to the
 * ftrace_ops that will be passed to the callback function.
 */
union ftrace_op_code_union {
	char code[OP_REF_SIZE];
	struct {
		char op[3];
		int offset;
	} __attribute__((packed));
};

static unsigned long create_trampoline(struct ftrace_ops *ops)
{
	unsigned const char *jmp;
	unsigned long start_offset;
	unsigned long end_offset;
	unsigned long op_offset;
	unsigned long offset;
	unsigned long size;
	unsigned long ip;
	unsigned long *ptr;
	void *trampoline;
	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
	union ftrace_op_code_union op_ptr;
	int ret;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
		start_offset = (unsigned long)ftrace_regs_caller;
		end_offset = (unsigned long)ftrace_regs_caller_end;
		op_offset = (unsigned long)ftrace_regs_caller_op_ptr;
	} else {
		start_offset = (unsigned long)ftrace_caller;
		end_offset = (unsigned long)ftrace_caller_end;
		op_offset = (unsigned long)ftrace_caller_op_ptr;
	}

	size = end_offset - start_offset;

	/*
	 * Allocate enough size to store the ftrace_caller code,
	 * the jmp to ftrace_return, as well as the address of
	 * the ftrace_ops this trampoline is used for.
	 */
	trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));
	if (!trampoline)
		return 0;

	/* Copy ftrace_caller onto the trampoline memory */
	ret = probe_kernel_read(trampoline, (void *)start_offset, size);
	if (WARN_ON(ret < 0)) {
		tramp_free(trampoline);
		return 0;
	}

	ip = (unsigned long)trampoline + size;

	/* The trampoline ends with a jmp to ftrace_return */
	jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_return);
	memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);

	/*
	 * The address of the ftrace_ops that is used for this trampoline
	 * is stored at the end of the trampoline. This will be used to
	 * load the third parameter for the callback. Basically, that
	 * location at the end of the trampoline takes the place of
	 * the global function_trace_op variable.
	 */

	ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE);
	*ptr = (unsigned long)ops;

	op_offset -= start_offset;
	memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE);

	/* Are we pointing to the reference? */
	if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0)) {
		tramp_free(trampoline);
		return 0;
	}

	/* Load the contents of ptr into the callback parameter */
	offset = (unsigned long)ptr;
	offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;

	op_ptr.offset = offset;

	/* put in the new offset to the ftrace_ops */
	memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);

	/* ALLOC_TRAMP flags lets us know we created it */
	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;

	return (unsigned long)trampoline;
}

800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
static unsigned long calc_trampoline_call_offset(bool save_regs)
{
	unsigned long start_offset;
	unsigned long call_offset;

	if (save_regs) {
		start_offset = (unsigned long)ftrace_regs_caller;
		call_offset = (unsigned long)ftrace_regs_call;
	} else {
		start_offset = (unsigned long)ftrace_caller;
		call_offset = (unsigned long)ftrace_call;
	}

	return call_offset - start_offset;
}

816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
	ftrace_func_t func;
	unsigned char *new;
	unsigned long offset;
	unsigned long ip;
	int ret;

	if (ops->trampoline) {
		/*
		 * The ftrace_ops caller may set up its own trampoline.
		 * In such a case, this code must not modify it.
		 */
		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
			return;
	} else {
		ops->trampoline = create_trampoline(ops);
		if (!ops->trampoline)
			return;
	}

837
	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
838 839 840 841 842 843 844 845 846 847 848
	ip = ops->trampoline + offset;

	func = ftrace_ops_get_func(ops);

	/* Do a safe modify in case the trampoline is executing */
	new = ftrace_call_replace(ip, (unsigned long)func);
	ret = update_ftrace_func(ip, new);

	/* The update should never fail */
	WARN_ON(ret);
}
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916

/* Return the address of the function the trampoline calls */
static void *addr_from_call(void *ptr)
{
	union ftrace_code_union calc;
	int ret;

	ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
	if (WARN_ON_ONCE(ret < 0))
		return NULL;

	/* Make sure this is a call */
	if (WARN_ON_ONCE(calc.e8 != 0xe8)) {
		pr_warn("Expected e8, got %x\n", calc.e8);
		return NULL;
	}

	return ptr + MCOUNT_INSN_SIZE + calc.offset;
}

void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
			   unsigned long frame_pointer);

/*
 * If the ops->trampoline was not allocated, then it probably
 * has a static trampoline func, or is the ftrace caller itself.
 */
static void *static_tramp_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	unsigned long offset;
	bool save_regs = rec->flags & FTRACE_FL_REGS_EN;
	void *ptr;

	if (ops && ops->trampoline) {
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
		/*
		 * We only know about function graph tracer setting as static
		 * trampoline.
		 */
		if (ops->trampoline == FTRACE_GRAPH_ADDR)
			return (void *)prepare_ftrace_return;
#endif
		return NULL;
	}

	offset = calc_trampoline_call_offset(save_regs);

	if (save_regs)
		ptr = (void *)FTRACE_REGS_ADDR + offset;
	else
		ptr = (void *)FTRACE_ADDR + offset;

	return addr_from_call(ptr);
}

void *arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	unsigned long offset;

	/* If we didn't allocate this trampoline, consider it static */
	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
		return static_tramp_func(ops, rec);

	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
	return addr_from_call((void *)ops->trampoline + offset);
}


917 918 919 920 921 922 923
#endif /* CONFIG_X86_64 */
#endif /* CONFIG_DYNAMIC_FTRACE */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER

#ifdef CONFIG_DYNAMIC_FTRACE
extern void ftrace_graph_call(void);
924

925 926 927
static int ftrace_mod_jmp(unsigned long ip, void *func)
{
	unsigned char *new;
928

929
	new = ftrace_jmp_replace(ip, (unsigned long)func);
930

931
	return update_ftrace_func(ip, new);
932 933 934 935 936 937
}

int ftrace_enable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);

938
	return ftrace_mod_jmp(ip, &ftrace_graph_caller);
939 940 941 942 943 944
}

int ftrace_disable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);

945
	return ftrace_mod_jmp(ip, &ftrace_stub);
946 947
}

948 949 950 951 952 953
#endif /* !CONFIG_DYNAMIC_FTRACE */

/*
 * Hook the return address and push it in the stack of return addrs
 * in current thread info.
 */
954 955
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
			   unsigned long frame_pointer)
956 957 958
{
	unsigned long old;
	int faulted;
959
	struct ftrace_graph_ent trace;
960 961 962
	unsigned long return_hooker = (unsigned long)
				&return_to_handler;

963 964 965
	if (unlikely(ftrace_graph_is_dead()))
		return;

966
	if (unlikely(atomic_read(&current->tracing_graph_pause)))
967 968 969 970 971 972 973 974
		return;

	/*
	 * Protect against fault, even if it shouldn't
	 * happen. This tool is too much intrusive to
	 * ignore such a protection.
	 */
	asm volatile(
975 976
		"1: " _ASM_MOV " (%[parent]), %[old]\n"
		"2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
977
		"   movl $0, %[faulted]\n"
978
		"3:\n"
979 980

		".section .fixup, \"ax\"\n"
981 982
		"4: movl $1, %[faulted]\n"
		"   jmp 3b\n"
983 984
		".previous\n"

985 986
		_ASM_EXTABLE(1b, 4b)
		_ASM_EXTABLE(2b, 4b)
987

988
		: [old] "=&r" (old), [faulted] "=r" (faulted)
989
		: [parent] "r" (parent), [return_hooker] "r" (return_hooker)
990 991 992
		: "memory"
	);

S
Steven Rostedt 已提交
993 994 995
	if (unlikely(faulted)) {
		ftrace_graph_stop();
		WARN_ON(1);
996 997 998
		return;
	}

999
	trace.func = self_addr;
1000
	trace.depth = current->curr_ret_stack + 1;
1001

1002 1003 1004
	/* Only trace if the calling function expects to */
	if (!ftrace_graph_entry(&trace)) {
		*parent = old;
1005 1006 1007 1008 1009 1010 1011
		return;
	}

	if (ftrace_push_return_trace(old, self_addr, &trace.depth,
		    frame_pointer) == -EBUSY) {
		*parent = old;
		return;
1012
	}
1013
}
1014
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */