ftrace.c 25.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2
/*
3
 * Dynamic function tracing support.
4 5 6 7 8 9 10 11 12
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 *
 * Thanks goes to Ingo Molnar, for suggesting the idea.
 * Mathieu Desnoyers, for suggesting postponing the modifications.
 * Arjan van de Ven, for keeping me straight, and explaining to me
 * the dangers of modifying code on the run.
 */

13 14
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

15 16
#include <linux/spinlock.h>
#include <linux/hardirq.h>
17
#include <linux/uaccess.h>
18 19
#include <linux/ftrace.h>
#include <linux/percpu.h>
20
#include <linux/sched.h>
21
#include <linux/slab.h>
22 23
#include <linux/init.h>
#include <linux/list.h>
24
#include <linux/module.h>
25

26 27
#include <trace/syscall.h>

L
Laura Abbott 已提交
28
#include <asm/set_memory.h>
29
#include <asm/kprobes.h>
30
#include <asm/ftrace.h>
31
#include <asm/nops.h>
32
#include <asm/text-patching.h>
33

34
#ifdef CONFIG_DYNAMIC_FTRACE
35

36 37 38
int ftrace_arch_code_modify_prepare(void)
{
	set_kernel_text_rw();
39
	set_all_modules_text_rw();
40 41 42 43 44
	return 0;
}

int ftrace_arch_code_modify_post_process(void)
{
45
	set_all_modules_text_ro();
46 47 48 49
	set_kernel_text_ro();
	return 0;
}

50
union ftrace_code_union {
51
	char code[MCOUNT_INSN_SIZE];
52
	struct {
53
		unsigned char e8;
54 55 56 57
		int offset;
	} __attribute__((packed));
};

58
static int ftrace_calc_offset(long ip, long addr)
59 60 61
{
	return (int)(addr - ip);
}
62

63
static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
64 65
{
	static union ftrace_code_union calc;
66

67
	calc.e8		= 0xe8;
68
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
69 70 71 72 73 74

	/*
	 * No locking needed, this must be called via kstop_machine
	 * which in essence is like running on a uniprocessor machine.
	 */
	return calc.code;
75 76
}

77 78 79 80 81 82
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
	return addr >= start && addr < end;
}

83
static unsigned long text_ip_addr(unsigned long ip)
84
{
85
	/*
86 87 88
	 * On x86_64, kernel text mappings are mapped read-only, so we use
	 * the kernel identity mapping instead of the kernel text mapping
	 * to modify the kernel text.
89 90 91 92 93
	 *
	 * For 32bit kernels, these mappings are same and we can use
	 * kernel identity mapping to modify code.
	 */
	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
94
		ip = (unsigned long)__va(__pa_symbol(ip));
95

96
	return ip;
97 98
}

99
static const unsigned char *ftrace_nop_replace(void)
100
{
101
	return ideal_nops[NOP_ATOMIC5];
102 103
}

104
static int
105
ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
106
		   unsigned const char *new_code)
107
{
108
	unsigned char replaced[MCOUNT_INSN_SIZE];
109

110 111
	ftrace_expected = old_code;

112
	/*
113 114 115 116 117
	 * Note:
	 * We are paranoid about modifying text, as if a bug was to happen, it
	 * could cause us to read or write to someplace that could cause harm.
	 * Carefully read and modify the code with probe_kernel_*(), and make
	 * sure what we read is what we expected it to be before modifying it.
118
	 */
S
Steven Rostedt 已提交
119 120

	/* read the text we want to modify */
S
Steven Rostedt 已提交
121
	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
122
		return -EFAULT;
123

S
Steven Rostedt 已提交
124
	/* Make sure it is what we expect it to be */
125
	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
126
		return -EINVAL;
127

128 129
	ip = text_ip_addr(ip);

S
Steven Rostedt 已提交
130
	/* replace the text with the new text */
131
	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
132
		return -EPERM;
133 134

	sync_core();
135

136
	return 0;
137 138
}

139 140 141
int ftrace_make_nop(struct module *mod,
		    struct dyn_ftrace *rec, unsigned long addr)
{
142
	unsigned const char *new, *old;
143 144 145 146 147
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);
	new = ftrace_nop_replace();

148 149 150 151 152 153 154 155 156 157 158
	/*
	 * On boot up, and when modules are loaded, the MCOUNT_ADDR
	 * is converted to a nop, and will never become MCOUNT_ADDR
	 * again. This code is either running before SMP (on boot up)
	 * or before the code will ever be executed (module load).
	 * We do not want to use the breakpoint version in this case,
	 * just modify the code directly.
	 */
	if (addr == MCOUNT_ADDR)
		return ftrace_modify_code_direct(rec->ip, old, new);

159 160
	ftrace_expected = NULL;

161 162 163
	/* Normal cases use add_brk_on_nop */
	WARN_ONCE(1, "invalid use of ftrace_make_nop");
	return -EINVAL;
164 165 166 167
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
168
	unsigned const char *new, *old;
169 170 171 172 173
	unsigned long ip = rec->ip;

	old = ftrace_nop_replace();
	new = ftrace_call_replace(ip, addr);

174 175
	/* Should only be called when module is loaded */
	return ftrace_modify_code_direct(rec->ip, old, new);
176 177
}

178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
/*
 * The modifying_ftrace_code is used to tell the breakpoint
 * handler to call ftrace_int3_handler(). If it fails to
 * call this handler for a breakpoint added by ftrace, then
 * the kernel may crash.
 *
 * As atomic_writes on x86 do not need a barrier, we do not
 * need to add smp_mb()s for this to work. It is also considered
 * that we can not read the modifying_ftrace_code before
 * executing the breakpoint. That would be quite remarkable if
 * it could do that. Here's the flow that is required:
 *
 *   CPU-0                          CPU-1
 *
 * atomic_inc(mfc);
 * write int3s
 *				<trap-int3> // implicit (r)mb
 *				if (atomic_read(mfc))
 *					call ftrace_int3_handler()
 *
 * Then when we are finished:
 *
 * atomic_dec(mfc);
 *
 * If we hit a breakpoint that was not set by ftrace, it does not
 * matter if ftrace_int3_handler() is called or not. It will
 * simply be ignored. But it is crucial that a ftrace nop/caller
 * breakpoint is handled. No other user should ever place a
 * breakpoint on an ftrace nop/caller location. It must only
 * be done by this code.
 */
atomic_t modifying_ftrace_code __read_mostly;
210

211 212 213 214
static int
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
		   unsigned const char *new_code);

215 216 217 218 219 220 221 222 223 224 225 226
/*
 * Should never be called:
 *  As it is only called by __ftrace_replace_code() which is called by
 *  ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
 *  which is called to turn mcount into nops or nops into function calls
 *  but not to convert a function from not using regs to one that uses
 *  regs, which ftrace_modify_call() is for.
 */
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
				 unsigned long addr)
{
	WARN_ON(1);
227
	ftrace_expected = NULL;
228 229 230
	return -EINVAL;
}

231
static unsigned long ftrace_update_func;
232
static unsigned long ftrace_update_func_call;
233 234

static int update_ftrace_func(unsigned long ip, void *new)
235
{
236
	unsigned char old[MCOUNT_INSN_SIZE];
237 238
	int ret;

239 240 241 242 243
	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);

	ftrace_update_func = ip;
	/* Make sure the breakpoints see the ftrace_update_func update */
	smp_wmb();
244 245 246 247 248 249

	/* See comment above by declaration of modifying_ftrace_code */
	atomic_inc(&modifying_ftrace_code);

	ret = ftrace_modify_code(ip, old, new);

250 251 252 253 254 255 256 257 258 259 260
	atomic_dec(&modifying_ftrace_code);

	return ret;
}

int ftrace_update_ftrace_func(ftrace_func_t func)
{
	unsigned long ip = (unsigned long)(&ftrace_call);
	unsigned char *new;
	int ret;

261 262
	ftrace_update_func_call = (unsigned long)func;

263 264 265
	new = ftrace_call_replace(ip, (unsigned long)func);
	ret = update_ftrace_func(ip, new);

266 267 268 269
	/* Also update the regs callback function */
	if (!ret) {
		ip = (unsigned long)(&ftrace_regs_call);
		new = ftrace_call_replace(ip, (unsigned long)func);
270
		ret = update_ftrace_func(ip, new);
271 272
	}

273 274 275
	return ret;
}

276 277
static int is_ftrace_caller(unsigned long ip)
{
278
	if (ip == ftrace_update_func)
279 280 281 282 283
		return 1;

	return 0;
}

284 285 286 287 288 289 290 291 292
/*
 * A breakpoint was added to the code address we are about to
 * modify, and this is the handle that will just skip over it.
 * We are either changing a nop into a trace call, or a trace
 * call to a nop. While the change is taking place, we treat
 * it just like it was a nop.
 */
int ftrace_int3_handler(struct pt_regs *regs)
{
293 294
	unsigned long ip;

295 296 297
	if (WARN_ON_ONCE(!regs))
		return 0;

298
	ip = regs->ip - INT3_INSN_SIZE;
299

300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
#ifdef CONFIG_X86_64
	if (ftrace_location(ip)) {
		int3_emulate_call(regs, (unsigned long)ftrace_regs_caller);
		return 1;
	} else if (is_ftrace_caller(ip)) {
		if (!ftrace_update_func_call) {
			int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
			return 1;
		}
		int3_emulate_call(regs, ftrace_update_func_call);
		return 1;
	}
#else
	if (ftrace_location(ip) || is_ftrace_caller(ip)) {
		int3_emulate_jmp(regs, ip + CALL_INSN_SIZE);
		return 1;
	}
#endif
318

319
	return 0;
320 321 322 323
}

static int ftrace_write(unsigned long ip, const char *val, int size)
{
324
	ip = text_ip_addr(ip);
325

326 327 328 329
	if (probe_kernel_write((void *)ip, val, size))
		return -EPERM;

	return 0;
330 331 332 333 334 335 336 337 338 339
}

static int add_break(unsigned long ip, const char *old)
{
	unsigned char replaced[MCOUNT_INSN_SIZE];
	unsigned char brk = BREAKPOINT_INSTRUCTION;

	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

340 341
	ftrace_expected = old;

342 343 344 345
	/* Make sure it is what we expect it to be */
	if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
		return -EINVAL;

346
	return ftrace_write(ip, &brk, 1);
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
}

static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned const char *old;
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);

	return add_break(rec->ip, old);
}


static int add_brk_on_nop(struct dyn_ftrace *rec)
{
	unsigned const char *old;

	old = ftrace_nop_replace();

	return add_break(rec->ip, old);
}

static int add_breakpoints(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

374
	ftrace_addr = ftrace_get_addr_curr(rec);
375

376
	ret = ftrace_test_record(rec, enable);
377 378 379 380 381 382 383 384 385

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return add_brk_on_nop(rec);

386
	case FTRACE_UPDATE_MODIFY_CALL:
387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return add_brk_on_call(rec, ftrace_addr);
	}
	return 0;
}

/*
 * On error, we need to remove breakpoints. This needs to
 * be done caefully. If the address does not currently have a
 * breakpoint, we know we are done. Otherwise, we look at the
 * remaining 4 bytes of the instruction. If it matches a nop
 * we replace the breakpoint with the nop. Otherwise we replace
 * it with the call instruction.
 */
static int remove_breakpoint(struct dyn_ftrace *rec)
{
	unsigned char ins[MCOUNT_INSN_SIZE];
	unsigned char brk = BREAKPOINT_INSTRUCTION;
	const unsigned char *nop;
	unsigned long ftrace_addr;
	unsigned long ip = rec->ip;

	/* If we fail the read, just give up */
	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	/* If this does not have a breakpoint, we are done */
	if (ins[0] != brk)
416
		return 0;
417 418 419 420 421 422 423 424 425 426 427 428 429 430

	nop = ftrace_nop_replace();

	/*
	 * If the last 4 bytes of the instruction do not match
	 * a nop, then we assume that this is a call to ftrace_addr.
	 */
	if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
		/*
		 * For extra paranoidism, we check if the breakpoint is on
		 * a call that would actually jump to the ftrace_addr.
		 * If not, don't touch the breakpoint, we make just create
		 * a disaster.
		 */
431
		ftrace_addr = ftrace_get_addr_new(rec);
432 433 434 435 436 437
		nop = ftrace_call_replace(ip, ftrace_addr);

		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
			goto update;

		/* Check both ftrace_addr and ftrace_old_addr */
438
		ftrace_addr = ftrace_get_addr_curr(rec);
439 440
		nop = ftrace_call_replace(ip, ftrace_addr);

441 442
		ftrace_expected = nop;

443 444 445 446
		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
			return -EINVAL;
	}

447
 update:
448
	return ftrace_write(ip, nop, 1);
449 450 451 452 453 454 455
}

static int add_update_code(unsigned long ip, unsigned const char *new)
{
	/* skip breakpoint */
	ip++;
	new++;
456
	return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
}

static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_call_replace(ip, addr);
	return add_update_code(ip, new);
}

static int add_update_nop(struct dyn_ftrace *rec)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_nop_replace();
	return add_update_code(ip, new);
}

static int add_update(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_test_record(rec, enable);

484
	ftrace_addr  = ftrace_get_addr_new(rec);
485 486 487 488 489

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

490
	case FTRACE_UPDATE_MODIFY_CALL:
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return add_update_call(rec, ftrace_addr);

	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return add_update_nop(rec);
	}

	return 0;
}

static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_call_replace(ip, addr);

510
	return ftrace_write(ip, new, 1);
511 512 513 514 515 516 517 518 519
}

static int finish_update_nop(struct dyn_ftrace *rec)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_nop_replace();

520
	return ftrace_write(ip, new, 1);
521 522 523 524 525 526 527 528 529
}

static int finish_update(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_update_record(rec, enable);

530
	ftrace_addr = ftrace_get_addr_new(rec);
531 532 533 534 535

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

536
	case FTRACE_UPDATE_MODIFY_CALL:
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return finish_update_call(rec, ftrace_addr);

	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return finish_update_nop(rec);
	}

	return 0;
}

static void do_sync_core(void *data)
{
	sync_core();
}

static void run_sync(void)
{
556 557 558 559 560 561 562
	int enable_irqs;

	/* No need to sync if there's only one CPU */
	if (num_online_cpus() == 1)
		return;

	enable_irqs = irqs_disabled();
563

564
	/* We may be called with interrupts disabled (on bootup). */
565 566 567 568 569 570 571
	if (enable_irqs)
		local_irq_enable();
	on_each_cpu(do_sync_core, NULL, 1);
	if (enable_irqs)
		local_irq_disable();
}

572
void ftrace_replace_code(int enable)
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
{
	struct ftrace_rec_iter *iter;
	struct dyn_ftrace *rec;
	const char *report = "adding breakpoints";
	int count = 0;
	int ret;

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = add_breakpoints(rec, enable);
		if (ret)
			goto remove_breakpoints;
		count++;
	}

	run_sync();

	report = "updating code";
592
	count = 0;
593 594 595 596 597 598 599

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = add_update(rec, enable);
		if (ret)
			goto remove_breakpoints;
600
		count++;
601 602 603 604 605
	}

	run_sync();

	report = "removing breakpoints";
606
	count = 0;
607 608 609 610 611 612 613

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = finish_update(rec, enable);
		if (ret)
			goto remove_breakpoints;
614
		count++;
615 616 617 618 619 620 621
	}

	run_sync();

	return;

 remove_breakpoints:
622
	pr_warn("Failed on %s (%d):\n", report, count);
623
	ftrace_bug(ret, rec);
624 625
	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);
626 627 628 629 630 631
		/*
		 * Breakpoints are handled only when this function is in
		 * progress. The system could not work with them.
		 */
		if (remove_breakpoint(rec))
			BUG();
632
	}
633
	run_sync();
634 635
}

636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
static int
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
		   unsigned const char *new_code)
{
	int ret;

	ret = add_break(ip, old_code);
	if (ret)
		goto out;

	run_sync();

	ret = add_update_code(ip, new_code);
	if (ret)
		goto fail_update;

	run_sync();

	ret = ftrace_write(ip, new_code, 1);
655 656 657 658 659
	/*
	 * The breakpoint is handled only when this function is in progress.
	 * The system could not work if we could not remove it.
	 */
	BUG_ON(ret);
660
 out:
661
	run_sync();
662 663 664
	return ret;

 fail_update:
665 666 667
	/* Also here the system could not work with the breakpoint */
	if (ftrace_write(ip, old_code, 1))
		BUG();
668 669 670
	goto out;
}

671 672
void arch_ftrace_update_code(int command)
{
673 674
	/* See comment above by declaration of modifying_ftrace_code */
	atomic_inc(&modifying_ftrace_code);
675

676
	ftrace_modify_all_code(command);
677

678
	atomic_dec(&modifying_ftrace_code);
679 680
}

681
int __init ftrace_dyn_arch_init(void)
682 683 684
{
	return 0;
}
685

686
#if defined(CONFIG_X86_64) || defined(CONFIG_FUNCTION_GRAPH_TRACER)
687
static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
688
{
689
	static union ftrace_code_union calc;
690

691 692 693
	/* Jmp not a call (ignore the .e8) */
	calc.e8		= 0xe9;
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
694

695 696 697 698 699
	/*
	 * ftrace external locks synchronize the access to the static variable.
	 */
	return calc.code;
}
700 701 702 703 704 705 706 707 708 709 710 711
#endif

/* Currently only x86_64 supports dynamic trampolines */
#ifdef CONFIG_X86_64

#ifdef CONFIG_MODULES
#include <linux/moduleloader.h>
/* Module allocation simplifies allocating memory for code */
static inline void *alloc_tramp(unsigned long size)
{
	return module_alloc(size);
}
712
static inline void tramp_free(void *tramp, int size)
713
{
714 715 716 717
	int npages = PAGE_ALIGN(size) >> PAGE_SHIFT;

	set_memory_nx((unsigned long)tramp, npages);
	set_memory_rw((unsigned long)tramp, npages);
718
	module_memfree(tramp);
719 720 721 722 723 724 725
}
#else
/* Trampolines can only be created if modules are supported */
static inline void *alloc_tramp(unsigned long size)
{
	return NULL;
}
726
static inline void tramp_free(void *tramp, int size) { }
727 728 729 730
#endif

/* Defined as markers to the end of the ftrace default trampolines */
extern void ftrace_regs_caller_end(void);
731
extern void ftrace_epilogue(void);
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754
extern void ftrace_caller_op_ptr(void);
extern void ftrace_regs_caller_op_ptr(void);

/* movq function_trace_op(%rip), %rdx */
/* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
#define OP_REF_SIZE	7

/*
 * The ftrace_ops is passed to the function callback. Since the
 * trampoline only services a single ftrace_ops, we can pass in
 * that ops directly.
 *
 * The ftrace_op_code_union is used to create a pointer to the
 * ftrace_ops that will be passed to the callback function.
 */
union ftrace_op_code_union {
	char code[OP_REF_SIZE];
	struct {
		char op[3];
		int offset;
	} __attribute__((packed));
};

755 756
#define RET_SIZE		1

757 758
static unsigned long
create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
759 760 761 762 763 764
{
	unsigned long start_offset;
	unsigned long end_offset;
	unsigned long op_offset;
	unsigned long offset;
	unsigned long size;
765
	unsigned long retq;
766 767
	unsigned long *ptr;
	void *trampoline;
768
	void *ip;
769 770 771 772 773 774 775 776 777 778 779
	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
	union ftrace_op_code_union op_ptr;
	int ret;

	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
		start_offset = (unsigned long)ftrace_regs_caller;
		end_offset = (unsigned long)ftrace_regs_caller_end;
		op_offset = (unsigned long)ftrace_regs_caller_op_ptr;
	} else {
		start_offset = (unsigned long)ftrace_caller;
780
		end_offset = (unsigned long)ftrace_epilogue;
781 782 783 784 785 786 787
		op_offset = (unsigned long)ftrace_caller_op_ptr;
	}

	size = end_offset - start_offset;

	/*
	 * Allocate enough size to store the ftrace_caller code,
788 789
	 * the iret , as well as the address of the ftrace_ops this
	 * trampoline is used for.
790
	 */
791
	trampoline = alloc_tramp(size + RET_SIZE + sizeof(void *));
792 793 794
	if (!trampoline)
		return 0;

795
	*tramp_size = size + RET_SIZE + sizeof(void *);
796

797 798
	/* Copy ftrace_caller onto the trampoline memory */
	ret = probe_kernel_read(trampoline, (void *)start_offset, size);
799 800
	if (WARN_ON(ret < 0))
		goto fail;
801

802
	ip = trampoline + size;
803

804 805 806 807 808
	/* The trampoline ends with ret(q) */
	retq = (unsigned long)ftrace_stub;
	ret = probe_kernel_read(ip, (void *)retq, RET_SIZE);
	if (WARN_ON(ret < 0))
		goto fail;
809 810 811 812 813 814 815 816 817

	/*
	 * The address of the ftrace_ops that is used for this trampoline
	 * is stored at the end of the trampoline. This will be used to
	 * load the third parameter for the callback. Basically, that
	 * location at the end of the trampoline takes the place of
	 * the global function_trace_op variable.
	 */

818
	ptr = (unsigned long *)(trampoline + size + RET_SIZE);
819 820 821 822 823 824
	*ptr = (unsigned long)ops;

	op_offset -= start_offset;
	memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE);

	/* Are we pointing to the reference? */
825 826
	if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0))
		goto fail;
827 828 829 830 831 832 833 834 835 836 837 838 839 840

	/* Load the contents of ptr into the callback parameter */
	offset = (unsigned long)ptr;
	offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;

	op_ptr.offset = offset;

	/* put in the new offset to the ftrace_ops */
	memcpy(trampoline + op_offset, &op_ptr, OP_REF_SIZE);

	/* ALLOC_TRAMP flags lets us know we created it */
	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;

	return (unsigned long)trampoline;
841 842 843
fail:
	tramp_free(trampoline, *tramp_size);
	return 0;
844 845
}

846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
static unsigned long calc_trampoline_call_offset(bool save_regs)
{
	unsigned long start_offset;
	unsigned long call_offset;

	if (save_regs) {
		start_offset = (unsigned long)ftrace_regs_caller;
		call_offset = (unsigned long)ftrace_regs_call;
	} else {
		start_offset = (unsigned long)ftrace_caller;
		call_offset = (unsigned long)ftrace_call;
	}

	return call_offset - start_offset;
}

862 863 864 865 866 867
void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
	ftrace_func_t func;
	unsigned char *new;
	unsigned long offset;
	unsigned long ip;
868
	unsigned int size;
869
	int ret, npages;
870 871 872 873 874 875 876 877

	if (ops->trampoline) {
		/*
		 * The ftrace_ops caller may set up its own trampoline.
		 * In such a case, this code must not modify it.
		 */
		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
			return;
878 879
		npages = PAGE_ALIGN(ops->trampoline_size) >> PAGE_SHIFT;
		set_memory_rw(ops->trampoline, npages);
880
	} else {
881
		ops->trampoline = create_trampoline(ops, &size);
882 883
		if (!ops->trampoline)
			return;
884
		ops->trampoline_size = size;
885
		npages = PAGE_ALIGN(size) >> PAGE_SHIFT;
886 887
	}

888
	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
889 890 891 892
	ip = ops->trampoline + offset;

	func = ftrace_ops_get_func(ops);

893 894
	ftrace_update_func_call = (unsigned long)func;

895 896 897
	/* Do a safe modify in case the trampoline is executing */
	new = ftrace_call_replace(ip, (unsigned long)func);
	ret = update_ftrace_func(ip, new);
898
	set_memory_ro(ops->trampoline, npages);
899 900 901 902

	/* The update should never fail */
	WARN_ON(ret);
}
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922

/* Return the address of the function the trampoline calls */
static void *addr_from_call(void *ptr)
{
	union ftrace_code_union calc;
	int ret;

	ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
	if (WARN_ON_ONCE(ret < 0))
		return NULL;

	/* Make sure this is a call */
	if (WARN_ON_ONCE(calc.e8 != 0xe8)) {
		pr_warn("Expected e8, got %x\n", calc.e8);
		return NULL;
	}

	return ptr + MCOUNT_INSN_SIZE + calc.offset;
}

923
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969
			   unsigned long frame_pointer);

/*
 * If the ops->trampoline was not allocated, then it probably
 * has a static trampoline func, or is the ftrace caller itself.
 */
static void *static_tramp_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	unsigned long offset;
	bool save_regs = rec->flags & FTRACE_FL_REGS_EN;
	void *ptr;

	if (ops && ops->trampoline) {
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
		/*
		 * We only know about function graph tracer setting as static
		 * trampoline.
		 */
		if (ops->trampoline == FTRACE_GRAPH_ADDR)
			return (void *)prepare_ftrace_return;
#endif
		return NULL;
	}

	offset = calc_trampoline_call_offset(save_regs);

	if (save_regs)
		ptr = (void *)FTRACE_REGS_ADDR + offset;
	else
		ptr = (void *)FTRACE_ADDR + offset;

	return addr_from_call(ptr);
}

void *arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
	unsigned long offset;

	/* If we didn't allocate this trampoline, consider it static */
	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
		return static_tramp_func(ops, rec);

	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);
	return addr_from_call((void *)ops->trampoline + offset);
}

970 971 972 973 974
void arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))
		return;

975
	tramp_free((void *)ops->trampoline, ops->trampoline_size);
976 977
	ops->trampoline = 0;
}
978

979 980 981 982 983 984 985
#endif /* CONFIG_X86_64 */
#endif /* CONFIG_DYNAMIC_FTRACE */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER

#ifdef CONFIG_DYNAMIC_FTRACE
extern void ftrace_graph_call(void);
986

987 988 989
static int ftrace_mod_jmp(unsigned long ip, void *func)
{
	unsigned char *new;
990

991
	ftrace_update_func_call = 0UL;
992
	new = ftrace_jmp_replace(ip, (unsigned long)func);
993

994
	return update_ftrace_func(ip, new);
995 996 997 998 999 1000
}

int ftrace_enable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);

1001
	return ftrace_mod_jmp(ip, &ftrace_graph_caller);
1002 1003 1004 1005 1006 1007
}

int ftrace_disable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);

1008
	return ftrace_mod_jmp(ip, &ftrace_stub);
1009 1010
}

1011 1012 1013 1014 1015 1016
#endif /* !CONFIG_DYNAMIC_FTRACE */

/*
 * Hook the return address and push it in the stack of return addrs
 * in current thread info.
 */
1017
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
1018
			   unsigned long frame_pointer)
1019 1020 1021 1022 1023 1024
{
	unsigned long old;
	int faulted;
	unsigned long return_hooker = (unsigned long)
				&return_to_handler;

1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
	/*
	 * When resuming from suspend-to-ram, this function can be indirectly
	 * called from early CPU startup code while the CPU is in real mode,
	 * which would fail miserably.  Make sure the stack pointer is a
	 * virtual address.
	 *
	 * This check isn't as accurate as virt_addr_valid(), but it should be
	 * good enough for this purpose, and it's fast.
	 */
	if (unlikely((long)__builtin_frame_address(0) >= 0))
		return;

1037 1038 1039
	if (unlikely(ftrace_graph_is_dead()))
		return;

1040
	if (unlikely(atomic_read(&current->tracing_graph_pause)))
1041 1042 1043 1044 1045 1046 1047 1048
		return;

	/*
	 * Protect against fault, even if it shouldn't
	 * happen. This tool is too much intrusive to
	 * ignore such a protection.
	 */
	asm volatile(
1049 1050
		"1: " _ASM_MOV " (%[parent]), %[old]\n"
		"2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
1051
		"   movl $0, %[faulted]\n"
1052
		"3:\n"
1053 1054

		".section .fixup, \"ax\"\n"
1055 1056
		"4: movl $1, %[faulted]\n"
		"   jmp 3b\n"
1057 1058
		".previous\n"

1059 1060
		_ASM_EXTABLE(1b, 4b)
		_ASM_EXTABLE(2b, 4b)
1061

1062
		: [old] "=&r" (old), [faulted] "=r" (faulted)
1063
		: [parent] "r" (parent), [return_hooker] "r" (return_hooker)
1064 1065 1066
		: "memory"
	);

S
Steven Rostedt 已提交
1067 1068 1069
	if (unlikely(faulted)) {
		ftrace_graph_stop();
		WARN_ON(1);
1070 1071 1072
		return;
	}

1073
	if (function_graph_enter(old, self_addr, frame_pointer, parent))
1074
		*parent = old;
1075
}
1076
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */