ftrace.c 18.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * Code for replacing ftrace calls with jumps.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 *
 * Thanks goes to Ingo Molnar, for suggesting the idea.
 * Mathieu Desnoyers, for suggesting postponing the modifications.
 * Arjan van de Ven, for keeping me straight, and explaining to me
 * the dangers of modifying code on the run.
 */

12 13
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

14 15
#include <linux/spinlock.h>
#include <linux/hardirq.h>
16
#include <linux/uaccess.h>
17 18
#include <linux/ftrace.h>
#include <linux/percpu.h>
19
#include <linux/sched.h>
20 21
#include <linux/init.h>
#include <linux/list.h>
22
#include <linux/module.h>
23

24 25
#include <trace/syscall.h>

26
#include <asm/cacheflush.h>
27
#include <asm/kprobes.h>
28
#include <asm/ftrace.h>
29
#include <asm/nops.h>
30

31
#ifdef CONFIG_DYNAMIC_FTRACE
32

33 34 35
int ftrace_arch_code_modify_prepare(void)
{
	set_kernel_text_rw();
36
	set_all_modules_text_rw();
37 38 39 40 41
	return 0;
}

int ftrace_arch_code_modify_post_process(void)
{
42
	set_all_modules_text_ro();
43 44 45 46
	set_kernel_text_ro();
	return 0;
}

47
union ftrace_code_union {
48
	char code[MCOUNT_INSN_SIZE];
49 50 51 52 53 54
	struct {
		char e8;
		int offset;
	} __attribute__((packed));
};

55
static int ftrace_calc_offset(long ip, long addr)
56 57 58
{
	return (int)(addr - ip);
}
59

60
static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
61 62
{
	static union ftrace_code_union calc;
63

64
	calc.e8		= 0xe8;
65
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
66 67 68 69 70 71

	/*
	 * No locking needed, this must be called via kstop_machine
	 * which in essence is like running on a uniprocessor machine.
	 */
	return calc.code;
72 73
}

74 75 76 77 78 79
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
	return addr >= start && addr < end;
}

80
static unsigned long text_ip_addr(unsigned long ip)
81
{
82 83 84 85 86 87 88 89 90
	/*
	 * On x86_64, kernel text mappings are mapped read-only with
	 * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
	 * of the kernel text mapping to modify the kernel text.
	 *
	 * For 32bit kernels, these mappings are same and we can use
	 * kernel identity mapping to modify code.
	 */
	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
91
		ip = (unsigned long)__va(__pa_symbol(ip));
92

93
	return ip;
94 95
}

96
static const unsigned char *ftrace_nop_replace(void)
97
{
98
	return ideal_nops[NOP_ATOMIC5];
99 100
}

101
static int
102
ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
103
		   unsigned const char *new_code)
104
{
105
	unsigned char replaced[MCOUNT_INSN_SIZE];
106 107 108 109

	/*
	 * Note: Due to modules and __init, code can
	 *  disappear and change, we need to protect against faulting
S
Steven Rostedt 已提交
110
	 *  as well as code changing. We do this by using the
S
Steven Rostedt 已提交
111
	 *  probe_kernel_* functions.
112 113
	 *
	 * No real locking needed, this code is run through
114
	 * kstop_machine, or before SMP starts.
115
	 */
S
Steven Rostedt 已提交
116 117

	/* read the text we want to modify */
S
Steven Rostedt 已提交
118
	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
119
		return -EFAULT;
120

S
Steven Rostedt 已提交
121
	/* Make sure it is what we expect it to be */
122
	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
123
		return -EINVAL;
124

125 126
	ip = text_ip_addr(ip);

S
Steven Rostedt 已提交
127
	/* replace the text with the new text */
128
	if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
129
		return -EPERM;
130 131

	sync_core();
132

133
	return 0;
134 135
}

136 137 138
int ftrace_make_nop(struct module *mod,
		    struct dyn_ftrace *rec, unsigned long addr)
{
139
	unsigned const char *new, *old;
140 141 142 143 144
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);
	new = ftrace_nop_replace();

145 146 147 148 149 150 151 152 153 154 155 156 157 158
	/*
	 * On boot up, and when modules are loaded, the MCOUNT_ADDR
	 * is converted to a nop, and will never become MCOUNT_ADDR
	 * again. This code is either running before SMP (on boot up)
	 * or before the code will ever be executed (module load).
	 * We do not want to use the breakpoint version in this case,
	 * just modify the code directly.
	 */
	if (addr == MCOUNT_ADDR)
		return ftrace_modify_code_direct(rec->ip, old, new);

	/* Normal cases use add_brk_on_nop */
	WARN_ONCE(1, "invalid use of ftrace_make_nop");
	return -EINVAL;
159 160 161 162
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
163
	unsigned const char *new, *old;
164 165 166 167 168
	unsigned long ip = rec->ip;

	old = ftrace_nop_replace();
	new = ftrace_call_replace(ip, addr);

169 170
	/* Should only be called when module is loaded */
	return ftrace_modify_code_direct(rec->ip, old, new);
171 172
}

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204
/*
 * The modifying_ftrace_code is used to tell the breakpoint
 * handler to call ftrace_int3_handler(). If it fails to
 * call this handler for a breakpoint added by ftrace, then
 * the kernel may crash.
 *
 * As atomic_writes on x86 do not need a barrier, we do not
 * need to add smp_mb()s for this to work. It is also considered
 * that we can not read the modifying_ftrace_code before
 * executing the breakpoint. That would be quite remarkable if
 * it could do that. Here's the flow that is required:
 *
 *   CPU-0                          CPU-1
 *
 * atomic_inc(mfc);
 * write int3s
 *				<trap-int3> // implicit (r)mb
 *				if (atomic_read(mfc))
 *					call ftrace_int3_handler()
 *
 * Then when we are finished:
 *
 * atomic_dec(mfc);
 *
 * If we hit a breakpoint that was not set by ftrace, it does not
 * matter if ftrace_int3_handler() is called or not. It will
 * simply be ignored. But it is crucial that a ftrace nop/caller
 * breakpoint is handled. No other user should ever place a
 * breakpoint on an ftrace nop/caller location. It must only
 * be done by this code.
 */
atomic_t modifying_ftrace_code __read_mostly;
205

206 207 208 209
static int
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
		   unsigned const char *new_code);

210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
/*
 * Should never be called:
 *  As it is only called by __ftrace_replace_code() which is called by
 *  ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
 *  which is called to turn mcount into nops or nops into function calls
 *  but not to convert a function from not using regs to one that uses
 *  regs, which ftrace_modify_call() is for.
 */
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
				 unsigned long addr)
{
	WARN_ON(1);
	return -EINVAL;
}

225 226 227
static unsigned long ftrace_update_func;

static int update_ftrace_func(unsigned long ip, void *new)
228
{
229
	unsigned char old[MCOUNT_INSN_SIZE];
230 231
	int ret;

232 233 234 235 236
	memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);

	ftrace_update_func = ip;
	/* Make sure the breakpoints see the ftrace_update_func update */
	smp_wmb();
237 238 239 240 241 242

	/* See comment above by declaration of modifying_ftrace_code */
	atomic_inc(&modifying_ftrace_code);

	ret = ftrace_modify_code(ip, old, new);

243 244 245 246 247 248 249 250 251 252 253 254 255 256
	atomic_dec(&modifying_ftrace_code);

	return ret;
}

int ftrace_update_ftrace_func(ftrace_func_t func)
{
	unsigned long ip = (unsigned long)(&ftrace_call);
	unsigned char *new;
	int ret;

	new = ftrace_call_replace(ip, (unsigned long)func);
	ret = update_ftrace_func(ip, new);

257 258 259 260
	/* Also update the regs callback function */
	if (!ret) {
		ip = (unsigned long)(&ftrace_regs_call);
		new = ftrace_call_replace(ip, (unsigned long)func);
261
		ret = update_ftrace_func(ip, new);
262 263
	}

264 265 266
	return ret;
}

267 268
static int is_ftrace_caller(unsigned long ip)
{
269
	if (ip == ftrace_update_func)
270 271 272 273 274
		return 1;

	return 0;
}

275 276 277 278 279 280 281 282 283
/*
 * A breakpoint was added to the code address we are about to
 * modify, and this is the handle that will just skip over it.
 * We are either changing a nop into a trace call, or a trace
 * call to a nop. While the change is taking place, we treat
 * it just like it was a nop.
 */
int ftrace_int3_handler(struct pt_regs *regs)
{
284 285
	unsigned long ip;

286 287 288
	if (WARN_ON_ONCE(!regs))
		return 0;

289 290
	ip = regs->ip - 1;
	if (!ftrace_location(ip) && !is_ftrace_caller(ip))
291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308
		return 0;

	regs->ip += MCOUNT_INSN_SIZE - 1;

	return 1;
}

static int ftrace_write(unsigned long ip, const char *val, int size)
{
	/*
	 * On x86_64, kernel text mappings are mapped read-only with
	 * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
	 * of the kernel text mapping to modify the kernel text.
	 *
	 * For 32bit kernels, these mappings are same and we can use
	 * kernel identity mapping to modify code.
	 */
	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
309
		ip = (unsigned long)__va(__pa_symbol(ip));
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351

	return probe_kernel_write((void *)ip, val, size);
}

static int add_break(unsigned long ip, const char *old)
{
	unsigned char replaced[MCOUNT_INSN_SIZE];
	unsigned char brk = BREAKPOINT_INSTRUCTION;

	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	/* Make sure it is what we expect it to be */
	if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
		return -EINVAL;

	if (ftrace_write(ip, &brk, 1))
		return -EPERM;

	return 0;
}

static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned const char *old;
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);

	return add_break(rec->ip, old);
}


static int add_brk_on_nop(struct dyn_ftrace *rec)
{
	unsigned const char *old;

	old = ftrace_nop_replace();

	return add_break(rec->ip, old);
}

352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
/*
 * If the record has the FTRACE_FL_REGS set, that means that it
 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
 * is not not set, then it wants to convert to the normal callback.
 */
static unsigned long get_ftrace_addr(struct dyn_ftrace *rec)
{
	if (rec->flags & FTRACE_FL_REGS)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

/*
 * The FTRACE_FL_REGS_EN is set when the record already points to
 * a function that saves all the regs. Basically the '_EN' version
 * represents the current state of the function.
 */
static unsigned long get_ftrace_old_addr(struct dyn_ftrace *rec)
{
	if (rec->flags & FTRACE_FL_REGS_EN)
		return (unsigned long)FTRACE_REGS_ADDR;
	else
		return (unsigned long)FTRACE_ADDR;
}

378 379 380 381 382 383 384
static int add_breakpoints(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_test_record(rec, enable);

385
	ftrace_addr = get_ftrace_addr(rec);
386 387 388 389 390 391 392 393 394

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return add_brk_on_nop(rec);

395 396 397 398
	case FTRACE_UPDATE_MODIFY_CALL_REGS:
	case FTRACE_UPDATE_MODIFY_CALL:
		ftrace_addr = get_ftrace_old_addr(rec);
		/* fall through */
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return add_brk_on_call(rec, ftrace_addr);
	}
	return 0;
}

/*
 * On error, we need to remove breakpoints. This needs to
 * be done caefully. If the address does not currently have a
 * breakpoint, we know we are done. Otherwise, we look at the
 * remaining 4 bytes of the instruction. If it matches a nop
 * we replace the breakpoint with the nop. Otherwise we replace
 * it with the call instruction.
 */
static int remove_breakpoint(struct dyn_ftrace *rec)
{
	unsigned char ins[MCOUNT_INSN_SIZE];
	unsigned char brk = BREAKPOINT_INSTRUCTION;
	const unsigned char *nop;
	unsigned long ftrace_addr;
	unsigned long ip = rec->ip;

	/* If we fail the read, just give up */
	if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	/* If this does not have a breakpoint, we are done */
	if (ins[0] != brk)
		return -1;

	nop = ftrace_nop_replace();

	/*
	 * If the last 4 bytes of the instruction do not match
	 * a nop, then we assume that this is a call to ftrace_addr.
	 */
	if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
		/*
		 * For extra paranoidism, we check if the breakpoint is on
		 * a call that would actually jump to the ftrace_addr.
		 * If not, don't touch the breakpoint, we make just create
		 * a disaster.
		 */
443 444 445 446 447 448 449 450
		ftrace_addr = get_ftrace_addr(rec);
		nop = ftrace_call_replace(ip, ftrace_addr);

		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
			goto update;

		/* Check both ftrace_addr and ftrace_old_addr */
		ftrace_addr = get_ftrace_old_addr(rec);
451 452 453 454 455 456
		nop = ftrace_call_replace(ip, ftrace_addr);

		if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
			return -EINVAL;
	}

457
 update:
458
	return ftrace_write(ip, nop, 1);
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
}

static int add_update_code(unsigned long ip, unsigned const char *new)
{
	/* skip breakpoint */
	ip++;
	new++;
	if (ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1))
		return -EPERM;
	return 0;
}

static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_call_replace(ip, addr);
	return add_update_code(ip, new);
}

static int add_update_nop(struct dyn_ftrace *rec)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_nop_replace();
	return add_update_code(ip, new);
}

static int add_update(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_test_record(rec, enable);

496
	ftrace_addr  = get_ftrace_addr(rec);
497 498 499 500 501

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

502 503
	case FTRACE_UPDATE_MODIFY_CALL_REGS:
	case FTRACE_UPDATE_MODIFY_CALL:
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return add_update_call(rec, ftrace_addr);

	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return add_update_nop(rec);
	}

	return 0;
}

static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_call_replace(ip, addr);

	if (ftrace_write(ip, new, 1))
		return -EPERM;

	return 0;
}

static int finish_update_nop(struct dyn_ftrace *rec)
{
	unsigned long ip = rec->ip;
	unsigned const char *new;

	new = ftrace_nop_replace();

	if (ftrace_write(ip, new, 1))
		return -EPERM;
	return 0;
}

static int finish_update(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr;
	int ret;

	ret = ftrace_update_record(rec, enable);

548
	ftrace_addr = get_ftrace_addr(rec);
549 550 551 552 553

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;

554 555
	case FTRACE_UPDATE_MODIFY_CALL_REGS:
	case FTRACE_UPDATE_MODIFY_CALL:
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
	case FTRACE_UPDATE_MAKE_CALL:
		/* converting nop to call */
		return finish_update_call(rec, ftrace_addr);

	case FTRACE_UPDATE_MAKE_NOP:
		/* converting a call to a nop */
		return finish_update_nop(rec);
	}

	return 0;
}

static void do_sync_core(void *data)
{
	sync_core();
}

static void run_sync(void)
{
	int enable_irqs = irqs_disabled();

	/* We may be called with interrupts disbled (on bootup). */
	if (enable_irqs)
		local_irq_enable();
	on_each_cpu(do_sync_core, NULL, 1);
	if (enable_irqs)
		local_irq_disable();
}

585
void ftrace_replace_code(int enable)
586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
{
	struct ftrace_rec_iter *iter;
	struct dyn_ftrace *rec;
	const char *report = "adding breakpoints";
	int count = 0;
	int ret;

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = add_breakpoints(rec, enable);
		if (ret)
			goto remove_breakpoints;
		count++;
	}

	run_sync();

	report = "updating code";

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = add_update(rec, enable);
		if (ret)
			goto remove_breakpoints;
	}

	run_sync();

	report = "removing breakpoints";

	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);

		ret = finish_update(rec, enable);
		if (ret)
			goto remove_breakpoints;
	}

	run_sync();

	return;

 remove_breakpoints:
	ftrace_bug(ret, rec ? rec->ip : 0);
	printk(KERN_WARNING "Failed on %s (%d):\n", report, count);
	for_ftrace_rec_iter(iter) {
		rec = ftrace_rec_iter_record(iter);
		remove_breakpoint(rec);
	}
637
	run_sync();
638 639
}

640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
static int
ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
		   unsigned const char *new_code)
{
	int ret;

	ret = add_break(ip, old_code);
	if (ret)
		goto out;

	run_sync();

	ret = add_update_code(ip, new_code);
	if (ret)
		goto fail_update;

	run_sync();

	ret = ftrace_write(ip, new_code, 1);
	if (ret) {
		ret = -EPERM;
		goto out;
	}
	run_sync();
 out:
	return ret;

 fail_update:
668
	ftrace_write(ip, old_code, 1);
669 670 671
	goto out;
}

672 673
void arch_ftrace_update_code(int command)
{
674 675
	/* See comment above by declaration of modifying_ftrace_code */
	atomic_inc(&modifying_ftrace_code);
676

677
	ftrace_modify_all_code(command);
678

679
	atomic_dec(&modifying_ftrace_code);
680 681
}

682
int __init ftrace_dyn_arch_init(void *data)
683
{
684 685
	/* The return code is retured via data */
	*(unsigned long *)data = 0;
S
Steven Rostedt 已提交
686

687 688
	return 0;
}
689
#endif
690

691
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
692

693 694 695
#ifdef CONFIG_DYNAMIC_FTRACE
extern void ftrace_graph_call(void);

696
static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
697
{
698
	static union ftrace_code_union calc;
699

700 701 702
	/* Jmp not a call (ignore the .e8) */
	calc.e8		= 0xe9;
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
703

704 705 706 707 708
	/*
	 * ftrace external locks synchronize the access to the static variable.
	 */
	return calc.code;
}
709

710 711 712
static int ftrace_mod_jmp(unsigned long ip, void *func)
{
	unsigned char *new;
713

714
	new = ftrace_jmp_replace(ip, (unsigned long)func);
715

716
	return update_ftrace_func(ip, new);
717 718 719 720 721 722
}

int ftrace_enable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);

723
	return ftrace_mod_jmp(ip, &ftrace_graph_caller);
724 725 726 727 728 729
}

int ftrace_disable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);

730
	return ftrace_mod_jmp(ip, &ftrace_stub);
731 732
}

733 734 735 736 737 738
#endif /* !CONFIG_DYNAMIC_FTRACE */

/*
 * Hook the return address and push it in the stack of return addrs
 * in current thread info.
 */
739 740
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
			   unsigned long frame_pointer)
741 742 743
{
	unsigned long old;
	int faulted;
744
	struct ftrace_graph_ent trace;
745 746 747
	unsigned long return_hooker = (unsigned long)
				&return_to_handler;

748
	if (unlikely(atomic_read(&current->tracing_graph_pause)))
749 750 751 752 753 754 755 756
		return;

	/*
	 * Protect against fault, even if it shouldn't
	 * happen. This tool is too much intrusive to
	 * ignore such a protection.
	 */
	asm volatile(
757 758
		"1: " _ASM_MOV " (%[parent]), %[old]\n"
		"2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
759
		"   movl $0, %[faulted]\n"
760
		"3:\n"
761 762

		".section .fixup, \"ax\"\n"
763 764
		"4: movl $1, %[faulted]\n"
		"   jmp 3b\n"
765 766
		".previous\n"

767 768
		_ASM_EXTABLE(1b, 4b)
		_ASM_EXTABLE(2b, 4b)
769

770
		: [old] "=&r" (old), [faulted] "=r" (faulted)
771
		: [parent] "r" (parent), [return_hooker] "r" (return_hooker)
772 773 774
		: "memory"
	);

S
Steven Rostedt 已提交
775 776 777
	if (unlikely(faulted)) {
		ftrace_graph_stop();
		WARN_ON(1);
778 779 780
		return;
	}

781
	trace.func = self_addr;
782
	trace.depth = current->curr_ret_stack + 1;
783

784 785 786
	/* Only trace if the calling function expects to */
	if (!ftrace_graph_entry(&trace)) {
		*parent = old;
787 788 789 790 791 792 793
		return;
	}

	if (ftrace_push_return_trace(old, self_addr, &trace.depth,
		    frame_pointer) == -EBUSY) {
		*parent = old;
		return;
794
	}
795
}
796
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */