ftrace.c 14.0 KB
Newer Older
1 2 3 4 5 6 7
/*
 * Code for replacing ftrace calls with jumps.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 *
 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
 *
8 9 10
 * Added function graph tracer code, taken from x86 that was written
 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
 *
11 12
 */

13 14
#define pr_fmt(fmt) "ftrace-powerpc: " fmt

15 16
#include <linux/spinlock.h>
#include <linux/hardirq.h>
17
#include <linux/uaccess.h>
18
#include <linux/module.h>
19 20 21 22 23 24
#include <linux/ftrace.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/list.h>

#include <asm/cacheflush.h>
25
#include <asm/code-patching.h>
26
#include <asm/ftrace.h>
27
#include <asm/syscall.h>
28 29


30
#ifdef CONFIG_DYNAMIC_FTRACE
31
static unsigned int
32
ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
33
{
34
	unsigned int op;
35

36
	addr = ppc_function_entry((void *)addr);
37

38
	/* if (link) set op to 'bl' else 'b' */
39
	op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
40

41
	return op;
42 43
}

44
static int
45
ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
46
{
47
	unsigned int replaced;
48 49 50 51

	/*
	 * Note: Due to modules and __init, code can
	 *  disappear and change, we need to protect against faulting
52 53
	 *  as well as code changing. We do this by using the
	 *  probe_kernel_* functions.
54 55
	 *
	 * No real locking needed, this code is run through
56
	 * kstop_machine, or before SMP starts.
57
	 */
58 59

	/* read the text we want to modify */
60
	if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
61 62 63
		return -EFAULT;

	/* Make sure it is what we expect it to be */
64 65 66
	if (replaced != old) {
		pr_err("%p: replaced (%#x) != old (%#x)",
		(void *)ip, replaced, old);
67
		return -EINVAL;
68
	}
69 70

	/* replace the text with the new text */
71
	if (patch_instruction((unsigned int *)ip, new))
72 73 74
		return -EPERM;

	return 0;
75 76
}

77 78 79
/*
 * Helper functions that are the same for both PPC64 and PPC32.
 */
80 81
static int test_24bit_addr(unsigned long ip, unsigned long addr)
{
82
	addr = ppc_function_entry((void *)addr);
83

84 85
	/* use the create_branch to verify that this offset can be branched */
	return create_branch((unsigned int *)ip, addr, 0);
86 87
}

88 89
#ifdef CONFIG_MODULES

90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
static int is_bl_op(unsigned int op)
{
	return (op & 0xfc000003) == 0x48000001;
}

static unsigned long find_bl_target(unsigned long ip, unsigned int op)
{
	static int offset;

	offset = (op & 0x03fffffc);
	/* make it signed */
	if (offset & 0x02000000)
		offset |= 0xfe000000;

	return ip + (long)offset;
}

#ifdef CONFIG_PPC64
static int
__ftrace_make_nop(struct module *mod,
		  struct dyn_ftrace *rec, unsigned long addr)
{
112
	unsigned long entry, ptr, tramp;
113
	unsigned long ip = rec->ip;
114
	unsigned int op, pop;
115 116

	/* read where this goes */
117 118
	if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
		pr_err("Fetching opcode failed.\n");
119
		return -EFAULT;
120
	}
121 122

	/* Make sure that that this is still a 24bit jump */
123
	if (!is_bl_op(op)) {
124
		pr_err("Not expected bl: opcode is %x\n", op);
125 126 127 128
		return -EINVAL;
	}

	/* lets find where the pointer goes */
129
	tramp = find_bl_target(ip, op);
130

131
	pr_devel("ip:%lx jumps to %lx", ip, tramp);
132

133
	if (module_trampoline_target(mod, tramp, &ptr)) {
134
		pr_err("Failed to get trampoline target\n");
135 136 137
		return -EFAULT;
	}

138
	pr_devel("trampoline target %lx", ptr);
139

140
	entry = ppc_global_function_entry((void *)addr);
141
	/* This should match what was called */
142
	if (ptr != entry) {
143
		pr_err("addr %lx does not match expected %lx\n", ptr, entry);
144 145 146 147
		return -EINVAL;
	}

	/*
148 149 150 151 152 153 154 155 156 157 158
	 * Our original call site looks like:
	 *
	 * bl <tramp>
	 * ld r2,XX(r1)
	 *
	 * Milton Miller pointed out that we can not simply nop the branch.
	 * If a task was preempted when calling a trace function, the nops
	 * will remove the way to restore the TOC in r2 and the r2 TOC will
	 * get corrupted.
	 *
	 * Use a b +8 to jump over the load.
159 160
	 */

161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
	pop = PPC_INST_BRANCH | 8;	/* b +8 */

	/*
	 * Check what is in the next instruction. We can see ld r2,40(r1), but
	 * on first pass after boot we will see mflr r0.
	 */
	if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
		pr_err("Fetching op failed.\n");
		return -EFAULT;
	}

	if (op != PPC_INST_LD_TOC) {
		unsigned int inst;

		if (probe_kernel_read(&inst, (void *)(ip - 4), 4)) {
			pr_err("Fetching instruction at %lx failed.\n", ip - 4);
			return -EFAULT;
		}

		/* We expect either a mlfr r0, or a std r0, LRSAVE(r1) */
		if (inst != PPC_INST_MFLR && inst != PPC_INST_STD_LR) {
			pr_err("Unexpected instructions around bl _mcount\n"
			       "when enabling dynamic ftrace!\t"
			       "(%08x,bl,%08x)\n", inst, op);
			return -EINVAL;
		}

		/* When using -mkernel_profile there is no load to jump over */
		pop = PPC_INST_NOP;
	}

	if (patch_instruction((unsigned int *)ip, pop)) {
		pr_err("Patching NOP failed.\n");
194
		return -EPERM;
195
	}
196 197 198 199 200 201 202 203 204

	return 0;
}

#else /* !PPC64 */
static int
__ftrace_make_nop(struct module *mod,
		  struct dyn_ftrace *rec, unsigned long addr)
{
205 206
	unsigned int op;
	unsigned int jmp[4];
207 208 209
	unsigned long ip = rec->ip;
	unsigned long tramp;

210
	if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
211 212 213
		return -EFAULT;

	/* Make sure that that this is still a 24bit jump */
214
	if (!is_bl_op(op)) {
215
		pr_err("Not expected bl: opcode is %x\n", op);
216 217 218 219
		return -EINVAL;
	}

	/* lets find where the pointer goes */
220
	tramp = find_bl_target(ip, op);
221 222 223

	/*
	 * On PPC32 the trampoline looks like:
224 225 226
	 *  0x3d, 0x80, 0x00, 0x00  lis r12,sym@ha
	 *  0x39, 0x8c, 0x00, 0x00  addi r12,r12,sym@l
	 *  0x7d, 0x89, 0x03, 0xa6  mtctr r12
227
	 *  0x4e, 0x80, 0x04, 0x20  bctr
228 229
	 */

230
	pr_devel("ip:%lx jumps to %lx", ip, tramp);
231 232

	/* Find where the trampoline jumps to */
233
	if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
234
		pr_err("Failed to read %lx\n", tramp);
235 236 237
		return -EFAULT;
	}

238
	pr_devel(" %08x %08x ", jmp[0], jmp[1]);
239 240

	/* verify that this is what we expect it to be */
241 242 243
	if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
	    ((jmp[1] & 0xffff0000) != 0x398c0000) ||
	    (jmp[2] != 0x7d8903a6) ||
244
	    (jmp[3] != 0x4e800420)) {
245
		pr_err("Not a trampoline\n");
246 247
		return -EINVAL;
	}
248

249 250
	tramp = (jmp[1] & 0xffff) |
		((jmp[0] & 0xffff) << 16);
251 252 253
	if (tramp & 0x8000)
		tramp -= 0x10000;

254
	pr_devel(" %lx ", tramp);
255 256

	if (tramp != addr) {
257
		pr_err("Trampoline location %08lx does not match addr\n",
258 259 260 261
		       tramp);
		return -EINVAL;
	}

262
	op = PPC_INST_NOP;
263

264
	if (patch_instruction((unsigned int *)ip, op))
265 266
		return -EPERM;

267 268 269
	return 0;
}
#endif /* PPC64 */
270
#endif /* CONFIG_MODULES */
271

272 273 274
int ftrace_make_nop(struct module *mod,
		    struct dyn_ftrace *rec, unsigned long addr)
{
275
	unsigned long ip = rec->ip;
276
	unsigned int old, new;
277 278 279 280 281 282

	/*
	 * If the calling address is more that 24 bits away,
	 * then we had to use a trampoline to make the call.
	 * Otherwise just update the call site.
	 */
283
	if (test_24bit_addr(ip, addr)) {
284
		/* within range */
285
		old = ftrace_call_replace(ip, addr, 1);
286
		new = PPC_INST_NOP;
287 288 289
		return ftrace_modify_code(ip, old, new);
	}

290
#ifdef CONFIG_MODULES
291 292 293 294 295 296 297
	/*
	 * Out of range jumps are called from modules.
	 * We should either already have a pointer to the module
	 * or it has been passed in.
	 */
	if (!rec->arch.mod) {
		if (!mod) {
298
			pr_err("No module loaded addr=%lx\n", addr);
299 300 301 302 303
			return -EFAULT;
		}
		rec->arch.mod = mod;
	} else if (mod) {
		if (mod != rec->arch.mod) {
304
			pr_err("Record mod %p not equal to passed in mod %p\n",
305 306 307 308 309 310 311 312
			       rec->arch.mod, mod);
			return -EINVAL;
		}
		/* nothing to do if mod == rec->arch.mod */
	} else
		mod = rec->arch.mod;

	return __ftrace_make_nop(mod, rec, addr);
313 314 315 316
#else
	/* We should not get here without modules */
	return -EINVAL;
#endif /* CONFIG_MODULES */
317 318
}

319
#ifdef CONFIG_MODULES
320
#ifdef CONFIG_PPC64
321 322 323 324 325 326
/*
 * Examine the existing instructions for __ftrace_make_call.
 * They should effectively be a NOP, and follow formal constraints,
 * depending on the ABI. Return false if they don't.
 */
#ifndef CC_USING_MPROFILE_KERNEL
327
static int
328
expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
329 330
{
	/*
331 332 333 334 335 336 337
	 * We expect to see:
	 *
	 * b +8
	 * ld r2,XX(r1)
	 *
	 * The load offset is different depending on the ABI. For simplicity
	 * just mask it out when doing the compare.
338
	 */
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
	if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
		return 0;
	return 1;
}
#else
static int
expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
{
	/* look for patched "NOP" on ppc64 with -mprofile-kernel */
	if (op0 != PPC_INST_NOP)
		return 0;
	return 1;
}
#endif

static int
__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned int op[2];
	void *ip = (void *)rec->ip;

	/* read where this goes */
	if (probe_kernel_read(op, ip, sizeof(op)))
		return -EFAULT;

	if (!expected_nop_sequence(ip, op[0], op[1])) {
		pr_err("Unexpected call sequence at %p: %x %x\n",
		ip, op[0], op[1]);
367 368 369 370 371
		return -EINVAL;
	}

	/* If we never set up a trampoline to ftrace_caller, then bail */
	if (!rec->arch.mod->arch.tramp) {
372
		pr_err("No ftrace trampoline\n");
373 374 375
		return -EINVAL;
	}

376
	/* Ensure branch is within 24 bits */
377
	if (!create_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
378
		pr_err("Branch out of range\n");
379
		return -EINVAL;
380 381
	}

382
	if (patch_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
383
		pr_err("REL24 out of range!\n");
384 385
		return -EINVAL;
	}
386

387 388
	return 0;
}
389 390 391 392 393 394 395 396 397 398

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
			unsigned long addr)
{
	return ftrace_make_call(rec, addr);
}
#endif

#else  /* !CONFIG_PPC64: */
399 400 401
static int
__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
402
	unsigned int op;
403 404 405
	unsigned long ip = rec->ip;

	/* read where this goes */
406
	if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
407 408 409
		return -EFAULT;

	/* It should be pointing to a nop */
410
	if (op != PPC_INST_NOP) {
411
		pr_err("Expected NOP but have %x\n", op);
412 413 414 415 416
		return -EINVAL;
	}

	/* If we never set up a trampoline to ftrace_caller, then bail */
	if (!rec->arch.mod->arch.tramp) {
417
		pr_err("No ftrace trampoline\n");
418 419 420
		return -EINVAL;
	}

421 422 423 424
	/* create the branch to the trampoline */
	op = create_branch((unsigned int *)ip,
			   rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
	if (!op) {
425
		pr_err("REL24 out of range!\n");
426 427 428
		return -EINVAL;
	}

429
	pr_devel("write to %lx\n", rec->ip);
430

431
	if (patch_instruction((unsigned int *)ip, op))
432 433
		return -EPERM;

434 435 436
	return 0;
}
#endif /* CONFIG_PPC64 */
437
#endif /* CONFIG_MODULES */
438 439 440

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
441
	unsigned long ip = rec->ip;
442
	unsigned int old, new;
443 444 445 446 447 448

	/*
	 * If the calling address is more that 24 bits away,
	 * then we had to use a trampoline to make the call.
	 * Otherwise just update the call site.
	 */
449
	if (test_24bit_addr(ip, addr)) {
450
		/* within range */
451
		old = PPC_INST_NOP;
452
		new = ftrace_call_replace(ip, addr, 1);
453
		return ftrace_modify_code(ip, old, new);
454 455
	}

456
#ifdef CONFIG_MODULES
457 458 459 460 461 462
	/*
	 * Out of range jumps are called from modules.
	 * Being that we are converting from nop, it had better
	 * already have a module defined.
	 */
	if (!rec->arch.mod) {
463
		pr_err("No module loaded\n");
464 465 466 467
		return -EINVAL;
	}

	return __ftrace_make_call(rec, addr);
468 469 470 471
#else
	/* We should not get here without modules */
	return -EINVAL;
#endif /* CONFIG_MODULES */
472 473
}

474
int ftrace_update_ftrace_func(ftrace_func_t func)
475 476
{
	unsigned long ip = (unsigned long)(&ftrace_call);
477
	unsigned int old, new;
478 479
	int ret;

480
	old = *(unsigned int *)&ftrace_call;
481
	new = ftrace_call_replace(ip, (unsigned long)func, 1);
482 483 484 485 486
	ret = ftrace_modify_code(ip, old, new);

	return ret;
}

487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
	unsigned long ftrace_addr = (unsigned long)FTRACE_ADDR;
	int ret;

	ret = ftrace_update_record(rec, enable);

	switch (ret) {
	case FTRACE_UPDATE_IGNORE:
		return 0;
	case FTRACE_UPDATE_MAKE_CALL:
		return ftrace_make_call(rec, ftrace_addr);
	case FTRACE_UPDATE_MAKE_NOP:
		return ftrace_make_nop(NULL, rec, ftrace_addr);
	}

	return 0;
}

void ftrace_replace_code(int enable)
{
	struct ftrace_rec_iter *iter;
	struct dyn_ftrace *rec;
	int ret;

	for (iter = ftrace_rec_iter_start(); iter;
	     iter = ftrace_rec_iter_next(iter)) {
		rec = ftrace_rec_iter_record(iter);
		ret = __ftrace_replace_code(rec, enable);
		if (ret) {
517
			ftrace_bug(ret, rec);
518 519 520 521 522
			return;
		}
	}
}

523 524 525 526
/*
 * Use the default ftrace_modify_all_code, but without
 * stop_machine().
 */
527 528
void arch_ftrace_update_code(int command)
{
529
	ftrace_modify_all_code(command);
530 531
}

532
int __init ftrace_dyn_arch_init(void)
533 534 535
{
	return 0;
}
536 537 538 539
#endif /* CONFIG_DYNAMIC_FTRACE */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER

540 541 542 543 544 545 546 547 548
#ifdef CONFIG_DYNAMIC_FTRACE
extern void ftrace_graph_call(void);
extern void ftrace_graph_stub(void);

int ftrace_enable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);
	unsigned long addr = (unsigned long)(&ftrace_graph_caller);
	unsigned long stub = (unsigned long)(&ftrace_graph_stub);
549
	unsigned int old, new;
550

551
	old = ftrace_call_replace(ip, stub, 0);
552 553 554 555 556 557 558 559 560 561
	new = ftrace_call_replace(ip, addr, 0);

	return ftrace_modify_code(ip, old, new);
}

int ftrace_disable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);
	unsigned long addr = (unsigned long)(&ftrace_graph_caller);
	unsigned long stub = (unsigned long)(&ftrace_graph_stub);
562
	unsigned int old, new;
563

564
	old = ftrace_call_replace(ip, addr, 0);
565 566 567 568 569 570
	new = ftrace_call_replace(ip, stub, 0);

	return ftrace_modify_code(ip, old, new);
}
#endif /* CONFIG_DYNAMIC_FTRACE */

571 572
/*
 * Hook the return address and push it in the stack of return addrs
573
 * in current thread info. Return the address we want to divert to.
574
 */
575
unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
576 577
{
	struct ftrace_graph_ent trace;
578
	unsigned long return_hooker;
579

580
	if (unlikely(ftrace_graph_is_dead()))
581
		goto out;
582

583
	if (unlikely(atomic_read(&current->tracing_graph_pause)))
584
		goto out;
585

586
	return_hooker = ppc_function_entry(return_to_handler);
587

588
	trace.func = ip;
589
	trace.depth = current->curr_ret_stack + 1;
590 591

	/* Only trace if the calling function expects to */
592 593 594 595 596
	if (!ftrace_graph_entry(&trace))
		goto out;

	if (ftrace_push_return_trace(parent, ip, &trace.depth, 0) == -EBUSY)
		goto out;
597

598 599 600
	parent = return_hooker;
out:
	return parent;
601 602
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
603 604 605 606 607 608 609

#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
unsigned long __init arch_syscall_addr(int nr)
{
	return sys_call_table[nr*2];
}
#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */
新手
引导
客服 返回
顶部